Adding upstream version 16.2.11+ds.upstream/16.2.11+dsupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

626 files changed, 289432 insertions, 0 deletions

diff --git a/doc/.gitignore b/doc/.gitignore
new file mode 100644
index 000000000..0c7c74746
--- /dev/null
+++ b/doc/.gitignore
@@ -0,0 +1,2 @@
+/overview.png
+/object_store.png
diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt
new file mode 100644
index 000000000..e0c30d175
--- /dev/null
+++ b/doc/CMakeLists.txt
@@ -0,0 +1,3 @@
+if(WITH_MANPAGE)
+  add_subdirectory(man)
+endif(WITH_MANPAGE)
diff --git a/doc/README.md b/doc/README.md
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/doc/README.md
diff --git a/doc/_ext/ceph_commands.py b/doc/_ext/ceph_commands.py
new file mode 100644
index 000000000..f1d7ee606
--- /dev/null
+++ b/doc/_ext/ceph_commands.py
@@ -0,0 +1,441 @@
+import io
+import os
+import sys
+import contextlib
+
+from docutils.parsers.rst import directives
+from docutils.parsers.rst import Directive
+from jinja2 import Template
+from pcpp.preprocessor import Preprocessor
+from sphinx.util import logging
+from sphinx.util.console import bold
+
+logger = logging.getLogger(__name__)
+
+
+class Flags:
+    NOFORWARD = (1 << 0)
+    OBSOLETE = (1 << 1)
+    DEPRECATED = (1 << 2)
+    MGR = (1 << 3)
+    POLL = (1 << 4)
+    HIDDEN = (1 << 5)
+
+    VALS = {
+        NOFORWARD: 'no_forward',
+        OBSOLETE: 'obsolete',
+        DEPRECATED: 'deprecated',
+        MGR: 'mgr',
+        POLL: 'poll',
+        HIDDEN: 'hidden',
+    }
+
+    def __init__(self, fs):
+        self.fs = fs
+
+    def __contains__(self, other):
+        return other in str(self)
+
+    def __str__(self):
+        keys = Flags.VALS.keys()
+        es = {Flags.VALS[k] for k in keys if self.fs & k == k}
+        return ', '.join(sorted(es))
+
+    def __bool__(self):
+        return bool(str(self))
+
+
+class CmdParam(object):
+    t = {
+        'CephInt': 'int',
+        'CephString': 'str',
+        'CephChoices': 'str',
+        'CephPgid': 'str',
+        'CephOsdName': 'str',
+        'CephPoolname': 'str',
+        'CephObjectname': 'str',
+        'CephUUID': 'str',
+        'CephEntityAddr': 'str',
+        'CephIPAddr': 'str',
+        'CephName': 'str',
+        'CephBool': 'bool',
+        'CephFloat': 'float',
+        'CephFilepath': 'str',
+    }
+
+    bash_example = {
+        'CephInt': '1',
+        'CephString': 'string',
+        'CephChoices': 'choice',
+        'CephPgid': '0',
+        'CephOsdName': 'osd.0',
+        'CephPoolname': 'poolname',
+        'CephObjectname': 'objectname',
+        'CephUUID': 'uuid',
+        'CephEntityAddr': 'entityaddr',
+        'CephIPAddr': '0.0.0.0',
+        'CephName': 'name',
+        'CephBool': 'true',
+        'CephFloat': '0.0',
+        'CephFilepath': '/path/to/file',
+    }
+
+    def __init__(self, type, name,
+                 who=None, n=None, req=True, range=None, strings=None,
+                 goodchars=None):
+        self.type = type
+        self.name = name
+        self.who = who
+        self.n = n == 'N'
+        self.req = req != 'false'
+        self.range = range.split('|') if range else []
+        self.strings = strings.split('|') if strings else []
+        self.goodchars = goodchars
+
+        assert who == None
+
+    def help(self):
+        advanced = []
+        if self.type != 'CephString':
+            advanced.append(self.type + ' ')
+        if self.range:
+            advanced.append('range= ``{}`` '.format('..'.join(self.range)))
+        if self.strings:
+            advanced.append('strings=({}) '.format(' '.join(self.strings)))
+        if self.goodchars:
+            advanced.append('goodchars= ``{}`` '.format(self.goodchars))
+        if self.n:
+            advanced.append('(can be repeated)')
+
+        advanced = advanced or ["(string)"]
+        return ' '.join(advanced)
+
+    def mk_example_value(self):
+        if self.type == 'CephChoices' and self.strings:
+            return self.strings[0]
+        if self.range:
+            return self.range[0]
+        return CmdParam.bash_example[self.type]
+
+    def mk_bash_example(self, simple):
+        val = self.mk_example_value()
+
+        if self.type == 'CephBool':
+            return '--' + self.name
+        if simple:
+            if self.type == "CephChoices" and self.strings:
+                return val
+            elif self.type == "CephString" and self.name != 'who':
+                return 'my_' + self.name
+            else:
+                return CmdParam.bash_example[self.type]
+        else:
+            return '--{}={}'.format(self.name, val)
+
+
+class CmdCommand(object):
+    def __init__(self, prefix, args, desc,
+                 module=None, perm=None, flags=0, poll=None):
+        self.prefix = prefix
+        self.params = sorted([CmdParam(**arg) for arg in args],
+                             key=lambda p: p.req, reverse=True)
+        self.help = desc
+        self.module = module
+        self.perm = perm
+        self.flags = Flags(flags)
+        self.needs_overload = False
+
+    def is_reasonably_simple(self):
+        if len(self.params) > 3:
+            return False
+        if any(p.n for p in self.params):
+            return False
+        return True
+
+    def mk_bash_example(self):
+        simple = self.is_reasonably_simple()
+        line = ' '.join(['ceph', self.prefix] + [p.mk_bash_example(simple) for p in self.params])
+        return line
+
+
+class Sig:
+    @staticmethod
+    def _parse_arg_desc(desc):
+        try:
+            return dict(kv.split('=') for kv in desc.split(',') if kv)
+        except ValueError:
+            return desc
+
+    @staticmethod
+    def parse_cmd(cmd):
+        parsed = [Sig._parse_arg_desc(s) or s for s in cmd.split()]
+        prefix = [s for s in parsed if isinstance(s, str)]
+        params = [s for s in parsed if not isinstance(s, str)]
+        return ' '.join(prefix), params
+
+    @staticmethod
+    def parse_args(args):
+        return [Sig._parse_arg_desc(arg) for arg in args.split()]
+
+
+TEMPLATE = '''
+.. This file is automatically generated. do not modify
+
+{% for command in commands %}
+
+{{ command.prefix }}
+{{ command.prefix | length * '^' }}
+
+{{ command.help | wordwrap(70)}}
+
+Example command:
+
+.. code-block:: bash
+
+    {{ command.mk_bash_example() }}
+{% if command.params %}
+Parameters:
+
+{% for param in command.params %}* **{{param.name}}**: {{ param.help() | wordwrap(70) | indent(2) }}
+{% endfor %}{% endif %}
+Ceph Module:
+
+* *{{ command.module }}*
+
+Required Permissions:
+
+* *{{ command.perm }}*
+
+{% if command.flags %}Command Flags:
+
+* *{{ command.flags }}*
+{% endif %}
+{% endfor %}
+
+'''
+
+
+class CephMgrCommands(Directive):
+    """
+    extracts commands from specified mgr modules
+    """
+    has_content = True
+    required_arguments = 1
+    optional_arguments = 0
+    final_argument_whitespace = False
+    option_spec = {'python_path': directives.unchanged}
+
+    def _normalize_path(self, dirname):
+        my_dir = os.path.dirname(os.path.realpath(__file__))
+        src_dir = os.path.abspath(os.path.join(my_dir, '../..'))
+        return os.path.join(src_dir, dirname)
+
+    def _is_mgr_module(self, dirname, name):
+        if not os.path.isdir(os.path.join(dirname, name)):
+            return False
+        if not os.path.isfile(os.path.join(dirname, name, '__init__.py')):
+            return False
+        return name not in ['tests']
+
+    @contextlib.contextmanager
+    def mocked_modules(self):
+        # src/pybind/mgr/tests
+        from tests import mock
+        mock_imports = ['rados',
+                        'rbd',
+                        'cephfs',
+                        'dateutil',
+                        'dateutil.parser']
+        # make dashboard happy
+        mock_imports += ['OpenSSL',
+                         'jwt',
+                         'bcrypt',
+                         'scipy',
+                         'jsonpatch',
+                         'rook.rook_client',
+                         'rook.rook_client.ceph',
+                         'rook.rook_client._helper',
+                         'cherrypy=3.2.3']
+
+        # make restful happy
+        mock_imports += ['pecan',
+                         'pecan.rest',
+                         'pecan.hooks',
+                         'werkzeug',
+                         'werkzeug.serving']
+
+        for m in mock_imports:
+            args = {}
+            parts = m.split('=', 1)
+            mocked = parts[0]
+            if len(parts) > 1:
+                args['__version__'] = parts[1]
+            sys.modules[mocked] = mock.Mock(**args)
+
+        try:
+            yield
+        finally:
+            for m in mock_imports:
+                mocked = m.split('=', 1)[0]
+                sys.modules.pop(mocked)
+
+    def _collect_module_commands(self, name):
+        with self.mocked_modules():
+            logger.info(bold(f"loading mgr module '{name}'..."))
+            mgr_mod = __import__(name, globals(), locals(), [], 0)
+            from tests import M
+
+            def subclass(x):
+                try:
+                    return issubclass(x, M)
+                except TypeError:
+                    return False
+            ms = [c for c in mgr_mod.__dict__.values()
+                  if subclass(c) and 'Standby' not in c.__name__]
+            [m] = ms
+            assert isinstance(m.COMMANDS, list)
+            return m.COMMANDS
+
+    def _normalize_command(self, command):
+        if 'handler' in command:
+            del command['handler']
+        if 'cmd' in command:
+            command['prefix'], command['args'] = Sig.parse_cmd(command['cmd'])
+            del command['cmd']
+        else:
+            command['args'] = Sig.parse_args(command['args'])
+        command['flags'] = (1 << 3)
+        command['module'] = 'mgr'
+        return command
+
+    def _render_cmds(self, commands):
+        rendered = Template(TEMPLATE).render(commands=list(commands))
+        lines = rendered.split("\n")
+        assert lines
+        lineno = self.lineno - self.state_machine.input_offset - 1
+        source = self.state_machine.input_lines.source(lineno)
+        self.state_machine.insert_input(lines, source)
+
+    def run(self):
+        module_path = self._normalize_path(self.arguments[0])
+        sys.path.insert(0, module_path)
+        for path in self.options.get('python_path', '').split(':'):
+            sys.path.insert(0, self._normalize_path(path))
+        os.environ['UNITTEST'] = 'true'
+        modules = [name for name in os.listdir(module_path)
+                   if self._is_mgr_module(module_path, name)]
+        commands = sum([self._collect_module_commands(name) for name in modules], [])
+        cmds = [CmdCommand(**self._normalize_command(c)) for c in commands]
+        cmds = [cmd for cmd in cmds if 'hidden' not in cmd.flags]
+        cmds = sorted(cmds, key=lambda cmd: cmd.prefix)
+        self._render_cmds(cmds)
+        return []
+
+
+class MyProcessor(Preprocessor):
+    def __init__(self):
+        super().__init__()
+        self.cmds = []
+        self.undef('__DATE__')
+        self.undef('__TIME__')
+        self.expand_linemacro = False
+        self.expand_filemacro = False
+        self.expand_countermacro = False
+        self.line_directive = '#line'
+        self.define("__PCPP_VERSION__ " + '')
+        self.define("__PCPP_ALWAYS_FALSE__ 0")
+        self.define("__PCPP_ALWAYS_TRUE__ 1")
+
+    def eval(self, src):
+        _cmds = []
+
+        NONE = 0
+        NOFORWARD = (1 << 0)
+        OBSOLETE = (1 << 1)
+        DEPRECATED = (1 << 2)
+        MGR = (1 << 3)
+        POLL = (1 << 4)
+        HIDDEN = (1 << 5)
+        TELL = (1 << 6)
+
+        def FLAG(a):
+            return a
+
+        def COMMAND(cmd, desc, module, perm):
+            _cmds.append({
+                'cmd': cmd,
+                'desc': desc,
+                'module': module,
+                'perm': perm
+            })
+
+        def COMMAND_WITH_FLAG(cmd, desc, module, perm, flag):
+            _cmds.append({
+                'cmd': cmd,
+                'desc': desc,
+                'module': module,
+                'perm': perm,
+                'flags': flag
+            })
+
+        self.parse(src)
+        out = io.StringIO()
+        self.write(out)
+        out.seek(0)
+        s = out.read()
+        exec(s, globals(), locals())
+        return _cmds
+
+
+class CephMonCommands(Directive):
+    """
+    extracts commands from specified header file
+    """
+    has_content = True
+    required_arguments = 1
+    optional_arguments = 0
+    final_argument_whitespace = True
+
+    def _src_dir(self):
+        my_dir = os.path.dirname(os.path.realpath(__file__))
+        return os.path.abspath(os.path.join(my_dir, '../..'))
+
+    def _parse_headers(self, headers):
+        src_dir = self._src_dir()
+        src = '\n'.join(f'#include "{src_dir}/{header}"' for header in headers)
+        return MyProcessor().eval(src)
+
+    def _normalize_command(self, command):
+        if 'handler' in command:
+            del command['handler']
+        command['prefix'], command['args'] = Sig.parse_cmd(command['cmd'])
+        del command['cmd']
+        return command
+
+    def _render_cmds(self, commands):
+        rendered = Template(TEMPLATE).render(commands=list(commands))
+        lines = rendered.split("\n")
+        assert lines
+        lineno = self.lineno - self.state_machine.input_offset - 1
+        source = self.state_machine.input_lines.source(lineno)
+        self.state_machine.insert_input(lines, source)
+
+    def run(self):
+        headers = self.arguments[0].split()
+        commands = self._parse_headers(headers)
+        cmds = [CmdCommand(**self._normalize_command(c)) for c in commands]
+        cmds = [cmd for cmd in cmds if 'hidden' not in cmd.flags]
+        cmds = sorted(cmds, key=lambda cmd: cmd.prefix)
+        self._render_cmds(cmds)
+        return []
+
+
+def setup(app):
+    app.add_directive("ceph-mgr-commands", CephMgrCommands)
+    app.add_directive("ceph-mon-commands", CephMonCommands)
+
+    return {
+        'version': '0.1',
+        'parallel_read_safe': True,
+        'parallel_write_safe': True,
+    }
diff --git a/doc/_ext/ceph_releases.py b/doc/_ext/ceph_releases.py
new file mode 100644
index 000000000..a45809e2a
--- /dev/null
+++ b/doc/_ext/ceph_releases.py
@@ -0,0 +1,201 @@
+# cobbled together from:
+#   https://github.com/sphinx-contrib/documentedlist/blob/master/sphinxcontrib/documentedlist.py
+#   https://github.com/sphinx-doc/sphinx/blob/v1.6.3/sphinx/ext/graphviz.py
+#   https://github.com/thewtex/sphinx-contrib/blob/master/exceltable/sphinxcontrib/exceltable.py
+#   https://bitbucket.org/prometheus/sphinxcontrib-htsql/src/331a542c29a102eec9f8cba44797e53a49de2a49/sphinxcontrib/htsql.py?at=default&fileviewer=file-view-default
+# into the glory that follows:
+import json
+import yaml
+import sphinx
+import datetime
+from docutils.parsers.rst import Directive
+from docutils import nodes
+from sphinx.util import logging
+
+class CephReleases(Directive):
+    has_content = False
+    required_arguments = 1
+    optional_arguments = 0
+    option_spec = {}
+
+    def run(self):
+        filename = self.arguments[0]
+        document = self.state.document
+        env = document.settings.env
+        rel_filename, filename = env.relfn2path(filename)
+        env.note_dependency(filename)
+        try:
+            with open(filename, 'r') as fp:
+                releases = yaml.safe_load(fp)
+                releases = releases["releases"]
+        except Exception as e:
+            return [document.reporter.warning(
+                "Failed to open Ceph releases file {}: {}".format(filename, e),
+                line=self.lineno)]
+
+        table = nodes.table()
+        tgroup = nodes.tgroup(cols=3)
+        table += tgroup
+
+        tgroup.extend(
+            nodes.colspec(colwidth=30, colname='c'+str(idx))
+            for idx, _ in enumerate(range(4)))
+
+        thead = nodes.thead()
+        tgroup += thead
+        row_node = nodes.row()
+        thead += row_node
+        row_node.extend(nodes.entry(h, nodes.paragraph(text=h))
+            for h in ["Version", "Initial release", "Latest", "End of life (estimated)"])
+
+        releases = releases.items()
+        releases = sorted(releases, key=lambda t: t[0], reverse=True)
+
+        tbody = nodes.tbody()
+        tgroup += tbody
+
+        rows = []
+        for code_name, info in releases:
+            actual_eol = info.get("actual_eol", None)
+            if actual_eol and actual_eol <= datetime.datetime.now().date():
+                continue
+            trow = nodes.row()
+
+            entry = nodes.entry()
+            para = nodes.paragraph(text="`{}`_".format(code_name))
+            sphinx.util.nodes.nested_parse_with_titles(
+                    self.state, para, entry)
+            #entry += para
+            trow += entry
+
+            sorted_releases = sorted(info["releases"],
+                    key=lambda t: [t["released"]] + list(map(lambda v: int(v), t["version"].split("."))))
+            oldest_release = sorted_releases[0]
+            newest_release = sorted_releases[-1]
+
+            entry = nodes.entry()
+            para = nodes.paragraph(text="{}".format(
+                oldest_release["released"].strftime("%b %Y")))
+            entry += para
+            trow += entry
+
+            entry = nodes.entry()
+            if newest_release.get("skip_ref", False):
+                para = nodes.paragraph(text="{}".format(
+                    newest_release["version"]))
+            else:
+                para = nodes.paragraph(text="`{}`_".format(
+                    newest_release["version"]))
+            sphinx.util.nodes.nested_parse_with_titles(
+                    self.state, para, entry)
+            #entry += para
+            trow += entry
+
+            entry = nodes.entry()
+            para = nodes.paragraph(text="{}".format(
+                info.get("target_eol", "--")))
+            entry += para
+            trow += entry
+
+            rows.append(trow)
+
+        tbody.extend(rows)
+
+        return [table]
+
+class CephTimeline(Directive):
+    has_content = False
+    required_arguments = 12
+    optional_arguments = 0
+    option_spec = {}
+
+    def run(self):
+        filename = self.arguments[0]
+        document = self.state.document
+        env = document.settings.env
+        rel_filename, filename = env.relfn2path(filename)
+        env.note_dependency(filename)
+        try:
+            with open(filename, 'r') as fp:
+                releases = yaml.safe_load(fp)
+        except Exception as e:
+            return [document.reporter.warning(
+                "Failed to open Ceph releases file {}: {}".format(filename, e),
+                line=self.lineno)]
+
+        display_releases = self.arguments[1:]
+
+        timeline = []
+        for code_name, info in releases["releases"].items():
+            if code_name in display_releases:
+                for release in info.get("releases", []):
+                    released = release["released"]
+                    timeline.append((released, code_name, release["version"],
+                        release.get("skip_ref", False)))
+
+        assert "development" not in releases["releases"]
+        if "development" in display_releases:
+            for release in releases["development"]["releases"]:
+                released = release["released"]
+                timeline.append((released, "development", release["version"],
+                    release.get("skip_ref", False)))
+
+        timeline = sorted(timeline, key=lambda t: t[0], reverse=True)
+
+        table = nodes.table()
+        tgroup = nodes.tgroup(cols=3)
+        table += tgroup
+
+        columns = ["Date"] + display_releases
+        tgroup.extend(
+            nodes.colspec(colwidth=30, colname='c'+str(idx))
+            for idx, _ in enumerate(range(len(columns))))
+
+        thead = nodes.thead()
+        tgroup += thead
+        row_node = nodes.row()
+        thead += row_node
+        for col in columns:
+            entry = nodes.entry()
+            if col.lower() in ["date", "development"]:
+                para = nodes.paragraph(text=col)
+            else:
+                para = nodes.paragraph(text="`{}`_".format(col))
+            sphinx.util.nodes.nested_parse_with_titles(
+                    self.state, para, entry)
+            row_node += entry
+
+        tbody = nodes.tbody()
+        tgroup += tbody
+
+        rows = []
+        for row_info in timeline:
+            trow = nodes.row()
+
+            entry = nodes.entry()
+            para = nodes.paragraph(text=row_info[0].strftime("%b %Y"))
+            entry += para
+            trow += entry
+
+            for release in display_releases:
+                entry = nodes.entry()
+                if row_info[1] == release:
+                    if row_info[3]: # if skip ref
+                        para = nodes.paragraph(text=row_info[2])
+                    else:
+                        para = nodes.paragraph(text="`{}`_".format(row_info[2]))
+                    sphinx.util.nodes.nested_parse_with_titles(
+                            self.state, para, entry)
+                else:
+                    para = nodes.paragraph(text="--")
+                    entry += para
+                trow += entry
+            rows.append(trow)
+
+        tbody.extend(rows)
+
+        return [table]
+
+def setup(app):
+    app.add_directive('ceph_releases', CephReleases)
+    app.add_directive('ceph_timeline', CephTimeline)
diff --git a/doc/_static/css/custom.css b/doc/_static/css/custom.css
new file mode 100644
index 000000000..2a37cab99
--- /dev/null
+++ b/doc/_static/css/custom.css
@@ -0,0 +1,28 @@
+dt { 
+    scroll-margin-top: 3em; 
+}
+
+h2 {
+    scroll-margin-top: 4em; 
+}
+
+h3 {
+    scroll-margin-top: 4em; 
+}
+
+section {
+    scroll-margin-top: 4em; 
+}
+
+span {
+    scroll-margin-top: 2em; 
+}
+
+ul.simple > li > ul > li:last-child {
+    margin-block-end : 1em;
+}
+
+div.section > ul > li > p {
+    margin-block-start : 0.6em;
+    margin-block-end : 0.6em;
+}
diff --git a/doc/_templates/page.html b/doc/_templates/page.html
new file mode 100644
index 000000000..3923975d7
--- /dev/null
+++ b/doc/_templates/page.html
@@ -0,0 +1,25 @@
+{% extends "!page.html" %}
+{% block body %}
+
+{%- if release == 'dev' %}
+<div id="dev-warning" class="admonition note">
+  <p class="first admonition-title">Notice</p>
+  <p class="last">This document is for a development version of Ceph.</p>
+</div>
+{%- endif %}
+
+{%- if is_release_eol %}
+<div id="eol-warning" class="admonition warning">
+  <p class="first admonition-title">Warning</p>
+  <p class="last">This document is for an unsupported version of Ceph.</p>
+</div>
+{%- endif %}
+
+{%- if not is_release_eol %}
+  <div id="docubetter" align="right" style="padding: 15px; font-weight: bold;">
+    <a href="https://pad.ceph.com/p/Report_Documentation_Bugs">Report a Documentation Bug</a>
+  </div>
+{%- endif %}
+
+  {{ super() }}
+{% endblock %}
diff --git a/doc/_templates/smarttoc.html b/doc/_templates/smarttoc.html
new file mode 100644
index 000000000..7a3fd3f2b
--- /dev/null
+++ b/doc/_templates/smarttoc.html
@@ -0,0 +1,16 @@
+{#
+    Sphinx sidebar template: smart table of contents.
+
+    Shows a sidebar ToC that gives you a more global view of the
+    documentation, and not the confusing cur/prev/next which is the
+    default sidebar.
+
+    The ToC will open and collapse automatically to show the part of the
+    hierarchy you are in. Top-level items will always be visible.
+
+#}
+<h3><a href="{{ pathto(master_doc) }}">{{ _('Table Of Contents') }}</a></h3>
+{{ toctree(maxdepth=-1, includehidden=True) }}
+
+<!-- ugly kludge to make genindex look like it's part of the toc -->
+<ul style="margin-top: -10px"><li class="toctree-l1"><a class="reference internal" href="{{ pathto('genindex') }}">Index</a></li></ul>
diff --git a/doc/_themes/ceph/static/nature.css_t b/doc/_themes/ceph/static/nature.css_t
new file mode 100644
index 000000000..f02a23298
--- /dev/null
+++ b/doc/_themes/ceph/static/nature.css_t
@@ -0,0 +1,316 @@
+/*
+ * nature.css_t
+ * ~~~~~~~~~~~~
+ *
+ * Sphinx stylesheet -- nature theme.
+ *
+ * :copyright: Copyright 2007-2019 by the Sphinx team, see AUTHORS.
+ * :license: BSD, see LICENSE for details.
+ *
+ */
+
+@import url("basic.css");
+
+/* -- page layout ----------------------------------------------------------- */
+
+@import url('https://fonts.googleapis.com/css?family=Titillium+Web:400,300,700');
+
+body {
+    font-family: Helvetica, Arial, sans-serif;
+    font-size: 100%;
+    background-color: #fff;
+    color: #555;
+    margin: 0;
+    padding: 0;
+    border-top: 5px solid #F05C56;
+}
+
+div.documentwrapper {
+    float: left;
+    width: 100%;
+}
+
+div.bodywrapper {
+    margin: 0 0 0 330px;
+}
+
+hr {
+    border: 1px solid #B1B4B6;
+}
+
+div.document {
+    background-color: #eee;
+}
+
+div.body {
+    background-color: #ffffff;
+    color: #3E4349;
+    padding: 0 30px 30px 30px;
+    font-size: 0.9em;
+}
+
+div.footer {
+    color: #555;
+    width: 100%;
+    padding: 13px 0;
+    text-align: center;
+    font-size: 75%;
+}
+
+div.footer a {
+    color: #444;
+    text-decoration: underline;
+}
+
+div.related {
+    background-color: #80D2DC;
+    line-height: 32px;
+    color: #37424A;
+    font-size: 0.9em;
+    border-top: #9C4850 5px solid;
+}
+
+div.related a {
+    color: #37424A;
+    text-decoration: none;
+}
+
+div.related a:hover {
+    color: #fff;
+    // text-decoration: underline;
+}
+
+div.sphinxsidebar {
+    font-size: 0.75em;
+    line-height: 1.5em;
+    width: 330px;
+}
+
+div.sphinxsidebarwrapper{
+    padding: 20px 0;
+}
+
+div.sphinxsidebar h3,
+div.sphinxsidebar h4 {
+    font-family: 'Titillium Web', sans-serif;
+    color: #e6e8e8;
+    font-size: 1.2em;
+    font-weight: normal;
+    margin: 0;
+    padding: 5px 10px;
+    background-color: #5e6a71;
+    // text-shadow: 1px 1px 0 white;
+    text-transform: uppercase;
+}
+
+div.sphinxsidebar h4{
+    font-size: 1.1em;
+}
+
+div.sphinxsidebar h3 a,
+div.sphinxsidebar h4 a {
+    color: #e6e8e8;
+}
+
+div.sphinxsidebar p {
+    color: #888;
+    padding: 5px 20px;
+}
+
+div.sphinxsidebar p.topless {
+}
+
+div.sphinxsidebar ul {
+    margin: 10px 20px;
+    padding: 0;
+    color: #000;
+}
+
+div.sphinxsidebar a {
+    color: #444;
+}
+
+div.sphinxsidebar input {
+    border: 1px solid #ccc;
+    font-family: sans-serif;
+    font-size: 1em;
+}
+
+div.sphinxsidebar input[type=text] {
+    margin-left: 20px;
+    margin-right: 20px;
+}
+
+div.sphinxsidebar input[type=submit] {
+    border-left: 1px solid #ccc !important;
+    margin-left: 20px;
+}
+
+/* -- body styles ----------------------------------------------------------- */
+
+a {
+    color: #F05C56;
+    text-decoration: none;
+}
+
+a:hover {
+    color: #E32E00;
+    text-decoration: underline;
+}
+
+div.body h1,
+div.body h2,
+div.body h3,
+div.body h4,
+div.body h5,
+div.body h6 {
+    background-color: #99DAE3;
+    font-weight: normal;
+    color: #212224;
+    margin: 30px 0px 10px 0px;
+    padding: 5px 0 5px 10px;
+    // text-shadow: 0px 1px 0 white;
+    text-transform: uppercase;
+}
+
+div.body h1 { font-family: Titillium Web; border-top: 20px solid white; margin-top: 0; font-size: 200%; }
+div.body h2 { font-family: Titillium Web; font-size: 150%; background-color: #B2E3EA; }
+div.body h3 { font-family: Titillium Web; font-size: 120%; background-color: #CCECF1; }
+div.body h4 { font-family: Helvetica, Arial, sans-serif; font-size: 110%; background-color: #CCECF1; }
+div.body h5 { font-family: Helvetica, Arial, sans-serif; font-size: 100%; background-color: #CCECF1; }
+div.body h6 { font-family: Helvetica, Arial, sans-serif; font-size: 100%; background-color: #CCECF1; }
+
+a.headerlink {
+    color: #c60f0f;
+    font-size: 0.8em;
+    padding: 0 4px 0 4px;
+    text-decoration: none;
+}
+
+a.headerlink:hover {
+    background-color: #c60f0f;
+    color: white;
+}
+
+div.body p, div.body dd, div.body li {
+    line-height: 1.5em;
+}
+
+div.admonition p.admonition-title + p {
+    display: inline;
+}
+
+div.highlight {
+    background-color: white;
+}
+
+div.note {
+    background-color: #eee;
+    border: 1px solid #ccc;
+}
+
+div.seealso {
+    background-color: #ffc;
+    border: 1px solid #ff6;
+}
+
+div.topic {
+    background-color: #eee;
+}
+
+div.warning {
+    background-color: #ffe4e4;
+    border: 1px solid #f66;
+}
+
+div.warning a {
+    color: black;
+}
+
+p.admonition-title {
+    display: inline;
+}
+
+p.admonition-title:after {
+    content: ":";
+}
+
+pre {
+    padding: 10px;
+    background-color: White;
+    color: #222;
+    line-height: 1.2em;
+    border: 1px solid #C6C9CB;
+    font-size: 1.1em;
+    margin: 1.5em 0 1.5em 0;
+    -webkit-box-shadow: 1px 1px 1px #d8d8d8;
+    -moz-box-shadow: 1px 1px 1px #d8d8d8;
+}
+
+tt {
+    background-color: #ecf0f3;
+    color: #222;
+    /* padding: 1px 2px; */
+    font-size: 1.1em;
+    font-family: monospace;
+}
+
+.viewcode-back {
+    font-family: Arial, sans-serif;
+}
+
+div.viewcode-block:target {
+    background-color: #f4debf;
+    border-top: 1px solid #ac9;
+    border-bottom: 1px solid #ac9;
+}
+
+table.docutils {
+    margin: 1.5em;
+}
+
+div.sidebar {
+    border: 1px solid #5E6A71;
+    background-color: #E6E8E8;
+}
+
+div.admonition.tip {
+    background-color: #80D2DC;
+    border: 1px solid #55AEBA;
+}
+
+div.admonition.important {
+    background-color: #F05C56;
+    border: 1px solid #9C4850;
+    color: #fff;
+}
+
+div.admonition.important a {
+    color: #E6E8E8;
+}
+
+div.tip tt.literal {
+    background-color: #55aeba;
+    color: #fff;
+}
+
+div.important tt.literal {
+    background-color: #9C4850;
+    color: #fff;
+}
+
+h2 .literal {
+    color: #fff;
+    background-color: #37424a;
+}
+
+dl.glossary dt {
+    font-size: 1.0em;
+    padding-top:20px;
+}
+
+p.breathe-sectiondef-title {
+    font-size: 1.3em;
+    font-weight: bold;
+    border-bottom: thin solid #5E6A71;
+}
diff --git a/doc/_themes/ceph/theme.conf b/doc/_themes/ceph/theme.conf
new file mode 100644
index 000000000..1cc400446
--- /dev/null
+++ b/doc/_themes/ceph/theme.conf
@@ -0,0 +1,4 @@
+[theme]
+inherit = basic
+stylesheet = nature.css
+pygments_style = tango
diff --git a/doc/api/index.rst b/doc/api/index.rst
new file mode 100644
index 000000000..e359b87f8
--- /dev/null
+++ b/doc/api/index.rst
@@ -0,0 +1,62 @@
+===================
+ API Documentation
+===================
+
+Ceph RESTful API
+================
+
+See `Ceph REST API`_.
+
+.. _Ceph REST API: ../mgr/ceph_api/
+
+Ceph Storage Cluster APIs
+=========================
+
+See `Ceph Storage Cluster APIs`_.
+
+.. _Ceph Storage Cluster APIs: ../rados/api/
+
+
+Ceph File System APIs
+=====================
+
+See `libcephfs`_
+
+.. _libcephfs: ../cephfs/api
+
+Ceph Block Device APIs
+======================
+
+See `librbdpy`_.
+
+.. _librbdpy: ../rbd/api/librbdpy
+
+Ceph RADOS Gateway APIs
+=======================
+
+See `librgw-py`_.
+
+.. _librgw-py: ../radosgw/api
+
+Ceph Object Store APIs
+======================
+
+- See `S3-compatible API`_.
+- See `Swift-compatible API`_.
+- See `Admin Ops API`_.
+
+Ceph MON Command API
+====================
+
+- See `Mon command API`_.
+
+.. _S3-compatible API: ../radosgw/s3/
+.. _Swift-compatible API: ../radosgw/swift/
+.. _Admin Ops API: ../radosgw/adminops
+.. _Mon command API: mon_command_api
+
+
+.. toctree::
+    :hidden:
+    
+    mon_command_api
diff --git a/doc/api/mon_command_api.rst b/doc/api/mon_command_api.rst
new file mode 100644
index 000000000..51a752a84
--- /dev/null
+++ b/doc/api/mon_command_api.rst
@@ -0,0 +1,3 @@
+.. ceph-mgr-commands:: src/pybind/mgr
+   :python_path: src/pybind
+.. ceph-mon-commands:: src/mon/MonCommands.h src/mgr/MgrCommands.h
diff --git a/doc/architecture.rst b/doc/architecture.rst
new file mode 100644
index 000000000..46be74603
--- /dev/null
+++ b/doc/architecture.rst
@@ -0,0 +1,1649 @@
+==============
+ Architecture
+==============
+
+:term:`Ceph` uniquely delivers **object, block, and file storage** in one
+unified system. Ceph is highly reliable, easy to manage, and free. The power of
+Ceph can transform your company's IT infrastructure and your ability to manage
+vast amounts of data. Ceph delivers extraordinary scalability–thousands of
+clients accessing petabytes to exabytes of data. A :term:`Ceph Node` leverages
+commodity hardware and intelligent daemons, and a :term:`Ceph Storage Cluster`
+accommodates large numbers of nodes, which communicate with each other to
+replicate and redistribute data dynamically.
+
+.. image:: images/stack.png
+
+.. _arch-ceph-storage-cluster:
+
+The Ceph Storage Cluster
+========================
+
+Ceph provides an infinitely scalable :term:`Ceph Storage Cluster` based upon
+:abbr:`RADOS (Reliable Autonomic Distributed Object Store)`, which you can read
+about in `RADOS - A Scalable, Reliable Storage Service for Petabyte-scale
+Storage Clusters`_.
+
+A Ceph Storage Cluster consists of multiple types of daemons:
+
+- :term:`Ceph Monitor`
+- :term:`Ceph OSD Daemon`
+- :term:`Ceph Manager`
+- :term:`Ceph Metadata Server`
+
+.. ditaa::
+
+            +---------------+ +---------------+ +---------------+ +---------------+ 
+            |      OSDs     | |    Monitors   | |    Managers   | |      MDS      |
+            +---------------+ +---------------+ +---------------+ +---------------+ 
+
+A Ceph Monitor maintains a master copy of the cluster map. A cluster of Ceph
+monitors ensures high availability should a monitor daemon fail. Storage cluster
+clients retrieve a copy of the cluster map from the Ceph Monitor.
+
+A Ceph OSD Daemon checks its own state and the state of other OSDs and reports 
+back to monitors.
+
+A Ceph Manager acts as an endpoint for monitoring, orchestration, and plug-in
+modules.
+
+A Ceph Metadata Server (MDS) manages file metadata when CephFS is used to
+provide file services.
+
+Storage cluster clients and each :term:`Ceph OSD Daemon` use the CRUSH algorithm
+to efficiently compute information about data location, instead of having to
+depend on a central lookup table. Ceph's high-level features include a
+native interface to the Ceph Storage Cluster via ``librados``, and a number of
+service interfaces built on top of ``librados``.
+
+
+
+Storing Data
+------------
+
+The Ceph Storage Cluster receives data from :term:`Ceph Client`\s--whether it
+comes through a :term:`Ceph Block Device`, :term:`Ceph Object Storage`, the
+:term:`Ceph File System` or a custom implementation you create using
+``librados``-- which is stored as RADOS objects. Each object is stored on an
+:term:`Object Storage Device`. Ceph OSD Daemons handle read, write, and
+replication operations on storage drives.  With the older Filestore back end,
+each RADOS object was stored as a separate file on a conventional filesystem
+(usually XFS).  With the new and default BlueStore back end, objects are
+stored in a monolithic database-like fashion.
+
+.. ditaa::
+
+           /-----\       +-----+       +-----+
+           | obj |------>| {d} |------>| {s} |
+           \-----/       +-----+       +-----+
+   
+            Object         OSD          Drive
+
+Ceph OSD Daemons store data as objects in a flat namespace (e.g., no
+hierarchy of directories). An object has an identifier, binary data, and
+metadata consisting of a set of name/value pairs. The semantics are completely
+up to :term:`Ceph Client`\s. For example, CephFS uses metadata to store file
+attributes such as the file owner, created date, last modified date, and so
+forth.
+
+
+.. ditaa::
+
+           /------+------------------------------+----------------\
+           | ID   | Binary Data                  | Metadata       |
+           +------+------------------------------+----------------+
+           | 1234 | 0101010101010100110101010010 | name1 = value1 | 
+           |      | 0101100001010100110101010010 | name2 = value2 |
+           |      | 0101100001010100110101010010 | nameN = valueN |
+           \------+------------------------------+----------------/    
+
+.. note:: An object ID is unique across the entire cluster, not just the local
+   filesystem.
+
+
+.. index:: architecture; high availability, scalability
+
+Scalability and High Availability
+---------------------------------
+
+In traditional architectures, clients talk to a centralized component (e.g., a
+gateway, broker, API, facade, etc.), which acts as a single point of entry to a
+complex subsystem. This imposes a limit to both performance and scalability,
+while introducing a single point of failure (i.e., if the centralized component
+goes down, the whole system goes down, too).
+
+Ceph eliminates the centralized gateway to enable clients to interact with 
+Ceph OSD Daemons directly. Ceph OSD Daemons create object replicas on other
+Ceph Nodes to ensure data safety and high availability. Ceph also uses a cluster
+of monitors to ensure high availability. To eliminate centralization, Ceph 
+uses an algorithm called CRUSH.
+
+
+.. index:: CRUSH; architecture
+
+CRUSH Introduction
+~~~~~~~~~~~~~~~~~~
+
+Ceph Clients and Ceph OSD Daemons both use the :abbr:`CRUSH (Controlled
+Replication Under Scalable Hashing)` algorithm to efficiently compute
+information about object location, instead of having to depend on a
+central lookup table. CRUSH provides a better data management mechanism compared
+to older approaches, and enables massive scale by cleanly distributing the work
+to all the clients and OSD daemons in the cluster. CRUSH uses intelligent data
+replication to ensure resiliency, which is better suited to hyper-scale storage.
+The following sections provide additional details on how CRUSH works. For a
+detailed discussion of CRUSH, see `CRUSH - Controlled, Scalable, Decentralized
+Placement of Replicated Data`_.
+
+.. index:: architecture; cluster map
+
+.. _architecture_cluster_map:
+
+Cluster Map
+~~~~~~~~~~~
+
+Ceph depends upon Ceph Clients and Ceph OSD Daemons having knowledge of the
+cluster topology, which is inclusive of 5 maps collectively referred to as the
+"Cluster Map":
+
+#. **The Monitor Map:** Contains the cluster ``fsid``, the position, name 
+   address and port of each monitor. It also indicates the current epoch, 
+   when the map was created, and the last time it changed. To view a monitor
+   map, execute ``ceph mon dump``.   
+   
+#. **The OSD Map:** Contains the cluster ``fsid``, when the map was created and
+   last modified, a list of pools, replica sizes, PG numbers, a list of OSDs
+   and their status (e.g., ``up``, ``in``). To view an OSD map, execute
+   ``ceph osd dump``. 
+   
+#. **The PG Map:** Contains the PG version, its time stamp, the last OSD
+   map epoch, the full ratios, and details on each placement group such as
+   the PG ID, the `Up Set`, the `Acting Set`, the state of the PG (e.g., 
+   ``active + clean``), and data usage statistics for each pool.
+
+#. **The CRUSH Map:** Contains a list of storage devices, the failure domain
+   hierarchy (e.g., device, host, rack, row, room, etc.), and rules for 
+   traversing the hierarchy when storing data. To view a CRUSH map, execute
+   ``ceph osd getcrushmap -o {filename}``; then, decompile it by executing
+   ``crushtool -d {comp-crushmap-filename} -o {decomp-crushmap-filename}``.
+   You can view the decompiled map in a text editor or with ``cat``. 
+
+#. **The MDS Map:** Contains the current MDS map epoch, when the map was 
+   created, and the last time it changed. It also contains the pool for 
+   storing metadata, a list of metadata servers, and which metadata servers
+   are ``up`` and ``in``. To view an MDS map, execute ``ceph fs dump``.
+
+Each map maintains an iterative history of its operating state changes. Ceph
+Monitors maintain a master copy of the cluster map including the cluster
+members, state, changes, and the overall health of the Ceph Storage Cluster.
+
+.. index:: high availability; monitor architecture
+
+High Availability Monitors
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Before Ceph Clients can read or write data, they must contact a Ceph Monitor
+to obtain the most recent copy of the cluster map. A Ceph Storage Cluster
+can operate with a single monitor; however, this introduces a single 
+point of failure (i.e., if the monitor goes down, Ceph Clients cannot
+read or write data).
+
+For added reliability and fault tolerance, Ceph supports a cluster of monitors.
+In a cluster of monitors, latency and other faults can cause one or more
+monitors to fall behind the current state of the cluster. For this reason, Ceph
+must have agreement among various monitor instances regarding the state of the
+cluster. Ceph always uses a majority of monitors (e.g., 1, 2:3, 3:5, 4:6, etc.)
+and the `Paxos`_ algorithm to establish a consensus among the monitors about the
+current state of the cluster.
+
+For details on configuring monitors, see the `Monitor Config Reference`_.
+
+.. index:: architecture; high availability authentication
+
+High Availability Authentication
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To identify users and protect against man-in-the-middle attacks, Ceph provides
+its ``cephx`` authentication system to authenticate users and daemons.
+
+.. note:: The ``cephx`` protocol does not address data encryption in transport 
+   (e.g., SSL/TLS) or encryption at rest.
+
+Cephx uses shared secret keys for authentication, meaning both the client and
+the monitor cluster have a copy of the client's secret key. The authentication
+protocol is such that both parties are able to prove to each other they have a
+copy of the key without actually revealing it. This provides mutual
+authentication, which means the cluster is sure the user possesses the secret
+key, and the user is sure that the cluster has a copy of the secret key.
+
+A key scalability feature of Ceph is to avoid a centralized interface to the
+Ceph object store, which means that Ceph clients must be able to interact with
+OSDs directly. To protect data, Ceph provides its ``cephx`` authentication
+system, which authenticates users operating Ceph clients. The ``cephx`` protocol
+operates in a manner with behavior similar to `Kerberos`_. 
+
+A user/actor invokes a Ceph client to contact a monitor. Unlike Kerberos, each
+monitor can authenticate users and distribute keys, so there is no single point
+of failure or bottleneck when using ``cephx``. The monitor returns an
+authentication data structure similar to a Kerberos ticket that contains a
+session key for use in obtaining Ceph services.  This session key is itself
+encrypted with the user's permanent  secret key, so that only the user can
+request services from the Ceph Monitor(s). The client then uses the session key
+to request its desired services from the monitor, and the monitor provides the
+client with a ticket that will authenticate the client to the OSDs that actually
+handle data. Ceph Monitors and OSDs share a secret, so the client can use the
+ticket provided by the monitor with any OSD or metadata server in the cluster.
+Like Kerberos, ``cephx`` tickets expire, so an attacker cannot use an expired
+ticket or session key obtained surreptitiously. This form of authentication will
+prevent attackers with access to the communications medium from either creating
+bogus messages under another user's identity or altering another user's
+legitimate messages, as long as the user's secret key is not divulged before it
+expires.
+
+To use ``cephx``, an administrator must set up users first. In the following
+diagram, the ``client.admin`` user invokes  ``ceph auth get-or-create-key`` from
+the command line to generate a username and secret key. Ceph's ``auth``
+subsystem generates the username and key, stores a copy with the monitor(s) and
+transmits the user's secret back to the ``client.admin`` user. This means that 
+the client and the monitor share a secret key.
+
+.. note:: The ``client.admin`` user must provide the user ID and 
+   secret key to the user in a secure manner. 
+
+.. ditaa::
+
+           +---------+     +---------+
+           | Client  |     | Monitor |
+           +---------+     +---------+
+                |  request to   |
+                | create a user |
+                |-------------->|----------+ create user
+                |               |          | and                 
+                |<--------------|<---------+ store key
+                | transmit key  |
+                |               |
+
+
+To authenticate with the monitor, the client passes in the user name to the
+monitor, and the monitor generates a session key and encrypts it with the secret
+key associated to the user name. Then, the monitor transmits the encrypted
+ticket back to the client. The client then decrypts the payload with the shared
+secret key to retrieve the session key. The session key identifies the user for
+the current session. The client then requests a ticket on behalf of the user
+signed by the session key. The monitor generates a ticket, encrypts it with the
+user's secret key and transmits it back to the client. The client decrypts the
+ticket and uses it to sign requests to OSDs and metadata servers throughout the
+cluster.
+
+.. ditaa::
+
+           +---------+     +---------+
+           | Client  |     | Monitor |
+           +---------+     +---------+
+                |  authenticate |
+                |-------------->|----------+ generate and
+                |               |          | encrypt                
+                |<--------------|<---------+ session key
+                | transmit      |
+                | encrypted     |
+                | session key   |
+                |               |             
+                |-----+ decrypt |
+                |     | session | 
+                |<----+ key     |              
+                |               |
+                |  req. ticket  |
+                |-------------->|----------+ generate and
+                |               |          | encrypt                
+                |<--------------|<---------+ ticket
+                | recv. ticket  |
+                |               |             
+                |-----+ decrypt |
+                |     | ticket  | 
+                |<----+         |              
+
+
+The ``cephx`` protocol authenticates ongoing communications between the client
+machine and the Ceph servers. Each message sent between a client and server,
+subsequent to the initial authentication, is signed using a ticket that the
+monitors, OSDs and metadata servers can verify with their shared secret.
+
+.. ditaa::
+
+           +---------+     +---------+     +-------+     +-------+
+           |  Client |     | Monitor |     |  MDS  |     |  OSD  |
+           +---------+     +---------+     +-------+     +-------+
+                |  request to   |              |             |
+                | create a user |              |             |               
+                |-------------->| mon and      |             |
+                |<--------------| client share |             |
+                |    receive    | a secret.    |             |
+                | shared secret |              |             |
+                |               |<------------>|             |
+                |               |<-------------+------------>|
+                |               | mon, mds,    |             |
+                | authenticate  | and osd      |             |  
+                |-------------->| share        |             |
+                |<--------------| a secret     |             |
+                |  session key  |              |             |
+                |               |              |             |
+                |  req. ticket  |              |             |
+                |-------------->|              |             |
+                |<--------------|              |             |
+                | recv. ticket  |              |             |
+                |               |              |             |
+                |   make request (CephFS only) |             |
+                |----------------------------->|             |
+                |<-----------------------------|             |
+                | receive response (CephFS only)             |
+                |                                            |
+                |                make request                |
+                |------------------------------------------->|  
+                |<-------------------------------------------|
+                               receive response
+
+The protection offered by this authentication is between the Ceph client and the
+Ceph server hosts. The authentication is not extended beyond the Ceph client. If
+the user accesses the Ceph client from a remote host, Ceph authentication is not
+applied to the connection between the user's host and the client host.
+
+
+For configuration details, see `Cephx Config Guide`_. For user management 
+details, see `User Management`_.
+
+
+.. index:: architecture; smart daemons and scalability
+
+Smart Daemons Enable Hyperscale
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In many clustered architectures, the primary purpose of cluster membership is 
+so that a centralized interface knows which nodes it can access. Then the
+centralized interface provides services to the client through a double
+dispatch--which is a **huge** bottleneck at the petabyte-to-exabyte scale.
+
+Ceph eliminates the bottleneck: Ceph's OSD Daemons AND Ceph Clients are cluster
+aware. Like Ceph clients, each Ceph OSD Daemon knows about other Ceph OSD
+Daemons in the cluster.  This enables Ceph OSD Daemons to interact directly with
+other Ceph OSD Daemons and Ceph Monitors. Additionally, it enables Ceph Clients
+to interact directly with Ceph OSD Daemons.
+
+The ability of Ceph Clients, Ceph Monitors and Ceph OSD Daemons to interact with
+each other means that Ceph OSD Daemons can utilize the CPU and RAM of the Ceph
+nodes to easily perform tasks that would bog down a centralized server. The
+ability to leverage this computing power leads to several major benefits:
+
+#. **OSDs Service Clients Directly:** Since any network device has a limit to 
+   the number of concurrent connections it can support, a centralized system 
+   has a low physical limit at high scales. By enabling Ceph Clients to contact 
+   Ceph OSD Daemons directly, Ceph increases both performance and total system 
+   capacity simultaneously, while removing a single point of failure. Ceph 
+   Clients can maintain a session when they need to, and with a particular Ceph 
+   OSD Daemon instead of a centralized server.
+
+#. **OSD Membership and Status**: Ceph OSD Daemons join a cluster and report 
+   on their status. At the lowest level, the Ceph OSD Daemon status is ``up`` 
+   or ``down`` reflecting whether or not it is running and able to service 
+   Ceph Client requests. If a Ceph OSD Daemon is ``down`` and ``in`` the Ceph 
+   Storage Cluster, this status may indicate the failure of the Ceph OSD 
+   Daemon. If a Ceph OSD Daemon is not running (e.g., it crashes), the Ceph OSD 
+   Daemon cannot notify the Ceph Monitor that it is ``down``. The OSDs
+   periodically send messages to the Ceph Monitor (``MPGStats`` pre-luminous,
+   and a new ``MOSDBeacon`` in luminous).  If the Ceph Monitor doesn't see that
+   message after a configurable period of time then it marks the OSD down.
+   This mechanism is a failsafe, however. Normally, Ceph OSD Daemons will
+   determine if a neighboring OSD is down and report it to the Ceph Monitor(s).
+   This assures that Ceph Monitors are lightweight processes.  See `Monitoring
+   OSDs`_ and `Heartbeats`_ for additional details.
+
+#. **Data Scrubbing:** As part of maintaining data consistency and cleanliness, 
+   Ceph OSD Daemons can scrub objects. That is, Ceph OSD Daemons can compare
+   their local objects metadata with its replicas stored on other OSDs. Scrubbing
+   happens on a per-Placement Group base. Scrubbing (usually performed daily)
+   catches mismatches in size and other metadata. Ceph OSD Daemons also perform deeper
+   scrubbing by comparing data in objects bit-for-bit with their checksums.
+   Deep scrubbing (usually performed weekly) finds bad sectors on a drive that
+   weren't apparent in a light scrub. See `Data Scrubbing`_ for details on
+   configuring scrubbing.
+
+#. **Replication:** Like Ceph Clients, Ceph OSD Daemons use the CRUSH 
+   algorithm, but the Ceph OSD Daemon uses it to compute where replicas of 
+   objects should be stored (and for rebalancing). In a typical write scenario, 
+   a client uses the CRUSH algorithm to compute where to store an object, maps 
+   the object to a pool and placement group, then looks at the CRUSH map to 
+   identify the primary OSD for the placement group.
+   
+   The client writes the object to the identified placement group in the 
+   primary OSD. Then, the primary OSD with its own copy of the CRUSH map 
+   identifies the secondary and tertiary OSDs for replication purposes, and 
+   replicates the object to the appropriate placement groups in the secondary 
+   and tertiary OSDs (as many OSDs as additional replicas), and responds to the
+   client once it has confirmed the object was stored successfully.
+
+.. ditaa::
+
+             +----------+
+             |  Client  |
+             |          |
+             +----------+
+                 *  ^
+      Write (1)  |  |  Ack (6)
+                 |  |
+                 v  *
+            +-------------+
+            | Primary OSD |
+            |             |
+            +-------------+
+              *  ^   ^  *
+    Write (2) |  |   |  |  Write (3)
+       +------+  |   |  +------+
+       |  +------+   +------+  |
+       |  | Ack (4)  Ack (5)|  | 
+       v  *                 *  v
+ +---------------+   +---------------+
+ | Secondary OSD |   | Tertiary OSD  |
+ |               |   |               |
+ +---------------+   +---------------+
+
+With the ability to perform data replication, Ceph OSD Daemons relieve Ceph
+clients from that duty, while ensuring high data availability and data safety.
+
+
+Dynamic Cluster Management
+--------------------------
+
+In the `Scalability and High Availability`_ section, we explained how Ceph uses
+CRUSH, cluster awareness and intelligent daemons to scale and maintain high
+availability. Key to Ceph's design is the autonomous, self-healing, and
+intelligent Ceph OSD Daemon. Let's take a deeper look at how CRUSH works to
+enable modern cloud storage infrastructures to place data, rebalance the cluster
+and recover from faults dynamically.
+
+.. index:: architecture; pools
+
+About Pools
+~~~~~~~~~~~
+
+The Ceph storage system supports the notion of 'Pools', which are logical
+partitions for storing objects.
+
+Ceph Clients retrieve a `Cluster Map`_ from a Ceph Monitor, and write objects to
+pools. The pool's ``size`` or number of replicas, the CRUSH rule and the
+number of placement groups determine how Ceph will place the data.
+
+.. ditaa::
+
+            +--------+  Retrieves  +---------------+
+            | Client |------------>|  Cluster Map  |
+            +--------+             +---------------+
+                 |
+                 v      Writes
+              /-----\
+              | obj |
+              \-----/
+                 |      To
+                 v
+            +--------+           +---------------+
+            |  Pool  |---------->|  CRUSH Rule   |
+            +--------+  Selects  +---------------+
+                 
+
+Pools set at least the following parameters:
+
+- Ownership/Access to Objects
+- The Number of Placement Groups, and 
+- The CRUSH Rule to Use.
+
+See `Set Pool Values`_ for details.
+
+
+.. index: architecture; placement group mapping
+
+Mapping PGs to OSDs
+~~~~~~~~~~~~~~~~~~~
+
+Each pool has a number of placement groups. CRUSH maps PGs to OSDs dynamically.
+When a Ceph Client stores objects, CRUSH will map each object to a placement
+group.
+
+Mapping objects to placement groups creates a layer of indirection between the
+Ceph OSD Daemon and the Ceph Client. The Ceph Storage Cluster must be able to
+grow (or shrink) and rebalance where it stores objects dynamically. If the Ceph
+Client "knew" which Ceph OSD Daemon had which object, that would create a tight
+coupling between the Ceph Client and the Ceph OSD Daemon. Instead, the CRUSH
+algorithm maps each object to a placement group and then maps each placement
+group to one or more Ceph OSD Daemons. This layer of indirection allows Ceph to
+rebalance dynamically when new Ceph OSD Daemons and the underlying OSD devices
+come online. The following diagram depicts how CRUSH maps objects to placement
+groups, and placement groups to OSDs.
+
+.. ditaa::
+
+           /-----\  /-----\  /-----\  /-----\  /-----\
+           | obj |  | obj |  | obj |  | obj |  | obj |
+           \-----/  \-----/  \-----/  \-----/  \-----/
+              |        |        |        |        |
+              +--------+--------+        +---+----+
+              |                              |
+              v                              v
+   +-----------------------+      +-----------------------+
+   |  Placement Group #1   |      |  Placement Group #2   |
+   |                       |      |                       |
+   +-----------------------+      +-----------------------+
+               |                              |
+               |      +-----------------------+---+
+        +------+------+-------------+             |
+        |             |             |             |
+        v             v             v             v
+   /----------\  /----------\  /----------\  /----------\ 
+   |          |  |          |  |          |  |          |
+   |  OSD #1  |  |  OSD #2  |  |  OSD #3  |  |  OSD #4  |
+   |          |  |          |  |          |  |          |
+   \----------/  \----------/  \----------/  \----------/  
+
+With a copy of the cluster map and the CRUSH algorithm, the client can compute
+exactly which OSD to use when reading or writing a particular object.
+
+.. index:: architecture; calculating PG IDs
+
+Calculating PG IDs
+~~~~~~~~~~~~~~~~~~
+
+When a Ceph Client binds to a Ceph Monitor, it retrieves the latest copy of the
+`Cluster Map`_. With the cluster map, the client knows about all of the monitors,
+OSDs, and metadata servers in the cluster. **However, it doesn't know anything
+about object locations.** 
+
+.. epigraph:: 
+
+	Object locations get computed.
+
+
+The only input required by the client is the object ID and the pool.
+It's simple: Ceph stores data in named pools (e.g., "liverpool"). When a client
+wants to store a named object (e.g., "john," "paul," "george," "ringo", etc.)
+it calculates a placement group using the object name, a hash code, the
+number of PGs in the pool and the pool name. Ceph clients use the following
+steps to compute PG IDs.
+
+#. The client inputs the pool name and the object ID. (e.g., pool = "liverpool" 
+   and object-id = "john")
+#. Ceph takes the object ID and hashes it.
+#. Ceph calculates the hash modulo the number of PGs. (e.g., ``58``) to get 
+   a PG ID.
+#. Ceph gets the pool ID given the pool name (e.g., "liverpool" = ``4``)
+#. Ceph prepends the pool ID to the PG ID (e.g., ``4.58``).
+
+Computing object locations is much faster than performing object location query
+over a chatty session. The :abbr:`CRUSH (Controlled Replication Under Scalable
+Hashing)` algorithm allows a client to compute where objects *should* be stored,
+and enables the client to contact the primary OSD to store or retrieve the
+objects.
+
+.. index:: architecture; PG Peering
+
+Peering and Sets
+~~~~~~~~~~~~~~~~
+
+In previous sections, we noted that Ceph OSD Daemons check each other's
+heartbeats and report back to the Ceph Monitor. Another thing Ceph OSD daemons
+do is called 'peering', which is the process of bringing all of the OSDs that
+store a Placement Group (PG) into agreement about the state of all of the
+objects (and their metadata) in that PG. In fact, Ceph OSD Daemons `Report
+Peering Failure`_ to the Ceph Monitors. Peering issues  usually resolve
+themselves; however, if the problem persists, you may need to refer to the
+`Troubleshooting Peering Failure`_ section.
+
+.. Note:: Agreeing on the state does not mean that the PGs have the latest contents.
+
+The Ceph Storage Cluster was designed to store at least two copies of an object
+(i.e., ``size = 2``), which is the minimum requirement for data safety. For high
+availability, a Ceph Storage Cluster should store more than two copies of an object
+(e.g., ``size = 3`` and ``min size = 2``) so that it can continue to run in a 
+``degraded`` state while maintaining data safety.
+
+Referring back to the diagram in `Smart Daemons Enable Hyperscale`_, we do not 
+name the Ceph OSD Daemons specifically (e.g., ``osd.0``, ``osd.1``, etc.), but 
+rather refer to them as *Primary*, *Secondary*, and so forth. By convention, 
+the *Primary* is the first OSD in the *Acting Set*, and is responsible for 
+coordinating the peering process for each placement group where it acts as 
+the *Primary*, and is the **ONLY** OSD that that will accept client-initiated 
+writes to objects for a given placement group where it acts as the *Primary*.
+
+When a series of OSDs are responsible for a placement group, that series of
+OSDs, we refer to them as an *Acting Set*. An *Acting Set* may refer to the Ceph
+OSD Daemons that are currently responsible for the placement group, or the Ceph
+OSD Daemons that were responsible  for a particular placement group as of some
+epoch.
+
+The Ceph OSD daemons that are part of an *Acting Set* may not always be  ``up``.
+When an OSD in the *Acting Set* is ``up``, it is part of the  *Up Set*. The *Up
+Set* is an important distinction, because Ceph can remap PGs to other Ceph OSD
+Daemons when an OSD fails. 
+
+.. note:: In an *Acting Set* for a PG containing ``osd.25``, ``osd.32`` and 
+   ``osd.61``, the first OSD, ``osd.25``, is the *Primary*. If that OSD fails,
+   the Secondary, ``osd.32``, becomes the *Primary*, and ``osd.25`` will be 
+   removed from the *Up Set*.
+
+
+.. index:: architecture; Rebalancing
+
+Rebalancing
+~~~~~~~~~~~
+
+When you add a Ceph OSD Daemon to a Ceph Storage Cluster, the cluster map gets
+updated with the new OSD. Referring back to `Calculating PG IDs`_, this changes
+the cluster map. Consequently, it changes object placement, because it changes
+an input for the calculations. The following diagram depicts the rebalancing
+process (albeit rather crudely, since it is substantially less impactful with
+large clusters) where some, but not all of the PGs migrate from existing OSDs
+(OSD 1, and OSD 2) to the new OSD (OSD 3). Even when rebalancing, CRUSH is
+stable. Many of the placement groups remain in their original configuration,
+and each OSD gets some added capacity, so there are no load spikes on the 
+new OSD after rebalancing is complete.
+
+
+.. ditaa::
+
+           +--------+     +--------+
+   Before  |  OSD 1 |     |  OSD 2 |
+           +--------+     +--------+
+           |  PG #1 |     | PG #6  |
+           |  PG #2 |     | PG #7  |
+           |  PG #3 |     | PG #8  |
+           |  PG #4 |     | PG #9  |
+           |  PG #5 |     | PG #10 |
+           +--------+     +--------+
+
+           +--------+     +--------+     +--------+
+    After  |  OSD 1 |     |  OSD 2 |     |  OSD 3 |
+           +--------+     +--------+     +--------+
+           |  PG #1 |     | PG #7  |     |  PG #3 |
+           |  PG #2 |     | PG #8  |     |  PG #6 |
+           |  PG #4 |     | PG #10 |     |  PG #9 |
+           |  PG #5 |     |        |     |        |
+           |        |     |        |     |        |
+           +--------+     +--------+     +--------+
+
+
+.. index:: architecture; Data Scrubbing
+
+Data Consistency
+~~~~~~~~~~~~~~~~
+
+As part of maintaining data consistency and cleanliness, Ceph OSDs also scrub
+objects within placement groups. That is, Ceph OSDs compare object metadata in
+one placement group with its replicas in placement groups stored in other
+OSDs. Scrubbing (usually performed daily) catches OSD bugs or filesystem
+errors, often as a result of hardware issues.  OSDs also perform deeper
+scrubbing by comparing data in objects bit-for-bit.  Deep scrubbing (by default
+performed weekly) finds bad blocks on a drive that weren't apparent in a light
+scrub.
+
+See `Data Scrubbing`_ for details on configuring scrubbing.
+
+
+
+
+
+.. index:: erasure coding
+
+Erasure Coding
+--------------
+
+An erasure coded pool stores each object as ``K+M`` chunks. It is divided into
+``K`` data chunks and ``M`` coding chunks. The pool is configured to have a size
+of ``K+M`` so that each chunk is stored in an OSD in the acting set. The rank of
+the chunk is stored as an attribute of the object.
+
+For instance an erasure coded pool can be created to use five OSDs (``K+M = 5``) and
+sustain the loss of two of them (``M = 2``).
+
+Reading and Writing Encoded Chunks
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When the object **NYAN** containing ``ABCDEFGHI`` is written to the pool, the erasure
+encoding function splits the content into three data chunks simply by dividing
+the content in three: the first contains ``ABC``, the second ``DEF`` and the
+last ``GHI``. The content will be padded if the content length is not a multiple
+of ``K``. The function also creates two coding chunks: the fourth with ``YXY``
+and the fifth with ``QGC``. Each chunk is stored in an OSD in the acting set.
+The chunks are stored in objects that have the same name (**NYAN**) but reside
+on different OSDs. The order in which the chunks were created must be preserved
+and is stored as an attribute of the object (``shard_t``), in addition to its
+name. Chunk 1 contains ``ABC`` and is stored on **OSD5** while chunk 4 contains
+``YXY`` and is stored on **OSD3**.
+
+
+.. ditaa::
+
+                            +-------------------+
+                       name |       NYAN        |
+                            +-------------------+
+                    content |     ABCDEFGHI     |
+                            +--------+----------+
+                                     |
+                                     |
+                                     v
+                              +------+------+
+              +---------------+ encode(3,2) +-----------+
+              |               +--+--+---+---+           |
+              |                  |  |   |               |
+              |          +-------+  |   +-----+         |
+              |          |          |         |         |
+           +--v---+   +--v---+   +--v---+  +--v---+  +--v---+
+     name  | NYAN |   | NYAN |   | NYAN |  | NYAN |  | NYAN |
+           +------+   +------+   +------+  +------+  +------+
+    shard  |  1   |   |  2   |   |  3   |  |  4   |  |  5   |
+           +------+   +------+   +------+  +------+  +------+
+  content  | ABC  |   | DEF  |   | GHI  |  | YXY  |  | QGC  |
+           +--+---+   +--+---+   +--+---+  +--+---+  +--+---+
+              |          |          |         |         |
+              |          |          v         |         |
+              |          |       +--+---+     |         |
+              |          |       | OSD1 |     |         |
+              |          |       +------+     |         |
+              |          |                    |         |
+              |          |       +------+     |         |
+              |          +------>| OSD2 |     |         |
+              |                  +------+     |         |
+              |                               |         |
+              |                  +------+     |         |
+              |                  | OSD3 |<----+         |
+              |                  +------+               |
+              |                                         |
+              |                  +------+               |
+              |                  | OSD4 |<--------------+
+              |                  +------+
+              |
+              |                  +------+
+              +----------------->| OSD5 |
+                                 +------+
+
+
+When the object **NYAN** is read from the erasure coded pool, the decoding
+function reads three chunks: chunk 1 containing ``ABC``, chunk 3 containing
+``GHI`` and chunk 4 containing ``YXY``. Then, it rebuilds the original content
+of the object ``ABCDEFGHI``. The decoding function is informed that the chunks 2
+and 5 are missing (they are called 'erasures'). The chunk 5 could not be read
+because the **OSD4** is out. The decoding function can be called as soon as
+three chunks are read: **OSD2** was the slowest and its chunk was not taken into
+account.
+
+.. ditaa::
+
+	                         +-------------------+
+	                    name |       NYAN        |
+	                         +-------------------+
+	                 content |     ABCDEFGHI     |
+	                         +---------+---------+
+	                                   ^
+	                                   |
+	                                   |
+	                           +-------+-------+
+	                           |  decode(3,2)  |
+	            +------------->+  erasures 2,5 +<-+
+	            |              |               |  |
+	            |              +-------+-------+  |
+	            |                      ^          |
+	            |                      |          | 
+	            |                      |          |
+	         +--+---+   +------+   +---+--+   +---+--+
+	   name  | NYAN |   | NYAN |   | NYAN |   | NYAN |
+	         +------+   +------+   +------+   +------+
+	  shard  |  1   |   |  2   |   |  3   |   |  4   |
+	         +------+   +------+   +------+   +------+
+	content  | ABC  |   | DEF  |   | GHI  |   | YXY  |
+	         +--+---+   +--+---+   +--+---+   +--+---+
+	            ^          .          ^          ^
+	            |    TOO   .          |          |
+	            |    SLOW  .       +--+---+      |
+	            |          ^       | OSD1 |      |
+	            |          |       +------+      |
+	            |          |                     |
+	            |          |       +------+      |
+	            |          +-------| OSD2 |      |
+	            |                  +------+      |
+	            |                                |
+	            |                  +------+      |
+	            |                  | OSD3 |------+
+	            |                  +------+
+	            |
+	            |                  +------+
+	            |                  | OSD4 | OUT
+	            |                  +------+
+	            |
+	            |                  +------+
+	            +------------------| OSD5 |
+	                               +------+
+
+
+Interrupted Full Writes
+~~~~~~~~~~~~~~~~~~~~~~~
+
+In an erasure coded pool, the primary OSD in the up set receives all write
+operations. It is responsible for encoding the payload into ``K+M`` chunks and
+sends them to the other OSDs. It is also responsible for maintaining an
+authoritative version of the placement group logs.
+
+In the following diagram, an erasure coded placement group has been created with
+``K = 2, M = 1`` and is supported by three OSDs, two for ``K`` and one for
+``M``. The acting set of the placement group is made of **OSD 1**, **OSD 2** and
+**OSD 3**. An object has been encoded and stored in the OSDs : the chunk
+``D1v1`` (i.e. Data chunk number 1, version 1) is on **OSD 1**, ``D2v1`` on
+**OSD 2** and ``C1v1`` (i.e. Coding chunk number 1, version 1) on **OSD 3**. The
+placement group logs on each OSD are identical (i.e. ``1,1`` for epoch 1,
+version 1).
+
+
+.. ditaa::
+
+     Primary OSD
+    
+   +-------------+
+   |    OSD 1    |             +-------------+
+   |         log |  Write Full |             |
+   |  +----+     |<------------+ Ceph Client |
+   |  |D1v1| 1,1 |      v1     |             |
+   |  +----+     |             +-------------+
+   +------+------+
+          |
+          |
+          |          +-------------+
+          |          |    OSD 2    |
+          |          |         log |
+          +--------->+  +----+     |
+          |          |  |D2v1| 1,1 |
+          |          |  +----+     |
+          |          +-------------+
+          |
+          |          +-------------+
+          |          |    OSD 3    |
+          |          |         log |
+          +--------->|  +----+     |
+                     |  |C1v1| 1,1 |
+                     |  +----+     |
+                     +-------------+
+
+**OSD 1** is the primary and receives a **WRITE FULL** from a client, which
+means the payload is to replace the object entirely instead of overwriting a
+portion of it. Version 2 (v2) of the object is created to override version 1
+(v1). **OSD 1** encodes the payload into three chunks: ``D1v2`` (i.e. Data
+chunk number 1 version 2) will be on **OSD 1**, ``D2v2`` on **OSD 2** and
+``C1v2`` (i.e. Coding chunk number 1 version 2) on **OSD 3**. Each chunk is sent
+to the target OSD, including the primary OSD which is responsible for storing
+chunks in addition to handling write operations and maintaining an authoritative
+version of the placement group logs. When an OSD receives the message
+instructing it to write the chunk, it also creates a new entry in the placement
+group logs to reflect the change. For instance, as soon as **OSD 3** stores
+``C1v2``, it adds the entry ``1,2`` ( i.e. epoch 1, version 2 ) to its logs.
+Because the OSDs work asynchronously, some chunks may still be in flight ( such
+as ``D2v2`` ) while others are acknowledged and persisted to storage drives
+(such as ``C1v1`` and ``D1v1``).
+
+.. ditaa::
+
+     Primary OSD
+    
+   +-------------+
+   |    OSD 1    |
+   |         log |
+   |  +----+     |             +-------------+
+   |  |D1v2| 1,2 |  Write Full |             |
+   |  +----+     +<------------+ Ceph Client |
+   |             |      v2     |             |
+   |  +----+     |             +-------------+
+   |  |D1v1| 1,1 |           
+   |  +----+     |           
+   +------+------+           
+          |                  
+          |                  
+          |           +------+------+
+          |           |    OSD 2    |
+          |  +------+ |         log |
+          +->| D2v2 | |  +----+     |
+          |  +------+ |  |D2v1| 1,1 |
+          |           |  +----+     |
+          |           +-------------+
+          |
+          |           +-------------+
+          |           |    OSD 3    |
+          |           |         log |
+          |           |  +----+     |
+          |           |  |C1v2| 1,2 |
+          +---------->+  +----+     |
+                      |             |
+                      |  +----+     |
+                      |  |C1v1| 1,1 |
+                      |  +----+     |
+                      +-------------+
+
+
+If all goes well, the chunks are acknowledged on each OSD in the acting set and
+the logs' ``last_complete`` pointer can move from ``1,1`` to ``1,2``.
+
+.. ditaa::
+
+     Primary OSD
+    
+   +-------------+
+   |    OSD 1    |
+   |         log |
+   |  +----+     |             +-------------+
+   |  |D1v2| 1,2 |  Write Full |             |
+   |  +----+     +<------------+ Ceph Client |
+   |             |      v2     |             |
+   |  +----+     |             +-------------+
+   |  |D1v1| 1,1 |           
+   |  +----+     |           
+   +------+------+           
+          |                  
+          |           +-------------+
+          |           |    OSD 2    |
+          |           |         log |
+          |           |  +----+     |
+          |           |  |D2v2| 1,2 |
+          +---------->+  +----+     |
+          |           |             |
+          |           |  +----+     |
+          |           |  |D2v1| 1,1 |
+          |           |  +----+     |
+          |           +-------------+
+          |                  
+          |           +-------------+
+          |           |    OSD 3    |
+          |           |         log |
+          |           |  +----+     |
+          |           |  |C1v2| 1,2 |
+          +---------->+  +----+     |
+                      |             |
+                      |  +----+     |
+                      |  |C1v1| 1,1 |
+                      |  +----+     |
+                      +-------------+
+
+
+Finally, the files used to store the chunks of the previous version of the
+object can be removed: ``D1v1`` on **OSD 1**, ``D2v1`` on **OSD 2** and ``C1v1``
+on **OSD 3**.
+
+.. ditaa::
+
+     Primary OSD
+    
+   +-------------+
+   |    OSD 1    |
+   |         log |
+   |  +----+     |
+   |  |D1v2| 1,2 |
+   |  +----+     |
+   +------+------+
+          |
+          |
+          |          +-------------+
+          |          |    OSD 2    |
+          |          |         log |
+          +--------->+  +----+     |
+          |          |  |D2v2| 1,2 |
+          |          |  +----+     |
+          |          +-------------+
+          |
+          |          +-------------+
+          |          |    OSD 3    |
+          |          |         log |
+          +--------->|  +----+     |
+                     |  |C1v2| 1,2 |
+                     |  +----+     |
+                     +-------------+
+
+
+But accidents happen. If **OSD 1** goes down while ``D2v2`` is still in flight,
+the object's version 2 is partially written: **OSD 3** has one chunk but that is
+not enough to recover. It lost two chunks: ``D1v2`` and ``D2v2`` and the
+erasure coding parameters ``K = 2``, ``M = 1`` require that at least two chunks are
+available to rebuild the third. **OSD 4** becomes the new primary and finds that
+the ``last_complete`` log entry (i.e., all objects before this entry were known
+to be available on all OSDs in the previous acting set ) is ``1,1`` and that
+will be the head of the new authoritative log.
+
+.. ditaa::
+
+   +-------------+
+   |    OSD 1    |
+   |   (down)    |
+   | c333        |
+   +------+------+
+          |                  
+          |           +-------------+
+          |           |    OSD 2    |
+          |           |         log |
+          |           |  +----+     |
+          +---------->+  |D2v1| 1,1 |
+          |           |  +----+     |
+          |           |             |
+          |           +-------------+
+          |                  
+          |           +-------------+
+          |           |    OSD 3    |
+          |           |         log |
+          |           |  +----+     |
+          |           |  |C1v2| 1,2 |
+          +---------->+  +----+     |
+                      |             |
+                      |  +----+     |
+                      |  |C1v1| 1,1 |
+                      |  +----+     |
+                      +-------------+
+     Primary OSD
+   +-------------+
+   |    OSD 4    |
+   |         log |
+   |             |
+   |         1,1 |
+   |             |
+   +------+------+
+          
+
+
+The log entry 1,2 found on **OSD 3** is divergent from the new authoritative log
+provided by **OSD 4**: it is discarded and the file containing the ``C1v2``
+chunk is removed. The ``D1v1`` chunk is rebuilt with the ``decode`` function of
+the erasure coding library during scrubbing and stored on the new primary 
+**OSD 4**.
+
+
+.. ditaa::
+
+     Primary OSD
+    
+   +-------------+
+   |    OSD 4    |
+   |         log |
+   |  +----+     |
+   |  |D1v1| 1,1 |
+   |  +----+     |
+   +------+------+
+          ^
+          |
+          |          +-------------+
+          |          |    OSD 2    |
+          |          |         log |
+          +----------+  +----+     |
+          |          |  |D2v1| 1,1 |
+          |          |  +----+     |
+          |          +-------------+
+          |
+          |          +-------------+
+          |          |    OSD 3    |
+          |          |         log |
+          +----------|  +----+     |
+                     |  |C1v1| 1,1 |
+                     |  +----+     |
+                     +-------------+
+
+   +-------------+
+   |    OSD 1    |
+   |   (down)    |
+   | c333        |
+   +-------------+
+
+See `Erasure Code Notes`_ for additional details.
+
+
+
+Cache Tiering
+-------------
+
+A cache tier provides Ceph Clients with better I/O performance for a subset of
+the data stored in a backing storage tier. Cache tiering involves creating a
+pool of relatively fast/expensive storage devices (e.g., solid state drives)
+configured to act as a cache tier, and a backing pool of either erasure-coded
+or relatively slower/cheaper devices configured to act as an economical storage
+tier. The Ceph objecter handles where to place the objects and the tiering
+agent determines when to flush objects from the cache to the backing storage
+tier. So the cache tier and the backing storage tier are completely transparent 
+to Ceph clients.
+
+
+.. ditaa::
+
+           +-------------+
+           | Ceph Client |
+           +------+------+
+                  ^
+     Tiering is   |  
+    Transparent   |              Faster I/O
+        to Ceph   |           +---------------+
+     Client Ops   |           |               |   
+                  |    +----->+   Cache Tier  |
+                  |    |      |               |
+                  |    |      +-----+---+-----+
+                  |    |            |   ^ 
+                  v    v            |   |   Active Data in Cache Tier
+           +------+----+--+         |   |
+           |   Objecter   |         |   |
+           +-----------+--+         |   |
+                       ^            |   |   Inactive Data in Storage Tier
+                       |            v   |
+                       |      +-----+---+-----+
+                       |      |               |
+                       +----->|  Storage Tier |
+                              |               |
+                              +---------------+
+                                 Slower I/O
+
+See `Cache Tiering`_ for additional details.  Note that Cache Tiers can be
+tricky and their use is now discouraged.
+
+
+.. index:: Extensibility, Ceph Classes
+
+Extending Ceph
+--------------
+
+You can extend Ceph by creating shared object classes called 'Ceph Classes'.
+Ceph loads ``.so`` classes stored in the ``osd class dir`` directory dynamically
+(i.e., ``$libdir/rados-classes`` by default). When you implement a class, you
+can create new object methods that have the ability to call the native methods
+in the Ceph Object Store, or other class methods you incorporate via libraries
+or create yourself.
+
+On writes, Ceph Classes can call native or class methods, perform any series of
+operations on the inbound data and generate a resulting write transaction  that
+Ceph will apply atomically.
+
+On reads, Ceph Classes can call native or class methods, perform any series of
+operations on the outbound data and return the data to the client.
+
+.. topic:: Ceph Class Example
+
+   A Ceph class for a content management system that presents pictures of a
+   particular size and aspect ratio could take an inbound bitmap image, crop it
+   to a particular aspect ratio, resize it and embed an invisible copyright or 
+   watermark to help protect the intellectual property; then, save the 
+   resulting bitmap image to the object store.
+
+See ``src/objclass/objclass.h``, ``src/fooclass.cc`` and ``src/barclass`` for 
+exemplary implementations.
+
+
+Summary
+-------
+
+Ceph Storage Clusters are dynamic--like a living organism. Whereas, many storage
+appliances do not fully utilize the CPU and RAM of a typical commodity server,
+Ceph does. From heartbeats, to  peering, to rebalancing the cluster or
+recovering from faults,  Ceph offloads work from clients (and from a centralized
+gateway which doesn't exist in the Ceph architecture) and uses the computing
+power of the OSDs to perform the work. When referring to `Hardware
+Recommendations`_ and the `Network Config Reference`_,  be cognizant of the
+foregoing concepts to understand how Ceph utilizes computing resources.
+
+.. index:: Ceph Protocol, librados
+
+Ceph Protocol
+=============
+
+Ceph Clients use the native protocol for interacting with the Ceph Storage
+Cluster. Ceph packages this functionality into the ``librados`` library so that
+you can create your own custom Ceph Clients. The following diagram depicts the
+basic architecture.
+
+.. ditaa::
+
+            +---------------------------------+
+            |  Ceph Storage Cluster Protocol  |
+            |           (librados)            |
+            +---------------------------------+
+            +---------------+ +---------------+
+            |      OSDs     | |    Monitors   |
+            +---------------+ +---------------+
+
+
+Native Protocol and ``librados``
+--------------------------------
+
+Modern applications need a simple object storage interface with asynchronous
+communication capability. The Ceph Storage Cluster provides a simple object
+storage interface with asynchronous communication capability. The interface
+provides direct, parallel access to objects throughout the cluster.
+
+
+- Pool Operations
+- Snapshots and Copy-on-write Cloning
+- Read/Write Objects
+  - Create or Remove
+  - Entire Object or Byte Range
+  - Append or Truncate
+- Create/Set/Get/Remove XATTRs
+- Create/Set/Get/Remove Key/Value Pairs
+- Compound operations and dual-ack semantics
+- Object Classes
+
+
+.. index:: architecture; watch/notify
+
+Object Watch/Notify
+-------------------
+
+A client can register a persistent interest with an object and keep a session to
+the primary OSD open. The client can send a notification message and a payload to
+all watchers and receive notification when the watchers receive the
+notification. This enables a client to use any object as a
+synchronization/communication channel.
+
+
+.. ditaa::
+
+           +----------+     +----------+     +----------+     +---------------+
+           | Client 1 |     | Client 2 |     | Client 3 |     | OSD:Object ID |
+           +----------+     +----------+     +----------+     +---------------+
+                 |                |                |                  |
+                 |                |                |                  |
+                 |                |  Watch Object  |                  |               
+                 |--------------------------------------------------->|
+                 |                |                |                  |
+                 |<---------------------------------------------------|
+                 |                |   Ack/Commit   |                  |
+                 |                |                |                  |
+                 |                |  Watch Object  |                  |
+                 |                |---------------------------------->|
+                 |                |                |                  |
+                 |                |<----------------------------------|
+                 |                |   Ack/Commit   |                  |
+                 |                |                |   Watch Object   |
+                 |                |                |----------------->|
+                 |                |                |                  |
+                 |                |                |<-----------------|
+                 |                |                |    Ack/Commit    |
+                 |                |     Notify     |                  |               
+                 |--------------------------------------------------->|
+                 |                |                |                  |
+                 |<---------------------------------------------------|
+                 |                |     Notify     |                  |
+                 |                |                |                  |
+                 |                |<----------------------------------|
+                 |                |     Notify     |                  |
+                 |                |                |<-----------------|
+                 |                |                |      Notify      |
+                 |                |       Ack      |                  |               
+                 |----------------+---------------------------------->|
+                 |                |                |                  |
+                 |                |       Ack      |                  |
+                 |                +---------------------------------->|
+                 |                |                |                  |
+                 |                |                |        Ack       |
+                 |                |                |----------------->|
+                 |                |                |                  | 
+                 |<---------------+----------------+------------------|
+                 |                     Complete
+
+.. index:: architecture; Striping
+
+Data Striping
+-------------
+
+Storage devices have throughput limitations, which impact performance and
+scalability. So storage systems often support `striping`_--storing sequential
+pieces of information across multiple storage devices--to increase throughput
+and performance. The most common form of data striping comes from `RAID`_.
+The RAID type most similar to Ceph's striping is `RAID 0`_, or a 'striped
+volume'. Ceph's striping offers the throughput of RAID 0 striping, the
+reliability of n-way RAID mirroring and faster recovery.
+
+Ceph provides three types of clients: Ceph Block Device, Ceph File System, and
+Ceph Object Storage. A Ceph Client converts its data from the representation 
+format it provides to its users (a block device image, RESTful objects, CephFS
+filesystem directories) into objects for storage in the Ceph Storage Cluster. 
+
+.. tip:: The objects Ceph stores in the Ceph Storage Cluster are not striped. 
+   Ceph Object Storage, Ceph Block Device, and the Ceph File System stripe their 
+   data over multiple Ceph Storage Cluster objects. Ceph Clients that write 
+   directly to the Ceph Storage Cluster via ``librados`` must perform the
+   striping (and parallel I/O) for themselves to obtain these benefits.
+
+The simplest Ceph striping format involves a stripe count of 1 object. Ceph
+Clients write stripe units to a Ceph Storage Cluster object until the object is
+at its maximum capacity, and then create another object for additional stripes
+of data. The simplest form of striping may be sufficient for small block device
+images, S3 or Swift objects and CephFS files. However, this simple form doesn't
+take maximum advantage of Ceph's ability to distribute data across placement
+groups, and consequently doesn't improve performance very much. The following
+diagram depicts the simplest form of striping:
+
+.. ditaa::
+
+                        +---------------+
+                        |  Client Data  |
+                        |     Format    |
+                        | cCCC          |
+                        +---------------+
+                                |
+                       +--------+-------+
+                       |                |
+                       v                v
+                 /-----------\    /-----------\
+                 | Begin cCCC|    | Begin cCCC|
+                 | Object  0 |    | Object  1 |
+                 +-----------+    +-----------+
+                 |  stripe   |    |  stripe   |
+                 |  unit 1   |    |  unit 5   |
+                 +-----------+    +-----------+
+                 |  stripe   |    |  stripe   |
+                 |  unit 2   |    |  unit 6   |
+                 +-----------+    +-----------+
+                 |  stripe   |    |  stripe   |
+                 |  unit 3   |    |  unit 7   |
+                 +-----------+    +-----------+
+                 |  stripe   |    |  stripe   |
+                 |  unit 4   |    |  unit 8   |
+                 +-----------+    +-----------+
+                 | End cCCC  |    | End cCCC  |
+                 | Object 0  |    | Object 1  |
+                 \-----------/    \-----------/
+   
+
+If you anticipate large images sizes, large S3 or Swift objects (e.g., video),
+or large CephFS directories, you may see considerable read/write performance
+improvements by striping client data over multiple objects within an object set.
+Significant write performance occurs when the client writes the stripe units to
+their corresponding objects in parallel. Since objects get mapped to different
+placement groups and further mapped to different OSDs, each write occurs in
+parallel at the maximum write speed. A write to a single drive would be limited
+by the head movement (e.g. 6ms per seek) and bandwidth of that one device (e.g.
+100MB/s).  By spreading that write over multiple objects (which map to different
+placement groups and OSDs) Ceph can reduce the number of seeks per drive and
+combine the throughput of multiple drives to achieve much faster write (or read)
+speeds.
+
+.. note:: Striping is independent of object replicas. Since CRUSH
+   replicates objects across OSDs, stripes get replicated automatically.
+
+In the following diagram, client data gets striped across an object set
+(``object set 1`` in the following diagram) consisting of 4 objects, where the
+first stripe unit is ``stripe unit 0`` in ``object 0``, and the fourth stripe
+unit is ``stripe unit 3`` in ``object 3``. After writing the fourth stripe, the
+client determines if the object set is full. If the object set is not full, the
+client begins writing a stripe to the first object again (``object 0`` in the
+following diagram). If the object set is full, the client creates a new object
+set (``object set 2`` in the following diagram), and begins writing to the first
+stripe (``stripe unit 16``) in the first object in the new object set (``object
+4`` in the diagram below).
+
+.. ditaa::
+
+                          +---------------+
+                          |  Client Data  |
+                          |     Format    |
+                          | cCCC          |
+                          +---------------+
+                                  |
+       +-----------------+--------+--------+-----------------+
+       |                 |                 |                 |     +--\
+       v                 v                 v                 v        |
+ /-----------\     /-----------\     /-----------\     /-----------\  |   
+ | Begin cCCC|     | Begin cCCC|     | Begin cCCC|     | Begin cCCC|  |
+ | Object 0  |     | Object  1 |     | Object  2 |     | Object  3 |  |
+ +-----------+     +-----------+     +-----------+     +-----------+  |
+ |  stripe   |     |  stripe   |     |  stripe   |     |  stripe   |  |
+ |  unit 0   |     |  unit 1   |     |  unit 2   |     |  unit 3   |  |
+ +-----------+     +-----------+     +-----------+     +-----------+  |
+ |  stripe   |     |  stripe   |     |  stripe   |     |  stripe   |  +-\ 
+ |  unit 4   |     |  unit 5   |     |  unit 6   |     |  unit 7   |    | Object
+ +-----------+     +-----------+     +-----------+     +-----------+    +- Set 
+ |  stripe   |     |  stripe   |     |  stripe   |     |  stripe   |    |   1
+ |  unit 8   |     |  unit 9   |     |  unit 10  |     |  unit 11  |  +-/
+ +-----------+     +-----------+     +-----------+     +-----------+  |
+ |  stripe   |     |  stripe   |     |  stripe   |     |  stripe   |  |
+ |  unit 12  |     |  unit 13  |     |  unit 14  |     |  unit 15  |  |
+ +-----------+     +-----------+     +-----------+     +-----------+  |
+ | End cCCC  |     | End cCCC  |     | End cCCC  |     | End cCCC  |  |
+ | Object 0  |     | Object 1  |     | Object 2  |     | Object 3  |  |  
+ \-----------/     \-----------/     \-----------/     \-----------/  |
+                                                                      |
+                                                                   +--/
+  
+                                                                   +--\
+                                                                      |
+ /-----------\     /-----------\     /-----------\     /-----------\  |   
+ | Begin cCCC|     | Begin cCCC|     | Begin cCCC|     | Begin cCCC|  |
+ | Object  4 |     | Object  5 |     | Object  6 |     | Object  7 |  |  
+ +-----------+     +-----------+     +-----------+     +-----------+  |
+ |  stripe   |     |  stripe   |     |  stripe   |     |  stripe   |  |
+ |  unit 16  |     |  unit 17  |     |  unit 18  |     |  unit 19  |  |
+ +-----------+     +-----------+     +-----------+     +-----------+  |
+ |  stripe   |     |  stripe   |     |  stripe   |     |  stripe   |  +-\ 
+ |  unit 20  |     |  unit 21  |     |  unit 22  |     |  unit 23  |    | Object
+ +-----------+     +-----------+     +-----------+     +-----------+    +- Set
+ |  stripe   |     |  stripe   |     |  stripe   |     |  stripe   |    |   2 
+ |  unit 24  |     |  unit 25  |     |  unit 26  |     |  unit 27  |  +-/
+ +-----------+     +-----------+     +-----------+     +-----------+  |
+ |  stripe   |     |  stripe   |     |  stripe   |     |  stripe   |  |
+ |  unit 28  |     |  unit 29  |     |  unit 30  |     |  unit 31  |  |
+ +-----------+     +-----------+     +-----------+     +-----------+  |
+ | End cCCC  |     | End cCCC  |     | End cCCC  |     | End cCCC  |  |
+ | Object 4  |     | Object 5  |     | Object 6  |     | Object 7  |  |  
+ \-----------/     \-----------/     \-----------/     \-----------/  |
+                                                                      |
+                                                                   +--/
+
+Three important variables determine how Ceph stripes data: 
+
+- **Object Size:** Objects in the Ceph Storage Cluster have a maximum
+  configurable size (e.g., 2MB, 4MB, etc.). The object size should be large
+  enough to accommodate many stripe units, and should be a multiple of
+  the stripe unit.
+
+- **Stripe Width:** Stripes have a configurable unit size (e.g., 64kb).
+  The Ceph Client divides the data it will write to objects into equally 
+  sized stripe units, except for the last stripe unit. A stripe width, 
+  should be a fraction of the Object Size so that an object may contain 
+  many stripe units.
+
+- **Stripe Count:** The Ceph Client writes a sequence of stripe units
+  over a series of objects determined by the stripe count. The series 
+  of objects is called an object set. After the Ceph Client writes to 
+  the last object in the object set, it returns to the first object in
+  the object set.
+  
+.. important:: Test the performance of your striping configuration before
+   putting your cluster into production. You CANNOT change these striping
+   parameters after you stripe the data and write it to objects.
+
+Once the Ceph Client has striped data to stripe units and mapped the stripe
+units to objects, Ceph's CRUSH algorithm maps the objects to placement groups,
+and the placement groups to Ceph OSD Daemons before the objects are stored as 
+files on a storage drive.
+
+.. note:: Since a client writes to a single pool, all data striped into objects
+   get mapped to placement groups in the same pool. So they use the same CRUSH
+   map and the same access controls.
+
+
+.. index:: architecture; Ceph Clients
+
+Ceph Clients
+============
+
+Ceph Clients include a number of service interfaces. These include:
+
+- **Block Devices:** The :term:`Ceph Block Device` (a.k.a., RBD) service 
+  provides resizable, thin-provisioned block devices with snapshotting and
+  cloning. Ceph stripes a block device across the cluster for high
+  performance. Ceph supports both kernel objects (KO) and a QEMU hypervisor 
+  that uses ``librbd`` directly--avoiding the kernel object overhead for 
+  virtualized systems.
+
+- **Object Storage:** The :term:`Ceph Object Storage` (a.k.a., RGW) service 
+  provides RESTful APIs with interfaces that are compatible with Amazon S3
+  and OpenStack Swift. 
+  
+- **Filesystem**: The :term:`Ceph File System` (CephFS) service provides 
+  a POSIX compliant filesystem usable with ``mount`` or as 
+  a filesystem in user space (FUSE).
+
+Ceph can run additional instances of OSDs, MDSs, and monitors for scalability
+and high availability. The following diagram depicts the high-level
+architecture. 
+
+.. ditaa::
+
+            +--------------+  +----------------+  +-------------+
+            | Block Device |  | Object Storage |  |   CephFS    |
+            +--------------+  +----------------+  +-------------+            
+
+            +--------------+  +----------------+  +-------------+
+            |    librbd    |  |     librgw     |  |  libcephfs  |
+            +--------------+  +----------------+  +-------------+
+
+            +---------------------------------------------------+
+            |      Ceph Storage Cluster Protocol (librados)     |
+            +---------------------------------------------------+
+
+            +---------------+ +---------------+ +---------------+
+            |      OSDs     | |      MDSs     | |    Monitors   |
+            +---------------+ +---------------+ +---------------+
+
+
+.. index:: architecture; Ceph Object Storage
+
+Ceph Object Storage
+-------------------
+
+The Ceph Object Storage daemon, ``radosgw``, is a FastCGI service that provides
+a RESTful_ HTTP API to store objects and metadata. It layers on top of the Ceph
+Storage Cluster with its own data formats, and maintains its own user database,
+authentication, and access control. The RADOS Gateway uses a unified namespace,
+which means you can use either the OpenStack Swift-compatible API or the Amazon
+S3-compatible API. For example, you can write data using the S3-compatible API
+with one application and then read data using the Swift-compatible API with
+another application.
+
+.. topic:: S3/Swift Objects and Store Cluster Objects Compared
+
+   Ceph's Object Storage uses the term *object* to describe the data it stores.
+   S3 and Swift objects are not the same as the objects that Ceph writes to the 
+   Ceph Storage Cluster. Ceph Object Storage objects are mapped to Ceph Storage
+   Cluster objects. The S3 and Swift objects do not necessarily 
+   correspond in a 1:1 manner with an object stored in the storage cluster. It 
+   is possible for an S3 or Swift object to map to multiple Ceph objects.
+
+See `Ceph Object Storage`_ for details.
+
+
+.. index:: Ceph Block Device; block device; RBD; Rados Block Device
+
+Ceph Block Device
+-----------------
+
+A Ceph Block Device stripes a block device image over multiple objects in the
+Ceph Storage Cluster, where each object gets mapped to a placement group and
+distributed, and the placement groups are spread across separate ``ceph-osd``
+daemons throughout the cluster.
+
+.. important:: Striping allows RBD block devices to perform better than a single 
+   server could!
+
+Thin-provisioned snapshottable Ceph Block Devices are an attractive option for
+virtualization and cloud computing. In virtual machine scenarios, people
+typically deploy a Ceph Block Device with the ``rbd`` network storage driver in
+QEMU/KVM, where the host machine uses ``librbd`` to provide a block device
+service to the guest. Many cloud computing stacks use ``libvirt`` to integrate
+with hypervisors. You can use thin-provisioned Ceph Block Devices with QEMU and
+``libvirt`` to support OpenStack and CloudStack among other solutions.
+
+While we do not provide ``librbd`` support with other hypervisors at this time,
+you may also use Ceph Block Device kernel objects to provide a block device to a
+client. Other virtualization technologies such as Xen can access the Ceph Block
+Device kernel object(s). This is done with the  command-line tool ``rbd``.
+
+
+.. index:: CephFS; Ceph File System; libcephfs; MDS; metadata server; ceph-mds
+
+.. _arch-cephfs:
+
+Ceph File System
+----------------
+
+The Ceph File System (CephFS) provides a POSIX-compliant filesystem as a
+service that is layered on top of the object-based Ceph Storage Cluster.
+CephFS files get mapped to objects that Ceph stores in the Ceph Storage
+Cluster. Ceph Clients mount a CephFS filesystem as a kernel object or as
+a Filesystem in User Space (FUSE).
+
+.. ditaa::
+
+            +-----------------------+  +------------------------+
+            | CephFS Kernel Object  |  |      CephFS FUSE       |
+            +-----------------------+  +------------------------+            
+
+            +---------------------------------------------------+
+            |            CephFS Library (libcephfs)             |
+            +---------------------------------------------------+
+
+            +---------------------------------------------------+
+            |      Ceph Storage Cluster Protocol (librados)     |
+            +---------------------------------------------------+
+
+            +---------------+ +---------------+ +---------------+
+            |      OSDs     | |      MDSs     | |    Monitors   |
+            +---------------+ +---------------+ +---------------+
+
+
+The Ceph File System service includes the Ceph Metadata Server (MDS) deployed
+with the Ceph Storage cluster. The purpose of the MDS is to store all the
+filesystem metadata (directories, file ownership, access modes, etc) in
+high-availability Ceph Metadata Servers where the metadata resides in memory.
+The reason for the MDS (a daemon called ``ceph-mds``) is that simple filesystem
+operations like listing a directory or changing a directory (``ls``, ``cd``)
+would tax the Ceph OSD Daemons unnecessarily. So separating the metadata from
+the data means that the Ceph File System can provide high performance services
+without taxing the Ceph Storage Cluster.
+
+CephFS separates the metadata from the data, storing the metadata in the MDS,
+and storing the file data in one or more objects in the Ceph Storage Cluster.
+The Ceph filesystem aims for POSIX compatibility. ``ceph-mds`` can run as a
+single process, or it can be distributed out to multiple physical machines,
+either for high availability or for scalability. 
+
+- **High Availability**: The extra ``ceph-mds`` instances can be `standby`, 
+  ready to take over the duties of any failed ``ceph-mds`` that was
+  `active`. This is easy because all the data, including the journal, is
+  stored on RADOS. The transition is triggered automatically by ``ceph-mon``.
+
+- **Scalability**: Multiple ``ceph-mds`` instances can be `active`, and they
+  will split the directory tree into subtrees (and shards of a single
+  busy directory), effectively balancing the load amongst all `active`
+  servers.
+
+Combinations of `standby` and `active` etc are possible, for example
+running 3 `active` ``ceph-mds`` instances for scaling, and one `standby`
+instance for high availability.
+
+
+
+.. _RADOS - A Scalable, Reliable Storage Service for Petabyte-scale Storage Clusters: https://ceph.io/assets/pdfs/weil-rados-pdsw07.pdf
+.. _Paxos: https://en.wikipedia.org/wiki/Paxos_(computer_science)
+.. _Monitor Config Reference: ../rados/configuration/mon-config-ref
+.. _Monitoring OSDs and PGs: ../rados/operations/monitoring-osd-pg
+.. _Heartbeats: ../rados/configuration/mon-osd-interaction
+.. _Monitoring OSDs: ../rados/operations/monitoring-osd-pg/#monitoring-osds
+.. _CRUSH - Controlled, Scalable, Decentralized Placement of Replicated Data: https://ceph.io/assets/pdfs/weil-crush-sc06.pdf
+.. _Data Scrubbing: ../rados/configuration/osd-config-ref#scrubbing
+.. _Report Peering Failure: ../rados/configuration/mon-osd-interaction#osds-report-peering-failure
+.. _Troubleshooting Peering Failure: ../rados/troubleshooting/troubleshooting-pg#placement-group-down-peering-failure
+.. _Ceph Authentication and Authorization: ../rados/operations/auth-intro/
+.. _Hardware Recommendations: ../start/hardware-recommendations
+.. _Network Config Reference: ../rados/configuration/network-config-ref
+.. _Data Scrubbing: ../rados/configuration/osd-config-ref#scrubbing
+.. _striping: https://en.wikipedia.org/wiki/Data_striping
+.. _RAID: https://en.wikipedia.org/wiki/RAID
+.. _RAID 0: https://en.wikipedia.org/wiki/RAID_0#RAID_0
+.. _Ceph Object Storage: ../radosgw/
+.. _RESTful: https://en.wikipedia.org/wiki/RESTful
+.. _Erasure Code Notes: https://github.com/ceph/ceph/blob/40059e12af88267d0da67d8fd8d9cd81244d8f93/doc/dev/osd_internals/erasure_coding/developer_notes.rst
+.. _Cache Tiering: ../rados/operations/cache-tiering
+.. _Set Pool Values: ../rados/operations/pools#set-pool-values
+.. _Kerberos: https://en.wikipedia.org/wiki/Kerberos_(protocol)
+.. _Cephx Config Guide: ../rados/configuration/auth-config-ref
+.. _User Management: ../rados/operations/user-management
diff --git a/doc/ceph-volume/drive-group.rst b/doc/ceph-volume/drive-group.rst
new file mode 100644
index 000000000..4b2b8ed94
--- /dev/null
+++ b/doc/ceph-volume/drive-group.rst
@@ -0,0 +1,12 @@
+.. _ceph-volume-drive-group:
+
+``drive-group``
+===============
+The drive-group subcommand allows for passing :ref:'drivegroups' specifications
+straight to ceph-volume as json. ceph-volume will then attempt to deploy this
+drive groups via the batch subcommand.
+
+The specification can be passed via a file, string argument or on stdin.
+See the subcommand help for further details::
+
+    # ceph-volume drive-group --help
diff --git a/doc/ceph-volume/index.rst b/doc/ceph-volume/index.rst
new file mode 100644
index 000000000..9271bc2a0
--- /dev/null
+++ b/doc/ceph-volume/index.rst
@@ -0,0 +1,87 @@
+.. _ceph-volume:
+
+ceph-volume
+===========
+Deploy OSDs with different device technologies like lvm or physical disks using
+pluggable tools (:doc:`lvm/index` itself is treated like a plugin) and trying to
+follow a predictable, and robust way of preparing, activating, and starting OSDs.
+
+:ref:`Overview <ceph-volume-overview>` |
+:ref:`Plugin Guide <ceph-volume-plugins>` |
+
+
+**Command Line Subcommands**
+
+There is currently support for ``lvm``, and plain disks (with GPT partitions)
+that may have been deployed with ``ceph-disk``.
+
+``zfs`` support is available for running a FreeBSD cluster.
+
+* :ref:`ceph-volume-lvm`
+* :ref:`ceph-volume-simple`
+* :ref:`ceph-volume-zfs`
+
+**Node inventory**
+
+The :ref:`ceph-volume-inventory` subcommand provides information and metadata
+about a nodes physical disk inventory.
+
+
+Migrating
+---------
+Starting on Ceph version 13.0.0, ``ceph-disk`` is deprecated. Deprecation
+warnings will show up that will link to this page. It is strongly suggested
+that users start consuming ``ceph-volume``. There are two paths for migrating:
+
+#. Keep OSDs deployed with ``ceph-disk``: The :ref:`ceph-volume-simple` command
+   provides a way to take over the management while disabling ``ceph-disk``
+   triggers.
+#. Redeploy existing OSDs with ``ceph-volume``: This is covered in depth on
+   :ref:`rados-replacing-an-osd`
+
+For details on why ``ceph-disk`` was removed please see the :ref:`Why was
+ceph-disk replaced? <ceph-disk-replaced>` section.
+
+
+New deployments
+^^^^^^^^^^^^^^^
+For new deployments, :ref:`ceph-volume-lvm` is recommended, it can use any
+logical volume as input for data OSDs, or it can setup a minimal/naive logical
+volume from a device.
+
+Existing OSDs
+^^^^^^^^^^^^^
+If the cluster has OSDs that were provisioned with ``ceph-disk``, then
+``ceph-volume`` can take over the management of these with
+:ref:`ceph-volume-simple`. A scan is done on the data device or OSD directory,
+and ``ceph-disk`` is fully disabled. Encryption is fully supported.
+
+
+.. toctree::
+   :hidden:
+   :maxdepth: 3
+   :caption: Contents:
+
+   intro
+   systemd
+   inventory
+   drive-group
+   lvm/index
+   lvm/activate
+   lvm/batch
+   lvm/encryption
+   lvm/prepare
+   lvm/create
+   lvm/scan
+   lvm/systemd
+   lvm/list
+   lvm/zap
+   lvm/migrate
+   lvm/newdb
+   lvm/newwal
+   simple/index
+   simple/activate
+   simple/scan
+   simple/systemd
+   zfs/index
+   zfs/inventory
diff --git a/doc/ceph-volume/intro.rst b/doc/ceph-volume/intro.rst
new file mode 100644
index 000000000..d59dd2b6c
--- /dev/null
+++ b/doc/ceph-volume/intro.rst
@@ -0,0 +1,84 @@
+.. _ceph-volume-overview:
+
+Overview
+--------
+The ``ceph-volume`` tool aims to be a single purpose command line tool to deploy
+logical volumes as OSDs, trying to maintain a similar API to ``ceph-disk`` when
+preparing, activating, and creating OSDs.
+
+It deviates from ``ceph-disk`` by not interacting or relying on the udev rules
+that come installed for Ceph. These rules allow automatic detection of
+previously setup devices that are in turn fed into ``ceph-disk`` to activate
+them.
+
+.. _ceph-disk-replaced:
+
+Replacing ``ceph-disk``
+-----------------------
+The ``ceph-disk`` tool was created at a time were the project was required to
+support many different types of init systems (upstart, sysvinit, etc...) while
+being able to discover devices. This caused the tool to concentrate initially
+(and exclusively afterwards) on GPT partitions. Specifically on GPT GUIDs,
+which were used to label devices in a unique way to answer questions like:
+
+* is this device a Journal?
+* an encrypted data partition?
+* was the device left partially prepared?
+
+To solve these, it used ``UDEV`` rules to match the GUIDs, that would call
+``ceph-disk``, and end up in a back and forth between the ``ceph-disk`` systemd
+unit and the ``ceph-disk`` executable. The process was very unreliable and time
+consuming (a timeout of close to three hours **per OSD** had to be put in
+place), and would cause OSDs to not come up at all during the boot process of
+a node.
+
+It was hard to debug, or even replicate these problems given the asynchronous
+behavior of ``UDEV``.
+
+Since the world-view of ``ceph-disk`` had to be GPT partitions exclusively, it meant
+that it couldn't work with other technologies like LVM, or similar device
+mapper devices. It was ultimately decided to create something modular, starting
+with LVM support, and the ability to expand on other technologies as needed.
+
+
+GPT partitions are simple?
+--------------------------
+Although partitions in general are simple to reason about, ``ceph-disk``
+partitions were not simple by any means. It required a tremendous amount of
+special flags in order to get them to work correctly with the device discovery
+workflow. Here is an example call to create a data partition::
+
+    /sbin/sgdisk --largest-new=1 --change-name=1:ceph data --partition-guid=1:f0fc39fd-eeb2-49f1-b922-a11939cf8a0f --typecode=1:89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be --mbrtogpt -- /dev/sdb
+
+Not only creating these was hard, but these partitions required devices to be
+exclusively owned by Ceph. For example, in some cases a special partition would
+be created when devices were encrypted, which would contain unencrypted keys.
+This was ``ceph-disk`` domain knowledge, which would not translate to a "GPT
+partitions are simple" understanding. Here is an example of that special
+partition being created::
+
+    /sbin/sgdisk --new=5:0:+10M --change-name=5:ceph lockbox --partition-guid=5:None --typecode=5:fb3aabf9-d25f-47cc-bf5e-721d181642be --mbrtogpt -- /dev/sdad
+
+
+Modularity
+----------
+``ceph-volume`` was designed to be a modular tool because we anticipate that
+there are going to be lots of ways that people provision the hardware devices
+that we need to consider. There are already two: legacy ceph-disk devices that
+are still in use and have GPT partitions (handled by :ref:`ceph-volume-simple`),
+and lvm. SPDK devices where we manage NVMe devices directly from userspace are
+on the immediate horizon, where LVM won't work there since the kernel isn't
+involved at all.
+
+``ceph-volume lvm``
+-------------------
+By making use of :term:`LVM tags`, the :ref:`ceph-volume-lvm` sub-command is
+able to store and later re-discover and query devices associated with OSDs so
+that they can later be activated.
+
+LVM performance penalty
+-----------------------
+In short: we haven't been able to notice any significant performance penalties
+associated with the change to LVM. By being able to work closely with LVM, the
+ability to work with other device mapper technologies was a given: there is no
+technical difficulty in working with anything that can sit below a Logical Volume.
diff --git a/doc/ceph-volume/inventory.rst b/doc/ceph-volume/inventory.rst
new file mode 100644
index 000000000..edb1fd205
--- /dev/null
+++ b/doc/ceph-volume/inventory.rst
@@ -0,0 +1,17 @@
+.. _ceph-volume-inventory:
+
+``inventory``
+=============
+The ``inventory`` subcommand queries a host's disc inventory and provides
+hardware information and metadata on every physical device.
+
+By default the command returns a short, human-readable report of all physical disks.
+
+For programmatic consumption of this report pass ``--format json`` to generate a
+JSON formatted report. This report includes extensive information on the
+physical drives such as disk metadata (like model and size), logical volumes
+and whether they are used by ceph, and if the disk is usable by ceph and
+reasons why not.
+
+A device path can be specified to report extensive information on a device in
+both plain and json format.
diff --git a/doc/ceph-volume/lvm/activate.rst b/doc/ceph-volume/lvm/activate.rst
new file mode 100644
index 000000000..eef5a0101
--- /dev/null
+++ b/doc/ceph-volume/lvm/activate.rst
@@ -0,0 +1,121 @@
+.. _ceph-volume-lvm-activate:
+
+``activate``
+============
+          
+Once :ref:`ceph-volume-lvm-prepare` is completed, and all the various steps
+that entails are done, the volume is ready to get "activated".
+
+This activation process enables a systemd unit that persists the OSD ID and its
+UUID (also called ``fsid`` in Ceph CLI tools), so that at boot time it can
+understand what OSD is enabled and needs to be mounted.
+
+.. note:: The execution of this call is fully idempotent, and there is no
+          side-effects when running multiple times
+
+For OSDs deployed by cephadm, please refer to :ref:cephadm-osd-activate: 
+instead.
+
+New OSDs
+--------
+To activate newly prepared OSDs both the :term:`OSD id` and :term:`OSD uuid`
+need to be supplied. For example::
+
+    ceph-volume lvm activate --bluestore 0 0263644D-0BF1-4D6D-BC34-28BD98AE3BC8
+
+.. note:: The UUID is stored in the ``fsid`` file in the OSD path, which is
+          generated when :ref:`ceph-volume-lvm-prepare` is used.
+
+Activating all OSDs
+-------------------
+
+.. note:: For OSDs deployed by cephadm, please refer to :ref:cephadm-osd-activate: 
+          instead.
+
+It is possible to activate all existing OSDs at once by using the ``--all``
+flag. For example::
+
+    ceph-volume lvm activate --all
+
+This call will inspect all the OSDs created by ceph-volume that are inactive
+and will activate them one by one. If any of the OSDs are already running, it
+will report them in the command output and skip them, making it safe to rerun
+(idempotent).
+
+requiring uuids
+^^^^^^^^^^^^^^^
+The :term:`OSD uuid` is being required as an extra step to ensure that the
+right OSD is being activated. It is entirely possible that a previous OSD with
+the same id exists and would end up activating the incorrect one.
+
+
+dmcrypt
+^^^^^^^
+If the OSD was prepared with dmcrypt by ceph-volume, there is no need to
+specify ``--dmcrypt`` on the command line again (that flag is not available for
+the ``activate`` subcommand). An encrypted OSD will be automatically detected.
+
+
+Discovery
+---------
+With OSDs previously created by ``ceph-volume``, a *discovery* process is
+performed using :term:`LVM tags` to enable the systemd units.
+
+The systemd unit will capture the :term:`OSD id` and :term:`OSD uuid` and
+persist it. Internally, the activation will enable it like::
+
+    systemctl enable ceph-volume@lvm-$id-$uuid
+
+For example::
+
+    systemctl enable ceph-volume@lvm-0-8715BEB4-15C5-49DE-BA6F-401086EC7B41
+
+Would start the discovery process for the OSD with an id of ``0`` and a UUID of
+``8715BEB4-15C5-49DE-BA6F-401086EC7B41``.
+
+.. note:: for more details on the systemd workflow see :ref:`ceph-volume-lvm-systemd`
+
+The systemd unit will look for the matching OSD device, and by looking at its
+:term:`LVM tags` will proceed to:
+
+# mount the device in the corresponding location (by convention this is
+  ``/var/lib/ceph/osd/<cluster name>-<osd id>/``)
+
+# ensure that all required devices are ready for that OSD. In the case of
+a journal (when ``--filestore`` is selected) the device will be queried (with
+``blkid`` for partitions, and lvm for logical volumes) to ensure that the
+correct device is being linked. The symbolic link will *always* be re-done to
+ensure that the correct device is linked.
+
+# start the ``ceph-osd@0`` systemd unit
+
+.. note:: The system infers the objectstore type (filestore or bluestore) by
+          inspecting the LVM tags applied to the OSD devices
+
+Existing OSDs
+-------------
+For existing OSDs that have been deployed with ``ceph-disk``, they need to be
+scanned and activated :ref:`using the simple sub-command <ceph-volume-simple>`.
+If a different tooling was used then the only way to port them over to the new
+mechanism is to prepare them again (losing data). See
+:ref:`ceph-volume-lvm-existing-osds` for details on how to proceed.
+
+Summary
+-------
+To recap the ``activate`` process for :term:`bluestore`:
+
+#. require both :term:`OSD id` and :term:`OSD uuid`
+#. enable the system unit with matching id and uuid
+#. Create the ``tmpfs`` mount at the OSD directory in
+   ``/var/lib/ceph/osd/$cluster-$id/``
+#. Recreate all the files needed with ``ceph-bluestore-tool prime-osd-dir`` by
+   pointing it to the OSD ``block`` device.
+#. the systemd unit will ensure all devices are ready and linked
+#. the matching ``ceph-osd`` systemd unit will get started
+
+And for :term:`filestore`:
+
+#. require both :term:`OSD id` and :term:`OSD uuid`
+#. enable the system unit with matching id and uuid
+#. the systemd unit will ensure all devices are ready and mounted (if needed)
+#. the matching ``ceph-osd`` systemd unit will get started
diff --git a/doc/ceph-volume/lvm/batch.rst b/doc/ceph-volume/lvm/batch.rst
new file mode 100644
index 000000000..6033b5aaf
--- /dev/null
+++ b/doc/ceph-volume/lvm/batch.rst
@@ -0,0 +1,178 @@
+.. _ceph-volume-lvm-batch:
+
+``batch``
+===========
+The subcommand allows to create multiple OSDs at the same time given
+an input of devices. The ``batch`` subcommand is closely related to
+drive-groups. One individual drive group specification translates to a single
+``batch`` invocation.
+
+The subcommand is based to :ref:`ceph-volume-lvm-create`, and will use the very
+same code path. All ``batch`` does is to calculate the appropriate sizes of all
+volumes and skip over already created volumes.
+
+All the features that ``ceph-volume lvm create`` supports, like ``dmcrypt``,
+avoiding ``systemd`` units from starting, defining bluestore or filestore,
+are supported.
+
+
+.. _ceph-volume-lvm-batch_auto:
+
+Automatic sorting of disks
+--------------------------
+If ``batch`` receives only a single list of data devices and other options are
+passed , ``ceph-volume`` will auto-sort disks by its rotational
+property and use non-rotating disks for ``block.db`` or ``journal`` depending
+on the objectstore used. If all devices are to be used for standalone OSDs,
+no matter if rotating or solid state, pass ``--no-auto``.
+For example assuming :term:`bluestore` is used and ``--no-auto`` is not passed,
+the deprecated behavior would deploy the following, depending on the devices
+passed:
+
+#. Devices are all spinning HDDs: 1 OSD is created per device
+#. Devices are all SSDs: 2 OSDs are created per device
+#. Devices are a mix of HDDs and SSDs: data is placed on the spinning device,
+   the ``block.db`` is created on the SSD, as large as possible.
+
+.. note:: Although operations in ``ceph-volume lvm create`` allow usage of
+          ``block.wal`` it isn't supported with the ``auto`` behavior.
+
+This default auto-sorting behavior is now DEPRECATED and will be changed in future releases.
+Instead devices are not automatically sorted unless the ``--auto`` option is passed
+
+It is recommended to make use of the explicit device lists for ``block.db``,
+   ``block.wal`` and ``journal``.
+
+.. _ceph-volume-lvm-batch_bluestore:
+
+Reporting
+=========
+By default ``batch`` will print a report of the computed OSD layout and ask the
+user to confirm. This can be overridden by passing ``--yes``.
+
+If one wants to try out several invocations with being asked to deploy
+``--report`` can be passed. ``ceph-volume`` will exit after printing the report.
+
+Consider the following invocation::
+
+    $ ceph-volume lvm batch --report /dev/sdb /dev/sdc /dev/sdd --db-devices /dev/nvme0n1
+
+This will deploy three OSDs with external ``db`` and ``wal`` volumes on
+an NVME device.
+
+**pretty reporting**
+The ``pretty`` report format (the default) would
+look like this::
+
+    $ ceph-volume lvm batch --report /dev/sdb /dev/sdc /dev/sdd --db-devices /dev/nvme0n1
+    --> passed data devices: 3 physical, 0 LVM
+    --> relative data size: 1.0
+    --> passed block_db devices: 1 physical, 0 LVM
+
+    Total OSDs: 3
+
+      Type            Path                                                    LV Size         % of device
+    ----------------------------------------------------------------------------------------------------
+      data            /dev/sdb                                              300.00 GB         100.00%
+      block_db        /dev/nvme0n1                                           66.67 GB         33.33%
+    ----------------------------------------------------------------------------------------------------
+      data            /dev/sdc                                              300.00 GB         100.00%
+      block_db        /dev/nvme0n1                                           66.67 GB         33.33%
+    ----------------------------------------------------------------------------------------------------
+      data            /dev/sdd                                              300.00 GB         100.00%
+      block_db        /dev/nvme0n1                                           66.67 GB         33.33%
+
+
+
+
+
+**JSON reporting**
+Reporting can produce a structured output with ``--format json`` or
+``--format json-pretty``::
+
+    $ ceph-volume lvm batch --report --format json-pretty /dev/sdb /dev/sdc /dev/sdd --db-devices /dev/nvme0n1
+    --> passed data devices: 3 physical, 0 LVM
+    --> relative data size: 1.0
+    --> passed block_db devices: 1 physical, 0 LVM
+    [
+        {
+            "block_db": "/dev/nvme0n1",
+            "block_db_size": "66.67 GB",
+            "data": "/dev/sdb",
+            "data_size": "300.00 GB",
+            "encryption": "None"
+        },
+        {
+            "block_db": "/dev/nvme0n1",
+            "block_db_size": "66.67 GB",
+            "data": "/dev/sdc",
+            "data_size": "300.00 GB",
+            "encryption": "None"
+        },
+        {
+            "block_db": "/dev/nvme0n1",
+            "block_db_size": "66.67 GB",
+            "data": "/dev/sdd",
+            "data_size": "300.00 GB",
+            "encryption": "None"
+        }
+    ]
+
+Sizing
+======
+When no sizing arguments are passed, `ceph-volume` will derive the sizing from
+the passed device lists (or the sorted lists when using the automatic sorting).
+`ceph-volume batch` will attempt to fully utilize a device's available capacity.
+Relying on automatic sizing is recommended.
+
+If one requires a different sizing policy for wal, db or journal devices,
+`ceph-volume` offers implicit and explicit sizing rules.
+
+Implicit sizing
+---------------
+Scenarios in which either devices are under-comitted or not all data devices are
+currently ready for use (due to a broken disk for example), one can still rely
+on `ceph-volume` automatic sizing.
+Users can provide hints to `ceph-volume` as to how many data devices should have
+their external volumes on a set of fast devices. These options are:
+
+* ``--block-db-slots``
+* ``--block-wal-slots``
+* ``--journal-slots``
+
+For example, consider an OSD host that is supposed to contain 5 data devices and
+one device for wal/db volumes. However, one data device is currently broken and
+is being replaced. Instead of calculating the explicit sizes for the wal/db
+volume, one can simply call::
+
+    $ ceph-volume lvm batch --report /dev/sdb /dev/sdc /dev/sdd /dev/sde --db-devices /dev/nvme0n1 --block-db-slots 5
+
+Explicit sizing
+---------------
+It is also possible to provide explicit sizes to `ceph-volume` via the arguments
+
+* ``--block-db-size``
+* ``--block-wal-size``
+* ``--journal-size``
+
+`ceph-volume` will try to satisfy the requested sizes given the passed disks. If
+this is not possible, no OSDs will be deployed.
+
+
+Idempotency and disk replacements
+=================================
+`ceph-volume lvm batch` intends to be idempotent, i.e. calling the same command
+repeatedly must result in the same outcome. For example calling::
+
+    $ ceph-volume lvm batch --report /dev/sdb /dev/sdc /dev/sdd --db-devices /dev/nvme0n1
+
+will result in three deployed OSDs (if all disks were available). Calling this
+command again, you will still end up with three OSDs and ceph-volume will exit
+with return code 0.
+
+Suppose /dev/sdc goes bad and needs to be replaced. After destroying the OSD and
+replacing the hardware, you can again call the same command and `ceph-volume`
+will detect that only two out of the three wanted OSDs are setup and re-create
+the missing OSD.
+
+This idempotency notion is tightly coupled to and extensively used by :ref:`drivegroups`.
diff --git a/doc/ceph-volume/lvm/create.rst b/doc/ceph-volume/lvm/create.rst
new file mode 100644
index 000000000..c90d1f6fa
--- /dev/null
+++ b/doc/ceph-volume/lvm/create.rst
@@ -0,0 +1,24 @@
+.. _ceph-volume-lvm-create:
+
+``create``
+===========
+This subcommand wraps the two-step process to provision a new osd (calling
+``prepare`` first and then ``activate``) into a single
+one. The reason to prefer ``prepare`` and then ``activate`` is to gradually
+introduce new OSDs into a cluster, and avoiding large amounts of data being
+rebalanced.
+
+The single-call process unifies exactly what :ref:`ceph-volume-lvm-prepare` and
+:ref:`ceph-volume-lvm-activate` do, with the convenience of doing it all at
+once.
+
+There is nothing different to the process except the OSD will become up and in
+immediately after completion.
+
+The backing objectstore can be specified with:
+
+* :ref:`--filestore <ceph-volume-lvm-prepare_filestore>`
+* :ref:`--bluestore <ceph-volume-lvm-prepare_bluestore>`
+
+All command line flags and options are the same as ``ceph-volume lvm prepare``.
+Please refer to :ref:`ceph-volume-lvm-prepare` for details.
diff --git a/doc/ceph-volume/lvm/encryption.rst b/doc/ceph-volume/lvm/encryption.rst
new file mode 100644
index 000000000..1483ef32e
--- /dev/null
+++ b/doc/ceph-volume/lvm/encryption.rst
@@ -0,0 +1,86 @@
+.. _ceph-volume-lvm-encryption:
+
+Encryption
+==========
+
+Logical volumes can be encrypted using ``dmcrypt`` by specifying the
+``--dmcrypt`` flag when creating OSDs. Encryption can be done in different ways,
+specially with LVM. ``ceph-volume`` is somewhat opinionated with the way it
+sets up encryption with logical volumes so that the process is consistent and
+robust.
+
+In this case, ``ceph-volume lvm`` follows these constraints:
+
+* only LUKS (version 1) is used
+* Logical Volumes are encrypted, while their underlying PVs (physical volumes)
+  aren't
+* Non-LVM devices like partitions are also encrypted with the same OSD key
+
+
+LUKS
+----
+There are currently two versions of LUKS, 1 and 2. Version 2 is a bit easier
+to implement but not widely available in all distros Ceph supports. LUKS 1 is
+not going to be deprecated in favor of LUKS 2, so in order to have as wide
+support as possible, ``ceph-volume`` uses LUKS version 1.
+
+.. note:: Version 1 of LUKS is just referenced as "LUKS" whereas version 2 is
+          referred to as LUKS2
+
+
+LUKS on LVM
+-----------
+Encryption is done on top of existing logical volumes (unlike encrypting the
+physical device). Any single logical volume can be encrypted while other
+volumes can remain unencrypted. This method also allows for flexible logical
+volume setups, since encryption will happen once the LV is created.
+
+
+Workflow
+--------
+When setting up the OSD, a secret key will be created, that will be passed
+along to the monitor in JSON format as ``stdin`` to prevent the key from being
+captured in the logs.
+
+The JSON payload looks something like::
+
+        {
+            "cephx_secret": CEPHX_SECRET,
+            "dmcrypt_key": DMCRYPT_KEY,
+            "cephx_lockbox_secret": LOCKBOX_SECRET,
+        }
+
+The naming convention for the keys is **strict**, and they are named like that
+for the hardcoded (legacy) names ceph-disk used.
+
+* ``cephx_secret`` : The cephx key used to authenticate
+* ``dmcrypt_key`` : The secret (or private) key to unlock encrypted devices
+* ``cephx_lockbox_secret`` : The authentication key used to retrieve the
+  ``dmcrypt_key``. It is named *lockbox* because ceph-disk used to have an
+  unencrypted partition named after it, used to store public keys and other
+  OSD metadata.
+
+The naming convention is strict because Monitors supported the naming
+convention by ceph-disk, which used these key names. In order to keep
+compatibility and prevent ceph-disk from breaking, ceph-volume will use the same
+naming convention *although they don't make sense for the new encryption
+workflow*.
+
+After the common steps of setting up the OSD during the prepare stage, either
+with :term:`filestore` or :term:`bluestore`, the logical volume is left ready
+to be activated, regardless of the state of the device (encrypted or decrypted).
+
+At activation time, the logical volume will get decrypted and the OSD started
+once the process completes correctly.
+
+Summary of the encryption workflow for creating a new OSD:
+
+#. OSD is created, both lockbox and dmcrypt keys are created, and sent along
+   with JSON to the monitors, indicating an encrypted OSD.
+
+#. All complementary devices (like journal, db, or wal) get created and
+   encrypted with the same OSD key. Key is stored in the LVM metadata of the
+   OSD
+
+#. Activation continues by ensuring devices are mounted, retrieving the dmcrypt
+   secret key from the monitors and decrypting before the OSD gets started.
diff --git a/doc/ceph-volume/lvm/index.rst b/doc/ceph-volume/lvm/index.rst
new file mode 100644
index 000000000..962e51a51
--- /dev/null
+++ b/doc/ceph-volume/lvm/index.rst
@@ -0,0 +1,34 @@
+.. _ceph-volume-lvm:
+
+``lvm``
+=======
+Implements the functionality needed to deploy OSDs from the ``lvm`` subcommand:
+``ceph-volume lvm``
+
+**Command Line Subcommands**
+
+* :ref:`ceph-volume-lvm-prepare`
+
+* :ref:`ceph-volume-lvm-activate`
+
+* :ref:`ceph-volume-lvm-create`
+
+* :ref:`ceph-volume-lvm-list`
+
+* :ref:`ceph-volume-lvm-migrate`
+
+* :ref:`ceph-volume-lvm-newdb`
+
+* :ref:`ceph-volume-lvm-newwal`
+
+.. not yet implemented
+.. * :ref:`ceph-volume-lvm-scan`
+
+**Internal functionality**
+
+There are other aspects of the ``lvm`` subcommand that are internal and not
+exposed to the user, these sections explain how these pieces work together,
+clarifying the workflows of the tool.
+
+:ref:`Systemd Units <ceph-volume-lvm-systemd>` |
+:ref:`lvm <ceph-volume-lvm-api>`
diff --git a/doc/ceph-volume/lvm/list.rst b/doc/ceph-volume/lvm/list.rst
new file mode 100644
index 000000000..718154b10
--- /dev/null
+++ b/doc/ceph-volume/lvm/list.rst
@@ -0,0 +1,184 @@
+.. _ceph-volume-lvm-list:
+
+``list``
+========
+This subcommand will list any devices (logical and physical) that may be
+associated with a Ceph cluster, as long as they contain enough metadata to
+allow for that discovery.
+
+Output is grouped by the OSD ID associated with the devices, and unlike
+``ceph-disk`` it does not provide any information for devices that aren't
+associated with Ceph.
+
+Command line options:
+
+* ``--format`` Allows a ``json`` or ``pretty`` value. Defaults to ``pretty``
+  which will group the device information in a human-readable format.
+
+Full Reporting
+--------------
+When no positional arguments are used, a full reporting will be presented. This
+means that all devices and logical volumes found in the system will be
+displayed.
+
+Full ``pretty`` reporting for two OSDs, one with a lv as a journal, and another
+one with a physical device may look similar to::
+
+    # ceph-volume lvm list
+
+
+    ====== osd.1 =======
+
+      [journal]    /dev/journals/journal1
+
+          journal uuid              C65n7d-B1gy-cqX3-vZKY-ZoE0-IEYM-HnIJzs
+          osd id                    1
+          cluster fsid              ce454d91-d748-4751-a318-ff7f7aa18ffd
+          type                      journal
+          osd fsid                  661b24f8-e062-482b-8110-826ffe7f13fa
+          data uuid                 SlEgHe-jX1H-QBQk-Sce0-RUls-8KlY-g8HgcZ
+          journal device            /dev/journals/journal1
+          data device               /dev/test_group/data-lv2
+          devices                   /dev/sda
+
+      [data]    /dev/test_group/data-lv2
+
+          journal uuid              C65n7d-B1gy-cqX3-vZKY-ZoE0-IEYM-HnIJzs
+          osd id                    1
+          cluster fsid              ce454d91-d748-4751-a318-ff7f7aa18ffd
+          type                      data
+          osd fsid                  661b24f8-e062-482b-8110-826ffe7f13fa
+          data uuid                 SlEgHe-jX1H-QBQk-Sce0-RUls-8KlY-g8HgcZ
+          journal device            /dev/journals/journal1
+          data device               /dev/test_group/data-lv2
+          devices                   /dev/sdb
+
+    ====== osd.0 =======
+
+      [data]    /dev/test_group/data-lv1
+
+          journal uuid              cd72bd28-002a-48da-bdf6-d5b993e84f3f
+          osd id                    0
+          cluster fsid              ce454d91-d748-4751-a318-ff7f7aa18ffd
+          type                      data
+          osd fsid                  943949f0-ce37-47ca-a33c-3413d46ee9ec
+          data uuid                 TUpfel-Q5ZT-eFph-bdGW-SiNW-l0ag-f5kh00
+          journal device            /dev/sdd1
+          data device               /dev/test_group/data-lv1
+          devices                   /dev/sdc
+
+      [journal]    /dev/sdd1
+
+          PARTUUID                  cd72bd28-002a-48da-bdf6-d5b993e84f3f
+
+
+For logical volumes the ``devices`` key is populated with the physical devices
+associated with the logical volume. Since LVM allows multiple physical devices
+to be part of a logical volume, the value will be comma separated when using
+``pretty``, but an array when using ``json``.
+
+.. note:: Tags are displayed in a readable format. The ``osd id`` key is stored
+          as a ``ceph.osd_id`` tag. For more information on lvm tag conventions
+          see :ref:`ceph-volume-lvm-tag-api`
+
+Single Reporting
+----------------
+Single reporting can consume both devices and logical volumes as input
+(positional parameters). For logical volumes, it is required to use the group
+name as well as the logical volume name.
+
+For example the ``data-lv2`` logical volume, in the ``test_group`` volume group
+can be listed in the following way::
+
+    # ceph-volume lvm list test_group/data-lv2
+
+
+    ====== osd.1 =======
+
+      [data]    /dev/test_group/data-lv2
+
+          journal uuid              C65n7d-B1gy-cqX3-vZKY-ZoE0-IEYM-HnIJzs
+          osd id                    1
+          cluster fsid              ce454d91-d748-4751-a318-ff7f7aa18ffd
+          type                      data
+          osd fsid                  661b24f8-e062-482b-8110-826ffe7f13fa
+          data uuid                 SlEgHe-jX1H-QBQk-Sce0-RUls-8KlY-g8HgcZ
+          journal device            /dev/journals/journal1
+          data device               /dev/test_group/data-lv2
+          devices                   /dev/sdc
+
+
+.. note:: Tags are displayed in a readable format. The ``osd id`` key is stored
+          as a ``ceph.osd_id`` tag. For more information on lvm tag conventions
+          see :ref:`ceph-volume-lvm-tag-api`
+
+
+For plain disks, the full path to the device is required. For example, for
+a device like ``/dev/sdd1`` it can look like::
+
+
+    # ceph-volume lvm list /dev/sdd1
+
+
+    ====== osd.0 =======
+
+      [journal]    /dev/sdd1
+
+          PARTUUID                  cd72bd28-002a-48da-bdf6-d5b993e84f3f
+
+
+
+``json`` output
+---------------
+All output using ``--format=json`` will show everything the system has stored
+as metadata for the devices, including tags.
+
+No changes for readability are done with ``json`` reporting, and all
+information is presented as-is. Full output as well as single devices can be
+listed.
+
+For brevity, this is how a single logical volume would look with ``json``
+output (note how tags aren't modified)::
+
+    # ceph-volume lvm list --format=json test_group/data-lv1
+    {
+        "0": [
+            {
+                "devices": ["/dev/sda"],
+                "lv_name": "data-lv1",
+                "lv_path": "/dev/test_group/data-lv1",
+                "lv_tags": "ceph.cluster_fsid=ce454d91-d748-4751-a318-ff7f7aa18ffd,ceph.data_device=/dev/test_group/data-lv1,ceph.data_uuid=TUpfel-Q5ZT-eFph-bdGW-SiNW-l0ag-f5kh00,ceph.journal_device=/dev/sdd1,ceph.journal_uuid=cd72bd28-002a-48da-bdf6-d5b993e84f3f,ceph.osd_fsid=943949f0-ce37-47ca-a33c-3413d46ee9ec,ceph.osd_id=0,ceph.type=data",
+                "lv_uuid": "TUpfel-Q5ZT-eFph-bdGW-SiNW-l0ag-f5kh00",
+                "name": "data-lv1",
+                "path": "/dev/test_group/data-lv1",
+                "tags": {
+                    "ceph.cluster_fsid": "ce454d91-d748-4751-a318-ff7f7aa18ffd",
+                    "ceph.data_device": "/dev/test_group/data-lv1",
+                    "ceph.data_uuid": "TUpfel-Q5ZT-eFph-bdGW-SiNW-l0ag-f5kh00",
+                    "ceph.journal_device": "/dev/sdd1",
+                    "ceph.journal_uuid": "cd72bd28-002a-48da-bdf6-d5b993e84f3f",
+                    "ceph.osd_fsid": "943949f0-ce37-47ca-a33c-3413d46ee9ec",
+                    "ceph.osd_id": "0",
+                    "ceph.type": "data"
+                },
+                "type": "data",
+                "vg_name": "test_group"
+            }
+        ]
+    }
+
+
+Synchronized information
+------------------------
+Before any listing type, the lvm API is queried to ensure that physical devices
+that may be in use haven't changed naming. It is possible that non-persistent
+devices like ``/dev/sda1`` could change to ``/dev/sdb1``.
+
+The detection is possible because the ``PARTUUID`` is stored as part of the
+metadata in the logical volume for the data lv. Even in the case of a journal
+that is a physical device, this information is still stored on the data logical
+volume associated with it.
+
+If the name is no longer the same (as reported by ``blkid`` when using the
+``PARTUUID``), the tag will get updated and the report will use the newly
+refreshed information.
diff --git a/doc/ceph-volume/lvm/migrate.rst b/doc/ceph-volume/lvm/migrate.rst
new file mode 100644
index 000000000..983d2e797
--- /dev/null
+++ b/doc/ceph-volume/lvm/migrate.rst
@@ -0,0 +1,47 @@
+.. _ceph-volume-lvm-migrate:
+
+``migrate``
+===========
+
+Moves BlueFS data from source volume(s) to the target one, source volumes
+(except the main, i.e. data or block one) are removed on success.
+
+LVM volumes are permitted for Target only, both already attached or new one.
+
+In the latter case it is attached to the OSD replacing one of the source
+devices.
+
+Following replacement rules apply (in the order of precedence, stop
+on the first match):
+
+    - if source list has DB volume - target device replaces it.
+    - if source list has WAL volume - target device replaces it.
+    - if source list has slow volume only - operation is not permitted,
+      requires explicit allocation via new-db/new-wal command.
+
+Moves BlueFS data from main device to LV already attached as DB::
+
+    ceph-volume lvm migrate --osd-id 1 --osd-fsid <uuid> --from data --target vgname/db
+
+Moves BlueFS data from shared main device to LV which will be attached as a
+new DB::
+
+    ceph-volume lvm migrate --osd-id 1 --osd-fsid <uuid> --from data --target vgname/new_db
+
+Moves BlueFS data from DB device to new LV, DB is replaced::
+
+    ceph-volume lvm migrate --osd-id 1 --osd-fsid <uuid> --from db --target vgname/new_db
+
+Moves BlueFS data from main and DB devices to new LV, DB is replaced::
+
+    ceph-volume lvm migrate --osd-id 1 --osd-fsid <uuid> --from data db --target vgname/new_db
+
+Moves BlueFS data from main, DB and WAL devices to new LV, WAL is  removed and
+DB is replaced::
+
+    ceph-volume lvm migrate --osd-id 1 --osd-fsid <uuid> --from data db wal --target vgname/new_db
+
+Moves BlueFS data from main, DB and WAL devices to main device, WAL and DB are
+removed::
+
+    ceph-volume lvm migrate --osd-id 1 --osd-fsid <uuid> --from db wal --target vgname/data
diff --git a/doc/ceph-volume/lvm/newdb.rst b/doc/ceph-volume/lvm/newdb.rst
new file mode 100644
index 000000000..dcc87fc8a
--- /dev/null
+++ b/doc/ceph-volume/lvm/newdb.rst
@@ -0,0 +1,11 @@
+.. _ceph-volume-lvm-newdb:
+
+``new-db``
+===========
+
+Attaches the given logical volume to OSD as a DB.
+Logical volume name format is vg/lv. Fails if OSD has already got attached DB.
+
+Attach vgname/lvname as a DB volume to OSD 1::
+
+    ceph-volume lvm new-db --osd-id 1 --osd-fsid 55BD4219-16A7-4037-BC20-0F158EFCC83D --target vgname/new_db
diff --git a/doc/ceph-volume/lvm/newwal.rst b/doc/ceph-volume/lvm/newwal.rst
new file mode 100644
index 000000000..05f87fff6
--- /dev/null
+++ b/doc/ceph-volume/lvm/newwal.rst
@@ -0,0 +1,11 @@
+.. _ceph-volume-lvm-newwal:
+
+``new-wal``
+===========
+
+Attaches the given logical volume to the given OSD as a WAL volume.
+Logical volume format is vg/lv. Fails if OSD has already got attached DB.
+
+Attach vgname/lvname as a WAL volume to OSD 1::
+
+    ceph-volume lvm new-wal --osd-id 1 --osd-fsid 55BD4219-16A7-4037-BC20-0F158EFCC83D --target vgname/new_wal
diff --git a/doc/ceph-volume/lvm/prepare.rst b/doc/ceph-volume/lvm/prepare.rst
new file mode 100644
index 000000000..ae6aac414
--- /dev/null
+++ b/doc/ceph-volume/lvm/prepare.rst
@@ -0,0 +1,350 @@
+.. _ceph-volume-lvm-prepare:
+
+``prepare``
+===========
+Before you run ``ceph-volume lvm prepare``, we recommend that you provision a
+logical volume. Then you can run ``prepare`` on that logical volume. 
+
+``prepare`` adds metadata to logical volumes but does not alter them in any
+other way. 
+
+.. note:: This is part of a two-step process to deploy an OSD. If you prefer 
+   to deploy an OSD by using only one command, see :ref:`ceph-volume-lvm-create`.
+
+``prepare`` uses :term:`LVM tags` to assign several pieces of metadata to a
+logical volume. Volumes tagged in this way are easier to identify and easier to
+use with Ceph. :term:`LVM tags` identify logical volumes by the role that they
+play in the Ceph cluster (for example: BlueStore data or BlueStore WAL+DB).
+
+:term:`BlueStore<bluestore>` is the default backend. Ceph permits changing
+the backend, which can be done by using the following flags and arguments:
+
+* :ref:`--filestore <ceph-volume-lvm-prepare_filestore>`
+* :ref:`--bluestore <ceph-volume-lvm-prepare_bluestore>`
+
+.. _ceph-volume-lvm-prepare_bluestore:
+
+``bluestore``
+-------------
+:term:`Bluestore<bluestore>` is the default backend for new OSDs. It
+offers more flexibility for devices than :term:`filestore` does.  Bluestore
+supports the following configurations:
+
+* a block device, a block.wal device, and a block.db device
+* a block device and a block.wal device
+* a block device and a block.db device
+* a single block device
+
+The ``bluestore`` subcommand accepts physical block devices, partitions on physical
+block devices, or logical volumes as arguments for the various device
+parameters. If a physical block device is provided, a logical volume will be
+created. If the provided volume group's name begins with `ceph`, it will be
+created if it does not yet exist and it will be clobbered and reused if it
+already exists. This allows for a simpler approach to using LVM but at the
+cost of flexibility: no option or configuration can be used to change how the
+logical volume is created.
+
+The ``block`` is specified with the ``--data`` flag, and in its simplest use
+case it looks like:
+
+.. prompt:: bash #
+
+    ceph-volume lvm prepare --bluestore --data vg/lv
+
+A raw device can be specified in the same way:
+
+.. prompt:: bash #
+
+    ceph-volume lvm prepare --bluestore --data /path/to/device
+
+For enabling :ref:`encryption <ceph-volume-lvm-encryption>`, the ``--dmcrypt`` flag is required:
+
+.. prompt:: bash #
+
+    ceph-volume lvm prepare --bluestore --dmcrypt --data vg/lv
+
+If a ``block.db`` device or a ``block.wal`` device is needed, it can be
+specified with ``--block.db`` or ``--block.wal``. These can be physical
+devices, partitions, or logical volumes. ``block.db`` and ``block.wal`` are
+optional for bluestore.
+
+For both ``block.db`` and ``block.wal``, partitions can be used as-is, and 
+therefore are not made into logical volumes.
+
+While creating the OSD directory, the process uses a ``tmpfs`` mount to hold
+the files needed for the OSD. These files are created by ``ceph-osd --mkfs``
+and are ephemeral.
+
+A symlink is created for the ``block`` device, and is optional for ``block.db``
+and ``block.wal``. For a cluster with a default name and an OSD ID of 0, the
+directory looks like this::
+
+    # ls -l /var/lib/ceph/osd/ceph-0
+    lrwxrwxrwx. 1 ceph ceph 93 Oct 20 13:05 block -> /dev/ceph-be2b6fbd-bcf2-4c51-b35d-a35a162a02f0/osd-block-25cf0a05-2bc6-44ef-9137-79d65bd7ad62
+    lrwxrwxrwx. 1 ceph ceph 93 Oct 20 13:05 block.db -> /dev/sda1
+    lrwxrwxrwx. 1 ceph ceph 93 Oct 20 13:05 block.wal -> /dev/ceph/osd-wal-0
+    -rw-------. 1 ceph ceph 37 Oct 20 13:05 ceph_fsid
+    -rw-------. 1 ceph ceph 37 Oct 20 13:05 fsid
+    -rw-------. 1 ceph ceph 55 Oct 20 13:05 keyring
+    -rw-------. 1 ceph ceph  6 Oct 20 13:05 ready
+    -rw-------. 1 ceph ceph 10 Oct 20 13:05 type
+    -rw-------. 1 ceph ceph  2 Oct 20 13:05 whoami
+
+In the above case, a device was used for ``block``, so ``ceph-volume`` created
+a volume group and a logical volume using the following conventions:
+
+* volume group name: ``ceph-{cluster fsid}`` (or if the volume group already
+  exists: ``ceph-{random uuid}``)
+
+* logical volume name: ``osd-block-{osd_fsid}``
+
+
+.. _ceph-volume-lvm-prepare_filestore:
+
+``filestore``
+-------------
+``Filestore<filestore>`` is the OSD backend that prepares logical volumes for a
+:term:`filestore`-backed object-store OSD.
+
+
+``Filestore<filestore>`` uses a logical volume to store OSD data and it uses
+physical devices, partitions, or logical volumes to store the journal.  If a
+physical device is used to create a filestore backend, a logical volume will be
+created on that physical device. If the provided volume group's name begins
+with `ceph`, it will be created if it does not yet exist and it will be
+clobbered and reused if it already exists. No special preparation is needed for
+these volumes, but be sure to meet the minimum size requirements for OSD data and
+for the journal.
+
+Use the following command to create a basic filestore OSD:
+
+.. prompt:: bash #
+
+   ceph-volume lvm prepare --filestore --data <data block device>
+
+Use this command to deploy filestore with an external journal:
+
+.. prompt:: bash #
+
+   ceph-volume lvm prepare --filestore --data <data block device> --journal <journal block device>
+
+Use this command to enable :ref:`encryption <ceph-volume-lvm-encryption>`, and note that the ``--dmcrypt`` flag is required:
+
+.. prompt:: bash #
+
+   ceph-volume lvm prepare --filestore --dmcrypt --data <data block device> --journal <journal block device>
+
+The data block device and the journal can each take one of three forms: 
+
+* a physical block device
+* a partition on a physical block device
+* a logical volume
+
+If you use a logical volume to deploy filestore, the value that you pass in the
+command *must* be of the format ``volume_group/logical_volume_name``. Since logical
+volume names are not enforced for uniqueness, using this format is an important 
+safeguard against accidentally choosing the wrong volume (and clobbering its data).
+
+If you use a partition to deploy filestore, the partition *must* contain a
+``PARTUUID`` that can be discovered by ``blkid``. This ensures that the
+partition can be identified correctly regardless of the device's name (or path).
+
+For example, to use a logical volume for OSD data and a partition
+(``/dev/sdc1``) for the journal, run a command of this form:
+
+.. prompt:: bash #
+
+   ceph-volume lvm prepare --filestore --data volume_group/logical_volume_name --journal /dev/sdc1
+
+Or, to use a bare device for data and a logical volume for the journal:
+
+.. prompt:: bash #
+
+   ceph-volume lvm prepare --filestore --data /dev/sdc --journal volume_group/journal_lv
+
+A generated UUID is used when asking the cluster for a new OSD. These two
+pieces of information (the OSD ID and the OSD UUID) are necessary for
+identifying a given OSD and will later be used throughout the
+:ref:`activation<ceph-volume-lvm-activate>` process.
+
+The OSD data directory is created using the following convention::
+
+    /var/lib/ceph/osd/<cluster name>-<osd id>
+
+To link the journal volume to the mounted data volume, use this command:
+
+.. prompt:: bash #
+
+   ln -s /path/to/journal /var/lib/ceph/osd/<cluster_name>-<osd-id>/journal
+
+To fetch the monmap by using the bootstrap key from the OSD, use this command:
+
+.. prompt:: bash #
+
+   /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring
+   /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o
+   /var/lib/ceph/osd/<cluster name>-<osd id>/activate.monmap
+
+To populate the OSD directory (which has already been mounted), use this ``ceph-osd`` command:  
+.. prompt:: bash #
+
+   ceph-osd --cluster ceph --mkfs --mkkey -i <osd id> \ --monmap
+   /var/lib/ceph/osd/<cluster name>-<osd id>/activate.monmap --osd-data \
+   /var/lib/ceph/osd/<cluster name>-<osd id> --osd-journal
+   /var/lib/ceph/osd/<cluster name>-<osd id>/journal \ --osd-uuid <osd uuid>
+   --keyring /var/lib/ceph/osd/<cluster name>-<osd id>/keyring \ --setuser ceph
+   --setgroup ceph
+
+All of the information from the previous steps is used in the above command.      
+
+
+
+.. _ceph-volume-lvm-partitions:
+
+Partitioning
+------------
+``ceph-volume lvm`` does not currently create partitions from a whole device.
+If using device partitions the only requirement is that they contain the
+``PARTUUID`` and that it is discoverable by ``blkid``. Both ``fdisk`` and
+``parted`` will create that automatically for a new partition.
+
+For example, using a new, unformatted drive (``/dev/sdd`` in this case) we can
+use ``parted`` to create a new partition. First we list the device
+information::
+
+    $ parted --script /dev/sdd print
+    Model: VBOX HARDDISK (scsi)
+    Disk /dev/sdd: 11.5GB
+    Sector size (logical/physical): 512B/512B
+    Disk Flags:
+
+This device is not even labeled yet, so we can use ``parted`` to create
+a ``gpt`` label before we create a partition, and verify again with ``parted
+print``::
+
+    $ parted --script /dev/sdd mklabel gpt
+    $ parted --script /dev/sdd print
+    Model: VBOX HARDDISK (scsi)
+    Disk /dev/sdd: 11.5GB
+    Sector size (logical/physical): 512B/512B
+    Partition Table: gpt
+    Disk Flags:
+
+Now lets create a single partition, and verify later if ``blkid`` can find
+a ``PARTUUID`` that is needed by ``ceph-volume``::
+
+    $ parted --script /dev/sdd mkpart primary 1 100%
+    $ blkid /dev/sdd1
+    /dev/sdd1: PARTLABEL="primary" PARTUUID="16399d72-1e1f-467d-96ee-6fe371a7d0d4"
+
+
+.. _ceph-volume-lvm-existing-osds:
+
+Existing OSDs
+-------------
+For existing clusters that want to use this new system and have OSDs that are
+already running there are a few things to take into account:
+
+.. warning:: this process will forcefully format the data device, destroying
+             existing data, if any.
+
+* OSD paths should follow this convention::
+
+     /var/lib/ceph/osd/<cluster name>-<osd id>
+
+* Preferably, no other mechanisms to mount the volume should exist, and should
+  be removed (like fstab mount points)
+
+The one time process for an existing OSD, with an ID of 0 and using
+a ``"ceph"`` cluster name would look like (the following command will **destroy
+any data** in the OSD)::
+
+    ceph-volume lvm prepare --filestore --osd-id 0 --osd-fsid E3D291C1-E7BF-4984-9794-B60D9FA139CB
+
+The command line tool will not contact the monitor to generate an OSD ID and
+will format the LVM device in addition to storing the metadata on it so that it
+can be started later (for detailed metadata description see
+:ref:`ceph-volume-lvm-tags`).
+
+
+Crush device class
+------------------
+
+To set the crush device class for the OSD, use the ``--crush-device-class`` flag. This will
+work for both bluestore and filestore OSDs::
+
+    ceph-volume lvm prepare --bluestore --data vg/lv --crush-device-class foo
+
+
+.. _ceph-volume-lvm-multipath:
+
+``multipath`` support
+---------------------
+``multipath`` devices are supported if ``lvm`` is configured properly.
+
+**Leave it to LVM**
+
+Most Linux distributions should ship their LVM2 package with
+``multipath_component_detection = 1`` in the default configuration. With this
+setting ``LVM`` ignores any device that is a multipath component and
+``ceph-volume`` will accordingly not touch these devices.
+
+**Using filters**
+
+Should this setting be unavailable, a correct ``filter`` expression must be
+provided in ``lvm.conf``. ``ceph-volume`` must not be able to use both the
+multipath device and its multipath components.
+
+Storing metadata
+----------------
+The following tags will get applied as part of the preparation process
+regardless of the type of volume (journal or data) or OSD objectstore:
+
+* ``cluster_fsid``
+* ``encrypted``
+* ``osd_fsid``
+* ``osd_id``
+* ``crush_device_class``
+
+For :term:`filestore` these tags will be added:
+
+* ``journal_device``
+* ``journal_uuid``
+
+For :term:`bluestore` these tags will be added:
+
+* ``block_device``
+* ``block_uuid``
+* ``db_device``
+* ``db_uuid``
+* ``wal_device``
+* ``wal_uuid``
+
+.. note:: For the complete lvm tag conventions see :ref:`ceph-volume-lvm-tag-api`
+
+
+Summary
+-------
+To recap the ``prepare`` process for :term:`bluestore`:
+
+#. Accepts raw physical devices, partitions on physical devices or logical volumes as arguments.
+#. Creates logical volumes on any raw physical devices.
+#. Generate a UUID for the OSD
+#. Ask the monitor get an OSD ID reusing the generated UUID
+#. OSD data directory is created on a tmpfs mount.
+#. ``block``, ``block.wal``, and ``block.db`` are symlinked if defined.
+#. monmap is fetched for activation
+#. Data directory is populated by ``ceph-osd``
+#. Logical Volumes are assigned all the Ceph metadata using lvm tags
+
+
+And the ``prepare`` process for :term:`filestore`:
+
+#. Accepts raw physical devices, partitions on physical devices or logical volumes as arguments.
+#. Generate a UUID for the OSD
+#. Ask the monitor get an OSD ID reusing the generated UUID
+#. OSD data directory is created and data volume mounted
+#. Journal is symlinked from data volume to journal location
+#. monmap is fetched for activation
+#. devices is mounted and data directory is populated by ``ceph-osd``
+#. data and journal volumes are assigned all the Ceph metadata using lvm tags
diff --git a/doc/ceph-volume/lvm/scan.rst b/doc/ceph-volume/lvm/scan.rst
new file mode 100644
index 000000000..aa9990f71
--- /dev/null
+++ b/doc/ceph-volume/lvm/scan.rst
@@ -0,0 +1,9 @@
+scan
+====
+This sub-command will allow to discover Ceph volumes previously setup by the
+tool by looking into the system's logical volumes and their tags.
+
+As part of the :ref:`ceph-volume-lvm-prepare` process, the logical volumes are assigned
+a few tags with important pieces of information.
+
+.. note:: This sub-command is not yet implemented
diff --git a/doc/ceph-volume/lvm/systemd.rst b/doc/ceph-volume/lvm/systemd.rst
new file mode 100644
index 000000000..30260de7e
--- /dev/null
+++ b/doc/ceph-volume/lvm/systemd.rst
@@ -0,0 +1,28 @@
+.. _ceph-volume-lvm-systemd:
+
+systemd
+=======
+Upon startup, it will identify the logical volume using :term:`LVM tags`,
+finding a matching ID and later ensuring it is the right one with
+the :term:`OSD uuid`.
+
+After identifying the correct volume it will then proceed to mount it by using
+the OSD destination conventions, that is::
+
+    /var/lib/ceph/osd/<cluster name>-<osd id>
+
+For our example OSD with an id of ``0``, that means the identified device will
+be mounted at::
+
+
+    /var/lib/ceph/osd/ceph-0
+
+
+Once that process is complete, a call will be made to start the OSD::
+
+    systemctl start ceph-osd@0
+
+The systemd portion of this process is handled by the ``ceph-volume lvm
+trigger`` sub-command, which is only in charge of parsing metadata coming from
+systemd and startup, and then dispatching to ``ceph-volume lvm activate`` which
+would proceed with activation.
diff --git a/doc/ceph-volume/lvm/zap.rst b/doc/ceph-volume/lvm/zap.rst
new file mode 100644
index 000000000..e737fc386
--- /dev/null
+++ b/doc/ceph-volume/lvm/zap.rst
@@ -0,0 +1,65 @@
+.. _ceph-volume-lvm-zap:
+
+``zap``
+=======
+
+This subcommand is used to zap lvs, partitions or raw devices that have been used
+by ceph OSDs so that they may be reused. If given a path to a logical
+volume it must be in the format of vg/lv. Any file systems present
+on the given lv or partition will be removed and all data will be purged.
+
+.. note:: The lv or partition will be kept intact.
+
+.. note:: If the logical volume, raw device or partition is being used for any ceph related
+          mount points they will be unmounted.
+
+Zapping a logical volume::
+
+    ceph-volume lvm zap {vg name/lv name}
+
+Zapping a partition::
+
+    ceph-volume lvm zap /dev/sdc1
+
+Removing Devices
+----------------
+When zapping, and looking for full removal of the device (lv, vg, or partition)
+use the ``--destroy`` flag. A common use case is to simply deploy OSDs using
+a whole raw device. If you do so and then wish to reuse that device for another
+OSD you must use the ``--destroy`` flag when zapping so that the vgs and lvs
+that ceph-volume created on the raw device will be removed.
+
+.. note:: Multiple devices can be accepted at once, to zap them all
+
+Zapping a raw device and destroying any vgs or lvs present::
+
+    ceph-volume lvm zap /dev/sdc --destroy
+
+
+This action can be performed on partitions, and logical volumes as well::
+
+    ceph-volume lvm zap /dev/sdc1 --destroy
+    ceph-volume lvm zap osd-vg/data-lv --destroy
+
+
+Finally, multiple devices can be detected if filtering by OSD ID and/or OSD
+FSID. Either identifier can be used or both can be used at the same time. This
+is useful in situations where multiple devices associated with a specific ID
+need to be purged. When using the FSID, the filtering is stricter, and might
+not match other (possibly invalid) devices associated to an ID.
+
+By ID only::
+
+    ceph-volume lvm zap --destroy --osd-id 1
+
+By FSID::
+
+    ceph-volume lvm zap --destroy --osd-fsid 2E8FBE58-0328-4E3B-BFB7-3CACE4E9A6CE
+
+By both::
+
+    ceph-volume lvm zap --destroy --osd-fsid 2E8FBE58-0328-4E3B-BFB7-3CACE4E9A6CE --osd-id 1
+
+
+.. warning:: If the systemd unit associated with the OSD ID to be zapped is
+             detected as running, the tool will refuse to zap until the daemon is stopped.
diff --git a/doc/ceph-volume/simple/activate.rst b/doc/ceph-volume/simple/activate.rst
new file mode 100644
index 000000000..2b2795d0b
--- /dev/null
+++ b/doc/ceph-volume/simple/activate.rst
@@ -0,0 +1,80 @@
+.. _ceph-volume-simple-activate:
+
+``activate``
+============
+Once :ref:`ceph-volume-simple-scan` has been completed, and all the metadata
+captured for an OSD has been persisted to ``/etc/ceph/osd/{id}-{uuid}.json``
+the OSD is now ready to get "activated".
+
+This activation process **disables** all ``ceph-disk`` systemd units by masking
+them, to prevent the UDEV/ceph-disk interaction that will attempt to start them
+up at boot time.
+
+The disabling of ``ceph-disk`` units is done only when calling ``ceph-volume
+simple activate`` directly, but is avoided when being called by systemd when
+the system is booting up.
+
+The activation process requires using both the :term:`OSD id` and :term:`OSD uuid`
+To activate parsed OSDs::
+
+    ceph-volume simple activate 0 6cc43680-4f6e-4feb-92ff-9c7ba204120e
+
+The above command will assume that a JSON configuration will be found in::
+
+    /etc/ceph/osd/0-6cc43680-4f6e-4feb-92ff-9c7ba204120e.json
+
+Alternatively, using a path to a JSON file directly is also possible::
+
+    ceph-volume simple activate --file /etc/ceph/osd/0-6cc43680-4f6e-4feb-92ff-9c7ba204120e.json
+
+requiring uuids
+^^^^^^^^^^^^^^^
+The :term:`OSD uuid` is being required as an extra step to ensure that the
+right OSD is being activated. It is entirely possible that a previous OSD with
+the same id exists and would end up activating the incorrect one.
+
+
+Discovery
+---------
+With OSDs previously scanned by ``ceph-volume``, a *discovery* process is
+performed using ``blkid`` and ``lvm``. There is currently support only for
+devices with GPT partitions and LVM logical volumes.
+
+The GPT partitions will have a ``PARTUUID`` that can be queried by calling out
+to ``blkid``, and the logical volumes will have a ``lv_uuid`` that can be
+queried against ``lvs`` (the LVM tool to list logical volumes).
+
+This discovery process ensures that devices can be correctly detected even if
+they are repurposed into another system or if their name changes (as in the
+case of non-persisting names like ``/dev/sda1``)
+
+The JSON configuration file used to map what devices go to what OSD will then
+coordinate the mounting and symlinking as part of activation.
+
+To ensure that the symlinks are always correct, if they exist in the OSD
+directory, the symlinks will be re-done.
+
+A systemd unit will capture the :term:`OSD id` and :term:`OSD uuid` and
+persist it. Internally, the activation will enable it like::
+
+    systemctl enable ceph-volume@simple-$id-$uuid
+
+For example::
+
+    systemctl enable ceph-volume@simple-0-8715BEB4-15C5-49DE-BA6F-401086EC7B41
+
+Would start the discovery process for the OSD with an id of ``0`` and a UUID of
+``8715BEB4-15C5-49DE-BA6F-401086EC7B41``.
+
+
+The systemd process will call out to activate passing the information needed to
+identify the OSD and its devices, and it will proceed to:
+
+# mount the device in the corresponding location (by convention this is
+  ``/var/lib/ceph/osd/<cluster name>-<osd id>/``)
+
+# ensure that all required devices are ready for that OSD and properly linked,
+regardless of objectstore used (filestore or bluestore). The symbolic link will
+**always** be re-done to ensure that the correct device is linked.
+
+# start the ``ceph-osd@0`` systemd unit
diff --git a/doc/ceph-volume/simple/index.rst b/doc/ceph-volume/simple/index.rst
new file mode 100644
index 000000000..315dea99a
--- /dev/null
+++ b/doc/ceph-volume/simple/index.rst
@@ -0,0 +1,32 @@
+.. _ceph-volume-simple:
+
+``simple``
+==========
+Implements the functionality needed to manage OSDs from the ``simple`` subcommand:
+``ceph-volume simple``
+
+**Command Line Subcommands**
+
+* :ref:`ceph-volume-simple-scan`
+
+* :ref:`ceph-volume-simple-activate`
+
+* :ref:`ceph-volume-simple-systemd`
+
+
+By *taking over* management, it disables all ``ceph-disk`` systemd units used
+to trigger devices at startup, relying on basic (customizable) JSON
+configuration and systemd for starting up OSDs.
+
+This process involves two steps:
+
+#. :ref:`Scan <ceph-volume-simple-scan>` the running OSD or the data device
+#. :ref:`Activate <ceph-volume-simple-activate>` the scanned OSD
+
+The scanning will infer everything that ``ceph-volume`` needs to start the OSD,
+so that when activation is needed, the OSD can start normally without getting
+interference from ``ceph-disk``.
+
+As part of the activation process the systemd units for ``ceph-disk`` in charge
+of reacting to ``udev`` events, are linked to ``/dev/null`` so that they are
+fully inactive.
diff --git a/doc/ceph-volume/simple/scan.rst b/doc/ceph-volume/simple/scan.rst
new file mode 100644
index 000000000..2749b14b6
--- /dev/null
+++ b/doc/ceph-volume/simple/scan.rst
@@ -0,0 +1,176 @@
+.. _ceph-volume-simple-scan:
+
+``scan``
+========
+Scanning allows to capture any important details from an already-deployed OSD
+so that ``ceph-volume`` can manage it without the need of any other startup
+workflows or tools (like ``udev`` or ``ceph-disk``). Encryption with LUKS or
+PLAIN formats is fully supported.
+
+The command has the ability to inspect a running OSD, by inspecting the
+directory where the OSD data is stored, or by consuming the data partition.
+The command can also scan all running OSDs if no path or device is provided.
+
+Once scanned, information will (by default) persist the metadata as JSON in
+a file in ``/etc/ceph/osd``. This ``JSON`` file will use the naming convention
+of: ``{OSD ID}-{OSD FSID}.json``. An OSD with an id of 1, and an FSID like
+``86ebd829-1405-43d3-8fd6-4cbc9b6ecf96`` the absolute path of the file would
+be::
+
+    /etc/ceph/osd/1-86ebd829-1405-43d3-8fd6-4cbc9b6ecf96.json
+
+The ``scan`` subcommand will refuse to write to this file if it already exists.
+If overwriting the contents is needed, the ``--force`` flag must be used::
+
+    ceph-volume simple scan --force {path}
+
+If there is no need to persist the ``JSON`` metadata, there is support to send
+the contents to ``stdout`` (no file will be written)::
+
+    ceph-volume simple scan --stdout {path}
+
+
+.. _ceph-volume-simple-scan-directory:
+
+Running OSDs scan
+-----------------
+Using this command without providing an OSD directory or device will scan the
+directories of any currently running OSDs. If a running OSD was not created
+by ceph-disk it will be ignored and not scanned.
+
+To scan all running ceph-disk OSDs, the command would look like::
+
+    ceph-volume simple scan
+
+Directory scan
+--------------
+The directory scan will capture OSD file contents from interesting files. There
+are a few files that must exist in order to have a successful scan:
+
+* ``ceph_fsid``
+* ``fsid``
+* ``keyring``
+* ``ready``
+* ``type``
+* ``whoami``
+
+If the OSD is encrypted, it will additionally add the following keys:
+
+* ``encrypted``
+* ``encryption_type``
+* ``lockbox_keyring``
+
+In the case of any other file, as long as it is not a binary or a directory, it
+will also get captured and persisted as part of the JSON object.
+
+The convention for the keys in the JSON object is that any file name will be
+a key, and its contents will be its value. If the contents are a single line
+(like in the case of the ``whoami``) the contents are trimmed, and the newline
+is dropped. For example with an OSD with an id of 1, this is how the JSON entry
+would look like::
+
+    "whoami": "1",
+
+For files that may have more than one line, the contents are left as-is, except
+for keyrings which are treated specially and parsed to extract the keyring. For
+example, a ``keyring`` that gets read as::
+
+    [osd.1]\n\tkey = AQBBJ/dZp57NIBAAtnuQS9WOS0hnLVe0rZnE6Q==\n
+
+Would get stored as::
+
+    "keyring": "AQBBJ/dZp57NIBAAtnuQS9WOS0hnLVe0rZnE6Q==",
+
+
+For a directory like ``/var/lib/ceph/osd/ceph-1``, the command could look
+like::
+
+    ceph-volume simple scan /var/lib/ceph/osd/ceph1
+
+
+.. _ceph-volume-simple-scan-device:
+
+Device scan
+-----------
+When an OSD directory is not available (OSD is not running, or device is not
+mounted) the ``scan`` command is able to introspect the device to capture
+required data. Just like :ref:`ceph-volume-simple-scan-directory`, it would
+still require a few files present. This means that the device to be scanned
+**must be** the data partition of the OSD.
+
+As long as the data partition of the OSD is being passed in as an argument, the
+sub-command can scan its contents.
+
+In the case where the device is already mounted, the tool can detect this
+scenario and capture file contents from that directory.
+
+If the device is not mounted, a temporary directory will be created, and the
+device will be mounted temporarily just for scanning the contents. Once
+contents are scanned, the device will be unmounted.
+
+For a device like ``/dev/sda1`` which **must** be a data partition, the command
+could look like::
+
+    ceph-volume simple scan /dev/sda1
+
+
+.. _ceph-volume-simple-scan-json:
+
+``JSON`` contents
+-----------------
+The contents of the JSON object is very simple. The scan not only will persist
+information from the special OSD files and their contents, but will also
+validate paths and device UUIDs. Unlike what ``ceph-disk`` would do, by storing
+them in ``{device type}_uuid`` files, the tool will persist them as part of the
+device type key.
+
+For example, a ``block.db`` device would look something like::
+
+    "block.db": {
+        "path": "/dev/disk/by-partuuid/6cc43680-4f6e-4feb-92ff-9c7ba204120e",
+        "uuid": "6cc43680-4f6e-4feb-92ff-9c7ba204120e"
+    },
+
+But it will also persist the ``ceph-disk`` special file generated, like so::
+
+    "block.db_uuid": "6cc43680-4f6e-4feb-92ff-9c7ba204120e",
+
+This duplication is in place because the tool is trying to ensure the
+following:
+
+# Support OSDs that may not have ceph-disk special files
+# Check the most up-to-date information on the device, by querying against LVM
+and ``blkid``
+# Support both logical volumes and GPT devices
+
+This is a sample ``JSON`` metadata, from an OSD that is using ``bluestore``::
+
+    {
+        "active": "ok",
+        "block": {
+            "path": "/dev/disk/by-partuuid/40fd0a64-caa5-43a3-9717-1836ac661a12",
+            "uuid": "40fd0a64-caa5-43a3-9717-1836ac661a12"
+        },
+        "block.db": {
+            "path": "/dev/disk/by-partuuid/6cc43680-4f6e-4feb-92ff-9c7ba204120e",
+            "uuid": "6cc43680-4f6e-4feb-92ff-9c7ba204120e"
+        },
+        "block.db_uuid": "6cc43680-4f6e-4feb-92ff-9c7ba204120e",
+        "block_uuid": "40fd0a64-caa5-43a3-9717-1836ac661a12",
+        "bluefs": "1",
+        "ceph_fsid": "c92fc9eb-0610-4363-aafc-81ddf70aaf1b",
+        "cluster_name": "ceph",
+        "data": {
+            "path": "/dev/sdr1",
+            "uuid": "86ebd829-1405-43d3-8fd6-4cbc9b6ecf96"
+        },
+        "fsid": "86ebd829-1405-43d3-8fd6-4cbc9b6ecf96",
+        "keyring": "AQBBJ/dZp57NIBAAtnuQS9WOS0hnLVe0rZnE6Q==",
+        "kv_backend": "rocksdb",
+        "magic": "ceph osd volume v026",
+        "mkfs_done": "yes",
+        "ready": "ready",
+        "systemd": "",
+        "type": "bluestore",
+        "whoami": "3"
+    }
diff --git a/doc/ceph-volume/simple/systemd.rst b/doc/ceph-volume/simple/systemd.rst
new file mode 100644
index 000000000..aa5bebffe
--- /dev/null
+++ b/doc/ceph-volume/simple/systemd.rst
@@ -0,0 +1,28 @@
+.. _ceph-volume-simple-systemd:
+
+systemd
+=======
+Upon startup, it will identify the logical volume by loading the JSON file in
+``/etc/ceph/osd/{id}-{uuid}.json`` corresponding to the instance name of the
+systemd unit.
+
+After identifying the correct volume it will then proceed to mount it by using
+the OSD destination conventions, that is::
+
+    /var/lib/ceph/osd/{cluster name}-{osd id}
+
+For our example OSD with an id of ``0``, that means the identified device will
+be mounted at::
+
+
+    /var/lib/ceph/osd/ceph-0
+
+
+Once that process is complete, a call will be made to start the OSD::
+
+    systemctl start ceph-osd@0
+
+The systemd portion of this process is handled by the ``ceph-volume simple
+trigger`` sub-command, which is only in charge of parsing metadata coming from
+systemd and startup, and then dispatching to ``ceph-volume simple activate`` which
+would proceed with activation.
diff --git a/doc/ceph-volume/systemd.rst b/doc/ceph-volume/systemd.rst
new file mode 100644
index 000000000..5b5273c9c
--- /dev/null
+++ b/doc/ceph-volume/systemd.rst
@@ -0,0 +1,49 @@
+.. _ceph-volume-systemd:
+
+systemd
+=======
+As part of the activation process (either with :ref:`ceph-volume-lvm-activate`
+or :ref:`ceph-volume-simple-activate`), systemd units will get enabled that
+will use the OSD id and uuid as part of their name. These units will be run
+when the system boots, and will proceed to activate their corresponding
+volumes via their sub-command implementation.
+
+The API for activation is a bit loose, it only requires two parts: the
+subcommand to use and any extra meta information separated by a dash. This
+convention makes the units look like::
+
+    ceph-volume@{command}-{extra metadata}
+
+The *extra metadata* can be anything needed that the subcommand implementing
+the processing might need. In the case of :ref:`ceph-volume-lvm` and
+:ref:`ceph-volume-simple`, both look to consume the :term:`OSD id` and :term:`OSD uuid`,
+but this is not a hard requirement, it is just how the sub-commands are
+implemented.
+
+Both the command and extra metadata gets persisted by systemd as part of the
+*"instance name"* of the unit.  For example an OSD with an ID of 0, for the
+``lvm`` sub-command would look like::
+
+    systemctl enable ceph-volume@lvm-0-0A3E1ED2-DA8A-4F0E-AA95-61DEC71768D6
+
+The enabled unit is a :term:`systemd oneshot` service, meant to start at boot
+after the local file system is ready to be used.
+
+
+Failure and Retries
+-------------------
+It is common to have failures when a system is coming up online. The devices
+are sometimes not fully available and this unpredictable behavior may cause an
+OSD to not be ready to be used.
+
+There are two configurable environment variables used to set the retry
+behavior:
+
+* ``CEPH_VOLUME_SYSTEMD_TRIES``: Defaults to 30
+* ``CEPH_VOLUME_SYSTEMD_INTERVAL``: Defaults to 5
+
+The *"tries"* is a number that sets the maximum number of times the unit will
+attempt to activate an OSD before giving up.
+
+The *"interval"* is a value in seconds that determines the waiting time before
+initiating another try at activating the OSD.
diff --git a/doc/ceph-volume/zfs/index.rst b/doc/ceph-volume/zfs/index.rst
new file mode 100644
index 000000000..c06228de9
--- /dev/null
+++ b/doc/ceph-volume/zfs/index.rst
@@ -0,0 +1,31 @@
+.. _ceph-volume-zfs:
+
+``zfs``
+=======
+Implements the functionality needed to deploy OSDs from the ``zfs`` subcommand:
+``ceph-volume zfs``
+
+The current implementation only works for ZFS on FreeBSD
+
+**Command Line Subcommands**
+
+* :ref:`ceph-volume-zfs-inventory`
+
+.. not yet implemented
+.. * :ref:`ceph-volume-zfs-prepare`
+
+.. * :ref:`ceph-volume-zfs-activate`
+
+.. * :ref:`ceph-volume-zfs-create`
+
+.. * :ref:`ceph-volume-zfs-list`
+
+.. * :ref:`ceph-volume-zfs-scan`
+
+**Internal functionality**
+
+There are other aspects of the ``zfs`` subcommand that are internal and not
+exposed to the user, these sections explain how these pieces work together,
+clarifying the workflows of the tool.
+
+:ref:`zfs <ceph-volume-zfs-api>`
diff --git a/doc/ceph-volume/zfs/inventory.rst b/doc/ceph-volume/zfs/inventory.rst
new file mode 100644
index 000000000..fd00325b6
--- /dev/null
+++ b/doc/ceph-volume/zfs/inventory.rst
@@ -0,0 +1,19 @@
+.. _ceph-volume-zfs-inventory:
+
+``inventory``
+=============
+The ``inventory`` subcommand queries a host's disc inventory through GEOM and provides
+hardware information and metadata on every physical device.
+
+This only works on a FreeBSD platform.
+
+By default the command returns a short, human-readable report of all physical disks.
+
+For programmatic consumption of this report pass ``--format json`` to generate a
+JSON formatted report. This report includes extensive information on the
+physical drives such as disk metadata (like model and size), logical volumes
+and whether they are used by ceph, and if the disk is usable by ceph and
+reasons why not.
+
+A device path can be specified to report extensive information on a device in
+both plain and json format.
diff --git a/doc/cephadm/adoption.rst b/doc/cephadm/adoption.rst
new file mode 100644
index 000000000..78d1343eb
--- /dev/null
+++ b/doc/cephadm/adoption.rst
@@ -0,0 +1,213 @@
+.. _cephadm-adoption:
+
+Converting an existing cluster to cephadm
+=========================================
+
+It is possible to convert some existing clusters so that they can be managed
+with ``cephadm``. This statment applies to some clusters that were deployed
+with ``ceph-deploy``, ``ceph-ansible``, or ``DeepSea``.
+
+This section of the documentation explains how to determine whether your
+clusters can be converted to a state in which they can be managed by
+``cephadm`` and how to perform those conversions.
+
+Limitations
+-----------
+
+* Cephadm works only with BlueStore OSDs. FileStore OSDs that are in your
+  cluster cannot be managed with ``cephadm``.
+
+Preparation
+-----------
+
+#. Make sure that the ``cephadm`` command line tool is available on each host
+   in the existing cluster.  See :ref:`get-cephadm` to learn how.
+
+#. Prepare each host for use by ``cephadm`` by running this command:
+
+   .. prompt:: bash #
+
+      cephadm prepare-host
+
+#. Choose a version of Ceph to use for the conversion. This procedure will work
+   with any release of Ceph that is Octopus (15.2.z) or later, inclusive.  The
+   latest stable release of Ceph is the default. You might be upgrading from an
+   earlier Ceph release at the same time that you're performing this
+   conversion; if you are upgrading from an earlier release, make sure to
+   follow any upgrade-related instructions for that release.
+
+   Pass the image to cephadm with the following command:
+
+   .. prompt:: bash #
+
+      cephadm --image $IMAGE <rest of command goes here>
+
+   The conversion begins.
+
+#. Confirm that the conversion is underway by running ``cephadm ls`` and
+   making sure that the style of the daemons is changed:
+
+   .. prompt:: bash #
+
+      cephadm ls
+
+   Before starting the converstion process, ``cephadm ls`` shows all existing
+   daemons to have a style of ``legacy``. As the adoption process progresses,
+   adopted daemons will appear with a style of ``cephadm:v1``.
+
+
+Adoption process
+----------------
+
+#. Make sure that the ceph configuration has been migrated to use the cluster
+   config database.  If the ``/etc/ceph/ceph.conf`` is identical on each host,
+   then the following command can be run on one single host and will affect all
+   hosts:
+
+   .. prompt:: bash #
+
+      ceph config assimilate-conf -i /etc/ceph/ceph.conf
+
+   If there are configuration variations between hosts, you will need to repeat
+   this command on each host. During this adoption process, view the cluster's
+   configuration to confirm that it is complete by running the following
+   command:
+
+   .. prompt:: bash #
+
+      ceph config dump
+
+#. Adopt each monitor:
+
+   .. prompt:: bash #
+
+      cephadm adopt --style legacy --name mon.<hostname>
+
+   Each legacy monitor should stop, quickly restart as a cephadm
+   container, and rejoin the quorum.
+
+#. Adopt each manager:
+
+   .. prompt:: bash #
+
+      cephadm adopt --style legacy --name mgr.<hostname>
+
+#. Enable cephadm:
+
+   .. prompt:: bash #
+
+      ceph mgr module enable cephadm
+      ceph orch set backend cephadm
+
+#. Generate an SSH key:
+
+   .. prompt:: bash #
+
+      ceph cephadm generate-key
+      ceph cephadm get-pub-key > ~/ceph.pub
+
+#. Install the cluster SSH key on each host in the cluster:
+
+   .. prompt:: bash #
+
+      ssh-copy-id -f -i ~/ceph.pub root@<host>
+
+   .. note::
+     It is also possible to import an existing SSH key. See
+     :ref:`SSH errors <cephadm-ssh-errors>` in the troubleshooting
+     document for instructions that describe how to import existing
+     SSH keys.
+
+   .. note::
+     It is also possible to have cephadm use a non-root user to SSH 
+     into cluster hosts. This user needs to have passwordless sudo access.
+     Use ``ceph cephadm set-user <user>`` and copy the SSH key to that user.
+     See :ref:`cephadm-ssh-user`
+
+#. Tell cephadm which hosts to manage:
+
+   .. prompt:: bash #
+
+      ceph orch host add <hostname> [ip-address]
+
+   This will perform a ``cephadm check-host`` on each host before adding it;
+   this check ensures that the host is functioning properly. The IP address
+   argument is recommended; if not provided, then the host name will be resolved
+   via DNS.
+
+#. Verify that the adopted monitor and manager daemons are visible:
+
+   .. prompt:: bash #
+
+      ceph orch ps
+
+#. Adopt all OSDs in the cluster:
+
+   .. prompt:: bash #
+
+      cephadm adopt --style legacy --name <name>
+
+   For example:
+
+   .. prompt:: bash #
+
+      cephadm adopt --style legacy --name osd.1
+      cephadm adopt --style legacy --name osd.2
+
+#. Redeploy MDS daemons by telling cephadm how many daemons to run for
+   each file system. List file systems by name with the command ``ceph fs
+   ls``. Run the following command on the master nodes to redeploy the MDS
+   daemons:
+
+   .. prompt:: bash #
+
+      ceph orch apply mds <fs-name> [--placement=<placement>]
+
+   For example, in a cluster with a single file system called `foo`:
+
+   .. prompt:: bash #
+
+      ceph fs ls
+
+   .. code-block:: bash
+
+      name: foo, metadata pool: foo_metadata, data pools: [foo_data ]
+
+   .. prompt:: bash #
+
+      ceph orch apply mds foo 2
+
+   Confirm that the new MDS daemons have started:
+
+   .. prompt:: bash #
+
+      ceph orch ps --daemon-type mds
+
+   Finally, stop and remove the legacy MDS daemons:
+
+   .. prompt:: bash #
+
+      systemctl stop ceph-mds.target
+      rm -rf /var/lib/ceph/mds/ceph-*
+
+#. Redeploy RGW daemons. Cephadm manages RGW daemons by zone. For each
+   zone, deploy new RGW daemons with cephadm:
+
+   .. prompt:: bash #
+
+      ceph orch apply rgw <svc_id> [--realm=<realm>] [--zone=<zone>] [--port=<port>] [--ssl] [--placement=<placement>]
+
+   where *<placement>* can be a simple daemon count, or a list of
+   specific hosts (see :ref:`orchestrator-cli-placement-spec`), and the
+   zone and realm arguments are needed only for a multisite setup.
+
+   After the daemons have started and you have confirmed that they are
+   functioning, stop and remove the old, legacy daemons:
+
+   .. prompt:: bash #
+
+      systemctl stop ceph-rgw.target
+      rm -rf /var/lib/ceph/radosgw/ceph-*
+
+#. Check the output of the command ``ceph health detail`` for cephadm warnings
+   about stray cluster daemons or hosts that are not yet managed by cephadm.
diff --git a/doc/cephadm/client-setup.rst b/doc/cephadm/client-setup.rst
new file mode 100644
index 000000000..f98ba798b
--- /dev/null
+++ b/doc/cephadm/client-setup.rst
@@ -0,0 +1,45 @@
+=======================
+Basic Ceph Client Setup
+=======================
+Client machines require some basic configuration to interact with
+Ceph clusters. This section describes how to configure a client machine
+so that it can interact with a Ceph cluster.
+
+.. note:: 
+   Most client machines need to install only the `ceph-common` package
+   and its dependencies. Such a setup supplies the basic `ceph` and
+   `rados` commands, as well as other commands including `mount.ceph`
+   and `rbd`.
+
+Config File Setup
+=================
+Client machines usually require smaller configuration files (here
+sometimes called "config files") than do full-fledged cluster members.
+To generate a minimal config file, log into a host that has been
+configured as a client or that is running a cluster daemon, and then run the following command:
+
+.. prompt:: bash #
+
+  ceph config generate-minimal-conf
+
+This command generates a minimal config file that tells the client how
+to reach the Ceph monitors. The contents of this file should usually 
+be installed in ``/etc/ceph/ceph.conf``.
+
+Keyring Setup
+=============
+Most Ceph clusters run with authentication enabled. This means that
+the client needs keys in order to communicate with the machines in the
+cluster. To generate a keyring file with credentials for `client.fs`,
+log into an running cluster member and run the following command:
+
+.. prompt:: bash $
+
+  ceph auth get-or-create client.fs
+
+The resulting output is directed into a keyring file, typically
+``/etc/ceph/ceph.keyring``.
+
+To gain a broader understanding of client keyring distribution and administration, you should read :ref:`client_keyrings_and_configs`.
+
+To see an example that explains how to distribute ``ceph.conf`` configuration files to hosts that are tagged with the ``bare_config`` label, you should read the section called "Distributing ceph.conf to hosts tagged with bare_config" in the section called :ref:`etc_ceph_conf_distribution`.
diff --git a/doc/cephadm/compatibility.rst b/doc/cephadm/compatibility.rst
new file mode 100644
index 000000000..7d9c763bb
--- /dev/null
+++ b/doc/cephadm/compatibility.rst
@@ -0,0 +1,58 @@
+
+===========================
+Compatibility and Stability
+===========================
+
+.. _cephadm-compatibility-with-podman:
+
+Compatibility with Podman Versions
+----------------------------------
+
+Podman and Ceph have different end-of-life strategies. This means that care
+must be taken in finding a version of Podman that is compatible with Ceph.
+
+These versions are expected to work:
+
+
++-----------+---------------------------------------+
+|  Ceph     |                 Podman                |
++-----------+-------+-------+-------+-------+-------+
+|           | 1.9   |  2.0  |  2.1  |  2.2  |  3.0  |
++===========+=======+=======+=======+=======+=======+
+| <= 15.2.5 | True  | False | False | False | False |
++-----------+-------+-------+-------+-------+-------+
+| >= 15.2.6 | True  | True  | True  | False | False |
++-----------+-------+-------+-------+-------+-------+
+| >= 16.2.1 | False | True  | True  | False | True  |
++-----------+-------+-------+-------+-------+-------+
+
+.. warning:: 
+
+   To use Podman with Ceph Pacific, you must use **a version of Podman that
+   is 2.0.0 or higher**. However, **Podman version 2.2.1 does not work with
+   Ceph Pacific**.
+   
+   "Kubic stable" is known to work with Ceph Pacific, but it must be run
+   with a newer kernel.
+
+
+.. _cephadm-stability:
+
+Stability
+---------
+
+Cephadm is under development. Some functionality is incomplete. Be aware
+that some of the components of Ceph may not work perfectly with cephadm.
+These include:
+
+- RGW
+
+Cephadm support remains under development for the following features:
+
+- Ingress
+- Cephadm exporter daemon
+- cephfs-mirror
+
+If a cephadm command fails or a service stops running properly, see
+:ref:`cephadm-pause` for instructions on how to pause the Ceph cluster's
+background activity and how to disable cephadm.
diff --git a/doc/cephadm/host-management.rst b/doc/cephadm/host-management.rst
new file mode 100644
index 000000000..df9525ca8
--- /dev/null
+++ b/doc/cephadm/host-management.rst
@@ -0,0 +1,436 @@
+.. _orchestrator-cli-host-management:
+
+===============
+Host Management
+===============
+
+Listing Hosts
+=============
+
+Run a command of this form to list hosts associated with the cluster:
+
+.. prompt:: bash #
+
+   ceph orch host ls [--format yaml] [--host-pattern <name>] [--label <label>] [--host-status <status>]
+
+In commands of this form, the arguments "host-pattern", "label" and
+"host-status" are optional and are used for filtering. 
+
+- "host-pattern" is a regex that matches against hostnames and returns only
+  matching hosts.
+- "label" returns only hosts with the specified label.
+- "host-status" returns only hosts with the specified status (currently
+  "offline" or "maintenance").
+- Any combination of these filtering flags is valid. It is possible to filter
+  against name, label and status simultaneously, or to filter against any
+  proper subset of name, label and status.
+
+.. _cephadm-adding-hosts:    
+    
+Adding Hosts
+============
+
+Hosts must have these :ref:`cephadm-host-requirements` installed.
+Hosts without all the necessary requirements will fail to be added to the cluster.
+
+To add each new host to the cluster, perform two steps:
+
+#. Install the cluster's public SSH key in the new host's root user's ``authorized_keys`` file:
+
+   .. prompt:: bash #
+
+      ssh-copy-id -f -i /etc/ceph/ceph.pub root@*<new-host>*
+
+   For example:
+
+   .. prompt:: bash #
+
+      ssh-copy-id -f -i /etc/ceph/ceph.pub root@host2
+      ssh-copy-id -f -i /etc/ceph/ceph.pub root@host3
+
+#. Tell Ceph that the new node is part of the cluster:
+
+   .. prompt:: bash #
+
+      ceph orch host add *<newhost>* [*<ip>*] [*<label1> ...*]
+
+   For example:
+
+   .. prompt:: bash #
+
+     ceph orch host add host2 10.10.0.102
+     ceph orch host add host3 10.10.0.103
+
+   It is best to explicitly provide the host IP address.  If an IP is
+   not provided, then the host name will be immediately resolved via
+   DNS and that IP will be used.
+
+   One or more labels can also be included to immediately label the
+   new host.  For example, by default the ``_admin`` label will make
+   cephadm maintain a copy of the ``ceph.conf`` file and a
+   ``client.admin`` keyring file in ``/etc/ceph``:
+
+   .. prompt:: bash #
+
+      ceph orch host add host4 10.10.0.104 --labels _admin
+
+.. _cephadm-removing-hosts:
+
+Removing Hosts
+==============
+
+A host can safely be removed from the cluster after all daemons are removed
+from it.
+
+To drain all daemons from a host, run a command of the following form:
+
+.. prompt:: bash #
+
+   ceph orch host drain *<host>*
+
+The ``_no_schedule`` label will be applied to the host. See
+:ref:`cephadm-special-host-labels`.
+
+All OSDs on the host will be scheduled to be removed. You can check the progress of the OSD removal operation with the following command:
+
+.. prompt:: bash #
+
+   ceph orch osd rm status
+
+See :ref:`cephadm-osd-removal` for more details about OSD removal.
+
+Use the following command to determine whether any daemons are still on the
+host:
+
+.. prompt:: bash #
+
+   ceph orch ps <host> 
+
+After all daemons have been removed from the host, remove the host from the
+cluster by running the following command: 
+
+.. prompt:: bash #
+
+   ceph orch host rm <host>
+
+Offline host removal
+--------------------
+
+Even if a host is offline and can not be recovered, it can be removed from the
+cluster by running a command of the following form:
+
+.. prompt:: bash #
+
+   ceph orch host rm <host> --offline --force
+
+.. warning:: This can potentially cause data loss. This command forcefully
+   purges OSDs from the cluster by calling ``osd purge-actual`` for each OSD.
+   Any service specs that still contain this host should be manually updated.
+
+.. _orchestrator-host-labels:
+
+Host labels
+===========
+
+The orchestrator supports assigning labels to hosts. Labels
+are free form and have no particular meaning by itself and each host
+can have multiple labels. They can be used to specify placement
+of daemons. See :ref:`orch-placement-by-labels`
+
+Labels can be added when adding a host with the ``--labels`` flag:
+
+.. prompt:: bash #
+
+   ceph orch host add my_hostname --labels=my_label1
+   ceph orch host add my_hostname --labels=my_label1,my_label2
+
+To add a label a existing host, run:
+
+.. prompt:: bash #
+
+   ceph orch host label add my_hostname my_label
+
+To remove a label, run:
+
+.. prompt:: bash #
+
+   ceph orch host label rm my_hostname my_label
+
+
+.. _cephadm-special-host-labels:
+
+Special host labels
+-------------------
+
+The following host labels have a special meaning to cephadm.  All start with ``_``.
+
+* ``_no_schedule``: *Do not schedule or deploy daemons on this host*.
+
+  This label prevents cephadm from deploying daemons on this host.  If it is added to
+  an existing host that already contains Ceph daemons, it will cause cephadm to move
+  those daemons elsewhere (except OSDs, which are not removed automatically).
+
+* ``_no_autotune_memory``: *Do not autotune memory on this host*.
+
+  This label will prevent daemon memory from being tuned even when the
+  ``osd_memory_target_autotune`` or similar option is enabled for one or more daemons
+  on that host.
+
+* ``_admin``: *Distribute client.admin and ceph.conf to this host*.
+
+  By default, an ``_admin`` label is applied to the first host in the cluster (where
+  bootstrap was originally run), and the ``client.admin`` key is set to be distributed
+  to that host via the ``ceph orch client-keyring ...`` function.  Adding this label
+  to additional hosts will normally cause cephadm to deploy config and keyring files
+  in ``/etc/ceph``.
+
+Maintenance Mode
+================
+
+Place a host in and out of maintenance mode (stops all Ceph daemons on host):
+
+.. prompt:: bash #
+
+   ceph orch host maintenance enter <hostname> [--force]
+   ceph orch host maintenance exit <hostname>
+
+Where the force flag when entering maintenance allows the user to bypass warnings (but not alerts)
+
+See also :ref:`cephadm-fqdn`
+
+Rescanning Host Devices
+=======================
+
+Some servers and external enclosures may not register device removal or insertion with the
+kernel. In these scenarios, you'll need to perform a host rescan. A rescan is typically
+non-disruptive, and can be performed with the following CLI command:
+
+.. prompt:: bash #
+
+   ceph orch host rescan <hostname> [--with-summary]
+
+The ``with-summary`` flag provides a breakdown of the number of HBAs found and scanned, together
+with any that failed:
+
+.. prompt:: bash [ceph:root@rh9-ceph1/]#
+
+   ceph orch host rescan rh9-ceph1 --with-summary
+   
+::
+
+   Ok. 2 adapters detected: 2 rescanned, 0 skipped, 0 failed (0.32s)
+
+Creating many hosts at once
+===========================
+
+Many hosts can be added at once using
+``ceph orch apply -i`` by submitting a multi-document YAML file:
+
+.. code-block:: yaml
+
+    service_type: host
+    hostname: node-00
+    addr: 192.168.0.10
+    labels:
+    - example1
+    - example2
+    ---
+    service_type: host
+    hostname: node-01
+    addr: 192.168.0.11
+    labels:
+    - grafana
+    ---
+    service_type: host
+    hostname: node-02
+    addr: 192.168.0.12
+
+This can be combined with service specifications (below) to create a cluster spec
+file to deploy a whole cluster in one command.  see ``cephadm bootstrap --apply-spec``
+also to do this during bootstrap. Cluster SSH Keys must be copied to hosts prior to adding them.
+
+Setting the initial CRUSH location of host
+==========================================
+
+Hosts can contain a ``location`` identifier which will instruct cephadm to 
+create a new CRUSH host located in the specified hierachy.
+
+.. code-block:: yaml
+
+    service_type: host
+    hostname: node-00
+    addr: 192.168.0.10
+    location:
+      rack: rack1
+
+.. note:: 
+
+  The ``location`` attribute will be only affect the initial CRUSH location. Subsequent
+  changes of the ``location`` property will be ignored. Also, removing a host will no remove
+  any CRUSH buckets.
+
+See also :ref:`crush_map_default_types`.
+
+SSH Configuration
+=================
+
+Cephadm uses SSH to connect to remote hosts.  SSH uses a key to authenticate
+with those hosts in a secure way.
+
+
+Default behavior
+----------------
+
+Cephadm stores an SSH key in the monitor that is used to
+connect to remote hosts.  When the cluster is bootstrapped, this SSH
+key is generated automatically and no additional configuration
+is necessary.
+
+A *new* SSH key can be generated with:
+
+.. prompt:: bash #
+
+   ceph cephadm generate-key
+
+The public portion of the SSH key can be retrieved with:
+
+.. prompt:: bash #
+
+   ceph cephadm get-pub-key
+
+The currently stored SSH key can be deleted with:
+
+.. prompt:: bash #
+
+   ceph cephadm clear-key
+
+You can make use of an existing key by directly importing it with:
+
+.. prompt:: bash #
+
+   ceph config-key set mgr/cephadm/ssh_identity_key -i <key>
+   ceph config-key set mgr/cephadm/ssh_identity_pub -i <pub>
+
+You will then need to restart the mgr daemon to reload the configuration with:
+
+.. prompt:: bash #
+
+   ceph mgr fail
+
+.. _cephadm-ssh-user:
+
+Configuring a different SSH user
+----------------------------------
+
+Cephadm must be able to log into all the Ceph cluster nodes as an user
+that has enough privileges to download container images, start containers
+and execute commands without prompting for a password. If you do not want
+to use the "root" user (default option in cephadm), you must provide
+cephadm the name of the user that is going to be used to perform all the
+cephadm operations. Use the command:
+
+.. prompt:: bash #
+
+   ceph cephadm set-user <user>
+
+Prior to running this the cluster SSH key needs to be added to this users
+authorized_keys file and non-root users must have passwordless sudo access.
+
+
+Customizing the SSH configuration
+---------------------------------
+
+Cephadm generates an appropriate ``ssh_config`` file that is
+used for connecting to remote hosts.  This configuration looks
+something like this::
+
+  Host *
+  User root
+  StrictHostKeyChecking no
+  UserKnownHostsFile /dev/null
+
+There are two ways to customize this configuration for your environment:
+
+#. Import a customized configuration file that will be stored
+   by the monitor with:
+
+   .. prompt:: bash #
+
+      ceph cephadm set-ssh-config -i <ssh_config_file>
+
+   To remove a customized SSH config and revert back to the default behavior:
+
+   .. prompt:: bash #
+
+      ceph cephadm clear-ssh-config
+
+#. You can configure a file location for the SSH configuration file with:
+
+   .. prompt:: bash #
+
+      ceph config set mgr mgr/cephadm/ssh_config_file <path>
+
+   We do *not recommend* this approach.  The path name must be
+   visible to *any* mgr daemon, and cephadm runs all daemons as
+   containers. That means that the file either need to be placed
+   inside a customized container image for your deployment, or
+   manually distributed to the mgr data directory
+   (``/var/lib/ceph/<cluster-fsid>/mgr.<id>`` on the host, visible at
+   ``/var/lib/ceph/mgr/ceph-<id>`` from inside the container).
+   
+.. _cephadm-fqdn:
+
+Fully qualified domain names vs bare host names
+===============================================
+
+.. note::
+
+  cephadm demands that the name of the host given via ``ceph orch host add`` 
+  equals the output of ``hostname`` on remote hosts.
+
+Otherwise cephadm can't be sure that names returned by
+``ceph * metadata`` match the hosts known to cephadm. This might result
+in a :ref:`cephadm-stray-host` warning.
+
+When configuring new hosts, there are two **valid** ways to set the 
+``hostname`` of a host:
+
+1. Using the bare host name. In this case:
+
+-  ``hostname`` returns the bare host name.
+-  ``hostname -f`` returns the FQDN.
+
+2. Using the fully qualified domain name as the host name. In this case:
+
+-  ``hostname`` returns the FQDN
+-  ``hostname -s`` return the bare host name
+
+Note that ``man hostname`` recommends ``hostname`` to return the bare
+host name:
+
+    The FQDN (Fully Qualified Domain Name) of the system is the
+    name that the resolver(3) returns for the host name, such as,
+    ursula.example.com. It is usually the hostname followed by the DNS
+    domain name (the part after the first dot). You can check the FQDN
+    using ``hostname --fqdn`` or the domain name using ``dnsdomainname``.
+
+    .. code-block:: none
+
+          You cannot change the FQDN with hostname or dnsdomainname.
+
+          The recommended method of setting the FQDN is to make the hostname
+          be an alias for the fully qualified name using /etc/hosts, DNS, or
+          NIS. For example, if the hostname was "ursula", one might have
+          a line in /etc/hosts which reads
+
+                 127.0.1.1    ursula.example.com ursula
+
+Which means, ``man hostname`` recommends ``hostname`` to return the bare
+host name. This in turn means that Ceph will return the bare host names
+when executing ``ceph * metadata``. This in turn means cephadm also
+requires the bare host name when adding a host to the cluster: 
+``ceph orch host add <bare-name>``.
+
+..
+  TODO: This chapter needs to provide way for users to configure
+  Grafana in the dashboard, as this is right now very hard to do.
diff --git a/doc/cephadm/index.rst b/doc/cephadm/index.rst
new file mode 100644
index 000000000..bfa3a4bb2
--- /dev/null
+++ b/doc/cephadm/index.rst
@@ -0,0 +1,50 @@
+.. _cephadm:
+
+=======
+Cephadm
+=======
+
+``cephadm`` is a utility that is used to manage a Ceph cluster. 
+
+Here is a list of some of the things that ``cephadm`` can do:
+
+- ``cephadm`` can add a Ceph container to the cluster.
+- ``cephadm`` can remove a Ceph container from the cluster.
+- ``cephadm`` can update Ceph containers.
+
+``cephadm`` does not rely on external configuration tools like Ansible, Rook,
+or Salt. However, those external configuration tools can be used to automate
+operations not performed by cephadm itself. To learn more about these external
+configuration tools, visit their pages:
+
+ * https://github.com/ceph/cephadm-ansible
+ * https://rook.io/docs/rook/v1.10/Getting-Started/intro/
+ * https://github.com/ceph/ceph-salt
+
+``cephadm`` manages the full lifecycle of a Ceph cluster. This lifecycle starts
+with the bootstrapping process, when ``cephadm`` creates a tiny Ceph cluster on
+a single node. This cluster consists of one monitor and one manager.
+``cephadm`` then uses the orchestration interface to expand the cluster, adding
+hosts and provisioning Ceph daemons and services. Management of this lifecycle
+can be performed either via the Ceph command-line interface (CLI) or via the
+dashboard (GUI).
+
+To use ``cephadm`` to get started with Ceph, follow the instructions in
+:ref:`cephadm_deploying_new_cluster`.
+
+``cephadm`` was introduced in Ceph release v15.2.0 (Octopus) and does not
+support older versions of Ceph.
+
+.. toctree::
+    :maxdepth: 2
+
+    compatibility
+    install
+    adoption
+    host-management
+    Service Management <services/index>
+    upgrade
+    Cephadm operations <operations>
+    Client Setup <client-setup>
+    troubleshooting
+    Cephadm Feature Planning <../dev/cephadm/index>
diff --git a/doc/cephadm/install.rst b/doc/cephadm/install.rst
new file mode 100644
index 000000000..4c6179614
--- /dev/null
+++ b/doc/cephadm/install.rst
@@ -0,0 +1,439 @@
+.. _cephadm_deploying_new_cluster:
+
+============================
+Deploying a new Ceph cluster
+============================
+
+Cephadm creates a new Ceph cluster by "bootstrapping" on a single
+host, expanding the cluster to encompass any additional hosts, and
+then deploying the needed services.
+
+.. highlight:: console
+
+.. _cephadm-host-requirements:
+
+Requirements
+============
+
+- Python 3
+- Systemd
+- Podman or Docker for running containers
+- Time synchronization (such as chrony or NTP)
+- LVM2 for provisioning storage devices
+
+Any modern Linux distribution should be sufficient.  Dependencies
+are installed automatically by the bootstrap process below.
+
+See the section :ref:`Compatibility With Podman
+Versions<cephadm-compatibility-with-podman>` for a table of Ceph versions that
+are compatible with Podman. Not every version of Podman is compatible with
+Ceph.
+
+
+
+.. _get-cephadm:
+
+Install cephadm
+===============
+
+There are two ways to install ``cephadm``:
+
+#. a :ref:`curl-based installation<cephadm_install_curl>` method
+#. :ref:`distribution-specific installation methods<cephadm_install_distros>`
+
+
+.. _cephadm_install_curl:
+
+curl-based installation
+-----------------------
+
+* Use ``curl`` to fetch the most recent version of the
+  standalone script. 
+  
+  .. prompt:: bash #
+     :substitutions:
+
+     curl --silent --remote-name --location https://github.com/ceph/ceph/raw/|stable-release|/src/cephadm/cephadm
+
+  Make the ``cephadm`` script executable:
+
+  .. prompt:: bash #
+
+   chmod +x cephadm
+
+  This script can be run directly from the current directory:
+
+  .. prompt:: bash #
+
+   ./cephadm <arguments...>
+
+* Although the standalone script is sufficient to get a cluster started, it is
+  convenient to have the ``cephadm`` command installed on the host.  To install
+  the packages that provide the ``cephadm`` command, run the following
+  commands:
+
+  .. prompt:: bash #
+     :substitutions:
+
+     ./cephadm add-repo --release |stable-release|
+     ./cephadm install
+
+  Confirm that ``cephadm`` is now in your PATH by running ``which``:
+
+  .. prompt:: bash #
+
+    which cephadm
+
+  A successful ``which cephadm`` command will return this:
+
+  .. code-block:: bash
+
+    /usr/sbin/cephadm
+
+.. _cephadm_install_distros:
+
+distribution-specific installations
+-----------------------------------
+
+.. important:: The methods of installing ``cephadm`` in this section are distinct from the curl-based method above. Use either the curl-based method above or one of the methods in this section, but not both the curl-based method and one of these.
+
+Some Linux distributions  may already include up-to-date Ceph packages.  In
+that case, you can install cephadm directly. For example:
+
+  In Ubuntu:
+
+  .. prompt:: bash #
+
+     apt install -y cephadm
+
+  In CentOS Stream:
+
+  .. prompt:: bash #
+
+     dnf install --assumeyes centos-release-ceph-pacific.noarch
+     dnf install --assumeyes cephadm
+
+  In Fedora:
+
+  .. prompt:: bash #
+
+     dnf -y install cephadm
+
+  In SUSE:
+
+  .. prompt:: bash #
+
+     zypper install -y cephadm
+
+
+
+Bootstrap a new cluster
+=======================
+
+What to know before you bootstrap
+---------------------------------
+
+The first step in creating a new Ceph cluster is running the ``cephadm
+bootstrap`` command on the Ceph cluster's first host. The act of running the
+``cephadm bootstrap`` command on the Ceph cluster's first host creates the Ceph
+cluster's first "monitor daemon", and that monitor daemon needs an IP address.
+You must pass the IP address of the Ceph cluster's first host to the ``ceph
+bootstrap`` command, so you'll need to know the IP address of that host.
+
+.. note:: If there are multiple networks and interfaces, be sure to choose one
+   that will be accessible by any host accessing the Ceph cluster.
+
+Running the bootstrap command
+-----------------------------
+
+Run the ``ceph bootstrap`` command:
+
+.. prompt:: bash # 
+
+   cephadm bootstrap --mon-ip *<mon-ip>*
+
+This command will:
+
+* Create a monitor and manager daemon for the new cluster on the local
+  host.
+* Generate a new SSH key for the Ceph cluster and add it to the root
+  user's ``/root/.ssh/authorized_keys`` file.
+* Write a copy of the public key to ``/etc/ceph/ceph.pub``.
+* Write a minimal configuration file to ``/etc/ceph/ceph.conf``. This
+  file is needed to communicate with the new cluster.
+* Write a copy of the ``client.admin`` administrative (privileged!)
+  secret key to ``/etc/ceph/ceph.client.admin.keyring``.
+* Add the ``_admin`` label to the bootstrap host.  By default, any host
+  with this label will (also) get a copy of ``/etc/ceph/ceph.conf`` and
+  ``/etc/ceph/ceph.client.admin.keyring``.
+
+Further information about cephadm bootstrap 
+-------------------------------------------
+
+The default bootstrap behavior will work for most users. But if you'd like
+immediately to know more about ``cephadm bootstrap``, read the list below.  
+
+Also, you can run ``cephadm bootstrap -h`` to see all of ``cephadm``'s
+available options.
+
+* By default, Ceph daemons send their log output to stdout/stderr, which is picked
+  up by the container runtime (docker or podman) and (on most systems) sent to
+  journald.  If you want Ceph to write traditional log files to ``/var/log/ceph/$fsid``,
+  use the ``--log-to-file`` option during bootstrap.
+
+* Larger Ceph clusters perform better when (external to the Ceph cluster)
+  public network traffic is separated from (internal to the Ceph cluster)
+  cluster traffic. The internal cluster traffic handles replication, recovery,
+  and heartbeats between OSD daemons.  You can define the :ref:`cluster
+  network<cluster-network>` by supplying the ``--cluster-network`` option to the ``bootstrap``
+  subcommand. This parameter must define a subnet in CIDR notation (for example
+  ``10.90.90.0/24`` or ``fe80::/64``).
+
+* ``cephadm bootstrap`` writes to ``/etc/ceph`` the files needed to access
+  the new cluster. This central location makes it possible for Ceph
+  packages installed on the host (e.g., packages that give access to the
+  cephadm command line interface) to find these files.
+
+  Daemon containers deployed with cephadm, however, do not need
+  ``/etc/ceph`` at all.  Use the ``--output-dir *<directory>*`` option
+  to put them in a different directory (for example, ``.``). This may help
+  avoid conflicts with an existing Ceph configuration (cephadm or
+  otherwise) on the same host.
+
+* You can pass any initial Ceph configuration options to the new
+  cluster by putting them in a standard ini-style configuration file
+  and using the ``--config *<config-file>*`` option.  For example::
+
+      $ cat <<EOF > initial-ceph.conf
+      [global]
+      osd crush chooseleaf type = 0
+      EOF
+      $ ./cephadm bootstrap --config initial-ceph.conf ...
+
+* The ``--ssh-user *<user>*`` option makes it possible to choose which SSH 
+  user cephadm will use to connect to hosts. The associated SSH key will be
+  added to ``/home/*<user>*/.ssh/authorized_keys``. The user that you 
+  designate with this option must have passwordless sudo access.
+
+* If you are using a container on an authenticated registry that requires
+  login, you may add the argument:
+
+  * ``--registry-json <path to json file>`` 
+
+  example contents of JSON file with login info::
+
+      {"url":"REGISTRY_URL", "username":"REGISTRY_USERNAME", "password":"REGISTRY_PASSWORD"}
+  
+  Cephadm will attempt to log in to this registry so it can pull your container
+  and then store the login info in its config database. Other hosts added to
+  the cluster will then also be able to make use of the authenticated registry.
+
+* See :ref:`cephadm-deployment-scenarios` for additional examples for using ``cephadm bootstrap``.
+
+.. _cephadm-enable-cli:
+
+Enable Ceph CLI
+===============
+
+Cephadm does not require any Ceph packages to be installed on the
+host.  However, we recommend enabling easy access to the ``ceph``
+command.  There are several ways to do this:
+
+* The ``cephadm shell`` command launches a bash shell in a container
+  with all of the Ceph packages installed. By default, if
+  configuration and keyring files are found in ``/etc/ceph`` on the
+  host, they are passed into the container environment so that the
+  shell is fully functional. Note that when executed on a MON host,
+  ``cephadm shell`` will infer the ``config`` from the MON container
+  instead of using the default configuration. If ``--mount <path>``
+  is given, then the host ``<path>`` (file or directory) will appear
+  under ``/mnt`` inside the container:
+
+  .. prompt:: bash #
+
+     cephadm shell
+
+* To execute ``ceph`` commands, you can also run commands like this:
+
+  .. prompt:: bash #
+
+     cephadm shell -- ceph -s
+
+* You can install the ``ceph-common`` package, which contains all of the
+  ceph commands, including ``ceph``, ``rbd``, ``mount.ceph`` (for mounting
+  CephFS file systems), etc.:
+
+  .. prompt:: bash #
+     :substitutions:
+
+     cephadm add-repo --release |stable-release|
+     cephadm install ceph-common
+
+Confirm that the ``ceph`` command is accessible with:
+
+.. prompt:: bash #
+ 
+  ceph -v
+
+
+Confirm that the ``ceph`` command can connect to the cluster and also
+its status with:
+
+.. prompt:: bash #
+
+  ceph status
+
+Adding Hosts
+============
+
+Next, add all hosts to the cluster by following :ref:`cephadm-adding-hosts`.
+
+By default, a ``ceph.conf`` file and a copy of the ``client.admin`` keyring
+are maintained in ``/etc/ceph`` on all hosts with the ``_admin`` label, which is initially
+applied only to the bootstrap host. We usually recommend that one or more other hosts be
+given the ``_admin`` label so that the Ceph CLI (e.g., via ``cephadm shell``) is easily
+accessible on multiple hosts.  To add the ``_admin`` label to additional host(s),
+
+  .. prompt:: bash #
+
+    ceph orch host label add *<host>* _admin
+
+Adding additional MONs
+======================
+
+A typical Ceph cluster has three or five monitor daemons spread
+across different hosts.  We recommend deploying five
+monitors if there are five or more nodes in your cluster.
+
+Please follow :ref:`deploy_additional_monitors` to deploy additional MONs.
+
+Adding Storage
+==============
+
+To add storage to the cluster, either tell Ceph to consume any
+available and unused device:
+
+  .. prompt:: bash #
+
+    ceph orch apply osd --all-available-devices
+
+See :ref:`cephadm-deploy-osds` for more detailed instructions.
+
+Enabling OSD memory autotuning
+------------------------------
+
+.. warning:: By default, cephadm enables ``osd_memory_target_autotune`` on bootstrap, with ``mgr/cephadm/autotune_memory_target_ratio`` set to ``.7`` of total host memory.
+
+See :ref:`osd_autotune`.
+
+To deploy hyperconverged Ceph with TripleO, please refer to the TripleO documentation: `Scenario: Deploy Hyperconverged Ceph <https://docs.openstack.org/project-deploy-guide/tripleo-docs/latest/features/cephadm.html#scenario-deploy-hyperconverged-ceph>`_
+
+In other cases where the cluster hardware is not exclusively used by Ceph (hyperconverged),
+reduce the memory consumption of Ceph like so:
+
+  .. prompt:: bash #
+
+    # hyperconverged only:
+    ceph config set mgr mgr/cephadm/autotune_memory_target_ratio 0.2
+
+Then enable memory autotuning:
+
+  .. prompt:: bash #
+
+    ceph config set osd osd_memory_target_autotune true
+
+
+Using Ceph
+==========
+
+To use the *Ceph Filesystem*, follow :ref:`orchestrator-cli-cephfs`.
+
+To use the *Ceph Object Gateway*, follow :ref:`cephadm-deploy-rgw`.
+
+To use *NFS*, follow :ref:`deploy-cephadm-nfs-ganesha`
+
+To use *iSCSI*, follow :ref:`cephadm-iscsi`
+
+.. _cephadm-deployment-scenarios:
+
+Different deployment scenarios
+==============================
+
+Single host
+-----------
+
+To configure a Ceph cluster to run on a single host, use the
+``--single-host-defaults`` flag when bootstrapping. For use cases of this, see
+:ref:`one-node-cluster`.
+
+The ``--single-host-defaults`` flag sets the following configuration options::
+
+  global/osd_crush_chooseleaf_type = 0
+  global/osd_pool_default_size = 2
+  mgr/mgr_standby_modules = False
+
+For more information on these options, see :ref:`one-node-cluster` and
+``mgr_standby_modules`` in :ref:`mgr-administrator-guide`.
+
+.. _cephadm-airgap:
+
+Deployment in an isolated environment
+-------------------------------------
+
+You might need to install cephadm in an environment that is not connected
+directly to the internet (such an environment is also called an "isolated
+environment"). This can be done if a custom container registry is used. Either
+of two kinds of custom container registry can be used in this scenario: (1) a
+Podman-based or Docker-based insecure registry, or (2) a secure registry.
+
+The practice of installing software on systems that are not connected directly
+to the internet is called "airgapping" and registries that are not connected
+directly to the internet are referred to as "airgapped".
+
+Make sure that your container image is inside the registry. Make sure that you
+have access to all hosts that you plan to add to the cluster.
+
+#. Run a local container registry:
+
+   .. prompt:: bash #
+
+      podman run --privileged -d --name registry -p 5000:5000 -v /var/lib/registry:/var/lib/registry --restart=always registry:2
+
+#. If you are using an insecure registry, configure Podman or Docker with the
+   hostname and port where the registry is running.
+
+   .. note:: You must repeat this step for every host that accesses the local
+             insecure registry.
+
+#. Push your container image to your local registry. Here are some acceptable
+   kinds of container images: 
+
+   * Ceph container image. See :ref:`containers`.
+   * Prometheus container image
+   * Node exporter container image
+   * Grafana container image
+   * Alertmanager container image
+
+#. Create a temporary configuration file to store the names of the monitoring
+   images. (See :ref:`cephadm_monitoring-images`):
+
+   .. prompt:: bash $
+
+      cat <<EOF > initial-ceph.conf
+
+   ::
+
+      [mgr]
+      mgr/cephadm/container_image_prometheus *<hostname>*:5000/prometheus
+      mgr/cephadm/container_image_node_exporter *<hostname>*:5000/node_exporter
+      mgr/cephadm/container_image_grafana *<hostname>*:5000/grafana
+      mgr/cephadm/container_image_alertmanager *<hostname>*:5000/alertmanger
+
+#. Run bootstrap using the ``--image`` flag and pass the name of your
+   container image as the argument of the image flag. For example:
+
+   .. prompt:: bash #
+
+      cephadm --image *<hostname>*:5000/ceph/ceph bootstrap --mon-ip *<mon-ip>*
+
+.. _cluster network: ../rados/configuration/network-config-ref#cluster-network
diff --git a/doc/cephadm/operations.rst b/doc/cephadm/operations.rst
new file mode 100644
index 000000000..23b396b51
--- /dev/null
+++ b/doc/cephadm/operations.rst
@@ -0,0 +1,545 @@
+==================
+Cephadm Operations
+==================
+
+.. _watching_cephadm_logs:
+
+Watching cephadm log messages
+=============================
+
+Cephadm writes logs to the ``cephadm`` cluster log channel. You can
+monitor Ceph's activity in real time by reading the logs as they fill
+up. Run the following command to see the logs in real time:
+
+.. prompt:: bash #
+
+  ceph -W cephadm
+
+By default, this command shows info-level events and above.  To see
+debug-level messages as well as info-level events, run the following
+commands:
+
+.. prompt:: bash #
+
+  ceph config set mgr mgr/cephadm/log_to_cluster_level debug
+  ceph -W cephadm --watch-debug
+
+.. warning::
+
+  The debug messages are very verbose!
+
+You can see recent events by running the following command:
+
+.. prompt:: bash #
+
+  ceph log last cephadm
+
+These events are also logged to the ``ceph.cephadm.log`` file on
+monitor hosts as well as to the monitor daemons' stderr.
+
+
+.. _cephadm-logs:
+
+Ceph daemon logs
+================
+
+Logging to journald
+-------------------
+
+Ceph daemons traditionally write logs to ``/var/log/ceph``. Ceph daemons log to
+journald by default and Ceph logs are captured by the container runtime
+environment. They are accessible via ``journalctl``.
+
+.. note:: Prior to Quincy, ceph daemons logged to stderr.
+
+Example of logging to journald
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+For example, to view the logs for the daemon ``mon.foo`` for a cluster
+with ID ``5c5a50ae-272a-455d-99e9-32c6a013e694``, the command would be
+something like:
+
+.. prompt:: bash #
+
+  journalctl -u ceph-5c5a50ae-272a-455d-99e9-32c6a013e694@mon.foo
+
+This works well for normal operations when logging levels are low.
+
+Logging to files
+----------------
+
+You can also configure Ceph daemons to log to files instead of to
+journald if you prefer logs to appear in files (as they did in earlier,
+pre-cephadm, pre-Octopus versions of Ceph).  When Ceph logs to files,
+the logs appear in ``/var/log/ceph/<cluster-fsid>``. If you choose to
+configure Ceph to log to files instead of to journald, remember to
+configure Ceph so that it will not log to journald (the commands for
+this are covered below).
+
+Enabling logging to files
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To enable logging to files, run the following commands:
+
+.. prompt:: bash #
+
+  ceph config set global log_to_file true
+  ceph config set global mon_cluster_log_to_file true
+
+Disabling logging to journald
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you choose to log to files, we recommend disabling logging to journald or else
+everything will be logged twice. Run the following commands to disable logging
+to stderr:
+
+.. prompt:: bash #
+
+  ceph config set global log_to_stderr false
+  ceph config set global mon_cluster_log_to_stderr false
+  ceph config set global log_to_journald false
+  ceph config set global mon_cluster_log_to_journald false
+
+.. note:: You can change the default by passing --log-to-file during
+   bootstrapping a new cluster.
+
+Modifying the log retention schedule
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+By default, cephadm sets up log rotation on each host to rotate these
+files.  You can configure the logging retention schedule by modifying
+``/etc/logrotate.d/ceph.<cluster-fsid>``.
+
+
+Data location
+=============
+
+Cephadm stores daemon data and logs in different locations than did
+older, pre-cephadm (pre Octopus) versions of ceph:
+
+* ``/var/log/ceph/<cluster-fsid>`` contains all cluster logs. By
+  default, cephadm logs via stderr and the container runtime. These
+  logs will not exist unless you have enabled logging to files as
+  described in `cephadm-logs`_.
+* ``/var/lib/ceph/<cluster-fsid>`` contains all cluster daemon data
+  (besides logs).
+* ``/var/lib/ceph/<cluster-fsid>/<daemon-name>`` contains all data for
+  an individual daemon.
+* ``/var/lib/ceph/<cluster-fsid>/crash`` contains crash reports for
+  the cluster.
+* ``/var/lib/ceph/<cluster-fsid>/removed`` contains old daemon
+  data directories for stateful daemons (e.g., monitor, prometheus)
+  that have been removed by cephadm.
+
+Disk usage
+----------
+
+Because a few Ceph daemons (notably, the monitors and prometheus) store a
+large amount of data in ``/var/lib/ceph`` , we recommend moving this
+directory to its own disk, partition, or logical volume so that it does not
+fill up the root file system.
+
+
+Health checks
+=============
+The cephadm module provides additional health checks to supplement the
+default health checks provided by the Cluster. These additional health
+checks fall into two categories:
+
+- **cephadm operations**: Health checks in this category are always
+  executed when the cephadm module is active.
+- **cluster configuration**: These health checks are *optional*, and
+  focus on the configuration of the hosts in the cluster.
+
+CEPHADM Operations
+------------------
+
+CEPHADM_PAUSED
+~~~~~~~~~~~~~~
+
+This indicates that cephadm background work has been paused with
+``ceph orch pause``.  Cephadm continues to perform passive monitoring
+activities (like checking host and daemon status), but it will not
+make any changes (like deploying or removing daemons).
+
+Resume cephadm work by running the following command:
+
+.. prompt:: bash #
+
+  ceph orch resume
+
+.. _cephadm-stray-host:
+
+CEPHADM_STRAY_HOST
+~~~~~~~~~~~~~~~~~~
+
+This indicates that one or more hosts have Ceph daemons that are
+running, but are not registered as hosts managed by *cephadm*.  This
+means that those services cannot currently be managed by cephadm
+(e.g., restarted, upgraded, included in `ceph orch ps`).
+
+* You can manage the host(s) by running the following command:
+
+  .. prompt:: bash #
+
+    ceph orch host add *<hostname>*
+
+  .. note::
+
+    You might need to configure SSH access to the remote host
+    before this will work.
+
+* See :ref:`cephadm-fqdn` for more information about host names and
+  domain names.
+
+* Alternatively, you can manually connect to the host and ensure that
+  services on that host are removed or migrated to a host that is
+  managed by *cephadm*.
+
+* This warning can be disabled entirely by running the following
+  command:
+
+  .. prompt:: bash #
+
+    ceph config set mgr mgr/cephadm/warn_on_stray_hosts false
+
+CEPHADM_STRAY_DAEMON
+~~~~~~~~~~~~~~~~~~~~
+
+One or more Ceph daemons are running but not are not managed by
+*cephadm*.  This may be because they were deployed using a different
+tool, or because they were started manually.  Those
+services cannot currently be managed by cephadm (e.g., restarted,
+upgraded, or included in `ceph orch ps`).
+
+* If the daemon is a stateful one (monitor or OSD), it should be adopted
+  by cephadm; see :ref:`cephadm-adoption`.  For stateless daemons, it is
+  usually easiest to provision a new daemon with the ``ceph orch apply``
+  command and then stop the unmanaged daemon.
+
+* If the stray daemon(s) are running on hosts not managed by cephadm, you can manage the host(s) by running the following command:
+
+  .. prompt:: bash #
+
+    ceph orch host add *<hostname>*
+
+  .. note::
+
+    You might need to configure SSH access to the remote host
+    before this will work.
+
+* See :ref:`cephadm-fqdn` for more information about host names and
+  domain names.
+
+* This warning can be disabled entirely by running the following command:
+
+  .. prompt:: bash #
+
+    ceph config set mgr mgr/cephadm/warn_on_stray_daemons false
+
+CEPHADM_HOST_CHECK_FAILED
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+One or more hosts have failed the basic cephadm host check, which verifies
+that (1) the host is reachable and cephadm can be executed there, and (2)
+that the host satisfies basic prerequisites, like a working container
+runtime (podman or docker) and working time synchronization.
+If this test fails, cephadm will no be able to manage services on that host.
+
+You can manually run this check by running the following command:
+
+.. prompt:: bash #
+
+  ceph cephadm check-host *<hostname>*
+
+You can remove a broken host from management by running the following command:
+
+.. prompt:: bash #
+
+  ceph orch host rm *<hostname>*
+
+You can disable this health warning by running the following command:
+
+.. prompt:: bash #
+
+  ceph config set mgr mgr/cephadm/warn_on_failed_host_check false
+
+Cluster Configuration Checks
+----------------------------
+Cephadm periodically scans each of the hosts in the cluster in order
+to understand the state of the OS, disks, NICs etc. These facts can
+then be analysed for consistency across the hosts in the cluster to
+identify any configuration anomalies.
+
+Enabling Cluster Configuration Checks
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The configuration checks are an **optional** feature, and are enabled
+by running the following command:
+
+.. prompt:: bash #
+
+  ceph config set mgr mgr/cephadm/config_checks_enabled true
+
+States Returned by Cluster Configuration Checks
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The configuration checks are triggered after each host scan (1m). The
+cephadm log entries will show the current state and outcome of the
+configuration checks as follows:
+
+Disabled state (config_checks_enabled false):
+
+.. code-block:: bash 
+
+  ALL cephadm checks are disabled, use 'ceph config set mgr mgr/cephadm/config_checks_enabled true' to enable
+
+Enabled state (config_checks_enabled true):
+
+.. code-block:: bash 
+
+  CEPHADM 8/8 checks enabled and executed (0 bypassed, 0 disabled). No issues detected
+
+Managing Configuration Checks (subcommands)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The configuration checks themselves are managed through several cephadm subcommands.
+
+To determine whether the configuration checks are enabled, run the following command:
+
+.. prompt:: bash #
+
+  ceph cephadm config-check status
+
+This command returns the status of the configuration checker as either "Enabled" or "Disabled".
+
+
+To list all the configuration checks and their current states, run the following command:
+
+.. code-block:: console
+
+  # ceph cephadm config-check ls
+
+    NAME             HEALTHCHECK                      STATUS   DESCRIPTION
+  kernel_security  CEPHADM_CHECK_KERNEL_LSM         enabled  checks SELINUX/Apparmor profiles are consistent across cluster hosts
+  os_subscription  CEPHADM_CHECK_SUBSCRIPTION       enabled  checks subscription states are consistent for all cluster hosts
+  public_network   CEPHADM_CHECK_PUBLIC_MEMBERSHIP  enabled  check that all hosts have a NIC on the Ceph public_network
+  osd_mtu_size     CEPHADM_CHECK_MTU                enabled  check that OSD hosts share a common MTU setting
+  osd_linkspeed    CEPHADM_CHECK_LINKSPEED          enabled  check that OSD hosts share a common linkspeed
+  network_missing  CEPHADM_CHECK_NETWORK_MISSING    enabled  checks that the cluster/public networks defined exist on the Ceph hosts
+  ceph_release     CEPHADM_CHECK_CEPH_RELEASE       enabled  check for Ceph version consistency - ceph daemons should be on the same release (unless upgrade is active)
+  kernel_version   CEPHADM_CHECK_KERNEL_VERSION     enabled  checks that the MAJ.MIN of the kernel on Ceph hosts is consistent
+
+The name of each configuration check can be used to enable or disable a specific check by running a command of the following form:
+:
+
+.. prompt:: bash #
+
+  ceph cephadm config-check disable <name>
+
+For example:
+
+.. prompt:: bash #
+
+  ceph cephadm config-check disable kernel_security
+
+CEPHADM_CHECK_KERNEL_LSM
+~~~~~~~~~~~~~~~~~~~~~~~~
+Each host within the cluster is expected to operate within the same Linux
+Security Module (LSM) state. For example, if the majority of the hosts are
+running with SELINUX in enforcing mode, any host not running in this mode is
+flagged as an anomaly and a healtcheck (WARNING) state raised.
+
+CEPHADM_CHECK_SUBSCRIPTION
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+This check relates to the status of vendor subscription. This check is
+performed only for hosts using RHEL, but helps to confirm that all hosts are
+covered by an active subscription, which ensures that patches and updates are
+available.
+
+CEPHADM_CHECK_PUBLIC_MEMBERSHIP
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+All members of the cluster should have NICs configured on at least one of the
+public network subnets. Hosts that are not on the public network will rely on
+routing, which may affect performance.
+
+CEPHADM_CHECK_MTU
+~~~~~~~~~~~~~~~~~
+The MTU of the NICs on OSDs can be a key factor in consistent performance. This
+check examines hosts that are running OSD services to ensure that the MTU is
+configured consistently within the cluster. This is determined by establishing
+the MTU setting that the majority of hosts is using. Any anomalies result in a
+Ceph health check.
+
+CEPHADM_CHECK_LINKSPEED
+~~~~~~~~~~~~~~~~~~~~~~~
+This check is similar to the MTU check. Linkspeed consistency is a factor in
+consistent cluster performance, just as the MTU of the NICs on the OSDs is.
+This check determines the linkspeed shared by the majority of OSD hosts, and a
+health check is run for any hosts that are set at a lower linkspeed rate.
+
+CEPHADM_CHECK_NETWORK_MISSING
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The `public_network` and `cluster_network` settings support subnet definitions
+for IPv4 and IPv6. If these settings are not found on any host in the cluster,
+a health check is raised.
+
+CEPHADM_CHECK_CEPH_RELEASE
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+Under normal operations, the Ceph cluster runs daemons under the same ceph
+release (that is, the Ceph cluster runs all daemons under (for example)
+Octopus).  This check determines the active release for each daemon, and
+reports any anomalies as a healthcheck. *This check is bypassed if an upgrade
+process is active within the cluster.*
+
+CEPHADM_CHECK_KERNEL_VERSION
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The OS kernel version (maj.min) is checked for consistency across the hosts.
+The kernel version of the majority of the hosts is used as the basis for 
+identifying anomalies.
+
+.. _client_keyrings_and_configs:
+
+Client keyrings and configs
+===========================
+
+Cephadm can distribute copies of the ``ceph.conf`` file and client keyring
+files to hosts. It is usually a good idea to store a copy of the config and
+``client.admin`` keyring on any host used to administer the cluster via the
+CLI.  By default, cephadm does this for any nodes that have the ``_admin``
+label (which normally includes the bootstrap host).
+
+When a client keyring is placed under management, cephadm will:
+
+  - build a list of target hosts based on the specified placement spec (see
+    :ref:`orchestrator-cli-placement-spec`)
+  - store a copy of the ``/etc/ceph/ceph.conf`` file on the specified host(s)
+  - store a copy of the keyring file on the specified host(s)
+  - update the ``ceph.conf`` file as needed (e.g., due to a change in the cluster monitors)
+  - update the keyring file if the entity's key is changed (e.g., via ``ceph
+    auth ...`` commands)
+  - ensure that the keyring file has the specified ownership and specified mode
+  - remove the keyring file when client keyring management is disabled
+  - remove the keyring file from old hosts if the keyring placement spec is
+    updated (as needed)
+
+Listing Client Keyrings
+-----------------------
+
+To see the list of client keyrings are currently under management, run the following command:
+
+.. prompt:: bash #
+
+  ceph orch client-keyring ls
+
+Putting a Keyring Under Management
+----------------------------------
+
+To put a keyring under management, run a command of the following form: 
+
+.. prompt:: bash #
+
+  ceph orch client-keyring set <entity> <placement> [--mode=<mode>] [--owner=<uid>.<gid>] [--path=<path>]
+
+- By default, the *path* is ``/etc/ceph/client.{entity}.keyring``, which is
+  where Ceph looks by default.  Be careful when specifying alternate locations,
+  as existing files may be overwritten.
+- A placement of ``*`` (all hosts) is common.
+- The mode defaults to ``0600`` and ownership to ``0:0`` (user root, group root).
+
+For example, to create a ``client.rbd`` key and deploy it to hosts with the
+``rbd-client`` label and make it group readable by uid/gid 107 (qemu), run the
+following commands:
+
+.. prompt:: bash #
+
+  ceph auth get-or-create-key client.rbd mon 'profile rbd' mgr 'profile rbd' osd 'profile rbd pool=my_rbd_pool'
+  ceph orch client-keyring set client.rbd label:rbd-client --owner 107:107 --mode 640
+
+The resulting keyring file is:
+
+.. code-block:: console
+
+  -rw-r-----. 1 qemu qemu 156 Apr 21 08:47 /etc/ceph/client.client.rbd.keyring
+
+Disabling Management of a Keyring File
+--------------------------------------
+
+To disable management of a keyring file, run a command of the following form:
+
+.. prompt:: bash #
+
+  ceph orch client-keyring rm <entity>
+
+.. note::
+
+  This deletes any keyring files for this entity that were previously written
+  to cluster nodes.
+
+.. _etc_ceph_conf_distribution:
+
+/etc/ceph/ceph.conf
+===================
+
+Distributing ceph.conf to hosts that have no keyrings
+-----------------------------------------------------
+
+It might be useful to distribute ``ceph.conf`` files to hosts without an
+associated client keyring file.  By default, cephadm deploys only a
+``ceph.conf`` file to hosts where a client keyring is also distributed (see
+above).  To write config files to hosts without client keyrings, run the
+following command:
+
+.. prompt:: bash #
+
+    ceph config set mgr mgr/cephadm/manage_etc_ceph_ceph_conf true
+
+Using Placement Specs to specify which hosts get keyrings
+---------------------------------------------------------
+
+By default, the configs are written to all hosts (i.e., those listed by ``ceph
+orch host ls``).  To specify which hosts get a ``ceph.conf``, run a command of
+the following form:
+
+.. prompt:: bash #
+
+  ceph config set mgr mgr/cephadm/manage_etc_ceph_ceph_conf_hosts <placement spec>
+
+For example, to distribute configs to hosts with the ``bare_config`` label, run
+the following command:
+
+Distributing ceph.conf to hosts tagged with bare_config 
+-------------------------------------------------------
+
+For example, to distribute configs to hosts with the ``bare_config`` label, run the following command:
+
+.. prompt:: bash #
+
+  ceph config set mgr mgr/cephadm/manage_etc_ceph_ceph_conf_hosts label:bare_config
+
+(See :ref:`orchestrator-cli-placement-spec` for more information about placement specs.)
+
+Purging a cluster
+=================
+
+.. danger:: THIS OPERATION WILL DESTROY ALL DATA STORED IN THIS CLUSTER
+
+In order to destory a cluster and delete all data stored in this cluster, pause 
+cephadm to avoid deploying new daemons.
+
+.. prompt:: bash #
+
+  ceph orch pause
+
+Then verify the FSID of the cluster:
+
+.. prompt:: bash #
+
+  ceph fsid 
+
+Purge ceph daemons from all hosts in the cluster
+
+.. prompt:: bash #
+
+  # For each host:
+  cephadm rm-cluster --force --zap-osds --fsid <fsid>
diff --git a/doc/cephadm/services/custom-container.rst b/doc/cephadm/services/custom-container.rst
new file mode 100644
index 000000000..3ece248c5
--- /dev/null
+++ b/doc/cephadm/services/custom-container.rst
@@ -0,0 +1,79 @@
+========================
+Custom Container Service
+========================
+
+The orchestrator enables custom containers to be deployed using a YAML file.
+A corresponding :ref:`orchestrator-cli-service-spec` must look like:
+
+.. code-block:: yaml
+
+    service_type: container
+    service_id: foo
+    placement:
+        ...
+    spec:
+      image: docker.io/library/foo:latest
+      entrypoint: /usr/bin/foo
+      uid: 1000
+      gid: 1000
+      args:
+        - "--net=host"
+        - "--cpus=2"
+      ports:
+        - 8080
+        - 8443
+      envs:
+        - SECRET=mypassword
+        - PORT=8080
+        - PUID=1000
+        - PGID=1000
+      volume_mounts:
+        CONFIG_DIR: /etc/foo
+      bind_mounts:
+        - ['type=bind', 'source=lib/modules', 'destination=/lib/modules', 'ro=true']
+      dirs:
+        - CONFIG_DIR
+      files:
+        CONFIG_DIR/foo.conf:
+          - refresh=true
+          - username=xyz
+          - "port: 1234"
+
+where the properties of a service specification are:
+
+* ``service_id``
+    A unique name of the service.
+* ``image``
+    The name of the Docker image.
+* ``uid``
+    The UID to use when creating directories and files in the host system.
+* ``gid``
+    The GID to use when creating directories and files in the host system.
+* ``entrypoint``
+    Overwrite the default ENTRYPOINT of the image.
+* ``args``
+    A list of additional Podman/Docker command line arguments.
+* ``ports``
+    A list of TCP ports to open in the host firewall.
+* ``envs``
+    A list of environment variables.
+* ``bind_mounts``
+    When you use a bind mount, a file or directory on the host machine
+    is mounted into the container. Relative `source=...` paths will be
+    located below `/var/lib/ceph/<cluster-fsid>/<daemon-name>`.
+* ``volume_mounts``
+    When you use a volume mount, a new directory is created within
+    Docker’s storage directory on the host machine, and Docker manages
+    that directory’s contents. Relative source paths will be located below
+    `/var/lib/ceph/<cluster-fsid>/<daemon-name>`.
+* ``dirs``
+    A list of directories that are created below
+    `/var/lib/ceph/<cluster-fsid>/<daemon-name>`.
+* ``files``
+    A dictionary, where the key is the relative path of the file and the
+    value the file content. The content must be double quoted when using
+    a string. Use '\\n' for line breaks in that case. Otherwise define
+    multi-line content as list of strings. The given files will be created
+    below the directory `/var/lib/ceph/<cluster-fsid>/<daemon-name>`.
+    The absolute path of the directory where the file will be created must
+    exist. Use the `dirs` property to create them if necessary.
diff --git a/doc/cephadm/services/index.rst b/doc/cephadm/services/index.rst
new file mode 100644
index 000000000..26fd8864a
--- /dev/null
+++ b/doc/cephadm/services/index.rst
@@ -0,0 +1,658 @@
+==================
+Service Management
+==================
+
+A service is a group of daemons configured together. See these chapters
+for details on individual services:
+
+.. toctree::
+    :maxdepth: 1
+
+    mon
+    mgr
+    osd
+    rgw
+    mds
+    nfs
+    iscsi
+    custom-container
+    monitoring
+    snmp-gateway
+
+Service Status
+==============
+
+
+To see the status of one
+of the services running in the Ceph cluster, do the following:
+
+#. Use the command line to print a list of services.
+#. Locate the service whose status you want to check.
+#. Print the status of the service.
+
+The following command prints a list of services known to the orchestrator. To
+limit the output to services only on a specified host, use the optional
+``--host`` parameter. To limit the output to services of only a particular
+type, use the optional ``--type`` parameter (mon, osd, mgr, mds, rgw):
+
+   .. prompt:: bash #
+
+     ceph orch ls [--service_type type] [--service_name name] [--export] [--format f] [--refresh]
+
+Discover the status of a particular service or daemon:
+
+   .. prompt:: bash #
+
+     ceph orch ls --service_type type --service_name <name> [--refresh]
+
+To export the service specifications knows to the orchestrator, run the following command.
+
+   .. prompt:: bash #
+
+     ceph orch ls --export
+
+The service specifications exported with this command will be exported as yaml
+and that yaml can be used with the ``ceph orch apply -i`` command.
+
+For information about retrieving the specifications of single services (including examples of commands), see :ref:`orchestrator-cli-service-spec-retrieve`.
+
+Daemon Status
+=============
+
+A daemon is a systemd unit that is running and part of a service.
+
+To see the status of a daemon, do the following:
+
+#. Print a list of all daemons known to the orchestrator.
+#. Query the status of the target daemon.
+
+First, print a list of all daemons known to the orchestrator:
+
+   .. prompt:: bash #
+
+    ceph orch ps [--hostname host] [--daemon_type type] [--service_name name] [--daemon_id id] [--format f] [--refresh]
+
+Then query the status of a particular service instance (mon, osd, mds, rgw).
+For OSDs the id is the numeric OSD ID. For MDS services the id is the file
+system name:
+
+   .. prompt:: bash #
+
+    ceph orch ps --daemon_type osd --daemon_id 0
+
+.. _orchestrator-cli-service-spec:
+
+Service Specification
+=====================
+
+A *Service Specification* is a data structure that is used to specify the
+deployment of services. In addition to parameters such as `placement` or
+`networks`, the user can set initial values of service configuration parameters
+by means of the `config` section. For each param/value configuration pair,
+cephadm calls the following command to set its value:
+
+   .. prompt:: bash #
+
+    ceph config set <service-name> <param> <value>
+
+cephadm raises health warnings in case invalid configuration parameters are
+found in the spec (`CEPHADM_INVALID_CONFIG_OPTION`) or if any error while
+trying to apply the new configuration option(s) (`CEPHADM_FAILED_SET_OPTION`).
+
+Here is an example of a service specification in YAML:
+
+.. code-block:: yaml
+
+    service_type: rgw
+    service_id: realm.zone
+    placement:
+      hosts:
+        - host1
+        - host2
+        - host3
+    config:
+      param_1: val_1
+      ...
+      param_N: val_N
+    unmanaged: false
+    networks:
+    - 192.169.142.0/24
+    spec:
+      # Additional service specific attributes.
+
+In this example, the properties of this service specification are:
+
+.. py:currentmodule:: ceph.deployment.service_spec
+
+.. autoclass:: ServiceSpec
+   :members:
+
+Each service type can have additional service-specific properties.
+
+Service specifications of type ``mon``, ``mgr``, and the monitoring
+types do not require a ``service_id``.
+
+A service of type ``osd`` is described in :ref:`drivegroups`
+
+Many service specifications can be applied at once using ``ceph orch apply -i``
+by submitting a multi-document YAML file::
+
+    cat <<EOF | ceph orch apply -i -
+    service_type: mon
+    placement:
+      host_pattern: "mon*"
+    ---
+    service_type: mgr
+    placement:
+      host_pattern: "mgr*"
+    ---
+    service_type: osd
+    service_id: default_drive_group
+    placement:
+      host_pattern: "osd*"
+    data_devices:
+      all: true
+    EOF
+
+.. _orchestrator-cli-service-spec-retrieve:
+
+Retrieving the running Service Specification
+--------------------------------------------
+
+If the services have been started via ``ceph orch apply...``, then directly changing
+the Services Specification is complicated. Instead of attempting to directly change
+the Services Specification, we suggest exporting the running Service Specification by
+following these instructions:
+
+   .. prompt:: bash #
+
+    ceph orch ls --service-name rgw.<realm>.<zone> --export > rgw.<realm>.<zone>.yaml
+    ceph orch ls --service-type mgr --export > mgr.yaml
+    ceph orch ls --export > cluster.yaml
+
+The Specification can then be changed and re-applied as above.
+
+Updating Service Specifications
+-------------------------------
+
+The Ceph Orchestrator maintains a declarative state of each
+service in a ``ServiceSpec``. For certain operations, like updating
+the RGW HTTP port, we need to update the existing
+specification.
+
+1. List the current ``ServiceSpec``:
+
+   .. prompt:: bash #
+
+    ceph orch ls --service_name=<service-name> --export > myservice.yaml
+
+2. Update the yaml file:
+
+   .. prompt:: bash #
+
+    vi myservice.yaml
+
+3. Apply the new ``ServiceSpec``:
+
+   .. prompt:: bash #
+
+    ceph orch apply -i myservice.yaml [--dry-run]
+
+.. _orchestrator-cli-placement-spec:
+
+Daemon Placement
+================
+
+For the orchestrator to deploy a *service*, it needs to know where to deploy
+*daemons*, and how many to deploy.  This is the role of a placement
+specification.  Placement specifications can either be passed as command line arguments
+or in a YAML files.
+
+.. note::
+
+   cephadm will not deploy daemons on hosts with the ``_no_schedule`` label; see :ref:`cephadm-special-host-labels`.
+
+.. note::
+   The **apply** command can be confusing. For this reason, we recommend using
+   YAML specifications.
+
+   Each ``ceph orch apply <service-name>`` command supersedes the one before it.
+   If you do not use the proper syntax, you will clobber your work
+   as you go.
+
+   For example:
+
+   .. prompt:: bash #
+
+        ceph orch apply mon host1
+        ceph orch apply mon host2
+        ceph orch apply mon host3
+
+   This results in only one host having a monitor applied to it: host 3.
+
+   (The first command creates a monitor on host1. Then the second command
+   clobbers the monitor on host1 and creates a monitor on host2. Then the
+   third command clobbers the monitor on host2 and creates a monitor on
+   host3. In this scenario, at this point, there is a monitor ONLY on
+   host3.)
+
+   To make certain that a monitor is applied to each of these three hosts,
+   run a command like this:
+
+   .. prompt:: bash #
+
+     ceph orch apply mon "host1,host2,host3"
+
+   There is another way to apply monitors to multiple hosts: a ``yaml`` file
+   can be used. Instead of using the "ceph orch apply mon" commands, run a
+   command of this form:
+
+   .. prompt:: bash #
+
+      ceph orch apply -i file.yaml
+
+   Here is a sample **file.yaml** file
+
+   .. code-block:: yaml
+
+        service_type: mon
+        placement:
+          hosts:
+           - host1
+           - host2
+           - host3
+
+Explicit placements
+-------------------
+
+Daemons can be explicitly placed on hosts by simply specifying them:
+
+   .. prompt:: bash #
+
+    orch apply prometheus --placement="host1 host2 host3"
+
+Or in YAML:
+
+.. code-block:: yaml
+
+    service_type: prometheus
+    placement:
+      hosts:
+        - host1
+        - host2
+        - host3
+
+MONs and other services may require some enhanced network specifications:
+
+   .. prompt:: bash #
+
+    orch daemon add mon --placement="myhost:[v2:1.2.3.4:3300,v1:1.2.3.4:6789]=name"
+
+where ``[v2:1.2.3.4:3300,v1:1.2.3.4:6789]`` is the network address of the monitor
+and ``=name`` specifies the name of the new monitor.
+
+.. _orch-placement-by-labels:
+
+Placement by labels
+-------------------
+
+Daemon placement can be limited to hosts that match a specific label. To set
+a label ``mylabel`` to the appropriate hosts, run this command:
+
+  .. prompt:: bash #
+
+    ceph orch host label add *<hostname>* mylabel
+
+  To view the current hosts and labels, run this command:
+
+  .. prompt:: bash #
+
+    ceph orch host ls
+
+  For example:
+
+  .. prompt:: bash #
+
+    ceph orch host label add host1 mylabel
+    ceph orch host label add host2 mylabel
+    ceph orch host label add host3 mylabel
+    ceph orch host ls
+
+  .. code-block:: bash
+
+    HOST   ADDR   LABELS  STATUS
+    host1         mylabel
+    host2         mylabel
+    host3         mylabel
+    host4
+    host5
+
+Now, Tell cephadm to deploy daemons based on the label by running
+this command:
+
+   .. prompt:: bash #
+
+    orch apply prometheus --placement="label:mylabel"
+
+Or in YAML:
+
+.. code-block:: yaml
+
+    service_type: prometheus
+    placement:
+      label: "mylabel"
+
+* See :ref:`orchestrator-host-labels`
+
+Placement by pattern matching
+-----------------------------
+
+Daemons can be placed on hosts as well:
+
+   .. prompt:: bash #
+
+    orch apply prometheus --placement='myhost[1-3]'
+
+Or in YAML:
+
+.. code-block:: yaml
+
+    service_type: prometheus
+    placement:
+      host_pattern: "myhost[1-3]"
+
+To place a service on *all* hosts, use ``"*"``:
+
+   .. prompt:: bash #
+
+    orch apply node-exporter --placement='*'
+
+Or in YAML:
+
+.. code-block:: yaml
+
+    service_type: node-exporter
+    placement:
+      host_pattern: "*"
+
+
+Changing the number of daemons
+------------------------------
+
+By specifying ``count``, only the number of daemons specified will be created:
+
+   .. prompt:: bash #
+
+    orch apply prometheus --placement=3
+
+To deploy *daemons* on a subset of hosts, specify the count:
+
+   .. prompt:: bash #
+
+    orch apply prometheus --placement="2 host1 host2 host3"
+
+If the count is bigger than the amount of hosts, cephadm deploys one per host:
+
+   .. prompt:: bash #
+
+    orch apply prometheus --placement="3 host1 host2"
+
+The command immediately above results in two Prometheus daemons.
+
+YAML can also be used to specify limits, in the following way:
+
+.. code-block:: yaml
+
+    service_type: prometheus
+    placement:
+      count: 3
+
+YAML can also be used to specify limits on hosts:
+
+.. code-block:: yaml
+
+    service_type: prometheus
+    placement:
+      count: 2
+      hosts:
+        - host1
+        - host2
+        - host3
+
+.. _cephadm_co_location:
+
+Co-location of daemons
+----------------------
+
+Cephadm supports the deployment of multiple daemons on the same host:
+
+.. code-block:: yaml
+
+    service_type: rgw
+    placement:
+      label: rgw
+      count_per_host: 2
+
+The main reason for deploying multiple daemons per host is an additional
+performance benefit for running multiple RGW and MDS daemons on the same host.
+
+See also: 
+
+* :ref:`cephadm_mgr_co_location`.
+* :ref:`cephadm-rgw-designated_gateways`.
+
+This feature was introduced in Pacific.
+
+Algorithm description
+---------------------
+
+Cephadm's declarative state consists of a list of service specifications
+containing placement specifications.
+
+Cephadm continually compares a list of daemons actually running in the cluster
+against the list in the service specifications. Cephadm adds new daemons and
+removes old daemons as necessary in order to conform to the service
+specifications.
+
+Cephadm does the following to maintain compliance with the service
+specifications.
+
+Cephadm first selects a list of candidate hosts. Cephadm seeks explicit host
+names and selects them. If cephadm finds no explicit host names, it looks for
+label specifications. If no label is defined in the specification, cephadm
+selects hosts based on a host pattern. If no host pattern is defined, as a last
+resort, cephadm selects all known hosts as candidates.
+
+Cephadm is aware of existing daemons running services and tries to avoid moving
+them.
+
+Cephadm supports the deployment of a specific amount of services.
+Consider the following service specification:
+
+.. code-block:: yaml
+
+    service_type: mds
+    service_name: myfs
+    placement:
+      count: 3
+      label: myfs
+
+This service specifcation instructs cephadm to deploy three daemons on hosts
+labeled ``myfs`` across the cluster.
+
+If there are fewer than three daemons deployed on the candidate hosts, cephadm
+randomly chooses hosts on which to deploy new daemons.
+
+If there are more than three daemons deployed on the candidate hosts, cephadm
+removes existing daemons.
+
+Finally, cephadm removes daemons on hosts that are outside of the list of
+candidate hosts.
+
+.. note::
+
+   There is a special case that cephadm must consider.
+
+   If there are fewer hosts selected by the placement specification than
+   demanded by ``count``, cephadm will deploy only on the selected hosts.
+
+Extra Container Arguments
+=========================
+
+.. warning:: 
+  The arguments provided for extra container args are limited to whatever arguments are available for a `run` command from whichever container engine you are using. Providing any arguments the `run` command does not support (or invalid values for arguments) will cause the daemon to fail to start.
+
+
+Cephadm supports providing extra miscellaneous container arguments for
+specific cases when they may be necessary. For example, if a user needed
+to limit the amount of cpus their mon daemons make use of they could apply
+a spec like
+
+.. code-block:: yaml
+
+    service_type: mon
+    service_name: mon
+    placement:
+      hosts:
+        - host1
+        - host2
+        - host3
+    extra_container_args:
+      - "--cpus=2"
+
+which would cause each mon daemon to be deployed with `--cpus=2`.
+
+Mounting Files with Extra Container Arguments
+---------------------------------------------
+
+A common use case for extra container arguments is to mount additional
+files within the container. However, some intuitive formats for doing
+so can cause deployment to fail (see https://tracker.ceph.com/issues/57338).
+The recommended syntax for mounting a file with extra container arguments is:
+
+.. code-block:: yaml
+
+    extra_container_args:
+      - "-v"
+      - "/absolute/file/path/on/host:/absolute/file/path/in/container"
+
+For example:
+
+.. code-block:: yaml
+
+    extra_container_args:
+      - "-v"
+      - "/opt/ceph_cert/host.cert:/etc/grafana/certs/cert_file:ro"
+
+.. _orch-rm:
+
+Removing a Service
+==================
+
+In order to remove a service including the removal
+of all daemons of that service, run
+
+.. prompt:: bash
+
+  ceph orch rm <service-name>
+
+For example:
+
+.. prompt:: bash
+
+  ceph orch rm rgw.myrgw
+
+.. _cephadm-spec-unmanaged:
+
+Disabling automatic deployment of daemons
+=========================================
+
+Cephadm supports disabling the automated deployment and removal of daemons on a
+per service basis. The CLI supports two commands for this.
+
+In order to fully remove a service, see :ref:`orch-rm`.
+
+Disabling automatic management of daemons
+-----------------------------------------
+
+To disable the automatic management of dameons, set ``unmanaged=True`` in the
+:ref:`orchestrator-cli-service-spec` (``mgr.yaml``).
+
+``mgr.yaml``:
+
+.. code-block:: yaml
+
+  service_type: mgr
+  unmanaged: true
+  placement:
+    label: mgr
+
+
+.. prompt:: bash #
+
+   ceph orch apply -i mgr.yaml
+
+
+.. note::
+
+  After you apply this change in the Service Specification, cephadm will no
+  longer deploy any new daemons (even if the placement specification matches
+  additional hosts).
+
+Deploying a daemon on a host manually
+-------------------------------------
+
+.. note::
+
+  This workflow has a very limited use case and should only be used
+  in rare circumstances.
+
+To manually deploy a daemon on a host, follow these steps:
+
+Modify the service spec for a service by getting the
+existing spec, adding ``unmanaged: true``, and applying the modified spec.
+
+Then manually deploy the daemon using the following:
+
+   .. prompt:: bash #
+
+     ceph orch daemon add <daemon-type>  --placement=<placement spec>
+
+For example :
+
+   .. prompt:: bash #
+
+     ceph orch daemon add mgr --placement=my_host
+
+.. note::
+
+  Removing ``unmanaged: true`` from the service spec will
+  enable the reconciliation loop for this service and will
+  potentially lead to the removal of the daemon, depending
+  on the placement spec.
+
+Removing a daemon from a host manually
+--------------------------------------
+
+To manually remove a daemon, run a command of the following form:
+
+   .. prompt:: bash #
+
+     ceph orch daemon rm <daemon name>... [--force]
+
+For example:
+
+   .. prompt:: bash #
+
+     ceph orch daemon rm mgr.my_host.xyzxyz
+
+.. note::
+
+  For managed services (``unmanaged=False``), cephadm will automatically
+  deploy a new daemon a few seconds later.
+
+See also
+--------
+
+* See :ref:`cephadm-osd-declarative` for special handling of unmanaged OSDs.
+* See also :ref:`cephadm-pause`
diff --git a/doc/cephadm/services/iscsi.rst b/doc/cephadm/services/iscsi.rst
new file mode 100644
index 000000000..e039e8d9a
--- /dev/null
+++ b/doc/cephadm/services/iscsi.rst
@@ -0,0 +1,80 @@
+=============
+iSCSI Service
+=============
+
+.. _cephadm-iscsi:
+
+Deploying iSCSI
+===============
+
+To deploy an iSCSI gateway, create a yaml file containing a
+service specification for iscsi:
+
+.. code-block:: yaml
+
+    service_type: iscsi
+    service_id: iscsi
+    placement:
+      hosts:
+        - host1
+        - host2
+    spec:
+      pool: mypool  # RADOS pool where ceph-iscsi config data is stored.
+      trusted_ip_list: "IP_ADDRESS_1,IP_ADDRESS_2"
+      api_port: ... # optional
+      api_user: ... # optional
+      api_password: ... # optional
+      api_secure: true/false # optional
+      ssl_cert: | # optional
+        ...
+      ssl_key: | # optional
+        ...
+
+For example:
+
+.. code-block:: yaml
+
+    service_type: iscsi
+    service_id: iscsi
+    placement:
+      hosts:
+      - [...]
+    spec:
+      pool: iscsi_pool
+      trusted_ip_list: "IP_ADDRESS_1,IP_ADDRESS_2,IP_ADDRESS_3,..."
+      api_user: API_USERNAME
+      api_password: API_PASSWORD
+      ssl_cert: |
+        -----BEGIN CERTIFICATE-----
+        MIIDtTCCAp2gAwIBAgIYMC4xNzc1NDQxNjEzMzc2MjMyXzxvQ7EcMA0GCSqGSIb3
+        DQEBCwUAMG0xCzAJBgNVBAYTAlVTMQ0wCwYDVQQIDARVdGFoMRcwFQYDVQQHDA5T
+        [...]
+        -----END CERTIFICATE-----
+      ssl_key: |
+        -----BEGIN PRIVATE KEY-----
+        MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC5jdYbjtNTAKW4
+        /CwQr/7wOiLGzVxChn3mmCIF3DwbL/qvTFTX2d8bDf6LjGwLYloXHscRfxszX/4h
+        [...]
+        -----END PRIVATE KEY-----
+
+.. py:currentmodule:: ceph.deployment.service_spec
+
+.. autoclass:: IscsiServiceSpec
+   :members:
+
+
+The specification can then be applied using:
+
+.. prompt:: bash #
+
+   ceph orch apply -i iscsi.yaml
+
+
+See :ref:`orchestrator-cli-placement-spec` for details of the placement specification.
+
+See also: :ref:`orchestrator-cli-service-spec`.
+
+Further Reading
+===============
+
+* RBD: :ref:`ceph-iscsi`
diff --git a/doc/cephadm/services/mds.rst b/doc/cephadm/services/mds.rst
new file mode 100644
index 000000000..949a0fa5d
--- /dev/null
+++ b/doc/cephadm/services/mds.rst
@@ -0,0 +1,49 @@
+===========
+MDS Service
+===========
+
+
+.. _orchestrator-cli-cephfs:
+
+Deploy CephFS
+=============
+
+One or more MDS daemons is required to use the :term:`CephFS` file system.
+These are created automatically if the newer ``ceph fs volume``
+interface is used to create a new file system. For more information,
+see :ref:`fs-volumes-and-subvolumes`.
+
+For example:
+
+.. prompt:: bash #
+
+  ceph fs volume create <fs_name> --placement="<placement spec>"
+
+where ``fs_name`` is the name of the CephFS and ``placement`` is a
+:ref:`orchestrator-cli-placement-spec`.
+
+For manually deploying MDS daemons, use this specification:
+
+.. code-block:: yaml
+
+    service_type: mds
+    service_id: fs_name
+    placement:
+      count: 3
+
+
+The specification can then be applied using:
+
+.. prompt:: bash #
+
+   ceph orch apply -i mds.yaml
+
+See :ref:`orchestrator-cli-stateless-services` for manually deploying
+MDS daemons on the CLI.
+
+Further Reading
+===============
+
+* :ref:`ceph-file-system`
+
+
diff --git a/doc/cephadm/services/mgr.rst b/doc/cephadm/services/mgr.rst
new file mode 100644
index 000000000..133a00d77
--- /dev/null
+++ b/doc/cephadm/services/mgr.rst
@@ -0,0 +1,43 @@
+.. _mgr-cephadm-mgr:
+
+===========
+MGR Service
+===========
+
+The cephadm MGR service is hosting different modules, like the :ref:`mgr-dashboard`
+and the cephadm manager module.
+
+.. _cephadm-mgr-networks:
+
+Specifying Networks
+-------------------
+
+The MGR service supports binding only to a specific IP within a network.
+
+example spec file (leveraging a default placement):
+
+.. code-block:: yaml
+
+    service_type: mgr
+    networks:
+    - 192.169.142.0/24
+
+.. _cephadm_mgr_co_location:
+
+Allow co-location of MGR daemons
+================================
+
+In deployment scenarios with just a single host, cephadm still needs
+to deploy at least two MGR daemons in order to allow an automated
+upgrade of the cluster. See ``mgr_standby_modules`` in
+the :ref:`mgr-administrator-guide` for further details.
+
+See also: :ref:`cephadm_co_location`.
+
+
+Further Reading
+===============
+
+* :ref:`ceph-manager-daemon`
+* :ref:`cephadm-manually-deploy-mgr`
+
diff --git a/doc/cephadm/services/mon.rst b/doc/cephadm/services/mon.rst
new file mode 100644
index 000000000..6326b73f4
--- /dev/null
+++ b/doc/cephadm/services/mon.rst
@@ -0,0 +1,179 @@
+===========
+MON Service
+===========
+
+.. _deploy_additional_monitors:
+
+Deploying additional monitors 
+=============================
+
+A typical Ceph cluster has three or five monitor daemons that are spread
+across different hosts.  We recommend deploying five monitors if there are
+five or more nodes in your cluster.
+
+.. _CIDR: https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#CIDR_notation
+
+Ceph deploys monitor daemons automatically as the cluster grows and Ceph
+scales back monitor daemons automatically as the cluster shrinks. The
+smooth execution of this automatic growing and shrinking depends upon
+proper subnet configuration.
+
+The cephadm bootstrap procedure assigns the first monitor daemon in the
+cluster to a particular subnet. ``cephadm`` designates that subnet as the
+default subnet of the cluster. New monitor daemons will be assigned by
+default to that subnet unless cephadm is instructed to do otherwise. 
+
+If all of the ceph monitor daemons in your cluster are in the same subnet,
+manual administration of the ceph monitor daemons is not necessary.
+``cephadm`` will automatically add up to five monitors to the subnet, as
+needed, as new hosts are added to the cluster.
+
+By default, cephadm will deploy 5 daemons on arbitrary hosts. See
+:ref:`orchestrator-cli-placement-spec` for details of specifying
+the placement of daemons.
+
+Designating a Particular Subnet for Monitors
+--------------------------------------------
+
+To designate a particular IP subnet for use by ceph monitor daemons, use a
+command of the following form, including the subnet's address in `CIDR`_
+format (e.g., ``10.1.2.0/24``):
+
+  .. prompt:: bash #
+
+     ceph config set mon public_network *<mon-cidr-network>*
+
+  For example:
+
+  .. prompt:: bash #
+
+     ceph config set mon public_network 10.1.2.0/24
+
+Cephadm deploys new monitor daemons only on hosts that have IP addresses in
+the designated subnet.
+
+You can also specify two public networks by using a list of networks:
+
+  .. prompt:: bash #
+
+     ceph config set mon public_network *<mon-cidr-network1>,<mon-cidr-network2>*
+
+  For example:
+
+  .. prompt:: bash #
+
+     ceph config set mon public_network 10.1.2.0/24,192.168.0.1/24
+
+
+Deploying Monitors on a Particular Network 
+------------------------------------------
+
+You can explicitly specify the IP address or CIDR network for each monitor and
+control where each monitor is placed.  To disable automated monitor deployment,
+run this command:
+
+  .. prompt:: bash #
+
+    ceph orch apply mon --unmanaged
+
+  To deploy each additional monitor:
+
+  .. prompt:: bash #
+
+    ceph orch daemon add mon *<host1:ip-or-network1>
+
+  For example, to deploy a second monitor on ``newhost1`` using an IP
+  address ``10.1.2.123`` and a third monitor on ``newhost2`` in
+  network ``10.1.2.0/24``, run the following commands:
+
+  .. prompt:: bash #
+
+    ceph orch apply mon --unmanaged
+    ceph orch daemon add mon newhost1:10.1.2.123
+    ceph orch daemon add mon newhost2:10.1.2.0/24
+
+  Now, enable automatic placement of Daemons
+
+  .. prompt:: bash #
+
+    ceph orch apply mon --placement="newhost1,newhost2,newhost3" --dry-run
+
+  See :ref:`orchestrator-cli-placement-spec` for details of specifying
+  the placement of daemons.
+
+  Finally apply this new placement by dropping ``--dry-run``
+
+  .. prompt:: bash #
+
+    ceph orch apply mon --placement="newhost1,newhost2,newhost3"
+
+
+Moving Monitors to a Different Network
+--------------------------------------
+
+To move Monitors to a new network, deploy new monitors on the new network and
+subsequently remove monitors from the old network. It is not advised to
+modify and inject the ``monmap`` manually.
+
+First, disable the automated placement of daemons:
+
+  .. prompt:: bash #
+
+    ceph orch apply mon --unmanaged
+
+To deploy each additional monitor:
+
+  .. prompt:: bash #
+
+    ceph orch daemon add mon *<newhost1:ip-or-network1>*
+
+For example, to deploy a second monitor on ``newhost1`` using an IP
+address ``10.1.2.123`` and a third monitor on ``newhost2`` in
+network ``10.1.2.0/24``, run the following commands:
+
+  .. prompt:: bash #
+
+    ceph orch apply mon --unmanaged
+    ceph orch daemon add mon newhost1:10.1.2.123
+    ceph orch daemon add mon newhost2:10.1.2.0/24
+
+  Subsequently remove monitors from the old network:
+
+  .. prompt:: bash #
+
+    ceph orch daemon rm *mon.<oldhost1>*
+
+  Update the ``public_network``:
+
+  .. prompt:: bash #
+
+     ceph config set mon public_network *<mon-cidr-network>*
+
+  For example:
+
+  .. prompt:: bash #
+
+     ceph config set mon public_network 10.1.2.0/24
+
+  Now, enable automatic placement of Daemons
+
+  .. prompt:: bash #
+
+    ceph orch apply mon --placement="newhost1,newhost2,newhost3" --dry-run
+
+  See :ref:`orchestrator-cli-placement-spec` for details of specifying
+  the placement of daemons.
+
+  Finally apply this new placement by dropping ``--dry-run``
+
+  .. prompt:: bash #
+
+    ceph orch apply mon --placement="newhost1,newhost2,newhost3" 
+
+Futher Reading
+==============
+
+* :ref:`rados-operations`
+* :ref:`rados-troubleshooting-mon`
+* :ref:`cephadm-restore-quorum`
+
diff --git a/doc/cephadm/services/monitoring.rst b/doc/cephadm/services/monitoring.rst
new file mode 100644
index 000000000..86e3e3f69
--- /dev/null
+++ b/doc/cephadm/services/monitoring.rst
@@ -0,0 +1,457 @@
+.. _mgr-cephadm-monitoring:
+
+Monitoring Services
+===================
+
+Ceph Dashboard uses `Prometheus <https://prometheus.io/>`_, `Grafana
+<https://grafana.com/>`_, and related tools to store and visualize detailed
+metrics on cluster utilization and performance.  Ceph users have three options:
+
+#. Have cephadm deploy and configure these services.  This is the default
+   when bootstrapping a new cluster unless the ``--skip-monitoring-stack``
+   option is used.
+#. Deploy and configure these services manually.  This is recommended for users
+   with existing prometheus services in their environment (and in cases where
+   Ceph is running in Kubernetes with Rook).
+#. Skip the monitoring stack completely.  Some Ceph dashboard graphs will
+   not be available.
+
+The monitoring stack consists of `Prometheus <https://prometheus.io/>`_,
+Prometheus exporters (:ref:`mgr-prometheus`, `Node exporter
+<https://prometheus.io/docs/guides/node-exporter/>`_), `Prometheus Alert
+Manager <https://prometheus.io/docs/alerting/alertmanager/>`_ and `Grafana
+<https://grafana.com/>`_.
+
+.. note::
+
+  Prometheus' security model presumes that untrusted users have access to the
+  Prometheus HTTP endpoint and logs. Untrusted users have access to all the
+  (meta)data Prometheus collects that is contained in the database, plus a
+  variety of operational and debugging information.
+
+  However, Prometheus' HTTP API is limited to read-only operations.
+  Configurations can *not* be changed using the API and secrets are not
+  exposed. Moreover, Prometheus has some built-in measures to mitigate the
+  impact of denial of service attacks.
+
+  Please see `Prometheus' Security model
+  <https://prometheus.io/docs/operating/security/>` for more detailed
+  information.
+
+Deploying monitoring with cephadm
+---------------------------------
+
+The default behavior of ``cephadm`` is to deploy a basic monitoring stack.  It
+is however possible that you have a Ceph cluster without a monitoring stack,
+and you would like to add a monitoring stack to it. (Here are some ways that
+you might have come to have a Ceph cluster without a monitoring stack: You
+might have passed the ``--skip-monitoring stack`` option to ``cephadm`` during
+the installation of the cluster, or you might have converted an existing
+cluster (which had no monitoring stack) to cephadm management.)
+
+To set up monitoring on a Ceph cluster that has no monitoring, follow the
+steps below:
+
+#. Deploy a node-exporter service on every node of the cluster.  The node-exporter provides host-level metrics like CPU and memory utilization:
+
+   .. prompt:: bash #
+
+     ceph orch apply node-exporter
+
+#. Deploy alertmanager:
+
+   .. prompt:: bash #
+
+     ceph orch apply alertmanager
+
+#. Deploy Prometheus. A single Prometheus instance is sufficient, but
+   for high availablility (HA) you might want to deploy two:
+
+   .. prompt:: bash #
+
+     ceph orch apply prometheus
+
+   or 
+
+   .. prompt:: bash #
+     
+     ceph orch apply prometheus --placement 'count:2'
+
+#. Deploy grafana:
+
+   .. prompt:: bash #
+
+     ceph orch apply grafana
+
+.. _cephadm-monitoring-networks-ports:
+
+Networks and Ports
+~~~~~~~~~~~~~~~~~~
+
+All monitoring services can have the network and port they bind to configured with a yaml service specification
+
+example spec file:
+
+.. code-block:: yaml
+
+    service_type: grafana
+    service_name: grafana
+    placement:
+      count: 1
+    networks:
+    - 192.169.142.0/24
+    spec:
+      port: 4200
+
+.. _cephadm_monitoring-images:
+
+Using custom images
+~~~~~~~~~~~~~~~~~~~
+
+It is possible to install or upgrade monitoring components based on other
+images.  To do so, the name of the image to be used needs to be stored in the
+configuration first.  The following configuration options are available.
+
+- ``container_image_prometheus``
+- ``container_image_grafana``
+- ``container_image_alertmanager``
+- ``container_image_node_exporter``
+
+Custom images can be set with the ``ceph config`` command
+
+.. code-block:: bash
+
+     ceph config set mgr mgr/cephadm/<option_name> <value>
+
+For example
+
+.. code-block:: bash
+
+     ceph config set mgr mgr/cephadm/container_image_prometheus prom/prometheus:v1.4.1
+
+If there were already running monitoring stack daemon(s) of the type whose
+image you've changed, you must redeploy the daemon(s) in order to have them
+actually use the new image.
+
+For example, if you had changed the prometheus image
+
+.. prompt:: bash #
+
+     ceph orch redeploy prometheus
+
+
+.. note::
+
+     By setting a custom image, the default value will be overridden (but not
+     overwritten).  The default value changes when updates become available.
+     By setting a custom image, you will not be able to update the component
+     you have set the custom image for automatically.  You will need to
+     manually update the configuration (image name and tag) to be able to
+     install updates.
+
+     If you choose to go with the recommendations instead, you can reset the
+     custom image you have set before.  After that, the default value will be
+     used again.  Use ``ceph config rm`` to reset the configuration option
+
+     .. code-block:: bash
+
+          ceph config rm mgr mgr/cephadm/<option_name>
+
+     For example
+
+     .. code-block:: bash
+
+          ceph config rm mgr mgr/cephadm/container_image_prometheus
+
+See also :ref:`cephadm-airgap`.
+
+.. _cephadm-overwrite-jinja2-templates:
+
+Using custom configuration files
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+By overriding cephadm templates, it is possible to completely customize the
+configuration files for monitoring services.
+
+Internally, cephadm already uses `Jinja2
+<https://jinja.palletsprojects.com/en/2.11.x/>`_ templates to generate the
+configuration files for all monitoring components. To be able to customize the
+configuration of Prometheus, Grafana or the Alertmanager it is possible to store
+a Jinja2 template for each service that will be used for configuration
+generation instead. This template will be evaluated every time a service of that
+kind is deployed or reconfigured. That way, the custom configuration is
+preserved and automatically applied on future deployments of these services.
+
+.. note::
+
+  The configuration of the custom template is also preserved when the default
+  configuration of cephadm changes. If the updated configuration is to be used,
+  the custom template needs to be migrated *manually* after each upgrade of Ceph.
+
+Option names
+""""""""""""
+
+The following templates for files that will be generated by cephadm can be
+overridden. These are the names to be used when storing with ``ceph config-key
+set``:
+
+- ``services/alertmanager/alertmanager.yml``
+- ``services/grafana/ceph-dashboard.yml``
+- ``services/grafana/grafana.ini``
+- ``services/prometheus/prometheus.yml``
+- ``services/prometheus/alerting/custom_alerts.yml``
+
+You can look up the file templates that are currently used by cephadm in
+``src/pybind/mgr/cephadm/templates``:
+
+- ``services/alertmanager/alertmanager.yml.j2``
+- ``services/grafana/ceph-dashboard.yml.j2``
+- ``services/grafana/grafana.ini.j2``
+- ``services/prometheus/prometheus.yml.j2``
+
+Usage
+"""""
+
+The following command applies a single line value:
+
+.. code-block:: bash
+
+  ceph config-key set mgr/cephadm/<option_name> <value>
+
+To set contents of files as template use the ``-i`` argument:
+
+.. code-block:: bash
+
+  ceph config-key set mgr/cephadm/<option_name> -i $PWD/<filename>
+
+.. note::
+
+  When using files as input to ``config-key`` an absolute path to the file must
+  be used.
+
+
+Then the configuration file for the service needs to be recreated.
+This is done using `reconfig`. For more details see the following example.
+
+Example
+"""""""
+
+.. code-block:: bash
+
+  # set the contents of ./prometheus.yml.j2 as template
+  ceph config-key set mgr/cephadm/services/prometheus/prometheus.yml \
+    -i $PWD/prometheus.yml.j2
+
+  # reconfig the prometheus service
+  ceph orch reconfig prometheus
+
+.. code-block:: bash
+
+  # set additional custom alerting rules for Prometheus
+  ceph config-key set mgr/cephadm/services/prometheus/alerting/custom_alerts.yml \
+    -i $PWD/custom_alerts.yml
+
+  # Note that custom alerting rules are not parsed by Jinja and hence escaping
+  # will not be an issue.
+
+Deploying monitoring without cephadm
+------------------------------------
+
+If you have an existing prometheus monitoring infrastructure, or would like
+to manage it yourself, you need to configure it to integrate with your Ceph
+cluster.
+
+* Enable the prometheus module in the ceph-mgr daemon
+
+  .. code-block:: bash
+
+     ceph mgr module enable prometheus
+
+  By default, ceph-mgr presents prometheus metrics on port 9283 on each host
+  running a ceph-mgr daemon.  Configure prometheus to scrape these.
+
+* To enable the dashboard's prometheus-based alerting, see :ref:`dashboard-alerting`.
+
+* To enable dashboard integration with Grafana, see :ref:`dashboard-grafana`.
+
+Disabling monitoring
+--------------------
+
+To disable monitoring and remove the software that supports it, run the following commands:
+
+.. code-block:: console
+
+  $ ceph orch rm grafana
+  $ ceph orch rm prometheus --force   # this will delete metrics data collected so far
+  $ ceph orch rm node-exporter
+  $ ceph orch rm alertmanager
+  $ ceph mgr module disable prometheus
+
+See also :ref:`orch-rm`.
+
+Setting up RBD-Image monitoring
+-------------------------------
+
+Due to performance reasons, monitoring of RBD images is disabled by default. For more information please see
+:ref:`prometheus-rbd-io-statistics`. If disabled, the overview and details dashboards will stay empty in Grafana
+and the metrics will not be visible in Prometheus.
+
+Setting up Prometheus
+-----------------------
+
+Setting Prometheus Retention Time
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Cephadm provides the option to set the Prometheus TDSB retention time using
+a ``retention_time`` field in the Prometheus service spec. The value defaults
+to 15 days (15d). If you would like a different value, such as 1 year (1y) you
+can apply a service spec similar to:
+
+.. code-block:: yaml
+
+    service_type: prometheus
+    placement:
+      count: 1
+    spec:
+      retention_time: "1y"
+
+.. note::
+
+  If you already had Prometheus daemon(s) deployed before and are updating an
+  existent spec as opposed to doing a fresh Prometheus deployment, you must also
+  tell cephadm to redeploy the Prometheus daemon(s) to put this change into effect.
+  This can be done with a ``ceph orch redeploy prometheus`` command.
+
+Setting up Grafana
+------------------
+
+Manually setting the Grafana URL
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Cephadm automatically configures Prometheus, Grafana, and Alertmanager in
+all cases except one.
+
+In a some setups, the Dashboard user's browser might not be able to access the
+Grafana URL that is configured in Ceph Dashboard. This can happen when the
+cluster and the accessing user are in different DNS zones.
+
+If this is the case, you can use a configuration option for Ceph Dashboard
+to set the URL that the user's browser will use to access Grafana. This
+value will never be altered by cephadm. To set this configuration option,
+issue the following command:
+
+   .. prompt:: bash $
+
+     ceph dashboard set-grafana-frontend-api-url <grafana-server-api>
+
+It might take a minute or two for services to be deployed. After the
+services have been deployed, you should see something like this when you issue the command ``ceph orch ls``:
+
+.. code-block:: console
+
+  $ ceph orch ls
+  NAME           RUNNING  REFRESHED  IMAGE NAME                                      IMAGE ID        SPEC
+  alertmanager       1/1  6s ago     docker.io/prom/alertmanager:latest              0881eb8f169f  present
+  crash              2/2  6s ago     docker.io/ceph/daemon-base:latest-master-devel  mix           present
+  grafana            1/1  0s ago     docker.io/pcuzner/ceph-grafana-el8:latest       f77afcf0bcf6   absent
+  node-exporter      2/2  6s ago     docker.io/prom/node-exporter:latest             e5a616e4b9cf  present
+  prometheus         1/1  6s ago     docker.io/prom/prometheus:latest                e935122ab143  present
+
+Configuring SSL/TLS for Grafana
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``cephadm`` deploys Grafana using the certificate defined in the ceph
+key/value store. If no certificate is specified, ``cephadm`` generates a
+self-signed certificate during the deployment of the Grafana service.
+
+A custom certificate can be configured using the following commands:
+
+.. prompt:: bash #
+
+  ceph config-key set mgr/cephadm/grafana_key -i $PWD/key.pem
+  ceph config-key set mgr/cephadm/grafana_crt -i $PWD/certificate.pem
+
+If you have already deployed Grafana, run ``reconfig`` on the service to
+update its configuration:
+
+.. prompt:: bash #
+
+  ceph orch reconfig grafana
+
+The ``reconfig`` command also sets the proper URL for Ceph Dashboard.
+
+Setting the initial admin password
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+By default, Grafana will not create an initial
+admin user. In order to create the admin user, please create a file
+``grafana.yaml`` with this content:
+
+.. code-block:: yaml
+
+  service_type: grafana
+  spec:
+    initial_admin_password: mypassword
+
+Then apply this specification:
+
+.. code-block:: bash
+
+  ceph orch apply -i grafana.yaml
+  ceph orch redeploy grafana
+
+Grafana will now create an admin user called ``admin`` with the
+given password.
+
+
+Setting up Alertmanager
+-----------------------
+
+Adding Alertmanager webhooks
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To add new webhooks to the Alertmanager configuration, add additional
+webhook urls like so:
+
+.. code-block:: yaml
+
+    service_type: alertmanager
+    spec:
+      user_data:
+        default_webhook_urls:
+        - "https://foo"
+        - "https://bar"
+
+Where ``default_webhook_urls`` is a list of additional URLs that are
+added to the default receivers' ``<webhook_configs>`` configuration.
+
+Run ``reconfig`` on the service to update its configuration:
+
+.. prompt:: bash #
+
+  ceph orch reconfig alertmanager
+
+Turn on Certificate Validation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you are using certificates for alertmanager and want to make sure
+these certs are verified, you should set the "secure" option to
+true in your alertmanager spec (this defaults to false).
+
+.. code-block:: yaml
+
+    service_type: alertmanager
+    spec:
+      secure: true
+
+If you already had alertmanager daemons running before applying the spec
+you must reconfigure them to update their configuration
+
+.. prompt:: bash #
+
+  ceph orch reconfig alertmanager
+
+Further Reading
+---------------
+
+* :ref:`mgr-prometheus`
diff --git a/doc/cephadm/services/nfs.rst b/doc/cephadm/services/nfs.rst
new file mode 100644
index 000000000..c48d0f765
--- /dev/null
+++ b/doc/cephadm/services/nfs.rst
@@ -0,0 +1,120 @@
+.. _deploy-cephadm-nfs-ganesha:
+
+===========
+NFS Service
+===========
+
+.. note:: Only the NFSv4 protocol is supported.
+
+The simplest way to manage NFS is via the ``ceph nfs cluster ...``
+commands; see :ref:`mgr-nfs`.  This document covers how to manage the
+cephadm services directly, which should only be necessary for unusual NFS
+configurations.
+
+Deploying NFS ganesha
+=====================
+
+Cephadm deploys NFS Ganesha daemon (or set of daemons).  The configuration for
+NFS is stored in the ``nfs-ganesha`` pool and exports are managed via the
+``ceph nfs export ...`` commands and via the dashboard.
+
+To deploy a NFS Ganesha gateway, run the following command:
+
+.. prompt:: bash #
+
+    ceph orch apply nfs *<svc_id>* [--port *<port>*] [--placement ...]
+
+For example, to deploy NFS with a service id of *foo* on the default
+port 2049 with the default placement of a single daemon:
+
+.. prompt:: bash #
+
+   ceph orch apply nfs foo
+
+See :ref:`orchestrator-cli-placement-spec` for the details of the placement
+specification.
+
+Service Specification
+=====================
+
+Alternatively, an NFS service can be applied using a YAML specification. 
+
+.. code-block:: yaml
+
+    service_type: nfs
+    service_id: mynfs
+    placement:
+      hosts:
+        - host1
+        - host2
+    spec:
+      port: 12345
+
+In this example, we run the server on the non-default ``port`` of
+12345 (instead of the default 2049) on ``host1`` and ``host2``.
+
+The specification can then be applied by running the following command:
+
+.. prompt:: bash #
+
+   ceph orch apply -i nfs.yaml
+
+.. _cephadm-ha-nfs:
+
+High-availability NFS
+=====================
+
+Deploying an *ingress* service for an existing *nfs* service will provide:
+
+* a stable, virtual IP that can be used to access the NFS server
+* fail-over between hosts if there is a host failure
+* load distribution across multiple NFS gateways (although this is rarely necessary)
+
+Ingress for NFS can be deployed for an existing NFS service
+(``nfs.mynfs`` in this example) with the following specification:
+
+.. code-block:: yaml
+
+    service_type: ingress
+    service_id: nfs.mynfs
+    placement:
+      count: 2
+    spec:
+      backend_service: nfs.mynfs
+      frontend_port: 2049
+      monitor_port: 9000
+      virtual_ip: 10.0.0.123/24
+
+A few notes:
+
+  * The *virtual_ip* must include a CIDR prefix length, as in the
+    example above.  The virtual IP will normally be configured on the
+    first identified network interface that has an existing IP in the
+    same subnet.  You can also specify a *virtual_interface_networks*
+    property to match against IPs in other networks; see
+    :ref:`ingress-virtual-ip` for more information.
+  * The *monitor_port* is used to access the haproxy load status
+    page.  The user is ``admin`` by default, but can be modified by
+    via an *admin* property in the spec.  If a password is not
+    specified via a *password* property in the spec, the auto-generated password
+    can be found with:
+
+    .. prompt:: bash #
+
+	ceph config-key get mgr/cephadm/ingress.*{svc_id}*/monitor_password
+
+    For example:
+
+    .. prompt:: bash #
+
+	ceph config-key get mgr/cephadm/ingress.nfs.myfoo/monitor_password
+	
+  * The backend service (``nfs.mynfs`` in this example) should include
+    a *port* property that is not 2049 to avoid conflicting with the
+    ingress service, which could be placed on the same host(s).
+
+Further Reading
+===============
+
+* CephFS: :ref:`cephfs-nfs`
+* MGR: :ref:`mgr-nfs`
diff --git a/doc/cephadm/services/osd.rst b/doc/cephadm/services/osd.rst
new file mode 100644
index 000000000..de0d4f82a
--- /dev/null
+++ b/doc/cephadm/services/osd.rst
@@ -0,0 +1,936 @@
+***********
+OSD Service
+***********
+.. _device management: ../rados/operations/devices
+.. _libstoragemgmt: https://github.com/libstorage/libstoragemgmt
+
+List Devices
+============
+
+``ceph-volume`` scans each host in the cluster from time to time in order
+to determine which devices are present and whether they are eligible to be
+used as OSDs.
+
+To print a list of devices discovered by ``cephadm``, run this command:
+
+.. prompt:: bash #
+
+    ceph orch device ls [--hostname=...] [--wide] [--refresh]
+
+Example
+::
+
+  Hostname  Path      Type  Serial              Size   Health   Ident  Fault  Available
+  srv-01    /dev/sdb  hdd   15P0A0YFFRD6         300G  Unknown  N/A    N/A    No
+  srv-01    /dev/sdc  hdd   15R0A08WFRD6         300G  Unknown  N/A    N/A    No
+  srv-01    /dev/sdd  hdd   15R0A07DFRD6         300G  Unknown  N/A    N/A    No
+  srv-01    /dev/sde  hdd   15P0A0QDFRD6         300G  Unknown  N/A    N/A    No
+  srv-02    /dev/sdb  hdd   15R0A033FRD6         300G  Unknown  N/A    N/A    No
+  srv-02    /dev/sdc  hdd   15R0A05XFRD6         300G  Unknown  N/A    N/A    No
+  srv-02    /dev/sde  hdd   15R0A0ANFRD6         300G  Unknown  N/A    N/A    No
+  srv-02    /dev/sdf  hdd   15R0A06EFRD6         300G  Unknown  N/A    N/A    No
+  srv-03    /dev/sdb  hdd   15R0A0OGFRD6         300G  Unknown  N/A    N/A    No
+  srv-03    /dev/sdc  hdd   15R0A0P7FRD6         300G  Unknown  N/A    N/A    No
+  srv-03    /dev/sdd  hdd   15R0A0O7FRD6         300G  Unknown  N/A    N/A    No
+
+Using the ``--wide`` option provides all details relating to the device,
+including any reasons that the device might not be eligible for use as an OSD.
+
+In the above example you can see fields named "Health", "Ident", and "Fault".
+This information is provided by integration with `libstoragemgmt`_. By default,
+this integration is disabled (because `libstoragemgmt`_ may not be 100%
+compatible with your hardware).  To make ``cephadm`` include these fields,
+enable cephadm's "enhanced device scan" option as follows;
+
+.. prompt:: bash #
+
+    ceph config set mgr mgr/cephadm/device_enhanced_scan true
+
+.. warning::
+    Although the libstoragemgmt library performs standard SCSI inquiry calls,
+    there is no guarantee that your firmware fully implements these standards.
+    This can lead to erratic behaviour and even bus resets on some older
+    hardware. It is therefore recommended that, before enabling this feature,
+    you test your hardware's compatibility with libstoragemgmt first to avoid
+    unplanned interruptions to services.
+
+    There are a number of ways to test compatibility, but the simplest may be
+    to use the cephadm shell to call libstoragemgmt directly - ``cephadm shell
+    lsmcli ldl``. If your hardware is supported you should see something like
+    this: 
+
+    ::
+
+      Path     | SCSI VPD 0x83    | Link Type | Serial Number      | Health Status
+      ----------------------------------------------------------------------------
+      /dev/sda | 50000396082ba631 | SAS       | 15P0A0R0FRD6       | Good
+      /dev/sdb | 50000396082bbbf9 | SAS       | 15P0A0YFFRD6       | Good
+
+
+After you have enabled libstoragemgmt support, the output will look something
+like this:
+
+::
+
+  # ceph orch device ls
+  Hostname   Path      Type  Serial              Size   Health   Ident  Fault  Available
+  srv-01     /dev/sdb  hdd   15P0A0YFFRD6         300G  Good     Off    Off    No
+  srv-01     /dev/sdc  hdd   15R0A08WFRD6         300G  Good     Off    Off    No
+  :
+
+In this example, libstoragemgmt has confirmed the health of the drives and the ability to
+interact with the Identification and Fault LEDs on the drive enclosures. For further
+information about interacting with these LEDs, refer to `device management`_.
+
+.. note::
+    The current release of `libstoragemgmt`_ (1.8.8) supports SCSI, SAS, and SATA based
+    local disks only. There is no official support for NVMe devices (PCIe)
+
+.. _cephadm-deploy-osds:
+
+Deploy OSDs
+===========
+
+Listing Storage Devices
+-----------------------
+
+In order to deploy an OSD, there must be a storage device that is *available* on
+which the OSD will be deployed.
+
+Run this command to display an inventory of storage devices on all cluster hosts:
+
+.. prompt:: bash #
+
+  ceph orch device ls
+
+A storage device is considered *available* if all of the following
+conditions are met:
+
+* The device must have no partitions.
+* The device must not have any LVM state.
+* The device must not be mounted.
+* The device must not contain a file system.
+* The device must not contain a Ceph BlueStore OSD.
+* The device must be larger than 5 GB.
+
+Ceph will not provision an OSD on a device that is not available.
+
+Creating New OSDs
+-----------------
+
+There are a few ways to create new OSDs:
+
+* Tell Ceph to consume any available and unused storage device:
+
+  .. prompt:: bash #
+
+    ceph orch apply osd --all-available-devices
+
+* Create an OSD from a specific device on a specific host:
+
+  .. prompt:: bash #
+
+    ceph orch daemon add osd *<host>*:*<device-path>*
+
+  For example:
+
+  .. prompt:: bash #
+
+    ceph orch daemon add osd host1:/dev/sdb
+
+  Advanced OSD creation from specific devices on a specific host:
+
+  .. prompt:: bash #
+
+    ceph orch daemon add osd host1:data_devices=/dev/sda,/dev/sdb,db_devices=/dev/sdc,osds_per_device=2
+
+* You can use :ref:`drivegroups` to categorize device(s) based on their
+  properties. This might be useful in forming a clearer picture of which
+  devices are available to consume. Properties include device type (SSD or
+  HDD), device model names, size, and the hosts on which the devices exist:
+
+  .. prompt:: bash #
+
+    ceph orch apply -i spec.yml
+
+Dry Run
+-------
+
+The ``--dry-run`` flag causes the orchestrator to present a preview of what
+will happen without actually creating the OSDs.
+
+For example:
+
+   .. prompt:: bash #
+
+     ceph orch apply osd --all-available-devices --dry-run
+
+   ::
+
+     NAME                  HOST  DATA      DB  WAL
+     all-available-devices node1 /dev/vdb  -   -
+     all-available-devices node2 /dev/vdc  -   -
+     all-available-devices node3 /dev/vdd  -   -
+
+.. _cephadm-osd-declarative:
+
+Declarative State
+-----------------
+
+The effect of ``ceph orch apply`` is persistent. This means that drives that
+are added to the system after the ``ceph orch apply`` command completes will be
+automatically found and added to the cluster.  It also means that drives that
+become available (by zapping, for example) after the ``ceph orch apply``
+command completes will be automatically found and added to the cluster.
+
+We will examine the effects of the following command:
+
+   .. prompt:: bash #
+
+     ceph orch apply osd --all-available-devices
+
+After running the above command: 
+
+* If you add new disks to the cluster, they will automatically be used to
+  create new OSDs.
+* If you remove an OSD and clean the LVM physical volume, a new OSD will be
+  created automatically.
+
+To disable the automatic creation of OSD on available devices, use the
+``unmanaged`` parameter:
+
+If you want to avoid this behavior (disable automatic creation of OSD on available devices), use the ``unmanaged`` parameter:
+
+.. prompt:: bash #
+
+   ceph orch apply osd --all-available-devices --unmanaged=true
+
+.. note::
+
+  Keep these three facts in mind:
+
+  - The default behavior of ``ceph orch apply`` causes cephadm constantly to reconcile. This means that cephadm creates OSDs as soon as new drives are detected.
+
+  - Setting ``unmanaged: True`` disables the creation of OSDs. If ``unmanaged: True`` is set, nothing will happen even if you apply a new OSD service.
+
+  - ``ceph orch daemon add`` creates OSDs, but does not add an OSD service.
+
+* For cephadm, see also :ref:`cephadm-spec-unmanaged`.
+
+.. _cephadm-osd-removal:
+
+Remove an OSD
+=============
+
+Removing an OSD from a cluster involves two steps:
+
+#. evacuating all placement groups (PGs) from the cluster
+#. removing the PG-free OSD from the cluster
+
+The following command performs these two steps:
+
+.. prompt:: bash #
+
+  ceph orch osd rm <osd_id(s)> [--replace] [--force]
+
+Example:
+
+.. prompt:: bash #
+
+  ceph orch osd rm 0
+
+Expected output::
+
+   Scheduled OSD(s) for removal
+
+OSDs that are not safe to destroy will be rejected.
+
+.. note::
+    After removing OSDs, if the drives the OSDs were deployed on once again
+    become available, cephadm may automatically try to deploy more OSDs
+    on these drives if they match an existing drivegroup spec. If you deployed
+    the OSDs you are removing with a spec and don't want any new OSDs deployed on
+    the drives after removal, it's best to modify the drivegroup spec before removal.
+    Either set ``unmanaged: true`` to stop it from picking up new drives at all,
+    or modify it in some way that it no longer matches the drives used for the
+    OSDs you wish to remove. Then re-apply the spec. For more info on drivegroup
+    specs see :ref:`drivegroups`. For more info on the declarative nature of
+    cephadm in reference to deploying OSDs, see :ref:`cephadm-osd-declarative`
+
+Monitoring OSD State
+--------------------
+
+You can query the state of OSD operation with the following command:
+
+.. prompt:: bash #
+
+   ceph orch osd rm status
+
+Expected output::
+
+    OSD_ID  HOST         STATE                    PG_COUNT  REPLACE  FORCE  STARTED_AT
+    2       cephadm-dev  done, waiting for purge  0         True     False  2020-07-17 13:01:43.147684
+    3       cephadm-dev  draining                 17        False    True   2020-07-17 13:01:45.162158
+    4       cephadm-dev  started                  42        False    True   2020-07-17 13:01:45.162158
+
+
+When no PGs are left on the OSD, it will be decommissioned and removed from the cluster.
+
+.. note::
+    After removing an OSD, if you wipe the LVM physical volume in the device used by the removed OSD, a new OSD will be created.
+    For more information on this, read about the ``unmanaged`` parameter in :ref:`cephadm-osd-declarative`.
+
+Stopping OSD Removal
+--------------------
+
+It is possible to stop queued OSD removals by using the following command:
+
+.. prompt:: bash #
+
+  ceph orch osd rm stop <osd_id(s)>
+
+Example:
+
+.. prompt:: bash #
+
+    ceph orch osd rm stop 4
+
+Expected output::
+
+    Stopped OSD(s) removal
+
+This resets the initial state of the OSD and takes it off the removal queue.
+
+.. _cephadm-replacing-an-osd:
+
+Replacing an OSD
+----------------
+
+.. prompt:: bash #
+
+  orch osd rm <osd_id(s)> --replace [--force]
+
+Example:
+
+.. prompt:: bash #
+
+  ceph orch osd rm 4 --replace
+
+Expected output::
+
+   Scheduled OSD(s) for replacement
+
+This follows the same procedure as the procedure in the "Remove OSD" section, with
+one exception: the OSD is not permanently removed from the CRUSH hierarchy, but is
+instead assigned a 'destroyed' flag.
+
+.. note::
+    The new OSD that will replace the removed OSD must be created on the same host 
+    as the OSD that was removed.
+
+**Preserving the OSD ID**
+
+The 'destroyed' flag is used to determine which OSD ids will be reused in the
+next OSD deployment.
+
+If you use OSDSpecs for OSD deployment, your newly added disks will be assigned
+the OSD ids of their replaced counterparts. This assumes that the new disks
+still match the OSDSpecs.
+
+Use the ``--dry-run`` flag to make certain that the ``ceph orch apply osd`` 
+command does what you want it to. The ``--dry-run`` flag shows you what the
+outcome of the command will be without making the changes you specify. When
+you are satisfied that the command will do what you want, run the command
+without the ``--dry-run`` flag.
+
+.. tip::
+
+  The name of your OSDSpec can be retrieved with the command ``ceph orch ls``
+
+Alternatively, you can use your OSDSpec file:
+
+.. prompt:: bash #
+
+  ceph orch apply -i <osd_spec_file> --dry-run
+
+Expected output::
+
+  NAME                  HOST  DATA     DB WAL
+  <name_of_osd_spec>    node1 /dev/vdb -  -
+
+
+When this output reflects your intention, omit the ``--dry-run`` flag to
+execute the deployment.
+
+
+Erasing Devices (Zapping Devices)
+---------------------------------
+
+Erase (zap) a device so that it can be reused. ``zap`` calls ``ceph-volume
+zap`` on the remote host.
+
+.. prompt:: bash #
+
+  ceph orch device zap <hostname> <path>
+
+Example command:
+
+.. prompt:: bash #
+
+  ceph orch device zap my_hostname /dev/sdx
+
+.. note::
+    If the unmanaged flag is unset, cephadm automatically deploys drives that
+    match the OSDSpec.  For example, if you use the
+    ``all-available-devices`` option when creating OSDs, when you ``zap`` a
+    device the cephadm orchestrator automatically creates a new OSD in the
+    device.  To disable this behavior, see :ref:`cephadm-osd-declarative`.
+
+
+.. _osd_autotune:
+
+Automatically tuning OSD memory
+===============================
+
+OSD daemons will adjust their memory consumption based on the
+``osd_memory_target`` config option (several gigabytes, by
+default).  If Ceph is deployed on dedicated nodes that are not sharing
+memory with other services, cephadm can automatically adjust the per-OSD
+memory consumption based on the total amount of RAM and the number of deployed
+OSDs.
+
+.. warning:: Cephadm sets ``osd_memory_target_autotune`` to ``true`` by default which is unsuitable for hyperconverged infrastructures.
+
+Cephadm will start with a fraction
+(``mgr/cephadm/autotune_memory_target_ratio``, which defaults to
+``.7``) of the total RAM in the system, subtract off any memory
+consumed by non-autotuned daemons (non-OSDs, for OSDs for which
+``osd_memory_target_autotune`` is false), and then divide by the
+remaining OSDs.
+
+The final targets are reflected in the config database with options like::
+
+  WHO   MASK      LEVEL   OPTION              VALUE
+  osd   host:foo  basic   osd_memory_target   126092301926
+  osd   host:bar  basic   osd_memory_target   6442450944
+
+Both the limits and the current memory consumed by each daemon are visible from
+the ``ceph orch ps`` output in the ``MEM LIMIT`` column::
+
+  NAME        HOST  PORTS  STATUS         REFRESHED  AGE  MEM USED  MEM LIMIT  VERSION                IMAGE ID      CONTAINER ID  
+  osd.1       dael         running (3h)     10s ago   3h    72857k     117.4G  17.0.0-3781-gafaed750  7015fda3cd67  9e183363d39c  
+  osd.2       dael         running (81m)    10s ago  81m    63989k     117.4G  17.0.0-3781-gafaed750  7015fda3cd67  1f0cc479b051  
+  osd.3       dael         running (62m)    10s ago  62m    64071k     117.4G  17.0.0-3781-gafaed750  7015fda3cd67  ac5537492f27  
+
+To exclude an OSD from memory autotuning, disable the autotune option
+for that OSD and also set a specific memory target.  For example,
+
+  .. prompt:: bash #
+
+    ceph config set osd.123 osd_memory_target_autotune false
+    ceph config set osd.123 osd_memory_target 16G
+
+
+.. _drivegroups:
+
+Advanced OSD Service Specifications
+===================================
+
+:ref:`orchestrator-cli-service-spec`\s of type ``osd`` are a way to describe a
+cluster layout, using the properties of disks. Service specifications give the
+user an abstract way to tell Ceph which disks should turn into OSDs with which
+configurations, without knowing the specifics of device names and paths.
+
+Service specifications make it possible to define a yaml or json file that can
+be used to reduce the amount of manual work involved in creating OSDs.
+
+For example, instead of running the following command:
+
+.. prompt:: bash [monitor.1]#
+
+  ceph orch daemon add osd *<host>*:*<path-to-device>*
+
+for each device and each host, we can define a yaml or json file that allows us
+to describe the layout. Here's the most basic example.
+
+Create a file called (for example) ``osd_spec.yml``:
+
+.. code-block:: yaml
+
+    service_type: osd
+    service_id: default_drive_group  # custom name of the osd spec
+    placement:
+      host_pattern: '*'              # which hosts to target
+    spec:
+      data_devices:                  # the type of devices you are applying specs to
+        all: true                    # a filter, check below for a full list
+
+This means :
+
+#. Turn any available device (ceph-volume decides what 'available' is) into an
+   OSD on all hosts that match the glob pattern '*'. (The glob pattern matches
+   against the registered hosts from `host ls`) A more detailed section on
+   host_pattern is available below.
+
+#. Then pass it to `osd create` like this:
+
+   .. prompt:: bash [monitor.1]#
+
+     ceph orch apply -i /path/to/osd_spec.yml
+
+   This instruction will be issued to all the matching hosts, and will deploy
+   these OSDs.
+
+   Setups more complex than the one specified by the ``all`` filter are
+   possible. See :ref:`osd_filters` for details.
+
+   A ``--dry-run`` flag can be passed to the ``apply osd`` command to display a
+   synopsis of the proposed layout.
+
+Example
+
+.. prompt:: bash [monitor.1]#
+
+   ceph orch apply -i /path/to/osd_spec.yml --dry-run
+
+
+
+.. _osd_filters:
+
+Filters
+-------
+
+.. note::
+   Filters are applied using an `AND` gate by default. This means that a drive
+   must fulfill all filter criteria in order to get selected. This behavior can
+   be adjusted by setting ``filter_logic: OR`` in the OSD specification. 
+
+Filters are used to assign disks to groups, using their attributes to group
+them. 
+
+The attributes are based off of ceph-volume's disk query. You can retrieve
+information about the attributes with this command:
+
+.. code-block:: bash
+
+  ceph-volume inventory </path/to/disk>
+
+Vendor or Model
+^^^^^^^^^^^^^^^
+
+Specific disks can be targeted by vendor or model:
+
+.. code-block:: yaml
+
+    model: disk_model_name
+
+or
+
+.. code-block:: yaml
+
+    vendor: disk_vendor_name
+
+
+Size
+^^^^
+
+Specific disks can be targeted by `Size`:
+
+.. code-block:: yaml
+
+    size: size_spec
+
+Size specs
+__________
+
+Size specifications can be of the following forms:
+
+* LOW:HIGH
+* :HIGH
+* LOW:
+* EXACT
+
+Concrete examples:
+
+To include disks of an exact size
+
+.. code-block:: yaml
+
+    size: '10G'
+
+To include disks within a given range of size: 
+
+.. code-block:: yaml
+
+    size: '10G:40G'
+
+To include disks that are less than or equal to 10G in size:
+
+.. code-block:: yaml
+
+    size: ':10G'
+
+To include disks equal to or greater than 40G in size:
+
+.. code-block:: yaml
+
+    size: '40G:'
+
+Sizes don't have to be specified exclusively in Gigabytes(G).
+
+Other units of size are supported: Megabyte(M), Gigabyte(G) and Terrabyte(T).
+Appending the (B) for byte is also supported: ``MB``, ``GB``, ``TB``.
+
+
+Rotational
+^^^^^^^^^^
+
+This operates on the 'rotational' attribute of the disk.
+
+.. code-block:: yaml
+
+    rotational: 0 | 1
+
+`1` to match all disks that are rotational
+
+`0` to match all disks that are non-rotational (SSD, NVME etc)
+
+
+All
+^^^
+
+This will take all disks that are 'available'
+
+.. note:: This is exclusive for the data_devices section.
+
+.. code-block:: yaml
+
+    all: true
+
+
+Limiter
+^^^^^^^
+
+If you have specified some valid filters but want to limit the number of disks that they match, use the ``limit`` directive:
+
+.. code-block:: yaml
+
+    limit: 2
+
+For example, if you used `vendor` to match all disks that are from `VendorA`
+but want to use only the first two, you could use `limit`:
+
+.. code-block:: yaml
+
+  data_devices:
+    vendor: VendorA
+    limit: 2
+
+.. note:: `limit` is a last resort and shouldn't be used if it can be avoided.
+
+
+Additional Options
+------------------
+
+There are multiple optional settings you can use to change the way OSDs are deployed.
+You can add these options to the base level of an OSD spec for it to take effect.
+
+This example would deploy all OSDs with encryption enabled.
+
+.. code-block:: yaml
+
+    service_type: osd
+    service_id: example_osd_spec
+    placement:
+      host_pattern: '*'
+    spec:
+      data_devices:
+        all: true
+      encrypted: true
+
+See a full list in the DriveGroupSpecs
+
+.. py:currentmodule:: ceph.deployment.drive_group
+
+.. autoclass:: DriveGroupSpec
+   :members:
+   :exclude-members: from_json
+
+Examples
+========
+
+The simple case
+---------------
+
+All nodes with the same setup
+
+.. code-block:: none
+
+    20 HDDs
+    Vendor: VendorA
+    Model: HDD-123-foo
+    Size: 4TB
+
+    2 SSDs
+    Vendor: VendorB
+    Model: MC-55-44-ZX
+    Size: 512GB
+
+This is a common setup and can be described quite easily:
+
+.. code-block:: yaml
+
+    service_type: osd
+    service_id: osd_spec_default
+    placement:
+      host_pattern: '*'
+    spec:
+      data_devices:
+        model: HDD-123-foo # Note, HDD-123 would also be valid
+      db_devices:
+        model: MC-55-44-XZ # Same here, MC-55-44 is valid
+
+However, we can improve it by reducing the filters on core properties of the drives:
+
+.. code-block:: yaml
+
+    service_type: osd
+    service_id: osd_spec_default
+    placement:
+      host_pattern: '*'
+    spec:
+      data_devices:
+        rotational: 1
+      db_devices:
+        rotational: 0
+
+Now, we enforce all rotating devices to be declared as 'data devices' and all non-rotating devices will be used as shared_devices (wal, db)
+
+If you know that drives with more than 2TB will always be the slower data devices, you can also filter by size:
+
+.. code-block:: yaml
+
+    service_type: osd
+    service_id: osd_spec_default
+    placement:
+      host_pattern: '*'
+    spec:
+      data_devices:
+        size: '2TB:'
+      db_devices:
+        size: ':2TB'
+
+.. note:: All of the above OSD specs  are equally valid. Which of those you want to use depends on taste and on how much you expect your node layout to change.
+
+
+Multiple OSD specs for a single host
+------------------------------------
+
+Here we have two distinct setups
+
+.. code-block:: none
+
+    20 HDDs
+    Vendor: VendorA
+    Model: HDD-123-foo
+    Size: 4TB
+
+    12 SSDs
+    Vendor: VendorB
+    Model: MC-55-44-ZX
+    Size: 512GB
+
+    2 NVMEs
+    Vendor: VendorC
+    Model: NVME-QQQQ-987
+    Size: 256GB
+
+
+* 20 HDDs should share 2 SSDs
+* 10 SSDs should share 2 NVMes
+
+This can be described with two layouts.
+
+.. code-block:: yaml
+
+    service_type: osd
+    service_id: osd_spec_hdd
+    placement:
+      host_pattern: '*'
+    spec:
+      data_devices:
+        rotational: 0
+      db_devices:
+        model: MC-55-44-XZ
+        limit: 2 # db_slots is actually to be favoured here, but it's not implemented yet
+    ---
+    service_type: osd
+    service_id: osd_spec_ssd
+    placement:
+      host_pattern: '*'
+    spec:
+      data_devices:
+        model: MC-55-44-XZ
+      db_devices:
+        vendor: VendorC
+
+This would create the desired layout by using all HDDs as data_devices with two SSD assigned as dedicated db/wal devices.
+The remaining SSDs(8) will be data_devices that have the 'VendorC' NVMEs assigned as dedicated db/wal devices.
+
+Multiple hosts with the same disk layout
+----------------------------------------
+
+Assuming the cluster has different kinds of hosts each with similar disk 
+layout, it is recommended to apply different OSD specs matching only one
+set of hosts. Typically you will have a spec for multiple hosts with the 
+same layout. 
+
+The sevice id as the unique key: In case a new OSD spec with an already
+applied service id is applied, the existing OSD spec will be superseeded.
+cephadm will now create new OSD daemons based on the new spec
+definition. Existing OSD daemons will not be affected. See :ref:`cephadm-osd-declarative`.
+
+Node1-5
+
+.. code-block:: none
+
+    20 HDDs
+    Vendor: Intel
+    Model: SSD-123-foo
+    Size: 4TB
+    2 SSDs
+    Vendor: VendorA
+    Model: MC-55-44-ZX
+    Size: 512GB
+
+Node6-10
+
+.. code-block:: none
+
+    5 NVMEs
+    Vendor: Intel
+    Model: SSD-123-foo
+    Size: 4TB
+    20 SSDs
+    Vendor: VendorA
+    Model: MC-55-44-ZX
+    Size: 512GB
+
+You can use the 'placement' key in the layout to target certain nodes.
+
+.. code-block:: yaml
+
+    service_type: osd
+    service_id: disk_layout_a
+    placement:
+      label: disk_layout_a
+    spec:
+      data_devices:
+        rotational: 1
+      db_devices:
+        rotational: 0
+    ---
+    service_type: osd
+    service_id: disk_layout_b
+    placement:
+      label: disk_layout_b
+    spec:
+      data_devices:
+        model: MC-55-44-XZ
+      db_devices:
+        model: SSD-123-foo
+
+This applies different OSD specs to different hosts depending on the `placement` key.
+See :ref:`orchestrator-cli-placement-spec`
+
+.. note::
+
+   Assuming each host has a unique disk layout, each OSD 
+   spec needs to have a different service id
+
+
+Dedicated wal + db
+------------------
+
+All previous cases co-located the WALs with the DBs.
+It's however possible to deploy the WAL on a dedicated device as well, if it makes sense.
+
+.. code-block:: none
+
+    20 HDDs
+    Vendor: VendorA
+    Model: SSD-123-foo
+    Size: 4TB
+
+    2 SSDs
+    Vendor: VendorB
+    Model: MC-55-44-ZX
+    Size: 512GB
+
+    2 NVMEs
+    Vendor: VendorC
+    Model: NVME-QQQQ-987
+    Size: 256GB
+
+
+The OSD spec for this case would look like the following (using the `model` filter):
+
+.. code-block:: yaml
+
+    service_type: osd
+    service_id: osd_spec_default
+    placement:
+      host_pattern: '*'
+    spec:
+      data_devices:
+        model: MC-55-44-XZ
+      db_devices:
+        model: SSD-123-foo
+      wal_devices:
+        model: NVME-QQQQ-987
+
+
+It is also possible to specify directly device paths in specific hosts like the following:
+
+.. code-block:: yaml
+
+    service_type: osd
+    service_id: osd_using_paths
+    placement:
+      hosts:
+        - Node01
+        - Node02
+    spec:
+      data_devices:
+        paths:
+        - /dev/sdb
+      db_devices:
+        paths:
+        - /dev/sdc
+      wal_devices:
+        paths:
+        - /dev/sdd
+
+
+This can easily be done with other filters, like `size` or `vendor` as well.
+
+.. _cephadm-osd-activate:
+
+Activate existing OSDs
+======================
+
+In case the OS of a host was reinstalled, existing OSDs need to be activated
+again. For this use case, cephadm provides a wrapper for :ref:`ceph-volume-lvm-activate` that
+activates all existing OSDs on a host.
+
+.. prompt:: bash #
+
+   ceph cephadm osd activate <host>...
+
+This will scan all existing disks for OSDs and deploy corresponding daemons.
+
+Futher Reading
+==============
+
+* :ref:`ceph-volume`
+* :ref:`rados-index`
diff --git a/doc/cephadm/services/rgw.rst b/doc/cephadm/services/rgw.rst
new file mode 100644
index 000000000..0f9b14650
--- /dev/null
+++ b/doc/cephadm/services/rgw.rst
@@ -0,0 +1,324 @@
+===========
+RGW Service
+===========
+
+.. _cephadm-deploy-rgw:
+
+Deploy RGWs
+===========
+
+Cephadm deploys radosgw as a collection of daemons that manage a
+single-cluster deployment or a particular *realm* and *zone* in a
+multisite deployment.  (For more information about realms and zones,
+see :ref:`multisite`.)
+
+Note that with cephadm, radosgw daemons are configured via the monitor
+configuration database instead of via a `ceph.conf` or the command line.  If
+that configuration isn't already in place (usually in the
+``client.rgw.<something>`` section), then the radosgw
+daemons will start up with default settings (e.g., binding to port
+80).
+
+To deploy a set of radosgw daemons, with an arbitrary service name
+*name*, run the following command:
+
+.. prompt:: bash #
+
+  ceph orch apply rgw *<name>* [--realm=*<realm-name>*] [--zone=*<zone-name>*] --placement="*<num-daemons>* [*<host1>* ...]"
+
+Trivial setup
+-------------
+
+For example, to deploy 2 RGW daemons (the default) for a single-cluster RGW deployment
+under the arbitrary service id *foo*:
+
+.. prompt:: bash #
+
+   ceph orch apply rgw foo
+
+.. _cephadm-rgw-designated_gateways:
+
+Designated gateways
+-------------------
+
+A common scenario is to have a labeled set of hosts that will act
+as gateways, with multiple instances of radosgw running on consecutive
+ports 8000 and 8001:
+
+.. prompt:: bash #
+
+   ceph orch host label add gwhost1 rgw  # the 'rgw' label can be anything
+   ceph orch host label add gwhost2 rgw
+   ceph orch apply rgw foo '--placement=label:rgw count-per-host:2' --port=8000
+
+See also: :ref:`cephadm_co_location`.
+
+.. _cephadm-rgw-networks:
+
+Specifying Networks
+-------------------
+
+The RGW service can have the network they bind to configured with a yaml service specification.
+
+example spec file:
+
+.. code-block:: yaml
+
+    service_type: rgw
+    service_id: foo
+    placement:
+      label: rgw
+      count_per_host: 2
+    networks:
+    - 192.169.142.0/24
+    spec:
+      rgw_frontend_port: 8080
+
+
+Multisite zones
+---------------
+
+To deploy RGWs serving the multisite *myorg* realm and the *us-east-1* zone on
+*myhost1* and *myhost2*:
+
+.. prompt:: bash #
+
+   ceph orch apply rgw east --realm=myorg --zone=us-east-1 --placement="2 myhost1 myhost2"
+
+Note that in a multisite situation, cephadm only deploys the daemons.  It does not create
+or update the realm or zone configurations.  To create a new realm and zone, you need to do
+something like:
+
+.. prompt:: bash #
+
+  radosgw-admin realm create --rgw-realm=<realm-name> --default
+  
+.. prompt:: bash #
+
+  radosgw-admin zonegroup create --rgw-zonegroup=<zonegroup-name>  --master --default
+
+.. prompt:: bash #
+
+  radosgw-admin zone create --rgw-zonegroup=<zonegroup-name> --rgw-zone=<zone-name> --master --default
+
+.. prompt:: bash #
+
+  radosgw-admin period update --rgw-realm=<realm-name> --commit
+
+See :ref:`orchestrator-cli-placement-spec` for details of the placement
+specification.  See :ref:`multisite` for more information of setting up multisite RGW.
+
+See also :ref:`multisite`.
+
+Setting up HTTPS
+----------------
+
+In order to enable HTTPS for RGW services, apply a spec file following this scheme:
+
+.. code-block:: yaml
+
+  service_type: rgw
+  service_id: myrgw
+  spec:
+    rgw_frontend_ssl_certificate: | 
+      -----BEGIN PRIVATE KEY-----
+      V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4gTG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFt
+      ZXQsIGNvbnNldGV0dXIgc2FkaXBzY2luZyBlbGl0ciwgc2VkIGRpYW0gbm9udW15
+      IGVpcm1vZCB0ZW1wb3IgaW52aWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWdu
+      YSBhbGlxdXlhbSBlcmF0LCBzZWQgZGlhbSB2b2x1cHR1YS4gQXQgdmVybyBlb3Mg
+      ZXQgYWNjdXNhbSBldCBqdXN0byBkdW8=
+      -----END PRIVATE KEY-----
+      -----BEGIN CERTIFICATE-----
+      V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4gTG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFt
+      ZXQsIGNvbnNldGV0dXIgc2FkaXBzY2luZyBlbGl0ciwgc2VkIGRpYW0gbm9udW15
+      IGVpcm1vZCB0ZW1wb3IgaW52aWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWdu
+      YSBhbGlxdXlhbSBlcmF0LCBzZWQgZGlhbSB2b2x1cHR1YS4gQXQgdmVybyBlb3Mg
+      ZXQgYWNjdXNhbSBldCBqdXN0byBkdW8=
+      -----END CERTIFICATE-----
+    ssl: true
+
+Then apply this yaml document:
+
+.. prompt:: bash #
+
+  ceph orch apply -i myrgw.yaml
+
+Note the value of ``rgw_frontend_ssl_certificate`` is a literal string as
+indicated by a ``|`` character preserving newline characters.
+
+Service specification
+---------------------
+
+.. py:currentmodule:: ceph.deployment.service_spec
+
+.. autoclass:: RGWSpec
+   :members:
+
+.. _orchestrator-haproxy-service-spec:
+
+High availability service for RGW
+=================================
+
+The *ingress* service allows you to create a high availability endpoint
+for RGW with a minumum set of configuration options.  The orchestrator will
+deploy and manage a combination of haproxy and keepalived to provide load
+balancing on a floating virtual IP.
+
+If SSL is used, then SSL must be configured and terminated by the ingress service
+and not RGW itself.
+
+.. image:: ../../images/HAProxy_for_RGW.svg
+
+There are N hosts where the ingress service is deployed.  Each host
+has a haproxy daemon and a keepalived daemon.  A virtual IP is
+automatically configured on only one of these hosts at a time.
+
+Each keepalived daemon checks every few seconds whether the haproxy
+daemon on the same host is responding.  Keepalived will also check
+that the master keepalived daemon is running without problems.  If the
+"master" keepalived daemon or the active haproxy is not responding,
+one of the remaining keepalived daemons running in backup mode will be
+elected as master, and the virtual IP will be moved to that node.
+
+The active haproxy acts like a load balancer, distributing all RGW requests
+between all the RGW daemons available.
+
+Prerequisites
+-------------
+
+* An existing RGW service, without SSL.  (If you want SSL service, the certificate
+  should be configured on the ingress service, not the RGW service.)
+
+Deploying
+---------
+
+Use the command::
+
+    ceph orch apply -i <ingress_spec_file>
+
+Service specification
+---------------------
+
+It is a yaml format file with the following properties:
+
+.. code-block:: yaml
+
+    service_type: ingress
+    service_id: rgw.something    # adjust to match your existing RGW service
+    placement:
+      hosts:
+        - host1
+        - host2
+        - host3
+    spec:
+      backend_service: rgw.something      # adjust to match your existing RGW service
+      virtual_ip: <string>/<string>       # ex: 192.168.20.1/24
+      frontend_port: <integer>            # ex: 8080
+      monitor_port: <integer>             # ex: 1967, used by haproxy for load balancer status
+      virtual_interface_networks: [ ... ] # optional: list of CIDR networks
+      ssl_cert: |                         # optional: SSL certificate and key
+        -----BEGIN CERTIFICATE-----
+        ...
+        -----END CERTIFICATE-----
+        -----BEGIN PRIVATE KEY-----
+        ...
+        -----END PRIVATE KEY-----
+
+.. code-block:: yaml
+
+    service_type: ingress
+    service_id: rgw.something    # adjust to match your existing RGW service
+    placement:
+      hosts:
+        - host1
+        - host2
+        - host3
+    spec:
+      backend_service: rgw.something      # adjust to match your existing RGW service
+      virtual_ips_list:
+      - <string>/<string>                 # ex: 192.168.20.1/24
+      - <string>/<string>                 # ex: 192.168.20.2/24
+      - <string>/<string>                 # ex: 192.168.20.3/24
+      frontend_port: <integer>            # ex: 8080
+      monitor_port: <integer>             # ex: 1967, used by haproxy for load balancer status
+      virtual_interface_networks: [ ... ] # optional: list of CIDR networks
+      ssl_cert: |                         # optional: SSL certificate and key
+        -----BEGIN CERTIFICATE-----
+        ...
+        -----END CERTIFICATE-----
+        -----BEGIN PRIVATE KEY-----
+        ...
+        -----END PRIVATE KEY-----
+
+
+where the properties of this service specification are:
+
+* ``service_type``
+    Mandatory and set to "ingress"
+* ``service_id``
+    The name of the service.  We suggest naming this after the service you are
+    controlling ingress for (e.g., ``rgw.foo``).
+* ``placement hosts``
+    The hosts where it is desired to run the HA daemons. An haproxy and a
+    keepalived container will be deployed on these hosts.  These hosts do not need
+    to match the nodes where RGW is deployed.
+* ``virtual_ip``
+    The virtual IP (and network) in CIDR format where the ingress service will be available.
+* ``virtual_ips_list``
+    The virtual IP address in CIDR format where the ingress service will be available.
+    Each virtual IP address will be primary on one node running the ingress service. The number
+    of virtual IP addresses must be less than or equal to the number of ingress nodes.
+* ``virtual_interface_networks``
+    A list of networks to identify which ethernet interface to use for the virtual IP.
+* ``frontend_port``
+    The port used to access the ingress service.
+* ``ssl_cert``:
+    SSL certificate, if SSL is to be enabled. This must contain the both the certificate and
+    private key blocks in .pem format.
+
+.. _ingress-virtual-ip:
+
+Selecting ethernet interfaces for the virtual IP
+------------------------------------------------
+
+You cannot simply provide the name of the network interface on which
+to configure the virtual IP because interface names tend to vary
+across hosts (and/or reboots).  Instead, cephadm will select
+interfaces based on other existing IP addresses that are already
+configured.
+
+Normally, the virtual IP will be configured on the first network
+interface that has an existing IP in the same subnet.  For example, if
+the virtual IP is 192.168.0.80/24 and eth2 has the static IP
+192.168.0.40/24, cephadm will use eth2.
+
+In some cases, the virtual IP may not belong to the same subnet as an existing static
+IP.  In such cases, you can provide a list of subnets to match against existing IPs,
+and cephadm will put the virtual IP on the first network interface to match.  For example,
+if the virtual IP is 192.168.0.80/24 and we want it on the same interface as the machine's
+static IP in 10.10.0.0/16, you can use a spec like::
+
+  service_type: ingress
+  service_id: rgw.something
+  spec:
+    virtual_ip: 192.168.0.80/24
+    virtual_interface_networks:
+      - 10.10.0.0/16
+    ...
+
+A consequence of this strategy is that you cannot currently configure the virtual IP
+on an interface that has no existing IP address.  In this situation, we suggest
+configuring a "dummy" IP address is an unroutable network on the correct interface
+and reference that dummy network in the networks list (see above).
+
+
+Useful hints for ingress
+------------------------
+
+* It is good to have at least 3 RGW daemons.
+* We recommend at least 3 hosts for the ingress service.
+
+Further Reading
+===============
+
+* :ref:`object-gateway`
diff --git a/doc/cephadm/services/snmp-gateway.rst b/doc/cephadm/services/snmp-gateway.rst
new file mode 100644
index 000000000..f927fdfd0
--- /dev/null
+++ b/doc/cephadm/services/snmp-gateway.rst
@@ -0,0 +1,171 @@
+====================
+SNMP Gateway Service
+====================
+
+SNMP_ is still a widely used protocol, to monitor distributed systems and devices across a variety of hardware
+and software platforms. Ceph's SNMP integration focuses on forwarding alerts from it's Prometheus Alertmanager
+cluster to a gateway daemon. The gateway daemon, transforms the alert into an SNMP Notification and sends
+it on to a designated SNMP management platform. The gateway daemon is from the snmp_notifier_ project,
+which provides SNMP V2c and V3 support (authentication and encryption).
+
+Ceph's SNMP gateway service deploys one instance of the gateway by default. You may increase this
+by providing placement information. However, bear in mind that if you enable multiple SNMP gateway daemons,
+your SNMP management platform will receive multiple notifications for the same event.
+
+.. _SNMP: https://en.wikipedia.org/wiki/Simple_Network_Management_Protocol
+.. _snmp_notifier: https://github.com/maxwo/snmp_notifier
+
+Compatibility
+=============
+The table below shows the SNMP versions that are supported by the gateway implementation
+
+================ =========== ===============================================
+ SNMP Version     Supported  Notes
+================ =========== ===============================================
+ V1                  �      Not supported by snmp_notifier
+ V2c                  ✔
+ V3 authNoPriv        ✔      uses username/password authentication, without
+                             encryption (NoPriv = no privacy)
+ V3 authPriv          ✔      uses username/password authentication with
+                             encryption to the SNMP management platform
+================ =========== ===============================================
+
+
+Deploying an SNMP Gateway
+=========================
+Both SNMP V2c and V3 provide credentials support. In the case of V2c, this is just the community string - but for V3
+environments you must provide additional authentication information. These credentials are not supported on the command
+line when deploying the service. Instead, you must create the service using a credentials file (in yaml format), or
+specify the complete service definition in a yaml file.
+
+Command format
+--------------
+
+.. prompt:: bash #
+
+   ceph orch apply snmp-gateway <snmp_version:V2c|V3> <destination> [<port:int>] [<engine_id>] [<auth_protocol: MD5|SHA>] [<privacy_protocol:DES|AES>] [<placement>] ...
+
+
+Usage Notes
+
+- you must supply the ``--snmp-version`` parameter
+- the ``--destination`` parameter must be of the format hostname:port (no default)
+- you may omit ``--port``. It defaults to 9464
+- the ``--engine-id`` is a unique identifier for the device (in hex) and required for SNMP v3 only.
+  Suggested value: 8000C53F<fsid> where the fsid is from your cluster, without the '-' symbols
+- for SNMP V3, the ``--auth-protocol`` setting defaults to **SHA**
+- for SNMP V3, with encryption you must define the ``--privacy-protocol``
+- you **must** provide a -i <filename> to pass the secrets/passwords to the orchestrator
+
+Deployment Examples
+===================
+
+SNMP V2c
+--------
+Here's an example for V2c, showing CLI and service based deployments
+
+.. prompt:: bash #
+
+   ceph orch apply snmp-gateway --port 9464 --snmp_version=V2c --destination=192.168.122.73:162 -i ./snmp_creds.yaml
+
+with a credentials file that contains;
+
+.. code-block:: yaml
+
+   ---
+   snmp_community: public
+
+Alternatively, you can create a yaml definition for the gateway and apply it from a single file
+
+.. prompt:: bash #
+
+   ceph orch apply -i snmp-gateway.yml
+
+with the file containing the following configuration
+
+.. code-block:: yaml
+
+    service_type: snmp-gateway
+    service_name: snmp-gateway
+    placement:
+      count: 1
+    spec:
+      credentials:
+        snmp_community: public
+      port: 9464
+      snmp_destination: 192.168.122.73:162
+      snmp_version: V2c
+
+
+SNMP V3 (authNoPriv)
+--------------------
+Deploying an snmp-gateway service supporting SNMP V3 with authentication only, would look like this;
+
+.. prompt:: bash #
+
+   ceph orch apply snmp-gateway --snmp-version=V3 --engine-id=800C53F000000 --destination=192.168.122.1:162 -i ./snmpv3_creds.yml
+
+with a credentials file as;
+
+.. code-block:: yaml
+
+   ---
+   snmp_v3_auth_username: myuser
+   snmp_v3_auth_password: mypassword
+
+or as a service configuration file
+
+.. code-block:: yaml
+
+   service_type: snmp-gateway
+   service_name: snmp-gateway
+   placement:
+     count: 1
+   spec:
+     credentials:
+       snmp_v3_auth_password: mypassword
+       snmp_v3_auth_username: myuser
+     engine_id: 800C53F000000
+     port: 9464
+     snmp_destination: 192.168.122.1:162
+     snmp_version: V3
+
+
+SNMP V3 (authPriv)
+------------------
+
+Defining an SNMP V3 gateway service that implements authentication and privacy (encryption), requires two additional values
+
+.. prompt:: bash #
+
+   ceph orch apply snmp-gateway --snmp-version=V3 --engine-id=800C53F000000 --destination=192.168.122.1:162 --privacy-protocol=AES -i ./snmpv3_creds.yml
+
+with a credentials file as;
+
+.. code-block:: yaml
+
+   ---
+   snmp_v3_auth_username: myuser
+   snmp_v3_auth_password: mypassword
+   snmp_v3_priv_password: mysecret
+
+
+.. note::
+
+   The credentials are stored on the host, restricted to the root user and passed to the snmp_notifier daemon as
+   an environment file (``--env-file``), to limit exposure.
+
+
+AlertManager Integration
+========================
+When an SNMP gateway service is deployed or updated, the Prometheus Alertmanager configuration is automatically updated to forward any
+alert that has an OID_ label to the SNMP gateway daemon for processing.
+
+.. _OID: https://en.wikipedia.org/wiki/Object_identifier
+
+Implementing the MIB
+======================
+To make sense of the SNMP Notification/Trap, you'll need to apply the MIB to your SNMP management platform. The MIB (CEPH-MIB.txt) can
+downloaded from the main Ceph repo_
+
+.. _repo: https://github.com/ceph/ceph/tree/master/monitoring/snmp
diff --git a/doc/cephadm/troubleshooting.rst b/doc/cephadm/troubleshooting.rst
new file mode 100644
index 000000000..9a534f633
--- /dev/null
+++ b/doc/cephadm/troubleshooting.rst
@@ -0,0 +1,370 @@
+Troubleshooting
+===============
+
+You might need to investigate why a cephadm command failed
+or why a certain service no longer runs properly.
+
+Cephadm deploys daemons as containers. This means that
+troubleshooting those containerized daemons might work
+differently than you expect (and that is certainly true if
+you expect this troubleshooting to work the way that
+troubleshooting does when the daemons involved aren't
+containerized). 
+
+Here are some tools and commands to help you troubleshoot
+your Ceph environment.
+
+.. _cephadm-pause:
+
+Pausing or disabling cephadm
+----------------------------
+
+If something goes wrong and cephadm is behaving badly, you can
+pause most of the Ceph cluster's background activity by running
+the following command: 
+
+.. prompt:: bash #
+
+  ceph orch pause
+
+This stops all changes in the Ceph cluster, but cephadm will
+still periodically check hosts to refresh its inventory of
+daemons and devices.  You can disable cephadm completely by
+running the following commands:
+
+.. prompt:: bash #
+
+  ceph orch set backend ''
+  ceph mgr module disable cephadm
+
+These commands disable all of the ``ceph orch ...`` CLI commands.
+All previously deployed daemon containers continue to exist and
+will start as they did before you ran these commands.
+
+See :ref:`cephadm-spec-unmanaged` for information on disabling
+individual services.
+
+
+Per-service and per-daemon events
+---------------------------------
+
+In order to help with the process of debugging failed daemon
+deployments, cephadm stores events per service and per daemon.
+These events often contain information relevant to
+troubleshooting
+your Ceph cluster. 
+
+Listing service events
+~~~~~~~~~~~~~~~~~~~~~~
+
+To see the events associated with a certain service, run a
+command of the and following form:
+
+.. prompt:: bash #
+
+  ceph orch ls --service_name=<service-name> --format yaml
+
+This will return something in the following form:
+
+.. code-block:: yaml
+
+  service_type: alertmanager
+  service_name: alertmanager
+  placement:
+    hosts:
+    - unknown_host
+  status:
+    ...
+    running: 1
+    size: 1
+  events:
+  - 2021-02-01T08:58:02.741162 service:alertmanager [INFO] "service was created"
+  - '2021-02-01T12:09:25.264584 service:alertmanager [ERROR] "Failed to apply: Cannot
+    place <AlertManagerSpec for service_name=alertmanager> on unknown_host: Unknown hosts"'
+
+Listing daemon events
+~~~~~~~~~~~~~~~~~~~~~
+
+To see the events associated with a certain daemon, run a
+command of the and following form:
+
+.. prompt:: bash #
+
+  ceph orch ps --service-name <service-name> --daemon-id <daemon-id> --format yaml
+
+This will return something in the following form:
+
+.. code-block:: yaml
+
+  daemon_type: mds
+  daemon_id: cephfs.hostname.ppdhsz
+  hostname: hostname
+  status_desc: running
+  ...
+  events:
+  - 2021-02-01T08:59:43.845866 daemon:mds.cephfs.hostname.ppdhsz [INFO] "Reconfigured
+    mds.cephfs.hostname.ppdhsz on host 'hostname'"
+
+
+Checking cephadm logs
+---------------------
+
+To learn how to monitor the cephadm logs as they are generated, read :ref:`watching_cephadm_logs`.
+
+If your Ceph cluster has been configured to log events to files, there will exist a
+cephadm log file called ``ceph.cephadm.log`` on all monitor hosts (see
+:ref:`cephadm-logs` for a more complete explanation of this).
+
+Gathering log files
+-------------------
+
+Use journalctl to gather the log files of all daemons:
+
+.. note:: By default cephadm now stores logs in journald. This means
+   that you will no longer find daemon logs in ``/var/log/ceph/``.
+
+To read the log file of one specific daemon, run::
+
+    cephadm logs --name <name-of-daemon>
+
+Note: this only works when run on the same host where the daemon is running. To
+get logs of a daemon running on a different host, give the ``--fsid`` option::
+
+    cephadm logs --fsid <fsid> --name <name-of-daemon>
+
+where the ``<fsid>`` corresponds to the cluster ID printed by ``ceph status``.
+
+To fetch all log files of all daemons on a given host, run::
+
+    for name in $(cephadm ls | jq -r '.[].name') ; do
+      cephadm logs --fsid <fsid> --name "$name" > $name;
+    done
+
+Collecting systemd status
+-------------------------
+
+To print the state of a systemd unit, run::
+
+      systemctl status "ceph-$(cephadm shell ceph fsid)@<service name>.service";
+
+
+To fetch all state of all daemons of a given host, run::
+
+    fsid="$(cephadm shell ceph fsid)"
+    for name in $(cephadm ls | jq -r '.[].name') ; do
+      systemctl status "ceph-$fsid@$name.service" > $name;
+    done
+
+
+List all downloaded container images
+------------------------------------
+
+To list all container images that are downloaded on a host:
+
+.. note:: ``Image`` might also be called `ImageID`
+
+::
+
+    podman ps -a --format json | jq '.[].Image'
+    "docker.io/library/centos:8"
+    "registry.opensuse.org/opensuse/leap:15.2"
+
+
+Manually running containers
+---------------------------
+
+Cephadm writes small wrappers that run a containers. Refer to
+``/var/lib/ceph/<cluster-fsid>/<service-name>/unit.run`` for the
+container execution command.
+
+.. _cephadm-ssh-errors:
+
+SSH errors
+----------
+
+Error message::
+
+  execnet.gateway_bootstrap.HostNotFound: -F /tmp/cephadm-conf-73z09u6g -i /tmp/cephadm-identity-ky7ahp_5 root@10.10.1.2
+  ...
+  raise OrchestratorError(msg) from e
+  orchestrator._interface.OrchestratorError: Failed to connect to 10.10.1.2 (10.10.1.2).
+  Please make sure that the host is reachable and accepts connections using the cephadm SSH key
+  ...
+
+Things users can do:
+
+1. Ensure cephadm has an SSH identity key::
+
+     [root@mon1~]# cephadm shell -- ceph config-key get mgr/cephadm/ssh_identity_key > ~/cephadm_private_key
+     INFO:cephadm:Inferring fsid f8edc08a-7f17-11ea-8707-000c2915dd98
+     INFO:cephadm:Using recent ceph image docker.io/ceph/ceph:v15 obtained 'mgr/cephadm/ssh_identity_key'
+     [root@mon1 ~] # chmod 0600 ~/cephadm_private_key
+
+ If this fails, cephadm doesn't have a key. Fix this by running the following command::
+
+     [root@mon1 ~]# cephadm shell -- ceph cephadm generate-ssh-key
+
+ or::
+
+     [root@mon1 ~]# cat ~/cephadm_private_key | cephadm shell -- ceph cephadm set-ssk-key -i -
+
+2. Ensure that the SSH config is correct::
+
+     [root@mon1 ~]# cephadm shell -- ceph cephadm get-ssh-config > config
+
+3. Verify that we can connect to the host::
+
+     [root@mon1 ~]# ssh -F config -i ~/cephadm_private_key root@mon1
+
+Verifying that the Public Key is Listed in the authorized_keys file
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To verify that the public key is in the authorized_keys file, run the following commands::
+
+     [root@mon1 ~]# cephadm shell -- ceph cephadm get-pub-key > ~/ceph.pub
+     [root@mon1 ~]# grep "`cat ~/ceph.pub`"  /root/.ssh/authorized_keys
+
+Failed to infer CIDR network error
+----------------------------------
+
+If you see this error::
+
+   ERROR: Failed to infer CIDR network for mon ip ***; pass --skip-mon-network to configure it later
+
+Or this error::
+
+   Must set public_network config option or specify a CIDR network, ceph addrvec, or plain IP
+
+This means that you must run a command of this form::
+
+  ceph config set mon public_network <mon_network>
+
+For more detail on operations of this kind, see :ref:`deploy_additional_monitors`
+
+Accessing the admin socket
+--------------------------
+
+Each Ceph daemon provides an admin socket that bypasses the
+MONs (See :ref:`rados-monitoring-using-admin-socket`).
+
+To access the admin socket, first enter the daemon container on the host::
+
+    [root@mon1 ~]# cephadm enter --name <daemon-name>
+    [ceph: root@mon1 /]# ceph --admin-daemon /var/run/ceph/ceph-<daemon-name>.asok config show
+
+Calling miscellaneous ceph tools
+--------------------------------
+
+To call miscellaneous like ``ceph-objectstore-tool`` or 
+``ceph-monstore-tool``, you can run them by calling 
+``cephadm shell --name <daemon-name>`` like so::
+
+    root@myhostname # cephadm unit --name mon.myhostname stop
+    root@myhostname # cephadm shell --name mon.myhostname
+    [ceph: root@myhostname /]# ceph-monstore-tool /var/lib/ceph/mon/ceph-myhostname get monmap > monmap         
+    [ceph: root@myhostname /]# monmaptool --print monmap
+    monmaptool: monmap file monmap
+    epoch 1
+    fsid 28596f44-3b56-11ec-9034-482ae35a5fbb
+    last_changed 2021-11-01T20:57:19.755111+0000
+    created 2021-11-01T20:57:19.755111+0000
+    min_mon_release 17 (quincy)
+    election_strategy: 1
+    0: [v2:127.0.0.1:3300/0,v1:127.0.0.1:6789/0] mon.myhostname
+
+This command sets up the environment in a way that is suitable
+for extended daemon maintenance and running the deamon interactively. 
+
+.. _cephadm-restore-quorum:
+
+Restoring the MON quorum
+------------------------
+
+In case the Ceph MONs cannot form a quorum, cephadm is not able
+to manage the cluster, until the quorum is restored.
+
+In order to restore the MON quorum, remove unhealthy MONs
+form the monmap by following these steps:
+
+1. Stop all MONs. For each MON host::
+
+    ssh {mon-host}
+    cephadm unit --name mon.`hostname` stop
+
+
+2. Identify a surviving monitor and log in to that host::
+
+    ssh {mon-host}
+    cephadm enter --name mon.`hostname`
+
+3. Follow the steps in :ref:`rados-mon-remove-from-unhealthy`
+
+.. _cephadm-manually-deploy-mgr:
+
+Manually deploying a MGR daemon
+-------------------------------
+cephadm requires a MGR daemon in order to manage the cluster. In case the cluster
+the last MGR of a cluster was removed, follow these steps in order to deploy 
+a MGR ``mgr.hostname.smfvfd`` on a random host of your cluster manually. 
+
+Disable the cephadm scheduler, in order to prevent cephadm from removing the new 
+MGR. See :ref:`cephadm-enable-cli`::
+
+  ceph config-key set mgr/cephadm/pause true
+
+Then get or create the auth entry for the new MGR::
+
+  ceph auth get-or-create mgr.hostname.smfvfd mon "profile mgr" osd "allow *" mds "allow *"
+
+Get the ceph.conf::
+
+  ceph config generate-minimal-conf
+
+Get the container image::
+
+  ceph config get "mgr.hostname.smfvfd" container_image
+
+Create a file ``config-json.json`` which contains the information neccessary to deploy
+the daemon:
+
+.. code-block:: json
+
+  {
+    "config": "# minimal ceph.conf for 8255263a-a97e-4934-822c-00bfe029b28f\n[global]\n\tfsid = 8255263a-a97e-4934-822c-00bfe029b28f\n\tmon_host = [v2:192.168.0.1:40483/0,v1:192.168.0.1:40484/0]\n",
+    "keyring": "[mgr.hostname.smfvfd]\n\tkey = V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4=\n"
+  }
+
+Deploy the daemon::
+
+  cephadm --image <container-image> deploy --fsid <fsid> --name mgr.hostname.smfvfd --config-json config-json.json
+
+Analyzing core dumps
+---------------------
+
+In case a Ceph daemon crashes, cephadm supports analyzing core dumps. To enable core dumps, run
+
+.. prompt:: bash #
+
+  ulimit -c unlimited
+
+core dumps will now be written to ``/var/lib/systemd/coredump``.
+
+.. note::
+
+  core dumps are not namespaced by the kernel, which means
+  they will be written to ``/var/lib/systemd/coredump`` on
+  the container host. 
+
+Now, wait for the crash to happen again. (To simulate the crash of a daemon, run e.g. ``killall -3 ceph-mon``)
+
+Install debug packages by entering the cephadm shell and install ``ceph-debuginfo``::
+
+  # cephadm shell --mount /var/lib/systemd/coredump
+  [ceph: root@host1 /]# dnf install ceph-debuginfo gdb zstd
+  [ceph: root@host1 /]# unzstd /mnt/coredump/core.ceph-*.zst
+  [ceph: root@host1 /]# gdb /usr/bin/ceph-mon /mnt/coredump/core.ceph-...
+  (gdb) bt
+  #0  0x00007fa9117383fc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
+  #1  0x00007fa910d7f8f0 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /lib64/libstdc++.so.6
+  #2  0x00007fa913d3f48f in AsyncMessenger::wait() () from /usr/lib64/ceph/libceph-common.so.2
+  #3  0x0000563085ca3d7e in main ()
diff --git a/doc/cephadm/upgrade.rst b/doc/cephadm/upgrade.rst
new file mode 100644
index 000000000..2bbfc30e5
--- /dev/null
+++ b/doc/cephadm/upgrade.rst
@@ -0,0 +1,270 @@
+==============
+Upgrading Ceph
+==============
+
+Cephadm can safely upgrade Ceph from one bugfix release to the next.  For
+example, you can upgrade from v15.2.0 (the first Octopus release) to the next
+point release, v15.2.1.
+
+The automated upgrade process follows Ceph best practices.  For example:
+
+* The upgrade order starts with managers, monitors, then other daemons.
+* Each daemon is restarted only after Ceph indicates that the cluster
+  will remain available.
+
+.. note::
+
+   The Ceph cluster health status is likely to switch to
+   ``HEALTH_WARNING`` during the upgrade.
+
+.. note:: 
+
+   In case a host of the cluster is offline, the upgrade is paused.
+
+
+Starting the upgrade
+====================
+
+Before you use cephadm to upgrade Ceph, verify that all hosts are currently online and that your cluster is healthy by running the following command:
+
+.. prompt:: bash #
+
+   ceph -s
+
+To upgrade (or downgrade) to a specific release, run the following command:
+
+.. prompt:: bash #
+
+  ceph orch upgrade start --ceph-version <version>
+
+For example, to upgrade to v16.2.6, run the following command:
+
+.. prompt:: bash #
+
+  ceph orch upgrade start --ceph-version 16.2.6
+
+.. note::
+
+    From version v16.2.6 the Docker Hub registry is no longer used, so if you use Docker you have to point it to the image in the quay.io registry:
+
+.. prompt:: bash #
+
+  ceph orch upgrade start --image quay.io/ceph/ceph:v16.2.6
+
+
+Monitoring the upgrade
+======================
+
+Determine (1) whether an upgrade is in progress and (2) which version the
+cluster is upgrading to by running the following command:
+
+.. prompt:: bash #
+
+  ceph orch upgrade status
+
+Watching the progress bar during a Ceph upgrade
+-----------------------------------------------
+
+During the upgrade, a progress bar is visible in the ceph status output. It
+looks like this:
+
+.. code-block:: console
+
+  # ceph -s
+
+  [...]
+    progress:
+      Upgrade to docker.io/ceph/ceph:v15.2.1 (00h 20m 12s)
+        [=======.....................] (time remaining: 01h 43m 31s)
+
+Watching the cephadm log during an upgrade
+------------------------------------------
+
+Watch the cephadm log by running the following command:
+
+.. prompt:: bash #
+
+  ceph -W cephadm
+
+
+Canceling an upgrade
+====================
+
+You can stop the upgrade process at any time by running the following command:
+
+.. prompt:: bash #
+
+  ceph orch upgrade stop
+
+Post upgrade actions
+====================
+
+In case the new version is based on ``cephadm``, once done with the upgrade the user
+has to update the ``cephadm`` package (or ceph-common package in case the user
+doesn't use ``cephadm shell``) to a version compatible with the new version.
+
+Potential problems
+==================
+
+There are a few health alerts that can arise during the upgrade process.
+
+UPGRADE_NO_STANDBY_MGR
+----------------------
+
+This alert (``UPGRADE_NO_STANDBY_MGR``) means that Ceph does not detect an
+active standby manager daemon. In order to proceed with the upgrade, Ceph
+requires an active standby manager daemon (which you can think of in this
+context as "a second manager").
+
+You can ensure that Cephadm is configured to run 2 (or more) managers by
+running the following command:
+
+.. prompt:: bash #
+
+  ceph orch apply mgr 2  # or more
+
+You can check the status of existing mgr daemons by running the following
+command:
+
+.. prompt:: bash #
+
+  ceph orch ps --daemon-type mgr
+
+If an existing mgr daemon has stopped, you can try to restart it by running the
+following command: 
+
+.. prompt:: bash #
+
+  ceph orch daemon restart <name>
+
+UPGRADE_FAILED_PULL
+-------------------
+
+This alert (``UPGRADE_FAILED_PULL``) means that Ceph was unable to pull the
+container image for the target version. This can happen if you specify a
+version or container image that does not exist (e.g. "1.2.3"), or if the
+container registry can not be reached by one or more hosts in the cluster.
+
+To cancel the existing upgrade and to specify a different target version, run
+the following commands: 
+
+.. prompt:: bash #
+
+  ceph orch upgrade stop
+  ceph orch upgrade start --ceph-version <version>
+
+
+Using customized container images
+=================================
+
+For most users, upgrading requires nothing more complicated than specifying the
+Ceph version number to upgrade to.  In such cases, cephadm locates the specific
+Ceph container image to use by combining the ``container_image_base``
+configuration option (default: ``docker.io/ceph/ceph``) with a tag of
+``vX.Y.Z``.
+
+But it is possible to upgrade to an arbitrary container image, if that's what
+you need. For example, the following command upgrades to a development build:
+
+.. prompt:: bash #
+
+  ceph orch upgrade start --image quay.io/ceph-ci/ceph:recent-git-branch-name
+
+For more information about available container images, see :ref:`containers`.
+
+Staggered Upgrade
+=================
+
+Some users may prefer to upgrade components in phases rather than all at once.
+The upgrade command, starting in 16.2.11 and 17.2.1 allows parameters
+to limit which daemons are upgraded by a single upgrade command. The options in
+include ``daemon_types``, ``services``, ``hosts`` and ``limit``. ``daemon_types``
+takes a comma-separated list of daemon types and will only upgrade daemons of those
+types. ``services`` is mutually exclusive with ``daemon_types``, only takes services
+of one type at a time (e.g. can't provide an OSD and RGW service at the same time), and
+will only upgrade daemons belonging to those services. ``hosts`` can be combined
+with ``daemon_types`` or ``services`` or provided on its own. The ``hosts`` parameter
+follows the same format as the command line options for :ref:`orchestrator-cli-placement-spec`.
+``limit`` takes an integer > 0 and provides a numerical limit on the number of
+daemons cephadm will upgrade. ``limit`` can be combined with any of the other
+parameters. For example, if you specify to upgrade daemons of type osd on host
+Host1 with ``limit`` set to 3, cephadm will upgrade (up to) 3 osd daemons on
+Host1.
+
+Example: specifying daemon types and hosts:
+
+.. prompt:: bash #
+
+  ceph orch upgrade start --image <image-name> --daemon-types mgr,mon --hosts host1,host2
+
+Example: specifying services and using limit:
+
+.. prompt:: bash #
+
+  ceph orch upgrade start --image <image-name> --services rgw.example1,rgw.example2 --limit 2
+
+.. note::
+
+   Cephadm strictly enforces an order to the upgrade of daemons that is still present
+   in staggered upgrade scenarios. The current upgrade ordering is
+   ``mgr -> mon -> crash -> osd -> mds -> rgw -> rbd-mirror -> cephfs-mirror -> iscsi -> nfs``.
+   If you specify parameters that would upgrade daemons out of order, the upgrade
+   command will block and note which daemons will be missed if you proceed.
+
+.. note::
+
+  Upgrade commands with limiting parameters will validate the options before beginning the
+  upgrade, which may require pulling the new container image. Do not be surprised
+  if the upgrade start command takes a while to return when limiting parameters are provided.
+
+.. note::
+
+   In staggered upgrade scenarios (when a limiting parameter is provided) monitoring
+   stack daemons including Prometheus and node-exporter are refreshed after the Manager
+   daemons have been upgraded. Do not be surprised if Manager upgrades thus take longer
+   than expected. Note that the versions of monitoring stack daemons may not change between
+   Ceph releases, in which case they are only redeployed.
+
+Upgrading to a version that supports staggered upgrade from one that doesn't
+----------------------------------------------------------------------------
+
+While upgrading from a version that already supports staggered upgrades the process
+simply requires providing the necessary arguments. However, if you wish to upgrade
+to a version that supports staggered upgrade from one that does not, there is a
+workaround. It requires first manually upgrading the Manager daemons and then passing
+the limiting parameters as usual.
+
+.. warning::
+  Make sure you have multiple running mgr daemons before attempting this procedure.
+
+To start with, determine which Manager is your active one and which are standby. This
+can be done in a variety of ways such as looking at the ``ceph -s`` output. Then,
+manually upgrade each standby mgr daemon with:
+
+.. prompt:: bash #
+
+  ceph orch daemon redeploy mgr.example1.abcdef --image <new-image-name>
+
+.. note::
+
+   If you are on a very early version of cephadm (early Octopus) the ``orch daemon redeploy``
+   command may not have the ``--image`` flag. In that case, you must manually set the
+   Manager container image ``ceph config set mgr container_image <new-image-name>`` and then
+   redeploy the Manager ``ceph orch daemon redeploy mgr.example1.abcdef``
+
+At this point, a Manager fail over should allow us to have the active Manager be one
+running the new version.
+
+.. prompt:: bash #
+
+  ceph mgr fail
+
+Verify the active Manager is now one running the new version. To complete the Manager
+upgrading:
+
+.. prompt:: bash #
+
+  ceph orch upgrade start --image <new-image-name> --daemon-types mgr
+
+You should now have all your Manager daemons on the new version and be able to
+specify the limiting parameters for the rest of the upgrade.
diff --git a/doc/cephfs/Makefile b/doc/cephfs/Makefile
new file mode 100644
index 000000000..eee2fa572
--- /dev/null
+++ b/doc/cephfs/Makefile
@@ -0,0 +1,7 @@
+TARGETS=mds-state-diagram.svg
+
+%.svg: %.dot
+	dot -Tsvg -o $@ $^
+
+
+all: $(TARGETS)
diff --git a/doc/cephfs/add-remove-mds.rst b/doc/cephfs/add-remove-mds.rst
new file mode 100644
index 000000000..4f5ee06aa
--- /dev/null
+++ b/doc/cephfs/add-remove-mds.rst
@@ -0,0 +1,118 @@
+.. _cephfs_add_remote_mds:
+
+.. note::
+   It is highly recommended to use :doc:`/cephadm/index` or another Ceph
+   orchestrator for setting up the ceph cluster. Use this approach only if you
+   are setting up the ceph cluster manually. If one still intends to use the
+   manual way for deploying MDS daemons, :doc:`/cephadm/services/mds/` can
+   also be used.
+
+============================
+ Deploying Metadata Servers
+============================
+
+Each CephFS file system requires at least one MDS. The cluster operator will
+generally use their automated deployment tool to launch required MDS servers as
+needed.  Rook and ansible (via the ceph-ansible playbooks) are recommended
+tools for doing this. For clarity, we also show the systemd commands here which
+may be run by the deployment technology if executed on bare-metal.
+
+See `MDS Config Reference`_ for details on configuring metadata servers.
+
+
+Provisioning Hardware for an MDS
+================================
+
+The present version of the MDS is single-threaded and CPU-bound for most
+activities, including responding to client requests. An MDS under the most
+aggressive client loads uses about 2 to 3 CPU cores. This is due to the other
+miscellaneous upkeep threads working in tandem.
+
+Even so, it is recommended that an MDS server be well provisioned with an
+advanced CPU with sufficient cores. Development is on-going to make better use
+of available CPU cores in the MDS; it is expected in future versions of Ceph
+that the MDS server will improve performance by taking advantage of more cores.
+
+The other dimension to MDS performance is the available RAM for caching. The
+MDS necessarily manages a distributed and cooperative metadata cache among all
+clients and other active MDSs. Therefore it is essential to provide the MDS
+with sufficient RAM to enable faster metadata access and mutation. The default
+MDS cache size (see also :doc:`/cephfs/cache-configuration`) is 4GB. It is
+recommended to provision at least 8GB of RAM for the MDS to support this cache
+size.
+
+Generally, an MDS serving a large cluster of clients (1000 or more) will use at
+least 64GB of cache. An MDS with a larger cache is not well explored in the
+largest known community clusters; there may be diminishing returns where
+management of such a large cache negatively impacts performance in surprising
+ways. It would be best to do analysis with expected workloads to determine if
+provisioning more RAM is worthwhile.
+
+In a bare-metal cluster, the best practice is to over-provision hardware for
+the MDS server. Even if a single MDS daemon is unable to fully utilize the
+hardware, it may be desirable later on to start more active MDS daemons on the
+same node to fully utilize the available cores and memory. Additionally, it may
+become clear with workloads on the cluster that performance improves with
+multiple active MDS on the same node rather than over-provisioning a single
+MDS.
+
+Finally, be aware that CephFS is a highly-available file system by supporting
+standby MDS (see also :ref:`mds-standby`) for rapid failover. To get a real
+benefit from deploying standbys, it is usually necessary to distribute MDS
+daemons across at least two nodes in the cluster. Otherwise, a hardware failure
+on a single node may result in the file system becoming unavailable.
+
+Co-locating the MDS with other Ceph daemons (hyperconverged) is an effective
+and recommended way to accomplish this so long as all daemons are configured to
+use available hardware within certain limits.  For the MDS, this generally
+means limiting its cache size.
+
+
+Adding an MDS
+=============
+
+#. Create an mds directory ``/var/lib/ceph/mds/ceph-${id}``. The daemon only uses this directory to store its keyring.
+
+#. Create the authentication key, if you use CephX: ::
+
+	$ sudo ceph auth get-or-create mds.${id} mon 'profile mds' mgr 'profile mds' mds 'allow *' osd 'allow *' > /var/lib/ceph/mds/ceph-${id}/keyring
+
+#. Start the service: ::
+
+	$ sudo systemctl start ceph-mds@${id}
+
+#. The status of the cluster should show: ::
+
+	mds: ${id}:1 {0=${id}=up:active} 2 up:standby
+
+#. Optionally, configure the file system the MDS should join (:ref:`mds-join-fs`): ::
+
+    $ ceph config set mds.${id} mds_join_fs ${fs}
+
+
+Removing an MDS
+===============
+
+If you have a metadata server in your cluster that you'd like to remove, you may use
+the following method.
+
+#. (Optionally:) Create a new replacement Metadata Server. If there are no
+   replacement MDS to take over once the MDS is removed, the file system will
+   become unavailable to clients.  If that is not desirable, consider adding a
+   metadata server before tearing down the metadata server you would like to
+   take offline.
+
+#. Stop the MDS to be removed. ::
+
+	$ sudo systemctl stop ceph-mds@${id}
+
+   The MDS will automatically notify the Ceph monitors that it is going down.
+   This enables the monitors to perform instantaneous failover to an available
+   standby, if one exists. It is unnecessary to use administrative commands to
+   effect this failover, e.g. through the use of ``ceph mds fail mds.${id}``.
+
+#. Remove the ``/var/lib/ceph/mds/ceph-${id}`` directory on the MDS. ::
+
+	$ sudo rm -rf /var/lib/ceph/mds/ceph-${id}
+
+.. _MDS Config Reference: ../mds-config-ref
diff --git a/doc/cephfs/administration.rst b/doc/cephfs/administration.rst
new file mode 100644
index 000000000..966b45ee5
--- /dev/null
+++ b/doc/cephfs/administration.rst
@@ -0,0 +1,375 @@
+.. _cephfs-administration:
+
+CephFS Administrative commands
+==============================
+
+File Systems
+------------
+
+.. note:: The names of the file systems, metadata pools, and data pools can
+          only have characters in the set [a-zA-Z0-9\_-.].
+
+These commands operate on the CephFS file systems in your Ceph cluster.
+Note that by default only one file system is permitted: to enable
+creation of multiple file systems use ``ceph fs flag set enable_multiple true``.
+
+::
+
+    fs new <file system name> <metadata pool name> <data pool name>
+
+This command creates a new file system. The file system name and metadata pool
+name are self-explanatory. The specified data pool is the default data pool and
+cannot be changed once set. Each file system has its own set of MDS daemons
+assigned to ranks so ensure that you have sufficient standby daemons available
+to accommodate the new file system.
+
+::
+
+    fs ls
+
+List all file systems by name.
+
+::
+
+    fs dump [epoch]
+
+This dumps the FSMap at the given epoch (default: current) which includes all
+file system settings, MDS daemons and the ranks they hold, and the list of
+standby MDS daemons.
+
+
+::
+
+    fs rm <file system name> [--yes-i-really-mean-it]
+
+Destroy a CephFS file system. This wipes information about the state of the
+file system from the FSMap. The metadata pool and data pools are untouched and
+must be destroyed separately.
+
+::
+
+    fs get <file system name>
+
+Get information about the named file system, including settings and ranks. This
+is a subset of the same information from the ``fs dump`` command.
+
+::
+
+    fs set <file system name> <var> <val>
+
+Change a setting on a file system. These settings are specific to the named
+file system and do not affect other file systems.
+
+::
+
+    fs add_data_pool <file system name> <pool name/id>
+
+Add a data pool to the file system. This pool can be used for file layouts
+as an alternate location to store file data.
+
+::
+
+    fs rm_data_pool <file system name> <pool name/id>
+
+This command removes the specified pool from the list of data pools for the
+file system.  If any files have layouts for the removed data pool, the file
+data will become unavailable. The default data pool (when creating the file
+system) cannot be removed.
+
+
+Settings
+--------
+
+::
+
+    fs set <fs name> max_file_size <size in bytes>
+
+CephFS has a configurable maximum file size, and it's 1TB by default.
+You may wish to set this limit higher if you expect to store large files
+in CephFS. It is a 64-bit field.
+
+Setting ``max_file_size`` to 0 does not disable the limit. It would
+simply limit clients to only creating empty files.
+
+
+Maximum file sizes and performance
+----------------------------------
+
+CephFS enforces the maximum file size limit at the point of appending to
+files or setting their size. It does not affect how anything is stored.
+
+When users create a file of an enormous size (without necessarily
+writing any data to it), some operations (such as deletes) cause the MDS
+to have to do a large number of operations to check if any of the RADOS
+objects within the range that could exist (according to the file size)
+really existed.
+
+The ``max_file_size`` setting prevents users from creating files that
+appear to be eg. exabytes in size, causing load on the MDS as it tries
+to enumerate the objects during operations like stats or deletes.
+
+
+Taking the cluster down
+-----------------------
+
+Taking a CephFS cluster down is done by setting the down flag:
+ 
+:: 
+ 
+    fs set <fs_name> down true
+ 
+To bring the cluster back online:
+ 
+:: 
+
+    fs set <fs_name> down false
+
+This will also restore the previous value of max_mds. MDS daemons are brought
+down in a way such that journals are flushed to the metadata pool and all
+client I/O is stopped.
+
+
+Taking the cluster down rapidly for deletion or disaster recovery
+-----------------------------------------------------------------
+
+To allow rapidly deleting a file system (for testing) or to quickly bring the
+file system and MDS daemons down, use the ``fs fail`` command:
+
+::
+
+    fs fail <fs_name>
+
+This command sets a file system flag to prevent standbys from
+activating on the file system (the ``joinable`` flag).
+
+This process can also be done manually by doing the following:
+
+::
+
+    fs set <fs_name> joinable false
+
+Then the operator can fail all of the ranks which causes the MDS daemons to
+respawn as standbys. The file system will be left in a degraded state.
+
+::
+
+    # For all ranks, 0-N:
+    mds fail <fs_name>:<n>
+
+Once all ranks are inactive, the file system may also be deleted or left in
+this state for other purposes (perhaps disaster recovery).
+
+To bring the cluster back up, simply set the joinable flag:
+
+::
+
+    fs set <fs_name> joinable true
+
+
+Daemons
+-------
+
+Most commands manipulating MDSs take a ``<role>`` argument which can take one
+of three forms:
+
+::
+
+    <fs_name>:<rank>
+    <fs_id>:<rank>
+    <rank>
+
+Commands to manipulate MDS daemons:
+
+::
+
+    mds fail <gid/name/role>
+
+Mark an MDS daemon as failed.  This is equivalent to what the cluster
+would do if an MDS daemon had failed to send a message to the mon
+for ``mds_beacon_grace`` second.  If the daemon was active and a suitable
+standby is available, using ``mds fail`` will force a failover to the standby.
+
+If the MDS daemon was in reality still running, then using ``mds fail``
+will cause the daemon to restart.  If it was active and a standby was
+available, then the "failed" daemon will return as a standby.
+
+
+::
+
+    tell mds.<daemon name> command ...
+
+Send a command to the MDS daemon(s). Use ``mds.*`` to send a command to all
+daemons. Use ``ceph tell mds.* help`` to learn available commands.
+
+::
+
+    mds metadata <gid/name/role>
+
+Get metadata about the given MDS known to the Monitors.
+
+::
+
+    mds repaired <role>
+
+Mark the file system rank as repaired. Unlike the name suggests, this command
+does not change a MDS; it manipulates the file system rank which has been
+marked damaged.
+
+
+Required Client Features
+------------------------
+
+It is sometimes desirable to set features that clients must support to talk to
+CephFS. Clients without those features may disrupt other clients or behave in
+surprising ways. Or, you may want to require newer features to prevent older
+and possibly buggy clients from connecting.
+
+Commands to manipulate required client features of a file system:
+
+::
+
+    fs required_client_features <fs name> add reply_encoding
+    fs required_client_features <fs name> rm reply_encoding
+
+To list all CephFS features
+
+::
+
+    fs feature ls
+
+Clients that are missing newly added features will be evicted automatically.
+
+Here are the current CephFS features and first release they came out:
+
++------------------+--------------+-----------------+
+| Feature          | Ceph release | Upstream Kernel |
++==================+==============+=================+
+| jewel            | jewel        | 4.5             |
++------------------+--------------+-----------------+
+| kraken           | kraken       | 4.13            |
++------------------+--------------+-----------------+
+| luminous         | luminous     | 4.13            |
++------------------+--------------+-----------------+
+| mimic            | mimic        | 4.19            |
++------------------+--------------+-----------------+
+| reply_encoding   | nautilus     | 5.1             |
++------------------+--------------+-----------------+
+| reclaim_client   | nautilus     | N/A             |
++------------------+--------------+-----------------+
+| lazy_caps_wanted | nautilus     | 5.1             |
++------------------+--------------+-----------------+
+| multi_reconnect  | nautilus     | 5.1             |
++------------------+--------------+-----------------+
+| deleg_ino        | octopus      | 5.6             |
++------------------+--------------+-----------------+
+| metric_collect   | pacific      | N/A             |
++------------------+--------------+-----------------+
+| alternate_name   | pacific      | PLANNED         |
++------------------+--------------+-----------------+
+
+CephFS Feature Descriptions
+
+
+::
+
+    reply_encoding
+
+MDS encodes request reply in extensible format if client supports this feature.
+
+
+::
+
+    reclaim_client
+
+MDS allows new client to reclaim another (dead) client's states. This feature
+is used by NFS-Ganesha.
+
+
+::
+
+    lazy_caps_wanted
+
+When a stale client resumes, if the client supports this feature, mds only needs
+to re-issue caps that are explicitly wanted.
+
+
+::
+
+    multi_reconnect
+
+When mds failover, client sends reconnect messages to mds, to reestablish cache
+states. If MDS supports this feature, client can split large reconnect message
+into multiple ones.
+
+
+::
+
+    deleg_ino
+
+MDS delegate inode numbers to client if client supports this feature. Having
+delegated inode numbers is a prerequisite for client to do async file creation.
+
+
+::
+
+    metric_collect
+
+Clients can send performance metric to MDS if MDS support this feature.
+
+::
+
+    alternate_name
+
+Clients can set and understand "alternate names" for directory entries. This is
+to be used for encrypted file name support.
+
+
+Global settings
+---------------
+
+
+::
+
+    fs flag set <flag name> <flag val> [<confirmation string>]
+
+Sets a global CephFS flag (i.e. not specific to a particular file system).
+Currently, the only flag setting is 'enable_multiple' which allows having
+multiple CephFS file systems.
+
+Some flags require you to confirm your intentions with "--yes-i-really-mean-it"
+or a similar string they will prompt you with. Consider these actions carefully
+before proceeding; they are placed on especially dangerous activities.
+
+.. _advanced-cephfs-admin-settings:
+
+Advanced
+--------
+
+These commands are not required in normal operation, and exist
+for use in exceptional circumstances.  Incorrect use of these
+commands may cause serious problems, such as an inaccessible
+file system.
+
+::
+
+    mds rmfailed
+
+This removes a rank from the failed set.
+
+::
+
+    fs reset <file system name>
+
+This command resets the file system state to defaults, except for the name and
+pools. Non-zero ranks are saved in the stopped set.
+
+
+::
+
+    fs new <file system name> <metadata pool name> <data pool name> --fscid <fscid> --force
+
+This command creates a file system with a specific **fscid** (file system cluster ID).
+You may want to do this when an application expects the file system's ID to be
+stable after it has been recovered, e.g., after monitor databases are lost and
+rebuilt. Consequently, file system IDs don't always keep increasing with newer
+file systems.
diff --git a/doc/cephfs/api/index.rst b/doc/cephfs/api/index.rst
new file mode 100644
index 000000000..2247610bb
--- /dev/null
+++ b/doc/cephfs/api/index.rst
@@ -0,0 +1,9 @@
+============
+ CephFS APIs
+============
+
+.. toctree::
+   :maxdepth: 2
+
+   libcephfs (Java) <libcephfs-java>
+   libcephfs (Python) <libcephfs-py>
diff --git a/doc/cephfs/api/libcephfs-java.rst b/doc/cephfs/api/libcephfs-java.rst
new file mode 100644
index 000000000..83b5a6638
--- /dev/null
+++ b/doc/cephfs/api/libcephfs-java.rst
@@ -0,0 +1,18 @@
+===================
+Libcephfs (JavaDoc)
+===================
+
+.. warning::
+
+    CephFS Java bindings are no longer tested by CI. They may not work properly
+    or corrupt data.
+
+    Developers interested in reviving these bindings by fixing and writing tests
+    are encouraged to contribute!
+
+..
+    The admin/build-docs script runs Ant to build the JavaDoc files, and
+    copies them to api/libcephfs-java/javadoc/.
+
+
+View the auto-generated `JavaDoc pages for the CephFS Java bindings <javadoc/>`_.
diff --git a/doc/cephfs/api/libcephfs-py.rst b/doc/cephfs/api/libcephfs-py.rst
new file mode 100644
index 000000000..039401f22
--- /dev/null
+++ b/doc/cephfs/api/libcephfs-py.rst
@@ -0,0 +1,13 @@
+===================
+ LibCephFS (Python)
+===================
+
+.. highlight:: python
+
+The `cephfs` python module provides access to CephFS service.
+
+API calls
+=========
+
+.. automodule:: cephfs
+    :members: DirEntry, DirResult, LibCephFS
diff --git a/doc/cephfs/app-best-practices.rst b/doc/cephfs/app-best-practices.rst
new file mode 100644
index 000000000..50bd3b689
--- /dev/null
+++ b/doc/cephfs/app-best-practices.rst
@@ -0,0 +1,82 @@
+
+Application best practices for distributed file systems
+=======================================================
+
+CephFS is POSIX compatible, and therefore should work with any existing
+applications that expect a POSIX file system.  However, because it is a
+network file system (unlike e.g. XFS) and it is highly consistent (unlike
+e.g. NFS), there are some consequences that application authors may
+benefit from knowing about.
+
+The following sections describe some areas where distributed file systems
+may have noticeably different performance behaviours compared with
+local file systems.
+
+
+ls -l
+-----
+
+When you run "ls -l", the ``ls`` program
+is first doing a directory listing, and then calling ``stat`` on every
+file in the directory.
+
+This is usually far in excess of what an application really needs, and
+it can be slow for large directories.  If you don't really need all
+this metadata for each file, then use a plain ``ls``.
+
+ls/stat on files being extended
+-------------------------------
+
+If another client is currently extending files in the listed directory,
+then an ``ls -l`` may take an exceptionally long time to complete, as
+the lister must wait for the writer to flush data in order to do a valid
+read of the every file's size.  So unless you *really* need to know the
+exact size of every file in the directory, just don't do it!
+
+This would also apply to any application code that was directly
+issuing ``stat`` system calls on files being appended from
+another node.
+
+Very large directories
+----------------------
+
+Do you really need that 10,000,000 file directory?  While directory
+fragmentation enables CephFS to handle it, it is always going to be
+less efficient than splitting your files into more modest-sized directories.
+
+Even standard userspace tools can become quite slow when operating on very
+large directories. For example, the default behaviour of ``ls``
+is to give an alphabetically ordered result, but ``readdir`` system
+calls do not give an ordered result (this is true in general, not just
+with CephFS).  So when you ``ls`` on a million file directory, it is
+loading a list of a million names into memory, sorting the list, then writing
+it out to the display.
+
+Hard links
+----------
+
+Hard links have an intrinsic cost in terms of the internal housekeeping
+that a file system has to do to keep two references to the same data.  In
+CephFS there is a particular performance cost, because with normal files
+the inode is embedded in the directory (i.e. there is no extra fetch of
+the inode after looking up the path).
+
+Working set size
+----------------
+
+The MDS acts as a cache for the metadata stored in RADOS.  Metadata
+performance is very different for workloads whose metadata fits within
+that cache.
+
+If your workload has more files than fit in your cache (configured using 
+``mds_cache_memory_limit`` settings), then make sure you test it
+appropriately: don't test your system with a small number of files and then
+expect equivalent performance when you move to a much larger number of files.
+
+Do you need a file system?
+--------------------------
+
+Remember that Ceph also includes an object storage interface.  If your
+application needs to store huge flat collections of files where you just
+read and write whole files at once, then you might well be better off
+using the :ref:`Object Gateway <object-gateway>`
diff --git a/doc/cephfs/cache-configuration.rst b/doc/cephfs/cache-configuration.rst
new file mode 100644
index 000000000..eabc61cc1
--- /dev/null
+++ b/doc/cephfs/cache-configuration.rst
@@ -0,0 +1,227 @@
+=======================
+MDS Cache Configuration
+=======================
+
+The Metadata Server coordinates a distributed cache among all MDS and CephFS
+clients. The cache serves to improve metadata access latency and allow clients
+to safely (coherently) mutate metadata state (e.g. via `chmod`). The MDS issues
+**capabilities** and **directory entry leases** to indicate what state clients
+may cache and what manipulations clients may perform (e.g. writing to a file).
+
+The MDS and clients both try to enforce a cache size. The mechanism for
+specifying the MDS cache size is described below. Note that the MDS cache size
+is not a hard limit. The MDS always allows clients to lookup new metadata
+which is loaded into the cache. This is an essential policy as it avoids
+deadlock in client requests (some requests may rely on held capabilities before
+capabilities are released).
+
+When the MDS cache is too large, the MDS will **recall** client state so cache
+items become unpinned and eligible to be dropped. The MDS can only drop cache
+state when no clients refer to the metadata to be dropped. Also described below
+is how to configure the MDS recall settings for your workload's needs. This is
+necessary if the internal throttles on the MDS recall can not keep up with the
+client workload.
+
+
+MDS Cache Size
+--------------
+
+You can limit the size of the Metadata Server (MDS) cache by a byte count. This
+is done through the `mds_cache_memory_limit` configuration. For example::
+
+    ceph config set mds mds_cache_memory_limit 8GB
+
+In addition, you can specify a cache reservation by using the
+`mds_cache_reservation` parameter for MDS operations. The cache reservation is
+limited as a percentage of the memory and is set to 5% by default. The intent
+of this parameter is to have the MDS maintain an extra reserve of memory for
+its cache for new metadata operations to use. As a consequence, the MDS should
+in general operate below its memory limit because it will recall old state from
+clients in order to drop unused metadata in its cache.
+
+If the MDS cannot keep its cache under the target size, the MDS will send a
+health alert to the Monitors indicating the cache is too large. This is
+controlled by the `mds_health_cache_threshold` configuration which is by
+default 150% of the maximum cache size.
+
+Because the cache limit is not a hard limit, potential bugs in the CephFS
+client, MDS, or misbehaving applications might cause the MDS to exceed its
+cache size. The health warnings are intended to help the operator detect this
+situation and make necessary adjustments or investigate buggy clients.
+
+MDS Cache Trimming
+------------------
+
+There are two configurations for throttling the rate of cache trimming in the MDS:
+
+::
+
+    mds_cache_trim_threshold (default 64k)
+
+
+and
+
+::
+
+    mds_cache_trim_decay_rate (default 1)
+
+
+The intent of the throttle is to prevent the MDS from spending too much time
+trimming its cache. This may limit its ability to handle client requests or
+perform other upkeep.
+
+The trim configurations control an internal **decay counter**. Anytime metadata
+is trimmed from the cache, the counter is incremented.  The threshold sets the
+maximum size of the counter while the decay rate indicates the exponential half
+life for the counter. If the MDS is continually removing items from its cache,
+it will reach a steady state of ``-ln(0.5)/rate*threshold`` items removed per
+second.
+
+.. note:: Increasing the value of the confguration setting
+          ``mds_cache_trim_decay_rate`` leads to the MDS spending less time
+          trimming the cache. To increase the cache trimming rate, set a lower
+          value.
+
+The defaults are conservative and may need to be changed for production MDS with
+large cache sizes.
+
+
+MDS Recall
+----------
+
+MDS limits its recall of client state (capabilities/leases) to prevent creating
+too much work for itself handling release messages from clients. This is controlled
+via the following configurations:
+
+
+The maximum number of capabilities to recall from a single client in a given recall
+event::
+
+    mds_recall_max_caps (default: 5000)
+
+The threshold and decay rate for the decay counter on a session::
+
+    mds_recall_max_decay_threshold (default: 16k)
+
+and::
+
+    mds_recall_max_decay_rate (default: 2.5 seconds)
+
+The session decay counter controls the rate of recall for an individual
+session. The behavior of the counter works the same as for cache trimming
+above. Each capability that is recalled increments the counter.
+
+There is also a global decay counter that throttles for all session recall::
+
+    mds_recall_global_max_decay_threshold (default: 64k)
+
+its decay rate is the same as ``mds_recall_max_decay_rate``. Any recalled
+capability for any session also increments this counter.
+
+If clients are slow to release state, the warning "failing to respond to cache
+pressure" or ``MDS_HEALTH_CLIENT_RECALL`` will be reported. Each session's rate
+of release is monitored by another decay counter configured by::
+
+    mds_recall_warning_threshold (default: 32k)
+
+and::
+
+    mds_recall_warning_decay_rate (default: 60.0 seconds)
+
+Each time a capability is released, the counter is incremented.  If clients do
+not release capabilities quickly enough and there is cache pressure, the
+counter will indicate if the client is slow to release state.
+
+Some workloads and client behaviors may require faster recall of client state
+to keep up with capability acquisition. It is recommended to increase the above
+counters as needed to resolve any slow recall warnings in the cluster health
+state.
+
+
+MDS Cap Acquisition Throttle
+----------------------------
+
+A trivial "find" command on a large directory hierarchy will cause the client
+to receive caps significantly faster than it will release. The MDS will try
+to have the client reduce its caps below the ``mds_max_caps_per_client`` limit
+but the recall throttles prevent it from catching up to the pace of acquisition.
+So the readdir is throttled to control cap acquisition via the following
+configurations:
+
+
+The threshold and decay rate for the readdir cap acquisition decay counter::
+
+    mds_session_cap_acquisition_throttle (default: 500K)
+
+and::
+
+    mds_session_cap_acquisition_decay_rate (default: 10 seconds)
+
+The cap acquisition decay counter controls the rate of cap acquisition via
+readdir. The behavior of the decay counter is the same as for cache trimming or
+caps recall. Each readdir call increments the counter by the number of files in
+the result.
+
+The ratio of ``mds_max_maps_per_client`` that client must exceed before readdir
+maybe throttled by cap acquisition throttle::
+
+    mds_session_max_caps_throttle_ratio (default: 1.1)
+
+The timeout in seconds after which a client request is retried due to cap
+acquisition throttling::
+
+    mds_cap_acquisition_throttle_retry_request_timeout (default: 0.5 seconds)
+
+If the number of caps acquired by the client per session is greater than the
+``mds_session_max_caps_throttle_ratio`` and cap acquisition decay counter is
+greater than ``mds_session_cap_acquisition_throttle``, the readdir is throttled.
+The readdir request is retried after ``mds_cap_acquisition_throttle_retry_request_timeout``
+seconds.
+
+
+Session Liveness
+----------------
+
+The MDS also keeps track of whether sessions are quiescent. If a client session
+is not utilizing its capabilities or is otherwise quiet, the MDS will begin
+recalling state from the session even if it's not under cache pressure. This
+helps the MDS avoid future work when the cluster workload is hot and cache
+pressure is forcing the MDS to recall state. The expectation is that a client
+not utilizing its capabilities is unlikely to use those capabilities anytime
+in the near future.
+
+Determining whether a given session is quiescent is controlled by the following
+configuration variables::
+
+    mds_session_cache_liveness_magnitude (default: 10)
+
+and::
+
+    mds_session_cache_liveness_decay_rate (default: 5min)
+
+The configuration ``mds_session_cache_liveness_decay_rate`` indicates the
+half-life for the decay counter tracking the use of capabilities by the client.
+Each time a client manipulates or acquires a capability, the MDS will increment
+the counter. This is a rough but effective way to monitor the utilization of the
+client cache.
+
+The ``mds_session_cache_liveness_magnitude`` is a base-2 magnitude difference
+of the liveness decay counter and the number of capabilities outstanding for
+the session. So if the client has ``1*2^20`` (1M) capabilities outstanding and
+only uses **less** than ``1*2^(20-mds_session_cache_liveness_magnitude)`` (1K
+using defaults), the MDS will consider the client to be quiescent and begin
+recall.
+
+
+Capability Limit
+----------------
+
+The MDS also tries to prevent a single client from acquiring too many
+capabilities. This helps prevent recovery from taking a long time in some
+situations.  It is not generally necessary for a client to have such a large
+cache. The limit is configured via::
+
+    mds_max_caps_per_client (default: 1M)
+
+It is not recommended to set this value above 5M but it may be helpful with
+some workloads.
diff --git a/doc/cephfs/capabilities.rst b/doc/cephfs/capabilities.rst
new file mode 100644
index 000000000..ac47bb12b
--- /dev/null
+++ b/doc/cephfs/capabilities.rst
@@ -0,0 +1,112 @@
+======================
+Capabilities in CephFS
+======================
+When a client wants to operate on an inode, it will query the MDS in various
+ways, which will then grant the client a set of **capabilities**. These
+grant the client permissions to operate on the inode in various ways. One
+of the major differences from other network file systems (e.g NFS or SMB) is
+that the capabilities granted are quite granular, and it's possible that
+multiple clients can hold different capabilities on the same inodes.
+
+Types of Capabilities
+---------------------
+There are several "generic" capability bits. These denote what sort of ability
+the capability grants.
+
+::
+
+        /* generic cap bits */
+        #define CEPH_CAP_GSHARED     1  /* client can reads (s) */
+        #define CEPH_CAP_GEXCL       2  /* client can read and update (x) */
+        #define CEPH_CAP_GCACHE      4  /* (file) client can cache reads (c) */
+        #define CEPH_CAP_GRD         8  /* (file) client can read (r) */
+        #define CEPH_CAP_GWR        16  /* (file) client can write (w) */
+        #define CEPH_CAP_GBUFFER    32  /* (file) client can buffer writes (b) */
+        #define CEPH_CAP_GWREXTEND  64  /* (file) client can extend EOF (a) */
+        #define CEPH_CAP_GLAZYIO   128  /* (file) client can perform lazy io (l) */
+
+These are then shifted by a particular number of bits. These denote a part of
+the inode's data or metadata on which the capability is being granted:
+
+::
+
+        /* per-lock shift */
+        #define CEPH_CAP_SAUTH      2 /* A */
+        #define CEPH_CAP_SLINK      4 /* L */
+        #define CEPH_CAP_SXATTR     6 /* X */
+        #define CEPH_CAP_SFILE      8 /* F */
+
+Only certain generic cap types are ever granted for some of those "shifts",
+however. In particular, only the FILE shift ever has more than the first two
+bits.
+
+::
+
+        | AUTH | LINK | XATTR | FILE
+        2      4      6       8
+
+From the above, we get a number of constants, that are generated by taking
+each bit value and shifting to the correct bit in the word:
+
+::
+
+        #define CEPH_CAP_AUTH_SHARED  (CEPH_CAP_GSHARED  << CEPH_CAP_SAUTH)
+
+These bits can then be or'ed together to make a bitmask denoting a set of
+capabilities.
+
+There is one exception:
+
+::
+
+        #define CEPH_CAP_PIN  1  /* no specific capabilities beyond the pin */
+
+The "pin" just pins the inode into memory, without granting any other caps.
+
+Graphically:
+
+::
+
+    +---+---+---+---+---+---+---+---+
+    | p | _ |As   x |Ls   x |Xs   x |
+    +---+---+---+---+---+---+---+---+
+    |Fs   x   c   r   w   b   a   l |
+    +---+---+---+---+---+---+---+---+
+
+The second bit is currently unused.
+
+Abilities granted by each cap
+-----------------------------
+While that is how capabilities are granted (and communicated), the important
+bit is what they actually allow the client to do:
+
+* PIN: this just pins the inode into memory. This is sufficient to allow the
+  client to get to the inode number, as well as other immutable things like
+  major or minor numbers in a device inode, or symlink contents.
+
+* AUTH: this grants the ability to get to the authentication-related metadata.
+  In particular, the owner, group and mode. Note that doing a full permission
+  check may require getting at ACLs as well, which are stored in xattrs.
+
+* LINK: the link count of the inode
+
+* XATTR: ability to access or manipulate xattrs. Note that since ACLs are
+  stored in xattrs, it's also sometimes necessary to access them when checking
+  permissions.
+
+* FILE: this is the big one. These allow the client to access and manipulate
+  file data. It also covers certain metadata relating to file data -- the
+  size, mtime, atime and ctime, in particular.
+
+Shorthand
+---------
+Note that the client logging can also present a compact representation of the
+capabilities. For example:
+
+::
+
+        pAsLsXsFs
+
+The 'p' represents the pin. Each capital letter corresponds to the shift
+values, and the lowercase letters after each shift are for the actual
+capabilities granted in each shift.
diff --git a/doc/cephfs/ceph-dokan.rst b/doc/cephfs/ceph-dokan.rst
new file mode 100644
index 000000000..b9fb6c592
--- /dev/null
+++ b/doc/cephfs/ceph-dokan.rst
@@ -0,0 +1,102 @@
+.. _ceph-dokan:
+
+=======================
+Mount CephFS on Windows
+=======================
+
+``ceph-dokan`` can be used for mounting CephFS filesystems on Windows.
+It leverages Dokany, a Windows driver that allows implementing filesystems in
+userspace, pretty much like FUSE.
+
+Please check the `installation guide`_ to get started.
+
+Usage
+=====
+
+Mounting filesystems
+--------------------
+
+In order to mount a ceph filesystem, the following command can be used::
+
+    ceph-dokan.exe -c c:\ceph.conf -l x
+
+This will mount the default ceph filesystem using the drive letter ``x``.
+If ``ceph.conf`` is placed at the default location, which is
+``%ProgramData%\ceph\ceph.conf``, then this argument becomes optional.
+
+The ``-l`` argument also allows using an empty folder as a mountpoint
+instead of a drive letter.
+
+The uid and gid used for mounting the filesystem default to 0 and may be
+changed using the following ``ceph.conf`` options::
+
+    [client]
+    # client_permissions = true
+    client_mount_uid = 1000
+    client_mount_gid = 1000
+
+If you have more than one FS on your Ceph cluster, use the option
+``--client_fs`` to mount the non-default FS::
+
+    mkdir -Force C:\mnt\mycephfs2
+    ceph-dokan.exe --mountpoint C:\mnt\mycephfs2 --client_fs mycephfs2
+
+CephFS subdirectories can be mounted using the ``--root-path`` parameter::
+
+    ceph-dokan -l y --root-path /a
+
+If the ``-o --removable`` flags are set, the mounts will show up in the
+``Get-Volume`` results::
+
+    PS C:\> Get-Volume -FriendlyName "Ceph*" | `
+            Select-Object -Property @("DriveLetter", "Filesystem", "FilesystemLabel")
+
+    DriveLetter Filesystem FilesystemLabel
+    ----------- ---------- ---------------
+              Z Ceph       Ceph
+              W Ceph       Ceph - new_fs
+
+Please use ``ceph-dokan --help`` for a full list of arguments.
+
+Credentials
+-----------
+
+The ``--id`` option passes the name of the CephX user whose keyring we intend to
+use for mounting CephFS. The following commands are equivalent::
+
+    ceph-dokan --id foo -l x
+    ceph-dokan --name client.foo -l x
+
+Unmounting filesystems
+----------------------
+
+The mount can be removed by either issuing ctrl-c or using the unmap command,
+like so::
+
+    ceph-dokan.exe unmap -l x
+
+Note that when unmapping Ceph filesystems, the exact same mountpoint argument
+must be used as when the mapping was created.
+
+Limitations
+-----------
+
+Be aware that Windows ACLs are ignored. Posix ACLs are supported but cannot be
+modified using the current CLI. In the future, we may add some command actions
+to change file ownership or permissions.
+
+Another thing to note is that cephfs doesn't support mandatory file locks, which
+Windows is heavily rely upon. At the moment, we're letting Dokan handle file
+locks, which are only enforced locally.
+
+Unlike ``rbd-wnbd``, ``ceph-dokan`` doesn't currently provide a ``service``
+command. In order for the cephfs mount to survive host reboots, consider using
+``NSSM``.
+
+Troubleshooting
+===============
+
+Please consult the `Windows troubleshooting`_ page.
+
+.. _Windows troubleshooting: ../../install/windows-troubleshooting
+.. _installation guide: ../../install/windows-install
diff --git a/doc/cephfs/cephfs-architecture.svg b/doc/cephfs/cephfs-architecture.svg
new file mode 100644
index 000000000..e44c52a3f
--- /dev/null
+++ b/doc/cephfs/cephfs-architecture.svg
@@ -0,0 +1 @@
+<svg version="1.1" viewBox="0.0 0.0 784.4199475065617 481.56692913385825" fill="none" stroke="none" stroke-linecap="square" stroke-miterlimit="10" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg"><clipPath id="p.0"><path d="m0 0l784.4199 0l0 481.56693l-784.4199 0l0 -481.56693z" clip-rule="nonzero"/></clipPath><g clip-path="url(#p.0)"><path fill="#000000" fill-opacity="0.0" d="m0 0l784.4199 0l0 481.56693l-784.4199 0z" fill-rule="evenodd"/><path fill="#b6d7a8" d="m30.272966 79.10533l0 0c0 -31.388912 24.684267 -56.83465 55.133858 -56.83465l0 0c14.622414 0 28.64592 5.987919 38.985527 16.646484c10.3396 10.658562 16.14833 25.114677 16.14833 40.188164l0 0c0 31.388908 -24.684265 56.83464 -55.133858 56.83464l0 0c-30.449589 0 -55.133858 -25.445732 -55.133858 -56.83464z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m30.272966 79.10533l0 0c0 -31.388912 24.684267 -56.83465 55.133858 -56.83465l0 0c14.622414 0 28.64592 5.987919 38.985527 16.646484c10.3396 10.658562 16.14833 25.114677 16.14833 40.188164l0 0c0 31.388908 -24.684265 56.83464 -55.133858 56.83464l0 0c-30.449589 0 -55.133858 -25.445732 -55.133858 -56.83464z" fill-rule="evenodd"/><path fill="#000000" d="m62.688076 85.08783q0.6875 0 1.3124962 -0.375q0.640625 -0.390625 1.03125 -1.03125l0.96875 0.625q-0.578125 0.953125 -1.359375 1.4375q-0.7812462 0.46875 -1.9062462 0.46875q-1.296875 0 -2.3125 -0.640625q-1.015625 -0.640625 -1.625 -1.96875q-0.609375 -1.328125 -0.609375 -3.328125q0 -2.140625 0.65625 -3.46875q0.65625 -1.34375 1.65625 -1.921875q1.0 -0.578125 2.171875 -0.578125q1.21875 0 2.1249962 0.625q0.90625 0.609375 1.375 1.671875l-1.125 0.515625q-0.015625 0 -0.015625 0q0 -0.015625 0 -0.015625q-0.484375 -0.96875 -1.0781212 -1.359375q-0.59375 -0.390625 -1.328125 -0.390625q-1.46875 0 -2.328125 1.28125q-0.84375 1.28125 -0.84375 3.546875q0 1.4375 0.421875 2.5625q0.4375 1.109375 1.171875 1.734375q0.734375 0.609375 1.640625 0.609375zm2.1406212 -8.015625q0.03125 -0.046875 0.03125 -0.046875q0.046875 0 0.15625 0.125l-0.125 0.046875l-0.0625 -0.125zm0.21875 0.046875q0.078125 0.171875 -0.03125 0.03125l0.03125 -0.03125zm3.171875 8.90625l0 -1.078125l2.53125 0l0 -10.25l-2.421875 0l0 -1.078125l3.78125 0l0 11.328125l2.5 0l0 1.078125l-6.390625 0zm9.71875 0l0 -1.078125l2.1875076 0l0 -6.359375l-2.0625076 0l0 -1.09375l3.4062576 0l0 7.453125l2.0 0l0 1.078125l-5.5312576 0zm2.7812576 -10.3125q-0.390625 0 -0.671875 -0.28125q-0.28125 -0.28125 -0.28125 -0.671875q0 -0.40625 0.265625 -0.6875q0.28125 -0.28125 0.6875 -0.28125q0.390625 0 0.671875 0.296875q0.296875 0.28125 0.296875 0.671875q0 0.390625 -0.296875 0.671875q-0.28125 0.28125 -0.671875 0.28125zm9.765625 10.5q-1.90625 0 -3.03125 -1.15625q-1.125 -1.15625 -1.125 -3.265625q0 -1.40625 0.515625 -2.421875q0.515625 -1.015625 1.40625 -1.546875q0.890625 -0.53125 1.984375 -0.53125q1.546875 0 2.5 1.03125q0.96875 1.03125 0.96875 3.015625q0 0.203125 -0.03125 0.625l-6.0625 0q0.078125 1.5625 0.875 2.375q0.796875 0.796875 2.03125 0.796875q1.34375 0 2.203125 -0.953125l0.75 0.71875q-1.078125 1.3125 -2.984375 1.3125zm1.859375 -5.296875q0 -1.203125 -0.609375 -1.890625q-0.59375 -0.703125 -1.59375 -0.703125q-0.921875 0 -1.625 0.65625q-0.6875 0.640625 -0.859375 1.9375l4.6875 0zm3.734375 -3.421875l1.3125 0l0 1.515625q0.5 -0.78125 1.265625 -1.25q0.765625 -0.46875 1.625 -0.46875q1.125 0 1.8125 0.875q0.6875 0.859375 0.6875 2.6875l0 5.171875l-1.3125 0l0 -5.125q0 -1.28125 -0.4375 -1.859375q-0.421875 -0.578125 -1.125 -0.578125q-0.578125 0 -1.171875 0.34375q-0.578125 0.328125 -0.96875 0.9375q-0.375 0.609375 -0.375 1.375l0 4.90625l-1.3125 0l0 -8.53125zm16.265625 7.75q-1.234375 0.90625 -2.703125 0.90625q-1.421875 0 -2.015625 -0.84375q-0.578125 -0.859375 -0.578125 -2.8125q0 -0.3125 0.03125 -1.09375l0.171875 -2.796875l-1.875 0l0 -1.109375l1.953125 0l0.125 -2.265625l1.5 -0.25l0.1875 -0.015625l0.015625 0.109375q-0.125 0.1875 -0.203125 0.328125q-0.0625 0.125 -0.078125 0.40625l-0.203125 1.6875l2.828125 0l0 1.109375l-2.90625 0l-0.171875 2.890625q-0.03125 0.75 -0.03125 0.984375q0 1.5 0.34375 2.03125q0.34375 0.53125 1.109375 0.53125q0.5625 0 1.03125 -0.203125q0.46875 -0.21875 1.0625 -0.65625l0.40625 1.0625z" fill-rule="nonzero"/><path fill="#e06666" d="m245.87401 93.125015l0 0c0 -33.441494 27.109695 -60.55118 60.551193 -60.55118l0 0c16.059174 0 31.460602 6.3794823 42.81613 17.735031c11.35556 11.355549 17.735046 26.756977 17.735046 42.81615l0 0c0 33.44149 -27.10968 60.551178 -60.551178 60.551178l0 0c-33.441498 0 -60.551193 -27.109688 -60.551193 -60.551178z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m245.87401 93.125015l0 0c0 -33.441494 27.109695 -60.55118 60.551193 -60.55118l0 0c16.059174 0 31.460602 6.3794823 42.81613 17.735031c11.35556 11.355549 17.735046 26.756977 17.735046 42.81615l0 0c0 33.44149 -27.10968 60.551178 -60.551178 60.551178l0 0c-33.441498 0 -60.551193 -27.109688 -60.551193 -60.551178z" fill-rule="evenodd"/><path fill="#000000" d="m286.12833 89.04501l-1.171875 -3.390625l-3.75 0l-1.1875 3.390625l-1.28125 0l4.296875 -11.828125l0.140625 0l4.296875 11.828125l-1.34375 0zm-4.578125 -4.40625l3.0625 0l-1.53125 -4.4375l-1.53125 4.4375zm11.609375 3.40625q1.28125 0 2.21875 -1.046875l0.78125 0.90625q-1.25 1.34375 -3.078125 1.34375q-1.234375 0 -2.203125 -0.578125q-0.96875 -0.578125 -1.515625 -1.59375q-0.546875 -1.015625 -0.546875 -2.28125q0 -1.265625 0.546875 -2.265625q0.546875 -1.015625 1.515625 -1.59375q0.96875 -0.578125 2.1875 -0.578125q1.03125 0 1.859375 0.421875q0.84375 0.40625 1.375 1.15625l-0.84375 0.828125l-0.015625 0.015625q-0.546875 -0.71875 -1.125 -1.0q-0.5625 -0.296875 -1.390625 -0.296875q-0.734375 0 -1.359375 0.40625q-0.625 0.40625 -1.0 1.140625q-0.375 0.71875 -0.375 1.671875q0 0.953125 0.375 1.71875q0.390625 0.765625 1.0625 1.203125q0.6875 0.421875 1.53125 0.421875zm2.078125 -5.25q0 -0.109375 0.15625 0.015625l-0.0625 0.078125l-0.09375 -0.09375zm0.203125 -0.015625q0.09375 0.125 0.046875 0.09375q-0.046875 -0.046875 -0.09375 -0.0625l0.046875 -0.03125zm9.9375 5.484375q-1.234375 0.90625 -2.703125 0.90625q-1.421875 0 -2.015625 -0.84375q-0.578125 -0.859375 -0.578125 -2.8125q0 -0.3125 0.03125 -1.09375l0.171875 -2.796875l-1.875 0l0 -1.109375l1.953125 0l0.125 -2.265625l1.5 -0.25l0.1875 -0.015625l0.015625 0.109375q-0.125 0.1875 -0.203125 0.328125q-0.0625 0.125 -0.078125 0.40625l-0.203125 1.6875l2.828125 0l0 1.109375l-2.90625 0l-0.171875 2.890625q-0.03125 0.75 -0.03125 0.984375q0 1.5 0.34375 2.03125q0.34375 0.53125 1.109375 0.53125q0.5625 0 1.03125 -0.203125q0.46875 -0.21875 1.0625 -0.65625l0.40625 1.0625zm2.90625 0.78125l0 -1.078125l2.1875 0l0 -6.359375l-2.0625 0l0 -1.09375l3.40625 0l0 7.453125l2.0 0l0 1.078125l-5.53125 0zm2.78125 -10.3125q-0.390625 0 -0.671875 -0.28125q-0.28125 -0.28125 -0.28125 -0.671875q0 -0.40625 0.265625 -0.6875q0.28125 -0.28125 0.6875 -0.28125q0.390625 0 0.671875 0.296875q0.296875 0.28125 0.296875 0.671875q0 0.390625 -0.296875 0.671875q-0.28125 0.28125 -0.671875 0.28125zm13.171875 1.78125q-0.296875 1.515625 -1.25 3.734375l-2.0625 4.796875l-1.046875 0l-3.375 -8.53125l1.34375 0l2.625 6.6875l1.375 -3.15625q0.859375 -1.9375 1.125 -3.53125l1.265625 0zm5.921875 8.71875q-1.90625 0 -3.03125 -1.15625q-1.125 -1.15625 -1.125 -3.265625q0 -1.40625 0.515625 -2.421875q0.515625 -1.015625 1.40625 -1.546875q0.890625 -0.53125 1.984375 -0.53125q1.546875 0 2.5 1.03125q0.96875 1.03125 0.96875 3.015625q0 0.203125 -0.03125 0.625l-6.0625 0q0.078125 1.5625 0.875 2.375q0.796875 0.796875 2.03125 0.796875q1.34375 0 2.203125 -0.953125l0.75 0.71875q-1.078125 1.3125 -2.984375 1.3125zm1.859375 -5.296875q0 -1.203125 -0.609375 -1.890625q-0.59375 -0.703125 -1.59375 -0.703125q-0.921875 0 -1.625 0.65625q-0.6875 0.640625 -0.859375 1.9375l4.6875 0z" fill-rule="nonzero"/><path fill="#000000" d="m293.19864 111.04501l0 -11.625l1.03125 0l2.875 5.6875l2.921875 -5.703125l0.984375 0l0 11.640625l-1.234375 0l0 -8.765625l-2.515625 4.6875l-0.5 0l-2.34375 -4.640625l0 8.71875l-1.21875 0zm9.5625 -11.625l2.71875 0q1.4375 0 2.265625 0.390625q0.84375 0.375 1.453125 1.203125q1.140625 1.53125 1.140625 4.28125q-0.109375 2.828125 -1.3125 4.3125q-1.1875 1.46875 -3.78125 1.453125l-2.484375 0l0 -11.640625zm2.4375 10.625q3.84375 0 3.84375 -4.671875q-0.03125 -2.375 -0.890625 -3.609375q-0.84375 -1.234375 -2.75 -1.234375l-1.40625 0l0 9.515625l1.203125 0zm11.453125 -5.421875q1.65625 0.6875 2.28125 1.421875q0.640625 0.734375 0.640625 1.828125q0 0.859375 -0.421875 1.625q-0.40625 0.765625 -1.296875 1.25q-0.875 0.484375 -2.15625 0.484375q-2.265625 0 -3.640625 -1.46875l0.671875 -1.1875l0 -0.015625q0.015625 0 0.015625 0.015625q0 0 0 0q0.515625 0.671875 1.296875 1.078125q0.796875 0.40625 1.84375 0.40625q1.03125 0 1.71875 -0.578125q0.6875 -0.578125 0.6875 -1.421875q0 -0.546875 -0.21875 -0.90625q-0.21875 -0.359375 -0.78125 -0.71875q-0.546875 -0.359375 -1.671875 -0.84375q-1.71875 -0.671875 -2.453125 -1.515625q-0.734375 -0.859375 -0.734375 -1.9375q0 -1.296875 0.953125 -2.078125q0.953125 -0.78125 2.59375 -0.78125q0.953125 0 1.78125 0.390625q0.828125 0.375 1.4375 1.0625l-0.71875 0.96875l-0.015625 0.015625q-0.5625 -0.765625 -1.171875 -1.0625q-0.609375 -0.296875 -1.515625 -0.296875q-0.90625 0 -1.453125 0.5q-0.546875 0.484375 -0.546875 1.1875q0 0.546875 0.234375 0.953125q0.234375 0.390625 0.84375 0.78125q0.625 0.375 1.796875 0.84375zm1.59375 -2.875q0 -0.046875 0.078125 0q0.078125 0.03125 0.09375 0.03125l-0.046875 0.0625l-0.125 -0.0625l0 -0.03125zm0.21875 -0.03125q0.109375 0.15625 0.03125 0.109375q-0.0625 -0.046875 -0.078125 -0.046875l0.046875 -0.0625zm-5.53125 6.796875q0 0.046875 -0.078125 0.015625q-0.0625 -0.046875 -0.09375 -0.046875l0.046875 -0.0625l0.125 0.0625l0 0.03125zm-0.203125 0.046875q-0.109375 -0.140625 0.03125 -0.078125l-0.03125 0.078125z" fill-rule="nonzero"/><path fill="#6fa8dc" d="m247.85564 203.24673l100.84248 0l0 55.47885c-50.418518 0 -50.418518 21.345428 -100.84248 10.672699zm8.308182 0l0 -7.1924896l100.15378 0l0 55.82686c-3.8070374 0 -7.6194763 0.3867035 -7.6194763 0.3867035l0 -49.021072zm7.809265 -7.1924896l0 -7.018463l101.021454 0l0 55.652832c-4.3384705 0 -8.676941 0.2900238 -8.676941 0.2900238l0 -48.924393z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m247.85564 203.24673l100.84248 0l0 55.47885c-50.418518 0 -50.418518 21.345428 -100.84248 10.672699zm8.308182 0l0 -7.1924896l100.15378 0l0 55.82686c-3.8070374 0 -7.6194763 0.3867035 -7.6194763 0.3867035m-84.72504 -56.213562l0 -7.018463l101.021454 0l0 55.652832c-4.3384705 0 -8.676941 0.2900238 -8.676941 0.2900238" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m247.85564 269.3983c50.423965 10.6727295 50.423965 -10.672699 100.84248 -10.672699l0 -6.457779c0 0 3.812439 -0.3867035 7.6194763 -0.3867035l0 -6.902466c0 0 4.3384705 -0.2900238 8.676941 -0.2900238l0 -55.652832l-101.021454 0l0 7.018463l-7.809265 0l0 7.1924896l-8.308182 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m247.85564 203.24673l100.84248 0l0 55.47885c-50.418518 0 -50.418518 21.345428 -100.84248 10.672699zm8.308182 0l0 -7.1924896l100.15378 0l0 55.82686c-3.8070374 0 -7.6194763 0.3867035 -7.6194763 0.3867035m-84.72504 -56.213562l0 -7.018463l101.021454 0l0 55.652832c-4.3384705 0 -8.676941 0.2900238 -8.676941 0.2900238" fill-rule="evenodd"/><path fill="#000000" d="m272.36282 232.7269l0 6.609375q0.03125 1.421875 -0.375 2.34375q-0.40625 0.90625 -1.109375 1.34375q-0.703125 0.421875 -1.5625 0.421875q-1.703125 0 -2.765625 -1.28125l0.734375 -0.9375l0.015625 -0.015625l0.03125 0.015625q0.515625 0.5625 0.953125 0.8125q0.4375 0.25 1.0 0.25q0.953125 0 1.375 -0.65625q0.421875 -0.671875 0.421875 -2.265625l0 -6.640625l-2.25 0l0 -1.109375l5.328125 0l0 1.109375l-1.796875 0zm-4.84375 8.421875q0 0.078125 -0.0625 0.078125q-0.0625 -0.015625 -0.109375 -0.0625l0.078125 -0.09375l0.09375 0.078125zm-0.21875 0.0625q-0.09375 -0.09375 -0.03125 -0.078125q0.0625 0.015625 0.078125 0.03125l-0.046875 0.046875zm12.328125 2.1875q-1.140625 0 -2.046875 -0.5625q-0.890625 -0.5625 -1.390625 -1.5625q-0.5 -1.015625 -0.5 -2.296875q0 -1.296875 0.5 -2.296875q0.5 -1.015625 1.390625 -1.578125q0.90625 -0.578125 2.046875 -0.578125q1.125 0 2.015625 0.578125q0.90625 0.5625 1.40625 1.578125q0.5 1.0 0.5 2.296875q0 1.28125 -0.5 2.296875q-0.5 1.0 -1.40625 1.5625q-0.890625 0.5625 -2.015625 0.5625zm0 -1.125q0.71875 0 1.28125 -0.421875q0.578125 -0.4375 0.90625 -1.1875q0.328125 -0.765625 0.328125 -1.71875q0 -1.453125 -0.71875 -2.375q-0.703125 -0.921875 -1.796875 -0.921875q-1.109375 0 -1.828125 0.921875q-0.703125 0.921875 -0.703125 2.375q0 0.953125 0.328125 1.71875q0.328125 0.75 0.890625 1.1875q0.578125 0.421875 1.3125 0.421875zm8.78125 1.171875q-1.34375 0 -2.15625 -1.0q-0.796875 -1.0 -0.78125 -2.96875l0.03125 -4.765625l1.296875 0l0 4.765625q0 1.53125 0.53125 2.21875q0.53125 0.671875 1.40625 0.671875q0.96875 0 1.625 -0.75q0.671875 -0.765625 0.671875 -2.203125l0 -4.703125l1.3125 0l0 7.203125q0 0.453125 0.015625 0.75q0.03125 0.28125 0.171875 0.578125l-1.296875 0q-0.125 -0.28125 -0.15625 -0.578125q-0.03125 -0.296875 -0.03125 -0.734375q-0.40625 0.71875 -1.109375 1.125q-0.6875 0.390625 -1.53125 0.390625zm13.15625 -6.953125l0 0.015625q-0.546875 -0.5 -0.921875 -0.671875q-0.359375 -0.171875 -0.84375 -0.171875q-0.671875 0 -1.265625 0.34375q-0.59375 0.328125 -0.96875 1.015625q-0.375 0.671875 -0.375 1.703125l0 4.53125l-1.359375 0l0 -8.546875l1.40625 0l-0.046875 1.578125q0.359375 -0.84375 1.09375 -1.3125q0.734375 -0.46875 1.609375 -0.46875q1.375 0 2.265625 0.9375l-0.59375 1.046875zm0 0.015625q0.125 0.109375 0.0625 0.09375q-0.0625 -0.015625 -0.109375 -0.03125l0.046875 -0.0625zm-0.203125 0.0625q0 -0.0625 0.046875 -0.046875q0.0625 0 0.109375 0.046875l-0.03125 0.09375l-0.125 -0.078125l0 -0.015625zm2.921875 -1.859375l1.3125 0l0 1.515625q0.5 -0.78125 1.265625 -1.25q0.765625 -0.46875 1.625 -0.46875q1.125 0 1.8125 0.875q0.6875 0.859375 0.6875 2.6875l0 5.171875l-1.3125 0l0 -5.125q0 -1.28125 -0.4375 -1.859375q-0.421875 -0.578125 -1.125 -0.578125q-0.578125 0 -1.171875 0.34375q-0.578125 0.328125 -0.96875 0.9375q-0.375 0.609375 -0.375 1.375l0 4.90625l-1.3125 0l0 -8.53125zm12.53125 -0.1875q1.828125 0 2.78125 0.953125q0.96875 0.953125 0.96875 3.234375l0 4.53125l-1.453125 0l0 -1.3125q-0.78125 1.515625 -2.90625 1.515625q-1.421875 0 -2.21875 -0.640625q-0.796875 -0.640625 -0.796875 -1.703125q0 -0.90625 0.578125 -1.578125q0.578125 -0.671875 1.546875 -1.03125q0.984375 -0.359375 2.15625 -0.359375q1.0625 0 1.921875 0.109375q-0.109375 -1.4375 -0.75 -2.015625q-0.640625 -0.59375 -1.921875 -0.59375q-0.625 0 -1.21875 0.25q-0.578125 0.234375 -1.0625 0.703125l-0.65625 -0.859375q1.1875 -1.203125 3.03125 -1.203125zm-0.484375 7.890625q1.390625 0 2.1875 -0.796875q0.796875 -0.8125 0.875 -2.34375q-0.84375 -0.140625 -1.8125 -0.140625q-1.40625 0 -2.234375 0.46875q-0.828125 0.46875 -0.828125 1.421875q0 1.390625 1.8125 1.390625zm6.734375 0.828125l0 -1.078125l2.53125 0l0 -10.25l-2.421875 0l0 -1.078125l3.78125 0l0 11.328125l2.5 0l0 1.078125l-6.390625 0z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m306.4252 153.6762l8.062988 35.370087" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m306.4252 153.6762l6.729431 29.520172" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m311.54422 183.56346l2.619049 4.05748l0.6017761 -4.7917023z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m673.7428 143.05284l100.28345 0l0 73.44881l-100.28345 0z" fill-rule="evenodd"/><path fill="#000000" d="m683.5084 169.97284l2.46875 -11.625l1.03125 0l1.671875 5.6875l4.125 -5.703125l0.984375 0l-2.46875 11.640625l-1.234375 0l1.859375 -8.765625l-3.515625 4.6875l-0.5 0l-1.34375 -4.640625l-1.859375 8.71875l-1.21875 0zm13.59375 0.1875q-1.90625 0 -2.78125 -1.15625q-0.875 -1.15625 -0.421875 -3.265625q0.296875 -1.40625 1.015625 -2.421875q0.734375 -1.015625 1.734375 -1.546875q1.015625 -0.53125 2.109375 -0.53125q1.546875 0 2.28125 1.03125q0.75 1.03125 0.328125 3.015625q-0.046875 0.203125 -0.171875 0.625l-6.0625 0q-0.25 1.5625 0.375 2.375q0.625 0.796875 1.859375 0.796875q1.34375 0 2.40625 -0.953125l0.59375 0.71875q-1.359375 1.3125 -3.265625 1.3125zm3.0 -5.296875q0.25 -1.203125 -0.203125 -1.890625q-0.453125 -0.703125 -1.453125 -0.703125q-0.921875 0 -1.765625 0.65625q-0.828125 0.640625 -1.265625 1.9375l4.6875 0zm9.75 4.328125q-1.4375 0.90625 -2.90625 0.90625q-1.421875 0 -1.828125 -0.84375q-0.40625 -0.859375 0.015625 -2.8125q0.0625 -0.3125 0.265625 -1.09375l0.765625 -2.796875l-1.875 0l0.234375 -1.109375l1.953125 0l0.609375 -2.265625l1.546875 -0.25l0.1875 -0.015625l0 0.109375q-0.171875 0.1875 -0.265625 0.328125q-0.09375 0.125 -0.171875 0.40625l-0.5625 1.6875l2.828125 0l-0.234375 1.109375l-2.90625 0l-0.78125 2.890625q-0.203125 0.75 -0.25 0.984375q-0.3125 1.5 -0.09375 2.03125q0.234375 0.53125 1.0 0.53125q0.5625 0 1.078125 -0.203125q0.515625 -0.21875 1.203125 -0.65625l0.1875 1.0625zm7.28125 -7.9375q1.828125 0 2.578125 0.953125q0.765625 0.953125 0.28125 3.234375l-0.96875 4.53125l-1.453125 0l0.28125 -1.3125q-1.109375 1.515625 -3.234375 1.515625q-1.421875 0 -2.078125 -0.640625q-0.65625 -0.640625 -0.4375 -1.703125q0.1875 -0.90625 0.90625 -1.578125q0.734375 -0.671875 1.78125 -1.03125q1.0625 -0.359375 2.234375 -0.359375q1.0625 0 1.890625 0.109375q0.203125 -1.4375 -0.3125 -2.015625q-0.515625 -0.59375 -1.796875 -0.59375q-0.625 0 -1.265625 0.25q-0.640625 0.234375 -1.21875 0.703125l-0.484375 -0.859375q1.453125 -1.203125 3.296875 -1.203125zm-2.171875 7.890625q1.390625 0 2.359375 -0.796875q0.96875 -0.8125 1.375 -2.34375q-0.8125 -0.140625 -1.78125 -0.140625q-1.40625 0 -2.34375 0.46875q-0.921875 0.46875 -1.125 1.421875q-0.296875 1.390625 1.515625 1.390625zm9.40625 1.015625q-0.90625 0 -1.609375 -0.515625q-0.6875 -0.515625 -0.96875 -1.53125q-0.28125 -1.03125 0.03125 -2.484375q0.3125 -1.484375 1.046875 -2.46875q0.734375 -0.984375 1.65625 -1.453125q0.921875 -0.484375 1.828125 -0.484375q0.859375 0 1.40625 0.390625q0.5625 0.390625 0.765625 1.078125l1.09375 -5.125l1.421875 0l-0.03125 0.125q-0.140625 0.125 -0.203125 0.25q-0.046875 0.125 -0.109375 0.453125l-2.171875 10.25q-0.09375 0.453125 -0.125 0.75q-0.03125 0.28125 0.03125 0.578125l-1.328125 0q-0.0625 -0.296875 -0.03125 -0.578125q0.03125 -0.296875 0.125 -0.75q-0.5625 0.71875 -1.296875 1.125q-0.71875 0.390625 -1.53125 0.390625zm0.46875 -1.171875q1.15625 0 1.890625 -0.90625q0.734375 -0.921875 1.0625 -2.421875q0.3125 -1.515625 -0.078125 -2.421875q-0.390625 -0.921875 -1.578125 -0.921875q-1.109375 0 -1.890625 0.84375q-0.78125 0.828125 -1.078125 2.265625q-0.34375 1.640625 0.0625 2.609375q0.421875 0.953125 1.609375 0.953125zm10.953125 -7.734375q1.828125 0 2.578125 0.953125q0.765625 0.953125 0.28125 3.234375l-0.96875 4.53125l-1.453125 0l0.28125 -1.3125q-1.109375 1.515625 -3.234375 1.515625q-1.421875 0 -2.078125 -0.640625q-0.65625 -0.640625 -0.4375 -1.703125q0.1875 -0.90625 0.90625 -1.578125q0.734375 -0.671875 1.78125 -1.03125q1.0625 -0.359375 2.234375 -0.359375q1.0625 0 1.890625 0.109375q0.203125 -1.4375 -0.3125 -2.015625q-0.515625 -0.59375 -1.796875 -0.59375q-0.625 0 -1.265625 0.25q-0.640625 0.234375 -1.21875 0.703125l-0.484375 -0.859375q1.453125 -1.203125 3.296875 -1.203125zm-2.171875 7.890625q1.390625 0 2.359375 -0.796875q0.96875 -0.8125 1.375 -2.34375q-0.8125 -0.140625 -1.78125 -0.140625q-1.40625 0 -2.34375 0.46875q-0.921875 0.46875 -1.125 1.421875q-0.296875 1.390625 1.515625 1.390625zm13.546875 0.046875q-1.4375 0.90625 -2.90625 0.90625q-1.421875 0 -1.828125 -0.84375q-0.40625 -0.859375 0.015625 -2.8125q0.0625 -0.3125 0.265625 -1.09375l0.765625 -2.796875l-1.875 0l0.234375 -1.109375l1.953125 0l0.609375 -2.265625l1.546875 -0.25l0.1875 -0.015625l0 0.109375q-0.171875 0.1875 -0.265625 0.328125q-0.09375 0.125 -0.171875 0.40625l-0.5625 1.6875l2.828125 0l-0.234375 1.109375l-2.90625 0l-0.78125 2.890625q-0.203125 0.75 -0.25 0.984375q-0.3125 1.5 -0.09375 2.03125q0.234375 0.53125 1.0 0.53125q0.5625 0 1.078125 -0.203125q0.515625 -0.21875 1.203125 -0.65625l0.1875 1.0625zm7.28125 -7.9375q1.828125 0 2.578125 0.953125q0.765625 0.953125 0.28125 3.234375l-0.96875 4.53125l-1.453125 0l0.28125 -1.3125q-1.109375 1.515625 -3.234375 1.515625q-1.421875 0 -2.078125 -0.640625q-0.65625 -0.640625 -0.4375 -1.703125q0.1875 -0.90625 0.90625 -1.578125q0.734375 -0.671875 1.78125 -1.03125q1.0625 -0.359375 2.234375 -0.359375q1.0625 0 1.890625 0.109375q0.203125 -1.4375 -0.3125 -2.015625q-0.515625 -0.59375 -1.796875 -0.59375q-0.625 0 -1.265625 0.25q-0.640625 0.234375 -1.21875 0.703125l-0.484375 -0.859375q1.453125 -1.203125 3.296875 -1.203125zm-2.171875 7.890625q1.390625 0 2.359375 -0.796875q0.96875 -0.8125 1.375 -2.34375q-0.8125 -0.140625 -1.78125 -0.140625q-1.40625 0 -2.34375 0.46875q-0.921875 0.46875 -1.125 1.421875q-0.296875 1.390625 1.515625 1.390625z" fill-rule="nonzero"/><path fill="#000000" d="m683.5084 191.97284l2.46875 -11.625l1.03125 0l1.671875 5.6875l4.125 -5.703125l0.984375 0l-2.46875 11.640625l-1.234375 0l1.859375 -8.765625l-3.515625 4.6875l-0.5 0l-1.34375 -4.640625l-1.859375 8.71875l-1.21875 0zm12.640625 0.203125q-1.34375 0 -1.9375 -1.0q-0.59375 -1.0 -0.15625 -2.96875l1.046875 -4.765625l1.296875 0l-1.015625 4.765625q-0.328125 1.53125 0.0625 2.21875q0.390625 0.671875 1.265625 0.671875q0.96875 0 1.796875 -0.75q0.828125 -0.765625 1.125 -2.203125l1.0 -4.703125l1.3125 0l-1.53125 7.203125q-0.09375 0.453125 -0.140625 0.75q-0.03125 0.28125 0.046875 0.578125l-1.296875 0q-0.0625 -0.28125 -0.03125 -0.578125q0.03125 -0.296875 0.125 -0.734375q-0.5625 0.71875 -1.34375 1.125q-0.78125 0.390625 -1.625 0.390625zm13.703125 -0.984375q-1.4375 0.90625 -2.90625 0.90625q-1.421875 0 -1.828125 -0.84375q-0.40625 -0.859375 0.015625 -2.8125q0.0625 -0.3125 0.265625 -1.09375l0.765625 -2.796875l-1.875 0l0.234375 -1.109375l1.953125 0l0.609375 -2.265625l1.546875 -0.25l0.1875 -0.015625l0 0.109375q-0.171875 0.1875 -0.265625 0.328125q-0.09375 0.125 -0.171875 0.40625l-0.5625 1.6875l2.828125 0l-0.234375 1.109375l-2.90625 0l-0.78125 2.890625q-0.203125 0.75 -0.25 0.984375q-0.3125 1.5 -0.09375 2.03125q0.234375 0.53125 1.0 0.53125q0.5625 0 1.078125 -0.203125q0.515625 -0.21875 1.203125 -0.65625l0.1875 1.0625zm7.28125 -7.9375q1.828125 0 2.578125 0.953125q0.765625 0.953125 0.28125 3.234375l-0.96875 4.53125l-1.453125 0l0.28125 -1.3125q-1.109375 1.515625 -3.234375 1.515625q-1.421875 0 -2.078125 -0.640625q-0.65625 -0.640625 -0.4375 -1.703125q0.1875 -0.90625 0.90625 -1.578125q0.734375 -0.671875 1.78125 -1.03125q1.0625 -0.359375 2.234375 -0.359375q1.0625 0 1.890625 0.109375q0.203125 -1.4375 -0.3125 -2.015625q-0.515625 -0.59375 -1.796875 -0.59375q-0.625 0 -1.265625 0.25q-0.640625 0.234375 -1.21875 0.703125l-0.484375 -0.859375q1.453125 -1.203125 3.296875 -1.203125zm-2.171875 7.890625q1.390625 0 2.359375 -0.796875q0.96875 -0.8125 1.375 -2.34375q-0.8125 -0.140625 -1.78125 -0.140625q-1.40625 0 -2.34375 0.46875q-0.921875 0.46875 -1.125 1.421875q-0.296875 1.390625 1.515625 1.390625zm13.546875 0.046875q-1.4375 0.90625 -2.90625 0.90625q-1.421875 0 -1.828125 -0.84375q-0.40625 -0.859375 0.015625 -2.8125q0.0625 -0.3125 0.265625 -1.09375l0.765625 -2.796875l-1.875 0l0.234375 -1.109375l1.953125 0l0.609375 -2.265625l1.546875 -0.25l0.1875 -0.015625l0 0.109375q-0.171875 0.1875 -0.265625 0.328125q-0.09375 0.125 -0.171875 0.40625l-0.5625 1.6875l2.828125 0l-0.234375 1.109375l-2.90625 0l-0.78125 2.890625q-0.203125 0.75 -0.25 0.984375q-0.3125 1.5 -0.09375 2.03125q0.234375 0.53125 1.0 0.53125q0.5625 0 1.078125 -0.203125q0.515625 -0.21875 1.203125 -0.65625l0.1875 1.0625zm2.734375 0.78125l0.234375 -1.078125l2.1875 0l1.34375 -6.359375l-2.0625 0l0.234375 -1.09375l3.40625 0l-1.578125 7.453125l2.0 0l-0.234375 1.078125l-5.53125 0zm4.96875 -10.3125q-0.390625 0 -0.609375 -0.28125q-0.21875 -0.28125 -0.140625 -0.671875q0.09375 -0.40625 0.421875 -0.6875q0.328125 -0.28125 0.734375 -0.28125q0.390625 0 0.625 0.296875q0.234375 0.28125 0.140625 0.671875q-0.078125 0.390625 -0.4375 0.671875q-0.34375 0.28125 -0.734375 0.28125zm7.140625 10.46875q-1.140625 0 -1.921875 -0.5625q-0.78125 -0.5625 -1.0625 -1.5625q-0.28125 -1.015625 -0.015625 -2.296875q0.28125 -1.296875 0.984375 -2.296875q0.71875 -1.015625 1.734375 -1.578125q1.03125 -0.578125 2.171875 -0.578125q1.125 0 1.890625 0.578125q0.78125 0.5625 1.0625 1.578125q0.296875 1.0 0.015625 2.296875q-0.265625 1.28125 -0.984375 2.296875q-0.71875 1.0 -1.734375 1.5625q-1.015625 0.5625 -2.140625 0.5625zm0.234375 -1.125q0.71875 0 1.375 -0.421875q0.671875 -0.4375 1.15625 -1.1875q0.484375 -0.765625 0.6875 -1.71875q0.3125 -1.453125 -0.203125 -2.375q-0.515625 -0.921875 -1.609375 -0.921875q-1.109375 0 -2.015625 0.921875q-0.90625 0.921875 -1.21875 2.375q-0.203125 0.953125 -0.03125 1.71875q0.171875 0.75 0.640625 1.1875q0.484375 0.421875 1.21875 0.421875zm7.609375 -7.5625l1.3125 0l-0.328125 1.515625q0.671875 -0.78125 1.53125 -1.25q0.875 -0.46875 1.734375 -0.46875q1.125 0 1.625 0.875q0.5 0.859375 0.109375 2.6875l-1.09375 5.171875l-1.3125 0l1.09375 -5.125q0.265625 -1.28125 -0.046875 -1.859375q-0.296875 -0.578125 -1.0 -0.578125q-0.578125 0 -1.234375 0.34375q-0.65625 0.328125 -1.171875 0.9375q-0.515625 0.609375 -0.671875 1.375l-1.046875 4.90625l-1.3125 0l1.8125 -8.53125z" fill-rule="nonzero"/><path fill="#000000" d="m689.08655 208.95721q1.515625 0.484375 2.046875 1.0625q0.53125 0.5625 0.34375 1.46875q-0.25 1.15625 -1.3125 1.921875q-1.0625 0.75 -2.78125 0.75q-2.171875 0 -3.328125 -1.34375l1.015625 -1.265625l0.015625 -0.015625l0.015625 0.015625q0.421875 0.75 0.953125 1.125q0.546875 0.359375 1.609375 0.359375q1.015625 0 1.671875 -0.359375q0.65625 -0.359375 0.78125 -1.0q0.109375 -0.53125 -0.28125 -0.890625q-0.390625 -0.359375 -1.515625 -0.75q-3.015625 -0.90625 -2.65625 -2.640625q0.21875 -1.0 1.15625 -1.578125q0.9375 -0.578125 2.453125 -0.578125q1.15625 0 1.875 0.328125q0.71875 0.328125 1.234375 1.015625l-0.96875 0.9375l-0.015625 0.015625q-0.265625 -0.59375 -0.921875 -0.9375q-0.640625 -0.34375 -1.390625 -0.34375q-0.796875 0 -1.375 0.28125q-0.578125 0.28125 -0.671875 0.78125q-0.109375 0.46875 0.328125 0.859375q0.453125 0.390625 1.71875 0.78125zm2.09375 -1.390625q0.015625 -0.09375 0.140625 0.03125l-0.078125 0.0625l-0.0625 -0.09375zm0.21875 -0.03125q0.03125 0.078125 0.015625 0.09375q-0.015625 0.015625 -0.046875 0q-0.03125 -0.015625 -0.046875 -0.03125l0.078125 -0.0625zm-6.09375 3.9375q-0.015625 0.078125 -0.171875 0l0.078125 -0.09375l0.09375 0.078125l0 0.015625zm-0.21875 0.0625q-0.046875 -0.09375 -0.03125 -0.09375q0.03125 0 0.078125 0.03125l-0.046875 0.0625z" fill-rule="nonzero"/><path fill="#e06666" d="m424.69684 98.478035l0 0c0 -33.441498 27.10971 -60.55118 60.551178 -60.55118l0 0c16.059174 0 31.460602 6.3794785 42.816193 17.735027c11.35553 11.355553 17.734985 26.75698 17.734985 42.816154l0 0c0 33.44149 -27.10968 60.551186 -60.551178 60.551186l0 0c-33.441467 0 -60.551178 -27.109695 -60.551178 -60.551186z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m424.69684 98.478035l0 0c0 -33.441498 27.10971 -60.55118 60.551178 -60.55118l0 0c16.059174 0 31.460602 6.3794785 42.816193 17.735027c11.35553 11.355553 17.734985 26.75698 17.734985 42.816154l0 0c0 33.44149 -27.10968 60.551186 -60.551178 60.551186l0 0c-33.441467 0 -60.551178 -27.109695 -60.551178 -60.551186z" fill-rule="evenodd"/><path fill="#000000" d="m458.16208 87.97616q1.65625 0.6875 2.28125 1.421875q0.640625 0.734375 0.640625 1.828125q0 0.859375 -0.421875 1.625q-0.40625 0.765625 -1.296875 1.25q-0.875 0.484375 -2.15625 0.484375q-2.265625 0 -3.640625 -1.46875l0.671875 -1.1875l0 -0.015625q0.015625 0 0.015625 0.015625q0 0 0 0q0.515625 0.671875 1.296875 1.078125q0.796875 0.40625 1.84375 0.40625q1.03125 0 1.71875 -0.578125q0.6875 -0.578125 0.6875 -1.421875q0 -0.546875 -0.21875 -0.90625q-0.21875 -0.359375 -0.78125 -0.71875q-0.546875 -0.359375 -1.671875 -0.84375q-1.71875 -0.671875 -2.453125 -1.515625q-0.734375 -0.859375 -0.734375 -1.9375q0 -1.296875 0.953125 -2.078125q0.953125 -0.78125 2.59375 -0.78125q0.953125 0 1.78125 0.390625q0.828125 0.375 1.4375 1.0625l-0.71875 0.96875l-0.015625 0.015625q-0.5625 -0.765625 -1.171875 -1.0625q-0.609375 -0.296875 -1.515625 -0.296875q-0.90625 0 -1.453125 0.5q-0.546875 0.484375 -0.546875 1.1875q0 0.546875 0.234375 0.953125q0.234375 0.390625 0.84375 0.78125q0.625 0.375 1.796875 0.84375zm1.59375 -2.875q0 -0.046875 0.078125 0q0.078125 0.03125 0.09375 0.03125l-0.046875 0.0625l-0.125 -0.0625l0 -0.03125zm0.21875 -0.03125q0.109375 0.15625 0.03125 0.109375q-0.0625 -0.046875 -0.078125 -0.046875l0.046875 -0.0625zm-5.53125 6.796875q0 0.046875 -0.078125 0.015625q-0.0625 -0.046875 -0.09375 -0.046875l0.046875 -0.0625l0.125 0.0625l0 0.03125zm-0.203125 0.046875q-0.109375 -0.140625 0.03125 -0.078125l-0.03125 0.078125zm15.96875 1.703125q-1.234375 0.90625 -2.703125 0.90625q-1.421875 0 -2.015625 -0.84375q-0.578125 -0.859375 -0.578125 -2.8125q0 -0.3125 0.03125 -1.09375l0.171875 -2.796875l-1.875 0l0 -1.109375l1.953125 0l0.125 -2.265625l1.5 -0.25l0.1875 -0.015625l0.015625 0.109375q-0.125 0.1875 -0.203125 0.328125q-0.0625 0.125 -0.078125 0.40625l-0.203125 1.6875l2.828125 0l0 1.109375l-2.90625 0l-0.171875 2.890625q-0.03125 0.75 -0.03125 0.984375q0 1.5 0.34375 2.03125q0.34375 0.53125 1.109375 0.53125q0.5625 0 1.03125 -0.203125q0.46875 -0.21875 1.0625 -0.65625l0.40625 1.0625zm5.59375 -7.9375q1.828125 0 2.78125 0.953125q0.96875 0.953125 0.96875 3.234375l0 4.53125l-1.453125 0l0 -1.3125q-0.78125 1.515625 -2.90625 1.515625q-1.421875 0 -2.21875 -0.640625q-0.796875 -0.640625 -0.796875 -1.703125q0 -0.90625 0.578125 -1.578125q0.578125 -0.671875 1.546875 -1.03125q0.984375 -0.359375 2.15625 -0.359375q1.0625 0 1.921875 0.109375q-0.109375 -1.4375 -0.75 -2.015625q-0.640625 -0.59375 -1.921875 -0.59375q-0.625 0 -1.21875 0.25q-0.578125 0.234375 -1.0625 0.703125l-0.65625 -0.859375q1.1875 -1.203125 3.03125 -1.203125zm-0.484375 7.890625q1.390625 0 2.1875 -0.796875q0.796875 -0.8125 0.875 -2.34375q-0.84375 -0.140625 -1.8125 -0.140625q-1.40625 0 -2.234375 0.46875q-0.828125 0.46875 -0.828125 1.421875q0 1.390625 1.8125 1.390625zm6.609375 -7.703125l1.3125 0l0 1.515625q0.5 -0.78125 1.265625 -1.25q0.765625 -0.46875 1.625 -0.46875q1.125 0 1.8125 0.875q0.6875 0.859375 0.6875 2.6875l0 5.171875l-1.3125 0l0 -5.125q0 -1.28125 -0.4375 -1.859375q-0.421875 -0.578125 -1.125 -0.578125q-0.578125 0 -1.171875 0.34375q-0.578125 0.328125 -0.96875 0.9375q-0.375 0.609375 -0.375 1.375l0 4.90625l-1.3125 0l0 -8.53125zm12.34375 8.71875q-0.90625 0 -1.71875 -0.515625q-0.796875 -0.515625 -1.296875 -1.53125q-0.5 -1.03125 -0.5 -2.484375q0 -1.484375 0.515625 -2.46875q0.53125 -0.984375 1.34375 -1.453125q0.828125 -0.484375 1.734375 -0.484375q0.859375 0 1.5 0.390625q0.640625 0.390625 0.984375 1.078125l0 -5.125l1.421875 0l0 0.125q-0.109375 0.125 -0.140625 0.25q-0.03125 0.125 -0.03125 0.453125l0.015625 10.25q0 0.453125 0.03125 0.75q0.03125 0.28125 0.15625 0.578125l-1.328125 0q-0.125 -0.296875 -0.15625 -0.578125q-0.03125 -0.296875 -0.03125 -0.75q-0.40625 0.71875 -1.046875 1.125q-0.640625 0.390625 -1.453125 0.390625zm0.21875 -1.171875q1.15625 0 1.6875 -0.90625q0.546875 -0.921875 0.546875 -2.421875q0 -1.515625 -0.59375 -2.421875q-0.578125 -0.921875 -1.765625 -0.921875q-1.109375 0 -1.71875 0.84375q-0.59375 0.828125 -0.59375 2.265625q0 1.640625 0.625 2.609375q0.625 0.953125 1.8125 0.953125zm9.71875 1.1875q-0.765625 0 -1.421875 -0.34375q-0.65625 -0.34375 -1.109375 -0.984375l-0.453125 1.125l-0.859375 0l0 -12.40625l1.53125 0l0 0.125q-0.125 0.125 -0.15625 0.25q-0.015625 0.125 -0.015625 0.453125l0 4.359375q0.40625 -0.6875 1.109375 -1.09375q0.703125 -0.421875 1.421875 -0.421875q1.609375 0 2.5625 1.125q0.953125 1.109375 0.953125 3.265625q0 1.453125 -0.515625 2.484375q-0.5 1.03125 -1.328125 1.546875q-0.8125 0.515625 -1.71875 0.515625zm-0.171875 -1.1875q1.0 0 1.671875 -0.796875q0.671875 -0.796875 0.671875 -2.484375q0 -1.640625 -0.625 -2.46875q-0.625 -0.84375 -1.6875 -0.84375q-1.046875 0 -1.703125 0.9375q-0.640625 0.9375 -0.640625 2.40625q0 3.25 2.3125 3.25zm12.7969055 -7.546875q-0.109375 0.765625 -0.34375 1.515625q-0.234375 0.75 -0.671875 1.984375l-2.125 6.0q-0.421875 1.203125 -1.109375 1.734375q-0.6875305 0.546875 -1.6406555 0.546875q-1.203125 0 -1.96875 -0.734375l0.578125 -0.90625l0.015625 -0.03125l0.03125 0.03125q0.296875 0.28125 0.625 0.40625q0.34375 0.125 0.78125 0.125q0.703125 0 1.1094055 -0.484375q0.40625 -0.484375 0.796875 -1.59375l-3.4844055 -8.59375l1.390625 0l2.6562805 6.828125l1.125 -3.421875q0.40625 -1.296875 0.59375 -1.96875q0.203125 -0.6875 0.296875 -1.4375l1.34375 0zm-7.0469055 10.0625q0 0.03125 -0.046875 0.03125l-0.140625 -0.03125l0.0625 -0.09375l0.125 0.078125l0 0.015625zm-0.21875 0.046875q-0.078125 -0.078125 -0.046875 -0.0625q0.046875 0.015625 0.078125 0.015625l-0.03125 0.046875z" fill-rule="nonzero"/><path fill="#000000" d="m472.02145 116.39803l0 -11.625l1.03125 0l2.875 5.6875l2.921875 -5.703125l0.984375 0l0 11.640625l-1.234375 0l0 -8.765625l-2.515625 4.6875l-0.5 0l-2.34375 -4.640625l0 8.71875l-1.21875 0zm9.5625 -11.625l2.71875 0q1.4375 0 2.265625 0.390625q0.84375 0.375 1.453125 1.203125q1.140625 1.53125 1.140625 4.28125q-0.109375 2.828125 -1.3125 4.3125q-1.1875 1.46875 -3.78125 1.453125l-2.484375 0l0 -11.640625zm2.4375 10.625q3.84375 0 3.84375 -4.671875q-0.03125 -2.375 -0.890625 -3.609375q-0.84375 -1.234375 -2.75 -1.234375l-1.40625 0l0 9.515625l1.203125 0zm11.453125 -5.421875q1.65625 0.6875 2.28125 1.421875q0.640625 0.734375 0.640625 1.828125q0 0.859375 -0.421875 1.625q-0.40625 0.765625 -1.296875 1.25q-0.875 0.484375 -2.15625 0.484375q-2.265625 0 -3.640625 -1.46875l0.671875 -1.1875l0 -0.015625q0.015625 0 0.015625 0.015625q0 0 0 0q0.515625 0.671875 1.296875 1.078125q0.796875 0.40625 1.84375 0.40625q1.03125 0 1.71875 -0.578125q0.6875 -0.578125 0.6875 -1.421875q0 -0.546875 -0.21875 -0.90625q-0.21875 -0.359375 -0.78125 -0.71875q-0.546875 -0.359375 -1.671875 -0.84375q-1.71875 -0.671875 -2.453125 -1.515625q-0.734375 -0.859375 -0.734375 -1.9375q0 -1.296875 0.953125 -2.078125q0.953125 -0.78125 2.59375 -0.78125q0.953125 0 1.78125 0.390625q0.828125 0.375 1.4375 1.0625l-0.71875 0.96875l-0.015625 0.015625q-0.5625 -0.765625 -1.171875 -1.0625q-0.609375 -0.296875 -1.515625 -0.296875q-0.90625 0 -1.453125 0.5q-0.546875 0.484375 -0.546875 1.1875q0 0.546875 0.234375 0.953125q0.234375 0.390625 0.84375 0.78125q0.625 0.375 1.796875 0.84375zm1.59375 -2.875q0 -0.046875 0.078125 0q0.078125 0.03125 0.09375 0.03125l-0.046875 0.0625l-0.125 -0.0625l0 -0.03125zm0.21875 -0.03125q0.109375 0.15625 0.03125 0.109375q-0.0625 -0.046875 -0.078125 -0.046875l0.046875 -0.0625zm-5.53125 6.796875q0 0.046875 -0.078125 0.015625q-0.0625 -0.046875 -0.09375 -0.046875l0.046875 -0.0625l0.125 0.0625l0 0.03125zm-0.203125 0.046875q-0.109375 -0.140625 0.03125 -0.078125l-0.03125 0.078125z" fill-rule="nonzero"/><path fill="#e06666" d="m603.52234 93.125015l0 0c0 -33.441494 27.10968 -60.55118 60.551147 -60.55118l0 0c16.059204 0 31.460632 6.3794823 42.816162 17.735031c11.35553 11.355549 17.735046 26.756977 17.735046 42.81615l0 0c0 33.44149 -27.10968 60.551178 -60.55121 60.551178l0 0c-33.441467 0 -60.551147 -27.109688 -60.551147 -60.551178z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m603.52234 93.125015l0 0c0 -33.441494 27.10968 -60.55118 60.551147 -60.55118l0 0c16.059204 0 31.460632 6.3794823 42.816162 17.735031c11.35553 11.355549 17.735046 26.756977 17.735046 42.81615l0 0c0 33.44149 -27.10968 60.551178 -60.55121 60.551178l0 0c-33.441467 0 -60.551147 -27.109688 -60.551147 -60.551178z" fill-rule="evenodd"/><path fill="#000000" d="m643.7766 89.04501l-1.171875 -3.390625l-3.75 0l-1.1875 3.390625l-1.28125 0l4.296875 -11.828125l0.140625 0l4.296875 11.828125l-1.34375 0zm-4.578125 -4.40625l3.0625 0l-1.53125 -4.4375l-1.53125 4.4375zm11.609375 3.40625q1.28125 0 2.21875 -1.046875l0.78125 0.90625q-1.25 1.34375 -3.078125 1.34375q-1.234375 0 -2.203125 -0.578125q-0.96875 -0.578125 -1.515625 -1.59375q-0.546875 -1.015625 -0.546875 -2.28125q0 -1.265625 0.546875 -2.265625q0.546875 -1.015625 1.515625 -1.59375q0.96875 -0.578125 2.1875 -0.578125q1.03125 0 1.859375 0.421875q0.84375 0.40625 1.375 1.15625l-0.84375 0.828125l-0.015625 0.015625q-0.546875 -0.71875 -1.125 -1.0q-0.5625 -0.296875 -1.390625 -0.296875q-0.734375 0 -1.359375 0.40625q-0.625 0.40625 -1.0 1.140625q-0.375 0.71875 -0.375 1.671875q0 0.953125 0.375 1.71875q0.390625 0.765625 1.0625 1.203125q0.6875 0.421875 1.53125 0.421875zm2.078125 -5.25q0 -0.109375 0.15625 0.015625l-0.0625 0.078125l-0.09375 -0.09375zm0.203125 -0.015625q0.09375 0.125 0.046875 0.09375q-0.046875 -0.046875 -0.09375 -0.0625l0.046875 -0.03125zm9.9375 5.484375q-1.234375 0.90625 -2.703125 0.90625q-1.421875 0 -2.015625 -0.84375q-0.578125 -0.859375 -0.578125 -2.8125q0 -0.3125 0.03125 -1.09375l0.171875 -2.796875l-1.875 0l0 -1.109375l1.953125 0l0.125 -2.265625l1.5 -0.25l0.1875 -0.015625l0.015625 0.109375q-0.125 0.1875 -0.203125 0.328125q-0.0625 0.125 -0.078125 0.40625l-0.203125 1.6875l2.828125 0l0 1.109375l-2.90625 0l-0.171875 2.890625q-0.03125 0.75 -0.03125 0.984375q0 1.5 0.34375 2.03125q0.34375 0.53125 1.109375 0.53125q0.5625 0 1.03125 -0.203125q0.46875 -0.21875 1.0625 -0.65625l0.40625 1.0625zm2.90625 0.78125l0 -1.078125l2.1875 0l0 -6.359375l-2.0625 0l0 -1.09375l3.40625 0l0 7.453125l2.0 0l0 1.078125l-5.53125 0zm2.78125 -10.3125q-0.390625 0 -0.671875 -0.28125q-0.28125 -0.28125 -0.28125 -0.671875q0 -0.40625 0.265625 -0.6875q0.28125 -0.28125 0.6875 -0.28125q0.390625 0 0.671875 0.296875q0.296875 0.28125 0.296875 0.671875q0 0.390625 -0.296875 0.671875q-0.28125 0.28125 -0.671875 0.28125zm13.171875 1.78125q-0.296875 1.515625 -1.25 3.734375l-2.0625 4.796875l-1.046875 0l-3.375 -8.53125l1.34375 0l2.625 6.6875l1.375 -3.15625q0.859375 -1.9375 1.125 -3.53125l1.265625 0zm5.921875 8.71875q-1.90625 0 -3.03125 -1.15625q-1.125 -1.15625 -1.125 -3.265625q0 -1.40625 0.515625 -2.421875q0.515625 -1.015625 1.40625 -1.546875q0.890625 -0.53125 1.984375 -0.53125q1.546875 0 2.5 1.03125q0.96875 1.03125 0.96875 3.015625q0 0.203125 -0.03125 0.625l-6.0625 0q0.078125 1.5625 0.875 2.375q0.796875 0.796875 2.03125 0.796875q1.34375 0 2.203125 -0.953125l0.75 0.71875q-1.078125 1.3125 -2.984375 1.3125zm1.859375 -5.296875q0 -1.203125 -0.609375 -1.890625q-0.59375 -0.703125 -1.59375 -0.703125q-0.921875 0 -1.625 0.65625q-0.6875 0.640625 -0.859375 1.9375l4.6875 0z" fill-rule="nonzero"/><path fill="#000000" d="m650.8469 111.04501l0 -11.625l1.03125 0l2.875 5.6875l2.921875 -5.703125l0.984375 0l0 11.640625l-1.234375 0l0 -8.765625l-2.515625 4.6875l-0.5 0l-2.34375 -4.640625l0 8.71875l-1.21875 0zm9.5625 -11.625l2.71875 0q1.4375 0 2.265625 0.390625q0.84375 0.375 1.453125 1.203125q1.140625 1.53125 1.140625 4.28125q-0.109375 2.828125 -1.3125 4.3125q-1.1875 1.46875 -3.78125 1.453125l-2.484375 0l0 -11.640625zm2.4375 10.625q3.84375 0 3.84375 -4.671875q-0.03125 -2.375 -0.890625 -3.609375q-0.84375 -1.234375 -2.75 -1.234375l-1.40625 0l0 9.515625l1.203125 0zm11.453125 -5.421875q1.65625 0.6875 2.28125 1.421875q0.640625 0.734375 0.640625 1.828125q0 0.859375 -0.421875 1.625q-0.40625 0.765625 -1.296875 1.25q-0.875 0.484375 -2.15625 0.484375q-2.265625 0 -3.640625 -1.46875l0.671875 -1.1875l0 -0.015625q0.015625 0 0.015625 0.015625q0 0 0 0q0.515625 0.671875 1.296875 1.078125q0.796875 0.40625 1.84375 0.40625q1.03125 0 1.71875 -0.578125q0.6875 -0.578125 0.6875 -1.421875q0 -0.546875 -0.21875 -0.90625q-0.21875 -0.359375 -0.78125 -0.71875q-0.546875 -0.359375 -1.671875 -0.84375q-1.71875 -0.671875 -2.453125 -1.515625q-0.734375 -0.859375 -0.734375 -1.9375q0 -1.296875 0.953125 -2.078125q0.953125 -0.78125 2.59375 -0.78125q0.953125 0 1.78125 0.390625q0.828125 0.375 1.4375 1.0625l-0.71875 0.96875l-0.015625 0.015625q-0.5625 -0.765625 -1.171875 -1.0625q-0.609375 -0.296875 -1.515625 -0.296875q-0.90625 0 -1.453125 0.5q-0.546875 0.484375 -0.546875 1.1875q0 0.546875 0.234375 0.953125q0.234375 0.390625 0.84375 0.78125q0.625 0.375 1.796875 0.84375zm1.59375 -2.875q0 -0.046875 0.078125 0q0.078125 0.03125 0.09375 0.03125l-0.046875 0.0625l-0.125 -0.0625l0 -0.03125zm0.21875 -0.03125q0.109375 0.15625 0.03125 0.109375q-0.0625 -0.046875 -0.078125 -0.046875l0.046875 -0.0625zm-5.53125 6.796875q0 0.046875 -0.078125 0.015625q-0.0625 -0.046875 -0.09375 -0.046875l0.046875 -0.0625l0.125 0.0625l0 0.03125zm-0.203125 0.046875q-0.109375 -0.140625 0.03125 -0.078125l-0.03125 0.078125z" fill-rule="nonzero"/><path fill="#6fa8dc" d="m597.18896 215.2218l100.84253 0l0 55.47882c-50.41858 0 -50.41858 21.345459 -100.84253 10.6727295zm8.3081665 0l0 -7.1924896l100.15381 0l0 55.82686c-3.8070068 0 -7.619446 0.38668823 -7.619446 0.38668823l0 -49.021057zm7.809265 -7.1924896l0 -7.0184784l101.021484 0l0 55.652863c-4.338501 0 -8.676941 0.29000854 -8.676941 0.29000854l0 -48.924393z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m597.18896 215.2218l100.84253 0l0 55.47882c-50.41858 0 -50.41858 21.345459 -100.84253 10.6727295zm8.3081665 0l0 -7.1924896l100.15381 0l0 55.82686c-3.8070068 0 -7.619446 0.38668823 -7.619446 0.38668823m-84.7251 -56.213547l0 -7.0184784l101.021484 0l0 55.652863c-4.338501 0 -8.676941 0.29000854 -8.676941 0.29000854" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m597.18896 281.37335c50.42395 10.6727295 50.42395 -10.6727295 100.84253 -10.6727295l0 -6.4577637c0 0 3.812439 -0.38668823 7.619446 -0.38668823l0 -6.902466c0 0 4.33844 -0.29000854 8.676941 -0.29000854l0 -55.652863l-101.021484 0l0 7.0184784l-7.809265 0l0 7.1924896l-8.3081665 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m597.18896 215.2218l100.84253 0l0 55.47882c-50.41858 0 -50.41858 21.345459 -100.84253 10.6727295zm8.3081665 0l0 -7.1924896l100.15381 0l0 55.82686c-3.8070068 0 -7.619446 0.38668823 -7.619446 0.38668823m-84.7251 -56.213547l0 -7.0184784l101.021484 0l0 55.652863c-4.338501 0 -8.676941 0.29000854 -8.676941 0.29000854" fill-rule="evenodd"/><path fill="#000000" d="m621.69617 244.70195l0 6.609375q0.03125 1.421875 -0.375 2.34375q-0.40625 0.90625 -1.109375 1.34375q-0.703125 0.421875 -1.5625 0.421875q-1.703125 0 -2.765625 -1.28125l0.734375 -0.9375l0.015625 -0.015625l0.03125 0.015625q0.515625 0.5625 0.953125 0.8125q0.4375 0.25 1.0 0.25q0.953125 0 1.375 -0.65625q0.421875 -0.671875 0.421875 -2.265625l0 -6.640625l-2.25 0l0 -1.109375l5.328125 0l0 1.109375l-1.796875 0zm-4.84375 8.421875q0 0.078125 -0.0625 0.078125q-0.0625 -0.015625 -0.109375 -0.0625l0.078125 -0.09375l0.09375 0.078125zm-0.21875 0.0625q-0.09375 -0.09375 -0.03125 -0.078125q0.0625 0.015625 0.078125 0.03125l-0.046875 0.046875zm12.328125 2.1875q-1.140625 0 -2.046875 -0.5625q-0.890625 -0.5625 -1.390625 -1.5625q-0.5 -1.015625 -0.5 -2.296875q0 -1.296875 0.5 -2.296875q0.5 -1.015625 1.390625 -1.578125q0.90625 -0.578125 2.046875 -0.578125q1.125 0 2.015625 0.578125q0.90625 0.5625 1.40625 1.578125q0.5 1.0 0.5 2.296875q0 1.28125 -0.5 2.296875q-0.5 1.0 -1.40625 1.5625q-0.890625 0.5625 -2.015625 0.5625zm0 -1.125q0.71875 0 1.28125 -0.421875q0.578125 -0.4375 0.90625 -1.1875q0.328125 -0.765625 0.328125 -1.71875q0 -1.453125 -0.71875 -2.375q-0.703125 -0.921875 -1.796875 -0.921875q-1.109375 0 -1.828125 0.921875q-0.703125 0.921875 -0.703125 2.375q0 0.953125 0.328125 1.71875q0.328125 0.75 0.890625 1.1875q0.578125 0.421875 1.3125 0.421875zm8.78125 1.171875q-1.34375 0 -2.15625 -1.0q-0.796875 -1.0 -0.78125 -2.96875l0.03125 -4.765625l1.296875 0l0 4.765625q0 1.53125 0.53125 2.21875q0.53125 0.671875 1.40625 0.671875q0.96875 0 1.625 -0.75q0.671875 -0.765625 0.671875 -2.203125l0 -4.703125l1.3125 0l0 7.203125q0 0.453125 0.015625 0.75q0.03125 0.28125 0.171875 0.578125l-1.296875 0q-0.125 -0.28125 -0.15625 -0.578125q-0.03125 -0.296875 -0.03125 -0.734375q-0.40625 0.71875 -1.109375 1.125q-0.6875 0.390625 -1.53125 0.390625zm13.15625 -6.953125l0 0.015625q-0.546875 -0.5 -0.921875 -0.671875q-0.359375 -0.171875 -0.84375 -0.171875q-0.671875 0 -1.265625 0.34375q-0.59375 0.328125 -0.96875 1.015625q-0.375 0.671875 -0.375 1.703125l0 4.53125l-1.359375 0l0 -8.546875l1.40625 0l-0.046875 1.578125q0.359375 -0.84375 1.09375 -1.3125q0.734375 -0.46875 1.609375 -0.46875q1.375 0 2.265625 0.9375l-0.59375 1.046875zm0 0.015625q0.125 0.109375 0.0625 0.09375q-0.0625 -0.015625 -0.109375 -0.03125l0.046875 -0.0625zm-0.203125 0.0625q0 -0.0625 0.046875 -0.046875q0.0625 0 0.109375 0.046875l-0.03125 0.09375l-0.125 -0.078125l0 -0.015625zm2.921875 -1.859375l1.3125 0l0 1.515625q0.5 -0.78125 1.265625 -1.25q0.765625 -0.46875 1.625 -0.46875q1.125 0 1.8125 0.875q0.6875 0.859375 0.6875 2.6875l0 5.171875l-1.3125 0l0 -5.125q0 -1.28125 -0.4375 -1.859375q-0.421875 -0.578125 -1.125 -0.578125q-0.578125 0 -1.171875 0.34375q-0.578125 0.328125 -0.96875 0.9375q-0.375 0.609375 -0.375 1.375l0 4.90625l-1.3125 0l0 -8.53125zm12.53125 -0.1875q1.828125 0 2.78125 0.953125q0.96875 0.953125 0.96875 3.234375l0 4.53125l-1.453125 0l0 -1.3125q-0.78125 1.515625 -2.90625 1.515625q-1.421875 0 -2.21875 -0.640625q-0.796875 -0.640625 -0.796875 -1.703125q0 -0.90625 0.578125 -1.578125q0.578125 -0.671875 1.546875 -1.03125q0.984375 -0.359375 2.15625 -0.359375q1.0625 0 1.921875 0.109375q-0.109375 -1.4375 -0.75 -2.015625q-0.640625 -0.59375 -1.921875 -0.59375q-0.625 0 -1.21875 0.25q-0.578125 0.234375 -1.0625 0.703125l-0.65625 -0.859375q1.1875 -1.203125 3.03125 -1.203125zm-0.484375 7.890625q1.390625 0 2.1875 -0.796875q0.796875 -0.8125 0.875 -2.34375q-0.84375 -0.140625 -1.8125 -0.140625q-1.40625 0 -2.234375 0.46875q-0.828125 0.46875 -0.828125 1.421875q0 1.390625 1.8125 1.390625zm6.734375 0.828125l0 -1.078125l2.53125 0l0 -10.25l-2.421875 0l0 -1.078125l3.78125 0l0 11.328125l2.5 0l0 1.078125l-6.390625 0z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m664.0735 153.6762l-0.25195312 47.338577" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m664.0735 153.6762l-0.22003174 41.33867" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m662.2017 195.00607l1.6275635 4.546829l1.6759033 -4.529236z" fill-rule="evenodd"/><path fill="#e6b8af" d="m3.406824 307.91898l777.6063 0l0 169.13385l-777.6063 0z" fill-rule="evenodd"/><path fill="#000000" d="m344.085 381.96402l7.890625 0q4.46875 0 6.59375 1.96875q2.125 1.953125 2.125 5.46875q0 1.375 -0.609375 2.75q-0.59375 1.359375 -1.671875 2.40625q-1.078125 1.03125 -2.484375 1.484375l5.828125 10.84375l-5.03125 0l-5.15625 -10.3125l-2.90625 0l0 10.3125l-4.578125 0l0 -24.921875zm8.171875 10.796875q1.921875 0 2.875 -0.890625q0.96875 -0.90625 0.96875 -2.46875q0 -1.75 -0.9375 -2.609375q-0.921875 -0.859375 -2.90625 -0.859375l-3.59375 0l0 6.828125l3.59375 0zm24.921875 14.125l-1.78125 -6.453125l-6.375 0l-1.8125 6.453125l-4.609375 0l8.328125 -24.921875l2.609375 0l8.328125 24.921875l-4.6875 0zm-7.265625 -9.546875l4.625 0l-2.28125 -8.28125l-2.34375 8.28125zm14.0625 -15.375l6.4375 0q3.1875 0 5.0625 0.859375q1.890625 0.84375 3.21875 2.609375q2.5625 3.484375 2.5625 9.171875q0 5.890625 -2.75 9.09375q-2.734375 3.1875 -8.578125 3.1875l-5.953125 0l0 -24.921875zm5.875 21.40625q3.5 0 5.203125 -2.203125q1.703125 -2.21875 1.703125 -6.453125q0 -9.015625 -6.5 -9.015625l-1.9375 0l0 17.671875l1.53125 0zm22.484375 4.234375q-2.953125 0 -5.078125 -1.5625q-2.109375 -1.578125 -3.234375 -4.546875q-1.125 -2.984375 -1.125 -7.125q0 -4.09375 1.125 -7.015625q1.125 -2.9375 3.234375 -4.484375q2.125 -1.546875 5.078125 -1.546875q2.9375 0 5.0625 1.546875q2.125 1.546875 3.234375 4.484375q1.125 2.921875 1.125 7.015625q0 4.140625 -1.125 7.125q-1.109375 2.96875 -3.234375 4.546875q-2.125 1.5625 -5.0625 1.5625zm0 -3.59375q1.484375 0 2.5625 -1.21875q1.078125 -1.21875 1.65625 -3.453125q0.578125 -2.234375 0.578125 -5.203125q0 -2.765625 -0.578125 -4.84375q-0.578125 -2.078125 -1.65625 -3.21875q-1.078125 -1.140625 -2.5625 -1.140625q-1.46875 0 -2.5625 1.140625q-1.09375 1.140625 -1.671875 3.21875q-0.578125 2.078125 -0.578125 4.84375q0 2.96875 0.578125 5.203125q0.578125 2.234375 1.65625 3.453125q1.09375 1.21875 2.578125 1.21875zm21.75 -11.5625q2.546875 1.21875 3.953125 2.34375q1.40625 1.109375 1.984375 2.34375q0.59375 1.234375 0.59375 2.875q0 1.984375 -0.921875 3.71875q-0.921875 1.734375 -2.90625 2.8125q-1.984375 1.0625 -5.03125 1.0625q-4.859375 0 -8.4375 -3.46875l1.953125 -3.40625l0.03125 -0.03125l0.03125 0.03125q2.90625 3.03125 6.796875 3.03125q1.875 0 2.890625 -0.875q1.03125 -0.890625 1.03125 -2.640625q0 -0.875 -0.4375 -1.59375q-0.4375 -0.71875 -1.5 -1.4375q-1.0625 -0.71875 -2.9375 -1.5625q-3.484375 -1.5 -5.046875 -3.375q-1.5625 -1.875 -1.5625 -4.359375q0 -1.796875 1.0 -3.3125q1.015625 -1.515625 2.78125 -2.390625q1.765625 -0.890625 3.984375 -0.890625q2.46875 0 4.265625 0.828125q1.796875 0.8125 3.4375 2.53125l-2.234375 3.125l-0.03125 0.03125q-0.015625 0 -0.015625 0q0 -0.015625 0 -0.015625q-0.921875 -1.46875 -2.34375 -2.140625q-1.421875 -0.671875 -3.125 -0.671875q-1.09375 0 -1.890625 0.390625q-0.78125 0.375 -1.1875 1.0q-0.390625 0.625 -0.390625 1.359375q0 0.875 0.453125 1.5625q0.453125 0.671875 1.5625 1.390625q1.125 0.703125 3.25 1.734375zm3.140625 -4.671875q0 -0.03125 0.046875 -0.03125q0.03125 0 0.15625 0.09375q0.140625 0.09375 0.234375 0.15625l-0.0625 0.09375l-0.375 -0.25l0 -0.0625zm0.546875 0.0625q0.140625 0.21875 0.140625 0.28125q0 0.015625 -0.015625 0.015625q-0.0625 0 -0.234375 -0.140625l0.109375 -0.15625zm-11.90625 12.875q0 0.03125 -0.046875 0.03125q-0.046875 0 -0.203125 -0.078125q-0.140625 -0.078125 -0.234375 -0.109375l0.078125 -0.140625l0.390625 0.234375l0.015625 0.0625zm-0.5625 -0.015625q-0.15625 -0.15625 -0.15625 -0.21875q0 0 0 0q0 -0.015625 0.015625 -0.015625l0.21875 0.09375l-0.078125 0.140625z" fill-rule="nonzero"/><path fill="#cfe2f3" d="m168.86089 338.05414l0 0c0 7.6123962 24.68428 13.783478 55.133865 13.783478c30.449585 0 55.13385 -6.1710815 55.13385 -13.783478l0 119.078735c0 7.6124268 -24.684265 13.783478 -55.13385 13.783478c-30.449585 0 -55.133865 -6.171051 -55.133865 -13.783478z" fill-rule="evenodd"/><path fill="#e2edf7" d="m168.86089 338.05414l0 0c0 -7.6123962 24.68428 -13.783447 55.133865 -13.783447c30.449585 0 55.13385 6.171051 55.13385 13.783447l0 0c0 7.6123962 -24.684265 13.783478 -55.13385 13.783478c-30.449585 0 -55.133865 -6.1710815 -55.133865 -13.783478z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m279.1286 338.05414l0 0c0 7.6123962 -24.684265 13.783478 -55.13385 13.783478c-30.449585 0 -55.133865 -6.1710815 -55.133865 -13.783478l0 0c0 -7.6123962 24.68428 -13.783447 55.133865 -13.783447c30.449585 0 55.13385 6.171051 55.13385 13.783447l0 119.078735c0 7.6124268 -24.684265 13.783478 -55.13385 13.783478c-30.449585 0 -55.133865 -6.171051 -55.133865 -13.783478l0 -119.078735" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m279.1286 338.05414l0 0c0 7.6123962 -24.684265 13.783478 -55.13385 13.783478c-30.449585 0 -55.133865 -6.1710815 -55.133865 -13.783478l0 0c0 -7.6123962 24.68428 -13.783447 55.133865 -13.783447c30.449585 0 55.13385 6.171051 55.13385 13.783447l0 119.078735c0 7.6124268 -24.684265 13.783478 -55.13385 13.783478c-30.449585 0 -55.133865 -6.171051 -55.133865 -13.783478l0 -119.078735" fill-rule="evenodd"/><path fill="#000000" d="m183.01819 399.78024l2.71875 0q1.4375 0 2.265625 0.390625q0.84375 0.375 1.453125 1.203125q1.140625 1.53125 1.140625 4.28125q-0.109375 2.828125 -1.3125 4.3125q-1.1875 1.46875 -3.78125 1.453125l-2.484375 0l0 -11.640625zm2.4375 10.625q3.84375 0 3.84375 -4.671875q-0.03125 -2.375 -0.890625 -3.609375q-0.84375 -1.234375 -2.75 -1.234375l-1.40625 0l0 9.515625l1.203125 0zm10.4375 -7.71875q1.828125 0 2.78125 0.953125q0.96875 0.953125 0.96875 3.234375l0 4.53125l-1.453125 0l0 -1.3125q-0.78125 1.515625 -2.90625 1.515625q-1.421875 0 -2.21875 -0.640625q-0.796875 -0.640625 -0.796875 -1.703125q0 -0.90625 0.578125 -1.578125q0.578125 -0.671875 1.546875 -1.03125q0.984375 -0.359375 2.15625 -0.359375q1.0625 0 1.921875 0.109375q-0.109375 -1.4375 -0.75 -2.015625q-0.640625 -0.59375 -1.921875 -0.59375q-0.625 0 -1.21875 0.25q-0.578125 0.234375 -1.0625 0.703125l-0.65625 -0.859375q1.1875 -1.203125 3.03125 -1.203125zm-0.484375 7.890625q1.390625 0 2.1875 -0.796875q0.796875 -0.8125 0.875 -2.34375q-0.84375 -0.140625 -1.8125 -0.140625q-1.40625 0 -2.234375 0.46875q-0.828125 0.46875 -0.828125 1.421875q0 1.390625 1.8125 1.390625zm13.546875 0.046875q-1.234375 0.90625 -2.703125 0.90625q-1.421875 0 -2.015625 -0.84375q-0.578125 -0.859375 -0.578125 -2.8125q0 -0.3125 0.03125 -1.09375l0.171875 -2.796875l-1.875 0l0 -1.109375l1.953125 0l0.125 -2.265625l1.5 -0.25l0.1875 -0.015625l0.015625 0.109375q-0.125 0.1875 -0.203125 0.328125q-0.0625 0.125 -0.078125 0.40625l-0.203125 1.6875l2.828125 0l0 1.109375l-2.90625 0l-0.171875 2.890625q-0.03125 0.75 -0.03125 0.984375q0 1.5 0.34375 2.03125q0.34375 0.53125 1.109375 0.53125q0.5625 0 1.03125 -0.203125q0.46875 -0.21875 1.0625 -0.65625l0.40625 1.0625zm5.59375 -7.9375q1.828125 0 2.78125 0.953125q0.96875 0.953125 0.96875 3.234375l0 4.53125l-1.453125 0l0 -1.3125q-0.78125 1.515625 -2.90625 1.515625q-1.421875 0 -2.21875 -0.640625q-0.796875 -0.640625 -0.796875 -1.703125q0 -0.90625 0.578125 -1.578125q0.578125 -0.671875 1.546875 -1.03125q0.984375 -0.359375 2.15625 -0.359375q1.0625 0 1.921875 0.109375q-0.109375 -1.4375 -0.75 -2.015625q-0.640625 -0.59375 -1.921875 -0.59375q-0.625 0 -1.21875 0.25q-0.578125 0.234375 -1.0625 0.703125l-0.65625 -0.859375q1.1875 -1.203125 3.03125 -1.203125zm-0.484375 7.890625q1.390625 0 2.1875 -0.796875q0.796875 -0.8125 0.875 -2.34375q-0.84375 -0.140625 -1.8125 -0.140625q-1.40625 0 -2.234375 0.46875q-0.828125 0.46875 -0.828125 1.421875q0 1.390625 1.8125 1.390625zm15.6875 -10.796875l3.734375 0q1.84375 0 2.75 0.921875q0.921875 0.90625 0.921875 2.359375q0 1.4375 -0.890625 2.328125q-0.890625 0.890625 -2.6875 0.890625l-2.484375 0l0 5.125l-1.34375 0l0 -11.625zm3.6875 5.34375q1.21875 0 1.796875 -0.53125q0.578125 -0.53125 0.578125 -1.484375q0 -0.921875 -0.59375 -1.5q-0.59375 -0.59375 -1.765625 -0.59375l-2.359375 0l0 4.109375l2.34375 0zm9.21875 6.4375q-1.140625 0 -2.046875 -0.5625q-0.890625 -0.5625 -1.390625 -1.5625q-0.5 -1.015625 -0.5 -2.296875q0 -1.296875 0.5 -2.296875q0.5 -1.015625 1.390625 -1.578125q0.90625 -0.578125 2.046875 -0.578125q1.125 0 2.015625 0.578125q0.90625 0.5625 1.40625 1.578125q0.5 1.0 0.5 2.296875q0 1.28125 -0.5 2.296875q-0.5 1.0 -1.40625 1.5625q-0.890625 0.5625 -2.015625 0.5625zm0 -1.125q0.71875 0 1.28125 -0.421875q0.578125 -0.4375 0.90625 -1.1875q0.328125 -0.765625 0.328125 -1.71875q0 -1.453125 -0.71875 -2.375q-0.703125 -0.921875 -1.796875 -0.921875q-1.109375 0 -1.828125 0.921875q-0.703125 0.921875 -0.703125 2.375q0 0.953125 0.328125 1.71875q0.328125 0.75 0.890625 1.1875q0.578125 0.421875 1.3125 0.421875zm9.328125 1.125q-1.140625 0 -2.046875 -0.5625q-0.890625 -0.5625 -1.390625 -1.5625q-0.5 -1.015625 -0.5 -2.296875q0 -1.296875 0.5 -2.296875q0.5 -1.015625 1.390625 -1.578125q0.90625 -0.578125 2.046875 -0.578125q1.125 0 2.015625 0.578125q0.90625 0.5625 1.40625 1.578125q0.5 1.0 0.5 2.296875q0 1.28125 -0.5 2.296875q-0.5 1.0 -1.40625 1.5625q-0.890625 0.5625 -2.015625 0.5625zm0 -1.125q0.71875 0 1.28125 -0.421875q0.578125 -0.4375 0.90625 -1.1875q0.328125 -0.765625 0.328125 -1.71875q0 -1.453125 -0.71875 -2.375q-0.703125 -0.921875 -1.796875 -0.921875q-1.109375 0 -1.828125 0.921875q-0.703125 0.921875 -0.703125 2.375q0 0.953125 0.328125 1.71875q0.328125 0.75 0.890625 1.1875q0.578125 0.421875 1.3125 0.421875zm6.125 0.96875l0 -1.078125l2.53125 0l0 -10.25l-2.421875 0l0 -1.078125l3.78125 0l0 11.328125l2.5 0l0 1.078125l-6.390625 0z" fill-rule="nonzero"/><path fill="#cfe2f3" d="m500.87402 338.05414l0 0c0 7.6123962 24.684265 13.783478 55.13385 13.783478c30.449585 0 55.13385 -6.1710815 55.13385 -13.783478l0 119.078735c0 7.6124268 -24.684265 13.783478 -55.13385 13.783478c-30.449585 0 -55.13385 -6.171051 -55.13385 -13.783478z" fill-rule="evenodd"/><path fill="#e2edf7" d="m500.87402 338.05414l0 0c0 -7.6123962 24.684265 -13.783447 55.13385 -13.783447c30.449585 0 55.13385 6.171051 55.13385 13.783447l0 0c0 7.6123962 -24.684265 13.783478 -55.13385 13.783478c-30.449585 0 -55.13385 -6.1710815 -55.13385 -13.783478z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m611.1417 338.05414l0 0c0 7.6123962 -24.684265 13.783478 -55.13385 13.783478c-30.449585 0 -55.13385 -6.1710815 -55.13385 -13.783478l0 0c0 -7.6123962 24.684265 -13.783447 55.13385 -13.783447c30.449585 0 55.13385 6.171051 55.13385 13.783447l0 119.078735c0 7.6124268 -24.684265 13.783478 -55.13385 13.783478c-30.449585 0 -55.13385 -6.171051 -55.13385 -13.783478l0 -119.078735" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m611.1417 338.05414l0 0c0 7.6123962 -24.684265 13.783478 -55.13385 13.783478c-30.449585 0 -55.13385 -6.1710815 -55.13385 -13.783478l0 0c0 -7.6123962 24.684265 -13.783447 55.13385 -13.783447c30.449585 0 55.13385 6.171051 55.13385 13.783447l0 119.078735c0 7.6124268 -24.684265 13.783478 -55.13385 13.783478c-30.449585 0 -55.13385 -6.171051 -55.13385 -13.783478l0 -119.078735" fill-rule="evenodd"/><path fill="#000000" d="m519.461 400.40524l0 -11.625l1.03125 0l2.875 5.6875l2.921875 -5.703125l0.984375 0l0 11.640625l-1.234375 0l0 -8.765625l-2.515625 4.6875l-0.5 0l-2.34375 -4.640625l0 8.71875l-1.21875 0zm13.640625 0.1875q-1.90625 0 -3.03125 -1.15625q-1.125 -1.15625 -1.125 -3.265625q0 -1.40625 0.515625 -2.421875q0.515625 -1.015625 1.40625 -1.546875q0.890625 -0.53125 1.984375 -0.53125q1.546875 0 2.5 1.03125q0.96875 1.03125 0.96875 3.015625q0 0.203125 -0.03125 0.625l-6.0625 0q0.078125 1.5625 0.875 2.375q0.796875 0.796875 2.03125 0.796875q1.34375 0 2.203125 -0.953125l0.75 0.71875q-1.078125 1.3125 -2.984375 1.3125zm1.859375 -5.296875q0 -1.203125 -0.609375 -1.890625q-0.59375 -0.703125 -1.59375 -0.703125q-0.921875 0 -1.625 0.65625q-0.6875 0.640625 -0.859375 1.9375l4.6875 0zm10.671875 4.328125q-1.234375 0.90625 -2.703125 0.90625q-1.421875 0 -2.015625 -0.84375q-0.578125 -0.859375 -0.578125 -2.8125q0 -0.3125 0.03125 -1.09375l0.171875 -2.796875l-1.875 0l0 -1.109375l1.953125 0l0.125 -2.265625l1.5 -0.25l0.1875 -0.015625l0.015625 0.109375q-0.125 0.1875 -0.203125 0.328125q-0.0625 0.125 -0.078125 0.40625l-0.203125 1.6875l2.828125 0l0 1.109375l-2.90625 0l-0.171875 2.890625q-0.03125 0.75 -0.03125 0.984375q0 1.5 0.34375 2.03125q0.34375 0.53125 1.109375 0.53125q0.5625 0 1.03125 -0.203125q0.46875 -0.21875 1.0625 -0.65625l0.40625 1.0625zm5.59375 -7.9375q1.828125 0 2.78125 0.953125q0.96875 0.953125 0.96875 3.234375l0 4.53125l-1.453125 0l0 -1.3125q-0.78125 1.515625 -2.90625 1.515625q-1.421875 0 -2.21875 -0.640625q-0.796875 -0.640625 -0.796875 -1.703125q0 -0.90625 0.578125 -1.578125q0.578125 -0.671875 1.546875 -1.03125q0.984375 -0.359375 2.15625 -0.359375q1.0625 0 1.921875 0.109375q-0.109375 -1.4375 -0.75 -2.015625q-0.640625 -0.59375 -1.921875 -0.59375q-0.625 0 -1.21875 0.25q-0.578125 0.234375 -1.0625 0.703125l-0.65625 -0.859375q1.1875 -1.203125 3.03125 -1.203125zm-0.484375 7.890625q1.390625 0 2.1875 -0.796875q0.796875 -0.8125 0.875 -2.34375q-0.84375 -0.140625 -1.8125 -0.140625q-1.40625 0 -2.234375 0.46875q-0.828125 0.46875 -0.828125 1.421875q0 1.390625 1.8125 1.390625zm9.625 1.015625q-0.90625 0 -1.71875 -0.515625q-0.796875 -0.515625 -1.296875 -1.53125q-0.5 -1.03125 -0.5 -2.484375q0 -1.484375 0.515625 -2.46875q0.53125 -0.984375 1.34375 -1.453125q0.828125 -0.484375 1.734375 -0.484375q0.859375 0 1.5 0.390625q0.640625 0.390625 0.984375 1.078125l0 -5.125l1.421875 0l0 0.125q-0.109375 0.125 -0.140625 0.25q-0.03125 0.125 -0.03125 0.453125l0.015625 10.25q0 0.453125 0.03125 0.75q0.03125 0.28125 0.15625 0.578125l-1.328125 0q-0.125 -0.296875 -0.15625 -0.578125q-0.03125 -0.296875 -0.03125 -0.75q-0.40625 0.71875 -1.046875 1.125q-0.640625 0.390625 -1.453125 0.390625zm0.21875 -1.171875q1.15625 0 1.6875 -0.90625q0.546875 -0.921875 0.546875 -2.421875q0 -1.515625 -0.59375 -2.421875q-0.578125 -0.921875 -1.765625 -0.921875q-1.109375 0 -1.71875 0.84375q-0.59375 0.828125 -0.59375 2.265625q0 1.640625 0.625 2.609375q0.625 0.953125 1.8125 0.953125zm9.296875 -7.734375q1.828125 0 2.78125 0.953125q0.96875 0.953125 0.96875 3.234375l0 4.53125l-1.453125 0l0 -1.3125q-0.78125 1.515625 -2.90625 1.515625q-1.421875 0 -2.21875 -0.640625q-0.796875 -0.640625 -0.796875 -1.703125q0 -0.90625 0.578125 -1.578125q0.578125 -0.671875 1.546875 -1.03125q0.984375 -0.359375 2.15625 -0.359375q1.0625 0 1.921875 0.109375q-0.109375 -1.4375 -0.75 -2.015625q-0.640625 -0.59375 -1.921875 -0.59375q-0.625 0 -1.21875 0.25q-0.578125 0.234375 -1.0625 0.703125l-0.65625 -0.859375q1.1875 -1.203125 3.03125 -1.203125zm-0.484375 7.890625q1.390625 0 2.1875 -0.796875q0.796875 -0.8125 0.875 -2.34375q-0.84375 -0.140625 -1.8125 -0.140625q-1.40625 0 -2.234375 0.46875q-0.828125 0.46875 -0.828125 1.421875q0 1.390625 1.8125 1.390625zm13.546875 0.046875q-1.234375 0.90625 -2.703125 0.90625q-1.421875 0 -2.015625 -0.84375q-0.578125 -0.859375 -0.578125 -2.8125q0 -0.3125 0.03125 -1.09375l0.171875 -2.796875l-1.875 0l0 -1.109375l1.953125 0l0.125 -2.265625l1.5 -0.25l0.1875 -0.015625l0.015625 0.109375q-0.125 0.1875 -0.203125 0.328125q-0.0625 0.125 -0.078125 0.40625l-0.203125 1.6875l2.828125 0l0 1.109375l-2.90625 0l-0.171875 2.890625q-0.03125 0.75 -0.03125 0.984375q0 1.5 0.34375 2.03125q0.34375 0.53125 1.109375 0.53125q0.5625 0 1.03125 -0.203125q0.46875 -0.21875 1.0625 -0.65625l0.40625 1.0625zm5.59375 -7.9375q1.828125 0 2.78125 0.953125q0.96875 0.953125 0.96875 3.234375l0 4.53125l-1.453125 0l0 -1.3125q-0.78125 1.515625 -2.90625 1.515625q-1.421875 0 -2.21875 -0.640625q-0.796875 -0.640625 -0.796875 -1.703125q0 -0.90625 0.578125 -1.578125q0.578125 -0.671875 1.546875 -1.03125q0.984375 -0.359375 2.15625 -0.359375q1.0625 0 1.921875 0.109375q-0.109375 -1.4375 -0.75 -2.015625q-0.640625 -0.59375 -1.921875 -0.59375q-0.625 0 -1.21875 0.25q-0.578125 0.234375 -1.0625 0.703125l-0.65625 -0.859375q1.1875 -1.203125 3.03125 -1.203125zm-0.484375 7.890625q1.390625 0 2.1875 -0.796875q0.796875 -0.8125 0.875 -2.34375q-0.84375 -0.140625 -1.8125 -0.140625q-1.40625 0 -2.234375 0.46875q-0.828125 0.46875 -0.828125 1.421875q0 1.390625 1.8125 1.390625z" fill-rule="nonzero"/><path fill="#000000" d="m538.4454 410.78024l3.734375 0q1.84375 0 2.75 0.921875q0.921875 0.90625 0.921875 2.359375q0 1.4375 -0.890625 2.328125q-0.890625 0.890625 -2.6875 0.890625l-2.484375 0l0 5.125l-1.34375 0l0 -11.625zm3.6875 5.34375q1.21875 0 1.796875 -0.53125q0.578125 -0.53125 0.578125 -1.484375q0 -0.921875 -0.59375 -1.5q-0.59375 -0.59375 -1.765625 -0.59375l-2.359375 0l0 4.109375l2.34375 0zm9.21875 6.4375q-1.140625 0 -2.046875 -0.5625q-0.890625 -0.5625 -1.390625 -1.5625q-0.5 -1.015625 -0.5 -2.296875q0 -1.296875 0.5 -2.296875q0.5 -1.015625 1.390625 -1.578125q0.90625 -0.578125 2.046875 -0.578125q1.125 0 2.015625 0.578125q0.90625 0.5625 1.40625 1.578125q0.5 1.0 0.5 2.296875q0 1.28125 -0.5 2.296875q-0.5 1.0 -1.40625 1.5625q-0.890625 0.5625 -2.015625 0.5625zm0 -1.125q0.71875 0 1.28125 -0.421875q0.578125 -0.4375 0.90625 -1.1875q0.328125 -0.765625 0.328125 -1.71875q0 -1.453125 -0.71875 -2.375q-0.703125 -0.921875 -1.796875 -0.921875q-1.109375 0 -1.828125 0.921875q-0.703125 0.921875 -0.703125 2.375q0 0.953125 0.328125 1.71875q0.328125 0.75 0.890625 1.1875q0.578125 0.421875 1.3125 0.421875zm9.328125 1.125q-1.140625 0 -2.046875 -0.5625q-0.890625 -0.5625 -1.390625 -1.5625q-0.5 -1.015625 -0.5 -2.296875q0 -1.296875 0.5 -2.296875q0.5 -1.015625 1.390625 -1.578125q0.90625 -0.578125 2.046875 -0.578125q1.125 0 2.015625 0.578125q0.90625 0.5625 1.40625 1.578125q0.5 1.0 0.5 2.296875q0 1.28125 -0.5 2.296875q-0.5 1.0 -1.40625 1.5625q-0.890625 0.5625 -2.015625 0.5625zm0 -1.125q0.71875 0 1.28125 -0.421875q0.578125 -0.4375 0.90625 -1.1875q0.328125 -0.765625 0.328125 -1.71875q0 -1.453125 -0.71875 -2.375q-0.703125 -0.921875 -1.796875 -0.921875q-1.109375 0 -1.828125 0.921875q-0.703125 0.921875 -0.703125 2.375q0 0.953125 0.328125 1.71875q0.328125 0.75 0.890625 1.1875q0.578125 0.421875 1.3125 0.421875zm6.125 0.96875l0 -1.078125l2.53125 0l0 -10.25l-2.421875 0l0 -1.078125l3.78125 0l0 11.328125l2.5 0l0 1.078125l-6.390625 0z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m85.40682 135.93997l138.58267 188.34647" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m85.40682 135.93997l135.0268 183.51367" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m219.10321 320.43256l4.0198975 2.676361l-1.3590851 -4.6341553z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m47.26509 193.37305l76.28346 0l0 73.44882l-76.28346 0z" fill-rule="evenodd"/><path fill="#000000" d="m76.1412 213.54305l0 0.015625q-0.453125 -0.5 -0.78125 -0.671875q-0.328125 -0.171875 -0.8125 -0.171875q-0.671875 0 -1.34375 0.34375q-0.65625 0.328125 -1.1875 1.015625q-0.515625 0.671875 -0.734375 1.703125l-0.953125 4.53125l-1.359375 0l1.8125 -8.546875l1.40625 0l-0.375 1.578125q0.53125 -0.84375 1.359375 -1.3125q0.84375 -0.46875 1.71875 -0.46875q1.375 0 2.0625 0.9375l-0.8125 1.046875zm0 0.015625q0.09375 0.109375 0.03125 0.09375q-0.046875 -0.015625 -0.09375 -0.03125l0.0625 -0.0625zm-0.21875 0.0625q0.015625 -0.0625 0.0625 -0.046875q0.046875 0 0.09375 0.046875l-0.0625 0.09375l-0.09375 -0.078125l0 -0.015625zm5.1875 6.859375q-1.90625 0 -2.78125 -1.15625q-0.875 -1.15625 -0.421875 -3.265625q0.296875 -1.40625 1.015625 -2.421875q0.734375 -1.015625 1.734375 -1.546875q1.015625 -0.53125 2.109375 -0.53125q1.546875 0 2.28125 1.03125q0.75 1.03125 0.328125 3.015625q-0.046875 0.203125 -0.171875 0.625l-6.0625 0q-0.25 1.5625 0.375 2.375q0.625 0.796875 1.859375 0.796875q1.34375 0 2.40625 -0.953125l0.59375 0.71875q-1.359375 1.3125 -3.265625 1.3125zm3.0 -5.296875q0.25 -1.203125 -0.203125 -1.890625q-0.453125 -0.703125 -1.453125 -0.703125q-0.921875 0 -1.765625 0.65625q-0.828125 0.640625 -1.265625 1.9375l4.6875 0zm7.703125 -3.609375q1.828125 0 2.578125 0.953125q0.765625 0.953125 0.28125 3.234375l-0.96875 4.53125l-1.453125 0l0.28125 -1.3125q-1.109375 1.515625 -3.234375 1.515625q-1.421875 0 -2.078125 -0.640625q-0.65625 -0.640625 -0.4375 -1.703125q0.1875 -0.90625 0.90625 -1.578125q0.734375 -0.671875 1.78125 -1.03125q1.0625 -0.359375 2.234375 -0.359375q1.0625 0 1.890625 0.109375q0.203125 -1.4375 -0.3125 -2.015625q-0.515625 -0.59375 -1.796875 -0.59375q-0.625 0 -1.265625 0.25q-0.640625 0.234375 -1.21875 0.703125l-0.484375 -0.859375q1.453125 -1.203125 3.296875 -1.203125zm-2.171875 7.890625q1.390625 0 2.359375 -0.796875q0.96875 -0.8125 1.375 -2.34375q-0.8125 -0.140625 -1.78125 -0.140625q-1.40625 0 -2.34375 0.46875q-0.921875 0.46875 -1.125 1.421875q-0.296875 1.390625 1.515625 1.390625zm9.40625 1.015625q-0.90625 0 -1.609375 -0.515625q-0.6875 -0.515625 -0.96875 -1.53125q-0.28125 -1.03125 0.03125 -2.484375q0.3125 -1.484375 1.046875 -2.46875q0.734375 -0.984375 1.65625 -1.453125q0.921875 -0.484375 1.828125 -0.484375q0.859375 0 1.40625 0.390625q0.5625 0.390625 0.765625 1.078125l1.09375 -5.125l1.421875 0l-0.03125 0.125q-0.140625 0.125 -0.203125 0.25q-0.046875 0.125 -0.109375 0.453125l-2.171875 10.25q-0.09375 0.453125 -0.125 0.75q-0.03125 0.28125 0.03125 0.578125l-1.328125 0q-0.0625 -0.296875 -0.03125 -0.578125q0.03125 -0.296875 0.125 -0.75q-0.5625 0.71875 -1.296875 1.125q-0.71875 0.390625 -1.53125 0.390625zm0.46875 -1.171875q1.15625 0 1.890625 -0.90625q0.734375 -0.921875 1.0625 -2.421875q0.3125 -1.515625 -0.078125 -2.421875q-0.390625 -0.921875 -1.578125 -0.921875q-1.109375 0 -1.890625 0.84375q-0.78125 0.828125 -1.078125 2.265625q-0.34375 1.640625 0.0625 2.609375q0.421875 0.953125 1.609375 0.953125z" fill-rule="nonzero"/><path fill="#000000" d="m64.195885 233.7618l1.171875 0l-0.0625 6.859375l2.734375 -6.046875l0.8125 0l0.53125 6.015625q1.28125 -3.484375 1.625 -4.578125q0.359375 -1.09375 0.546875 -1.96875l0.0625 -0.28125l1.25 0q-1.484375 4.328125 -3.21875 8.546875l-1.28125 0l-0.390625 -5.515625l-2.65625 5.515625l-1.21875 0l0.09375 -8.546875zm16.609375 1.78125l0 0.015625q-0.453125 -0.5 -0.78125 -0.671875q-0.328125 -0.171875 -0.8125 -0.171875q-0.671875 0 -1.34375 0.34375q-0.65625 0.328125 -1.1875 1.015625q-0.515625 0.671875 -0.734375 1.703125l-0.953125 4.53125l-1.359375 0l1.8125 -8.546875l1.40625 0l-0.375 1.578125q0.53125 -0.84375 1.359375 -1.3125q0.84375 -0.46875 1.71875 -0.46875q1.375 0 2.0625 0.9375l-0.8125 1.046875zm0 0.015625q0.09375 0.109375 0.03125 0.09375q-0.046875 -0.015625 -0.09375 -0.03125l0.0625 -0.0625zm-0.21875 0.0625q0.015625 -0.0625 0.0625 -0.046875q0.046875 0 0.09375 0.046875l-0.0625 0.09375l-0.09375 -0.078125l0 -0.015625zm2.015625 6.671875l0.234375 -1.078125l2.1875 0l1.34375 -6.359375l-2.0625 0l0.234375 -1.09375l3.40625 0l-1.578125 7.453125l2.0 0l-0.234375 1.078125l-5.53125 0zm4.96875 -10.3125q-0.390625 0 -0.609375 -0.28125q-0.21875 -0.28125 -0.140625 -0.671875q0.09375 -0.40625 0.421875 -0.6875q0.328125 -0.28125 0.734375 -0.28125q0.390625 0 0.625 0.296875q0.234375 0.28125 0.140625 0.671875q-0.078125 0.390625 -0.4375 0.671875q-0.34375 0.28125 -0.734375 0.28125zm10.953125 9.53125q-1.4375 0.90625 -2.90625 0.90625q-1.421875 0 -1.828125 -0.84375q-0.40625 -0.859375 0.015625 -2.8125q0.0625 -0.3125 0.265625 -1.09375l0.765625 -2.796875l-1.875 0l0.234375 -1.109375l1.953125 0l0.609375 -2.265625l1.546875 -0.25l0.1875 -0.015625l0 0.109375q-0.171875 0.1875 -0.265625 0.328125q-0.09375 0.125 -0.171875 0.40625l-0.5625 1.6875l2.828125 0l-0.234375 1.109375l-2.90625 0l-0.78125 2.890625q-0.203125 0.75 -0.25 0.984375q-0.3125 1.5 -0.09375 2.03125q0.234375 0.53125 1.0 0.53125q0.5625 0 1.078125 -0.203125q0.515625 -0.21875 1.203125 -0.65625l0.1875 1.0625zm5.90625 0.96875q-1.90625 0 -2.78125 -1.15625q-0.875 -1.15625 -0.421875 -3.265625q0.296875 -1.40625 1.015625 -2.421875q0.734375 -1.015625 1.734375 -1.546875q1.015625 -0.53125 2.109375 -0.53125q1.546875 0 2.28125 1.03125q0.75 1.03125 0.328125 3.015625q-0.046875 0.203125 -0.171875 0.625l-6.0625 0q-0.25 1.5625 0.375 2.375q0.625 0.796875 1.859375 0.796875q1.34375 0 2.40625 -0.953125l0.59375 0.71875q-1.359375 1.3125 -3.265625 1.3125zm3.0 -5.296875q0.25 -1.203125 -0.203125 -1.890625q-0.453125 -0.703125 -1.453125 -0.703125q-0.921875 0 -1.765625 0.65625q-0.828125 0.640625 -1.265625 1.9375l4.6875 0z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m647.6129 281.37335l-91.590515 42.897644" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m647.6129 281.37335l-86.15698 40.352753" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m560.7554 320.23032l-3.4091187 3.4206238l4.8103027 -0.4290161z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m298.2796 269.3983l257.7323 54.86615" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m298.2796 269.3983l251.86383 53.616882" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m549.7995 324.63068l4.7825317 -0.6706238l-4.0947266 -2.5604248z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m621.2573 135.94116l-65.25983 188.31497" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m621.2573 135.94116l-63.295166 182.64575" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m556.4015 318.04605l0.07470703 4.828766l3.0466309 -3.7470703z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m420.3832 219.50296l161.32285 0l0 52.503937l-161.32285 0z" fill-rule="evenodd"/><path fill="#000000" d="m460.40915 235.90733l-1.40625 6.609375q-0.265625 1.421875 -0.875 2.34375q-0.59375 0.90625 -1.390625 1.34375q-0.796875 0.421875 -1.65625 0.421875q-1.703125 0 -2.484375 -1.28125l0.921875 -0.9375l0.03125 -0.015625l0.015625 0.015625q0.40625 0.5625 0.78125 0.8125q0.390625 0.25 0.953125 0.25q0.953125 0 1.515625 -0.65625q0.5625 -0.671875 0.90625 -2.265625l1.40625 -6.640625l-2.25 0l0.234375 -1.109375l5.328125 0l-0.234375 1.109375l-1.796875 0zm-6.640625 8.421875q-0.015625 0.078125 -0.078125 0.078125q-0.046875 -0.015625 -0.09375 -0.0625l0.109375 -0.09375l0.0625 0.078125zm-0.21875 0.0625q-0.078125 -0.09375 -0.03125 -0.078125q0.0625 0.015625 0.078125 0.03125l-0.046875 0.046875zm11.859375 2.1875q-1.140625 0 -1.921875 -0.5625q-0.78125 -0.5625 -1.0625 -1.5625q-0.28125 -1.015625 -0.015625 -2.296875q0.28125 -1.296875 0.984375 -2.296875q0.71875 -1.015625 1.734375 -1.578125q1.03125 -0.578125 2.171875 -0.578125q1.125 0 1.890625 0.578125q0.78125 0.5625 1.0625 1.578125q0.296875 1.0 0.015625 2.296875q-0.265625 1.28125 -0.984375 2.296875q-0.71875 1.0 -1.734375 1.5625q-1.015625 0.5625 -2.140625 0.5625zm0.234375 -1.125q0.71875 0 1.375 -0.421875q0.671875 -0.4375 1.15625 -1.1875q0.484375 -0.765625 0.6875 -1.71875q0.3125 -1.453125 -0.203125 -2.375q-0.515625 -0.921875 -1.609375 -0.921875q-1.109375 0 -2.015625 0.921875q-0.90625 0.921875 -1.21875 2.375q-0.203125 0.953125 -0.03125 1.71875q0.171875 0.75 0.640625 1.1875q0.484375 0.421875 1.21875 0.421875zm8.53125 1.171875q-1.34375 0 -1.9375 -1.0q-0.59375 -1.0 -0.15625 -2.96875l1.046875 -4.765625l1.296875 0l-1.015625 4.765625q-0.328125 1.53125 0.0625 2.21875q0.390625 0.671875 1.265625 0.671875q0.96875 0 1.796875 -0.75q0.828125 -0.765625 1.125 -2.203125l1.0 -4.703125l1.3125 0l-1.53125 7.203125q-0.09375 0.453125 -0.140625 0.75q-0.03125 0.28125 0.046875 0.578125l-1.296875 0q-0.0625 -0.28125 -0.03125 -0.578125q0.03125 -0.296875 0.125 -0.734375q-0.5625 0.71875 -1.34375 1.125q-0.78125 0.390625 -1.625 0.390625zm14.640625 -6.953125l0 0.015625q-0.453125 -0.5 -0.78125 -0.671875q-0.328125 -0.171875 -0.8125 -0.171875q-0.671875 0 -1.34375 0.34375q-0.65625 0.328125 -1.1875 1.015625q-0.515625 0.671875 -0.734375 1.703125l-0.953125 4.53125l-1.359375 0l1.8125 -8.546875l1.40625 0l-0.375 1.578125q0.53125 -0.84375 1.359375 -1.3125q0.84375 -0.46875 1.71875 -0.46875q1.375 0 2.0625 0.9375l-0.8125 1.046875zm0 0.015625q0.09375 0.109375 0.03125 0.09375q-0.046875 -0.015625 -0.09375 -0.03125l0.0625 -0.0625zm-0.21875 0.0625q0.015625 -0.0625 0.0625 -0.046875q0.046875 0 0.09375 0.046875l-0.0625 0.09375l-0.09375 -0.078125l0 -0.015625zm3.3125 -1.859375l1.3125 0l-0.328125 1.515625q0.671875 -0.78125 1.53125 -1.25q0.875 -0.46875 1.734375 -0.46875q1.125 0 1.625 0.875q0.5 0.859375 0.109375 2.6875l-1.09375 5.171875l-1.3125 0l1.09375 -5.125q0.265625 -1.28125 -0.046875 -1.859375q-0.296875 -0.578125 -1.0 -0.578125q-0.578125 0 -1.234375 0.34375q-0.65625 0.328125 -1.171875 0.9375q-0.515625 0.609375 -0.671875 1.375l-1.046875 4.90625l-1.3125 0l1.8125 -8.53125zm12.578125 -0.1875q1.828125 0 2.578125 0.953125q0.765625 0.953125 0.28125 3.234375l-0.96875 4.53125l-1.453125 0l0.28125 -1.3125q-1.109375 1.515625 -3.234375 1.515625q-1.421875 0 -2.078125 -0.640625q-0.65625 -0.640625 -0.4375 -1.703125q0.1875 -0.90625 0.90625 -1.578125q0.734375 -0.671875 1.78125 -1.03125q1.0625 -0.359375 2.234375 -0.359375q1.0625 0 1.890625 0.109375q0.203125 -1.4375 -0.3125 -2.015625q-0.515625 -0.59375 -1.796875 -0.59375q-0.625 0 -1.265625 0.25q-0.640625 0.234375 -1.21875 0.703125l-0.484375 -0.859375q1.453125 -1.203125 3.296875 -1.203125zm-2.171875 7.890625q1.390625 0 2.359375 -0.796875q0.96875 -0.8125 1.375 -2.34375q-0.8125 -0.140625 -1.78125 -0.140625q-1.40625 0 -2.34375 0.46875q-0.921875 0.46875 -1.125 1.421875q-0.296875 1.390625 1.515625 1.390625zm6.5625 0.828125l0.234375 -1.078125l2.53125 0l2.1719055 -10.25l-2.4219055 0l0.234375 -1.078125l3.7812805 0l-2.40625 11.328125l2.5 0l-0.234375 1.078125l-6.3906555 0zm18.640656 0l2.46875 -11.640625l6.703125 0l-0.234375 1.140625l-5.390625 0l-0.78125 3.65625l4.34375 0l-0.234375 1.140625l-4.34375 0l-1.21875 5.703125l-1.3125 0zm9.34375 0l0.234375 -1.078125l2.53125 0l2.171875 -10.25l-2.421875 0l0.234375 -1.078125l3.78125 0l-2.40625 11.328125l2.5 0l-0.234375 1.078125l-6.390625 0zm11.9375 0.203125q-1.34375 0 -1.9375 -1.0q-0.59375 -1.0 -0.15625 -2.96875l1.046875 -4.765625l1.296875 0l-1.015625 4.765625q-0.328125 1.53125 0.0625 2.21875q0.390625 0.671875 1.265625 0.671875q0.96875 0 1.796875 -0.75q0.828125 -0.765625 1.125 -2.203125l1.0 -4.703125l1.3125 0l-1.53125 7.203125q-0.09375 0.453125 -0.140625 0.75q-0.03125 0.28125 0.046875 0.578125l-1.296875 0q-0.0625 -0.28125 -0.03125 -0.578125q0.03125 -0.296875 0.125 -0.734375q-0.5625 0.71875 -1.34375 1.125q-0.78125 0.390625 -1.625 0.390625zm11.59375 -5.21875q1.515625 0.484375 2.046875 1.0625q0.53125 0.5625 0.34375 1.46875q-0.25 1.15625 -1.3125 1.921875q-1.0625 0.75 -2.78125 0.75q-2.171875 0 -3.328125 -1.34375l1.015625 -1.265625l0.015625 -0.015625l0.015625 0.015625q0.421875 0.75 0.953125 1.125q0.546875 0.359375 1.609375 0.359375q1.015625 0 1.671875 -0.359375q0.65625 -0.359375 0.78125 -1.0q0.109375 -0.53125 -0.28125 -0.890625q-0.390625 -0.359375 -1.515625 -0.75q-3.015625 -0.90625 -2.65625 -2.640625q0.21875 -1.0 1.15625 -1.578125q0.9375 -0.578125 2.453125 -0.578125q1.15625 0 1.875 0.328125q0.71875 0.328125 1.234375 1.015625l-0.96875 0.9375l-0.015625 0.015625q-0.265625 -0.59375 -0.921875 -0.9375q-0.640625 -0.34375 -1.390625 -0.34375q-0.796875 0 -1.375 0.28125q-0.578125 0.28125 -0.671875 0.78125q-0.109375 0.46875 0.328125 0.859375q0.453125 0.390625 1.71875 0.78125zm2.09375 -1.390625q0.015625 -0.09375 0.140625 0.03125l-0.078125 0.0625l-0.0625 -0.09375zm0.21875 -0.03125q0.03125 0.078125 0.015625 0.09375q-0.015625 0.015625 -0.046875 0q-0.03125 -0.015625 -0.046875 -0.03125l0.078125 -0.0625zm-6.09375 3.9375q-0.015625 0.078125 -0.171875 0l0.078125 -0.09375l0.09375 0.078125l0 0.015625zm-0.21875 0.0625q-0.046875 -0.09375 -0.03125 -0.09375q0.03125 0 0.078125 0.03125l-0.046875 0.0625zm10.984375 -9.96875l1.46875 0l-0.03125 0.125q-0.140625 0.125 -0.1875 0.25q-0.046875 0.125 -0.109375 0.453125l-0.984375 4.5625q0.671875 -0.78125 1.53125 -1.25q0.859375 -0.46875 1.640625 -0.46875q1.203125 0 1.703125 0.859375q0.515625 0.859375 0.125 2.703125l-1.09375 5.171875l-1.3125 0l1.09375 -5.125q0.265625 -1.28125 -0.046875 -1.859375q-0.296875 -0.578125 -1.0 -0.578125q-0.59375 0 -1.25 0.328125q-0.640625 0.328125 -1.15625 0.9375q-0.515625 0.609375 -0.671875 1.390625l-1.046875 4.90625l-1.3125 0l2.640625 -12.40625z" fill-rule="nonzero"/><path fill="#ffffff" d="m341.60367 41.995308l0 0c0 -5.917248 4.796875 -10.714127 10.714142 -10.714127l42.855194 0l0 0c2.8415833 0 5.566742 1.1288071 7.57605 3.1380959c2.0092773 2.0092888 3.138092 4.7344666 3.138092 7.5760307l0 42.855217c0 5.917244 -4.796875 10.714119 -10.714142 10.714119l-42.855194 0c-5.917267 0 -10.714142 -4.796875 -10.714142 -10.714119z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m341.60367 41.995308l0 0c0 -5.917248 4.796875 -10.714127 10.714142 -10.714127l42.855194 0l0 0c2.8415833 0 5.566742 1.1288071 7.57605 3.1380959c2.0092773 2.0092888 3.138092 4.7344666 3.138092 7.5760307l0 42.855217c0 5.917244 -4.796875 10.714119 -10.714142 10.714119l-42.855194 0c-5.917267 0 -10.714142 -4.796875 -10.714142 -10.714119z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m371.98032 52.367798l22.11023 22.110237" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m374.66028 55.04773l16.750305 16.750374" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m372.33386 52.72135c0.64242554 -0.6424141 1.68396 -0.6424141 2.3263855 0c0.64242554 0.6424103 0.64242554 1.6839676 0 2.3263817c-0.64242554 0.6424103 -1.68396 0.6424103 -2.3263855 0c-0.642395 -0.6424141 -0.642395 -1.6839714 0 -2.3263817z" fill-rule="nonzero"/><path stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m393.737 74.12448c-0.64242554 0.6424103 -1.68396 0.6424103 -2.3263855 0c-0.642395 -0.6424103 -0.642395 -1.6839676 0 -2.3263855c0.64242554 -0.6424103 1.68396 -0.6424103 2.3263855 0c0.64242554 0.6424179 0.64242554 1.6839752 0 2.3263855z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m374.6601 52.79299l-21.259827 21.259842" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m374.6601 52.792995l-18.579865 18.579906" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m357.24338 72.536095l-2.3263855 2.3263779l-2.326355 -2.3263779l2.326355 -2.3263855l2.3263855 2.3263855z" fill-rule="nonzero"/><path fill="#ffffff" d="m702.1522 41.993996l0 0c0 -5.917248 4.796875 -10.714127 10.714111 -10.714127l42.855225 0l0 0c2.8415527 0 5.5667725 1.1288071 7.57605 3.1380959c2.0092773 2.0092888 3.1380615 4.7344666 3.1380615 7.5760307l0 42.855217c0 5.917244 -4.796875 10.714119 -10.714111 10.714119l-42.855225 0c-5.9172363 0 -10.714111 -4.796875 -10.714111 -10.714119z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m702.1522 41.993996l0 0c0 -5.917248 4.796875 -10.714127 10.714111 -10.714127l42.855225 0l0 0c2.8415527 0 5.5667725 1.1288071 7.57605 3.1380959c2.0092773 2.0092888 3.1380615 4.7344666 3.1380615 7.5760307l0 42.855217c0 5.917244 -4.796875 10.714119 -10.714111 10.714119l-42.855225 0c-5.9172363 0 -10.714111 -4.796875 -10.714111 -10.714119z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m716.458 43.87436l22.11023 22.110237" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m719.13794 46.554295l16.750366 16.750366" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m716.8116 44.227913c0.642395 -0.6424141 1.68396 -0.6424141 2.326355 0c0.642395 0.6424103 0.642395 1.6839676 0 2.3263817c-0.642395 0.6424103 -1.68396 0.6424103 -2.326355 0c-0.64245605 -0.6424141 -0.64245605 -1.6839714 0 -2.3263817z" fill-rule="nonzero"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m738.21466 65.63104c-0.642395 0.6424103 -1.68396 0.6424103 -2.326355 0c-0.642395 -0.6424103 -0.642395 -1.6839676 0 -2.3263817c0.642395 -0.6424141 1.68396 -0.6424141 2.326355 0c0.64245605 0.6424141 0.64245605 1.6839714 0 2.3263817z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m738.14307 61.709003l-21.259888 21.259846" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m738.14307 61.709003l-18.579895 18.579914" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m720.7263 81.4521l-2.326355 2.3263855l-2.326416 -2.3263855l2.326416 -2.3263779l2.326355 2.3263779z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m735.96716 61.34155l21.259888 21.259846" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m735.96716 61.34155l18.579895 18.579914" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m755.71027 78.75827l2.326416 2.3263779l-2.326416 2.3263855l-2.326355 -2.3263855l2.326355 -2.3263779z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m361.7454 127.52721c13.486847 7.3713837 49.210907 36.856964 80.92093 44.228348c31.710052 7.371399 81.85028 8.16481 109.33923 0c27.489014 -8.164795 46.328796 -40.82402 55.594604 -48.988815" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m366.87332 130.64238l0.061187744 0.03933716c0.28182983 0.18249512 0.5683594 0.36921692 0.8595886 0.5599823c2.3294678 1.5260925 4.9551697 3.3113708 7.8257446 5.269394c5.7411804 3.9160461 12.462006 8.523178 19.75232 13.130295c14.580627 9.214233 31.439148 18.428467 47.29419 22.114166c31.709991 7.371399 81.85025 8.16481 109.3392 0c13.744507 -4.0823975 25.326721 -14.288406 34.63098 -24.494415c4.6521606 -5.102997 8.734863 -10.205994 12.233582 -14.543549c0.8746948 -1.0843811 1.7128906 -2.1209412 2.5143433 -3.0976868c0.40075684 -0.4883728 0.79229736 -0.961792 1.1746216 -1.4187622c0.19122314 -0.22849274 0.38006592 -0.45287323 0.5666504 -0.6729584l0.34716797 -0.40646362" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m367.7309 129.23071l-4.7360535 -0.9445038l3.0209045 3.7678375z" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m604.6718 128.2579l1.9227295 -4.4300766l-4.3204956 2.1577148z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m388.79266 166.79956l192.91339 0l0 43.338577l-192.91339 0z" fill-rule="evenodd"/><path fill="#000000" d="m406.72592 193.71956l2.46875 -11.625l1.03125 0l1.671875 5.6875l4.125 -5.703125l0.984375 0l-2.46875 11.640625l-1.234375 0l1.859375 -8.765625l-3.515625 4.6875l-0.5 0l-1.34375 -4.640625l-1.859375 8.71875l-1.21875 0zm13.59375 0.1875q-1.90625 0 -2.78125 -1.15625q-0.875 -1.15625 -0.421875 -3.265625q0.296875 -1.40625 1.015625 -2.421875q0.734375 -1.015625 1.734375 -1.546875q1.015625 -0.53125 2.109375 -0.53125q1.546875 0 2.28125 1.03125q0.75 1.03125 0.328125 3.015625q-0.046875 0.203125 -0.171875 0.625l-6.0625 0q-0.25 1.5625 0.375 2.375q0.625 0.796875 1.859375 0.796875q1.34375 0 2.40625 -0.953125l0.59375 0.71875q-1.359375 1.3125 -3.265625 1.3125zm3.0 -5.296875q0.25 -1.203125 -0.203125 -1.890625q-0.453125 -0.703125 -1.453125 -0.703125q-0.921875 0 -1.765625 0.65625q-0.828125 0.640625 -1.265625 1.9375l4.6875 0zm9.75 4.328125q-1.4375 0.90625 -2.90625 0.90625q-1.421875 0 -1.828125 -0.84375q-0.40625 -0.859375 0.015625 -2.8125q0.0625 -0.3125 0.265625 -1.09375l0.765625 -2.796875l-1.875 0l0.234375 -1.109375l1.953125 0l0.609375 -2.265625l1.546875 -0.25l0.1875 -0.015625l0 0.109375q-0.171875 0.1875 -0.265625 0.328125q-0.09375 0.125 -0.171875 0.40625l-0.5625 1.6875l2.828125 0l-0.234375 1.109375l-2.90625 0l-0.78125 2.890625q-0.203125 0.75 -0.25 0.984375q-0.3125 1.5 -0.09375 2.03125q0.234375 0.53125 1.0 0.53125q0.5625 0 1.078125 -0.203125q0.515625 -0.21875 1.203125 -0.65625l0.1875 1.0625zm7.28125 -7.9375q1.828125 0 2.578125 0.953125q0.765625 0.953125 0.28125 3.234375l-0.96875 4.53125l-1.453125 0l0.28125 -1.3125q-1.109375 1.515625 -3.234375 1.515625q-1.421875 0 -2.078125 -0.640625q-0.65625 -0.640625 -0.4375 -1.703125q0.1875 -0.90625 0.90625 -1.578125q0.734375 -0.671875 1.78125 -1.03125q1.0625 -0.359375 2.234375 -0.359375q1.0625 0 1.890625 0.109375q0.203125 -1.4375 -0.3125 -2.015625q-0.515625 -0.59375 -1.796875 -0.59375q-0.625 0 -1.265625 0.25q-0.640625 0.234375 -1.21875 0.703125l-0.484375 -0.859375q1.453125 -1.203125 3.296875 -1.203125zm-2.171875 7.890625q1.390625 0 2.359375 -0.796875q0.96875 -0.8125 1.375 -2.34375q-0.8125 -0.140625 -1.78125 -0.140625q-1.40625 0 -2.34375 0.46875q-0.921875 0.46875 -1.125 1.421875q-0.296875 1.390625 1.515625 1.390625zm9.40625 1.015625q-0.90625 0 -1.609375 -0.515625q-0.6875 -0.515625 -0.96875 -1.53125q-0.28125 -1.03125 0.03125 -2.484375q0.3125 -1.484375 1.046875 -2.46875q0.734375 -0.984375 1.65625 -1.453125q0.921875 -0.484375 1.828125 -0.484375q0.859375 0 1.40625 0.390625q0.5625 0.390625 0.765625 1.078125l1.09375 -5.125l1.421875 0l-0.03125 0.125q-0.140625 0.125 -0.203125 0.25q-0.046875 0.125 -0.109375 0.453125l-2.171875 10.25q-0.09375 0.453125 -0.125 0.75q-0.03125 0.28125 0.03125 0.578125l-1.328125 0q-0.0625 -0.296875 -0.03125 -0.578125q0.03125 -0.296875 0.125 -0.75q-0.5625 0.71875 -1.296875 1.125q-0.71875 0.390625 -1.53125 0.390625zm0.46875 -1.171875q1.15625 0 1.890625 -0.90625q0.734375 -0.921875 1.0625 -2.421875q0.3125 -1.515625 -0.078125 -2.421875q-0.390625 -0.921875 -1.578125 -0.921875q-1.109375 0 -1.890625 0.84375q-0.78125 0.828125 -1.078125 2.265625q-0.34375 1.640625 0.0625 2.609375q0.421875 0.953125 1.609375 0.953125zm10.953125 -7.734375q1.828125 0 2.578125 0.953125q0.765625 0.953125 0.28125 3.234375l-0.96875 4.53125l-1.453125 0l0.28125 -1.3125q-1.109375 1.515625 -3.234375 1.515625q-1.421875 0 -2.078125 -0.640625q-0.65625 -0.640625 -0.4375 -1.703125q0.1875 -0.90625 0.90625 -1.578125q0.734375 -0.671875 1.78125 -1.03125q1.0625 -0.359375 2.234375 -0.359375q1.0625 0 1.890625 0.109375q0.203125 -1.4375 -0.3125 -2.015625q-0.515625 -0.59375 -1.796875 -0.59375q-0.625 0 -1.265625 0.25q-0.640625 0.234375 -1.21875 0.703125l-0.484375 -0.859375q1.453125 -1.203125 3.296875 -1.203125zm-2.171875 7.890625q1.390625 0 2.359375 -0.796875q0.96875 -0.8125 1.375 -2.34375q-0.8125 -0.140625 -1.78125 -0.140625q-1.40625 0 -2.34375 0.46875q-0.921875 0.46875 -1.125 1.421875q-0.296875 1.390625 1.515625 1.390625zm13.546875 0.046875q-1.4375 0.90625 -2.90625 0.90625q-1.421875 0 -1.828125 -0.84375q-0.40625 -0.859375 0.015625 -2.8125q0.0625 -0.3125 0.265625 -1.09375l0.765625 -2.796875l-1.875 0l0.234375 -1.109375l1.953125 0l0.609375 -2.265625l1.546875 -0.25l0.1875 -0.015625l0 0.109375q-0.171875 0.1875 -0.265625 0.328125q-0.09375 0.125 -0.171875 0.40625l-0.5625 1.6875l2.828125 0l-0.234375 1.109375l-2.90625 0l-0.78125 2.890625q-0.203125 0.75 -0.25 0.984375q-0.3125 1.5 -0.09375 2.03125q0.234375 0.53125 1.0 0.53125q0.5625 0 1.078125 -0.203125q0.515625 -0.21875 1.203125 -0.65625l0.1875 1.0625zm7.28125 -7.9375q1.828125 0 2.578125 0.953125q0.765625 0.953125 0.28125 3.234375l-0.96875 4.53125l-1.453125 0l0.28125 -1.3125q-1.109375 1.515625 -3.234375 1.515625q-1.421875 0 -2.078125 -0.640625q-0.65625 -0.640625 -0.4375 -1.703125q0.1875 -0.90625 0.90625 -1.578125q0.734375 -0.671875 1.78125 -1.03125q1.0625 -0.359375 2.234375 -0.359375q1.0625 0 1.890625 0.109375q0.203125 -1.4375 -0.3125 -2.015625q-0.515625 -0.59375 -1.796875 -0.59375q-0.625 0 -1.265625 0.25q-0.640625 0.234375 -1.21875 0.703125l-0.484375 -0.859375q1.453125 -1.203125 3.296875 -1.203125zm-2.171875 7.890625q1.390625 0 2.359375 -0.796875q0.96875 -0.8125 1.375 -2.34375q-0.8125 -0.140625 -1.78125 -0.140625q-1.40625 0 -2.34375 0.46875q-0.921875 0.46875 -1.125 1.421875q-0.296875 1.390625 1.515625 1.390625zm15.515625 0.828125l2.46875 -11.640625l7.140625 0l-0.234375 1.15625l-5.90625 0l-0.8125 3.828125l4.875 0l-0.265625 1.1875l-4.875 0l-0.90625 4.3125l5.84375 0l-0.25 1.15625l-7.078125 0zm9.109375 0l3.921875 -4.328125l-2.046875 -4.203125l1.46875 0l1.546875 3.15625l2.796875 -3.15625l1.390625 0l-3.640625 4.140625l2.1875 4.390625l-1.515625 0l-1.65625 -3.328125l-2.921875 3.328125l-1.53125 0zm14.0624695 -1.0q1.28125 0 2.4375 -1.046875l0.59375 0.90625q-1.546875 1.34375 -3.375 1.34375q-1.234375 0 -2.0780945 -0.578125q-0.84375 -0.578125 -1.171875 -1.59375q-0.328125 -1.015625 -0.0625 -2.28125q0.265625 -1.265625 1.015625 -2.265625q0.7655945 -1.015625 1.8593445 -1.59375q1.09375 -0.578125 2.3125 -0.578125q1.03125 0 1.78125 0.421875q0.75 0.40625 1.125 1.15625l-1.03125 0.828125l-0.015625 0.015625q-0.390625 -0.71875 -0.90625 -1.0q-0.5 -0.296875 -1.328125 -0.296875q-0.734375 0 -1.453125 0.40625q-0.703125 0.40625 -1.234375 1.140625q-0.53125 0.71875 -0.7343445 1.671875q-0.203125 0.953125 0.015625 1.71875q0.21871948 0.765625 0.8124695 1.203125q0.59375 0.421875 1.4375 0.421875zm3.1875 -5.25q0.03125 -0.109375 0.15625 0.015625l-0.078125 0.078125l-0.078125 -0.09375zm0.203125 -0.015625q0.078125 0.125 0.03125 0.09375q-0.03125 -0.046875 -0.078125 -0.0625l0.046875 -0.03125zm4.328125 -6.140625l1.46875 0l-0.03125 0.125q-0.140625 0.125 -0.1875 0.25q-0.046875 0.125 -0.109375 0.453125l-0.984375 4.5625q0.671875 -0.78125 1.53125 -1.25q0.859375 -0.46875 1.640625 -0.46875q1.203125 0 1.703125 0.859375q0.515625 0.859375 0.125 2.703125l-1.09375 5.171875l-1.3125 0l1.09375 -5.125q0.265625 -1.28125 -0.046875 -1.859375q-0.296875 -0.578125 -1.0 -0.578125q-0.59375 0 -1.25 0.328125q-0.640625 0.328125 -1.15625 0.9375q-0.515625 0.609375 -0.671875 1.390625l-1.046875 4.90625l-1.3125 0l2.640625 -12.40625zm11.734375 3.6875q1.828125 0 2.578125 0.953125q0.765625 0.953125 0.28125 3.234375l-0.96875 4.53125l-1.453125 0l0.28125 -1.3125q-1.109375 1.515625 -3.234375 1.515625q-1.421875 0 -2.078125 -0.640625q-0.65625 -0.640625 -0.4375 -1.703125q0.1875 -0.90625 0.90625 -1.578125q0.734375 -0.671875 1.78125 -1.03125q1.0625 -0.359375 2.234375 -0.359375q1.0625 0 1.890625 0.109375q0.203125 -1.4375 -0.3125 -2.015625q-0.515625 -0.59375 -1.796875 -0.59375q-0.625 0 -1.265625 0.25q-0.640625 0.234375 -1.21875 0.703125l-0.484375 -0.859375q1.453125 -1.203125 3.296875 -1.203125zm-2.171875 7.890625q1.390625 0 2.359375 -0.796875q0.96875 -0.8125 1.375 -2.34375q-0.8125 -0.140625 -1.78125 -0.140625q-1.40625 0 -2.34375 0.46875q-0.921875 0.46875 -1.125 1.421875q-0.296875 1.390625 1.515625 1.390625zm8.25 -7.703125l1.3125 0l-0.328125 1.515625q0.671875 -0.78125 1.53125 -1.25q0.875 -0.46875 1.734375 -0.46875q1.125 0 1.625 0.875q0.5 0.859375 0.109375 2.6875l-1.09375 5.171875l-1.3125 0l1.09375 -5.125q0.265625 -1.28125 -0.046875 -1.859375q-0.296875 -0.578125 -1.0 -0.578125q-0.578125 0 -1.234375 0.34375q-0.65625 0.328125 -1.171875 0.9375q-0.515625 0.609375 -0.671875 1.375l-1.046875 4.90625l-1.3125 0l1.8125 -8.53125zm16.65625 0.875q-0.171875 -0.015625 -0.484375 -0.015625q-0.890625 0 -1.5625 0.390625q0.171875 0.640625 0.015625 1.40625q-0.1875 0.84375 -0.703125 1.53125q-0.5 0.671875 -1.265625 1.0625q-0.765625 0.390625 -1.65625 0.390625q-0.71875 0 -1.265625 -0.265625q-0.421875 0.40625 -0.5 0.78125q-0.09375 0.453125 0.421875 0.65625q0.53125 0.1875 2.015625 0.1875q1.8125 0 2.34375 0.5625q0.546875 0.546875 0.3125 1.609375q-0.234375 1.09375 -1.28125 1.828125q-1.046875 0.734375 -2.953125 0.734375q-1.828125 0 -2.75 -0.5625q-0.90625 -0.546875 -0.6875 -1.640625q0.15625 -0.671875 0.625 -1.15625q0.484375 -0.484375 1.125 -0.765625q-0.53125 -0.40625 -0.375 -1.125q0.15625 -0.75 1.09375 -1.578125q-0.375 -0.40625 -0.5 -0.984375q-0.109375 -0.578125 0.03125 -1.265625q0.171875 -0.828125 0.671875 -1.5q0.515625 -0.6875 1.28125 -1.078125q0.765625 -0.390625 1.65625 -0.390625q1.359375 0 2.0 0.875q0.53125 -0.40625 1.046875 -0.59375q0.515625 -0.203125 1.09375 -0.203125l0.3125 0.015625l-0.0625 1.09375zm-5.40625 3.640625q0.796875 0 1.4375 -0.53125q0.640625 -0.546875 0.8125 -1.328125q0.171875 -0.78125 -0.25 -1.328125q-0.421875 -0.546875 -1.21875 -0.546875q-0.796875 0 -1.453125 0.546875q-0.640625 0.546875 -0.8125 1.328125q-0.171875 0.78125 0.25 1.328125q0.4375 0.53125 1.234375 0.53125zm1.828125 4.859375q0.09375 -0.453125 -0.03125 -0.703125q-0.109375 -0.25 -0.546875 -0.375q-0.421875 -0.125 -1.359375 -0.125q-1.34375 0 -2.09375 -0.234375q-0.515625 0.296875 -0.765625 0.609375q-0.234375 0.3125 -0.359375 0.84375q-0.125 0.640625 0.5 0.984375q0.640625 0.359375 1.90625 0.359375q1.234375 0 1.921875 -0.375q0.6875 -0.359375 0.828125 -0.984375zm7.453125 -0.65625q-1.90625 0 -2.78125 -1.15625q-0.875 -1.15625 -0.421875 -3.265625q0.296875 -1.40625 1.015625 -2.421875q0.734375 -1.015625 1.734375 -1.546875q1.015625 -0.53125 2.109375 -0.53125q1.546875 0 2.28125 1.03125q0.75 1.03125 0.328125 3.015625q-0.046875 0.203125 -0.171875 0.625l-6.0625 0q-0.25 1.5625 0.375 2.375q0.625 0.796875 1.859375 0.796875q1.34375 0 2.40625 -0.953125l0.59375 0.71875q-1.359375 1.3125 -3.265625 1.3125zm3.0 -5.296875q0.25 -1.203125 -0.203125 -1.890625q-0.453125 -0.703125 -1.453125 -0.703125q-0.921875 0 -1.765625 0.65625q-0.828125 0.640625 -1.265625 1.9375l4.6875 0z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m110.45702 4.5455894l171.33856 -0.031496048l0 73.44882l-171.33856 0.03149414z" fill-rule="evenodd"/><path fill="#000000" d="m182.11069 31.608667q-1.140625 2.07901E-4 -1.921875 -0.56214714q-0.78125 -0.56235695 -1.0625 -1.5623055q-0.28125 -1.0155735 -0.015625 -2.2968712q0.28125 -1.2969265 0.984375 -2.2970562q0.71875 -1.0157585 1.734375 -1.5784435q1.03125 -0.57831573 2.171875 -0.57852554q1.125 -2.0599365E-4 1.890625 0.57777786q0.78125 0.56235695 1.0625 1.5779305q0.296875 0.9999447 0.015625 2.2968712q-0.265625 1.2812996 -0.984375 2.2970562q-0.71875 1.0001316 -1.734375 1.5628185q-1.015625 0.5626869 -2.140625 0.5628948zm0.234375 -1.1250439q0.71875 -1.3160706E-4 1.375 -0.42212868q0.671875 -0.43762207 1.15625 -1.1877117q0.484375 -0.76571465 0.6875 -1.7188759q0.3125 -1.4531822 -0.203125 -2.3749638q-0.515625 -0.92177963 -1.609375 -0.92157936q-1.109375 2.040863E-4 -2.015625 0.92224693q-0.90625 0.92204094 -1.21875 2.3752232q-0.203125 0.95316315 -0.03125 1.7187557q0.171875 0.7499695 0.640625 1.1873817q0.484375 0.42178726 1.21875 0.42165184zm7.390625 -7.563858l1.359375 -2.4986267E-4l-0.359375 1.6875648q0.484375 -0.93758774 1.265625 -1.4064827q0.796875 -0.46889687 1.734375 -0.46906853q0.96875 -1.7738342E-4 1.671875 0.49969292q0.703125 0.4998703 0.96875 1.4841976q0.28125 0.9843235 -0.015625 2.406252q-0.3125 1.4375572 -1.046875 2.4689426q-0.71875 1.0313816 -1.65625 1.5628052q-0.9375 0.51579666 -1.875 0.5159683q-0.765625 1.411438E-4 -1.359375 -0.34350014q-0.59375 -0.35926437 -0.890625 -0.99983597l-0.890625 4.234541l-1.375 2.5177002E-4l2.46875 -11.641079zm2.078125 7.5621166q1.03125 -1.8882751E-4 1.890625 -0.76597214q0.859375 -0.7814083 1.234375 -2.531477q0.34375 -1.593813 -0.09375 -2.4374828q-0.4375 -0.8592949 -1.5625 -0.859087q-1.0625 1.9454956E-4 -1.921875 0.89097786q-0.84375 0.8907795 -1.234375 2.703352q-0.28125 1.4688015 0.171875 2.2343426q0.453125 0.765543 1.515625 0.7653465zm9.34375 1.1545334q-1.90625 3.5095215E-4 -2.78125 -1.1557388q-0.875 -1.1560898 -0.421875 -3.2655468q0.296875 -1.4063053 1.015625 -2.422062q0.734375 -1.0157604 1.734375 -1.5471954q1.015625 -0.531435 2.109375 -0.5316372q1.546875 -2.8419495E-4 2.28125 1.0308304q0.75 1.0311127 0.328125 3.0155659q-0.046875 0.20313263 -0.171875 0.6250305l-6.0625 0.001115799q-0.25 1.5625458 0.375 2.3749294q0.625 0.79676056 1.859375 0.7965336q1.34375 -2.4604797E-4 2.40625 -0.9535675l0.59375 0.7186413q-1.359375 1.3127499 -3.265625 1.3131008zm3.0 -5.297426q0.25 -1.2031708 -0.203125 -1.8905888q-0.453125 -0.7030411 -1.453125 -0.702858q-0.921875 1.6975403E-4 -1.765625 0.65657425q-0.828125 0.6407776 -1.265625 1.9377327l4.6875 -8.6021423E-4zm4.453125 -3.4226952l1.3125 -2.4032593E-4l-0.328125 1.515686q0.671875 -0.781374 1.53125 -1.2502823q0.875 -0.46891022 1.734375 -0.46906853q1.125 -2.07901E-4 1.625 0.87470055q0.5 0.85928345 0.109375 2.687481l-1.09375 5.1720753l-1.3125 2.4223328E-4l1.09375 -5.125202q0.265625 -1.2812977 -0.046875 -1.8593655q-0.296875 -0.5780716 -1.0 -0.5779419q-0.578125 1.0681152E-4 -1.234375 0.34397697q-0.65625 0.32824516 -1.171875 0.93771553q-0.515625 0.60947037 -0.671875 1.375124l-1.046875 4.9064426l-1.3125 2.4032593E-4l1.8125 -8.531584z" fill-rule="nonzero"/><path fill="#000000" d="m175.32162 44.922413l1.234375 -2.2506714E-4l-0.171875 0.8437805q0.390625 -0.46882248 0.921875 -0.75016785q0.53125 -0.29697418 1.03125 -0.29706573q0.5625 -1.02996826E-4 0.90625 0.35920715q0.34375 0.35931396 0.34375 0.89056396q0.34375 -0.56256485 0.9375 -0.90642166q0.609375 -0.34386444 1.25 -0.34397888q0.84375 -1.5640259E-4 1.125 0.6091652q0.28125 0.6093254 0.046875 1.5781174l-1.390625 6.5471306l-1.234375 2.2888184E-4l1.28125 -6.0471115q0.234375 -1.062542 0.09375 -1.3750191q-0.140625 -0.3280983 -0.53125 -0.32802582q-0.34375 6.1035156E-5 -0.71875 0.3282585q-0.375 0.31256866 -0.6875 0.8126259q-0.296875 0.4844284 -0.390625 0.96881866l-1.203125 5.6408463l-1.25 2.3269653E-4l1.265625 -5.9377327q0.21875 -1.046917 0.09375 -1.4062691q-0.109375 -0.35935593 -0.65625 -0.35925293q-0.28125 4.9591064E-5 -0.625 0.25011444q-0.34375 0.25006104 -0.640625 0.71886826q-0.28125 0.4531746 -0.40625 1.0469475l-1.203125 5.6877213l-1.234375 2.2888184E-4l1.8125 -8.531586zm11.4375 4.263523l-1.34375 1.031498l-0.6875 3.234501l-1.359375 2.5177002E-4l2.640625 -12.406738l1.53125 -2.784729E-4l-0.03125 0.12500381q-0.140625 0.1250267 -0.203125 0.25003815q-0.046875 0.12500763 -0.109375 0.45314407l-1.484375 6.9690247l5.09375 -3.9540634q0.453125 0.09366608 0.984375 0.09357071l0.484375 -8.773804E-5l-4.3125 3.3757896l3.109375 5.0931816l-1.75 0.078445435l-2.5625 -4.343281zm9.015625 4.4514694q-0.90625 1.6784668E-4 -1.609375 -0.51533127q-0.6875 -0.5154953 -0.96875 -1.5310707q-0.28125 -1.0311966 0.03125 -2.4843788q0.3125 -1.4844322 1.046875 -2.4689445q0.734375 -0.9845085 1.65625 -1.4534302q0.921875 -0.48454285 1.828125 -0.4847107q0.859375 -1.5640259E-4 1.40625 0.3903656q0.5625 0.390522 0.765625 1.0779877l1.09375 -5.125202l1.421875 -2.632141E-4l-0.03125 0.12500763q-0.140625 0.1250267 -0.203125 0.25003815q-0.046875 0.12500763 -0.109375 0.45314407l-2.171875 10.250401q-0.09375 0.45314026 -0.125 0.7500229q-0.03125 0.2812538 0.03125 0.5781174l-1.328125 2.4414062E-4q-0.0625 -0.29686356 -0.03125 -0.5781174q0.03125 -0.29688263 0.125 -0.7500229q-0.5625 0.718853 -1.296875 1.1252365q-0.71875 0.3907585 -1.53125 0.3909073zm0.46875 -1.1719627q1.15625 -2.0980835E-4 1.890625 -0.90659714q0.734375 -0.9220085 1.0625 -2.4220695q0.3125 -1.5156822 -0.078125 -2.4218597q-0.390625 -0.9218025 -1.578125 -0.9215851q-1.109375 2.0217896E-4 -1.890625 0.84409714q-0.78125 0.82826996 -1.078125 2.2658234q-0.34375 1.640686 0.0625 2.6093636q0.421875 0.9530487 1.609375 0.95282745zm6.40625 0.9832001l0.234375 -1.0781708l2.1875 -4.005432E-4l1.34375 -6.359623l-2.0625 3.8146973E-4l0.234375 -1.0937958l3.40625 -6.2561035E-4l-1.578125 7.453415l2.0 -3.6621094E-4l-0.234375 1.078167l-5.53125 0.0010185242zm4.96875 -10.313416q-0.390625 7.247925E-5 -0.609375 -0.28113556q-0.21875 -0.28121185 -0.140625 -0.6718521q0.09375 -0.40626526 0.421875 -0.6875763q0.328125 -0.28131104 0.734375 -0.2813835q0.390625 -7.247925E-5 0.625 0.29675674q0.234375 0.28120804 0.140625 0.6718521q-0.078125 0.39063644 -0.4375 0.6719551q-0.34375 0.28131104 -0.734375 0.2813835zm11.890625 3.5603142l0 0.015625q-0.453125 -0.49991608 -0.78125 -0.67173004q-0.328125 -0.17181396 -0.8125 -0.17172623q-0.671875 1.2207031E-4 -1.34375 0.34399796q-0.65625 0.32824326 -1.1875 1.0158424q-0.515625 0.67197037 -0.734375 1.7032585l-0.953125 4.5314255l-1.359375 2.5177002E-4l1.8125 -8.547211l1.40625 -2.5558472E-4l-0.375 1.5781937q0.53125 -0.8438492 1.359375 -1.3127518q0.84375 -0.4689026 1.71875 -0.46906662q1.375 -2.5177002E-4 2.0625 0.93712234l-0.8125 1.0470238zm0 0.015625q0.09375 0.10935974 0.03125 0.093746185q-0.046875 -0.015617371 -0.09375 -0.031234741l0.0625 -0.062511444zm-0.21875 0.06254196q0.015625 -0.062503815 0.0625 -0.046886444q0.046875 -1.1444092E-5 0.09375 0.046855927l-0.0625 0.093761444l-0.09375 -0.07810593l0 -0.015625z" fill-rule="nonzero"/><path fill="#000000" d="m164.94662 75.455574l0.234375 -1.0781708l2.53125 -4.6539307E-4l2.171875 -10.250397l-2.421875 4.4250488E-4l0.234375 -1.078167l3.78125 -6.942749E-4l-2.40625 11.328564l2.5 -4.5776367E-4l-0.234375 1.0781708l-6.390625 0.0011749268zm9.71875 -0.0017852783l0.234375 -1.0781708l2.1875 -4.043579E-4l1.34375 -6.359619l-2.0625 3.8146973E-4l0.234375 -1.0937958l3.40625 -6.2561035E-4l-1.578125 7.453415l2.0 -3.6621094E-4l-0.234375 1.0781631l-5.53125 0.0010223389zm4.96875 -10.313416q-0.390625 6.866455E-5 -0.609375 -0.28113556q-0.21875 -0.28121185 -0.140625 -0.6718521q0.09375 -0.40626907 0.421875 -0.6875763q0.328125 -0.28131104 0.734375 -0.28138733q0.390625 -7.247925E-5 0.625 0.29676056q0.234375 0.28120804 0.140625 0.6718521q-0.078125 0.39064026 -0.4375 0.6719513q-0.34375 0.28131104 -0.734375 0.28138733zm8.84375 5.29525q1.515625 0.4840927 2.046875 1.0621185q0.53125 0.56240845 0.34375 1.468689q-0.25 1.1562958 -1.3125 1.9221191q-1.0625 0.75019073 -2.78125 0.75051117q-2.171875 3.9672852E-4 -3.328125 -1.3431396l1.015625 -1.2658081l0.015625 -0.01563263l0.015625 0.015625q0.421875 0.7499237 0.953125 1.1248245q0.546875 0.35927582 1.609375 0.35907745q1.015625 -1.8310547E-4 1.671875 -0.35968018q0.65625 -0.35949707 0.78125 -1.000145q0.109375 -0.5312729 -0.28125 -0.8905716q-0.390625 -0.35930634 -1.515625 -0.74972534q-3.015625 -0.90569305 -2.65625 -2.6401367q0.21875 -1.0000381 1.15625 -1.5783386q0.9375 -0.57829285 2.453125 -0.57857513q1.15625 -2.1362305E-4 1.875 0.32778168q0.71875 0.3279953 1.234375 1.0153961l-0.96875 0.9376831l-0.015625 0.015625q-0.265625 -0.5937042 -0.921875 -0.93733215q-0.640625 -0.34362793 -1.390625 -0.3434906q-0.796875 1.449585E-4 -1.375 0.28150177q-0.578125 0.2813568 -0.671875 0.7813721q-0.109375 0.4687729 0.328125 0.85931396q0.453125 0.39054108 1.71875 0.7809372zm2.09375 -1.3910141q0.015625 -0.09375 0.140625 0.031227112l-0.078125 0.06251526l-0.0625 -0.09374237zm0.21875 -0.031288147q0.03125 0.07811737 0.015625 0.09375q-0.015625 0.015625 -0.046875 7.6293945E-6q-0.03125 -0.015617371 -0.046875 -0.03124237l0.078125 -0.06251526zm-6.09375 3.9386215q-0.015625 0.078125 -0.171875 3.0517578E-5l0.078125 -0.09376526l0.09375 0.07810974l0 0.015625zm-0.21875 0.06253815q-0.046875 -0.09374237 -0.03125 -0.09374237q0.03125 -7.6293945E-6 0.078125 0.031234741l-0.046875 0.06250763zm15.4375 1.6534119q-1.4375 0.906517 -2.90625 0.90678406q-1.421875 2.670288E-4 -1.828125 -0.8434143q-0.40625 -0.8592987 0.015625 -2.8125q0.0625 -0.31251526 0.265625 -1.0937958l0.765625 -2.79702l-1.875 3.4332275E-4l0.234375 -1.1094131l1.953125 -3.5858154E-4l0.609375 -2.2657394l1.546875 -0.2502823l0.1875 -0.015663147l0 0.109375q-0.171875 0.18753052 -0.265625 0.32817078q-0.09375 0.12502289 -0.171875 0.40628815l-0.5625 1.6875992l2.828125 -5.187988E-4l-0.234375 1.1094208l-2.90625 5.340576E-4l-0.78125 2.89077q-0.203125 0.7500305 -0.25 0.9844208q-0.3125 1.5000534 -0.09375 2.0312653q0.234375 0.5312042 1.0 0.5310669q0.5625 -1.0681152E-4 1.078125 -0.20332336q0.515625 -0.21884918 1.203125 -0.65647125l0.1875 1.0624619zm5.1875 0.9677963q-0.90625 1.6784668E-4 -1.609375 -0.51532745q-0.6875 -0.5154953 -0.96875 -1.5310745q-0.28125 -1.0311966 0.03125 -2.484375q0.3125 -1.484436 1.046875 -2.4689484q0.734375 -0.9845047 1.65625 -1.4534302q0.921875 -0.48454285 1.828125 -0.4847107q0.859375 -1.5258789E-4 1.40625 0.39037323q0.5625 0.3905182 0.765625 1.07798l1.09375 -5.125202l1.421875 -2.593994E-4l-0.03125 0.12500381q-0.140625 0.1250267 -0.203125 0.25003815q-0.046875 0.12501144 -0.109375 0.4531479l-2.171875 10.250397q-0.09375 0.45314026 -0.125 0.7500229q-0.03125 0.28125763 0.03125 0.5781174l-1.328125 2.4414062E-4q-0.0625 -0.29685974 -0.03125 -0.5781174q0.03125 -0.29688263 0.125 -0.7500229q-0.5625 0.7188568 -1.296875 1.1252365q-0.71875 0.39076233 -1.53125 0.3909073zm0.46875 -1.1719589q1.15625 -2.1362305E-4 1.890625 -0.90660095q0.734375 -0.9220047 1.0625 -2.4220657q0.3125 -1.515686 -0.078125 -2.4218597q-0.390625 -0.92180634 -1.578125 -0.9215851q-1.109375 1.9836426E-4 -1.890625 0.8440933q-0.78125 0.82826996 -1.078125 2.2658234q-0.34375 1.6406937 0.0625 2.6093674q0.421875 0.9530487 1.609375 0.95282745zm6.40625 0.9832001l0.234375 -1.0781708l2.1875 -4.043579E-4l1.34375 -6.359619l-2.0625 3.8146973E-4l0.234375 -1.0937958l3.40625 -6.2561035E-4l-1.578125 7.453415l2.0 -3.6621094E-4l-0.234375 1.0781631l-5.53125 0.0010223389zm4.96875 -10.313416q-0.390625 6.866455E-5 -0.609375 -0.28113556q-0.21875 -0.28121185 -0.140625 -0.6718521q0.09375 -0.40626907 0.421875 -0.6875763q0.328125 -0.28131104 0.734375 -0.28138733q0.390625 -7.247925E-5 0.625 0.29676056q0.234375 0.28120804 0.140625 0.6718521q-0.078125 0.39064026 -0.4375 0.6719513q-0.34375 0.28131104 -0.734375 0.28138733zm11.890625 3.5603104l0 0.015625q-0.453125 -0.49991608 -0.78125 -0.67173004q-0.328125 -0.17181396 -0.8125 -0.17172241q-0.671875 1.2207031E-4 -1.34375 0.34399414q-0.65625 0.32824707 -1.1875 1.0158463q-0.515625 0.67196655 -0.734375 1.7032547l-0.953125 4.5314255l-1.359375 2.5177002E-4l1.8125 -8.547203l1.40625 -2.593994E-4l-0.375 1.5781937q0.53125 -0.8438492 1.359375 -1.3127518q0.84375 -0.4689026 1.71875 -0.46907043q1.375 -2.5177002E-4 2.0625 0.93712616l-0.8125 1.04702zm0 0.015625q0.09375 0.10935974 0.03125 0.09375q-0.046875 -0.015617371 -0.09375 -0.031234741l0.0625 -0.06251526zm-0.21875 0.06254578q0.015625 -0.06250763 0.0625 -0.04689026q0.046875 -7.6293945E-6 0.09375 0.04685974l-0.0625 0.09375763l-0.09375 -0.07810211l0 -0.015625z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m124.98425 40.490856c3.682846 -3.459755 6.026245 -16.741034 22.097107 -20.758532c16.070877 -4.017497 54.685913 -8.257217 74.328094 -3.3464565c19.642166 4.910761 36.270782 27.342522 43.524933 32.811024" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m124.98425 40.490856c3.6828613 -3.459755 6.026245 -16.741034 22.097107 -20.758532c16.070877 -4.017497 54.68593 -8.257218 74.328094 -3.3464565c9.82106 2.4553795 18.888779 9.29101 26.407974 16.196358c3.7595825 3.4526749 7.1320496 6.922779 10.018021 9.871494c0.72146606 0.7371788 1.4125671 1.4417686 2.0716858 2.1053543l0.594635 0.594059" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m259.38858 46.373383l4.46579 1.8382454l-2.23938 -4.27874z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m220.74016 15.71573c65.288284 0 317.17804 -2.5669298 391.7296 0c74.551636 2.566928 46.31671 12.834645 55.58008 15.401574" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m220.74016 15.71573c65.288284 0 317.17804 -2.5669298 391.7296 0c37.27588 1.2834635 48.85504 4.4921255 52.273132 7.7007875c0.42730713 0.401083 0.7270508 0.802166 0.9335327 1.1994877c0.10321045 0.1986618 0.1831665 0.3963833 0.24407959 0.59269524c0.015197754 0.04907608 0.02923584 0.09806633 0.042175293 0.14696121l0.030273438 0.12522316" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m664.44135 26.04712l3.1073608 3.6968708l-0.0040893555 -4.829342z" fill-rule="evenodd"/></g></svg>
\ No newline at end of file
diff --git a/doc/cephfs/cephfs-io-path.rst b/doc/cephfs/cephfs-io-path.rst
new file mode 100644
index 000000000..8c7810ba0
--- /dev/null
+++ b/doc/cephfs/cephfs-io-path.rst
@@ -0,0 +1,50 @@
+=========================
+ Ceph File System IO Path
+=========================
+
+All file data in CephFS is stored as RADOS objects. CephFS clients can directly
+access RADOS to operate on file data. MDS only handles metadata operations.
+
+To read/write a CephFS file, client needs to have 'file read/write' capabilities
+for corresponding inode. If client does not have required capabilities, it sends
+a 'cap message' to MDS, telling MDS what it wants. MDS will issue capabilities
+to client when it is possible. Once client has 'file read/write' capabilities,
+it can directly access RADOS to read/write file data. File data are stored as
+RADOS objects in the form of <inode number>.<object index>. See 'Data Striping'
+section of `Architecture`_ for more information. If the file is only opened by
+one client, MDS also issues 'file cache/buffer' capabilities to the only client.
+The 'file cache' capability means that file read can be satisfied by client
+cache. The 'file buffer' capability means that file write can be buffered in
+client cache.
+
+
+.. ditaa::
+
+            +---------------------+
+            |     Application     |
+            +---------------------+
+              |
+              V
+            +---------------------+  Data IOs  +--------------------+
+            |    CephFS Library   | ---------> |       LibRados     |
+            +---------------------+            +--------------------+
+              |                                  |
+              | Metadata Operations              | Objects Read/Write
+              V                                  V
+            +---------------------+            +--------------------+
+            |         MDSs        | -=-------> |         OSDs       |
+            +---------------------+            +--------------------+
+
+
+            +----------------------+           +---------------------+
+            | CephFS kernel client | Data IOs  | Ceph kernel library |
+            |      (ceph.ko)       | --------> |     (libceph.ko)    |
+            +----------------------+           +---------------------+
+              |                                  |
+              | Metadata Operations              | Objects Read/Write
+              v                                  v
+            +---------------------+            +--------------------+
+            |         MDSs        | -=-------> |         OSDs       |
+            +---------------------+            +--------------------+
+
+.. _Architecture: ../architecture
diff --git a/doc/cephfs/cephfs-journal-tool.rst b/doc/cephfs/cephfs-journal-tool.rst
new file mode 100644
index 000000000..64a113091
--- /dev/null
+++ b/doc/cephfs/cephfs-journal-tool.rst
@@ -0,0 +1,238 @@
+
+cephfs-journal-tool
+===================
+
+Purpose
+-------
+
+If a CephFS journal has become damaged, expert intervention may be required
+to restore the file system to a working state.
+
+The ``cephfs-journal-tool`` utility provides functionality to aid experts in
+examining, modifying, and extracting data from journals.
+
+.. warning::
+
+    This tool is **dangerous** because it directly modifies internal
+    data structures of the file system.  Make backups, be careful, and
+    seek expert advice.  If you are unsure, do not run this tool.
+
+Syntax
+------
+
+::
+
+    cephfs-journal-tool journal <inspect|import|export|reset>
+    cephfs-journal-tool header <get|set>
+    cephfs-journal-tool event <get|splice|apply> [filter] <list|json|summary|binary>
+
+
+The tool operates in three modes: ``journal``, ``header`` and ``event``,
+meaning the whole journal, the header, and the events within the journal
+respectively.
+
+Journal mode
+------------
+
+This should be your starting point to assess the state of a journal.
+
+* ``inspect`` reports on the health of the journal.  This will identify any
+  missing objects or corruption in the stored journal.  Note that this does
+  not identify inconsistencies in the events themselves, just that events are
+  present and can be decoded.
+
+* ``import`` and ``export`` read and write binary dumps of the journal
+  in a sparse file format.  Pass the filename as the last argument.  The
+  export operation may not work reliably for journals which are damaged (missing
+  objects).
+
+* ``reset`` truncates a journal, discarding any information within it.
+
+
+Example: journal inspect
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+    # cephfs-journal-tool journal inspect
+    Overall journal integrity: DAMAGED
+    Objects missing:
+      0x1
+    Corrupt regions:
+      0x400000-ffffffffffffffff
+
+Example: Journal import/export
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+    # cephfs-journal-tool journal export myjournal.bin
+    journal is 4194304~80643
+    read 80643 bytes at offset 4194304
+    wrote 80643 bytes at offset 4194304 to myjournal.bin
+    NOTE: this is a _sparse_ file; you can
+        $ tar cSzf myjournal.bin.tgz myjournal.bin
+          to efficiently compress it while preserving sparseness.
+
+    # cephfs-journal-tool journal import myjournal.bin
+    undump myjournal.bin
+    start 4194304 len 80643
+    writing header 200.00000000
+     writing 4194304~80643
+    done.
+
+.. note::
+
+    It is wise to use the ``journal export <backup file>`` command to make a journal backup
+    before any further manipulation.
+
+Header mode
+-----------
+
+* ``get`` outputs the current content of the journal header
+
+* ``set`` modifies an attribute of the header.  Allowed attributes are
+  ``trimmed_pos``, ``expire_pos`` and ``write_pos``.
+
+Example: header get/set
+~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+    # cephfs-journal-tool header get
+    { "magic": "ceph fs volume v011",
+      "write_pos": 4274947,
+      "expire_pos": 4194304,
+      "trimmed_pos": 4194303,
+      "layout": { "stripe_unit": 4194304,
+          "stripe_count": 4194304,
+          "object_size": 4194304,
+          "cas_hash": 4194304,
+          "object_stripe_unit": 4194304,
+          "pg_pool": 4194304}}
+
+    # cephfs-journal-tool header set trimmed_pos 4194303
+    Updating trimmed_pos 0x400000 -> 0x3fffff
+    Successfully updated header.
+
+
+Event mode
+----------
+
+Event mode allows detailed examination and manipulation of the contents of the journal.  Event
+mode can operate on all events in the journal, or filters may be applied.
+
+The arguments following ``cephfs-journal-tool event`` consist of an action, optional filter
+parameters, and an output mode:
+
+::
+
+    cephfs-journal-tool event <action> [filter] <output>
+
+Actions:
+
+* ``get`` read the events from the log
+* ``splice`` erase events or regions in the journal
+* ``apply`` extract file system metadata from events and attempt to apply it to the metadata store.
+
+Filtering:
+
+* ``--range <int begin>..[int end]`` only include events within the range begin (inclusive) to end (exclusive)
+* ``--path <path substring>`` only include events referring to metadata containing the specified string
+* ``--inode <int>`` only include events referring to metadata containing the specified inode
+* ``--type <type string>`` only include events of this type
+* ``--frag <ino>[.frag id]`` only include events referring to this directory fragment
+* ``--dname <string>`` only include events referring to this named dentry within a directory
+  fragment (may only be used in conjunction with ``--frag``
+* ``--client <int>`` only include events from this client session ID
+
+Filters may be combined on an AND basis (i.e. only the intersection of events from each filter).
+
+Output modes:
+
+* ``binary``: write each event as a binary file, within a folder whose name is controlled by ``--path``
+* ``json``: write all events to a single file, as a JSON serialized list of objects
+* ``summary``: write a human readable summary of the events read to standard out
+* ``list``: write a human readable terse listing of the type of each event, and
+  which file paths the event affects.
+
+
+Example: event mode
+~~~~~~~~~~~~~~~~~~~
+
+::
+
+    # cephfs-journal-tool event get json --path output.json
+    Wrote output to JSON file 'output.json'
+
+    # cephfs-journal-tool event get summary
+    Events by type:
+      NOOP: 2
+      OPEN: 2
+      SESSION: 2
+      SUBTREEMAP: 1
+      UPDATE: 43
+
+    # cephfs-journal-tool event get list
+    0x400000 SUBTREEMAP:  ()
+    0x400308 SESSION:  ()
+    0x4003de UPDATE:  (setattr)
+      /
+    0x40068b UPDATE:  (mkdir)
+      diralpha
+    0x400d1b UPDATE:  (mkdir)
+      diralpha/filealpha1
+    0x401666 UPDATE:  (unlink_local)
+      stray0/10000000001
+      diralpha/filealpha1
+    0x40228d UPDATE:  (unlink_local)
+      diralpha
+      stray0/10000000000
+    0x402bf9 UPDATE:  (scatter_writebehind)
+      stray0
+    0x403150 UPDATE:  (mkdir)
+      dirbravo
+    0x4037e0 UPDATE:  (openc)
+      dirbravo/.filebravo1.swp
+    0x404032 UPDATE:  (openc)
+      dirbravo/.filebravo1.swpx
+
+    # cephfs-journal-tool event get --path filebravo1 list
+    0x40785a UPDATE:  (openc)
+      dirbravo/filebravo1
+    0x4103ee UPDATE:  (cap update)
+      dirbravo/filebravo1
+
+    # cephfs-journal-tool event splice --range 0x40f754..0x410bf1 summary
+    Events by type:
+      OPEN: 1
+      UPDATE: 2
+
+    # cephfs-journal-tool event apply --range 0x410bf1.. summary
+    Events by type:
+      NOOP: 1
+      SESSION: 1
+      UPDATE: 9
+
+    # cephfs-journal-tool event get --inode=1099511627776 list
+    0x40068b UPDATE:  (mkdir)
+      diralpha
+    0x400d1b UPDATE:  (mkdir)
+      diralpha/filealpha1
+    0x401666 UPDATE:  (unlink_local)
+      stray0/10000000001
+      diralpha/filealpha1
+    0x40228d UPDATE:  (unlink_local)
+      diralpha
+      stray0/10000000000
+
+    # cephfs-journal-tool event get --frag=1099511627776 --dname=filealpha1 list
+    0x400d1b UPDATE:  (mkdir)
+      diralpha/filealpha1
+    0x401666 UPDATE:  (unlink_local)
+      stray0/10000000001
+      diralpha/filealpha1
+
+    # cephfs-journal-tool event get binary --path bin_events
+    Wrote output to binary files in directory 'bin_events'
+
diff --git a/doc/cephfs/cephfs-mirroring.rst b/doc/cephfs/cephfs-mirroring.rst
new file mode 100644
index 000000000..3dbaa5d1a
--- /dev/null
+++ b/doc/cephfs/cephfs-mirroring.rst
@@ -0,0 +1,385 @@
+.. _cephfs-mirroring:
+
+=========================
+CephFS Snapshot Mirroring
+=========================
+
+CephFS supports asynchronous replication of snapshots to a remote CephFS file system via
+`cephfs-mirror` tool. Snapshots are synchronized by mirroring snapshot data followed by
+creating a snapshot with the same name (for a given directory on the remote file system) as
+the snapshot being synchronized.
+
+Requirements
+------------
+
+The primary (local) and secondary (remote) Ceph clusters version should be Pacific or later.
+
+Creating Users
+--------------
+
+Start by creating a user (on the primary/local cluster) for the mirror daemon. This user
+requires write capability on the metadata pool to create RADOS objects (index objects)
+for watch/notify operation and read capability on the data pool(s)::
+
+  $ ceph auth get-or-create client.mirror mon 'profile cephfs-mirror' mds 'allow r' osd 'allow rw tag cephfs metadata=*, allow r tag cephfs data=*' mgr 'allow r'
+
+Create a user for each file system peer (on the secondary/remote cluster). This user needs
+to have full capabilities on the MDS (to take snapshots) and the OSDs::
+
+  $ ceph fs authorize <fs_name> client.mirror_remote / rwps
+
+This user should be used (as part of peer specification) when adding a peer.
+
+Starting Mirror Daemon
+----------------------
+
+Mirror daemon should be spawned using `systemctl(1)` unit files::
+
+  $ systemctl enable cephfs-mirror@mirror
+  $ systemctl start cephfs-mirror@mirror
+
+`cephfs-mirror` daemon can be run in foreground using::
+
+  $ cephfs-mirror --id mirror --cluster site-a -f
+
+.. note:: User used here is `mirror` created in the `Creating Users` section.
+
+Interface
+---------
+
+`Mirroring` module (manager plugin) provides interfaces for managing directory snapshot
+mirroring. Manager interfaces are (mostly) wrappers around monitor commands for managing
+file system mirroring and is the recommended control interface.
+
+Mirroring Module
+----------------
+
+The mirroring module is responsible for assigning directories to mirror daemons for
+synchronization. Multiple mirror daemons can be spawned to achieve concurrency in
+directory snapshot synchronization. When mirror daemons are spawned (or terminated)
+, the mirroring module discovers the modified set of mirror daemons and rebalances
+the directory assignment amongst the new set thus providing high-availability.
+
+.. note:: Multiple mirror daemons is currently untested. Only a single mirror daemon
+          is recommended.
+
+Mirroring module is disabled by default. To enable mirroring use::
+
+  $ ceph mgr module enable mirroring
+
+Mirroring module provides a family of commands to control mirroring of directory
+snapshots. To add or remove directories, mirroring needs to be enabled for a given
+file system. To enable mirroring use::
+
+  $ ceph fs snapshot mirror enable <fs_name>
+
+.. note:: Mirroring module commands use `fs snapshot mirror` prefix as compared to
+          the monitor commands which `fs mirror` prefix. Make sure to use module
+          commands.
+
+To disable mirroring, use::
+
+  $ ceph fs snapshot mirror disable <fs_name>
+
+Once mirroring is enabled, add a peer to which directory snapshots are to be mirrored.
+Peers follow `<client>@<cluster>` specification and get assigned a unique-id (UUID)
+when added. See `Creating Users` section on how to create Ceph users for mirroring.
+
+To add a peer use::
+
+  $ ceph fs snapshot mirror peer_add <fs_name> <remote_cluster_spec> [<remote_fs_name>] [<remote_mon_host>] [<cephx_key>]
+
+`<remote_fs_name>` is optional, and defaults to `<fs_name>` (on the remote cluster).
+
+This requires the remote cluster ceph configuration and user keyring to be available in
+the primary cluster. See `Bootstrap Peers` section to avoid this. `peer_add` additionally
+supports passing the remote cluster monitor address and the user key. However, bootstrapping
+a peer is the recommended way to add a peer.
+
+.. note:: Only a single peer is supported right now.
+
+To remove a peer use::
+
+  $ ceph fs snapshot mirror peer_remove <fs_name> <peer_uuid>
+
+To list file system mirror peers use::
+
+  $ ceph fs snapshot mirror peer_list <fs_name>
+
+To configure a directory for mirroring, use::
+
+  $ ceph fs snapshot mirror add <fs_name> <path>
+
+To stop a mirroring directory snapshots use::
+
+  $ ceph fs snapshot mirror remove <fs_name> <path>
+
+Only absolute directory paths are allowed. Also, paths are normalized by the mirroring
+module, therfore, `/a/b/../b` is equivalent to `/a/b`.
+
+  $ mkdir -p /d0/d1/d2
+  $ ceph fs snapshot mirror add cephfs /d0/d1/d2
+  {}
+  $ ceph fs snapshot mirror add cephfs /d0/d1/../d1/d2
+  Error EEXIST: directory /d0/d1/d2 is already tracked
+
+Once a directory is added for mirroring, its subdirectory or ancestor directories are
+disallowed to be added for mirorring::
+
+  $ ceph fs snapshot mirror add cephfs /d0/d1
+  Error EINVAL: /d0/d1 is a ancestor of tracked path /d0/d1/d2
+  $ ceph fs snapshot mirror add cephfs /d0/d1/d2/d3
+  Error EINVAL: /d0/d1/d2/d3 is a subtree of tracked path /d0/d1/d2
+
+Commands to check directory mapping (to mirror daemons) and directory distribution are
+detailed in `Mirroring Status` section.
+
+Bootstrap Peers
+---------------
+
+Adding a peer (via `peer_add`) requires the peer cluster configuration and user keyring
+to be available in the primary cluster (manager host and hosts running the mirror daemon).
+This can be avoided by bootstrapping and importing a peer token. Peer bootstrap involves
+creating a bootstrap token on the peer cluster via::
+
+  $ ceph fs snapshot mirror peer_bootstrap create <fs_name> <client_entity> <site-name>
+
+e.g.::
+
+  $ ceph fs snapshot mirror peer_bootstrap create backup_fs client.mirror_remote site-remote
+  {"token": "eyJmc2lkIjogIjBkZjE3MjE3LWRmY2QtNDAzMC05MDc5LTM2Nzk4NTVkNDJlZiIsICJmaWxlc3lzdGVtIjogImJhY2t1cF9mcyIsICJ1c2VyIjogImNsaWVudC5taXJyb3JfcGVlcl9ib290c3RyYXAiLCAic2l0ZV9uYW1lIjogInNpdGUtcmVtb3RlIiwgImtleSI6ICJBUUFhcDBCZ0xtRmpOeEFBVnNyZXozai9YYUV0T2UrbUJEZlJDZz09IiwgIm1vbl9ob3N0IjogIlt2MjoxOTIuMTY4LjAuNTo0MDkxOCx2MToxOTIuMTY4LjAuNTo0MDkxOV0ifQ=="}
+
+`site-name` refers to a user-defined string to identify the remote filesystem. In context
+of `peer_add` interface, `site-name` is the passed in `cluster` name from `remote_cluster_spec`.
+
+Import the bootstrap token in the primary cluster via::
+
+  $ ceph fs snapshot mirror peer_bootstrap import <fs_name> <token>
+
+e.g.::
+
+  $ ceph fs snapshot mirror peer_bootstrap import cephfs eyJmc2lkIjogIjBkZjE3MjE3LWRmY2QtNDAzMC05MDc5LTM2Nzk4NTVkNDJlZiIsICJmaWxlc3lzdGVtIjogImJhY2t1cF9mcyIsICJ1c2VyIjogImNsaWVudC5taXJyb3JfcGVlcl9ib290c3RyYXAiLCAic2l0ZV9uYW1lIjogInNpdGUtcmVtb3RlIiwgImtleSI6ICJBUUFhcDBCZ0xtRmpOeEFBVnNyZXozai9YYUV0T2UrbUJEZlJDZz09IiwgIm1vbl9ob3N0IjogIlt2MjoxOTIuMTY4LjAuNTo0MDkxOCx2MToxOTIuMTY4LjAuNTo0MDkxOV0ifQ==
+
+Mirroring Status
+----------------
+
+CephFS mirroring module provides `mirror daemon status` interface to check mirror daemon status::
+
+  $ ceph fs snapshot mirror daemon status
+  [
+    {
+      "daemon_id": 284167,
+      "filesystems": [
+        {
+          "filesystem_id": 1,
+          "name": "a",
+          "directory_count": 1,
+          "peers": [
+            {
+              "uuid": "02117353-8cd1-44db-976b-eb20609aa160",
+              "remote": {
+                "client_name": "client.mirror_remote",
+                "cluster_name": "ceph",
+                "fs_name": "backup_fs"
+              },
+              "stats": {
+                "failure_count": 1,
+                "recovery_count": 0
+              }
+            }
+          ]
+        }
+      ]
+    }
+  ]
+
+An entry per mirror daemon instance is displayed along with information such as configured
+peers and basic stats. For more detailed stats, use the admin socket interface as detailed
+below.
+
+CephFS mirror daemons provide admin socket commands for querying mirror status. To check
+available commands for mirror status use::
+
+  $ ceph --admin-daemon /path/to/mirror/daemon/admin/socket help
+  {
+      ....
+      ....
+      "fs mirror status cephfs@360": "get filesystem mirror status",
+      ....
+      ....
+  }
+
+Commands with `fs mirror status` prefix provide mirror status for mirror enabled
+file systems. Note that `cephfs@360` is of format `filesystem-name@filesystem-id`.
+This format is required since mirror daemons get asynchronously notified regarding
+file system mirror status (A file system can be deleted and recreated with the same
+name).
+
+Right now, the command provides minimal information regarding mirror status::
+
+  $ ceph --admin-daemon /var/run/ceph/cephfs-mirror.asok fs mirror status cephfs@360
+  {
+    "rados_inst": "192.168.0.5:0/1476644347",
+    "peers": {
+        "a2dc7784-e7a1-4723-b103-03ee8d8768f8": {
+            "remote": {
+                "client_name": "client.mirror_remote",
+                "cluster_name": "site-a",
+                "fs_name": "backup_fs"
+            }
+        }
+    },
+    "snap_dirs": {
+        "dir_count": 1
+    }
+  }
+
+`Peers` section in the command output above shows the peer information such as unique
+peer-id (UUID) and specification. The peer-id is required to remove an existing peer
+as mentioned in the `Mirror Module and Interface` section.
+
+Command with `fs mirror peer status` prefix provide peer synchronization status. This
+command is of format `filesystem-name@filesystem-id peer-uuid`::
+
+  $ ceph --admin-daemon /var/run/ceph/cephfs-mirror.asok fs mirror peer status cephfs@360 a2dc7784-e7a1-4723-b103-03ee8d8768f8
+  {
+    "/d0": {
+        "state": "idle",
+        "last_synced_snap": {
+            "id": 120,
+            "name": "snap1",
+            "sync_duration": 0.079997898999999997,
+            "sync_time_stamp": "274900.558797s"
+        },
+        "snaps_synced": 2,
+        "snaps_deleted": 0,
+        "snaps_renamed": 0
+    }
+  }
+
+Synchronization stats such as `snaps_synced`, `snaps_deleted` and `snaps_renamed` are reset
+on daemon restart and/or when a directory is reassigned to another mirror daemon (when
+multiple mirror daemons are deployed).
+
+A directory can be in one of the following states::
+
+  - `idle`: The directory is currently not being synchronized
+  - `syncing`: The directory is currently being synchronized
+  - `failed`: The directory has hit upper limit of consecutive failures
+
+When a directory hits a configured number of consecutive synchronization failures, the
+mirror daemon marks it as `failed`. Synchronization for these directories are retried.
+By default, the number of consecutive failures before a directory is marked as failed
+is controlled by `cephfs_mirror_max_consecutive_failures_per_directory` configuration
+option (default: 10) and the retry interval for failed directories is controlled via
+`cephfs_mirror_retry_failed_directories_interval` configuration option (default: 60s).
+
+E.g., adding a regular file for synchronization would result in failed status::
+
+  $ ceph fs snapshot mirror add cephfs /f0
+  $ ceph --admin-daemon /var/run/ceph/cephfs-mirror.asok fs mirror peer status cephfs@360 a2dc7784-e7a1-4723-b103-03ee8d8768f8
+  {
+    "/d0": {
+        "state": "idle",
+        "last_synced_snap": {
+            "id": 120,
+            "name": "snap1",
+            "sync_duration": 0.079997898999999997,
+            "sync_time_stamp": "274900.558797s"
+        },
+        "snaps_synced": 2,
+        "snaps_deleted": 0,
+        "snaps_renamed": 0
+    },
+    "/f0": {
+        "state": "failed",
+        "snaps_synced": 0,
+        "snaps_deleted": 0,
+        "snaps_renamed": 0
+    }
+  }
+
+This allows a user to add a non-existent directory for synchronization. The mirror daemon
+would mark the directory as failed and retry (less frequently). When the directory comes
+to existence, the mirror daemons would unmark the failed state upon successfull snapshot
+synchronization.
+
+When mirroring is disabled, the respective `fs mirror status` command for the file system
+will not show up in command help.
+
+Configuration Options
+---------------------
+
+``cephfs_mirror_max_concurrent_directory_syncs``
+
+:Description: Maximum number of directory snapshots that can be synchronized concurrently by
+              cephfs-mirror daemon. Controls the number of synchronization threads.
+:Type:  64-bit Integer Unsigned
+:Default: ``3``
+
+``cephfs_mirror_action_update_interval``
+
+:Description: Interval in seconds to process pending mirror update actions.
+:Type:  Float
+:Default: ``2``
+
+``cephfs_mirror_restart_mirror_on_blocklist_interval``
+
+:Description: Interval in seconds to restart blocklisted mirror instances. Setting to zero (0)
+              disables restarting blocklisted instances.
+:Type:  Float
+:Default: ``30``
+
+``cephfs_mirror_max_snapshot_sync_per_cycle``
+
+:Description: Maximum number of snapshots to mirror when a directory is picked up for mirroring
+              by worker threads.
+:Type:  64-bit Integer Unsigned
+:Default: ``3``
+
+``cephfs_mirror_directory_scan_interval``
+
+:Description: Interval in seconds to scan configured directories for snapshot mirroring.
+:Type:  64-bit Integer Unsigned
+:Default: ``10``
+
+``cephfs_mirror_max_consecutive_failures_per_directory``
+
+:Description: Number of consecutive snapshot synchronization failues to mark a directory as
+              "failed". Failed directories are retried for synchronization less frequently.
+:Type:  64-bit Integer Unsigned
+:Default: ``10``
+
+``cephfs_mirror_retry_failed_directories_interval``
+
+:Description: Interval in seconds to retry synchronization for failed directories.
+:Type:  64-bit Integer Unsigned
+:Default: ``60``
+
+``cephfs_mirror_restart_mirror_on_failure_interval``
+
+:Description: Interval in seconds to restart failed mirror instances. Setting to zero (0)
+              disables restarting failed mirror instances.
+:Type:  Float
+:Default: ``20``
+
+``cephfs_mirror_mount_timeout``
+
+:Description: Timeout in seconds for mounting primary or secondary (remote) ceph file system
+              by the cephfs-mirror daemon. Setting this to a higher value could result in the
+              mirror daemon getting stalled when mounting a file system if the cluster is not
+              reachable. This option is used to override the usual client_mount_timeout.
+:Type:  Float
+:Default: ``10``
+
+
+Re-adding Peers
+---------------
+
+When re-adding (reassigning) a peer to a file system in another cluster, ensure that
+all mirror daemons have stopped synchronization to the peer. This can be checked
+via `fs mirror status` admin socket command (the `Peer UUID` should not show up
+in the command output). Also, it is recommended to purge synchronized directories
+from the peer  before re-adding it to another file system (especially those directories
+which might exist in the new primary file system). This is not required if re-adding
+a peer to the same primary file system it was earlier synchronized from.
diff --git a/doc/cephfs/cephfs-shell.rst b/doc/cephfs/cephfs-shell.rst
new file mode 100644
index 000000000..2fa1decbb
--- /dev/null
+++ b/doc/cephfs/cephfs-shell.rst
@@ -0,0 +1,588 @@
+
+=============
+CephFS Shell
+=============
+
+CephFS Shell provides shell-like commands that directly interact with the
+:term:`Ceph File System`.
+
+This tool can be used in interactive mode as well as in non-interactive mode.
+In former mode, cephfs-shell opens a shell session and after the given command
+is finished, it prints the prompt string and waits indefinitely. When the
+shell session is finished, cephfs-shell quits with the return value of last
+executed command. In non-interactive mode, cephfs-shell issues a command and
+exits right after the command's execution is complete with the command's
+return value.
+
+Behaviour of CephFS Shell can be tweaked using ``cephfs-shell.conf``. Refer to
+`CephFS Shell Configuration File`_ for details.
+
+Usage :
+
+    cephfs-shell [options] [command]
+    cephfs-shell [options] -- [command, command,...]
+
+Options :
+    -c, --config FILE     Path to cephfs-shell.conf
+    -b, --batch FILE      Path to batch file.
+    -t, --test FILE       Path to transcript(s) in FILE for testing
+
+
+.. note::
+
+    Latest version of the cmd2 module is required for running cephfs-shell.
+    If CephFS is installed through source, execute cephfs-shell in the build
+    directory. It can also be executed as following using virtualenv:
+
+.. code:: bash
+
+    [build]$ python3 -m venv venv && source venv/bin/activate && pip3 install cmd2
+    [build]$ source vstart_environment.sh && source venv/bin/activate && python3 ../src/tools/cephfs/cephfs-shell
+
+Commands
+========
+
+mkdir
+-----
+
+Create the directory(ies), if they do not already exist.
+
+Usage : 
+        
+    mkdir [-option] <directory>... 
+
+* directory - name of the directory to be created.
+
+Options :
+  -m MODE    Sets the access mode for the new directory.
+  -p, --parent         Create parent directories as necessary. When this option is specified, no error is reported if a directory already exists.
+ 
+put
+---
+
+Copy a file/directory to Ceph File System from Local File System.
+
+Usage : 
+    
+        put [options] <source_path> <target_path>
+
+* source_path - local file/directory path to be copied to cephfs.
+    * if `.` copies all the file/directories in the local working directory.
+    * if `-`  Reads the input from stdin. 
+
+* target_path - remote directory path where the files/directories are to be copied to.
+    * if `.` files/directories are copied to the remote working directory.
+
+Options :
+   -f, --force        Overwrites the destination if it already exists.
+
+
+get
+---
+ 
+Copy a file from Ceph File System to Local File System.
+
+Usage : 
+
+    get [options] <source_path> <target_path>
+
+* source_path - remote file/directory path which is to be copied to local file system.
+    * if `.` copies all the file/directories in the remote working directory.
+                    
+* target_path - local directory path where the files/directories are to be copied to.
+    * if `.` files/directories are copied to the local working directory. 
+    * if `-` Writes output to stdout.
+
+Options:
+  -f, --force        Overwrites the destination if it already exists.
+
+ls
+--
+
+List all the files and directories in the current working directory.
+
+Usage : 
+    
+    ls [option] [directory]...
+
+* directory - name of directory whose files/directories are to be listed. 
+    * By default current working directory's files/directories are listed.
+
+Options:
+  -l, --long	    list with long format - show permissions
+  -r, --reverse     reverse sort     
+  -H                human readable
+  -a, -all          ignore entries starting with .
+  -S                Sort by file_size
+
+
+cat
+---
+
+Concatenate files and print on the standard output
+
+Usage : 
+
+    cat  <file>....
+
+* file - name of the file
+
+cd
+--
+
+Change current working directory.
+
+Usage : 
+    
+    cd [directory]
+        
+* directory - path/directory name. If no directory is mentioned it is changed to the root directory.
+    * If '.' moves to the parent directory of the current directory.
+
+cwd
+---
+
+Get current working directory.
+ 
+Usage : 
+    
+    cwd
+
+
+quit/Ctrl + D
+-------------
+
+Close the shell.
+
+chmod
+-----
+
+Change the permissions of file/directory.
+ 
+Usage : 
+    
+    chmod <mode> <file/directory>
+
+mv
+--
+
+Moves files/Directory from source to destination.
+
+Usage : 
+    
+    mv <source_path> <destination_path>
+
+rmdir
+-----
+
+Delete a directory(ies).
+
+Usage : 
+    
+    rmdir <directory_name>.....
+
+rm
+--
+
+Remove a file(es).
+
+Usage : 
+    
+    rm <file_name/pattern>...
+
+
+write
+-----
+
+Create and Write a file.
+
+Usage : 
+        
+        write <file_name>
+        <Enter Data>
+        Ctrl+D Exit.
+
+lls
+---
+
+Lists all files and directories in the specified directory.Current local directory files and directories are listed if no     path is mentioned
+
+Usage: 
+    
+    lls <path>.....
+
+lcd
+---
+
+Moves into the given local directory.
+
+Usage : 
+    
+    lcd <path>
+
+lpwd
+----
+
+Prints the absolute path of the current local directory.
+
+Usage : 
+    
+    lpwd
+
+
+umask
+-----
+
+Set and get the file mode creation mask 
+
+Usage : 
+    
+    umask [mode]
+
+alias
+-----
+
+Define or display aliases
+
+Usage: 
+
+    alias [name] | [<name> <value>]
+
+* name - name of the alias being looked up, added, or replaced
+* value - what the alias will be resolved to (if adding or replacing) this can contain spaces and does not need to be quoted
+
+run_pyscript
+------------
+
+Runs a python script file inside the console
+
+Usage: 
+    
+    run_pyscript <script_path> [script_arguments]
+
+* Console commands can be executed inside this script with cmd ("your command")
+  However, you cannot run nested "py" or "pyscript" commands from within this
+  script. Paths or arguments that contain spaces must be enclosed in quotes
+
+.. note:: This command is available as ``pyscript`` for cmd2 versions 0.9.13
+   or less.
+
+py
+--
+
+Invoke python command, shell, or script
+
+Usage : 
+
+        py <command>: Executes a Python command.
+        py: Enters interactive Python mode.
+
+shortcuts
+---------
+
+Lists shortcuts (aliases) available
+
+Usage :
+
+    shortcuts
+
+history
+-------
+
+View, run, edit, and save previously entered commands.
+
+Usage : 
+    
+    history [-h] [-r | -e | -s | -o FILE | -t TRANSCRIPT] [arg]
+
+Options:
+   -h             show this help message and exit
+   -r             run selected history items
+   -e             edit and then run selected history items
+   -s             script format; no separation lines
+   -o FILE        output commands to a script file
+   -t TRANSCRIPT  output commands and results to a transcript file
+
+unalias
+-------
+
+Unsets aliases
+ 
+Usage : 
+    
+    unalias [-a] name [name ...]
+
+* name - name of the alias being unset
+
+Options:
+   -a     remove all alias definitions
+
+set
+---
+
+Sets a settable parameter or shows current settings of parameters.
+
+Usage : 
+
+    set [-h] [-a] [-l] [settable [settable ...]]
+
+* Call without arguments for a list of settable parameters with their values.
+
+Options :
+  -h     show this help message and exit
+  -a     display read-only settings as well
+  -l     describe function of parameter
+
+edit
+----
+
+Edit a file in a text editor.
+
+Usage:  
+
+    edit [file_path]
+
+* file_path - path to a file to open in editor
+
+run_script
+----------
+
+Runs commands in script file that is encoded as either ASCII or UTF-8 text.
+Each command in the script should be separated by a newline.
+
+Usage:  
+    
+    run_script <file_path>
+
+
+* file_path - a file path pointing to a script
+
+.. note:: This command is available as ``load`` for cmd2 versions 0.9.13
+   or less.
+
+shell
+-----
+
+Execute a command as if at the OS prompt.
+
+Usage:  
+    
+    shell <command> [arguments]
+
+locate
+------
+
+Find an item in File System
+
+Usage:
+
+     locate [options] <name>
+
+Options :
+  -c       Count number of items found
+  -i       Ignore case 
+
+stat
+------
+
+Display file status.
+
+Usage :
+
+     stat [-h] <file_name> [file_name ...]
+
+Options :
+  -h     Shows the help message
+
+snap
+----
+
+Create or Delete Snapshot
+
+Usage:
+
+     snap {create|delete} <snap_name> <dir_name>
+
+* snap_name - Snapshot name to be created or deleted
+
+* dir_name - directory under which snapshot should be created or deleted
+
+setxattr
+--------
+
+Set extended attribute for a file
+
+Usage :
+
+     setxattr [-h] <path> <name> <value>
+
+*  path - Path to the file
+
+*  name - Extended attribute name to get or set
+
+*  value - Extended attribute value to be set
+
+Options:
+  -h, --help   Shows the help message
+
+getxattr
+--------
+
+Get extended attribute value for the name associated with the path
+
+Usage :
+
+     getxattr [-h] <path> <name>
+
+*  path - Path to the file
+
+*  name - Extended attribute name to get or set
+
+Options:
+  -h, --help   Shows the help message
+
+listxattr
+---------
+
+List extended attribute names associated with the path
+
+Usage :
+
+     listxattr [-h] <path>
+
+*  path - Path to the file
+
+Options:
+  -h, --help   Shows the help message
+
+df
+--
+
+Display amount of available disk space
+
+Usage :
+
+    df [-h] [file [file ...]]
+
+* file - name of the file
+
+Options:
+  -h, --help   Shows the help message
+
+du
+--
+
+Show disk usage of a directory
+
+Usage :
+
+    du [-h] [-r] [paths [paths ...]]
+
+* paths - name of the directory
+
+Options:
+  -h, --help   Shows the help message
+
+  -r     Recursive Disk usage of all directories
+
+
+quota
+-----
+
+Quota management for a Directory
+
+Usage :
+
+    quota [-h] [--max_bytes [MAX_BYTES]] [--max_files [MAX_FILES]] {get,set} path
+
+* {get,set} - quota operation type.
+
+* path - name of the directory.
+
+Options :
+  -h, --help   Shows the help message
+
+  --max_bytes MAX_BYTES    Set max cumulative size of the data under this directory
+
+  --max_files MAX_FILES    Set total number of files under this directory tree
+
+CephFS Shell Configuration File
+===============================
+By default, CephFS Shell looks for ``cephfs-shell.conf`` in the path provided
+by the environment variable ``CEPHFS_SHELL_CONF`` and then in user's home
+directory (``~/.cephfs-shell.conf``).
+
+Right now, CephFS Shell inherits all its options from its dependency ``cmd2``.
+Therefore, these options might vary with the version of ``cmd2`` installed on
+your system. Refer to ``cmd2`` docs for a description of these options.
+
+Following is a sample ``cephfs-shell.conf``::
+
+    [cephfs-shell]
+    prompt = CephFS:~/>>>
+    continuation_prompt = >
+
+    quiet = False
+    timing = False
+    colors = True
+    debug = False
+
+    abbrev = False
+    autorun_on_edit = False
+    echo = False
+    editor = vim
+    feedback_to_output = False
+    locals_in_py = True
+
+Exit Code
+=========
+
+Following exit codes are returned by cephfs shell
+
++-----------------------------------------------+-----------+
+| Error Type                                    | Exit Code |
++===============================================+===========+
+| Miscellaneous                                 |     1     |
++-----------------------------------------------+-----------+
+| Keyboard Interrupt                            |     2     |
++-----------------------------------------------+-----------+
+| Operation not permitted                       |     3     |
++-----------------------------------------------+-----------+
+| Permission denied                             |     4     |
++-----------------------------------------------+-----------+
+| No such file or directory                     |     5     |
++-----------------------------------------------+-----------+
+| I/O error                                     |     6     |
++-----------------------------------------------+-----------+
+| No space left on device                       |     7     |
++-----------------------------------------------+-----------+
+| File exists                                   |     8     |
++-----------------------------------------------+-----------+
+| No data available                             |     9     |
++-----------------------------------------------+-----------+
+| Invalid argument                              |     10    |
++-----------------------------------------------+-----------+
+| Operation not supported on transport endpoint |     11    |
++-----------------------------------------------+-----------+
+| Range error                                   |     12    |
++-----------------------------------------------+-----------+
+| Operation would block                         |     13    |
++-----------------------------------------------+-----------+
+| Directory not empty                           |     14    |
++-----------------------------------------------+-----------+
+| Not a directory                               |     15    |
++-----------------------------------------------+-----------+
+| Disk quota exceeded                           |     16    |
++-----------------------------------------------+-----------+
+| Broken pipe                                   |     17    |
++-----------------------------------------------+-----------+
+| Cannot send after transport endpoint shutdown |     18    |
++-----------------------------------------------+-----------+
+| Connection aborted                            |     19    |
++-----------------------------------------------+-----------+
+| Connection refused                            |     20    |
++-----------------------------------------------+-----------+
+| Connection reset                              |     21    |
++-----------------------------------------------+-----------+
+| Interrupted function call                     |     22    |
++-----------------------------------------------+-----------+
diff --git a/doc/cephfs/cephfs-top.png b/doc/cephfs/cephfs-top.png
new file mode 100644
index 000000000..79f298b8c
--- /dev/null
+++ b/doc/cephfs/cephfs-top.png
diff --git a/doc/cephfs/cephfs-top.rst b/doc/cephfs/cephfs-top.rst
new file mode 100644
index 000000000..ae8c71ae1
--- /dev/null
+++ b/doc/cephfs/cephfs-top.rst
@@ -0,0 +1,97 @@
+.. _cephfs-top:
+
+==================
+CephFS Top Utility
+==================
+
+CephFS provides `top(1)` like utility to display various Ceph Filesystem metrics
+in realtime. `cephfs-top` is a curses based python script which makes use of `stats`
+plugin in Ceph Manager to fetch (and display) metrics.
+
+Manager Plugin
+==============
+
+Ceph Filesystem clients periodically forward various metrics to Ceph Metadata Servers (MDS)
+which in turn get forwarded to Ceph Manager by MDS rank zero. Each active MDS forward its
+respective set of metrics to MDS rank zero. Metrics are aggergated and forwarded to Ceph
+Manager.
+
+Metrics are divided into two categories - global and per-mds. Global metrics represent
+set of metrics for the filesystem as a whole (e.g., client read latency) whereas per-mds
+metrics are for a particular MDS rank (e.g., number of subtrees handled by an MDS).
+
+.. note:: Currently, only global metrics are tracked.
+
+`stats` plugin is disabled by default and should be enabled via::
+
+  $ ceph mgr module enable stats
+
+Once enabled, Ceph Filesystem metrics can be fetched via::
+
+  $ ceph fs perf stats
+
+The output format is JSON and contains fields as follows:
+
+- `version`: Version of stats output
+- `global_counters`: List of global performance metrics
+- `counters`: List of per-mds performance metrics
+- `client_metadata`: Ceph Filesystem client metadata
+- `global_metrics`: Global performance counters
+- `metrics`: Per-MDS performance counters (currently, empty) and delayed ranks
+
+.. note:: `delayed_ranks` is the set of active MDS ranks that are reporting stale metrics.
+          This can happen in cases such as (temporary) network issue between MDS rank zero
+          and other active MDSs.
+
+Metrics can be fetched for a partcilar client and/or for a set of active MDSs. To fetch metrics
+for a particular client (e.g., for client-id: 1234)::
+
+  $ ceph fs perf stats --client_id=1234
+
+To fetch metrics only for a subset of active MDSs (e.g., MDS rank 1 and 2)::
+
+  $ ceph fs perf stats --mds_rank=1,2
+
+`cephfs-top`
+============
+
+`cephfs-top` utility relies on `stats` plugin to fetch performance metrics and display in
+`top(1)` like format. `cephfs-top` is available as part of `cephfs-top` package.
+
+By default, `cephfs-top` uses `client.fstop` user to connect to a Ceph cluster::
+
+  $ ceph auth get-or-create client.fstop mon 'allow r' mds 'allow r' osd 'allow r' mgr 'allow r'
+  $ cephfs-top
+
+Command-Line Options
+--------------------
+
+To use a non-default user (other than `client.fstop`) use::
+
+  $ cephfs-top --id <name>
+
+By default, `cephfs-top` connects to cluster name `ceph`. To use a non-default cluster name::
+
+  $ cephfs-top --cluster <cluster>
+
+`cephfs-top` refreshes stats every second by default. To chose a different refresh interval use::
+
+  $ cephfs-top -d <seconds>
+
+Interval should be greater or equal to 0.5 second. Fractional seconds are honoured.
+
+Interactive Commands
+--------------------
+
+1. m : Filesystem selection
+      Displays a menu of filesystems for selection.
+
+2. q : Quit
+      Exit the utility if you are at the home screen (All Filesystem Info),
+      otherwise escape back to the home screen.
+
+The metrics display can be scrolled using the Arrow Keys, PgUp/PgDn, Home/End and mouse.
+
+Sample screenshot running `cephfs-top` with 2 filesystems:
+
+.. image:: cephfs-top.png
diff --git a/doc/cephfs/client-auth.rst b/doc/cephfs/client-auth.rst
new file mode 100644
index 000000000..fd0faa839
--- /dev/null
+++ b/doc/cephfs/client-auth.rst
@@ -0,0 +1,251 @@
+================================
+CephFS Client Capabilities
+================================
+
+Use Ceph authentication capabilities to restrict your file system clients
+to the lowest possible level of authority needed.
+
+.. note:: Path restriction and layout modification restriction are new features
+    in the Jewel release of Ceph.
+
+.. note:: Using Erasure Coded(EC) pools with CephFS is supported only with the
+   BlueStore Backend. They cannot be used as metadata pools and overwrites must
+   be enabled on the data pools.
+
+
+Path restriction
+================
+
+By default, clients are not restricted in what paths they are allowed to
+mount. Further, when clients mount a subdirectory, e.g., ``/home/user``, the
+MDS does not by default verify that subsequent operations are ‘locked’ within
+that directory.
+
+To restrict clients to only mount and work within a certain directory, use
+path-based MDS authentication capabilities.
+
+Syntax
+------
+
+To grant rw access to the specified directory only, we mention the specified
+directory while creating key for a client using the following syntax::
+
+ ceph fs authorize <fs_name> client.<client_id> <path-in-cephfs> rw
+
+For example, to restrict client ``foo`` to writing only in the ``bar``
+directory of file system ``cephfs_a``, use ::
+
+ ceph fs authorize cephfs_a client.foo / r /bar rw
+
+ results in:
+
+ client.foo
+   key: *key*
+   caps: [mds] allow r, allow rw path=/bar
+   caps: [mon] allow r
+   caps: [osd] allow rw tag cephfs data=cephfs_a
+
+To completely restrict the client to the ``bar`` directory, omit the
+root directory ::
+
+ ceph fs authorize cephfs_a client.foo /bar rw
+
+Note that if a client's read access is restricted to a path, they will only
+be able to mount the file system when specifying a readable path in the
+mount command (see below).
+
+Supplying ``all`` or ``*`` as the file system name will grant access to every
+file system. Note that it is usually necessary to quote ``*`` to protect it
+from the shell.
+
+See `User Management - Add a User to a Keyring`_. for additional details on
+user management
+
+To restrict a client to the specified sub-directory only, we mention the
+specified directory while mounting using the following syntax::
+
+ ceph-fuse -n client.<client_id> <mount-path> -r *directory_to_be_mounted*
+
+For example, to restrict client ``foo`` to ``mnt/bar`` directory, we will
+use::
+
+ ceph-fuse -n client.foo mnt -r /bar
+
+Free space reporting
+--------------------
+
+By default, when a client is mounting a sub-directory, the used space (``df``)
+will be calculated from the quota on that sub-directory, rather than reporting
+the overall amount of space used on the cluster.
+
+If you would like the client to report the overall usage of the file system,
+and not just the quota usage on the sub-directory mounted, then set the
+following config option on the client::
+
+
+    client quota df = false
+
+If quotas are not enabled, or no quota is set on the sub-directory mounted,
+then the overall usage of the file system will be reported irrespective of
+the value of this setting.
+
+Layout and Quota restriction (the 'p' flag)
+===========================================
+
+To set layouts or quotas, clients require the 'p' flag in addition to 'rw'.
+This restricts all the attributes that are set by special extended attributes
+with a "ceph." prefix, as well as restricting other means of setting
+these fields (such as openc operations with layouts).
+
+For example, in the following snippet client.0 can modify layouts and quotas
+on the file system cephfs_a, but client.1 cannot::
+
+    client.0
+        key: AQAz7EVWygILFRAAdIcuJ12opU/JKyfFmxhuaw==
+        caps: [mds] allow rwp
+        caps: [mon] allow r
+        caps: [osd] allow rw tag cephfs data=cephfs_a
+
+    client.1
+        key: AQAz7EVWygILFRAAdIcuJ12opU/JKyfFmxhuaw==
+        caps: [mds] allow rw
+        caps: [mon] allow r
+        caps: [osd] allow rw tag cephfs data=cephfs_a
+
+
+Snapshot restriction (the 's' flag)
+===========================================
+
+To create or delete snapshots, clients require the 's' flag in addition to
+'rw'. Note that when capability string also contains the 'p' flag, the 's'
+flag must appear after it (all flags except 'rw' must be specified in
+alphabetical order).
+
+For example, in the following snippet client.0 can create or delete snapshots
+in the ``bar`` directory of file system ``cephfs_a``::
+
+    client.0
+        key: AQAz7EVWygILFRAAdIcuJ12opU/JKyfFmxhuaw==
+        caps: [mds] allow rw, allow rws path=/bar
+        caps: [mon] allow r
+        caps: [osd] allow rw tag cephfs data=cephfs_a
+
+
+.. _User Management - Add a User to a Keyring: ../../rados/operations/user-management/#add-a-user-to-a-keyring
+
+Network restriction
+===================
+
+::
+
+ client.foo
+   key: *key*
+   caps: [mds] allow r network 10.0.0.0/8, allow rw path=/bar network 10.0.0.0/8
+   caps: [mon] allow r network 10.0.0.0/8
+   caps: [osd] allow rw tag cephfs data=cephfs_a network 10.0.0.0/8
+
+The optional ``{network/prefix}`` is a standard network name and
+prefix length in CIDR notation (e.g., ``10.3.0.0/16``).  If present,
+the use of this capability is restricted to clients connecting from
+this network.
+
+.. _fs-authorize-multifs:
+
+File system Information Restriction
+===================================
+
+If desired, the monitor cluster can present a limited view of the file systems
+available. In this case, the monitor cluster will only inform clients about
+file systems specified by the administrator. Other file systems will not be
+reported and commands affecting them will fail as if the file systems do
+not exist.
+
+Consider following example. The Ceph cluster has 2 FSs::
+
+    $ ceph fs ls
+    name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ]
+    name: cephfs2, metadata pool: cephfs2_metadata, data pools: [cephfs2_data ]
+
+But we authorize client ``someuser`` for only one FS::
+
+    $ ceph fs authorize cephfs client.someuser / rw
+    [client.someuser]
+        key = AQAmthpf89M+JhAAiHDYQkMiCq3x+J0n9e8REQ==
+    $ cat ceph.client.someuser.keyring
+    [client.someuser]
+        key = AQAmthpf89M+JhAAiHDYQkMiCq3x+J0n9e8REQ==
+        caps mds = "allow rw fsname=cephfs"
+        caps mon = "allow r fsname=cephfs"
+        caps osd = "allow rw tag cephfs data=cephfs"
+
+And the client can only see the FS that it has authorization for::
+
+    $ ceph fs ls -n client.someuser -k ceph.client.someuser.keyring
+    name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ]
+
+Standby MDS daemons will always be displayed. Note that the information about
+restricted MDS daemons and file systems may become available by other means,
+such as ``ceph health detail``.
+
+MDS communication restriction
+=============================
+
+By default, user applications may communicate with any MDS, whether or not
+they are allowed to modify data on an associated file system (see
+`Path restriction` above). Client's communication can be restricted to MDS
+daemons associated with particular file system(s) by adding MDS caps for that
+particular file system. Consider the following example where the Ceph cluster
+has 2 FSs::
+
+    $ ceph fs ls
+    name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ]
+    name: cephfs2, metadata pool: cephfs2_metadata, data pools: [cephfs2_data ]
+
+Client ``someuser`` is authorized only for one FS::
+
+    $ ceph fs authorize cephfs client.someuser / rw
+    [client.someuser]
+        key = AQBPSARfg8hCJRAAEegIxjlm7VkHuiuntm6wsA==
+    $ ceph auth get client.someuser > ceph.client.someuser.keyring
+    exported keyring for client.someuser
+    $ cat ceph.client.someuser.keyring
+    [client.someuser]
+        key = AQBPSARfg8hCJRAAEegIxjlm7VkHuiuntm6wsA==
+        caps mds = "allow rw fsname=cephfs"
+        caps mon = "allow r"
+        caps osd = "allow rw tag cephfs data=cephfs"
+
+Mounting ``cephfs1`` with ``someuser`` works::
+
+    $ sudo ceph-fuse /mnt/cephfs1 -n client.someuser -k ceph.client.someuser.keyring --client-fs=cephfs
+    ceph-fuse[96634]: starting ceph client
+    ceph-fuse[96634]: starting fuse
+    $ mount | grep ceph-fuse
+    ceph-fuse on /mnt/cephfs1 type fuse.ceph-fuse (rw,nosuid,nodev,relatime,user_id=0,group_id=0,allow_other)
+
+But mounting ``cephfs2`` does not::
+
+    $ sudo ceph-fuse /mnt/cephfs2 -n client.someuser -k ceph.client.someuser.keyring --client-fs=cephfs2
+    ceph-fuse[96599]: starting ceph client
+    ceph-fuse[96599]: ceph mount failed with (1) Operation not permitted
+
+Root squash
+===========
+
+The ``root squash`` feature is implemented as a safety measure to prevent
+scenarios such as accidental ``sudo rm -rf /path``. You can enable
+``root_squash`` mode in MDS caps to disallow clients with uid=0 or gid=0 to
+perform write access operations -- e.g., rm, rmdir, rmsnap, mkdir, mksnap.
+However, the mode allows the read operations of a root client unlike in
+other file systems.
+
+Following is an example of enabling root_squash in a filesystem except within
+'/volumes' directory tree in the filesystem::
+
+    $ ceph fs authorize a client.test_a / rw root_squash /volumes rw
+    $ ceph auth get client.test_a
+    [client.test_a]
+	key = AQBZcDpfEbEUKxAADk14VflBXt71rL9D966mYA==
+	caps mds = "allow rw fsname=a root_squash, allow rw fsname=a path=/volumes"
+	caps mon = "allow r fsname=a"
+	caps osd = "allow rw tag cephfs data=a"
diff --git a/doc/cephfs/client-config-ref.rst b/doc/cephfs/client-config-ref.rst
new file mode 100644
index 000000000..067726b76
--- /dev/null
+++ b/doc/cephfs/client-config-ref.rst
@@ -0,0 +1,261 @@
+Client Configuration
+====================
+
+Updating Client Configuration
+-----------------------------
+
+Certain client configurations can be applied at runtime. To check if a configuration option can be applied (taken into affect by a client) at runtime, use the `config help` command::
+
+   ceph config help debug_client
+    debug_client - Debug level for client
+    (str, advanced)                                                                                                                      Default: 0/5
+    Can update at runtime: true
+
+    The value takes the form 'N' or 'N/M' where N and M are values between 0 and 99.  N is the debug level to log (all values below this are included), and M is the level to gather and buffer in memory.  In the event of a crash, the most recent items <= M are dumped to the log file.
+
+`config help` tells if a given configuration can be applied at runtime along with the defaults and a description of the configuration option.
+
+To update a configuration option at runtime, use the `config set` command::
+
+   ceph config set client debug_client 20/20
+
+Note that this changes a given configuration for all clients.
+
+To check configured options use the `config get` command::
+
+   ceph config get client
+    WHO    MASK LEVEL    OPTION                    VALUE     RO 
+    client      advanced debug_client              20/20          
+    global      advanced osd_pool_default_min_size 1            
+    global      advanced osd_pool_default_size     3            
+
+Client Config Reference
+------------------------
+
+``client_acl_type``
+
+:Description: Set the ACL type. Currently, only possible value is ``"posix_acl"`` to enable POSIX ACL, or an empty string. This option only takes effect when the ``fuse_default_permissions`` is set to ``false``.
+
+:Type: String
+:Default: ``""`` (no ACL enforcement)
+
+``client_cache_mid``
+
+:Description: Set client cache midpoint. The midpoint splits the least recently used lists into a hot and warm list.
+:Type: Float
+:Default: ``0.75``
+
+``client_cache_size``
+
+:Description: Set the number of inodes that the client keeps in the metadata cache.
+:Type: Integer
+:Default: ``16384``
+
+``client_caps_release_delay``
+
+:Description: Set the delay between capability releases in seconds. The delay sets how many seconds a client waits to release capabilities that it no longer needs in case the capabilities are needed for another user space operation.
+:Type: Integer
+:Default: ``5`` (seconds)
+
+``client_debug_force_sync_read``
+
+:Description: If set to ``true``, clients read data directly from OSDs instead of using a local page cache.
+:Type: Boolean
+:Default: ``false``
+
+``client_dirsize_rbytes``
+
+:Description: If set to ``true``, use the recursive size of a directory (that is, total of all descendants).
+:Type: Boolean
+:Default: ``true``
+
+``client_max_inline_size``
+
+:Description: Set the maximum size of inlined data stored in a file inode rather than in a separate data object in RADOS. This setting only applies if the ``inline_data`` flag is set on the MDS map.
+:Type: Integer
+:Default: ``4096``
+
+``client_metadata``
+
+:Description: Comma-delimited strings for client metadata sent to each MDS, in addition to the automatically generated version, host name, and other metadata.
+:Type: String
+:Default: ``""`` (no additional metadata)
+
+``client_mount_gid``
+
+:Description: Set the group ID of CephFS mount.
+:Type: Integer
+:Default: ``-1``
+
+``client_mount_timeout``
+
+:Description: Set the timeout for CephFS mount in seconds.
+:Type: Float
+:Default: ``300.0``
+
+``client_mount_uid``
+
+:Description: Set the user ID of CephFS mount.
+:Type: Integer
+:Default: ``-1``
+
+``client_mountpoint``
+
+:Description: Directory to mount on the CephFS file system. An alternative to the ``-r`` option of the ``ceph-fuse`` command.
+:Type: String
+:Default: ``"/"``
+
+``client_oc``
+
+:Description: Enable object caching.
+:Type: Boolean
+:Default: ``true``
+
+``client_oc_max_dirty``
+
+:Description: Set the maximum number of dirty bytes in the object cache.
+:Type: Integer
+:Default: ``104857600`` (100MB)
+
+``client_oc_max_dirty_age``
+
+:Description: Set the maximum age in seconds of dirty data in the object cache before writeback.
+:Type: Float
+:Default: ``5.0`` (seconds)
+
+``client_oc_max_objects``
+
+:Description: Set the maximum number of objects in the object cache.
+:Type: Integer
+:Default: ``1000``
+
+``client_oc_size``
+
+:Description: Set how many bytes of data will the client cache.
+:Type: Integer
+:Default: ``209715200`` (200 MB)
+
+``client_oc_target_dirty``
+
+:Description: Set the target size of dirty data. We recommend to keep this number low.
+:Type: Integer
+:Default: ``8388608`` (8MB)
+
+``client_permissions``
+
+:Description: Check client permissions on all I/O operations.
+:Type: Boolean
+:Default: ``true``
+
+``client_quota``
+
+:Description: Enable client quota checking if set to ``true``.
+:Type: Boolean
+:Default: ``true``
+
+``client_quota_df``
+
+:Description: Report root directory quota for the ``statfs`` operation.
+:Type: Boolean
+:Default: ``true``
+
+``client_readahead_max_bytes``
+
+:Description: Set the maximum number of bytes that the client reads ahead for future read operations. Overridden by the ``client_readahead_max_periods`` setting.
+:Type: Integer
+:Default: ``0`` (unlimited)
+
+``client_readahead_max_periods``
+
+:Description: Set the number of file layout periods (object size * number of stripes) that the client reads ahead. Overrides the ``client_readahead_max_bytes`` setting.
+:Type: Integer
+:Default: ``4``
+
+``client_readahead_min``
+
+:Description: Set the minimum number bytes that the client reads ahead.
+:Type: Integer
+:Default: ``131072`` (128KB)
+
+``client_reconnect_stale``
+
+:Description: Automatically reconnect stale session.
+:Type: Boolean
+:Default: ``false``
+
+``client_snapdir``
+
+:Description: Set the snapshot directory name.
+:Type: String
+:Default: ``".snap"``
+
+``client_tick_interval``
+
+:Description: Set the interval in seconds between capability renewal and other upkeep.
+:Type: Float
+:Default: ``1.0`` (seconds)
+
+``client_use_random_mds``
+
+:Description: Choose random MDS for each request.
+:Type: Boolean
+:Default: ``false``
+
+``fuse_default_permissions``
+
+:Description: When set to ``false``, ``ceph-fuse`` utility checks does its own permissions checking, instead of relying on the permissions enforcement in FUSE. Set to ``false`` together with the ``client acl type=posix_acl`` option to enable POSIX ACL.
+:Type: Boolean
+:Default: ``true``
+
+``fuse_max_write``
+
+:Description: Set the maximum number of bytes in a single write operation.
+              A value of 0 indicates no change; the
+              FUSE default of 128 kbytes remains in force.
+:Type: Integer
+:Default: ``0``
+
+``fuse_disable_pagecache``
+
+:Description: If set to ``true``, kernel page cache is disabled for ``ceph-fuse``
+              mounts. When multiple clients read/write to a file at the same
+              time, readers may get stale data from page cache. Due to
+              limitations of FUSE, ``ceph-fuse`` can't disable page cache dynamically.
+:Type: Boolean
+:Default: ``false``
+
+Developer Options
+#################
+
+.. important:: These options are internal. They are listed here only to complete the list of options.
+
+``client_debug_getattr_caps``
+
+:Description: Check if the reply from the MDS contains required capabilities.
+:Type: Boolean
+:Default: ``false``
+
+``client_debug_inject_tick_delay``
+
+:Description: Add artificial delay between client ticks.
+:Type: Integer
+:Default: ``0``
+
+``client_inject_fixed_oldest_tid``
+
+:Description:
+:Type: Boolean
+:Default: ``false``
+
+``client_inject_release_failure``
+
+:Description:
+:Type: Boolean
+:Default: ``false``
+
+``client_trace``
+
+:Description: The path to the trace file for all file operations. The output is designed to be used by the Ceph `synthetic client <../../man/8/ceph-syn>`_.
+:Type: String
+:Default: ``""`` (disabled)
+
diff --git a/doc/cephfs/createfs.rst b/doc/cephfs/createfs.rst
new file mode 100644
index 000000000..537a46b25
--- /dev/null
+++ b/doc/cephfs/createfs.rst
@@ -0,0 +1,100 @@
+=========================
+Create a Ceph file system
+=========================
+
+Creating pools
+==============
+
+A Ceph file system requires at least two RADOS pools, one for data and one for metadata.
+When configuring these pools, you might consider:
+
+- Using a higher replication level for the metadata pool, as any data loss in
+  this pool can render the whole file system inaccessible.
+- Using lower-latency storage such as SSDs for the metadata pool, as this will
+  directly affect the observed latency of file system operations on clients.
+- The data pool used to create the file system is the "default" data pool and
+  the location for storing all inode backtrace information, used for hard link
+  management and disaster recovery. For this reason, all inodes created in
+  CephFS have at least one object in the default data pool. If erasure-coded
+  pools are planned for the file system, it is usually better to use a
+  replicated pool for the default data pool to improve small-object write and
+  read performance for updating backtraces. Separately, another erasure-coded
+  data pool can be added (see also :ref:`ecpool`) that can be used on an entire
+  hierarchy of directories and files (see also :ref:`file-layouts`).
+
+Refer to :doc:`/rados/operations/pools` to learn more about managing pools.  For
+example, to create two pools with default settings for use with a file system, you
+might run the following commands:
+
+.. code:: bash
+
+    $ ceph osd pool create cephfs_data
+    $ ceph osd pool create cephfs_metadata
+
+Generally, the metadata pool will have at most a few gigabytes of data. For
+this reason, a smaller PG count is usually recommended. 64 or 128 is commonly
+used in practice for large clusters.
+
+.. note:: The names of the file systems, metadata pools, and data pools can
+          only have characters in the set [a-zA-Z0-9\_-.].
+
+Creating a file system
+======================
+
+Once the pools are created, you may enable the file system using the ``fs new`` command:
+
+.. code:: bash
+
+    $ ceph fs new <fs_name> <metadata> <data>
+
+For example:
+
+.. code:: bash
+
+    $ ceph fs new cephfs cephfs_metadata cephfs_data
+    $ ceph fs ls
+    name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ]
+
+Once a file system has been created, your MDS(s) will be able to enter
+an *active* state.  For example, in a single MDS system:
+
+.. code:: bash
+
+    $ ceph mds stat
+    cephfs-1/1/1 up {0=a=up:active}
+
+Once the file system is created and the MDS is active, you are ready to mount
+the file system.  If you have created more than one file system, you will
+choose which to use when mounting.
+
+  - `Mount CephFS`_
+  - `Mount CephFS as FUSE`_
+  - `Mount CephFS on Windows`_
+
+.. _Mount CephFS: ../../cephfs/mount-using-kernel-driver
+.. _Mount CephFS as FUSE: ../../cephfs/mount-using-fuse
+.. _Mount CephFS on Windows: ../../cephfs/ceph-dokan
+
+If you have created more than one file system, and a client does not
+specify a file system when mounting, you can control which file system
+they will see by using the `ceph fs set-default` command.
+
+Adding a Data Pool to the File System 
+-------------------------------------
+
+See :ref:`adding-data-pool-to-file-system`.
+
+
+Using Erasure Coded pools with CephFS
+=====================================
+
+You may use Erasure Coded pools as CephFS data pools as long as they have overwrites enabled, which is done as follows:
+
+.. code:: bash
+
+    ceph osd pool set my_ec_pool allow_ec_overwrites true
+    
+Note that EC overwrites are only supported when using OSDS with the BlueStore backend.
+
+You may not use Erasure Coded pools as CephFS metadata pools, because CephFS metadata is stored using RADOS *OMAP* data structures, which EC pools cannot store.
+
diff --git a/doc/cephfs/dirfrags.rst b/doc/cephfs/dirfrags.rst
new file mode 100644
index 000000000..d53e0c060
--- /dev/null
+++ b/doc/cephfs/dirfrags.rst
@@ -0,0 +1,97 @@
+
+===================================
+Configuring Directory fragmentation
+===================================
+
+In CephFS, directories are *fragmented* when they become very large
+or very busy.  This splits up the metadata so that it can be shared
+between multiple MDS daemons, and between multiple objects in the
+metadata pool.
+
+In normal operation, directory fragmentation is invisible to
+users and administrators, and all the configuration settings mentioned
+here should be left at their default values.
+
+While directory fragmentation enables CephFS to handle very large
+numbers of entries in a single directory, application programmers should
+remain conservative about creating very large directories, as they still
+have a resource cost in situations such as a CephFS client listing
+the directory, where all the fragments must be loaded at once.
+
+.. tip:: The root directory cannot be fragmented.
+
+All directories are initially created as a single fragment.  This fragment
+may be *split* to divide up the directory into more fragments, and these
+fragments may be *merged* to reduce the number of fragments in the directory.
+
+Splitting and merging
+=====================
+
+When an MDS identifies a directory fragment to be split, it does not
+do the split immediately.  Because splitting interrupts metadata IO,
+a short delay is used to allow short bursts of client IO to complete
+before the split begins.  This delay is configured with
+``mds_bal_fragment_interval``, which defaults to 5 seconds.
+
+When the split is done, the directory fragment is broken up into
+a power of two number of new fragments.  The number of new
+fragments is given by two to the power ``mds_bal_split_bits``, i.e.
+if ``mds_bal_split_bits`` is 2, then four new fragments will be
+created.  The default setting is 3, i.e. splits create 8 new fragments.
+
+The criteria for initiating a split or a merge are described in the
+following sections.
+
+Size thresholds
+===============
+
+A directory fragment is eligible for splitting when its size exceeds
+``mds_bal_split_size`` (default 10000).  Ordinarily this split is
+delayed by ``mds_bal_fragment_interval``, but if the fragment size
+exceeds a factor of ``mds_bal_fragment_fast_factor`` the split size,
+the split will happen immediately (holding up any client metadata
+IO on the directory).
+
+``mds_bal_fragment_size_max`` is the hard limit on the size of
+directory fragments.  If it is reached, clients will receive
+ENOSPC errors if they try to create files in the fragment.  On
+a properly configured system, this limit should never be reached on
+ordinary directories, as they will have split long before.  By default,
+this is set to 10 times the split size, giving a dirfrag size limit of
+100000.  Increasing this limit may lead to oversized directory fragment
+objects in the metadata pool, which the OSDs may not be able to handle.
+
+A directory fragment is eligible for merging when its size is less
+than ``mds_bal_merge_size``.  There is no merge equivalent of the
+"fast splitting" explained above: fast splitting exists to avoid
+creating oversized directory fragments, there is no equivalent issue
+to avoid when merging.  The default merge size is 50.
+
+Activity thresholds
+===================
+
+In addition to splitting fragments based
+on their size, the MDS may split directory fragments if their
+activity exceeds a threshold.
+
+The MDS maintains separate time-decaying load counters for read and write
+operations on directory fragments.  The decaying load counters have an
+exponential decay based on the ``mds_decay_halflife`` setting.
+
+On writes, the write counter is
+incremented, and compared with ``mds_bal_split_wr``, triggering a 
+split if the threshold is exceeded.  Write operations include metadata IO
+such as renames, unlinks and creations. 
+
+The ``mds_bal_split_rd`` threshold is applied based on the read operation
+load counter, which tracks readdir operations.
+
+By the default, the read threshold is 25000 and the write threshold is
+10000, i.e. 2.5x as many reads as writes would be required to trigger
+a split.
+
+After fragments are split due to the activity thresholds, they are only
+merged based on the size threshold (``mds_bal_merge_size``), so 
+a spike in activity may cause a directory to stay fragmented
+forever unless some entries are unlinked.
+
diff --git a/doc/cephfs/disaster-recovery-experts.rst b/doc/cephfs/disaster-recovery-experts.rst
new file mode 100644
index 000000000..11df16e38
--- /dev/null
+++ b/doc/cephfs/disaster-recovery-experts.rst
@@ -0,0 +1,318 @@
+
+.. _disaster-recovery-experts:
+
+Advanced: Metadata repair tools
+===============================
+
+.. warning::
+
+    If you do not have expert knowledge of CephFS internals, you will
+    need to seek assistance before using any of these tools.
+
+    The tools mentioned here can easily cause damage as well as fixing it.
+
+    It is essential to understand exactly what has gone wrong with your
+    file system before attempting to repair it.
+
+    If you do not have access to professional support for your cluster,
+    consult the ceph-users mailing list or the #ceph IRC channel.
+
+
+Journal export
+--------------
+
+Before attempting dangerous operations, make a copy of the journal like so:
+
+::
+
+    cephfs-journal-tool journal export backup.bin
+
+Note that this command may not always work if the journal is badly corrupted,
+in which case a RADOS-level copy should be made (http://tracker.ceph.com/issues/9902).
+
+
+Dentry recovery from journal
+----------------------------
+
+If a journal is damaged or for any reason an MDS is incapable of replaying it,
+attempt to recover what file metadata we can like so:
+
+::
+
+    cephfs-journal-tool event recover_dentries summary
+
+This command by default acts on MDS rank 0, pass --rank=<n> to operate on other ranks.
+
+This command will write any inodes/dentries recoverable from the journal
+into the backing store, if these inodes/dentries are higher-versioned
+than the previous contents of the backing store.  If any regions of the journal
+are missing/damaged, they will be skipped.
+
+Note that in addition to writing out dentries and inodes, this command will update
+the InoTables of each 'in' MDS rank, to indicate that any written inodes' numbers
+are now in use.  In simple cases, this will result in an entirely valid backing
+store state.
+
+.. warning::
+
+    The resulting state of the backing store is not guaranteed to be self-consistent,
+    and an online MDS scrub will be required afterwards.  The journal contents
+    will not be modified by this command, you should truncate the journal
+    separately after recovering what you can.
+
+Journal truncation
+------------------
+
+If the journal is corrupt or MDSs cannot replay it for any reason, you can
+truncate it like so:
+
+::
+
+    cephfs-journal-tool [--rank=N] journal reset
+
+Specify the MDS rank using the ``--rank`` option when the file system has/had
+multiple active MDS.
+
+.. warning::
+
+    Resetting the journal *will* lose metadata unless you have extracted
+    it by other means such as ``recover_dentries``.  It is likely to leave
+    some orphaned objects in the data pool.  It may result in re-allocation
+    of already-written inodes, such that permissions rules could be violated.
+
+MDS table wipes
+---------------
+
+After the journal has been reset, it may no longer be consistent with respect
+to the contents of the MDS tables (InoTable, SessionMap, SnapServer).
+
+To reset the SessionMap (erase all sessions), use:
+
+::
+
+    cephfs-table-tool all reset session
+
+This command acts on the tables of all 'in' MDS ranks.  Replace 'all' with an MDS
+rank to operate on that rank only.
+
+The session table is the table most likely to need resetting, but if you know you
+also need to reset the other tables then replace 'session' with 'snap' or 'inode'.
+
+MDS map reset
+-------------
+
+Once the in-RADOS state of the file system (i.e. contents of the metadata pool)
+is somewhat recovered, it may be necessary to update the MDS map to reflect
+the contents of the metadata pool.  Use the following command to reset the MDS
+map to a single MDS:
+
+::
+
+    ceph fs reset <fs name> --yes-i-really-mean-it
+
+Once this is run, any in-RADOS state for MDS ranks other than 0 will be ignored:
+as a result it is possible for this to result in data loss.
+
+One might wonder what the difference is between 'fs reset' and 'fs remove; fs new'.  The
+key distinction is that doing a remove/new will leave rank 0 in 'creating' state, such
+that it would overwrite any existing root inode on disk and orphan any existing files.  In
+contrast, the 'reset' command will leave rank 0 in 'active' state such that the next MDS
+daemon to claim the rank will go ahead and use the existing in-RADOS metadata.
+
+Recovery from missing metadata objects
+--------------------------------------
+
+Depending on what objects are missing or corrupt, you may need to
+run various commands to regenerate default versions of the
+objects.
+
+::
+
+    # Session table
+    cephfs-table-tool 0 reset session
+    # SnapServer
+    cephfs-table-tool 0 reset snap
+    # InoTable
+    cephfs-table-tool 0 reset inode
+    # Journal
+    cephfs-journal-tool --rank=0 journal reset
+    # Root inodes ("/" and MDS directory)
+    cephfs-data-scan init
+
+Finally, you can regenerate metadata objects for missing files
+and directories based on the contents of a data pool.  This is
+a three-phase process.  First, scanning *all* objects to calculate
+size and mtime metadata for inodes.  Second, scanning the first
+object from every file to collect this metadata and inject it into
+the metadata pool. Third, checking inode linkages and fixing found
+errors.
+
+::
+
+    cephfs-data-scan scan_extents <data pool>
+    cephfs-data-scan scan_inodes <data pool>
+    cephfs-data-scan scan_links
+
+'scan_extents' and 'scan_inodes' commands may take a *very long* time
+if there are many files or very large files in the data pool.
+
+To accelerate the process, run multiple instances of the tool.
+
+Decide on a number of workers, and pass each worker a number within
+the range 0-(worker_m - 1).
+
+The example below shows how to run 4 workers simultaneously:
+
+::
+
+    # Worker 0
+    cephfs-data-scan scan_extents --worker_n 0 --worker_m 4 <data pool>
+    # Worker 1
+    cephfs-data-scan scan_extents --worker_n 1 --worker_m 4 <data pool>
+    # Worker 2
+    cephfs-data-scan scan_extents --worker_n 2 --worker_m 4 <data pool>
+    # Worker 3
+    cephfs-data-scan scan_extents --worker_n 3 --worker_m 4 <data pool>
+
+    # Worker 0
+    cephfs-data-scan scan_inodes --worker_n 0 --worker_m 4 <data pool>
+    # Worker 1
+    cephfs-data-scan scan_inodes --worker_n 1 --worker_m 4 <data pool>
+    # Worker 2
+    cephfs-data-scan scan_inodes --worker_n 2 --worker_m 4 <data pool>
+    # Worker 3
+    cephfs-data-scan scan_inodes --worker_n 3 --worker_m 4 <data pool>
+
+It is **important** to ensure that all workers have completed the
+scan_extents phase before any workers enter the scan_inodes phase.
+
+After completing the metadata recovery, you may want to run cleanup
+operation to delete ancillary data geneated during recovery.
+
+::
+
+    cephfs-data-scan cleanup <data pool>
+
+
+
+Using an alternate metadata pool for recovery
+---------------------------------------------
+
+.. warning::
+
+   There has not been extensive testing of this procedure. It should be
+   undertaken with great care.
+
+If an existing file system is damaged and inoperative, it is possible to create
+a fresh metadata pool and attempt to reconstruct the file system metadata into
+this new pool, leaving the old metadata in place. This could be used to make a
+safer attempt at recovery since the existing metadata pool would not be
+modified.
+
+.. caution::
+
+   During this process, multiple metadata pools will contain data referring to
+   the same data pool. Extreme caution must be exercised to avoid changing the
+   data pool contents while this is the case. Once recovery is complete, the
+   damaged metadata pool should be archived or deleted.
+
+To begin, the existing file system should be taken down, if not done already,
+to prevent further modification of the data pool. Unmount all clients and then
+mark the file system failed:
+
+::
+
+    ceph fs fail <fs_name>
+
+Next, create a recovery file system in which we will populate a new metadata pool
+backed by the original data pool.
+
+::
+
+    ceph fs flag set enable_multiple true --yes-i-really-mean-it
+    ceph osd pool create cephfs_recovery_meta
+    ceph fs new cephfs_recovery recovery <data_pool> --allow-dangerous-metadata-overlay
+
+
+The recovery file system starts with an MDS rank that will initialize the new
+metadata pool with some metadata. This is necessary to bootstrap recovery.
+However, now we will take the MDS down as we do not want it interacting with
+the metadata pool further.
+
+::
+
+    ceph fs fail cephfs_recovery
+
+Next, we will reset the initial metadata the MDS created:
+
+::
+
+    cephfs-table-tool cephfs_recovery:all reset session
+    cephfs-table-tool cephfs_recovery:all reset snap
+    cephfs-table-tool cephfs_recovery:all reset inode
+
+Now perform the recovery of the metadata pool from the data pool:
+
+::
+
+    cephfs-data-scan init --force-init --filesystem cephfs_recovery --alternate-pool cephfs_recovery_meta
+    cephfs-data-scan scan_extents --alternate-pool cephfs_recovery_meta --filesystem <fs_name> <data_pool>
+    cephfs-data-scan scan_inodes --alternate-pool cephfs_recovery_meta --filesystem <fs_name> --force-corrupt <data_pool>
+    cephfs-data-scan scan_links --filesystem cephfs_recovery
+
+.. note::
+
+   Each scan procedure above goes through the entire data pool. This may take a
+   significant amount of time. See the previous section on how to distribute
+   this task among workers.
+
+If the damaged file system contains dirty journal data, it may be recovered next
+with:
+
+::
+
+    cephfs-journal-tool --rank=<fs_name>:0 event recover_dentries list --alternate-pool cephfs_recovery_meta
+    cephfs-journal-tool --rank cephfs_recovery:0 journal reset --force
+
+After recovery, some recovered directories will have incorrect statistics.
+Ensure the parameters ``mds_verify_scatter`` and ``mds_debug_scatterstat`` are
+set to false (the default) to prevent the MDS from checking the statistics:
+
+::
+
+    ceph config rm mds mds_verify_scatter
+    ceph config rm mds mds_debug_scatterstat
+
+(Note, the config may also have been set globally or via a ceph.conf file.)
+Now, allow an MDS to join the recovery file system:
+
+::
+
+    ceph fs set cephfs_recovery joinable true
+
+Finally, run a forward :doc:`scrub </cephfs/scrub>` to repair the statistics.
+Ensure you have an MDS running and issue:
+
+::
+
+    ceph fs status # get active MDS
+    ceph tell mds.<id> scrub start / recursive repair
+
+.. note::
+
+   Symbolic links are recovered as empty regular files. `Symbolic link recovery
+   <https://tracker.ceph.com/issues/46166>`_ is scheduled to be supported in
+   Pacific.
+
+It is recommended to migrate any data from the recovery file system as soon as
+possible. Do not restore the old file system while the recovery file system is
+operational.
+
+.. note::
+
+    If the data pool is also corrupt, some files may not be restored because
+    backtrace information is lost. If any data objects are missing (due to
+    issues like lost Placement Groups on the data pool), the recovered files
+    will contain holes in place of the missing data.
+
+.. _Symbolic link recovery: https://tracker.ceph.com/issues/46166
diff --git a/doc/cephfs/disaster-recovery.rst b/doc/cephfs/disaster-recovery.rst
new file mode 100644
index 000000000..a728feb55
--- /dev/null
+++ b/doc/cephfs/disaster-recovery.rst
@@ -0,0 +1,61 @@
+.. _cephfs-disaster-recovery:
+
+Disaster recovery
+=================
+
+Metadata damage and repair
+--------------------------
+
+If a file system has inconsistent or missing metadata, it is considered
+*damaged*.  You may find out about damage from a health message, or in some
+unfortunate cases from an assertion in a running MDS daemon.
+
+Metadata damage can result either from data loss in the underlying RADOS
+layer (e.g. multiple disk failures that lose all copies of a PG), or from
+software bugs.
+
+CephFS includes some tools that may be able to recover a damaged file system,
+but to use them safely requires a solid understanding of CephFS internals.
+The documentation for these potentially dangerous operations is on a
+separate page: :ref:`disaster-recovery-experts`.
+
+Data pool damage (files affected by lost data PGs)
+--------------------------------------------------
+
+If a PG is lost in a *data* pool, then the file system will continue
+to operate normally, but some parts of some files will simply
+be missing (reads will return zeros).
+
+Losing a data PG may affect many files.  Files are split into many objects,
+so identifying which files are affected by loss of particular PGs requires
+a full scan over all object IDs that may exist within the size of a file. 
+This type of scan may be useful for identifying which files require
+restoring from a backup.
+
+.. danger::
+
+    This command does not repair any metadata, so when restoring files in
+    this case you must *remove* the damaged file, and replace it in order
+    to have a fresh inode.  Do not overwrite damaged files in place.
+
+If you know that objects have been lost from PGs, use the ``pg_files``
+subcommand to scan for files that may have been damaged as a result:
+
+::
+
+    cephfs-data-scan pg_files <path> <pg id> [<pg id>...]
+
+For example, if you have lost data from PGs 1.4 and 4.5, and you would like
+to know which files under /home/bob might have been damaged:
+
+::
+
+    cephfs-data-scan pg_files /home/bob 1.4 4.5
+
+The output will be a list of paths to potentially damaged files, one
+per line.
+
+Note that this command acts as a normal CephFS client to find all the
+files in the file system and read their layouts, so the MDS must be
+up and running.
+
diff --git a/doc/cephfs/dynamic-metadata-management.rst b/doc/cephfs/dynamic-metadata-management.rst
new file mode 100644
index 000000000..5ef8d3ee7
--- /dev/null
+++ b/doc/cephfs/dynamic-metadata-management.rst
@@ -0,0 +1,90 @@
+==================================
+CephFS Dynamic Metadata Management
+==================================
+Metadata operations usually take up more than 50 percent of all
+file system operations. Also the metadata scales in a more complex
+fashion when compared to scaling storage (which in turn scales I/O
+throughput linearly). This is due to the hierarchical and
+interdependent nature of the file system metadata. So in CephFS,
+the metadata workload is decoupled from data workload so as to
+avoid placing unnecessary strain on the RADOS cluster. The metadata
+is hence handled by a cluster of Metadata Servers (MDSs). 
+CephFS distributes metadata across MDSs via `Dynamic Subtree Partitioning <https://ceph.com/wp-content/uploads/2016/08/weil-mds-sc04.pdf>`__.
+
+Dynamic Subtree Partitioning
+----------------------------
+In traditional subtree partitioning, subtrees of the file system
+hierarchy are assigned to individual MDSs. This metadata distribution
+strategy provides good hierarchical locality, linear growth of
+cache and horizontal scaling across MDSs and a fairly good distribution
+of metadata across MDSs.
+
+.. image:: subtree-partitioning.svg
+
+The problem with traditional subtree partitioning is that the workload
+growth by depth (across a single MDS) leads to a hotspot of activity.
+This results in lack of vertical scaling and wastage of non-busy resources/MDSs. 
+
+This led to the adoption of a more dynamic way of handling
+metadata: Dynamic Subtree Partitioning, where load intensive portions
+of the directory hierarchy from busy MDSs are migrated to non busy MDSs. 
+
+This strategy ensures that activity hotspots are relieved as they
+appear and so leads to vertical scaling of the metadata workload in
+addition to horizontal scaling.
+
+Export Process During Subtree Migration
+---------------------------------------
+
+Once the exporter verifies that the subtree is permissible to be exported
+(Non degraded cluster, non-frozen subtree root), the subtree root
+directory is temporarily auth pinned, the subtree freeze is initiated,
+and the exporter is committed to the subtree migration, barring an
+intervening failure of the importer or itself.
+
+The MExportDiscover message is exchanged to ensure that the inode for the
+base directory being exported is open on the destination node. It is
+auth pinned by the importer to prevent it from being trimmed. This occurs
+before the exporter completes the freeze of the subtree to ensure that
+the importer is able to replicate the necessary metadata. When the
+exporter receives the MDiscoverAck, it allows the freeze to proceed by
+removing its temporary auth pin.
+
+A warning stage occurs only if the base subtree directory is open by
+nodes other than the importer and exporter. If it is not, then this
+implies that no metadata within or nested beneath the subtree is
+replicated by any node other than the importer and exporter. If it is,
+then an MExportWarning message informs any bystanders that the
+authority for the region is temporarily ambiguous, and lists both the
+exporter and importer as authoritative MDS nodes. In particular,
+bystanders who are trimming items from their cache must send
+MCacheExpire messages to both the old and new authorities. This is
+necessary to ensure that the surviving authority reliably receives all
+expirations even if the importer or exporter fails. While the subtree
+is frozen (on both the importer and exporter), expirations will not be
+immediately processed; instead, they will be queued until the region
+is unfrozen and it can be determined that the node is or is not
+authoritative.
+
+The exporter then packages an MExport message containing all metadata
+of the subtree and flags the objects as non-authoritative. The MExport message sends
+the actual subtree metadata to the importer. Upon receipt, the
+importer inserts the data into its cache, marks all objects as
+authoritative, and logs a copy of all metadata in an EImportStart
+journal message. Once that has safely flushed, it replies with an
+MExportAck. The exporter can now log an EExport journal entry, which
+ultimately specifies that the export was a success. In the presence
+of failures, it is the existence of the EExport entry only that
+disambiguates authority during recovery.
+
+Once logged, the exporter will send an MExportNotify to any
+bystanders, informing them that the authority is no longer ambiguous
+and cache expirations should be sent only to the new authority (the
+importer). Once these are acknowledged back to the exporter,
+implicitly flushing the bystander to exporter message streams of any
+stray expiration notices, the exporter unfreezes the subtree, cleans
+up its migration-related state, and sends a final MExportFinish to the
+importer. Upon receipt, the importer logs an EImportFinish(true)
+(noting locally that the export was indeed a success), unfreezes its
+subtree, processes any queued cache expierations, and cleans up its
+state.
diff --git a/doc/cephfs/eviction.rst b/doc/cephfs/eviction.rst
new file mode 100644
index 000000000..eb6f70a8e
--- /dev/null
+++ b/doc/cephfs/eviction.rst
@@ -0,0 +1,190 @@
+
+================================
+Ceph file system client eviction
+================================
+
+When a file system client is unresponsive or otherwise misbehaving, it
+may be necessary to forcibly terminate its access to the file system.  This
+process is called *eviction*.
+
+Evicting a CephFS client prevents it from communicating further with MDS
+daemons and OSD daemons.  If a client was doing buffered IO to the file system,
+any un-flushed data will be lost.
+
+Clients may either be evicted automatically (if they fail to communicate
+promptly with the MDS), or manually (by the system administrator).
+
+The client eviction process applies to clients of all kinds, this includes
+FUSE mounts, kernel mounts, nfs-ganesha gateways, and any process using
+libcephfs.
+
+Automatic client eviction
+=========================
+
+There are three situations in which a client may be evicted automatically.
+
+#. On an active MDS daemon, if a client has not communicated with the MDS for over
+   ``session_autoclose`` (a file system variable) seconds (300 seconds by
+   default), then it will be evicted automatically.
+
+#. On an active MDS daemon, if a client has not responded to cap revoke messages
+   for over ``mds_cap_revoke_eviction_timeout`` (configuration option) seconds.
+   This is disabled by default.
+
+#. During MDS startup (including on failover), the MDS passes through a
+   state called ``reconnect``.  During this state, it waits for all the
+   clients to connect to the new MDS daemon.  If any clients fail to do
+   so within the time window (``mds_reconnect_timeout``, 45 seconds by default)
+   then they will be evicted.
+
+A warning message is sent to the cluster log if either of these situations
+arises.
+
+Manual client eviction
+======================
+
+Sometimes, the administrator may want to evict a client manually.  This
+could happen if a client has died and the administrator does not
+want to wait for its session to time out, or it could happen if
+a client is misbehaving and the administrator does not have access to
+the client node to unmount it.
+
+It is useful to inspect the list of clients first:
+
+::
+
+    ceph tell mds.0 client ls
+
+    [
+        {
+            "id": 4305,
+            "num_leases": 0,
+            "num_caps": 3,
+            "state": "open",
+            "replay_requests": 0,
+            "completed_requests": 0,
+            "reconnecting": false,
+            "inst": "client.4305 172.21.9.34:0/422650892",
+            "client_metadata": {
+                "ceph_sha1": "ae81e49d369875ac8b569ff3e3c456a31b8f3af5",
+                "ceph_version": "ceph version 12.0.0-1934-gae81e49 (ae81e49d369875ac8b569ff3e3c456a31b8f3af5)",
+                "entity_id": "0",
+                "hostname": "senta04",
+                "mount_point": "/tmp/tmpcMpF1b/mnt.0",
+                "pid": "29377",
+                "root": "/"
+            }
+        }
+    ]
+    
+
+
+Once you have identified the client you want to evict, you can
+do that using its unique ID, or various other attributes to identify it:
+
+::
+    
+    # These all work
+    ceph tell mds.0 client evict id=4305
+    ceph tell mds.0 client evict client_metadata.=4305
+
+
+Advanced: Un-blocklisting a client
+==================================
+
+Ordinarily, a blocklisted client may not reconnect to the servers: it
+must be unmounted and then mounted anew.
+
+However, in some situations it may be useful to permit a client that
+was evicted to attempt to reconnect.
+
+Because CephFS uses the RADOS OSD blocklist to control client eviction,
+CephFS clients can be permitted to reconnect by removing them from
+the blocklist:
+
+::
+
+    $ ceph osd blocklist ls
+    listed 1 entries
+    127.0.0.1:0/3710147553 2018-03-19 11:32:24.716146
+    $ ceph osd blocklist rm 127.0.0.1:0/3710147553
+    un-blocklisting 127.0.0.1:0/3710147553
+
+
+Doing this may put data integrity at risk if other clients have accessed
+files that the blocklisted client was doing buffered IO to.  It is also not
+guaranteed to result in a fully functional client -- the best way to get
+a fully healthy client back after an eviction is to unmount the client
+and do a fresh mount.
+
+If you are trying to reconnect clients in this way, you may also
+find it useful to set ``client_reconnect_stale`` to true in the
+FUSE client, to prompt the client to try to reconnect.
+
+Advanced: Configuring blocklisting
+==================================
+
+If you are experiencing frequent client evictions, due to slow
+client hosts or an unreliable network, and you cannot fix the underlying
+issue, then you may want to ask the MDS to be less strict.
+
+It is possible to respond to slow clients by simply dropping their
+MDS sessions, but permit them to re-open sessions and permit them
+to continue talking to OSDs.  To enable this mode, set
+``mds_session_blocklist_on_timeout`` to false on your MDS nodes.
+
+For the equivalent behaviour on manual evictions, set
+``mds_session_blocklist_on_evict`` to false.
+
+Note that if blocklisting is disabled, then evicting a client will
+only have an effect on the MDS you send the command to.  On a system
+with multiple active MDS daemons, you would need to send an
+eviction command to each active daemon.  When blocklisting is enabled 
+(the default), sending an eviction command to just a single
+MDS is sufficient, because the blocklist propagates it to the others.
+
+.. _background_blocklisting_and_osd_epoch_barrier:
+
+Background: Blocklisting and OSD epoch barrier
+==============================================
+
+After a client is blocklisted, it is necessary to make sure that
+other clients and MDS daemons have the latest OSDMap (including
+the blocklist entry) before they try to access any data objects
+that the blocklisted client might have been accessing.
+
+This is ensured using an internal "osdmap epoch barrier" mechanism.
+
+The purpose of the barrier is to ensure that when we hand out any
+capabilities which might allow touching the same RADOS objects, the
+clients we hand out the capabilities to must have a sufficiently recent
+OSD map to not race with cancelled operations (from ENOSPC) or
+blocklisted clients (from evictions).
+
+More specifically, the cases where an epoch barrier is set are:
+
+ * Client eviction (where the client is blocklisted and other clients
+   must wait for a post-blocklist epoch to touch the same objects).
+ * OSD map full flag handling in the client (where the client may
+   cancel some OSD ops from a pre-full epoch, so other clients must
+   wait until the full epoch or later before touching the same objects).
+ * MDS startup, because we don't persist the barrier epoch, so must
+   assume that latest OSD map is always required after a restart.
+
+Note that this is a global value for simplicity. We could maintain this on
+a per-inode basis. But we don't, because:
+
+ * It would be more complicated.
+ * It would use an extra 4 bytes of memory for every inode.
+ * It would not be much more efficient as, almost always, everyone has
+   the latest OSD map. And, in most cases everyone will breeze through this
+   barrier rather than waiting.
+ * This barrier is done in very rare cases, so any benefit from per-inode
+   granularity would only very rarely be seen.
+
+The epoch barrier is transmitted along with all capability messages, and
+instructs the receiver of the message to avoid sending any more RADOS
+operations to OSDs until it has seen this OSD epoch.  This mainly applies
+to clients (doing their data writes directly to files), but also applies
+to the MDS because things like file size probing and file deletion are
+done directly from the MDS.
diff --git a/doc/cephfs/experimental-features.rst b/doc/cephfs/experimental-features.rst
new file mode 100644
index 000000000..ba60d12c7
--- /dev/null
+++ b/doc/cephfs/experimental-features.rst
@@ -0,0 +1,42 @@
+=====================
+Experimental Features
+=====================
+
+CephFS includes a number of experimental features which are not fully
+stabilized or qualified for users to turn on in real deployments. We generally
+do our best to clearly demarcate these and fence them off so they cannot be
+used by mistake.
+
+Some of these features are closer to being done than others, though. We
+describe each of them with an approximation of how risky they are and briefly
+describe what is required to enable them. Note that doing so will
+*irrevocably* flag maps in the monitor as having once enabled this flag to
+improve debugging and support processes.
+
+Inline data
+-----------
+By default, all CephFS file data is stored in RADOS objects. The inline data
+feature enables small files (generally <2KB) to be stored in the inode
+and served out of the MDS. This may improve small-file performance but increases
+load on the MDS. It is not sufficiently tested to support at this time, although
+failures within it are unlikely to make non-inlined data inaccessible
+
+Inline data has always been off by default and requires setting
+the ``inline_data`` flag.
+
+Inline data has been declared deprecated for the Octopus release, and will
+likely be removed altogether in the Q release.
+
+Mantle: Programmable Metadata Load Balancer
+-------------------------------------------
+
+Mantle is a programmable metadata balancer built into the MDS. The idea is to
+protect the mechanisms for balancing load (migration, replication,
+fragmentation) but stub out the balancing policies using Lua. For details, see
+:doc:`/cephfs/mantle`.
+
+LazyIO
+------
+LazyIO relaxes POSIX semantics. Buffered reads/writes are allowed even when a
+file is opened by multiple applications on multiple clients. Applications are
+responsible for managing cache coherency themselves.
diff --git a/doc/cephfs/file-layouts.rst b/doc/cephfs/file-layouts.rst
new file mode 100644
index 000000000..2cdb26efc
--- /dev/null
+++ b/doc/cephfs/file-layouts.rst
@@ -0,0 +1,272 @@
+.. _file-layouts:
+
+File layouts
+============
+
+The layout of a file controls how its contents are mapped to Ceph RADOS objects.  You can
+read and write a file's layout using *virtual extended attributes* or xattrs.
+
+The name of the layout xattrs depends on whether a file is a regular file or a directory.  Regular
+files' layout xattrs are called ``ceph.file.layout``, whereas directories' layout xattrs are called
+``ceph.dir.layout``.  Where subsequent examples refer to ``ceph.file.layout``, substitute ``dir`` as appropriate
+when dealing with directories.
+
+.. tip::
+
+    Your linux distribution may not ship with commands for manipulating xattrs by default,
+    the required package is usually called ``attr``.
+
+Layout fields
+-------------
+
+pool
+    String, giving ID or name. String can only have characters in the set [a-zA-Z0-9\_-.]. Which RADOS pool a file's data objects will be stored in.
+
+pool_id
+    String of digits. This is the system assigned pool id for the RADOS pool whenever it is created.
+
+pool_name
+    String, given name. This is the user defined name for the RADOS pool whenever user creates it.
+
+pool_namespace
+    String with only characters in the set [a-zA-Z0-9\_-.].  Within the data pool, which RADOS namespace the objects will
+    be written to.  Empty by default (i.e. default namespace).
+
+stripe_unit
+    Integer in bytes.  The size (in bytes) of a block of data used in the RAID 0 distribution of a file. All stripe units for a file have equal size. The last stripe unit is typically incomplete–i.e. it represents the data at the end of the file as well as unused “space� beyond it up to the end of the fixed stripe unit size.
+
+stripe_count
+    Integer.  The number of consecutive stripe units that constitute a RAID 0 “stripe� of file data.
+
+object_size
+    Integer in bytes.  File data is chunked into RADOS objects of this size.
+
+.. tip::
+
+    RADOS enforces a configurable limit on object sizes: if you increase CephFS
+    object sizes beyond that limit then writes may not succeed.  The OSD
+    setting is ``osd_max_object_size``, which is 128MB by default.
+    Very large RADOS objects may prevent smooth operation of the cluster,
+    so increasing the object size limit past the default is not recommended.
+
+Reading layouts with ``getfattr``
+---------------------------------
+
+Read the layout information as a single string:
+
+.. code-block:: bash
+
+    $ touch file
+    $ getfattr -n ceph.file.layout file
+    # file: file
+    ceph.file.layout="stripe_unit=4194304 stripe_count=1 object_size=4194304 pool=cephfs_data"
+
+Read individual layout fields:
+
+.. code-block:: bash
+
+    $ getfattr -n ceph.file.layout.pool_name file
+    # file: file
+    ceph.file.layout.pool_name="cephfs_data"
+    $ getfattr -n ceph.file.layout.pool_id file
+    # file: file
+    ceph.file.layout.pool_id="5"
+    $ getfattr -n ceph.file.layout.pool file
+    # file: file
+    ceph.file.layout.pool="cephfs_data"
+    $ getfattr -n ceph.file.layout.stripe_unit file
+    # file: file
+    ceph.file.layout.stripe_unit="4194304"
+    $ getfattr -n ceph.file.layout.stripe_count file
+    # file: file
+    ceph.file.layout.stripe_count="1"
+    $ getfattr -n ceph.file.layout.object_size file
+    # file: file
+    ceph.file.layout.object_size="4194304"    
+
+.. note::
+
+    When reading layouts, the pool will usually be indicated by name.  However, in 
+    rare cases when pools have only just been created, the ID may be output instead.
+
+Directories do not have an explicit layout until it is customized.  Attempts to read
+the layout will fail if it has never been modified: this indicates that layout of the
+next ancestor directory with an explicit layout will be used.
+
+.. code-block:: bash
+
+    $ mkdir dir
+    $ getfattr -n ceph.dir.layout dir
+    dir: ceph.dir.layout: No such attribute
+    $ setfattr -n ceph.dir.layout.stripe_count -v 2 dir
+    $ getfattr -n ceph.dir.layout dir
+    # file: dir
+    ceph.dir.layout="stripe_unit=4194304 stripe_count=2 object_size=4194304 pool=cephfs_data"
+
+Getting the layout in json format. If there's no specific layout set for the
+particular inode, the system traverses the directory path backwards and finds
+the closest ancestor directory with a layout and returns it in json format.
+A file layout also can be retrieved in json format using ``ceph.file.layout.json`` vxattr.
+
+A virtual field named ``inheritance`` is added to the json output to show the status of layout.
+The ``inheritance`` field can have the following values:
+
+``@default`` implies the system default layout
+``@set`` implies that a specific layout has been set for that particular inode
+``@inherited`` implies that the returned layout has been inherited from an ancestor
+
+.. code-block:: bash
+
+   $ getfattr -n ceph.dir.layout.json --only-values /mnt/mycephs/accounts
+   {"stripe_unit": 4194304, "stripe_count": 1, "object_size": 4194304, "pool_name": "cephfs.a.data", "pool_id": 3, "pool_namespace": "", "inheritance": "@default"}
+
+
+Writing layouts with ``setfattr``
+---------------------------------
+
+Layout fields are modified using ``setfattr``:
+
+.. code-block:: bash
+
+    $ ceph osd lspools
+    0 rbd
+    1 cephfs_data
+    2 cephfs_metadata
+
+    $ setfattr -n ceph.file.layout.stripe_unit -v 1048576 file2
+    $ setfattr -n ceph.file.layout.stripe_count -v 8 file2
+    $ setfattr -n ceph.file.layout.object_size -v 10485760 file2
+    $ setfattr -n ceph.file.layout.pool -v 1 file2  # Setting pool by ID
+    $ setfattr -n ceph.file.layout.pool -v cephfs_data file2  # Setting pool by name
+    $ setfattr -n ceph.file.layout.pool_id -v 1 file2  # Setting pool by ID
+    $ setfattr -n ceph.file.layout.pool_name -v cephfs_data file2  # Setting pool by name
+
+.. note::
+
+    When the layout fields of a file are modified using ``setfattr``, this file must be empty, otherwise an error will occur.
+
+.. code-block:: bash
+
+    # touch an empty file
+    $ touch file1
+    # modify layout field successfully
+    $ setfattr -n ceph.file.layout.stripe_count -v 3 file1
+
+    # write something to file1
+    $ echo "hello world" > file1
+    $ setfattr -n ceph.file.layout.stripe_count -v 4 file1
+    setfattr: file1: Directory not empty
+
+File and Directory layouts can also be set using the json format.
+The ``inheritance`` field is ignored when setting the layout.
+Also, if both, ``pool_name`` and ``pool_id`` fields are specified, then the
+``pool_name`` is given preference for better disambiguation.
+
+.. code-block:: bash
+
+   $ setfattr -n ceph.file.layout.json -v '{"stripe_unit": 4194304, "stripe_count": 1, "object_size": 4194304, "pool_name": "cephfs.a.data", "pool_id": 3, "pool_namespace": "", "inheritance": "@default"}' file1
+    
+Clearing layouts
+----------------
+
+If you wish to remove an explicit layout from a directory, to revert to
+inheriting the layout of its ancestor, you can do so:
+
+.. code-block:: bash
+
+    setfattr -x ceph.dir.layout mydir
+
+Similarly, if you have set the ``pool_namespace`` attribute and wish
+to modify the layout to use the default namespace instead:
+
+.. code-block:: bash
+
+    # Create a dir and set a namespace on it
+    mkdir mydir
+    setfattr -n ceph.dir.layout.pool_namespace -v foons mydir
+    getfattr -n ceph.dir.layout mydir
+    ceph.dir.layout="stripe_unit=4194304 stripe_count=1 object_size=4194304 pool=cephfs_data_a pool_namespace=foons"
+
+    # Clear the namespace from the directory's layout
+    setfattr -x ceph.dir.layout.pool_namespace mydir
+    getfattr -n ceph.dir.layout mydir
+    ceph.dir.layout="stripe_unit=4194304 stripe_count=1 object_size=4194304 pool=cephfs_data_a"
+
+
+Inheritance of layouts
+----------------------
+
+Files inherit the layout of their parent directory at creation time.  However, subsequent
+changes to the parent directory's layout do not affect children.
+
+.. code-block:: bash
+
+    $ getfattr -n ceph.dir.layout dir
+    # file: dir
+    ceph.dir.layout="stripe_unit=4194304 stripe_count=2 object_size=4194304 pool=cephfs_data"
+
+    # Demonstrate file1 inheriting its parent's layout
+    $ touch dir/file1
+    $ getfattr -n ceph.file.layout dir/file1
+    # file: dir/file1
+    ceph.file.layout="stripe_unit=4194304 stripe_count=2 object_size=4194304 pool=cephfs_data"
+
+    # Now update the layout of the directory before creating a second file
+    $ setfattr -n ceph.dir.layout.stripe_count -v 4 dir
+    $ touch dir/file2
+
+    # Demonstrate that file1's layout is unchanged
+    $ getfattr -n ceph.file.layout dir/file1
+    # file: dir/file1
+    ceph.file.layout="stripe_unit=4194304 stripe_count=2 object_size=4194304 pool=cephfs_data"
+
+    # ...while file2 has the parent directory's new layout
+    $ getfattr -n ceph.file.layout dir/file2
+    # file: dir/file2
+    ceph.file.layout="stripe_unit=4194304 stripe_count=4 object_size=4194304 pool=cephfs_data"
+
+
+Files created as descendents of the directory also inherit the layout, if the intermediate
+directories do not have layouts set:
+
+.. code-block:: bash
+
+    $ getfattr -n ceph.dir.layout dir
+    # file: dir
+    ceph.dir.layout="stripe_unit=4194304 stripe_count=4 object_size=4194304 pool=cephfs_data"
+    $ mkdir dir/childdir
+    $ getfattr -n ceph.dir.layout dir/childdir
+    dir/childdir: ceph.dir.layout: No such attribute
+    $ touch dir/childdir/grandchild
+    $ getfattr -n ceph.file.layout dir/childdir/grandchild
+    # file: dir/childdir/grandchild
+    ceph.file.layout="stripe_unit=4194304 stripe_count=4 object_size=4194304 pool=cephfs_data"
+
+
+.. _adding-data-pool-to-file-system:
+    
+Adding a data pool to the File System 
+-------------------------------------
+
+Before you can use a pool with CephFS you have to add it to the Metadata Servers.
+
+.. code-block:: bash
+
+    $ ceph fs add_data_pool cephfs cephfs_data_ssd
+    $ ceph fs ls  # Pool should now show up
+    .... data pools: [cephfs_data cephfs_data_ssd ]
+
+Make sure that your cephx keys allows the client to access this new pool.
+
+You can then update the layout on a directory in CephFS to use the pool you added:
+
+.. code-block:: bash
+
+    $ mkdir /mnt/cephfs/myssddir
+    $ setfattr -n ceph.dir.layout.pool -v cephfs_data_ssd /mnt/cephfs/myssddir
+
+All new files created within that directory will now inherit its layout and place their data in your newly added pool. 
+
+You may notice that object counts in your primary data pool (the one passed to ``fs new``) continue to increase, even if files are being created in the pool you added.  This is normal: the file data is stored in the pool specified by the layout, but a small amount of metadata is kept in the primary data pool for all files.
+
+
diff --git a/doc/cephfs/fs-volumes.rst b/doc/cephfs/fs-volumes.rst
new file mode 100644
index 000000000..ee5a7dc87
--- /dev/null
+++ b/doc/cephfs/fs-volumes.rst
@@ -0,0 +1,627 @@
+.. _fs-volumes-and-subvolumes:
+
+FS volumes and subvolumes
+=========================
+
+A  single source of truth for CephFS exports is implemented in the volumes
+module of the :term:`Ceph Manager` daemon (ceph-mgr). The OpenStack shared
+file system service (manila_), Ceph Container Storage Interface (CSI_),
+storage administrators among others can use the common CLI provided by the
+ceph-mgr volumes module to manage the CephFS exports.
+
+The ceph-mgr volumes module implements the following file system export
+abstactions:
+
+* FS volumes, an abstraction for CephFS file systems
+
+* FS subvolumes, an abstraction for independent CephFS directory trees
+
+* FS subvolume groups, an abstraction for a directory level higher than FS
+  subvolumes to effect policies (e.g., :doc:`/cephfs/file-layouts`) across a
+  set of subvolumes
+
+Some possible use-cases for the export abstractions:
+
+* FS subvolumes used as manila shares or CSI volumes
+
+* FS subvolume groups used as manila share groups
+
+Requirements
+------------
+
+* Nautilus (14.2.x) or a later version of Ceph
+
+* Cephx client user (see :doc:`/rados/operations/user-management`) with
+  the following minimum capabilities::
+
+    mon 'allow r'
+    mgr 'allow rw'
+
+
+FS Volumes
+----------
+
+Create a volume using::
+
+    $ ceph fs volume create <vol_name> [<placement>]
+
+This creates a CephFS file system and its data and metadata pools. It can also
+try to create MDSes for the filesystem using the enabled ceph-mgr orchestrator
+module (see :doc:`/mgr/orchestrator`), e.g. rook.
+
+<vol_name> is the volume name (an arbitrary string), and
+
+<placement> is an optional string signifying which hosts should have NFS Ganesha
+daemon containers running on them and, optionally, the total number of NFS
+Ganesha daemons the cluster (should you want to have more than one NFS Ganesha
+daemon running per node). For example, the following placement string means
+"deploy NFS Ganesha daemons on nodes host1 and host2 (one daemon per host):
+
+    "host1,host2"
+
+and this placement specification says to deploy two NFS Ganesha daemons each
+on nodes host1 and host2 (for a total of four NFS Ganesha daemons in the
+cluster):
+
+    "4 host1,host2"
+
+For more details on placement specification refer to the `orchestrator doc
+<https://docs.ceph.com/docs/master/mgr/orchestrator/#placement-specification>`_
+but keep in mind that specifying the placement via a YAML file is not supported.
+
+Remove a volume using::
+
+    $ ceph fs volume rm <vol_name> [--yes-i-really-mean-it]
+
+This removes a file system and its data and metadata pools. It also tries to
+remove MDSes using the enabled ceph-mgr orchestrator module.
+
+List volumes using::
+
+    $ ceph fs volume ls
+
+Fetch the information of a CephFS volume using::
+
+    $ ceph fs volume info vol_name [--human_readable]
+
+The ``--human_readable`` flag shows used and available pool capacities in KB/MB/GB.
+
+The output format is JSON and contains fields as follows:
+
+* pools: Attributes of data and metadata pools
+        * avail: The amount of free space available in bytes
+        * used: The amount of storage consumed in bytes
+        * name: Name of the pool
+* mon_addrs: List of monitor addresses
+* used_size: Current used size of the CephFS volume in bytes
+* pending_subvolume_deletions: Number of subvolumes pending deletion
+
+Sample output of volume info command::
+
+  $ ceph fs volume info vol_name
+  {
+      "mon_addrs": [
+          "192.168.1.7:40977"
+      ],
+      "pending_subvolume_deletions": 0,
+      "pools": {
+          "data": [
+              {
+                  "avail": 106288709632,
+                  "name": "cephfs.vol_name.data",
+                  "used": 4096
+              }
+          ],
+          "metadata": [
+              {
+                  "avail": 106288709632,
+                  "name": "cephfs.vol_name.meta",
+                  "used": 155648
+              }
+          ]
+      },
+      "used_size": 0
+  }
+
+FS Subvolume groups
+-------------------
+
+Create a subvolume group using::
+
+    $ ceph fs subvolumegroup create <vol_name> <group_name> [--size <size_in_bytes>] [--pool_layout <data_pool_name>] [--uid <uid>] [--gid <gid>] [--mode <octal_mode>]
+
+The command succeeds even if the subvolume group already exists.
+
+When creating a subvolume group you can specify its data pool layout (see
+:doc:`/cephfs/file-layouts`), uid, gid, file mode in octal numerals and
+size in bytes. The size of the subvolume group is specified by setting
+a quota on it (see :doc:`/cephfs/quota`). By default, the subvolume group
+is created with an octal file mode '755', uid '0', gid '0' and the data pool
+layout of its parent directory.
+
+
+Remove a subvolume group using::
+
+    $ ceph fs subvolumegroup rm <vol_name> <group_name> [--force]
+
+The removal of a subvolume group fails if it is not empty or non-existent.
+'--force' flag allows the non-existent subvolume group remove command to succeed.
+
+
+Fetch the absolute path of a subvolume group using::
+
+    $ ceph fs subvolumegroup getpath <vol_name> <group_name>
+
+List subvolume groups using::
+
+    $ ceph fs subvolumegroup ls <vol_name>
+
+.. note:: Subvolume group snapshot feature is no longer supported in mainline CephFS (existing group
+          snapshots can still be listed and deleted)
+
+Fetch the metadata of a subvolume group using::
+
+    $ ceph fs subvolumegroup info <vol_name> <group_name>
+
+The output format is JSON and contains fields as follows.
+
+* atime: access time of subvolume group path in the format "YYYY-MM-DD HH:MM:SS"
+* mtime: modification time of subvolume group path in the format "YYYY-MM-DD HH:MM:SS"
+* ctime: change time of subvolume group path in the format "YYYY-MM-DD HH:MM:SS"
+* uid: uid of subvolume group path
+* gid: gid of subvolume group path
+* mode: mode of subvolume group path
+* mon_addrs: list of monitor addresses
+* bytes_pcent: quota used in percentage if quota is set, else displays "undefined"
+* bytes_quota: quota size in bytes if quota is set, else displays "infinite"
+* bytes_used: current used size of the subvolume group in bytes
+* created_at: time of creation of subvolume group in the format "YYYY-MM-DD HH:MM:SS"
+* data_pool: data pool the subvolume group belongs to
+
+Check the presence of any subvolume group using::
+
+    $ ceph fs subvolumegroup exist <vol_name>
+
+The strings returned by the 'exist' command:
+
+* "subvolumegroup exists": if any subvolumegroup is present
+* "no subvolumegroup exists": if no subvolumegroup is present
+
+.. note:: It checks for the presence of custom groups and not the default one. To validate the emptiness of the volume, subvolumegroup existence check alone is not sufficient. The subvolume existence also needs to be checked as there might be subvolumes in the default group.
+
+Resize a subvolume group using::
+
+    $ ceph fs subvolumegroup resize <vol_name> <group_name> <new_size> [--no_shrink]
+
+The command resizes the subvolume group quota using the size specified by 'new_size'.
+The '--no_shrink' flag prevents the subvolume group to shrink below the current used
+size of the subvolume group.
+
+The subvolume group can be resized to an unlimited size by passing 'inf' or 'infinite'
+as the new_size.
+
+Remove a snapshot of a subvolume group using::
+
+    $ ceph fs subvolumegroup snapshot rm <vol_name> <group_name> <snap_name> [--force]
+
+Using the '--force' flag allows the command to succeed that would otherwise
+fail if the snapshot did not exist.
+
+List snapshots of a subvolume group using::
+
+    $ ceph fs subvolumegroup snapshot ls <vol_name> <group_name>
+
+
+FS Subvolumes
+-------------
+
+Create a subvolume using::
+
+    $ ceph fs subvolume create <vol_name> <subvol_name> [--size <size_in_bytes>] [--group_name <subvol_group_name>] [--pool_layout <data_pool_name>] [--uid <uid>] [--gid <gid>] [--mode <octal_mode>] [--namespace-isolated]
+
+
+The command succeeds even if the subvolume already exists.
+
+When creating a subvolume you can specify its subvolume group, data pool layout,
+uid, gid, file mode in octal numerals, and size in bytes. The size of the subvolume is
+specified by setting a quota on it (see :doc:`/cephfs/quota`). The subvolume can be
+created in a separate RADOS namespace by specifying --namespace-isolated option. By
+default a subvolume is created within the default subvolume group, and with an octal file
+mode '755', uid of its subvolume group, gid of its subvolume group, data pool layout of
+its parent directory and no size limit.
+
+Remove a subvolume using::
+
+    $ ceph fs subvolume rm <vol_name> <subvol_name> [--group_name <subvol_group_name>] [--force] [--retain-snapshots]
+
+
+The command removes the subvolume and its contents. It does this in two steps.
+First, it moves the subvolume to a trash folder, and then asynchronously purges
+its contents.
+
+The removal of a subvolume fails if it has snapshots, or is non-existent.
+'--force' flag allows the non-existent subvolume remove command to succeed.
+
+A subvolume can be removed retaining existing snapshots of the subvolume using the
+'--retain-snapshots' option. If snapshots are retained, the subvolume is considered
+empty for all operations not involving the retained snapshots.
+
+.. note:: Snapshot retained subvolumes can be recreated using 'ceph fs subvolume create'
+
+.. note:: Retained snapshots can be used as a clone source to recreate the subvolume, or clone to a newer subvolume.
+
+Resize a subvolume using::
+
+    $ ceph fs subvolume resize <vol_name> <subvol_name> <new_size> [--group_name <subvol_group_name>] [--no_shrink]
+
+The command resizes the subvolume quota using the size specified by 'new_size'.
+'--no_shrink' flag prevents the subvolume to shrink below the current used size of the subvolume.
+
+The subvolume can be resized to an infinite size by passing 'inf' or 'infinite' as the new_size.
+
+Authorize cephx auth IDs, the read/read-write access to fs subvolumes::
+
+    $ ceph fs subvolume authorize <vol_name> <sub_name> <auth_id> [--group_name=<group_name>] [--access_level=<access_level>]
+
+The 'access_level' takes 'r' or 'rw' as value.
+
+Deauthorize cephx auth IDs, the read/read-write access to fs subvolumes::
+
+    $ ceph fs subvolume deauthorize <vol_name> <sub_name> <auth_id> [--group_name=<group_name>]
+
+List cephx auth IDs authorized to access fs subvolume::
+
+    $ ceph fs subvolume authorized_list <vol_name> <sub_name> [--group_name=<group_name>]
+
+Evict fs clients based on auth ID and subvolume mounted::
+
+    $ ceph fs subvolume evict <vol_name> <sub_name> <auth_id> [--group_name=<group_name>]
+
+Fetch the absolute path of a subvolume using::
+
+    $ ceph fs subvolume getpath <vol_name> <subvol_name> [--group_name <subvol_group_name>]
+
+Fetch the information of a subvolume using::
+
+    $ ceph fs subvolume info <vol_name> <subvol_name> [--group_name <subvol_group_name>]
+
+The output format is json and contains fields as follows.
+
+* atime: access time of subvolume path in the format "YYYY-MM-DD HH:MM:SS"
+* mtime: modification time of subvolume path in the format "YYYY-MM-DD HH:MM:SS"
+* ctime: change time of subvolume path in the format "YYYY-MM-DD HH:MM:SS"
+* uid: uid of subvolume path
+* gid: gid of subvolume path
+* mode: mode of subvolume path
+* mon_addrs: list of monitor addresses
+* bytes_pcent: quota used in percentage if quota is set, else displays "undefined"
+* bytes_quota: quota size in bytes if quota is set, else displays "infinite"
+* bytes_used: current used size of the subvolume in bytes
+* created_at: time of creation of subvolume in the format "YYYY-MM-DD HH:MM:SS"
+* data_pool: data pool the subvolume belongs to
+* path: absolute path of a subvolume
+* type: subvolume type indicating whether it's clone or subvolume
+* pool_namespace: RADOS namespace of the subvolume
+* features: features supported by the subvolume
+* state: current state of the subvolume
+
+If a subvolume has been removed retaining its snapshots, the output only contains fields as follows.
+
+* type: subvolume type indicating whether it's clone or subvolume
+* features: features supported by the subvolume
+* state: current state of the subvolume
+
+The subvolume "features" are based on the internal version of the subvolume and is a list containing
+a subset of the following features,
+
+* "snapshot-clone": supports cloning using a subvolumes snapshot as the source
+* "snapshot-autoprotect": supports automatically protecting snapshots, that are active clone sources, from deletion
+* "snapshot-retention": supports removing subvolume contents, retaining any existing snapshots
+
+The subvolume "state" is based on the current state of the subvolume and contains one of the following values.
+
+* "complete": subvolume is ready for all operations
+* "snapshot-retained": subvolume is removed but its snapshots are retained
+
+List subvolumes using::
+
+    $ ceph fs subvolume ls <vol_name> [--group_name <subvol_group_name>]
+
+.. note:: subvolumes that are removed but have snapshots retained, are also listed.
+
+Check the presence of any subvolume using::
+
+    $ ceph fs subvolume exist <vol_name> [--group_name <subvol_group_name>]
+
+The strings returned by the 'exist' command:
+
+* "subvolume exists": if any subvolume of given group_name is present
+* "no subvolume exists": if no subvolume of given group_name is present
+
+Set custom metadata on the subvolume as a key-value pair using::
+
+    $ ceph fs subvolume metadata set <vol_name> <subvol_name> <key_name> <value> [--group_name <subvol_group_name>]
+
+.. note:: If the key_name already exists then the old value will get replaced by the new value.
+
+.. note:: key_name and value should be a string of ASCII characters (as specified in python's string.printable). key_name is case-insensitive and always stored in lower case.
+
+.. note:: Custom metadata on a subvolume is not preserved when snapshotting the subvolume, and hence, is also not preserved when cloning the subvolume snapshot.
+
+Get custom metadata set on the subvolume using the metadata key::
+
+    $ ceph fs subvolume metadata get <vol_name> <subvol_name> <key_name> [--group_name <subvol_group_name>]
+
+List custom metadata (key-value pairs) set on the subvolume using::
+
+    $ ceph fs subvolume metadata ls <vol_name> <subvol_name> [--group_name <subvol_group_name>]
+
+Remove custom metadata set on the subvolume using the metadata key::
+
+    $ ceph fs subvolume metadata rm <vol_name> <subvol_name> <key_name> [--group_name <subvol_group_name>] [--force]
+
+Using the '--force' flag allows the command to succeed that would otherwise
+fail if the metadata key did not exist.
+
+Create a snapshot of a subvolume using::
+
+    $ ceph fs subvolume snapshot create <vol_name> <subvol_name> <snap_name> [--group_name <subvol_group_name>]
+
+
+Remove a snapshot of a subvolume using::
+
+    $ ceph fs subvolume snapshot rm <vol_name> <subvol_name> <snap_name> [--group_name <subvol_group_name>] [--force]
+
+Using the '--force' flag allows the command to succeed that would otherwise
+fail if the snapshot did not exist.
+
+.. note:: if the last snapshot within a snapshot retained subvolume is removed, the subvolume is also removed
+
+List snapshots of a subvolume using::
+
+    $ ceph fs subvolume snapshot ls <vol_name> <subvol_name> [--group_name <subvol_group_name>]
+
+Fetch the information of a snapshot using::
+
+    $ ceph fs subvolume snapshot info <vol_name> <subvol_name> <snap_name> [--group_name <subvol_group_name>]
+
+The output format is JSON and contains fields as follows.
+
+* created_at: time of creation of snapshot in the format "YYYY-MM-DD HH:MM:SS:ffffff"
+* data_pool: data pool the snapshot belongs to
+* has_pending_clones: "yes" if snapshot clone is in progress otherwise "no"
+* pending_clones: list of in progress or pending clones and their target group if exist otherwise this field is not shown
+* orphan_clones_count: count of orphan clones if snapshot has orphan clones otherwise this field is not shown
+
+Sample output when snapshot clones are in progress or pending state::
+
+  $ ceph fs subvolume snapshot info cephfs subvol snap
+  {
+      "created_at": "2022-06-14 13:54:58.618769",
+      "data_pool": "cephfs.cephfs.data",
+      "has_pending_clones": "yes",
+      "pending_clones": [
+          {
+              "name": "clone_1",
+              "target_group": "target_subvol_group"
+          },
+          {
+              "name": "clone_2"
+          },
+          {
+              "name": "clone_3",
+              "target_group": "target_subvol_group"
+          }
+      ]
+  }
+
+Sample output when no snapshot clone is in progress or pending state::
+
+  $ ceph fs subvolume snapshot info cephfs subvol snap
+  {
+      "created_at": "2022-06-14 13:54:58.618769",
+      "data_pool": "cephfs.cephfs.data",
+      "has_pending_clones": "no"
+  }
+
+Set custom metadata on the snapshot as a key-value pair using::
+
+    $ ceph fs subvolume snapshot metadata set <vol_name> <subvol_name> <snap_name> <key_name> <value> [--group_name <subvol_group_name>]
+
+.. note:: If the key_name already exists then the old value will get replaced by the new value.
+
+.. note:: The key_name and value should be a string of ASCII characters (as specified in python's string.printable). The key_name is case-insensitive and always stored in lower case.
+
+.. note:: Custom metadata on a snapshots is not preserved when snapshotting the subvolume, and hence, is also not preserved when cloning the subvolume snapshot.
+
+Get custom metadata set on the snapshot using the metadata key::
+
+    $ ceph fs subvolume snapshot metadata get <vol_name> <subvol_name> <snap_name> <key_name> [--group_name <subvol_group_name>]
+
+List custom metadata (key-value pairs) set on the snapshot using::
+
+    $ ceph fs subvolume snapshot metadata ls <vol_name> <subvol_name> <snap_name> [--group_name <subvol_group_name>]
+
+Remove custom metadata set on the snapshot using the metadata key::
+
+    $ ceph fs subvolume snapshot metadata rm <vol_name> <subvol_name> <snap_name> <key_name> [--group_name <subvol_group_name>] [--force]
+
+Using the '--force' flag allows the command to succeed that would otherwise
+fail if the metadata key did not exist.
+
+Cloning Snapshots
+-----------------
+
+Subvolumes can be created by cloning subvolume snapshots. Cloning is an asynchronous operation involving copying
+data from a snapshot to a subvolume. Due to this bulk copy nature, cloning is currently inefficient for very huge
+data sets.
+
+.. note:: Removing a snapshot (source subvolume) would fail if there are pending or in progress clone operations.
+
+Protecting snapshots prior to cloning was a pre-requisite in the Nautilus release, and the commands to protect/unprotect
+snapshots were introduced for this purpose. This pre-requisite, and hence the commands to protect/unprotect, is being
+deprecated in mainline CephFS, and may be removed from a future release.
+
+The commands being deprecated are:
+  $ ceph fs subvolume snapshot protect <vol_name> <subvol_name> <snap_name> [--group_name <subvol_group_name>]
+  $ ceph fs subvolume snapshot unprotect <vol_name> <subvol_name> <snap_name> [--group_name <subvol_group_name>]
+
+.. note:: Using the above commands would not result in an error, but they serve no useful function.
+
+.. note:: Use subvolume info command to fetch subvolume metadata regarding supported "features" to help decide if protect/unprotect of snapshots is required, based on the "snapshot-autoprotect" feature availability.
+
+To initiate a clone operation use::
+
+  $ ceph fs subvolume snapshot clone <vol_name> <subvol_name> <snap_name> <target_subvol_name>
+
+If a snapshot (source subvolume) is a part of non-default group, the group name needs to be specified as per::
+
+  $ ceph fs subvolume snapshot clone <vol_name> <subvol_name> <snap_name> <target_subvol_name> --group_name <subvol_group_name>
+
+Cloned subvolumes can be a part of a different group than the source snapshot (by default, cloned subvolumes are created in default group). To clone to a particular group use::
+
+  $ ceph fs subvolume snapshot clone <vol_name> <subvol_name> <snap_name> <target_subvol_name> --target_group_name <subvol_group_name>
+
+Similar to specifying a pool layout when creating a subvolume, pool layout can be specified when creating a cloned subvolume. To create a cloned subvolume with a specific pool layout use::
+
+  $ ceph fs subvolume snapshot clone <vol_name> <subvol_name> <snap_name> <target_subvol_name> --pool_layout <pool_layout>
+
+Configure maximum number of concurrent clones. The default is set to 4::
+
+  $ ceph config set mgr mgr/volumes/max_concurrent_clones <value>
+
+To check the status of a clone operation use::
+
+  $ ceph fs clone status <vol_name> <clone_name> [--group_name <group_name>]
+
+A clone can be in one of the following states:
+
+#. `pending`     : Clone operation has not started
+#. `in-progress` : Clone operation is in progress
+#. `complete`    : Clone operation has successfully finished
+#. `failed`      : Clone operation has failed
+#. `canceled`    : Clone operation is cancelled by user
+
+The reason for a clone failure is shown as below:
+
+#. `errno`     : error number
+#. `error_msg` : failure error string
+
+Sample output of an `in-progress` clone operation::
+
+  $ ceph fs subvolume snapshot clone cephfs subvol1 snap1 clone1
+  $ ceph fs clone status cephfs clone1
+  {
+    "status": {
+      "state": "in-progress",
+      "source": {
+        "volume": "cephfs",
+        "subvolume": "subvol1",
+        "snapshot": "snap1"
+      }
+    }
+  }
+
+.. note:: The `failure` section will be shown only if the clone is in failed or cancelled state
+
+Sample output of a `failed` clone operation::
+
+  $ ceph fs subvolume snapshot clone cephfs subvol1 snap1 clone1
+  $ ceph fs clone status cephfs clone1
+  {
+    "status": {
+      "state": "failed",
+      "source": {
+        "volume": "cephfs",
+        "subvolume": "subvol1",
+        "snapshot": "snap1"
+        "size": "104857600"
+      },
+      "failure": {
+        "errno": "122",
+        "errstr": "Disk quota exceeded"
+      }
+    }
+  }
+
+(NOTE: since `subvol1` is in default group, `source` section in `clone status` does not include group name)
+
+.. note:: Cloned subvolumes are accessible only after the clone operation has successfully completed.
+
+For a successful clone operation, `clone status` would look like so::
+
+  $ ceph fs clone status cephfs clone1
+  {
+    "status": {
+      "state": "complete"
+    }
+  }
+
+or `failed` state when clone is unsuccessful.
+
+On failure of a clone operation, the partial clone needs to be deleted and the clone operation needs to be retriggered.
+To delete a partial clone use::
+
+  $ ceph fs subvolume rm <vol_name> <clone_name> [--group_name <group_name>] --force
+
+.. note:: Cloning only synchronizes directories, regular files and symbolic links. Also, inode timestamps (access and
+          modification times) are synchronized upto seconds granularity.
+
+An `in-progress` or a `pending` clone operation can be canceled. To cancel a clone operation use the `clone cancel` command::
+
+  $ ceph fs clone cancel <vol_name> <clone_name> [--group_name <group_name>]
+
+On successful cancelation, the cloned subvolume is moved to `canceled` state::
+
+  $ ceph fs subvolume snapshot clone cephfs subvol1 snap1 clone1
+  $ ceph fs clone cancel cephfs clone1
+  $ ceph fs clone status cephfs clone1
+  {
+    "status": {
+      "state": "canceled",
+      "source": {
+        "volume": "cephfs",
+        "subvolume": "subvol1",
+        "snapshot": "snap1"
+      }
+    }
+  }
+
+.. note:: The canceled cloned can be deleted by using --force option in `fs subvolume rm` command.
+
+
+.. _subvol-pinning:
+
+Pinning Subvolumes and Subvolume Groups
+---------------------------------------
+
+
+Subvolumes and subvolume groups can be automatically pinned to ranks according
+to policies. This can help distribute load across MDS ranks in predictable and
+stable ways.  Review :ref:`cephfs-pinning` and :ref:`cephfs-ephemeral-pinning`
+for details on how pinning works.
+
+Pinning is configured by::
+
+  $ ceph fs subvolumegroup pin <vol_name> <group_name> <pin_type> <pin_setting>
+
+or for subvolumes::
+
+  $ ceph fs subvolume pin <vol_name> <group_name> <pin_type> <pin_setting>
+
+Typically you will want to set subvolume group pins. The ``pin_type`` may be
+one of ``export``, ``distributed``, or ``random``. The ``pin_setting``
+corresponds to the extended attributed "value" as in the pinning documentation
+referenced above.
+
+So, for example, setting a distributed pinning strategy on a subvolume group::
+
+  $ ceph fs subvolumegroup pin cephfilesystem-a csi distributed 1
+
+Will enable distributed subtree partitioning policy for the "csi" subvolume
+group.  This will cause every subvolume within the group to be automatically
+pinned to one of the available ranks on the file system.
+
+
+.. _manila: https://github.com/openstack/manila
+.. _CSI: https://github.com/ceph/ceph-csi
diff --git a/doc/cephfs/full.rst b/doc/cephfs/full.rst
new file mode 100644
index 000000000..fe0616cb6
--- /dev/null
+++ b/doc/cephfs/full.rst
@@ -0,0 +1,60 @@
+
+Handling a full Ceph file system
+================================
+
+When a RADOS cluster reaches its ``mon_osd_full_ratio`` (default
+95%) capacity, it is marked with the OSD full flag.  This flag causes
+most normal RADOS clients to pause all operations until it is resolved
+(for example by adding more capacity to the cluster).
+
+The file system has some special handling of the full flag, explained below.
+
+Hammer and later
+----------------
+
+Since the hammer release, a full file system will lead to ENOSPC
+results from:
+
+ * Data writes on the client
+ * Metadata operations other than deletes and truncates
+
+Because the full condition may not be encountered until
+data is flushed to disk (sometime after a ``write`` call has already
+returned 0), the ENOSPC error may not be seen until the application
+calls ``fsync`` or ``fclose`` (or equivalent) on the file handle.
+
+Calling ``fsync`` is guaranteed to reliably indicate whether the data
+made it to disk, and will return an error if it doesn't.  ``fclose`` will
+only return an error if buffered data happened to be flushed since
+the last write -- a successful ``fclose`` does not guarantee that the
+data made it to disk, and in a full-space situation, buffered data
+may be discarded after an ``fclose`` if no space is available to persist it.
+
+.. warning::
+    If an application appears to be misbehaving on a full file system,
+    check that it is performing ``fsync()`` calls as necessary to ensure
+    data is on disk before proceeding.
+
+Data writes may be cancelled by the client if they are in flight at the
+time the OSD full flag is sent.  Clients update the ``osd_epoch_barrier``
+when releasing capabilities on files affected by cancelled operations, in
+order to ensure that these cancelled operations do not interfere with
+subsequent access to the data objects by the MDS or other clients.  For
+more on the epoch barrier mechanism, see :ref:`background_blocklisting_and_osd_epoch_barrier`.
+
+Legacy (pre-hammer) behavior
+----------------------------
+
+In versions of Ceph earlier than hammer, the MDS would ignore
+the full status of the RADOS cluster, and any data writes from
+clients would stall until the cluster ceased to be full.
+
+There are two dangerous conditions to watch for with this behaviour:
+
+* If a client had pending writes to a file, then it was not possible
+  for the client to release the file to the MDS for deletion: this could
+  lead to difficulty clearing space on a full file system
+* If clients continued to create a large number of empty files, the
+  resulting metadata writes from the MDS could lead to total exhaustion
+  of space on the OSDs such that no further deletions could be performed.
+
diff --git a/doc/cephfs/health-messages.rst b/doc/cephfs/health-messages.rst
new file mode 100644
index 000000000..28ceb704a
--- /dev/null
+++ b/doc/cephfs/health-messages.rst
@@ -0,0 +1,170 @@
+
+.. _cephfs-health-messages:
+
+======================
+CephFS health messages
+======================
+
+Cluster health checks
+=====================
+
+The Ceph monitor daemons will generate health messages in response
+to certain states of the file system map structure (and the enclosed MDS maps).
+
+Message: mds rank(s) *ranks* have failed
+Description: One or more MDS ranks are not currently assigned to
+an MDS daemon; the cluster will not recover until a suitable replacement
+daemon starts.
+
+Message: mds rank(s) *ranks* are damaged
+Description: One or more MDS ranks has encountered severe damage to
+its stored metadata, and cannot start again until it is repaired.
+
+Message: mds cluster is degraded
+Description: One or more MDS ranks are not currently up and running, clients
+may pause metadata IO until this situation is resolved.  This includes
+ranks being failed or damaged, and additionally includes ranks
+which are running on an MDS but have not yet made it to the *active*
+state (e.g. ranks currently in *replay* state).
+
+Message: mds *names* are laggy
+Description: The named MDS daemons have failed to send beacon messages
+to the monitor for at least ``mds_beacon_grace`` (default 15s), while
+they are supposed to send beacon messages every ``mds_beacon_interval``
+(default 4s).  The daemons may have crashed.  The Ceph monitor will
+automatically replace laggy daemons with standbys if any are available.
+
+Message: insufficient standby daemons available
+Description: One or more file systems are configured to have a certain number
+of standby daemons available (including daemons in standby-replay) but the
+cluster does not have enough standby daemons. The standby daemons not in replay
+count towards any file system (i.e. they may overlap). This warning can
+configured by setting ``ceph fs set <fs> standby_count_wanted <count>``.  Use
+zero for ``count`` to disable.
+
+
+Daemon-reported health checks
+=============================
+
+MDS daemons can identify a variety of unwanted conditions, and
+indicate these to the operator in the output of ``ceph status``.
+These conditions have human readable messages, and additionally
+a unique code starting with ``MDS_``.
+
+.. highlight:: console
+
+``ceph health detail`` shows the details of the conditions. Following
+is a typical health report from a cluster experiencing MDS related
+performance issues::
+
+  $ ceph health detail
+  HEALTH_WARN 1 MDSs report slow metadata IOs; 1 MDSs report slow requests
+  MDS_SLOW_METADATA_IO 1 MDSs report slow metadata IOs
+     mds.fs-01(mds.0): 3 slow metadata IOs are blocked > 30 secs, oldest blocked for 51123 secs
+  MDS_SLOW_REQUEST 1 MDSs report slow requests
+     mds.fs-01(mds.0): 5 slow requests are blocked > 30 secs
+
+Where, for intance, ``MDS_SLOW_REQUEST`` is the unique code representing the
+condition where requests are taking long time to complete. And the following
+description shows its severity and the MDS daemons which are serving these
+slow requests.
+
+This page lists the health checks raised by MDS daemons. For the checks from
+other daemons, please see :ref:`health-checks`.
+
+* ``MDS_TRIM``
+
+  Message
+    "Behind on trimming..."
+  Description
+    CephFS maintains a metadata journal that is divided into
+    *log segments*.  The length of journal (in number of segments) is controlled
+    by the setting ``mds_log_max_segments``, and when the number of segments
+    exceeds that setting the MDS starts writing back metadata so that it
+    can remove (trim) the oldest segments.  If this writeback is happening
+    too slowly, or a software bug is preventing trimming, then this health
+    message may appear.  The threshold for this message to appear is controlled by
+    the config option ``mds_log_warn_factor``, the default is 2.0.
+* ``MDS_HEALTH_CLIENT_LATE_RELEASE``, ``MDS_HEALTH_CLIENT_LATE_RELEASE_MANY``
+
+  Message
+    "Client *name* failing to respond to capability release"
+  Description
+    CephFS clients are issued *capabilities* by the MDS, which
+    are like locks.  Sometimes, for example when another client needs access,
+    the MDS will request clients release their capabilities.  If the client
+    is unresponsive or buggy, it might fail to do so promptly or fail to do
+    so at all.  This message appears if a client has taken longer than
+    ``session_timeout`` (default 60s) to comply.
+* ``MDS_CLIENT_RECALL``, ``MDS_HEALTH_CLIENT_RECALL_MANY``
+
+  Message
+    "Client *name* failing to respond to cache pressure"
+  Description
+    Clients maintain a metadata cache.  Items (such as inodes) in the
+    client cache are also pinned in the MDS cache, so when the MDS needs to shrink
+    its cache (to stay within ``mds_cache_memory_limit``), it sends messages to
+    clients to shrink their caches too.  If the client is unresponsive or buggy,
+    this can prevent the MDS from properly staying within its cache limits and it
+    may eventually run out of memory and crash.  This message appears if a client
+    has failed to release more than
+    ``mds_recall_warning_threshold`` capabilities (decaying with a half-life of
+    ``mds_recall_max_decay_rate``) within the last
+    ``mds_recall_warning_decay_rate`` second.
+* ``MDS_CLIENT_OLDEST_TID``, ``MDS_CLIENT_OLDEST_TID_MANY``
+
+  Message
+    "Client *name* failing to advance its oldest client/flush tid"
+  Description
+    The CephFS client-MDS protocol uses a field called the
+    *oldest tid* to inform the MDS of which client requests are fully
+    complete and may therefore be forgotten about by the MDS.  If a buggy
+    client is failing to advance this field, then the MDS may be prevented
+    from properly cleaning up resources used by client requests.  This message
+    appears if a client appears to have more than ``max_completed_requests``
+    (default 100000) requests that are complete on the MDS side but haven't
+    yet been accounted for in the client's *oldest tid* value.
+* ``MDS_DAMAGE``
+
+  Message
+    "Metadata damage detected"
+  Description
+    Corrupt or missing metadata was encountered when reading
+    from the metadata pool.  This message indicates that the damage was
+    sufficiently isolated for the MDS to continue operating, although
+    client accesses to the damaged subtree will return IO errors.  Use
+    the ``damage ls`` admin socket command to get more detail on the damage.
+    This message appears as soon as any damage is encountered.
+* ``MDS_HEALTH_READ_ONLY``
+
+  Message
+    "MDS in read-only mode"
+  Description
+    The MDS has gone into readonly mode and will return EROFS
+    error codes to client operations that attempt to modify any metadata.  The
+    MDS will go into readonly mode if it encounters a write error while
+    writing to the metadata pool, or if forced to by an administrator using
+    the *force_readonly* admin socket command.
+* ``MDS_SLOW_REQUEST``
+
+  Message
+    "*N* slow requests are blocked"
+
+  Description
+    One or more client requests have not been completed promptly,
+    indicating that the MDS is either running very slowly, or that the RADOS
+    cluster is not acknowledging journal writes promptly, or that there is a bug.
+    Use the ``ops`` admin socket command to list outstanding metadata operations.
+    This message appears if any client requests have taken longer than
+    ``mds_op_complaint_time`` (default 30s).
+* ``MDS_CACHE_OVERSIZED``
+
+  Message
+    "Too many inodes in cache"
+  Description
+    The MDS is not succeeding in trimming its cache to comply with the
+    limit set by the administrator.  If the MDS cache becomes too large, the daemon
+    may exhaust available memory and crash.  By default, this message appears if
+    the actual cache size (in memory) is at least 50% greater than
+    ``mds_cache_memory_limit`` (default 1GB). Modify ``mds_health_cache_threshold``
+    to set the warning ratio.
diff --git a/doc/cephfs/index.rst b/doc/cephfs/index.rst
new file mode 100644
index 000000000..ad3863985
--- /dev/null
+++ b/doc/cephfs/index.rst
@@ -0,0 +1,211 @@
+.. _ceph-file-system:
+
+=================
+ Ceph File System
+=================
+
+The Ceph File System, or **CephFS**, is a POSIX-compliant file system built on
+top of Ceph's distributed object store, **RADOS**. CephFS endeavors to provide
+a state-of-the-art, multi-use, highly available, and performant file store for
+a variety of applications, including traditional use-cases like shared home
+directories, HPC scratch space, and distributed workflow shared storage.
+
+CephFS achieves these goals through the use of some novel architectural
+choices.  Notably, file metadata is stored in a separate RADOS pool from file
+data and served via a resizable cluster of *Metadata Servers*, or **MDS**,
+which may scale to support higher throughput metadata workloads.  Clients of
+the file system have direct access to RADOS for reading and writing file data
+blocks. For this reason, workloads may linearly scale with the size of the
+underlying RADOS object store; that is, there is no gateway or broker mediating
+data I/O for clients.
+
+Access to data is coordinated through the cluster of MDS which serve as
+authorities for the state of the distributed metadata cache cooperatively
+maintained by clients and MDS. Mutations to metadata are aggregated by each MDS
+into a series of efficient writes to a journal on RADOS; no metadata state is
+stored locally by the MDS. This model allows for coherent and rapid
+collaboration between clients within the context of a POSIX file system.
+
+.. image:: cephfs-architecture.svg
+
+CephFS is the subject of numerous academic papers for its novel designs and
+contributions to file system research. It is the oldest storage interface in
+Ceph and was once the primary use-case for RADOS.  Now it is joined by two
+other storage interfaces to form a modern unified storage system: RBD (Ceph
+Block Devices) and RGW (Ceph Object Storage Gateway).
+
+
+Getting Started with CephFS
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+For most deployments of Ceph, setting up a CephFS file system is as simple as:
+
+.. code:: bash
+
+    ceph fs volume create <fs name>
+
+The Ceph `Orchestrator`_  will automatically create and configure MDS for
+your file system if the back-end deployment technology supports it (see
+`Orchestrator deployment table`_). Otherwise, please `deploy MDS manually
+as needed`_.
+
+Finally, to mount CephFS on your client nodes, see `Mount CephFS:
+Prerequisites`_ page. Additionally, a command-line shell utility is available
+for interactive access or scripting via the `cephfs-shell`_.
+
+.. _Orchestrator: ../mgr/orchestrator
+.. _deploy MDS manually as needed: add-remove-mds
+.. _Orchestrator deployment table: ../mgr/orchestrator/#current-implementation-status
+.. _Mount CephFS\: Prerequisites: mount-prerequisites
+.. _cephfs-shell: cephfs-shell
+
+
+.. raw:: html
+
+   <!---
+
+Administration
+^^^^^^^^^^^^^^
+
+.. raw:: html
+
+   --->
+
+.. toctree:: 
+   :maxdepth: 1
+   :hidden:
+
+    Create a CephFS file system <createfs>
+    Administrative commands <administration>
+    Creating Multiple File Systems <multifs>
+    Provision/Add/Remove MDS(s) <add-remove-mds>
+    MDS failover and standby configuration <standby>
+    MDS Cache Configuration <cache-configuration>
+    MDS Configuration Settings <mds-config-ref>
+    Manual: ceph-mds <../../man/8/ceph-mds>
+    Export over NFS <nfs>
+    Application best practices <app-best-practices>
+    FS volume and subvolumes <fs-volumes>
+    CephFS Quotas <quota>
+    Health messages <health-messages>
+    Upgrading old file systems <upgrading>
+    CephFS Top Utility <cephfs-top>
+    Scheduled Snapshots <snap-schedule>
+    CephFS Snapshot Mirroring <cephfs-mirroring>
+
+.. raw:: html
+
+   <!---
+
+Mounting CephFS
+^^^^^^^^^^^^^^^
+
+.. raw:: html
+
+   --->
+
+.. toctree:: 
+   :maxdepth: 1
+   :hidden:
+
+    Client Configuration Settings <client-config-ref>
+    Client Authentication <client-auth>
+    Mount CephFS: Prerequisites <mount-prerequisites>
+    Mount CephFS using Kernel Driver <mount-using-kernel-driver>
+    Mount CephFS using FUSE <mount-using-fuse>
+    Mount CephFS on Windows <ceph-dokan>
+    Use the CephFS Shell <cephfs-shell>
+    Supported Features of Kernel Driver <kernel-features>
+    Manual: ceph-fuse <../../man/8/ceph-fuse>
+    Manual: mount.ceph <../../man/8/mount.ceph>
+    Manual: mount.fuse.ceph <../../man/8/mount.fuse.ceph>
+
+
+.. raw:: html
+
+   <!---
+
+CephFS Concepts
+^^^^^^^^^^^^^^^
+
+.. raw:: html
+
+   --->
+
+.. toctree:: 
+   :maxdepth: 1
+   :hidden:
+
+    MDS States <mds-states>
+    POSIX compatibility <posix>
+    MDS Journaling <mds-journaling>
+    File layouts <file-layouts>
+    Distributed Metadata Cache <mdcache>
+    Dynamic Metadata Management in CephFS <dynamic-metadata-management>
+    CephFS IO Path <cephfs-io-path>
+    LazyIO <lazyio>
+    Directory fragmentation <dirfrags>
+    Multiple active MDS daemons <multimds>
+
+
+.. raw:: html
+
+   <!---
+
+Troubleshooting and Disaster Recovery
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. raw:: html
+
+   --->
+
+.. toctree:: 
+   :hidden:
+
+    Client eviction <eviction>
+    Scrubbing the File System <scrub>
+    Handling full file systems <full>
+    Metadata repair <disaster-recovery-experts>
+    Troubleshooting <troubleshooting>
+    Disaster recovery <disaster-recovery>
+    cephfs-journal-tool <cephfs-journal-tool>
+    Recovering file system after monitor store loss <recover-fs-after-mon-store-loss>
+
+
+.. raw:: html
+
+   <!---
+
+Developer Guides
+^^^^^^^^^^^^^^^^
+
+.. raw:: html
+
+   --->
+
+.. toctree:: 
+   :maxdepth: 1
+   :hidden:
+
+    Journaler Configuration <journaler>
+    Client's Capabilities <capabilities>
+    Java and Python bindings <api/index>
+    Mantle <mantle>
+
+
+.. raw:: html
+
+   <!---
+
+Additional Details
+^^^^^^^^^^^^^^^^^^
+
+.. raw:: html
+
+   --->
+
+.. toctree::
+   :maxdepth: 1
+   :hidden:
+
+    Experimental Features <experimental-features>
diff --git a/doc/cephfs/journaler.rst b/doc/cephfs/journaler.rst
new file mode 100644
index 000000000..2121532f5
--- /dev/null
+++ b/doc/cephfs/journaler.rst
@@ -0,0 +1,41 @@
+===========
+ Journaler
+===========
+
+``journaler write head interval``
+
+:Description: How frequently to update the journal head object
+:Type: Integer
+:Required: No
+:Default: ``15``
+
+
+``journaler prefetch periods``
+
+:Description: How many stripe periods to read-ahead on journal replay
+:Type: Integer
+:Required: No
+:Default: ``10``
+
+
+``journal prezero periods``
+
+:Description: How many stripe periods to zero ahead of write position
+:Type: Integer
+:Required: No
+:Default: ``10``
+
+``journaler batch interval``
+
+:Description: Maximum additional latency in seconds we incur artificially. 
+:Type: Double
+:Required: No
+:Default: ``.001``
+
+
+``journaler batch max``
+
+:Description: Maximum bytes we will delay flushing. 
+:Type: 64-bit Unsigned Integer 
+:Required: No
+:Default: ``0``
diff --git a/doc/cephfs/kernel-features.rst b/doc/cephfs/kernel-features.rst
new file mode 100644
index 000000000..c025c7a52
--- /dev/null
+++ b/doc/cephfs/kernel-features.rst
@@ -0,0 +1,47 @@
+
+Supported Features of the Kernel Driver
+========================================
+The kernel driver is developed separately from the core ceph code, and as
+such it sometimes differs from the FUSE driver in feature implementation.
+The following details the implementation status of various CephFS features
+in the kernel driver.
+
+Inline data
+-----------
+Inline data was introduced by the Firefly release. This feature is being
+deprecated in mainline CephFS, and may be removed from a future kernel
+release.
+
+Linux kernel clients >= 3.19 can read inline data and convert existing
+inline data to RADOS objects when file data is modified. At present,
+Linux kernel clients do not store file data as inline data.
+
+See `Experimental Features`_ for more information.
+
+Quotas
+------
+Quota was first introduced by the hammer release. Quota disk format got renewed
+by the Mimic release. Linux kernel clients >= 4.17 can support the new format
+quota. At present, no Linux kernel client support the old format quota.
+
+See `Quotas`_ for more information.
+
+Multiple file systems within a Ceph cluster
+-------------------------------------------
+The feature was introduced by the Jewel release. Linux kernel clients >= 4.7
+can support it.
+
+See `Experimental Features`_ for more information.
+
+Multiple active metadata servers
+--------------------------------
+The feature has been supported since the Luminous release. It is recommended to
+use Linux kernel clients >= 4.14 when there are multiple active MDS.
+
+Snapshots
+---------
+The feature has been supported since the Mimic release. It is recommended to
+use Linux kernel clients >= 4.17 if snapshot is used.
+
+.. _Experimental Features: ../experimental-features
+.. _Quotas: ../quota
diff --git a/doc/cephfs/lazyio.rst b/doc/cephfs/lazyio.rst
new file mode 100644
index 000000000..9cd9ae74a
--- /dev/null
+++ b/doc/cephfs/lazyio.rst
@@ -0,0 +1,76 @@
+======
+LazyIO
+======
+
+LazyIO relaxes POSIX semantics. Buffered reads/writes are allowed even when a
+file is opened by multiple applications on multiple clients. Applications are
+responsible for managing cache coherency themselves.
+
+Libcephfs supports LazyIO since nautilus release.
+
+Enable LazyIO
+=============
+
+LazyIO can be enabled by following ways.
+
+- ``client_force_lazyio`` option enables LAZY_IO globally for libcephfs and
+  ceph-fuse mount.
+
+- ``ceph_lazyio(...)`` and ``ceph_ll_lazyio(...)`` enable LAZY_IO for file handle
+  in libcephfs.
+
+Using LazyIO
+============
+
+LazyIO includes two methods ``lazyio_propagate()`` and ``lazyio_synchronize()``.
+With LazyIO enabled, writes may not be visble to other clients until
+``lazyio_propagate()`` is called. Reads may come from local cache (irrespective of
+changes to the file by other clients) until ``lazyio_synchronize()`` is called.
+
+- ``lazyio_propagate(int fd, loff_t offset, size_t count)`` - Ensures that any
+  buffered writes of the client, in the specific region (offset to offset+count),
+  has been propagated to the shared file. If offset and count are both 0, the 
+  operation is performed on the entire file. Currently only this is supported.
+
+- ``lazyio_synchronize(int fd, loff_t offset, size_t count)`` - Ensures that the
+  client is, in a subsequent read call, able to read the updated file with all 
+  the propagated writes of the other clients. In CephFS this is facilitated by
+  invalidating the file caches pertaining to the inode and hence forces the
+  client to refetch/recache the data from the updated file. Also if the write cache
+  of the calling client is dirty (not propagated), lazyio_synchronize() flushes it as well.
+
+An example usage (utilizing libcephfs) is given below. This is a sample I/O loop for a
+particular client/file descriptor in a parallel application:
+
+::
+
+        /* Client a (ca) opens the shared file file.txt */
+        int fda = ceph_open(ca, "shared_file.txt", O_CREAT|O_RDWR, 0644); 
+
+        /* Enable LazyIO for fda */
+        ceph_lazyio(ca, fda, 1));
+
+        for(i = 0; i < num_iters; i++) {
+            char out_buf[] = "fooooooooo";
+            
+            ceph_write(ca, fda, out_buf, sizeof(out_buf), i);
+            /* Propagate the writes associated with fda to the backing storage*/
+            ceph_propagate(ca, fda, 0, 0);
+            
+            /* The barrier makes sure changes associated with all file descriptors
+            are propagated so that there is certainty that the backing file 
+            is upto date */
+            application_specific_barrier();
+
+            char in_buf[40];
+            /* Calling ceph_lazyio_synchronize here will ascertain that ca will
+            read the updated file with the propagated changes and not read
+            stale cached data */
+            ceph_lazyio_synchronize(ca, fda, 0, 0);
+            ceph_read(ca, fda, in_buf, sizeof(in_buf), 0);
+
+            /* A barrier is required here before returning to the next write
+            phase so as to avoid overwriting the portion of the shared file still
+            being read by another file descriptor */
+            application_specific_barrier();
+        }
diff --git a/doc/cephfs/mantle.rst b/doc/cephfs/mantle.rst
new file mode 100644
index 000000000..064408f71
--- /dev/null
+++ b/doc/cephfs/mantle.rst
@@ -0,0 +1,263 @@
+Mantle
+======
+
+.. warning::
+
+    Mantle is for research and development of metadata balancer algorithms,
+    not for use on production CephFS clusters.
+
+Multiple, active MDSs can migrate directories to balance metadata load. The
+policies for when, where, and how much to migrate are hard-coded into the
+metadata balancing module. Mantle is a programmable metadata balancer built
+into the MDS. The idea is to protect the mechanisms for balancing load
+(migration, replication, fragmentation) but stub out the balancing policies
+using Lua. Mantle is based on [1] but the current implementation does *NOT*
+have the following features from that paper:
+
+1. Balancing API: in the paper, the user fills in when, where, how much, and
+   load calculation policies; currently, Mantle only requires that Lua policies
+   return a table of target loads (e.g., how much load to send to each MDS)
+2. "How much" hook: in the paper, there was a hook that let the user control
+   the fragment selector policy; currently, Mantle does not have this hook
+3. Instantaneous CPU utilization as a metric
+
+[1] Supercomputing '15 Paper:
+http://sc15.supercomputing.org/schedule/event_detail-evid=pap168.html
+
+Quickstart with vstart
+----------------------
+
+.. warning::
+
+    Developing balancers with vstart is difficult because running all daemons
+    and clients on one node can overload the system. Let it run for a while, even
+    though you will likely see a bunch of lost heartbeat and laggy MDS warnings.
+    Most of the time this guide will work but sometimes all MDSs lock up and you
+    cannot actually see them spill. It is much better to run this on a cluster.
+
+As a prerequisite, we assume you have installed `mdtest
+<https://sourceforge.net/projects/mdtest/>`_ or pulled the `Docker image
+<https://hub.docker.com/r/michaelsevilla/mdtest/>`_. We use mdtest because we
+need to generate enough load to get over the MIN_OFFLOAD threshold that is
+arbitrarily set in the balancer. For example, this does not create enough
+metadata load:
+
+::
+
+    while true; do
+      touch "/cephfs/blah-`date`"
+    done
+
+
+Mantle with `vstart.sh`
+~~~~~~~~~~~~~~~~~~~~~~~
+
+1. Start Ceph and tune the logging so we can see migrations happen:
+
+::
+
+    cd build
+    ../src/vstart.sh -n -l
+    for i in a b c; do 
+      bin/ceph --admin-daemon out/mds.$i.asok config set debug_ms 0
+      bin/ceph --admin-daemon out/mds.$i.asok config set debug_mds 2
+      bin/ceph --admin-daemon out/mds.$i.asok config set mds_beacon_grace 1500
+    done
+
+
+2. Put the balancer into RADOS:
+
+::
+
+    bin/rados put --pool=cephfs_metadata_a greedyspill.lua ../src/mds/balancers/greedyspill.lua
+
+
+3. Activate Mantle:
+
+::
+
+    bin/ceph fs set cephfs max_mds 5
+    bin/ceph fs set cephfs_a balancer greedyspill.lua
+
+
+4. Mount CephFS in another window:
+
+::
+
+     bin/ceph-fuse /cephfs -o allow_other &
+     tail -f out/mds.a.log
+
+
+   Note that if you look at the last MDS (which could be a, b, or c -- it's
+   random), you will see an attempt to index a nil value. This is because the
+   last MDS tries to check the load of its neighbor, which does not exist.
+
+5. Run a simple benchmark. In our case, we use the Docker mdtest image to
+   create load:
+
+::
+
+    for i in 0 1 2; do
+      docker run -d \
+        --name=client$i \
+        -v /cephfs:/cephfs \
+        michaelsevilla/mdtest \
+        -F -C -n 100000 -d "/cephfs/client-test$i"
+    done
+
+
+6. When you are done, you can kill all the clients with:
+
+::
+
+    for i in 0 1 2 3; do docker rm -f client$i; done
+
+
+Output
+~~~~~~
+
+Looking at the log for the first MDS (could be a, b, or c), we see that
+everyone has no load:
+
+::
+
+    2016-08-21 06:44:01.763930 7fd03aaf7700  0 lua.balancer MDS0: < auth.meta_load=0.0 all.meta_load=0.0 req_rate=1.0 queue_len=0.0 cpu_load_avg=1.35 > load=0.0
+    2016-08-21 06:44:01.763966 7fd03aaf7700  0 lua.balancer MDS1: < auth.meta_load=0.0 all.meta_load=0.0 req_rate=0.0 queue_len=0.0 cpu_load_avg=1.35 > load=0.0
+    2016-08-21 06:44:01.763982 7fd03aaf7700  0 lua.balancer MDS2: < auth.meta_load=0.0 all.meta_load=0.0 req_rate=0.0 queue_len=0.0 cpu_load_avg=1.35 > load=0.0
+    2016-08-21 06:44:01.764010 7fd03aaf7700  2 lua.balancer when: not migrating! my_load=0.0 hisload=0.0
+    2016-08-21 06:44:01.764033 7fd03aaf7700  2 mds.0.bal  mantle decided that new targets={}
+
+
+After the jobs starts, MDS0 gets about 1953 units of load. The greedy spill
+balancer dictates that half the load goes to your neighbor MDS, so we see that
+Mantle tries to send 1953 load units to MDS1.
+
+::
+
+    2016-08-21 06:45:21.869994 7fd03aaf7700  0 lua.balancer MDS0: < auth.meta_load=5834.188908912 all.meta_load=1953.3492228857 req_rate=12591.0 queue_len=1075.0 cpu_load_avg=3.05 > load=1953.3492228857
+    2016-08-21 06:45:21.870017 7fd03aaf7700  0 lua.balancer MDS1: < auth.meta_load=0.0 all.meta_load=0.0 req_rate=0.0 queue_len=0.0 cpu_load_avg=3.05 > load=0.0
+    2016-08-21 06:45:21.870027 7fd03aaf7700  0 lua.balancer MDS2: < auth.meta_load=0.0 all.meta_load=0.0 req_rate=0.0 queue_len=0.0 cpu_load_avg=3.05 > load=0.0
+    2016-08-21 06:45:21.870034 7fd03aaf7700  2 lua.balancer when: migrating! my_load=1953.3492228857 hisload=0.0
+    2016-08-21 06:45:21.870050 7fd03aaf7700  2 mds.0.bal  mantle decided that new targets={0=0,1=976.675,2=0}
+    2016-08-21 06:45:21.870094 7fd03aaf7700  0 mds.0.bal    - exporting [0,0.52287 1.04574] 1030.88 to mds.1 [dir 100000006ab /client-test2/ [2,head] auth pv=33 v=32 cv=32/0 ap=2+3+4 state=1610612802|complete f(v0 m2016-08-21 06:44:20.366935 1=0+1) n(v2 rc2016-08-21 06:44:30.946816 3790=3788+2) hs=1+0,ss=0+0 dirty=1 | child=1 dirty=1 authpin=1 0x55d2762fd690]
+    2016-08-21 06:45:21.870151 7fd03aaf7700  0 mds.0.migrator nicely exporting to mds.1 [dir 100000006ab /client-test2/ [2,head] auth pv=33 v=32 cv=32/0 ap=2+3+4 state=1610612802|complete f(v0 m2016-08-21 06:44:20.366935 1=0+1) n(v2 rc2016-08-21 06:44:30.946816 3790=3788+2) hs=1+0,ss=0+0 dirty=1 | child=1 dirty=1 authpin=1 0x55d2762fd690]
+
+
+Eventually load moves around:
+
+::
+
+    2016-08-21 06:47:10.210253 7fd03aaf7700  0 lua.balancer MDS0: < auth.meta_load=415.77414300449 all.meta_load=415.79000078186 req_rate=82813.0 queue_len=0.0 cpu_load_avg=11.97 > load=415.79000078186
+    2016-08-21 06:47:10.210277 7fd03aaf7700  0 lua.balancer MDS1: < auth.meta_load=228.72023977691 all.meta_load=186.5606496623 req_rate=28580.0 queue_len=0.0 cpu_load_avg=11.97 > load=186.5606496623
+    2016-08-21 06:47:10.210290 7fd03aaf7700  0 lua.balancer MDS2: < auth.meta_load=0.0 all.meta_load=0.0 req_rate=1.0 queue_len=0.0 cpu_load_avg=11.97 > load=0.0
+    2016-08-21 06:47:10.210298 7fd03aaf7700  2 lua.balancer when: not migrating! my_load=415.79000078186 hisload=186.5606496623
+    2016-08-21 06:47:10.210311 7fd03aaf7700  2 mds.0.bal  mantle decided that new targets={}
+
+
+Implementation Details
+----------------------
+
+Most of the implementation is in MDBalancer. Metrics are passed to the balancer
+policies via the Lua stack and a list of loads is returned back to MDBalancer.
+It sits alongside the current balancer implementation and it's enabled with a
+Ceph CLI command ("ceph fs set cephfs balancer mybalancer.lua"). If the Lua policy
+fails (for whatever reason), we fall back to the original metadata load
+balancer. The balancer is stored in the RADOS metadata pool and a string in the
+MDSMap tells the MDSs which balancer to use.
+
+Exposing Metrics to Lua
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Metrics are exposed directly to the Lua code as global variables instead of
+using a well-defined function signature. There is a global "mds" table, where
+each index is an MDS number (e.g., 0) and each value is a dictionary of metrics
+and values. The Lua code can grab metrics using something like this:
+
+::
+
+    mds[0]["queue_len"]
+
+
+This is in contrast to cls-lua in the OSDs, which has well-defined arguments
+(e.g., input/output bufferlists). Exposing the metrics directly makes it easier
+to add new metrics without having to change the API on the Lua side; we want
+the API to grow and shrink as we explore which metrics matter. The downside of
+this approach is that the person programming Lua balancer policies has to look
+at the Ceph source code to see which metrics are exposed. We figure that the
+Mantle developer will be in touch with MDS internals anyways.
+
+The metrics exposed to the Lua policy are the same ones that are already stored
+in mds_load_t: auth.meta_load(), all.meta_load(), req_rate, queue_length,
+cpu_load_avg.
+
+Compile/Execute the Balancer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Here we use `lua_pcall` instead of `lua_call` because we want to handle errors
+in the MDBalancer. We do not want the error propagating up the call chain. The
+cls_lua class wants to handle the error itself because it must fail gracefully.
+For Mantle, we don't care if a Lua error crashes our balancer -- in that case,
+we will fall back to the original balancer.
+
+The performance improvement of using `lua_call` over `lua_pcall` would not be
+leveraged here because the balancer is invoked every 10 seconds by default. 
+
+Returning Policy Decision to C++
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We force the Lua policy engine to return a table of values, corresponding to
+the amount of load to send to each MDS. These loads are inserted directly into
+the MDBalancer "my_targets" vector. We do not allow the MDS to return a table
+of MDSs and metrics because we want the decision to be completely made on the
+Lua side.
+
+Iterating through tables returned by Lua is done through the stack. In Lua
+jargon: a dummy value is pushed onto the stack and the next iterator replaces
+the top of the stack with a (k, v) pair. After reading each value, pop that
+value but keep the key for the next call to `lua_next`. 
+
+Reading from RADOS
+~~~~~~~~~~~~~~~~~~
+
+All MDSs will read balancing code from RADOS when the balancer version changes
+in the MDS Map. The balancer pulls the Lua code from RADOS synchronously. We do
+this with a timeout: if the asynchronous read does not come back within half
+the balancing tick interval the operation is cancelled and a Connection Timeout
+error is returned. By default, the balancing tick interval is 10 seconds, so
+Mantle will use a 5 second second timeout. This design allows Mantle to
+immediately return an error if anything RADOS-related goes wrong.
+
+We use this implementation because we do not want to do a blocking OSD read
+from inside the global MDS lock. Doing so would bring down the MDS cluster if
+any of the OSDs are not responsive -- this is tested in the ceph-qa-suite by
+setting all OSDs to down/out and making sure the MDS cluster stays active.
+
+One approach would be to asynchronously fire the read when handling the MDS Map
+and fill in the Lua code in the background. We cannot do this because the MDS
+does not support daemon-local fallbacks and the balancer assumes that all MDSs
+come to the same decision at the same time (e.g., importers, exporters, etc.).
+
+Debugging
+~~~~~~~~~
+
+Logging in a Lua policy will appear in the MDS log. The syntax is the same as
+the cls logging interface:
+
+::
+
+    BAL_LOG(0, "this is a log message")
+
+
+It is implemented by passing a function that wraps the `dout` logging framework
+(`dout_wrapper`) to Lua with the `lua_register()` primitive. The Lua code is
+actually calling the `dout` function in C++.
+
+Warning and Info messages are centralized using the clog/Beacon. Successful
+messages are only sent on version changes by the first MDS to avoid spamming
+the `ceph -w` utility. These messages are used for the integration tests.
+
+Testing
+~~~~~~~
+
+Testing is done with the ceph-qa-suite (tasks.cephfs.test_mantle). We do not
+test invalid balancer logging and loading the actual Lua VM.
diff --git a/doc/cephfs/mdcache.rst b/doc/cephfs/mdcache.rst
new file mode 100644
index 000000000..f2e20238c
--- /dev/null
+++ b/doc/cephfs/mdcache.rst
@@ -0,0 +1,77 @@
+=================================
+CephFS Distributed Metadata Cache
+=================================
+While the data for inodes in a Ceph file system is stored in RADOS and
+accessed by the clients directly, inode metadata and directory
+information is managed by the Ceph metadata server (MDS). The MDS's
+act as mediator for all metadata related activity, storing the resulting
+information in a separate RADOS pool from the file data.
+
+CephFS clients can request that the MDS fetch or change inode metadata
+on its behalf, but an MDS can also grant the client **capabilities**
+(aka **caps**) for each inode (see :doc:`/cephfs/capabilities`).
+
+A capability grants the client the ability to cache and possibly
+manipulate some portion of the data or metadata associated with the
+inode. When another client needs access to the same information, the MDS
+will revoke the capability and the client will eventually return it,
+along with an updated version of the inode's metadata (in the event that
+it made changes to it while it held the capability).
+
+Clients can request capabilities and will generally get them, but when
+there is competing access or memory pressure on the MDS, they may be
+**revoked**. When a capability is revoked, the client is responsible for
+returning it as soon as it is able. Clients that fail to do so in a
+timely fashion may end up **blocklisted** and unable to communicate with
+the cluster.
+
+Since the cache is distributed, the MDS must take great care to ensure
+that no client holds capabilities that may conflict with other clients'
+capabilities, or operations that it does itself. This allows cephfs
+clients to rely on much greater cache coherence than a filesystem like
+NFS, where the client may cache data and metadata beyond the point where
+it has changed on the server.
+
+Client Metadata Requests
+------------------------
+When a client needs to query/change inode metadata or perform an
+operation on a directory, it has two options. It can make a request to
+the MDS directly, or serve the information out of its cache. With
+CephFS, the latter is only possible if the client has the necessary
+caps.
+
+Clients can send simple requests to the MDS to query or request changes
+to certain metadata. The replies to these requests may also grant the
+client a certain set of caps for the inode, allowing it to perform
+subsequent requests without consulting the MDS.
+
+Clients can also request caps directly from the MDS, which is necessary
+in order to read or write file data.
+
+Distributed Locks in an MDS Cluster
+-----------------------------------
+When an MDS wants to read or change information about an inode, it must
+gather the appropriate locks for it. The MDS cluster may have a series
+of different types of locks on the given inode and each MDS may have
+disjoint sets of locks.
+
+If there are outstanding caps that would conflict with these locks, then
+they must be revoked before the lock can be acquired. Once the competing
+caps are returned to the MDS, then it can get the locks and do the
+operation.
+
+On a filesystem served by multiple MDS', the metadata cache is also
+distributed among the MDS' in the cluster. For every inode, at any given
+time, only one MDS in the cluster is considered **authoritative**. Any
+requests to change that inode must be done by the authoritative MDS,
+though non-authoritative MDS can forward requests to the authoritative
+one.
+
+Non-auth MDS' can also obtain read locks that prevent the auth MDS from
+changing the data until the lock is dropped, so that they can serve
+inode info to the clients.
+
+The auth MDS for an inode can change over time as well. The MDS' will
+actively balance responsibility for the inode cache amongst
+themselves, but this can be overridden by **pinning** certain subtrees
+to a single MDS.
diff --git a/doc/cephfs/mds-config-ref.rst b/doc/cephfs/mds-config-ref.rst
new file mode 100644
index 000000000..2efc83b41
--- /dev/null
+++ b/doc/cephfs/mds-config-ref.rst
@@ -0,0 +1,543 @@
+======================
+ MDS Config Reference
+======================
+
+``mds_cache_memory_limit``
+
+:Description: The memory limit the MDS should enforce for its cache.
+:Type:  64-bit Integer Unsigned
+:Default: ``4G``
+
+``mds_cache_reservation``
+
+:Description: The cache reservation (memory or inodes) for the MDS cache to maintain.
+              Once the MDS begins dipping into its reservation, it will recall
+              client state until its cache size shrinks to restore the
+              reservation.
+:Type:  Float
+:Default: ``0.05``
+
+
+``mds_cache_mid``
+
+:Description: The insertion point for new items in the cache LRU 
+              (from the top).
+
+:Type:  Float
+:Default: ``0.7``
+
+
+``mds_dir_commit_ratio``
+
+:Description: The fraction of directory that is dirty before Ceph commits using 
+              a full update (instead of partial update).
+
+:Type:  Float
+:Default: ``0.5``
+
+
+``mds_dir_max_commit_size``
+
+:Description: The maximum size of a directory update before Ceph breaks it into 
+              smaller transactions (MB).
+              
+:Type:  32-bit Integer
+:Default: ``10``
+
+``mds_dir_max_entries``
+
+:Description: The maximum number of entries before any new entries are rejected
+              with ENOSPC.
+:Type:  64-bit Integer Unsigned
+:Default: ``0``
+
+``mds_decay_halflife``
+
+:Description: The half-life of MDS cache temperature.
+:Type:  Float
+:Default: ``5``
+
+``mds_beacon_interval``
+
+:Description: The frequency (in seconds) of beacon messages sent 
+              to the monitor.
+
+:Type:  Float
+:Default: ``4``
+
+
+``mds_beacon_grace``
+
+:Description: The interval without beacons before Ceph declares an MDS laggy 
+              (and possibly replace it).
+              
+:Type:  Float
+:Default: ``15``
+
+
+``mds_blocklist_interval``
+
+:Description: The blocklist duration for failed MDSs in the OSD map. Note,
+              this controls how long failed MDS daemons will stay in the
+              OSDMap blocklist. It has no effect on how long something is
+              blocklisted when the administrator blocklists it manually. For
+              example, ``ceph osd blocklist add`` will still use the default
+              blocklist time.
+:Type:  Float
+:Default: ``24.0*60.0``
+
+
+``mds_reconnect_timeout``
+
+:Description: The interval (in seconds) to wait for clients to reconnect 
+              during MDS restart.
+
+:Type:  Float
+:Default: ``45``
+
+
+``mds_tick_interval``
+
+:Description: How frequently the MDS performs internal periodic tasks.
+:Type:  Float
+:Default: ``5``
+
+
+``mds_dirstat_min_interval``
+
+:Description: The minimum interval (in seconds) to try to avoid propagating 
+              recursive stats up the tree.
+              
+:Type:  Float
+:Default: ``1``
+
+``mds_scatter_nudge_interval``
+
+:Description: How quickly dirstat changes propagate up.
+:Type:  Float
+:Default: ``5``
+
+
+``mds_client_prealloc_inos``
+
+:Description: The number of inode numbers to preallocate per client session.
+:Type:  32-bit Integer
+:Default: ``1000``
+
+
+``mds_early_reply``
+
+:Description: Determines whether the MDS should allow clients to see request 
+              results before they commit to the journal.
+
+:Type:  Boolean
+:Default: ``true``
+
+
+``mds_default_dir_hash``
+
+:Description: The function to use for hashing files across directory fragments.
+:Type:  32-bit Integer
+:Default: ``2`` (i.e., rjenkins)
+
+
+``mds_log_skip_corrupt_events``
+
+:Description: Determines whether the MDS should try to skip corrupt journal 
+              events during journal replay.
+              
+:Type:  Boolean
+:Default:  ``false``
+
+
+``mds_log_max_events``
+
+:Description: The maximum events in the journal before we initiate trimming.
+              Set to ``-1`` to disable limits.
+              
+:Type:  32-bit Integer
+:Default: ``-1``
+
+
+``mds_log_max_segments``
+
+:Description: The maximum number of segments (objects) in the journal before 
+              we initiate trimming. Set to ``-1`` to disable limits.
+
+:Type:  32-bit Integer
+:Default: ``128``
+
+
+``mds_bal_sample_interval``
+
+:Description: Determines how frequently to sample directory temperature 
+              (for fragmentation decisions).
+              
+:Type:  Float
+:Default: ``3``
+
+
+``mds_bal_replicate_threshold``
+
+:Description: The maximum temperature before Ceph attempts to replicate 
+              metadata to other nodes.
+              
+:Type:  Float
+:Default: ``8000``
+
+
+``mds_bal_unreplicate_threshold``
+
+:Description: The minimum temperature before Ceph stops replicating 
+              metadata to other nodes.
+              
+:Type:  Float
+:Default: ``0``
+
+
+``mds_bal_split_size``
+
+:Description: The maximum directory size before the MDS will split a directory 
+              fragment into smaller bits.
+              
+:Type:  32-bit Integer
+:Default: ``10000``
+
+
+``mds_bal_split_rd``
+
+:Description: The maximum directory read temperature before Ceph splits 
+              a directory fragment.
+              
+:Type:  Float
+:Default: ``25000``
+
+
+``mds_bal_split_wr``
+
+:Description: The maximum directory write temperature before Ceph splits 
+              a directory fragment.
+              
+:Type:  Float
+:Default: ``10000``
+
+
+``mds_bal_split_bits``
+
+:Description: The number of bits by which to split a directory fragment.
+:Type:  32-bit Integer
+:Default: ``3``
+
+
+``mds_bal_merge_size``
+
+:Description: The minimum directory size before Ceph tries to merge 
+              adjacent directory fragments.
+              
+:Type:  32-bit Integer
+:Default: ``50``
+
+
+``mds_bal_interval``
+
+:Description: The frequency (in seconds) of workload exchanges between MDSs.
+:Type:  32-bit Integer
+:Default: ``10``
+
+
+``mds_bal_fragment_interval``
+
+:Description: The delay (in seconds) between a fragment being eligible for split
+              or merge and executing the fragmentation change.
+:Type:  32-bit Integer
+:Default: ``5``
+
+
+``mds_bal_fragment_fast_factor``
+
+:Description: The ratio by which frags may exceed the split size before
+              a split is executed immediately (skipping the fragment interval)
+:Type:  Float
+:Default: ``1.5``
+
+``mds_bal_fragment_size_max``
+
+:Description: The maximum size of a fragment before any new entries
+              are rejected with ENOSPC.
+:Type:  32-bit Integer
+:Default: ``100000``
+
+``mds_bal_idle_threshold``
+
+:Description: The minimum temperature before Ceph migrates a subtree 
+              back to its parent.
+              
+:Type:  Float
+:Default: ``0``
+
+
+``mds_bal_max``
+
+:Description: The number of iterations to run balancer before Ceph stops. 
+              (used for testing purposes only)
+
+:Type:  32-bit Integer
+:Default: ``-1``
+
+
+``mds_bal_max_until``
+
+:Description: The number of seconds to run balancer before Ceph stops. 
+              (used for testing purposes only)
+
+:Type:  32-bit Integer
+:Default: ``-1``
+
+
+``mds_bal_mode``
+
+:Description: The method for calculating MDS load. 
+
+              - ``0`` = Hybrid.
+              - ``1`` = Request rate and latency. 
+              - ``2`` = CPU load.
+              
+:Type:  32-bit Integer
+:Default: ``0``
+
+
+``mds_bal_min_rebalance``
+
+:Description: The minimum subtree temperature before Ceph migrates.
+:Type:  Float
+:Default: ``0.1``
+
+
+``mds_bal_min_start``
+
+:Description: The minimum subtree temperature before Ceph searches a subtree.
+:Type:  Float
+:Default: ``0.2``
+
+
+``mds_bal_need_min``
+
+:Description: The minimum fraction of target subtree size to accept.
+:Type:  Float
+:Default: ``0.8``
+
+
+``mds_bal_need_max``
+
+:Description: The maximum fraction of target subtree size to accept.
+:Type:  Float
+:Default: ``1.2``
+
+
+``mds_bal_midchunk``
+
+:Description: Ceph will migrate any subtree that is larger than this fraction 
+              of the target subtree size.
+              
+:Type:  Float
+:Default: ``0.3``
+
+
+``mds_bal_minchunk``
+
+:Description: Ceph will ignore any subtree that is smaller than this fraction 
+              of the target subtree size.
+              
+:Type:  Float
+:Default: ``0.001``
+
+
+``mds_bal_target_removal_min``
+
+:Description: The minimum number of balancer iterations before Ceph removes
+              an old MDS target from the MDS map.
+              
+:Type:  32-bit Integer
+:Default: ``5``
+
+
+``mds_bal_target_removal_max``
+
+:Description: The maximum number of balancer iterations before Ceph removes 
+              an old MDS target from the MDS map.
+              
+:Type:  32-bit Integer
+:Default: ``10``
+
+
+``mds_replay_interval``
+
+:Description: The journal poll interval when in standby-replay mode.
+              ("hot standby")
+              
+:Type:  Float
+:Default: ``1``
+
+
+``mds_shutdown_check``
+
+:Description: The interval for polling the cache during MDS shutdown.
+:Type:  32-bit Integer
+:Default: ``0``
+
+
+``mds_thrash_exports``
+
+:Description: Ceph will randomly export subtrees between nodes (testing only).
+:Type:  32-bit Integer
+:Default: ``0``
+
+
+``mds_thrash_fragments``
+
+:Description: Ceph will randomly fragment or merge directories.
+:Type:  32-bit Integer
+:Default: ``0``
+
+
+``mds_dump_cache_on_map``
+
+:Description: Ceph will dump the MDS cache contents to a file on each MDSMap.
+:Type:  Boolean
+:Default:  ``false``
+
+
+``mds_dump_cache_after_rejoin``
+
+:Description: Ceph will dump MDS cache contents to a file after 
+              rejoining the cache (during recovery).
+              
+:Type:  Boolean
+:Default:  ``false``
+
+
+``mds_verify_scatter``
+
+:Description: Ceph will assert that various scatter/gather invariants 
+              are ``true`` (developers only).
+              
+:Type:  Boolean
+:Default:  ``false``
+
+
+``mds_debug_scatterstat``
+
+:Description: Ceph will assert that various recursive stat invariants 
+              are ``true`` (for developers only).
+              
+:Type:  Boolean
+:Default:  ``false``
+
+
+``mds_debug_frag``
+
+:Description: Ceph will verify directory fragmentation invariants 
+              when convenient (developers only).
+              
+:Type:  Boolean
+:Default:  ``false``
+
+
+``mds_debug_auth_pins``
+
+:Description: The debug auth pin invariants (for developers only).
+:Type:  Boolean
+:Default:  ``false``
+
+
+``mds_debug_subtrees``
+
+:Description: The debug subtree invariants (for developers only).
+:Type:  Boolean
+:Default:  ``false``
+
+
+``mds_kill_mdstable_at``
+
+:Description: Ceph will inject MDS failure in MDSTable code 
+              (for developers only).
+              
+:Type:  32-bit Integer
+:Default: ``0``
+
+
+``mds_kill_export_at``
+
+:Description: Ceph will inject MDS failure in the subtree export code 
+              (for developers only).
+              
+:Type:  32-bit Integer
+:Default: ``0``
+
+
+``mds_kill_import_at``
+
+:Description: Ceph will inject MDS failure in the subtree import code 
+              (for developers only).
+              
+:Type:  32-bit Integer
+:Default: ``0``
+
+
+``mds_kill_link_at``
+
+:Description: Ceph will inject MDS failure in hard link code 
+              (for developers only).
+              
+:Type:  32-bit Integer
+:Default: ``0``
+
+
+``mds_kill_rename_at``
+
+:Description: Ceph will inject MDS failure in the rename code 
+              (for developers only).
+              
+:Type:  32-bit Integer
+:Default: ``0``
+
+
+``mds_wipe_sessions``
+
+:Description: Ceph will delete all client sessions on startup 
+              (for testing only).
+              
+:Type:  Boolean
+:Default: ``false``
+
+
+``mds_wipe_ino_prealloc``
+
+:Description: Ceph will delete ino preallocation metadata on startup 
+              (for testing only).
+              
+:Type:  Boolean
+:Default: ``false``
+
+
+``mds_skip_ino``
+
+:Description: The number of inode numbers to skip on startup 
+              (for testing only).
+              
+:Type:  32-bit Integer
+:Default: ``0``
+
+
+``mds_min_caps_per_client``
+
+:Description: Set the minimum number of capabilities a client may hold.
+:Type: Integer
+:Default: ``100``
+
+
+``mds_max_ratio_caps_per_client``
+
+:Description: Set the maximum ratio of current caps that may be recalled during MDS cache pressure.
+:Type: Float
+:Default: ``0.8``
diff --git a/doc/cephfs/mds-journaling.rst b/doc/cephfs/mds-journaling.rst
new file mode 100644
index 000000000..92b32bb45
--- /dev/null
+++ b/doc/cephfs/mds-journaling.rst
@@ -0,0 +1,90 @@
+MDS Journaling
+==============
+
+CephFS Metadata Pool
+--------------------
+
+CephFS uses a separate (metadata) pool for managing file metadata (inodes and
+dentries) in a Ceph File System. The metadata pool has all the information about
+files in a Ceph File System including the File System hierarchy. Additionally,
+CephFS maintains meta information related to other entities in a file system
+such as file system journals, open file table, session map, etc.
+
+This document describes how Ceph Metadata Servers use and rely on journaling.
+
+CephFS MDS Journaling
+---------------------
+
+CephFS metadata servers stream a journal of metadata events into RADOS in the metadata
+pool prior to executing a file system operation. Active MDS daemon(s) manage metadata
+for files and directories in CephFS.
+
+CephFS uses journaling for couple of reasons:
+
+#. Consistency: On an MDS failover, the journal events can be replayed to reach a
+   consistent file system state. Also, metadata operations that require multiple
+   updates to the backing store need to be journaled for crash consistency (along
+   with other consistency mechanisms such as locking, etc..).
+
+#. Performance: Journal updates are (mostly) sequential, hence updates to journals
+   are fast. Furthermore, updates can be batched into single write, thereby saving
+   disk seek time involved in updates to different parts of a file. Having a large
+   journal also helps a standby MDS to warm its cache which helps indirectly during
+   MDS failover.
+
+Each active metadata server maintains its own journal in the metadata pool. Journals
+are striped over multiple objects. Journal entries which are not required (deemed as
+old) are trimmed by the metadata server.
+
+Journal Events
+--------------
+
+Apart from journaling file system metadata updates, CephFS journals various other events
+such as client session info and directory import/export state to name a few. These events
+are used by the metadata sever to reestablish correct state as required, e.g., Ceph MDS
+tries to reconnect clients on restart when journal events get replayed and a specific
+event type in the journal specifies that a client entity type has a session with the MDS
+before it was restarted.
+
+To examine the list of such events recorded in the journal, CephFS provides a command
+line utility `cephfs-journal-tool` which can be used as follows:
+
+::
+
+   cephfs-journal-tool --rank=<fs>:<rank> event get list
+
+`cephfs-journal-tool` is also used to discover and repair a damaged Ceph File System.
+(See :doc:`/cephfs/cephfs-journal-tool` for more details)
+
+Journal Event Types
+-------------------
+
+Following are various event types that are journaled by the MDS.
+
+#. `EVENT_COMMITTED`: Mark a request (id) as committed.
+
+#. `EVENT_EXPORT`: Maps directories to an MDS rank.
+
+#. `EVENT_FRAGMENT`: Tracks various stages of directory fragmentation (split/merge).
+
+#. `EVENT_IMPORTSTART`: Logged when an MDS rank starts importing directory fragments.
+
+#. `EVENT_IMPORTFINISH`: Logged when an MDS rank finishes importing directory fragments.
+
+#. `EVENT_NOOP`: No operation event type for skipping over a journal region.
+
+#. `EVENT_OPEN`: Tracks which inodes have open file handles.
+
+#. `EVENT_RESETJOURNAL`: Used to mark a journal as `reset` post truncation.
+
+#. `EVENT_SESSION`: Tracks open client sessions.
+
+#. `EVENT_SLAVEUPDATE`: Logs various stages of an operation that has been forwarded to a (slave) mds.
+
+#. `EVENT_SUBTREEMAP`: Map of directory inodes to directory contents (subtree partition).
+
+#. `EVENT_TABLECLIENT`: Log transition states of MDSs view of client tables (snap/anchor).
+
+#. `EVENT_TABLESERVER`: Log transition states of MDSs view of server tables (snap/anchor).
+
+#. `EVENT_UPDATE`: Log file operations on an inode.
diff --git a/doc/cephfs/mds-state-diagram.dot b/doc/cephfs/mds-state-diagram.dot
new file mode 100644
index 000000000..8c9fa25d0
--- /dev/null
+++ b/doc/cephfs/mds-state-diagram.dot
@@ -0,0 +1,75 @@
+digraph {
+
+node [shape=circle,style=unfilled,fixedsize=true,width=2.0]
+
+node [color=blue,peripheries=1];
+N0 [label="up:boot"]
+
+node [color=green,peripheries=1];
+S1 [label="up:standby"]
+N0 -> S1 [color=green,penwidth=2.0];
+S2 [label="up:standby_replay"]
+S1 -> S2 [color=green,penwidth=2.0];
+
+node [color=orange,peripheries=2];
+N1 [label="up:creating"]
+S1 -> N1 [color=orange,penwidth=2.0];
+N2 [label="up:starting"]
+S1 -> N2 [color=orange,penwidth=2.0];
+N3 [label="up:replay"]
+S1 -> N3 [color=orange,penwidth=2.0];
+S2 -> N3 [color=orange,penwidth=2.0];
+N4 [label="up:resolve"]
+N3 -> N4 [color=orange,penwidth=2.0];
+N5 [label="up:reconnect"]
+N3 -> N5 [color=orange,penwidth=2.0];
+N4 -> N5 [color=orange,penwidth=2.0];
+N6 [label="up:rejoin"]
+N5 -> N6 [color=orange,penwidth=2.0];
+N7 [label="up:clientreplay"]
+N6 -> N7 [color=orange,penwidth=2.0];
+
+node [color=green,peripheries=2];
+S0 [label="up:active"]
+N7 -> S0 [color=green,penwidth=2.0];
+N1 -> S0 [color=green,penwidth=2.0];
+N2 -> S0 [color=green,penwidth=2.0];
+N6 -> S0 [color=green,penwidth=2.0];
+
+// going down but still accessible by clients
+node [color=purple,peripheries=2];
+S3 [label="up:stopping"]
+S0 -> S3 [color=purple,penwidth=2.0];
+
+// terminal (but "in")
+node [shape=polygon,sides=6,color=red,peripheries=2];
+D0 [label="down:failed"]
+N2 -> D0 [color=red,penwidth=2.0];
+N3 -> D0 [color=red,penwidth=2.0];
+N4 -> D0 [color=red,penwidth=2.0];
+N5 -> D0 [color=red,penwidth=2.0];
+N6 -> D0 [color=red,penwidth=2.0];
+N7 -> D0 [color=red,penwidth=2.0];
+S0 -> D0 [color=red,penwidth=2.0];
+S3 -> D0 [color=red,penwidth=2.0];
+D0 -> N3 [color=red,penwidth=2.0];
+
+// terminal (but not "in")
+node [shape=polygon,sides=6,color=black,peripheries=1];
+D1 [label="down:damaged"]
+S2 -> D1 [color=black,penwidth=2.0];
+N3 -> D1 [color=black,penwidth=2.0];
+N4 -> D1 [color=black,penwidth=2.0];
+N5 -> D1 [color=black,penwidth=2.0];
+N6 -> D1 [color=black,penwidth=2.0];
+N7 -> D1 [color=black,penwidth=2.0];
+S0 -> D1 [color=black,penwidth=2.0];
+S3 -> D1 [color=black,penwidth=2.0];
+D1 -> D0 [color=red,penwidth=2.0]
+
+node [shape=polygon,sides=6,color=purple,peripheries=1];
+D3 [label="down:stopped"]
+S3 -> D3 [color=purple,penwidth=2.0];
+N6 -> D3 [color=purple,penwidth=2.0];
+
+}
diff --git a/doc/cephfs/mds-state-diagram.svg b/doc/cephfs/mds-state-diagram.svg
new file mode 100644
index 000000000..d870a6e9b
--- /dev/null
+++ b/doc/cephfs/mds-state-diagram.svg
@@ -0,0 +1,317 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by graphviz version 2.40.1 (20161225.0304)
+ -->
+<!-- Title: %3 Pages: 1 -->
+<svg width="756pt" height="1980pt"
+ viewBox="0.00 0.00 756.37 1980.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 1976)">
+<title>%3</title>
+<polygon fill="#ffffff" stroke="transparent" points="-4,4 -4,-1976 752.3688,-1976 752.3688,4 -4,4"/>
+<!-- N0 -->
+<g id="node1" class="node">
+<title>N0</title>
+<ellipse fill="none" stroke="#0000ff" cx="529.3688" cy="-1900" rx="72" ry="72"/>
+<text text-anchor="middle" x="529.3688" y="-1896.3" font-family="Times,serif" font-size="14.00" fill="#000000">up:boot</text>
+</g>
+<!-- S1 -->
+<g id="node2" class="node">
+<title>S1</title>
+<ellipse fill="none" stroke="#00ff00" cx="529.3688" cy="-1720" rx="72" ry="72"/>
+<text text-anchor="middle" x="529.3688" y="-1716.3" font-family="Times,serif" font-size="14.00" fill="#000000">up:standby</text>
+</g>
+<!-- N0&#45;&gt;S1 -->
+<g id="edge1" class="edge">
+<title>N0&#45;&gt;S1</title>
+<path fill="none" stroke="#00ff00" stroke-width="2" d="M529.3688,-1827.8243C529.3688,-1819.4801 529.3688,-1810.9229 529.3688,-1802.4555"/>
+<polygon fill="#00ff00" stroke="#00ff00" stroke-width="2" points="532.8689,-1802.1919 529.3688,-1792.1919 525.8689,-1802.192 532.8689,-1802.1919"/>
+</g>
+<!-- S2 -->
+<g id="node3" class="node">
+<title>S2</title>
+<ellipse fill="none" stroke="#00ff00" cx="402.3688" cy="-1540" rx="72" ry="72"/>
+<text text-anchor="middle" x="402.3688" y="-1536.3" font-family="Times,serif" font-size="14.00" fill="#000000">up:standby_replay</text>
+</g>
+<!-- S1&#45;&gt;S2 -->
+<g id="edge2" class="edge">
+<title>S1&#45;&gt;S2</title>
+<path fill="none" stroke="#00ff00" stroke-width="2" d="M487.7842,-1661.0613C475.7237,-1643.9675 462.4609,-1625.1699 450.0576,-1607.5904"/>
+<polygon fill="#00ff00" stroke="#00ff00" stroke-width="2" points="452.6717,-1605.2243 444.0468,-1599.0711 446.952,-1609.2599 452.6717,-1605.2243"/>
+</g>
+<!-- N1 -->
+<g id="node4" class="node">
+<title>N1</title>
+<ellipse fill="none" stroke="#ffa500" cx="568.3688" cy="-1356" rx="72" ry="72"/>
+<ellipse fill="none" stroke="#ffa500" cx="568.3688" cy="-1356" rx="76" ry="76"/>
+<text text-anchor="middle" x="568.3688" y="-1352.3" font-family="Times,serif" font-size="14.00" fill="#000000">up:creating</text>
+</g>
+<!-- S1&#45;&gt;N1 -->
+<g id="edge3" class="edge">
+<title>S1&#45;&gt;N1</title>
+<path fill="none" stroke="#ffa500" stroke-width="2" d="M537.0739,-1648.0853C543.4139,-1588.912 552.4184,-1504.87 559.1421,-1442.1155"/>
+<polygon fill="#ffa500" stroke="#ffa500" stroke-width="2" points="562.6577,-1442.1568 560.243,-1431.8408 555.6975,-1441.411 562.6577,-1442.1568"/>
+</g>
+<!-- N2 -->
+<g id="node5" class="node">
+<title>N2</title>
+<ellipse fill="none" stroke="#ffa500" cx="672.3688" cy="-604" rx="72" ry="72"/>
+<ellipse fill="none" stroke="#ffa500" cx="672.3688" cy="-604" rx="76" ry="76"/>
+<text text-anchor="middle" x="672.3688" y="-600.3" font-family="Times,serif" font-size="14.00" fill="#000000">up:starting</text>
+</g>
+<!-- S1&#45;&gt;N2 -->
+<g id="edge4" class="edge">
+<title>S1&#45;&gt;N2</title>
+<path fill="none" stroke="#ffa500" stroke-width="2" d="M569.2023,-1659.7191C611.5148,-1589.7402 672.3688,-1469.3866 672.3688,-1356 672.3688,-1356 672.3688,-1356 672.3688,-980 672.3688,-880.6681 672.3688,-766.5754 672.3688,-690.3655"/>
+<polygon fill="#ffa500" stroke="#ffa500" stroke-width="2" points="675.8689,-690.3343 672.3688,-680.3343 668.8689,-690.3344 675.8689,-690.3343"/>
+</g>
+<!-- N3 -->
+<g id="node6" class="node">
+<title>N3</title>
+<ellipse fill="none" stroke="#ffa500" cx="398.3688" cy="-1356" rx="72" ry="72"/>
+<ellipse fill="none" stroke="#ffa500" cx="398.3688" cy="-1356" rx="76" ry="76"/>
+<text text-anchor="middle" x="398.3688" y="-1352.3" font-family="Times,serif" font-size="14.00" fill="#000000">up:replay</text>
+</g>
+<!-- S1&#45;&gt;N3 -->
+<g id="edge5" class="edge">
+<title>S1&#45;&gt;N3</title>
+<path fill="none" stroke="#ffa500" stroke-width="2" d="M525.1622,-1647.8711C520.0317,-1596.0056 508.6442,-1525.3873 483.3688,-1468 476.1463,-1451.6015 466.077,-1435.3985 455.3873,-1420.6272"/>
+<polygon fill="#ffa500" stroke="#ffa500" stroke-width="2" points="458.1754,-1418.5111 449.3989,-1412.576 452.5587,-1422.6887 458.1754,-1418.5111"/>
+</g>
+<!-- S2&#45;&gt;N3 -->
+<g id="edge6" class="edge">
+<title>S2&#45;&gt;N3</title>
+<path fill="none" stroke="#ffa500" stroke-width="2" d="M400.7994,-1467.8064C400.6169,-1459.4122 400.4294,-1450.7869 400.2433,-1442.2286"/>
+<polygon fill="#ffa500" stroke="#ffa500" stroke-width="2" points="403.7426,-1442.1519 400.026,-1432.2304 396.7442,-1442.3041 403.7426,-1442.1519"/>
+</g>
+<!-- S0 -->
+<g id="node11" class="node">
+<title>S0</title>
+<ellipse fill="none" stroke="#00ff00" cx="474.3688" cy="-416" rx="72" ry="72"/>
+<ellipse fill="none" stroke="#00ff00" cx="474.3688" cy="-416" rx="76" ry="76"/>
+<text text-anchor="middle" x="474.3688" y="-412.3" font-family="Times,serif" font-size="14.00" fill="#000000">up:active</text>
+</g>
+<!-- N1&#45;&gt;S0 -->
+<g id="edge13" class="edge">
+<title>N1&#45;&gt;S0</title>
+<path fill="none" stroke="#00ff00" stroke-width="2" d="M568.3688,-1279.6657C568.3688,-1203.6994 568.3688,-1083.7917 568.3688,-980 568.3688,-980 568.3688,-980 568.3688,-792 568.3688,-687.305 533.6439,-570.9765 506.4053,-495.8318"/>
+<polygon fill="#00ff00" stroke="#00ff00" stroke-width="2" points="509.6832,-494.6043 502.9547,-486.4202 503.1109,-497.0139 509.6832,-494.6043"/>
+</g>
+<!-- N2&#45;&gt;S0 -->
+<g id="edge14" class="edge">
+<title>N2&#45;&gt;S0</title>
+<path fill="none" stroke="#00ff00" stroke-width="2" d="M617.2148,-551.6315C592.3238,-527.9977 562.7987,-499.9638 537.0702,-475.5347"/>
+<polygon fill="#00ff00" stroke="#00ff00" stroke-width="2" points="539.2319,-472.7608 529.5701,-468.4134 534.412,-477.8371 539.2319,-472.7608"/>
+</g>
+<!-- D0 -->
+<g id="node13" class="node">
+<title>D0</title>
+<polygon fill="none" stroke="#ff0000" points="400.3193,-22 364.344,-40 292.3936,-40 256.4183,-22 292.3936,-4 364.344,-4 400.3193,-22"/>
+<polygon fill="none" stroke="#ff0000" points="409.2574,-22 365.2876,-44 291.4499,-44 247.4802,-22 291.4499,0 365.2876,0 409.2574,-22"/>
+<text text-anchor="middle" x="328.3688" y="-18.3" font-family="Times,serif" font-size="14.00" fill="#000000">down:failed</text>
+</g>
+<!-- N2&#45;&gt;D0 -->
+<g id="edge17" class="edge">
+<title>N2&#45;&gt;D0</title>
+<path fill="none" stroke="#ff0000" stroke-width="2" d="M694.3918,-531.0749C723.206,-419.3057 758.0861,-205.4415 648.3688,-80 618.1861,-45.4916 497.5217,-31.4247 413.3193,-25.7515"/>
+<polygon fill="#ff0000" stroke="#ff0000" stroke-width="2" points="413.4449,-22.2524 403.2401,-25.1004 412.9936,-29.2378 413.4449,-22.2524"/>
+</g>
+<!-- N4 -->
+<g id="node7" class="node">
+<title>N4</title>
+<ellipse fill="none" stroke="#ffa500" cx="211.3688" cy="-1168" rx="72" ry="72"/>
+<ellipse fill="none" stroke="#ffa500" cx="211.3688" cy="-1168" rx="76" ry="76"/>
+<text text-anchor="middle" x="211.3688" y="-1164.3" font-family="Times,serif" font-size="14.00" fill="#000000">up:resolve</text>
+</g>
+<!-- N3&#45;&gt;N4 -->
+<g id="edge7" class="edge">
+<title>N3&#45;&gt;N4</title>
+<path fill="none" stroke="#ffa500" stroke-width="2" d="M344.5315,-1301.8748C321.9439,-1279.1664 295.5653,-1252.6468 272.272,-1229.2289"/>
+<polygon fill="#ffa500" stroke="#ffa500" stroke-width="2" points="274.6834,-1226.6902 265.1497,-1222.0686 269.7205,-1231.6268 274.6834,-1226.6902"/>
+</g>
+<!-- N5 -->
+<g id="node8" class="node">
+<title>N5</title>
+<ellipse fill="none" stroke="#ffa500" cx="303.3688" cy="-980" rx="72" ry="72"/>
+<ellipse fill="none" stroke="#ffa500" cx="303.3688" cy="-980" rx="76" ry="76"/>
+<text text-anchor="middle" x="303.3688" y="-976.3" font-family="Times,serif" font-size="14.00" fill="#000000">up:reconnect</text>
+</g>
+<!-- N3&#45;&gt;N5 -->
+<g id="edge8" class="edge">
+<title>N3&#45;&gt;N5</title>
+<path fill="none" stroke="#ffa500" stroke-width="2" d="M379.7132,-1282.1631C363.857,-1219.406 341.0993,-1129.3335 324.5591,-1063.869"/>
+<polygon fill="#ffa500" stroke="#ffa500" stroke-width="2" points="327.9058,-1062.8267 322.0628,-1053.9888 321.1191,-1064.5415 327.9058,-1062.8267"/>
+</g>
+<!-- N3&#45;&gt;D0 -->
+<g id="edge18" class="edge">
+<title>N3&#45;&gt;D0</title>
+<path fill="none" stroke="#ff0000" stroke-width="2" d="M429.2219,-1286.1488C463.1132,-1212.2797 512.3688,-1091.2645 512.3688,-980 512.3688,-980 512.3688,-980 512.3688,-792 512.3688,-474.3192 783.7821,-319.7602 575.3688,-80 552.9853,-54.2499 469.2016,-38.7797 405.305,-30.3735"/>
+<polygon fill="#ff0000" stroke="#ff0000" stroke-width="2" points="405.5966,-26.8823 395.2332,-29.0845 404.708,-33.8257 405.5966,-26.8823"/>
+</g>
+<!-- D1 -->
+<g id="node14" class="node">
+<title>D1</title>
+<polygon fill="none" stroke="#000000" points="350.3688,-98 314.3688,-116 242.3688,-116 206.3688,-98 242.3688,-80 314.3688,-80 350.3688,-98"/>
+<text text-anchor="middle" x="278.3688" y="-94.3" font-family="Times,serif" font-size="14.00" fill="#000000">down:damaged</text>
+</g>
+<!-- N3&#45;&gt;D1 -->
+<g id="edge26" class="edge">
+<title>N3&#45;&gt;D1</title>
+<path fill="none" stroke="#000000" stroke-width="2" d="M322.8204,-1347.7263C258.5111,-1336.2473 169.8464,-1308.9217 126.3688,-1244 59.1011,-1143.5544 -158.2652,-797.8876 204.3688,-152 211.1318,-139.9545 221.8244,-129.8664 233.0015,-121.7941"/>
+<polygon fill="#000000" stroke="#000000" stroke-width="2" points="235.2451,-124.5029 241.5861,-116.0151 231.3361,-118.6961 235.2451,-124.5029"/>
+</g>
+<!-- N4&#45;&gt;N5 -->
+<g id="edge9" class="edge">
+<title>N4&#45;&gt;N5</title>
+<path fill="none" stroke="#ffa500" stroke-width="2" d="M244.8432,-1099.5959C251.5443,-1085.9023 258.6292,-1071.4243 265.4677,-1057.4501"/>
+<polygon fill="#ffa500" stroke="#ffa500" stroke-width="2" points="268.6675,-1058.8738 269.9194,-1048.3532 262.38,-1055.7969 268.6675,-1058.8738"/>
+</g>
+<!-- N4&#45;&gt;D0 -->
+<g id="edge19" class="edge">
+<title>N4&#45;&gt;D0</title>
+<path fill="none" stroke="#ff0000" stroke-width="2" d="M182.2708,-1097.5883C130.6372,-966.0773 28.3688,-673.1423 28.3688,-416 28.3688,-416 28.3688,-416 28.3688,-228 28.3688,-154.2584 42.6488,-121.8439 103.3688,-80 128.4366,-62.7251 202.1607,-45.6583 258.4514,-34.5595"/>
+<polygon fill="#ff0000" stroke="#ff0000" stroke-width="2" points="259.2404,-37.9717 268.3878,-32.6261 257.9034,-31.1006 259.2404,-37.9717"/>
+</g>
+<!-- N4&#45;&gt;D1 -->
+<g id="edge27" class="edge">
+<title>N4&#45;&gt;D1</title>
+<path fill="none" stroke="#000000" stroke-width="2" d="M209.2083,-1091.7553C205.5506,-945.7608 199.6177,-616.649 211.3688,-340 214.9276,-256.2179 193.1085,-228.5397 227.3688,-152 232.1284,-141.3666 239.8803,-131.5131 247.9197,-123.2133"/>
+<polygon fill="#000000" stroke="#000000" stroke-width="2" points="250.564,-125.5245 255.3061,-116.0501 245.6908,-120.4994 250.564,-125.5245"/>
+</g>
+<!-- N6 -->
+<g id="node9" class="node">
+<title>N6</title>
+<ellipse fill="none" stroke="#ffa500" cx="303.3688" cy="-792" rx="72" ry="72"/>
+<ellipse fill="none" stroke="#ffa500" cx="303.3688" cy="-792" rx="76" ry="76"/>
+<text text-anchor="middle" x="303.3688" y="-788.3" font-family="Times,serif" font-size="14.00" fill="#000000">up:rejoin</text>
+</g>
+<!-- N5&#45;&gt;N6 -->
+<g id="edge10" class="edge">
+<title>N5&#45;&gt;N6</title>
+<path fill="none" stroke="#ffa500" stroke-width="2" d="M303.3688,-903.8042C303.3688,-895.4826 303.3688,-886.9678 303.3688,-878.5337"/>
+<polygon fill="#ffa500" stroke="#ffa500" stroke-width="2" points="306.8689,-878.3042 303.3688,-868.3043 299.8689,-878.3043 306.8689,-878.3042"/>
+</g>
+<!-- N5&#45;&gt;D0 -->
+<g id="edge20" class="edge">
+<title>N5&#45;&gt;D0</title>
+<path fill="none" stroke="#ff0000" stroke-width="2" d="M254.0847,-921.7614C241.397,-905.0698 228.485,-886.3944 218.3688,-868 116.2346,-682.2886 66.3688,-627.9436 66.3688,-416 66.3688,-416 66.3688,-416 66.3688,-228 66.3688,-152.5844 87.2542,-122.7701 149.3688,-80 181.6933,-57.7424 223.1014,-43.6017 257.948,-34.8615"/>
+<polygon fill="#ff0000" stroke="#ff0000" stroke-width="2" points="259.1567,-38.1706 268.0605,-32.4284 257.5192,-31.3648 259.1567,-38.1706"/>
+</g>
+<!-- N5&#45;&gt;D1 -->
+<g id="edge28" class="edge">
+<title>N5&#45;&gt;D1</title>
+<path fill="none" stroke="#000000" stroke-width="2" d="M355.2,-924.1401C368.1413,-907.2479 380.5465,-887.8749 388.3688,-868 419.1254,-789.8535 403.1412,-763.8747 407.3688,-680 410.7695,-612.5301 422.8899,-593.7484 407.3688,-528 403.2588,-510.59 395.5752,-508.7774 389.3688,-492 335.3278,-345.9134 357.3181,-298.4858 304.3688,-152 301.1307,-143.0417 296.8836,-133.5849 292.7633,-125.1317"/>
+<polygon fill="#000000" stroke="#000000" stroke-width="2" points="295.8407,-123.4606 288.2256,-116.0943 289.5849,-126.6016 295.8407,-123.4606"/>
+</g>
+<!-- N7 -->
+<g id="node10" class="node">
+<title>N7</title>
+<ellipse fill="none" stroke="#ffa500" cx="322.3688" cy="-604" rx="72" ry="72"/>
+<ellipse fill="none" stroke="#ffa500" cx="322.3688" cy="-604" rx="76" ry="76"/>
+<text text-anchor="middle" x="322.3688" y="-600.3" font-family="Times,serif" font-size="14.00" fill="#000000">up:clientreplay</text>
+</g>
+<!-- N6&#45;&gt;N7 -->
+<g id="edge11" class="edge">
+<title>N6&#45;&gt;N7</title>
+<path fill="none" stroke="#ffa500" stroke-width="2" d="M311.0147,-716.3459C311.8988,-707.5982 312.8051,-698.6301 313.7015,-689.7605"/>
+<polygon fill="#ffa500" stroke="#ffa500" stroke-width="2" points="317.1843,-690.1066 314.7076,-679.8054 310.2197,-689.4027 317.1843,-690.1066"/>
+</g>
+<!-- N6&#45;&gt;S0 -->
+<g id="edge15" class="edge">
+<title>N6&#45;&gt;S0</title>
+<path fill="none" stroke="#00ff00" stroke-width="2" d="M360.4621,-741.3761C377.7051,-723.4948 395.2541,-702.2572 407.3688,-680 437.473,-624.6922 454.3874,-555.4885 463.6883,-501.8117"/>
+<polygon fill="#00ff00" stroke="#00ff00" stroke-width="2" points="467.1928,-502.0756 465.3932,-491.6348 460.289,-500.919 467.1928,-502.0756"/>
+</g>
+<!-- N6&#45;&gt;D0 -->
+<g id="edge21" class="edge">
+<title>N6&#45;&gt;D0</title>
+<path fill="none" stroke="#ff0000" stroke-width="2" d="M248.8296,-738.5591C234.4494,-721.2875 220.6505,-701.1069 212.3688,-680 114.7809,-431.287 23.5812,-299.8683 175.3688,-80 186.1466,-64.3881 227.326,-49.2787 264.2499,-38.3971"/>
+<polygon fill="#ff0000" stroke="#ff0000" stroke-width="2" points="265.3806,-41.7138 274.022,-35.5839 263.444,-34.987 265.3806,-41.7138"/>
+</g>
+<!-- N6&#45;&gt;D1 -->
+<g id="edge29" class="edge">
+<title>N6&#45;&gt;D1</title>
+<path fill="none" stroke="#000000" stroke-width="2" d="M259.0732,-730.0838C250.1412,-714.4524 242.0601,-697.2016 237.3688,-680 180.9547,-473.1485 245.862,-210.4698 270.032,-125.7018"/>
+<polygon fill="#000000" stroke="#000000" stroke-width="2" points="273.41,-126.6196 272.8319,-116.0405 266.6866,-124.671 273.41,-126.6196"/>
+</g>
+<!-- N7&#45;&gt;S0 -->
+<g id="edge12" class="edge">
+<title>N7&#45;&gt;S0</title>
+<path fill="none" stroke="#00ff00" stroke-width="2" d="M370.2539,-544.7737C386.0793,-525.2002 403.7979,-503.285 420.0683,-483.1611"/>
+<polygon fill="#00ff00" stroke="#00ff00" stroke-width="2" points="422.8978,-485.2284 426.4633,-475.2515 417.4543,-480.8273 422.8978,-485.2284"/>
+</g>
+<!-- N7&#45;&gt;D0 -->
+<g id="edge22" class="edge">
+<title>N7&#45;&gt;D0</title>
+<path fill="none" stroke="#ff0000" stroke-width="2" d="M278.6936,-541.4308C212.0474,-437.4794 102.2193,-226.1856 197.3688,-80 211.4404,-58.3807 235.6107,-44.7591 259.3879,-36.2034"/>
+<polygon fill="#ff0000" stroke="#ff0000" stroke-width="2" points="260.7836,-39.4276 269.1659,-32.9478 258.5723,-32.7861 260.7836,-39.4276"/>
+</g>
+<!-- N7&#45;&gt;D1 -->
+<g id="edge30" class="edge">
+<title>N7&#45;&gt;D1</title>
+<path fill="none" stroke="#000000" stroke-width="2" d="M315.7815,-528.2467C305.8418,-413.94 287.4667,-202.6265 280.8359,-126.3719"/>
+<polygon fill="#000000" stroke="#000000" stroke-width="2" points="284.2984,-125.788 279.9452,-116.1288 277.3248,-126.3945 284.2984,-125.788"/>
+</g>
+<!-- S3 -->
+<g id="node12" class="node">
+<title>S3</title>
+<ellipse fill="none" stroke="#a020f0" cx="474.3688" cy="-228" rx="72" ry="72"/>
+<ellipse fill="none" stroke="#a020f0" cx="474.3688" cy="-228" rx="76" ry="76"/>
+<text text-anchor="middle" x="474.3688" y="-224.3" font-family="Times,serif" font-size="14.00" fill="#000000">up:stopping</text>
+</g>
+<!-- S0&#45;&gt;S3 -->
+<g id="edge16" class="edge">
+<title>S0&#45;&gt;S3</title>
+<path fill="none" stroke="#a020f0" stroke-width="2" d="M474.3688,-339.8042C474.3688,-331.4826 474.3688,-322.9678 474.3688,-314.5337"/>
+<polygon fill="#a020f0" stroke="#a020f0" stroke-width="2" points="477.8689,-314.3042 474.3688,-304.3043 470.8689,-314.3043 477.8689,-314.3042"/>
+</g>
+<!-- S0&#45;&gt;D0 -->
+<g id="edge23" class="edge">
+<title>S0&#45;&gt;D0</title>
+<path fill="none" stroke="#ff0000" stroke-width="2" d="M527.0686,-361.1971C540.1945,-344.1784 552.4791,-324.4748 559.3688,-304 591.1196,-209.6433 620.5965,-158.5014 559.3688,-80 540.2875,-55.5355 464.1975,-39.8225 404.4932,-31.0076"/>
+<polygon fill="#ff0000" stroke="#ff0000" stroke-width="2" points="404.7058,-27.5023 394.3101,-29.5466 403.7116,-34.4314 404.7058,-27.5023"/>
+</g>
+<!-- S0&#45;&gt;D1 -->
+<g id="edge31" class="edge">
+<title>S0&#45;&gt;D1</title>
+<path fill="none" stroke="#000000" stroke-width="2" d="M424.3129,-358.5726C411.5474,-341.8108 398.7888,-322.897 389.3688,-304 358.0472,-241.1673 385.2932,-210.4284 346.3688,-152 338.776,-140.6027 328.0091,-130.4943 317.2769,-122.1953"/>
+<polygon fill="#000000" stroke="#000000" stroke-width="2" points="319.2611,-119.31 309.1298,-116.2103 315.1168,-124.9514 319.2611,-119.31"/>
+</g>
+<!-- S3&#45;&gt;D0 -->
+<g id="edge24" class="edge">
+<title>S3&#45;&gt;D0</title>
+<path fill="none" stroke="#ff0000" stroke-width="2" d="M430.366,-165.9139C403.7723,-128.3912 371.0479,-82.2184 349.9551,-52.4574"/>
+<polygon fill="#ff0000" stroke="#ff0000" stroke-width="2" points="352.7039,-50.2828 344.0659,-44.148 346.9928,-54.3305 352.7039,-50.2828"/>
+</g>
+<!-- S3&#45;&gt;D1 -->
+<g id="edge32" class="edge">
+<title>S3&#45;&gt;D1</title>
+<path fill="none" stroke="#000000" stroke-width="2" d="M417.8114,-176.8892C407.0411,-168.1353 395.6128,-159.467 384.3688,-152 366.1181,-139.88 344.7922,-128.5324 326.0732,-119.3982"/>
+<polygon fill="#000000" stroke="#000000" stroke-width="2" points="327.366,-116.1365 316.8361,-114.965 324.3371,-122.4473 327.366,-116.1365"/>
+</g>
+<!-- D3 -->
+<g id="node15" class="node">
+<title>D3</title>
+<polygon fill="none" stroke="#a020f0" points="550.3688,-98 514.3688,-116 442.3688,-116 406.3688,-98 442.3688,-80 514.3688,-80 550.3688,-98"/>
+<text text-anchor="middle" x="478.3688" y="-94.3" font-family="Times,serif" font-size="14.00" fill="#000000">down:stopped</text>
+</g>
+<!-- S3&#45;&gt;D3 -->
+<g id="edge34" class="edge">
+<title>S3&#45;&gt;D3</title>
+<path fill="none" stroke="#a020f0" stroke-width="2" d="M476.7072,-152.0023C476.9861,-142.9385 477.2559,-134.1686 477.4942,-126.4252"/>
+<polygon fill="#a020f0" stroke="#a020f0" stroke-width="2" points="480.9983,-126.3415 477.8076,-116.2385 474.0016,-126.1261 480.9983,-126.3415"/>
+</g>
+<!-- D0&#45;&gt;N3 -->
+<g id="edge25" class="edge">
+<title>D0&#45;&gt;N3</title>
+<path fill="none" stroke="#ff0000" stroke-width="2" d="M397.758,-27.8117C467.676,-35.4452 568.4799,-51.3676 593.3688,-80 801.7821,-319.7602 530.3688,-474.3192 530.3688,-792 530.3688,-980 530.3688,-980 530.3688,-980 530.3688,-1089.5002 482.6629,-1208.4443 444.4976,-1282.5994"/>
+<polygon fill="#ff0000" stroke="#ff0000" stroke-width="2" points="441.1858,-1281.3802 439.6575,-1291.8642 447.3902,-1284.6215 441.1858,-1281.3802"/>
+</g>
+<!-- D1&#45;&gt;D0 -->
+<g id="edge33" class="edge">
+<title>D1&#45;&gt;D0</title>
+<path fill="none" stroke="#ff0000" stroke-width="2" d="M290.2181,-79.9891C295.5579,-71.8726 302.0166,-62.0553 308.0897,-52.8243"/>
+<polygon fill="#ff0000" stroke="#ff0000" stroke-width="2" points="311.1727,-54.5061 313.745,-44.2282 305.3248,-50.6587 311.1727,-54.5061"/>
+</g>
+</g>
+</svg>
diff --git a/doc/cephfs/mds-states.rst b/doc/cephfs/mds-states.rst
new file mode 100644
index 000000000..aba066ae2
--- /dev/null
+++ b/doc/cephfs/mds-states.rst
@@ -0,0 +1,227 @@
+
+MDS States
+==========
+
+
+The Metadata Server (MDS) goes through several states during normal operation
+in CephFS. For example, some states indicate that the MDS is recovering from a
+failover by a previous instance of the MDS. Here we'll document all of these
+states and include a state diagram to visualize the transitions.
+
+State Descriptions
+------------------
+
+Common states
+~~~~~~~~~~~~~~
+
+
+::
+
+    up:active
+
+This is the normal operating state of the MDS. It indicates that the MDS
+and its rank in the file system is available.
+
+
+::
+
+    up:standby
+
+The MDS is available to takeover for a failed rank (see also :ref:`mds-standby`).
+The monitor will automatically assign an MDS in this state to a failed rank
+once available.
+
+
+::
+
+    up:standby_replay
+
+The MDS is following the journal of another ``up:active`` MDS. Should the
+active MDS fail, having a standby MDS in replay mode is desirable as the MDS is
+replaying the live journal and will more quickly takeover. A downside to having
+standby replay MDSs is that they are not available to takeover for any other
+MDS that fails, only the MDS they follow.
+
+
+Less common or transitory states
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+
+::
+
+    up:boot
+
+This state is broadcast to the Ceph monitors during startup. This state is
+never visible as the Monitor immediately assign the MDS to an available rank or
+commands the MDS to operate as a standby. The state is documented here for
+completeness.
+
+
+::
+
+    up:creating
+
+The MDS is creating a new rank (perhaps rank 0) by constructing some per-rank
+metadata (like the journal) and entering the MDS cluster.
+
+
+::
+
+    up:starting
+
+The MDS is restarting a stopped rank. It opens associated per-rank metadata
+and enters the MDS cluster.
+
+
+::
+
+    up:stopping
+
+When a rank is stopped, the monitors command an active MDS to enter the
+``up:stopping`` state. In this state, the MDS accepts no new client
+connections, migrates all subtrees to other ranks in the file system, flush its
+metadata journal, and, if the last rank (0), evict all clients and shutdown
+(see also :ref:`cephfs-administration`).
+
+
+::
+
+    up:replay
+
+The MDS taking over a failed rank. This state represents that the MDS is
+recovering its journal and other metadata.
+
+
+::
+
+    up:resolve
+
+The MDS enters this state from ``up:replay`` if the Ceph file system has
+multiple ranks (including this one), i.e. it's not a single active MDS cluster.
+The MDS is resolving any uncommitted inter-MDS operations. All ranks in the
+file system must be in this state or later for progress to be made, i.e. no
+rank can be failed/damaged or ``up:replay``.
+
+
+::
+
+    up:reconnect
+
+An MDS enters this state from ``up:replay`` or ``up:resolve``. This state is to
+solicit reconnections from clients. Any client which had a session with this
+rank must reconnect during this time, configurable via
+``mds_reconnect_timeout``.
+
+
+::
+
+    up:rejoin
+
+The MDS enters this state from ``up:reconnect``. In this state, the MDS is
+rejoining the MDS cluster cache. In particular, all inter-MDS locks on metadata
+are reestablished.
+
+If there are no known client requests to be replayed, the MDS directly becomes
+``up:active`` from this state.
+
+
+::
+
+    up:clientreplay
+
+The MDS may enter this state from ``up:rejoin``. The MDS is replaying any
+client requests which were replied to but not yet durable (not journaled).
+Clients resend these requests during ``up:reconnect`` and the requests are
+replayed once again. The MDS enters ``up:active`` after completing replay.
+
+
+Failed states
+~~~~~~~~~~~~~
+
+::
+
+    down:failed
+
+No MDS actually holds this state. Instead, it is applied to the rank in the file system. For example:
+
+::
+
+    $ ceph fs dump
+    ...
+    max_mds 1
+    in      0
+    up      {}
+    failed  0
+    ...
+
+Rank 0 is part of the failed set.
+
+
+::
+
+    down:damaged
+
+No MDS actually holds this state. Instead, it is applied to the rank in the file system. For example:
+
+::
+
+    $ ceph fs dump
+    ...
+    max_mds 1
+    in      0
+    up      {}
+    failed  
+    damaged 0
+    ...
+
+Rank 0 has become damaged (see also :ref:`cephfs-disaster-recovery`) and placed in
+the ``damaged`` set. An MDS which was running as rank 0 found metadata damage
+that could not be automatically recovered. Operator intervention is required.
+
+
+::
+
+    down:stopped
+    
+No MDS actually holds this state. Instead, it is applied to the rank in the file system. For example:
+
+::
+
+    $ ceph fs dump
+    ...
+    max_mds 1
+    in      0
+    up      {}
+    failed  
+    damaged 
+    stopped 1
+    ...
+
+The rank has been stopped by reducing ``max_mds`` (see also :ref:`cephfs-multimds`).
+
+State Diagram
+-------------
+
+This state diagram shows the possible state transitions for the MDS/rank. The legend is as follows:
+
+Color
+~~~~~
+
+- Green: MDS is active.
+- Orange: MDS is in transient state trying to become active.
+- Red: MDS is indicating a state that causes the rank to be marked failed.
+- Purple: MDS and rank is stopping.
+- Black: MDS is indicating a state that causes the rank to be marked damaged.
+
+Shape
+~~~~~
+
+- Circle: an MDS holds this state.
+- Hexagon: no MDS holds this state (it is applied to the rank).
+
+Lines
+~~~~~
+
+- A double-lined shape indicates the rank is "in".
+
+.. image:: mds-state-diagram.svg
diff --git a/doc/cephfs/mount-prerequisites.rst b/doc/cephfs/mount-prerequisites.rst
new file mode 100644
index 000000000..6ed8a19b6
--- /dev/null
+++ b/doc/cephfs/mount-prerequisites.rst
@@ -0,0 +1,75 @@
+Mount CephFS: Prerequisites
+===========================
+
+You can use CephFS by mounting it to your local filesystem or by using
+`cephfs-shell`_. Mounting CephFS requires superuser privileges to trim
+dentries by issuing a remount of itself. CephFS can be mounted
+`using kernel`_ as well as `using FUSE`_. Both have their own
+advantages. Read the following section to understand more about both of
+these ways to mount CephFS.
+
+For Windows CephFS mounts, please check the `ceph-dokan`_ page.
+
+Which CephFS Client?
+--------------------
+
+The FUSE client is the most accessible and the easiest to upgrade to the
+version of Ceph used by the storage cluster, while the kernel client will
+always gives better performance.
+
+When encountering bugs or performance issues, it is often instructive to
+try using the other client, in order to find out whether the bug was
+client-specific or not (and then to let the developers know).
+
+General Pre-requisite for Mounting CephFS
+-----------------------------------------
+Before mounting CephFS, ensure that the client host (where CephFS has to be
+mounted and used) has a copy of the Ceph configuration file (i.e.
+``ceph.conf``) and a keyring of the CephX user that has permission to access
+the MDS. Both of these files must already be present on the host where the
+Ceph MON resides.
+
+#. Generate a minimal conf file for the client host and place it at a
+   standard location::
+
+    # on client host
+    mkdir -p -m 755 /etc/ceph
+    ssh {user}@{mon-host} "sudo ceph config generate-minimal-conf" | sudo tee /etc/ceph/ceph.conf
+
+   Alternatively, you may copy the conf file. But the above method generates
+   a conf with minimal details which is usually sufficient. For more
+   information, see `Client Authentication`_ and :ref:`bootstrap-options`.
+
+#. Ensure that the conf has appropriate permissions::
+
+    chmod 644 /etc/ceph/ceph.conf
+
+#. Create a CephX user and get its secret key::
+
+    ssh {user}@{mon-host} "sudo ceph fs authorize cephfs client.foo / rw" | sudo tee /etc/ceph/ceph.client.foo.keyring
+
+   In above command, replace ``cephfs`` with the name of your CephFS, ``foo``
+   by the name you want for your CephX user and ``/`` by the path within your
+   CephFS for which you want to allow access to the client host and ``rw``
+   stands for both read and write permissions. Alternatively, you may copy the
+   Ceph keyring from the MON host to client host at ``/etc/ceph`` but creating
+   a keyring specific to the client host is better. While creating a CephX
+   keyring/client, using same client name across multiple machines is perfectly
+   fine.
+
+   .. note:: If you get 2 prompts for password while running above any of 2
+             above command, run ``sudo ls`` (or any other trivial command with
+             sudo) immediately before these commands.
+
+#. Ensure that the keyring has appropriate permissions::
+
+    chmod 600 /etc/ceph/ceph.client.foo.keyring
+
+.. note:: There might be few more prerequisites for kernel and FUSE mounts
+   individually, please check respective mount documents.
+
+.. _Client Authentication: ../client-auth
+.. _cephfs-shell: ../cephfs-shell
+.. _using kernel: ../mount-using-kernel-driver
+.. _using FUSE: ../mount-using-fuse
+.. _ceph-dokan: ../ceph-dokan
diff --git a/doc/cephfs/mount-using-fuse.rst b/doc/cephfs/mount-using-fuse.rst
new file mode 100644
index 000000000..27768f503
--- /dev/null
+++ b/doc/cephfs/mount-using-fuse.rst
@@ -0,0 +1,102 @@
+========================
+ Mount CephFS using FUSE
+========================
+
+`ceph-fuse`_ is an alternate way of mounting CephFS, although it mounts it
+in userspace. Therefore, performance of FUSE can be relatively lower but FUSE
+clients can be more manageable, especially while upgrading CephFS.
+
+Prerequisites
+=============
+
+Go through the prerequisites required by both, kernel as well as FUSE mounts,
+in `Mount CephFS: Prerequisites`_ page.
+
+.. note:: Mounting CephFS using FUSE requires superuser privileges to trim dentries
+   by issuing a remount of itself.
+
+Synopsis
+========
+In general, the command to mount CephFS via FUSE looks like this::
+
+    ceph-fuse {mountpoint} {options}
+
+Mounting CephFS
+===============
+To FUSE-mount the Ceph file system, use the ``ceph-fuse`` command::
+
+    mkdir /mnt/mycephfs
+    ceph-fuse --id foo /mnt/mycephfs
+
+Option ``-id`` passes the name of the CephX user whose keyring we intend to
+use for mounting CephFS. In the above command, it's ``foo``. You can also use
+``-n`` instead, although ``--id`` is evidently easier::
+
+    ceph-fuse -n client.foo /mnt/mycephfs
+
+In case the keyring is not present in standard locations, you may pass it
+too::
+
+    ceph-fuse --id foo -k /path/to/keyring /mnt/mycephfs
+
+You may pass the MON's socket too, although this is not mandatory::
+
+    ceph-fuse --id foo -m 192.168.0.1:6789 /mnt/mycephfs
+
+You can also mount a specific directory within CephFS instead of mounting
+root of CephFS on your local FS::
+
+    ceph-fuse --id foo -r /path/to/dir /mnt/mycephfs
+
+If you have more than one FS on your Ceph cluster, use the option
+``--client_fs`` to mount the non-default FS::
+
+    ceph-fuse --id foo --client_fs mycephfs2 /mnt/mycephfs2
+
+You may also add a ``client_fs`` setting to your ``ceph.conf``
+
+Unmounting CephFS
+=================
+
+Use ``umount`` to unmount CephFS like any other FS::
+
+    umount /mnt/mycephfs
+
+.. tip:: Ensure that you are not within the file system directories before
+   executing this command.
+
+Persistent Mounts
+=================
+
+To mount CephFS as a file system in user space, add the following to ``/etc/fstab``::
+
+       #DEVICE PATH       TYPE      OPTIONS
+       none    /mnt/mycephfs  fuse.ceph ceph.id={user-ID}[,ceph.conf={path/to/conf.conf}],_netdev,defaults  0 0
+
+For example::
+
+       none    /mnt/mycephfs  fuse.ceph ceph.id=myuser,_netdev,defaults  0 0
+       none    /mnt/mycephfs  fuse.ceph ceph.id=myuser,ceph.conf=/etc/ceph/foo.conf,_netdev,defaults  0 0
+
+Ensure you use the ID (e.g., ``myuser``, not ``client.myuser``). You can pass
+any valid ``ceph-fuse`` option to the command line this way.
+
+To mount a subdirectory of the CephFS, add the following to ``/etc/fstab``::
+
+       none    /mnt/mycephfs  fuse.ceph ceph.id=myuser,ceph.client_mountpoint=/path/to/dir,_netdev,defaults  0 0
+
+``ceph-fuse@.service`` and ``ceph-fuse.target`` systemd units are available.
+As usual, these unit files declare the default dependencies and recommended
+execution context for ``ceph-fuse``. After making the fstab entry shown above,
+run following commands::
+
+    systemctl start ceph-fuse@/mnt/mycephfs.service
+    systemctl enable ceph-fuse.target
+    systemctl enable ceph-fuse@-mnt-mycephfs.service
+
+See :ref:`User Management <user-management>` for details on CephX user management and `ceph-fuse`_
+manual for more options it can take. For troubleshooting, see
+:ref:`ceph_fuse_debugging`.
+
+.. _ceph-fuse: ../../man/8/ceph-fuse/#options
+.. _Mount CephFS\: Prerequisites: ../mount-prerequisites
diff --git a/doc/cephfs/mount-using-kernel-driver.rst b/doc/cephfs/mount-using-kernel-driver.rst
new file mode 100644
index 000000000..5bd98dd51
--- /dev/null
+++ b/doc/cephfs/mount-using-kernel-driver.rst
@@ -0,0 +1,131 @@
+=================================
+ Mount CephFS using Kernel Driver
+=================================
+
+The CephFS kernel driver is part of the Linux kernel. It allows mounting
+CephFS as a regular file system with native kernel performance. It is the
+client of choice for most use-cases.
+
+Prerequisites
+=============
+
+Complete General Prerequisites
+------------------------------
+Go through the prerequisites required by both, kernel as well as FUSE mounts,
+in `Mount CephFS: Prerequisites`_ page.
+
+Is mount helper is present?
+---------------------------
+``mount.ceph`` helper is installed by Ceph packages. The helper passes the
+monitor address(es) and CephX user keyrings automatically saving the Ceph
+admin the effort to pass these details explicitly while mountng CephFS. In
+case the helper is not present on the client machine, CephFS can still be
+mounted using kernel but by passing these details explicitly to the ``mount``
+command. To check whether it is present on your system, do::
+
+    stat /sbin/mount.ceph
+
+Which Kernel Version?
+---------------------
+
+Because the kernel client is distributed as part of the linux kernel (not
+as part of packaged ceph releases), you will need to consider which kernel
+version to use on your client nodes. Older kernels are known to include buggy
+ceph clients, and may not support features that more recent Ceph clusters
+support.
+
+Remember that the "latest" kernel in a stable linux distribution is likely
+to be years behind the latest upstream linux kernel where Ceph development
+takes place (including bug fixes).
+
+As a rough guide, as of Ceph 10.x (Jewel), you should be using a least a 4.x
+kernel. If you absolutely have to use an older kernel, you should use the
+fuse client instead of the kernel client.
+
+This advice does not apply if you are using a linux distribution that
+includes CephFS support, as in this case the distributor will be responsible
+for backporting fixes to their stable kernel: check with your vendor.
+
+Synopsis
+========
+In general, the command to mount CephFS via kernel driver looks like this::
+
+    mount -t ceph {device-string}:{path-to-mounted} {mount-point} -o {key-value-args} {other-args}
+
+Mounting CephFS
+===============
+On Ceph clusters, CephX is enabled by default. Use ``mount`` command to
+mount CephFS with the kernel driver::
+
+    mkdir /mnt/mycephfs
+    mount -t ceph :/ /mnt/mycephfs -o name=foo
+
+The key-value argument right after option ``-o`` is CephX credential;
+``name`` is the username of the CephX user we are using to mount CephFS. The
+default value for ``name`` is ``guest``.
+
+The kernel driver also requires MON's socket and the secret key for the CephX
+user. In case of the above command, ``mount.ceph`` helper figures out these
+details automatically by finding and reading Ceph conf file and keyring. In
+case you don't have these files on the host where you're running mount
+command, you can pass these details yourself too::
+
+    mount -t ceph 192.168.0.1:6789,192.168.0.2:6789:/ /mnt/mycephfs -o name=foo,secret=AQATSKdNGBnwLhAAnNDKnH65FmVKpXZJVasUeQ==
+
+Passing a single MON socket in above command works too. A potential problem
+with the command above is that the secret key is left in your shell's command
+history. To prevent that you can copy the secret key inside a file and pass
+the file by using the option ``secretfile`` instead of ``secret``::
+
+    mount -t ceph :/ /mnt/mycephfs -o name=foo,secretfile=/etc/ceph/foo.secret
+
+Ensure the permissions on the secret key file are appropriate (preferably,
+``600``).
+
+In case CephX is disabled, you can omit ``-o`` and the list of key-value
+arguments that follow it::
+
+    mount -t ceph :/ /mnt/mycephfs
+
+To mount a subtree of the CephFS root, append the path to the device string::
+
+    mount -t ceph :/subvolume/dir1/dir2 /mnt/mycephfs -o name=fs
+
+If you have more than one file system on your Ceph cluster, you can mount the
+non-default FS as follows::
+
+    mount -t ceph :/ /mnt/mycephfs2 -o name=fs,fs=mycephfs2
+
+Unmounting CephFS
+=================
+To unmount the Ceph file system, use the ``umount`` command as usual::
+
+    umount /mnt/mycephfs
+
+.. tip:: Ensure that you are not within the file system directories before
+   executing this command.
+
+Persistent Mounts
+==================
+
+To mount CephFS in your file systems table as a kernel driver, add the
+following to ``/etc/fstab``::
+
+    [{ipaddress}:{port}]:/ {mount}/{mountpoint} ceph [name=username,secret=secretkey|secretfile=/path/to/secretfile],[{mount.options}]
+
+For example::
+
+    :/     /mnt/ceph    ceph    name=admin,noatime,_netdev    0       2
+
+The default for the ``name=`` parameter is ``guest``. If the ``secret`` or
+``secretfile`` options are not specified then the mount helper will attempt to
+find a secret for the given ``name`` in one of the configured keyrings.
+
+See `User Management`_ for details on CephX user management and mount.ceph_
+manual for more options it can take. For troubleshooting, see
+:ref:`kernel_mount_debugging`.
+
+.. _fstab: ../fstab/#kernel-driver
+.. _Mount CephFS\: Prerequisites: ../mount-prerequisites
+.. _mount.ceph: ../../man/8/mount.ceph/
+.. _User Management: ../../rados/operations/user-management/
diff --git a/doc/cephfs/multifs.rst b/doc/cephfs/multifs.rst
new file mode 100644
index 000000000..2dcba7ae0
--- /dev/null
+++ b/doc/cephfs/multifs.rst
@@ -0,0 +1,54 @@
+.. _cephfs-multifs:
+
+Multiple Ceph File Systems
+==========================
+
+
+Beginning with the Pacific release, multiple file system support is stable
+and ready to use. This functionality allows configuring separate file systems
+with full data separation on separate pools.
+
+Existing clusters must set a flag to enable multiple file systems::
+
+    ceph fs flag set enable_multiple true
+
+New Ceph clusters automatically set this.
+
+
+Creating a new Ceph File System
+-------------------------------
+
+The new ``volumes`` plugin interface (see: :doc:`/cephfs/fs-volumes`) automates
+most of the work of configuring a new file system. The "volume" concept is
+simply a new file system. This can be done via::
+
+    ceph fs volume create <fs_name>
+
+Ceph will create the new pools and automate the deployment of new MDS to
+support the new file system. The deployment technology used, e.g. cephadm, will
+also configure the MDS affinity (see: :ref:`mds-join-fs`) of new MDS daemons to
+operate the new file system.
+
+
+Securing access
+---------------
+
+The ``fs authorize`` command allows configuring the client's access to a
+particular file system. See also in :ref:`fs-authorize-multifs`. The client will
+only have visibility of authorized file systems and the Monitors/MDS will
+reject access to clients without authorization.
+
+
+Other Notes
+-----------
+
+Multiple file systems do not share pools. This is particularly important for
+snapshots but also because no measures are in place to prevent duplicate
+inodes. The Ceph commands prevent this dangerous configuration.
+
+Each file system has its own set of MDS ranks. Consequently, each new file
+system requires more MDS daemons to operate and increases operational costs.
+This can be useful for increasing metadata throughput by application or user
+base but also adds cost to the creation of a file system. Generally, a single
+file system with subtree pinning is a better choice for isolating load between
+applications.
diff --git a/doc/cephfs/multimds.rst b/doc/cephfs/multimds.rst
new file mode 100644
index 000000000..db9b52f3a
--- /dev/null
+++ b/doc/cephfs/multimds.rst
@@ -0,0 +1,227 @@
+.. _cephfs-multimds:
+
+Configuring multiple active MDS daemons
+---------------------------------------
+
+*Also known as: multi-mds, active-active MDS*
+
+Each CephFS file system is configured for a single active MDS daemon
+by default.  To scale metadata performance for large scale systems, you
+may enable multiple active MDS daemons, which will share the metadata
+workload with one another.
+
+When should I use multiple active MDS daemons?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You should configure multiple active MDS daemons when your metadata performance
+is bottlenecked on the single MDS that runs by default.
+
+Adding more daemons may not increase performance on all workloads.  Typically,
+a single application running on a single client will not benefit from an
+increased number of MDS daemons unless the application is doing a lot of
+metadata operations in parallel.
+
+Workloads that typically benefit from a larger number of active MDS daemons
+are those with many clients, perhaps working on many separate directories.
+
+
+Increasing the MDS active cluster size
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Each CephFS file system has a *max_mds* setting, which controls how many ranks
+will be created.  The actual number of ranks in the file system will only be
+increased if a spare daemon is available to take on the new rank. For example,
+if there is only one MDS daemon running, and max_mds is set to two, no second
+rank will be created. (Note that such a configuration is not Highly Available
+(HA) because no standby is available to take over for a failed rank. The
+cluster will complain via health warnings when configured this way.)
+
+Set ``max_mds`` to the desired number of ranks.  In the following examples
+the "fsmap" line of "ceph status" is shown to illustrate the expected
+result of commands.
+
+::
+
+    # fsmap e5: 1/1/1 up {0=a=up:active}, 2 up:standby
+
+    ceph fs set <fs_name> max_mds 2
+
+    # fsmap e8: 2/2/2 up {0=a=up:active,1=c=up:creating}, 1 up:standby
+    # fsmap e9: 2/2/2 up {0=a=up:active,1=c=up:active}, 1 up:standby
+
+The newly created rank (1) will pass through the 'creating' state
+and then enter this 'active state'.
+
+Standby daemons
+~~~~~~~~~~~~~~~
+
+Even with multiple active MDS daemons, a highly available system **still
+requires standby daemons** to take over if any of the servers running
+an active daemon fail.
+
+Consequently, the practical maximum of ``max_mds`` for highly available systems
+is at most one less than the total number of MDS servers in your system.
+
+To remain available in the event of multiple server failures, increase the
+number of standby daemons in the system to match the number of server failures
+you wish to withstand.
+
+Decreasing the number of ranks
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Reducing the number of ranks is as simple as reducing ``max_mds``:
+
+::
+    
+    # fsmap e9: 2/2/2 up {0=a=up:active,1=c=up:active}, 1 up:standby
+    ceph fs set <fs_name> max_mds 1
+    # fsmap e10: 2/2/1 up {0=a=up:active,1=c=up:stopping}, 1 up:standby
+    # fsmap e10: 2/2/1 up {0=a=up:active,1=c=up:stopping}, 1 up:standby
+    ...
+    # fsmap e10: 1/1/1 up {0=a=up:active}, 2 up:standby
+
+The cluster will automatically stop extra ranks incrementally until ``max_mds``
+is reached.
+
+See :doc:`/cephfs/administration` for more details which forms ``<role>`` can
+take.
+
+Note: stopped ranks will first enter the stopping state for a period of
+time while it hands off its share of the metadata to the remaining active
+daemons.  This phase can take from seconds to minutes.  If the MDS appears to
+be stuck in the stopping state then that should be investigated as a possible
+bug.
+
+If an MDS daemon crashes or is killed while in the ``up:stopping`` state, a
+standby will take over and the cluster monitors will against try to stop
+the daemon.
+
+When a daemon finishes stopping, it will respawn itself and go back to being a
+standby.
+
+
+.. _cephfs-pinning:
+
+Manually pinning directory trees to a particular rank
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In multiple active metadata server configurations, a balancer runs which works
+to spread metadata load evenly across the cluster. This usually works well
+enough for most users but sometimes it is desirable to override the dynamic
+balancer with explicit mappings of metadata to particular ranks. This can allow
+the administrator or users to evenly spread application load or limit impact of
+users' metadata requests on the entire cluster.
+
+The mechanism provided for this purpose is called an ``export pin``, an
+extended attribute of directories. The name of this extended attribute is
+``ceph.dir.pin``.  Users can set this attribute using standard commands:
+
+::
+
+    setfattr -n ceph.dir.pin -v 2 path/to/dir
+
+The value of the extended attribute is the rank to assign the directory subtree
+to. A default value of ``-1`` indicates the directory is not pinned.
+
+A directory's export pin is inherited from its closest parent with a set export
+pin.  In this way, setting the export pin on a directory affects all of its
+children. However, the parents pin can be overridden by setting the child
+directory's export pin. For example:
+
+::
+
+    mkdir -p a/b
+    # "a" and "a/b" both start without an export pin set
+    setfattr -n ceph.dir.pin -v 1 a/
+    # a and b are now pinned to rank 1
+    setfattr -n ceph.dir.pin -v 0 a/b
+    # a/b is now pinned to rank 0 and a/ and the rest of its children are still pinned to rank 1
+
+
+.. _cephfs-ephemeral-pinning:
+
+Setting subtree partitioning policies
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+It is also possible to setup **automatic** static partitioning of subtrees via
+a set of **policies**. In CephFS, this automatic static partitioning is
+referred to as **ephemeral pinning**. Any directory (inode) which is
+ephemerally pinned will be automatically assigned to a particular rank
+according to a consistent hash of its inode number. The set of all
+ephemerally pinned directories should be uniformly distributed across all
+ranks.
+
+Ephemerally pinned directories are so named because the pin may not persist
+once the directory inode is dropped from cache. However, an MDS failover does
+not affect the ephemeral nature of the pinned directory. The MDS records what
+subtrees are ephemerally pinned in its journal so MDS failovers do not drop
+this information.
+
+A directory is either ephemerally pinned or not. Which rank it is pinned to is
+derived from its inode number and a consistent hash. This means that
+ephemerally pinned directories are somewhat evenly spread across the MDS
+cluster. The **consistent hash** also minimizes redistribution when the MDS
+cluster grows or shrinks. So, growing an MDS cluster may automatically increase
+your metadata throughput with no other administrative intervention.
+
+Presently, there are two types of ephemeral pinning:
+
+**Distributed Ephemeral Pins**: This policy causes a directory to fragment
+(even well below the normal fragmentation thresholds) and distribute its
+fragments as ephemerally pinned subtrees. This has the effect of distributing
+immediate children across a range of MDS ranks.  The canonical example use-case
+would be the ``/home`` directory: we want every user's home directory to be
+spread across the entire MDS cluster. This can be set via:
+
+::
+
+    setfattr -n ceph.dir.pin.distributed -v 1 /cephfs/home
+
+
+**Random Ephemeral Pins**: This policy indicates any descendent sub-directory
+may be ephemerally pinned. This is set through the extended attribute
+``ceph.dir.pin.random`` with the value set to the percentage of directories
+that should be pinned. For example:
+
+::
+
+    setfattr -n ceph.dir.pin.random -v 0.5 /cephfs/tmp
+
+Would cause any directory loaded into cache or created under ``/tmp`` to be
+ephemerally pinned 50 percent of the time.
+
+It is recommended to only set this to small values, like ``.001`` or ``0.1%``.
+Having too many subtrees may degrade performance. For this reason, the config
+``mds_export_ephemeral_random_max`` enforces a cap on the maximum of this
+percentage (default: ``.01``). The MDS returns ``EINVAL`` when attempting to
+set a value beyond this config.
+
+Both random and distributed ephemeral pin policies are off by default in
+Octopus. The features may be enabled via the
+``mds_export_ephemeral_random`` and ``mds_export_ephemeral_distributed``
+configuration options.
+
+Ephemeral pins may override parent export pins and vice versa. What determines
+which policy is followed is the rule of the closest parent: if a closer parent
+directory has a conflicting policy, use that one instead. For example:
+
+::
+
+    mkdir -p foo/bar1/baz foo/bar2
+    setfattr -n ceph.dir.pin -v 0 foo
+    setfattr -n ceph.dir.pin.distributed -v 1 foo/bar1
+
+The ``foo/bar1/baz`` directory will be ephemerally pinned because the
+``foo/bar1`` policy overrides the export pin on ``foo``. The ``foo/bar2``
+directory will obey the pin on ``foo`` normally.
+
+For the reverse situation:
+
+::
+
+    mkdir -p home/{patrick,john}
+    setfattr -n ceph.dir.pin.distributed -v 1 home
+    setfattr -n ceph.dir.pin -v 2 home/patrick
+
+The ``home/patrick`` directory and its children will be pinned to rank 2
+because its export pin overrides the policy on ``home``.
diff --git a/doc/cephfs/nfs.rst b/doc/cephfs/nfs.rst
new file mode 100644
index 000000000..6c44b8650
--- /dev/null
+++ b/doc/cephfs/nfs.rst
@@ -0,0 +1,87 @@
+.. _cephfs-nfs:
+
+===
+NFS
+===
+
+CephFS namespaces can be exported over NFS protocol using the NFS-Ganesha NFS
+server.  This document provides information on configuring NFS-Ganesha
+clusters manually.  The simplest and preferred way of managing NFS-Ganesha
+clusters and CephFS exports is using ``ceph nfs ...`` commands. See
+:doc:`/mgr/nfs` for more details. As the deployment is done using cephadm or
+rook.
+
+Requirements
+============
+
+-  Ceph file system (preferably latest stable luminous or higher versions)
+-  In the NFS server host machine, 'libcephfs2' (preferably latest stable
+   luminous or higher), 'nfs-ganesha' and 'nfs-ganesha-ceph' packages (latest
+   ganesha v2.5 stable or higher versions)
+-  NFS-Ganesha server host connected to the Ceph public network
+
+.. note::
+   It is recommended to use 3.5 or later stable version of NFS-Ganesha
+   packages with pacific (16.2.x) or later stable version of Ceph packages.
+
+Configuring NFS-Ganesha to export CephFS
+========================================
+
+NFS-Ganesha provides a File System Abstraction Layer (FSAL) to plug in different
+storage backends. `FSAL_CEPH <https://github.com/nfs-ganesha/nfs-ganesha/tree/next/src/FSAL/FSAL_CEPH>`_
+is the plugin FSAL for CephFS. For each NFS-Ganesha export, FSAL_CEPH uses a
+libcephfs client, user-space CephFS client, to mount the CephFS path that
+NFS-Ganesha exports.
+
+Setting up NFS-Ganesha with CephFS, involves setting up NFS-Ganesha's
+configuration file, and also setting up a Ceph configuration file and cephx
+access credentials for the Ceph clients created by NFS-Ganesha to access
+CephFS.
+
+NFS-Ganesha configuration
+-------------------------
+
+A sample ganesha.conf configured with FSAL_CEPH can be found here,
+`<https://github.com/nfs-ganesha/nfs-ganesha/blob/next/src/config_samples/ceph.conf>`_.
+It is suitable for a standalone NFS-Ganesha server, or an active/passive
+configuration of NFS-Ganesha servers managed by some sort of clustering
+software (e.g., Pacemaker). Important details about the options are
+added as comments in the sample conf. There are options to do the following:
+
+- minimize Ganesha caching wherever possible since the libcephfs clients
+  (of FSAL_CEPH) also cache aggressively
+
+- read from Ganesha config files stored in RADOS objects
+
+- store client recovery data in RADOS OMAP key-value interface
+
+- mandate NFSv4.1+ access
+
+- enable read delegations (need at least v13.0.1 'libcephfs2' package
+  and v2.6.0 stable 'nfs-ganesha' and 'nfs-ganesha-ceph' packages)
+
+Configuration for libcephfs clients
+-----------------------------------
+
+Required ceph.conf for libcephfs clients includes:
+
+* a [client] section with ``mon_host`` option set to let the clients connect
+  to the Ceph cluster's monitors, usually generated via ``ceph config generate-minimal-conf``, e.g., ::
+
+    [global]
+            mon host = [v2:192.168.1.7:3300,v1:192.168.1.7:6789], [v2:192.168.1.8:3300,v1:192.168.1.8:6789], [v2:192.168.1.9:3300,v1:192.168.1.9:6789]
+
+Mount using NFSv4 clients
+=========================
+
+It is preferred to mount the NFS-Ganesha exports using NFSv4.1+ protocols
+to get the benefit of sessions.
+
+Conventions for mounting NFS resources are platform-specific. The
+following conventions work on Linux and some Unix platforms:
+
+.. code:: bash
+
+    mount -t nfs -o nfsvers=4.1,proto=tcp <ganesha-host-name>:<ganesha-pseudo-path> <mount-point>
+
+
diff --git a/doc/cephfs/posix.rst b/doc/cephfs/posix.rst
new file mode 100644
index 000000000..d80cbe67e
--- /dev/null
+++ b/doc/cephfs/posix.rst
@@ -0,0 +1,99 @@
+========================
+ Differences from POSIX
+========================
+
+CephFS aims to adhere to POSIX semantics wherever possible.  For
+example, in contrast to many other common network file systems like
+NFS, CephFS maintains strong cache coherency across clients.  The goal
+is for processes communicating via the file system to behave the same
+when they are on different hosts as when they are on the same host.
+
+However, there are a few places where CephFS diverges from strict
+POSIX semantics for various reasons:
+
+- If a client is writing to a file and fails, its writes are not
+  necessarily atomic. That is, the client may call write(2) on a file
+  opened with O_SYNC with an 8 MB buffer and then crash and the write
+  may be only partially applied.  (Almost all file systems, even local
+  file systems, have this behavior.)
+- In shared simultaneous writer situations, a write that crosses
+  object boundaries is not necessarily atomic. This means that you
+  could have writer A write "aa|aa" and writer B write "bb|bb"
+  simultaneously (where | is the object boundary), and end up with
+  "aa|bb" rather than the proper "aa|aa" or "bb|bb".
+- Sparse files propagate incorrectly to the stat(2) st_blocks field.
+  Because CephFS does not explicitly track which parts of a file are
+  allocated/written, the st_blocks field is always populated by the
+  file size divided by the block size.  This will cause tools like
+  du(1) to overestimate consumed space.  (The recursive size field,
+  maintained by CephFS, also includes file "holes" in its count.)
+- When a file is mapped into memory via mmap(2) on multiple hosts,
+  writes are not coherently propagated to other clients' caches.  That
+  is, if a page is cached on host A, and then updated on host B, host
+  A's page is not coherently invalidated.  (Shared writable mmap
+  appears to be quite rare--we have yet to hear any complaints about this
+  behavior, and implementing cache coherency properly is complex.)
+- CephFS clients present a hidden ``.snap`` directory that is used to
+  access, create, delete, and rename snapshots.  Although the virtual
+  directory is excluded from readdir(2), any process that tries to
+  create a file or directory with the same name will get an error
+  code.  The name of this hidden directory can be changed at mount
+  time with ``-o snapdirname=.somethingelse`` (Linux) or the config
+  option ``client_snapdir`` (libcephfs, ceph-fuse).
+
+Perspective
+-----------
+
+People talk a lot about "POSIX compliance," but in reality most file
+system implementations do not strictly adhere to the spec, including
+local Linux file systems like ext4 and XFS.  For example, for
+performance reasons, the atomicity requirements for reads are relaxed:
+processing reading from a file that is also being written may see torn
+results.
+
+Similarly, NFS has extremely weak consistency semantics when multiple
+clients are interacting with the same files or directories, opting
+instead for "close-to-open".  In the world of network attached
+storage, where most environments use NFS, whether or not the server's
+file system is "fully POSIX" may not be relevant, and whether client
+applications notice depends on whether data is being shared between
+clients or not.  NFS may also "tear" the results of concurrent writers
+as client data may not even be flushed to the server until the file is
+closed (and more generally writes will be significantly more
+time-shifted than CephFS, leading to less predictable results).
+
+Regardless, these are all similar enough to POSIX, and applications still work
+most of the time. Many other storage systems (e.g., HDFS) claim to be
+"POSIX-like" but diverge significantly from the standard by dropping support
+for things like in-place file modifications, truncate, or directory renames.
+
+Bottom line
+-----------
+
+CephFS relaxes more than local Linux kernel file systems (for example, writes
+spanning object boundaries may be torn).  It relaxes strictly less
+than NFS when it comes to multiclient consistency, and generally less
+than NFS when it comes to write atomicity.
+
+In other words, when it comes to POSIX, ::
+
+  HDFS < NFS < CephFS < {XFS, ext4}
+
+
+fsync() and error reporting
+---------------------------
+
+POSIX is somewhat vague about the state of an inode after fsync reports
+an error. In general, CephFS uses the standard error-reporting
+mechanisms in the client's kernel, and therefore follows the same
+conventions as other file systems.
+
+In modern Linux kernels (v4.17 or later), writeback errors are reported
+once to every file description that is open at the time of the error. In
+addition, unreported errors that occurred before the file description was
+opened will also be returned on fsync.
+
+See `PostgreSQL's summary of fsync() error reporting across operating systems
+<https://wiki.postgresql.org/wiki/Fsync_Errors>`_ and `Matthew Wilcox's
+presentation on Linux IO error handling
+<https://www.youtube.com/watch?v=74c19hwY2oE>`_ for more information.
diff --git a/doc/cephfs/quota.rst b/doc/cephfs/quota.rst
new file mode 100644
index 000000000..78c0887d9
--- /dev/null
+++ b/doc/cephfs/quota.rst
@@ -0,0 +1,87 @@
+Quotas
+======
+
+CephFS allows quotas to be set on any directory in the system.  The
+quota can restrict the number of *bytes* or the number of *files*
+stored beneath that point in the directory hierarchy.
+
+Limitations
+-----------
+
+#. *Quotas are cooperative and non-adversarial.* CephFS quotas rely on
+   the cooperation of the client who is mounting the file system to
+   stop writers when a limit is reached.  A modified or adversarial
+   client cannot be prevented from writing as much data as it needs.
+   Quotas should not be relied on to prevent filling the system in
+   environments where the clients are fully untrusted.
+
+#. *Quotas are imprecise.* Processes that are writing to the file
+   system will be stopped a short time after the quota limit is
+   reached.  They will inevitably be allowed to write some amount of
+   data over the configured limit.  How far over the quota they are
+   able to go depends primarily on the amount of time, not the amount
+   of data.  Generally speaking writers will be stopped within 10s of
+   seconds of crossing the configured limit.
+
+#. *Quotas are implemented in the kernel client 4.17 and higher.*
+   Quotas are supported by the userspace client (libcephfs, ceph-fuse).
+   Linux kernel clients >= 4.17 support CephFS quotas but only on
+   mimic+ clusters.  Kernel clients (even recent versions) will fail
+   to handle quotas on older clusters, even if they may be able to set
+   the quotas extended attributes.
+
+#. *Quotas must be configured carefully when used with path-based
+   mount restrictions.* The client needs to have access to the
+   directory inode on which quotas are configured in order to enforce
+   them.  If the client has restricted access to a specific path
+   (e.g., ``/home/user``) based on the MDS capability, and a quota is
+   configured on an ancestor directory they do not have access to
+   (e.g., ``/home``), the client will not enforce it.  When using
+   path-based access restrictions be sure to configure the quota on
+   the directory the client is restricted too (e.g., ``/home/user``)
+   or something nested beneath it.
+
+   Kernel clients need access to the parent of the directory inode on
+   which quotas are configured in order to enforce them. If quota is
+   configured on a directory path (e.g., ``/home/volumes/group``), the
+   kclient needs to have access to the parent (e.g., ``/home/volumes``).
+
+   An example command to create such an user is as below::
+
+     $ ceph auth get-or-create client.guest mds 'allow r path=/home/volumes, allow rw path=/home/volumes/group' mgr 'allow rw' osd 'allow rw tag cephfs metadata=*' mon 'allow r'
+
+   See also: https://tracker.ceph.com/issues/55090
+
+#. *Snapshot file data which has since been deleted or changed does not count
+   towards the quota.* See also: http://tracker.ceph.com/issues/24284
+
+Configuration
+-------------
+
+Like most other things in CephFS, quotas are configured using virtual
+extended attributes:
+
+ * ``ceph.quota.max_files`` -- file limit
+ * ``ceph.quota.max_bytes`` -- byte limit
+
+If the attributes appear on a directory inode that means a quota is
+configured there.  If they are not present then no quota is set on
+that directory (although one may still be configured on a parent directory).
+
+To set a quota::
+
+  setfattr -n ceph.quota.max_bytes -v 100000000 /some/dir     # 100 MB
+  setfattr -n ceph.quota.max_files -v 10000 /some/dir         # 10,000 files
+
+To view quota settings::
+
+  getfattr -n ceph.quota.max_bytes /some/dir
+  getfattr -n ceph.quota.max_files /some/dir
+
+Note that if the value of the extended attribute is ``0`` that means
+the quota is not set.
+
+To remove a quota::
+
+  setfattr -n ceph.quota.max_bytes -v 0 /some/dir
+  setfattr -n ceph.quota.max_files -v 0 /some/dir
diff --git a/doc/cephfs/recover-fs-after-mon-store-loss.rst b/doc/cephfs/recover-fs-after-mon-store-loss.rst
new file mode 100644
index 000000000..d44269c83
--- /dev/null
+++ b/doc/cephfs/recover-fs-after-mon-store-loss.rst
@@ -0,0 +1,51 @@
+Recovering the file system after catastrophic Monitor store loss
+================================================================
+
+During rare occasions, all the monitor stores of a cluster may get corrupted
+or lost. To recover the cluster in such a scenario, you need to rebuild the
+monitor stores using the OSDs (see :ref:`mon-store-recovery-using-osds`),
+and get back the pools intact (active+clean state). However, the rebuilt monitor
+stores don't restore the file system maps ("FSMap"). Additional steps are required
+to bring back the file system. The steps to recover a multiple active MDS file
+system or multiple file systems are yet to be identified. Currently, only the steps
+to recover a **single active MDS** file system with no additional file systems
+in the cluster have been identified and tested. Briefly the steps are:
+recreate the FSMap with basic defaults; and allow MDSs to recover from
+the journal/metadata stored in the filesystem's pools. The steps are described
+in more detail below.
+
+First up, recreate the file system using the recovered file system pools. The
+new FSMap will have the filesystem's default settings. However, the user defined
+file system settings such as ``standby_count_wanted``, ``required_client_features``,
+extra data pools, etc., are lost and need to be reapplied later.
+
+::
+
+    ceph fs new <fs_name> <metadata_pool> <data_pool> --force --recover
+
+The ``recover`` flag sets the state of file system's rank 0 to existing but
+failed. So when a MDS daemon eventually picks up rank 0, the daemon reads the
+existing in-RADOS metadata and doesn't overwrite it. The flag also prevents the
+standby MDS daemons to activate the file system.
+
+The file system cluster ID, fscid, of the file system will not be preserved.
+This behaviour may not be desirable for certain applications (e.g., Ceph CSI)
+that expect the file system to be unchanged across recovery. To fix this, you
+can optionally set the ``fscid`` option in the above command (see
+:ref:`advanced-cephfs-admin-settings`).
+
+Allow standby MDS daemons to join the file system.
+
+::
+
+    ceph fs set <fs_name> joinable true
+
+
+Check that the file system is no longer in degraded state and has an active
+MDS.
+
+::
+
+    ceph fs status
+
+Reapply any other custom file system settings.
diff --git a/doc/cephfs/scrub.rst b/doc/cephfs/scrub.rst
new file mode 100644
index 000000000..7f55df950
--- /dev/null
+++ b/doc/cephfs/scrub.rst
@@ -0,0 +1,133 @@
+.. _mds-scrub:
+
+======================
+Ceph File System Scrub
+======================
+
+CephFS provides the cluster admin (operator) to check consistency of a file system
+via a set of scrub commands. Scrub can be classified into two parts:
+
+#. Forward Scrub: In which the scrub operation starts at the root of the file system
+   (or a sub directory) and looks at everything that can be touched in the hierarchy
+   to ensure consistency.
+
+#. Backward Scrub: In which the scrub operation looks at every RADOS object in the
+   file system pools and maps it back to the file system hierarchy.
+
+This document details commands to initiate and control forward scrub (referred as
+scrub thereafter).
+
+.. warning::
+
+   CephFS forward scrubs are started and manipulated on rank 0. All scrub
+   commands must be directed at rank 0.
+
+Initiate File System Scrub
+==========================
+
+To start a scrub operation for a directory tree use the following command::
+
+   ceph tell mds.<fsname>:0 scrub start <path> [scrubopts] [tag]
+
+where ``scrubopts`` is a comma delimited list of ``recursive``, ``force``, or
+``repair`` and ``tag`` is an optional custom string tag (the default is a generated
+UUID). An example command is::
+
+   ceph tell mds.cephfs:0 scrub start / recursive
+   {
+       "return_code": 0,
+       "scrub_tag": "6f0d204c-6cfd-4300-9e02-73f382fd23c1",
+       "mode": "asynchronous"
+   }
+
+Recursive scrub is asynchronous (as hinted by `mode` in the output above).
+Asynchronous scrubs must be polled using ``scrub status`` to determine the
+status.
+
+The scrub tag is used to differentiate scrubs and also to mark each inode's
+first data object in the default data pool (where the backtrace information is
+stored) with a ``scrub_tag`` extended attribute with the value of the tag. You
+can verify an inode was scrubbed by looking at the extended attribute using the
+RADOS utilities.
+
+Scrubs work for multiple active MDS (multiple ranks). The scrub is managed by
+rank 0 and distributed across MDS as appropriate.
+
+
+Monitor (ongoing) File System Scrubs
+====================================
+
+Status of ongoing scrubs can be monitored and polled using in `scrub status`
+command. This commands lists out ongoing scrubs (identified by the tag) along
+with the path and options used to initiate the scrub::
+
+   ceph tell mds.cephfs:0 scrub status
+   {
+       "status": "scrub active (85 inodes in the stack)",
+       "scrubs": {
+           "6f0d204c-6cfd-4300-9e02-73f382fd23c1": {
+               "path": "/",
+               "options": "recursive"
+           }
+       }
+   }
+
+`status` shows the number of inodes that are scheduled to be scrubbed at any point in time,
+hence, can change on subsequent `scrub status` invocations. Also, a high level summary of
+scrub operation (which includes the operation state and paths on which scrub is triggered)
+gets displayed in `ceph status`::
+
+   ceph status
+   [...]
+
+   task status:
+     scrub status:
+         mds.0: active [paths:/]
+
+   [...]
+
+A scrub is complete when it no longer shows up in this list (although that may
+change in future releases). Any damage will be reported via cluster health warnings.
+
+Control (ongoing) File System Scrubs
+====================================
+
+- Pause: Pausing ongoing scrub operations results in no new or pending inodes being
+  scrubbed after in-flight RADOS ops (for the inodes that are currently being scrubbed)
+  finish::
+
+   ceph tell mds.cephfs:0 scrub pause
+   {
+       "return_code": 0
+   }
+
+  The ``scrub status`` after pausing reflects the paused state. At this point,
+  initiating new scrub operations (via ``scrub start``) would just queue the
+  inode for scrub::
+
+   ceph tell mds.cephfs:0 scrub status
+   {
+       "status": "PAUSED (66 inodes in the stack)",
+       "scrubs": {
+           "6f0d204c-6cfd-4300-9e02-73f382fd23c1": {
+               "path": "/",
+               "options": "recursive"
+           }
+       }
+   }
+
+- Resume: Resuming kick starts a paused scrub operation::
+
+   ceph tell mds.cephfs:0 scrub resume
+   {
+       "return_code": 0
+   }
+
+- Abort: Aborting ongoing scrub operations removes pending inodes from the scrub
+  queue (thereby aborting the scrub) after in-flight RADOS ops (for the inodes that
+  are currently being scrubbed) finish::
+
+   ceph tell mds.cephfs:0 scrub abort
+   {
+       "return_code": 0
+   }
diff --git a/doc/cephfs/snap-schedule.rst b/doc/cephfs/snap-schedule.rst
new file mode 100644
index 000000000..2728203f4
--- /dev/null
+++ b/doc/cephfs/snap-schedule.rst
@@ -0,0 +1,182 @@
+.. _snap-schedule:
+
+==========================
+Snapshot Scheduling Module
+==========================
+This module implements scheduled snapshots for CephFS.
+It provides a user interface to add, query and remove snapshots schedules and
+retention policies, as well as a scheduler that takes the snapshots and prunes
+existing snapshots accordingly.
+
+
+How to enable
+=============
+
+The *snap_schedule* module is enabled with::
+
+  ceph mgr module enable snap_schedule
+
+Usage
+=====
+
+This module uses :doc:`/dev/cephfs-snapshots`, please consider this documentation
+as well.
+
+This module's subcommands live under the `ceph fs snap-schedule` namespace.
+Arguments can either be supplied as positional arguments or as keyword
+arguments. Once a keyword argument was encountered, all following arguments are
+assumed to be keyword arguments too.
+
+Snapshot schedules are identified by path, their repeat interval and their start
+time. The
+repeat interval defines the time between two subsequent snapshots. It is
+specified by a number and a period multiplier, one of `h(our)`, `d(ay)` and
+`w(eek)`. E.g. a repeat interval of `12h` specifies one snapshot every 12
+hours.
+The start time is specified as a time string (more details about passing times
+below). By default
+the start time is last midnight. So when a snapshot schedule with repeat
+interval `1h` is added at 13:50
+with the default start time, the first snapshot will be taken at 14:00.
+
+Retention specifications are identified by path and the retention spec itself. A
+retention spec consists of either a number and a time period separated by a
+space or concatenated pairs of `<number><time period>`.
+The semantics are that a spec will ensure `<number>` snapshots are kept that are
+at least `<time period>` apart. For Example `7d` means the user wants to keep 7
+snapshots that are at least one day (but potentially longer) apart from each other.
+The following time periods are recognized: `h(our), d(ay), w(eek), m(onth),
+y(ear)` and `n`. The latter is a special modifier where e.g. `10n` means keep
+the last 10 snapshots regardless of timing,
+
+All subcommands take optional `fs` argument to specify paths in
+multi-fs setups and :doc:`/cephfs/fs-volumes` managed setups. If not
+passed `fs` defaults to the first file system listed in the fs_map.
+When using :doc:`/cephfs/fs-volumes` the argument `fs` is equivalent to a
+`volume`.
+
+When a timestamp is passed (the `start` argument in the `add`, `remove`,
+`activate` and `deactivate` subcommands) the ISO format `%Y-%m-%dT%H:%M:%S` will
+always be accepted. When either python3.7 or newer is used or
+https://github.com/movermeyer/backports.datetime_fromisoformat is installed, any
+valid ISO timestamp that is parsed by python's `datetime.fromisoformat` is valid.
+
+When no subcommand is supplied a synopsis is printed::
+
+  #> ceph fs snap-schedule
+  no valid command found; 8 closest matches:
+  fs snap-schedule status [<path>] [<fs>] [<format>]
+  fs snap-schedule list <path> [--recursive] [<fs>] [<format>]
+  fs snap-schedule add <path> <snap_schedule> [<start>] [<fs>]
+  fs snap-schedule remove <path> [<repeat>] [<start>] [<fs>]
+  fs snap-schedule retention add <path> <retention_spec_or_period> [<retention_count>] [<fs>]
+  fs snap-schedule retention remove <path> <retention_spec_or_period> [<retention_count>] [<fs>]
+  fs snap-schedule activate <path> [<repeat>] [<start>] [<fs>]
+  fs snap-schedule deactivate <path> [<repeat>] [<start>] [<fs>]
+  Error EINVAL: invalid command
+
+Note:
+^^^^^
+A `subvolume` argument is no longer accepted by the commands.
+
+
+Inspect snapshot schedules
+--------------------------
+
+The module offers two subcommands to inspect existing schedules: `list` and
+`status`. Bother offer plain and json output via the optional `format` argument.
+The default is plain.
+The `list` sub-command will list all schedules on a path in a short single line
+format. It offers a `recursive` argument to list all schedules in the specified
+directory and all contained directories.
+The `status` subcommand prints all available schedules and retention specs for a
+path.
+
+Examples::
+
+  ceph fs snap-schedule status /
+  ceph fs snap-schedule status /foo/bar --format=json
+  ceph fs snap-schedule list /
+  ceph fs snap-schedule list / --recursive=true # list all schedules in the tree
+
+
+Add and remove schedules
+------------------------
+The `add` and `remove` subcommands add and remove snapshots schedules
+respectively. Both require at least a `path` argument, `add` additionally
+requires a `schedule` argument as described in the USAGE section.
+
+Multiple different schedules can be added to a path. Two schedules are considered
+different from each other if they differ in their repeat interval and their
+start time.
+
+If multiple schedules have been set on a path, `remove` can remove individual
+schedules on a path by specifying the exact repeat interval and start time, or
+the subcommand can remove all schedules on a path when just a `path` is
+specified.
+
+Examples::
+
+  ceph fs snap-schedule add / 1h
+  ceph fs snap-schedule add / 1h 11:55
+  ceph fs snap-schedule add / 2h 11:55
+  ceph fs snap-schedule remove / 1h 11:55 # removes one single schedule
+  ceph fs snap-schedule remove / 1h # removes all schedules with --repeat=1h
+  ceph fs snap-schedule remove / # removes all schedules on path /
+
+Add and remove retention policies
+---------------------------------
+The `retention add` and `retention remove` subcommands allow to manage
+retention policies. One path has exactly one retention policy. A policy can
+however contain multiple count-time period pairs in order to specify complex
+retention policies.
+Retention policies can be added and removed individually or in bulk via the
+forms `ceph fs snap-schedule retention add <path> <time period> <count>` and
+`ceph fs snap-schedule retention add <path> <countTime period>[countTime period]`
+
+Examples::
+
+  ceph fs snap-schedule retention add / h 24 # keep 24 snapshots at least an hour apart
+  ceph fs snap-schedule retention add / d 7 # and 7 snapshots at least a day apart
+  ceph fs snap-schedule retention remove / h 24 # remove retention for 24 hourlies
+  ceph fs snap-schedule retention add / 24h4w # add 24 hourly and 4 weekly to retention
+  ceph fs snap-schedule retention remove / 7d4w # remove 7 daily and 4 weekly, leaves 24 hourly
+
+Active and inactive schedules
+-----------------------------
+Snapshot schedules can be added for a path that doesn't exist yet in the
+directory tree. Similarly a path can be removed without affecting any snapshot
+schedules on that path.
+If a directory is not present when a snapshot is scheduled to be taken, the
+schedule will be set to inactive and will be excluded from scheduling until
+it is activated again.
+A schedule can manually be set to inactive to pause the creating of scheduled
+snapshots.
+The module provides the `activate` and `deactivate` subcommands for this
+purpose.
+
+Examples::
+
+  ceph fs snap-schedule activate / # activate all schedules on the root directory
+  ceph fs snap-schedule deactivate / 1d # deactivates daily snapshots on the root directory
+
+Limitations
+-----------
+Snapshots are scheduled using python Timers. Under normal circumstances
+specifying 1h as the schedule will result in snapshots 1 hour apart fairly
+precisely. If the mgr daemon is under heavy load however, the Timer threads
+might not get scheduled right away, resulting in a slightly delayed snapshot. If
+this happens, the next snapshot will be schedule as if the previous one was not
+delayed, i.e. one or more delayed snapshots will not cause drift in the overall
+schedule.
+
+In order to somewhat limit the overall number of snapshots in a file system, the
+module will only keep a maximum of 50 snapshots per directory. If the retention
+policy results in more then 50 retained snapshots, the retention list will be
+shortened to the newest 50 snapshots.
+
+Data storage
+------------
+The snapshot schedule data is stored in a rados object in the cephfs metadata
+pool. At runtime all data lives in a sqlite database that is serialized and
+stored as a rados object.
diff --git a/doc/cephfs/standby.rst b/doc/cephfs/standby.rst
new file mode 100644
index 000000000..22216c36f
--- /dev/null
+++ b/doc/cephfs/standby.rst
@@ -0,0 +1,189 @@
+.. _mds-standby:
+
+Terminology
+-----------
+
+A Ceph cluster may have zero or more CephFS *file systems*.  CephFS
+file systems have a human readable name (set in ``fs new``)
+and an integer ID.  The ID is called the file system cluster ID,
+or *FSCID*.
+
+Each CephFS file system has a number of *ranks*, one by default,
+which start at zero.  A rank may be thought of as a metadata shard.
+Controlling the number of ranks in a file system is described
+in :doc:`/cephfs/multimds`
+
+Each CephFS ceph-mds process (a *daemon*) initially starts up
+without a rank.  It may be assigned one by the monitor cluster.
+A daemon may only hold one rank at a time.  Daemons only give up
+a rank when the ceph-mds process stops.
+
+If a rank is not associated with a daemon, the rank is
+considered *failed*.  Once a rank is assigned to a daemon,
+the rank is considered *up*.
+
+A daemon has a *name* that is set statically by the administrator
+when the daemon is first configured.  Typical configurations
+use the hostname where the daemon runs as the daemon name.
+
+A ceph-mds daemons can be assigned to a particular file system by
+setting the `mds_join_fs` configuration option to the file system
+name.
+
+Each time a daemon starts up, it is also assigned a *GID*, which
+is unique to this particular process lifetime of the daemon.  The
+GID is an integer.
+
+Referring to MDS daemons
+------------------------
+
+Most of the administrative commands that refer to an MDS daemon
+accept a flexible argument format that may contain a rank, a GID
+or a name.
+
+Where a rank is used, this may optionally be qualified with
+a leading file system name or ID.  If a daemon is a standby (i.e.
+it is not currently assigned a rank), then it may only be
+referred to by GID or name.
+
+For example, if we had an MDS daemon which was called 'myhost',
+had GID 5446, and was assigned rank 0 in the file system 'myfs'
+which had FSCID 3, then any of the following would be suitable
+forms of the 'fail' command:
+
+::
+
+    ceph mds fail 5446     # GID
+    ceph mds fail myhost   # Daemon name
+    ceph mds fail 0        # Unqualified rank
+    ceph mds fail 3:0      # FSCID and rank
+    ceph mds fail myfs:0   # File System name and rank
+
+Managing failover
+-----------------
+
+If an MDS daemon stops communicating with the monitor, the monitor will wait
+``mds_beacon_grace`` seconds (default 15 seconds) before marking the daemon as
+*laggy*. If a standby is available, the monitor will immediately replace the
+laggy daemon.
+
+Each file system may specify a number of standby daemons to be considered
+healthy. This number includes daemons in standby-replay waiting for a rank to
+fail (remember that a standby-replay daemon will not be assigned to take over a
+failure for another rank or a failure in a another CephFS file system). The
+pool of standby daemons not in replay count towards any file system count.
+Each file system may set the number of standby daemons wanted using:
+
+::
+
+    ceph fs set <fs name> standby_count_wanted <count>
+
+Setting ``count`` to 0 will disable the health check.
+
+
+.. _mds-standby-replay:
+
+Configuring standby-replay
+--------------------------
+
+Each CephFS file system may be configured to add standby-replay daemons.  These
+standby daemons follow the active MDS's metadata journal to reduce failover
+time in the event the active MDS becomes unavailable. Each active MDS may have
+only one standby-replay daemon following it.
+
+Configuring standby-replay on a file system is done using:
+
+::
+
+    ceph fs set <fs name> allow_standby_replay <bool>
+
+Once set, the monitors will assign available standby daemons to follow the
+active MDSs in that file system.
+
+Once an MDS has entered the standby-replay state, it will only be used as a
+standby for the rank that it is following. If another rank fails, this
+standby-replay daemon will not be used as a replacement, even if no other
+standbys are available. For this reason, it is advised that if standby-replay
+is used then every active MDS should have a standby-replay daemon.
+
+.. _mds-join-fs:
+
+Configuring MDS file system affinity
+------------------------------------
+
+You may want to have an MDS used for a particular file system. Or, perhaps you
+have larger MDSs on better hardware that should be preferred over a last-resort
+standby on lesser or over-provisioned hardware. To express this preference,
+CephFS provides a configuration option for MDS called ``mds_join_fs`` which
+enforces this `affinity`.
+
+As part of any failover, the Ceph monitors will prefer standby daemons with
+``mds_join_fs`` equal to the file system name with the failed rank.  If no
+standby exists with ``mds_join_fs`` equal to the file system name, it will
+choose a `vanilla` standby (no setting for ``mds_join_fs``) for the replacement
+or any other available standby as a last resort. Note, this does not change the
+behavior that ``standby-replay`` daemons are always selected before looking at
+other standbys.
+
+Even further, the monitors will regularly examine the CephFS file systems when
+stable to check if a standby with stronger affinity is available to replace an
+MDS with lower affinity. This process is also done for standby-replay daemons:
+if a regular standby has stronger affinity than the standby-replay MDS, it will
+replace the standby-replay MDS.
+
+For example, given this stable and healthy file system:
+
+::
+
+    $ ceph fs dump
+    dumped fsmap epoch 399
+    ...
+    Filesystem 'cephfs' (27)
+    ...
+    e399
+    max_mds 1
+    in      0
+    up      {0=20384}
+    failed
+    damaged
+    stopped
+    ...
+    [mds.a{0:20384} state up:active seq 239 addr [v2:127.0.0.1:6854/966242805,v1:127.0.0.1:6855/966242805]]
+
+    Standby daemons:
+
+    [mds.b{-1:10420} state up:standby seq 2 addr [v2:127.0.0.1:6856/2745199145,v1:127.0.0.1:6857/2745199145]]
+
+
+You may set ``mds_join_fs`` on the standby to enforce your preference: ::
+
+    $ ceph config set mds.b mds_join_fs cephfs
+
+after automatic failover: ::
+
+    $ ceph fs dump
+    dumped fsmap epoch 405
+    e405
+    ...
+    Filesystem 'cephfs' (27)
+    ...
+    max_mds 1
+    in      0
+    up      {0=10420}
+    failed
+    damaged
+    stopped
+    ...
+    [mds.b{0:10420} state up:active seq 274 join_fscid=27 addr [v2:127.0.0.1:6856/2745199145,v1:127.0.0.1:6857/2745199145]]
+
+    Standby daemons:
+
+    [mds.a{-1:10720} state up:standby seq 2 addr [v2:127.0.0.1:6854/1340357658,v1:127.0.0.1:6855/1340357658]]
+
+Note in the above example that ``mds.b`` now has ``join_fscid=27``. In this
+output, the file system name from ``mds_join_fs`` is changed to the file system
+identifier (27). If the file system is recreated with the same name, the
+standby will follow the new file system as expected.
+
+Finally, if the file system is degraded or undersized, no failover will occur
+to enforce ``mds_join_fs``.
diff --git a/doc/cephfs/subtree-partitioning.svg b/doc/cephfs/subtree-partitioning.svg
new file mode 100644
index 000000000..20c60de92
--- /dev/null
+++ b/doc/cephfs/subtree-partitioning.svg
@@ -0,0 +1 @@
+<svg version="1.1" viewBox="0.0 0.0 908.7690288713911 373.36482939632543" fill="none" stroke="none" stroke-linecap="square" stroke-miterlimit="10" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg"><clipPath id="p.0"><path d="m0 0l908.76904 0l0 373.36484l-908.76904 0l0 -373.36484z" clip-rule="nonzero"/></clipPath><g clip-path="url(#p.0)"><path fill="#000000" fill-opacity="0.0" d="m0 0l908.76904 0l0 373.36484l-908.76904 0z" fill-rule="evenodd"/><g filter="url(#shadowFilter-p.1)"><use xlink:href="#p.1" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.1" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.1"><path fill="#ebeded" d="m259.5493 158.67822c-0.29309082 0 -0.5570679 0.2373352 -0.5570679 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570679 0.5012207l26.164001 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.164001 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m259.5493 158.67822c-0.29309082 0 -0.5570679 0.2373352 -0.5570679 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570679 0.5012207l26.164001 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.164001 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.2)"><use xlink:href="#p.2" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.2" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.2"><path fill="#717d84" d="m437.25455 5.221119c-0.43972778 0 -0.8501892 0.36909866 -0.8501892 0.7646594l0 13.396389c0 0.39556122 0.41046143 0.79112244 0.8501892 0.79112244l58.752655 0c0.44024658 0 0.87994385 -0.39556122 0.87994385 -0.79112244l0 -13.396389c0 -0.39556074 -0.43969727 -0.7646594 -0.87994385 -0.7646594l-58.752655 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m437.25455 5.221119c-0.43972778 0 -0.8501892 0.36909866 -0.8501892 0.7646594l0 13.396389c0 0.39556122 0.41046143 0.79112244 0.8501892 0.79112244l58.752655 0c0.44024658 0 0.87994385 -0.39556122 0.87994385 -0.79112244l0 -13.396389c0 -0.39556074 -0.43969727 -0.7646594 -0.87994385 -0.7646594l-58.752655 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.3)"><use xlink:href="#p.3" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.3" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.3"><path fill="#90dae3" d="m595.4631 55.41167c-0.46917725 0 -0.90875244 0.3957138 -0.90875244 0.81785583l0 14.326885c0 0.4221344 0.4395752 0.8442764 0.90875244 0.8442764l41.4339 0c0.46917725 0 0.9383545 -0.42214203 0.9383545 -0.8442764l0 -14.326885c0 -0.42214203 -0.46917725 -0.81785583 -0.9383545 -0.81785583l-41.4339 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m595.4631 55.41167c-0.46917725 0 -0.90875244 0.3957138 -0.90875244 0.81785583l0 14.326885c0 0.4221344 0.4395752 0.8442764 0.90875244 0.8442764l41.4339 0c0.46917725 0 0.9383545 -0.42214203 0.9383545 -0.8442764l0 -14.326885c0 -0.42214203 -0.46917725 -0.81785583 -0.9383545 -0.81785583l-41.4339 0" fill-rule="evenodd"/></g><path fill="#000000" fill-opacity="0.0" d="m315.78055 55.438137l150.90213 -50.2082" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m315.78055 55.438137l150.90213 -50.2082" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m466.6484 5.2211194l149.58078 50.208202" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m466.6484 5.2211194l149.58078 50.208202" fill-rule="evenodd"/><g filter="url(#shadowFilter-p.4)"><use xlink:href="#p.4" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.4" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.4"><path fill="#717d84" d="m295.0487 55.41167c-0.46920776 0 -0.9088135 0.3957138 -0.9088135 0.81785583l0 14.326885c0 0.4221344 0.4396057 0.8442764 0.9088135 0.8442764l41.433838 0c0.46920776 0 0.9384155 -0.42214203 0.9384155 -0.8442764l0 -14.326885c0 -0.42214203 -0.46920776 -0.81785583 -0.9384155 -0.81785583l-41.433838 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m295.0487 55.41167c-0.46920776 0 -0.9088135 0.3957138 -0.9088135 0.81785583l0 14.326885c0 0.4221344 0.4396057 0.8442764 0.9088135 0.8442764l41.433838 0c0.46920776 0 0.9384155 -0.42214203 0.9384155 -0.8442764l0 -14.326885c0 -0.42214203 -0.46920776 -0.81785583 -0.9384155 -0.81785583l-41.433838 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.5)"><use xlink:href="#p.5" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.5" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.5"><path fill="#717d84" d="m302.62457 101.419685c-0.29312134 0 -0.5566406 0.23719788 -0.5566406 0.5007782l0 9.093201c0 0.26358032 0.2635193 0.52716064 0.5566406 0.52716064l26.135773 0c0.2928772 0 0.585968 -0.26358032 0.585968 -0.52716064l0 -9.093201c0 -0.26358032 -0.29309082 -0.5007782 -0.585968 -0.5007782l-26.135773 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m302.62457 101.419685c-0.29312134 0 -0.5566406 0.23719788 -0.5566406 0.5007782l0 9.093201c0 0.26358032 0.2635193 0.52716064 0.5566406 0.52716064l26.135773 0c0.2928772 0 0.585968 -0.26358032 0.585968 -0.52716064l0 -9.093201c0 -0.26358032 -0.29309082 -0.5007782 -0.585968 -0.5007782l-26.135773 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.6)"><use xlink:href="#p.6" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.6" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.6"><path fill="#ebeded" d="m445.97543 55.41167c-0.46939087 0 -0.90893555 0.3957138 -0.90893555 0.81785583l0 14.326885c0 0.4221344 0.43954468 0.8442764 0.90893555 0.8442764l41.286926 0c0.4694214 0 0.93844604 -0.42214203 0.93844604 -0.8442764l0 -14.326885c0 -0.42214203 -0.46902466 -0.81785583 -0.93844604 -0.81785583l-41.286926 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m445.97543 55.41167c-0.46939087 0 -0.90893555 0.3957138 -0.90893555 0.81785583l0 14.326885c0 0.4221344 0.43954468 0.8442764 0.90893555 0.8442764l41.286926 0c0.4694214 0 0.93844604 -0.42214203 0.93844604 -0.8442764l0 -14.326885c0 -0.42214203 -0.46902466 -0.81785583 -0.93844604 -0.81785583l-41.286926 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.7)"><use xlink:href="#p.7" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.7" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.7"><path fill="#717d84" d="m361.4975 101.419685c-0.2929077 0 -0.556427 0.23719788 -0.556427 0.5007782l0 9.093201c0 0.26358032 0.2635193 0.52716064 0.556427 0.52716064l26.160553 0c0.2929077 0 0.5858154 -0.26358032 0.5858154 -0.52716064l0 -9.093201c0 -0.26358032 -0.2929077 -0.5007782 -0.5858154 -0.5007782l-26.160553 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m361.4975 101.419685c-0.2929077 0 -0.556427 0.23719788 -0.556427 0.5007782l0 9.093201c0 0.26358032 0.2635193 0.52716064 0.556427 0.52716064l26.160553 0c0.2929077 0 0.5858154 -0.26358032 0.5858154 -0.52716064l0 -9.093201c0 -0.26358032 -0.2929077 -0.5007782 -0.5858154 -0.5007782l-26.160553 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.8)"><use xlink:href="#p.8" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.8" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.8"><path fill="#90dae3" d="m542.25726 101.419685c-0.29309082 0 -0.5566406 0.23719788 -0.5566406 0.5007782l0 9.093201c0 0.26358032 0.2635498 0.52716064 0.5566406 0.52716064l26.135742 0c0.2929077 0 0.58599854 -0.26358032 0.58599854 -0.52716064l0 -9.093201c0 -0.26358032 -0.29309082 -0.5007782 -0.58599854 -0.5007782l-26.135742 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m542.25726 101.419685c-0.29309082 0 -0.5566406 0.23719788 -0.5566406 0.5007782l0 9.093201c0 0.26358032 0.2635498 0.52716064 0.5566406 0.52716064l26.135742 0c0.2929077 0 0.58599854 -0.26358032 0.58599854 -0.52716064l0 -9.093201c0 -0.26358032 -0.29309082 -0.5007782 -0.58599854 -0.5007782l-26.135742 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.9)"><use xlink:href="#p.9" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.9" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.9"><path fill="#ebeded" d="m302.62503 131.78285c-0.29312134 0 -0.5570984 0.2373352 -0.5570984 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570984 0.5012207l26.16397 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.16397 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m302.62503 131.78285c-0.29312134 0 -0.5570984 0.2373352 -0.5570984 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570984 0.5012207l26.16397 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.16397 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.10)"><use xlink:href="#p.10" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.10" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.10"><path fill="#ebeded" d="m602.9806 101.419685c-0.29315186 0 -0.5569458 0.23719788 -0.5569458 0.5007782l0 9.093201c0 0.26358032 0.26379395 0.52716064 0.5569458 0.52716064l26.359985 0c0.26379395 0 0.55718994 -0.26358032 0.55718994 -0.52716064l0 -9.093201c0 -0.26358032 -0.293396 -0.5007782 -0.55718994 -0.5007782l-26.359985 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m602.9806 101.419685c-0.29315186 0 -0.5569458 0.23719788 -0.5569458 0.5007782l0 9.093201c0 0.26358032 0.26379395 0.52716064 0.5569458 0.52716064l26.359985 0c0.26379395 0 0.55718994 -0.26358032 0.55718994 -0.52716064l0 -9.093201c0 -0.26358032 -0.293396 -0.5007782 -0.55718994 -0.5007782l-26.359985 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.11)"><use xlink:href="#p.11" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.11" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.11"><path fill="#ff4b47" d="m581.3394 131.78285c-0.2929077 0 -0.5564575 0.2373352 -0.5564575 0.50120544l0 9.074539c0 0.23744202 0.2635498 0.5012207 0.5564575 0.5012207l26.307251 0c0.2929077 0 0.58599854 -0.2637787 0.58599854 -0.5012207l0 -9.074539c0 -0.26387024 -0.29309082 -0.50120544 -0.58599854 -0.50120544l-26.307251 0" fill-rule="evenodd"/><path stroke="#e93317" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m581.3394 131.78285c-0.2929077 0 -0.5564575 0.2373352 -0.5564575 0.50120544l0 9.074539c0 0.23744202 0.2635498 0.5012207 0.5564575 0.5012207l26.307251 0c0.2929077 0 0.58599854 -0.2637787 0.58599854 -0.5012207l0 -9.074539c0 -0.26387024 -0.29309082 -0.50120544 -0.58599854 -0.50120544l-26.307251 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.12)"><use xlink:href="#p.12" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.12" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.12"><path fill="#ebeded" d="m245.95436 101.419685c-0.29327393 0 -0.55729675 0.23719788 -0.55729675 0.5007782l0 9.093201c0 0.26358032 0.26402283 0.52716064 0.55729675 0.52716064l26.285995 0c0.2640381 0 0.557312 -0.26358032 0.557312 -0.52716064l0 -9.093201c0 -0.26358032 -0.29327393 -0.5007782 -0.557312 -0.5007782l-26.285995 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m245.95436 101.419685c-0.29327393 0 -0.55729675 0.23719788 -0.55729675 0.5007782l0 9.093201c0 0.26358032 0.26402283 0.52716064 0.55729675 0.52716064l26.285995 0c0.2640381 0 0.557312 -0.26358032 0.557312 -0.52716064l0 -9.093201c0 -0.26358032 -0.29327393 -0.5007782 -0.557312 -0.5007782l-26.285995 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.13)"><use xlink:href="#p.13" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.13" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.13"><path fill="#717d84" d="m259.5493 131.78285c-0.29309082 0 -0.5570679 0.2373352 -0.5570679 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570679 0.5012207l26.164001 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.164001 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m259.5493 131.78285c-0.29309082 0 -0.5570679 0.2373352 -0.5570679 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570679 0.5012207l26.164001 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.164001 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.14)"><use xlink:href="#p.14" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.14" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.14"><path fill="#ebeded" d="m541.8753 131.78285c-0.2929077 0 -0.5563965 0.2373352 -0.5563965 0.50120544l0 9.074539c0 0.23744202 0.26348877 0.5012207 0.5563965 0.5012207l26.160583 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.160583 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m541.8753 131.78285c-0.2929077 0 -0.5563965 0.2373352 -0.5563965 0.50120544l0 9.074539c0 0.23744202 0.26348877 0.5012207 0.5563965 0.5012207l26.160583 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.160583 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.15)"><use xlink:href="#p.15" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.15" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.15"><path fill="#ebeded" d="m581.3394 158.67822c-0.2929077 0 -0.5564575 0.2373352 -0.5564575 0.50120544l0 9.074539c0 0.23744202 0.2635498 0.5012207 0.5564575 0.5012207l26.307251 0c0.2929077 0 0.58599854 -0.2637787 0.58599854 -0.5012207l0 -9.074539c0 -0.26387024 -0.29309082 -0.50120544 -0.58599854 -0.50120544l-26.307251 0" fill-rule="evenodd"/><path stroke="#e93317" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m581.3394 158.67822c-0.2929077 0 -0.5564575 0.2373352 -0.5564575 0.50120544l0 9.074539c0 0.23744202 0.2635498 0.5012207 0.5564575 0.5012207l26.307251 0c0.2929077 0 0.58599854 -0.2637787 0.58599854 -0.5012207l0 -9.074539c0 -0.26387024 -0.29309082 -0.50120544 -0.58599854 -0.50120544l-26.307251 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.16)"><use xlink:href="#p.16" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.16" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.16"><path fill="#ebeded" d="m622.8298 158.67822c-0.29309082 0 -0.5566406 0.2373352 -0.5566406 0.50120544l0 9.074539c0 0.23744202 0.2635498 0.5012207 0.5566406 0.5012207l26.28247 0c0.29309082 0 0.5859375 -0.2637787 0.5859375 -0.5012207l0 -9.074539c0 -0.26387024 -0.29284668 -0.50120544 -0.5859375 -0.50120544l-26.28247 0" fill-rule="evenodd"/><path stroke="#e93317" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m622.8298 158.67822c-0.29309082 0 -0.5566406 0.2373352 -0.5566406 0.50120544l0 9.074539c0 0.23744202 0.2635498 0.5012207 0.5566406 0.5012207l26.28247 0c0.29309082 0 0.5859375 -0.2637787 0.5859375 -0.5012207l0 -9.074539c0 -0.26387024 -0.29284668 -0.50120544 -0.5859375 -0.50120544l-26.28247 0" fill-rule="evenodd"/></g><path fill="#000000" fill-opacity="0.0" d="m315.7218 101.44615l0.09790039 -46.03889" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m315.7218 101.44615l0.09790039 -46.03889" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m374.6243 101.44615l-58.82419 -46.03889" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m374.6243 101.44615l-58.82419 -46.03889" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m259.10968 101.44615l56.695343 -46.03889" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m259.10968 101.44615l56.695343 -46.03889" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m315.7218 131.80933l0.024475098 -30.39846" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m315.7218 131.80933l0.024475098 -30.39846" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m376.35672 131.80933l-1.7617798 -30.39846" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m376.35672 131.80933l-1.7617798 -30.39846" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m315.7512 101.419685l-43.1149 30.39846" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m315.7512 101.419685l-43.1149 30.39846" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m272.6461 158.70467l0.02444458 -26.890945" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m272.6461 158.70467l0.02444458 -26.890945" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m466.6484 55.438137l0.024475098 -50.2082" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m466.6484 55.438137l0.024475098 -50.2082" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m555.3545 101.44615l60.879578 -46.03889" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m555.3545 101.44615l60.879578 -46.03889" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m616.1655 101.44615l0.07342529 -46.03889" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m616.1655 101.44615l0.07342529 -46.03889" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m679.0027 101.975586l-62.812683 -46.590385" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m679.0027 101.975586l-62.812683 -46.590385" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m554.9728 131.80933l0.4159546 -30.39846" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m554.9728 131.80933l0.4159546 -30.39846" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m594.5543 131.80933l-39.224304 -30.39846" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m594.5543 131.80933l-39.224304 -30.39846" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m636.0151 158.70467l-41.49994 -26.890945" fill-rule="evenodd"/><path stroke="#e93317" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m636.0151 158.70467l-41.49994 -26.890945" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m594.5249 158.70467l0.024475098 -26.890945" fill-rule="evenodd"/><path stroke="#e93317" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m594.5249 158.70467l0.024475098 -26.890945" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m510.6638 131.80933l44.729828 -30.39846" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m510.6638 131.80933l44.729828 -30.39846" fill-rule="evenodd"/><g filter="url(#shadowFilter-p.17)"><use xlink:href="#p.17" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.17" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.17"><path fill="#ebeded" d="m407.98 131.78285c-0.29309082 0 -0.5570679 0.2373352 -0.5570679 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570679 0.5012207l26.164001 0c0.26397705 0 0.5572815 -0.2637787 0.5572815 -0.5012207l0 -9.074539c0 -0.26387024 -0.29330444 -0.50120544 -0.5572815 -0.50120544l-26.164001 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m407.98 131.78285c-0.29309082 0 -0.5570679 0.2373352 -0.5570679 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570679 0.5012207l26.164001 0c0.26397705 0 0.5572815 -0.2637787 0.5572815 -0.5012207l0 -9.074539c0 -0.26387024 -0.29330444 -0.50120544 -0.5572815 -0.50120544l-26.164001 0" fill-rule="evenodd"/></g><path fill="#000000" fill-opacity="0.0" d="m421.10617 131.80933l-46.516113 -30.39846" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m421.10617 131.80933l-46.516113 -30.39846" fill-rule="evenodd"/><g filter="url(#shadowFilter-p.18)"><use xlink:href="#p.18" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.18" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.18"><path fill="#717d84" d="m363.2306 131.78285c-0.29312134 0 -0.5570984 0.2373352 -0.5570984 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570984 0.5012207l26.16397 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.16397 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m363.2306 131.78285c-0.29312134 0 -0.5570984 0.2373352 -0.5570984 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570984 0.5012207l26.16397 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.16397 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.19)"><use xlink:href="#p.19" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.19" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.19"><path fill="#f9afaa" d="m192.04312 101.419685c-0.29301453 0 -0.5567932 0.23719788 -0.5567932 0.5007782l0 9.093201c0 0.26358032 0.2637787 0.52716064 0.5567932 0.52716064l26.233093 0c0.2932434 0 0.58625793 -0.26358032 0.58625793 -0.52716064l0 -9.093201c0 -0.26358032 -0.29301453 -0.5007782 -0.58625793 -0.5007782l-26.233093 0" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m192.04312 101.419685c-0.29301453 0 -0.5567932 0.23719788 -0.5567932 0.5007782l0 9.093201c0 0.26358032 0.2637787 0.52716064 0.5567932 0.52716064l26.233093 0c0.2932434 0 0.58625793 -0.26358032 0.58625793 -0.52716064l0 -9.093201c0 -0.26358032 -0.29301453 -0.5007782 -0.58625793 -0.5007782l-26.233093 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.20)"><use xlink:href="#p.20" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.20" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.20"><path fill="#f9afaa" d="m184.55559 55.41167c-0.46939087 0 -0.9092102 0.3957138 -0.9092102 0.81785583l0 14.326885c0 0.4221344 0.43981934 0.8442764 0.9092102 0.8442764l41.384506 0c0.46903992 0 0.9384308 -0.42214203 0.9384308 -0.8442764l0 -14.326885c0 -0.42214203 -0.46939087 -0.81785583 -0.9384308 -0.81785583l-41.384506 0" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m184.55559 55.41167c-0.46939087 0 -0.9092102 0.3957138 -0.9092102 0.81785583l0 14.326885c0 0.4221344 0.43981934 0.8442764 0.9092102 0.8442764l41.384506 0c0.46903992 0 0.9384308 -0.42214203 0.9384308 -0.8442764l0 -14.326885c0 -0.42214203 -0.46939087 -0.81785583 -0.9384308 -0.81785583l-41.384506 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.21)"><use xlink:href="#p.21" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.21" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.21"><path fill="#ebeded" d="m736.84656 55.41167c-0.46899414 0 -0.90875244 0.3957138 -0.90875244 0.81785583l0 14.326885c0 0.4221344 0.4397583 0.8442764 0.90875244 0.8442764l41.483215 0c0.46899414 0 0.9379883 -0.42214203 0.9379883 -0.8442764l0 -14.326885c0 -0.42214203 -0.46899414 -0.81785583 -0.9379883 -0.81785583l-41.483215 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m736.84656 55.41167c-0.46899414 0 -0.90875244 0.3957138 -0.90875244 0.81785583l0 14.326885c0 0.4221344 0.4397583 0.8442764 0.90875244 0.8442764l41.483215 0c0.46899414 0 0.9379883 -0.42214203 0.9379883 -0.8442764l0 -14.326885c0 -0.42214203 -0.46899414 -0.81785583 -0.9379883 -0.81785583l-41.483215 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.22)"><use xlink:href="#p.22" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.22" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.22"><path fill="#90dae3" d="m665.8761 101.94912c-0.29315186 0 -0.5566406 0.23719788 -0.5566406 0.5007782l0 9.093201c0 0.26358032 0.26348877 0.5271683 0.5566406 0.5271683l26.135742 0c0.2929077 0 0.58599854 -0.26358795 0.58599854 -0.5271683l0 -9.093201c0 -0.26358032 -0.29309082 -0.5007782 -0.58599854 -0.5007782l-26.135742 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m665.8761 101.94912c-0.29315186 0 -0.5566406 0.23719788 -0.5566406 0.5007782l0 9.093201c0 0.26358032 0.26348877 0.5271683 0.5566406 0.5271683l26.135742 0c0.2929077 0 0.58599854 -0.26358795 0.58599854 -0.5271683l0 -9.093201c0 -0.26358032 -0.29309082 -0.5007782 -0.58599854 -0.5007782l-26.135742 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.23)"><use xlink:href="#p.23" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.23" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.23"><path fill="#ebeded" d="m192.04282 131.78285c-0.2929077 0 -0.55648804 0.2373352 -0.55648804 0.50120544l0 9.074539c0 0.23744202 0.26358032 0.5012207 0.55648804 0.5012207l26.331833 0c0.2929077 0 0.5858307 -0.2637787 0.5858307 -0.5012207l0 -9.074539c0 -0.26387024 -0.29292297 -0.50120544 -0.5858307 -0.50120544l-26.331833 0" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m192.04282 131.78285c-0.2929077 0 -0.55648804 0.2373352 -0.55648804 0.50120544l0 9.074539c0 0.23744202 0.26358032 0.5012207 0.55648804 0.5012207l26.331833 0c0.2929077 0 0.5858307 -0.2637787 0.5858307 -0.5012207l0 -9.074539c0 -0.26387024 -0.29292297 -0.50120544 -0.5858307 -0.50120544l-26.331833 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.24)"><use xlink:href="#p.24" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.24" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.24"><path fill="#ebeded" d="m149.17256 158.67822c-0.2929077 0 -0.556427 0.2373352 -0.556427 0.50120544l0 9.074539c0 0.23744202 0.2635193 0.5012207 0.556427 0.5012207l26.160568 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.160568 0" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m149.17256 158.67822c-0.2929077 0 -0.556427 0.2373352 -0.556427 0.50120544l0 9.074539c0 0.23744202 0.2635193 0.5012207 0.556427 0.5012207l26.160568 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.160568 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.25)"><use xlink:href="#p.25" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.25" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.25"><path fill="#ebeded" d="m665.8765 131.78285c-0.29309082 0 -0.5570679 0.2373352 -0.5570679 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570679 0.5012207l26.164001 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.164001 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m665.8765 131.78285c-0.29309082 0 -0.5570679 0.2373352 -0.5570679 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570679 0.5012207l26.164001 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.164001 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.26)"><use xlink:href="#p.26" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.26" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.26"><path fill="#ebeded" d="m707.5716 131.78285c-0.2929077 0 -0.5564575 0.2373352 -0.5564575 0.50120544l0 9.074539c0 0.23744202 0.2635498 0.5012207 0.5564575 0.5012207l26.184875 0c0.29296875 0 0.5859375 -0.2637787 0.5859375 -0.5012207l0 -9.074539c0 -0.26387024 -0.29296875 -0.50120544 -0.5859375 -0.50120544l-26.184875 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m707.5716 131.78285c-0.2929077 0 -0.5564575 0.2373352 -0.5564575 0.50120544l0 9.074539c0 0.23744202 0.2635498 0.5012207 0.5564575 0.5012207l26.184875 0c0.29296875 0 0.5859375 -0.2637787 0.5859375 -0.5012207l0 -9.074539c0 -0.26387024 -0.29296875 -0.50120544 -0.5859375 -0.50120544l-26.184875 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.27)"><use xlink:href="#p.27" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.27" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.27"><path fill="#ebeded" d="m750.88226 158.67822c-0.2929077 0 -0.55651855 0.2373352 -0.55651855 0.50120544l0 9.074539c0 0.23744202 0.26361084 0.5012207 0.55651855 0.5012207l26.331848 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.331848 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m750.88226 158.67822c-0.2929077 0 -0.55651855 0.2373352 -0.55651855 0.50120544l0 9.074539c0 0.23744202 0.26361084 0.5012207 0.55651855 0.5012207l26.331848 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.331848 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.28)"><use xlink:href="#p.28" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.28" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.28"><path fill="#ebeded" d="m106.53792 158.67822c-0.29310608 0 -0.55708313 0.2373352 -0.55708313 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.55708313 0.5012207l26.163986 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.163986 0" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m106.53792 158.67822c-0.29310608 0 -0.55708313 0.2373352 -0.55708313 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.55708313 0.5012207l26.163986 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.163986 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.29)"><use xlink:href="#p.29" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.29" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.29"><path fill="#ebeded" d="m191.01506 158.67822c-0.2928772 0 -0.55644226 0.2373352 -0.55644226 0.50120544l0 9.074539c0 0.23744202 0.26356506 0.5012207 0.55644226 0.5012207l26.307266 0c0.2928772 0 0.58599854 -0.2637787 0.58599854 -0.5012207l0 -9.074539c0 -0.26387024 -0.29312134 -0.50120544 -0.58599854 -0.50120544l-26.307266 0" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m191.01506 158.67822c-0.2928772 0 -0.55644226 0.2373352 -0.55644226 0.50120544l0 9.074539c0 0.23744202 0.26356506 0.5012207 0.55644226 0.5012207l26.307266 0c0.2928772 0 0.58599854 -0.2637787 0.58599854 -0.5012207l0 -9.074539c0 -0.26387024 -0.29312134 -0.50120544 -0.58599854 -0.50120544l-26.307266 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.30)"><use xlink:href="#p.30" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.30" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.30"><path fill="#0433ff" d="m303.35846 158.67822c-0.2929077 0 -0.556427 0.2373352 -0.556427 0.50120544l0 9.074539c0 0.23744202 0.2635193 0.5012207 0.556427 0.5012207l26.160553 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.160553 0" fill-rule="evenodd"/><path stroke="#021ca1" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m303.35846 158.67822c-0.2929077 0 -0.556427 0.2373352 -0.556427 0.50120544l0 9.074539c0 0.23744202 0.2635193 0.5012207 0.556427 0.5012207l26.160553 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.160553 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.31)"><use xlink:href="#p.31" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.31" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.31"><path fill="#ebeded" d="m363.2306 158.67822c-0.29312134 0 -0.5570984 0.2373352 -0.5570984 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570984 0.5012207l26.16397 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.16397 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m363.2306 158.67822c-0.29312134 0 -0.5570984 0.2373352 -0.5570984 0.50120544l0 9.074539c0 0.23744202 0.26397705 0.5012207 0.5570984 0.5012207l26.16397 0c0.26397705 0 0.557312 -0.2637787 0.557312 -0.5012207l0 -9.074539c0 -0.26387024 -0.29333496 -0.50120544 -0.557312 -0.50120544l-26.16397 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.32)"><use xlink:href="#p.32" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.32" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.32"><path fill="#ebeded" d="m409.41852 158.67822c-0.2932129 0 -0.5567932 0.2373352 -0.5567932 0.50120544l0 9.074539c0 0.23744202 0.26358032 0.5012207 0.5567932 0.5012207l26.380066 0c0.29296875 0 0.5861511 -0.2637787 0.5861511 -0.5012207l0 -9.074539c0 -0.26387024 -0.29318237 -0.50120544 -0.5861511 -0.50120544l-26.380066 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m409.41852 158.67822c-0.2932129 0 -0.5567932 0.2373352 -0.5567932 0.50120544l0 9.074539c0 0.23744202 0.26358032 0.5012207 0.5567932 0.5012207l26.380066 0c0.29296875 0 0.5861511 -0.2637787 0.5861511 -0.5012207l0 -9.074539c0 -0.26387024 -0.29318237 -0.50120544 -0.5861511 -0.50120544l-26.380066 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.33)"><use xlink:href="#p.33" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.33" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.33"><path fill="#ebeded" d="m707.5716 158.67822c-0.2929077 0 -0.5564575 0.2373352 -0.5564575 0.50120544l0 9.074539c0 0.23744202 0.2635498 0.5012207 0.5564575 0.5012207l26.184875 0c0.29296875 0 0.5859375 -0.2637787 0.5859375 -0.5012207l0 -9.074539c0 -0.26387024 -0.29296875 -0.50120544 -0.5859375 -0.50120544l-26.184875 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m707.5716 158.67822c-0.2929077 0 -0.5564575 0.2373352 -0.5564575 0.50120544l0 9.074539c0 0.23744202 0.2635498 0.5012207 0.5564575 0.5012207l26.184875 0c0.29296875 0 0.5859375 -0.2637787 0.5859375 -0.5012207l0 -9.074539c0 -0.26387024 -0.29296875 -0.50120544 -0.5859375 -0.50120544l-26.184875 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.34)"><use xlink:href="#p.34" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.34" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.34"><path fill="#ebeded" d="m497.4783 158.67822c-0.2929077 0 -0.55648804 0.2373352 -0.55648804 0.50120544l0 9.074539c0 0.23744202 0.26358032 0.5012207 0.55648804 0.5012207l26.331818 0c0.2929077 0 0.58587646 -0.2637787 0.58587646 -0.5012207l0 -9.074539c0 -0.26387024 -0.29296875 -0.50120544 -0.58587646 -0.50120544l-26.331818 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m497.4783 158.67822c-0.2929077 0 -0.55648804 0.2373352 -0.55648804 0.50120544l0 9.074539c0 0.23744202 0.26358032 0.5012207 0.55648804 0.5012207l26.331818 0c0.2929077 0 0.58587646 -0.2637787 0.58587646 -0.5012207l0 -9.074539c0 -0.26387024 -0.29296875 -0.50120544 -0.58587646 -0.50120544l-26.331818 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.35)"><use xlink:href="#p.35" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.35" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.35"><path fill="#f9afaa" d="m149.17256 131.78285c-0.2929077 0 -0.556427 0.2373352 -0.556427 0.50120544l0 9.074539c0 0.23744202 0.2635193 0.5012207 0.556427 0.5012207l26.160568 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.160568 0" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m149.17256 131.78285c-0.2929077 0 -0.556427 0.2373352 -0.556427 0.50120544l0 9.074539c0 0.23744202 0.2635193 0.5012207 0.556427 0.5012207l26.160568 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.160568 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.36)"><use xlink:href="#p.36" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.36" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.36"><path fill="#90dae3" d="m497.4783 131.78285c-0.2929077 0 -0.55648804 0.2373352 -0.55648804 0.50120544l0 9.074539c0 0.23744202 0.26358032 0.5012207 0.55648804 0.5012207l26.331818 0c0.2929077 0 0.58587646 -0.2637787 0.58587646 -0.5012207l0 -9.074539c0 -0.26387024 -0.29296875 -0.50120544 -0.58587646 -0.50120544l-26.331818 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m497.4783 131.78285c-0.2929077 0 -0.55648804 0.2373352 -0.55648804 0.50120544l0 9.074539c0 0.23744202 0.26358032 0.5012207 0.55648804 0.5012207l26.331818 0c0.2929077 0 0.58587646 -0.2637787 0.58587646 -0.5012207l0 -9.074539c0 -0.26387024 -0.29296875 -0.50120544 -0.58587646 -0.50120544l-26.331818 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.37)"><use xlink:href="#p.37" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.37" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.37"><path fill="#90dae3" d="m750.88226 131.78285c-0.2929077 0 -0.55651855 0.2373352 -0.55651855 0.50120544l0 9.074539c0 0.23744202 0.26361084 0.5012207 0.55651855 0.5012207l26.331848 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.331848 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m750.88226 131.78285c-0.2929077 0 -0.55651855 0.2373352 -0.55651855 0.50120544l0 9.074539c0 0.23744202 0.26361084 0.5012207 0.55651855 0.5012207l26.331848 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.331848 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.38)"><use xlink:href="#p.38" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.38" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.38"><path fill="#ebeded" d="m459.74677 158.67822c-0.29284668 0 -0.5565796 0.2373352 -0.5565796 0.50120544l0 9.074539c0 0.23744202 0.2637329 0.5012207 0.5565796 0.5012207l26.111206 0c0.2930603 0 0.5861206 -0.2637787 0.5861206 -0.5012207l0 -9.074539c0 -0.26387024 -0.2930603 -0.50120544 -0.5861206 -0.50120544l-26.111206 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m459.74677 158.67822c-0.29284668 0 -0.5565796 0.2373352 -0.5565796 0.50120544l0 9.074539c0 0.23744202 0.2637329 0.5012207 0.5565796 0.5012207l26.111206 0c0.2930603 0 0.5861206 -0.2637787 0.5861206 -0.5012207l0 -9.074539c0 -0.26387024 -0.2930603 -0.50120544 -0.5861206 -0.50120544l-26.111206 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.39)"><use xlink:href="#p.39" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.39" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.39"><path fill="#ebeded" d="m536.7074 158.67822c-0.2929077 0 -0.5564575 0.2373352 -0.5564575 0.50120544l0 9.074539c0 0.23744202 0.2635498 0.5012207 0.5564575 0.5012207l26.160583 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.160583 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m536.7074 158.67822c-0.2929077 0 -0.5564575 0.2373352 -0.5564575 0.50120544l0 9.074539c0 0.23744202 0.2635498 0.5012207 0.5564575 0.5012207l26.160583 0c0.2929077 0 0.5858154 -0.2637787 0.5858154 -0.5012207l0 -9.074539c0 -0.26387024 -0.2929077 -0.50120544 -0.5858154 -0.50120544l-26.160583 0" fill-rule="evenodd"/></g><path fill="#000000" fill-opacity="0.0" d="m205.25764 55.438137l261.40546 -50.2082" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m205.25764 55.438137l261.40546 -50.2082" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m757.63727 55.438137l-290.98883 -50.2082" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m757.63727 55.438137l-290.98883 -50.2082" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m205.16956 101.44615l0.12234497 -46.03889" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m205.16956 101.44615l0.12234497 -46.03889" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m205.25764 131.80933l-0.09786987 -30.39846" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m205.25764 131.80933l-0.09786987 -30.39846" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m162.26999 101.975586l43.017014 -46.590385" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m162.26999 101.975586l43.017014 -46.590385" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m162.26999 131.80933l42.919144 -30.39846" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m162.26999 131.80933l42.919144 -30.39846" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m678.9733 131.80933l0.024475098 -29.869026" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m678.9733 131.80933l0.024475098 -29.869026" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m720.69836 131.80933l43.6532 -30.39846" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m720.69836 131.80933l43.6532 -30.39846" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m764.06775 131.80933l0.29364014 -30.39846" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m764.06775 131.80933l0.29364014 -30.39846" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m119.634705 158.70467l42.67444 -26.890945" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m119.634705 158.70467l42.67444 -26.890945" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m204.22993 158.70467l-41.96483 -26.890945" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m204.22993 158.70467l-41.96483 -26.890945" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m316.48526 158.70467l-43.84897 -26.890945" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m316.48526 158.70467l-43.84897 -26.890945" fill-rule="evenodd"/><g filter="url(#shadowFilter-p.40)"><use xlink:href="#p.40" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.40" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.40"><path fill="#ebeded" d="m149.17256 101.94912c-0.2929077 0 -0.556427 0.23719788 -0.556427 0.5007782l0 9.093201c0 0.26358032 0.2635193 0.5271683 0.556427 0.5271683l26.160568 0c0.2929077 0 0.5858154 -0.26358795 0.5858154 -0.5271683l0 -9.093201c0 -0.26358032 -0.2929077 -0.5007782 -0.5858154 -0.5007782l-26.160568 0" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m149.17256 101.94912c-0.2929077 0 -0.556427 0.23719788 -0.556427 0.5007782l0 9.093201c0 0.26358032 0.2635193 0.5271683 0.556427 0.5271683l26.160568 0c0.2929077 0 0.5858154 -0.26358795 0.5858154 -0.5271683l0 -9.093201c0 -0.26358032 -0.2929077 -0.5007782 -0.5858154 -0.5007782l-26.160568 0" fill-rule="evenodd"/></g><path fill="#000000" fill-opacity="0.0" d="m422.66238 158.70467l-46.320343 -26.890945" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m422.66238 158.70467l-46.320343 -26.890945" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m549.8342 158.70467l-39.175354 -26.890945" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m549.8342 158.70467l-39.175354 -26.890945" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m472.81467 158.70467l37.87845 -26.890945" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m472.81467 158.70467l37.87845 -26.890945" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m720.69836 158.70467l43.40851 -26.890945" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m720.69836 158.70467l43.40851 -26.890945" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m804.6476 158.70467l-40.57007 -26.890945" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m804.6476 158.70467l-40.57007 -26.890945" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m764.06775 158.70467l0.024475098 -26.890945" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m764.06775 158.70467l0.024475098 -26.890945" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m376.32736 158.70467l0.024475098 -26.890945" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m376.32736 158.70467l0.024475098 -26.890945" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m510.6638 158.70467l0.024475098 -26.890945" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m510.6638 158.70467l0.024475098 -26.890945" fill-rule="evenodd"/><g filter="url(#shadowFilter-p.41)"><use xlink:href="#p.41" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.41" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.41"><path fill="#90dae3" d="m751.14703 101.419685c-0.29315186 0 -0.5569458 0.23719788 -0.5569458 0.5007782l0 9.093201c0 0.26358032 0.26379395 0.52716064 0.5569458 0.52716064l26.360046 0c0.26379395 0 0.5571289 -0.26358032 0.5571289 -0.52716064l0 -9.093201c0 -0.26358032 -0.29333496 -0.5007782 -0.5571289 -0.5007782l-26.360046 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m751.14703 101.419685c-0.29315186 0 -0.5569458 0.23719788 -0.5569458 0.5007782l0 9.093201c0 0.26358032 0.26379395 0.52716064 0.5569458 0.52716064l26.360046 0c0.26379395 0 0.5571289 -0.26358032 0.5571289 -0.52716064l0 -9.093201c0 -0.26358032 -0.29333496 -0.5007782 -0.5571289 -0.5007782l-26.360046 0" fill-rule="evenodd"/></g><path fill="#000000" fill-opacity="0.0" d="m764.3614 101.44615l-148.16156 -46.03889" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="4.005249343832021" stroke-linejoin="round" stroke-linecap="butt" d="m764.3614 101.44615l-148.16156 -46.03889" fill-rule="evenodd"/><g filter="url(#shadowFilter-p.42)"><use xlink:href="#p.42" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.42" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.42"><path fill="#d7dada" d="m276.6083 192.50917c-1.1743164 0 -2.3179016 1.0310669 -2.3179016 2.0888672l0 36.888687c0 1.0578003 1.1435852 2.115265 2.3179016 2.115265l107.05585 0c1.1734009 0 2.3477173 -1.0574646 2.3477173 -2.115265l0 -36.888687c0 -1.0578003 -1.1743164 -2.0888672 -2.3477173 -2.0888672l-107.05585 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m276.6083 192.50917c-1.1743164 0 -2.3179016 1.0310669 -2.3179016 2.0888672l0 36.888687c0 1.0578003 1.1435852 2.115265 2.3179016 2.115265l107.05585 0c1.1734009 0 2.3477173 -1.0574646 2.3477173 -2.115265l0 -36.888687c0 -1.0578003 -1.1743164 -2.0888672 -2.3477173 -2.0888672l-107.05585 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.43)"><use xlink:href="#p.43" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.43" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.43"><path fill="#717d84" d="m287.06165 197.2741c-0.9097595 0 -1.8187866 0.8192291 -1.8187866 1.6384583l0 28.965668c0 0.8192291 0.9090271 1.6648407 1.8187866 1.6648407l85.71365 0c0.9097595 0 1.8485718 -0.8456116 1.8485718 -1.6648407l0 -28.965668c0 -0.8192291 -0.93881226 -1.6384583 -1.8485718 -1.6384583l-85.71365 0" fill-rule="evenodd"/><path stroke="#65bbc6" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m287.06165 197.2741c-0.9097595 0 -1.8187866 0.8192291 -1.8187866 1.6384583l0 28.965668c0 0.8192291 0.9090271 1.6648407 1.8187866 1.6648407l85.71365 0c0.9097595 0 1.8485718 -0.8456116 1.8485718 -1.6648407l0 -28.965668c0 -0.8192291 -0.93881226 -1.6384583 -1.8485718 -1.6384583l-85.71365 0" fill-rule="evenodd"/></g><path fill="#000000" fill-opacity="0.0" d="m305.03363 202.62143l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m305.03363 202.62143l66.60541 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m305.03363 206.5922l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m305.03363 206.5922l66.60541 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m305.03363 210.53648l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m305.03363 210.53648l66.60541 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m305.03363 214.50725l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m305.03363 214.50725l66.60541 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m305.03363 218.45154l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m305.03363 218.45154l66.60541 0" fill-rule="evenodd"/><g filter="url(#shadowFilter-p.44)"><use xlink:href="#p.44" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.44" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.44"><path fill="#d7dada" d="m139.9815 192.50917c-1.1738129 0 -2.3178253 1.0310669 -2.3178253 2.0888672l0 36.888687c0 1.0578003 1.1440125 2.115265 2.3178253 2.115265l107.00708 0c1.1738129 0 2.3476105 -1.0574646 2.3476105 -2.115265l0 -36.888687c0 -1.0578003 -1.1737976 -2.0888672 -2.3476105 -2.0888672l-107.00708 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m139.9815 192.50917c-1.1738129 0 -2.3178253 1.0310669 -2.3178253 2.0888672l0 36.888687c0 1.0578003 1.1440125 2.115265 2.3178253 2.115265l107.00708 0c1.1738129 0 2.3476105 -1.0574646 2.3476105 -2.115265l0 -36.888687c0 -1.0578003 -1.1737976 -2.0888672 -2.3476105 -2.0888672l-107.00708 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.45)"><use xlink:href="#p.45" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.45" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.45"><path fill="#717d84" d="m150.37743 197.2741c-0.9100189 0 -1.8200226 0.8192291 -1.8200226 1.6384583l0 28.965668c0 0.8192291 0.91000366 1.6648407 1.8200226 1.6648407l85.838806 0c0.91000366 0 1.8200073 -0.8456116 1.8200073 -1.6648407l0 -28.965668c0 -0.8192291 -0.91000366 -1.6384583 -1.8200073 -1.6384583l-85.838806 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m150.37743 197.2741c-0.9100189 0 -1.8200226 0.8192291 -1.8200226 1.6384583l0 28.965668c0 0.8192291 0.91000366 1.6648407 1.8200226 1.6648407l85.838806 0c0.91000366 0 1.8200073 -0.8456116 1.8200073 -1.6648407l0 -28.965668c0 -0.8192291 -0.91000366 -1.6384583 -1.8200073 -1.6384583l-85.838806 0" fill-rule="evenodd"/></g><path fill="#000000" fill-opacity="0.0" d="m168.37753 202.62143l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m168.37753 202.62143l66.60541 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m168.37753 206.5922l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m168.37753 206.5922l66.60541 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m168.37753 210.53648l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m168.37753 210.53648l66.60541 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m168.37753 214.50725l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m168.37753 214.50725l66.60541 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m168.37753 218.45154l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m168.37753 218.45154l66.60541 0" fill-rule="evenodd"/><g filter="url(#shadowFilter-p.46)"><use xlink:href="#p.46" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.46" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.46"><path fill="#d7dada" d="m413.99884 192.72095c-1.174469 0 -2.3182373 1.0310669 -2.3182373 2.0888672l0 36.888687c0 1.0578003 1.1437683 2.115265 2.3182373 2.115265l106.98181 0c1.173523 0 2.3480225 -1.0574646 2.3480225 -2.115265l0 -36.888687c0 -1.0578003 -1.1744995 -2.0888672 -2.3480225 -2.0888672l-106.98181 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m413.99884 192.72095c-1.174469 0 -2.3182373 1.0310669 -2.3182373 2.0888672l0 36.888687c0 1.0578003 1.1437683 2.115265 2.3182373 2.115265l106.98181 0c1.173523 0 2.3480225 -1.0574646 2.3480225 -2.115265l0 -36.888687c0 -1.0578003 -1.1744995 -2.0888672 -2.3480225 -2.0888672l-106.98181 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.47)"><use xlink:href="#p.47" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.47" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.47"><path fill="#717d84" d="m424.36423 197.45943c-0.91000366 0 -1.8192749 0.8189697 -1.8192749 1.6382294l0 28.988144c0 0.8192444 0.90927124 1.6646271 1.8192749 1.6646271l85.810455 0c0.91000366 0 1.8491211 -0.8453827 1.8491211 -1.6646271l0 -28.988144c0 -0.81925964 -0.93911743 -1.6382294 -1.8491211 -1.6382294l-85.810455 0" fill-rule="evenodd"/><path stroke="#f57469" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m424.36423 197.45943c-0.91000366 0 -1.8192749 0.8189697 -1.8192749 1.6382294l0 28.988144c0 0.8192444 0.90927124 1.6646271 1.8192749 1.6646271l85.810455 0c0.91000366 0 1.8491211 -0.8453827 1.8491211 -1.6646271l0 -28.988144c0 -0.81925964 -0.93911743 -1.6382294 -1.8491211 -1.6382294l-85.810455 0" fill-rule="evenodd"/></g><path fill="#000000" fill-opacity="0.0" d="m442.36508 202.83318l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m442.36508 202.83318l66.60541 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m442.36508 206.7775l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m442.36508 206.7775l66.60541 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m442.36508 210.74826l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m442.36508 210.74826l66.60541 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m442.36508 214.69255l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m442.36508 214.69255l66.60541 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m442.36508 218.66333l66.60541 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m442.36508 218.66333l66.60541 0" fill-rule="evenodd"/><g filter="url(#shadowFilter-p.48)"><use xlink:href="#p.48" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.48" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.48"><path fill="#d7dada" d="m551.53564 192.72095c-1.1734619 0 -2.3180542 1.0310669 -2.3180542 2.0888672l0 36.888687c0 1.0578003 1.1445923 2.115265 2.3180542 2.115265l107.15332 0c1.1744385 0 2.3479004 -1.0574646 2.3479004 -2.115265l0 -36.888687c0 -1.0578003 -1.1734619 -2.0888672 -2.3479004 -2.0888672l-107.15332 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m551.53564 192.72095c-1.1734619 0 -2.3180542 1.0310669 -2.3180542 2.0888672l0 36.888687c0 1.0578003 1.1445923 2.115265 2.3180542 2.115265l107.15332 0c1.1744385 0 2.3479004 -1.0574646 2.3479004 -2.115265l0 -36.888687c0 -1.0578003 -1.1734619 -2.0888672 -2.3479004 -2.0888672l-107.15332 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.49)"><use xlink:href="#p.49" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.49" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.49"><path fill="#717d84" d="m561.9303 197.45943c-0.90948486 0 -1.8189697 0.8189697 -1.8189697 1.6382294l0 28.988144c0 0.8192444 0.90948486 1.6646271 1.8189697 1.6646271l85.909546 0c0.90948486 0 1.8481445 -0.8453827 1.8481445 -1.6646271l0 -28.988144c0 -0.81925964 -0.93865967 -1.6382294 -1.8481445 -1.6382294l-85.909546 0" fill-rule="evenodd"/><path stroke="#e93317" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m561.9303 197.45943c-0.90948486 0 -1.8189697 0.8189697 -1.8189697 1.6382294l0 28.988144c0 0.8192444 0.90948486 1.6646271 1.8189697 1.6646271l85.909546 0c0.90948486 0 1.8481445 -0.8453827 1.8481445 -1.6646271l0 -28.988144c0 -0.81925964 -0.93865967 -1.6382294 -1.8481445 -1.6382294l-85.909546 0" fill-rule="evenodd"/></g><path fill="#000000" fill-opacity="0.0" d="m579.9902 202.83318l66.65436 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m579.9902 202.83318l66.65436 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m579.9902 206.7775l66.65436 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m579.9902 206.7775l66.65436 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m579.9902 210.74826l66.65436 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m579.9902 210.74826l66.65436 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m579.9902 214.69255l66.65436 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m579.9902 214.69255l66.65436 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m579.9902 218.66333l66.65436 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m579.9902 218.66333l66.65436 0" fill-rule="evenodd"/><g filter="url(#shadowFilter-p.50)"><use xlink:href="#p.50" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.50" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.50"><path fill="#d7dada" d="m686.0774 192.72095c-1.1738281 0 -2.317871 1.0310669 -2.317871 2.0888672l0 36.888687c0 1.0578003 1.144043 2.115265 2.317871 2.115265l107.00708 0c1.1737671 0 2.3475952 -1.0574646 2.3475952 -2.115265l0 -36.888687c0 -1.0578003 -1.1738281 -2.0888672 -2.3475952 -2.0888672l-107.00708 0" fill-rule="evenodd"/><path stroke="#47545c" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m686.0774 192.72095c-1.1738281 0 -2.317871 1.0310669 -2.317871 2.0888672l0 36.888687c0 1.0578003 1.144043 2.115265 2.317871 2.115265l107.00708 0c1.1737671 0 2.3475952 -1.0574646 2.3475952 -2.115265l0 -36.888687c0 -1.0578003 -1.1738281 -2.0888672 -2.3475952 -2.0888672l-107.00708 0" fill-rule="evenodd"/></g><g filter="url(#shadowFilter-p.51)"><use xlink:href="#p.51" transform="matrix(1.0 0.0 0.0 1.0 0.0 2.418897573716371)"/></g><defs><filter id="shadowFilter-p.51" filterUnits="userSpaceOnUse"><feGaussianBlur in="SourceAlpha" stdDeviation="0.0" result="blur"/><feComponentTransfer in="blur" color-interpolation-filters="sRGB"><feFuncR type="linear" slope="0" intercept="0.0"/><feFuncG type="linear" slope="0" intercept="0.0"/><feFuncB type="linear" slope="0" intercept="0.0"/><feFuncA type="linear" slope="0.349" intercept="0"/></feComponentTransfer></filter></defs><g id="p.51"><path fill="#717d84" d="m696.53125 197.45943c-0.9100342 0 -1.8192749 0.8189697 -1.8192749 1.6382294l0 28.988144c0 0.8192444 0.9092407 1.6646271 1.8192749 1.6646271l85.810486 0c0.90997314 0 1.84906 -0.8453827 1.84906 -1.6646271l0 -28.988144c0 -0.81925964 -0.9390869 -1.6382294 -1.84906 -1.6382294l-85.810486 0" fill-rule="evenodd"/><path stroke="#021ca1" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m696.53125 197.45943c-0.9100342 0 -1.8192749 0.8189697 -1.8192749 1.6382294l0 28.988144c0 0.8192444 0.9092407 1.6646271 1.8192749 1.6646271l85.810486 0c0.90997314 0 1.84906 -0.8453827 1.84906 -1.6646271l0 -28.988144c0 -0.81925964 -0.9390869 -1.6382294 -1.84906 -1.6382294l-85.810486 0" fill-rule="evenodd"/></g><path fill="#000000" fill-opacity="0.0" d="m714.50275 202.83318l66.55646 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m714.50275 202.83318l66.55646 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m714.50275 206.7775l66.55646 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m714.50275 206.7775l66.55646 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m714.50275 210.74826l66.55646 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m714.50275 210.74826l66.55646 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m714.50275 214.69255l66.55646 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m714.50275 214.69255l66.55646 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m714.50275 218.66333l66.55646 0" fill-rule="evenodd"/><path stroke="#ebeded" stroke-width="2.6824146981627295" stroke-linejoin="round" stroke-linecap="butt" d="m714.50275 218.66333l66.55646 0" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m158.46217 234.2839l70.10451 0l0 32.91327l-70.10451 0z" fill-rule="evenodd"/><path fill="#000000" d="m168.8528 261.2039l0 -13.35936l2.65625 0l3.15625 9.45311q0.4375 1.328125 0.640625 1.984375q0.234375 -0.734375 0.703125 -2.140625l3.203125 -9.29686l2.375 0l0 13.35936l-1.703125 0l0 -11.17186l-3.875 11.17186l-1.59375 0l-3.859375 -11.374985l0 11.374985l-1.703125 0zm15.587677 0l0 -13.35936l4.609375 0q1.546875 0 2.375 0.203125q1.140625 0.25 1.953125 0.953125q1.0625 0.890625 1.578125 2.28125q0.53125 1.390625 0.53125 3.171875q0 1.515625 -0.359375 2.7031097q-0.359375 1.171875 -0.921875 1.9375q-0.546875 0.765625 -1.203125 1.21875q-0.65625 0.4375 -1.59375 0.671875q-0.9375 0.21875 -2.140625 0.21875l-4.828125 0zm1.765625 -1.578125l2.859375 0q1.3125 0 2.0625 -0.234375q0.75 -0.25 1.203125 -0.703125q0.625 -0.625 0.96875 -1.6875q0.359375 -1.0624847 0.359375 -2.5781097q0 -2.09375 -0.6875 -3.21875q-0.6875 -1.125 -1.671875 -1.5q-0.703125 -0.28125 -2.28125 -0.28125l-2.8125 0l0 10.20311zm11.113571 -2.71875l1.65625 -0.140625q0.125 1.0 0.546875 1.640625q0.4375 0.640625 1.34375 1.046875q0.921875 0.390625 2.0625 0.390625q1.0 0 1.78125 -0.296875q0.78125 -0.296875 1.15625 -0.8125q0.375 -0.53125 0.375 -1.15625q0 -0.625 -0.375 -1.09375q-0.359375 -0.46875 -1.1875 -0.79685974q-0.546875 -0.203125 -2.390625 -0.640625q-1.828125 -0.453125 -2.5625 -0.84375q-0.96875 -0.5 -1.4375 -1.234375q-0.46875 -0.75 -0.46875 -1.671875q0 -1.0 0.578125 -1.875q0.578125 -0.890625 1.671875 -1.34375q1.109375 -0.453125 2.453125 -0.453125q1.484375 0 2.609375 0.484375q1.140625 0.46875 1.75 1.40625q0.609375 0.921875 0.65625 2.09375l-1.6875 0.125q-0.140625 -1.265625 -0.9375 -1.90625q-0.78125 -0.65625 -2.3125 -0.65625q-1.609375 0 -2.34375 0.59375q-0.734375 0.59375 -0.734375 1.421875q0 0.71875 0.53125 1.171875q0.5 0.46875 2.65625 0.96875q2.15625 0.484375 2.953125 0.84375q1.171875 0.53125 1.71875 1.359375q0.5625 0.82810974 0.5625 1.9062347q0 1.0625 -0.609375 2.015625q-0.609375 0.9375 -1.75 1.46875q-1.140625 0.515625 -2.578125 0.515625q-1.8125 0 -3.046875 -0.53125q-1.21875 -0.53125 -1.921875 -1.59375q-0.6875 -1.0625 -0.71875 -2.40625z" fill-rule="nonzero"/><path fill="#000000" d="m168.24342 276.61014q0 -2.359375 0.484375 -3.796875q0.484375 -1.453125 1.4375 -2.234375q0.96875 -0.78125 2.421875 -0.78125q1.078125 0 1.890625 0.4375q0.8125 0.421875 1.328125 1.25q0.53125 0.8125 0.828125 1.984375q0.3125 1.15625 0.3125 3.140625q0 2.359375 -0.484375 3.8125q-0.484375 1.4375 -1.453125 2.234375q-0.953125 0.78125 -2.421875 0.78125q-1.921875 0 -3.03125 -1.390625q-1.3125 -1.671875 -1.3125 -5.4375zm1.671875 0q0 3.296875 0.765625 4.390625q0.78125 1.078125 1.90625 1.078125q1.140625 0 1.90625 -1.09375q0.765625 -1.09375 0.765625 -4.375q0 -3.296875 -0.765625 -4.375q-0.765625 -1.078125 -1.921875 -1.078125q-1.125 0 -1.796875 0.953125q-0.859375 1.21875 -0.859375 4.5z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m295.11298 234.2839l70.10452 0l0 32.91327l-70.10452 0z" fill-rule="evenodd"/><path fill="#000000" d="m305.5036 261.2039l0 -13.35936l2.65625 0l3.15625 9.45311q0.4375 1.328125 0.640625 1.984375q0.234375 -0.734375 0.703125 -2.140625l3.203125 -9.29686l2.375 0l0 13.35936l-1.703125 0l0 -11.17186l-3.875 11.17186l-1.59375 0l-3.859375 -11.374985l0 11.374985l-1.703125 0zm15.587677 0l0 -13.35936l4.609375 0q1.546875 0 2.375 0.203125q1.140625 0.25 1.953125 0.953125q1.0625 0.890625 1.578125 2.28125q0.53125 1.390625 0.53125 3.171875q0 1.515625 -0.359375 2.7031097q-0.359375 1.171875 -0.921875 1.9375q-0.546875 0.765625 -1.203125 1.21875q-0.65625 0.4375 -1.59375 0.671875q-0.9375 0.21875 -2.140625 0.21875l-4.828125 0zm1.765625 -1.578125l2.859375 0q1.3125 0 2.0625 -0.234375q0.75 -0.25 1.203125 -0.703125q0.625 -0.625 0.96875 -1.6875q0.359375 -1.0624847 0.359375 -2.5781097q0 -2.09375 -0.6875 -3.21875q-0.6875 -1.125 -1.671875 -1.5q-0.703125 -0.28125 -2.28125 -0.28125l-2.8125 0l0 10.20311zm11.113556 -2.71875l1.65625 -0.140625q0.125 1.0 0.546875 1.640625q0.4375 0.640625 1.34375 1.046875q0.921875 0.390625 2.0625 0.390625q1.0 0 1.78125 -0.296875q0.78125 -0.296875 1.15625 -0.8125q0.375 -0.53125 0.375 -1.15625q0 -0.625 -0.375 -1.09375q-0.359375 -0.46875 -1.1875 -0.79685974q-0.546875 -0.203125 -2.390625 -0.640625q-1.828125 -0.453125 -2.5625 -0.84375q-0.96875 -0.5 -1.4375 -1.234375q-0.46875 -0.75 -0.46875 -1.671875q0 -1.0 0.578125 -1.875q0.578125 -0.890625 1.671875 -1.34375q1.109375 -0.453125 2.453125 -0.453125q1.484375 0 2.609375 0.484375q1.140625 0.46875 1.75 1.40625q0.609375 0.921875 0.65625 2.09375l-1.6875 0.125q-0.140625 -1.265625 -0.9375 -1.90625q-0.78125 -0.65625 -2.3125 -0.65625q-1.609375 0 -2.34375 0.59375q-0.734375 0.59375 -0.734375 1.421875q0 0.71875 0.53125 1.171875q0.5 0.46875 2.65625 0.96875q2.15625 0.484375 2.953125 0.84375q1.171875 0.53125 1.71875 1.359375q0.5625 0.82810974 0.5625 1.9062347q0 1.0625 -0.609375 2.015625q-0.609375 0.9375 -1.75 1.46875q-1.140625 0.515625 -2.578125 0.515625q-1.8125 0 -3.046875 -0.53125q-1.21875 -0.53125 -1.921875 -1.59375q-0.6875 -1.0625 -0.71875 -2.40625z" fill-rule="nonzero"/><path fill="#000000" d="m311.0661 283.2039l-1.640625 0l0 -10.453125q-0.59375 0.5625 -1.5625 1.140625q-0.953125 0.5625 -1.71875 0.84375l0 -1.59375q1.375 -0.640625 2.40625 -1.5625q1.03125 -0.921875 1.453125 -1.78125l1.0625 0l0 13.40625z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m437.7791 234.2839l70.10452 0l0 32.91327l-70.10452 0z" fill-rule="evenodd"/><path fill="#000000" d="m448.16974 261.2039l0 -13.35936l2.65625 0l3.15625 9.45311q0.4375 1.328125 0.640625 1.984375q0.234375 -0.734375 0.703125 -2.140625l3.203125 -9.29686l2.375 0l0 13.35936l-1.703125 0l0 -11.17186l-3.875 11.17186l-1.59375 0l-3.859375 -11.374985l0 11.374985l-1.703125 0zm15.587677 0l0 -13.35936l4.609375 0q1.546875 0 2.375 0.203125q1.140625 0.25 1.953125 0.953125q1.0625 0.890625 1.578125 2.28125q0.53125 1.390625 0.53125 3.171875q0 1.515625 -0.359375 2.7031097q-0.359375 1.171875 -0.921875 1.9375q-0.546875 0.765625 -1.203125 1.21875q-0.65625 0.4375 -1.59375 0.671875q-0.9375 0.21875 -2.140625 0.21875l-4.828125 0zm1.765625 -1.578125l2.859375 0q1.3125 0 2.0625 -0.234375q0.75 -0.25 1.203125 -0.703125q0.625 -0.625 0.96875 -1.6875q0.359375 -1.0624847 0.359375 -2.5781097q0 -2.09375 -0.6875 -3.21875q-0.6875 -1.125 -1.671875 -1.5q-0.703125 -0.28125 -2.28125 -0.28125l-2.8125 0l0 10.20311zm11.113586 -2.71875l1.65625 -0.140625q0.125 1.0 0.546875 1.640625q0.4375 0.640625 1.34375 1.046875q0.921875 0.390625 2.0625 0.390625q1.0 0 1.78125 -0.296875q0.78125 -0.296875 1.15625 -0.8125q0.375 -0.53125 0.375 -1.15625q0 -0.625 -0.375 -1.09375q-0.359375 -0.46875 -1.1875 -0.79685974q-0.546875 -0.203125 -2.390625 -0.640625q-1.828125 -0.453125 -2.5625 -0.84375q-0.96875 -0.5 -1.4375 -1.234375q-0.46875 -0.75 -0.46875 -1.671875q0 -1.0 0.578125 -1.875q0.578125 -0.890625 1.671875 -1.34375q1.109375 -0.453125 2.453125 -0.453125q1.484375 0 2.609375 0.484375q1.140625 0.46875 1.75 1.40625q0.609375 0.921875 0.65625 2.09375l-1.6875 0.125q-0.140625 -1.265625 -0.9375 -1.90625q-0.78125 -0.65625 -2.3125 -0.65625q-1.609375 0 -2.34375 0.59375q-0.734375 0.59375 -0.734375 1.421875q0 0.71875 0.53125 1.171875q0.5 0.46875 2.65625 0.96875q2.15625 0.484375 2.953125 0.84375q1.171875 0.53125 1.71875 1.359375q0.5625 0.82810974 0.5625 1.9062347q0 1.0625 -0.609375 2.015625q-0.609375 0.9375 -1.75 1.46875q-1.140625 0.515625 -2.578125 0.515625q-1.8125 0 -3.046875 -0.53125q-1.21875 -0.53125 -1.921875 -1.59375q-0.6875 -1.0625 -0.71875 -2.40625z" fill-rule="nonzero"/><path fill="#000000" d="m456.16974 281.62576l0 1.578125l-8.828125 0q-0.015625 -0.59375 0.1875 -1.140625q0.34375 -0.90625 1.078125 -1.78125q0.75 -0.875 2.15625 -2.015625q2.171875 -1.78125 2.9375 -2.828125q0.765625 -1.046875 0.765625 -1.96875q0 -0.984375 -0.703125 -1.640625q-0.6875 -0.671875 -1.8125 -0.671875q-1.1875 0 -1.90625 0.71875q-0.703125 0.703125 -0.703125 1.953125l-1.6875 -0.171875q0.171875 -1.890625 1.296875 -2.875q1.140625 -0.984375 3.03125 -0.984375q1.921875 0 3.046875 1.0625q1.125 1.0625 1.125 2.640625q0 0.796875 -0.328125 1.578125q-0.328125 0.78125 -1.09375 1.640625q-0.75 0.84375 -2.53125 2.34375q-1.46875 1.234375 -1.890625 1.6875q-0.421875 0.4375 -0.6875 0.875l6.546875 0z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m570.0907 234.2839l70.10455 0l0 32.91327l-70.10455 0z" fill-rule="evenodd"/><path fill="#000000" d="m580.4813 261.2039l0 -13.35936l2.65625 0l3.15625 9.45311q0.4375 1.328125 0.640625 1.984375q0.234375 -0.734375 0.703125 -2.140625l3.203125 -9.29686l2.375 0l0 13.35936l-1.703125 0l0 -11.17186l-3.875 11.17186l-1.59375 0l-3.859375 -11.374985l0 11.374985l-1.703125 0zm15.5877075 0l0 -13.35936l4.609375 0q1.546875 0 2.375 0.203125q1.140625 0.25 1.953125 0.953125q1.0625 0.890625 1.578125 2.28125q0.53125 1.390625 0.53125 3.171875q0 1.515625 -0.359375 2.7031097q-0.359375 1.171875 -0.921875 1.9375q-0.546875 0.765625 -1.203125 1.21875q-0.65625 0.4375 -1.59375 0.671875q-0.9375 0.21875 -2.140625 0.21875l-4.828125 0zm1.765625 -1.578125l2.859375 0q1.3125 0 2.0625 -0.234375q0.75 -0.25 1.203125 -0.703125q0.625 -0.625 0.96875 -1.6875q0.359375 -1.0624847 0.359375 -2.5781097q0 -2.09375 -0.6875 -3.21875q-0.6875 -1.125 -1.671875 -1.5q-0.703125 -0.28125 -2.28125 -0.28125l-2.8125 0l0 10.20311zm11.113525 -2.71875l1.65625 -0.140625q0.125 1.0 0.546875 1.640625q0.4375 0.640625 1.34375 1.046875q0.921875 0.390625 2.0625 0.390625q1.0 0 1.78125 -0.296875q0.78125 -0.296875 1.15625 -0.8125q0.375 -0.53125 0.375 -1.15625q0 -0.625 -0.375 -1.09375q-0.359375 -0.46875 -1.1875 -0.79685974q-0.546875 -0.203125 -2.390625 -0.640625q-1.828125 -0.453125 -2.5625 -0.84375q-0.96875 -0.5 -1.4375 -1.234375q-0.46875 -0.75 -0.46875 -1.671875q0 -1.0 0.578125 -1.875q0.578125 -0.890625 1.671875 -1.34375q1.109375 -0.453125 2.453125 -0.453125q1.484375 0 2.609375 0.484375q1.140625 0.46875 1.75 1.40625q0.609375 0.921875 0.65625 2.09375l-1.6875 0.125q-0.140625 -1.265625 -0.9375 -1.90625q-0.78125 -0.65625 -2.3125 -0.65625q-1.609375 0 -2.34375 0.59375q-0.734375 0.59375 -0.734375 1.421875q0 0.71875 0.53125 1.171875q0.5 0.46875 2.65625 0.96875q2.15625 0.484375 2.953125 0.84375q1.171875 0.53125 1.71875 1.359375q0.5625 0.82810974 0.5625 1.9062347q0 1.0625 -0.609375 2.015625q-0.609375 0.9375 -1.75 1.46875q-1.140625 0.515625 -2.578125 0.515625q-1.8125 0 -3.046875 -0.53125q-1.21875 -0.53125 -1.921875 -1.59375q-0.6875 -1.0625 -0.71875 -2.40625z" fill-rule="nonzero"/><path fill="#000000" d="m579.87195 279.67264l1.640625 -0.21875q0.28125 1.40625 0.953125 2.015625q0.6875 0.609375 1.65625 0.609375q1.15625 0 1.953125 -0.796875q0.796875 -0.796875 0.796875 -1.984375q0 -1.125 -0.734375 -1.859375q-0.734375 -0.734375 -1.875 -0.734375q-0.46875 0 -1.15625 0.171875l0.1875 -1.4375q0.15625 0.015625 0.265625 0.015625q1.046875 0 1.875 -0.546875q0.84375 -0.546875 0.84375 -1.671875q0 -0.90625 -0.609375 -1.5q-0.609375 -0.59375 -1.578125 -0.59375q-0.953125 0 -1.59375 0.609375q-0.640625 0.59375 -0.8125 1.796875l-1.640625 -0.296875q0.296875 -1.640625 1.359375 -2.546875q1.0625 -0.90625 2.65625 -0.90625q1.09375 0 2.0 0.46875q0.921875 0.46875 1.40625 1.28125q0.5 0.8125 0.5 1.71875q0 0.859375 -0.46875 1.578125q-0.46875 0.703125 -1.375 1.125q1.1875 0.28125 1.84375 1.140625q0.65625 0.859375 0.65625 2.15625q0 1.734375 -1.28125 2.953125q-1.265625 1.21875 -3.21875 1.21875q-1.765625 0 -2.921875 -1.046875q-1.15625 -1.046875 -1.328125 -2.71875z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m704.5576 234.2839l70.10449 0l0 32.91327l-70.10449 0z" fill-rule="evenodd"/><path fill="#000000" d="m714.94824 261.2039l0 -13.35936l2.65625 0l3.15625 9.45311q0.4375 1.328125 0.640625 1.984375q0.234375 -0.734375 0.703125 -2.140625l3.203125 -9.29686l2.375 0l0 13.35936l-1.703125 0l0 -11.17186l-3.875 11.17186l-1.59375 0l-3.859375 -11.374985l0 11.374985l-1.703125 0zm15.5876465 0l0 -13.35936l4.609375 0q1.546875 0 2.375 0.203125q1.140625 0.25 1.953125 0.953125q1.0625 0.890625 1.578125 2.28125q0.53125 1.390625 0.53125 3.171875q0 1.515625 -0.359375 2.7031097q-0.359375 1.171875 -0.921875 1.9375q-0.546875 0.765625 -1.203125 1.21875q-0.65625 0.4375 -1.59375 0.671875q-0.9375 0.21875 -2.140625 0.21875l-4.828125 0zm1.765625 -1.578125l2.859375 0q1.3125 0 2.0625 -0.234375q0.75 -0.25 1.203125 -0.703125q0.625 -0.625 0.96875 -1.6875q0.359375 -1.0624847 0.359375 -2.5781097q0 -2.09375 -0.6875 -3.21875q-0.6875 -1.125 -1.671875 -1.5q-0.703125 -0.28125 -2.28125 -0.28125l-2.8125 0l0 10.20311zm11.113586 -2.71875l1.65625 -0.140625q0.125 1.0 0.546875 1.640625q0.4375 0.640625 1.34375 1.046875q0.921875 0.390625 2.0625 0.390625q1.0 0 1.78125 -0.296875q0.78125 -0.296875 1.15625 -0.8125q0.375 -0.53125 0.375 -1.15625q0 -0.625 -0.375 -1.09375q-0.359375 -0.46875 -1.1875 -0.79685974q-0.546875 -0.203125 -2.390625 -0.640625q-1.828125 -0.453125 -2.5625 -0.84375q-0.96875 -0.5 -1.4375 -1.234375q-0.46875 -0.75 -0.46875 -1.671875q0 -1.0 0.578125 -1.875q0.578125 -0.890625 1.671875 -1.34375q1.109375 -0.453125 2.453125 -0.453125q1.484375 0 2.609375 0.484375q1.140625 0.46875 1.75 1.40625q0.609375 0.921875 0.65625 2.09375l-1.6875 0.125q-0.140625 -1.265625 -0.9375 -1.90625q-0.78125 -0.65625 -2.3125 -0.65625q-1.609375 0 -2.34375 0.59375q-0.734375 0.59375 -0.734375 1.421875q0 0.71875 0.53125 1.171875q0.5 0.46875 2.65625 0.96875q2.15625 0.484375 2.953125 0.84375q1.171875 0.53125 1.71875 1.359375q0.5625 0.82810974 0.5625 1.9062347q0 1.0625 -0.609375 2.015625q-0.609375 0.9375 -1.75 1.46875q-1.140625 0.515625 -2.578125 0.515625q-1.8125 0 -3.046875 -0.53125q-1.21875 -0.53125 -1.921875 -1.59375q-0.6875 -1.0625 -0.71875 -2.40625z" fill-rule="nonzero"/><path fill="#000000" d="m719.58887 283.2039l0 -3.203125l-5.796875 0l0 -1.5l6.09375 -8.65625l1.34375 0l0 8.65625l1.796875 0l0 1.5l-1.796875 0l0 3.203125l-1.640625 0zm0 -4.703125l0 -6.015625l-4.1875 6.015625l4.1875 0z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m27.929134 282.41995l877.6693 0l0 27.937012l-877.6693 0z" fill-rule="evenodd"/><path fill="#000000" d="m38.429134 306.77994l0 -11.453125l1.515625 0l0 11.453125l-1.515625 0zm4.0078125 0l0 -8.296875l1.265625 0l0 1.171875q0.90625 -1.359375 2.640625 -1.359375q0.75 0 1.375 0.265625q0.625 0.265625 0.9375 0.703125q0.3125 0.4375 0.4375 1.046875q0.078125 0.390625 0.078125 1.359375l0 5.109375l-1.40625 0l0 -5.046875q0 -0.859375 -0.171875 -1.28125q-0.15625 -0.4375 -0.578125 -0.6875q-0.40625 -0.25 -0.96875 -0.25q-0.90625 0 -1.5625 0.578125q-0.640625 0.5625 -0.640625 2.15625l0 4.53125l-1.40625 0zm16.40625 -1.265625l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3828125 1.265625l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.4843712 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.2968712 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm8.898434 -9.84375l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm2.9921875 -2.484375l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm13.3359375 2.484375l0 -7.203125l-1.234375 0l0 -1.09375l1.234375 0l0 -0.890625q0 -0.828125 0.15625 -1.234375q0.203125 -0.546875 0.703125 -0.890625q0.515625 -0.34375 1.4375 -0.34375q0.59375 0 1.3125 0.140625l-0.203125 1.234375q-0.4375 -0.078125 -0.828125 -0.078125q-0.640625 0 -0.90625 0.28125q-0.265625 0.265625 -0.265625 1.015625l0 0.765625l1.609375 0l0 1.09375l-1.609375 0l0 7.203125l-1.40625 0zm4.1171875 -9.84375l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm3.5234375 0l0 -11.453125l1.40625 0l0 11.453125l-1.40625 0zm9.2578125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm11.71875 2.46875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm8.5 5.6875l-0.15625 -1.328125q0.453125 0.125 0.796875 0.125q0.46875 0 0.75 -0.15625q0.28125 -0.15625 0.46875 -0.4375q0.125 -0.203125 0.421875 -1.046875q0.046875 -0.109375 0.125 -0.34375l-3.140625 -8.3125l1.515625 0l1.71875 4.796875q0.34375 0.921875 0.609375 1.921875q0.234375 -0.96875 0.578125 -1.890625l1.765625 -4.828125l1.40625 0l-3.15625 8.4375q-0.5 1.375 -0.78125 1.890625q-0.375 0.6875 -0.859375 1.015625q-0.484375 0.328125 -1.15625 0.328125q-0.40625 0 -0.90625 -0.171875zm7.5 -5.6875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.5468826 0.4375 1.5000076 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.5468826 -0.390625 -2.1562576 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.5781326 -0.171875 1.2187576 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.9687576 0 -1.3906326 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.2500076 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.6250076 0 -2.4687576 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm11.625008 1.21875l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm7.0546875 -1.40625l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm7.8359375 4.953125l0 -8.296875l1.25 0l0 1.15625q0.390625 -0.609375 1.03125 -0.96875q0.65625 -0.375 1.484375 -0.375q0.921875 0 1.515625 0.390625q0.59375 0.375 0.828125 1.0625q0.984375 -1.453125 2.5625 -1.453125q1.234375 0 1.890625 0.6875q0.671875 0.671875 0.671875 2.09375l0 5.703125l-1.390625 0l0 -5.234375q0 -0.84375 -0.140625 -1.203125q-0.140625 -0.375 -0.5 -0.59375q-0.359375 -0.234375 -0.84375 -0.234375q-0.875 0 -1.453125 0.578125q-0.578125 0.578125 -0.578125 1.859375l0 4.828125l-1.40625 0l0 -5.390625q0 -0.9375 -0.34375 -1.40625q-0.34375 -0.46875 -1.125 -0.46875q-0.59375 0 -1.09375 0.3125q-0.5 0.3125 -0.734375 0.921875q-0.21875 0.59375 -0.21875 1.71875l0 4.3125l-1.40625 0zm17.773438 0l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm8.8984375 -9.84375l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm9.2265625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm7.8203125 4.953125l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm10.75 -1.03125q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm3.5859375 4.171875l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm10.75 -3.046875l1.390625 0.1875q-0.234375 1.421875 -1.171875 2.234375q-0.921875 0.8125 -2.28125 0.8125q-1.703125 0 -2.75 -1.109375q-1.03125 -1.125 -1.03125 -3.203125q0 -1.34375 0.4375 -2.34375q0.453125 -1.015625 1.359375 -1.515625q0.921875 -0.5 1.984375 -0.5q1.359375 0 2.21875 0.6875q0.859375 0.671875 1.09375 1.9375l-1.359375 0.203125q-0.203125 -0.828125 -0.703125 -1.25q-0.484375 -0.421875 -1.1875 -0.421875q-1.0625 0 -1.734375 0.765625q-0.65625 0.75 -0.65625 2.40625q0 1.671875 0.640625 2.4375q0.640625 0.75 1.671875 0.75q0.828125 0 1.375 -0.5q0.5625 -0.515625 0.703125 -1.578125zm2.59375 3.046875l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm8.8359375 3.203125l-0.15625 -1.328125q0.453125 0.125 0.796875 0.125q0.46875 0 0.75 -0.15625q0.28125 -0.15625 0.46875 -0.4375q0.125 -0.203125 0.421875 -1.046875q0.046875 -0.109375 0.125 -0.34375l-3.140625 -8.3125l1.515625 0l1.71875 4.796875q0.34375 0.921875 0.609375 1.921875q0.234375 -0.96875 0.578125 -1.890625l1.765625 -4.828125l1.40625 0l-3.15625 8.4375q-0.5 1.375 -0.78125 1.890625q-0.375 0.6875 -0.859375 1.015625q-0.484375 0.328125 -1.15625 0.328125q-0.40625 0 -0.90625 -0.171875zm15.5703125 -4.46875l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3671875 1.265625l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.5078125 2.28125l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm8.1953125 4.953125l0 -1.609375l1.609375 0l0 1.609375q0 0.890625 -0.3125 1.421875q-0.3125 0.546875 -1.0 0.84375l-0.390625 -0.59375q0.453125 -0.203125 0.65625 -0.578125q0.21875 -0.375 0.234375 -1.09375l-0.796875 0zm11.59375 -1.265625l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3828125 1.265625l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm14.5703125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm11.71875 2.46875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm8.5625 2.484375l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm14.3046875 -1.03125q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm8.9765625 4.171875l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm13.6328125 1.46875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.2109375 4.953125l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm12.40625 4.140625l0 -8.296875l1.265625 0l0 1.171875q0.90625 -1.359375 2.640625 -1.359375q0.75 0 1.375 0.265625q0.625 0.265625 0.9375 0.703125q0.3125 0.4375 0.4375 1.046875q0.078125 0.390625 0.078125 1.359375l0 5.109375l-1.40625 0l0 -5.046875q0 -0.859375 -0.171875 -1.28125q-0.15625 -0.4375 -0.578125 -0.6875q-0.40625 -0.25 -0.96875 -0.25q-0.90625 0 -1.5625 0.578125q-0.640625 0.5625 -0.640625 2.15625l0 4.53125l-1.40625 0zm8.3671875 -4.15625q0 -2.296875 1.28125 -3.40625q1.078125 -0.921875 2.609375 -0.921875q1.71875 0 2.796875 1.125q1.09375 1.109375 1.09375 3.09375q0 1.59375 -0.484375 2.515625q-0.484375 0.921875 -1.40625 1.4375q-0.90625 0.5 -2.0 0.5q-1.734375 0 -2.8125 -1.109375q-1.078125 -1.125 -1.078125 -3.234375zm1.453125 0q0 1.59375 0.6875 2.390625q0.703125 0.796875 1.75 0.796875q1.046875 0 1.734375 -0.796875q0.703125 -0.796875 0.703125 -2.4375q0 -1.53125 -0.703125 -2.328125q-0.6875 -0.796875 -1.734375 -0.796875q-1.046875 0 -1.75 0.796875q-0.6875 0.78125 -0.6875 2.375zm13.3515625 4.15625l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm13.6328125 1.46875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm7.2734375 2.46875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm12.9921875 2.484375l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm7.8359375 8.140625l0 -11.484375l1.28125 0l0 1.078125q0.453125 -0.640625 1.015625 -0.953125q0.578125 -0.3125 1.390625 -0.3125q1.0625 0 1.875 0.546875q0.8125 0.546875 1.21875 1.546875q0.421875 0.984375 0.421875 2.171875q0 1.28125 -0.46875 2.296875q-0.453125 1.015625 -1.328125 1.5625q-0.859375 0.546875 -1.828125 0.546875q-0.703125 0 -1.265625 -0.296875q-0.546875 -0.296875 -0.90625 -0.75l0 4.046875l-1.40625 0zm1.265625 -7.296875q0 1.609375 0.640625 2.375q0.65625 0.765625 1.578125 0.765625q0.9375 0 1.609375 -0.796875q0.671875 -0.796875 0.671875 -2.453125q0 -1.59375 -0.65625 -2.375q-0.65625 -0.796875 -1.5625 -0.796875q-0.890625 0 -1.59375 0.84375q-0.6875 0.84375 -0.6875 2.4375zm7.6171875 4.109375l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm7.2734375 2.46875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm14.234375 -0.1875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm7.8359375 4.953125l0 -8.296875l1.265625 0l0 1.171875q0.90625 -1.359375 2.640625 -1.359375q0.75 0 1.375 0.265625q0.625 0.265625 0.9375 0.703125q0.3125 0.4375 0.4375 1.046875q0.078125 0.390625 0.078125 1.359375l0 5.109375l-1.40625 0l0 -5.046875q0 -0.859375 -0.171875 -1.28125q-0.15625 -0.4375 -0.578125 -0.6875q-0.40625 -0.25 -0.96875 -0.25q-0.90625 0 -1.5625 0.578125q-0.640625 0.5625 -0.640625 2.15625l0 4.53125l-1.40625 0zm11.9609375 -1.265625l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm11.203125 1.265625l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm7.9609375 -5.703125l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm3.5390625 0l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.2421875 1.90625l1.390625 0.1875q-0.234375 1.421875 -1.171875 2.234375q-0.921875 0.8125 -2.28125 0.8125q-1.703125 0 -2.75 -1.109375q-1.03125 -1.125 -1.03125 -3.203125q0 -1.34375 0.4375 -2.34375q0.453125 -1.015625 1.359375 -1.515625q0.921875 -0.5 1.984375 -0.5q1.359375 0 2.21875 0.6875q0.859375 0.671875 1.09375 1.9375l-1.359375 0.203125q-0.203125 -0.828125 -0.703125 -1.25q-0.484375 -0.421875 -1.1875 -0.421875q-1.0625 0 -1.734375 0.765625q-0.65625 0.75 -0.65625 2.40625q0 1.671875 0.640625 2.4375q0.640625 0.75 1.671875 0.75q0.828125 0 1.375 -0.5q0.5625 -0.515625 0.703125 -1.578125zm5.65625 1.78125l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm0.8515625 -2.890625q0 -2.296875 1.28125 -3.40625q1.078125 -0.921875 2.609375 -0.921875q1.71875 0 2.796875 1.125q1.09375 1.109375 1.09375 3.09375q0 1.59375 -0.484375 2.515625q-0.484375 0.921875 -1.40625 1.4375q-0.90625 0.5 -2.0 0.5q-1.734375 0 -2.8125 -1.109375q-1.078125 -1.125 -1.078125 -3.234375zm1.453125 0q0 1.59375 0.6875 2.390625q0.703125 0.796875 1.75 0.796875q1.046875 0 1.734375 -0.796875q0.703125 -0.796875 0.703125 -2.4375q0 -1.53125 -0.703125 -2.328125q-0.6875 -0.796875 -1.734375 -0.796875q-1.046875 0 -1.75 0.796875q-0.6875 0.78125 -0.6875 2.375zm7.9609375 4.15625l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm5.34375 -9.84375l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm9.2265625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm7.2734375 2.46875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm18.414062 1.453125q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm3.6015625 4.171875l0 -8.296875l1.265625 0l0 1.171875q0.90625 -1.359375 2.640625 -1.359375q0.75 0 1.375 0.265625q0.625 0.265625 0.9375 0.703125q0.3125 0.4375 0.4375 1.046875q0.078125 0.390625 0.078125 1.359375l0 5.109375l-1.40625 0l0 -5.046875q0 -0.859375 -0.171875 -1.28125q-0.15625 -0.4375 -0.578125 -0.6875q-0.40625 -0.25 -0.96875 -0.25q-0.90625 0 -1.5625 0.578125q-0.640625 0.5625 -0.640625 2.15625l0 4.53125l-1.40625 0zm14.2734375 0l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm17.84375 4.140625l0 -1.21875q-0.96875 1.40625 -2.640625 1.40625q-0.734375 0 -1.375 -0.28125q-0.625 -0.28125 -0.9375 -0.703125q-0.3125 -0.4375 -0.4375 -1.046875q-0.078125 -0.421875 -0.078125 -1.3125l0 -5.140625l1.40625 0l0 4.59375q0 1.109375 0.078125 1.484375q0.140625 0.5625 0.5625 0.875q0.4375 0.3125 1.0625 0.3125q0.640625 0 1.1875 -0.3125q0.5625 -0.328125 0.78125 -0.890625q0.234375 -0.5625 0.234375 -1.625l0 -4.4375l1.40625 0l0 8.296875l-1.25 0zm3.4609375 0l0 -8.296875l1.265625 0l0 1.171875q0.90625 -1.359375 2.640625 -1.359375q0.75 0 1.375 0.265625q0.625 0.265625 0.9375 0.703125q0.3125 0.4375 0.4375 1.046875q0.078125 0.390625 0.078125 1.359375l0 5.109375l-1.40625 0l0 -5.046875q0 -0.859375 -0.171875 -1.28125q-0.15625 -0.4375 -0.578125 -0.6875q-0.40625 -0.25 -0.96875 -0.25q-0.90625 0 -1.5625 0.578125q-0.640625 0.5625 -0.640625 2.15625l0 4.53125l-1.40625 0zm8.3359375 -2.484375l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm8.5625 2.484375l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm14.3046875 -1.03125q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm8.9765625 4.171875l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm13.6328125 1.46875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.2109375 4.953125l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm12.40625 4.140625l0 -8.296875l1.265625 0l0 1.171875q0.90625 -1.359375 2.640625 -1.359375q0.75 0 1.375 0.265625q0.625 0.265625 0.9375 0.703125q0.3125 0.4375 0.4375 1.046875q0.078125 0.390625 0.078125 1.359375l0 5.109375l-1.40625 0l0 -5.046875q0 -0.859375 -0.171875 -1.28125q-0.15625 -0.4375 -0.578125 -0.6875q-0.40625 -0.25 -0.96875 -0.25q-0.90625 0 -1.5625 0.578125q-0.640625 0.5625 -0.640625 2.15625l0 4.53125l-1.40625 0zm8.3671875 -4.15625q0 -2.296875 1.28125 -3.40625q1.078125 -0.921875 2.609375 -0.921875q1.71875 0 2.796875 1.125q1.09375 1.109375 1.09375 3.09375q0 1.59375 -0.484375 2.515625q-0.484375 0.921875 -1.40625 1.4375q-0.90625 0.5 -2.0 0.5q-1.734375 0 -2.8125 -1.109375q-1.078125 -1.125 -1.078125 -3.234375zm1.453125 0q0 1.59375 0.6875 2.390625q0.703125 0.796875 1.75 0.796875q1.046875 0 1.734375 -0.796875q0.703125 -0.796875 0.703125 -2.4375q0 -1.53125 -0.703125 -2.328125q-0.6875 -0.796875 -1.734375 -0.796875q-1.046875 0 -1.75 0.796875q-0.6875 0.78125 -0.6875 2.375zm13.3515625 4.15625l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm13.6328125 1.46875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm7.2734375 2.46875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm13.3359375 2.484375l0 -7.203125l-1.234375 0l0 -1.09375l1.234375 0l0 -0.890625q0 -0.828125 0.15625 -1.234375q0.203125 -0.546875 0.703125 -0.890625q0.515625 -0.34375 1.4375 -0.34375q0.59375 0 1.3125 0.140625l-0.203125 1.234375q-0.4375 -0.078125 -0.828125 -0.078125q-0.640625 0 -0.90625 0.28125q-0.265625 0.265625 -0.265625 1.015625l0 0.765625l1.609375 0l0 1.09375l-1.609375 0l0 7.203125l-1.40625 0zm4.1171875 -9.84375l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm3.5234375 0l0 -11.453125l1.40625 0l0 11.453125l-1.40625 0zm9.2578125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm7.2734375 2.46875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm8.953125 2.484375l0 -1.609375l1.609375 0l0 1.609375l-1.609375 0zm11.3046875 0l0 -10.109375l-3.78125 0l0 -1.34375l9.078125 0l0 1.34375l-3.78125 0l0 10.109375l-1.515625 0zm6.6796875 0l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm14.5703125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm11.71875 2.46875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm14.0 2.484375l0 -1.21875q-0.96875 1.40625 -2.640625 1.40625q-0.734375 0 -1.375 -0.28125q-0.625 -0.28125 -0.9375 -0.703125q-0.3125 -0.4375 -0.4375 -1.046875q-0.078125 -0.421875 -0.078125 -1.3125l0 -5.140625l1.40625 0l0 4.59375q0 1.109375 0.078125 1.484375q0.140625 0.5625 0.5625 0.875q0.4375 0.3125 1.0625 0.3125q0.640625 0 1.1875 -0.3125q0.5625 -0.328125 0.78125 -0.890625q0.234375 -0.5625 0.234375 -1.625l0 -4.4375l1.40625 0l0 8.296875l-1.25 0zm4.7578125 0l-1.3125 0l0 -11.453125l1.40625 0l0 4.078125q0.890625 -1.109375 2.28125 -1.109375q0.765625 0 1.4375 0.3125q0.6875 0.296875 1.125 0.859375q0.453125 0.5625 0.703125 1.359375q0.25 0.78125 0.25 1.671875q0 2.140625 -1.0625 3.3125q-1.046875 1.15625 -2.53125 1.15625q-1.46875 0 -2.296875 -1.234375l0 1.046875zm-0.015625 -4.21875q0 1.5 0.40625 2.15625q0.65625 1.09375 1.796875 1.09375q0.921875 0 1.59375 -0.796875q0.671875 -0.8125 0.671875 -2.390625q0 -1.625 -0.65625 -2.390625q-0.640625 -0.78125 -1.546875 -0.78125q-0.921875 0 -1.59375 0.796875q-0.671875 0.796875 -0.671875 2.3125zm10.6796875 2.953125l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3671875 1.265625l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.5078125 2.28125l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm7.2734375 2.46875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm18.414062 1.453125q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm3.5859375 4.171875l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875z" fill-rule="nonzero"/><path fill="#000000" d="m37.991634 328.96744l0 -11.484375l1.28125 0l0 1.078125q0.453125 -0.640625 1.015625 -0.953125q0.578125 -0.3125 1.390625 -0.3125q1.0625 0 1.875 0.546875q0.8125 0.546875 1.21875 1.546875q0.421875 0.984375 0.421875 2.171875q0 1.28125 -0.46875 2.296875q-0.453125 1.015625 -1.328125 1.5625q-0.859375 0.546875 -1.828125 0.546875q-0.703125 0 -1.265625 -0.296875q-0.546875 -0.296875 -0.90625 -0.75l0 4.046875l-1.40625 0zm1.265625 -7.296875q0 1.609375 0.640625 2.375q0.65625 0.765625 1.578125 0.765625q0.9375 0 1.609375 -0.796875q0.671875 -0.796875 0.671875 -2.453125q0 -1.59375 -0.65625 -2.375q-0.65625 -0.796875 -1.5625 -0.796875q-0.890625 0 -1.59375 0.84375q-0.6875 0.84375 -0.6875 2.4375zm13.0390625 3.078125q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm3.5859375 4.171875l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm8.406246 -1.265625l0.203125 1.25q-0.5937462 0.125 -1.0624962 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.4062462 0l0 1.09375l-1.4062462 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.6093712 -0.0625zm1.3828125 -8.578125l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm6.6171875 -1.265625l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3828125 -8.578125l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm3.0234375 -4.15625q0 -2.296875 1.28125 -3.40625q1.078125 -0.921875 2.609375 -0.921875q1.71875 0 2.796875 1.125q1.09375 1.109375 1.09375 3.09375q0 1.59375 -0.484375 2.515625q-0.484375 0.921875 -1.40625 1.4375q-0.90625 0.5 -2.0 0.5q-1.734375 0 -2.8125 -1.109375q-1.078125 -1.125 -1.078125 -3.234375zm1.453125 0q0 1.59375 0.6875 2.390625q0.703125 0.796875 1.75 0.796875q1.046875 0 1.734375 -0.796875q0.703125 -0.796875 0.703125 -2.4375q0 -1.53125 -0.703125 -2.328125q-0.6875 -0.796875 -1.734375 -0.796875q-1.046875 0 -1.75 0.796875q-0.6875 0.78125 -0.6875 2.375zm7.9765625 4.15625l0 -8.296875l1.265625 0l0 1.171875q0.90625 -1.359375 2.640625 -1.359375q0.75 0 1.375 0.265625q0.625 0.265625 0.9375 0.703125q0.3125 0.4375 0.4375 1.046875q0.078125 0.390625 0.078125 1.359375l0 5.109375l-1.40625 0l0 -5.046875q0 -0.859375 -0.171875 -1.28125q-0.15625 -0.4375 -0.578125 -0.6875q-0.40625 -0.25 -0.96875 -0.25q-0.90625 0 -1.5625 0.578125q-0.640625 0.5625 -0.640625 2.15625l0 4.53125l-1.40625 0zm14.5703125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.2109375 4.953125l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm12.40625 -5.703125l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm3.5546875 0l0 -8.296875l1.265625 0l0 1.171875q0.90625 -1.359375 2.640625 -1.359375q0.75 0 1.375 0.265625q0.625 0.265625 0.9375 0.703125q0.3125 0.4375 0.4375 1.046875q0.078125 0.390625 0.078125 1.359375l0 5.109375l-1.40625 0l0 -5.046875q0 -0.859375 -0.171875 -1.28125q-0.15625 -0.4375 -0.578125 -0.6875q-0.40625 -0.25 -0.96875 -0.25q-0.90625 0 -1.5625 0.578125q-0.640625 0.5625 -0.640625 2.15625l0 4.53125l-1.40625 0zm12.781258 -2.484375l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm14.0 2.484375l0 -1.21875q-0.96875 1.40625 -2.640625 1.40625q-0.734375 0 -1.375 -0.28125q-0.625 -0.28125 -0.9375 -0.703125q-0.3125 -0.4375 -0.4375 -1.046875q-0.078125 -0.421875 -0.078125 -1.3125l0 -5.140625l1.40625 0l0 4.59375q0 1.109375 0.078125 1.484375q0.140625 0.5625 0.5625 0.875q0.4375 0.3125 1.0625 0.3125q0.640625 0 1.1875 -0.3125q0.5625 -0.328125 0.78125 -0.890625q0.234375 -0.5625 0.234375 -1.625l0 -4.4375l1.40625 0l0 8.296875l-1.25 0zm8.8671875 -3.046875l1.390625 0.1875q-0.234375 1.421875 -1.171875 2.234375q-0.921875 0.8125 -2.28125 0.8125q-1.703125 0 -2.75 -1.109375q-1.03125 -1.125 -1.03125 -3.203125q0 -1.34375 0.4375 -2.34375q0.453125 -1.015625 1.359375 -1.515625q0.921875 -0.5 1.984375 -0.5q1.359375 0 2.21875 0.6875q0.859375 0.671875 1.09375 1.9375l-1.359375 0.203125q-0.203125 -0.828125 -0.703125 -1.25q-0.484375 -0.421875 -1.1875 -0.421875q-1.0625 0 -1.734375 0.765625q-0.65625 0.75 -0.65625 2.40625q0 1.671875 0.640625 2.4375q0.640625 0.75 1.671875 0.75q0.828125 0 1.375 -0.5q0.5625 -0.515625 0.703125 -1.578125zm2.59375 3.046875l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm18.75 -1.03125q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm9.578125 4.171875l-2.546875 -8.296875l1.453125 0l1.328125 4.78125l0.484375 1.78125q0.03125 -0.125 0.4375 -1.703125l1.3125 -4.859375l1.453125 0l1.234375 4.8125l0.421875 1.578125l0.46875 -1.59375l1.421875 -4.796875l1.375 0l-2.59375 8.296875l-1.46875 0l-1.3125 -4.96875l-0.328125 -1.421875l-1.671875 6.390625l-1.46875 0zm15.4296875 -1.03125q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm3.5390625 7.375l-0.15625 -1.328125q0.453125 0.125 0.796875 0.125q0.46875 0 0.75 -0.15625q0.28125 -0.15625 0.46875 -0.4375q0.125 -0.203125 0.421875 -1.046875q0.046875 -0.109375 0.125 -0.34375l-3.140625 -8.3125l1.515625 0l1.71875 4.796875q0.34375 0.921875 0.609375 1.921875q0.234375 -0.96875 0.578125 -1.890625l1.765625 -4.828125l1.40625 0l-3.15625 8.4375q-0.5 1.375 -0.78125 1.890625q-0.375 0.6875 -0.859375 1.015625q-0.484375 0.328125 -1.15625 0.328125q-0.40625 0 -0.90625 -0.171875zm15.5703125 -4.46875l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3828125 1.265625l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm14.3046875 -1.03125q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm6.6640625 2.90625l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm8.890625 0l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3828125 1.265625l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm14.5703125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm12.28125 4.953125l0 -8.296875l1.25 0l0 1.15625q0.390625 -0.609375 1.03125 -0.96875q0.65625 -0.375 1.484375 -0.375q0.921875 0 1.515625 0.390625q0.59375 0.375 0.828125 1.0625q0.984375 -1.453125 2.5625 -1.453125q1.234375 0 1.890625 0.6875q0.671875 0.671875 0.671875 2.09375l0 5.703125l-1.390625 0l0 -5.234375q0 -0.84375 -0.140625 -1.203125q-0.140625 -0.375 -0.5 -0.59375q-0.359375 -0.234375 -0.84375 -0.234375q-0.875 0 -1.453125 0.578125q-0.578125 0.578125 -0.578125 1.859375l0 4.828125l-1.40625 0l0 -5.390625q0 -0.9375 -0.34375 -1.40625q-0.34375 -0.46875 -1.125 -0.46875q-0.59375 0 -1.09375 0.3125q-0.5 0.3125 -0.734375 0.921875q-0.21875 0.59375 -0.21875 1.71875l0 4.3125l-1.40625 0zm19.0 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm10.8984375 3.6875l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm6.7890625 0.234375q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm8.9765625 4.171875l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm13.3671875 3.109375q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm6.6640625 2.90625l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm6.7890625 0.234375q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm8.375 4.171875l0 -7.203125l-1.234375 0l0 -1.09375l1.234375 0l0 -0.890625q0 -0.828125 0.15625 -1.234375q0.203125 -0.546875 0.703125 -0.890625q0.515625 -0.34375 1.4375 -0.34375q0.59375 0 1.3125 0.140625l-0.203125 1.234375q-0.4375 -0.078125 -0.828125 -0.078125q-0.640625 0 -0.90625 0.28125q-0.265625 0.265625 -0.265625 1.015625l0 0.765625l1.609375 0l0 1.09375l-1.609375 0l0 7.203125l-1.40625 0zm3.5859375 -4.15625q0 -2.296875 1.28125 -3.40625q1.078125 -0.921875 2.609375 -0.921875q1.71875 0 2.796875 1.125q1.09375 1.109375 1.09375 3.09375q0 1.59375 -0.484375 2.515625q-0.484375 0.921875 -1.40625 1.4375q-0.90625 0.5 -2.0 0.5q-1.734375 0 -2.8125 -1.109375q-1.078125 -1.125 -1.078125 -3.234375zm1.453125 0q0 1.59375 0.6875 2.390625q0.703125 0.796875 1.75 0.796875q1.046875 0 1.734375 -0.796875q0.703125 -0.796875 0.703125 -2.4375q0 -1.53125 -0.703125 -2.328125q-0.6875 -0.796875 -1.734375 -0.796875q-1.046875 0 -1.75 0.796875q-0.6875 0.78125 -0.6875 2.375zm7.9609375 4.15625l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm12.8515625 -1.265625l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3828125 1.265625l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm14.5703125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm12.015625 5.640625l1.375 0.203125q0.078125 0.640625 0.46875 0.921875q0.53125 0.390625 1.4375 0.390625q0.96875 0 1.5 -0.390625q0.53125 -0.390625 0.71875 -1.09375q0.109375 -0.421875 0.109375 -1.8125q-0.921875 1.09375 -2.296875 1.09375q-1.71875 0 -2.65625 -1.234375q-0.9375 -1.234375 -0.9375 -2.96875q0 -1.1875 0.421875 -2.1875q0.4375 -1.0 1.25 -1.546875q0.828125 -0.546875 1.921875 -0.546875q1.46875 0 2.421875 1.1875l0 -1.0l1.296875 0l0 7.171875q0 1.9375 -0.390625 2.75q-0.390625 0.8125 -1.25 1.28125q-0.859375 0.46875 -2.109375 0.46875q-1.484375 0 -2.40625 -0.671875q-0.90625 -0.671875 -0.875 -2.015625zm1.171875 -4.984375q0 1.625 0.640625 2.375q0.65625 0.75 1.625 0.75q0.96875 0 1.625 -0.734375q0.65625 -0.75 0.65625 -2.34375q0 -1.53125 -0.671875 -2.296875q-0.671875 -0.78125 -1.625 -0.78125q-0.9375 0 -1.59375 0.765625q-0.65625 0.765625 -0.65625 2.265625zm7.9765625 4.296875l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm7.7734375 8.15625l-0.15625 -1.328125q0.453125 0.125 0.796875 0.125q0.46875 0 0.75 -0.15625q0.28125 -0.15625 0.46875 -0.4375q0.125 -0.203125 0.421875 -1.046875q0.046875 -0.109375 0.125 -0.34375l-3.140625 -8.3125l1.515625 0l1.71875 4.796875q0.34375 0.921875 0.609375 1.921875q0.234375 -0.96875 0.578125 -1.890625l1.765625 -4.828125l1.40625 0l-3.15625 8.4375q-0.5 1.375 -0.78125 1.890625q-0.375 0.6875 -0.859375 1.015625q-0.484375 0.328125 -1.15625 0.328125q-0.40625 0 -0.90625 -0.171875zm11.9453125 -5.6875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm14.0 2.484375l0 -1.21875q-0.96875 1.40625 -2.640625 1.40625q-0.734375 0 -1.375 -0.28125q-0.625 -0.28125 -0.9375 -0.703125q-0.3125 -0.4375 -0.4375 -1.046875q-0.078125 -0.421875 -0.078125 -1.3125l0 -5.140625l1.40625 0l0 4.59375q0 1.109375 0.078125 1.484375q0.140625 0.5625 0.5625 0.875q0.4375 0.3125 1.0625 0.3125q0.640625 0 1.1875 -0.3125q0.5625 -0.328125 0.78125 -0.890625q0.234375 -0.5625 0.234375 -1.625l0 -4.4375l1.40625 0l0 8.296875l-1.25 0zm4.7578125 0l-1.3125 0l0 -11.453125l1.40625 0l0 4.078125q0.890625 -1.109375 2.28125 -1.109375q0.765625 0 1.4375 0.3125q0.6875 0.296875 1.125 0.859375q0.453125 0.5625 0.703125 1.359375q0.25 0.78125 0.25 1.671875q0 2.140625 -1.0625 3.3125q-1.046875 1.15625 -2.53125 1.15625q-1.46875 0 -2.296875 -1.234375l0 1.046875zm-0.015625 -4.21875q0 1.5 0.40625 2.15625q0.65625 1.09375 1.796875 1.09375q0.921875 0 1.59375 -0.796875q0.671875 -0.8125 0.671875 -2.390625q0 -1.625 -0.65625 -2.390625q-0.640625 -0.78125 -1.546875 -0.78125q-0.921875 0 -1.59375 0.796875q-0.671875 0.796875 -0.671875 2.3125zm10.6796875 2.953125l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3671875 1.265625l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.5078125 2.28125l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm12.28125 -4.890625l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm2.9921875 -2.484375l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm13.0078125 2.484375l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm14.3046875 -1.03125q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm3.6015625 4.171875l0 -8.296875l1.265625 0l0 1.171875q0.90625 -1.359375 2.640625 -1.359375q0.75 0 1.375 0.265625q0.625 0.265625 0.9375 0.703125q0.3125 0.4375 0.4375 1.046875q0.078125 0.390625 0.078125 1.359375l0 5.109375l-1.40625 0l0 -5.046875q0 -0.859375 -0.171875 -1.28125q-0.15625 -0.4375 -0.578125 -0.6875q-0.40625 -0.25 -0.96875 -0.25q-0.90625 0 -1.5625 0.578125q-0.640625 0.5625 -0.640625 2.15625l0 4.53125l-1.40625 0zm14.2734375 0l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm7.9296875 4.140625l0 -11.453125l1.40625 0l0 11.453125l-1.40625 0zm9.2578125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.2109375 4.953125l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm13.703125 4.140625l-1.3125 0l0 -11.453125l1.40625 0l0 4.078125q0.890625 -1.109375 2.28125 -1.109375q0.765625 0 1.4375 0.3125q0.6875 0.296875 1.125 0.859375q0.453125 0.5625 0.703125 1.359375q0.25 0.78125 0.25 1.671875q0 2.140625 -1.0625 3.3125q-1.046875 1.15625 -2.53125 1.15625q-1.46875 0 -2.296875 -1.234375l0 1.046875zm-0.015625 -4.21875q0 1.5 0.40625 2.15625q0.65625 1.09375 1.796875 1.09375q0.921875 0 1.59375 -0.796875q0.671875 -0.8125 0.671875 -2.390625q0 -1.625 -0.65625 -2.390625q-0.640625 -0.78125 -1.546875 -0.78125q-0.921875 0 -1.59375 0.796875q-0.671875 0.796875 -0.671875 2.3125zm7.5546875 7.421875l-0.15625 -1.328125q0.453125 0.125 0.796875 0.125q0.46875 0 0.75 -0.15625q0.28125 -0.15625 0.46875 -0.4375q0.125 -0.203125 0.421875 -1.046875q0.046875 -0.109375 0.125 -0.34375l-3.140625 -8.3125l1.515625 0l1.71875 4.796875q0.34375 0.921875 0.609375 1.921875q0.234375 -0.96875 0.578125 -1.890625l1.765625 -4.828125l1.40625 0l-3.15625 8.4375q-0.5 1.375 -0.78125 1.890625q-0.375 0.6875 -0.859375 1.015625q-0.484375 0.328125 -1.15625 0.328125q-0.40625 0 -0.90625 -0.171875zm12.6328125 -3.203125l0 -11.453125l2.28125 0l2.71875 8.109375q0.375 1.125 0.546875 1.6875q0.1875 -0.625 0.609375 -1.828125l2.734375 -7.96875l2.046875 0l0 11.453125l-1.46875 0l0 -9.59375l-3.328125 9.59375l-1.359375 0l-3.3125 -9.75l0 9.75l-1.46875 0zm13.375 0l0 -11.453125l3.953125 0q1.328125 0 2.03125 0.15625q0.984375 0.234375 1.6875 0.828125q0.90625 0.765625 1.34375 1.953125q0.453125 1.1875 0.453125 2.71875q0 1.3125 -0.3125 2.328125q-0.296875 1.0 -0.78125 1.65625q-0.46875 0.65625 -1.03125 1.046875q-0.5625 0.375 -1.375 0.578125q-0.796875 0.1875 -1.828125 0.1875l-4.140625 0zm1.515625 -1.359375l2.453125 0q1.125 0 1.765625 -0.203125q0.65625 -0.21875 1.03125 -0.59375q0.546875 -0.546875 0.84375 -1.453125q0.296875 -0.90625 0.296875 -2.203125q0 -1.796875 -0.59375 -2.765625q-0.578125 -0.96875 -1.421875 -1.296875q-0.609375 -0.234375 -1.96875 -0.234375l-2.40625 0l0 8.75zm9.5234375 -2.328125l1.4375 -0.125q0.09375 0.859375 0.46875 1.421875q0.375 0.546875 1.15625 0.890625q0.78125 0.328125 1.75 0.328125q0.875 0 1.53125 -0.25q0.671875 -0.265625 0.984375 -0.703125q0.328125 -0.453125 0.328125 -0.984375q0 -0.546875 -0.3125 -0.9375q-0.3125 -0.40625 -1.03125 -0.6875q-0.453125 -0.171875 -2.03125 -0.546875q-1.578125 -0.390625 -2.21875 -0.71875q-0.8125 -0.4375 -1.21875 -1.0625q-0.40625 -0.640625 -0.40625 -1.4375q0 -0.859375 0.484375 -1.609375q0.5 -0.765625 1.4375 -1.15625q0.953125 -0.390625 2.109375 -0.390625q1.28125 0 2.25 0.421875q0.96875 0.40625 1.484375 1.203125q0.53125 0.796875 0.578125 1.796875l-1.453125 0.109375q-0.125 -1.078125 -0.796875 -1.625q-0.671875 -0.5625 -2.0 -0.5625q-1.375 0 -2.0 0.5q-0.625 0.5 -0.625 1.21875q0 0.609375 0.4375 1.015625q0.4375 0.390625 2.28125 0.8125q1.859375 0.421875 2.546875 0.734375q1.0 0.453125 1.46875 1.171875q0.484375 0.703125 0.484375 1.625q0 0.90625 -0.53125 1.71875q-0.515625 0.8125 -1.5 1.265625q-0.984375 0.453125 -2.203125 0.453125q-1.5625 0 -2.609375 -0.453125q-1.046875 -0.46875 -1.65625 -1.375q-0.59375 -0.90625 -0.625 -2.0625zm15.0703125 -1.96875q0 -2.03125 0.40625 -3.265625q0.421875 -1.234375 1.25 -1.90625q0.828125 -0.671875 2.078125 -0.671875q0.921875 0 1.609375 0.375q0.703125 0.359375 1.15625 1.0625q0.453125 0.703125 0.703125 1.703125q0.265625 1.0 0.265625 2.703125q0 2.015625 -0.421875 3.265625q-0.40625 1.234375 -1.234375 1.921875q-0.828125 0.671875 -2.078125 0.671875q-1.65625 0 -2.609375 -1.203125q-1.125 -1.421875 -1.125 -4.65625zm1.4375 0q0 2.828125 0.65625 3.765625q0.671875 0.921875 1.640625 0.921875q0.96875 0 1.625 -0.9375q0.65625 -0.9375 0.65625 -3.75q0 -2.828125 -0.65625 -3.75q-0.65625 -0.9375 -1.640625 -0.9375q-0.96875 0 -1.546875 0.828125q-0.734375 1.046875 -0.734375 3.859375zm8.2109375 5.65625l0 -1.609375l1.609375 0l0 1.609375q0 0.890625 -0.3125 1.421875q-0.3125 0.546875 -1.0 0.84375l-0.390625 -0.59375q0.453125 -0.203125 0.65625 -0.578125q0.21875 -0.375 0.234375 -1.09375l-0.796875 0zm11.59375 -1.265625l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3828125 1.265625l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm14.5703125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm12.25 4.953125l0 -11.453125l1.40625 0l0 11.453125l-1.40625 0zm3.5859375 -9.84375l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm3.2890625 0.6875l1.375 0.203125q0.078125 0.640625 0.46875 0.921875q0.53125 0.390625 1.4375 0.390625q0.96875 0 1.5 -0.390625q0.53125 -0.390625 0.71875 -1.09375q0.109375 -0.421875 0.109375 -1.8125q-0.921875 1.09375 -2.296875 1.09375q-1.71875 0 -2.65625 -1.234375q-0.9375 -1.234375 -0.9375 -2.96875q0 -1.1875 0.421875 -2.1875q0.4375 -1.0 1.25 -1.546875q0.828125 -0.546875 1.921875 -0.546875q1.46875 0 2.421875 1.1875l0 -1.0l1.296875 0l0 7.171875q0 1.9375 -0.390625 2.75q-0.390625 0.8125 -1.25 1.28125q-0.859375 0.46875 -2.109375 0.46875q-1.484375 0 -2.40625 -0.671875q-0.90625 -0.671875 -0.875 -2.015625zm1.171875 -4.984375q0 1.625 0.640625 2.375q0.65625 0.75 1.625 0.75q0.96875 0 1.625 -0.734375q0.65625 -0.75 0.65625 -2.34375q0 -1.53125 -0.671875 -2.296875q-0.671875 -0.78125 -1.625 -0.78125q-0.9375 0 -1.59375 0.765625q-0.65625 0.765625 -0.65625 2.265625zm7.9921875 4.296875l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm11.9609375 -1.265625l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm7.125 1.265625l-1.3125 0l0 -11.453125l1.40625 0l0 4.078125q0.890625 -1.109375 2.28125 -1.109375q0.765625 0 1.4375 0.3125q0.6875 0.296875 1.125 0.859375q0.453125 0.5625 0.703125 1.359375q0.25 0.78125 0.25 1.671875q0 2.140625 -1.0625 3.3125q-1.046875 1.15625 -2.53125 1.15625q-1.46875 0 -2.296875 -1.234375l0 1.046875zm-0.015625 -4.21875q0 1.5 0.40625 2.15625q0.65625 1.09375 1.796875 1.09375q0.921875 0 1.59375 -0.796875q0.671875 -0.8125 0.671875 -2.390625q0 -1.625 -0.65625 -2.390625q-0.640625 -0.78125 -1.546875 -0.78125q-0.921875 0 -1.59375 0.796875q-0.671875 0.796875 -0.671875 2.3125zm7.5859375 4.21875l0 -11.453125l1.40625 0l0 11.453125l-1.40625 0zm9.0234375 0l0 -1.21875q-0.96875 1.40625 -2.640625 1.40625q-0.734375 0 -1.375 -0.28125q-0.625 -0.28125 -0.9375 -0.703125q-0.3125 -0.4375 -0.4375 -1.046875q-0.078125 -0.421875 -0.078125 -1.3125l0 -5.140625l1.40625 0l0 4.59375q0 1.109375 0.078125 1.484375q0.140625 0.5625 0.5625 0.875q0.4375 0.3125 1.0625 0.3125q0.640625 0 1.1875 -0.3125q0.5625 -0.328125 0.78125 -0.890625q0.234375 -0.5625 0.234375 -1.625l0 -4.4375l1.40625 0l0 8.296875l-1.25 0zm9.1328125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm11.71875 2.46875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm14.0 2.484375l0 -1.21875q-0.96875 1.40625 -2.640625 1.40625q-0.734375 0 -1.375 -0.28125q-0.625 -0.28125 -0.9375 -0.703125q-0.3125 -0.4375 -0.4375 -1.046875q-0.078125 -0.421875 -0.078125 -1.3125l0 -5.140625l1.40625 0l0 4.59375q0 1.109375 0.078125 1.484375q0.140625 0.5625 0.5625 0.875q0.4375 0.3125 1.0625 0.3125q0.640625 0 1.1875 -0.3125q0.5625 -0.328125 0.78125 -0.890625q0.234375 -0.5625 0.234375 -1.625l0 -4.4375l1.40625 0l0 8.296875l-1.25 0zm4.7578125 0l-1.3125 0l0 -11.453125l1.40625 0l0 4.078125q0.890625 -1.109375 2.28125 -1.109375q0.765625 0 1.4375 0.3125q0.6875 0.296875 1.125 0.859375q0.453125 0.5625 0.703125 1.359375q0.25 0.78125 0.25 1.671875q0 2.140625 -1.0625 3.3125q-1.046875 1.15625 -2.53125 1.15625q-1.46875 0 -2.296875 -1.234375l0 1.046875zm-0.015625 -4.21875q0 1.5 0.40625 2.15625q0.65625 1.09375 1.796875 1.09375q0.921875 0 1.59375 -0.796875q0.671875 -0.8125 0.671875 -2.390625q0 -1.625 -0.65625 -2.390625q-0.640625 -0.78125 -1.546875 -0.78125q-0.921875 0 -1.59375 0.796875q-0.671875 0.796875 -0.671875 2.3125zm10.6796875 2.953125l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3671875 1.265625l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.5078125 2.28125l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.578125 4.953125l-1.3125 0l0 -11.453125l1.40625 0l0 4.078125q0.890625 -1.109375 2.28125 -1.109375q0.765625 0 1.4375 0.3125q0.6875 0.296875 1.125 0.859375q0.453125 0.5625 0.703125 1.359375q0.25 0.78125 0.25 1.671875q0 2.140625 -1.0625 3.3125q-1.046875 1.15625 -2.53125 1.15625q-1.46875 0 -2.296875 -1.234375l0 1.046875zm-0.015625 -4.21875q0 1.5 0.40625 2.15625q0.65625 1.09375 1.796875 1.09375q0.921875 0 1.59375 -0.796875q0.671875 -0.8125 0.671875 -2.390625q0 -1.625 -0.65625 -2.390625q-0.640625 -0.78125 -1.546875 -0.78125q-0.921875 0 -1.59375 0.796875q-0.671875 0.796875 -0.671875 2.3125zm7.5546875 7.421875l-0.15625 -1.328125q0.453125 0.125 0.796875 0.125q0.46875 0 0.75 -0.15625q0.28125 -0.15625 0.46875 -0.4375q0.125 -0.203125 0.421875 -1.046875q0.046875 -0.109375 0.125 -0.34375l-3.140625 -8.3125l1.515625 0l1.71875 4.796875q0.34375 0.921875 0.609375 1.921875q0.234375 -0.96875 0.578125 -1.890625l1.765625 -4.828125l1.40625 0l-3.15625 8.4375q-0.5 1.375 -0.78125 1.890625q-0.375 0.6875 -0.859375 1.015625q-0.484375 0.328125 -1.15625 0.328125q-0.40625 0 -0.90625 -0.171875zm12.6328125 -3.203125l0 -11.453125l2.28125 0l2.71875 8.109375q0.375 1.125 0.546875 1.6875q0.1875 -0.625 0.609375 -1.828125l2.734375 -7.96875l2.046875 0l0 11.453125l-1.46875 0l0 -9.59375l-3.328125 9.59375l-1.359375 0l-3.3125 -9.75l0 9.75l-1.46875 0zm13.375 0l0 -11.453125l3.953125 0q1.328125 0 2.03125 0.15625q0.984375 0.234375 1.6875 0.828125q0.90625 0.765625 1.34375 1.953125q0.453125 1.1875 0.453125 2.71875q0 1.3125 -0.3125 2.328125q-0.296875 1.0 -0.78125 1.65625q-0.46875 0.65625 -1.03125 1.046875q-0.5625 0.375 -1.375 0.578125q-0.796875 0.1875 -1.828125 0.1875l-4.140625 0zm1.515625 -1.359375l2.453125 0q1.125 0 1.765625 -0.203125q0.65625 -0.21875 1.03125 -0.59375q0.546875 -0.546875 0.84375 -1.453125q0.296875 -0.90625 0.296875 -2.203125q0 -1.796875 -0.59375 -2.765625q-0.578125 -0.96875 -1.421875 -1.296875q-0.609375 -0.234375 -1.96875 -0.234375l-2.40625 0l0 8.75zm9.5234375 -2.328125l1.4375 -0.125q0.09375 0.859375 0.46875 1.421875q0.375 0.546875 1.15625 0.890625q0.78125 0.328125 1.75 0.328125q0.875 0 1.53125 -0.25q0.671875 -0.265625 0.984375 -0.703125q0.328125 -0.453125 0.328125 -0.984375q0 -0.546875 -0.3125 -0.9375q-0.3125 -0.40625 -1.03125 -0.6875q-0.453125 -0.171875 -2.03125 -0.546875q-1.578125 -0.390625 -2.21875 -0.71875q-0.8125 -0.4375 -1.21875 -1.0625q-0.40625 -0.640625 -0.40625 -1.4375q0 -0.859375 0.484375 -1.609375q0.5 -0.765625 1.4375 -1.15625q0.953125 -0.390625 2.109375 -0.390625q1.28125 0 2.25 0.421875q0.96875 0.40625 1.484375 1.203125q0.53125 0.796875 0.578125 1.796875l-1.453125 0.109375q-0.125 -1.078125 -0.796875 -1.625q-0.671875 -0.5625 -2.0 -0.5625q-1.375 0 -2.0 0.5q-0.625 0.5 -0.625 1.21875q0 0.609375 0.4375 1.015625q0.4375 0.390625 2.28125 0.8125q1.859375 0.421875 2.546875 0.734375q1.0 0.453125 1.46875 1.171875q0.484375 0.703125 0.484375 1.625q0 0.90625 -0.53125 1.71875q-0.515625 0.8125 -1.5 1.265625q-0.984375 0.453125 -2.203125 0.453125q-1.5625 0 -2.609375 -0.453125q-1.046875 -0.46875 -1.65625 -1.375q-0.59375 -0.90625 -0.625 -2.0625zm20.367188 3.6875l-1.40625 0l0 -8.96875q-0.515625 0.484375 -1.34375 0.96875q-0.8125 0.484375 -1.46875 0.734375l0 -1.359375q1.171875 -0.5625 2.046875 -1.34375q0.890625 -0.796875 1.265625 -1.53125l0.90625 0l0 11.5zm4.3515625 0l0 -1.609375l1.609375 0l0 1.609375q0 0.890625 -0.3125 1.421875q-0.3125 0.546875 -1.0 0.84375l-0.390625 -0.59375q0.453125 -0.203125 0.65625 -0.578125q0.21875 -0.375 0.234375 -1.09375l-0.796875 0z" fill-rule="nonzero"/><path fill="#000000" d="m41.054134 343.5143l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3828125 1.265625l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm14.5703125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm11.749996 0.796875q0 -2.296875 1.28125 -3.40625q1.078125 -0.921875 2.609375 -0.921875q1.71875 0 2.796875 1.125q1.09375 1.109375 1.09375 3.09375q0 1.59375 -0.484375 2.515625q-0.484375 0.921875 -1.40625 1.4375q-0.90625 0.5 -2.0 0.5q-1.734375 0 -2.8125 -1.109375q-1.078125 -1.125 -1.078125 -3.234375zm1.453125 0q0 1.59375 0.6875 2.390625q0.703125 0.796875 1.75 0.796875q1.046875 0 1.734375 -0.796875q0.703125 -0.796875 0.703125 -2.4375q0 -1.53125 -0.703125 -2.328125q-0.6875 -0.796875 -1.734375 -0.796875q-1.046875 0 -1.75 0.796875q-0.6875 0.78125 -0.6875 2.375zm7.9609375 4.15625l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm10.75 -1.03125q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm3.6015625 4.171875l0 -8.296875l1.265625 0l0 1.171875q0.90625 -1.359375 2.640625 -1.359375q0.75 0 1.375 0.265625q0.625 0.265625 0.9375 0.703125q0.3125 0.4375 0.4375 1.046875q0.078125 0.390625 0.078125 1.359375l0 5.109375l-1.40625 0l0 -5.046875q0 -0.859375 -0.171875 -1.28125q-0.15625 -0.4375 -0.578125 -0.6875q-0.40625 -0.25 -0.96875 -0.25q-0.90625 0 -1.5625 0.578125q-0.640625 0.5625 -0.640625 2.15625l0 4.53125l-1.40625 0zm8.6328125 0.6875l1.375 0.203125q0.078125 0.640625 0.46875 0.921875q0.53125 0.390625 1.4375 0.390625q0.96875 0 1.5 -0.390625q0.53125 -0.390625 0.71875 -1.09375q0.109375 -0.421875 0.109375 -1.8125q-0.921875 1.09375 -2.296875 1.09375q-1.71875 0 -2.65625 -1.234375q-0.9375 -1.234375 -0.9375 -2.96875q0 -1.1875 0.421875 -2.1875q0.4375 -1.0 1.25 -1.546875q0.828125 -0.546875 1.921875 -0.546875q1.46875 0 2.421875 1.1875l0 -1.0l1.296875 0l0 7.171875q0 1.9375 -0.390625 2.75q-0.390625 0.8125 -1.25 1.28125q-0.859375 0.46875 -2.109375 0.46875q-1.484375 0 -2.40625 -0.671875q-0.90625 -0.671875 -0.875 -2.015625zm1.171875 -4.984375q0 1.625 0.640625 2.375q0.65625 0.75 1.625 0.75q0.96875 0 1.625 -0.734375q0.65625 -0.75 0.65625 -2.34375q0 -1.53125 -0.671875 -2.296875q-0.671875 -0.78125 -1.625 -0.78125q-0.9375 0 -1.59375 0.765625q-0.65625 0.765625 -0.65625 2.265625zm13.6640625 1.625l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm11.71875 2.46875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm14.000008 2.484375l0 -1.21875q-0.96875 1.40625 -2.640625 1.40625q-0.734375 0 -1.375 -0.28125q-0.6250076 -0.28125 -0.9375076 -0.703125q-0.3125 -0.4375 -0.4375 -1.046875q-0.078125 -0.421875 -0.078125 -1.3125l0 -5.140625l1.4062576 0l0 4.59375q0 1.109375 0.078125 1.484375q0.140625 0.5625 0.5625 0.875q0.4375 0.3125 1.0625 0.3125q0.640625 0 1.1875 -0.3125q0.5625 -0.328125 0.78125 -0.890625q0.234375 -0.5625 0.234375 -1.625l0 -4.4375l1.40625 0l0 8.296875l-1.25 0zm4.7578125 0l-1.3125 0l0 -11.453125l1.40625 0l0 4.078125q0.890625 -1.109375 2.28125 -1.109375q0.765625 0 1.4375 0.3125q0.6875 0.296875 1.125 0.859375q0.453125 0.5625 0.703125 1.359375q0.25 0.78125 0.25 1.671875q0 2.140625 -1.0625 3.3125q-1.046875 1.15625 -2.53125 1.15625q-1.46875 0 -2.296875 -1.234375l0 1.046875zm-0.015625 -4.21875q0 1.5 0.40625 2.15625q0.65625 1.09375 1.796875 1.09375q0.921875 0 1.59375 -0.796875q0.671875 -0.8125 0.671875 -2.390625q0 -1.625 -0.65625 -2.390625q-0.640625 -0.78125 -1.546875 -0.78125q-0.921875 0 -1.59375 0.796875q-0.671875 0.796875 -0.671875 2.3125zm10.6796875 2.953125l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3671875 1.265625l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.5078125 2.28125l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.578125 4.953125l-1.3125 0l0 -11.453125l1.40625 0l0 4.078125q0.890625 -1.109375 2.28125 -1.109375q0.765625 0 1.4375 0.3125q0.6875 0.296875 1.125 0.859375q0.453125 0.5625 0.703125 1.359375q0.25 0.78125 0.25 1.671875q0 2.140625 -1.0625 3.3125q-1.046875 1.15625 -2.53125 1.15625q-1.46875 0 -2.296875 -1.234375l0 1.046875zm-0.015625 -4.21875q0 1.5 0.40625 2.15625q0.65625 1.09375 1.796875 1.09375q0.921875 0 1.59375 -0.796875q0.671875 -0.8125 0.671875 -2.390625q0 -1.625 -0.65625 -2.390625q-0.640625 -0.78125 -1.546875 -0.78125q-0.921875 0 -1.59375 0.796875q-0.671875 0.796875 -0.671875 2.3125zm7.5546875 7.421875l-0.15625 -1.328125q0.453125 0.125 0.796875 0.125q0.46875 0 0.75 -0.15625q0.28125 -0.15625 0.46875 -0.4375q0.125 -0.203125 0.421875 -1.046875q0.046875 -0.109375 0.125 -0.34375l-3.140625 -8.3125l1.515625 0l1.71875 4.796875q0.34375 0.921875 0.609375 1.921875q0.234375 -0.96875 0.578125 -1.890625l1.765625 -4.828125l1.40625 0l-3.15625 8.4375q-0.5 1.375 -0.78125 1.890625q-0.375 0.6875 -0.859375 1.015625q-0.484375 0.328125 -1.15625 0.328125q-0.40625 0 -0.90625 -0.171875zm12.6328125 -3.203125l0 -11.453125l2.28125 0l2.71875 8.109375q0.375 1.125 0.546875 1.6875q0.1875 -0.625 0.609375 -1.828125l2.734375 -7.96875l2.046875 0l0 11.453125l-1.46875 0l0 -9.59375l-3.328125 9.59375l-1.359375 0l-3.3125 -9.75l0 9.75l-1.46875 0zm13.375 0l0 -11.453125l3.953125 0q1.328125 0 2.03125 0.15625q0.984375 0.234375 1.6875 0.828125q0.90625 0.765625 1.34375 1.953125q0.453125 1.1875 0.453125 2.71875q0 1.3125 -0.3125 2.328125q-0.296875 1.0 -0.78125 1.65625q-0.46875 0.65625 -1.03125 1.046875q-0.5625 0.375 -1.375 0.578125q-0.796875 0.1875 -1.828125 0.1875l-4.140625 0zm1.515625 -1.359375l2.453125 0q1.125 0 1.765625 -0.203125q0.65625 -0.21875 1.03125 -0.59375q0.546875 -0.546875 0.84375 -1.453125q0.296875 -0.90625 0.296875 -2.203125q0 -1.796875 -0.59375 -2.765625q-0.578125 -0.96875 -1.421875 -1.296875q-0.609375 -0.234375 -1.96875 -0.234375l-2.40625 0l0 8.75zm9.5234375 -2.328125l1.4375 -0.125q0.09375 0.859375 0.46875 1.421875q0.375 0.546875 1.15625 0.890625q0.78125 0.328125 1.75 0.328125q0.875 0 1.53125 -0.25q0.671875 -0.265625 0.984375 -0.703125q0.328125 -0.453125 0.328125 -0.984375q0 -0.546875 -0.3125 -0.9375q-0.3125 -0.40625 -1.03125 -0.6875q-0.453125 -0.171875 -2.03125 -0.546875q-1.578125 -0.390625 -2.21875 -0.71875q-0.8125 -0.4375 -1.21875 -1.0625q-0.40625 -0.640625 -0.40625 -1.4375q0 -0.859375 0.484375 -1.609375q0.5 -0.765625 1.4375 -1.15625q0.953125 -0.390625 2.109375 -0.390625q1.28125 0 2.25 0.421875q0.96875 0.40625 1.484375 1.203125q0.53125 0.796875 0.578125 1.796875l-1.453125 0.109375q-0.125 -1.078125 -0.796875 -1.625q-0.671875 -0.5625 -2.0 -0.5625q-1.375 0 -2.0 0.5q-0.625 0.5 -0.625 1.21875q0 0.609375 0.4375 1.015625q0.4375 0.390625 2.28125 0.8125q1.859375 0.421875 2.546875 0.734375q1.0 0.453125 1.46875 1.171875q0.484375 0.703125 0.484375 1.625q0 0.90625 -0.53125 1.71875q-0.515625 0.8125 -1.5 1.265625q-0.984375 0.453125 -2.203125 0.453125q-1.5625 0 -2.609375 -0.453125q-1.046875 -0.46875 -1.65625 -1.375q-0.59375 -0.90625 -0.625 -2.0625zm22.460938 2.328125l0 1.359375l-7.578125 0q-0.015625 -0.515625 0.171875 -0.984375q0.28125 -0.765625 0.921875 -1.515625q0.640625 -0.75 1.84375 -1.734375q1.859375 -1.53125 2.515625 -2.421875q0.65625 -0.90625 0.65625 -1.703125q0 -0.828125 -0.59375 -1.40625q-0.59375 -0.578125 -1.5625 -0.578125q-1.015625 0 -1.625 0.609375q-0.609375 0.609375 -0.609375 1.6875l-1.453125 -0.140625q0.15625 -1.625 1.125 -2.46875q0.96875 -0.84375 2.59375 -0.84375q1.65625 0 2.609375 0.921875q0.96875 0.90625 0.96875 2.25q0 0.6875 -0.28125 1.359375q-0.28125 0.65625 -0.9375 1.390625q-0.65625 0.734375 -2.171875 2.015625q-1.265625 1.0625 -1.625 1.453125q-0.359375 0.375 -0.59375 0.75l5.625 0zm2.2578125 1.359375l0 -1.609375l1.609375 0l0 1.609375q0 0.890625 -0.3125 1.421875q-0.3125 0.546875 -1.0 0.84375l-0.390625 -0.59375q0.453125 -0.203125 0.65625 -0.578125q0.21875 -0.375 0.234375 -1.09375l-0.796875 0zm11.59375 -1.265625l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3828125 1.265625l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm14.5703125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm12.265625 4.953125l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.2109375 4.953125l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm11.84375 1.65625l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm14.0 2.484375l0 -1.21875q-0.96875 1.40625 -2.640625 1.40625q-0.734375 0 -1.375 -0.28125q-0.625 -0.28125 -0.9375 -0.703125q-0.3125 -0.4375 -0.4375 -1.046875q-0.078125 -0.421875 -0.078125 -1.3125l0 -5.140625l1.40625 0l0 4.59375q0 1.109375 0.078125 1.484375q0.140625 0.5625 0.5625 0.875q0.4375 0.3125 1.0625 0.3125q0.640625 0 1.1875 -0.3125q0.5625 -0.328125 0.78125 -0.890625q0.234375 -0.5625 0.234375 -1.625l0 -4.4375l1.40625 0l0 8.296875l-1.25 0zm4.7578125 0l-1.3125 0l0 -11.453125l1.40625 0l0 4.078125q0.890625 -1.109375 2.28125 -1.109375q0.765625 0 1.4375 0.3125q0.6875 0.296875 1.125 0.859375q0.453125 0.5625 0.703125 1.359375q0.25 0.78125 0.25 1.671875q0 2.140625 -1.0625 3.3125q-1.046875 1.15625 -2.53125 1.15625q-1.46875 0 -2.296875 -1.234375l0 1.046875zm-0.015625 -4.21875q0 1.5 0.40625 2.15625q0.65625 1.09375 1.796875 1.09375q0.921875 0 1.59375 -0.796875q0.671875 -0.8125 0.671875 -2.390625q0 -1.625 -0.65625 -2.390625q-0.640625 -0.78125 -1.546875 -0.78125q-0.921875 0 -1.59375 0.796875q-0.671875 0.796875 -0.671875 2.3125zm10.6796875 2.953125l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3671875 1.265625l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.5078125 2.28125l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.578125 4.953125l-1.3125 0l0 -11.453125l1.40625 0l0 4.078125q0.890625 -1.109375 2.28125 -1.109375q0.765625 0 1.4375 0.3125q0.6875 0.296875 1.125 0.859375q0.453125 0.5625 0.703125 1.359375q0.25 0.78125 0.25 1.671875q0 2.140625 -1.0625 3.3125q-1.046875 1.15625 -2.53125 1.15625q-1.46875 0 -2.296875 -1.234375l0 1.046875zm-0.015625 -4.21875q0 1.5 0.40625 2.15625q0.65625 1.09375 1.796875 1.09375q0.921875 0 1.59375 -0.796875q0.671875 -0.8125 0.671875 -2.390625q0 -1.625 -0.65625 -2.390625q-0.640625 -0.78125 -1.546875 -0.78125q-0.921875 0 -1.59375 0.796875q-0.671875 0.796875 -0.671875 2.3125zm7.5546875 7.421875l-0.15625 -1.328125q0.453125 0.125 0.796875 0.125q0.46875 0 0.75 -0.15625q0.28125 -0.15625 0.46875 -0.4375q0.125 -0.203125 0.421875 -1.046875q0.046875 -0.109375 0.125 -0.34375l-3.140625 -8.3125l1.515625 0l1.71875 4.796875q0.34375 0.921875 0.609375 1.921875q0.234375 -0.96875 0.578125 -1.890625l1.765625 -4.828125l1.40625 0l-3.15625 8.4375q-0.5 1.375 -0.78125 1.890625q-0.375 0.6875 -0.859375 1.015625q-0.484375 0.328125 -1.15625 0.328125q-0.40625 0 -0.90625 -0.171875zm12.6328125 -3.203125l0 -11.453125l2.28125 0l2.71875 8.109375q0.375 1.125 0.546875 1.6875q0.1875 -0.625 0.609375 -1.828125l2.734375 -7.96875l2.046875 0l0 11.453125l-1.46875 0l0 -9.59375l-3.328125 9.59375l-1.359375 0l-3.3125 -9.75l0 9.75l-1.46875 0zm13.375 0l0 -11.453125l3.953125 0q1.328125 0 2.03125 0.15625q0.984375 0.234375 1.6875 0.828125q0.90625 0.765625 1.34375 1.953125q0.453125 1.1875 0.453125 2.71875q0 1.3125 -0.3125 2.328125q-0.296875 1.0 -0.78125 1.65625q-0.46875 0.65625 -1.03125 1.046875q-0.5625 0.375 -1.375 0.578125q-0.796875 0.1875 -1.828125 0.1875l-4.140625 0zm1.515625 -1.359375l2.453125 0q1.125 0 1.765625 -0.203125q0.65625 -0.21875 1.03125 -0.59375q0.546875 -0.546875 0.84375 -1.453125q0.296875 -0.90625 0.296875 -2.203125q0 -1.796875 -0.59375 -2.765625q-0.578125 -0.96875 -1.421875 -1.296875q-0.609375 -0.234375 -1.96875 -0.234375l-2.40625 0l0 8.75zm9.5234375 -2.328125l1.4375 -0.125q0.09375 0.859375 0.46875 1.421875q0.375 0.546875 1.15625 0.890625q0.78125 0.328125 1.75 0.328125q0.875 0 1.53125 -0.25q0.671875 -0.265625 0.984375 -0.703125q0.328125 -0.453125 0.328125 -0.984375q0 -0.546875 -0.3125 -0.9375q-0.3125 -0.40625 -1.03125 -0.6875q-0.453125 -0.171875 -2.03125 -0.546875q-1.578125 -0.390625 -2.21875 -0.71875q-0.8125 -0.4375 -1.21875 -1.0625q-0.40625 -0.640625 -0.40625 -1.4375q0 -0.859375 0.484375 -1.609375q0.5 -0.765625 1.4375 -1.15625q0.953125 -0.390625 2.109375 -0.390625q1.28125 0 2.25 0.421875q0.96875 0.40625 1.484375 1.203125q0.53125 0.796875 0.578125 1.796875l-1.453125 0.109375q-0.125 -1.078125 -0.796875 -1.625q-0.671875 -0.5625 -2.0 -0.5625q-1.375 0 -2.0 0.5q-0.625 0.5 -0.625 1.21875q0 0.609375 0.4375 1.015625q0.4375 0.390625 2.28125 0.8125q1.859375 0.421875 2.546875 0.734375q1.0 0.453125 1.46875 1.171875q0.484375 0.703125 0.484375 1.625q0 0.90625 -0.53125 1.71875q-0.515625 0.8125 -1.5 1.265625q-0.984375 0.453125 -2.203125 0.453125q-1.5625 0 -2.609375 -0.453125q-1.046875 -0.46875 -1.65625 -1.375q-0.59375 -0.90625 -0.625 -2.0625zm15.0703125 0.65625l1.40625 -0.1875q0.25 1.203125 0.828125 1.734375q0.578125 0.515625 1.421875 0.515625q0.984375 0 1.671875 -0.6875q0.6875 -0.6875 0.6875 -1.703125q0 -0.96875 -0.640625 -1.59375q-0.625 -0.625 -1.609375 -0.625q-0.390625 0 -0.984375 0.15625l0.15625 -1.234375q0.140625 0.015625 0.21875 0.015625q0.90625 0 1.625 -0.46875q0.71875 -0.46875 0.71875 -1.453125q0 -0.765625 -0.53125 -1.265625q-0.515625 -0.515625 -1.34375 -0.515625q-0.828125 0 -1.375 0.515625q-0.546875 0.515625 -0.703125 1.546875l-1.40625 -0.25q0.265625 -1.421875 1.171875 -2.1875q0.921875 -0.78125 2.28125 -0.78125q0.9375 0 1.71875 0.40625q0.796875 0.390625 1.203125 1.09375q0.421875 0.6875 0.421875 1.46875q0 0.75 -0.40625 1.359375q-0.390625 0.609375 -1.171875 0.96875q1.015625 0.234375 1.578125 0.96875q0.5625 0.734375 0.5625 1.84375q0 1.5 -1.09375 2.546875q-1.09375 1.046875 -2.765625 1.046875q-1.5 0 -2.5 -0.890625q-1.0 -0.90625 -1.140625 -2.34375zm19.140625 2.0q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm3.6015625 4.171875l0 -8.296875l1.265625 0l0 1.171875q0.90625 -1.359375 2.640625 -1.359375q0.75 0 1.375 0.265625q0.625 0.265625 0.9375 0.703125q0.3125 0.4375 0.4375 1.046875q0.078125 0.390625 0.078125 1.359375l0 5.109375l-1.40625 0l0 -5.046875q0 -0.859375 -0.171875 -1.28125q-0.15625 -0.4375 -0.578125 -0.6875q-0.40625 -0.25 -0.96875 -0.25q-0.90625 0 -1.5625 0.578125q-0.640625 0.5625 -0.640625 2.15625l0 4.53125l-1.40625 0zm14.2734375 0l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm15.46875 2.875l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3828125 1.265625l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm14.5703125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm17.65625 4.953125l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm13.3671875 3.109375q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm3.5859375 4.171875l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm5.34375 0l0 -11.453125l1.40625 0l0 6.53125l3.328125 -3.375l1.828125 0l-3.171875 3.078125l3.484375 5.21875l-1.734375 0l-2.734375 -4.25l-1.0 0.953125l0 3.296875l-1.40625 0zm13.7421875 0l-1.3125 0l0 -11.453125l1.40625 0l0 4.078125q0.890625 -1.109375 2.28125 -1.109375q0.765625 0 1.4375 0.3125q0.6875 0.296875 1.125 0.859375q0.453125 0.5625 0.703125 1.359375q0.25 0.78125 0.25 1.671875q0 2.140625 -1.0625 3.3125q-1.046875 1.15625 -2.53125 1.15625q-1.46875 0 -2.296875 -1.234375l0 1.046875zm-0.015625 -4.21875q0 1.5 0.40625 2.15625q0.65625 1.09375 1.796875 1.09375q0.921875 0 1.59375 -0.796875q0.671875 -0.8125 0.671875 -2.390625q0 -1.625 -0.65625 -2.390625q-0.640625 -0.78125 -1.546875 -0.78125q-0.921875 0 -1.59375 0.796875q-0.671875 0.796875 -0.671875 2.3125zm7.5859375 4.21875l0 -11.453125l1.40625 0l0 11.453125l-1.40625 0zm9.0234375 0l0 -1.21875q-0.96875 1.40625 -2.640625 1.40625q-0.734375 0 -1.375 -0.28125q-0.625 -0.28125 -0.9375 -0.703125q-0.3125 -0.4375 -0.4375 -1.046875q-0.078125 -0.421875 -0.078125 -1.3125l0 -5.140625l1.40625 0l0 4.59375q0 1.109375 0.078125 1.484375q0.140625 0.5625 0.5625 0.875q0.4375 0.3125 1.0625 0.3125q0.640625 0 1.1875 -0.3125q0.5625 -0.328125 0.78125 -0.890625q0.234375 -0.5625 0.234375 -1.625l0 -4.4375l1.40625 0l0 8.296875l-1.25 0zm9.1328125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm11.71875 2.46875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm14.0 2.484375l0 -1.21875q-0.96875 1.40625 -2.640625 1.40625q-0.734375 0 -1.375 -0.28125q-0.625 -0.28125 -0.9375 -0.703125q-0.3125 -0.4375 -0.4375 -1.046875q-0.078125 -0.421875 -0.078125 -1.3125l0 -5.140625l1.40625 0l0 4.59375q0 1.109375 0.078125 1.484375q0.140625 0.5625 0.5625 0.875q0.4375 0.3125 1.0625 0.3125q0.640625 0 1.1875 -0.3125q0.5625 -0.328125 0.78125 -0.890625q0.234375 -0.5625 0.234375 -1.625l0 -4.4375l1.40625 0l0 8.296875l-1.25 0zm4.7578125 0l-1.3125 0l0 -11.453125l1.40625 0l0 4.078125q0.890625 -1.109375 2.28125 -1.109375q0.765625 0 1.4375 0.3125q0.6875 0.296875 1.125 0.859375q0.453125 0.5625 0.703125 1.359375q0.25 0.78125 0.25 1.671875q0 2.140625 -1.0625 3.3125q-1.046875 1.15625 -2.53125 1.15625q-1.46875 0 -2.296875 -1.234375l0 1.046875zm-0.015625 -4.21875q0 1.5 0.40625 2.15625q0.65625 1.09375 1.796875 1.09375q0.921875 0 1.59375 -0.796875q0.671875 -0.8125 0.671875 -2.390625q0 -1.625 -0.65625 -2.390625q-0.640625 -0.78125 -1.546875 -0.78125q-0.921875 0 -1.59375 0.796875q-0.671875 0.796875 -0.671875 2.3125zm10.6796875 2.953125l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm1.3671875 1.265625l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.5078125 2.28125l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm10.5234375 8.328125q-1.171875 -1.46875 -1.984375 -3.4375q-0.796875 -1.984375 -0.796875 -4.09375q0 -1.859375 0.609375 -3.5625q0.703125 -1.96875 2.171875 -3.9375l1.0 0q-0.9375 1.625 -1.25 2.328125q-0.46875 1.078125 -0.75 2.25q-0.328125 1.453125 -0.328125 2.9375q0 3.75 2.328125 7.515625l-1.0 0zm8.6171875 -3.375l-2.546875 -8.296875l1.453125 0l1.328125 4.78125l0.484375 1.78125q0.03125 -0.125 0.4375 -1.703125l1.3125 -4.859375l1.453125 0l1.234375 4.8125l0.421875 1.578125l0.46875 -1.59375l1.421875 -4.796875l1.375 0l-2.59375 8.296875l-1.46875 0l-1.3125 -4.96875l-0.328125 -1.421875l-1.671875 6.390625l-1.46875 0zm10.0234375 0l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm8.8984375 -9.84375l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm8.9609375 -3.046875l1.390625 0.1875q-0.234375 1.421875 -1.171875 2.234375q-0.921875 0.8125 -2.28125 0.8125q-1.703125 0 -2.75 -1.109375q-1.03125 -1.125 -1.03125 -3.203125q0 -1.34375 0.4375 -2.34375q0.453125 -1.015625 1.359375 -1.515625q0.921875 -0.5 1.984375 -0.5q1.359375 0 2.21875 0.6875q0.859375 0.671875 1.09375 1.9375l-1.359375 0.203125q-0.203125 -0.828125 -0.703125 -1.25q-0.484375 -0.421875 -1.1875 -0.421875q-1.0625 0 -1.734375 0.765625q-0.65625 0.75 -0.65625 2.40625q0 1.671875 0.640625 2.4375q0.640625 0.75 1.671875 0.75q0.828125 0 1.375 -0.5q0.5625 -0.515625 0.703125 -1.578125zm2.59375 3.046875l0 -11.453125l1.40625 0l0 4.109375q0.984375 -1.140625 2.484375 -1.140625q0.921875 0 1.59375 0.359375q0.6875 0.359375 0.96875 1.0q0.296875 0.640625 0.296875 1.859375l0 5.265625l-1.40625 0l0 -5.265625q0 -1.046875 -0.453125 -1.53125q-0.453125 -0.484375 -1.296875 -0.484375q-0.625 0 -1.171875 0.328125q-0.546875 0.328125 -0.78125 0.890625q-0.234375 0.546875 -0.234375 1.515625l0 4.546875l-1.40625 0zm13.34375 -9.84375l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm2.9921875 -2.484375l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm12.9921875 -7.34375l0 -1.625l1.40625 0l0 1.625l-1.40625 0zm-1.78125 13.046875l0.265625 -1.1875q0.421875 0.109375 0.671875 0.109375q0.421875 0 0.625 -0.296875q0.21875 -0.28125 0.21875 -1.421875l0 -8.71875l1.40625 0l0 8.75q0 1.53125 -0.390625 2.140625q-0.515625 0.78125 -1.6875 0.78125q-0.578125 0 -1.109375 -0.15625zm10.7890625 -3.21875l0 -1.21875q-0.96875 1.40625 -2.640625 1.40625q-0.734375 0 -1.375 -0.28125q-0.625 -0.28125 -0.9375 -0.703125q-0.3125 -0.4375 -0.4375 -1.046875q-0.078125 -0.421875 -0.078125 -1.3125l0 -5.140625l1.40625 0l0 4.59375q0 1.109375 0.078125 1.484375q0.140625 0.5625 0.5625 0.875q0.4375 0.3125 1.0625 0.3125q0.640625 0 1.1875 -0.3125q0.5625 -0.328125 0.78125 -0.890625q0.234375 -0.5625 0.234375 -1.625l0 -4.4375l1.40625 0l0 8.296875l-1.25 0zm2.8984375 -2.484375l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm11.625 1.21875l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm11.234375 0.234375q-0.78125 0.671875 -1.5 0.953125q-0.71875 0.265625 -1.546875 0.265625q-1.375 0 -2.109375 -0.671875q-0.734375 -0.671875 -0.734375 -1.703125q0 -0.609375 0.28125 -1.109375q0.28125 -0.515625 0.71875 -0.8125q0.453125 -0.3125 1.015625 -0.46875q0.421875 -0.109375 1.25 -0.203125q1.703125 -0.203125 2.515625 -0.484375q0 -0.296875 0 -0.375q0 -0.859375 -0.390625 -1.203125q-0.546875 -0.484375 -1.609375 -0.484375q-0.984375 0 -1.46875 0.359375q-0.46875 0.34375 -0.6875 1.21875l-1.375 -0.1875q0.1875 -0.875 0.609375 -1.421875q0.4375 -0.546875 1.25 -0.828125q0.8125 -0.296875 1.875 -0.296875q1.0625 0 1.71875 0.25q0.671875 0.25 0.984375 0.625q0.3125 0.375 0.4375 0.953125q0.078125 0.359375 0.078125 1.296875l0 1.875q0 1.96875 0.078125 2.484375q0.09375 0.515625 0.359375 1.0l-1.46875 0q-0.21875 -0.4375 -0.28125 -1.03125zm-0.109375 -3.140625q-0.765625 0.3125 -2.296875 0.53125q-0.875 0.125 -1.234375 0.28125q-0.359375 0.15625 -0.5625 0.46875q-0.1875 0.296875 -0.1875 0.65625q0 0.5625 0.421875 0.9375q0.4375 0.375 1.25 0.375q0.8125 0 1.4375 -0.34375q0.640625 -0.359375 0.9375 -0.984375q0.234375 -0.46875 0.234375 -1.40625l0 -0.515625zm7.484375 1.6875l1.390625 -0.21875q0.109375 0.84375 0.640625 1.296875q0.546875 0.4375 1.5 0.4375q0.96875 0 1.4375 -0.390625q0.46875 -0.40625 0.46875 -0.9375q0 -0.46875 -0.40625 -0.75q-0.296875 -0.1875 -1.4375 -0.46875q-1.546875 -0.390625 -2.15625 -0.671875q-0.59375 -0.296875 -0.90625 -0.796875q-0.296875 -0.5 -0.296875 -1.109375q0 -0.5625 0.25 -1.03125q0.25 -0.46875 0.6875 -0.78125q0.328125 -0.25 0.890625 -0.40625q0.578125 -0.171875 1.21875 -0.171875q0.984375 0 1.71875 0.28125q0.734375 0.28125 1.078125 0.765625q0.359375 0.46875 0.5 1.28125l-1.375 0.1875q-0.09375 -0.640625 -0.546875 -1.0q-0.453125 -0.359375 -1.265625 -0.359375q-0.96875 0 -1.390625 0.328125q-0.40625 0.3125 -0.40625 0.734375q0 0.28125 0.171875 0.5q0.171875 0.21875 0.53125 0.375q0.21875 0.078125 1.25 0.359375q1.484375 0.390625 2.078125 0.65625q0.59375 0.25 0.921875 0.734375q0.34375 0.484375 0.34375 1.203125q0 0.703125 -0.421875 1.328125q-0.40625 0.609375 -1.1875 0.953125q-0.765625 0.34375 -1.734375 0.34375q-1.625 0 -2.46875 -0.671875q-0.84375 -0.671875 -1.078125 -2.0zm8.5625 -7.359375l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm3.5546875 0l0 -8.296875l1.265625 0l0 1.171875q0.90625 -1.359375 2.640625 -1.359375q0.75 0 1.375 0.265625q0.625 0.265625 0.9375 0.703125q0.3125 0.4375 0.4375 1.046875q0.078125 0.390625 0.078125 1.359375l0 5.109375l-1.40625 0l0 -5.046875q0 -0.859375 -0.171875 -1.28125q-0.15625 -0.4375 -0.578125 -0.6875q-0.40625 -0.25 -0.96875 -0.25q-0.90625 0 -1.5625 0.578125q-0.640625 0.5625 -0.640625 2.15625l0 4.53125l-1.40625 0zm8.6328125 0.6875l1.375 0.203125q0.078125 0.640625 0.46875 0.921875q0.53125 0.390625 1.4375 0.390625q0.96875 0 1.5 -0.390625q0.53125 -0.390625 0.71875 -1.09375q0.109375 -0.421875 0.109375 -1.8125q-0.921875 1.09375 -2.296875 1.09375q-1.71875 0 -2.65625 -1.234375q-0.9375 -1.234375 -0.9375 -2.96875q0 -1.1875 0.421875 -2.1875q0.4375 -1.0 1.25 -1.546875q0.828125 -0.546875 1.921875 -0.546875q1.46875 0 2.421875 1.1875l0 -1.0l1.296875 0l0 7.171875q0 1.9375 -0.390625 2.75q-0.390625 0.8125 -1.25 1.28125q-0.859375 0.46875 -2.109375 0.46875q-1.484375 0 -2.40625 -0.671875q-0.90625 -0.671875 -0.875 -2.015625zm1.171875 -4.984375q0 1.625 0.640625 2.375q0.65625 0.75 1.625 0.75q0.96875 0 1.625 -0.734375q0.65625 -0.75 0.65625 -2.34375q0 -1.53125 -0.671875 -2.296875q-0.671875 -0.78125 -1.625 -0.78125q-0.9375 0 -1.59375 0.765625q-0.65625 0.765625 -0.65625 2.265625zm7.9609375 4.296875l0 -11.453125l1.40625 0l0 11.453125l-1.40625 0zm9.2578125 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm17.65625 4.953125l0 -1.046875q-0.78125 1.234375 -2.3125 1.234375q-1.0 0 -1.828125 -0.546875q-0.828125 -0.546875 -1.296875 -1.53125q-0.453125 -0.984375 -0.453125 -2.25q0 -1.25 0.40625 -2.25q0.421875 -1.015625 1.25 -1.546875q0.828125 -0.546875 1.859375 -0.546875q0.75 0 1.328125 0.3125q0.59375 0.3125 0.953125 0.828125l0 -4.109375l1.40625 0l0 11.453125l-1.3125 0zm-4.4375 -4.140625q0 1.59375 0.671875 2.390625q0.671875 0.78125 1.578125 0.78125q0.921875 0 1.5625 -0.75q0.65625 -0.765625 0.65625 -2.3125q0 -1.703125 -0.65625 -2.5q-0.65625 -0.796875 -1.625 -0.796875q-0.9375 0 -1.5625 0.765625q-0.625 0.765625 -0.625 2.421875zm7.9609375 -5.703125l0 -1.609375l1.40625 0l0 1.609375l-1.40625 0zm0 9.84375l0 -8.296875l1.40625 0l0 8.296875l-1.40625 0zm3.5390625 0l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm11.015625 -2.671875l1.453125 0.171875q-0.34375 1.28125 -1.28125 1.984375q-0.921875 0.703125 -2.359375 0.703125q-1.828125 0 -2.890625 -1.125q-1.0625 -1.125 -1.0625 -3.140625q0 -2.09375 1.078125 -3.25q1.078125 -1.15625 2.796875 -1.15625q1.65625 0 2.703125 1.140625q1.0625 1.125 1.0625 3.171875q0 0.125 0 0.375l-6.1875 0q0.078125 1.375 0.765625 2.109375q0.703125 0.71875 1.734375 0.71875q0.78125 0 1.328125 -0.40625q0.546875 -0.40625 0.859375 -1.296875zm-4.609375 -2.28125l4.625 0q-0.09375 -1.046875 -0.53125 -1.5625q-0.671875 -0.8125 -1.734375 -0.8125q-0.96875 0 -1.640625 0.65625q-0.65625 0.640625 -0.71875 1.71875zm13.2421875 1.90625l1.390625 0.1875q-0.234375 1.421875 -1.171875 2.234375q-0.921875 0.8125 -2.28125 0.8125q-1.703125 0 -2.75 -1.109375q-1.03125 -1.125 -1.03125 -3.203125q0 -1.34375 0.4375 -2.34375q0.453125 -1.015625 1.359375 -1.515625q0.921875 -0.5 1.984375 -0.5q1.359375 0 2.21875 0.6875q0.859375 0.671875 1.09375 1.9375l-1.359375 0.203125q-0.203125 -0.828125 -0.703125 -1.25q-0.484375 -0.421875 -1.1875 -0.421875q-1.0625 0 -1.734375 0.765625q-0.65625 0.75 -0.65625 2.40625q0 1.671875 0.640625 2.4375q0.640625 0.75 1.671875 0.75q0.828125 0 1.375 -0.5q0.5625 -0.515625 0.703125 -1.578125zm5.65625 1.78125l0.203125 1.25q-0.59375 0.125 -1.0625 0.125q-0.765625 0 -1.1875 -0.234375q-0.421875 -0.25 -0.59375 -0.640625q-0.171875 -0.40625 -0.171875 -1.671875l0 -4.765625l-1.03125 0l0 -1.09375l1.03125 0l0 -2.0625l1.40625 -0.84375l0 2.90625l1.40625 0l0 1.09375l-1.40625 0l0 4.84375q0 0.609375 0.0625 0.78125q0.078125 0.171875 0.25 0.28125q0.171875 0.09375 0.484375 0.09375q0.234375 0 0.609375 -0.0625zm0.8515625 -2.890625q0 -2.296875 1.28125 -3.40625q1.078125 -0.921875 2.609375 -0.921875q1.71875 0 2.796875 1.125q1.09375 1.109375 1.09375 3.09375q0 1.59375 -0.484375 2.515625q-0.484375 0.921875 -1.40625 1.4375q-0.90625 0.5 -2.0 0.5q-1.734375 0 -2.8125 -1.109375q-1.078125 -1.125 -1.078125 -3.234375zm1.453125 0q0 1.59375 0.6875 2.390625q0.703125 0.796875 1.75 0.796875q1.046875 0 1.734375 -0.796875q0.703125 -0.796875 0.703125 -2.4375q0 -1.53125 -0.703125 -2.328125q-0.6875 -0.796875 -1.734375 -0.796875q-1.046875 0 -1.75 0.796875q-0.6875 0.78125 -0.6875 2.375zm7.9609375 4.15625l0 -8.296875l1.265625 0l0 1.25q0.484375 -0.875 0.890625 -1.15625q0.40625 -0.28125 0.90625 -0.28125q0.703125 0 1.4375 0.453125l-0.484375 1.296875q-0.515625 -0.296875 -1.03125 -0.296875q-0.453125 0 -0.828125 0.28125q-0.359375 0.265625 -0.515625 0.765625q-0.234375 0.75 -0.234375 1.640625l0 4.34375l-1.40625 0zm5.28125 3.203125l-0.15625 -1.328125q0.453125 0.125 0.796875 0.125q0.46875 0 0.75 -0.15625q0.28125 -0.15625 0.46875 -0.4375q0.125 -0.203125 0.421875 -1.046875q0.046875 -0.109375 0.125 -0.34375l-3.140625 -8.3125l1.515625 0l1.71875 4.796875q0.34375 0.921875 0.609375 1.921875q0.234375 -0.96875 0.578125 -1.890625l1.765625 -4.828125l1.40625 0l-3.15625 8.4375q-0.5 1.375 -0.78125 1.890625q-0.375 0.6875 -0.859375 1.015625q-0.484375 0.328125 -1.15625 0.328125q-0.40625 0 -0.90625 -0.171875zm8.984375 0.171875l-1.015625 0q2.34375 -3.765625 2.34375 -7.515625q0 -1.46875 -0.34375 -2.921875q-0.265625 -1.171875 -0.734375 -2.25q-0.3125 -0.703125 -1.265625 -2.34375l1.015625 0q1.46875 1.96875 2.171875 3.9375q0.59375 1.703125 0.59375 3.5625q0 2.109375 -0.8125 4.09375q-0.796875 1.96875 -1.953125 3.4375zm10.1484375 -3.375l-1.3125 0l0 -11.453125l1.40625 0l0 4.078125q0.890625 -1.109375 2.28125 -1.109375q0.765625 0 1.4375 0.3125q0.6875 0.296875 1.125 0.859375q0.453125 0.5625 0.703125 1.359375q0.25 0.78125 0.25 1.671875q0 2.140625 -1.0625 3.3125q-1.046875 1.15625 -2.53125 1.15625q-1.46875 0 -2.296875 -1.234375l0 1.046875zm-0.015625 -4.21875q0 1.5 0.40625 2.15625q0.65625 1.09375 1.796875 1.09375q0.921875 0 1.59375 -0.796875q0.671875 -0.8125 0.671875 -2.390625q0 -1.625 -0.65625 -2.390625q-0.640625 -0.78125 -1.546875 -0.78125q-0.921875 0 -1.59375 0.796875q-0.671875 0.796875 -0.671875 2.3125zm7.5546875 7.421875l-0.15625 -1.328125q0.453125 0.125 0.796875 0.125q0.46875 0 0.75 -0.15625q0.28125 -0.15625 0.46875 -0.4375q0.125 -0.203125 0.421875 -1.046875q0.046875 -0.109375 0.125 -0.34375l-3.140625 -8.3125l1.515625 0l1.71875 4.796875q0.34375 0.921875 0.609375 1.921875q0.234375 -0.96875 0.578125 -1.890625l1.765625 -4.828125l1.40625 0l-3.15625 8.4375q-0.5 1.375 -0.78125 1.890625q-0.375 0.6875 -0.859375 1.015625q-0.484375 0.328125 -1.15625 0.328125q-0.40625 0 -0.90625 -0.171875z" fill-rule="nonzero"/><path fill="#000000" d="m38.116634 366.33994l0 -11.453125l2.28125 0l2.71875 8.109375q0.375 1.125 0.546875 1.6875q0.1875 -0.625 0.609375 -1.828125l2.734375 -7.96875l2.046875 0l0 11.453125l-1.46875 0l0 -9.59375l-3.328125 9.59375l-1.359375 0l-3.3125 -9.75l0 9.75l-1.46875 0zm13.375 0l0 -11.453125l3.953125 0q1.328125 0 2.03125 0.15625q0.984375 0.234375 1.6875 0.828125q0.90625 0.765625 1.34375 1.953125q0.453125 1.1875 0.453125 2.71875q0 1.3125 -0.3125 2.328125q-0.296875 1.0 -0.78125 1.65625q-0.46875 0.65625 -1.03125 1.046875q-0.5625 0.375 -1.375 0.578125q-0.796875 0.1875 -1.828125 0.1875l-4.140625 0zm1.515625 -1.359375l2.453125 0q1.125 0 1.765625 -0.203125q0.65625 -0.21875 1.03125 -0.59375q0.546875 -0.546875 0.84375 -1.453125q0.296875 -0.90625 0.296875 -2.203125q0 -1.796875 -0.59375 -2.765625q-0.578125 -0.96875 -1.421875 -1.296875q-0.609375 -0.234375 -1.96875 -0.234375l-2.40625 0l0 8.75zm9.5234375 -2.328125l1.4375 -0.125q0.093746185 0.859375 0.4687462 1.421875q0.375 0.546875 1.15625 0.890625q0.78125 0.328125 1.75 0.328125q0.875 0 1.53125 -0.25q0.671875 -0.265625 0.984375 -0.703125q0.328125 -0.453125 0.328125 -0.984375q0 -0.546875 -0.3125 -0.9375q-0.3125 -0.40625 -1.03125 -0.6875q-0.453125 -0.171875 -2.03125 -0.546875q-1.578125 -0.390625 -2.21875 -0.71875q-0.8124962 -0.4375 -1.2187462 -1.0625q-0.40625 -0.640625 -0.40625 -1.4375q0 -0.859375 0.484375 -1.609375q0.5 -0.765625 1.4374962 -1.15625q0.953125 -0.390625 2.109375 -0.390625q1.28125 0 2.25 0.421875q0.96875 0.40625 1.484375 1.203125q0.53125 0.796875 0.578125 1.796875l-1.453125 0.109375q-0.125 -1.078125 -0.796875 -1.625q-0.671875 -0.5625 -2.0 -0.5625q-1.375 0 -2.0 0.5q-0.625 0.5 -0.625 1.21875q0 0.609375 0.4375 1.015625q0.4375 0.390625 2.28125 0.8125q1.859375 0.421875 2.546875 0.734375q1.0 0.453125 1.46875 1.171875q0.484375 0.703125 0.484375 1.625q0 0.90625 -0.53125 1.71875q-0.515625 0.8125 -1.5 1.265625q-0.984375 0.453125 -2.203125 0.453125q-1.5625 0 -2.609375 -0.453125q-1.0468712 -0.46875 -1.6562462 -1.375q-0.59375 -0.90625 -0.625 -2.0625zm19.570309 3.6875l0 -2.75l-4.96875 0l0 -1.28125l5.234375 -7.421875l1.140625 0l0 7.421875l1.546875 0l0 1.28125l-1.546875 0l0 2.75l-1.40625 0zm0 -4.03125l0 -5.171875l-3.578125 5.171875l3.578125 0zm5.1796875 4.03125l0 -1.609375l1.609375 0l0 1.609375l-1.609375 0z" fill-rule="nonzero"/></g></svg>
\ No newline at end of file
diff --git a/doc/cephfs/troubleshooting.rst b/doc/cephfs/troubleshooting.rst
new file mode 100644
index 000000000..78ad18dde
--- /dev/null
+++ b/doc/cephfs/troubleshooting.rst
@@ -0,0 +1,210 @@
+=================
+ Troubleshooting
+=================
+
+Slow/stuck operations
+=====================
+
+If you are experiencing apparent hung operations, the first task is to identify
+where the problem is occurring: in the client, the MDS, or the network connecting
+them. Start by looking to see if either side has stuck operations
+(:ref:`slow_requests`, below), and narrow it down from there.
+
+We can get hints about what's going on by dumping the MDS cache ::
+
+  ceph daemon mds.<name> dump cache /tmp/dump.txt
+
+.. note:: The file `dump.txt` is on the machine executing the MDS and for systemd
+	  controlled MDS services, this is in a tmpfs in the MDS container.
+	  Use `nsenter(1)` to locate `dump.txt` or specify another system-wide path.
+
+If high logging levels are set on the MDS, that will almost certainly hold the
+information we need to diagnose and solve the issue.
+
+RADOS Health
+============
+
+If part of the CephFS metadata or data pools is unavailable and CephFS is not
+responding, it is probably because RADOS itself is unhealthy. Resolve those
+problems first (:doc:`../../rados/troubleshooting/index`).
+
+The MDS
+=======
+
+If an operation is hung inside the MDS, it will eventually show up in ``ceph health``,
+identifying "slow requests are blocked". It may also identify clients as
+"failing to respond" or misbehaving in other ways. If the MDS identifies
+specific clients as misbehaving, you should investigate why they are doing so.
+
+Generally it will be the result of
+
+#. Overloading the system (if you have extra RAM, increase the
+   "mds cache memory limit" config from its default 1GiB; having a larger active
+   file set than your MDS cache is the #1 cause of this!).
+
+#. Running an older (misbehaving) client.
+
+#. Underlying RADOS issues.
+
+Otherwise, you have probably discovered a new bug and should report it to
+the developers!
+
+.. _slow_requests:
+
+Slow requests (MDS)
+-------------------
+You can list current operations via the admin socket by running::
+
+  ceph daemon mds.<name> dump_ops_in_flight
+
+from the MDS host. Identify the stuck commands and examine why they are stuck.
+Usually the last "event" will have been an attempt to gather locks, or sending
+the operation off to the MDS log. If it is waiting on the OSDs, fix them. If
+operations are stuck on a specific inode, you probably have a client holding
+caps which prevent others from using it, either because the client is trying
+to flush out dirty data or because you have encountered a bug in CephFS'
+distributed file lock code (the file "capabilities" ["caps"] system).
+
+If it's a result of a bug in the capabilities code, restarting the MDS
+is likely to resolve the problem.
+
+If there are no slow requests reported on the MDS, and it is not reporting
+that clients are misbehaving, either the client has a problem or its
+requests are not reaching the MDS.
+
+.. _ceph_fuse_debugging:
+
+ceph-fuse debugging
+===================
+
+ceph-fuse also supports ``dump_ops_in_flight``. See if it has any and where they are
+stuck.
+
+Debug output
+------------
+
+To get more debugging information from ceph-fuse, try running in the foreground
+with logging to the console (``-d``) and enabling client debug
+(``--debug-client=20``), enabling prints for each message sent
+(``--debug-ms=1``).
+
+If you suspect a potential monitor issue, enable monitor debugging as well
+(``--debug-monc=20``).
+
+.. _kernel_mount_debugging:
+
+Kernel mount debugging
+======================
+
+If there is an issue with the kernel client, the most important thing is
+figuring out whether the problem is with the kernel client or the MDS. Generally,
+this is easy to work out. If the kernel client broke directly, there will be
+output in ``dmesg``. Collect it and any inappropriate kernel state.
+
+Slow requests
+-------------
+
+Unfortunately the kernel client does not support the admin socket, but it has
+similar (if limited) interfaces if your kernel has debugfs enabled. There
+will be a folder in ``sys/kernel/debug/ceph/``, and that folder (whose name will
+look something like ``28f7427e-5558-4ffd-ae1a-51ec3042759a.client25386880``)
+will contain a variety of files that output interesting output when you ``cat``
+them. These files are described below; the most interesting when debugging
+slow requests are probably the ``mdsc`` and ``osdc`` files.
+
+* bdi: BDI info about the Ceph system (blocks dirtied, written, etc)
+* caps: counts of file "caps" structures in-memory and used
+* client_options: dumps the options provided to the CephFS mount
+* dentry_lru: Dumps the CephFS dentries currently in-memory
+* mdsc: Dumps current requests to the MDS
+* mdsmap: Dumps the current MDSMap epoch and MDSes
+* mds_sessions: Dumps the current sessions to MDSes
+* monc: Dumps the current maps from the monitor, and any "subscriptions" held
+* monmap: Dumps the current monitor map epoch and monitors
+* osdc: Dumps the current ops in-flight to OSDs (ie, file data IO)
+* osdmap: Dumps the current OSDMap epoch, pools, and OSDs
+
+If there are no stuck requests but you have file IO which is not progressing,
+you might have a...
+
+Disconnected+Remounted FS
+=========================
+Because CephFS has a "consistent cache", if your network connection is
+disrupted for a long enough time, the client will be forcibly
+disconnected from the system. At this point, the kernel client is in
+a bind: it cannot safely write back dirty data, and many applications
+do not handle IO errors correctly on close().
+At the moment, the kernel client will remount the FS, but outstanding file system
+IO may or may not be satisfied. In these cases, you may need to reboot your
+client system.
+
+You can identify you are in this situation if dmesg/kern.log report something like::
+
+   Jul 20 08:14:38 teuthology kernel: [3677601.123718] ceph: mds0 closed our session
+   Jul 20 08:14:38 teuthology kernel: [3677601.128019] ceph: mds0 reconnect start
+   Jul 20 08:14:39 teuthology kernel: [3677602.093378] ceph: mds0 reconnect denied
+   Jul 20 08:14:39 teuthology kernel: [3677602.098525] ceph:  dropping dirty+flushing Fw state for ffff8802dc150518 1099935956631
+   Jul 20 08:14:39 teuthology kernel: [3677602.107145] ceph:  dropping dirty+flushing Fw state for ffff8801008e8518 1099935946707
+   Jul 20 08:14:39 teuthology kernel: [3677602.196747] libceph: mds0 172.21.5.114:6812 socket closed (con state OPEN)
+   Jul 20 08:14:40 teuthology kernel: [3677603.126214] libceph: mds0 172.21.5.114:6812 connection reset
+   Jul 20 08:14:40 teuthology kernel: [3677603.132176] libceph: reset on mds0
+
+This is an area of ongoing work to improve the behavior. Kernels will soon
+be reliably issuing error codes to in-progress IO, although your application(s)
+may not deal with them well. In the longer-term, we hope to allow reconnect
+and reclaim of data in cases where it won't violate POSIX semantics (generally,
+data which hasn't been accessed or modified by other clients).
+
+Mounting
+========
+
+Mount 5 Error
+-------------
+
+A mount 5 error typically occurs if a MDS server is laggy or if it crashed.
+Ensure at least one MDS is up and running, and the cluster is ``active +
+healthy``. 
+
+Mount 12 Error
+--------------
+
+A mount 12 error with ``cannot allocate memory`` usually occurs if you  have a
+version mismatch between the :term:`Ceph Client` version and the :term:`Ceph
+Storage Cluster` version. Check the versions using::
+
+	ceph -v
+	
+If the Ceph Client is behind the Ceph cluster, try to upgrade it::
+
+	sudo apt-get update && sudo apt-get install ceph-common 
+
+You may need to uninstall, autoclean and autoremove ``ceph-common`` 
+and then reinstall it so that you have the latest version.
+
+Dynamic Debugging
+=================
+
+You can enable dynamic debug against the CephFS module.
+
+Please see: https://github.com/ceph/ceph/blob/master/src/script/kcon_all.sh
+
+Reporting Issues
+================
+
+If you have identified a specific issue, please report it with as much
+information as possible. Especially important information:
+
+* Ceph versions installed on client and server
+* Whether you are using the kernel or fuse client
+* If you are using the kernel client, what kernel version?
+* How many clients are in play, doing what kind of workload?
+* If a system is 'stuck', is that affecting all clients or just one?
+* Any ceph health messages
+* Any backtraces in the ceph logs from crashes
+
+If you are satisfied that you have found a bug, please file it on `the bug
+tracker`. For more general queries, please write to the `ceph-users mailing
+list`.
+
+.. _the bug tracker: http://tracker.ceph.com
+.. _ceph-users mailing list:  http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com/
diff --git a/doc/cephfs/upgrading.rst b/doc/cephfs/upgrading.rst
new file mode 100644
index 000000000..2dc29e129
--- /dev/null
+++ b/doc/cephfs/upgrading.rst
@@ -0,0 +1,88 @@
+Upgrading the MDS Cluster
+=========================
+
+Currently the MDS cluster does not have built-in versioning or file system
+flags to support seamless upgrades of the MDSs without potentially causing
+assertions or other faults due to incompatible messages or other functional
+differences. For this reason, it's necessary during any cluster upgrade to
+reduce the number of active MDS for a file system to one first so that two
+active MDS do not communicate with different versions.
+
+The proper sequence for upgrading the MDS cluster is:
+
+1. For each file system, disable and stop standby-replay daemons.
+
+::
+
+    ceph fs set <fs_name> allow_standby_replay false
+
+In Pacific, the standby-replay daemons are stopped for you after running this
+command. Older versions of Ceph require you to stop these daemons manually.
+
+::
+
+    ceph fs dump # find standby-replay daemons
+    ceph mds fail mds.<X>
+
+
+2. For each file system, reduce the number of ranks to 1:
+
+::
+
+    ceph fs set <fs_name> max_mds 1
+
+3. Wait for cluster to stop non-zero ranks where only rank 0 is active and the rest are standbys.
+
+::
+
+    ceph status # wait for MDS to finish stopping
+
+4. For each MDS, upgrade packages and restart. Note: to reduce failovers, it is
+   recommended -- but not strictly necessary -- to first upgrade standby daemons.
+
+::
+
+    # use package manager to update cluster
+    systemctl restart ceph-mds.target
+
+5. For each file system, restore the previous max_mds and allow_standby_replay settings for your cluster:
+
+::
+
+    ceph fs set <fs_name> max_mds <old_max_mds>
+    ceph fs set <fs_name> allow_standby_replay <old_allow_standby_replay>
+
+
+Upgrading pre-Firefly file systems past Jewel
+=============================================
+
+.. tip::
+
+    This advice only applies to users with file systems
+    created using versions of Ceph older than *Firefly* (0.80).
+    Users creating new file systems may disregard this advice.
+
+Pre-firefly versions of Ceph used a now-deprecated format
+for storing CephFS directory objects, called TMAPs.  Support
+for reading these in RADOS will be removed after the Jewel
+release of Ceph, so for upgrading CephFS users it is important
+to ensure that any old directory objects have been converted.
+
+After installing Jewel on all your MDS and OSD servers, and restarting
+the services, run the following command:
+
+::
+    
+    cephfs-data-scan tmap_upgrade <metadata pool name>
+
+This only needs to be run once, and it is not necessary to
+stop any other services while it runs.  The command may take some
+time to execute, as it iterates overall objects in your metadata
+pool.  It is safe to continue using your file system as normal while
+it executes.  If the command aborts for any reason, it is safe
+to simply run it again.
+
+If you are upgrading a pre-Firefly CephFS file system to a newer Ceph version
+than Jewel, you must first upgrade to Jewel and run the ``tmap_upgrade``
+command before completing your upgrade to the latest version.
+
diff --git a/doc/changelog/v0.48.1argonaut.txt b/doc/changelog/v0.48.1argonaut.txt
new file mode 100644
index 000000000..cdd557f97
--- /dev/null
+++ b/doc/changelog/v0.48.1argonaut.txt
@@ -0,0 +1,1286 @@
+commit a7ad701b9bd479f20429f19e6fea7373ca6bba7c
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Aug 13 14:58:51 2012 -0700
+
+    v0.48.1argonaut
+
+commit d4849f2f8a8c213c266658467bc5f22763010bc2
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Aug 1 13:22:38 2012 -0700
+
+    rgw: fix usage trim call encoding
+    
+    Fixes: #2841.
+    Usage trim operation was encoding the wrong op structure (usage read).
+    Since the structures somewhat overlapped it somewhat worked, but user
+    info wasn't encoded.
+    
+    Backport: argonaut
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 515952d07107d442889754ec3bd6a344fad25d58
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Aug 8 15:21:53 2012 -0700
+
+    cls_rgw: fix rgw_cls_usage_log_trim_op encode/decode
+    
+    It was not encoding user, adding that and reset version
+    compatibility.
+    This changes affects command interface, makes use of
+    radosgw-admin usage trim incompatible. Use of old
+    radosgw-admin usage trim should be avoided, as it may
+    remove more data than requested. In any case, upgraded
+    server code will not handle old client's trim requests.
+    
+    backport: argonaut
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 2e77130d5c80220be1612b5499d422de620d2d0b
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Jul 31 16:17:22 2012 -0700
+
+    rgw: expand date format support
+    
+    Relaxing the date format parsing function to allow UTC
+    instead of GMT.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 14fa77d9277b5ef5d0c6683504b368773b39ccc4
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Aug 2 11:13:05 2012 -0700
+
+    rgw: complete multipart upload can handle chunked encoding
+    
+    Fixes: #2878
+    We now allow complete multipart upload to use chunked encoding
+    when sending request data. With chunked encoding the HTTP_LENGTH
+    header is not required.
+    
+    Backport: argonaut
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit a06f7783fbcc02e775fc36f30e422fe0f9e0ec2d
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Aug 1 11:19:32 2012 -0700
+
+    rgw_xml: xml_handle_data() appends data string
+    
+    Fixes: #2879.
+    xml_handle_data() appends data to the object instead of just
+    replacing it. Parsed data can arrive in pieces, specifically
+    when data is escaped.
+    
+    Backport: argonaut
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit a8b224b9c4877a559ce420a2e04f19f68c8c5680
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Aug 1 13:09:41 2012 -0700
+
+    rgw: ETag is unquoted in multipart upload complete
+    
+    Fixes #2877.
+    Removing quotes from ETag before comparing it to what we
+    have when completing a multipart upload.
+    
+    Backport: argonaut
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 22259c6efda9a5d55221fd036c757bf123796753
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Aug 8 15:24:57 2012 -0700
+
+    MonMap: return error on failure in build_initial
+    
+    If mon_host fails to parse, return an error instead of success.
+    This avoids failing later on an assert monmap.size() > 0 in the
+    monmap in MonClient.
+    
+    Fixes: #2913
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 49b2c7b5a79b8fb4a3941eca2cb0dbaf22f658b7
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Aug 8 15:10:27 2012 -0700
+
+    addr_parsing: report correct error message
+    
+    getaddrinfo uses its return code to report failures.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 7084f29544f431b7c6a3286356f2448ae0333eda
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Aug 8 14:01:53 2012 -0700
+
+    mkcephfs: use default osd_data, _journal values
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit 96b1a496cdfda34a5efdb6686becf0d2e7e3a1c0
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Aug 8 14:01:35 2012 -0700
+
+    mkcephfs: use new default keyring locations
+    
+    The ceph-conf command only parses the conf; it does not apply default
+    config values.  This breaks mkcephfs if values are not specified in the
+    config.
+    
+    Let ceph-osd create its own key, fix copying, and fix creation/copying for
+    the mds.
+    
+    Fixes: #2845
+    Reported-by: Florian Haas <florian@hastexo.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit 4bd466d6ed49c7192df4a5bf0d63bda5d7d7dd9a
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 31 14:01:57 2012 -0700
+
+    osd: peering: detect when log source osd goes down
+    
+    The Peering state has a generic check based on the prior set osds that
+    will restart peering if one of them goes down (or one of the interesting
+    down ones comes up).  The GetLog state, however, can pull the log from
+    a peer that is not in the prior set if it got a notify from them (e.g., an
+    osd in an old interval that was down when the prior set was calculated).
+    If that osd goes down, we don't detect it and will block forward.
+    
+    Fix by adding a simple check in GetLog for the newest_update_osd going
+    down.
+    
+    (BTW GetMissing does not suffer from this problem because
+    peer_missing_requested is a subset of the prior set, so the Peering check
+    is sufficient.)
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+
+commit 87defa88a0c6d6aafaa65437a6e4ddd92418f834
+Author: Sylvain Munaut <tnt@246tNt.com>
+Date:   Tue Jul 31 11:55:56 2012 -0700
+
+    rbd: fix off-by-one error in key name
+    
+    Fixes: #2846
+    Signed-off-by: Sylvain Munaut <tnt@246tNt.com>
+
+commit 37d5b46269c8a4227e5df61a88579d94f7b56772
+Author: Sylvain Munaut <tnt@246tNt.com>
+Date:   Tue Jul 31 11:54:29 2012 -0700
+
+    secret: return error on empty secret
+    
+    Signed-off-by: Sylvain Munaut <tnt@246tNt.com>
+
+commit 7b9d37c662313929b52011ddae47cc8abab99095
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jul 28 10:05:47 2012 -0700
+
+    osd: set STRAY on pg load when non-primary
+    
+    The STRAY bit indicates that we should annouce ourselves to the primary,
+    but it is only set in start_peering_interval().  We also need to set it
+    initially, so that a PG that is loaded but whose role does not change
+    (e.g., the stray replica stays a stray) will notify the primary.
+    
+    Observed:
+     - osd starts up
+     - mapping does not change, STRAY not set
+     - does not announce to primary
+     - primary does not re-check must_have_unfound, objects appear unfound
+    
+    Fix this by initializing STRAY when pg is loaded or created whenever we
+    are not the primary.
+    
+    Fixes: #2866
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 96feca450c5505a06868bc012fe998a03371b77f
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 27 16:03:26 2012 -0700
+
+    osd: peering: make Incomplete a Peering substate
+    
+    This allows us to still catch changes in the prior set that would affect
+    our conclusions (that we are incomplete) and, when they happen, restart
+    peering.
+    
+    Consider:
+     - calc prior set, osd A is down
+     - query everyone else, no good info
+     - set down, go to Incomplete (previously WaitActingChange) state.
+     - osd A comes back up (we do nothing)
+     - osd A sends notify message with good info (we ignore)
+    
+    By making this a Peering substate, we catch the Peering AdvMap reaction,
+    which will notice a prior set down osd is now up and move to Reset.
+    
+    Fixes: #2860
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit a71e442fe620fa3a22ad9302413d8344a3a1a969
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 27 15:39:40 2012 -0700
+
+    osd: peering: move to Incomplete when.. incomplete
+    
+    PG::choose_acting() may return false and *not* request an acting set change
+    if it can't find any suitable peers with enough info to recover.  In that
+    case, we should move to Incomplete, not WaitActingChange, just like we do
+    a bit lower in GetLog() if we have non-contiguous logs.  The state name is
+    more accurate, and this is also needed to fix bug #2860.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 623026d9bc8ea4c845eb3b06d79e0ca9bef50deb
+Merge: 87b6e80 9db7809
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 27 14:00:52 2012 -0700
+
+    Merge remote-tracking branch 'gh/stable' into stable-next
+
+commit 9db78090451e609e3520ac3e57a5f53da03f9ee2
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 26 16:35:00 2012 -0700
+
+    osd: fixing sharing of past_intervals on backfill restart
+    
+    We need to share past_intervals whenever we instantiate the PG on a peer.
+    In the PG activation case, this is based on whether our peer_info[] value
+    for that peer is dne().  However, the backfill code was updating the
+    peer info (history) in the block preceeding the dne() check, which meant
+    we never shared past_intervals in this case and the peer would have to
+    chew through a potentially large number of maps if the PG has not been
+    clean recently.
+    
+    Fix by checking dne() prior to the backfill block.  We still need to fill
+    in the message later because it isn't yet instantiated.
+    
+    Fixes: #2849
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 87b6e8045a3a1ff6439d2684e960ad0dc8988b33
+Merge: 81d72e5 7dfdf4f
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 26 15:04:12 2012 -0700
+
+    Merge remote-tracking branch 'gh/wip-rbd-bid' into stable-next
+
+commit 81d72e5d7ba4713eb7c290878d901e21c0709028
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 23 10:47:10 2012 -0700
+
+    mon: make 'ceph osd rm ...' wipe out all state bits, not just EXISTS
+    
+    This ensures that when a new osd reclaims that id it behaves as if it were
+    really new.
+    
+    Backport: argonaut
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit ad9c37f2c029f6eb372efb711b234014397057e9
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 9 20:54:19 2012 -0700
+
+    test_stress_watch: just one librados instance
+    
+    This was creating a new cluster connection/session per iteration, and
+    along with it a few service threads and sockets and so forth.
+    
+    Unfortunately, librados leaks like a sieve, starting with CephContext
+    and ceph::crypto::init().  See #845 and #2067.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit c60afe1842a48dd75944822c0872fce6a7229f5a
+Merge: 8833050 35b1326
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 26 15:03:50 2012 -0700
+
+    Merge commit '35b13266923f8095650f45562d66372e618c8824' into stable-next
+    
+    First batch of msgr fixes.
+
+commit 88330505cc772a5528e9405d515aa2b945b0819e
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Jul 9 15:53:31 2012 -0700
+
+    ReplicatedPG: fix replay op ordering
+    
+    After a client reconnect, the client replays outstanding ops.  The
+    OSD then immediately responds with success if the op has already
+    committed (version < ReplicatedPG::get_first_in_progress).
+    Otherwise, we stick it in waiting_for_ondisk to be replied to when
+    eval_repop concludes that waitfor_disk is empty.
+    
+    Fixes #2508
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    
+    Conflicts:
+    
+    	src/osd/ReplicatedPG.cc
+
+commit 682609a9343d0488788b1c6b03bc437b7905e4d6
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 18 12:55:35 2012 -0700
+
+    objecter: always resend linger registrations
+    
+    If a linger op (watch) is sent to the OSD and updates the object, and then
+    the client loses the reply, it will resend the request.  The OSD will see
+    that it is a dup, however, and not set up the in-memory session state for
+    the watch.  This in turn will break the watch (i.e., notifies won't
+    get delivered).
+    
+    Instead, always resend linger registration ops, so that we always have a
+    unique reqid and do the correct session registeration for each session.
+    
+     * track the tid of the registation op for each LingerOp
+     * mark registrations ops as should_resend=false; cancel as needed
+     * when we send a new registration op, cancel the old one to ensure we
+       ignore the reply.  This is needed becuase we resend linger ops on any
+       pg change, not just a primary change.
+     * drop the first_send arg to send_linger(), as we can now infer that
+       from register_tid == 0.
+    
+    The bug was easily reproduced with ms inject socket failures = 500 and the
+    test_stress_watch utility.
+    
+    Fixes: #2796
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 4d7d3e276967d555fed8a689976047f72c96c2db
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 9 13:22:42 2012 -0700
+
+    osd: guard class call decoding
+    
+    Backport: argonaut
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 7fbbe4652ffb2826978aa1f1cacce4456d2ef1fc
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 5 18:08:58 2012 -0700
+
+    librados: take lock when signaling notify cond
+    
+    When we are signaling the cond to indicate that a notify is complete,
+    take the appropriate lock.  This removes the possibility of a race
+    that loses our signal.  (That would be very difficult given that there
+    are network round trips involved, but this makes the lock/cond usage
+    "correct.")
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 6ed01df412b4f4745c8f427a94446987c88b6bef
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jul 22 07:46:11 2012 -0700
+
+    workqueue: kick -> wake or _wake, depending on locking
+    
+    Break kick() into wake() and _wake() methods, depending on whether the
+    lock is already held.  (The rename ensures that we audit/fix all
+    callers.)
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    
+    Conflicts:
+    
+    	src/common/WorkQueue.h
+    	src/osd/OSD.cc
+
+commit d2d40dc3059d91450925534f361f2c03eec9ef88
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 4 15:11:21 2012 -0700
+
+    client: fix locking for SafeCond users
+    
+    Need to wait on flock, not client_lock.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit c963a21a8620779d97d6cbb51572551bdbb50d0b
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 26 15:01:05 2012 -0700
+
+    filestore: check for EIO in read path
+    
+    Check for EIO in read methods and helpers.  Try to do checks in low-level
+    methods (e.g., lfn_*()) to avoid duplication in higher-level methods.
+    
+    The transaction apply function already checks for EIO on writes, and will
+    generate a nicer error message, so we can largely ignore the write path,
+    as long as errors get passed up correctly.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 6bd89aeb1bf3b1cbb663107ae6bcda8a84dd8601
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 26 09:07:46 2012 -0700
+
+    filestore: add 'filestore fail eio' option, default true
+    
+    By default we will assert/fail/crash on EIO from the underlying fs.  We
+    already do this in the write path, but not the read path, or in various
+    internal infrastructure.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit e9b5a289838f17f75efbf9d1640b949e7485d530
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 24 13:53:03 2012 -0700
+
+    config: fix 'config set' admin socket command
+    
+    Fixes: #2832
+    Backport: argonaut
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 1a6cd9659abcdad0169fe802ed47967467c448b3
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 25 16:35:09 2012 -0700
+
+    osd: break potentially large transaction into pieces
+    
+    We do a similar trick elsewhere.  Control this via a tunable.  Eventually
+    we'll control the others (in a non-stable branch).
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 15e1622959f5a46f7a98502cdbaebfda2247a35b
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 25 14:53:34 2012 -0700
+
+    osd: only commit past intervals at end of parallel build
+    
+    We don't check for gaps in the past intervals, so we should only commit
+    this when we are completely done.  Otherwise a partial run and rsetart will
+    leave the gap in place, which may confuse the peering code that relies on
+    this information.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 16302acefd8def98fc4597366d6ba2845e17fcb6
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 25 10:57:35 2012 -0700
+
+    osd: generate past intervals in parallel on boot
+    
+    Even though we aggressively share past_intervals with notifies etc, it is
+    still possible for an osd to get buried behind a pile of old maps and need
+    to generate these if it has been out of the cluster for a while.  This has
+    happened to us in the past but, sadly, we did not merge the work then.
+    On the bright side, this implementation is much much much cleaner than the
+    old one because of the pg_interval_t helper we've since switched to.
+    
+    On bootup, we look at the intervals each pg needs and calclate the union,
+    and then iterate over that map range.  The inner bit of the loop is
+    functionally identical to PG::build_past_intervals(), keeping the per-pg
+    state in the pistate struct.
+    
+    Backport: argonaut
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit fca65ff52a5f7d49bcac83b3b2232963a879e446
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 25 10:58:07 2012 -0700
+
+    osd: move calculation of past_interval range into helper
+    
+    PG::generate_past_intervals() first calculates the range over which it
+    needs to generate past intervals.  Do this in a helper function.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 5979351ef3d3d03bced9286f79cbc22524c4a8de
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 25 10:58:28 2012 -0700
+
+    osd: fix map epoch boot condition
+    
+    We only want to join the cluster if we can catch up to the latest
+    osdmap with a small number of maps, in this case a single map message.
+    
+    Backport: argonaut
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 8c7186d02627f8255273009269d50955172efb52
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 24 20:18:01 2012 -0700
+
+    mon: ignore pgtemp messages from down osds
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit b17f54671f350fd4247f895f7666d46860736728
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 24 20:16:04 2012 -0700
+
+    mon: ignore osd_alive messages from down osds
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 7dfdf4f8de16155edd434534e161e06ba7c79d7d
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Jul 23 14:05:53 2012 -0700
+
+    librbd: replace assign_bid with client id and random number
+    
+    The assign_bid method has issues with replay because it is a write
+    that also returns data. This means that the replayed operation would
+    return success, but no data, and cause a create to fail. Instead, let
+    the client set the bid based on its global id and a random number.
+    
+    This only affects the creation of new images, since the bid is put
+    into an opaque string as part of the object prefix.
+    
+    Keep the server side assign_bid around in case there are old clients
+    still using it.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit dc2d67112163bee8b111f75ae3e3ca42884b09b4
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Mon Jul 9 14:11:23 2012 -0700
+
+    librados: add new constructor to form a Rados object from IoCtx
+    
+    This creates a separate reference to an existing connection, for
+    use when a client holding IoCtx needs to consult another (say,
+    for rbd cloning)
+    
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit c99671201de9d9cdf03bbf0f4e28e8afb70c280c
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 18 19:49:58 2012 -0700
+
+    add CRUSH_TUNABLES feature bit
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 0b579546cfddec35095b2aec753028d8e63f3533
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Jul 18 10:24:58 2012 -0700
+
+    ObjectCacher: fix cache_bytes_hit accounting
+    
+    Misses are not hits!
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 2869039b79027e530c2863ebe990662685e4bbe6
+Author: Pascal de Bruijn | Unilogic Networks B.V <pascal@unilogicnetworks.net>
+Date:   Wed Jul 11 15:23:16 2012 +0200
+
+    Robustify ceph-rbdnamer and adapt udev rules
+    
+    Below is a patch which makes the ceph-rbdnamer script more robust and
+    fixes a problem with the rbd udev rules.
+    
+    On our setup we encountered a symlink which was linked to the wrong rbd:
+    
+      /dev/rbd/mypool/myrbd -> /dev/rbd1
+    
+    While that link should have gone to /dev/rbd3 (on which a
+    partition /dev/rbd3p1 was present).
+    
+    Now the old udev rule passes %n to the ceph-rbdnamer script, the problem
+    with %n is that %n results in a value of 3 (for rbd3), but in a value of
+    1 (for rbd3p1), so it seems it can't be depended upon for rbdnaming.
+    
+    In the patch below the ceph-rbdnamer script is made more robust and it
+    now it can be called in various ways:
+    
+      /usr/bin/ceph-rbdnamer /dev/rbd3
+      /usr/bin/ceph-rbdnamer /dev/rbd3p1
+      /usr/bin/ceph-rbdnamer rbd3
+      /usr/bin/ceph-rbdnamer rbd3p1
+      /usr/bin/ceph-rbdnamer 3
+    
+    Even with all these different styles of calling the modified script, it
+    should now return the same rbdname. This change "has" to be combined
+    with calling it from udev with %k though.
+    
+    With that fixed, we hit the second problem. We ended up with:
+    
+      /dev/rbd/mypool/myrbd -> /dev/rbd3p1
+    
+    So the rbdname was symlinked to the partition on the rbd instead of the
+    rbd itself. So what probably went wrong is udev discovering the disk and
+    running ceph-rbdnamer which resolved it to myrbd so the following
+    symlink was created:
+    
+      /dev/rbd/mypool/myrbd -> /dev/rbd3
+    
+    However partitions would be discovered next and ceph-rbdnamer would be
+    run with rbd3p1 (%k) as parameter, resulting in the name myrbd too, with
+    the previous correct symlink being overwritten with a faulty one:
+    
+      /dev/rbd/mypool/myrbd -> /dev/rbd3p1
+    
+    The solution to the problem is in differentiating between disks and
+    partitions in udev and handling them slightly differently. So with the
+    patch below partitions now get their own symlinks in the following style
+    (which is fairly consistent with other udev rules):
+    
+      /dev/rbd/mypool/myrbd-part1 -> /dev/rbd3p1
+    
+    Please let me know any feedback you have on this patch or the approach
+    used.
+    
+    Regards,
+    Pascal de Bruijn
+    Unilogic B.V.
+    
+    Signed-off-by: Pascal de Bruijn <pascal@unilogicnetworks.net>
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 426384f6beccabf9e9b9601efcb8147904ec97c2
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 16 16:02:14 2012 -0700
+
+    log: apply log_level to stderr/syslog logic
+    
+    In non-crash situations, we want to make sure the message is both below the
+    syslog/stderr threshold and also below the normal log threshold.  Otherwise
+    we get anything we gather on those channels, even when the log level is
+    low.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 8dafcc5c1906095cb7d15d648a7c1d7524df3768
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 16 15:40:53 2012 -0700
+
+    log: fix event gather condition
+    
+    We should gather an event if it is below the log or gather threshold.
+    
+    Previously we were only gathering if we were going to print it, which makes
+    the dump no more useful than what was already logged.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit ec5cd6def9817039704b6cc010f2797a700d8500
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Jul 16 13:11:24 2012 -0700
+
+    PG::RecoveryState::Stray::react(LogEvt&): reset last_pg_scrub
+    
+    We need to reset the last_pg_scrub data in the osd since we
+    are replacing the info.
+    
+    Probably fixes #2453
+    
+    In cases like 2453, we hit the following backtrace:
+    
+         0> 2012-05-19 17:24:09.113684 7fe66be3d700 -1 osd/OSD.h: In function 'void OSD::unreg_last_pg_scrub(pg_t, utime_t)' thread 7fe66be3d700 time 2012-05-19 17:24:09.095719
+    osd/OSD.h: 840: FAILED assert(last_scrub_pg.count(p))
+    
+     ceph version 0.46-313-g4277d4d (commit:4277d4d3378dde4264e2b8d211371569219c6e4b)
+     1: (OSD::unreg_last_pg_scrub(pg_t, utime_t)+0x149) [0x641f49]
+     2: (PG::proc_primary_info(ObjectStore::Transaction&, pg_info_t const&)+0x5e) [0x63383e]
+     3: (PG::RecoveryState::ReplicaActive::react(PG::RecoveryState::MInfoRec const&)+0x4a) [0x633eda]
+     4: (boost::statechart::detail::reaction_result boost::statechart::simple_state<PG::RecoveryState::ReplicaActive, PG::RecoveryState::Started, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl<boost::mpl::list3<boost::statechart::custom_reaction<PG::RecoveryState::MQuery>, boost::statechart::custom_reaction<PG::RecoveryState::MInfoRec>, boost::statechart::custom_reaction<PG::RecoveryState::MLogRec> >, boost::statechart::simple_state<PG::RecoveryState::ReplicaActive, PG::RecoveryState::Started, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> >(boost::statechart::simple_state<PG::RecoveryState::ReplicaActive, PG::RecoveryState::Started, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>&, boost::statechart::event_base const&, void const*)+0x130) [0x6466a0]
+     5: (boost::statechart::simple_state<PG::RecoveryState::ReplicaActive, PG::RecoveryState::Started, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x81) [0x646791]
+     6: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::send_event(boost::statechart::event_base const&)+0x5b) [0x63dfcb]
+     7: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x11) [0x63e0f1]
+     8: (PG::RecoveryState::handle_info(int, pg_info_t&, PG::RecoveryCtx*)+0x177) [0x616987]
+     9: (OSD::handle_pg_info(std::tr1::shared_ptr<OpRequest>)+0x665) [0x5d3d15]
+     10: (OSD::dispatch_op(std::tr1::shared_ptr<OpRequest>)+0x2a0) [0x5d7370]
+     11: (OSD::_dispatch(Message*)+0x191) [0x5dd4a1]
+     12: (OSD::ms_dispatch(Message*)+0x153) [0x5ddda3]
+     13: (SimpleMessenger::dispatch_entry()+0x863) [0x77fbc3]
+     14: (SimpleMessenger::DispatchThread::entry()+0xd) [0x746c5d]
+     15: (()+0x7efc) [0x7fe679b1fefc]
+     16: (clone()+0x6d) [0x7fe67815089d]
+     NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
+    
+    Because we don't clear the scrub state before reseting info,
+    the last_scrub_stamp state in the info.history structure
+    changes without updating the osd state resulting in the
+    above assert failure.
+    
+    Backport: stable
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 248cfaddd0403c7bae8e1533a3d2e27d1a335b9b
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Jul 9 17:57:03 2012 -0700
+
+    ReplicatedPG: don't warn if backfill peer stats don't match
+    
+    pinfo.stats might be wrong if we did log-based recovery on the
+    backfilled portion in addition to continuing backfill.
+    
+    bug #2750
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit bcb1073f9171253adc37b67ee8d302932ba1667b
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jul 15 20:30:34 2012 -0700
+
+    mon/MonitorStore: always O_TRUNC when writing states
+    
+    It is possible for a .new file to already exist, potentially with a
+    larger size.  This would happen if:
+    
+     - we were proposing a different value
+     - we crashed (or were stopped) before it got renamed into place
+     - after restarting, a different value was proposed and accepted.
+    
+    This isn't so unlikely for the log state machine, where we're
+    aggregating random messages.  O_TRUNC ensure we avoid getting the tail
+    end of some previous junk.
+    
+    I observed #2593 and found that a logm state value had a larger size on
+    one mon (after slurping) than the others, pointing to put_bl_sn_map().
+    
+    While we are at it, O_TRUNC put_int() too; the same type of bug is
+    possible there, too.
+    
+    Fixes: #2593
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 41a570778a51fe9a36a5b67a177d173889e58363
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jul 14 14:31:34 2012 -0700
+
+    osd: based misdirected op role calc on acting set
+    
+    We want to look at the acting set here, nothing else.  This was causing us
+    to erroneously queue ops for later (wasting memory) and to erroneously
+    print out a 'misdrected op' message in the cluster log (confusion and
+    incorrect [but ignored] -ENXIO reply).
+    
+    Fixes: #2022
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit b3d077c61e977e8ebb91288aa2294fb21c197fe7
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Fri Jul 13 09:42:20 2012 -0700
+
+    qa: download tests from specified branch
+    
+    These python tests aren't installed, so they need to be downloaded
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit e855cb247b5a9eda6845637e2da5b6358f69c2ed
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Jun 25 09:47:37 2012 -0700
+
+    rgw: don't override subuser perm mask if perm not specified
+    
+    Bug #2650. We were overriding subuser perm mask whenever subuser
+    was modified, even if perm mask was not passed.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit d6c766ea425d87a2f2405c08dcec66f000a4e1a0
+Author: James Page <james.page@ubuntu.com>
+Date:   Wed Jul 11 11:34:21 2012 -0700
+
+    debian: fix ceph-fs-common-dbg depends
+    
+    Signed-off-by: James Page <james.page@ubuntu.com>
+
+commit 95e8d87bc3fb12580e4058401674b93e19df6e02
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Jul 11 11:52:24 2012 -0700
+
+    rados tool: remove -t param option for target pool
+    
+    Bug #2772. This fixes an issue that was introduced when we
+    added the 'rados cp' command. The -t param was already used
+    for rados bench. With this change the only way to specify
+    a target pool is using --target-pool.
+    Though this problem is post argonaut, the 'rados cp' command
+    has been backported, so we need this fix there too.
+    
+    Backport: argonaut
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 5b10778399d5bee602e57035df7d40092a649c06
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 11 09:19:00 2012 -0700
+
+    Makefile: don't install crush headers
+    
+    This is leftover from when we built a libcrush.so.  We can re-add when we
+    start doing that again.
+    
+    Reported-by: Laszlo Boszormenyi <gcs@debian.hu>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 35b13266923f8095650f45562d66372e618c8824
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 10 13:18:27 2012 -0700
+
+    msgr: take over existing Connection on Pipe replacement
+    
+    If a new pipe/socket is taking over an existing session, it should also
+    take over the Connection* associated with the existing session.  Because
+    we cannot clear existing->connection_state, we just take another reference.
+    
+    Clean up the comments a bit while we're here.
+    
+    This affects MDS<->client sessions when reconnecting after a socket fault.
+    It probably also affects intra-cluster (osd/osd, mds/mds, mon/mon)
+    sessions as well, but I did not confirm that.
+    
+    Backport: argonaut
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit b387077b1d019ee52b28bc3bc5305bfb53dfd892
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jul 8 20:33:12 2012 -0700
+
+    debian: include librados-config in librados-dev
+    
+    Reported-by: Laszlo Boszormenyi <gcs@debian.hu>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 03c2dc244af11b711e2514fd5f32b9bfa34183f6
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 3 13:04:28 2012 -0700
+
+    lockdep: increase max locks
+    
+    Hit this limit with the rados api tests.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit b554d112c107efe78ec64f85b5fe588f1e7137ce
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 3 12:07:28 2012 -0700
+
+    config: add unlocked version of get_my_sections; use it internally
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 01da287b8fdc07262be252f1a7c115734d3cc328
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 3 08:20:06 2012 -0700
+
+    config: fix lock recursion in get_val_from_conf_file()
+    
+    Introduce a private, already-locked version.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit c73c64a0f722477a5b0db93da2e26e313a5f52ba
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 3 08:15:08 2012 -0700
+
+    config: fix recursive lock in parse_config_files()
+    
+    The _impl() helper is only called from parse_config_files(); don't retake
+    the lock.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 6646e891ff0bd31c935d1ce0870367b1e086ddfd
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 3 18:51:02 2012 -0700
+
+    rgw: initialize fields of RGWObjEnt
+    
+    This fixes various valgrind warnings triggered by the s3test
+    test_object_create_unreadable.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit b33553aae63f70ccba8e3d377ad3068c6144c99a
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Jul 6 13:14:53 2012 -0700
+
+    rgw: handle response-* params
+    
+    Handle response-* params that set response header field values.
+    Fixes #2734, #2735.
+    Backport: argonaut
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 74f687501a8a02ef248a76f061fbc4d862a9abc4
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 4 13:59:04 2012 -0700
+
+    osd: add missing formatter close_section() to scrub status
+    
+    Also add braces to make the open/close matchups easier to see.  Broken
+    by f36617392710f9b3538bfd59d45fd72265993d57.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 020b29961303b12224524ddf78c0c6763a61242e
+Author: Mike Ryan <mike.ryan@inktank.com>
+Date:   Wed Jun 27 14:14:30 2012 -0700
+
+    pg: report scrub status
+    
+    Signed-off-by: Mike Ryan <mike.ryan@inktank.com>
+
+commit db6d83b3ed51c07b361b27d2e5ce3227a51e2c60
+Author: Mike Ryan <mike.ryan@inktank.com>
+Date:   Wed Jun 27 13:30:45 2012 -0700
+
+    pg: track who we are waiting for maps from
+    
+    Signed-off-by: Mike Ryan <mike.ryan@inktank.com>
+
+commit e1d4855fa18b1cda85923ad9debd95768260d4eb
+Author: Mike Ryan <mike.ryan@inktank.com>
+Date:   Tue Jun 26 16:25:27 2012 -0700
+
+    pg: reduce scrub write lock window
+    
+    Wait for all replicas to construct the base scrub map before finalizing
+    the scrub and locking out writes.
+    
+    Signed-off-by: Mike Ryan <mike.ryan@inktank.com>
+
+commit 27409aa1612c1512bf393de22b62bbfe79b104c1
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jul 5 15:52:51 2012 -0700
+
+    rgw: don't store bucket info indexed by bucket_id
+    
+    Issue #2701. This info wasn't really used anywhere and we weren't
+    removing it. It was also sharing the same pool namespace as the
+    info indexed by bucket name, which is bad.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 9814374a2b40e15c13eb03ce6b8e642b0f7f93e4
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jul 5 14:59:22 2012 -0700
+
+    test_rados_tool.sh: test copy pool
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit d75100667a539baf47c79d752b787ed5dcb51d7a
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jul 5 13:42:23 2012 -0700
+
+    rados tool: copy object in chunks
+    
+    Instead of reading the entire object and then writing it,
+    we read it in chunks.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 16ea64fbdebb7a74e69e80a18d98f35d68b8d9a1
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Jun 29 14:43:00 2012 -0700
+
+    rados tool: copy entire pool
+    
+    A new rados tool command that copies an entire pool
+    into another existing pool.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 960c2124804520e81086df97905a299c8dd4e08c
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Jun 29 14:09:08 2012 -0700
+
+    rados tool: copy object
+    
+    New rados command: rados cp <src-obj> [dest-obj]
+    
+    Requires specifying source pool. Target pool and locator can be specified.
+    The new command preserves object xattrs and omap data.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 23d31d3e2aa7f2b474a7b8e9d40deb245d8be9de
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 6 08:47:44 2012 -0700
+
+    ceph.spec.in: add ceph-disk-{activate,prepare}
+    
+    Reported-by: Jimmy Tang <jtang@tchpc.tcd.ie>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit ea11c7f9d8fd9795e127cfd7e8a1f28d4f5472e9
+Author: Wido den Hollander <wido@widodh.nl>
+Date:   Thu Jul 5 15:29:54 2012 +0200
+
+    Allow URL-safe base64 cephx keys to be decoded.
+    
+    In these cases + and / are replaced by - and _ to prevent problems when using
+    the base64 strings in URLs.
+    
+    Signed-off-by: Wido den Hollander <wido@widodh.nl>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit f67fe4e368b5f250f0adfb183476f5f294e8a529
+Author: Wido den Hollander <wido@widodh.nl>
+Date:   Wed Jul 4 15:46:04 2012 +0200
+
+    librados: Bump the version to 0.48
+    
+    Signed-off-by: Wido den Hollander <wido@widodh.nl>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 35b9ec881aecf84b3a49ec0395d7208de36dc67d
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Jun 26 17:28:51 2012 -0700
+
+    rgw-admin: use correct modifier with strptime
+    
+    Bug #2658: used %I (12h) instead of %H (24h)
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit da251fe88503d32b86113ee0618db7c446d34853
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jun 21 15:40:27 2012 -0700
+
+    rgw: send both swift x-storage-token and x-auth-token
+    
+    older clients need x-storage-token, newer x-auth-token
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 4c19ecb9a34e77e71d523a0a97e17f747bd5767d
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jun 21 15:17:19 2012 -0700
+
+    rgw: radosgw-admin date params now also accept time
+    
+    The date format now is "YYYY-MM-DD[ hh:mm:ss]". Got rid of
+    the --time param for the old ops log stuff.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    
+    Conflicts:
+    
+    	src/test/cli/radosgw-admin/help.t
+
+commit 6958aeb898fc683159483bfbb798f069a9b5330a
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jun 21 13:14:47 2012 -0700
+
+    rgw-admin: fix usage help
+    
+    s/show/trim
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 83c043f803ab2ed74fa9a84ae9237dd7df2a0c57
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 3 14:07:16 2012 -0700
+
+    radosgw-admin: fix clit test
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 5674158163e9c1d50985796931240b237676b74d
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 3 11:32:57 2012 -0700
+
+    ceph: fix cli help test
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 151bf0eef59acae2d1fcf3f0feb8b6aa963dc2f6
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Jul 3 11:23:16 2012 -0700
+
+    ReplicatedPG: remove faulty scrub assert in sub_op_modify_applied
+    
+    This assert assumed that all ops submitted before MOSDRepScrub was
+    submitted were processed by the time that MOSDRepScrub was
+    processed.  In fact, MOSDRepScrub's scrub_to may refer to a
+    last_update yet to be seen by the replica.
+    
+    Bug #2693
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 32833e88a1ad793fa4be86101ce9c22b6f677c06
+Author: Kyle Bader <kyle.bader@dreamhost.com>
+Date:   Tue Jul 3 11:20:38 2012 -0700
+
+    ceph: better usage
+    
+    Signed-off-by: Kyle Bader <kyle.bader@dreamhost.com>
+
+commit 67455c21879c9c117f6402259b5e2da84524e169
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 3 09:20:35 2012 -0700
+
+    debian: strip new ceph-mds package
+    
+    Reported-by: Amon Ott <a.ott@m-privacy.de>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit b53cdb97d15f9276a9b26bec9f29034149f93358
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 3 06:46:10 2012 -0700
+
+    config: remove bad argparse_flag argument in parse_option()
+    
+    This is wrong, and thankfully valgrind picks it up.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit f7d4e39740fd2afe82ac40c711bd3fe7a282e816
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jul 1 17:23:28 2012 -0700
+
+    msgr: restart_queue when replacing existing pipe and taking over the queue
+    
+    The queue may have been previously stopped (by discard_queue()), and needs
+    to be restarted.
+    
+    Fixes consistent failures from the mon_recovery.py integration tests.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 5dfd2a512d309f7f641bcf7c43277f08cf650b01
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jul 1 15:37:31 2012 -0700
+
+    msgr: choose incoming connection if ours is STANDBY
+    
+    If the connect_seq matches, but our existing connection is in STANDBY, take
+    the incoming one.  Otherwise, the other end will wait indefinitely for us
+    to connect but we won't.
+    
+    Alternatively, we could "win" the race and trigger a connection by sending
+    a keepalive (or similar), but that is more work; we may as well accept the
+    incoming connection we have now.
+    
+    This removes STANDBY from the acceptable WAIT case states.  It also keeps
+    responsibility squarely on the shoulders of the peer with something to
+    deliver.
+    
+    Without this patch, a 3-osd vstart cluster with
+    'ms inject socket failures = 100' and rados bench write -b 4096 would start
+    generating slow request warnings after a few minutes due to the osds
+    failing to connect to each other.  With the patch, I complete a 10 minute
+    run without problems.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit b7007a159f6d941fa8313a24af5810ce295b36ca
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 28 17:50:47 2012 -0700
+
+    msgr: preserve incoming message queue when replacing pipes
+    
+    If we replace an existing pipe with a new one, move the incoming queue
+    of messages that have not yet been dispatched over to the new Pipe so that
+    they are not lost.  This prevents messages from being lost.
+    
+    Alternatively, we could set in_seq = existing->in_seq - existing->in_qlen,
+    but that would make the other end resend those messages, which is a waste
+    of bandwidth.
+    
+    Very easy to reproduce the original bug with 'ms inject socket failures'.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 1f3a722e150f9f27fe7919e9579b5a88dcd15639
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 28 17:45:24 2012 -0700
+
+    msgr: move dispatch_entry into DispatchQueue class
+    
+    A bit cleaner.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 03445290dad5b1213dd138cacf46e379400201c9
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 28 17:38:34 2012 -0700
+
+    msgr: move incoming queue to separate class
+    
+    This extricates the incoming queue and its funky relationship with
+    DispatchQueue from Pipe and moves it into IncomingQueue.  There is now a
+    single IncomingQueue attached to each Pipe.  DispatchQueue is now no
+    longer tied to Pipe.
+    
+    This modularizes the code a bit better (tho that is still a work in
+    progress) and (more importantly) will make it possible to move the
+    incoming messages from one pipe to another in accept().
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 0dbc54169512da776c16161ec3b8fa0b3f08e248
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jun 27 17:06:40 2012 -0700
+
+    msgr: make D_CONNECT constant non-zero, fix ms_handle_connect() callback
+    
+    A while ago we inadvertantly broke ms_handle_connect() callbacks because
+    of a check for m being non-zero in the dispatch_entry() thread.  Adjust the
+    enums so that they get delivered again.
+    
+    This fixes hangs when, for example, the ceph tool sends a command, gets a
+    connection reset, and doesn't get the connect callback to resend after
+    reconnecting to a new monitor.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 2429556a51e8f60b0d9bdee71ef7b34b367f2f38
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 26 17:10:40 2012 -0700
+
+    msgr: fix pipe replacement assert
+    
+    We may replace an existing pipe in the STANDBY state if the previous
+    attempt failed during accept() (see previous patches).
+    
+    This might fix #1378.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 204bc594be1a6046d1b362693d086b49294c2a27
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 26 17:07:31 2012 -0700
+
+    msgr: do not try to reconnect con with CLOSED pipe
+    
+    If we have a con with a closed pipe, drop the message.  For lossless
+    sessions, the state will be STANDBY if we should reconnect.  For lossy
+    sessions, we will end up with CLOSED and we *should* drop the message.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit e6ad6d25a58b8e34a220d090d01e26293c2437b4
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 26 17:06:41 2012 -0700
+
+    msgr: move to STANDBY if we replace during accept and then fail
+    
+    If we replace an existing pipe during accept() and then fail, move to
+    STANDBY so that our connection state (connect_seq, etc.) is preserved.
+    Otherwise, we will throw out that information and falsely trigger a
+    RESETSESSION on the next connection attempt.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
diff --git a/doc/changelog/v0.48.2argonaut.txt b/doc/changelog/v0.48.2argonaut.txt
new file mode 100644
index 000000000..2884e60db
--- /dev/null
+++ b/doc/changelog/v0.48.2argonaut.txt
@@ -0,0 +1,476 @@
+commit 3e02b2fad88c2a95d9c0c86878f10d1beb780bfe
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Sep 11 13:04:50 2012 -0700
+
+    v0.48.2argonaut
+
+commit 343e607980c923324de241d2cf8396d7c4857dd8
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Sep 18 13:45:27 2012 -0700
+
+    cls_rgw: if stats drop below zero, set them to zero
+    
+    This complements fix for #3127. This is only a band aid
+    solution for argonaut, the real solution fixes the original
+    issue that made this possible.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 92be908c7f22c0ccf2092024fd54096362032394
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Sep 12 16:41:17 2012 -0700
+
+    cls_rgw: change scoping of suggested changes vars
+    
+    Fixes: #3127
+    Bad variable scoping made it so that specific variables
+    weren't initialized between suggested changes iterations.
+    This specifically affected a case where in a specific
+    change we had an updated followed by a remove, and the
+    remove was on a non-existent key (e.g., was already
+    removed earlier). We ended up re-substracting the
+    object stats, as the entry wasn't reset between
+    the iterations (and we didn't read it because the
+    key didn't exist).
+    
+    backport:argonaut
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit d3a8497b44296484fa18ac10e776c93701365a8b
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Sep 4 11:29:21 2012 -0700
+
+    objecter: fix osdmap wait
+    
+    When we get a pool_op_reply, we find out which osdmap we need to wait for.
+    The wait_for_new_map() code was feeding that epoch into
+    maybe_request_map(), which was feeding it to the monitor with the subscribe
+    request.  However, that epoch is the *start* epoch, not what we want.  Fix
+    this code to always subscribe to what we have (+1), and ensure we keep
+    asking for more until we catch up to what we know we should eventually
+    get.
+    
+    Bug: #3075
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit e09b26555c6132ffce08b565780a39e4177cbc1c)
+
+commit de026f31f7f31c2b79c41910b83570f052f354d5
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Aug 27 07:38:34 2012 -0700
+
+    objecter: send queued requests when we get first osdmap
+    
+    If we get our first osdmap and already have requests queued, send them.
+    
+    Backported from 8d1efd1b829ae50eab7f7f4c07da04e03fce7c45.
+    
+    Fixes: #3050
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 379aa80ac3a313025e433cefd239ddbeec45f9e7
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 21 21:12:33 2012 -0700
+
+    objecter: use ordered map<> for tracking tids to preserve order on resend
+    
+    We are using a hash_map<> to map tids to Op*'s.  In handle_osd_map(),
+    we will recalc_op_target() on each Op in a random (hash) order.  These
+    will get put in a temp map<tid,Op*> to ensure they are resent in the
+    correct order, but their order on the session->ops list will be random.
+    
+    Then later, if we reset an OSD connection, we will resend everything for
+    that session in ops order, which is be incorrect.
+    
+    Fix this by explicitly reordering the requests to resend in
+    kick_requests(), much like we do in handle_osd_map().  This lets us
+    continue to use a hash_map<>, which is faster for reasonable numbers of
+    requests.  A simpler but slower fix would be to just use map<> instead.
+    
+    This is one of many bugs contributing to #2947.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 1113a6c56739a56871f01fa13da881dab36a32c4)
+
+commit 54788d0da40608fb8ccf8f16039536729881d542
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Mon Aug 20 15:02:57 2012 -0700
+
+    rbd: force all exiting paths through main()/return
+    
+    This properly destroys objects.  In the process, remove usage_exit();
+    also kill error-handling in set_conf_param (never relevant for rbd.cc,
+    and if you call it with both pointers NULL, well...)
+    Also switch to EXIT_FAILURE for consistency.
+    
+    Backported from fed8aea662bf919f35a5a72e4e2a2a685af2b2ed.
+    
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Fixes: #2948
+
+commit a179ccee8848fe9bc8d622cba13fab1a99f6eb63
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Sep 18 09:37:44 2012 -0700
+
+    rbd: only open the destination pool for import
+    
+    Otherwise importing into another pool when the default pool, rbd,
+    doesn't exist results in an error trying to open the rbd pool.
+    
+    Reported-by: Sébastien Han <han.sebastien@gmail.com>
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 16aca749989b006f4d5d2190ed1e8480c1bf0282
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Mon Sep 17 08:55:14 2012 -0700
+
+    ceph-disk-activate, upstart: Use "initctl emit" to start OSDs.
+    
+    This avoids an error if the daemon was running already, and is
+    already being done with the other services.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 34d8eec9e23edec174a96d16bf42a0b8df118183
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Fri Sep 14 17:13:57 2012 -0700
+
+    rbd: make --pool/--image args easier to understand for import
+    
+    There's no need to set the default pool in set_pool_image_name - this
+    is done later, in a way that doesn't ignore --pool if --dest-pool
+    is not specified.
+    
+    This means --pool and --image can be used with import, just like
+    the rest of the commands. Without this change, --dest and --dest-pool
+    had to be used, and --pool would be silently ignored for rbd import.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit f390ee68233e9c6db1202aa95665adf27ba4399a
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Thu Sep 13 14:06:04 2012 -0700
+
+    ceph-create-keys: Create a bootstrap-osd key too.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 9348ea0de953a4fd2eed239437d14c366c8e2cdd
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Thu Sep 13 11:34:03 2012 -0700
+
+    ceph-create-keys: Refactor to share wait_for_quorum call.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit c9d4c58b9d71e6d84b644376684bcb72eae7e11c
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Sep 12 11:38:07 2012 -0700
+
+    objecter: fix skipped map handling
+    
+    If we skip a map, we want to translate NO_ACTION to NEED_RESEND, but leave
+    POOL_DNE alone.
+    
+    Backported from 2a3b7961c021b19a035f8a6cc4fc3cc90f88f367.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit d8dff93a2497bd578116eb20ed65a0595acdf341
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Jul 30 15:19:29 2012 -0700
+
+    librbd, cls_rbd: close snapshot creation race with old format
+    
+    If two clients created a snapshot at the same time, the one with the
+    higher snapshot id might be created first, so the lower snapshot id
+    would be added to the snapshot context and the snaphot seq would be
+    set to the lower one.
+    
+    Instead of allowing this to happen, return -ESTALE if the snapshot id
+    is lower than the currently stored snapshot sequence number. On the
+    client side, get a new id and retry if this error is encountered.
+    
+    Backport: argonaut
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit 771ca71c0357bd8149677795ac934ab09945a3a3
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Tue Sep 11 16:31:57 2012 -0700
+
+    upstart: Give everything a stop on stanza.
+    
+    These are all tasks, and expected to exit somewhat quickly,
+    but e.g. ceph-create-keys has a loop where it waits for mon
+    to reach quorum, so it might still be in that loop when the
+    machine is shut down.
+
+commit 9f967e3ac255b0037b598061f5cbb4050db79472
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Tue Sep 11 16:28:41 2012 -0700
+
+    upstart: Start mds,mon,radosgw after a reboot.
+    
+    They had no "start on" stanzas, so they didn't get started earlier.
+
+commit ce1e7d1b0291759950abf02f5bae064994d2ec34
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Tue Sep 11 15:31:06 2012 -0700
+
+    upstart: Add ceph-create-keys.conf to package.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 0ff22ba0508d43cd78aeae6736f2db002cc8de8e
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Sep 11 14:50:53 2012 -0700
+
+    obsync: if OrdinaryCallingFormat fails, try SubdomainCallingFormat
+    
+    This blindly tries the Subdomain calling format if the ordinary method
+    fails.  In particular, this works around buckets that present a
+    PermanentRedirect message.
+    
+    See bug #3128.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Matthew Wodrich <matthew.wodrich@dreamhost.com>
+
+commit 0e58d95134dd95f33b9b1a3b1286a0719ad20707
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 17 16:04:20 2012 -0700
+
+    librbd: add test for discard of nonexistent objects
+    
+    This verifies librbd properly handles ENOENT during discard.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 97d8a734ce937ce2a683c7c36f5b72395c6456c2
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Sep 10 13:19:53 2012 -0700
+
+    librbd: ignore -ENOENT during discard
+    
+    This is a backport of a3ad98a3eef062e9ed51dd2d1e58c593e12c9703
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 61d705e2d67f83c81aa7c6362ec6703e014ecb87
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Aug 15 18:42:56 2012 -0700
+
+    objectcacher: fix bh leak on discard
+    
+    Fixes: #2950
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 787a78e8b1d8e5d92ee447b970ec8b79baa46f0b
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Thu Aug 30 10:16:52 2012 -0400
+
+    upstart, ceph-create-keys: Make client.admin key generation automatic.
+    
+    This should help simplify Chef etc deployments. Now (when using the
+    Upstart jobs), when a ceph-mon is started, ceph-create-admin-key is
+    triggered. If /etc/ceph/$cluster.client.admin.keyring already exists,
+    it does nothing; otherwise, it waits for ceph-mon to reach quorum, and
+    then does a "ceph auth get-or-create" to create the key, and writes it
+    atomically to disk.
+    
+    The equivalent code can be removed from the Chef cookbook once this is
+    in.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 66aecb1e410a59f3e0ed89a30212b8dc546a3d4a
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Thu Aug 30 10:21:29 2012 -0400
+
+    config: Add a per-name default keyring to front of keyring search path.
+    
+    This lets us have e.g. /etc/ceph/ceph.client.admin.keyring that is
+    owned by root:admin and mode u=rw,g=r,o= without making every non-root
+    run of the command line tools complain and fail.
+    
+    This is what the Chef cookbook has been doing for a while already.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 6e34b8c109322690151b42efb745bc96a210dda4
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Thu Aug 30 10:11:09 2012 -0400
+
+    upstart: Make instance jobs export their cluster and id variables.
+    
+    This allows other jobs listening to Upstart "started ceph-mon" events
+    to see what instance started.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 27372dc4e99c17a7a2d5ad6646e5ae54392d5955
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Thu Jul 12 10:47:29 2012 -0700
+
+    upstart: Make ceph-osd always set the crush location.
+    
+    This used to be conditional on config having osd_crush_location set,
+    but with that, minimal configuration left the OSD completely out of
+    the crush map, and prevented the OSD from starting properly.
+    
+    Note: Ceph does not currently let this mechanism automatically move
+    hosts to another location in the CRUSH hierarchy. This means if you
+    let this run with defaults, setting osd_crush_location later will not
+    take effect. Set up your config file (or Chef environment) fully
+    before starting the OSDs the first time.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 471105a966f873aef2361b1ed48d088c490fe1aa
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Tue Jul 3 15:24:26 2012 -0700
+
+    ceph-disk-prepare: Partition and format OSD data disks automatically.
+    
+    Uses gdisk, as it seems to be the only tool that can automate GPT uuid
+    changes. Needs to run as root.
+    
+    Adds Recommends: gdisk to ceph.deb.
+    
+    Closes: #2547
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 24fe265c594308d5679e845fcfe8b0fe6a7be612
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Tue Jul 3 09:22:28 2012 -0700
+
+    ceph-disk-prepare: Take fsid from config file.
+    
+    Closes: #2546.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 1f06b490b4187055a5bfabc20746c87dfa075e0d
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Mon Jun 25 15:14:33 2012 -0700
+
+    upstart: fix regex
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+
+commit 657ca118a7658617b9117311d9ee1cbe00103c06
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Aug 28 16:17:21 2012 -0700
+
+    rgw: clear usage map before reading usage
+    
+    Fixes: #3057
+    Since we read usage in chunks we need to clear the
+    usage map before reading the next chunk, otherwise
+    we're going to aggregate the old data as well.
+    
+    Backport: argonaut
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit c49284c98de1df1a889e0c88e6b1157133f5e0a6
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Thu Aug 23 11:48:50 2012 -0700
+
+    Don't package crush header files.
+
+commit b6fb3e37a68d48434a81ee4356cde48cad187e94
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Aug 17 17:34:23 2012 -0700
+
+    rgw: dump content_range using 64 bit formatters
+    
+    Fixes: #2961
+    Also make sure that size is 64 bit.
+    
+    backport: argonaut
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 08d975146cbe6796799266f03361240a62acb297
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 21 10:58:38 2012 -0700
+
+    Revert "rgw: dump content_range using 64 bit formatters"
+    
+    This reverts commit faf9fa5744b459abc2eda829a48a4e07b9c97a08.
+
+commit faf9fa5744b459abc2eda829a48a4e07b9c97a08
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Aug 17 17:34:23 2012 -0700
+
+    rgw: dump content_range using 64 bit formatters
+    
+    Fixes: #2961
+    Also make sure that size is 64 bit.
+    
+    backport: argonaut
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit 47b24c0562bcb44964a0b8f6c4847bb0f05924e0
+Author: Matthew Wodrich <matthew.wodrich@dreamhost.com>
+Date:   Tue Jul 31 19:13:03 2012 -0700
+
+    obsync: add missing package specifier to format_exc
+    
+    Fixes: #2873
+    Signed-off-by: Matthew Wodrich <matthew.wodrich@dreamhost.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+
+commit a14214dc9b7c15581a0664dbe259389867f88e72
+Author: Danny Kukawka <danny.kukawka@bisect.de>
+Date:   Thu Aug 16 12:56:58 2012 +0200
+
+    fix keyring generation for mds and osd
+    
+        [ The following text is in the "UTF-8" character set. ]
+        [ Your display is set for the "ANSI_X3.4-1968" character set.  ]
+        [ Some characters may be displayed incorrectly. ]
+    
+    Fix config keys for OSD/MDS data dirs. As in documentation and other
+    places of the scripts the keys are 'osd data'/'mds data' and not
+    'osd_data'
+    
+    In case if MDS: if 'mds data' doesn't exist, create it.
+    
+    Signed-off-by: Danny Kukawka <danny.kukawka@bisect.de>
+
+commit d1a31ce1ca2a5e1c53146c6e9063a123a1fe8cdb
+Author: Danny Kukawka <danny.kukawka@bisect.de>
+Date:   Thu Aug 16 12:56:32 2012 +0200
+
+    fix ceph osd create help
+    
+        [ The following text is in the "UTF-8" character set. ]
+        [ Your display is set for the "ANSI_X3.4-1968" character set.  ]
+        [ Some characters may be displayed incorrectly. ]
+    
+    Change ceph osd create <osd-id> to ceph osd create <uuid>, since this
+    is what the command is really doing.
+    
+    Signed-off-by: Danny Kukawka <danny.kukawka@bisect.de>
+
+commit dc2a232bd37b7202c3d6e94396b3d85cec5225cd
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 9 17:24:19 2012 -0700
+
+    mon: simplify logmonitor check_subs; less noise
+    
+     * simple helper to translate name to id
+     * verify sub type is valid in caller
+     * assert sub type is valid in method
+     * simplify iterator usage
+    
+    Among other things, this gets rid of this noise in the logs:
+    
+    2012-07-10 20:51:42.617152 7facb23f1700  1 mon.a@1(peon).log v310 check_sub sub monmap not log type
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
diff --git a/doc/changelog/v0.48.3argonaut.txt b/doc/changelog/v0.48.3argonaut.txt
new file mode 100644
index 000000000..22f0dc967
--- /dev/null
+++ b/doc/changelog/v0.48.3argonaut.txt
@@ -0,0 +1,895 @@
+commit 920f82e805efec2cae05b79c155c07df0f3ed5dd
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Mon Jan 7 21:08:08 2013 -0800
+
+    v0.48.3argonaut
+
+commit 213e3559dd260a2e19324f2a671c808261249f96
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jan 6 20:43:21 2013 -0800
+
+    osd: fix race in do_recovery()
+    
+    Verify that the PG is still RECOVERING or BACKFILL when we take the pg
+    lock in the recovery thread.  This prevents a crash from an invalid
+    state machine event when the recovery queue races with a PG state change
+    (e.g., due to peering).
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+
+commit e410d1a066b906cad3103a5bbfa5b4509be9ac37
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Jan 4 21:19:45 2013 -0800
+
+    ReplicatedPG: requeue waiting_for_ondisk in apply_and_flush_repops
+    
+    Fixes: #3722
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit cd5f2bfdbb7fbf9237bef482845644cc41fa66de
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Nov 15 16:20:33 2012 -0800
+
+    ObjectCacher: fix off-by-one error in split
+    
+    This error left a completion that should have been attached
+    to the right BufferHead on the left BufferHead, which would
+    result in the completion never being called unless the buffers
+    were merged before it's original read completed. This would cause
+    a hang in any higher level waiting for a read to complete.
+    
+    The existing loop went backwards (using a forward iterator),
+    but stopped when the iterator reached the beginning of the map,
+    or when a waiter belonged to the left BufferHead.
+    
+    If the first list of waiters should have been moved to the right
+    BufferHead, it was skipped because at that point the iterator
+    was at the beginning of the map, which was the main condition
+    of the loop.
+    
+    Restructure the waiters-moving loop to go forward in the map instead,
+    so it's harder to make an off-by-one error.
+    
+    Possibly-fixes: #3286
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 2e862f4d183d8b57b43b0777737886f18f68bf00)
+
+commit 802c486fd0913e9fe58ae7ff72df135f99712358
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 4 11:07:48 2013 -0800
+
+    config: change default log_max_recent to 10,000
+    
+    Commit c34e38bcdc0460219d19b21ca7a0554adf7f7f84 meant to do this but got
+    the wrong number of zeros.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 39a734fbf34ccd121f17023bcec814e61c8bdaab
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jan 3 17:15:07 2013 -0800
+
+    os/FileStore: fix non-btrfs op_seq commit order
+    
+    The op_seq file is the starting point for journal replay.  For stable btrfs
+    commit mode, which is using a snapshot as a reference, we should write this
+    file before we take the snap.  We normally ignore current/ contents anyway.
+    
+    On non-btrfs file systems, however, we should only write this file *after*
+    we do a full sync, and we should then fsync(2) it before we continue
+    (and potentially trim anything from the journal).
+    
+    This fixes a serious bug that could cause data loss and corruption after
+    a power loss event.  For a 'kill -9' or crash, however, there was little
+    risk, since the writes were still captured by the host's cache.
+    
+    Fixes: #3721
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 28d59d374b28629a230d36b93e60a8474c902aa5)
+
+commit 49416619d733572368e5d2ba7f2b34150c754b23
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Dec 28 13:07:18 2012 -0800
+
+    log: broadcast cond signals
+    
+    We were using a single cond, and only signalling one waiter.  That means
+    that if the flusher and several logging threads are waiting, and we hit
+    a limit, we the logger could signal another logger instead of the flusher,
+    and we could deadlock.
+    
+    Similarly, if the flusher empties the queue, it might signal only a single
+    logger, and that logger could re-signal the flusher, and the other logger
+    could wait forever.
+    
+    Intead, break the single cond into two: one for loggers, and one for the
+    flusher.  Always signal the (one) flusher, and always broadcast to all
+    loggers.
+    
+    Backport: bobtail, argonaut
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 813787af3dbb99e42f481af670c4bb0e254e4432)
+
+commit f8bb4814d0fd7e08c9b6cdcde02940bf1b42cadd
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jan 2 13:58:44 2013 -0800
+
+    log: fix locking typo/stupid for dump_recent()
+    
+    We weren't locking m_flush_mutex properly, which in turn was leading to
+    racing threads calling dump_recent() and garbling the crash dump output.
+    
+    Backport: bobtail, argonaut
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 43cba617aa0247d714632bddf31b9271ef3a1b50)
+
+commit 9483a032f750572586f146c696ec6501d3df0383
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Dec 27 16:06:24 2012 -0800
+
+    init-ceph: fix status version check across machines
+    
+    The local state isn't propagated into the backtick shell, resulting in
+    'unknown' for all remote daemons.  Avoid backticks altogether.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6c7b667badc5e7608b69c533a119a2afc062e257)
+
+commit 8fef9360fe949fbf4ee6e468c36cc1c84b369b6e
+Author: Travis Rhoden <trhoden@gmail.com>
+Date:   Mon Aug 20 13:29:11 2012 -0700
+
+    init-ceph: use SSH in "service ceph status -a" to get version
+    
+    When running "service ceph status -a", a version number was never
+    returned for remote hosts, only for the local.  This was because
+    the command to query the version number didn't use the do_cmd
+    function, which is responsible for running the command over SSH
+    when needed.
+    
+    Modify the ceph init.d script to use do_cmd for querying the
+    Ceph version.
+    
+    Signed-off-by: Travis Rhoden <trhoden@gmail.com>
+    (cherry picked from commit 60fdb6fda6233b01dae4ed8a34427d5960840b84)
+
+commit c34e38bcdc0460219d19b21ca7a0554adf7f7f84
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Nov 28 13:00:36 2012 -0800
+
+    log: 10,000 recent log entries
+    
+    This is what we were (wrongly) doing before, so there are no memory
+    utilization surprises.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 78286b1403a5e0f14f95fe6b92f2fdb163e909f1)
+
+commit 4daede79f578cccd340210752f912e685b1ff03d
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Nov 28 12:59:43 2012 -0800
+
+    log: fix log_max_recent config
+    
+    <facepalm>
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4de7748b72d4f90eb1197a70015c199c15203354)
+
+commit fdae0552a0b8b1a634775609b473aff9221904f3
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Dec 20 13:48:06 2012 -0800
+
+    log: fix flush/signal race
+    
+    We need to signal the cond in the same interval where we hold the lock
+    *and* modify the queue.  Otherwise, we can have a race like:
+    
+     queue has 1 item, max is 1.
+     A: enter submit_entry, signal cond, wait on condition
+     B: enter submit_entry, signal cond, wait on condition
+     C: flush wakes up, flushes 1 previous item
+     A: retakes lock, enqueues something, exits
+     B: retakes lock, condition fails, waits
+      -> C is never woken up as there are 2 items waiting
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 50914e7a429acddb981bc3344f51a793280704e6)
+
+commit bdcf6647dec05617d3da46ad00459498ede94f1d
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Fri Dec 21 17:12:07 2012 -0800
+
+    .gitignore:  Add ar-lib to ignore list
+
+commit 51a900cfd7cc009f03fdb80be028015fc34547d2
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Fri Dec 21 16:55:27 2012 -0800
+
+    autogen.sh:  Create m4 directory for leveldb
+    
+    Signed-off-by: Gary Lowell <gary.lowell@inktank.com>
+
+commit 74473bb667df0ee0bb1adc6c43242733b80aa10d
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Fri Dec 21 16:17:33 2012 -0800
+
+    leveldb:  Update submodule
+    
+    Signed-off-by: Gary Lowell <gary.lowell@inktank.com>
+
+commit bc4f74c7b9ec4e6bbcf4ab4ab68a82a69b8eaca5
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Thu Dec 20 16:49:32 2012 -0800
+
+    ceph.spec.in:  Fedora builds debuginfo by default.
+    
+    Signed-off-by: Gary Lowell <gary.lowell@inktank.com>
+
+commit 3ed2d59e98494a0da29f4bb8537471237a6f8e93
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Dec 19 17:07:18 2012 -0800
+
+    rgw: fix error handling with swift
+    
+    Fixes: #3649
+    verify_swift_token returns a bool and not an int.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit dfd310368267df848f2b65cc536b4ffcb039d353
+Author: Sam Lang <sam.lang@inktank.com>
+Date:   Mon Sep 24 09:55:25 2012 -0700
+
+    client: Fix for #3184 cfuse segv with no keyring
+    
+    Fixes bug #3184 where the ceph-fuse client segfaults if authx is
+    enabled but no keyring file is present.  This was due to the
+    client->init() return value not getting checked.
+    
+    Signed-off-by: Sam Lang <sam.lang@inktank.com>
+    (cherry picked from commit 47983df4cbd31f299eef896b4612d3837bd7c7bd)
+
+commit acebcce91448421c34a72850a380fcd1aabe9f41
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Oct 9 21:25:54 2012 +0100
+
+    mon: Monitor: resolve keyring option to a file before loading keyring
+    
+    Otherwise our keyring default location, or any other similarly formatted
+    location, will be taken as the actual location for the keyring and fail.
+    
+    Reported-by: tziOm (at) #ceph
+    Fixes: 3276
+    Backport: argonaut
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 7ef0df25e001bfae303feb3ae36514608767b1f2)
+
+commit 7ab00a796d2233c3804f52fbba16c2f3ed0c297b
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Wed Dec 5 19:39:11 2012 -0800
+
+    .gitignore:  Add m4 macro directories to ignore list
+
+commit b781ac00e16fd802b6e055dd9d453271db6efa39
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Thu Nov 8 12:43:24 2012 -0800
+
+    build:  Add RPM release string generated from git describe.
+    
+    Fix for bug 3451.  Use the commit count and sha1 from git describe to
+    construct a release string for rpm packages.
+    
+    Conflicts:
+    
+    	configure.ac
+
+commit f60c9b6600a85b840a02e145bac77326d60fd03f
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Fri Nov 9 13:28:13 2012 -0800
+
+    ceph.spec.in:  Build debuginfo subpackage.
+    
+    This is a partial fix for bug 3471.  Enable building of debuginfo package.
+    Some distributions enable this automatically by installing additional rpm
+    macros, on others it needs to be explicity added to the spec file.
+
+commit 7fbf9476cb44ce256f4bb2366f46fa20cb570eda
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Dec 3 14:32:28 2012 -0800
+
+    rgw: fix swift auth concurrency issue
+    
+    Fixes: #3565
+    Originally ops were using static structures, but that
+    has since changed. Switching swift auth handler to do
+    the same.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 0639cd9c479d69b077175f0385eb569ebb839349
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Nov 29 13:39:22 2012 -0800
+
+    rgw: fix rgw_tools get_obj()
+    
+    The original implementation broke whenever data exceeded
+    the chunk size. Also don't keep cache for objects that
+    exceed the chunk size as cache is not designed for
+    it. Increased chunk size to 512k.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 84983387a220a29540aca64af774bbe7bc6b5ee6
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Nov 29 12:47:59 2012 -0800
+
+    rgw: fix PUT acls
+    
+    This fixes a regression introduced at
+    17e4c0df44781f5ff1d74f3800722452b6a0fc58. The original
+    patch fixed error leak, however it also removed the
+    operation's send_response() call.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 6c1e2b663227cadff4255db3321976882df6b261
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Nov 19 17:10:11 2012 -0800
+
+    rgw: fix xml parser leak
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit f86522cdfcd81b2d28c581ac8b8de6226bc8d1a4)
+
+commit 17e4c0df44781f5ff1d74f3800722452b6a0fc58
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Nov 19 16:52:38 2012 -0800
+
+    rgw: fix memory leaks
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 98a04d76ebffa61c3ba4b033cdd57ac57b2f29f3)
+    
+    Conflicts:
+    	src/rgw/rgw_op.cc
+    	src/rgw/rgw_op.h
+
+commit cd18b9f43cce68ac3d8f109e82a5b603f9caad9b
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Nov 7 13:21:15 2012 -0800
+
+    rgw: don't convert object mtime to UTC
+    
+    Fixes: #3452
+    When we read object info, don't try to convert mtime to
+    UTC, it's already in UTC.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit ddb40dec6e8d930fd63a71ef1c98279d6472e3e4
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Nov 14 11:30:34 2012 -0800
+
+    rgw: relax date format check
+    
+    Don't try to parse beyond the GMT or UTC. Some clients use
+    special date formatting. If we end up misparsing the date
+    it'll fail in the authorization, so don't need to be too
+    restrictive.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit e35e7e2bcb749e3ee00f1f93bc6b17bde8329414
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Oct 30 14:17:56 2012 -0700
+
+    ceph-disk-activate: avoid duplicating mounts if already activated
+    
+    If the given device is already mounted at the target location, do not
+    mount --move it again and create a bunch of dup entries in the /etc/mtab
+    and kernel mount table.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c435d314caeb5424c1f4482ad02f8a085317ad5b)
+
+commit 0c9bbb3b4b4dbe6f0a12960785e35af9c748701a
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Oct 25 21:21:18 2012 -0700
+
+    ceph-disk-prepare: poke kernel into refreshing partition tables
+    
+    Prod the kernel to refresh the partition table after we create one.  The
+    partprobe program is packaged with parted, which we already use, so this
+    introduces no new dependency.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 402e1f5319a52c309eca936081fddede1f107268)
+
+commit b4004806ec387915ba4e039b9bc1c563273ef08f
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Oct 25 21:20:21 2012 -0700
+
+    ceph-disk-prepare: fix journal partition creation
+    
+    The end value needs to have + to indicate it is relative to wherever the
+    start is.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2e32a0ee2d9e2a3bf5b138f50efc5fba8d5b8660)
+
+commit efac1bb19f1ca3851c7b51d96cb4e82d8d142a45
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Oct 25 18:14:47 2012 -0700
+
+    ceph-disk-prepare: assume parted failure means no partition table
+    
+    If the disk has no valid label we get an error like
+    
+      Error: /dev/sdi: unrecognised disk label
+    
+    Assume any error we get is that and go with an id label of 1.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 8921fc7c7bc28fb98334c06f1f0c10af58085085)
+
+commit 39b9da403541617fb088aa4ae03925850aa2de65
+Merge: 83ab366 8c115d3
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Nov 12 11:24:00 2012 -0800
+
+    Merge remote-tracking branch 'gh/wip-mds-stable' into stable
+
+commit 8c115d3e81543a4c723bae217ba5b9d2c0e988f7
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Nov 9 05:28:12 2012 -0800
+
+    mds: re-try_set_loner() after doing evals in eval(CInode*, int mask)
+    
+    Consider a case where current loner is A and wanted loner is B.
+    At the top of the function we try to set the loner, but that may fail
+    because we haven't processed the gathered caps yet for the previous
+    loner.  In the body we do that and potentially drop the old loner, but we
+    do not try_set_loner() again on the desired loner.
+    
+    Try after our drop.  If it succeeds, loop through the eval's one more time
+    so that we can issue caps approriately.
+    
+    This fixes a hang induced by a simple loop like:
+    
+     while true ; do echo asdf >> mnt.a/foo ; tail mnt.b/foo ; done &
+     while true ; do ls mnt.a mnt.b ; done
+    
+    (The second loop may not be necessary.)
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 1a23bd6981e2a308be4a905021453a6d5feac9e3
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Jul 13 14:23:27 2012 -0700
+
+    CompatSet: users pass bit indices rather than masks
+    
+    CompatSet users number the Feature objects rather than
+    providing masks.  Thus, we should do
+    
+    mask |= (1 << f.id) rather than mask |= f.id.
+    
+    In order to detect old, broken encodings, the lowest
+    bit will be set in memory but not set in the encoding.
+    We can reconstruct the correct mask from the names map.
+    
+    This bug can cause an incompat bit to not be detected
+    since 1|2 == 1|2|3.
+    
+    fixes: #2748
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 83ab36655f397b7d781a611b4d82679c8484710f
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Wed Nov 7 12:41:10 2012 -0800
+
+    ceph.spec.in:  Remove ceph version requirement from ceph-fuse package.
+    
+    The ceph-fuse rpm package now only requires ceph as a pre-req, not a specific
+    version.
+
+commit 578b5764d5b26d190c295da88c1be5f0c77682a7
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Oct 24 13:15:46 2012 -0700
+
+    rgw: fix multipart overwrite
+    
+    Fixes: #3400
+    Removed a few lines of code that prematurely created the head
+    part of the final object (before creating the manifest).
+    
+    backport:argonaut
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 96888ddf154231daa6c7138bb2b97674771cc610
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Nov 5 23:27:13 2012 -0800
+
+    mds: move to from loner -> mix if *anyone* wants rd|wr
+    
+    We were either going to MIX or SYNC depending on whether non-loners wanted
+    to read/write, but it may be that the loner wants to if our logic for
+    choosing loner vs not longer is based on anything other that just rd|wr
+    wanted.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit b940d7750d14285fc85ce63b0f0444b58ad3ad93
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Nov 5 23:26:09 2012 -0800
+
+    mds: base loner decision on wanted RD|WR|EXCL, not CACHE|BUFFER
+    
+    Observed instance where one client wanted the Fc cap and prevented the
+    loner from getting RD|WR caps.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 7a82a3214a38cbc7b5750adba23b760a4f51643a
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Oct 30 09:00:11 2012 -0700
+
+    osd: make pool_snap_info_t encoding backward compatible
+    
+    Way back in fc869dee1e8a1c90c93cb7e678563772fb1c51fb (v0.42) when we redid
+    the osd type encoding we forgot to make this conditionally encode the old
+    format for old clients.  In particular, this means that kernel clients
+    will fail to decode the osdmap if there is a rados pool with a pool-level
+    snapshot defined.
+    
+    Fixes: #3290
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    
+    Conflicts:
+    
+    	src/test/encoding/types.h
+
+commit 8d7ca76182fdc04aaf8b698afbbd2d9b22e07fdc
+Author: Yan, Zheng <zheng.z.yan@intel.com>
+Date:   Fri Sep 7 13:49:27 2012 +0800
+
+    osd/OSD.cc: Fix typo in OSD::heartbeat_check()
+    
+    The check 'p->second.last_tx > cutoff' should always be false
+    since last_tx is periodically updated by OSD::heartbeat()
+    
+    Signed-off-by: Yan, Zheng <zheng.z.yan@intel.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit fc2b3ced4f7ed8d6e065c159fbeb8670fcdaca2a
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Oct 22 16:52:11 2012 -0700
+
+    rgw: dump an error message if FCGX_Accept fails
+    
+    Adding missing debug info.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit d9b51b3b7011f4551fe27e33301b26cd23a72378
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Oct 22 15:38:30 2012 -0700
+
+    workqueue: make debug output include active threads
+    
+    Include active thread count in threadpool debug output.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 852e27d3e19239a696fa693e2a30c3226d27a611
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Oct 22 13:16:59 2012 -0700
+
+    rgw: don't continue processing of GET request on error
+    
+    Fixes #3381
+    We continued processing requests long after the client
+    has died. This fix appliese to both s3 and swift.
+    
+    backport: argonaut
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 5f2b9b451da5a96fdbf9cf292648feb40dc13d28
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Oct 19 08:46:19 2012 -0700
+
+    osd: be quiet about watches
+    
+    Useless log noise.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit ea42eb1f7866fed0e843257ce05e34503fd0129c
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Oct 17 17:44:12 2012 -0700
+
+    addr_parsing: make , and ; and ' ' all delimiters
+    
+    Instead of just ,.  Currently "foo.com, bar.com" will fail because of the
+    space after the comma.  This patches fixes that, and makes all delim
+    chars interchangeable.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit b0164d9902c834c95187f6b277f58a30d26cc4d3
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Fri Oct 5 10:57:42 2012 -0700
+
+    ceph-disk-prepare, debian/control: Support external journals.
+    
+    Previously, ceph-disk-* would only let you use a journal that was a
+    file inside the OSD data directory. With this, you can do:
+    
+      ceph-disk-prepare /dev/sdb /dev/sdb
+    
+    to put the journal as a second partition on the same disk as the OSD
+    data (might save some file system overhead), or, more interestingly:
+    
+      ceph-disk-prepare /dev/sdb /dev/sdc
+    
+    which makes it create a new partition on /dev/sdc to use as the
+    journal. Size of the partition is decided by $osd_journal_size.
+    /dev/sdc must be a GPT-format disk. Multiple OSDs may share the same
+    journal disk (using separate partitions); this way, a single fast SSD
+    can serve as journal for multiple spinning disks.
+    
+    The second use case currently requires parted, so a Recommends: for
+    parted has been added to Debian packaging.
+    
+    Closes: #3078
+    Closes: #3079
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 5a3076fd51706e32bc55b734099037520299003b
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Oct 15 09:43:47 2012 -0700
+
+    rgw: don't add port to url if already has one
+    
+    Fixes: #3296
+    Specifically, is host name string already has ':', then
+    don't try to append theport (swift auth).
+    
+    backport: argonaut
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit d76b71caa99a39035ea6ed9db6ca0994975d38af
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Oct 15 16:37:05 2012 -0700
+
+    admin_socket: fix '0' protocol version
+    
+    Broken by 895e24d198ced83ab7fed3725f12f75e3bc97b0b.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 714254cb0522b194118eb854f90d4c3971e8c6b3
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Oct 8 17:14:22 2012 -0700
+
+    mon: drop command replies on paxos reset
+    
+    If paxos resets, do not send the reply for the commit we were waiting for;
+    let the command be reprocessed and re-proposed.
+    
+    Among other things, this could lead to nondeterministic results for
+    'ceph osd create <uuid>'.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit d9e981f4209da9a0e000d434e28686a133b25f11
+Merge: 5d84b4a 13b448b
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Oct 8 21:02:51 2012 -0700
+
+    Merge remote-tracking branch 'gh/for-stable-fstypes-and-ext-journal' into stable
+
+commit 5d84b4a22b2ff975679711719f336a312a6edff3
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Thu Aug 2 13:02:04 2012 -0700
+
+    ceph-authtool: Fix usage, it's --print-key not --print.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 13b448b8ce959f31d07cd1827614c445c410aa62
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Fri Oct 5 09:22:34 2012 -0700
+
+    upstart: OSD journal can be a symlink; if it's dangling, don't start.
+    
+    This lets a $osd_data/journal symlink point to
+    /dev/disk/by-partuuid/UUID and the osd will not attempt to start until
+    that disk is available.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit d9dfa2d17859ca07d735b8e35c5dc00a25995153
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Oct 5 09:10:31 2012 -0700
+
+    osd: Make --get-journal-fsid not really start the osd.
+    
+    This way, it won't need -i ID and it won't access the osd_data_dir.
+    That makes it useful for locating the right osd to use with an
+    external journal partition.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit f65daf3f24830a28e85ebe7c6fce2642f5fc6dd8
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Fri Oct 5 09:08:56 2012 -0700
+
+    osd: Make --get-journal-fsid not attempt aio or direct_io.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 86546d7dec6cd5080162d1f3d7de2c579736dd65
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Thu Oct 4 16:03:40 2012 -0700
+
+    ceph-disk-prepare: Use the OSD uuid as the partition GUID.
+    
+    This will make locating the right data partition for a given journal
+    partition a lot easier.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit fa3aa41ead0be5a27695e3c208c5af00af90bc53
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Wed Oct 3 12:38:38 2012 -0700
+
+    debian/control, ceph-disk-prepare: Depend on xfsprogs, use xfs by default.
+    
+    Ext4 as a default is a bad choice, as we don't perform enough QA with
+    it. To use XFS as the default for ceph-disk-prepare, we need to depend
+    on xfsprogs.
+    
+    btrfs-tools is already recommended, so no change there. If you set
+    osd_fs_type=btrfs, and don't have the package installed, you'll just
+    get an error message.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 4c3997dc8bbd3985a7b86d8c54b71fbe15af1591
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Wed Oct 3 10:13:17 2012 -0700
+
+    ceph-disk-{prepare,activate}: Default mkfs arguments and mount options.
+    
+    The values for the settings were copied from teuthology task "ceph".
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit a6229b901785ef0a88d1b1886b5b5ae339ac7fcb
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Wed Oct 3 08:47:20 2012 -0700
+
+    ceph-disk-prepare: Avoid triggering activate before prepare is done.
+    
+    Earlier testing never saw this, but now a mount of a disk triggers a
+    udev blockdev-added event, causing ceph-disk-activate to run even
+    before ceph-disk-prepare has had a chance to write the files and
+    unmount the disk.
+    
+    Avoid this by using a temporary partition type uuid ("ceph 2 be"), and
+    only setting it to the permanent ("ceph osd"). The hotplug event won't
+    match the type uuid, and thus won't trigger ceph-disk-activate.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 03cec77e2b0b4536349880de6890ad9537da0783
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Tue Oct 2 17:06:11 2012 -0700
+
+    ceph-disk-activate: Add a comment about user_xattr being default now.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit 4aa13c6d228b9256870e02a57496f00a75e17349
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Tue Oct 2 16:53:35 2012 -0700
+
+    ceph-disk-activate: Use mount options from ceph.conf
+    
+    Always uses default cluster name ("ceph") for now, see
+    http://tracker.newdream.net/issues/3253
+    
+    Closes: #2548
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit b684ab1d1727b23f75f232123f1da947df2e5e63
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Tue Oct 2 16:43:08 2012 -0700
+
+    ceph-disk-activate: Refactor to extract detect_fstype call.
+    
+    This allows us to use the fstype for a config lookup.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit ce5e1f8dfaa8675b0a1e9fa22ebf91be9652aa01
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Tue Oct 2 16:37:07 2012 -0700
+
+    ceph-disk-activate: Unmount on errors (if it did the mount).
+    
+    This cleans up the error handling to not leave disks mounted
+    in /var/lib/ceph/tmp/mnt.* when something fails, e.g. when
+    the ceph command line tool can't talk to mons.
+    
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit da86e4e43676ceaf0b39635954489c297de88025
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Tue Oct 2 16:23:55 2012 -0700
+
+    ceph-disk-prepare: Allow setting mkfs arguments and mount options in ceph.conf
+    
+    Tested with meaningless but easy-to-verify values:
+    
+      [global]
+      osd_fs_type = xfs
+      osd_fs_mkfs_arguments_xfs = -i size=512
+      osd_fs_mount_options_xfs = noikeep
+    
+    ceph-disk-activate does not respect the mount options yet.
+    
+    Closes: #2549
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit cf2e3b15cea99cc100169e7a67103676d8990aee
+Author: Tommi Virtanen <tv@inktank.com>
+Date:   Tue Oct 2 16:04:15 2012 -0700
+
+    ceph-disk-prepare: Allow specifying fs type to use.
+    
+    Either use ceph.conf variable osd_fs_type or command line option
+    --fs-type=
+    
+    Default is still ext4, as currently nothing guarantees xfsprogs
+    or btrfs-tools are installed.
+    
+    Currently both btrfs and xfs seems to trigger a disk hotplug event at
+    mount time, thus triggering a useless and unwanted ceph-disk-activate
+    run. This will be worked around in a later commit.
+    
+    Currently mkfs and mount options cannot be configured.
+    
+    Bug: #2549
+    Signed-off-by: Tommi Virtanen <tv@inktank.com>
+
+commit f031046c4d600ce0a0c2c9ad5856dc100b0901fb
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Sep 26 15:43:56 2012 -0700
+
+    rgw: copy_object should not override ETAG implicitly
+    
+    When copying an object with new attrs, we still need to
+    maintain the ETAG.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 2fddfb0a95288d39cee2e82376297ce43296743b
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Sep 24 18:10:24 2012 -0700
+
+    rgw: url_decode should allocate extra byte for dest
+    
+    Was missing extra byte for null termination
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
diff --git a/doc/changelog/v0.56.1.txt b/doc/changelog/v0.56.1.txt
new file mode 100644
index 000000000..e2cfe9bbc
--- /dev/null
+++ b/doc/changelog/v0.56.1.txt
@@ -0,0 +1,316 @@
+commit e4a541624df62ef353e754391cbbb707f54b16f7
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Mon Jan 7 13:33:30 2013 -0800
+
+    v0.56.1
+
+commit 9aecacda7fbf07f12b210f87cf3dbb53021b068d
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jan 6 08:38:27 2013 -0800
+
+    msg/Pipe: prepare Message data for wire under pipe_lock
+    
+    We cannot trust the Message bufferlists or other structures to be
+    stable without pipe_lock, as another Pipe may claim and modify the sent
+    list items while we are writing to the socket.
+    
+    Related to #3678.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d16ad9263d7b1d3c096f56c56e9631fae8509651)
+
+commit 299dbad490df5e98c04f17fa8e486a718f3c121f
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jan 6 08:33:01 2013 -0800
+
+    msgr: update Message envelope in encode, not write_message
+    
+    Fill out the Message header, footer, and calculate CRCs during
+    encoding, not write_message().  This removes most modifications from
+    Pipe::write_message().
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 40706afc66f485b2bd40b2b4b1cd5377244f8758)
+
+commit 35d2f58305eab6c9b57a92269598b9729e2d8681
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jan 6 08:25:40 2013 -0800
+
+    msg/Pipe: encode message inside pipe_lock
+    
+    This modifies bufferlists in the Message struct, and it is possible
+    for multiple instances of the Pipe to get references on the Message;
+    make sure they don't modify those bufferlists concurrently.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4cfc4903c6fb130b6ac9105baf1f66fbda797f14)
+
+commit 9b23f195df43589d062da95a11abc07c79f3109b
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 5 10:39:08 2013 -0800
+
+    msg/Pipe: associate sending msgs to con inside lock
+    
+    Associate a sending message with the connection inside the pipe_lock.
+    This way if a racing thread tries to steal these messages it will
+    be sure to reset the con point *after* we do such that it the con
+    pointer is valid in encode_payload() (and later).
+    
+    This may be part of #3678.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a058f16113efa8f32eb5503d5443aa139754d479)
+
+commit 6229b5a06f449a470d3211ea94c1c5faf7100876
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 5 09:29:50 2013 -0800
+
+    msg/Pipe: fix msg leak in requeue_sent()
+    
+    The sent list owns a reference to each message.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2a1eb466d3f8e25ec8906b3ca6118a14c4e269d2)
+
+commit 6a00ce0dc24626fdfa210ddec6334bde3c8a20db
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 7 12:58:39 2013 -0800
+
+    osdc/Objecter: fix linger_ops iterator invalidation on pool deletion
+    
+    The call to check_linger_pool_dne() may unregister the linger request,
+    invalidating the iterator.  To avoid this, increment the iterator at
+    the top of the loop.
+    
+    This mirror the fix in 4bf9078286d58c2cd4e85cb8b31411220a377092 for
+    regular non-linger ops.
+    
+    Fixes: #3734
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 62586884afd56f2148205bdadc5a67037a750a9b)
+
+commit a10950f91e6ba9c1620d8fd00a84fc59f983fcee
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 5 20:53:49 2013 -0800
+
+    os/FileJournal: include limits.h
+    
+    Needed for IOV_MAX.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ce49968938ca3636f48fe543111aa219f36914d8)
+
+commit cd194ef3c7082993cae0892a97494f2a917ce2a7
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 4 17:43:41 2013 -0800
+
+    osd: special case CALL op to not have RD bit effects
+    
+    In commit 20496b8d2b2c3779a771695c6f778abbdb66d92a we treat a CALL as
+    different from a normal "read", but we did not adjust the behavior
+    determined by the RD bit in the op.  We tried to fix that in
+    91e941aef9f55425cc12204146f26d79c444cfae, but changing the op code breaks
+    compatibility, so that was reverted.
+    
+    Instead, special-case CALL in the helper--the only point in the code that
+    actually checks for the RD bit.  (And fix one lingering user to use that
+    helper appropriately.)
+    
+    Fixes: #3731
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 988a52173522e9a410ba975a4e8b7c25c7801123)
+
+commit 921e06decebccc913c0e4f61916d00e62e7e1635
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 4 20:46:48 2013 -0800
+
+    Revert "OSD: remove RD flag from CALL ops"
+    
+    This reverts commit 91e941aef9f55425cc12204146f26d79c444cfae.
+    
+    We cannot change this op code without breaking compatibility
+    with old code (client and server).  We'll have to special case
+    this op code instead.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit d3abd0fe0bb402ff403259d4b1a718a56331fc39)
+
+commit 7513e9719a532dc538d838f68e47c83cc51fef82
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Jan 4 12:43:52 2013 -0800
+
+    ReplicatedPG: remove old-head optization from push_to_replica
+    
+    This optimization allowed the primary to push a clone as a single push in the
+    case that the head object on the replica is old and happens to be at the same
+    version as the clone.  In general, using head in clone_subsets is tricky since
+    we might be writing to head during the push.  calc_clone_subsets does not
+    consider head (probably for this reason).  Handling the clone from head case
+    properly would require blocking writes on head in the interim which is probably
+    a bad trade off anyway.
+    
+    Because the old-head optimization only comes into play if the replica's state
+    happens to fall on the last write to head prior to the snap that caused the
+    clone in question, it's not worth the complexity.
+    
+    Fixes: #3698
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e89b6ade63cdad315ab754789de24008cfe42b37)
+
+commit c63c66463a567e8095711e7c853ac8feb065c5c5
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jan 3 17:15:07 2013 -0800
+
+    os/FileStore: fix non-btrfs op_seq commit order
+    
+    The op_seq file is the starting point for journal replay.  For stable btrfs
+    commit mode, which is using a snapshot as a reference, we should write this
+    file before we take the snap.  We normally ignore current/ contents anyway.
+    
+    On non-btrfs file systems, however, we should only write this file *after*
+    we do a full sync, and we should then fsync(2) it before we continue
+    (and potentially trim anything from the journal).
+    
+    This fixes a serious bug that could cause data loss and corruption after
+    a power loss event.  For a 'kill -9' or crash, however, there was little
+    risk, since the writes were still captured by the host's cache.
+    
+    Fixes: #3721
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 28d59d374b28629a230d36b93e60a8474c902aa5)
+
+commit b8f061dcdb808a6fc5ec01535b37560147b537de
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Jan 3 09:59:45 2013 -0800
+
+    OSD: for old osds, dispatch peering messages immediately
+    
+    Normally, we batch up peering messages until the end of
+    process_peering_events to allow us to combine many notifies, etc
+    to the same osd into the same message.  However, old osds assume
+    that the actiavtion message (log or info) will be _dispatched
+    before the first sub_op_modify of the interval.  Thus, for those
+    peers, we need to send the peering messages before we drop the
+    pg lock, lest we issue a client repop from another thread before
+    activation message is sent.
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4ae4dce5c5bb547c1ff54d07c8b70d287490cae9)
+
+commit 67968d115daf51762dce65af46b9b843eda592b5
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jan 2 22:38:53 2013 -0800
+
+    osd: move common active vs booting code into consume_map
+    
+    Push osdmaps to PGs in separate method from activate_map() (whose name
+    is becoming less and less accurate).
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a32d6c5dca081dcd8266f4ab51581ed6b2755685)
+
+commit 34266e6bde9f36b1c46144d2341b13605eaa9abe
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jan 2 22:20:06 2013 -0800
+
+    osd: let pgs process map advances before booting
+    
+    The OSD deliberate consumes and processes most OSDMaps from while it
+    was down before it marks itself up, as this is can be slow.  The new
+    threading code does this asynchronously in peering_wq, though, and
+    does not let it drain before booting the OSD.  The OSD can get into
+    a situation where it marks itself up but is not responsive or useful
+    because of the backlog, and only makes the situation works by
+    generating more osdmaps as result.
+    
+    Fix this by calling activate_map() even when booting, and when booting
+    draining the peering_wq on each call.  This is harmless since we are
+    not yet processing actual ops; we only need to be async when active.
+    
+    Fixes: #3714
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 0bfad8ef2040a0dd4a0dc1d3abf3ab5b2019d179)
+
+commit 4034f6c817d1efce5fb9eb8cc0a9327f9f7d7910
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Dec 28 13:07:18 2012 -0800
+
+    log: broadcast cond signals
+    
+    We were using a single cond, and only signalling one waiter.  That means
+    that if the flusher and several logging threads are waiting, and we hit
+    a limit, we the logger could signal another logger instead of the flusher,
+    and we could deadlock.
+    
+    Similarly, if the flusher empties the queue, it might signal only a single
+    logger, and that logger could re-signal the flusher, and the other logger
+    could wait forever.
+    
+    Intead, break the single cond into two: one for loggers, and one for the
+    flusher.  Always signal the (one) flusher, and always broadcast to all
+    loggers.
+    
+    Backport: bobtail, argonaut
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 813787af3dbb99e42f481af670c4bb0e254e4432)
+
+commit 2141454eee3a1727706d48f8efef92f8a2b98278
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jan 2 13:58:44 2013 -0800
+
+    log: fix locking typo/stupid for dump_recent()
+    
+    We weren't locking m_flush_mutex properly, which in turn was leading to
+    racing threads calling dump_recent() and garbling the crash dump output.
+    
+    Backport: bobtail, argonaut
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 43cba617aa0247d714632bddf31b9271ef3a1b50)
+
+commit 936560137516a1fd5e55b52ccab59c408ac2c245
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Dec 28 16:48:22 2012 -0800
+
+    test_filejournal: optionally specify journal filename as an argument
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 483c6f76adf960017614a8641c4dcdbd7902ce33)
+
+commit be0473bbb1feb8705be4fa8f827704694303a930
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Dec 28 16:48:05 2012 -0800
+
+    test_filejournal: test journaling bl with >IOV_MAX segments
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c461e7fc1e34fdddd8ff8833693d067451df906b)
+
+commit de61932793c5791c770855e470e3b5b9ebb53dba
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Dec 28 16:47:28 2012 -0800
+
+    os/FileJournal: limit size of aio submission
+    
+    Limit size of each aio submission to IOV_MAX-1 (to be safe).  Take care to
+    only mark the last aio with the seq to signal completion.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit dda7b651895ab392db08e98bf621768fd77540f0)
+
+commit ded454c669171d4038b087cfdad52a57da222c1f
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Dec 28 15:44:51 2012 -0800
+
+    os/FileJournal: logger is optional
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 076b418c7f03c5c62f811fdc566e4e2b776389b7)
diff --git a/doc/changelog/v0.56.2.txt b/doc/changelog/v0.56.2.txt
new file mode 100644
index 000000000..cd8e402f7
--- /dev/null
+++ b/doc/changelog/v0.56.2.txt
@@ -0,0 +1,1294 @@
+commit 586538e22afba85c59beda49789ec42024e7a061
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Tue Jan 29 23:54:47 2013 -0800
+
+    v0.56.2
+
+commit bcb8dfad9cbb4c6af7ae7f9584e36449a03cd1b6
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Tue Jan 29 23:05:49 2013 -0800
+
+    cls_rbd, cls_rgw: use PRI*64 when printing/logging 64-bit values
+
+    caused segfaults in 32-bit build
+
+    Fixes: #3961
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e253830abac76af03c63239302691f7fac1af381)
+
+commit 5a7c5088cc8f57f75eb594a21bf5fb6661e50978
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Tue Jan 29 15:18:53 2013 -0800
+
+    init-ceph: make ulimit -n be part of daemon command
+
+    ulimit -n from 'max open files' was being set only on the machine
+    running /etc/init.d/ceph.  It needs to be added to the commands to
+    start the daemons, and run both locally and remotely.
+
+    Verified by examining /proc/<pid>/limits on local and remote hosts
+
+    Fixes: #3900
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    Reviewed-by: Loïc Dachary <loic@dachary.org>
+    Reviewed-by: Gary Lowell <gary.lowell@inktank.com>
+    (cherry picked from commit 84a024b647c0ac2ee5a91bacdd4b8c966e44175c)
+
+commit 95677fc599b9bf37ab4c2037b3675fd68f92ebcf
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Sat Jan 12 01:06:36 2013 +0000
+
+    mon: OSDMonitor: only share osdmap with up OSDs
+
+    Try to share the map with a randomly picked OSD; if the picked monitor is
+    not 'up', then try to find the nearest 'up' OSD in the map by doing a
+    backward and a forward linear search on the map -- this would be O(n) in
+    the worst case scenario, as we only do a single iteration starting on the
+    picked position, incrementing and decrementing two different iterators
+    until we find an appropriate OSD or we exhaust the map.
+
+    Fixes: #3629
+    Backport: bobtail
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3610e72e4f9117af712f34a2e12c5e9537a5746f)
+
+commit e4d76cb8594c0ec901f89c2f2e8cc53e00eb2a06
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Sun Jan 27 21:57:31 2013 +0100
+
+    utime: fix narrowing conversion compiler warning in sleep()
+
+    Fix compiler warning:
+    ./include/utime.h: In member function 'void utime_t::sleep()':
+    ./include/utime.h:139:50: warning: narrowing conversion of
+     '((utime_t*)this)->utime_t::tv.utime_t::<anonymous struct>::tv_sec' from
+     '__u32 {aka unsigned int}' to '__time_t {aka long int}' inside { } is
+     ill-formed in C++11 [-Wnarrowing]
+    ./include/utime.h:139:50: warning: narrowing conversion of
+     '((utime_t*)this)->utime_t::tv.utime_t::<anonymous struct>::tv_nsec' from
+     '__u32 {aka unsigned int}' to 'long int' inside { } is
+     ill-formed in C++11 [-Wnarrowing]
+
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 014fc6d6c1c68e2e3ad0117d08c4e46e4030d49e)
+
+commit a8964107ddf02ac4a6707a997e1b634c1084a3b9
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Jan 28 17:13:23 2013 -0800
+
+    rgw: fix crash when missing content-type in POST object
+
+    Fixes: #3941
+    This fixes a crash when handling S3 POST request and content type
+    is not provided.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit f41010c44b3a4489525d25cd35084a168dc5f537)
+
+commit 11e1f3acf0953e9ac38322c0423144eaabd7bb61
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Jan 11 15:00:02 2013 -0800
+
+    ReplicatedPG: make_snap_collection when moving snap link in snap_trimmer
+
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 88956e3186798058a1170803f8abfc0f3cf77a07)
+
+commit c9201d0e9de5f4766a2d9f4715eb7c69691964de
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Jan 11 16:43:14 2013 -0800
+
+    ReplicatedPG: correctly handle new snap collections on replica
+
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 9e44fca13bf1ba39dbcad29111b29f46c49d59f7)
+
+commit 2efdfb41c1bc9128b76416630ee00a75de90c020
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Sun Jan 27 18:08:15 2013 +0000
+
+    mon: Elector: reset the acked leader when the election finishes and we lost
+
+    Failure to do so will mean that we will always ack the same leader during
+    an election started by another monitor.  This had been working so far
+    because we were still acking the existing leader if he was supposed to
+    still be the leader; or we were acking a new potentially leader; or we
+    would eventually fall behind on an election and start a new election
+    ourselves, thus resetting the previously acked leader.  While this wasn't
+    something that mattered much until now, the timechecks code stumbled into
+    this tiny issue and was failing hard at completing a round because there
+    wouldn't be a reset before the election started -- timechecks are bound
+    to election epochs.
+
+    Fixes: #3854
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit c54781618569680898e77e151dd7364f22ac4aa1)
+
+commit a16c6f3dc278e19e66776ffde45de3ff0db46a6c
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Dec 26 14:24:22 2012 -0800
+
+    rbd: fix bench-write infinite loop
+
+    I/O was continously submitted as long as there were few enough ops in
+    flight. If the number of 'threads' was high, or caching was turned on,
+    there would never be that many ops in flight, so the loop would continue
+    indefinitely. Instead, submit at most io_threads ops per offset.
+
+    Fixes: #3413
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    Reviewed-by: Sage Weil <sage.weil@inktank.com>
+    (cherry picked from commit d81ac8418f9e6bbc9adcc69b2e7cb98dd4db6abb)
+
+commit 76f93751d3603e3fb5c4b9e14bfdac406d8d1a58
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Fri Jan 4 18:00:24 2013 -0800
+
+    rbd: Don't call ProgressContext's finish() if there's an error.
+
+    do_copy was different from the others; call pc.fail() on error and
+    do not call pc.finish().
+
+    Fixes: #3729
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 0978dc4963fe441fb67afecb074bc7b01798d59d)
+
+commit 10053b14623f9c19727cb4d2d3a6b62945bef5c1
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Jan 2 14:15:24 2013 -0800
+
+    librbd: establish watch before reading header
+
+    This eliminates a window in which a race could occur when we have an
+    image open but no watch established. The previous fix (using
+    assert_version) did not work well with resend operations.
+
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit c4370ff03f8ab655a009cfd9ba3a0827d8c58b11)
+
+commit f666c617f6a5f8d94ce81461942c9f94a0775fb2
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Jan 2 12:32:33 2013 -0800
+
+    Revert "librbd: ensure header is up to date after initial read"
+
+    Using assert version for linger ops doesn't work with retries,
+    since the version will change after the first send.
+    This reverts commit e1776809031c6dad441cfb2b9fac9612720b9083.
+
+    Conflicts:
+
+	qa/workunits/rbd/watch_correct_version.sh
+    (cherry picked from commit e0858fa89903cf4055889c405f17515504e917a0)
+
+commit 575a58666adbca83d15468899272e8c369e903e1
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jan 23 22:16:49 2013 -0800
+
+    os/FileStore: only adjust up op queue for btrfs
+
+    We only need to adjust up the op queue limits during commit for btrfs,
+    because the snapshot initiation (async create) is currently
+    high-latency and the op queue is quiesced during that period.
+
+    This lets us revert 44dca5c, which disabled the extra allowance because
+    it is generally bad for non-btrfs writeahead mode.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 38871e27eca5a34de78db23aa3663f6cb045d461)
+
+commit c9eb1b0a99b0e55f7d7343176dad17d1a53589a1
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jan 24 10:52:46 2013 -0800
+
+    common/HeartbeatMap: fix uninitialized variable
+
+    Introduced by me in 132045ce085e8584a3e177af552ee7a5205b13d8.  Thank you,
+    valgrind!
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 00cfe1d3af286ffab7660933415684f18449720c)
+
+commit e6bceeedb0b77d23416560bd951326587470aacb
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Jan 25 11:31:29 2013 -0800
+
+    sharedptr_registry: remove extaneous Mutex::Locker declaration
+
+    For some reason, the lookup() retry loop (for when happened to
+    race with a removal and grab an invalid WeakPtr) locked
+    the lock again.  This causes the #3836 crash since the lock
+    is already locked.  It's rare since it requires a lookup between
+    invalidation of the WeakPtr and removal of the WeakPtr entry.
+
+    Fixes: #3836
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 037900dc7a051ce2293a4ef9d0e71911b29ec159)
+
+commit 60888cafdc53d6b381cd634170646c12669e1754
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Jan 24 12:02:09 2013 -0800
+
+    FileStore: ping TPHandle after each operation in _do_transactions
+
+    Each completed operation in the transaction proves thread
+    liveness, a stuck thread should still trigger the timeouts.
+
+    Fixes: #3928
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 0c1cc687b6a40d3c6a26671f0652e1b51c3fd1af)
+
+commit 6b8a673f88cbaca2891834dd5d2137a0e076fd1e
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Jan 24 11:07:37 2013 -0800
+
+    OSD: use TPHandle in peering_wq
+
+    Implement _process overload with TPHandle argument and use
+    that to ping the hb map between pgs and between map epochs
+    when advancing a pg.  The thread will still timeout if
+    genuinely stuck at any point.
+
+    Fixes: 3905
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit e0511f4f4773766d04e845af2d079f82f3177cb6)
+
+commit aa6d20aac22d4c14ff059dbc28e06b7a5e5d6de1
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Jan 24 11:04:04 2013 -0800
+
+    WorkQueue: add TPHandle to allow _process to ping the hb map
+
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 4f653d23999b24fc8c65a59f14905db6630be5b5)
+
+commit e66a75052a340b15693f08b05f7f9f5d975b0978
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jan 23 12:49:04 2013 -0800
+
+    ReplicatedPG: handle omap > max_recovery_chunk
+
+    span_of fails if len == 0.
+
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 8a97eef1f7004988449bd7ace4c69d5796495139)
+
+commit 44f0407a6b259e87803539ec9e942043de0cf35d
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jan 23 12:18:31 2013 -0800
+
+    ReplicatedPG: correctly handle omap key larger than max chunk
+
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit c3dec3e30a85ecad0090c75a38f28cb83e36232e)
+
+commit 50fd6ac9f147a4418d64dfe08843402e7cfb4910
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jan 23 12:15:10 2013 -0800
+
+    ReplicatedPG: start scanning omap at omap_recovered_to
+
+    Previously, we started scanning omap after omap_recovered_to.
+    This is a problem since the break in the loop implies that
+    omap_recovered_to is the first key not recovered.
+
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 09c71f2f5ee9929ac4574f4c35fb8c0211aad097)
+
+commit 4b32eecba2e2bd8e8ea17e1888e6971d31e71439
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jan 23 11:50:13 2013 -0800
+
+    ReplicatedPG: don't finish_recovery_op until the transaction completes
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 62a4b96831c1726043699db86a664dc6a0af8637)
+
+commit da34c77b93e3f880c01329711ab8eca7776b1830
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jan 23 11:35:47 2013 -0800
+
+    ReplicatedPG: ack push only after transaction has completed
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 20278c4f77b890d5b2b95d2ccbeb4fbe106667ac)
+
+commit f9381c74931b80294e5df60f6d2e69c946b8fe88
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jan 23 11:13:28 2013 -0800
+
+    ObjectStore: add queue_transactions with oncomplete
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 4d6ba06309b80fb21de7bb5d12d5482e71de5f16)
+
+commit e2560554f0568c30c786632723c5ce0c86043359
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jan 22 21:18:45 2013 -0800
+
+    common/HeartbeatMap: inject unhealthy heartbeat for N seconds
+
+    This lets us test code that is triggered by an unhealthy heartbeat in a
+    generic way.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 132045ce085e8584a3e177af552ee7a5205b13d8)
+
+commit cbe8b5bca40fd63a382b1a903087e7c34b314985
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jan 22 18:08:22 2013 -0800
+
+    os/FileStore: add stall injection into filestore op queue
+
+    Allow admin to artificially induce a stall in the op queue.  Forces the
+    thread(s) to sleep for N seconds.  We pause for 1 second increments and
+    recheck the value so that a previously stalled thread can be unwedged by
+    reinjecting a lower value (or 0).  To stall indefinitely, just injust
+    very large number.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 657df852e9c89bfacdbce25ea014f7830d61e6aa)
+
+commit beb6ca44cd0e7fc405360e6da974252cb76e7039
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jan 22 18:03:10 2013 -0800
+
+    osd: do not join cluster if not healthy
+
+    If our internal heartbeats are failing, do not send a boot message and try
+    to join the cluster.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a4e78652cdd1698e8dd72dda51599348d013e5e0)
+
+commit 1ecdfca3a3b4985ebd182a5f399c7b15af258663
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jan 22 18:01:07 2013 -0800
+
+    osd: hold lock while calling start_boot on startup
+
+    This probably doesn't strictly matter because start_boot doesn't need the
+    lock (currently) and few other threads should be running, but it is
+    better to be consistent.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c406476c0309792c43df512dddb2fe0f19835e71)
+
+commit e120bf20b3c7213fbde20907e158792dd36c8e54
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jan 22 17:56:32 2013 -0800
+
+    osd: do not reply to ping if internal heartbeat is not healthy
+
+    If we find that our internal threads are stalled, do not reply to ping
+    requests.  If we do this long enough, peers will mark us down.  If we are
+    only transiently unhealthy, we will reply to the next ping and they will
+    be satisfied.  If we are unhealthy and marked down, and eventually recover,
+    we will mark ourselves back up.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ad6b231127a6bfcbed600a7493ca3b66c68484d2)
+
+commit 5f396e2b9360401dfe4dc2afa6acc37df8580c80
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jan 22 17:53:40 2013 -0800
+
+    osd: reduce op thread heartbeat default 30 -> 15 seconds
+
+    If the thread stalls for 15 seconds, let our internal heartbeat fail.
+    This will let us internally respond more quickly to a stalled or failing
+    disk.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 61eafffc3242357d9add48be9308222085536898)
+
+commit fca288b718ef4582d65ff4b9d1fc87ba53d7fd8d
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 21:02:01 2013 -0800
+
+    osd: improve sub_op flag points
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 73a969366c8bbd105579611320c43e2334907fef)
+
+commit f13ddc8a2df401c37f6dc792eb93fc0cc45705e2
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 20:55:20 2013 -0800
+
+    osd: refactor ReplicatedPG::do_sub_op
+
+    PULL is the only case where we don't wait for active.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 23c02bce90c9725ccaf4295de3177e8146157723)
+
+commit d5e00f963f177745f0e0684d5977460b7ab59fbd
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 16:36:36 2013 -0800
+
+    osd: make last state for slow requests more informative
+
+    Report on the last event string, and pass in important context for the
+    op event list, including:
+
+     - which peers were sent sub ops and we are waiting for
+     - which pg queue we are delayed by
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a1137eb3e168c2d00f93789e4d565c1584790df0)
+
+commit ab3a110cbe16b548bb96225656b64507aa67e78f
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 15:59:07 2013 -0800
+
+    osd: dump op priority queue state via admin socket
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 24d0d7eb0165c8b8f923f2d8896b156bfb5e0e60)
+
+commit 43a65d04d8a13621a856baec85fb741971c13cb0
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 15:50:33 2013 -0800
+
+    osd: simplify asok to single callback
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 33efe32151e04beaafd9435d7f86dc2eb046214d)
+
+commit d040798637da03e3df937181de156714fc62a550
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 15:58:57 2013 -0800
+
+    common/PrioritizedQueue: dump state to Formatter
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 514af15e95604bd241d2a98a97b938889c6876db)
+
+commit 691fd505ad606bd8befd2b19113ee51a17a0a543
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 15:29:28 2013 -0800
+
+    common/PrioritizedQueue: add min cost, max tokens per bucket
+
+    Two problems.
+
+    First, we need to cap the tokens per bucket.  Otherwise, a stream of
+    items at one priority over time will indefinitely inflate the tokens
+    available at another priority.  The cap should represent how "bursty"
+    we allow a given bucket to be.  Start with 4MB for now.
+
+    Second, set a floor on the item cost.  Otherwise, we can have an
+    infinite queue of 0 cost items that start over queues.  More
+    realistically, we need to balance the overhead of processing small items
+    with the cost of large items.  I.e., a 4 KB item is not 1/1000th as
+    expensive as a 4MB item.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6e3363b20e590cd9df89f2caebe71867b94cc291)
+
+commit a2b03fe08044b5c121ea6b4c2f9d19e73e4c83d1
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 14:52:54 2013 -0800
+
+    common/PrioritizedQueue: buckets -> tokens
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c549a0cf6fae78c8418a3b4b0702fd8a1e4ce482)
+
+commit 612d75cdee0daf9dfca97831c249e1ac3fbd59fc
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 14:31:00 2013 -0800
+
+    note puller's max chunk in pull requests
+
+    this lets us calculate a cost value
+    (cherry picked from commit 128fcfcac7d3fb66ca2c799df521591a98b82e05)
+
+commit 2224e413fba11795693025fa8f11c3f1fba4bbaa
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 14:14:25 2013 -0800
+
+    osd: add OpRequest flag point when commit is sent
+
+    With writeahead journaling in particular, we can get requests that
+    stay in the queue for a long time even after the commit is sent to the
+    client while we are waiting for the transaction to apply to the fs.
+    Instead of showing up as 'waiting for subops', make it clear that the
+    client has gotten its reply and it is local state that is slow.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b685f727d4c37a26cb78bd4a04cce041428ceb52)
+
+commit 5b5ca5926258e4f0b5041fb2c15b1c2f904c4adb
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 13:57:59 2013 -0800
+
+    osd: set PULL subop cost to size of requested data
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a1bf8220e545f29b83d965f07b1abfbea06238b3)
+
+commit 10651e4f500d7b55d8c689a10a61d2239b3ecd26
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 13:57:38 2013 -0800
+
+    osd: use Message::get_cost() function for queueing
+
+    The data payload is a decent proxy for cost in most cases, but not all.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e8e0da1a577e24cd4aad71fb94d8b244e2ac7300)
+
+commit 9735c6b163f4d226d8de6508d5c1534d18f1c300
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 13:25:21 2013 -0800
+
+    osd: debug msg prio, cost, latency
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit bec96a234c160bebd9fd295df5b431dc70a2cfb3)
+
+commit c48279da7ad98013ce97eab89c17fe9fae1ba866
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 21:05:00 2013 -0800
+
+    filestore: filestore_queue_max_ops 500 -> 50
+
+    Having a deep queue limits the effectiveness of the priority queues
+    above by adding additional latency.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 40654d6d53436c210b2f80911217b044f4d7643a)
+
+commit f47b2e8b607cc0d56a42ec7b1465ce6b8c0ca68c
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 20:00:26 2013 -0800
+
+    osd: target transaction size 300 -> 30
+
+    Small transactions make pg removal nicer to the op queue.  It also slows
+    down PG deletion a bit, which may exacerbate the PG resurrection case
+    until #3884 is addressed.
+
+    At least on user reported this fixed an osd that kept failing due to
+    an internal heartbeat failure.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1233e8617098766c95100aa9a6a07db1a688e290)
+
+commit 4947f0efadf9ef209d02fd17f5f86b9a7d6523ef
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 21 19:55:26 2013 -0800
+
+    os/FileStore: allow filestore_queue_max_{ops,bytes} to be adjusted at runtime
+
+    The 'committing' ones too.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cfe4b8519363f92f84f724a812aa41257402865f)
+
+commit ad6e6c91f61c092bfc9f88b788ccbee6438fd40b
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 19 22:06:27 2013 -0800
+
+    osd: make osd_max_backfills dynamically adjustable
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 101955a6b8bfdf91f4229f4ecb5d5b3da096e160)
+
+commit 939b1855245bc9cb31f5762027f2ed3f2317eb55
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 19 18:28:35 2013 -0800
+
+    osd: make OSD a config observer
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 9230c863b3dc2bdda12c23202682a84c48f070a1)
+
+    Conflicts:
+
+	src/osd/OSD.cc
+
+commit b0f27a8f81feb401407bed784bf5d4d799998ee0
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Tue Jan 8 11:21:22 2013 -0800
+
+    librbd: Allow get_lock_info to fail
+
+    If the lock class isn't present, EOPNOTSUPP is returned for lock calls
+    on newer OSDs, but sadly EIO on older; we need to treat both as
+    acceptable failures for RBD images.  rados lock list will still fail.
+
+    Fixes #3744.
+
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4483285c9fb16f09986e2e48b855cd3db869e33c)
+
+commit 022a5254b4fac3f76220abdde2a2e81de33cb8dc
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 4 13:00:56 2013 -0800
+
+    osd: drop newlines from event descriptions
+
+    These produce extra newlines in the log.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 9a1f574283804faa6dbba9165a40558e1a6a1f13)
+
+commit ebc93a878c8b0697004a619d6aa957a80b8b7e35
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Jan 18 14:35:51 2013 -0800
+
+    OSD: do deep_scrub for repair
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 0cb760f31b0cb26f022fe8b9341e41cd5351afac)
+
+commit 32527fa3eb48a7d7d5d67c39bfa05087dbc0e41b
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Jan 14 12:52:04 2013 -0800
+
+    ReplicatedPG: ignore snap link info in scrub if nlinks==0
+
+    links==0 implies that the replica did not sent snap link information.
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 70c3512037596a42ba6eb5eb7f96238843095db9)
+
+commit 13e42265db150b19511a5a618c7a95ad801290c8
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 11 12:25:22 2013 -0800
+
+    osd/PG: fix osd id in error message on snap collection errors
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 381e25870f26fad144ecc2fb99710498e3a7a1d4)
+
+commit e3b6191fc45c7d2c27ec75c867be822a6da17e9a
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jan 9 22:34:12 2013 -0800
+
+    osd/ReplicatedPG: validate ino when scrubbing snap collections
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 665577a88b98390b9db0f9991836d10ebdd8f4cf)
+
+commit 353b7341caff86f936a429669de52e6949a89c2b
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jan 9 16:41:40 2013 -0800
+
+    ReplicatedPG: compare nlinks to snapcolls
+
+    nlinks gives us the number of hardlinks to the object.
+    nlinks should be 1 + snapcolls.size().  This will allow
+    us to detect links which remain in an erroneous snap
+    collection.
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit e65ea70ea64025fbb0709ee8596bb2878be0bbdc)
+
+commit 33d5cfc8c080a270d65275f8e010a6468c77381a
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Jan 10 15:35:10 2013 -0800
+
+    ReplicatedPG/PG: check snap collections during _scan_list
+
+    During _scan_list check the snapcollections corresponding to the
+    object_info attr on the object.  Report inconsistencies during
+    scrub_finalize.
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 57352351bb86e0ae9f64f9ba0d460c532d882de6)
+
+commit bea783bd722d862a5018477a637c843fe4b18a58
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jan 9 11:53:52 2013 -0800
+
+    osd_types: add nlink and snapcolls fields to ScrubMap::object
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit b85687475fa2ec74e5429d92ee64eda2051a256c)
+
+commit 0c48407bf46b39b2264a7be14e9d3caa2c1e5875
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Jan 3 20:16:50 2013 -0800
+
+    PG: move auth replica selection to helper in scrub
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 39bc65492af1bf1da481a8ea0a70fe7d0b4b17a3)
+
+commit c3433ce60ec3683217d8b4cd2b6e75fb749af2c6
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 14 18:23:52 2013 -0800
+
+    mon: note scrub errors in health summary
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 8e33a8b9e1fef757bbd901d55893e9b84ce6f3fc)
+
+commit 90c6edd0155b327c48a5b178d848d9e5839bd928
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 14 18:31:06 2013 -0800
+
+    osd: fix rescrub after repair
+
+    We were rescrubbing if INCONSISTENT is set, but that is now persistent.
+    Add a new scrub_after_recovery flag that is reset on each peering interval
+    and set that when repair encounters errors.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a586966a3cfb10b5ffec0e9140053a7e4ff105d2)
+
+commit 0696cf57283e6e9a3500c56ca5fc9f981475ca26
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 14 18:22:02 2013 -0800
+
+    osd: note must_scrub* flags in PG operator<<
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d56af797f996ac92bf4e0886d416fd358a2aa08e)
+
+commit 1541ffe4bec6cce607c505271ff074fd0a292d30
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 14 18:21:46 2013 -0800
+
+    osd: based INCONSISTENT pg state on persistent scrub errors
+
+    This makes the state persistent across PG peering and OSD restarts.
+
+    This has the side-effect that, on recovery, we rescrub any PGs marked
+    inconsistent.  This is new behavior!
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2baf1253eed630a7c4ae4cb43aab6475efd82425)
+
+commit 609101255c81d977072b2ab741ac47167d9b1b16
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 14 18:20:29 2013 -0800
+
+    osd: fix scrub scheduling for 0.0
+
+    The initial value for pair<utime_t,pg_t> can match pg 0.0, preventing it
+    from being manually scrubbed.  Fix!
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 26a63df97b2a12fd1a7c1e3cc9ccd34ca2ef9834)
+
+commit 0961a3a85c286a31ec2e8bba23217bbd3974572c
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jan 13 23:03:01 2013 -0800
+
+    osd: note last_clean_scrub_stamp, last_scrub_errors
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 389bed5d338cf32ab14c9fc2abbc7bcc386b8a28)
+
+commit 8d823045538bf4c51506e349b5c6705fd76450f8
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jan 13 22:59:39 2013 -0800
+
+    osd: add num_scrub_errors to object_stat_t
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2475066c3247774a2ad048a2e32968e47da1b0f5)
+
+commit 3a1cd6e07b4e2a4714de159f69afd689495e2927
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jan 13 22:43:35 2013 -0800
+
+    osd: add last_clean_scrub_stamp to pg_stat_t, pg_history_t
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d738328488de831bf090f23e3fa6d25f6fa819df)
+
+commit 7e5a899bdcf6c08a5f6f5c98cd2fff7fa2dacaca
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jan 13 22:56:14 2013 -0800
+
+    osd: fix object_stat_sum_t dump signedness
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6f6a41937f1bd05260a8d70b4c4a58ecadb34a2f)
+
+commit e252a313d465006d3fe4db97939ad307ebe91c71
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jan 13 22:04:58 2013 -0800
+
+    osd: change scrub min/max thresholds
+
+    The previous 'osd scrub min interval' was mostly meaningless and useless.
+    Meanwhile, the 'osd scrub max interval' would only trigger a scrub if the
+    load was sufficiently low; if it was high, the PG might *never* scrub.
+
+    Instead, make the 'min' what the max used to be.  If it has been more than
+    this many seconds, and the load is low, scrub.  And add an additional
+    condition that if it has been more than the max threshold, scrub the PG
+    no matter what--regardless of the load.
+
+    Note that this does not change the default scrub interval for less-loaded
+    clusters, but it *does* change the meaning of existing config options.
+
+    Fixes: #3786
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 299548024acbf8123a4e488424c06e16365fba5a)
+
+    Conflicts:
+
+	PendingReleaseNotes
+
+commit 33aa64eee34f4759f6000130de4d1306de49d087
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jan 13 20:27:59 2013 -0800
+
+    osd/PG: remove useless osd_scrub_min_interval check
+
+    This was already a no-op: we don't call PG::scrub_sched() unless it has
+    been osd_scrub_max_interval seconds since we last scrubbed.  Unless we
+    explicitly requested in, in which case we don't want this check anyway.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 16d67c798b6f752a6e03084bafe861396b86baae)
+
+commit fdd0c1ec3519376980a205b94e65187833634e2e
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jan 13 20:25:39 2013 -0800
+
+    osd: move scrub schedule random backoff to seperate helper
+
+    Separate this from the load check, which will soon vary dependon on the
+    PG.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a148120776d0930b265411332a60e93abfbf0423)
+
+commit 9ffbe268f785e1a74c0d893735117edb7a3ef377
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 12 09:18:38 2013 -0800
+
+    osd/PG: trigger scrub via scrub schedule, must_ flags
+
+    When a scrub is requested, flag it and move it to the front of the
+    scrub schedule instead of immediately queuing it.  This avoids
+    bypassing the scrub reservation framework, which can lead to a heavier
+    impact on performance.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 62ee6e099a8e4873287b54f9bba303ea9523d040)
+
+commit cffb1b22d5df7300ec411d2b620bf3c4a08351cd
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 12 09:15:16 2013 -0800
+
+    osd/PG: introduce flags to indicate explicitly requested scrubs
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1441095d6babfacd781929e8a54ed2f8a4444467)
+
+commit 438e3dfc88bfdc8eb36b5b5f7b728b2610476724
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 12 09:14:01 2013 -0800
+
+    osd/PG: move scrub schedule registration into a helper
+
+    Simplifies callers, and will let us easily modify the decision of when
+    to schedule the PG for scrub.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 796907e2159371f84a16cbd35f6caa8ac868acf6)
+
+commit acb47e4d7dc9682937984661a9d754131d806630
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 18 12:14:48 2013 -0800
+
+    os/FileStore: only flush inline if write is sufficiently large
+
+    Honor filestore_flush_min in the inline flush case.
+
+    Backport: bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 49726dcf973c38c7313ab78743b45ccc879671ea)
+
+commit 15a1ced859629c361da127799b05620bee84c9a8
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 18 12:14:40 2013 -0800
+
+    os/FileStore: fix compile when sync_file_range is missing;
+
+    If sync_file_range is not present, we always close inline, and flush
+    via fdatasync(2).
+
+    Fixes compile on ancient platforms like RHEL5.8.
+
+    Backport: bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 8ddb55d34c72e6df1023cf427cbd41f3f98da402)
+
+commit 9dddb9d855e6d5fd804b54bff1f726c1d2fb566c
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 18 15:23:22 2013 -0800
+
+    osd: set pg removal transactions based on configurable
+
+    Use the osd_target_transaction_size knob, and gracefully tolerate bogus
+    values (e.g., <= 0).
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5e00af406b89c9817e9a429f92a05ca9c29b19c3)
+
+commit c30d231e40a17c3fb08d1db5e01133466170e90c
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 18 15:30:06 2013 -0800
+
+    osd: make pg removal thread more friendly
+
+    For a large PG these are saturating the filestore and journal queues.  Do
+    them synchronously to make them more friendly.  They don't need to be fast.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4712e984d3f62cdf51ea67da8197eed18a5983dd)
+
+commit b2bc4b95fefaeb0cfc31ce0bc95b77062d0777c7
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 18 15:27:24 2013 -0800
+
+    os: move apply_transactions() sync wrapper into ObjectStore
+
+    This has nothing to do with the backend implementation.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit bc994045ad67fb70c7a0457b8cd29273dd5d1654)
+
+commit 6d161b57979246ddea4e6309e0e489ab729eec4b
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 18 15:28:24 2013 -0800
+
+    os: add apply_transaction() variant that takes a sequencer
+
+    Also, move the convenience wrappers into the interface and funnel through
+    a single implementation.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f6c69c3f1ac35546b90315fff625993ba5cd8c07)
+
+commit c5fe0965572c074a2a33660719ce3222d18c1464
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jan 20 16:11:10 2013 -0800
+
+    osd: calculate initial PG mapping from PG's osdmap
+
+    The initial values of up/acting need to be based on the PG's osdmap, not
+    the OSD's latest.  This can cause various confusion in
+    pg_interval_t::check_new_interval() when calling OSDMap methods due to the
+    up/acting OSDs not existing yet (for example).
+
+    Fixes: #3879
+    Reported-by: Jens Kristian S?gaard <jens@mermaidconsulting.dk>
+    Tested-by: Jens Kristian S?gaard <jens@mermaidconsulting.dk>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 17160843d0c523359d8fa934418ff2c1f7bffb25)
+
+commit 6008b1d8e4587d5a3aea60684b1d871401496942
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jan 17 15:01:35 2013 -0800
+
+    osdmap: make replica separate in default crush map configurable
+
+    Add 'osd crush chooseleaf type' option to control what the default
+    CRUSH rule separates replicas across.  Default to 1 (host), and set it
+    to 0 in vstart.sh.
+
+    Fixes: #3785
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit c236a51a8040508ee893e4c64b206e40f9459a62)
+
+commit 5fb77bf1d1b241b4f9c1fe9e57288bbc84d8d97d
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jan 16 14:09:53 2013 -0800
+
+    ceph: adjust crush tunables via 'ceph osd crush tunables <profile>'
+
+    Make it easy to adjust crush tunables.  Create profiles:
+
+     legacy: the legacy values
+     argonaut: the argonaut defaults, and what is supported.. legacy! (*(
+     bobtail: best that bobtail supports
+     optimal: the current optimal values
+     default: the current default values
+
+    * In actuality, argonaut supports some of the tunables, but it doesn't
+      say so via the feature bits.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 19ee23111585f15a39ee2907fa79e2db2bf523f0)
+
+commit 8c0d702e6f2ba0ed0fe31c06c7a028260ae08e42
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Dec 28 17:20:43 2012 -0800
+
+    msg/Pipe: use state_closed atomic_t for _lookup_pipe
+
+    We shouldn't look at Pipe::state in SimpleMessenger::_lookup_pipe() without
+    holding pipe_lock.  Instead, use an atomic that we set to non-zero only
+    when transitioning to the terminal STATE_CLOSED state.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 82f8bcddb5fa09913eb477ee26c71d6b4bb8d97c)
+
+commit 8e0359c3e586c0edcce769c8ed1a03444a521165
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Dec 23 13:43:15 2012 -0800
+
+    msgr: inject delays at inconvenient times
+
+    Exercise some rare races by injecting delays before taking locks
+    via the 'ms inject internal delays' option.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a5d692a7b9b4bec2c27993ca37aa3fec4065292b)
+
+commit 34e2d4024700f633c2c586265efb61537342db18
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Dec 23 09:22:18 2012 -0800
+
+    msgr: fix race on Pipe removal from hash
+
+    When a pipe is faulting and shutting down, we have to drop pipe_lock to
+    take msgr lock and then remove the entry.  The Pipe in this case will
+    have STATE_CLOSED.  Handle this case in all places we do a lookup on
+    the rank_pipe hash so that we effectively ignore entries that are
+    CLOSED.
+
+    This fixes a race introduced by the previous commit where we won't use
+    the CLOSED pipe and try to register a new one, but the old one is still
+    registered.
+
+    See bug #3675.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e99b4a307b4427945a4eb5ec50e65d6239af4337)
+
+commit ae1882e7efc91b770ac0ac8682ee6c5792a63a93
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Dec 23 09:19:05 2012 -0800
+
+    msgr: don't queue message on closed pipe
+
+    If we have a con that refs a pipe but it is closed, don't use it.  If
+    the ref is still there, it is only because we are racing with fault()
+    and it is about to (or just was) be detached.  Either way,
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6339c5d43974f4b495f15d199e01a141e74235f5)
+
+commit 373f1671b6cb64dba5a9172967b27177515be1fd
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Dec 22 21:24:52 2012 -0800
+
+    msgr: atomically queue first message with connect_rank
+
+    Atomically queue the first message on the new pipe, without dropping
+    and retaking pipe_lock.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 7bf0b0854d1f2706a3a2302bcbf92dd5c8c888ef)
+
+commit 82f22b38c5dc0b636574679ba1fee1b36a3c0478
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Jan 10 11:06:02 2013 -0800
+
+    config_opts.h: default osd_recovery_delay_start to 0
+
+    This setting was intended to prevent recovery from overwhelming peering traffic
+    by delaying the recovery_wq until osd_recovery_delay_start seconds after pgs
+    stop being added to it.  This should be less necessary now that recovery
+    messages are sent with strictly lower priority then peering messages.
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Gregory Farnum <greg@inktank.com>
+    (cherry picked from commit 44625d4460f61effe2d63d8280752f10f159e7b4)
+
+commit 81e8bb55e28384048fd82116a791a65ca52ef999
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jan 16 21:19:18 2013 -0800
+
+    osdmaptool: more fix cli test
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b0162fab3d927544885f2b9609b9ab3dc4aaff74)
+
+commit 2b5b2657579abdf5b1228f4c5c5ac8cec3706726
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jan 16 21:10:26 2013 -0800
+
+    osdmaptool: fix cli test
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5bd8765c918174aea606069124e43c480c809943)
+
+commit f739d1238a8a67598c037b6e2ed5d539a2d79996
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jan 16 14:21:47 2013 -0800
+
+    osdmaptool: allow user to specify pool for test-map-object
+
+    Fixes: #3820
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Gregory Farnum <greg@inktank.com>
+    (cherry picked from commit 85eb8e382a26dfc53df36ae1a473185608b282aa)
+
+commit 00759ee08f5dc62cbe4f237399f298472f6d8f4a
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed Jan 16 12:41:16 2013 -0800
+
+    rados.cc: fix rmomapkey usage: val not needed
+
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    Reviewed-by: Samuel Just <samuel.just@inktank.com>
+    (cherry picked from commit 625c3cb9b536a0cff7249b8181b7a4f09b1b4f4f)
+
+commit 06b3270f679be496df41810dacf863128b0cfcaa
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Jan 15 21:27:23 2013 -0800
+
+    librados.hpp: fix omap_get_vals and omap_get_keys comments
+
+    We list keys greater than start_after.
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 3f0ad497b3c4a5e9bef61ecbae5558ae72d4ce8b)
+
+commit 75072965201380aa55a8e15f9db4ccaf4d34d954
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Jan 15 21:26:22 2013 -0800
+
+    rados.cc: use omap_get_vals_by_keys in getomapval
+
+    Fixes: #3811
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit cb5e2be418924cf8b2c6a6d265a7a0327f08d00a)
+
+commit a3c2980fccfe95b7d094a7c93945437c3911b858
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Jan 15 21:24:50 2013 -0800
+
+    rados.cc: fix listomapvals usage: key,val are not needed
+
+    Fixes: #3812
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 44c45e520cc2e60c6c803bb245edb9330bff37e4)
+
+commit 20b27a1ce71c379a3b2a29d282dc0689a3a0df46
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Jan 16 15:01:47 2013 -0800
+
+    rgw: copy object should not copy source acls
+
+    Fixes: #3802
+    Backport: argonaut, bobtail
+
+    When using the S3 api and x-amz-metadata-directive is
+    set to COPY we used to copy complete metadata of source
+    object. However, this shouldn't include the source ACLs.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 37dbf7d9df93dd0e92019be31eaa1a19dd9569c7)
+
+commit 3293b31b44c9adad2b5e37da9d5342a6e4b72ade
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Jan 11 11:02:15 2013 -0800
+
+    OSD: only trim up to the oldest map still in use by a pg
+
+    map_cache.cached_lb() provides us with a lower bound across
+    all pgs for in-use osdmaps.  We cannot trim past this since
+    those maps are still in use.
+
+    backport: bobtail
+    Fixes: #3770
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 66eb93b83648b4561b77ee6aab5b484e6dba4771)
+
+commit 898a4b19ecc6fffc33feb198f37182ec0a6e77e9
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 14 08:15:02 2013 -0800
+
+    Revert "osdmap: spread replicas across hosts with default crush map"
+
+    This reverts commit 503917f0049d297218b1247dc0793980c39195b3.
+
+    This breaks vstart and teuthology configs.  A better fix is coming.
+
+commit 55b7dd3248f35929ea097525798e8667fafbf161
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Thu Jan 10 18:54:12 2013 +0000
+
+    mon: OSDMonitor: don't output to stdout in plain text if json is specified
+
+    Fixes: #3748
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 410906e04936c935903526f26fb7db16c412a711)
+
+commit 015a454a0c046cb678991cc4f4d53fb58c41dbe4
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 11 17:23:22 2013 -0800
+
+    osdmap: spread replicas across hosts with default crush map
+
+    This is more often the case than not, and we don't have a good way to
+    magically know what size of cluster the user will be creating.  Better to
+    err on the side of doing the right thing for more people.
+
+    Fixes: #3785
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 7ea5d84fa3d0ed3db61eea7eb9fa8dbee53244b6)
+
+commit d882d053927c319274be38a247f2beabb4e06b64
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jan 9 19:17:23 2013 -0800
+
+    ReplicatedPG: fix snapdir trimming
+
+    The previous logic was both complicated and not correct.  Consequently,
+    we have been tending to drop snapcollection links in some cases.  This
+    has resulted in clones incorrectly not being trimmed.  This patch
+    replaces the logic with something less efficient but hopefully a bit
+    clearer.
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 0f42c37359d976d1fe90f2d3b877b9b0268adc0b)
diff --git a/doc/changelog/v0.56.3.txt b/doc/changelog/v0.56.3.txt
new file mode 100644
index 000000000..c87675a96
--- /dev/null
+++ b/doc/changelog/v0.56.3.txt
@@ -0,0 +1,562 @@
+commit 6eb7e15a4783b122e9b0c85ea9ba064145958aa5
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Wed Feb 13 10:10:20 2013 -0800
+
+    v0.56.3
+
+commit f5eb845a0f7a2c28d3a88a37479bcb34f882f40c
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Feb 8 13:14:49 2013 -0800
+
+    rgw: change json formatting for swift list container
+    
+    Fixes: #4048
+    There is some difference in the way swift formats the
+    xml output and the json output for list container. In
+    xml the entity is named 'name' and in json it is named
+    'subdir'.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 3e4d79fe42dfc3ca70dc4d5d2aff5223f62eb34b)
+
+commit f21543f0d88f7bacb69cef3712b0ce087f386e93
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Feb 11 17:08:55 2013 -0800
+
+    librbd: unprotect any non-unprotected snapshot
+    
+    Include snapshots in the UNPROTECTING state as well, which can occur
+    after an unprotect is interrupted.
+    
+    Fixes: #4100
+    Backport: bobtail
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit fe283813b44a7c45def6768ea0788a3a0635957e)
+
+commit 65969f8fbef02ee39f6c2365fffbcd3f633f4b37
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Feb 8 21:36:13 2013 -0800
+
+    java: make CephMountTest use user.* xattr names
+    
+    Changes to the xattr code in Ceph require
+    a few tweaks to existing test cases.
+    Specifically, there is now a ceph.file.layout
+    xattr by default and user defined xattrs
+    are prepended with "user."
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joe Buck <jbbuck@gmail.com>
+    Reviewed-by: Noah Watkins <noahwatkins@gmail.com>
+
+commit 14fddc3ce85d3695aad9d3597f8f50dba5960a86
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Feb 8 09:59:25 2013 -0800
+
+    mon: fix typo in C_Stats
+    
+    Broken by previous commit.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3cf3710be0b4cccc8de152a97be50d983c35116d)
+
+commit 0453140d187016a61950a8836da57f54d2c34602
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 7 23:13:11 2013 -0800
+
+    mon: retry PGStats message on EAGAIN
+    
+    If we get EAGAIN from a paxos restart/election/whatever, we should
+    restart the message instead of just blindly acking it.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Luis <joao.luis@inktank.com>
+    (cherry picked from commit 4837063d447afb45554f55bb6fde1c97559acd4b)
+
+commit e68fcec78286363935cf731015108b9ea36b50a6
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 7 22:06:14 2013 -0800
+
+    mon: handle -EAGAIN in completion contexts
+    
+    We can get ECANCELED, EAGAIN, or success out of the completion contexts,
+    but in the EAGAIN case (meaning there was an election) we were sending
+    a success to the client.  This resulted in client hangs and all-around
+    confusion when the monitor cluster was thrashing.
+    
+    Backport: bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Luis <joao.luis@inktank.com>
+    (cherry picked from commit 17827769f1fe6d7c4838253fcec3b3a4ad288f41)
+
+commit 20ec490555728251444833520a40b20dc8015216
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Feb 12 14:11:09 2013 -0800
+
+    osd: only share maps on hb connection of OSD_HBMSGS feature is set
+    
+    Back in 1bc419a7affb056540ba8f9b332b6ff9380b37af we started sharing maps
+    with dead osds via the heartbeat connection, but old code will crash on an
+    unexpected message.  Only do this if the OSD_HBMSGS feature is present.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 302b26ff70ee5539da3dcb2e5614e2b7e83b9dcd)
+
+commit cbf63b633e7a59456f503af487fd4ad2607bbd76
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Feb 12 14:10:51 2013 -0800
+
+    osd: tolerate unexpected messages on the heartbeat interface
+    
+    We should note but not crash on unexpected messages.  Announce this awesome
+    new "capability" via a feature bit.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit afda30aeaae0a65f83c6886658354ad2b57c4c43)
+    
+    Conflicts:
+    
+    	src/include/ceph_features.h
+
+commit 102a519632f1b7a0fede9a3fbd4a5c1df0e732a5
+Merge: 2c6afa0 2ebf4d0
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Feb 12 13:39:52 2013 -0800
+
+    Merge remote-tracking branch 'gh/wip-bobtail-osd-msgr' into bobtail
+
+commit 2c6afa058e8b1738c1400392320482945834de86
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jan 30 11:32:23 2013 -0800
+
+    test_libcephfs: fix xattr test
+    
+    Ignore the ceph.*.layout xattrs.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b0d4dd21c7be86eb47728a4702a3c67ca44424ac)
+
+commit f11beb954976f66bfae75e847937f84958ebeaf3
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 7 22:51:29 2013 -0800
+
+    radosgw-admin: fix cli test
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1b05b0edbac09d1d7cf0da2e536829df05e48573)
+
+commit ec1085e534eb39d999775bebdcdb997f893a04ae
+Merge: 66d7758 62ed62f
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 7 23:25:30 2013 -0800
+
+    Merge remote-tracking branch 'gh/wip-bobtail-vxattrs' into bobtail
+
+commit 66d775858004d1d4e8a138b8d33a3799e03ce26e
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Feb 4 09:14:39 2013 -0800
+
+    mon: enforce reweight be between 0..1
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Luis <joao.luis@inktank.com>
+    (cherry picked from commit 4e29c95d6f61daa838888840cef0cceedc0fcfdd)
+
+commit 8bab3a1c3d0d2f619ddf885bb9050ad9a1c43517
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Feb 7 10:38:00 2013 -0800
+
+    PG: dirty_info on handle_activate_map
+    
+    We need to make sure the pg epoch is persisted during
+    activate_map.
+    
+    Backport: bobtail
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit dbce1d0dc919e221523bd44e1d0834711da1577d)
+
+commit dffa386bc13370c0ef56acf740b5200b2054980f
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 7 10:21:49 2013 -0800
+
+    osd: flush peering queue (consume maps) prior to boot
+    
+    If the osd itself is behind on many maps during boot, it will get more and
+    (as part of that) flush the peering wq to ensure the pgs consume them.
+    However, it is possible for OSD to have latest/recnet maps, but pgs to be
+    behind, and to jump directly to boot and join.  The OSD is then laggy and
+    unresponsive because the peering wq is way behind.
+    
+    To avoid this, call consume_map() (kick the peering wq) at the end of
+    init and flush it to ensure we are *internally* all caught up before we
+    consider joining the cluster.
+    
+    I'm pretty sure this is the root cause of #3905 and possibly #3995.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit af95d934b039d65d3667fc022e2ecaebba107b01)
+
+commit 47c9f46aac4afac37fb6ec72f0482e61f5e0d798
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Feb 6 17:10:00 2013 -0800
+
+    rgw: a tool to fix clobbered bucket info in user's bucket list
+    
+    This fixes bad entries in user's bucket list that may have occured
+    due to issue #4039. Syntax:
+    
+     $ radosgw-admin user check --uid=<uid> [--fix]
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 9cb6c33f0e2281b66cc690a28e08459f2e62ca13)
+    
+    Conflicts:
+    	src/rgw/rgw_admin.cc
+
+commit 6c8d63819fde1b6854f8fc03351465b420ff1bdc
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Feb 6 16:43:48 2013 -0800
+
+    rgw: bucket recreation should not clobber bucket info
+    
+    Fixes: #4039
+    User's list of buckets is getting modified even if bucket already
+    exists. This fix removes the newly created directory object, and
+    makes sure that user info's data points at the correct bucket.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 9d006ec40ced9d97b590ee07ca9171f0c9bec6e9)
+    
+    Conflicts:
+    	src/rgw/rgw_op.cc
+    	src/rgw/rgw_rados.cc
+
+commit cc167914ac9603f87083c63f2cbc8dac9441329f
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Feb 5 14:50:54 2013 -0800
+
+    rgw: a tool to fix buckets with leaked multipart references
+    
+    Checks specified bucket for the #4011 symptoms, optionally fix
+    the issue.
+    
+    sytax:
+      radosgw-admin bucket check --bucket=<bucket> [--fix]
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 2d8faf8e5f15e833e6b556b0f3c4ac92e4a4151e)
+    
+    Conflicts:
+    	src/rgw/rgw_admin.cc
+    	src/rgw/rgw_rados.h
+
+commit 4d6964fc7ddd23806e225c95bcb90ef93e4d23a1
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Feb 5 13:54:11 2013 -0800
+
+    rgw: radosgw-admin object unlink
+    
+    Add a radosgw-admin option to remove object from bucket index
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 16235a7acb9543d60470170bb2a09956364626cd)
+    
+    Conflicts:
+    	src/rgw/rgw_admin.cc
+    	src/rgw/rgw_rados.h
+    	src/test/cli/radosgw-admin/help.t
+
+commit 2ebf4d065af3dc2e581a25b921071af3efb57f8a
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 25 09:30:00 2013 -0800
+
+    osd: kill unused addr-based send_map()
+    
+    Not used, old API, bad.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e359a862199c8a94cb238f7271ba1b0edcc0863c)
+
+commit bac5b144b27f32da306161ae7018ccc337704121
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 25 09:29:37 2013 -0800
+
+    osd: share incoming maps via Connection*, not addrs
+    
+    Kill a set of parallel methods that are using the old addr/inst-based
+    msgr APIs, and instead use Connection handles.  This is much safer and gets
+    us closer to killing the old msgr API.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5e2fab54a4fdf2f59e2b635cbddef8a5909acb7c)
+
+commit 9ca3a165ded62313ba153d7bab89dadf3f73999f
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 25 09:27:00 2013 -0800
+
+    osd: pass new maps to dead osds via existing Connection
+    
+    Previously we were sending these maps to dead osds via their old addrs
+    using a new outgoing connection and setting the flags so that the msgr
+    would clean up.  That mechanism is possibly buggy and fragile, and we can
+    avoid it entirely if we just reuse the existing heartbeat Connection.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1bc419a7affb056540ba8f9b332b6ff9380b37af)
+
+commit 4cb28b6ed5a702fdac99b8ec71233ef7f877a7a2
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 25 09:25:28 2013 -0800
+
+    osd: requeue osdmaps on heartbeat connections for cluster connection
+    
+    If we receive an OSDMap on the cluster connection, requeue it for the
+    cluster messenger, and process it there where we normally do.  This avoids
+    any concerns about locking and ordering rules.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 76705ace2e9767939aa9acf5d9257c800f838854)
+
+commit e4f7ff8c288eac8a8b57382f11a4b6f93682315a
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 25 09:23:23 2013 -0800
+
+    msgr: add get_loopback_connection() method
+    
+    Return the Connection* for ourselves, so we can queue messages for
+    ourselves.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a7059eb3f3922cf08c1e5bb5958acc2d45952482)
+
+commit 62ed62f5e2fb068cee38612d7974526aa1b3c759
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 19 11:33:04 2013 -0800
+
+    qa: add layout_vxattrs.sh test script
+    
+    Test virtual xattrs for file and directory layouts.
+    
+    TODO: create a data pool, add it to the fs, and make sure we can use it.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 61fbe27a52d12ecd98ddeb5fc0965c4f8ee7841a)
+
+commit d386622c3961a3b57eea42fdb82611cd2e904f4d
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 19 10:11:18 2013 -0800
+
+    mds: allow dir layout/policy to be removed via removexattr on ceph.dir.layout
+    
+    This lets a user remove a policy that was previously set on a dir.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit db31a1f9f27416e4d531fda716e32d42a275e84f)
+
+commit 6af5da7ae2c4ef95c16c6460770b6244d1aa1a6e
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 19 10:09:39 2013 -0800
+
+    mds: handle ceph.*.layout.* setxattr
+    
+    Allow individual fields of file or dir layouts to be set via setxattr.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ebebf72f0993d028e795c78a986e1aee542ca5e0)
+
+commit c0af056eb9bdb62cfd8a6f9054a3a3c78c8e7447
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Feb 4 22:03:32 2013 -0800
+
+    mdsmap: backported is_data_pool()
+    
+    This roughly corresponds to mainline commit 99d9e1d.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 0407af4641ea19697f8feb0f48a92cde8dd4fbe4
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 19 10:04:05 2013 -0800
+
+    mds: fix client view of dir layout when layout is removed
+    
+    We weren't handling the case where the projected node has NULL for the
+    layout properly.  Fixes the client's view when we remove the dir layout.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 09f28541e374ffac198e4d48082b064aae93cb2c)
+
+commit 8ce834d3f50b00fdd59cd237f3fb5fef1d57e1dd
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 19 10:04:39 2013 -0800
+
+    client: note presence of dir layout in inode operator<<
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 84751489ca208964e617516e04556722008ddf67)
+
+commit 99824b93cec93daaa0d536f031eb3b6180f94e3b
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 19 09:05:59 2013 -0800
+
+    client: list only aggregate xattr, but allow setting subfield xattrs
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ba32ea9454d36072ec5ea3e6483dc3daf9199903)
+
+commit 809cff488ea1ffa299edd678ba6260993771bde3
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 18 22:26:00 2013 -0800
+
+    client: implement ceph.file.* and ceph.dir.* vxattrs
+    
+    Display ceph.file.* vxattrs on any regular file, and ceph.dir.* vxattrs
+    on any directory that has a policy set.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3f82912a891536dd7e930f98e28d9a8c18fab756)
+
+commit 13babca354d9fbe255de8bae9608a0c158bf6c40
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 18 17:21:37 2013 -0800
+
+    client: move xattr namespace enforcement into internal method
+    
+    This captures libcephfs users now too.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit febb96509559084357bfaabf7e4d28e494c274aa)
+
+commit 65ab51740175254ba3ee050f0fd97332dffe2eb7
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 18 17:20:22 2013 -0800
+
+    client: allow ceph.* xattrs
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ad7ebad70bf810fde45067f78f316f130a243b9c)
+
+commit 6f3c1cd2cc07d951dfc23e523b9c6400b7c77c72
+Author: caleb miles <caselim@gmail.com>
+Date:   Mon Jan 14 12:16:12 2013 -0500
+
+    rgw_rest: Make fallback uri configurable.
+    
+    Some HTTP servers, notabily lighttp, do not set SCRIPT_URI, make the fallback
+    string configurable.
+    
+    Signed-off-by: caleb miles <caleb.miles@inktank.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit b3a2e7e955547a863d29566aab62bcc480e27a65)
+    
+    Conflicts:
+    	src/rgw/rgw_rest.cc
+
+commit f57d1b4c8cc4d08c6147423d7881be55ed2e88d9
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Feb 1 10:56:11 2013 -0800
+
+    rgw: fix setting of NULL to string
+    
+    Fixes: #3777
+    s->env->get() returns char * and not string and can return NULL.
+    Also, remove some old unused code.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 9019fbbe8f84f530b6a8700dfe99dfeb03e0ed3d)
+
+commit 55687240b2de20185524de07e67f42c3b1ae6592
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Jan 11 10:44:04 2013 -0800
+
+    OSD: check for empty command in do_command
+    
+    Fixes: #3878
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 8cf79f252a1bcea5713065390180a36f31d66dfd)
+
+commit c3468f76a5e68a6426f03e508d8ecf26950fca2a
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Jan 30 18:52:24 2013 +0100
+
+    PGMap: fix -Wsign-compare warning
+    
+    Fix -Wsign-compare compiler warning:
+    
+    mon/PGMap.cc: In member function 'void PGMap::apply_incremental
+     (CephContext*, const PGMap::Incremental&)':
+    mon/PGMap.cc:247:30: warning: comparison between signed and
+     unsigned integer expressions [-Wsign-compare]
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit b571f8ee2d22a3894120204bc5f119ff37e1de53)
+
+commit 5a6b9af90f00d08ef97b34ee0b5abc7b0b63e72b
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jan 28 19:46:33 2013 -0800
+
+    mon: smooth pg stat rates over last N pgmaps
+    
+    This smooths the recovery and throughput stats over the last N pgmaps,
+    defaulting to 2.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a7d15afb529615db56bae038b18b66e60d827a96)
+
+commit 7fd7a5eed19d5ab508d5fe11ff8734bc2bc8c565
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 25 19:51:40 2013 -0800
+
+    mon/PGMap: report IO rates
+    
+    This does not appear to be very accurate; probably the stat values we're
+    displaying are not being calculated correctly.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3f6837e022176ec4b530219043cf12e009d1ed6e)
+
+commit 7f149cf6730280f0e633d9f5ef3f0f95c5a5e430
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 25 19:51:14 2013 -0800
+
+    mon/PGMap: report recovery rates
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 208b02a748d97378f312beaa5110d8630c853ced)
+
+commit 8d2d396c6d02bff72aca53920e9ac93fe91428d3
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 25 19:50:45 2013 -0800
+
+    mon/PGMap: include timestamp
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 76e9fe5f06411eb0e96753dcd708dd6e43ab2c02)
+
+commit 8ab77bd4b510149f4df6b3134de0ef59272cec71
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 25 19:49:16 2013 -0800
+
+    osd: track recovery ops in stats
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a2495f658c6d17f56ea0a2ab1043299a59a7115b)
+
+commit 8fd8534b4b808292a4b7c6b9f2f866c431cf9645
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 25 19:06:52 2013 -0800
+
+    osd_types: add recovery counts to object_sum_stats_t
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4aea19ee60fbe1106bdd71de2d172aa2941e8aab)
diff --git a/doc/changelog/v0.56.4.txt b/doc/changelog/v0.56.4.txt
new file mode 100644
index 000000000..f48fd834a
--- /dev/null
+++ b/doc/changelog/v0.56.4.txt
@@ -0,0 +1,1126 @@
+commit 63b0f854d1cef490624de5d6cf9039735c7de5ca
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Mon Mar 25 11:02:31 2013 -0700
+
+    v0.56.4
+
+commit 90ab120a3c08f30654207c9c67fb0a4ff1476aff
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Mar 25 09:50:33 2013 -0700
+
+    rgw: bucket index ops on system buckets shouldn't do anything
+    
+    Fixes: #4508
+    Backport: bobtail
+    On certain bucket index operations we didn't check whether
+    the bucket was a system bucket, which caused the operations
+    to fail. This triggered an error message on bucket removal
+    operations.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 70e0ee8ba955322832f0c366537ddf7a0288761e)
+
+commit b8657fa08d0fe176c71a60c1654c53c3156a9c53
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Feb 25 15:02:50 2013 -0800
+
+    systest: restrict list error acceptance
+    
+    Only ignore errors after the midway point if the midway_sem_post is
+    defined.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 5b24a68b6e7d57bac688021b822fb2f73494c3e9)
+
+commit 5af3d64f62b30cbaac13ccc7c23fc884ee1b7ae3
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Feb 25 14:55:34 2013 -0800
+
+    systest: fix race with pool deletion
+    
+    The second test have pool deletion and object listing wait on the same
+    semaphore to connect and start. This led to errors sometimes when the
+    pool was deleted before it could be opened by the listing process. Add
+    another semaphore so the pool deletion happens only after the listing
+    has begun.
+    
+    Fixes: #4147
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit b0271e390564119e998e18189282252d54f75eb6)
+
+commit 3569489b541ac0643520d20b08c788c26dfaff7f
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Mar 19 14:26:16 2013 -0700
+
+    os/FileJournal: fix aio self-throttling deadlock
+    
+    This block of code tries to limit the number of aios in flight by waiting
+    for the amount of data to be written to grow relative to a function of the
+    number of aios.  Strictly speaking, the condition we are waiting for is a
+    function of both aio_num and the write queue, but we are only woken by
+    changes in aio_num, and were (in rare cases) waiting when aio_num == 0 and
+    there was no possibility of being woken.
+    
+    Fix this by verifying that aio_num > 0, and restructuring the loop to
+    recheck that condition on each wakeup.
+    
+    Fixes: #4079
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit e5940da9a534821d0d8f872c13f9ac26fb05a0f5)
+
+commit 664ffa7d2178e486b00fa0706067f19b1bb9ab82
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 22 13:25:49 2013 -0700
+
+    common/MemoryModel: remove logging to /tmp/memlog
+    
+    This was a hack for dev purposes ages ago; remove it.  The predictable
+    filename is a security issue.
+    
+    CVE-2013-1882
+    
+    Reported-by: Michael Scherer <misc@zarb.org>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit c524e2e01da41ab5b6362c117939ea1efbd98095)
+
+commit 901fa435bb4be29b72cec39e5ff77570d13cbc6c
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 22 13:25:43 2013 -0700
+
+    init-ceph: clean up temp ceph.conf filename on exit
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 6a7ad2eac1db6abca3d7edb23ca9b80751400a23)
+
+commit 951f7e562afb4ae91b8d037ef793a96779461e96
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 22 13:25:33 2013 -0700
+
+    init-ceph: push temp conf file to a unique location on remote host
+    
+    The predictable file name is a security problem.
+    
+    CVE-2013-1882
+    
+    Reported-by: Michael Scherer <misc@zarb.org>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 051734522fea92878dd8139f28ec4e6b01371ede)
+
+commit dd670afccfd3ae58c03ea3b4c213dd8db22f618e
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 22 13:25:23 2013 -0700
+
+    mkcephfs: make remote temp directory name unique
+    
+    The predictable file name is a security problem.
+    
+    CVE-2013-1882
+    
+    Reported-by: Michael Scherer <misc@zarb.org>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit f463ef78d77b11b5ad78b31e9a3a88d0a6e62bca)
+
+commit 1cd51eb63a91fa0aa395c41572cfee4d53fbc7f5
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Mar 22 13:51:14 2013 -0700
+
+    PG::GetMissing: need to check need_up_thru in MLogRec handler
+    
+    Backport: bobtail
+    Fixes: #4534
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4fe4deafbe1758a6b3570048aca57485bd562440)
+
+commit d866814285667c7f9cd9205dca6cf95f3634c5fc
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Mar 22 13:48:49 2013 -0700
+
+    PG,osd_types: improve check_new_interval debugging
+    
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d611eba9caf45f2d977c627b123462a073f523a4)
+
+commit fc3406f395103737e5784611e2b38fd1c4f26369
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Mar 5 16:06:20 2013 -0800
+
+    FileStore: fix reversed collection_empty return value
+    
+    Backport: bobtail
+    Fixes: #4380
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 7a434d10da0f77e1b329de0b06b6645cd73cc81b)
+    
+    Conflicts:
+    	src/os/FileStore.cc
+
+commit 9f7c6ad395653b1039ea957cfaa7689699f510e7
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Feb 11 12:52:07 2013 -0800
+
+    FileStore: set replay guard on create_collection
+    
+    This should prevent sequences like:
+    
+    rmcoll a
+    mkcoll a
+    touch a foo
+    <crash>
+    
+    from causing trouble by preventing the rmcoll
+    and mkcoll from being replayed.
+    
+    Fixes: 4064
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 411770c45734c9827745ddc4018d86c14f2858a6)
+
+commit 9932c9f0a0503f1902549a3c39e5d51488fff3ef
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Feb 11 12:24:14 2013 -0800
+
+    FileStore: _split_collection should not create the collection
+    
+    This will simplify adding a replay guard to create_collection.
+    
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit b184ff581a08c9e6ce5b858f06ccbe9d0e2a170b)
+
+commit 7e7ea8266ee45d96863e1edc5a567dcc2977680c
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Feb 22 15:15:27 2013 -0800
+
+    client: use 4MB f_bsize and f_frsize for statfs
+    
+    Old stat(1) reports:
+    
+      Block size: 1048576    Fundamental block size: 1048576
+    
+    and the df(1) arithmetic works out.  New stat(1) reports:
+    
+      Block size: 1048576    Fundamental block size: 4096
+    
+    which is what we are shoving into statvfs, but we have the b_size and
+    fr_size arithmetic swapped.  However, doing the *correct* reporting would
+    then break the old stat by making both sizes appear to be 4KB (or
+    whatever).
+    
+    Sidestep the issue by making *both* values 4MB.. which is both large enough
+    to report large FS sizes, and also the default stripe size and thus a
+    "reasonable" value to report for a block size.
+    
+    Perhaps in the future, when we no longer care about old userland, we can
+    report the page size for f_bsize, which is probably the "most correct"
+    thing to do.
+    
+    Fixes: #3794.  See also #3793.
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 7c94083643891c9d66a117352f312b268bdb1135)
+
+commit 66dca7730006dffbce12e6b27016ea5469b7691b
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Feb 18 17:39:46 2013 -0800
+
+    os/FileStore: check replay guard on src for collection rename
+    
+    This avoids a problematic sequence like:
+    
+         - rename A/ -> B/
+         - remove B/1...100
+         - destroy B/
+         - create A/
+         - write A/101...
+         <crash>
+         - replay A/ -> B/
+         - remove B/1...100  (fails but tolerated)
+         - destroy B/        (fails with ENOTEMPTY)
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 5fc83c8d9887d2a916af11436ccc94fcbfe59b7a)
+
+commit ea570591b0d9dda102bece0f4cc7d263c3e2a3b0
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Feb 22 14:12:28 2013 -0800
+
+    PG::proc_replica_log: oinfo.last_complete must be *before* first entry in omissing
+    
+    Fixes: #4189
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 2dae6a68ee85a20220ee940dbe33a2144d43457b)
+
+commit aa38d87c6237ccfb8b3b03bc7c28dcadb927237b
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 21 17:55:21 2013 -0800
+
+    osd/PG: fix typo, missing -> omissing
+    
+    From ce7ffc34408bf32c66dc07e6f42d54b7ec489d41.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit dc181224abf6fb8fc583730ae3d90acdf0b80f39)
+
+commit ac914834ba4fb0e4770f63c60713be7a48aa3fa1
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Feb 21 15:31:36 2013 -0800
+
+    PG::proc_replica_log: adjust oinfo.last_complete based on omissing
+    
+    Otherwise, search_for_missing may neglect to check the missing
+    set for some objects assuming that if the need version is
+    prior to last_complete, the replica must have it.
+    
+    Fixes: #4994
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit ce7ffc34408bf32c66dc07e6f42d54b7ec489d41)
+
+commit 7f39704b0903fb11e3f1e102599bf0c540f93623
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Feb 9 00:05:33 2013 -0800
+
+    osd: fix load_pgs collection handling
+    
+    On a _TEMP pg, is_pg() would succeed, which meant we weren't actually
+    hitting the cleanup checks.  Instead, restructure this loop as positive
+    checks and handle each type of collection we understand.
+    
+    This fixes _TEMP cleanup.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit b19b6dced85617d594c15631571202aab2f94ae8)
+
+commit 0a09be99f3e9d2b68a052ff63ea858a7c8e5626c
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Feb 9 00:04:29 2013 -0800
+
+    osd: fix load_pgs handling of pg dirs without a head
+    
+    If there is a pgid that passes coll_t::is_pg() but there is no head, we
+    will populate the pgs map but then fail later when we try to do
+    read_state.  This is a side-effect of 55f8579.
+    
+    Take explicit note of _head collections we see, and then warn when we
+    find stray snap collections.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 1f80a0b576c0af1931f743ad988b6293cbf2d6d9)
+
+commit 2dc25dfd47de8d3602bdb3699c82b8b118b0fb28
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Feb 7 13:34:47 2013 -0800
+
+    OSD::load_pgs: first scan colls before initing PGs
+    
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 073f58ede2e473af91f76d01679631c169274af7)
+
+commit 2fd8db29ccf56fd5473826f20e351edfe200419d
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Tue Jan 8 19:24:13 2013 -0800
+
+    osd: Add digest of omap for deep-scrub
+    
+    Add ScrubMap encode/decode v4 message with omap digest
+    Compute digest of header and key/value.  Use bufferlist
+    to reflect structure and compute as we go, clearing
+    bufferlist to reduce memory usage.
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 509a93e89f04d7e9393090563cf7be8e0ea53891)
+
+commit ee59f517ccc195e591e0b1b6ddf6d1d054c4e2fd
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Mar 15 15:13:46 2013 -0700
+
+    OSD: split temp collection as well
+    
+    Otherwise, when we eventually remove the temp collection, there might be
+    objects in the temp collection which were independently pulled into the child
+    pg collection.  Thus, removing the old stale parent link from its temp
+    collection also blasts the omap entries and snap mappings for the real child
+    object.
+    
+    Backport: bobtail
+    Fixes: #4452
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit f8d66e87a5c155b027cc6249006b83b4ac9b6c9b)
+
+commit 7ec68e2d45b431a3a77458939622e6ea1e396d0e
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Mar 14 19:59:36 2013 -0700
+
+    PG: ignore non MISSING pg query in ReplicaActive
+    
+    1) Replica sends notify
+    2) Prior to processing notify, primary queues query to replica
+    3) Primary processes notify and activates sending MOSDPGLog
+    to replica.
+    4) Primary does do_notifies at end of process_peering_events
+    and sends to Query.
+    5) Replica sees MOSDPGLog and activates
+    6) Replica sees Query and asserts.
+    
+    In the above case, the Replica should simply ignore the old
+    Query.
+    
+    Fixes: #4050
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 8222cbc8f35c359a35f8381ad90ff0eed5615dac)
+
+commit b279b148b38c755d2dcca6524d7b8a95daf123a0
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Mar 13 16:04:23 2013 -0700
+
+    FileJournal: queue_pos \in [get_top(), header.max_size)
+    
+    If queue_pos == header.max_size when we create the entry
+    header magic, the entry will be rejected at get_top() on
+    replay.
+    
+    Fixes: #4436
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit de8edb732e3a5ce4471670e43cfe6357ae6a2758)
+
+commit 56e0a129b0d9ad463f4300999ed2c15173cbe603
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Mar 14 18:52:02 2013 -0700
+
+    OSD: expand_pg_num after pg removes
+    
+    Otherwise:
+    1) expand_pg_num removes a splitting pg entry
+    2) peering thread grabs pg lock and starts split
+    3) OSD::consume_map grabs pg lock and starts removal
+    
+    At step 2), we run afoul of the assert(is_splitting)
+    check in split_pgs.  This way, the would be splitting
+    pg is marked as removed prior to the splitting state
+    being updated.
+    
+    Backport: bobtail
+    Fixes: #4449
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f1b031b3cf195cf6df3d3c47c7d606fba63ed4c4)
+
+commit d6e678a62950ccc16d010a76500705e176ba9c73
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Feb 11 06:23:54 2013 -0800
+
+    osd: update snap collections for sub_op_modify log records conditionaly
+    
+    The only remaining caller is sub_op_modify().  If we do have a non-empty
+    op transaction, we want to do this update, regardless of what we think
+    last_backfill is (our notion may be not completely in sync with the
+    primary).  In particular, our last_backfill may be the same object but
+    a different snapid, but the primary disagrees and is pushing an op
+    transaction through.
+    
+    Instead, update the collections if we have a non-empty transaction.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 31e911b63d326bdd06981ec4029ad71b7479ed70)
+
+commit 51c2c4d7ccb25617967689d7e531eb145ee7011d
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Feb 10 17:02:45 2013 -0800
+
+    osd: include snaps in pg_log_entry_t::dump()
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 715d8717a0e8a08fbe97a3e7d3ffd33aa9529d90)
+
+commit f0bf68ff500f7337103aef2a9d6d10c3597e410f
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Feb 10 16:59:48 2013 -0800
+
+    osd: unconditionally encode snaps buffer
+    
+    Previously we would only encode the updated snaps vector for CLONE ops.
+    This doesn't work for MODIFY ops generated by the snap trimmer, which
+    may also adjust the clone collections.  It is also possible that other
+    operations may need to populate this field in the future (e.g.,
+    LOST_REVERT may, although it currently does not).
+    
+    Fixes: #4071, and possibly #4051.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 54b6dd924fea3af982f3d729150b6449f318daf2)
+
+commit aa74fabd90b776e9938f3d59ea5ed04bd4027dbb
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Feb 10 10:57:12 2013 -0800
+
+    osd: improve debug output on snap collections
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 8b05492ca5f1479589bb19c1ce058b0d0988b74f)
+
+commit a30903c6adaa023587d3147179d6038ad37ca520
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Mar 7 12:53:51 2013 -0800
+
+    PG: check_recovery_sources must happen even if not active
+    
+    missing_loc/missing_loc_sources also must be cleaned up
+    if a peer goes down during peering:
+    
+    1) pg is in GetInfo, acting is [3,1]
+    2) we find object A on osd [0] in GetInfo
+    3) 0 goes down, no new peering interval since it is neither up nor
+    acting, but peer_missing[0] is removed.
+    4) pg goes active and try to pull A from 0 since missing_loc did not get
+    cleaned up.
+    
+    Backport: bobtail
+    Fixes: #4371
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit de22b186c497ce151217aecf17a8d35cdbf549bb)
+
+commit 19f6df28337307ce6d8617524f88f222c0ffcab3
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Mar 5 15:49:26 2013 -0800
+
+    HashIndex: _collection_list_partial must tolerate NULL next
+    
+    Backport: bobtail
+    Fixes: #4379
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit ce4432adc67dc2fc06dd21ea08e59d179496bcc6)
+
+commit 1b5c1564bad3574db257cb17d48190958d870f51
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Mar 5 14:35:39 2013 -0800
+
+    OSD: lock not needed in ~DeletingState()
+    
+    No further refs to the object can remain at this point.
+    Furthermore, the callbacks might lock mutexes of their
+    own.
+    
+    Backport: bobtail
+    Fixes: #4378
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit e4bf1bcab159d7c5b720f5da01877c0f67c16d16)
+
+commit 50d2d4a77f339d489703c1125dc5c1159d08ca8a
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Sun Mar 10 12:50:01 2013 -0700
+
+    ReplicatedPG: don't leak reservation on removal
+    
+    Fixes: 4431
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 32bf131e0141faf407b5ff993f75f97516b27c12)
+    
+    Conflicts:
+    
+    	src/osd/ReplicatedPG.cc
+
+commit c6b7c4be3fc44d065c29ea76b744b39272c2ba05
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Mar 12 12:56:01 2013 -0700
+
+    rgw: set up curl with CURL_NOSIGNAL
+    
+    Fixes: #4425
+    Backport: bobtail
+    Apparently, libcurl needs that in order to be thread safe. Side
+    effect is that if libcurl is not compiled with c-ares support,
+    domain name lookups are not going to time out.
+    Issue affected keystone.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 88725316ddcfa02ff110e659f7a8131dc1ea2cfc)
+
+commit 8c16a376a9cbfd6812824fb71e2cc9935e14e667
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 8 08:56:44 2013 -0800
+
+    osd: mark down connections from old peers
+    
+    Close out any connection with an old peer.  This avoids a race like:
+    
+    - peer marked down
+    - we get map, mark down the con
+    - they reconnect and try to send us some stuff
+    - we share our map to tell them they are old and dead, but leave the con
+      open
+    ...
+    - peer marks itself up a few times, eventually reuses the same port
+    - sends messages on their fresh con
+    - we discard because of our old con
+    
+    This could cause a tight reconnect loop, but it is better than wrong
+    behavior.
+    
+    Other possible fixes:
+     - make addr nonce truly unique (augment pid in nonce)
+     - make a smarter 'disposable' msgr state (bleh)
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 881e9d850c6762290f8be24da9e74b9dc112f1c9)
+
+commit 1affc8b6e8cb82510aed524f90a6165c6edf7513
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 8 08:53:40 2013 -0800
+
+    osd/PG: rename require_same_or_newer_map -> is_same_or_newer_map
+    
+    This avoids confusion with the OSD method of the same name, and better
+    matches what the function tests (and does not do).
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ba7e815a18cad110525f228db1b3fe39e011409e)
+    
+    Conflicts:
+    
+    	src/osd/ReplicatedPG.cc
+
+commit 7ca55d36c4007be5880102fa1ed314d3d66e9d87
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Mar 11 16:25:16 2013 -0700
+
+    log: drop default 'log max recent' from 100k -> 10k
+    
+    Use less memory.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c021c5ccf0c063cccd7314964420405cea6406de)
+
+commit 33ac786ddc14d0904f4835cbf684a51f4815d45b
+Author: Jan Harkes <jaharkes@cs.cmu.edu>
+Date:   Fri Mar 8 12:45:57 2013 -0500
+
+    Fix radosgw actually reloading after rotating logs.
+    
+    The --signal argument to Debian's start-stop-daemon doesn't
+    make it send a signal, but defines which signal should be send
+    when --stop is specified.
+    
+    Signed-off-by: Jan Harkes <jaharkes@cs.cmu.edu>
+    (cherry picked from commit 44f1cc5bc42f9bb6d5a386037408d2de17dc5413)
+
+commit 6838b8db9a3d4ceaf121a261e2ded4af9bfeaeb9
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Mar 6 17:42:03 2013 -0800
+
+    common: reduce default in-memory logs for non-daemons
+    
+    The default of 100000 can result in hundreds of MBs of extra memory
+    used. This was most obvious when using librbd with caching enabled,
+    since there was a dout(0) accidentally left in the ObjectCacher.
+    
+    refs: #4352
+    backport: bobtail
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 7c208d2f8e3f28f4055a4ae51eceae892dcef1dc)
+
+commit f80f64cf024bd7519d5a1fb2a5698db97a003ce8
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Feb 22 17:01:53 2013 -0800
+
+    osd: allow (some) log trim when degraded, but not during recovery
+    
+    We allow some trim during degraded, although we keep more entries around to
+    improve our chances of a restarting OSD of doing log-based recovery.
+    
+    Still disallow during recovery...
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 6d89b34e5608c71b49ef33ab58340e90bd8da6e4)
+
+commit e502a65da1779564059e74c09ab87aca1d901bd2
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Feb 25 15:33:35 2013 -0800
+
+    osd: restructure calc_trim
+    
+    No functional change, except that we log more debug, yay!
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 86df164d04f6e31a0f20bbb94dbce0599c0e8b3d)
+
+commit 3a3123033d7d9e1574be3fb18c807eeb0f4678eb
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Feb 22 16:48:02 2013 -0800
+
+    osd: allow pg log trim during (non-classic) scrub
+    
+    Chunky (and deep) scrub do not care about PG log trimming.  Classic scrub
+    still does.
+    
+    Deep scrub can take a long time, so not trimming the log during that period
+    may eat lots of RAM; avoid that!
+    
+    Might fix: #4179
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 0ba8db6b664205348d5499937759916eac0997bf)
+
+commit cde0f224dac21b2e17e06d396ac52e18034a4262
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 28 12:46:00 2013 -0800
+
+    msgr: drop messages on cons with CLOSED Pipes
+    
+    Back in commit 6339c5d43974f4b495f15d199e01a141e74235f5, we tried to make
+    this deal with a race between a faulting pipe and new messages being
+    queued.  The sequence is
+    
+    - fault starts on pipe
+    - fault drops pipe_lock to unregister the pipe
+    - user (objecter) queues new message on the con
+    - submit_message reopens a Pipe (due to this bug)
+    - the message managed to make it out over the wire
+    - fault finishes faulting, calls ms_reset
+    - user (objecter) closes the con
+    - user (objecter) resends everything
+    
+    It appears as though the previous patch *meant* to drop *m on the floor in
+    this case, which is what this patch does.  And that fixes the crash I am
+    hitting; see #4271.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 0f42eddef5da6c1babe9ed51ceaa3212a42c2ec4)
+
+commit 2fa2546c90e7a6eab9a3de7ab4e4271f8b25bad6
+Author: Concubidated <tyler.brekke@inktank.com>
+Date:   Fri Mar 8 13:44:39 2013 -0800
+
+    Fix output of 'ceph osd tree --format=json'
+    
+    Signed-off-by: Tyler Brekke <tyler.brekke@inktank.com>
+    (cherry picked from commit 9bcba944c6586ad5f007c0a30e69c6b5a886510b)
+
+commit b70e2c270b9eb3fce673b7e51b527ebf88214f14
+Author: Sam Lang <sam.lang@inktank.com>
+Date:   Tue Feb 12 11:32:29 2013 -0600
+
+    deb:  Add ceph-coverage to ceph-test deb package
+    
+    Teuthology uses the ceph-coverage script extensively
+    and expects it to be installed by the ceph task.  Add
+    the script to the ceph-test debian package so that it
+    gets installed for that use case.
+    
+    Signed-off-by: Sam Lang <sam.lang@inktank.com>
+    (cherry picked from commit 376cca2d4d4f548ce6b00b4fc2928d2e6d41038f)
+
+commit ca9aac785eb9e2d1ee955792d2f4d1d911727fb3
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Feb 22 15:04:37 2013 -0800
+
+    rgw: set attrs on various list bucket xml results (swift)
+    
+    Fixes: #4247
+    The list buckets operation was missing some attrs on the different
+    xml result entities. This fixes it.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 4384e59ad046afc9ec53a2d2f1fff6a86e645505)
+
+commit e39660c901756d5e722308e72a9d8ee4893f70f7
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Feb 22 15:02:02 2013 -0800
+
+    formatter: add the ability to dump attrs in xml entities
+    
+    xml entities may have attrs assigned to them. Add the ability
+    to set them. A usage example:
+    
+    formatter->open_array_section_with_attrs("container",
+         FormatterAttrs("name", "foo", NULL));
+    
+    This will generate the following xml entity:
+    <container name="foo">
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 7cb6ee28073824591d8132a87ea09a11c44efd66)
+    
+    Conflicts:
+    	src/common/Formatter.cc
+
+commit 0304b85e5c12c30459b94ec5d332dfaa04d20d5b
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Mar 6 19:32:21 2013 -0800
+
+    rgw: don't iterate through all objects when in namespace
+    
+    Fixes: #4363
+    Backport: argonaut, bobtail
+    When listing objects in namespace don't iterate through all the
+    objects, only go though the ones that starts with the namespace
+    prefix
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 6669e73fa50e3908ec825ee030c31a6dbede6ac0)
+
+commit d0dafaa77f96fa91785df0796806ea07fd93f319
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Feb 28 12:13:45 2013 -0800
+
+    ObjectCacher: fix debug log level in split
+    
+    Level 0 should never be used for this kind of debugging.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit cb3ee33532fb60665f39f6ccb1d69d67279fd5e1)
+
+commit ff014ed26bfa4fbf40acffec4eb4805b95a9982c
+Author: Dan Mick <dmick@danceorelse.org>
+Date:   Thu Jan 24 13:38:25 2013 -0800
+
+    rados: remove unused "check_stdio" parameter
+    
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit bb860e49a7faeaf552538a9492ef0ba738c99760)
+
+commit e89884da9f76b713372a79b772ba3cc2f3b03048
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jan 23 21:31:11 2013 -0800
+
+    rados: obey op_size for 'get'
+    
+    Otherwise we try to read the whole object in one go, which doesn't bode
+    well for large objects (either non-optimal or simply broken).
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 234becd3447a679a919af458440bc31c8bd6b84f)
+
+commit ee943c8bcf36f1e2218d8e25edfa38ec5fe4bec2
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Feb 27 16:58:45 2013 -0800
+
+    FileJournal::wrap_read_bl: adjust pos before returning
+    
+    Otherwise, we may feed an offset past the end of the journal to
+    check_header in read_entry and incorrectly determine that the entry is
+    corrupt.
+    
+    Fixes: 4296
+    Backport: bobtail
+    Backport: argonaut
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 5d54ab154ca790688a6a1a2ad5f869c17a23980a)
+
+commit 8c6f52215240f48b5e4d5bb99a5f2f451e7ce70a
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jan 16 13:14:00 2013 -0800
+
+    osd: leave osd_lock locked in shutdown()
+    
+    No callers expect the lock to be dropped.
+    
+    Fixes: #3816
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 98a763123240803741ac9f67846b8f405f1b005b)
+
+commit a324d999804b4648f245ee36b3bd611b3d139d5d
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Feb 26 14:07:12 2013 -0800
+
+    msg: fix entity_addr_t::is_same_host() for IPv6
+    
+    We weren't checking the memcmp return value properly!  Aie...
+    
+    Backport: bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c8dd2b67b39a8c70e48441ecd1a5cc3c6200ae97)
+
+commit 79d68ae8581141c3fb4cfafd76d5111ff009b762
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Feb 17 22:35:50 2013 -0800
+
+    osd: requeue pg waiters at the front of the finished queue
+    
+    We could have a sequence like:
+    
+    - op1
+    - notify
+    - op2
+    
+    in the finished queue.  Op1 gets put on waiting_for_pg, the notify
+    creates the pg and requeues op1 (and the end), op2 is handled, and
+    finally op1 is handled.  That breaks ordering; see #2947.
+    
+    Instead, when we wake up a pg, queue the waiting messages at the front
+    of the dispatch queue.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 56c5a07708d52de1699585c9560cff8b4e993d0a)
+
+commit 79c4e7e91becc497843d96251776bdc176706aa0
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Feb 17 20:49:52 2013 -0800
+
+    osd: pull requeued requests off one at a time
+    
+    Pull items off the finished queue on at a time.  In certain cases, an
+    event may result in new items betting added to the finished queue that
+    will be put at the *front* instead of the back.  See latest incarnation
+    of #2947.
+    
+    Note that this is a significant changed in behavior in that we can
+    theoretically starve if an event keeps resulting in new events getting
+    generated.  Beware!
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit f1841e4189fce70ef5722d508289e516faa9af6a)
+
+commit 3a6ce5d0355beaa56199465e94666cae40bd8da1
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jan 17 22:00:42 2013 -0800
+
+    mds: open mydir after replay
+    
+    In certain cases, we may replay the journal and not end up with the
+    dirfrag for mydir open.  This is fine--we just need to open it up and
+    fetch it below.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e51299fbce6bdc3d6ec736e949ba8643afc965ec)
+
+commit 36ed407e0f939a9bca57c3ffc0ee5608d50ab7ed
+Author: Greg Farnum <greg@inktank.com>
+Date:   Thu Feb 21 09:21:01 2013 -0800
+
+    mds: use inode_t::layout for dir layout policy
+    
+    Remove the default_file_layout struct, which was just a ceph_file_layout,
+    and store it in the inode_t.  Rip out all the annoying code that put this
+    on the heap.
+    
+    To aid in this usage, add a clear_layout() function to inode_t.
+    
+    Signed-off-by: Sage Weil <sage.weil@dreamhost.com>
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit b4fd196cae5ab76aeb8f82f0581d0a6a2133a3ed
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jan 20 21:53:37 2013 -0800
+
+    mds: parse ceph.*.layout vxattr key/value content
+    
+    Use qi to parse a strictly formatted set of key/value pairs.  Be picky
+    about whitespace.  Any subset of recognized keys is allowed.  Parse the
+    same set of keys as the ceph.*.layout.* vxattrs.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5551aa5b3b5c2e9e7006476b9cd8cc181d2c9a04)
+
+commit 5f92adca26b509aed5655e85ad53174fa7b574b9
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Feb 18 09:10:43 2013 -0800
+
+    rgw: fix multipart uploads listing
+    
+    Fixes: #4177
+    Backport: bobtail
+    Listing multipart uploads had a typo, and was requiring the
+    wrong resource (uploadId instead of uploads).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit db99fb4417b87301a69cb37b00c35c838b77197e)
+
+commit a44df9343594099fecb3897df393249d3d1992e2
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Feb 15 10:22:54 2013 -0800
+
+    rgw: don't copy object when it's copied into itself
+    
+    Fixes: #4150
+    Backport: bobtail
+    
+    When object copied into itself, object will not be fully copied: tail
+    reference count stays the same, head part is rewritten.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 34f885be536d0ac89c10fd29b1518751d2ffc547)
+
+commit 91d6be8353259ca30dc87062422e9ae334c3c344
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Feb 19 10:49:33 2013 -0800
+
+    PG: remove weirdness log for last_complete < log.tail
+    
+    In the case of a divergent object prior to log.tail,
+    last_complete may end up before log.tail.
+    
+    Backport: bobtail
+    Fixes #4174
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit dbadb3e2921297882c5836c67ca32bb8ecdc75db)
+    
+    Conflicts:
+    
+    	src/osd/PG.cc
+
+commit 7889c5412deaab1e7ba53d66814d0f25659229c1
+Author: James Page <james.page@ubuntu.com>
+Date:   Mon Feb 18 16:24:54 2013 +0000
+
+    Strip any trailing whitespace from rbd showmapped
+    
+    More recent versions of ceph append a bit of whitespace to the line
+    after the name of the /dev/rbdX device; this causes the monitor check
+    to fail as it can't find the device name due to the whitespace.
+    
+    This fix excludes any characters after the /dev/rbdN match.
+    (cherry picked from commit ad84ea07cac5096de38b51b8fc452c99f016b8d8)
+
+commit 42fba772c3c0e2396124a9d6ec39297dd9af4cc9
+Merge: 290a352 ad6655e
+Author: Sage Weil <sage@newdream.net>
+Date:   Sun Feb 17 21:52:55 2013 -0800
+
+    Merge pull request #64 from dalgaaf/wip-bobtail-memleaks
+    
+    cherry-pick some memleak fixes from master to bobtail
+
+commit ad6655e976916d3b62e0c91fd469eeb49fe7da3e
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Fri Feb 8 17:17:59 2013 +0100
+
+    rgw/rgw_rest.cc: fix 4K memory leak
+    
+    Fix 4K memory leak in case RGWClientIO::read() fails in
+    read_all_chunked_input().
+    
+    Error from cppcheck was:
+    Checking src/rgw/rgw_rest.cc...
+    [src/rgw/rgw_rest.cc:688]: (error) Memory leak: data
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 89df090e04ef9fc5aae29122df106b0347786fab)
+
+commit 3de692753c28ddbeb3c86b51466be16e94c9c458
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Fri Feb 8 17:14:19 2013 +0100
+
+    SyntheticClient.cc: fix some memory leaks in the error handling
+    
+    Fix some memory leaks in case of error handling due to failed
+    client->open() calls.
+    
+    Error from cppcheck was:
+    [src/client/SyntheticClient.cc:1980]: (error) Memory leak: buf
+    [src/client/SyntheticClient.cc:2040]: (error) Memory leak: buf
+    [src/client/SyntheticClient.cc:2090]: (error) Memory leak: buf
+    (cherry picked from commit f0ba80756d1c3c313014ad7be18191981fb545be)
+
+commit f19d228c6a49222659c769099aaa4e755b80331d
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Fri Feb 8 16:57:20 2013 +0100
+
+    rgw/rgw_xml.cc: fix realloc memory leak in error case
+    
+    Fix error from cppcheck:
+    
+    [src/rgw/rgw_xml.cc:212]: (error) Common realloc mistake: 'buf'
+      nulled but not freed upon failure
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit d48cc789ea075ba2745754035640ada4131b2119)
+
+commit b0c6be95b03d9f3dd2badcdcff359ae7bc9684f4
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Fri Feb 8 16:54:33 2013 +0100
+
+    os/FileStore.cc: fix realloc memory leak in error case
+    
+    Fix error from cppcheck:
+    
+    [src/os/FileStore.cc:512]: (error) Common realloc mistake: 'fiemap'
+      nulled but not freed upon failure
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit c92a0f552587a232f66620170660d6b2ab6fb3a5)
+
+commit f3e5cedbc11005701ac0a8e70909a6372cd2fe6f
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Fri Feb 8 16:49:36 2013 +0100
+
+    common/fiemap.cc: fix realloc memory leak
+    
+    Fix error from cppcheck:
+    
+    [src/common/fiemap.cc:73]: (error) Common realloc mistake: 'fiemap'
+      nulled but not freed upon failure
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit f26f1470e7af36fa1eb8dc59c8a7c62c3c3a22ba)
+
+commit 290a352c3f9e241deac562e980ac8c6a74033ba6
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 14 11:37:57 2013 -0800
+
+    osd/OSDCap: add unit test for parsing pools/objects with _ and -
+    
+    Hunting #4122, where a user saw
+    
+    2013-02-13 19:39:25.467916 7f766fdb4700 10 osd.0 10  session 0x2c8cc60 client.libvirt has caps osdcap[grant(object_prefix rbd^@children  class-read),grant(pool libvirt^@pool^@test rwx)] 'allow class-read object_prefix rbd_children, allow pool libvirt-pool-test rwx'
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2ce28ef1d7f95e71e1043912dfa269ea3b0d1599)
+    (cherry picked from commit a6534bc8a0247418d5263b765772d5266f99229c)
+
+commit f47916353547c77a65c7b70e609a9096f09c6d28
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 14 15:39:43 2013 -0800
+
+    osd/OSDCap: tweak unquoted_word parsing in osd caps
+    
+    Newer versions of spirit (1.49.0-3.1ubuntu1.1 in quantal, in particular)
+    dislike the construct with alnum and replace the - and _ with '\0' in the
+    resulting string.
+    
+    Fixes: #4122
+    Backport: bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 6c504d96c1e4fbb67578fba0666ca453b939c218)
diff --git a/doc/changelog/v0.56.5.txt b/doc/changelog/v0.56.5.txt
new file mode 100644
index 000000000..0e9ce5ed2
--- /dev/null
+++ b/doc/changelog/v0.56.5.txt
@@ -0,0 +1,1972 @@
+commit df884bb7b59e5796c996ab806f5ec8b8322e292f
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Thu May 2 18:08:06 2013 -0700
+
+    v0.56.5
+
+commit b38cbabb2a38e4476de1135c33c25c673d62cd67
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Thu Apr 4 18:23:40 2013 +0200
+
+    ceph.spec.in: fix udev rules.d files handling
+    
+    Move 50-rbd.rules into the ceph base package since the related
+    ceph-rbdnamer binary is part of this package. Use correct install
+    pattern.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+
+commit d0678a062a1baf738ce96114114d99495be19478
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Apr 29 17:01:55 2013 -0700
+
+    debian: only start/stop upstart jobs if upstart is present
+    
+    This avoids errors on non-upstart distros (like wheezy).
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 030bf8aaa15837f898e453161eeaf1d52fc5779d)
+    
+    Conflicts:
+    	debian/ceph-mds.postinst
+
+commit 209ce34a420cc5cdbe7219393f2f150ac28f612c
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Mar 11 17:09:37 2013 -0700
+
+    debian: stop ceph-mds before uninstalling ceph-mds
+    
+    Fixes: #4384
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 9eb0d91b867ab980135d7c6ff6347d69d0a8a794)
+
+commit 0c91becfa6cc5f05c5b2eaa89299dbf1aa202344
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Thu Apr 25 15:53:51 2013 -0700
+
+    Makefile.am: Add -lpthread to fix build on newer ld in Raring Ringtail
+    
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 98f532e8000af281fa03b24da9ad2fda81755270)
+    
+    Conflicts:
+    	src/Makefile.am
+
+commit 15e6544f0892e8472f436149d15fc30257496c0b
+Merge: fdc0534 418cff5
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 1 12:57:43 2013 -0700
+
+    Merge remote-tracking branch 'gh/bobtail-deploy' into bobtail-next
+
+commit fdc05346177a60f064fe351ca81c6078cd065179
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Apr 25 11:13:33 2013 -0700
+
+    init-ceph: use remote config when starting daemons on remote nodes (-a)
+    
+    If you use -a to start a remote daemon, assume the remote config is present
+    instead of pushing the local config.  This makes more sense and simplifies
+    things.
+    
+    Note that this means that -a in concert with -c foo means that foo must
+    also be present on the remote node in the same path.  That, however, is a
+    use case that I don't particularly care about right now.  :)
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit cd7e52cc76878eed0f084f7b9a6cf7c792b716c6)
+
+commit 55c87e821b6acc1cd46056ca27c6e7f7030466e7
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Apr 24 12:20:17 2013 -0700
+
+    PG: call check_recovery_sources in remove_down_peer_info
+    
+    If we transition out of peering due to affected
+    prior set, we won't trigger start_peering_interval
+    and check_recovery_sources won't get called.  This
+    will leave an entry in missing_loc_sources without
+    a matching missing set.  We always want to
+    check_recovery_sources with remove_down_peer_info.
+    
+    Fixes: 4805
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 81a6165c13c533e9c1c6684ab7beac09d52ca6b5)
+
+commit a28c2f55bfc768a502fbc8ec35265e7f1a53e044
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Apr 25 14:08:57 2013 -0700
+
+    PG: clear want_acting when we leave Primary
+    
+    This is somewhat annoying actually.  Intuitively we want to
+    clear_primary_state when we leave primary, but when we restart
+    peering due to a change in prior set status, we can't afford
+    to forget most of our peering state.  want_acting, on the
+    other hand, should never persist across peering attempts.
+    In fact, in the future, want_acting should be pulled into
+    the Primary state structure.
+    
+    Fixes: #3904
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit a5cade1fe7338602fb2bbfa867433d825f337c87)
+
+commit 849ed5982d4547dd651313919fb090944f568214
+Author: Greg Farnum <greg@inktank.com>
+Date:   Tue Apr 30 18:12:10 2013 -0700
+
+    mon: communicate the quorum_features properly when declaring victory.
+    
+    Fixes #4747.
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit fe68afe9d10bc5d49a05a8bafa644d57783447cf)
+
+commit ea9c76b8f457d44e63551d47b6dd30bd16a56468
+Author: Greg Farnum <greg@inktank.com>
+Date:   Tue Apr 30 11:01:54 2013 -0700
+
+    elector: trigger a mon reset whenever we bump the epoch
+    
+    We need to call reset during every election cycle; luckily we
+    can call it more than once. bump_epoch is (by definition!) only called
+    once per cycle, and it's called at the beginning, so we put it there.
+    
+    Fixes #4858.
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6ae9bbb5d03cb5695a4ebb7a3c20f729de1bd67a)
+    
+    Conflicts:
+    
+    	src/mon/Elector.cc
+
+commit 418cff585bf6fc292d260a464369d82f7cdb3b79
+Author: Alexandre Marangone <alexandre.marangone@inktank.com>
+Date:   Fri Apr 19 15:09:28 2013 -0700
+
+    Fix journal partition creation
+    
+    With OSD sharing data and journal, the previous code created the
+    journal partiton from the end of the device. A uint32_t is
+    used in sgdisk to get the last sector, with large HD, uint32_t
+    is too small.
+    The journal partition will be created backwards from the
+    a sector in the midlle of the disk leaving space before
+    and after it. The data partition will use whichever of
+    these spaces is greater. The remaining will not be used.
+    
+    This patch creates the journal partition from the start as a workaround.
+    
+    Signed-off-by: Alexandre Marangone <alexandre.marangone@inktank.com>
+    (cherry picked from commit 56619ab91732bd2eacbef388311954f4e0230a30)
+
+commit 1a6b87ea48c3bcd3904d888a2ad1e5774aa2de8e
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Apr 26 15:37:21 2013 -0700
+
+    ceph.spec.in: put ceph-disk-* et al in correct sbindir
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 86337936059d6b17c887e82f79167324eafa083c
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Apr 26 15:29:12 2013 -0700
+
+    debian: fix ceph.install
+    
+    This got out of sync somewhere in cherry-picking all of these patches.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 0b42b1edb306a9763bcd02bd962bd284f6b7b3a3
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Thu Apr 4 15:54:31 2013 +0200
+
+    Makefile.am: install ceph-* python scripts to /usr/bin directly
+    
+    Install ceph-* scripts directly to $(prefix)$(sbindir) (which
+    normaly would be /usr/sbin) instead of moving it around after
+    installation in SPEC file or debian files.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 4d16f38f48e276497190c8bc03abc55c40e18eed)
+
+commit c71fb8d5b39667fc2c9f7e898ced53059d3f1eec
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Tue Apr 2 12:11:10 2013 -0700
+
+    ceph-disk:  CalledProcessError has no output keyword on 2.6
+    
+    Signed-off-by: Gary Lowell  <gary.lowell@inktank.com>
+    (cherry picked from commit a793853850ee135de14b9237f7023cadcdb8575c)
+
+commit 1b86b1c7618b1ab93e25e1abf79bbe0a992c8526
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Apr 2 17:54:53 2013 +0200
+
+    ceph-disk: fix some (local) variable names
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit c4eb7e6ddd593cd45ab8343da01355be7382723e)
+
+commit ee452ebe6ff37087714c085dacd1ca2bcf66bd49
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Apr 2 17:36:37 2013 +0200
+
+    ceph-disk: fix naming of local variable in is_mounted()
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 3dd8b461219e64bb0f7a210dba5a9ab7c644f014)
+
+commit 0e47d312dcec2deb26a960e6097708d8742d85cb
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Apr 2 17:33:08 2013 +0200
+
+    ceph-disk: merge twice defined function is_mounted(dev)
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit eaf31bf9f90ba9709a57a6870dbafa21142dae2c)
+
+commit 7326ea6397160607d78914444f7f0598a6ba346f
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Apr 2 17:26:12 2013 +0200
+
+    ceph-disk: define exception type
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 4c6d6442a89adc5b56e99cb4d2ed572f2ad192c9)
+
+commit 14a348dc8ea9c012b6843825fd7beb5896a6d92e
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Apr 2 17:17:38 2013 +0200
+
+    ceph-disk: fix Redefining name 'uuid' from outer scope
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 058eb923c5b7dab611901fdd1724ce2a7c180827)
+
+commit 9419dca69292146ea1fd216a1ad08b2e07bdc1a0
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Apr 2 17:14:23 2013 +0200
+
+    ceph-disk: add missing space after comma
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 0080d1df7c7950e051840a543fc4bdabe6cc53e5)
+
+commit 7406981aa5515a3be8af0b9a0abef83a40c432b2
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 29 17:30:28 2013 -0700
+
+    ceph-disk list: say 'unknown cluster $UUID' when cluster is unknown
+    
+    This makes it clearer that an old osd is in fact old.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e9b3f2e6e93426d631d4f8101fb431a523b0d88a)
+
+commit 0182973b66d449552d17248e27cd12c470137ea9
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 29 13:59:04 2013 -0700
+
+    ceph-disk: handle missing journal_uuid field gracefully
+    
+    Only lower if we know it's not None.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 562e1716bd4b5372716e502a5e4dd872d381bfb7)
+
+commit 24d729c591430349b033992f87141e596dcd6bff
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Mar 28 20:49:24 2013 -0700
+
+    ceph-disk: implement 'list'
+    
+    This is based on Sandon's initial patch, but much-modified.
+    
+    Mounts ceph data volumes temporarily to see what is inside.  Attempts to
+    associated journals with osds.
+    
+    Resolves: #3120
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 6a65b9131c444041d16b880c6a7f332776063a78)
+
+commit bf3f8702c580c6b9647878837355688a2a4f954c
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Mar 27 18:44:32 2013 -0700
+
+    ceph-disk: reimplement list_all_partitions
+    
+    Use /dev/disk/by-id to list disks and their partitions.  This is more
+    accurate and correct than the previous (as-yet unused) implementation.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d3e49047ff405573aa41f45864cf315be23f5c50)
+
+commit 9da81e4e798b8e5593f3a7eda6dfa8586307121f
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Mar 27 18:43:59 2013 -0700
+
+    ceph-disk: reimplement is_partition
+    
+    Previously we were assuming any device that ended in a digit was a
+    partition, but this is not at all correct (e.g., /dev/sr0, /dev/rbd1).
+    Instead, look in /dev/disk/by-id and see if there is a symlink that ends in
+    -partNN that links to our device.
+    
+    There is probably still a better way...
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 20d594a889d62110ad03b761d8703f79f8eea6ad)
+
+commit 0c8efc0664f068a1952fe54b8034d005cf3f9ce8
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Mar 27 12:45:29 2013 -0700
+
+    ceph-disk: conditionally remove mount path
+    
+    umount removes it on success; only remove it here if it is still there.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4362934a94c5a6f7521f06aa1255e111094e1d88)
+
+commit 153994cdcf9bc4a9719849d951bec5f4983340b5
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Mar 26 17:04:14 2013 -0700
+
+    ceph-disk: ignore udevadm settle return code
+    
+    If we time out, just continue and let the next step fail.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2ae297ccb812c00204ec0c4dc614821c506062b9)
+
+commit d714049d530dc9611cf5f2fcb9b272c355dbe9f1
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Mon Mar 25 17:45:32 2013 +0100
+
+    ceph-disk: rename some local variabels in list_*partitions
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit b7d7e6894c550a7afa8dfb5bfa2bc54b5d331178)
+
+commit ecb34b81b843adaf9455503eb3ecc69ea94652a3
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Mon Mar 25 16:24:00 2013 +0100
+
+    ceph-disk: fix naming of a local variable in find_cluster_by_uuid
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 0b5fcfffe6d2f69bd4318cc93ef73195d948b9c5)
+
+commit 63eb85072f6a61552725bd04dfb261e5cd52a60a
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Mon Mar 25 16:18:17 2013 +0100
+
+    ceph-disk: rename some constants to upper case variable names
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 8a999ded088e688fd3f4a7c27127b7c06f0b2f66)
+
+commit d26a03422a37f16d609de12f8973f3c32ffedae0
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Mon Mar 25 16:15:29 2013 +0100
+
+    ceph-disk: add some more docstrings
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 86e55f5448c4b5b46b74d2d89b01d1e64b1ea826)
+
+commit bd8bb984806a1dbc3514c3a2a8980a03cfb2bc23
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Mon Mar 25 14:36:41 2013 +0100
+
+    ceph-disk: print subprocess.CalledProcessError on error
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 543327b1f2a9efe8083bb196433c4bcf83883269)
+
+commit 970348fc46a7e100731ffdc63d20f343dcb1b787
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Mon Mar 25 13:55:56 2013 +0100
+
+    ceph-disk: fix indention
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 2d26bcc07162a5176cdbc1748b829e3f39653697)
+
+commit 0113e5335e1552294bbf075cfff85c204b164e95
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Tue Mar 26 11:31:16 2013 -0700
+
+    ceph-disk:  udevadm settle before partprobe
+    
+    After changing the partition table, allow the udev event to be
+    processed before calling partprobe.  This helps prevent partprobe
+    from getting a resource busy error on some platforms.
+    
+    Signed-off-by: Gary Lowell  <gary.lowell@inktank.com>
+    (cherry picked from commit 9eda8e5d5abf0743a2ad484806cfb2018243515f)
+
+commit 02d48351f73330917185d8980f8ad123b99138b5
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 22 10:09:55 2013 -0700
+
+    ceph-disk: re-add python 2.7 dependency comment
+    
+    FIXME!
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6d63752c8fde91cdab306d1ca689690b269fe977)
+
+commit e92baf5081a43ff3290eb934612a11fbed946469
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Mar 20 14:36:58 2013 +0100
+
+    ceph-disk: cast output of subprocess.Popen() to str()
+    
+    Cast output of subprocess.Popen() to str() to be able to use
+    str.split() and str.splitlines() without warnings from pylint.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 87691dc46edfba11c370592dbb533772190be4b2)
+
+commit 1ffc89af1954643426708987a09291463bd7525b
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Mar 20 14:02:28 2013 +0100
+
+    ceph-disk: fix adjust_symlink() replace 'journal' with 'target'
+    
+    Replace 'journal' variable with 'target' since journal doesn't
+    exist in this function.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 3575feb7d2e5f4e35c5df193a1f8c9f08f88fcf4)
+
+commit 690ab6b3e284e9beed74b3e77612e804464e467d
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Mar 20 14:01:02 2013 +0100
+
+    ceph-disk: fix adjust_symlink() replace 'canonical' with 'path'
+    
+    Replace 'canonical' variable with 'path' since canonical doesn't
+    exist in this function.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit a27cb85b21610b4a9f8eb24f8f6cd04902349728)
+
+commit 329f279cd108c837e434ee3aae6c8ad502ba802f
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Mar 20 13:56:23 2013 +0100
+
+    ceph-disk: there is no os.path.lstat use os.lstat
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 6d3247b5c02c39a66666a5833106dbc23044e436)
+
+commit ffe024b8b5024cdac256bacb9df433b69385cc0a
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Mar 20 13:30:21 2013 +0100
+
+    ceph-disk: remove unused variable key from prepare_journal_dev()
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 97b4f8d66bef2328fa53f9e508eb38f8b8d49632)
+
+commit 9464284f3ac9737345c595847e165db48adc0a70
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Mar 20 13:27:27 2013 +0100
+
+    ceph-disk: fix except to catch OSError
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 9daf6cfce2d57509d896eae28bb97146a682a306)
+
+commit 01152115f7bf4a307296c9126174ce09bb0ce22d
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Mar 20 13:15:36 2013 +0100
+
+    ceph-disk: add missing space after >> operator
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 0ada43f79d2b1f9f84367e558c6d1a3e90e5c9b2)
+
+commit b9f86d96f5dd6ca18eb39882caf69e76369b5e0b
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Mar 20 12:33:09 2013 +0100
+
+    fix: Redefining name 'uuid' from outer scope (line 14)
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit d3c60dc8cad1db1d5df1c740bc805aaf9ba606ba)
+
+commit 6fa6cd85b55935739e16b8ba537f7848dbb0218f
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Mar 20 12:17:45 2013 +0100
+
+    ceph-disk: remove unused variables from list_partitions()
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 6a8120d4b0c4cfa851d473532eb2366534f8653d)
+
+commit 0b4e85fe23e8aa24e8aef8548df2b508bda24752
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Mar 20 12:10:22 2013 +0100
+
+    ceph-disk: fix /dev/dm-[0-9] handling list_all_partitions()
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 3af7a1ac5bf24bf832d7180002281d6b5853e85a)
+
+commit 3ec61f85f6a8a8af8a2b921f94e77a77be77a1d7
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Mar 19 22:07:36 2013 +0100
+
+    ceph-disk: rename local variable shadowing builtin
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 9bcf5b64f45ab6c4bdedf820ed111319b2dbd778)
+
+commit bd1036dd83fa2ee4807047fb9c2b414ccc9375eb
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Mar 19 22:05:23 2013 +0100
+
+    ceph-disk: remove twice defined identical function unmount
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 4adf088107586be7b356d1e963570cdab232809e)
+
+commit 8dd8cbac6f88fcb9d2c6838b9f961c70cccb9288
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Mar 19 22:02:49 2013 +0100
+
+    ceph-disk: remove twice defined function mount
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit ea26ea0d81a23aa76076ad5441c3b1aadfba3b44)
+
+commit 0da87db1c0e4b44b69c19b718d89e7a621979009
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Mar 19 21:58:22 2013 +0100
+
+    ceph-disk: remove double defined function get_conf
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit c57daa3c6e03e8974e133d3a2d9bc3d6f06f4faf)
+
+commit 9c46dfb2512d3b55345faed1bebb9204ba07d030
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Mar 19 21:51:54 2013 +0100
+
+    ceph-disk: rename local variable shadowing builtin
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 57dde5c8b18ff4ccd53a30bb94119c0ffceb52b7)
+
+commit b807d8ba0a0c23c7dab6893bc5297e07c0fd902f
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Mar 18 14:55:24 2013 -0700
+
+    ceph-disk: install and package
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c2602d749023b24ac22d8cfce6e04889078f14d8)
+    
+    Conflicts:
+    
+    	debian/ceph.install
+
+commit 8901e02db05837b12d5e1d0ec6f14f909cc03315
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 15 16:50:05 2013 -0700
+
+    ceph-disk: simplify command dispatch
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f287c6f90af0dfdd41358846b069aa3c54b600b3)
+
+commit 3cbc0d0c92ad19d5022c94d9d1ac00aff4eb9b2d
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 15 16:40:32 2013 -0700
+
+    ceph-disk: consolidate exceptions
+    
+    Use a single exception type, and catch it at the top level.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a019753bd3897ee0f5d9c3563c653c5457d5e67d)
+
+commit b4176bafa2b0721664332a1bb8b73da54151f685
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 15 16:36:45 2013 -0700
+
+    ceph-disk: consolidate ceph-disk-* into a single binary
+    
+     ceph-disk prepare ...
+     ceph-disk activate ...
+     ceph-disk ...
+    
+    This let's us share code (we were already duplicating a bunch of stuff!)
+    and to add new commands easily.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 20e4ba5cebb7b4989d1745932f7a412c28504ee1)
+
+commit 455cb32543df8036a605dc7b6c194e58b10bc4ec
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Mar 18 21:13:34 2013 -0700
+
+    ceph-disk-prepare: 'mkfs -t' instead of 'mkfs --type='
+    
+    Older mkfs (el6) doesn't like --type=.
+    
+    Fixes: #4495
+    Reported-by: Alexandre Maragone <alexandre.maragone@inktank.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d47759429a6e2fdd392265e90b67046302c97644)
+
+commit 19a2cf58504a3305ac9afe3d6642cf68e1b0f6c1
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Mar 14 21:05:07 2013 -0700
+
+    ceph-disk-activate: identify cluster .conf by fsid
+    
+    Determine what cluster the disk belongs to by checking the fsid defined
+    in /etc/ceph/*.conf.  Previously we hard-coded 'ceph'.
+    
+    Note that this has the nice side-effect that if we have a disk with a
+    bad/different fsid, we now fail to activate it.  Previously, we would
+    mount and start ceph-osd, but the daemon would fail to authenticate
+    because it was part of the wrong cluster.
+    
+    Fixes: #3253
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 80af5fb887f30792c342ac16da9ed95d7e89e491)
+
+commit 568485bea8ae9f0f36571de259c7254fa4227b15
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Mar 14 16:18:26 2013 -0700
+
+    ceph-disk-activate: abort if target position is already mounted
+    
+    If the target position is already a mount point, fail to move our mount
+    over to it.  This usually indicates that a different osd.N from a
+    different cluster instances is in that position.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 7370b5564606474f11b9ac5afb7cc60e0ac36ed1)
+
+commit a6ecf9288af248656bc712c1a4de910ff107a9a2
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Mar 14 12:05:52 2013 -0700
+
+    Revert "ceph-disk-activate: rely on default/configured keyring path"
+    
+    This reverts commit 936b8f20af1d390976097c427b6e92da4b39b218.
+    
+    This is necessary because we mount the osd in a temporary location.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b6102c0945a0d68b7c5ff7f2254a1a994788ca0f)
+
+commit 7c1edc0c927a27c07b6f1611c2cf9c48f2dc95f6
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Mar 14 12:04:44 2013 -0700
+
+    Revert "ceph-disk-activate: don't override default or configured osd journal path"
+    
+    This reverts commit 813e9fe2b4291a1c1922ef78f031daa9b78fe53b.
+    
+    We run --mkfs with the osd disk mounted in a temporary location, so it is
+    necessary to explicitly pass in these paths.
+    
+    If we want to support journals in a different location, we need to make
+    ceph-disk-prepare update the journal symlink accordingly.. not control it via
+    the config option.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3e628eee770508e750f64ea50179bbce52e7b8e0)
+
+commit 739b013cfa4b84d36750b80ce44579498e7ea4f4
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Mar 11 21:15:11 2013 -0700
+
+    ceph-disk-activate: rely on default/configured keyring path
+    
+    No reason to override the default or configured value here.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 936b8f20af1d390976097c427b6e92da4b39b218)
+
+commit 897413f90c0e802c6824c0ee8d06bdae33979293
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Mar 11 21:14:53 2013 -0700
+
+    ceph-disk-activate: don't override default or configured osd journal path
+    
+    There is no reason not to rely on the default or obey any configured
+    value here.
+    
+    Fixes: #4031
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 813e9fe2b4291a1c1922ef78f031daa9b78fe53b)
+
+commit 35eac085182e569b17c287c86e1415880909be22
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Mar 5 13:08:26 2013 -0800
+
+    ceph-disk-prepare: move in-use checks to the top, before zap
+    
+    Move the in-use checks to the very top, before we (say) zap!
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 32407c994f309cd788bf13fe9af27e17a422309a)
+
+commit 5ad4120a4825d6615778a005cff990afccf30691
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Mar 4 20:57:52 2013 -0800
+
+    ceph-disk-prepare: verify device is not in use by device-mapper
+    
+    Be nice and tell the user which devices/mappings are consuming the device,
+    too.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a6196de9e2f3ca9d67691f79d44e9a9f669443e9)
+
+commit e4a520029196cc76f1ec45695554ca6330bf14aa
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Mar 4 16:38:57 2013 -0800
+
+    ceph-disk-prepare: verify device is not mounted before using
+    
+    Make sure the data and/or journal device(s) are not in use (mounted)
+    before using them.  Make room for additional "in-use" checks in the future.
+    
+    Closes: #3256
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3bd0ac0ab011c4cdf0121f0d9732938d085fb8bf)
+
+commit d05b439142a24d28f35c6a09a8ab7c9eed503cd5
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Mar 4 16:29:57 2013 -0800
+
+    ceph-disk-prepare: clean up stupid check for a digit
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f03f62697f170d42b4b62c53d2860ff2f24a2d73)
+
+commit ea07b0e1047a4ac1b4dde88c795709190bd048be
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Mar 4 16:08:15 2013 -0800
+
+    ceph-disk-prepare: use os.path.realpath()
+    
+    My janky symlink resolution is broken in various ways.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 59505546e52a175435881b431bd349d532ae627e)
+
+commit 8f7e3e7dbbb9cab869366927c1687c38c012a1ef
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Thu Feb 28 14:16:26 2013 +0100
+
+    ceph.spec.in: add new Requires from ceph-disk-prepare
+    
+    Added new Requires from ceph-disk-prepare: cryptsetup, gptfdisk,
+    parted and util-linux.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 5c3f5c5b69a9edc99138d4f1ddb016689303dc28)
+    
+    Conflicts:
+    
+    	ceph.spec.in
+
+commit 3441acf3749dbd99f1e00e5306e8180647085d2a
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Feb 27 13:31:12 2013 -0800
+
+    debian: require cryptsetup-bin
+    
+    This is needed for ceph-disk-prepare's dmcrypt support.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cfcaceac44d6b7b7c55e81d0bfb05f4893f3b1d0)
+    
+    Conflicts:
+    
+    	debian/control
+
+commit abdac6fddd25bc71f59c1097246cfa32b33b0f5f
+Author: Alexandre Marangone <alexandre.marangone@inktank.com>
+Date:   Mon Apr 15 15:57:00 2013 -0700
+
+    Fix: use absolute path with udev
+    
+    Avoids the following: udevd[61613]: failed to execute '/lib/udev/bash'
+    'bash -c 'while [ ! -e /dev/mapper/....
+    
+    Signed-off-by: Alexandre Marangone <alexandre.marangone@inktank.com>
+    (cherry picked from commit 785b25f53dc7f8035eeba2aae8a196e3b102d930)
+
+commit d1775daf68d9aa887116a357cbcaf7c44e091cce
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Feb 16 20:55:03 2013 -0800
+
+    ceph-disk-prepare: -f for mkfs.xfs only
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit fecc3c3abf1176f4c7938e161559ea2db59f1cff)
+
+commit 405e0ea1c692b1d62f948734e07c4f919b90d4e0
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Feb 16 16:49:50 2013 -0800
+
+    debian: fix start of ceph-all
+    
+    Tolerate failure, and do ceph-all, not ceph-osd-all.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit aff0bb6fdc8ca358f7ac1e941bb9cfecbefb4bb6)
+
+commit 632be442261cabd56da0dc897b8a47a8e5493dce
+Author: Alexandre Marangone <alexandre.marangone@inktank.com>
+Date:   Fri Feb 15 12:24:01 2013 -0800
+
+    ceph-disk-prepare: always force mkfs.xfs
+    
+    Signed-off-by: Alexandre Marangone <alexandre.marangone@inktank.com>
+    (cherry picked from commit d950d83250db3a179c4b629fd32cd7bc8149997e)
+
+commit 28d1193836edee622763e308aad69c7bfe7a4971
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Feb 13 18:22:45 2013 -0800
+
+    udev: trigger on dmcrypted osd partitions
+    
+    Automatically map encrypted journal partitions.
+    
+    For encrypted OSD partitions, map them, wait for the mapped device to
+    appear, and then ceph-disk-activate.
+    
+    This is much simpler than doing the work in ceph-disk-activate.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e090a92a20f4161f473d16bc966f7d6aacac75ee)
+
+commit 5c5021b43ab70a08d791020262f76879dc77effb
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Feb 12 21:35:56 2013 -0800
+
+    ceph-disk-prepare: add initial support for dm-crypt
+    
+    Keep keys in /etc/ceph/dmcrypt-keys.
+    
+    Identify partition instances by the partition UUID.  Identify encrypted
+    partitions by a parallel set of type UUIDs.
+    
+    Signed-off-by: Alexandre Marangone <alexandre.maragone@inktank.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c6ac0ddf91915ba2aeae46d21367f017e18e82cd)
+
+commit d7084037d7a2395acfe8181f94963c1337c0f95b
+Author: Alexandre Marangone <alexandre.marangone@inktank.com>
+Date:   Fri Feb 15 12:22:33 2013 -0800
+
+    ceph-disk-activate: pull mount options from ceph.conf
+    
+    Signed-off-by: Alexandre Marangone <alexandre.marangone@inktank.com>
+    (cherry picked from commit e7040f55f01db3de7d5cebfc79de50c8b6ad5d45)
+
+commit 34fba357ca8b662c7b943d426ed37875c173003f
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 14 17:05:32 2013 -0800
+
+    ceph-disk-activate: use full paths for everything
+    
+    We are run from udev, which doesn't get a decent PATH.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b1c0fccba42dd184a2891ee873c0d6d8f8c79d14)
+
+commit caad18745cd8553a012d9e7f92e59d5b4450dcf2
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 14 17:04:55 2013 -0800
+
+    ceph-disk-prepare: do partprobe after setting final partition type
+    
+    This is necessary to kick udev into processing the updated partition and
+    running its rules.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 95835de9f80712eb26652ac6b66ba9c5eeb093d6)
+
+commit e1624e4626b4e67909f3bb83920c99ad323ed083
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Feb 13 21:47:30 2013 -0800
+
+    debian: start/stop ceph-all event on install/uninstall
+    
+    This helps us avoid the confusing situation with upstart where an individual
+    daemon job is running (like ceph-osd id=2) but the container jobs ceph-osd-all
+    and ceph-all are not.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b7b9af5c0d531dcee7ce9b10043a29b0a1b31f47)
+
+commit 656305f65eb33d68b95f4d29f37b031cf6cf889a
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Feb 13 21:37:08 2013 -0800
+
+    ceph-disk-activate: catch daemon start errors
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 690ae05309db118fb3fe390a48df33355fd068a0)
+
+commit 8b771bf929c4a9ab3fd4eb6f40b9222a463fd1a4
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Feb 13 17:30:19 2013 -0800
+
+    udev: trigger ceph-disk-activate directly from udev
+    
+    There is no need to depend on upstart for this.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5bd85ee5aa31bfd1f4f0e434f08c2a19414358ef)
+    
+    Conflicts:
+    
+    	ceph.spec.in
+
+commit ffb0613efaa9cf919992640322785eedf95ca25f
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Feb 13 16:02:10 2013 -0800
+
+    ceph-disk-activate: auto detect init system
+    
+    Look for an option 'init' in ceph.conf.  Otherwise, check if we're ubuntu.
+    If so, use upstart.  Otherwise, use sysvinit.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d1904b2a848af3c02d2065ac2a42abe0e2699d0f)
+
+commit 74b562703c422890c67b2a88d7b18242f64949dc
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Feb 13 15:49:42 2013 -0800
+
+    ceph-disk-activate: specify full path for blkid, initctl, service
+    
+    /sbin apparently isn't in the path when udev runs us.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f06b45e66315310abb0720e021da377186455048)
+
+commit 9ea32e5f066bee75923212d8009fb37a365ef32e
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 26 19:18:23 2013 -0800
+
+    upstart: ceph-hotplug -> ceph-osd-activate
+    
+    This is a more meaningful name.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e011ad128e7f302cb6955d9a7171ac0ec8890ddf)
+
+commit 494533a55010a59a52f6dcbe64f76e677a7dfe8e
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 26 19:14:22 2013 -0800
+
+    upstart/ceph-hotplug: tell activate to start via upstart
+    
+    This will mark the OSD data dir as upstart-managed.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 792e45c63dd7a9622fddd6e15ee4c075f995ea56)
+
+commit 5e0892fd8cbc2571d94cffc0b64daaa4ec6c38f8
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 26 19:08:22 2013 -0800
+
+    ceph-disk-prepare: refactor to support DIR, DISK, or PARTITION for data or journal
+    
+    Lots of code reorganization collapsed into a single commit here.
+    
+    - detect whether the user gave us a directory, disk, or partition, and Do The
+    Right Thing
+    - allow them to force that the input was of type X, for the careful/paranoid.
+    - make --zap-disk an option -- no longer the default
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b2ff6e8c9d96dee2c063b126de7030a5c2ae0d02)
+
+commit aa428017e4aad9872a54cd87036ef2bb528aac7b
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 26 20:33:16 2013 -0800
+
+    ceph-disk-activate: detect whether PATH is mount or dir
+    
+    remove in-the-way symlinks in /var/lib/ceph/osd
+    
+    This is simpler.  Just detect what the path is and Do The Right Thing.
+    
+    Closes #3341 (which wanted to make --mount the default)
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 191d5f7535f8d96d493e1b35b43a421c67c168ea)
+
+commit e6d5aa053dbaaf07c94e35406c23f4ca009fe2da
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 26 20:32:47 2013 -0800
+
+    ceph-disk-activate: add --mark-init INITSYSTEM option
+    
+    Do not assume we will manage via upstart; let that be passed down via the
+    command line.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit fd4a921085a861e4aa428376219bb39055731f2b)
+
+commit fa23919eae363291b4807afe3fadd34f94e9ddcb
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 26 14:44:26 2013 -0800
+
+    ceph-disk-activate: factor mounting out of activate
+    
+    The activate stuff is generic for any OSD, regardless of whether we want
+    to mount it or not.  Pull that part out.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 07655288281c9c6f691f87352dc26b7c11ae07e8)
+
+commit 181ebdee0addfe3d13c13342def94ecf9415d6d5
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Feb 12 18:17:55 2013 -0800
+
+    debian: put ceph-mds upstart conf in ceph-mds package
+    
+    Fixes: #3157
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 23ad3a46a0099e263f43e0f0c1df1d21cfe58b3f)
+
+commit 919b0aedf7286e85e6ddc23f99f80f29e1d35db6
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Feb 12 17:02:52 2013 -0800
+
+    debian: include /var/lib/ceph/bootstrap-mds in package
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e80675a0f333c04452d4822fd0eb3c6e92eda3df)
+
+commit f97f49b1e7ca8555c945b93fbc8241f5fa620dd8
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Feb 12 16:26:14 2013 -0800
+
+    ceph-create-keys: create mds bootstrap key
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 809143f16c70483ba5bb429dea812d31b67f2b49)
+
+commit f43c339d67b78e6afc3737d26cea4bc05dcea900
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 26 14:45:43 2013 -0800
+
+    upstart/ceph-hotplug: drop -- in ceph-disk-activate args
+    
+    We would like to transition to
+    
+     ceph-disk-activate --mount DEV
+    
+    and away from a generic multi-definition PATH argument.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4698b6a1035dee8509ce2d4dab7b34a16b78f7cd)
+
+commit 39df4c81f2166b7e9ac89c591b268b60de4042e6
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 25 22:53:23 2013 -0800
+
+    init-ceph: iterate/locate local sysvinit-tagged directories
+    
+    Search /var/lib/ceph/$type/ceph-$id and start/stop those daemons if
+    present and tagged with the sysvinit file.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c8f528a4070dd3aa0b25c435c6234032aee39b21)
+
+commit 05efb7ab5b289472a1af05d6aa100a7385be304e
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 25 22:52:03 2013 -0800
+
+    init-ceph: consider sysvinit-tagged dirs as local
+    
+    If there is a 'sysvinit' file in the daemon directory in the default
+    location (/var/lib/ceph/$type/ceph-$id), consider it sysvinit-managed.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b8aa4769a62e0d88174678cbefd89d9ee2baceea)
+
+commit 8c4c53abf84d30ff4a605225a22575f56d946971
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Feb 11 17:39:03 2013 -0800
+
+    ceph-disk-prepare: align mkfs, mount config options with mkcephfs
+    
+    'osd mkfs ...', not 'osd fs mkfs ...'.  Sigh.  Support both.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit af2372ca4a702da70275edd1b1357fcff51e6ae2)
+
+commit a8e7e9df61a7229d9e2b4b4dedc68b5c1bf15c38
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Apr 23 10:00:38 2013 -0700
+
+    init-ceph: fix (and simplify) pushing ceph.conf to remote unique name
+    
+    The old code would only do the push once per remote node (due to the
+    list in $pushed_to) but would reset $unique on each attempt.  This would
+    break if a remote host was processed twice.
+    
+    Fix by just skipping the $pushed_to optimization entirely.
+    
+    Fixes: #4794
+    Reported-by: Andreas Friedrich <andreas.friedrich@ts.fujitsu.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ccbc4dbc6edf09626459ca52a53a72682f541e86)
+
+commit 9374baccb8a71e1fe5ea2e614feac93e865dd027
+Merge: b6b4ebe aa37726
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Apr 23 12:04:45 2013 -0700
+
+    Merge pull request #238 from ceph/wip-bobtail-rbd-backports-req-order
+    
+    Reviewed-by: Sage Weil <sage.weil@inktank.com>
+
+commit aa37726b8d43d08b495d55dc2aef93854e9539c5
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Apr 23 09:18:30 2013 -0700
+
+    rbd: only set STRIPINGV2 feature when needed
+    
+    Only set the STRIPINGV2 feature if the striping parameters are non-default.
+    Specifically, fix the case where the passed-in size and count are == 0.
+    
+    Fixes: #4710
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5926ffa576e9477324ca00eaec731a224195e7db)
+    
+    Conflicts:
+    
+    	src/rbd.cc
+
+commit 0e2266dba912cde2c4a86ba7055c8bec3a5bc1a5
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Apr 10 14:16:56 2013 -0700
+
+    LibrbdWriteback:  complete writes strictly in order
+    
+    RADOS returns writes to the same object in the same order. The
+    ObjectCacher relies on this assumption to make sure previous writes
+    are complete and maintain consistency. Reads, however, may be
+    reordered with respect to each other. When writing to an rbd clone,
+    reads to the parent must be performed when the object does not exist
+    in the child yet. These reads may be reordered, resulting in the
+    original writes being reordered. This breaks the assmuptions of the
+    ObjectCacher, causing an assert to fail.
+    
+    To fix this, keep a per-object queue of outstanding writes to an
+    object in the LibrbdWriteback handler, and finish them in the order in
+    which they were sent.
+    
+    Fixes: #4531
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 06d05e5ed7e09fa873cc05021d16f21317a1f8ef)
+
+commit 7a11c2505a8b74babee90cca945c2a068a656ec1
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Apr 10 12:22:02 2013 -0700
+
+    LibrbdWriteback: removed unused and undefined method
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 909dfb7d183f54f7583a70c05550bec07856d4e4)
+
+commit 884438febaf12dc380d4a1a3c3c8d3cbefd32399
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Apr 10 12:06:36 2013 -0700
+
+    LibrbdWriteback: use a tid_t for tids
+    
+    An int could be much smaller, leading to overflow and bad behavior.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 9d19961539b2d50d0c9edee1e3d5ac6912a37f24)
+
+commit 124f81cc5fa5a3ac9e7777ad49dd42d864fd7f68
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Apr 10 12:03:04 2013 -0700
+
+    WritebackHandler: make read return nothing
+    
+    The tid returned by reads is ignored, and would make tracking writes
+    internally more difficult by using the same id-space as them. Make read
+    void and update all implementations.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 870f9cd421ca7b0094f9f89e13b1898a8302c494)
+
+commit 13ba07a0f3af479861c0e83e3e6441fef1bc8a99
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Apr 1 14:51:46 2013 -0700
+
+    ObjectCacher: deduplicate final part of flush_set()
+    
+    Both versions of flush_set() did the same thing. Move it into a
+    helper called from both.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit f5b81d8d167d1aa7f82a5776bbb1f319063ab809)
+
+commit 7bc8df1f3e3b60b59672a082eba374729a6866b8
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Apr 10 11:35:46 2013 -0700
+
+    test_stress_watch: remove bogus asserts
+    
+    There's no reason to check the duration of a watch. The notify will
+    timeout after 30s on the OSD, but there's no guarantee the client will
+    see that in any bounded time. This test is really meant as a stress
+    test of the OSDs anyway, not of the clients, so just remove asserts
+    about operation duration.
+    
+    Fixes: #4591
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Sam Just <sam.just@inktank.com>
+    (cherry picked from commit 4b656730ffff21132f358c2b9a63504dfbf0998d)
+
+commit 4a1c27c0e6e2e70be2b58e01872e321352c8e484
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Apr 1 11:09:52 2013 -0700
+
+    librados: don't use lockdep for AioCompletionImpl
+    
+    This is a quick workaround for the next branch. A more complete fix
+    will be done for the master branch. This does not affect correctness,
+    just what qa runs with lockdep enabled do.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Sage Weil <sage.weil@inktank.com>
+    (cherry picked from commit 267ce0d90b8f3afaaddfdc0556c9bafbf4628426)
+
+commit d36c5b5bd30a9d7e60403a09cedaea28bdc29e99
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Fri Mar 29 12:46:27 2013 -0700
+
+    librados: move snapc creation to caller for aio_operate
+    
+    The common case already has a snapshot context, so avoid duplicating
+    it (copying a potentially large vector) in IoCtxImpl::aio_operate().
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 4c4d5591bdb048cd9ffa25b529c6127356e7f9a7)
+
+commit 31a45e8e23ebf9815ce5a4e0c89ddbf384702aad
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Mar 21 16:04:10 2013 -0700
+
+    librbd: add an async flush
+    
+    At this point it's a simple wrapper around the ObjectCacher or
+    librados.
+    
+    This is needed for QEMU so that its main thread can continue while a
+    flush is occurring. Since this will be backported, don't update the
+    librbd version yet, just add a #define that QEMU and others can use to
+    detect the presence of aio_flush().
+    
+    Refs: #3737
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 302b93c478b3f4bc2c82bfb08329e3c98389dd97)
+
+commit f2e490cb5b1f322e637d787e788733ac4b2496f4
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Mar 27 15:42:10 2013 -0700
+
+    librbd: use the same IoCtx for each request
+    
+    Before we were duplicating the IoCtx for each new request since they
+    could have a different snapshot context or read from a different
+    snapshot id. Since librados now supports setting these explicitly
+    for a given request, do that instead.
+    
+    Since librados tracks outstanding requests on a per-IoCtx basis, this
+    also fixes a bug that causes flush() without caching to ignore
+    all the outstanding requests, since they were to separate,
+    duplicate IoCtxs.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 860493e7ff0d87d02069b243fc1c8326ce0721f9)
+
+commit cbb37fb5470f22949a4dbb747f428943cce4aca7
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Mar 27 15:37:27 2013 -0700
+
+    librbd: add an is_complete() method to AioCompletions
+    
+    Mainly this is useful for testing, like flushing and checking that
+    all pending writes are complete after the flush finishes.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 2ae32068dee22a0ca0698e230ead98f2eeeff3e6)
+
+commit f9bcffa2482220202e68d6bac70216d44ee5046b
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Mar 27 15:32:29 2013 -0700
+
+    librados: add versions of a couple functions taking explicit snap args
+    
+    Usually the snapid to read from or the snapcontext to send with a write
+    are determined implicitly by the IoCtx the operations are done on.
+    
+    This makes it difficult to have multiple ops in flight to the same
+    IoCtx using different snapcontexts or reading from different snapshots,
+    particularly when more than one operation may be needed past the initial
+    scheduling.
+    
+    Add versions of aio_read, aio_sparse_read, and aio_operate
+    that don't depend on the snap id or snapcontext stored in the IoCtx,
+    but get them from the caller. Specifying this information for each
+    operation can be a more useful interface in general, but for now just
+    add it for the methods used by librbd.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit f06debef6c293750539501ec4e6103e5ae078392)
+
+commit fb95b800a4cc431a8bd98692fffbfc150f6f8d67
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Mar 27 14:48:31 2013 -0700
+
+    librados: add async flush interface
+    
+    Sometimes you don't want flush to block, and can't modify
+    already scheduled aio_writes. This will be useful for a
+    librbd async flush interface.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 7cc0940f89070dadab5b9102b1e78362f762f402)
+    
+    Conflicts:
+    
+    	src/include/rados/librados.h
+    	src/include/rados/librados.hpp
+
+commit 3a61d17b6729b73d0bf2df106f54dafb603af646
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Mar 28 10:34:37 2013 -0700
+
+    ObjectCacher: remove unneeded var from flush_set()
+    
+    The gather will only have subs if there is something to flush. Remove
+    the safe variable, which indicates the same thing, and convert the
+    conditionals that used it to an else branch. Movinig gather.activate()
+    inside the has_subs() check has no effect since activate() does
+    nothing when there are no subs.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 553aaac8a19e2359acf6d9d2e1bb4ef0bdba7801)
+
+commit ee7bf281005cce671e467ca9b9eff0ba7eb9899e
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Fri Mar 22 12:17:43 2013 -0700
+
+    ObjectCacher: remove NULL checks in flush_set()
+    
+    Callers will always pass a callback, so assert this and remove the
+    checks for it being NULL.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 41568b904de6d155e5ee87c68e9c31cbb69508e5)
+
+commit d86f9b1dd370563170ab875c6a7de511f05acbf7
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Fri Mar 22 12:13:36 2013 -0700
+
+    ObjectCacher: always complete flush_set() callback
+    
+    This removes the last remnants of
+    b5e9995f59d363ba00d9cac413d9b754ee44e370. If there's nothing to flush,
+    immediately call the callback instead of deleting it. Callers were
+    assuming they were responsible for completing the callback whenever
+    flush_set() returned true, and always called complete(0) in this
+    case. Simplify the interface and just do this in flush_set(), so that
+    it always calls the callback.
+    
+    Since C_GatherBuilder deletes its finisher if there are no subs,
+    only set its finisher when subs are present. This way we can still
+    call ->complete() for the callback.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 92db06c05dc2cad8ed31648cb08866781aee2855)
+    
+    Conflicts:
+    
+    	src/client/Client.cc
+
+commit 00dfb3f06fb6c2581a3f71126adcde0126e5e207
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Jan 29 14:22:15 2013 -0800
+
+    ObjectCacher: fix flush_set when no flushing is needed
+    
+    C_GatherBuilder takes ownership of the Context we pass it. Deleting it
+    in flush_set after constructing the C_GatherBuilder results in a
+    double delete.
+    
+    Fixes: #3946
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Sam Lang <sam.lang@inktank.com>
+    (cherry picked from commit 3bc21143552b35698c9916c67494336de8964d2a)
+
+commit 0f2e5d36e4dcd073174715c310fa22a835ca1259
+Author: Sam Lang <sam.lang@inktank.com>
+Date:   Fri Jan 18 14:59:12 2013 -0600
+
+    objectcacher: Remove commit_set, use flush_set
+    
+    commit_set() and flush_set() are identical in functionality,
+    so use flush_set everywhere and remove commit_set from
+    the code.
+    
+    Also fixes a bug in flush_set where the finisher context was
+    getting freed twice if no objects needed to be flushed.
+    
+    Signed-off-by: Sam Lang <sam.lang@inktank.com>
+    (cherry picked from commit 72147fd3a1da8ecbcb31ddf6b66a158d71933909)
+
+commit 3b0c565dceb41dcaf53965321ef420641bcb699b
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Mar 13 09:42:43 2013 -0700
+
+    librbd: make aio_writes to the cache always non-blocking by default
+    
+    When the ObjectCacher's writex blocks, it affects the thread requesting
+    the aio, which can cause starvation for other I/O when used by QEMU.
+    
+    Preserve the old behavior via a config option in case this has any
+    bad side-effects, like too much memory usage under heavy write loads.
+    
+    Fixes: #4091
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 03ac01fa6a94fa7a66ede057e9267e0a562c3cdb)
+
+commit e237dfc742bba76664861a8868c502946fe53548
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Mar 13 09:37:21 2013 -0700
+
+    ObjectCacher: optionally make writex always non-blocking
+    
+    Add a callback argument to writex, and a finisher to run the
+    callbacks. Move the check for dirty+tx > max_dirty into a helper that
+    can be called from a wrapper around the callbacks from writex, or from
+    the current place in _wait_for_write().
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit c21250406eced8e5c467f492a2148c57978634f4)
+
+commit 7bc1596b90d2d5853659cb3ffbe9653b633345f4
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Mar 27 17:30:42 2013 -0700
+
+    librbd: flush cache when set_snap() is called
+    
+    If there are writes pending, they should be sent while the image
+    is still writeable. If the image becomes read-only, flushing the
+    cache will just mark everything dirty again due to -EROFS.
+    
+    Fixes: #4525
+    Backport: bobtail
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 613b7085bb48cde1e464b7a97c00b8751e0e917f)
+
+commit 9facdcac7bb8ab92f1ab486eaabee681e6b6519d
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Fri Mar 15 17:28:13 2013 -0700
+
+    librbd: optionally wait for a flush before enabling writeback
+    
+    Older guests may not send flushes properly (i.e. never), so if this is
+    enabled, rbd_cache=true is safe for them transparently.
+    
+    Disable by default, since it will unnecessarily slow down newer guest
+    boot, and prevent writeback caching for things that don't need to send
+    flushes, like the command line tool.
+    
+    Refs: #3817
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1597b3e3a1d776b56e05c57d7c3de396f4f2b5b2)
+
+commit 1c44b66fb371a5864860c067bb5f59374f43005d
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Fri Mar 8 18:57:24 2013 -0800
+
+    librbd: invalidate cache when flattening
+    
+    The cache stores which objects don't exist. Flatten bypasses the cache
+    when doing its copyups, so when it is done the -ENOENT from the cache
+    is treated as zeroes instead of 'need to read from parent'.
+    
+    Clients that have the image open need to forgot about the cached
+    non-existent objects as well. Do this during ictx_refresh, while the
+    parent_lock is held exclusively so no new reads from the parent can
+    happen until the updated parent metadata is visible, so no new reads
+    from the parent will occur.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 46e8fc00b2dc8eb17d8777b6ef5ad1cfcc389cea)
+
+commit d9ca1b00a4dacb4715a69a276a644d645d0d5bc1
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Fri Mar 8 17:53:31 2013 -0800
+
+    ObjectCacher: add a method to clear -ENOENT caching
+    
+    Clear the exists and complete flags for any objects that have exists
+    set to false, and force any in-flight reads to retry if they get
+    -ENOENT instead of generating zeros.
+    
+    This is useful for getting the cache into a consistent state for rbd
+    after an image has been flattened, since many objects which previously
+    did not exist and went up to the parent to retrieve data may now exist
+    in the child.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit f2a23dc0b092c5ac081893e8f28c6d4bcabd0c2e)
+
+commit 1e51be05b7909a676ae499ca84e4cce94bb50da4
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Fri Mar 8 17:49:27 2013 -0800
+
+    ObjectCacher: keep track of outstanding reads on an object
+    
+    Reads always use C_ReadFinish as a callback (and they are the only
+    user of this callback). Keep an xlist of these for each object, so
+    they can remove themselves as they finish. To prevent racing requests
+    and with discard removing objects from the cache, clear the xlist in
+    the object destructor, so if the Object is still valid the set_item
+    will still be on the list.
+    
+    Make the ObjectCacher constructor take an Object* instead of the pool
+    and object id, which are derived from the Object* anyway.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit f6f876fe51e40570596c25ac84ba3689f72776c2)
+
+commit f2bcf241257500f5784b216b7a377ddec4669531
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Feb 25 16:09:26 2013 -0800
+
+    test_rbd: move flatten tests back into TestClone
+    
+    They need the same setup, and it's easy enough to run specific
+    subtests. Making them a separate subclass accidentally duplicated
+    tests from TestClone.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 9c693d7e8312026f6d8d9586381b026ada35d808)
+
+commit cd989681823a2b8521c789f5d4fea5525ac53911
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Feb 26 13:20:08 2013 -0800
+
+    librbd: fix rollback size
+    
+    The duplicate calls to get_image_size() and get_snap_size() replaced
+    by 5806226cf0743bb44eaf7bc815897c6846d43233 uncovered this. The first
+    call was using the currently set snap_id instead of the snapshot being
+    rolled back to.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit d6c126e2131fefab6df676f2b9d0addf78f7a488)
+
+commit 796066b7a8324b12a660d9b80b927e643db7529e
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Feb 25 12:05:16 2013 -0800
+
+    Merge branch 'wip-4249' into wip-4249-master
+    
+    Make snap_rollback() only take a read lock on snap_lock, since
+    it does not modify snapshot-related fields.
+    Conflicts:
+    	src/librbd/internal.cc
+    (cherry picked from commit db5fc2270f91aae220fc3c97b0c62e92e263527b)
+
+commit 34e9030e50fe15a320cbfbc9854c86deb3a9c57c
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Feb 21 11:26:45 2013 -0800
+
+    librbd: make sure racing flattens don't crash
+    
+    The only way for a parent to disappear is a racing flatten completing,
+    or possibly in the future the image being forcibly removed. In either
+    case, continuing to flatten makes no sense, so stop early.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit a1ae8562877d1b902918e866a1699214090c40bd)
+
+commit 6e6636d50840e97022c8e4a1072f42fee15050a9
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Feb 21 11:17:18 2013 -0800
+
+    librbd: use rwlocks instead of mutexes for several fields
+    
+    Image metadata like snapshots, size, and parent is frequently read,
+    but rarely updated. During flatten, we were depending on the parent
+    lock to prevent the parent ImageCtx from disappearing out from under
+    us while we read from it. The copy-up path also needed the parent lock
+    to be able to read from the parent image, which lead to a deadlock.
+    
+    Convert parent_lock, snap_lock, and md_lock to RWLocks, and change
+    their use to read instead of exclusive locks where appropriate. The
+    main place exclusive locks are needed is in ictx_refresh, so this is
+    pretty simple. This fixes the deadlock, since parent_lock is only
+    needed for read access in both flatten and the copy-up operation.
+    
+    cache_lock and refresh_lock are only really used for exclusive access,
+    so leave them as regular mutexes.
+    
+    One downside to this is that there's no way to assert is_locked()
+    for RWLocks, so we'll have to be very careful about changing code
+    in the future.
+    
+    Fixes: #3665
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 995ff0e3eaa560b242da8c019a2e11e735e854f7)
+
+commit 9b292199ef63822462c67cb6cf1cca6a53cf9fd7
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Feb 21 11:15:41 2013 -0800
+
+    common: add lockers for RWLocks
+    
+    This makes them easier to use, especially instead of existing mutexes.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit e0f8e5a80d6d22bd4dee79a4996ea7265d11b0c1)
+
+commit d8ac6cbf883fce0f39d661a879ff2462479d5494
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Feb 21 23:22:59 2013 -0800
+
+    objecter: initialize linger op snapid
+    
+    Since they are write ops now, it must be CEPH_NOSNAP or the OSD
+    returns EINVAL.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 15bb9ba9fbb4185708399ed6deee070d888ef6d2)
+
+commit 9ea4dac1db46c20cd83c5af5d29d9ce7a5768380
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 21 15:44:19 2013 -0800
+
+    objecter: separate out linger_read() and linger_mutate()
+    
+    A watch is a mutation, while a notify is a read.  The mutations need to
+    pass in a proper snap context to be fully correct.
+    
+    Also, make the WRITE flag implicit so the caller doesn't need to pass it
+    in.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6c08c7c1c6d354d090eb16df279d4b63ca7a355a)
+
+commit d9636faabfdb2c0a876f0afa092a2ab925cf8983
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 21 15:31:08 2013 -0800
+
+    osd: make watch OSDOp print sanely
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit de4fa95f03b99a55b5713911c364d7e2a4588679)
+
+commit 959bfe9097a2cfa48e2ab21f11bd32c1349807aa
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 21 13:28:47 2013 -0800
+
+    osdc/Objecter: unwatch is a mutation, not a read
+    
+    This was causing librados to unblock after the ACK on unwatch, which meant
+    that librbd users raced and tried to delete the image before the unwatch
+    change was committed..and got EBUSY.  See #3958.
+    
+    The watch operation has a similar problem.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit fea77682a6cf9c7571573bc9791c03373d1d976d)
+    
+    Conflicts:
+    
+    	src/librados/IoCtxImpl.cc
+
+commit b6b4ebed559db8bf74da4f482201948b5abc9225
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 21 11:15:58 2013 -0800
+
+    osd: an interval can't go readwrite if its acting is empty
+    
+    Let's not forget that min_size can be zero.
+    
+    Fixes: #4159
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4277265d99647c9fe950ba627e5d86234cfd70a9)
+
+commit 055d746c3bca0d7067b71ed1ff6d1c1cce42b5ed
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Feb 19 08:29:53 2013 -0800
+
+    mon: restrict pool size to 1..10
+    
+    See: #4159
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 30b8d653751acb4bc4be5ca611f154e19afe910a)
+
+commit c073bd25610ba556f3b30e6c583d1d74a10f09e2
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Apr 19 13:05:43 2013 -0700
+
+    init-ceph: do not stop start on first failure
+    
+    When starting we often loop over many daemon instances.  Currently we stop
+    on the first error and do not try to start other daemons.
+    
+    Instead, try them all, but return a failure if anything did not start.
+    
+    Fixes: #2545
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Gary Lowell <gary.lowell@inktank.com>
+    (cherry picked from commit d395aa521e8a4b295ed2b08dd7cfb7d9f995fcf7)
+    
+    Conflicts:
+    
+    	src/init-ceph.in
+
+commit daa6ed2bfe790f83f986f51d0917d6268d3e9d78
+Merge: 1d0bf61 9fe57e2
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Apr 11 13:00:27 2013 -0700
+
+    Merge pull request #210 from dalgaaf/wip-da-bobtail-pybind
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 9fe57e2bea89243a0080e667f90b7d4fb15ed883
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Fri Apr 5 15:55:34 2013 +0200
+
+    rados.py: fix create_pool()
+    
+    Call rados_pool_create_with_all() only if auid and crush_rule
+    are set properly. In case only crush_rule is set call
+    rados_pool_create_with_crush_rule() on librados, not the other
+    way around.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 94a1f25e7230a700f06a2699c9c2b99ec1bf7144)
+
+commit 1d0bf61da98bec7318f44b95a49298e13b792b70
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Mon Apr 8 13:52:32 2013 -0700
+
+    mon: Use _daemon version of argparse functions
+    
+    Allow argparse functions to fail if no argument given by using
+    special versions that avoid the default CLI behavior of "cerr/exit"
+    
+    Fixes: #4678
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit be801f6c506d9fbfb6c06afe94663abdb0037be5)
+    
+    Conflicts:
+    	src/mon/Monitor.cc
+
+commit 3769250acf42a751ee8dfa95207a4ffafd25574a
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Mon Apr 8 13:49:22 2013 -0700
+
+    ceph_argparse: add _daemon versions of argparse calls
+    
+    mon needs to call argparse for a couple of -- options, and the
+    argparse_witharg routines were attempting to cerr/exit on missing
+    arguments.  This is appropriate for the CLI usage, but not the daemon
+    usage.  Add a 'cli' flag that can be set false for the daemon usage
+    (and cause the parsing routine to return false instead of exit).
+    
+    The daemon's parsing code due for a rewrite soon.
+    
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit c76bbc2e6df16d283cac3613628a44937e38bed8)
+
+commit 1f3b4917cc5c4d796a96637912d2ae5ce2877861
+Author: Alexandre Oliva <oliva@gnu.org>
+Date:   Wed Feb 6 15:27:13 2013 -0200
+
+    silence logrotate some more
+    
+    I was getting email with logrotate error output from “which invoke-rc.d�
+    on systems without an invoke-rc.d.  This patch silences it.
+    
+    Silence stderr from which when running logrotate
+    
+    From: Alexandre Oliva <oliva@gnu.org>
+    
+    Signed-off-by: Alexandre Oliva <oliva@gnu.org>
+    (cherry picked from commit d02340d90c9d30d44c962bea7171db3fe3bfba8e)
+
+commit e6eace2334d04fc9292c96a843304d1748b5bbe7
+Merge: 8cb13a1 1507443
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Mar 29 12:14:22 2013 -0700
+
+    Merge remote-tracking branch 'upstream/bobtail-4556' into bobtail
+    
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+
+commit 8cb13a195c592fd2f17889e77d717c1864f97dca
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Feb 14 14:03:56 2013 -0800
+
+    OSD: always activate_map in advance_pgs, only send messages if up
+    
+    We should always handle_activate_map() after handle_advance_map() in
+    order to kick the pg into a valid peering state for processing requests
+    prior to dropping the lock.
+    
+    Additionally, we would prefer to avoid sending irrelevant messages
+    during boot, so only send if we are up according to the current service
+    osdmap.
+    
+    Fixes: #4572
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4dfcad44431855ba7d68a1ccb41dc3cb5db6bb50)
+
+commit d89ab0ea6fa8d0961cad82f6a81eccbd3bbd3f55
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Mar 28 14:09:17 2013 -0700
+
+    PG: update PGPool::name in PGPool::update
+    
+    Fixes: #4471
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f804892d725cfa25c242bdc577b12ee81dcc0dcc)
+
+commit 750626108616761512271d5a4f10dee82a54e460
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Mar 26 15:10:37 2013 -0700
+
+    ReplicatedPG: send entire stats on OP_BACKFILL_FINISH
+    
+    Otherwise, we update the stat.stat structure, but not the
+    stat.invalid_stats part.  This will result in a recently
+    split primary propogating the invalid stats but not the
+    invalid marker.  Sending the whole pg_stat_t structure
+    also mirrors MOSDSubOp.
+    
+    Fixes: #4557
+    Backport: bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 76b296f01fd0d337c8fc9f79013883e62146f0c6)
+
+commit 1507443271fda933032ef0877aff1890d4fd5b63
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Mar 27 13:19:03 2013 -0700
+
+    osd: disallow classes with flags==0
+    
+    They must be RD, WR, or something....
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 89c69016e1dddb9f3ca40fd699e4a995ef1e3eee)
+
+commit 6b6e0cef99e66b46ade35ab262f071982049bc22
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Mar 27 12:59:41 2013 -0700
+
+    osd: EINVAL when rmw_flags is 0
+    
+    A broken client (e.g., v0.56) can send a request that ends up with an
+    rmw_flags of 0.  Treat this as invalid and return EINVAL.
+    
+    Fixes: #4556
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f2dda43c9ed4fda9cfa87362514985ee79e0ae15)
+
+commit 4bdd37495cedb1cf30a5311548492fe3f5db6e92
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Mar 27 13:08:38 2013 -0700
+
+    osd: fix detection of non-existent class method
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 50b831e3641c21cd5b145271688189e199f432d1)
+
+commit f28800f8b65e90a3b96429f07197236e6c9bf1a2
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Mar 27 13:12:38 2013 -0700
+
+    osd: tolerate rmw_flags==0
+    
+    We will let OSD return a proper error instead of asserting.
+    
+    This is effectively a backport of c313423cfda55a2231e000cd5ff20729310867f8.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 94321ccdff81d5d6cea1acdb54344c3d930a49eb
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Feb 21 17:39:19 2013 -0800
+
+    test_librbd_fsx: fix image closing
+    
+    Always close the image we opened in check_clone(), and check the
+    return code of the rbd_close() called before cloning.
+    
+    Refs: #3958
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 94ae72546507799667197fd941633bb1fd2520c2)
+
+commit 7fbc1ab69267e6be96665b0d4e277dc4a07f9220
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Mar 13 17:05:42 2013 -0700
+
+    rbd: remove fiemap use from import
+    
+    On some kernels and filesystems fiemap can be racy and provide
+    incorrect data even after an fsync. Later we can use SEEK_HOLE and
+    SEEK_DATA, but for now just detect zero runs like we do with stdin.
+    
+    Basically this adapts import from stdin to work in the case of a file
+    or block device, and gets rid of other cruft in the import that used
+    fiemap.
+    
+    Fixes: #4388
+    Backport: bobtail
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 3091283895e8ffa3e4bda13399318a6e720d498f)
diff --git a/doc/changelog/v0.56.6.txt b/doc/changelog/v0.56.6.txt
new file mode 100644
index 000000000..17818d278
--- /dev/null
+++ b/doc/changelog/v0.56.6.txt
@@ -0,0 +1,40 @@
+commit 95a0bda7f007a33b0dc7adf4b330778fa1e5d70c
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Fri May 3 12:53:00 2013 -0700
+
+    v0.56.6
+
+commit 6dbdcf5a210febb5e0dd585e0e599ac807642210
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Fri May 3 12:45:24 2013 -0700
+
+    ceph.spec.in:  Fix platform dependecies
+
+    Picked up an incorrect dependency merging the rbd udev rules update.
+
+    Signed-off-by: Gary Lowell  <gary.lowell@inktank.com>
+
+commit 05af17e697eb95b2a807d9c05cde39106c5ecee9
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Apr 22 12:48:56 2013 -0700
+
+    rgw: don't send tail to gc if copying object to itself
+
+    Fixes: #4776
+    Backport: bobtail
+    Need to make sure that when copying an object into itself we don't
+    send the tail to the garbage collection.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit de5d1da810732ee48f41e8be18257053d862301b)
+
+commit f0eb20a7b0f7c8afadc21cc063f1f289b5092bab
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 3 12:24:21 2013 -0700
+
+    ceph_common.sh: re-sync get_name_list with master
+
+    We backported various items but didn't catch all the changes!  :(
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
diff --git a/doc/changelog/v0.56.7.txt b/doc/changelog/v0.56.7.txt
new file mode 100644
index 000000000..b01c0de10
--- /dev/null
+++ b/doc/changelog/v0.56.7.txt
@@ -0,0 +1,454 @@
+commit 14f23ab86b0058a8651895b3dc972a29459f3a33
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Tue Aug 27 10:44:32 2013 -0700
+
+    v0.56.7
+
+commit 8551be345c86837e0893fdf6c9c5b0af523f50f8
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Aug 21 14:28:49 2013 -0700
+
+    objecter: resend unfinished lingers when osdmap is no longer paused
+
+    Plain Ops that haven't finished yet need to be resent if the osdmap
+    transitions from full or paused to unpaused.  If these Ops are
+    triggered by LingerOps, they will be cancelled instead (since
+    should_resend = false), but the LingerOps that triggered them will not
+    be resent.
+
+    Fix this by checking the registered flag for all linger ops, and
+    resending any of them that aren't paused anymore.
+
+    Fixes: #6070
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Sage Weil <sage.weil@inktank.com>
+    (cherry picked from commit 38a0ca66a79af4b541e6322467ae3a8a4483cc72)
+
+commit 1670a73b56e0b407b65334d8f03d3ebb9558ac8b
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 13 12:52:41 2013 -0700
+
+    librados: fix async aio completion wakeup
+
+    For aio flush, we register a wait on the most recent write.  The write
+    completion code, however, was *only* waking the waiter if they were waiting
+    on that write, without regard to previous writes (completed or not).
+    For example, we might have 6 and 7 outstanding and wait on 7.  If they
+    finish in order all is well, but if 7 finishes first we do the flush
+    completion early.  Similarly, if we
+
+     - start 6
+     - start 7
+     - finish 7
+     - flush; wait on 7
+     - finish 6
+
+    we can hang forever.
+
+    Fix by doing any completions that are prior to the oldest pending write in
+    the aio write completion handler.
+
+    Refs: #5919
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Tested-by: Oliver Francke <Oliver.Francke@filoo.de>
+    (cherry picked from commit 16ed0b9af8bc08c7dabead1c1a7c1a22b1fb02fb)
+
+commit 6cf05375abea26d2645e3301c9082c64ddf31a55
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Aug 12 19:17:09 2013 -0700
+
+    librados: fix locking for AioCompletionImpl refcounting
+
+    Add an already-locked helper so that C_Aio{Safe,Complete} can
+    increment the reference count when their caller holds the
+    lock. C_AioCompleteAndSafe's caller is not holding the lock, so call
+    regular get() to ensure no racing updates can occur.
+
+    This eliminates all direct manipulations of AioCompletionImpl->ref,
+    and makes the necessary locking clear.
+
+    The only place C_AioCompleteAndSafe is used is in handling
+    aio_flush_async(). This could cause a missing completion.
+
+    Refs: #5919
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Tested-by: Oliver Francke <Oliver.Francke@filoo.de>
+    (cherry picked from commit 7a52e2ff5025754f3040eff3fc52d4893cafc389)
+
+commit 6a37a62b6f794026b82b88630519ec2cde4f20d6
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Aug 12 10:05:44 2013 -0700
+
+    rgw: fix multi delete
+
+    Fixes: #5931
+    Backport: bobtail, cuttlefish
+
+    Fix a bad check, where we compare the wrong field. Instead of
+    comparing the ret code to 0, we compare the string value to 0
+    which generates implicit casting, hence the crash.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit f9f1c48ad799da2b4be0077bf9d61ae116da33d7)
+
+    Conflicts:
+	src/rgw/rgw_rest_s3.cc
+
+commit 586c68f544c95f9e379df7e4d2705a3090baca49
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jun 2 21:21:09 2013 -0700
+
+    ceph-fuse: create finisher threads after fork()
+
+    The ObjectCacher and MonClient classes both instantiate Finisher
+    threads.  We need to make sure they are created *after* the fork(2)
+    or else the process will fail to join() them on shutdown, and the
+    threads will not exist while fuse is doing useful work.
+
+    Put CephFuse on the heap and move all this initalization into the child
+    block, and make sure errors are passed back to the parent.
+
+    Fix-proposed-by: Alexandre Marangone <alexandre.maragone@inktank.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit c1198d680587928b390bb82c87442384331afd40
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 21 16:22:08 2013 -0700
+
+    debian: update postinst, prerm hooks
+
+    This syncs up the hooks with the latest master versions.  In particular,
+    do not blindly stop/restart daemons on package upgrade!
+
+    Fixes: #5414
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit de8900dcd079207852b6ce0b51473037be9ae956
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 18 21:33:09 2013 -0700
+
+    os/FileStore: use fdatasync(2) instead of sync_file_range(2)
+
+    This fixes data corruption on XFS.  Backported from
+    ffade3c85dfffa13a16edd9630a52d99eb8a413d.
+
+    Fixes: #4976
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+
+commit cbbad5b5d917fe74d6cbc50a259f9dbaeda54ca8
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 17 20:06:59 2013 -0700
+
+    chmod +x iogen.sh
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit fcd65139135d907098a47427d8503ac6d6042f81
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Jun 14 14:53:54 2013 -0700
+
+    rgw: escape prefix correctly when listing objects
+
+    Fixes: #5362
+    When listing objects prefix needs to be escaped correctly (the
+    same as with the marker). Otherwise listing objects with prefix
+    that starts with underscore doesn't work.
+    Backport: bobtail, cuttlefish
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit a8f9d57a15ad7a69d53aa8fc6090fd1b394b616a
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Sun Mar 31 00:02:15 2013 -0700
+
+    rgw: translate object marker to raw format
+
+    Fixes: #4600
+    Object marker should be treated as an object, so that name is formatted
+    correctly when getting the raw oid.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit b083dece36a050ec15ac41a275aeef0ece1ac009)
+
+commit e1d41901cde97a77fc1fda2d7f6e78a7cea61c5c
+Author: tamil <tamil.muthamizhan@inktank.com>
+Date:   Thu Jun 13 13:50:56 2013 -0700
+
+    adding iogen.sh
+
+    Signed-off-by: tamil <tamil.muthamizhan@inktank.com>
+
+commit 32b2f11366418f9a0cc0659be22cf15fbafbe3b1
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 6 16:35:54 2013 -0700
+
+    osd: do not include logbl in scrub map
+
+    This is a potentially use object/file, usually prefixed by a zeroed region
+    on disk, that is not used by scrub at all.  It dates back to
+    f51348dc8bdd5071b7baaf3f0e4d2e0496618f08 (2008) and the original version of
+    scrub.
+
+    This *might* fix #4179.  It is not a leak per se, but I observed 1GB
+    scrub messages going over the write.  Maybe the allocations are causing
+    fragmentation, or the sub_op queues are growing.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 0b036ecddbfd82e651666326d6f16b3c000ade18)
+
+commit 5047a4ff16f556ec25b6624fe8f1a07e3ec5f864
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jun 6 21:53:00 2013 -0700
+
+    rgw: handle deep uri resources
+
+    In case of deep uri resources (ones created beyond a single level
+    of hierarchy, e.g. auth/v1.0) we want to create a new empty
+    handlers for the path if no handlers exists. E.g., for
+    auth/v1.0 we need to have a handler for 'auth', otherwise
+    the default S3 handler will be used, which we don't want.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit ad3934e335399f7844e45fcfd17f7802800d2cb3)
+
+commit 3ea45533c18174cb49af52024dae27533971fa01
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jun 6 21:47:21 2013 -0700
+
+    rgw: fix get_resource_mgr() to correctly identify resource
+
+    Fixes: #5262
+    The original test was not comparing the correct string, ended up
+    with the effect of just checking the substring of the uri to match
+    the resource.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 8d55b87f95d59dbfcfd0799c4601ca37ebb025f5)
+
+commit 4ee638cc3b1d74e678a3f78b3a57baf57a5d407c
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Apr 15 16:33:48 2013 -0700
+
+    PG: don't write out pg map epoch every handle_activate_map
+
+    We don't actually need to write out the pg map epoch on every
+    activate_map as long as:
+    a) the osd does not trim past the oldest pg map persisted
+    b) the pg does update the persisted map epoch from time
+    to time.
+
+    To that end, we now keep a reference to the last map persisted.
+    The OSD already does not trim past the oldest live OSDMapRef.
+    Second, handle_activate_map will trim if the difference between
+    the current map and the last_persisted_map is large enough.
+
+    Fixes: #4731
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit 8c6a912ae46c4d3aeb7c1000d221f67e158ec5c8
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu May 30 12:58:11 2013 -0700
+
+    rgw: only append prefetched data if reading from head
+
+    Fixes: #5209
+    Backport: bobtail, cuttlefish
+    If the head object wrongfully contains data, but according to the
+    manifest we don't read from the head, we shouldn't copy the prefetched
+    data. Also fix the length calculation for that data.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit c5fc52ae0fc851444226abd54a202af227d7cf17)
+
+commit f42e84ee3148dcc59a05c8a12ce39996eb854e26
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu May 30 09:34:21 2013 -0700
+
+    rgw: don't copy object idtag when copying object
+
+    Fixes: #5204
+    When copying object we ended up also copying the original
+    object idtag which overrode the newly generated one. When
+    refcount put is called with the wrong idtag the count
+    does't go down.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit b1312f94edc016e604f1d05ccfe2c788677f51d1)
+
+commit d86b9a852b57553448709c2b978ac3a8637cd63f
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 29 16:50:04 2013 -0700
+
+    osd: initialize new_state field when we use it
+
+    If we use operator[] on a new int field its value is undefined; avoid
+    reading it or using |= et al until we initialize it.
+
+    Fixes: #4967
+    Backport: cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 50ac8917f175d1b107c18ecb025af1a7b103d634)
+
+commit fca54bfbb35f0c200fac52963cd2d1780467f59a
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue May 28 11:10:05 2013 -0700
+
+    HashIndex: sync top directory during start_split,merge,col_split
+
+    Otherwise, the links might be ordered after the in progress
+    operation tag write.  We need the in progress operation tag to
+    correctly recover from an interrupted merge, split, or col_split.
+
+    Fixes: #5180
+    Backport: cuttlefish, bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5bca9c38ef5187c7a97916970a7fa73b342755ac)
+
+commit ac6c77830ad9182c7345f10943ea4f537c1629b0
+Merge: 3984b98 0b50d07
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu May 23 19:29:06 2013 -0700
+
+    Merge remote-tracking branch 'upstream/wip_scrub_tphandle_bobtail' into bobtail
+
+    Fixes: #5159
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit 0b50d073fab11f64682e20fec7bb71d4e931bd3c
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu May 23 17:40:44 2013 -0700
+
+    PG: ping tphandle during omap loop as well
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 5979a26d521918d74fc3b62b75c9fb6afe109933
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu May 23 15:24:39 2013 -0700
+
+    PG: reset timeout in _scan_list for each object, read chunk
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit bde6b58baf5e78550980ee5076bbc2d72a52c0fc
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu May 23 15:23:05 2013 -0700
+
+    OSD,PG: pass tphandle down to _scan_list
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 3984b98804bf18711a7a4fc3940143ab599ad5b5
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed May 22 21:34:52 2013 -0700
+
+    rgw: iterate usage entries from correct entry
+
+    Fixes: #5152
+    When iterating through usage entries, and when user id was
+    provided, we started at the user's first entry and not from
+    the entry indexed by the request start time.
+    This commit fixes the issue.
+
+    Backport: bobtail
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 8b3a04dec8be13559716667d4b16cde9e9543feb)
+
+commit 3d7f8f840fb9fd975089af32c85093eeb1eac338
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 22 12:45:27 2013 -0700
+
+    mon: be a bit more verbose about osd mark down events
+
+    Put these in the cluster log; they are interesting.
+
+    This is a backport of 87767fb1fb9a52d11b11f0b641cebbd9998f089e.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit f21a7f7bf3db61bd533e777297bff1346112a0db
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri May 3 12:57:00 2013 -0700
+
+    rgw: protect ops log socket formatter
+
+    Fixes: #4905
+    Ops log (through the unix domain socket) uses a formatter, which wasn't
+    protected.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit d48f1edb07a4d8727ac956f70e663c1b4e33e1dd)
+
+commit 49c04c623466df45003ed2a18445c354c424a078
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu May 16 15:28:40 2013 -0700
+
+    librbd: make image creation defaults configurable
+
+    Programs using older versions of the image creation functions can't
+    set newer parameters like image format and fancier striping.
+
+    Setting these options lets them use all the new functionality without
+    being patched and recompiled to use e.g. rbd_create3().
+    This is particularly useful for things like qemu-img, which does not
+    know how to create format 2 images yet.
+
+    Refs: #5067
+    backport: cuttlefish, bobtail
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit aacc9adc4e9ca90bbe73ac153cc754a3a5b2c0a1)
+
+commit 4d7058fe2254f335969f05bef649b1a27d470aa4
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu May 16 15:21:24 2013 -0700
+
+    rbd.py: fix stripe_unit() and stripe_count()
+
+    These matched older versions of the functions, but would segfault
+    using the current versions.
+
+    backport: cuttlefish, bobtail
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 53ee6f965e8f06c7256848210ad3c4f89d0cb5a0)
+
+commit 82a16c32a37dc46e3019cedc2a5407ae34f806e2
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu May 16 15:19:46 2013 -0700
+
+    cls_rbd: make sure stripe_unit is not larger than object size
+
+    Test a few other cases too.
+
+    backport: cuttlefish, bobtail
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 810306a2a76eec1c232fd28ec9c351e827fa3031)
+
+commit e0de00897468a434e94790a86fc812b77a59614c
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 10 22:14:05 2013 -0700
+
+    mon: fix validatation of mds ids in mon commands
+
+    Fixes: #4996
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5c305d63043762027323052b4bb3ae3063665c6f)
+
+    Conflicts:
+
+	src/mon/MDSMonitor.cc
diff --git a/doc/changelog/v0.61.1.txt b/doc/changelog/v0.61.1.txt
new file mode 100644
index 000000000..4dbbe9725
--- /dev/null
+++ b/doc/changelog/v0.61.1.txt
@@ -0,0 +1,139 @@
+commit 56c4847ba82a92023700e2d4920b59cdaf23428d
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Wed May 8 17:23:47 2013 -0700
+
+    v0.61.1
+
+commit 3b94f03ec58abe3d7a6d0359ff9b4d75826f3777
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu May 2 14:13:07 2013 -0700
+
+    mon: dump MonitorDBStore transactions to file
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 797089ef082b99910eebfd9454c03d1f027c93bb)
+
+commit 9143d6d0d0ebaaee81622587dc21a3d562b05c9c
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon May 6 14:21:28 2013 -0700
+
+    osd: optionally enable leveldb logging
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 0b4c5c1a3349670d11cc3c4fb3c4b3c1a80b2502)
+
+commit 8f456e89ec38f37ae4b444f5cabccd9fc0e415b8
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon May 6 14:13:50 2013 -0700
+
+    mon: allow leveldb logging
+    
+    'mon leveldb log = filename'
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c1d5f815546b731e10bfcb81cbcd48b7d432e9c4)
+
+commit 306ebc6a9164feab084123f91a0fe928125f4b74
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Wed May 8 16:33:05 2013 -0700
+
+    debian/control:  squeeze requres cryptsetup package
+    
+    Squeeze requires the cryptsetup package which has been renamed
+    cryptsetup-bin in later versions.  Allow either package to
+    satisfy the dependency.
+    
+    Signed-off-by: Gary Lowell  <gary.lowell@inktank.com>
+    (cherry picked from commit 83bbae415de16f708ca1cb24861ddbb0bd514a7f)
+
+commit e0c0a5c1a711263d234b70b454aca07c23e98243
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed May 8 12:18:49 2013 -0700
+
+    osd: don't assert if get_omap_iterator() returns NULL
+    
+    Fixes: #4949
+    This can happen if the object does not exist and it's
+    a write operation. Just return -ENOENT.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 36ec6f9bce63641f4fc2e4ab04d03d3ec1638ea0)
+
+commit 82b9299512537b8d921caa044107472adacbdabe
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 8 14:54:33 2013 -0700
+
+    ceph-create-keys: gracefully handle no data from admin socket
+    
+    Old ceph-mon (prior to 393c9372f82ef37fc6497dd46fc453507a463d42) would
+    return an empty string and success if the command was not registered yet.
+    Gracefully handle that case by retrying.
+    
+    If we still fail to parse, exit entirely with EINVAL.
+    
+    Fixes: #4952
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@intank.com>
+    (cherry picked from commit e2528ae42c455c522154c9f68b5032a3362fca8e)
+
+commit ee3da880fbc0488467c30b1d0b635e3c50e23cc3
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 8 14:35:54 2013 -0700
+
+    init-ceph: fix osd_data location when checking df utilization
+    
+    Do not assume default osd data location.
+    
+    Fixes: #4951
+    Backport: cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Gary Lowelll <gary.lowell@inktank.com>
+    (cherry picked from commit f2a54cc9c98a9f31aef049c74ea932b2d9000d3c)
+
+commit 4848fac24224879bcdc4fcf202d6ab689d8d990f
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue May 7 16:41:22 2013 -0700
+
+    OSD: handle stray snap collections from upgrade bug
+    
+    Previously, we failed to clear snap_collections, which causes split to
+    spawn a bunch of snap collections.  In load_pgs, we now clear any such
+    snap collections and then snap_collections field on the PG itself.
+    
+    Related: #4927
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 8e89db89cb36a217fd97cbc1f24fd643b62400dc)
+
+commit dc6b9e6b2e8c1b41e982430795ff41a65a5813dc
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue May 7 16:35:57 2013 -0700
+
+    PG: clear snap_collections on upgrade
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 252d71a81ef4536830a74897c84a7015ae6ec9fe)
+
+commit b514941ba5f49b6465082e4906b7f4022631bfb3
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue May 7 16:34:57 2013 -0700
+
+    OSD: snap collections can be ignored on split
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 438d9aa152e546b2008ec355b481df71aa1c51a5)
+
+commit 30ffca77df006a244044604074779af538721f14
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 8 11:05:29 2013 -0700
+
+    ceph: return error code when failing to get result from admin socket
+    
+    Make sure we return a non-zero result code when we fail to read something
+    from the admin socket.
+    
+    Backport: cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 393c9372f82ef37fc6497dd46fc453507a463d42)
diff --git a/doc/changelog/v0.61.2.txt b/doc/changelog/v0.61.2.txt
new file mode 100644
index 000000000..b57c7e634
--- /dev/null
+++ b/doc/changelog/v0.61.2.txt
@@ -0,0 +1,27 @@
+commit fea782543a844bb277ae94d3391788b76c5bee60
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Mon May 13 11:58:35 2013 -0700
+
+    v0.61.2
+
+commit 8464c06412b72673ca9d01d35e8f68451e1982b3
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Mon May 13 15:36:59 2013 +0100
+
+    mon: Monitor: tolerate GV duplicates during conversion
+    
+    Fixes: #4974
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit ba05b16ee2b6e25141f2ab88265a1cf92dcd493c)
+
+commit 703bc2fd8aec512fb2b58d5ef263e37647c1f047
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Fri May 10 20:09:34 2013 -0700
+
+    config_opts: default mon_debug_dump_transactions to 'false'
+    
+    otherwise, it chews mon log space at an alarming rate.
+    
+    Fixes: #5024
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
diff --git a/doc/changelog/v0.61.3.txt b/doc/changelog/v0.61.3.txt
new file mode 100644
index 000000000..c190d40ed
--- /dev/null
+++ b/doc/changelog/v0.61.3.txt
@@ -0,0 +1,831 @@
+commit 92b1e398576d55df8e5888dd1a9545ed3fd99532
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Wed Jun 5 11:10:05 2013 -0700
+
+    v0.61.3
+
+commit 7d549cb82ab8ebcf1cc104fc557d601b486c7635
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 4 10:42:13 2013 -0700
+
+    os/LevelDBStore: only remove logger if non-null
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ce67c58db7d3e259ef5a8222ef2ebb1febbf7362)
+    Fixes: #5255
+
+commit 8544ea751884617616addc17b4467b9a86bd9d8a
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Jun 3 15:57:23 2013 -0700
+
+    test_librbd: use correct type for varargs snap test
+
+    uint64_t is passed in, but int was extracted. This fails on 32-bit builds.
+
+    Fixes: #5220
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 17029b270dee386e12e5f42c2494a5feffd49b08)
+
+commit b226e117b5a72c3b04b74aec50a9198601f3730b
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jun 2 18:07:34 2013 -0700
+
+    os/LevelDBStore: fix merge loop
+
+    We were double-incrementing p, both in the for statement and in the
+    body.  While we are here, drop the unnecessary else's.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit eb6d5fcf994d2a25304827d7384eee58f40939af)
+
+commit d017afb5b075a1958cb19213dd95e41d54065e45
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jun 2 17:27:10 2013 -0700
+
+    msgr: add get_messenger() to Connection
+
+    This was part of commit 27381c0c6259ac89f5f9c592b4bfb585937a1cfc.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit ffb87918fa7b829a5199eec08804dc540a819bf2
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 31 17:09:19 2013 -0700
+
+    mon: start lease timer from peon_init()
+
+    In the scenario:
+
+     - leader wins, peons lose
+     - leader sees it is too far behind on paxos and bootstraps
+     - leader tries to sync with someone, waits for a quorum of the others
+     - peons sit around forever waiting
+
+    The problem is that they never time out because paxos never issues a lease,
+    which is the normal timeout that lets them detect a leader failure.
+
+    Avoid this by starting the lease timeout as soon as we lose the election.
+    The timeout callback just does a bootstrap and does not rely on any other
+    state.
+
+    I see one possible danger here: there may be some "normal" cases where the
+    leader takes a long time to issue its first lease that we currently
+    tolerate, but won't with this new check in place.  I hope that raising
+    the lease interval/timeout or reducing the allowed paxos drift will make
+    that a non-issue.  If it is problematic, we will need a separate explicit
+    "i am alive" from the leader while it is getting ready to issue the lease
+    to prevent a live-lock.
+
+    Backport: cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit f1ccb2d808453ad7ef619c2faa41a8f6e0077bd9)
+
+commit 38f8d850d35500e3d8751cd14c5cdaaff682c7d7
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 30 22:52:21 2013 -0700
+
+    mon: discard messages from disconnected clients
+
+    If the client is not connected, discard the message.  They will
+    reconnect and resend anyway, so there is no point in processing it
+    twice (now and later).
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit fb3cd0c2a8f27a1c8d601a478fd896cc0b609011)
+
+commit 36d948981685114d2fe807f480c19aade7497194
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 22 08:13:21 2013 -0700
+
+    msgr: add Messenger reference to Connection
+
+    This allows us to get the messenger associated with a connection.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 92a558bf0e5fee6d5250e1085427bff22fe4bbe4)
+
+commit 61135964419ecf5165366724d064b623b517fb4e
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 30 15:59:49 2013 -0700
+
+    mon/Paxos: adjust trimming defaults up; rename options
+
+    - trim more at a time (by an order of magnitude)
+    - rename fields to paxos_trim_{min,max}; only trim when there are min items
+      that are trimmable, and trim at most max items at a time.
+    - adjust the paxos_service_trim_{min,max} values up by a factor of 2.
+
+    Since we are compacting every time we trim, adjusting these up mean less
+    frequent compactions and less overall work for the monitor.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 6b8e74f0646a7e0d31db24eb29f3663fafed4ecc)
+
+commit 2dc402815f71204cfe592cfb3d6758486d84166d
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 8 16:42:24 2013 -0700
+
+    common/Preforker: fix warnings
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a284c9ece85f11d020d492120be66a9f4c997416)
+
+commit 48ee9283de288c101d3387cc48df6eb8ea889fb7
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 30 15:53:35 2013 -0700
+
+    fix test users of LevelDBStore
+
+    Need to pass in cct.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 446e0770c77de5d72858dcf7a95c5b19f642cf98)
+
+commit 3372235d307172b404a48e4c4d24702a96116d00
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 30 21:43:50 2013 -0700
+
+    mon: destroy MonitorDBStore before g_ceph_context
+
+    Put it on the heap so that we can destroy it before the g_ceph_context
+    cct that it references.  This fixes a crash like
+
+    *** Caught signal (Segmentation fault) **
+    in thread 4034a80
+    ceph version 0.63-204-gcf9aa7a (cf9aa7a0037e56eada8b3c1bb59d59d0bfe7bba5)
+    1: ceph-mon() [0x59932a]
+    2: (()+0xfcb0) [0x4e41cb0]
+    3: (Mutex::Lock(bool)+0x1b) [0x6235bb]
+    4: (PerfCountersCollection::remove(PerfCounters*)+0x27) [0x6a0877]
+    5: (LevelDBStore::~LevelDBStore()+0x1b) [0x582b2b]
+    6: (LevelDBStore::~LevelDBStore()+0x9) [0x582da9]
+    7: (main()+0x1386) [0x48db16]
+    8: (__libc_start_main()+0xed) [0x658076d]
+    9: ceph-mon() [0x4909ad]
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit df2d06db6f3f7e858bdadcc8cd2b0ade432df413)
+
+commit d284eaf9ce7d2022ba62562236d5fa41c26c1eb3
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 30 11:07:06 2013 -0700
+
+    mon: fix leak of health_monitor and config_key_service
+
+    Switch to using regular pointers here.  The lifecycle of these services is
+    very simple such that refcounting is overkill.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c888d1d3f1b77e62d1a8796992e918d12a009b9d)
+
+commit cc88cdfe8de02da675f9051e95b70da11f7fbe9e
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 29 17:54:17 2013 -0700
+
+    mon: return instead of exit(3) via preforker
+
+    This lets us run all the locally-scoped dtors so that leak checking will
+    work.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3c5706163b72245768958155d767abf561e6d96d)
+
+commit 85d629a5f8e1deef9a562d9a7b371707d90c5ba1
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 30 14:57:42 2013 -0700
+
+    os/LevelDBStore: add perfcounters
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 7802292e0a49be607d7ba139b44d5ea1f98e07e6)
+
+commit 0109fa8ae70671c2d8ca19bcc95662d5f41cac66
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 30 14:36:41 2013 -0700
+
+    mon: make compaction bounds overlap
+
+    When we trim items N to M, compact over range (N-1) to M so that the
+    items in the queue will share bounds and get merged.  There is no harm in
+    compacting over a larger range here when the lower bound is a key that
+    doesn't exist anyway.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a47ca583980523ee0108774b466718b303bd3f46)
+
+commit 83b1edac07dd74b91ba2cdfe8b63236d7930c9b1
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 30 14:26:42 2013 -0700
+
+    os/LevelDBStore: merge adjacent ranges in compactionqueue
+
+    If we get behind and multiple adjacent ranges end up in the queue, merge
+    them so that we fire off compaction on larger ranges.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f628dd0e4a5ace079568773edfab29d9f764d4f0)
+
+commit d14665e550d9b2dfc47684b73427042b0744127f
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 29 08:40:32 2013 -0700
+
+    mon: compact trimmed range, not entire prefix
+
+    This will reduce the work that leveldb is asked to do by only triggering
+    compaction of the keys that were just trimmed.
+
+    We ma want to further reduce the work by compacting less frequently, but
+    this is at least a step in that direction.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6da4b20ca53fc8161485c8a99a6b333e23ace30e)
+
+commit dcd9b793fb0b05976b55be029315114d6f1df0e5
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 29 08:35:44 2013 -0700
+
+    mon/MonitorDBStore: allow compaction of ranges
+
+    Allow a transaction to describe the compaction of a range of keys.  Do this
+    in a backward compatible say, such that older code will interpret the
+    compaction of a prefix + range as compaction of the entire prefix.  This
+    allows us to avoid introducing any new feature bits.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ab09f1e5c1305a64482ebbb5a6156a0bb12a63a4)
+
+    Conflicts:
+
+	src/mon/MonitorDBStore.h
+
+commit 6543da740a12c6ad085b807c9038d5b7b5aeaba6
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 29 08:34:13 2013 -0700
+
+    os/LevelDBStore: allow compaction of key ranges
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e20c9a3f79ccfeb816ed634ca25de29fc5975ea8)
+
+commit 87dcba2dd12a673fdc63ad64fb23e6e9f841d74f
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue May 28 16:35:55 2013 -0700
+
+    os/LevelDBStore: do compact_prefix() work asynchronously
+
+    We generally do not want to block while compacting a range of leveldb.
+    Push the blocking+waiting off to a separate thread.  (leveldb will do what
+    it can to avoid blocking internally; no reason for us to wait explicitly.)
+
+    This addresses part of #5176.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4af917d4478ec07734a69447420280880d775fa2)
+
+commit bac5720b2a583e799c6961c733c4a9132a002440
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat May 11 17:36:13 2013 -0700
+
+    qa: rsync test: exclude /usr/local
+
+    Some plana have non-world-readable crap in /usr/local/samba.  Avoid
+    /usr/local entirely for that and any similar landmines.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 82211f2197241c4f3d3135fd5d7f0aa776eaeeb6)
+
+commit 9f1afe1a8e7906bdc54158a4813f011933f6a78f
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 31 21:16:54 2013 -0700
+
+    mon: fix uninitialized fields in MMonHealth
+
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d7e2ab1451e284cd4273cca47eec75e1d323f113)
+
+commit d5b67d49c87b4fe6a90cdd0bf4fac4f9d9377e81
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri May 31 13:44:39 2013 -0700
+
+    PGLog: only add entry to caller_ops in add() if reqid_is_indexed()
+
+    Fixes: #5216
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit a5f6d8967fbbe87f34b23cfcceea84317b647f62
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Apr 15 16:33:48 2013 -0700
+
+    PG: don't write out pg map epoch every handle_activate_map
+
+    We don't actually need to write out the pg map epoch on every
+    activate_map as long as:
+    a) the osd does not trim past the oldest pg map persisted
+    b) the pg does update the persisted map epoch from time
+    to time.
+
+    To that end, we now keep a reference to the last map persisted.
+    The OSD already does not trim past the oldest live OSDMapRef.
+    Second, handle_activate_map will trim if the difference between
+    the current map and the last_persisted_map is large enough.
+
+    Fixes: #4731
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 2c5a9f0e178843e7ed514708bab137def840ab89)
+
+    Conflicts:
+
+	src/common/config_opts.h
+	src/osd/PG.cc
+	- last_persisted_osdmap_ref gets set in the non-static
+	  PG::write_info
+
+commit 9aaff1492a00e137f84af9b03e3a4608cea4b520
+Author: Alexandre Marangone <alexandre.marangone@inktank.com>
+Date:   Fri May 31 12:33:11 2013 -0700
+
+    upstart: handle upper case in cluster name and id
+
+    Signed-off-by: Alexandre Marangone <alexandre.marangone@inktank.com>
+    (cherry picked from commit 851619ab6645967e5d7659d9b0eea63d5c402b15)
+
+commit f87a19d34f9a03493eaca654dd176992676c5812
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue May 21 15:22:56 2013 -0700
+
+    OSDMonitor: skip new pools in update_pools_status() and get_pools_health()
+
+    New pools won't be full.  mon->pgmon()->pg_map.pg_pool_sum[poolid] will
+    implicitly create an entry for poolid causing register_new_pgs() to assume that
+    the newly created pgs in the new pool are in fact a result of a split
+    preventing MOSDPGCreate messages from being sent out.
+
+    Fixes: #4813
+    Backport: cuttlefish
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 0289c445be0269157fa46bbf187c92639a13db46)
+
+commit 34733bdc3b9da13195dfac8a25f98c6de64070d7
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu May 30 12:58:11 2013 -0700
+
+    rgw: only append prefetched data if reading from head
+
+    Fixes: #5209
+    Backport: bobtail, cuttlefish
+    If the head object wrongfully contains data, but according to the
+    manifest we don't read from the head, we shouldn't copy the prefetched
+    data. Also fix the length calculation for that data.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit c5fc52ae0fc851444226abd54a202af227d7cf17)
+
+commit 3d91301e5d788f9f9f52ba4067006f39a89e1531
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu May 30 09:34:21 2013 -0700
+
+    rgw: don't copy object idtag when copying object
+
+    Fixes: #5204
+    When copying object we ended up also copying the original
+    object idtag which overrode the newly generated one. When
+    refcount put is called with the wrong idtag the count
+    does't go down.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit b1312f94edc016e604f1d05ccfe2c788677f51d1)
+
+commit 131dca7d10d5b0b7dca5c5cbe9b0b54938805015
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 30 08:53:22 2013 -0700
+
+    debian: sync up postinst and prerm with latest
+
+    - do not use invoke-rc.d for upstart
+    - do not stop daemons on upgrade
+    - misc other cleanups
+
+    This corresponds to the state of master as of cf9aa7a.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 8b7ca687ded06fe0b67d98e81fa1dabbed440853
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Thu May 30 18:17:28 2013 +0100
+
+    mon: Monitor: backup monmap using all ceph features instead of quorum's
+
+    When a monitor is freshly created and for some reason its initial sync is
+    aborted, it will end up with an incorrect backup monmap.  This monmap is
+    incorrect in the sense that it will not contain the monitor's names as
+    it will expect on the next run.
+
+    This results from us being using the quorum features to encode the monmap
+    when backing it up, instead of CEPH_FEATURES_ALL.
+
+    Fixes: #5203
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 626de387e617db457d6d431c16327c275b0e8a34)
+
+commit 02ef6e918e4fe0368b02fbc700a4d921ae298dc3
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 29 09:49:11 2013 -0700
+
+    osd: do not assume head obc object exists when getting snapdir
+
+    For a list-snaps operation on the snapdir, do not assume that the obc for the
+    head means the object exists.  This fixes a race between a head deletion and
+    a list-snaps that wrongly returns ENOENT, triggered by the DiffItersateStress
+    test when thrashing OSDs.
+
+    Fixes: #5183
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 29e4e7e316fe3f3028e6930bb5987cfe3a5e59ab)
+
+commit 85ad65e294f2b3d4bd1cfef6ae613e31d1cea635
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 29 16:50:04 2013 -0700
+
+    osd: initialize new_state field when we use it
+
+    If we use operator[] on a new int field its value is undefined; avoid
+    reading it or using |= et al until we initialize it.
+
+    Fixes: #4967
+    Backport: cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 50ac8917f175d1b107c18ecb025af1a7b103d634)
+
+commit 65236a4a2541799a0d946df7f3048d4404678f64
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue May 28 11:10:05 2013 -0700
+
+    HashIndex: sync top directory during start_split,merge,col_split
+
+    Otherwise, the links might be ordered after the in progress
+    operation tag write.  We need the in progress operation tag to
+    correctly recover from an interrupted merge, split, or col_split.
+
+    Fixes: #5180
+    Backport: cuttlefish, bobtail
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5bca9c38ef5187c7a97916970a7fa73b342755ac)
+
+commit 0e127dc4ef16d19d56a3603ca90fa6b076d905b0
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed May 22 13:59:08 2013 +0100
+
+    mon: Paxos: get rid of the 'prepare_bootstrap()' mechanism
+
+    We don't need it after all.  If we are in the middle of some proposal,
+    then we guarantee that said proposal is likely to be retried.  If we
+    haven't yet proposed, then it's forever more likely that a client will
+    eventually retry the message that triggered this proposal.
+
+    Basically, this mechanism attempted at fixing a non-problem, and was in
+    fact triggering some unforeseen issues that would have required increasing
+    the code complexity for no good reason.
+
+    Fixes: #5102
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit e15d29094503f279d444eda246fc45c09f5535c9)
+
+commit f29206bdd950d1e916a1422b4826caaf4a8cef3a
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed May 22 13:51:13 2013 +0100
+
+    mon: Paxos: finish queued proposals instead of clearing the list
+
+    By finishing these Contexts, we make sure the Contexts they enclose (to be
+    called once the proposal goes through) will behave as their were initially
+    planned:  for instance, a C_Command() may retry the command if a -EAGAIN
+    is passed to 'finish_contexts', while a C_Trimmed() will simply set
+    'going_to_trim' to false.
+
+    This aims at fixing at least a bug in which Paxos will stop trimming if an
+    election is triggered while a trim is queued but not yet finished.  Such
+    happens because it is the C_Trimmed() context that is responsible for
+    resetting 'going_to_trim' back to false.  By clearing all the contexts on
+    the proposal list instead of finishing them, we stay forever unable to
+    trim Paxos again as 'going_to_trim' will stay True till the end of time as
+    we know it.
+
+    Fixes: #4895
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 586e8c2075f721456fbd40f738dab8ccfa657aa8)
+
+commit b73d7c6b7e83fe8ce29e7150f25a4cca7647cccd
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri May 17 18:23:36 2013 +0100
+
+    mon: Paxos: finish_proposal() when we're finished recovering
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 2ff23fe784245f3b86bc98e0434b21a5318e0a7b)
+
+commit 79eb61c4eadd9d94b3b8087d85b7210f4ab71d54
+Merge: a460e53 8682248
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu May 23 20:09:29 2013 -0700
+
+    Merge branch 'wip_scrub_tphandle' into cuttlefish
+
+    Fixes: #5159
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit 86822485e518d61d7b2c02a6ff25eb2c4b4bc307
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu May 23 17:40:44 2013 -0700
+
+    PG: ping tphandle during omap loop as well
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit d62716dd4ceb29032759bf84e864d214fe38a17c
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu May 23 15:24:39 2013 -0700
+
+    PG: reset timeout in _scan_list for each object, read chunk
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit b8a25e08a638c31b9cfc2c1bf6d9bad40e921a9f
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu May 23 15:23:05 2013 -0700
+
+    OSD,PG: pass tphandle down to _scan_list
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit a460e53ecac03e9c8f54c402a790e6d8cf75b38c
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed May 22 21:34:52 2013 -0700
+
+    rgw: iterate usage entries from correct entry
+
+    Fixes: #5152
+    When iterating through usage entries, and when user id was
+    provided, we started at the user's first entry and not from
+    the entry indexed by the request start time.
+    This commit fixes the issue.
+
+    Backport: bobtail
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 8b3a04dec8be13559716667d4b16cde9e9543feb)
+
+commit 48e1b11bcbdbdf8661cb560c8563f91738034287
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 16 20:37:05 2013 -0700
+
+    sysvinit: fix enumeration of local daemons when specifying type only
+
+    - prepend $local to the $allconf list at the top
+    - remove $local special case for all case
+    - fix the type prefix checks to explicitly check for prefixes
+
+    Fugly bash, but works!
+
+    Backport: cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit c80c6a032c8112eab4f80a01ea18e1fa2c7aa6ed)
+
+commit 6e5f0db75a2bf591bd1fc0a5c2a93051f83ebc5d
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 22 09:47:29 2013 -0700
+
+    sysvinit: fix osd weight calculation on remote hosts
+
+    We need to do df on the remote host, not locally.
+
+    Simlarly, the ceph command uses the osd key, which exists remotely; run it there.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d81d0ea5c442699570bd93a90bea0d97a288a1e9)
+
+commit 674e75bc0783cd9befe9c20e36fbc2cfdac62e5a
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 22 09:47:03 2013 -0700
+
+    sysvinit: use known hostname $host instead of (incorrectly) recalculating
+
+    We would need to do hostname -s on the remote node, not the local one.
+    But we already have $host; use it!
+
+    Reported-by: Xiaoxi Chen <xiaoxi.chen@intel.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit caa15a34cb5d918c0c8b052cd012ec8a12fca150)
+
+commit f4cd61d339419738355a26d7f4fce04eac9dedee
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon May 20 12:41:30 2013 -0700
+
+    mon: be a bit more verbose about osd mark down events
+
+    Put these in the cluster log; they are interesting.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 87767fb1fb9a52d11b11f0b641cebbd9998f089e)
+
+commit e04b1894ed7f3a6e95007e58dae5b35357e5c507
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon May 13 14:23:00 2013 -0700
+
+    PG: subset_last_update must be at least log.tail
+
+    Fixes: 5020
+    Backport: bobtail, cuttlefish
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 72bf5f4813c273210b5ced7f7793bc1bf813690c)
+
+commit 3a02e00d1d3b904b312db283faedf4dff37b0f6f
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue May 14 16:35:48 2013 -0700
+
+    FileJournal: adjust write_pos prior to unlocking write_lock
+
+    In committed_thru, we use write_pos to reset the header.start value in cases
+    where seq is past the end of our journalq.  It is therefore important that the
+    journalq be updated atomically with write_pos (that is, under the write_lock).
+
+    The call to align_bl() is moved into do_write in order to ensure that write_pos
+    is adjusted correctly prior to write_bl().
+
+    Also, we adjust pos at the end of write_bl() such that pos \in [get_top(),
+    header.max_size) after write_bl().
+
+    Fixes: #5020
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit eaf3abf3f9a7b13b81736aa558c9084a8f07fdbe)
+
+commit 8a030eb0e1b61121f7b1e2b7a551bb213d1c428b
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue May 21 14:36:11 2013 -0700
+
+    mon: implement --extract-monmap <filename>
+
+    This will make for a simpler process for
+      http://docs.ceph.com/docs/master/rados/operations/add-or-rm-mons/#removing-monitors-from-an-unhealthy-cluster
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c0268e27497a4d8228ef54da9d4ca12f3ac1f1bf)
+
+commit 4ad13c945fd29a2d183f7ea6c6ac8a51d24dabe1
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu May 16 15:28:40 2013 -0700
+
+    librbd: make image creation defaults configurable
+
+    Programs using older versions of the image creation functions can't
+    set newer parameters like image format and fancier striping.
+
+    Setting these options lets them use all the new functionality without
+    being patched and recompiled to use e.g. rbd_create3().
+    This is particularly useful for things like qemu-img, which does not
+    know how to create format 2 images yet.
+
+    Refs: #5067
+    backport: cuttlefish, bobtail
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit aacc9adc4e9ca90bbe73ac153cc754a3a5b2c0a1)
+
+commit 684444f88f2a7cf28f2e685c18f0771730a1d48f
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu May 16 15:21:24 2013 -0700
+
+    rbd.py: fix stripe_unit() and stripe_count()
+
+    These matched older versions of the functions, but would segfault
+    using the current versions.
+
+    backport: cuttlefish, bobtail
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 53ee6f965e8f06c7256848210ad3c4f89d0cb5a0)
+
+commit 9c7faf957fffb2721ccb915b68ca90ffb0d04a9f
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu May 16 15:19:46 2013 -0700
+
+    cls_rbd: make sure stripe_unit is not larger than object size
+
+    Test a few other cases too.
+
+    backport: cuttlefish, bobtail
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 810306a2a76eec1c232fd28ec9c351e827fa3031)
+
+commit 4071d7a79585ee2768d0a63819d99405a083369f
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri May 3 12:57:00 2013 -0700
+
+    rgw: protect ops log socket formatter
+
+    Fixes: #4905
+    Ops log (through the unix domain socket) uses a formatter, which wasn't
+    protected.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit d48f1edb07a4d8727ac956f70e663c1b4e33e1dd)
+
+commit 57a6a54db4020debd826a493cccca4695934f693
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 15 23:02:10 2013 -0700
+
+    Makefle: force char to be signed
+
+    On an armv7l build, we see errors like
+
+     warning: rgw/rgw_common.cc:626:16: comparison is always false due to limited range of data type [-Wtype-limits]
+
+    from code
+
+          char c1 = hex_to_num(*src++);
+    ...
+          if (c1 < 0)
+
+    Force char to be signed (regardless of any weird architecture's default)
+    to avoid risk of this leading to misbehavior.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 769a16d6674122f3b537f03e17514ad974bf2a2f)
+
+commit 85fb422a084785176af3b694882964841e02195d
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon May 20 13:34:27 2013 -0700
+
+    debian: stop sysvinit on ceph.prerm
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2f193fb931ed09d921e6fa5a985ab87aa4874589)
+
+commit eac3cd2d8334db7c70cbba433610136ff76e447f
+Author: Mike Kelly <pioto@pioto.org>
+Date:   Thu May 16 12:29:50 2013 -0400
+
+    ceph df: fix si units for 'global' stats
+
+    si_t expects bytes, but it was being given kilobytes.
+
+    Signed-off-by: Mike Kelly <pioto@pioto.org>
+    (cherry picked from commit 0c2b738d8d07994fee4c73dd076ac9364a64bdb2)
+
+commit 7bc7c9d4bc6f5bbadbb5e6f9844a0e1f66373c90
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 16 18:40:29 2013 -0700
+
+    udev: install disk/by-partuuid rules
+
+    Wheezy's udev (175-7.2) has broken rules for the /dev/disk/by-partuuid/
+    symlinks that ceph-disk relies on.  Install parallel rules that work.  On
+    new udev, this is harmless; old older udev, this will make life better.
+
+    Fixes: #4865
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d8d7113c35b59902902d487738888567e3a6b933)
+
+commit c2acecbdab93177227a80a114ecc4ce71dbdbe49
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 16 13:17:45 2013 -0700
+
+    debian: make radosgw require matching version of librados2
+
+    ...indirectly via ceph-common.  We get bad behavior when they diverge, I
+    think because of libcommon.la being linked both statically and dynamically.
+
+    Fixes: #4997
+    Backport: cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Gary Lowell <gary.lowell@inktank.com>
+    (cherry picked from commit 604c83ff18f9a40c4f44bc8483ef22ff41efc8ad)
+
+commit 4c0d3eb72c045c39e8b38fd5abf9ddfbb2219f9c
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 10 22:14:05 2013 -0700
+
+    mon: fix validatation of mds ids in mon commands
+
+    Fixes: #4996
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5c305d63043762027323052b4bb3ae3063665c6f)
diff --git a/doc/changelog/v0.61.4.txt b/doc/changelog/v0.61.4.txt
new file mode 100644
index 000000000..0964274f0
--- /dev/null
+++ b/doc/changelog/v0.61.4.txt
@@ -0,0 +1,823 @@
+commit 1669132fcfc27d0c0b5e5bb93ade59d147e23404
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Wed Jun 19 13:51:38 2013 -0700
+
+    v0.61.4
+
+commit b76540f6e2db7a08dee86f84358d56c4ea0b3293
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 22 14:29:37 2013 -0700
+
+    messages/MOSDMarkMeDown: fix uninit field
+    
+    Fixes valgrind warning:
+    ==14803== Use of uninitialised value of size 8
+    ==14803==    at 0x12E7614: sctp_crc32c_sb8_64_bit (sctp_crc32.c:567)
+    ==14803==    by 0x12E76F8: update_crc32 (sctp_crc32.c:609)
+    ==14803==    by 0x12E7720: ceph_crc32c_le (sctp_crc32.c:733)
+    ==14803==    by 0x105085F: ceph::buffer::list::crc32c(unsigned int) (buffer.h:427)
+    ==14803==    by 0x115D7B2: Message::calc_front_crc() (Message.h:441)
+    ==14803==    by 0x1159BB0: Message::encode(unsigned long, bool) (Message.cc:170)
+    ==14803==    by 0x1323934: Pipe::writer() (Pipe.cc:1524)
+    ==14803==    by 0x13293D9: Pipe::Writer::entry() (Pipe.h:59)
+    ==14803==    by 0x120A398: Thread::_entry_func(void*) (Thread.cc:41)
+    ==14803==    by 0x503BE99: start_thread (pthread_create.c:308)
+    ==14803==    by 0x6C6E4BC: clone (clone.S:112)
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit eb91f41042fa31df2bef9140affa6eac726f6187)
+
+commit 2434ecca6f8e1f2af62c74cb1b7df277f4a076f7
+Merge: f8973bd 3f32ded
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jun 19 10:56:51 2013 -0700
+
+    Merge remote-tracking branch 'gh/wip-4976-cuttlefish' into cuttlefish
+    
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+
+commit f8973bda801dbb2412c4cd83869c0203930f38e9
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 17 20:32:15 2013 -0700
+
+    common/Preforker: fix warning
+    
+    common/Preforker.h: In member function ‘int Preforker::signal_exit(int)’:
+    warning: common/Preforker.h:82:45: ignoring return value of ‘ssize_t safe_write(int, const void*, size_t)’, declared with attribute warn_unused_result [-Wunused-result]
+    
+    This is harder than it should be to fix.  :(
+      http://stackoverflow.com/questions/3614691/casting-to-void-doesnt-remove-warn-unused-result-error
+    
+    Whatever, I guess we can do something useful with this return value.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit ce7b5ea7d5c30be32e4448ab0e7e6bb6147af548)
+
+commit 6427985b746b7c29f5337d87b60aab9008a8bc6e
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Jun 19 02:50:45 2013 +0100
+
+    mon: Monitor: make sure we backup a monmap during sync start
+    
+    First of all, we must find a monmap to backup.  The newest version.
+    
+    Secondly, we must make sure we back it up before clearing the store.
+    
+    Finally, we must make sure that we don't remove said backup while
+    clearing the store; otherwise, we would be out of a backup monmap if the
+    sync happened to fail (and if the monitor happened to be killed before a
+    new sync had finished).
+    
+    This patch makes sure these conditions are met.
+    
+    Fixes: #5256 (partially)
+    Backport: cuttlefish
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5e6dc4ea21b452e34599678792cd36ce1ba3edb3)
+
+commit f8f83e78218ee5b8602cfe6758caf2ecd3e06602
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Jun 19 02:36:44 2013 +0100
+
+    mon: Monitor: obtain latest monmap on sync store init
+    
+    Always use the highest version amongst all the typically available
+    monmaps: whatever we have in memory, whatever we have under the
+    MonmapMonitor's store, and whatever we have backed up from a previous
+    sync.  This ensures we always use the newest version we came across
+    with.
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6284fdce794b73adcc757fee910e975b6b4bd054)
+
+commit 41e1a6beeef69136fe6ec3f31d07ea8c33d64cba
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Jun 19 02:21:58 2013 +0100
+
+    mon: Monitor: don't remove 'mon_sync' when clearing the store during abort
+    
+    Otherwise, we will end up losing the monmap we backed up when we started
+    the sync, and the monitor may be unable to start if it is killed or
+    crashes in-between the sync abort and finishing a new sync.
+    
+    Fixes: #5256 (partially)
+    Backport: cuttlefish
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit af5a9861d7c6b4527b0d2312d0efa792910bafd9)
+
+commit 3f32ded41818ad3c607339d723c095f166e36f7e
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 18 21:31:23 2013 -0700
+
+    os/FileStore: drop posix_fadvise(...DONTNEED)
+    
+    On XFS this call is problematic because it directly calls the filemap
+    writeback without vectoring through xfs.  This can break the delicate
+    ordering of writeback and range zeroing; see #4976 and this thread
+    
+          http://oss.sgi.com/archives/xfs/2013-06/msg00066.html
+    
+    Drop this behavior for now to avoid subtle data corruption.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit ffade3c85dfffa13a16edd9630a52d99eb8a413d
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 18 21:24:16 2013 -0700
+
+    os/FileStore: use fdatasync(2) instead of sync_file_range(2)
+    
+    The use of sync_file_range(2) on XFS screws up XFS' delicate ordering
+    of writeback and range zeroing; see #4976 and this thread:
+    
+      http://oss.sgi.com/archives/xfs/2013-06/msg00066.html
+    
+    Instead, replace all sync_file_range(2) calls with fdatasync(2), which
+    *does* do ordered writeback and should not leak unzeroed blocks.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 3383ba6326aeff6500df4792fcef65740dd6b3ea
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jun 12 21:47:09 2013 -0700
+
+    config: fix run_dir typo
+    
+    From 654299108bfb11e7dce45f54946d1505f71d2de8.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e9689ac6f5f50b077a6ac874f811d204ef996c96)
+
+commit 41e1416c3585507db75f10ed69f65a5a80370827
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 18 14:51:08 2013 -0700
+
+    ceph.spec: create /var/run on package install
+    
+    The %ghost %dir ... line will make this get cleaned up but won't install
+    it.
+    
+    Reported-by: Derek Yarnell <derek@umiacs.umd.edu>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Gary Lowell <gary.lowell@inktank.com>
+    (cherry picked from commit 64ee0148a5b7324c7df7de2d5f869b880529d452)
+
+commit 2df3619169d2e83617fc3005b68de395e982b39a
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 7 17:03:41 2013 -0700
+
+    global: create /var/run/ceph on daemon startup
+    
+    This handles cases where the daemon is started without the benefit of
+    sysvinit or upstart (as with teuthology or ceph-fuse).
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 654299108bfb11e7dce45f54946d1505f71d2de8)
+
+commit eace9987c19f4da5cc89ca84abec5e54035b4887
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jun 5 11:10:34 2013 -0700
+
+    PG: don't dirty log unconditionally in activate()
+    
+    merge_log and friends all take care of dirtying the log
+    as necessary.
+    
+    Fixes: #5238
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 5deece1d034749bf72b7bd04e4e9c5d97e5ad6ce)
+
+commit fda618e96690b2693c7a8dd523ece7c1c5a90b67
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri Jun 14 17:11:43 2013 +0100
+
+    mon: OSDMonitor: don't ignore apply_incremental()'s return on UfP [1]
+    
+    apply_incremental() may return -EINVAL.  Don't ignore it.
+    
+    [1] UfP = Update from Paxos
+    
+    Fixes: #5343
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit e3c33f4315cbf8718f61eb79e15dd6d44fc908b7)
+
+commit c14dd154584eef97075e4a260719b6cbe686d4c7
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 17 16:38:26 2013 -0700
+
+    client: handle reset during initial mds session open
+    
+    If we get a reset during our attempt to open an MDS session, close out the
+    Connection* and retry to open the session, moving the waiters over.
+    
+    Fixes: #5379
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit df8a3e5591948dfd94de2e06640cfe54d2de4322)
+
+commit aa8834831dbfcdebb96841686e3707e552c349f6
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 17 15:43:40 2013 -0700
+
+    ceph-disk: add some notes on wth we are up to
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 8c6b24e9039079e897108f28d6af58cbc703a15a)
+
+commit 261964adc5a7efe7c0efac2d527bc10a95498e45
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 14 16:29:10 2013 -0700
+
+    ceph-disk: clear TERM to avoid libreadline hijinx
+    
+    The weird output from libreadline users is related to the TERM variable.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e538829f16ce19d57d63229921afa01cc687eb86)
+
+commit e36fb3267cdbe0ec5bb01f8a012485de1bbf1652
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 17 09:49:46 2013 -0700
+
+    ceph-disk-udev: set up by-partuuid, -typeuuid symlinks on ancient udev
+    
+    Make the ancient-udev/blkid workaround script for RHEL/CentOS create the
+    symlinks for us too.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d7f7d613512fe39ec883e11d201793c75ee05db1)
+
+commit c3936ebb159b419115bfdc5dc7a23f5807cff9fa
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jun 15 20:06:33 2013 -0700
+
+    ceph-disk: do not stop activate-all on first failure
+    
+    Keep going even if we hit one activation error.  This avoids failing to
+    start some disks when only one of them won't start (e.g., because it
+    doesn't belong to the current cluster).
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c9074375bfbe1e3757b9c423a5ff60e8013afbce)
+
+commit 3cb783c31541cebe928b97cf27eb326bb4b54f9b
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 14 16:30:24 2013 -0700
+
+    ceph.spec: include partuuid rules in package
+    
+    Commit f3234c147e083f2904178994bc85de3d082e2836 missed this.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 253069e04707c5bf46869f4ff5a47ea6bb0fde3e)
+
+commit 2d771b57082545e139086f3c5c9f79ba47101a14
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 14 15:01:14 2013 -0700
+
+    ceph.spec: install/uninstall init script
+    
+    This was commented out almost years ago in commit 9baf5ef4 but it is not
+    clear to me that it was correct to do so.  In any case, we are not
+    installing the rc.d links for ceph, which means it does not start up after
+    a reboot.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cc9b83a80262d014cc37f0c974963cf7402a577a)
+
+commit dfe421e44cfa13178950258509a0cf71e308daa1
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 14 13:39:03 2013 -0700
+
+    sysvinit, upstart: ceph-disk activate-all on start
+    
+    On 'service ceph start' or 'service ceph start osd' or start ceph-osd-all
+    we should activate any osd GPT partitions.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 13680976ef6899cb33109f6f841e99d4d37bb168)
+
+commit ef47cf9c9dd7f4a1f549c14f984cd052fc95bb64
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 14 13:34:40 2013 -0700
+
+    ceph-disk: add 'activate-all'
+    
+    Scan /dev/disk/by-parttypeuuid for ceph OSDs and activate them all.  This
+    is useful when the event didn't trigger on the initial udev event for
+    some reason.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5c7a23687a1a21bec5cca7b302ac4ba47c78e041)
+
+commit 65ec1d87f96694ab08c99a7104329065c7c5ebae
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 14 13:23:52 2013 -0700
+
+    udev: /dev/disk/by-parttypeuuid/$type-$uuid
+    
+    We need this to help trigger OSD activations.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d512dc9eddef3299167d4bf44e2018b3b6031a22)
+
+commit 4a739b13f420acf4f0af21b4cf133ac08c97ee18
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Jun 14 14:53:54 2013 -0700
+
+    rgw: escape prefix correctly when listing objects
+    
+    Fixes: #5362
+    When listing objects prefix needs to be escaped correctly (the
+    same as with the marker). Otherwise listing objects with prefix
+    that starts with underscore doesn't work.
+    Backport: bobtail, cuttlefish
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit d582ee2438a3bd307324c5f44491f26fd6a56704)
+
+commit 6edba9e9ad1c115a462aeab858063fe3939a40e4
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 10 17:28:22 2013 -0700
+
+    messages/MMonSync: initialize crc in ctor
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cd1c289b96a874ff99a83a44955d05efc9f2765a)
+
+commit 2ad85576850bcb61c3ac8fee38609f9cd51bfac2
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jun 15 08:48:37 2013 -0700
+
+    client: fix ancient typo in caps revocation path
+    
+    If we have dropped all references to a revoked capability, send the ack
+    to the MDS.  This typo has been there since v0.7 (early 2009)!
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit b7143c2f84daafbe2c27d5b2a2d5dc40c3a68d15)
+
+commit d3d88983bceaf508f2585cc6bf8fc304a0943ffe
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jun 5 08:42:25 2013 -0700
+
+    messages/MMonHealth: remove unused flag field
+    
+    This was initialized in (one of) the ctor(s), but not encoded/decoded,
+    and not used.  Remove it.  This makes valgrind a happy.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 08bb8d510b5abd64f5b9f8db150bfc8bccaf9ce8)
+
+commit 646d9ccc575bf91cf73fbfaf91ab4d04ed2b5981
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jun 5 08:34:20 2013 -0700
+
+    messages/MMonProbe: fix uninitialized variables
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4974b29e251d433101b69955091e22393172bcd8)
+
+commit 08304a7c46da7517319b7db0b64d1c4f54771472
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jun 15 08:14:40 2013 -0700
+
+    common/Preforker: fix broken recursion on exit(3)
+    
+    If we exit via preforker, call exit(3) and not recursively back into
+    Preforker::exit(r).  Otherwise you get a hang with the child blocked
+    at:
+    
+    Thread 1 (Thread 0x7fa08962e7c0 (LWP 5419)):
+    #0  0x000000309860e0cd in write () from /lib64/libpthread.so.0
+    #1  0x00000000005cc906 in Preforker::exit(int) ()
+    #2  0x00000000005c8dfb in main ()
+    
+    and the parent at
+    
+    #0  0x000000309860eba7 in waitpid () from /lib64/libpthread.so.0
+    #1  0x00000000005cc87a in Preforker::parent_wait() ()
+    #2  0x00000000005c75ae in main ()
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 7e7ff7532d343c473178799e37f4b83cf29c4eee)
+
+commit f202d332dc86844caa9076c7866cc009d87f76b6
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Thu Jun 13 16:38:26 2013 -0700
+
+    rules:  Don't disable tcmalloc on ARM (and other non-intel)
+    
+    Fixes #5342
+    
+    Signed-off-by: Gary Lowell  <gary.lowell@inktank.com>
+
+commit cbca9232618e85bd15b8677c9306042caad876a6
+Author: Guilhem Lettron <guilhem+github@lettron.fr>
+Date:   Mon May 27 12:41:53 2013 +0200
+
+    Remove mon socket in post-stop
+    
+    If ceph-mon segfault, socket file isn't removed.
+    
+    By adding a remove in post-stop, upstart clean run directory properly.
+    
+    Signed-off-by: Guilhem Lettron <guilhem@lettron.fr>
+    (cherry picked from commit 554b41b171eab997038e83928c462027246c24f4)
+
+commit 3f8b3cf1d4042ae6bb64a840ff19f859e6ce6dc2
+Author: James Page <james.page@ubuntu.com>
+Date:   Mon May 20 10:26:06 2013 +0100
+
+    Remove stop on from upstart tasks
+    
+    Upstart tasks don't have to concept of 'stop on' as they
+    are not long running.
+    (cherry picked from commit 17f6fccabc262b9a6d59455c524b550e77cd0fe3)
+
+commit 0b9b8368aec049bfdd4cf72b481f3a4aa8cb031e
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Wed Jun 12 22:22:42 2013 -0700
+
+    ceph-disk: extra dash in error message
+    
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit f86b4e7a4831c684033363ddd335d2f3fb9a189a)
+
+commit ecddd1e0062ae58e8cc75e125cc683fc04b8befe
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Fri May 24 12:41:11 2013 +0200
+
+    ceph-disk: cast output of _check_output()
+    
+    Cast output of _check_output() to str() to be able to use
+    str.split().
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 16ecae153d260407085aaafbad1c1c51f4486c9a)
+
+commit 43855e3788cc90900f1b3424c4e4e2c28e8f351e
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Fri May 24 12:46:15 2013 +0200
+
+    ceph-disk: remove unnecessary semicolons
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 9785478a2aae7bf5234fbfe443603ba22b5a50d2)
+
+commit 53927f94f3dcf7c57b73b66159d924557d92506e
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Fri May 24 12:33:16 2013 +0200
+
+    ceph-disk: fix undefined variable
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 9429ff90a06368fc98d146e065a7b9d1b68e9822)
+
+commit f1beeb267d6e19754a9f2f483b5849a6b900f3ce
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Fri May 24 12:29:07 2013 +0200
+
+    ceph-disk: add missing spaces around operator
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit c127745cc021c8b244d721fa940319158ef9e9d4)
+
+commit fe23ae0042629f493792eadb88a30514c29fc1a0
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 13 22:02:03 2013 -0700
+
+    udev: drop useless --mount argument to ceph-disk
+    
+    It doesn't mean anything anymore; drop it.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit bcfd2f31a50d27038bc02e645795f0ec99dd3b32)
+
+commit d99e0a8908d4feccc0a73952ee31586c83174589
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 13 22:01:34 2013 -0700
+
+    ceph-disk-udev: activate-journal
+    
+    Trigger 'ceph-disk activate-journal' from the alt udev rules.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b139152039bfc0d190f855910d44347c9e79b22a)
+
+commit dfbd20104e01ce22c7e501f748b2bf13382ee7f6
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 13 21:56:23 2013 -0700
+
+    ceph-disk: do not use mount --move (or --bind)
+    
+    The kernel does not let you mount --move when the parent mount is
+    shared (see, e.g., https://bugzilla.redhat.com/show_bug.cgi?id=917008
+    for another person this also confused).  We can't use --bind either
+    since that (on RHEL at least) screws up /etc/mtab so that the final
+    result looks like
+    
+     /var/lib/ceph/tmp/mnt.HNHoXU /var/lib/ceph/osd/ceph-0 none rw,bind 0 0
+    
+    Instead, mount the original dev in the final location and then umount
+    from the old location.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e5ffe0d2484eb6cbcefcaeb5d52020b1130871a5)
+
+commit 98a990d796e71c62fa64c13230a18a4a40aedcdf
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 13 21:22:53 2013 -0700
+
+    ceph.spec: include by-partuuid udev workaround rules
+    
+    These are need for old or buggy udev.  Having them for new and unbroken
+    udev is harmless.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f3234c147e083f2904178994bc85de3d082e2836)
+
+commit 1504e23f177fff00a794c0db76db8a8c72bc8eb7
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 14 12:10:49 2013 -0700
+
+    ceph-disk: work around buggy rhel/centos parted
+    
+    parted on RHEL/Centos prefixes the *machine readable output* with
+    
+     1b 5b 3f 31 30 33 34 68
+    
+    Note that the same thing happens when you 'import readline' in python.
+    
+    Work around it!
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 82ff72f827b9bd7f91d30a09d35e42b25d2a7344)
+
+commit 0b9482bd631aada9e2ca730b6cddd51d7954fb11
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 13 15:54:58 2013 -0700
+
+    ceph-disk: implement 'activate-journal'
+    
+    Activate an osd via its journal device.  udev populates its symlinks and
+    triggers events in an order that is not related to whether the device is
+    an osd data partition or a journal.  That means that triggering
+    'ceph-disk activate' can happen before the journal (or journal symlink)
+    is present and then fail.
+    
+    Similarly, it may be that they are on different disks that are hotplugged
+    with the journal second.
+    
+    This can be wired up to the journal partition type to ensure that osds are
+    started when the journal appears second.
+    
+    Include the udev rules to trigger this.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a2a78e8d16db0a71b13fc15457abc5fe0091c84c)
+
+commit 062b9296c98a03f4f291615dce010aebc55aaa32
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 11 18:35:01 2013 -0700
+
+    ceph-disk: call partprobe outside of the prepare lock; drop udevadm settle
+    
+    After we change the final partition type, sgdisk may or may not trigger a
+    udev event, depending on how well udev is behaving (it varies between
+    distros, it seems).  The old code would often settle and wait for udev to
+    activate the device, and then partprobe would uselessly fail because it
+    was already mounted.
+    
+    Call partprobe only at the very end, after prepare is done.  This ensures
+    that if partprobe calls udevadm settle (which is sometimes does) we do not
+    get stuck.
+    
+    Drop the udevadm settle.  I'm not sure what this accomplishes; take it out,
+    at least until we determine we need it.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 8b3b59e01432090f7ae774e971862316203ade68)
+
+commit 9f9dc8b85a81529deb6865ec5f1cebb45f04ddc4
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 13 11:03:37 2013 -0700
+
+    ceph-disk: add 'zap' command
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 10ba60cd088c15d4b4ea0b86ad681aa57f1051b6)
+
+commit 4fab29c1c5e4dec79f089e9803adf0e855e5ec4f
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue May 21 12:52:03 2013 -0700
+
+    ceph-disk: fix stat errors with new suppress code
+    
+    Broken by 225fefe5e7c997b365f481b6c4f66312ea28ed61.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit bcc8bfdb672654c6a6b48a2aa08267a894debc32)
+
+commit 01ba391b079a494272a071b76f8dee89b5f5c44d
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon May 13 12:35:32 2013 -0700
+
+    ceph-disk: add '[un]suppress-activate <dev>' command
+    
+    It is often useful to prepare but not activate a device, for example when
+    preparing a bunch of spare disks.  This marks a device as 'do not
+    activate' so that it can be prepared without activating.
+    
+    Fixes: #3255
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 225fefe5e7c997b365f481b6c4f66312ea28ed61)
+
+commit 41a15a63349abc5f3e1c0b8de8fcc543e2567035
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 14 11:21:25 2013 -0700
+
+    upstart: start ceph-all on runlevel [2345]
+    
+    Starting when only one network interface has started breaks machines with
+    multiple nics in very problematic ways.
+    
+    There may be an earlier trigger that we can use for cases where other
+    services on the local machine depend on ceph, but for now this is better
+    than the existing behavior.
+    
+    See #5248
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 7e08ed1bf154f5556b3c4e49f937c1575bf992b8)
+
+commit 60e4bb0acb41febb75c866646c7102e9a25782f7
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jun 8 17:38:07 2013 -0700
+
+    client: set issue_seq (not seq) in cap release
+    
+    We regularly have been observing a stall where the MDS is blocked waiting
+    for a cap revocation (Ls, in our case) and never gets a reply.  We finally
+    tracked down the sequence:
+    
+     - mds issues cap seq 1 to client
+     - mds does revocation (seq 2)
+     - client replies
+     - much time goes by
+     - client trims inode from cache, sends release with seq == 2
+     - mds ignores release because its issue_seq is 1
+     - mds later tries to revoke other caps
+     - client discards message because it doesn't have the inode in cache
+    
+    The problem is simply that we are using seq instead of issue_seq in the
+    cap release message.  Note that the other release call site in
+    encode_inode_release() is correct.  That one is much more commonly
+    triggered by short tests, as compared to this case where the inode needs to
+    get pushed out of the client cache.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 9b012e234a924efd718826ab6a53b9aeb7cd6649)
+
+commit ae522291d2a9b994a65fed5e63e9fec74eece5be
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 22 15:03:50 2013 -0700
+
+    osd: skip mark-me-down message if osd is not up
+    
+    Fixes crash when the OSD has not successfully booted and gets a
+    SIGINT or SIGTERM.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c2e262fc9493b4bb22c2b7b4990aa1ee7846940e)
+
+commit 48f8d6e3ffe9fc5bb71b362e3059ffb2da69926b
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jun 2 21:21:51 2013 -0700
+
+    ceph-fuse: create finisher threads after fork()
+    
+    The ObjectCacher and MonClient classes both instantiate Finisher
+    threads.  We need to make sure they are created *after* the fork(2)
+    or else the process will fail to join() them on shutdown, and the
+    threads will not exist while fuse is doing useful work.
+    
+    Put CephFuse on the heap and move all this initalization into the child
+    block, and make sure errors are passed back to the parent.
+    
+    Fix-proposed-by: Alexandre Marangone <alexandre.maragone@inktank.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4fa5f99a40792341d247e51488c37301da3c4e4f)
+
+commit 7012d192e6a31d91e57bc9f5b00af6c541ceb3e0
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 6 16:35:54 2013 -0700
+
+    osd: do not include logbl in scrub map
+    
+    This is a potentially use object/file, usually prefixed by a zeroed region
+    on disk, that is not used by scrub at all.  It dates back to
+    f51348dc8bdd5071b7baaf3f0e4d2e0496618f08 (2008) and the original version of
+    scrub.
+    
+    This *might* fix #4179.  It is not a leak per se, but I observed 1GB
+    scrub messages going over the write.  Maybe the allocations are causing
+    fragmentation, or the sub_op queues are growing.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 0b036ecddbfd82e651666326d6f16b3c000ade18)
+
+commit c75760e39d8df5b1971343e9f9186fd67e154af6
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jun 6 21:53:00 2013 -0700
+
+    rgw: handle deep uri resources
+    
+    In case of deep uri resources (ones created beyond a single level
+    of hierarchy, e.g. auth/v1.0) we want to create a new empty
+    handlers for the path if no handlers exists. E.g., for
+    auth/v1.0 we need to have a handler for 'auth', otherwise
+    the default S3 handler will be used, which we don't want.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit ad3934e335399f7844e45fcfd17f7802800d2cb3)
+
+commit bd12e81e48014024171c55f5984c9183c8e363cb
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jun 6 21:47:21 2013 -0700
+
+    rgw: fix get_resource_mgr() to correctly identify resource
+    
+    Fixes: #5262
+    The original test was not comparing the correct string, ended up
+    with the effect of just checking the substring of the uri to match
+    the resource.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 8d55b87f95d59dbfcfd0799c4601ca37ebb025f5)
+
+commit b1d436e752c9c20e7dbff91b769cb2ba47383571
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jun 6 11:22:38 2013 -0700
+
+    rgw: add 'cors' to the list of sub-resources
+    
+    Fixes: #5261
+    Backport: cuttlefish
+    Add 'cors' to the list of sub-resources, otherwise auth signing
+    is wrong.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 9a0a9c205b8c24ca9c1e05b0cf9875768e867a9e)
+
+commit 4bf75015ae7539a9dbabbbfc0c784dbd58aa6004
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 31 21:23:45 2013 -0700
+
+    mon: fix preforker exit behavior behavior
+    
+    In 3c5706163b72245768958155d767abf561e6d96d we made exit() not actually
+    exit so that the leak checking would behave for a non-forking case.
+    That is only needed for the normal exit case; every other case expects
+    exit() to actually terminate and not continue execution.
+    
+    Instead, make a signal_exit() method that signals the parent (if any)
+    and then lets you return.  exit() goes back to it's usual behavior,
+    fixing the many other calls in main().
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 92d085f7fd6224ffe5b7651c1f83b093f964b5cd)
+
+commit d398a1850d5895ad409f5229f0dc06651c3d4d75
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Jun 4 13:23:36 2013 -0700
+
+    rados.py: correct some C types
+    
+    trunc was getting size_t instead of uint64_t, leading to bad results
+    in 32-bit environments. Explicitly cast to the desired type
+    everywhere, so it's clear the correct type is being used.
+    
+    Fixes: #5233
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 6dd7d469000144b499af84bda9b735710bb5cec3)
diff --git a/doc/changelog/v0.61.5.txt b/doc/changelog/v0.61.5.txt
new file mode 100644
index 000000000..9d99f910c
--- /dev/null
+++ b/doc/changelog/v0.61.5.txt
@@ -0,0 +1,1199 @@
+commit 8ee10dc4bb73bdd918873f29c70eedc3c7ef1979
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Wed Jul 17 16:39:08 2013 -0700
+
+    v0.61.5
+
+commit 39bffac6b6c898882d03de392f7f2218933d942b
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 16 13:14:50 2013 -0700
+
+    ceph-disk: rely on /dev/disk/by-partuuid instead of special-casing journal symlinks
+    
+    This was necessary when ceph-disk-udev didn't create the by-partuuid (and
+    other) symlinks for us, but now it is fragile and error-prone.  (It also
+    appears to be broken on a certain customer RHEL VM.)  See
+    d7f7d613512fe39ec883e11d201793c75ee05db1.
+    
+    Instead, just use the by-partuuid symlinks that we spent all that ugly
+    effort generating.
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 64379e701b3ed862c05f156539506d3382f77aa8)
+
+commit 363d54288254b5e2311cd28fce5988d68cfd5773
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jul 16 16:49:48 2013 +0100
+
+    mon: Monitor: StoreConverter: clearer debug message on 'needs_conversion()'
+    
+    The previous debug message outputted the function's name, as often our
+    functions do.  This was however a source of bewilderment, as users would
+    see those in logs and think their stores would need conversion.  Changing
+    this message is trivial enough and it will make ceph users happier log
+    readers.
+    
+    Backport: cuttlefish
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ad1392f68170b391d11df0ce5523c2d1fb57f60e)
+
+commit 0ea89760def73f76d8100889eca3c25b0a6eb772
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jul 16 16:45:39 2013 +0100
+
+    mon: Monitor: do not reopen MonitorDBStore during conversion
+    
+    We already open the store on ceph_mon.cc, before we start the conversion.
+    Given we are unable to reproduce this every time a conversion is triggered,
+    we are led to believe that this causes a race in leveldb that will lead
+    to 'store.db/LOCK' being locked upon the open this patch removes.
+    
+    Regardless, reopening the db here is pointless as we already did it when
+    we reach Monitor::StoreConverter::convert().
+    
+    Fixes: #5640
+    Backport: cuttlefish
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 036e6739a4e873863bae3d7d00f310c015dfcdb3)
+
+commit 595c09df9134fb0d62144fe1594914c90e567dca
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jul 14 15:54:29 2013 -0700
+
+    messages/MClientReconnect: clear data when encoding
+    
+    The MClientReconnect puts everything in the data payload portion of
+    the message and nothing in the front portion.  That means that if the
+    message is resent (socket failure or something), the messenger thinks it
+    hasn't been encoded yet (front empty) and reencodes, which means
+    everything gets added (again) to the data portion.
+    
+    Decoding keep decoding until it runs out of data, so the second copy
+    means we decode garbage snap realms, leading to the crash in bug
+    
+    Clearing data each time around resolves the problem, although it does
+    mean we do the encoding work multiple times.  We could alternatively
+    (or also) stick some data in the front portion of the payload
+    (ignored), but that changes the wire protocol and I would rather not
+    do that.
+    
+    Fixes: #4565
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 314cf046b0b787ca69665e8751eab6fe7adb4037)
+
+commit 8c178e0d39d8d4a4820eb061f79d74f95e60199f
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 15 10:05:37 2013 -0700
+
+    mon: once sync full is chosen, make sure we don't change our mind
+    
+    It is possible for a sequence like:
+    
+     - probe
+     - first probe reply has paxos trim that indicates a full sync is
+       needed
+     - start sync
+     - clear store
+     - something happens that makes us abort and bootstrap (e.g., the
+       provider mon restarts
+     - probe
+     - first probe reply has older paxos trim bound and we call an election
+     - on election completion, we crash because we have no data.
+    
+    Non-determinism of the probe decision aside, we need to ensure that
+    the info we share during probe (fc, lc) is accurate, and that once we
+    clear the store we know we *must* do a full sync.
+    
+    This is a backport of aa60f940ec1994a61624345586dc70d261688456.
+    
+    Fixes: #5621
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit 6af0ed9bc4cc955f8c30ad9dc6e9095599f323d0
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 9 14:12:15 2013 -0700
+
+    mon: do not scrub if scrub is in progress
+    
+    This prevents an assert from unexpected scrub results from the previous
+    scrub on the leader.
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 00ae543b3e32f89d906a0e934792cc5309f57696)
+
+commit 5642155ba5ca9b384a7af058a6538ac00c2a592d
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 10 10:06:20 2013 -0700
+
+    messages/MPGStats: do not set paxos version to osdmap epoch
+    
+    The PaxosServiceMessage version field is meant for client-coordinated
+    ordering of messages when switching between monitors (and is rarely
+    used).  Do not fill it with the osdmap epoch lest it be compared to a
+    pgmap version, which may cause the mon to (near) indefinitely put it on
+    a wait queue until the pgmap version catches up.
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit b36338be43f43b6dd4ee87c97f2eaa23b467c386)
+
+commit 06c65988bb0b1d1ec626fe31e9d806a1c4e24b28
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 11 18:43:24 2013 -0700
+
+    osd/OSDmap: fix OSDMap::Incremental::dump() for new pool names
+    
+    The name is always present when pools are created, but not when they are
+    modified.  Also, a name may be present with a new_pools entry if the pool
+    is just renamed.  Separate it out completely in the dump.
+    
+    Backport: cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3e4a29111e89588385e63f8d92ce3d67739dd679)
+
+commit 658240710baaf9c661b8fbf856322907a0d394ee
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 8 10:49:28 2013 -0700
+
+    mon/PaxosService: prevent reads until initial service commit is done
+    
+    Do not process reads (or, by PaxosService::dispatch() implication, writes)
+    until we have committed the initial service state.  This avoids things like
+    EPERM due to missing keys when we race with mon creation, triggered by
+    teuthology tests doing their health check after startup.
+    
+    Fixes: #5515
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit d08b6d6df7dba06dad73bdec2c945f24afc02717)
+
+commit 5c3ff33771e227b3fb5cc354323846fe8db4ecc1
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 28 12:21:58 2013 -0700
+
+    client: send all request put's through put_request()
+    
+    Make sure all MetaRequest reference put's go through the same path that
+    releases inode references, including all of the error paths.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 87217e1e3cb2785b79d0dec49bd3f23a827551f5)
+
+commit 1df78ad73df581bc7537688ae28bda820b089a13
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 28 11:50:11 2013 -0700
+
+    client: fix remaining Inode::put() caller, and make method psuedo-private
+    
+    Not sure I can make this actually private and make Client::put_inode() a
+    friend method (making all of Client a friend would defeat the purpose).
+    This works well enough, though!
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 9af3b86b25574e4d2cdfd43e61028cffa19bdeb1)
+
+commit fea024cc3dd2c6fd9ff322d1cd15e0d75c92eca5
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 27 21:39:35 2013 -0700
+
+    client: use put_inode on MetaRequest inode refs
+    
+    When we drop the request inode refs, we need to use put_inode() to ensure
+    they get cleaned up properly (removed from inode_map, caps released, etc.).
+    Do this explicitly here (as we do with all other inode put() paths that
+    matter).
+    
+    Fixes: #5381
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 81bee6487fb1ce9e090b030d61bda128a3cf4982)
+
+commit 62ae39ec8f208cb8f89e43ba844b9a20b4315c61
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 8 15:57:48 2013 -0700
+
+    mon: be smarter about calculating last_epoch_clean lower bound
+    
+    We need to take PGs whose mapping has not changed in a long time into
+    account.  For them, the pg state will indicate it was clean at the time of
+    the report, in which case we can use that as a lower-bound on their actual
+    latest epoch clean.  If they are not currently clean (at report time), use
+    the last_epoch_clean value.
+    
+    Fixes: #5519
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cc0006deee3153e06ddd220bf8a40358ba830135)
+
+commit da725852190245d2f91b7b21e72baee70e4342bd
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 8 13:27:58 2013 -0700
+
+    osd: report pg stats to mon at least every N (=500) epochs
+    
+    The mon needs a moderately accurate last_epoch_clean value in order to trim
+    old osdmaps.  To prevent a PG that hasn't peered or received IO in forever
+    from preventing this, send pg stats at some minimum frequency.  This will
+    increase the pg stat report workload for the mon over an idle pool, but
+    should be no worse that a cluster that is getting actual IO and sees these
+    updates from normal stat updates.
+    
+    This makes the reported update a bit more aggressive/useful in that the epoch
+    is the last map epoch processed by this PG and not just one that is >= the
+    currenting interval.  Note that the semantics of this field are pretty useless
+    at this point.
+    
+    See #5519
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit da81228cc73c95737f26c630e5c3eccf6ae1aaec)
+
+commit 757af91b2af0da6bbfeeb53551fa1ef4ef9118ea
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 10 11:32:34 2013 -0700
+
+    osd: fix warning
+    
+    From 653e04a79430317e275dd77a46c2b17c788b860b
+    
+    Backport: cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit bc291d3fc3fc1cac838565cbe0f25f71d855a6e3)
+
+commit 65af2538329472d2fd078bb961863c40cdabda12
+Merge: e537699 804314b
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 12 15:21:20 2013 -0700
+
+    Merge remote-tracking branch 'gh/wip-mon-sync-2' into cuttlefish
+    
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit e537699b33f84c14f027b56372fbcb0a99bbe88d
+Author: Sandon Van Ness <sandon@inktank.com>
+Date:   Wed Jul 10 14:55:52 2013 -0700
+
+    Get device-by-path by looking for it instead of assuming 3rd entry.
+    
+    On some systems (virtual machines so far) the device-by-path entry
+    from udevadm is not always in the same spot so instead actually
+    look for the right output instead of blindy assuming that its a
+    specific field in the output.
+    
+    Signed-off-by: Sandon Van Ness <sandon@inktank.com>
+    Reviewed-by: Gary Lowell  <gary.lowell@inktank.com>
+
+commit 804314b8bfa5ec75cc9653e2928874c457395c92
+Merge: 6ad9fe1 78f2266
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 10 11:40:37 2013 -0700
+
+    Merge remote-tracking branch 'gh/cuttlefish' into wip-mon-sync-2
+
+commit 78f226634bd80f6678b1f74ccf785bc52fcd6b62
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 10 11:02:08 2013 -0700
+
+    osd: limit number of inc osdmaps send to peers, clients
+    
+    We should not send an unbounded number of inc maps to our peers or clients.
+    In particular, if a peer is not contacted for a while, we may think they
+    have a very old map (say, 10000 epochs ago) and send thousands of inc maps
+    when the distribution shifts and we need to peer.
+    
+    Note that if we do not send enough maps, the peers will make do by
+    requesting the map from somewhere else (currently the mon).  Regardless
+    of the source, however, we must limit the amount that we speculatively
+    share as it usually is not needed.
+    
+    Backport: cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 653e04a79430317e275dd77a46c2b17c788b860b)
+
+commit 54ee2dc80ed032c286546da51442340ec9991cdf
+Author: Christophe Courtaut <christophe.courtaut@gmail.com>
+Date:   Mon Jul 1 14:57:17 2013 +0200
+
+    rgw: Fix return value for swift user not found
+    
+    http://tracker.ceph.com/issues/1779 fixes #1779
+    
+    Adjust the return value from rgw_get_user_info_by_swift call
+    in RGW_SWIFT_Auth_Get::execute() to have the correct
+    return code in response.
+    (cherry picked from commit 4089001de1f22d6acd0b9f09996b71c716235551)
+
+commit 47852c263831707fff1570317a7446b0700c5962
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 9 21:55:51 2013 -0700
+
+    mon/OSDMonitor: make 'osd crush rm ...' slightly more idempotent
+    
+    This is a manual backport of 18a624fd8b90d9959de51f07622cf0839e6bd9aa.
+    Do not return immediately if we are looking at uncommitted state.t
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit bfc26c656d183fbcc90a352391e47f9f51c96052
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 8 17:46:40 2013 -0700
+
+    mon/OSDMonitor: fix base case for loading full osdmap
+    
+    Right after cluster creation, first_committed is 1 and latest stashed in 0,
+    but we don't have the initial full map yet.  Thereafter, we do (because we
+    write it with trim).  Fixes afd6c7d8247075003e5be439ad59976c3d123218.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 43fa7aabf1f7e5deb844c1f52d451bab9e7d1006)
+
+commit 7fb3804fb860dcd0340dd3f7c39eec4315f8e4b6
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 8 15:04:59 2013 -0700
+
+    mon: fix osdmap stash, trim to retain complete history of full maps
+    
+    The current interaction between sync and stashing full osdmaps only on
+    active mons means that a sync can result in an incomplete osdmap_full
+    history:
+    
+     - mon.c starts a full sync
+     - during sync, active osdmap service should_stash_full() is true and
+       includes a full in the txn
+     - mon.c sync finishes
+     - mon.c update_from_paxos gets "latest" stashed that it got from the
+       paxos txn
+     - mon.c does *not* walk to previous inc maps to complete it's collection
+       of full maps.
+    
+    To fix this, we disable the periodic/random stash of full maps by the
+    osdmap service.
+    
+    This introduces a new problem: we must have at least one full map (the first
+    one) in order for a mon that just synced to build it's full collection.
+    Extend the encode_trim() process to allow the osdmap service to include
+    the oldest full map with the trim txn.  This is more complex than just
+    writing the full maps in the txn, but cheaper--we only write the full
+    map at trim time.
+    
+    This *might* be related to previous bugs where the full osdmap was
+    missing, or case where leveldb keys seemed to 'disappear'.
+    
+    Fixes: #5512
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit afd6c7d8247075003e5be439ad59976c3d123218)
+
+commit 24f90b832c695ef13021db66a178c18369ac356d
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 8 15:07:57 2013 -0700
+
+    mon: implement simple 'scrub' command
+    
+    Compare all keys within the sync'ed prefixes across members of the quorum
+    and compare the key counts and CRC for inconsistencies.
+    
+    Currently this is a one-shot inefficient hammer.  We'll want to make this
+    work in chunks before it is usable in production environments.
+    
+    Protect with a feature bit to avoid sending MMonScrub to mons who can't
+    decode it.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit a9906641a1dce150203b72682da05651e4d68ff5)
+    
+    Conflicts:
+    
+    	src/mon/MonCommands.h
+    	src/mon/Monitor.cc
+
+commit 926f723c12428a034545c6c4ff6641e1d5e05d24
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jul 3 11:18:33 2013 -0700
+
+    Elector.h: features are 64 bit
+    
+    Fixes: #5497
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Luis <joao.luis@inktank.com>
+    (cherry picked from commit 3564e304e3f50642e4d9ff25e529d5fc60629093)
+
+commit c2b38291e706c9d1d4d337cee3a944f34bf66525
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jul 3 11:18:19 2013 -0700
+
+    ceph_features.h: declare all features as ULL
+    
+    Otherwise, the first 32 get |'d together as ints.  Then, the result
+    ((int)-1) is sign extended to ((long long int)-1) before being |'d
+    with the 1LL entries.  This results in ~((uint64_t)0).
+    
+    Fixes: #5497
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Luis <joao.luis@inktank.com>
+    (cherry picked from commit 4255b5c2fb54ae40c53284b3ab700fdfc7e61748)
+
+commit 95ef961d8537fc369efd0634262ffb8f288d6e9e
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Jul 2 21:09:36 2013 -0700
+
+    Pipe: use uint64_t not unsigned when setting features
+    
+    Fixes: #5497
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Luis <joao.luis@inktank.com>
+    (cherry picked from commit bc3e2f09f8860555d8b3b49b2eea164b4118d817)
+
+commit 09d258b70a28e5cea555b9d7e215fe41d6b84577
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 8 11:24:48 2013 -0700
+
+    client: remove O_LAZY
+    
+    The once-upon-a-time unique O_LAZY value I chose forever ago is now
+    O_NOATIME, which means that some clients are choosing relaxed
+    consistency without meaning to.
+    
+    It is highly unlikely that a real O_LAZY will ever exist, and we can
+    select it in the ceph case with the ioctl or libcephfs call, so drop
+    any support for doing this via open(2) flags.
+    
+    Update doc/lazy_posix.txt file re: lazy io.
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 94afedf02d07ad4678222aa66289a74b87768810)
+
+commit c3b684932bad31fc853ad556d16e1e4a9926486e
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 8 12:55:20 2013 -0700
+
+    osd/osd_types: fix pg_stat_t::dump for last_epoch_clean
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 69a55445439fce0dd6a3d32ff4bf436da42f1b11)
+
+commit a02f2510fcc800b9f2cf2a06401a7b97d5985409
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 5 16:03:49 2013 -0700
+
+    mon: remove bad assert about monmap version
+    
+    It is possible to start a sync when our newest monmap is 0.  Usually we see
+    e0 from probe, but that isn't always published as part of the very first
+    paxos transaction due to the way PaxosService::_active generates it's
+    first initial commit.
+    
+    In any case, having e0 here is harmless.
+    
+    Fixes: #5509
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 85a1d6cc5d3852c94d1287b566656c5b5024fa13)
+
+commit 6ad9fe17a674ba65bbeb4052cb1ac47f3113e7bf
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 4 19:33:06 2013 -0700
+
+    mon/Paxos: fix sync restart
+    
+    If we have a sync going, and an election intervenes, the client will
+    try to continue by sending a new start_chunks request.  In order to
+    ensure that we get all of the paxos commits from our original starting
+    point (and thus properly update the keys from which they started),
+    only pay attention if they *also* send their current last_committed
+    version.  Otherwise, start them at the beginning.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit c5812b1c893305a7d20f9eaec2695c8b1691f0c9
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 4 14:57:06 2013 -0700
+
+    mon: uninline _trim_enable and Paxos::trim_{enable,disable} so we can debug them
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 6fbcbd7fddf35a5be4b38e536871903bff4f9bf1
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 4 14:55:34 2013 -0700
+
+    mon/Paxos: increase paxos max join drift
+    
+    A value of 10 is too aggressive for large, long-running syncs. 100 is
+    about 2 minutes of activity at most, which should be a more forgiving
+    buffer.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit f3a51fa30e5ce1656853b40d831409f195f6e4ca
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 4 14:21:04 2013 -0700
+
+    mon/Paxos: configure minimum paxos txns separately
+    
+    We were using paxos_max_join_drift to control the minimum number of
+    paxos transactions to keep around.  Instead, make this explicit, and
+    separate from the join drift.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 1156721f22f5f337241eef3d0276ca74fe6352d1
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 4 17:09:07 2013 -0700
+
+    mon: include any new paxos commits in each sync CHUNK message
+    
+    We already take note of the paxos version when we begin the sync.  As
+    sync progresses and there are new paxos commits/txns, include those
+    and update last_committed, so that when sync completes we will have
+    a full view of everything that happened during sync.
+    
+    Note that this does not introduce any compatibility change.  This change
+    *only* affects the provider.  The key difference is that at the end
+    of the sync, the provide will set version to the latest version, and
+    not the version from the start of the sync (as was done previously).
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 40672219a081f0dc2dd536977290ef05cfc9f097
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 4 12:17:28 2013 -0700
+
+    mon/MonitorDBStore: expose get_chunk_tx()
+    
+    Allow users get the transaction unencoded.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit db2bb270e93ed44f9252d65d1d4c9b36875d0ea5
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 3 17:15:56 2013 -0700
+
+    mon: enable leveldb cache by default
+    
+    256 is not as large as the upstream 512 MB, but will help signficiantly and
+    be less disruptive for existing cuttlefish clusters.
+    
+    Sort-of backport of e93730b7ffa48b53c8da2f439a60cb6805facf5a.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 123f676e3ae8154ca94cb076c4c4ec5389d2a643
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 3 16:56:06 2013 -0700
+
+    mon/Paxos: make 'paxos trim disabled max versions' much much larger
+    
+    108000 is about 3 hours if paxos is going full-bore (1 proposal/second).
+    That ought to be pretty safe.  Otherwise, we start trimming to soon and a
+    slow sync will just have to restart when it finishes.
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 71ebfe7e1abe4795b46cf00dfe1b03d1893368b0)
+    
+    Conflicts:
+    
+    	src/common/config_opts.h
+
+commit 03393c0df9f54e4f1db60e1058ca5a7cd89f44e6
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jun 26 06:01:40 2013 -0700
+
+    mon: do not reopen MonitorDBStore during startup
+    
+    level doesn't seem to like this when it races with an internal compaction
+    attempt (see below).  Instead, let the store get opened by the ceph_mon
+    caller, and pull a bit of the logic into the caller to make the flow a
+    little easier to follow.
+    
+        -2> 2013-06-25 17:49:25.184490 7f4d439f8780 10 needs_conversion
+        -1> 2013-06-25 17:49:25.184495 7f4d4065c700  5 asok(0x13b1460) entry start
+         0> 2013-06-25 17:49:25.316908 7f4d3fe5b700 -1 *** Caught signal (Segmentation fault) **
+     in thread 7f4d3fe5b700
+    
+     ceph version 0.64-667-g089cba8 (089cba8fc0e8ae8aef9a3111cba7342ecd0f8314)
+     1: ceph-mon() [0x649f0a]
+     2: (()+0xfcb0) [0x7f4d435dccb0]
+     3: (leveldb::Table::BlockReader(void*, leveldb::ReadOptions const&, leveldb::Slice const&)+0x154) [0x806e54]
+     4: ceph-mon() [0x808840]
+     5: ceph-mon() [0x808b39]
+     6: ceph-mon() [0x806540]
+     7: (leveldb::DBImpl::DoCompactionWork(leveldb::DBImpl::CompactionState*)+0xdd) [0x7f363d]
+     8: (leveldb::DBImpl::BackgroundCompaction()+0x2c0) [0x7f4210]
+     9: (leveldb::DBImpl::BackgroundCall()+0x68) [0x7f4cc8]
+     10: ceph-mon() [0x80b3af]
+     11: (()+0x7e9a) [0x7f4d435d4e9a]
+     12: (clone()+0x6d) [0x7f4d4196bccd]
+     NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ea1f316e5de21487ae034a1aa929068ba23ac525)
+
+commit 0143acc49bc5834836afc2c5a9d8f67030bec85f
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 2 14:43:17 2013 -0700
+
+    sysvinit, upstart: handle symlinks to dirs in /var/lib/ceph/*
+    
+    Match a symlink to a dir, not just dirs.  This fixes the osd case of e.g.,
+    creating an osd in /data/osd$id in which ceph-disk makes a symlink from
+    /var/lib/ceph/osd/ceph-$id.
+    
+    Fix proposed by Matt Thompson <matt.thompson@mandiant.com>; extended to
+    include the upstart users too.
+    
+    Fixes: #5490
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 87c98e92d1375c8bc76196bbbf06f677bef95e64)
+
+commit 7e878bcc8c1b51538f3c05f854a9dac74c09b116
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 1 17:33:11 2013 -0700
+
+    rgw: add RGWFormatter_Plain allocation to sidestep cranky strlen()
+    
+    Valgrind complains about an invalid read when we don't pad the allocation,
+    and because it is inlined we can't whitelist it for valgrind.  Workaround
+    the warning by just padding our allocations a bit.
+    
+    Fixes: #5346
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 49ff63b1750789070a8c6fef830c9526ae0f6d9f)
+
+commit ca61402855966210ba1598239eaf454eaad0f5f2
+Author: Yan, Zheng <zheng.z.yan@intel.com>
+Date:   Wed May 15 11:24:36 2013 +0800
+
+    mds: warn on unconnected snap realms
+    
+    When there are more than one active MDS, restarting MDS triggers
+    assertion "reconnected_snaprealms.empty()" quite often. If there
+    is no snapshot in the FS, the items left in reconnected_snaprealms
+    should be other MDS' mdsdir. I think it's harmless.
+    
+    If there are snapshots in the FS, the assertion probably can catch
+    real bugs. But at present, snapshot feature is broken, fixing it is
+    non-trivial. So replace the assertion with a warning.
+    
+    Signed-off-by: Yan, Zheng <zheng.z.yan@intel.com>
+    (cherry picked from commit 26effc0e583b0a3dade6ec81ef26dec1c94ac8b2)
+
+commit e11f258831e14dc3755e09c0fd4f9bfdf79022a7
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jun 26 06:53:08 2013 -0700
+
+    mon/PGMonitor: use post_paxos_update, not init, to refresh from osdmap
+    
+    We do two things here:
+     - make init an one-time unconditional init method, which is what the
+       health service expects/needs.
+     - switch PGMonitor::init to be post_paxos_update() which is called after
+       the other services update, which is what PGMonitor really needs.
+    
+    This is a new version of the fix originally in commit
+    a2fe0137946541e7b3b537698e1865fbce974ca6 (and those around it).  That is,
+    this re-fixes a problem where osds do not see pg creates from their
+    subscribe due to map_pg_creates() not getting called.
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e635c47851d185eda557e36bdc4bf3775f7b87a2)
+    
+    Conflicts:
+    	src/mon/PGMonitor.cc
+    	src/mon/PGMonitor.h
+
+commit 4d07fb014178da3c88edeb8765e1aaacb8cb8ffa
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jun 26 06:52:01 2013 -0700
+
+    mon/PaxosService: add post_paxos_update() hook
+    
+    Some services need to update internal state based on other service's
+    state, and thus need to be run after everyone has pulled their info out of
+    paxos.
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 131686980f0a930d5de7cbce8234fead5bd438b6)
+
+commit 90f5c448abeb127ae5a5528a79bd7bdbc74cb497
+Author: Greg Farnum <greg@inktank.com>
+Date:   Thu Jun 27 14:58:14 2013 -0700
+
+    ceph-disk: s/else if/elif/
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Joao Luis <joao.luis@inktank.com>
+    (cherry picked from commit bd8255a750de08c1b8ee5e9c9a0a1b9b16171462)
+    (cherry picked from commit 9e604ee6943fdb131978afbec51321050faddfc6)
+
+commit 5c4bb463dca5aa61ea5f02f7592d5a3cc82cf6f4
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Jun 26 11:28:57 2013 -0700
+
+    rgw: fix radosgw-admin buckets list
+    
+    Fixes: #5455
+    Backport: cuttlefish
+    This commit fixes a regression, where radosgw-admin buckets list
+    operation wasn't returning any data.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e1f9fe58d2860fcbb18c92d3eb3946236b49a6ce)
+
+commit b2fb48762f32279e73feb83b220339fea31275e9
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jun 19 17:27:49 2013 -0700
+
+    ceph-disk: use unix lock instead of lockfile class
+    
+    The lockfile class relies on file system trickery to get safe mutual
+    exclusion.  However, the unix syscalls do this for us.  More
+    importantly, the unix locks go away when the owning process dies, which
+    is behavior that we want here.
+    
+    Fixes: #5387
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 2a4953b697a3464862fd3913336edfd7eede2487)
+
+commit 26e7a6fffde4abcb685f34247e8491c05ee2a68d
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jun 26 18:27:49 2013 -0700
+
+    ceph-disk: do not mount over an osd directly in /var/lib/ceph/osd/$cluster-$id
+    
+    If we see a 'ready' file in the target OSD dir, do not mount our device
+    on top of it.
+    
+    Among other things, this prevents ceph-disk activate on stray disks from
+    stepping on teuthology osds.
+    
+    Fixes: #5445
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 8a17f33b14d858235dfeaa42be1f4842dcfd66d2)
+
+commit ccb3dd5ad5533ca4e9b656b4e3df31025a5f2017
+Author: Yan, Zheng <zheng.z.yan@intel.com>
+Date:   Tue Apr 2 15:46:51 2013 +0800
+
+    mds: fix underwater dentry cleanup
+    
+    If the underwater dentry is a remove link, we shouldn't mark the
+    inode clean
+    
+    Signed-off-by: Yan, Zheng <zheng.z.yan@intel.com>
+    (cherry picked from commit 81d073fecb58e2294df12b71351321e6d2e69652)
+
+commit 3020c5ea07a91475a7261dc2b810f5b61a1ae1f2
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 24 18:51:07 2013 -0700
+
+    mon/Elector: cancel election timer if we bootstrap
+    
+    If we short-circuit and bootstrap, cancel our timer.  Otherwise it will
+    go off some time later when we are in who knows what state.
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 9ae0ec83dabe37ac15e5165559debdfef7a5f91d)
+
+commit 305f0c50a5f0ffabc73e10bdf4590217d5d5d211
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 24 18:12:11 2013 -0700
+
+    mon: cancel probe timeout on reset
+    
+    If we are probing and get (say) an election timeout that calls reset(),
+    cancel the timer.  Otherwise, we assert later with a splat like
+    
+    2013-06-24 01:09:33.675882 7fb9627e7700  4 mon.b@0(leader) e1 probe_timeout 0x307a520
+    2013-06-24 01:09:33.676956 7fb9627e7700 -1 mon/Monitor.cc: In function 'void Monitor::probe_timeout(int)' thread 7fb9627e7700 time 2013-06-24 01:09:43.675904
+    mon/Monitor.cc: 1888: FAILED assert(is_probing() || is_synchronizing())
+    
+     ceph version 0.64-613-g134d08a (134d08a9654f66634b893d493e4a92f38acc63cf)
+     1: (Monitor::probe_timeout(int)+0x161) [0x56f5c1]
+     2: (Context::complete(int)+0xa) [0x574a2a]
+     3: (SafeTimer::timer_thread()+0x425) [0x7059a5]
+     4: (SafeTimerThread::entry()+0xd) [0x7065dd]
+     5: (()+0x7e9a) [0x7fb966f62e9a]
+     6: (clone()+0x6d) [0x7fb9652f9ccd]
+     NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
+    
+    Fixes: #5438
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 03d3be3eaa96a8e72754c36abd6f355c68d52d59)
+
+commit a8f601d543168f4cdbddf674479d8de4b8dfc732
+Author: Alexandre Maragone <alexandre.marangone@inktank.com>
+Date:   Tue Jun 18 16:18:01 2013 -0700
+
+    ceph-disk: make list_partition behave with unusual device names
+    
+    When you get device names like sdaa you do not want to mistakenly conclude that
+    sdaa is a partition of sda.  Use /sys/block/$device/$partition existence
+    instead.
+    
+    Fixes: #5211
+    Backport: cuttlefish
+    Signed-off-by: Alexandre Maragone <alexandre.maragone@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 8c0daafe003935881c5192e0b6b59b949269e5ae)
+
+commit 1c890f5cdfc596588e54fffeb016b4a5e9e2124c
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 17 20:28:24 2013 -0700
+
+    client: fix warning
+    
+    client/Client.cc: In member function 'virtual void Client::ms_handle_remote_reset(Connection*)':
+    warning: client/Client.cc:7892:9: enumeration value 'STATE_NEW' not handled in switch [-Wswitch]
+    warning: client/Client.cc:7892:9: enumeration value 'STATE_OPEN' not handled in switch [-Wswitch]
+    warning: client/Client.cc:7892:9: enumeration value 'STATE_CLOSED' not handled in switch [-Wswitch]
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 8bd936f077530dfeb2e699164e4492b1c0973088)
+
+commit c3b97591fd8206825bcfe65bdb24fbc75a2a9b42
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 24 17:58:48 2013 -0700
+
+    mon/AuthMonitor: ensure initial rotating keys get encoded when create_initial called 2x
+    
+    The create_initial() method may get called multiple times; make sure it
+    will unconditionally generate new/initial rotating keys.  Move the block
+    up so that we can easily assert as much.
+    
+    Broken by commit cd98eb0c651d9ee62e19c2cc92eadae9bed678cd.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 521fdc2a4e65559b3da83283e6ca607b6e55406f)
+
+commit 0cc826c385edb2e327505696491d3ff1c3bfe8fd
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 24 17:42:04 2013 -0700
+
+    init-radosgw.sysv: remove -x debug mode
+    
+    Fixes: #5443
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 31d6062076fdbcd2691c07a23b381b26abc59f65)
+
+commit 4d57c12faceb7f591f10776c6850d98da55c667b
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 24 12:52:44 2013 -0700
+
+    common/pick_addresses: behave even after internal_safe_to_start_threads
+    
+    ceph-mon recently started using Preforker to working around forking issues.
+    As a result, internal_safe_to_start_threads got set sooner and calls to
+    pick_addresses() which try to set string config values now fail because
+    there are no config observers for them.
+    
+    Work around this by observing the change while we adjust the value.  We
+    assume pick_addresses() callers are smart enough to realize that their
+    result will be reflected by cct->_conf and not magically handled elsewhere.
+    
+    Fixes: #5195, #5205
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit eb86eebe1ba42f04b46f7c3e3419b83eb6fe7f9a)
+
+commit e1ac7c6c3ca673d08710829aa5a3c03735710486
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 20 15:39:23 2013 -0700
+
+    mon/PaxosService: allow paxos service writes while paxos is updating
+    
+    In commit f985de28f86675e974ac7842a49922a35fe24c6c I mistakenly made
+    is_writeable() false while paxos was updating due to a misread of
+    Paxos::propose_new_value() (I didn't see that it would queue).
+    This is problematic because it narrows the window during which each service
+    is writeable for no reason.
+    
+    Allow service to be writeable both when paxos is active and updating.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 11169693d086e67dcf168ce65ef6e13eebd1a1ab)
+
+commit 02b0b4a9acb439b2ee5deadc8b02492006492931
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 7 11:41:21 2013 -0700
+
+    mon/PaxosService: not active during paxos UPDATING_PREVIOUS
+    
+    Treat this as an extension of the recovery process, e.g.
+    
+     RECOVERING -> ACTIVE
+    or
+     RECOVERING -> UPDATING_PREVIOUS -> ACTIVE
+    
+    and we are not active until we get to "the end" in both cases.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 392a8e21f8571b410c85be2129ef62dd6fc52b54)
+
+commit c6d5dc4d47838c8c8f4d059b7d018dea3f9c4425
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 7 11:40:22 2013 -0700
+
+    mon: simplify states
+    
+    - make states mutually exclusive (an enum)
+    - rename locked -> updating_previous
+    - set state prior to begin() to simplify things a bit
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ee34a219605d1943740fdae0d84cfb9020302dd6)
+
+commit c43b1f4dff254df96144b0b4d569cc72421a8fff
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 7 11:14:58 2013 -0700
+
+    mon/Paxos: not readable when LOCKED
+    
+    If we are re-proposing a previously accepted value from a previous quorum,
+    we should not consider it readable, because it is possible it was exposed
+    to clients as committed (2/3 accepted) but not recored to be committed, and
+    we do not want to expose old state as readable when new state was
+    previously readable.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ec2ea86ed55e00265c2cc5ad0c94460b4c92865c)
+
+commit 10d41200622d76dbf276602828584e7153cb22b5
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 7 11:07:38 2013 -0700
+
+    mon/Paxos: cleanup: drop unused PREPARING state bit
+    
+    This is never set when we block, and nobody looks at it.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 7b7ea8e30e20704caad9a841332ecb2e39819a41)
+
+commit 9d7c40e3f4ea2dd969aa0264ea8a6ad74f3e678a
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 6 15:20:05 2013 -0700
+
+    mon/PaxosService: simplify is_writeable
+    
+    Recast this in terms of paxos check + our conditions, and make it
+    match wait_for_writeable().
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f985de28f86675e974ac7842a49922a35fe24c6c)
+
+commit 35745cba8985c5f3238e3c28fd28b194fae043d9
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 4 17:03:15 2013 -0700
+
+    mon/PaxosService: simplify readable check
+    
+    Recast this in terms of the paxos check and our additional conditions,
+    which match wait_for_readable().
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3aa61a0beb540e48bf61ceded766d6ff52c95eb2)
+
+commit 57c89291a48c319907fb3029746d9f5a4bd9dd61
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 31 16:45:08 2013 -0700
+
+    mon: simplify Monitor::init_paxos()
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e832e76a4af04b091c806ad412bcfd0326d75a2d)
+
+commit fd1769cb2d61e8f2c7921a78760e8f12b28258fb
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 31 16:39:37 2013 -0700
+
+    mon/Paxos: go active *after* refreshing
+    
+    The update_from_paxos() methods occasionally like to trigger new activity.
+    As long as they check is_readable() and is_writeable(), they will defer
+    until we go active and that activity will happen in the normal callbacks.
+    
+    This fixes the problem where we active but is_writeable() is still false,
+    triggered by PGMonitor::check_osd_map().
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e68b1bd36ed285e38a558899f83cf224d3aa60ed)
+
+commit cf75478d027dfd377424988745230d096dae79ac
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 31 15:32:06 2013 -0700
+
+    mon: safely signal bootstrap from MonmapMonitor::update_from_paxos()
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit dc83430124a5fd37573202a4cc0986c3c03739ef)
+
+commit 6ac58cd9c1f9c80c5f3cbe97e19cfcd8427db46d
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jun 2 16:57:11 2013 -0700
+
+    mon/Paxos: do paxos refresh in finish_proposal; and refactor
+    
+    Do the paxos refresh inside finish_proposal, ordered *after* the leader
+    assertion so that MonmapMonitor::update_from_paxos() calling bootstrap()
+    does not kill us.
+    
+    Also, remove unnecessary finish_queued_proposal() and move the logic inline
+    where the bad leader assertion is obvious.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a42d7582f816b45f5d19c393fd45447555e78fdd)
+
+commit 054e96d96533b1c4078402e43184f13b97329905
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Sun Jun 2 16:15:02 2013 -0700
+
+    mon/PaxosService: cache {first,last}_committed
+    
+    Refresh the in-memory values when we are told the on-disk paxos state
+    may have changed.
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 2fccb300bdf6ffd44db3462eb05115da11322ed4)
+
+commit 265212a7384399bf85e15e6978bc7543824c0e92
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 31 14:30:48 2013 -0700
+
+    mon: no need to refresh from _active
+    
+    The refresh is done explicitly by the monitor, independent of the more
+    fragile PaxosService callbacks.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d941363d6e4249e97b64faff0e573f75e918ac0c)
+
+commit 1d8662504299babec22c714662cefbb86a0acb8b
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jun 2 16:10:57 2013 -0700
+
+    mon: remove unnecessary update_from_paxos calls
+    
+    The refresh() will do this when the state changes; no need to
+    opportunistically call this method all of the time.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 03014a4ecc06cde420fad0c6c2a0177ebd7b839d)
+
+commit 34acc5a3161b6bcda2b9f7ce18d89a8618fff1c5
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jun 2 16:14:01 2013 -0700
+
+    mon: explicitly refresh_from_paxos() when leveldb state changes
+    
+    Instead of opportunistically calling each service's update_from_paxos(),
+    instead explicitly refresh all in-memory state whenever we know the
+    paxos state may have changed.  This is simpler and less fragile.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cc339c07312006e65854207523f50542d00ecf87)
+
+commit 4474a0cc6c009a566ecf46efadb39d80343a7c68
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jun 23 09:25:55 2013 -0700
+
+    mon/AuthMonitor: make initial auth include rotating keys
+    
+    This closes a very narrow race during mon creation where there are no
+    service keys.
+    
+    Fixes: #5427
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cd98eb0c651d9ee62e19c2cc92eadae9bed678cd)
+
+commit d572cf6f77418f217a5a8e37f1124dc566e24d0b
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 21 11:53:29 2013 -0700
+
+    mds: fix iterator invalidation for backtrace removal
+    
+    - Don't increment before we dereference!
+    - We need to update the iterator before we delete the item.
+    
+    This code is changed in master, so this fix is for cuttlefish only.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit 50957772c3582290331f69ba4a985b1cdf86834d
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 9 09:44:20 2013 -0700
+
+    osd: init test_ops_hook
+    
+    CID 1019628 (#1 of 1): Uninitialized pointer field (UNINIT_CTOR)
+    2. uninit_member: Non-static class member "test_ops_hook" is not initialized in this constructor nor in any functions that it calls.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e30a03210c3efb768b1653df5ae58917ef26e579)
+
+commit 17d2745f095e7bb640dece611d7824d370ea3b81
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 9 09:45:51 2013 -0700
+
+    osd: initialize OSDService::next_notif_id
+    
+    CID 1019627 (#1 of 1): Uninitialized scalar field (UNINIT_CTOR)
+    2. uninit_member: Non-static class member "next_notif_id" is not initialized in this constructor nor in any functions that it calls.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 499edd8bfc355c2d590f5fa1ef197d1ea5680351)
+
+commit ffdb7236a994aa20b5f75860b9c81dac0f131f9a
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 20 09:46:42 2013 -0700
+
+    mon: more fix dout use in sync_requester_abort()
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d60534b8f59798feaeeaa17adba2a417d7777cbf)
+
+commit 38ddae04bb974a93f1718c509363f1afbe6b612d
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 10 11:48:25 2013 -0700
+
+    mon: fix raw use of *_dout in sync_requester_abort()
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 8a4ed58e39b287fd8667c62b45848487515bdc80)
diff --git a/doc/changelog/v0.61.6.txt b/doc/changelog/v0.61.6.txt
new file mode 100644
index 000000000..3a1e5bd4b
--- /dev/null
+++ b/doc/changelog/v0.61.6.txt
@@ -0,0 +1,75 @@
+commit 59ddece17e36fef69ecf40e239aeffad33c9db35
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Tue Jul 23 13:52:19 2013 -0700
+
+    v0.61.6
+
+commit 38c3271d3fc415919f0856398bd94eb87a0776b5
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 23 13:32:12 2013 -0700
+
+    mon/OSDMonitor: fix base case for 7fb3804fb workaround
+    
+    After cluster creation, we have no full map stored and first_committed ==
+    1.  In that case, there is no need for a full map, since we can get there
+    from OSDMap() and the incrementals.
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao@inktank.com>
+    (cherry picked from commit e807770784175b05130bba938113fdbf874f152e)
+
+commit f94956cb1a56ff62e01b7ae218a93c4004470ae5
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jul 23 17:25:13 2013 +0100
+
+    mon: OSDMonitor: work around a full version bug introduced in 7fb3804fb
+    
+    In 7fb3804fb860dcd0340dd3f7c39eec4315f8e4b6 we moved the full version
+    stashing logic to the encode_trim_extra() function.  However, we forgot
+    to update the osdmap's 'latest_full' key that should always point to
+    the latest osdmap full version.  This eventually degenerated in a missing
+    full version after a trim.  This patch works around this bug by looking
+    for the latest available full osdmap version in the store and updating
+    'latest_full' to its proper value.
+    
+    Related-to: #5704
+    Backport: cuttlefish
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 97462a3213e5e15812c79afc0f54d697b6c498b1)
+
+commit 10e1de45dc8ace793ecf921f884f90c9daa99c48
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jul 23 16:36:52 2013 +0100
+
+    mon: OSDMonitor: update the osdmap's latest_full with the new full version
+    
+    We used to do this on encode_full(), but since [1] we no longer rely on
+    PaxosService to manage the full maps for us.  And we forgot to write down
+    the latest_full version to the store, leaving it in a truly outdated state.
+    
+    [1] - 7fb3804fb860dcd0340dd3f7c39eec4315f8e4b6
+    
+    Fixes: #5704
+    Backport: cuttlefish
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a815547ed3e5ffdbbb96c8c0c1b8d6dd8c62bfba)
+
+commit a0cb40b45c4f2f921a63c2d7bb5a28572381d793
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 18 14:35:19 2013 -0700
+
+    mon: decline to scrub when paxos is not active
+    
+    In f1ce8d7c955a2443111bf7d9e16b4c563d445712 we close a race between scrub
+    and paxos commit completion on the leader.  The fix is nontrivial to
+    backport and probably not worthwhile; just avoid scrubbing at that time
+    for now.
+    
+    Note that the actual fix for this is in commit
+    f1ce8d7c955a2443111bf7d9e16b4c563d445712.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
diff --git a/doc/changelog/v0.61.7.txt b/doc/changelog/v0.61.7.txt
new file mode 100644
index 000000000..5836c6a32
--- /dev/null
+++ b/doc/changelog/v0.61.7.txt
@@ -0,0 +1,220 @@
+commit 8f010aff684e820ecc837c25ac77c7a05d7191ff
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Wed Jul 24 20:44:12 2013 -0700
+
+    v0.61.7
+
+commit 24a56a9637afd8c64b71d264359c78a25d52be02
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 24 14:46:24 2013 -0700
+
+    ceph-disk: use new get_dev_path helper for list
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    Tested-by: Olivier Bonvalet <ob.ceph@daevel.fr>
+    (cherry picked from commit fd1fd664d6102a2a96b27e8ca9933b54ac626ecb)
+
+commit 1f8e4b15eeb132fd7f389318009b19f8f13adbf5
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 11 12:59:56 2013 -0700
+
+    ceph-disk: use /sys/block to determine partition device names
+    
+    Not all devices are basename + number; some have intervening character(s),
+    like /dev/cciss/c0d1p2.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2ea8fac441141d64ee0d26c5dd2b441f9782d840)
+
+commit 0a08c8226cb3e461301beade9bab2e264d1b960e
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 3 11:01:58 2013 -0700
+
+    ceph-disk: reimplement is_partition() using /sys/block
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5b031e100b40f597752b4917cdbeebb366eb98d7)
+
+commit 056000346db09ea7274a22e57cf4b86a7ea4090e
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 3 11:01:39 2013 -0700
+
+    ceph-disk: use get_dev_name() helper throughout
+    
+    This is more robust than the broken split trick.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3359aaedde838c98d1155611e157fd2da9e8b9f5)
+
+commit f3ee3e690c42769229a6cd9ae8dec43f2aa22ecd
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 3 10:55:36 2013 -0700
+
+    ceph-disk: refactor list_[all_]partitions
+    
+    Make these methods work in terms of device *names*, not paths, and fix up
+    the only direct list_partitions() caller to do the same.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 35d3f2d84808efda3d2ac868afe03e6959d51c03)
+
+commit be12811b4cb98ff1c2c691c67af7ad3586c436ff
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 3 10:52:29 2013 -0700
+
+    ceph-disk: add get_dev_name, path helpers
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e0401591e352ea9653e3276d66aebeb41801eeb3)
+
+commit f46dbc462f623e9ab6c00394abb4d890e5d90890
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 18 16:21:48 2013 -0700
+
+    ceph-disk: handle /dev/foo/bar devices throughout
+    
+    Assume the last component is the unique device name, even if it appears
+    under a subdir of /dev.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cb97338b1186939deecb78e9d949c38c3ef59026)
+
+commit f799dac7bdf7cf0824a177131473cf59ef3c5205
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 17 20:54:15 2013 -0700
+
+    ceph-disk: make is_held() smarter about full disks
+    
+    Handle the case where the device is a full disk.  Make the partition
+    check a bit more robust (don't make assumptions about naming aside from
+    the device being a prefix of the partition).
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e082f1247fb6ddfb36c4223cbfdf500d6b45c978)
+
+commit 27f31895664fa7f10c1617d486f2a6ece0f97091
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 24 11:55:42 2013 -0700
+
+    mon/OSDMonitor: search for latest full osdmap if record version is missing
+    
+    In 97462a3213e5e15812c79afc0f54d697b6c498b1 we tried to search for a
+    recent full osdmap but were looking at the wrong key.  If full_0 was
+    present we could record that the latest full map was last_committed even
+    though it wasn't present.  This is fixed in 76cd7ac1c, but we need to
+    compensate for when get_version_latest_full() gives us a back version
+    number by repeating the search.
+    
+    Fixes: #5737
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit c2131d4047156aa2964581c9dbd93846382a07e7)
+
+commit 5b0967f03efb1be210b52f24f095f023fe1bc539
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Mon Jun 17 14:43:36 2013 +0100
+
+    test: test_store_tool: global init before using LevelDBStore
+    
+    Fixes a segfault
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a7a7d3fc8a2ba4a30ef136a32f2903d157b3e19a)
+
+commit 115468c73f121653eec2efc030d5ba998d834e43
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Jul 24 12:00:28 2013 +0100
+
+    mon: OSDMonitor: fix a bug introduced on 97462a32
+    
+    Fixes: #5737
+    Backport: cuttlefish
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 76cd7ac1c2094b34ad36bea89b2246fa90eb2f6d)
+
+commit 938a639e2cb6abd22c2c588e619c1aae32c6521f
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jul 21 08:48:18 2013 -0700
+
+    mon/Paxos: fix pn for uncommitted value during collect/last phase
+    
+    During the collect/last exchange, peers share any uncommitted values
+    with the leader.  They are supposed to also share the pn under which
+    that value was accepted, but were instead using the just-accepted pn
+    value.  This effectively meant that we *always* took the uncommitted
+    value; if there were multiples, which one we accepted depended on what
+    order the LAST messages arrived, not which pn the values were generated
+    under.
+    
+    The specific failure sequence I observed:
+    
+     - collect
+      - learned uncommitted value for 262 from myself
+      - send collect with pn 901
+     - got last with pn 901 (incorrect) for 200 (old) from peer
+      - discard our own value, remember the other
+     - finish collect phase
+      - ignore old uncommitted value
+    
+    Fix this by storing a pending_v and pending_pn value whenever we accept
+    a value.  Use this to send an appropriate pn value in the LAST reply
+    so that the leader can make it's decision about which uncommitted value
+    to accept based on accurate information.  Also use it when we learn
+    the uncommitted value from ourselves.
+    
+    We could probably be more clever about storing less information here,
+    for example by omitting pending_v and clearing pending_pn at the
+    appropriate point, but that would be more fragile.  Similarly, we could
+    store a pn for *every* commit if we wanted to lay some groundwork for
+    having multiple uncommitted proposals in flight, but I don't want to
+    speculate about what is necessary or sufficient for a correct solution
+    there.
+    
+    Fixes: #5698
+    Backport: cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 20baf662112dd5f560bc3a2d2114b469444c3de8)
+
+commit 18596340f020be1f21bdc9bcc752ae1da4a93a46
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jul 21 08:12:46 2013 -0700
+
+    mon/Paxos: debug ignored uncommitted values
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 19b29788966eb80ed847630090a16a3d1b810969)
+
+commit f598245f1355d7791162c03d90bdd97b013e56f3
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jul 21 08:11:22 2013 -0700
+
+    mon/Paxos: only learn uncommitted value if it is in the future
+    
+    If an older peer sends an uncommitted value, make sure we only take it
+    if it is in the future, and at least as new as any current uncommitted
+    value.
+    
+    (Prior to the previous patch, peers could send values from long-past
+    rounds.  The pn values are also bogus.)
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b3253a453c057914753846c77499f98d3845c58e)
+
+commit 732286a28cd8a643593d490a7a84a590d372f78d
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 22 14:13:23 2013 -0700
+
+    mon/Paxos: only share uncommitted value if it is next
+    
+    We may have an uncommitted value from our perspective (it is our lc + 1)
+    when the collector has a much larger lc (because we have been out for
+    the last few rounds).  Only share an uncommitted value if it is in fact
+    the next value.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b26b7f6e5e02ac6beb66e3e34e177e6448cf91cf)
diff --git a/doc/changelog/v0.61.8.txt b/doc/changelog/v0.61.8.txt
new file mode 100644
index 000000000..ef01d4551
--- /dev/null
+++ b/doc/changelog/v0.61.8.txt
@@ -0,0 +1,810 @@
+commit d783e33b672ec324eb48d588f956da0c51ff5dac
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Sun Aug 18 23:54:49 2013 -0700
+
+    v0.61.8
+
+commit 21a6e2479133a3debb9ab9057ff9fae70c9eede9
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Aug 8 15:12:46 2013 -0700
+
+    RadosClient: shutdown monclient after dropping lock
+
+    Otherwise, the monclient shutdown may deadlock waiting
+    on a context trying to take the RadosClient lock.
+
+    Fixes: #5897
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 0aacd10e2557c55021b5be72ddf39b9cea916be4)
+
+commit 64bef4ae4bab28b0b82a1481381b0c68a22fe1a4
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Aug 17 09:05:32 2013 -0700
+
+    mon/OSDMonitor: make 'osd pool mksnap ...' not expose uncommitted state
+
+    [This is a backport of d1501938f5d07c067d908501fc5cfe3c857d7281]
+
+    We were returning success without waiting if the pending pool state had
+    the snap.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+
+commit 411871f6bcc9a4b81140c2e98d13dc123860f6f7
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 16 10:52:02 2013 -0700
+
+    mon/OSDMonitor: make 'osd pool rmsnap ...' not racy/crashy
+
+    NOTE: This is a manual backport of d90683fdeda15b726dcf0a7cab7006c31e99f14.
+    Due to all kinds of collateral changes in the mon the original patch
+    doesn't cleanly apply.
+
+    Ensure that the snap does in fact exist before we try to remove it.  This
+    avoids a crash where a we get two dup rmsnap requests (due to thrashing, or
+    a reconnect, or something), the committed (p) value does have the snap, but
+    the uncommitted (pp) does not.  This fails the old test such that we try
+    to remove it from pp again, and assert.
+
+    Restructure the flow so that it is easier to distinguish the committed
+    short return from the uncommitted return (which must still wait for the
+    commit).
+
+             0> 2013-07-16 14:21:27.189060 7fdf301e9700 -1 osd/osd_types.cc: In function 'void pg_pool_t::remove_snap(snapid_t)' thread 7fdf301e9700 time 2013-07-16 14:21:27.187095
+        osd/osd_types.cc: 662: FAILED assert(snaps.count(s))
+
+         ceph version 0.66-602-gcd39d8a (cd39d8a6727d81b889869e98f5869e4227b50720)
+         1: (pg_pool_t::remove_snap(snapid_t)+0x6d) [0x7ad6dd]
+         2: (OSDMonitor::prepare_command(MMonCommand*)+0x6407) [0x5c1517]
+         3: (OSDMonitor::prepare_update(PaxosServiceMessage*)+0x1fb) [0x5c41ab]
+         4: (PaxosService::dispatch(PaxosServiceMessage*)+0x937) [0x598c87]
+         5: (Monitor::handle_command(MMonCommand*)+0xe56) [0x56ec36]
+         6: (Monitor::_ms_dispatch(Message*)+0xd1d) [0x5719ad]
+         7: (Monitor::handle_forward(MForward*)+0x821) [0x572831]
+         8: (Monitor::_ms_dispatch(Message*)+0xe44) [0x571ad4]
+         9: (Monitor::ms_dispatch(Message*)+0x32) [0x588c52]
+         10: (DispatchQueue::entry()+0x549) [0x7cf1d9]
+         11: (DispatchQueue::DispatchThread::entry()+0xd) [0x7060fd]
+         12: (()+0x7e9a) [0x7fdf35165e9a]
+         13: (clone()+0x6d) [0x7fdf334fcccd]
+         NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+
+commit 50698d1862065c8d74338fd08c7e5af66e222490
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 13 12:52:41 2013 -0700
+
+    librados: fix async aio completion wakeup
+
+    For aio flush, we register a wait on the most recent write.  The write
+    completion code, however, was *only* waking the waiter if they were waiting
+    on that write, without regard to previous writes (completed or not).
+    For example, we might have 6 and 7 outstanding and wait on 7.  If they
+    finish in order all is well, but if 7 finishes first we do the flush
+    completion early.  Similarly, if we
+
+     - start 6
+     - start 7
+     - finish 7
+     - flush; wait on 7
+     - finish 6
+
+    we can hang forever.
+
+    Fix by doing any completions that are prior to the oldest pending write in
+    the aio write completion handler.
+
+    Refs: #5919
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Tested-by: Oliver Francke <Oliver.Francke@filoo.de>
+    (cherry picked from commit 16ed0b9af8bc08c7dabead1c1a7c1a22b1fb02fb)
+
+commit ef731dfc84a71d3c3262f5cff9a9d33a60255485
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Aug 12 19:17:09 2013 -0700
+
+    librados: fix locking for AioCompletionImpl refcounting
+
+    Add an already-locked helper so that C_Aio{Safe,Complete} can
+    increment the reference count when their caller holds the
+    lock. C_AioCompleteAndSafe's caller is not holding the lock, so call
+    regular get() to ensure no racing updates can occur.
+
+    This eliminates all direct manipulations of AioCompletionImpl->ref,
+    and makes the necessary locking clear.
+
+    The only place C_AioCompleteAndSafe is used is in handling
+    aio_flush_async(). This could cause a missing completion.
+
+    Refs: #5919
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Tested-by: Oliver Francke <Oliver.Francke@filoo.de>
+    (cherry picked from commit 7a52e2ff5025754f3040eff3fc52d4893cafc389)
+
+commit 32631685199f2e47c2ba0ed27d16eff80fa6917d
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 12 14:27:04 2013 -0700
+
+    mon/Paxos: bootstrap peon too if monmap updates
+
+    If we get a monmap update, the leader bootstraps.  Peons should do the
+    same.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit efe5b67bb700ef6218d9579abf43cc9ecf25ef52)
+
+commit 1ea6b56170fc9e223e7c30635db02fa2ad8f4b4e
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 25 13:16:45 2013 -0700
+
+    osd: fix race when queuing recovery ops
+
+    Previously we would sample how many ops to start under the lock, drop it,
+    and start that many.  This is racy because multiple threads can jump in
+    and we start too many ops.  Instead, claim as many slots as we can and
+    release them back later if we do not end up using them.
+
+    Take care to re-wake the work-queue since we are releasing more resources
+    for wq use.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 01d3e094823d716be0b39e15323c2506c6f0cc3b)
+
+commit 4433f9ad8b338b6a55e205602434b307287bfaa3
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 24 16:37:29 2013 -0700
+
+    osd: tolerate racing threads starting recovery ops
+
+    We sample the (max - active) recovery ops to know how many to start, but
+    do not hold the lock over the full duration, such that it is possible to
+    start too many ops.  This isn't problematic except that our condition
+    checks for being == max but not beyond it, and we will continue to start
+    recovery ops when we shouldn't.  Fix this by adjusting the conditional
+    to be <=.
+
+    Reported-by: Stefan Priebe <s.priebe@profihost.ag>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 3791a1e55828ba541f9d3e8e3df0da8e79c375f9)
+
+commit 0964d53ef3e8e386e0a1635d2240aefad7b8e2c1
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 9 18:02:32 2013 -0700
+
+    ceph-disk: fix mount options passed to move_mount
+
+    Commit 6cbe0f021f62b3ebd5f68fcc01a12fde6f08cff5 added a mount_options but
+    in certain cases it may be blank.  Fill in with the defaults, just as we
+    do in mount().
+
+    Backport: cuttlefish
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cb50b5a7f1ab2d4e7fdad623a0e7769000755a70)
+
+commit d6be5ed2601b8cf45570afe7ca75ce5aba3f8b4f
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Aug 12 10:05:44 2013 -0700
+
+    rgw: fix multi delete
+
+    Fixes: #5931
+    Backport: bobtail, cuttlefish
+
+    Fix a bad check, where we compare the wrong field. Instead of
+    comparing the ret code to 0, we compare the string value to 0
+    which generates implicit casting, hence the crash.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit f9f1c48ad799da2b4be0077bf9d61ae116da33d7)
+
+    Conflicts:
+	src/rgw/rgw_rest_s3.cc
+
+commit ecaa46a13837305b9382ab319d43890729c54f1e
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Jul 23 21:56:09 2013 +0200
+
+    ceph.spec.in: obsolete ceph-libs only on the affected distro
+
+    The ceph-libs package existed only on Redhat based distro,
+    there was e.g. never such a package on SUSE. Therefore: make
+    sure the 'Obsoletes' is only set on these affected distros.
+
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+
+commit 81aa68c309a6d4eaecc54f8d735efde8843fed8c
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Wed Jul 3 11:28:28 2013 -0700
+
+    ceph.spec.in:  Obsolete ceph-libs
+
+    Signed-off-by: Gary Lowell  <gary.lowell@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit 2a34df68bb02d14f6a25bd13dff600a4d629ad05
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri Aug 9 14:48:15 2013 -0700
+
+    common: pick_addresses: fix bug with observer class that triggered #5205
+
+    The Observer class we defined to observe conf changes and thus avoid
+    triggering #5205 (as fixed by eb86eebe1ba42f04b46f7c3e3419b83eb6fe7f9a),
+    was returning always the same const static array, which would lead us to
+    always populate the observer's list with an observer for 'public_addr'.
+
+    This would of course become a problem when trying to obtain the observer
+    for 'cluster_add' during md_config_t::set_val() -- thus triggering the
+    same assert as initially reported on #5205.
+
+    Backport: cuttlefish
+    Fixes: #5205
+
+    Signed-off-by: Joao Eduardo Luis <jecluis@gmail.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 7ed6de9dd7aed59f3c5dd93e012cf080bcc36d8a)
+
+commit 1243c9749ed27850c5d041023780efcdf7b31a68
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Thu Aug 8 16:09:26 2013 -0700
+
+    make sure we are using the mount options
+
+    Signed-off-by: Alfredo Deza <alfredo@deza.pe>
+    (cherry picked from commit 34831d0989d4bcec4920068b6ee09ab6b3234c91)
+
+commit a9a370be2d8155b696ebe2866febb0571da5740f
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Aug 2 11:58:52 2013 -0700
+
+    PG: set !flushed in Reset()
+
+    Otherwise, we might serve a pull before we start_flush in the
+    ReplicaActive constructor.
+
+    Fixes: #5799
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 9e7d6d547e0e8a6db6ba611882afa9bf74ea0195)
+
+commit 65bfa4941f983c988837cd010f731966ff53fd19
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 26 14:02:07 2013 -0700
+
+    osd: make open classes on start optional
+
+    This is cuttlefish; default to the old behavior!
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6f996223fb34650771772b88355046746f238cf2)
+
+commit e8253ae5451b1c8e3d7d50199b8db7b2d4c66486
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 26 13:58:46 2013 -0700
+
+    osd: load all classes on startup
+
+    This avoid creating a wide window between when ceph-osd is started and
+    when a request arrives needing a class and it is loaded.  In particular,
+    upgrading the packages in that window may cause linkage errors (if the
+    class API has changed, for example).
+
+    Fixes: #5752
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit c24e652d8c5e693498814ebe38c6adbec079ea36)
+
+    Conflicts:
+	src/osd/ClassHandler.cc
+
+commit 7a1d6d3e727fd8b6947c658e171bf7ec31cd7966
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jul 28 15:42:08 2013 -0700
+
+    ceph_test_rados: print version banner on startup
+
+    It is helpful when looking at qa run logs to see what version of the
+    tester is running.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 12c1f1157c7b9513a3d9f716a8ec62fce00d28f5)
+
+commit 86769f05ccc54bfec403bb9ea9a3a951bbcea301
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 13 22:08:36 2013 -0700
+
+    ceph_test_rados: add --pool <name> arg
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit bcfbd0a3ffae6947464d930f636c8b35d1331e9d)
+
+commit b70a9abc5e3ae01204256f414bd7e69d083ed7c6
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 26 14:07:02 2013 -0700
+
+    upstart: stop ceph-create-keys when the monitor stops
+
+    This avoids lingering ceph-create-keys tasks.
+
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a90a2b42db8de134b8ea5d81cab7825fb9ec50b4)
+
+commit 5af48dc7c7e3a0d7f7bc22af58831d58d165e657
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Jul 26 13:42:27 2013 -0700
+
+    FileStore: fix fd leak in _check_global_replay_guard
+
+    Bug introduced in f3f92fe21061e21c8b259df5ef283a61782a44db.
+
+    Fixes: #5766
+    Backport: cuttlefish
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c562b72e703f671127d0ea2173f6a6907c825cd1)
+
+commit 17aa2d6d16c77028bae1d2a77903cdfd81efa096
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 25 11:10:53 2013 -0700
+
+    mon/Paxos: share uncommitted value when leader is/was behind
+
+    If the leader has and older lc than we do, and we are sharing states to
+    bring them up to date, we still want to also share our uncommitted value.
+    This particular case was broken by b26b7f6e, which was only contemplating
+    the case where the leader was ahead of us or at the same point as us, but
+    not the case where the leader was behind.  Note that the call to
+    share_state() a few lines up will bring them fully up to date, so
+    after they receive and store_state() for this message they will be at the
+    same lc as we are.
+
+    Fixes: #5750
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 05b6c7e8645081f405c616735238ae89602d3cc6)
+
+commit 09a664e25391dbad9a479bae33904d28231f429d
+Merge: 8f010af b0535fc
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 25 15:21:31 2013 -0700
+
+    Merge remote-tracking branch 'gh/cuttlefish-next' into cuttlefish
+
+commit b0535fcf854c5042d6b5ff481aabca08026d8f7f
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Jul 23 18:04:40 2013 -0700
+
+    HashIndex: reset attr upon split or merge completion
+
+    A replay of an in progress merge or split might make
+    our counts unreliable.
+
+    Fixes: #5723
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 0dc3efdd885377a07987d868af5bb7a38245c90b)
+
+commit 8f73302b4e637ca8b85d68ea7503279faecb57d8
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Jul 23 17:34:25 2013 -0700
+
+    test/filestore/store_test: add test for 5723
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 37a4c4af54879512429bb114285bcb4c7c3488d5)
+
+    Conflicts:
+	src/os/LFNIndex.cc
+	src/test/filestore/store_test.cc
+
+commit 6a7b9e5f0c1d2344209c69ab9992f94221a16468
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Jul 23 13:51:26 2013 -0700
+
+    FileStore::_collection_rename: fix global replay guard
+
+    If the replay is being replayed, we might have already
+    performed the rename, skip it.  Also, we must set the
+    collection replay guard only after we have done the
+    rename.
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 870c474c5348831fcb13797d164f49682918fb30)
+
+commit 7d98651775265896c22bacfc4afcfccbb0128470
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Jul 22 13:46:10 2013 -0700
+
+    PGLog::rewind_divergent_log: unindex only works from tail, index() instead
+
+    Fixes: #5714
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6957dbc75cc2577652b542aa3eae69f03060cb63)
+
+    The original patch covered the same code in PGLog.cc.
+
+    Conflicts:
+
+	src/osd/PGLog.cc
+	src/osd/PG.cc
+
+commit 611a06ae6c9cba468db206dfc82ec883c7a394af
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 18 09:55:43 2013 -0700
+
+    msg/Pipe: do not hold pipe_lock for verify_authorizer()
+
+    We shouldn't hold the pipe_lock while doing the ms_verify_authorizer
+    upcalls.
+
+    Fix by unlocking a bit earlier, and verifying our state is still correct
+    in the failure path.
+
+    This regression was introduced by ecab4bb9513385bd765cca23e4e2fadb7ac4bac2.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 723d691f7a1f53888618dfc311868d1988f61f56)
+
+    Conflicts:
+
+	src/msg/Pipe.cc
+
+commit 45bda482fa8a23f4b80d115e29d6f04cb5e226d6
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 16 14:17:05 2013 -0700
+
+    msg/Pipe: a bit of additional debug output
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 16568d9e1fb8ac0c06ebaa1e1dc1d6a432a5e4d4)
+
+commit 806eab59ad1a32aedb662c51de3b4a1d61fcbb62
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 16 13:13:46 2013 -0700
+
+    msg/Pipe: hold pipe_lock during important parts of accept()
+
+    Previously we did not bother with locking for accept() because we were
+    not visible to any other threads.  However, we need to close accepting
+    Pipes from mark_down_all(), which means we need to handle interference.
+
+    Fix up the locking so that we hold pipe_lock when looking at Pipe state
+    and verify that we are still in the ACCEPTING state any time we retake
+    the lock.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ecab4bb9513385bd765cca23e4e2fadb7ac4bac2)
+
+commit ce6a0b74459996f91a0511a4a7147179bcd47876
+Author: Greg Farnum <greg@inktank.com>
+Date:   Wed Jul 17 15:23:12 2013 -0700
+
+    msgr: fix a typo/goto-cross from dd4addef2d
+
+    We didn't build or review carefully enough!
+
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1a84411209b13084b3edb87897d5d678937e3299)
+
+commit 1ed51ad535612d5c444a3cc35a331f5e6a68ce30
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 15 17:16:23 2013 -0700
+
+    msgr: close accepting_pipes from mark_down_all()
+
+    We need to catch these pipes too, particularly when doing a rebind(),
+    to avoid them leaking through.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 687fe888b32ac9d41595348dfc82111c8dbf2fcb)
+
+commit 2f696f17a413015a3038d5aa76d18fe94f503f03
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 15 17:14:25 2013 -0700
+
+    msgr: maintain list of accepting pipes
+
+    New pipes exist in a sort of limbo before we know who the peer is and
+    add them to rank_pipe.  Keep a list of them in accepting_pipes for that
+    period.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit dd4addef2d5b457cc9a58782fe42af6b13c68b81)
+
+commit 540a6f49d402c1990f0e0fe9f8897dd664e79501
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 16 16:25:28 2013 -0700
+
+    msgr: adjust nonce on rebind()
+
+    We can have a situation where:
+
+     - we have a pipe to a peer
+     - pipe goes to standby (on peer)
+     - we rebind to a new port
+     - ....
+     - we rebind again to the same old port
+     - we connect to peer
+
+    and get reattached to the ancient pipe from two instances back.  Avoid that
+    by picking a new nonce each time we rebind.
+
+    Add 1,000,000 each time so that the port is still legible in the printed
+    output.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 994e2bf224ab7b7d5b832485ee14de05354d2ddf)
+
+    Conflicts:
+
+	src/msg/Accepter.cc
+
+commit f938a5bf604885ffba65a9b86e19258ca254e58c
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 15 17:10:23 2013 -0700
+
+    msgr: mark_down_all() after, not before, rebind
+
+    If we are shutting down all old connections and binding to new ports,
+    we want to avoid a sequence like:
+
+     - close all prevoius connections
+     - new connection comes in on old port
+     - rebind to new ports
+     -> connection from old port leaks through
+
+    As a first step, close all connections after we shut down the old
+    accepter and before we start the new one.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 07a0860a1899c7353bb506e33de72fdd22b857dd)
+
+    Conflicts:
+
+	src/msg/SimpleMessenger.cc
+
+commit 07b9ebf4212d53606ce332ff927a2ff68ed26978
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 16 13:01:18 2013 -0700
+
+    msg/Pipe: unlock msgr->lock earlier in accept()
+
+    Small cleanup.  Nothing needs msgr->lock for the previously larger
+    window.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ad548e72fd94b4a16717abd3b3f1d1be4a3476cf)
+
+commit ae85a0a101d624363fe761c06ecd52d3d38ba4a2
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 16 10:09:02 2013 -0700
+
+    msg/Pipe: avoid creating empty out_q entry
+
+    We need to maintain the invariant that all sub queues in out_q are never
+    empty.  Fix discard_requeued_up_to() to avoid creating an entry unless we
+    know it is already present.
+
+    This bug leads to an incorrect reconnect attempt when
+
+     - we accept a pipe (lossless peer)
+     - they send some stuff, maybe
+     - fault
+     - we initiate reconnect, even tho we have nothing queued
+
+    In particular, we shouldn't reconnect because we aren't checking for
+    resets, and the fact that our out_seq is 0 while the peer's might be
+    something else entirely will trigger asserts later.
+
+    This fixes at least one source of #5626, and possibly #5517.
+
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 9f1c27261811733f40acf759a72958c3689c8516)
+
+commit 21e27262edc6f5f090ea8915517ee867e30b9066
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 15 14:47:05 2013 -0700
+
+    msg/Pipe: assert lock is held in various helpers
+
+    These all require that we hold pipe_lock.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 579d858aabbe5df88543d096ef4dbddcfc023cca)
+
+commit 25f4786ac41869b3f135bd072000634765bb8919
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jul 14 08:55:52 2013 -0700
+
+    msg/Pipe: be a bit more explicit about encoding outgoing messages
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4282971d47b90484e681ff1a71ae29569dbd1d32)
+
+commit 48105a32605aa59b6970eb89fce4ecc4201e8d04
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 12 16:21:24 2013 -0700
+
+    msg/Pipe: fix RECONNECT_SEQ behavior
+
+    Calling handle_ack() here has no effect because we have already
+    spliced sent messages back into our out queue.  Instead, pull them out
+    of there and discard.  Add a few assertions along the way.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 495ee108dbb39d63e44cd3d4938a6ec7d11b12e3)
+
+commit 1eab069017ce6b71e4bc2bb9679dbe31b50ae938
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 17 13:32:38 2013 -0700
+
+    msgr: reaper: make sure pipe has been cleared (under pipe_lock)
+
+    All paths to pipe shutdown should have cleared the con->pipe reference
+    already.  Assert as much.
+
+    Also, do it under pipe_lock!
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 9586305a2317c7d6bbf31c9cf5b67dc93ccab50d)
+
+commit db06a5092bc45d0479fe492a5d592713a7c53494
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 17 14:14:02 2013 -0700
+
+    msg/Pipe: goto fail_unlocked on early failures in accept()
+
+    Instead of duplicating an incomplete cleanup sequence (that does not
+    clear_pipe()), goto fail_unlocked and do the cleanup in a generic way.
+    s/rc/r/ while we are here.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ec612a5bda119cea52bbac9b2a49ecf1e83b08e5)
+
+commit 8612e50fd70bfceebd6c291e6cab10d9dfd39e8c
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 17 13:32:07 2013 -0700
+
+    msgr: clear con->pipe inside pipe_lock on mark_down
+
+    We need to do this under protection of the pipe_lock.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit afafb87e8402242d3897069f4b94ba46ffe0c413)
+
+commit 8aafe131acadc22cb069f3d98bba6922ab09c749
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 17 12:47:11 2013 -0700
+
+    msgr: clear_pipe inside pipe_lock on mark_down_all
+
+    Observed a segfault in rebind -> mark_down_all -> clear_pipe -> put that
+    may have been due to a racing thread clearing the connection_state pointer.
+    Do the clear_pipe() call under the protection of pipe_lock, as we do in
+    all other contexts.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5fc1dabfb3b2cbffdee3214d24d7769d6e440e45)
+
+    Conflicts:
+
+	src/msg/SimpleMessenger.cc
+
+commit 2f7979d1262e9d4899be76963a1620db46b334e8
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Jul 18 19:26:02 2013 -0700
+
+    ReplicatedPG: track temp collection contents, clear during on_change
+
+    We also assert in on_flushed() that the temp collection is actually
+    empty.
+
+    Fixes: #5670
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 47516d9c4b7f023f3a16e166749fa7b1c7b3b24c)
+
+    Conflicts:
+
+	src/osd/ReplicatedPG.cc
+
+commit c7e2945a42541f966017180684dd969389eef3ac
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Jul 18 19:25:14 2013 -0700
+
+    PG, ReplicatedPG: pass a transaction down to ReplicatedPG::on_change
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 9f56a7b8bfcb63cb4fbbc0c9b8ff01de9e518c57)
+
+commit 7ffc65fc4d7d842954cf791c016fd2711f644a9c
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jul 17 15:04:10 2013 -0700
+
+    PG: start flush on primary only after we process the master log
+
+    Once we start serving reads, stray objects must have already
+    been removed.  Therefore, we have to flush all operations
+    up to the transaction writing out the authoritative log.
+    On replicas, we flush in Stray() if we will not eventually
+    be activated and in ReplicaActive if we are in the acting
+    set.  This way a replica won't serve a replica read until
+    the store is consistent.
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b41f1ba48563d1d3fd17c2f62d10103b5d63f305)
+
+commit 850da0890da5df7e670df9268afe420d0c906c38
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Jul 17 12:51:19 2013 -0700
+
+    ReplicatedPG: replace clean_up_local with a debug check
+
+    Stray objects should have been cleaned up in the merge_log
+    transactions.  Only on the primary have those operations
+    necessarily been flushed at activate().
+
+    Fixes: 5084
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 278c7b59228f614addf830cb0afff4988c9bc8cb)
+
+commit 95b1b5da439f1b7e2fb1886aaeec2d61532183f0
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Jul 18 10:12:17 2013 -0700
+
+    FileStore: add global replay guard for split, collection_rename
+
+    In the event of a split or collection rename, we need to ensure that
+    we don't replay any operations on objects within those collections
+    prior to that point.  Thus, we mark a global replay guard on the
+    collection after doing a syncfs and make sure to check that in
+    _check_replay_guard() for all object operations.
+
+    Fixes: #5154
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f3f92fe21061e21c8b259df5ef283a61782a44db)
+
+    Conflicts:
+
+	src/os/FileStore.cc
+
+commit d92a43d8ff0123b234e47a94c2ce73fcaae7f625
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Jul 15 13:44:20 2013 -0700
+
+    OSD: add config option for peering_wq batch size
+
+    Large peering_wq batch sizes may excessively delay
+    peering messages resulting in unreasonably long
+    peering.  This may speed up peering.
+
+    Backport: cuttlefish
+    Related: #5084
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 39e5a2a406b77fa82e9a78c267b679d49927e3c3)
diff --git a/doc/changelog/v0.61.9.txt b/doc/changelog/v0.61.9.txt
new file mode 100644
index 000000000..fe2a7e733
--- /dev/null
+++ b/doc/changelog/v0.61.9.txt
@@ -0,0 +1,571 @@
+commit 7440dcd135750839fa0f00263f80722ff6f51e90
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Wed Oct 16 18:57:51 2013 +0000
+
+    v0.61.9
+
+commit fcf5f117a9111c2d88b8fa5d00c975a8e377df7e
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Oct 15 10:20:48 2013 -0700
+
+    rgw: fix authenticated users acl group check
+    
+    Fixes: #6553
+    Backport: bobtail, cuttlefish, dumpling
+    Authenticated users group acl bit was not working correctly. Check to
+    test whether user is anonymous was wrong.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit bebbd6cb7b71697b34b8f27652cabdc40c97a33b)
+
+commit 991ed515480114c476cd3c4d761f256d1708fb39
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Oct 15 10:55:07 2013 -0700
+
+    rgw: change default log level
+    
+    Fixes: #6554
+    Backport: cuttlefish, dumpling
+    Default log level was just too high, bring it down a bit.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 8d7dbf85472cfca9268d81ecf057ea078cf345b3)
+
+commit ebb9b0cb7e4ab60fdbbc410ecfb35e51cf11434d
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jul 6 09:21:47 2013 -0700
+
+    mds: do not allow GLAZYIO in mix->sync state
+    
+    GLAZYIO is not allowed in SYNC, so we cannot allow it in the preceding
+    gather state.
+    
+    I verified the other GLAZYIO rules look ok.  We should make a validater
+    to confirm that no gather state includes caps that its target state
+    does not... or at least assert as much in eval_gather().
+    
+    Backport: cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b88938e5a646fbf175a7135e872bcb2d1afafbb8)
+
+commit 33da08f683d40f33061cefa0cf145f3ff21ea089
+Author: Yan, Zheng <zheng.z.yan@intel.com>
+Date:   Thu Sep 12 10:36:39 2013 +0800
+
+    osdc/ObjectCacher: finish contexts after dropping object reference
+    
+    The context to finish can be class C_Client_PutInode, which may drop
+    inode's last reference. So we should first drop object's reference,
+    then finish contexts.
+    
+    Signed-off-by: Yan, Zheng <zheng.z.yan@intel.com>
+    (cherry picked from commit b66ac77fa7aa3ff37804918c4308a348f239af09)
+
+commit 346b43d80f728e6b389208ccd8054d96b76b093c
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 7 22:04:09 2013 -0700
+
+    mds: fix filelock eval_gather
+    
+    Broken by a08d62045657713bf0a5372bf14136082ec3b17e
+    
+    Reported-by: Yan, Zheng <yan.zheng@intel.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e8300d0afb5154d4d13536abdcf47bd5cc8ce810)
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit ffdc7fce132b3b98463b4222d2c51ccef6b94d82
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 6 21:38:56 2013 -0700
+
+    mds: do not double-queue file recovery in eval_gather
+    
+    This fixes a specific case of double-queuing seen in #4832:
+    
+     - client goes stale, inode marked NEEDSRECOVER
+     - eval does sync, queued, -> RECOVERING
+     - client resumes
+     - client goes stale (again), inode marked NEEDSRECOVER
+     - eval_gather queues *again*
+    
+    Note that a cursory look at the recovery code makes me think this needs
+    a much more serious overhaul.  In particular, I don't think we should
+    be triggering recovery when transitioning *from* a stable state, but
+    explicitly when we are flagged, or when gathering.  We should probably
+    also hold a wrlock over the recovery period and remove the force_wrlock
+    kludge from the final size check.  Opened ticket #5268.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a08d62045657713bf0a5372bf14136082ec3b17e)
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit 60033c31381d36cbbc6c873d7055cbe735f5deb2
+Author: Sandon Van Ness <sandon@inktank.com>
+Date:   Tue Oct 8 11:58:57 2013 -0700
+
+    Go back to $PWD in fsstress.sh if compiling from source.
+    
+    Although fsstress was being called with a static path the directory
+    it was writing to was in the current directory so doing a cd to the
+    source directory that is made in /tmp and then removing it later
+    caused it to be unable to write the files in a non-existent dir.
+    
+    This change gets the current path first and cd's back into it after
+    it is done compiling fsstress.
+    
+    Issue #6479.
+    
+    Signed-off-by: Sandon Van Ness <sandon@inktank.com>
+    Reviewed-by: Alfredo Deza <alfredo.deza@inktank.com>
+
+commit eb06f3738851d27914704821897ed80104c4c29c
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Tue Aug 27 09:53:12 2013 -0700
+
+    ceph.spec.in:  radosgw package doesn't require mod_fcgi
+    
+    Fixes #5702
+    
+    Signed-off-by: Gary Lowell  <gary.lowell@inktank.com>
+
+commit 5a426a1f1f34d3f5a510009cc3f3b219d3cbc74b
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Oct 1 15:53:42 2013 -0700
+
+    crush: invalidate rmap on create (and thus decode)
+    
+    If we have an existing CrushWrapper object and decode from a bufferlist,
+    reset build_rmaps so that they get rebuilt.
+    
+    Remove the build_rmaps() all in decode that was useless on a redecode
+    (because have_rmaps == true in that case and it did nothing).
+    
+    Fixes: #6442
+    Backport: dumpling, maybe cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 9b7a2ae329b6a511064dd3d6e549ba61f52cfd21)
+
+commit 6f342872cdd211e24deb19f5e00380494514c437
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Sep 24 19:04:23 2013 +0200
+
+    osd: change warn_interval_multiplier to uint32_t
+    
+    to prevent overflow in OpTracker::check_ops_in_flight when
+    multiplying warn_interval_multiplier *= 2
+    
+    Backport: cuttlefish, dumpling
+    
+    http://tracker.ceph.com/issues/6370 fixes #6370
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 1bce1f009bffd3e28025a08775fec189907a81db)
+
+commit be2907ef85a31c2be8be7446fe71f5d2e1410ec0
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Sep 11 22:30:12 2013 -0700
+
+    rgw: don't call list::size() in ObjectCache
+    
+    Fixes: #6286
+    Use an external counter instead of calling list::size()
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 31e3a51e933429d286104fe077e98ea883437ad6)
+
+commit bbfbb097e2f9efbf4f7ec997c70befa20c79d27c
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Sep 10 12:18:55 2013 -0700
+
+    rgw: drain pending requests before completing write
+    
+    Fixes: #6268
+    When doing aio write of objects (either regular or multipart parts) we
+    need to drain pending aio requests. Otherwise if gateway goes down then
+    object might end up corrupted.
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit b16f812362ccb1d9bdd4900d155e248d695ef0d7
+Merge: 97a97c3 5f16ea6
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Sep 7 13:32:40 2013 -0700
+
+    Merge pull request #573 from dalgaaf/fix/da-cuttlefish-fixes-and-cherry-picks
+    
+    Cherry-pick some smaller changes from master to cuttlefish
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit 5f16ea62cee4fad9be6e44f3562da31908303ae5
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Sat Sep 7 20:32:40 2013 +0200
+
+    tools/ceph.cc: add missig 'ceph osd lspools' command to help
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+
+commit 59f02ecf0b91a2248d8b7b75dc27b517f04ac292
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Sat Sep 7 11:30:15 2013 +0200
+
+    init-radosgw*: fix status return value if radosgw isn't running
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit b5137baf651eaaa9f67e3864509e437f9d5c3d5a)
+
+commit c25770c39ae006ab4ad14a5d75bf7a2dffe0279e
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Thu Jun 6 15:34:54 2013 +0200
+
+    init-radosgw*: add all sections to usage output
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit a0c5095be3640e98d5541920c19387bf3764a350)
+
+commit 1a8347e0d1cafc38259adc1f1a6154fa0d48f1d2
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Thu Jun 6 15:33:23 2013 +0200
+
+    init-radosgw*: add status
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 385457f8d871238a896229d0c2cbb25646969f6a)
+
+commit b1c2aa2c4a8c0266a01903eab5539e7929ea0431
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Thu Jun 6 15:21:30 2013 +0200
+
+    fix init-radosgw* to use the same indentation
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit b4d4e92ed2deae435a24b36d086c1a73e5997855)
+
+commit 794ed1faec7ced23b5b46d114f5320d718c9e9fb
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Sun Jul 28 23:25:58 2013 +0200
+
+    ceph_authtool.cc: update help/usage text
+    
+    Added implemented but not listed commands to the help/usage text:
+    * -g  shortcut for --gen-key
+    * -a  shortcut for --add-key
+    * -u/--set-uid to set auid
+    * --gen-print-key
+    * --import-keyring
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 9a9a0ef3f9f39909eaeb95eb99db4711a2425af5)
+
+commit 97a97c3c554f689dd3f987e63eaa2b9c5ec1dd0a
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Aug 26 19:46:43 2013 -0700
+
+    rgw: check object name after rebuilding it in S3 POST
+    
+    Fixes: #6088
+    Backport: bobtail, cuttlefish, dumpling
+    
+    When posting an object it is possible to provide a key
+    name that refers to the original filename, however we
+    need to verify that in the end we don't end up with an
+    empty object name.
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit c8ec532fadc0df36e4b265fe20a2ff3e35319744)
+
+commit 7a0bd5bc2c6e5464f70b19154834448ac1e4c369
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Thu Aug 22 13:29:32 2013 -0700
+
+    ceph.spec.in:  remove trailing paren in previous commit
+    
+    Signed-off-by: Gary Lowell  <gary.lowell@inktank.com>
+
+commit f1507d23707e7929f7a55fe2ea9418dcc715d8b2
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Thu Aug 22 11:07:16 2013 -0700
+
+    ceph.spec.in:  Don't invoke debug_package macro on centos.
+    
+    If the redhat-rpm-config package is installed, the debuginfo rpms will
+    be built by default.   The build will fail when the package installed
+    and the specfile also invokes the macro.
+    
+    Signed-off-by: Gary Lowell  <gary.lowell@inktank.com>
+
+commit 65a10862feec199d14f17627d0c42fa7c85766fa
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jul 28 08:59:21 2013 -0700
+
+    osd: get initial full map after a map gap
+    
+    If there is a gap in our map history, get the full range of maps that
+    the mon has.  Make sure the first one is a full map.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit a6cd9fea50a4bd7048a222617a2bfe0680f7a969)
+
+commit aceef04f7fd56935e691c7deb05f25ace653bb76
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jul 28 08:55:38 2013 -0700
+
+    osd: fix off-by-one in map gap logic
+    
+    If we have map 250, and monitor's first is 251, but sends 260, we can
+    request the intervening range.
+    
+    Fixes: #5784
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit e24b50225c841a650d9303041bbe811e04bdd668)
+
+commit cdbfd66249cdf91c02a88af5df5a6517688a78df
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Jul 22 16:00:07 2013 -0700
+
+    OSD: tolerate holes in stored maps
+    
+    We may have holes in stored maps during init_splits_between
+    and advance_pg.  In either case, we should simply skip the
+    missing maps.
+    
+    Fixes: #5677
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6951d2345a5d837c3b14103bd4d8f5ee4407c937)
+    
+    Conflicts:
+    
+    	src/osd/OSD.cc
+
+commit 234d68c68028fcf9c2665cb9f45b9b76556241ba
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 20 22:39:09 2013 -0700
+
+    ceph-disk: partprobe after creating journal partition
+    
+    At least one user reports that a partprobe is needed after creating the
+    journal partition.  It is not clear why sgdisk is not doing it, but this
+    fixes ceph-disk for them, and should be harmless for other users.
+    
+    Fixes: #5599
+    Tested-by: lurbs in #ceph
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2af59d5e81c5e3e3d7cfc50d9330d7364659c5eb)
+
+commit cf2f31ac23b6eb43a81a1c8157907b9cae4d58a7
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Aug 15 21:48:06 2013 -0700
+
+    osdc/ObjectCacher: do not merge rx buffers
+    
+    We do not try to merge rx buffers currently.  Make that explicit and
+    documented in the code that it is not supported.  (Otherwise the
+    last_read_tid values will get lost and read results won't get applied
+    to the cache properly.)
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1c50c446152ab0e571ae5508edb4ad7c7614c310)
+
+commit 02da55757a9fb53df4746db5dd14724e77da95b6
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Aug 15 21:47:18 2013 -0700
+
+    osdc/ObjectCacher: match reads with their original rx buffers
+    
+    Consider a sequence like:
+    
+     1- start read on 100~200
+           100~200 state rx
+     2- truncate to 200
+           100~100 state rx
+     3- start read on 200~200
+           100~100 state rx
+           200~200 state rx
+     4- get 100~200 read result
+    
+    Currently this makes us crash on
+    
+    osdc/ObjectCacher.cc: 738: FAILED assert(bh->length() <= start+(loff_t)length-opos)
+    
+    when processing the second 200~200 bufferhead (it is too big).  The
+    larger issue, though, is that we should not be looking at this data at
+    all; it has been truncated away.
+    
+    Fix this by marking each rx buffer with the read request that is sent to
+    fill it, and only fill it from that read request.  Then the first reply
+    will fill the first 100~100 extend but not touch the other extent; the
+    second read will do that.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b59f930ae147767eb4c9ff18c3821f6936a83227)
+
+commit 43e7ad989dcb4deb18b32ec31f76c8755354d2a6
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Aug 22 15:54:48 2013 -0700
+
+    mon/Paxos: fix another uncommitted value corner case
+    
+    It is possible that we begin the paxos recovery with an uncommitted
+    value for, say, commit 100.  During last/collect we discover 100 has been
+    committed already.  But also, another node provides an uncommitted value
+    for 101 with the same pn.  Currently, we refuse to learn it, because the
+    pn is not strictly > than our current uncommitted pn... even though it is
+    the next last_committed+1 value that we need.
+    
+    There are two possible fixes here:
+    
+     - make this a >= as we can accept newer values from the same pn.
+     - discard our uncommitted value metadata when we commit the value.
+    
+    Let's do both!
+    
+    Fixes: #6090
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit fe5010380a3a18ca85f39403e8032de1dddbe905)
+
+commit 2de1515289f49f2e388448506f4788db56d0e25a
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 23 11:45:35 2013 -0700
+
+    os: make readdir_r buffers larger
+    
+    PATH_MAX isn't quite big enough.
+    
+    Backport: dumpling, cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 99a2ff7da99f8cf70976f05d4fe7aa28dd7afae5)
+
+commit af9818c486484c7617c07f26beaded8a3bc88043
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 23 11:45:08 2013 -0700
+
+    os: fix readdir_r buffer size
+    
+    The buffer needs to be big or else we're walk all over the stack.
+    
+    Backport: dumpling, cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2df66d9fa214e90eb5141df4d5755b57e8ba9413)
+    
+    Conflicts:
+    
+    	src/os/BtrfsFileStoreBackend.cc
+
+commit cce1d1f9cd8b034deee29d8566780763beb0155f
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Fri Aug 23 08:56:07 2013 -0400
+
+    ceph-disk: specify the filetype when mounting
+    
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f040020fb2a7801ebbed23439159755ff8a3edbd)
+
+commit c25e7da57d704d4a8db59a2e97fb687968520c69
+Author: Sandon Van Ness <sandon@inktank.com>
+Date:   Thu Aug 22 19:44:40 2013 -0700
+
+    QA: Compile fsstress if missing on machine.
+    
+    Some distro's have a lack of ltp-kernel packages and all we need is
+    fstress. This just modified the shell script to download/compile
+    fstress from source and copy it to the right location if it doesn't
+    currently exist where it is expected. It is a very small/quick
+    compile and currently only SLES and debian do not have it already.
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Signed-off-by: Sandon Van Ness <sandon@inktank.com>
+
+commit c807f27c391d336a7223fcfdd3daad9bb374a3dc
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Aug 5 12:52:44 2013 -0700
+
+    mds: fix locking, use-after-free/race in handle_accept
+    
+    We need to hold mds_lock here.
+    
+    Normally the con also holds a reference, but an ill-timed connection reset
+    could drop it.
+    
+    Fixes: #5883
+    Backport: dumpling, cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a0929955cb84fb8cfdeb551d6863e4955b8e2a71)
+
+commit bd71192eaa6f884e879b1711e5937b1e3609d86d
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Aug 22 10:14:59 2013 -0700
+
+    .gitignore: ignore test-driver
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit edf2c3449ec96d91d3d7ad01c50f7a79b7b2f7cc)
+    
+    Conflicts:
+    
+    	.gitignore
+
+commit bc997ebea3263c2bc7df83661ae3a966470ba35e
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 9 12:42:49 2013 -0700
+
+    fuse: fix warning when compiled against old fuse versions
+    
+    client/fuse_ll.cc: In function 'void invalidate_cb(void*, vinodeno_t, int64_t, int64_t)':
+    warning: client/fuse_ll.cc:540: unused variable 'fino'
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 9833e9dabe010e538cb98c51d79b6df58ce28f9e)
+
+commit 9cb2c2eb3627b52c3413b39b45e7fb7e0e9a074c
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 9 12:40:34 2013 -0700
+
+    json_spirit: remove unused typedef
+    
+    In file included from json_spirit/json_spirit_writer.cpp:7:0:
+    json_spirit/json_spirit_writer_template.h: In function 'String_type json_spirit::non_printable_to_string(unsigned int)':
+    json_spirit/json_spirit_writer_template.h:37:50: warning: typedef 'Char_type' locally defined but not used [-Wunused-local-typedefs]
+             typedef typename String_type::value_type Char_type;
+    
+    (Also, ha ha, this file uses \r\n.)
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6abae35a3952e5b513895267711fea63ff3bad09)
+
+commit d774559f118d26cd15ecf1a49468ce1a3d260efc
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 9 12:31:41 2013 -0700
+
+    gtest: add build-aux/test-driver to .gitignore
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c9cdd19d1cd88b84e8a867f5ab85cb51fdc6f8e4)
+
+commit 1a2d9edde0311b51d3d68b87c20dea3061b2395b
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Aug 21 14:28:49 2013 -0700
+
+    objecter: resend unfinished lingers when osdmap is no longer paused
+    
+    Plain Ops that haven't finished yet need to be resent if the osdmap
+    transitions from full or paused to unpaused.  If these Ops are
+    triggered by LingerOps, they will be cancelled instead (since
+    should_resend = false), but the LingerOps that triggered them will not
+    be resent.
+    
+    Fix this by checking the registered flag for all linger ops, and
+    resending any of them that aren't paused anymore.
+    
+    Fixes: #6070
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Sage Weil <sage.weil@inktank.com>
+    (cherry picked from commit 38a0ca66a79af4b541e6322467ae3a8a4483cc72)
diff --git a/doc/changelog/v0.67.1.txt b/doc/changelog/v0.67.1.txt
new file mode 100644
index 000000000..9b9405f82
--- /dev/null
+++ b/doc/changelog/v0.67.1.txt
@@ -0,0 +1,142 @@
+commit e23b817ad0cf1ea19c0a7b7c9999b30bed37d533
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Fri Aug 16 19:39:41 2013 -0700
+
+    v0.67.1
+
+commit 1aa01910957e967e87e46b86f1e67844148703e3
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Thu Aug 15 17:10:56 2013 -0700
+
+    ceph.in: --admin-daemon was not returning EINVAL on bad command
+    
+    Fix by restructuring code to hoist common code and have only one
+    place where admin_socket is actually called.
+    
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 266460e97ec9ef9711e9eaa4bd954f3188d8da69)
+
+commit d290a91525c3ebc6941dae2bb55f0bfbf120cb60
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Aug 15 14:37:07 2013 -0700
+
+    mon: use str_join instead of std::copy
+    
+    The std::copy method leaves a trailing separator.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 35565ee64e41d7fddc7849c6006692c78227132c)
+
+commit b99921746ef2e1e15777c748a15e929c72888db1
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Aug 15 14:36:57 2013 -0700
+
+    config: fix stringification of config values
+    
+    The std::copy construct leaves a trailing separator character, which breaks
+    parsing for booleans (among other things) and probably mangles everything
+    else too.
+    
+    Backport: dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit fc23cfe3fe567b30413d8af0c614a32fec238939)
+
+commit d212bba6bd0d7d234097122988e4d973064b5645
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Aug 15 14:36:49 2013 -0700
+
+    common: add str_join helper
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit ce3a0944d9b47f7b178fe7775c9d105305b238e0)
+
+commit a99fef9189086f5dd6ddacaecf967619dc5fe407
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Aug 14 15:50:59 2013 -0700
+
+    rados.py: fix Rados() unicode checking
+    
+    Check new parameters and check that rados_id is not None again to
+    catch the empty string.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Sage Weil <sage.weil@inktank.com>
+    (cherry picked from commit 4422f21a6586467a63ce6841552d0f60aa849cf1)
+
+commit dd0df583e2661444287a36acc240a8ac0ec381e8
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Aug 14 15:28:19 2013 -0700
+
+    rados.py: fix Rados() backwards compatibility
+    
+    Previously it had no name parameter, so the default will be used by
+    old clients. However, if an old client set rados_id, a new check that
+    both rados_id and name are set would result in an error. Fix this by
+    only applying the default names after the check, and add tests of this
+    behavior.
+    
+    This was introduced in 783b7ec847c7f987ac1814c9c41c91921cac4eba,
+    so it does not affect cuttlefish.
+    
+    Fixes: #5970
+    Reported-by: Michael Morgan <mmorgan@dca.net>
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Sage Weil <sage.weil@inktank.com>
+    (cherry picked from commit 34da9cbc33205623cf64aee1989f53dfb2c5bddd)
+
+commit b9d1bf51610159a88ad257f29a81691e6b178e17
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 13 12:52:41 2013 -0700
+
+    librados: fix async aio completion wakeup
+    
+    For aio flush, we register a wait on the most recent write.  The write
+    completion code, however, was *only* waking the waiter if they were waiting
+    on that write, without regard to previous writes (completed or not).
+    For example, we might have 6 and 7 outstanding and wait on 7.  If they
+    finish in order all is well, but if 7 finishes first we do the flush
+    completion early.  Similarly, if we
+    
+     - start 6
+     - start 7
+     - finish 7
+     - flush; wait on 7
+     - finish 6
+    
+    we can hang forever.
+    
+    Fix by doing any completions that are prior to the oldest pending write in
+    the aio write completion handler.
+    
+    Refs: #5919
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Tested-by: Oliver Francke <Oliver.Francke@filoo.de>
+    (cherry picked from commit 16ed0b9af8bc08c7dabead1c1a7c1a22b1fb02fb)
+
+commit 29ae033b435581a64fc92a26106b4ea1c3dfc0b1
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Aug 12 19:17:09 2013 -0700
+
+    librados: fix locking for AioCompletionImpl refcounting
+    
+    Add an already-locked helper so that C_Aio{Safe,Complete} can
+    increment the reference count when their caller holds the
+    lock. C_AioCompleteAndSafe's caller is not holding the lock, so call
+    regular get() to ensure no racing updates can occur.
+    
+    This eliminates all direct manipulations of AioCompletionImpl->ref,
+    and makes the necessary locking clear.
+    
+    The only place C_AioCompleteAndSafe is used is in handling
+    aio_flush_async(). This could cause a missing completion.
+    
+    Refs: #5919
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Tested-by: Oliver Francke <Oliver.Francke@filoo.de>
+    (cherry picked from commit 7a52e2ff5025754f3040eff3fc52d4893cafc389)
diff --git a/doc/changelog/v0.67.10.txt b/doc/changelog/v0.67.10.txt
new file mode 100644
index 000000000..165c8e0bb
--- /dev/null
+++ b/doc/changelog/v0.67.10.txt
@@ -0,0 +1,669 @@
+commit 9d446bd416c52cd785ccf048ca67737ceafcdd7f (tag: refs/tags/v0.67.10)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Mon Aug 11 05:30:10 2014 -0700
+
+    0.67.10
+
+commit a86e497a343b24425a1237e65ec2776398d07a95
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Mon Jul 14 14:27:17 2014 +0800
+
+    Add rbdcache max dirty object option
+    
+    Librbd will calculate max dirty object according to rbd_cache_max_size, it
+    doesn't suitable for every case. If user set image order 24, the calculating
+    result is too small for reality. It will increase the overhead of trim call
+    which is called each read/write op.
+    
+    Now we make it as option for tunning, by default this value is calculated.
+    
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit 3c7229a2fea98b30627878c86b1410c8eef2b5d7)
+
+commit d02381316ba96746a00b2b6f8c5c665179fe7a79
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Jun 4 23:22:18 2014 +0200
+
+    librbd/internal.cc: check earlier for null pointer
+    
+    Fix potential null ponter deref, move check for 'order != NULL'
+    to the beginning of the function to prevent a) deref in ldout() call
+    and b) to leave function as early as possible if check fails.
+    
+    [src/librbd/internal.cc:843] -> [src/librbd/internal.cc:865]: (warning)
+     Possible null pointer dereference: order - otherwise it is redundant
+     to check it against null.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 3ee3e66a9520a5fcafa7d8c632586642f7bdbd29)
+
+commit dda98b9afb5d74c8c5f101b792daca9ca344fe87
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Apr 24 14:47:24 2014 -0700
+
+    librbd: add an interface to invalidate cached data
+    
+    This is useful for qemu to guarantee live migration with caching is
+    safe, by invalidating the cache on the destination before starting it.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 5d340d26dd70192eb0e4f3f240e3433fb9a24154)
+
+commit d93579514db74eed61580a6166c2787304a8a9fb
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Apr 24 14:43:35 2014 -0700
+
+    librbd: check return code and error out if invalidate_cache fails
+    
+    This will only happen when shrinking or rolling back an image is done
+    while other I/O is in flight to the same ImageCtx. This is unsafe, so
+    return an error before performing the resize or rollback.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit e08b8b66c77be3a3d7f79d91c20b1619571149ee)
+
+commit 617914209694f67150264358da888c1ca9ff42e1
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Wed May 21 18:12:22 2014 +0800
+
+    Avoid extra check for clean object
+    
+    We needn't to check clean object via buffer state, skip the clean object.
+    
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit f51e33bd9c5a8e1cfc7065b30785696dc45918bc)
+
+commit d474443b12b978dbe267e63cd8e34e8c835e4f6c
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Fri Mar 28 18:10:43 2014 -0700
+
+    rbd.cc: yes, cover formatted output as well.  sigh.
+    
+    Fixes: #7577
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit bd6e35c1b171e46cc3e026c59b076b73440a8502)
+
+commit 051d1b4e56b37b2293b54d8cdb58a974684c8464
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Tue Mar 25 17:09:48 2014 -0700
+
+    rbd.cc: tolerate lack of NUL-termination on block_name_prefix
+    
+    Fixes: #7577
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit fd76fec589be13a4a6362ef388929d3e3d1d21f6)
+
+commit 61e2219dd07ebb856a61f54e8cd992dc1e16f5d9
+Author: Ilya Dryomov <ilya.dryomov@inktank.com>
+Date:   Wed Jan 29 16:12:01 2014 +0200
+
+    rbd: don't forget to call close_image() if remove_child() fails
+    
+    close_image() among other things unregisters a watcher that's been
+    registered by open_image().  Even though it'll timeout in 30 or so
+    seconds, it's not nice now that we check for watchers before starting
+    the removal process.
+    
+    Signed-off-by: Ilya Dryomov <ilya.dryomov@inktank.com>
+    (cherry picked from commit 4ebc32f37a4860bdc676491bf8b042c18fd619cf)
+
+commit 2c6c23cee7ff2a8f56f3986051d43824b7bd3526
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Aug 9 13:23:06 2014 -0700
+
+    os/FileStore: dump open fds before asserting
+    
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4e8de1792731cf30f2744ab0659d036adc0565a3)
+
+commit 3663233a63df3a1fb1584d9800e1c6a8feac2fe0
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Feb 18 16:43:48 2014 -0800
+
+    rgw: return error if accessing object in non-existent bucket
+    
+    Fixes: #7064
+    Instead of trying to access the object, which is impossible as we don't
+    even have a proper bucket info. Up until now we ended up creating an
+    empty pool and eventually returning ENOENT, this fix catches the issue
+    earlier in the process.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 3ed68eb9fac9b3d0bf9111867d609f9ea08fb202)
+
+commit 0a1cea58641c4584b85165755145d8d0288f4f22
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 6 17:04:02 2014 -0700
+
+    os/FileStore: force any new xattr into omap on E2BIG
+    
+    If we have a huge xattr (or many little ones), the _fgetattrs() for the
+    inline_set will fail with E2BIG.  The conditions later where we decide
+    whether to clean up the old xattr will then also fail.  Will *will* put
+    the xattr in omap, but the non-omap version isn't cleaned up.
+    
+    Fix this by setting a flag if we get E2BIG that the inline_set is known
+    to be incomplete.  In that case, take the conservative step of assuming
+    the xattr might be present and chain_fremovexattr().  Ignore any error
+    because it might not be there.
+    
+    This is clearly harmless in the general case because it won't be there.  If
+    it is, we will hopefully remove enough xattrs that the E2BIG condition
+    will go away (usually by removing some really big chained xattr).
+    
+    See bug #7779.
+    
+    This is a backport of 26750fcfe8d766874513e57981565adde2e6d8c7.
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 64d5c406995bedbb6a4bc9c851f5d25fe94749ee
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri May 23 14:58:54 2014 -0700
+
+    rgw: calc md5 and compare if user provided appropriate header
+    
+    Fixes: #8436
+    Backport: firefly
+    
+    This was broken in ddc2e1a8e39a5c6b9b224c3eebd1c0e762ca5782. The fix
+    resurrects and old check that was dropped.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 9c56c86bdac6bcb8e76c3f04e7d393e4eaadd721)
+
+commit ebecd80d3a082ac87e7cd6a63cbd00259c5a8baf
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Apr 21 15:07:12 2014 -0700
+
+    rgw: calculate user manifest
+    
+    Fixes: #8169
+    Backport: firefly
+    We didn't calculate the user manifest's object etag at all. The etag
+    needs to be the md5 of the contantenation of all the parts' etags.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit ddc2e1a8e39a5c6b9b224c3eebd1c0e762ca5782)
+    
+    Conflicts:
+    	src/rgw/rgw_op.cc
+
+commit 1f3f2982224f68109cce8b5564d99977b22c8f60
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Jul 29 15:25:47 2014 -0700
+
+    rgw: fix crash in swift CORS preflight request
+    
+    Fixes: #8586
+    
+    This fixes error handling, in accordance with commit 6af5a537 that fixed
+    the same issue for the S3 case.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 18ea2a869791b4894f93fdafde140285f2e4fb65)
+
+commit 6e4921500aeb256da0fa20da6ee47fb7d4e171f6
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jul 30 11:53:16 2014 -0700
+
+    cls_rgw: fix object name of objects removed on object creation
+    
+    Fixes: #8972
+    Backport: firefly, dumpling
+    
+    Reported-by: Patrycja Szabłowska <szablowska.patrycja@gmail.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 0f8929a68aed9bc3e50cf15765143a9c55826cd2)
+
+commit 4aa2b87385e94be0147401918c332d526029e481
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 2 10:38:43 2014 -0700
+
+    qa/workunits/rest/test.py: make osd create test idempotent
+    
+    Avoid possibility that we create multiple OSDs do to retries by passing in
+    the optional uuid arg.  (A stray osd id will make the osd tell tests a
+    few lines down fail.)
+    
+    Fixes: #8728
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit bb3e1c92b6682ed39968dc5085b69c117f43cbb0)
+
+commit 935e75a850b3363536cc09b68f45f2a497da4bb4
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Thu Apr 10 15:14:19 2014 +0100
+
+    mon: Monitor: suicide on start if mon has been removed from monmap
+    
+    If the monitor has been marked as having been part of an existing quorum
+    and is no longer in the monmap, then it is safe to assume the monitor
+    was removed from the monmap.  In that event, do not allow the monitor
+    to start, as it will try to find its way into the quorum again (and
+    someone clearly stated they don't really want them there), unless
+    'mon force quorum join' is specified.
+    
+    Fixes: 6789
+    Backport: dumpling, emperor
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 86b85947a2148c2e73886c1a7edd09093966ada0)
+    
+    Conflicts:
+    	src/common/config_opts.h
+
+commit 4aca6c0d7540c2bf5fb54df2d70c4649a9b94100
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Jul 16 16:05:58 2014 -0700
+
+    utf8: export encode_utf8() and decode_utf8()
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 49fc68cf8c3122c878ea9503c9c74d7046bc9c6f)
+
+commit ffa1f5200061c48de6315b5e7bd335ab5f718a6f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jul 18 14:52:48 2014 -0700
+
+    rgw: dump prefix unconditionally
+    
+    As part of issue #8858, and to be more in line with S3, dump the Prefix
+    field when listing bucket even if bucket is empty.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d7209c11251d42227608bc54cc69232ef62ffe80)
+    
+    Conflicts:
+    	src/rgw/rgw_rest_s3.cc
+
+commit 6df17a92283da9933bf5a3fca1278b728c79b8ee
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jul 17 15:48:26 2014 -0700
+
+    rgw: list extra objects to set truncation flag correctly
+    
+    Otherwise we end up returning wrong truncated value, and no data on the
+    next iteration.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit dc417e477d4ad262885c6b5f5987cf06d63b159d)
+
+commit 77c20879d02c9793241da508b5ad59f8b927e9f7
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jul 17 11:45:44 2014 -0700
+
+    rgw: account common prefixes for MaxKeys in bucket listing
+    
+    To be more in line with the S3 api. Beforehand we didn't account the
+    common prefixes towards the MaxKeys (a single common prefix counts as a
+    single key). Also need to adjust the marker now if it is pointing at a
+    common prefix.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 82d2d612e700f94a0bb2d9fb7555abf327be379b)
+
+commit 7068d8595ff462323dedd9b56bce28a76405585a
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jul 17 11:24:51 2014 -0700
+
+    rgw: add NextMarker param for bucket listing
+    
+    Partially fixes #8858.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 924686f0b6593deffcd1d4e80ab06b1e7af00dcb)
+
+commit 9604425b86f5839a109faa1f396b0d114e9b9391
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Jul 16 15:21:09 2014 -0700
+
+    rgw: improve delmited listing of bucket
+    
+    If found a prefix, calculate a string greater than that so that next
+    request we can skip to that. This is still not the most efficient way to
+    do it. It'll be better to push it down to the objclass, but that'll
+    require a much bigger change.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit e6cf618c257f26f97f60a4c1df1d23a14496cab0)
+
+commit b287c677ba1a5573e66c8a287f6a1f420c3ba46f
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Jul 16 12:23:31 2014 -0700
+
+    rgw: don't try to wait for pending if list is empty
+    
+    Fixes: #8846
+    Backport: firefly, dumpling
+    
+    This was broken at ea68b9372319fd0bab40856db26528d36359102e. We ended
+    up calling wait_pending_front() when pending list was empty.
+    This commit also moves the need_to_wait check to a different place,
+    where we actually throttle (and not just drain completed IOs).
+    
+    Reported-by: Sylvain Munaut <s.munaut@whatever-company.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit f9f2417d7db01ecf2425039539997901615816a9)
+
+commit 4b13caf5be33d4644811d387d1350f2f11ac5409
+Author: Warren Usui <warren.usui@inktank.com>
+Date:   Thu Apr 24 12:55:26 2014 -0700
+
+    Use new git mirror for qemu-iotests
+    
+    Fixes: 8191
+    Signed-off-by: Warren Usui <warren.usui@inktank.com>
+    (cherry picked from commit ddf37d903f826f3e153d8009c716780453b68b05)
+
+commit 3f251024bf056f5a8c22f29333b3a81589431424
+Author: Warren Usui <warren.usui@inktank.com>
+Date:   Wed Apr 23 13:20:14 2014 -0700
+
+    Support latest qemu iotest code
+    
+    Modified qemu-iotests workunit script to check for versions
+    that use the latest qemu (currently only Trusty).  Limit the
+    tests to those that are applicable to rbd.
+    
+    Fixes: 7882
+    Signed-off-by: Warren Usui <warren.usui@inktank.com>
+    (cherry picked from commit 606e725eb5204e76e602d26ffd113e40af2ee812)
+
+commit d06137841f9911ce4dc0d9ab9be73a131ede879d
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Mar 31 14:53:31 2014 -0700
+
+    librbd: skip zeroes when copying an image
+    
+    This is the simple coarse-grained solution, but it works well in
+    common cases like a small base image resized with a bunch of empty
+    space at the end. Finer-grained sparseness can be copied by using rbd
+    {export,import}-diff.
+    
+    Fixes: #6257
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 824da2029613a6f4b380b6b2f16a0bd0903f7e3c)
+
+commit 8649cbbc96a4de9de169b0203f35e0ac6c36a2ef
+Author: Greg Farnum <greg@inktank.com>
+Date:   Tue Jul 1 15:19:21 2014 -0700
+
+    Revert "qa/workunits/suites/fsx.sh: don't use zero range"
+    
+    This reverts commit 583e6e3ef7f28bf34fe038e8a2391f9325a69adf.
+    
+    We're using a different fsx source, which doesn't support the
+    same options as our git-based one does.
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+
+commit 583e6e3ef7f28bf34fe038e8a2391f9325a69adf
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 30 07:05:04 2014 -0700
+
+    qa/workunits/suites/fsx.sh: don't use zero range
+    
+    Zero range is not supported by cephfs.
+    
+    Fixes: #8542
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2dec8a810060f65d022c06e82090b4aa5ccec0cb)
+
+commit 49c0b46153f96a94407ef2487b2bd9525dfaa1e6
+Merge: 3c63601 4ca12b4
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Jun 30 19:19:24 2014 +0200
+
+    Merge pull request #2014 from ceph/wip-scrub-dumpling
+    
+    osd: scrub priority updates for dumpling
+    
+    Reviewed-by: Loic Dachary <loic@dachary.org>
+
+commit 3c63601888c5c825f33ffc6797474eb8dc6935a4
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Jun 16 11:48:24 2014 -0700
+
+    rgw: allocate enough space for bucket instance id
+    
+    Fixes: #8608
+    Backport: dumpling, firefly
+    Bucket instance id is a concatenation of zone name, rados instance id,
+    and a running counter. We need to allocate enough space to account zone
+    name length.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit d2e86a66ca55685e04ffbfaa58452af59f381277)
+
+commit 15360a900c8746c28006d6f5c8b093685ddac796
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 8 08:52:51 2014 -0700
+
+    ceph-disk: partprobe before settle when preparing dev
+    
+    Two users have reported this fixes a problem with using --dmcrypt.
+    
+    Fixes: #6966
+    Tested-by: Eric Eastman <eric0e@aol.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 0f196265f049d432e399197a3af3f90d2e916275)
+
+commit ef736f530405a2c4f9a9a1b4cc98d1160524e1ab
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 17 13:33:14 2014 -0700
+
+    osd: fix filestore perf stats update
+    
+    Update the struct we are about to send, not the (unlocked!) one we will
+    send the next time around.
+    
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4afffb4a10a0bbf7f2018ef3ed6b167c7921e46b)
+
+commit 4ca12b40206f64d32369a78ce443a0dbf5d2b757
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 18 11:02:09 2014 -0700
+
+    common/WorkQueue: allow io priority to be set for wq
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5e4b3b1f1cb870f39fc7cfb3adeae93e078d9057)
+    
+    Conflicts:
+    	src/common/WorkQueue.cc
+
+commit c9b3067f9b78a65e0b9c2ce2488d8906eb73c869
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 18 11:01:42 2014 -0700
+
+    common/Thread: allow io priority to be set for a Thread
+    
+    Ideally, set this before starting the thread.  If you set it after, we
+    could potentially race with create() itself.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 01533183e7455b713640e001962339907fb6f980)
+
+commit 0796057f1c13c467dd3eadca036e3034da20e500
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 18 11:01:09 2014 -0700
+
+    common/io_priority: wrap ioprio_set() and gettid()
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 705713564bebd84ad31cc91698311cf2fbd51a48)
+    
+    Conflicts:
+    	src/common/Makefile.am
+
+commit c46fe8e9ec54d4decd1f70256a8ab05c9b183f8a
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 17 10:47:24 2014 -0700
+
+    osd: introduce simple sleep during scrub
+    
+    This option is similar to osd_snap_trim_sleep: simply inject an optional
+    sleep in the thread that is doing scrub work.  This is a very kludgey and
+    coarse knob for limiting the impact of scrub on the cluster, but can help
+    until we have a more robust and elegant solution.
+    
+    Only sleep if we are in the NEW_CHUNK state to avoid delaying processing of
+    an in-progress chunk.  In this state nothing is blocked on anything.
+    Conveniently, chunky_scrub() requeues itself for each new chunk.
+    
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c4e8451cc5b4ec5ed07e09c08fb13221e31a7ac6)
+
+commit 3cf01a0e56c5599731e25898c1d9f2d2b3d6a9ba
+Merge: 8fd2fc1 c340085
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 16 09:27:03 2014 -0700
+
+    Merge pull request #1963 from dachary/wip-8599-ruleset-dumpling
+    
+    mon: pool set <pool> crush_ruleset must not use rule_exists (dumpling)
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit c34008532362c3afadff6287bc8746c141a221a3
+Author: John Spray <john.spray@inktank.com>
+Date:   Tue May 20 16:50:18 2014 +0100
+
+    mon: pool set <pool> crush_ruleset must not use rule_exists
+    
+    Implement CrushWrapper::ruleset_exists that iterates over the existing
+    rulesets to find the one matching the ruleset argument.
+    
+    ceph osd pool set <pool> crush_ruleset must not use
+    CrushWrapper::rule_exists, which checks for a *rule* existing, whereas
+    the value being set is a *ruleset*. (cherry picked from commit
+    fb504baed98d57dca8ec141bcc3fd021f99d82b0)
+    
+    A test via ceph osd pool set data crush_ruleset verifies the ruleset
+    argument is accepted.
+    
+    http://tracker.ceph.com/issues/8599 fixes: #8599
+    
+    Backport: firefly, emperor, dumpling
+    Signed-off-by: John Spray <john.spray@inktank.com>
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit d02d46e25080d5f7bb8ddd4874d9019a078b816b)
+    
+    Conflicts:
+    	src/mon/OSDMonitor.cc
+
+commit 8fd2fc1aec4ba08c086a7db2208dbdebfcb4e92e
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Mar 3 07:03:01 2014 -0800
+
+    osd: 'status' admin socket command
+    
+    Basic stuff, like what state is the OSD in, and what osdmap epoch are
+    we on.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 09099c9e4c7d2aa31eb8a0b7c18e43272fae7ce2)
+
+commit 323492451c2d29b90c30a4e133dc8ca9780035a5
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jun 8 20:18:49 2014 -0700
+
+    init-ceph: continue after failure doing osd data mount
+    
+    If we are starting many daemons and hit an error, we normally note it and
+    move on.  Do the same when doing the pre-mount step.
+    
+    Fixes: #8554
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6a7e20147cc39ed4689809ca7d674d3d408f2a17)
+
+commit 558249c519225d5e6d0784da20fe12d8f5d71cd2
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue May 6 11:06:29 2014 -0700
+
+    rgw: cut short object read if a chunk returns error
+    
+    Fixes: #8289
+    Backport: firefly, dumpling
+    When reading an object, if we hit an error when trying to read one of
+    the rados objects then we should just stop. Otherwise we're just going
+    to continue reading the rest of the object, and since it can't be sent
+    back to the client (as we have a hole in the middle), we end up
+    accumulating everything in memory.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 03b0d1cfb7bd30a77fedcf75eb06476b21b14e95)
+
+commit a948ce2f9ccf67d5ccc690b5c326af19b6e6da4d
+Merge: 95a464a 7ad2fbf
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jun 8 21:12:32 2014 -0700
+
+    Merge pull request #1931 from ceph/wip-7068-dumpling
+    
+    Wip 7068 dumpling
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit 95a464a443e3a249889151807065736fff44ea3a
+Merge: a79a787 94e6c08
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Jun 6 08:45:58 2014 -0700
+
+    Merge remote-tracking branch 'origin/wip-8269-dumpling' into dumpling
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit a79a787b924ef7e04efb7f3c13b9559e06bebfd8
+Author: John Wilkins <john.wilkins@inktank.com>
+Date:   Thu Jun 5 11:41:41 2014 -0700
+
+    doc: Added requiretty comment to preflight checklist.
+    
+    Signed-off-by: John Wilkins <john.wilkins@inktank.com>
+
+commit 5c5eb21fd1a12666e1ce79718c8166eb7bdd6748
+Author: John Wilkins <john.wilkins@inktank.com>
+Date:   Thu Jun 5 11:34:46 2014 -0700
+
+    doc: Added Disable requiretty to quick start.
+    
+    Signed-off-by: John Wilkins <john.wilkins@inktank.com>
+
+commit 7ad2fbf3bd3208f47aa2a76d3fd16479e2c845ab
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Oct 2 18:00:04 2013 -0700
+
+    ReplicatedPG: lock snapdir obc during write
+    
+    Otherwise, we won't block properly in prep_push_backfill_object.
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit b87bc2311aa4da065477f402a869e2edc1558e2f)
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.h
+
+commit 94e6c0858fb07c0edb8fc4d08e02542d22823ed2
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri May 2 17:06:05 2014 -0700
+
+    rgw: don't allow multiple writers to same multiobject part
+    
+    Fixes: #8269
+    
+    A client might need to retry a multipart part write. The original thread
+    might race with the new one, trying to clean up after it, clobbering the
+    part's data.
+    The fix is to detect whether an original part already existed, and if so
+    use a different part name for it.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
diff --git a/doc/changelog/v0.67.11.txt b/doc/changelog/v0.67.11.txt
new file mode 100644
index 000000000..6e581ece8
--- /dev/null
+++ b/doc/changelog/v0.67.11.txt
@@ -0,0 +1,215 @@
+commit bc8b67bef6309a32361be76cd11fb56b057ea9d2 (tag: refs/tags/v0.67.11, refs/remotes/gh/dumpling)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Wed Sep 24 06:25:55 2014 -0700
+
+    0.67.11
+
+commit 0e90b04d9452999d7d91305c80585782d492c91b (refs/remotes/gh/dumpling-jni)
+Author: Greg Farnum <greg@inktank.com>
+Date:   Wed May 21 21:41:23 2014 -0700
+
+    cephfs-java: build against older jni headers
+    
+    Older versions of the JNI interface expected non-const parameters
+    to their memory move functions. It's unpleasant, but won't actually
+    change the memory in question, to do a cast_const in order to satisfy
+    those older headers. (And even if it *did* modify the memory, that
+    would be okay given our single user.)
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 4d4b77e5b6b923507ec4a0ad9d5c7018e4542a3c)
+
+commit 3f020443c8d92e61d8593049147a79a6696c9c93
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Jul 24 15:29:40 2014 -0700
+
+    librbd: fix crash using clone of flattened image
+    
+    The crash occurs due to ImageCtx->parent->parent being uninitialized,
+    since the inital open_parent() -> open_image(parent) ->
+    ictx_refresh(parent) occurs before ImageCtx->parent->snap_id is set,
+    so refresh_parent() is not called to open an ImageCtx for the parent
+    of the parent. This leaves the ImageCtx->parent->parent NULL, but the
+    rest of ImageCtx->parent updated to point at the correct parent snapshot.
+    
+    Setting the parent->snap_id earlier has some unintended side effects
+    currently, so for now just call refresh_parent() during
+    open_parent(). This is the easily backportable version of the
+    fix. Further patches can clean up this whole initialization process.
+    
+    Fixes: #8845
+    Backport: firefly, dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 2545e80d274b23b6715f4d8b1f4c6b96182996fb)
+
+commit d3e880af5f3ae71d13159514c33c6b41fc648d54
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 10 08:00:50 2014 -0700
+
+    test/cli-integration/rbd: fix trailing space
+    
+    Newer versions of json.tool remove the trailing ' ' after the comma.  Add
+    it back in with sed so that the .t works on both old and new versions, and
+    so that we don't have to remove the trailing spaces from all of the test
+    cases.
+    
+    Fixes: #8920
+    
+    Backports commit 605064dc685aa25cc7d58ec18b6449a3ce476d01
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 5336398618f49d10e497392ab942d43062ed772d
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Aug 16 12:42:33 2014 -0700
+
+    os/FileStore: fix mount/remount force_sync race
+    
+    Consider:
+    
+     - mount
+     - sync_entry is doing some work
+     - umount
+       - set force_sync = true
+       - set done = true
+     - sync_entry exits (due to done)
+       - ..but does not set force_sync = false
+     - mount
+     - journal replay starts
+     - sync_entry sees force_sync and does a commit while op_seq == 0
+     ...crash...
+    
+    Fixes: #9144
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit dd11042f969b94f7a461d02e1475794031c79f61)
+    
+    Conflicts:
+    	src/os/FileStore.cc
+
+commit f7498c47ed90db71e50a3d4ff9ce13626e863719
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 8 13:44:57 2014 -0700
+
+    osdc/Objecter: revoke rx_buffer on op_cancel
+    
+    If we cancel a read, revoke the rx buffers to avoid a use-after-free and/or
+    other undefined badness by using user buffers that may no longer be
+    present.
+    
+    Fixes: #9362
+    Backport: firefly, dumpling
+    Reported-by: Matthias Kiefer <matthias.kiefer@1und1.de>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2305b2897acba38384358c33ca3bbfcae6f1c74e)
+    
+    (adjusted for op->con instead of s->con)
+
+commit 4dbb5010bd73ea8c7f9ed85e90e645ba1e85ea0a
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 8 06:58:45 2014 -0700
+
+    mon/Paxos: don't spam log with is_readable at dout level 1
+    
+    Backport: firefly, dumpling
+    Reported-by: Aanchal Agrawal <Aanchal.Agrawal@sandisk.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 62ca27d0b119b597ebad40dde64c4d86599e466d)
+
+commit 5315cf0a47e0a21e514df0d85be170dbca7ffc92
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Thu Sep 4 13:58:14 2014 -0400
+
+    doc: add note on soft JS dependency for navigating docs
+    
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit 657be818375bea2d8b5998ea1e5505eedc2f294d)
+
+commit 707076e3a9294169c3042afeb11b71b84727094b
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Wed Sep 3 21:21:45 2014 -0400
+
+    doc: fix missing bracket
+    
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit 69638dfaeb0dcd96dac4b5f5c00ed08042432487)
+
+commit 2ebdcfd5490462d77e32ffe3ef162f70183d53ab
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Wed Sep 3 20:47:54 2014 -0400
+
+    doc: attempt to get the ayni JS into all head tags
+    
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit 35663fa55ac1579a3b0c8b67028a3a8dfea87b48)
+
+commit c7948afa8232117c8a6e13b17774045a5e1bea6d
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Aug 17 20:54:28 2014 -0700
+
+    qa/workunits/rbd/qemu-iotests: touch common.env
+    
+    This seems to be necessary on trusty.
+    
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 055be68cf8e1b84287ab3631a02e89a9f3ae6cca)
+
+commit aaeebceb2189336f5f957f2a797d0448c2754b15
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 13 13:32:29 2014 -0700
+
+    mon: fix divide by zero when pg_num adjusted and no osds
+    
+    Fixes: #9052
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    
+    Manual backport of 239401db7b51541a57c59a261b89e0f05347c32d
+
+commit e7c5fb58d85ed0d1be359e616137666c5e207c38
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 12 07:05:34 2014 -0700
+
+    common/LogClient: fix sending dup log items
+    
+    We need to skip even the most recently sent item in order to get to the
+    ones we haven't sent yet.
+    
+    Fixes: #9080
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 057c6808be5bc61c3f1ac2b956c1522f18411245)
+
+commit cbc9218edc122337fd6d9dbbdca18670e20c6169
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Aug 11 16:41:26 2014 -0700
+
+    librbd: fix error path cleanup for opening an image
+    
+    If the image doesn't exist and caching is enabled, the ObjectCacher
+    was not being shutdown, and the ImageCtx was leaked. The IoCtx could
+    later be closed while the ObjectCacher was still running, resulting in
+    a segfault. Simply use the usual cleanup path in open_image(), which
+    works fine here.
+    
+    Fixes: #8912
+    Backport: dumpling, firefly
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 3dfa72d5b9a1f54934dc8289592556d30430959d)
+
+commit b5dafe1c0f7ecf7c3a25d0be5dfddcbe3d07e69e
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 18 11:02:58 2014 -0700
+
+    osd: allow io priority to be set for the disk_tp
+    
+    The disk_tp covers scrubbing, pg deletion, and snap trimming
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 84b3003119eeb8acfb3faacf357e6c6a452950e3)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+    
+    (cherry picked from commit 987ad133415aa988061c95259f9412b05ce8ac7e)
diff --git a/doc/changelog/v0.67.2.txt b/doc/changelog/v0.67.2.txt
new file mode 100644
index 000000000..e5d0f3d5e
--- /dev/null
+++ b/doc/changelog/v0.67.2.txt
@@ -0,0 +1,207 @@
+commit eb4380dd036a0b644c6283869911d615ed729ac8
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Thu Aug 22 19:10:55 2013 -0700
+
+    v0.67.2
+
+commit 242e43dae5b7c935b8f92c09e8dfe4704ba13787
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 9 12:49:57 2013 -0700
+
+    .gitignore: ignore test-driver
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit edf2c3449ec96d91d3d7ad01c50f7a79b7b2f7cc)
+
+commit 88aef702fb77c0a176caf37646a11ef480621412
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 9 12:42:49 2013 -0700
+
+    fuse: fix warning when compiled against old fuse versions
+    
+    client/fuse_ll.cc: In function 'void invalidate_cb(void*, vinodeno_t, int64_t, int64_t)':
+    warning: client/fuse_ll.cc:540: unused variable 'fino'
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 9833e9dabe010e538cb98c51d79b6df58ce28f9e)
+
+commit 48e104c9486f7a532455df108dbc225c00796097
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 9 12:40:34 2013 -0700
+
+    json_spirit: remove unused typedef
+    
+    In file included from json_spirit/json_spirit_writer.cpp:7:0:
+    json_spirit/json_spirit_writer_template.h: In function 'String_type json_spirit::non_printable_to_string(unsigned int)':
+    json_spirit/json_spirit_writer_template.h:37:50: warning: typedef 'Char_type' locally defined but not used [-Wunused-local-typedefs]
+             typedef typename String_type::value_type Char_type;
+    
+    (Also, ha ha, this file uses \r\n.)
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6abae35a3952e5b513895267711fea63ff3bad09)
+
+commit ae42619ca710d737bf4d8c63f39d1102326c903c
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 9 12:31:41 2013 -0700
+
+    gtest: add build-aux/test-driver to .gitignore
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c9cdd19d1cd88b84e8a867f5ab85cb51fdc6f8e4)
+
+commit 2c122be08db2f233d66214eb804734ae45646084
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Aug 21 14:28:49 2013 -0700
+
+    objecter: resend unfinished lingers when osdmap is no longer paused
+    
+    Plain Ops that haven't finished yet need to be resent if the osdmap
+    transitions from full or paused to unpaused.  If these Ops are
+    triggered by LingerOps, they will be cancelled instead (since
+    should_resend = false), but the LingerOps that triggered them will not
+    be resent.
+    
+    Fix this by checking the registered flag for all linger ops, and
+    resending any of them that aren't paused anymore.
+    
+    Fixes: #6070
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Sage Weil <sage.weil@inktank.com>
+    (cherry picked from commit 38a0ca66a79af4b541e6322467ae3a8a4483cc72)
+
+commit f6fe74ff51f679e7245b02462822d9ef1e15d28c
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 20 11:23:46 2013 -0700
+
+    pybind: fix Rados.conf_parse_env test
+    
+    This happens after we connect, which means we get ENOSYS always.
+    Instead, parse_env inside the normal setup method, which had the added
+    benefit of being able to debug these tests.
+    
+    Backport: dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6ef1970340c57d6e02f947348fb38882b51d131c)
+
+commit 47c89497b7f69cbf1557cd05b89837c388e2ba2f
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 13 13:14:59 2013 -0700
+
+    librados: fix MWatchNotify leak
+    
+    Do not leak the message if the watcher is not registered.  Also, simplify
+    this block.
+    
+    Fixes (part of): #5949
+    Backport: dumpling, cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 6f5d8036f3e70c5e30edf7e36fb8ff9a56197f60)
+
+commit b3a9a8c4e5edff5431d8da71033047eced6bf985
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Aug 19 17:23:44 2013 -0700
+
+    PG: remove old log when we upgrade log version
+    
+    Otherwise the log_oid will be non-empty and the next
+    boot will cause us to try to upgrade again.
+    
+    Fixes: #6057
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1f851cb2489a95526de932ec6734ebf413e43490)
+
+commit c6005ccbaa482c62d7a6cbb387bdcf17f0e308d5
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Aug 19 00:02:24 2013 -0700
+
+    PGLog: add a config to disable PGLog::check()
+    
+    This is a debug check which may be causing excessive
+    cpu usage.
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 00080d785f6695b800f71317a3048a21064e61cb)
+
+commit 96d719eeecceaa06078a29c2f868e50e6bc9ab31
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Aug 19 12:48:50 2013 -0700
+
+    ceph: parse CEPH_ARGS environment variable
+    
+    Fixes: #6052
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 67a95b9880c9bc6e858150352318d68d64ed74ad)
+
+commit d348cf5d135d099fe0490c1519196cd83a04831e
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Aug 19 12:48:40 2013 -0700
+
+    rados pybind: add conf_parse_env()
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit eef7cacdb19313907a9367187b742db5382ee584)
+
+commit 290bcd8a718887eb0e28aa2d97bceeee79068ea9
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Aug 13 13:16:07 2013 -0700
+
+    rgw: drain requests before exiting
+    
+    Fixes: #5953
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3cbf6a7b031c2ce8072733c5c0b7ceb53fdcb090)
+
+commit 863df08a43dff99797453040eb1ef6071b0432f9
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 13 11:16:17 2013 -0700
+
+    rgw: do not leak handler in get_handler() error path
+    
+    If we fail to initialize, delete the handler.
+    
+    Fixes (part of): #5949
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 810c52de36719c3ee6cf2bdf59d5cde8840bbe55)
+
+commit 9ac003f793b6cc72059110aac44014ddf2372bee
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 26 23:20:54 2013 -0700
+
+    rgw: fix leak of RGWDataChangesLog::renew_thread
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 25948319c4d256c4aeb0137eb88947e54d14cc79)
+
+commit 89cd9dc403e97b4bd08920fbb5d6e2b8b9b7dac2
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 26 23:17:10 2013 -0700
+
+    rgw: free resolver on shutdown
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a31356338b8ae55df59d829d9080ffad70b97d10)
+
+commit 5b26ca7fa5beb87cbbe6bbb26d70789ff2aa7661
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 26 23:22:20 2013 -0700
+
+    rgw: fix up signal handling
+    
+    OMG libfcgi is annoying with shutdown and signals.  You need to close
+    the fd *and* resend a signal to ensure that you kick the accept loop
+    hard enough to make it shut down.
+    
+    Document this, and switch to the async signal handlers.  Put them
+    tightly around the runtime loop as we do with other daemons.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 59b13cebee600dad2551d2c7dc3482b05eaf8b22)
diff --git a/doc/changelog/v0.67.3.txt b/doc/changelog/v0.67.3.txt
new file mode 100644
index 000000000..d6b1f2b27
--- /dev/null
+++ b/doc/changelog/v0.67.3.txt
@@ -0,0 +1,700 @@
+commit 408cd61584c72c0d97b774b3d8f95c6b1b06341a
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Mon Sep 9 12:50:11 2013 -0700
+
+    v0.67.3
+
+commit 17a7342b3b935c06610c58ab92a9a1d086923d32
+Merge: b4252bf 10433bb
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Sep 7 13:34:45 2013 -0700
+
+    Merge pull request #574 from dalgaaf/fix/da-dumpling-cherry-picks
+    
+    init-radosgw*: fix status return value if radosgw isn't running
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit 10433bbe72dbf8eae8fae836e557a043610eb54e
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Sat Sep 7 11:30:15 2013 +0200
+
+    init-radosgw*: fix status return value if radosgw isn't running
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit b5137baf651eaaa9f67e3864509e437f9d5c3d5a)
+
+commit b4252bff79150a95e9d075dd0b5e146ba9bf2ee5
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Aug 22 11:19:37 2013 -0700
+
+    FileStore: add config option to disable the wbthrottle
+    
+    Backport: dumpling
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3528100a53724e7ae20766344e467bf762a34163)
+
+commit 699324e0910e5e07a1ac68df8cf1108e5671ec15
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Aug 22 11:19:52 2013 -0700
+
+    WBThrottle: use fdatasync instead of fsync
+    
+    Backport: dumpling
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d571825080f0bff1ed3666e95e19b78a738ecfe8)
+
+commit 074717b4b49ae1a55bc867e5c34d43c51edc84a5
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Aug 29 15:08:58 2013 -0700
+
+    PGLog: initialize writeout_from in PGLog constructor
+    
+    Fixes: 6151
+    Backport: dumpling
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Introduced: f808c205c503f7d32518c91619f249466f84c4cf
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 42d65b0a7057696f4b8094f7c686d467c075a64d)
+
+commit c22d980cf42e580818dc9f526327518c0ddf8ff5
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Aug 27 08:49:14 2013 -0700
+
+    PGLog: maintain writeout_from and trimmed
+    
+    This way, we can avoid omap_rmkeyrange in the common append
+    and trim cases.
+    
+    Fixes: #6040
+    Backport: Dumpling
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit f808c205c503f7d32518c91619f249466f84c4cf)
+
+commit 53c7ab4db00ec7034f5aa555231f9ee167f43201
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Aug 27 07:27:26 2013 -0700
+
+    PGLog: don't maintain log_keys_debug if the config is disabled
+    
+    Fixes: #6040
+    Backport: Dumpling
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 1c0d75db1075a58d893d30494a5d7280cb308899)
+
+commit 40dc489351383c2e35b91c3d4e76b633309716df
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Aug 26 23:19:45 2013 -0700
+
+    PGLog: move the log size check after the early return
+    
+    There really are stl implementations (like the one on my ubuntu 12.04
+    machine) which have a list::size() which is linear in the size of the
+    list.  That assert, therefore, is quite expensive!
+    
+    Fixes: #6040
+    Backport: Dumpling
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit fe68b15a3d82349f8941f5b9f70fcbb5d4bc7f97)
+
+commit 4261eb5ec105b9c27605360910602dc367fd79f5
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 13 17:16:08 2013 -0700
+
+    rbd.cc: relicense as LGPL2
+    
+    All past authors for rbd.cc have consented to relicensing from GPL to
+    LGPL2 via email:
+    
+    ---
+    
+    Date: Sat, 27 Jul 2013 01:59:36 +0200
+    From: Sylvain Munaut <s.munaut@whatever-company.com>
+    Subject: Re: Ceph rbd.cc GPL -> LGPL2 license change
+    
+    I hereby consent to the relicensing of any contribution I made to the
+    aforementioned rbd.cc file from GPL to LGPL2.1.
+    
+    (I hope that'll be impressive enough, I did my best :p)
+    
+    btw, tnt@246tNt.com and s.munaut@whatever-company.com are both me.
+    
+    Cheers,
+    
+        Sylvain
+    
+    ---
+    
+    Date: Fri, 26 Jul 2013 17:00:48 -0700
+    From: Yehuda Sadeh <yehuda@inktank.com>
+    Subject: Re: Ceph rbd.cc GPL -> LGPL2 license change
+    
+    I consent.
+    
+    ---
+    
+    Date: Fri, 26 Jul 2013 17:02:24 -0700
+    From: Josh Durgin <josh.durgin@inktank.com>
+    Subject: Re: Ceph rbd.cc GPL -> LGPL2 license change
+    
+    I consent.
+    
+    ---
+    
+    Date: Fri, 26 Jul 2013 18:17:46 -0700
+    From: Stanislav Sedov <stas@freebsd.org>
+    Subject: Re: Ceph rbd.cc GPL -> LGPL2 license change
+    
+    I consent.
+    
+    Thanks for taking care of it!
+    
+    ---
+    
+    Date: Fri, 26 Jul 2013 18:24:15 -0700
+    From: Colin McCabe <cmccabe@alumni.cmu.edu>
+    
+    I consent.
+    
+    cheers,
+    Colin
+    
+    ---
+    
+    Date: Sat, 27 Jul 2013 07:08:12 +0200
+    From: Christian Brunner <christian@brunner-muc.de>
+    Subject: Re: Ceph rbd.cc GPL -> LGPL2 license change
+    
+    I consent
+    
+    Christian
+    
+    ---
+    
+    Date: Sat, 27 Jul 2013 12:17:34 +0300
+    From: Stratos Psomadakis <psomas@grnet.gr>
+    Subject: Re: Ceph rbd.cc GPL -> LGPL2 license change
+    
+    Hi,
+    
+    I consent with the GPL -> LGL2.1 re-licensing.
+    
+    Thanks
+    Stratos
+    
+    ---
+    
+    Date: Sat, 27 Jul 2013 16:13:13 +0200
+    From: Wido den Hollander <wido@42on.com>
+    Subject: Re: Ceph rbd.cc GPL -> LGPL2 license change
+    
+    I consent!
+    
+    You have my permission to re-license the code I wrote for rbd.cc to LGPL2.1
+    
+    ---
+    
+    Date: Sun, 11 Aug 2013 10:40:32 +0200
+    From: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    Subject: Re: btw
+    
+    Hi Sage,
+    
+    I agree to switch the license of ceph_argparse.py and rbd.cc from GPL2
+    to LGPL2.
+    
+    Regards
+    
+    Danny Al-Gaaf
+    
+    ---
+    
+    Date: Tue, 13 Aug 2013 17:15:24 -0700
+    From: Dan Mick <dan.mick@inktank.com>
+    Subject: Re: Ceph rbd.cc GPL -> LGPL2 license change
+    
+    I consent to relicense any contributed code that I wrote under LGPL2.1 license.
+    
+    ---
+    
+    ...and I consent too.  Drop the exception from COPYING and debian/copyright
+    files.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2206f55761c675b31078dea4e7dd66f2666d7d03)
+
+commit 211c5f13131e28b095a1f3b72426128f1db22218
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Aug 23 15:39:20 2013 -0700
+
+    rgw: flush pending data when completing multipart part upload
+    
+    Fixes: #6111
+    Backport: dumpling
+    When completing the part upload we need to flush any data that we
+    aggregated and didn't flush yet. With earlier code didn't have to deal
+    with it as for multipart upload we didn't have any pending data.
+    What we do now is we call the regular atomic data completion
+    function that takes care of it.
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 9a551296e0811f2b65972377b25bb28dbb42f575)
+
+commit 1a9651010aab51c9be2edeccd80e9bd11f5177ce
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Aug 26 19:46:43 2013 -0700
+
+    rgw: check object name after rebuilding it in S3 POST
+    
+    Fixes: #6088
+    Backport: bobtail, cuttlefish, dumpling
+    
+    When posting an object it is possible to provide a key
+    name that refers to the original filename, however we
+    need to verify that in the end we don't end up with an
+    empty object name.
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit c8ec532fadc0df36e4b265fe20a2ff3e35319744)
+
+commit 1bd74a020b93f154b2d4129d512f6334387de7c7
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Aug 22 17:46:45 2013 -0700
+
+    mon/MonClient: release pending outgoing messages on shutdown
+    
+    This fixes a small memory leak when we have messages queued for the mon
+    when we shut down.  It is harmless except for the valgrind leak check
+    noise that obscures real leaks.
+    
+    Backport: dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 309569a6d0b7df263654b7f3f15b910a72f2918d)
+
+commit 24f2669783e2eb9d9af5ecbe106efed93366ba63
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Aug 29 13:06:33 2013 -0700
+
+    rgw: change watch init ordering, don't distribute if can't
+    
+    Backport: dumpling
+    
+    Moving back the watch initialization after the zone init,
+    as the zone info holds the control pool name. Since zone
+    init might need to create a new system object (that needs
+    to distribute cache), don't try to distribute cache if
+    watch is not yet initialized.
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 1d1f7f18dfbdc46fdb09a96ef973475cd29feef5)
+
+commit a708c8ab52e5b1476405a1f817c23b8845fbaab3
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 30 09:41:29 2013 -0700
+
+    ceph-post-file: use mktemp instead of tempfile
+    
+    tempfile is a debian thing, apparently; mktemp is present everywhere.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e60d4e09e9f11e3c34a05cd122341e06c7c889bb)
+
+commit 625f13ee0d6cca48d61dfd65e00517d092552d1c
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Aug 28 09:50:11 2013 -0700
+
+    mon: discover mon addrs, names during election state too
+    
+    Currently we only detect new mon addrs and names during the probing phase.
+    For non-trivial clusters, this means we can get into a sticky spot when
+    we discover enough peers to form an quorum, but not all of them, and the
+    undiscovered ones are enough to break the mon ranks and prevent an
+    election.
+    
+    One way to work around this is to continue addr and name discovery during
+    the election.  We should also consider making the ranks less sensitive to
+    the undefined addrs; that is a separate change.
+    
+    Fixes: #4924
+    Backport: dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Tested-by: Bernhard Glomm <bernhard.glomm@ecologic.eu>
+    (cherry picked from commit c24028570015cacf1d9e154ffad80bec06a61e7c)
+
+commit 83cfd4386c1fd0fa41aea345704e27f82b524ece
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Thu Aug 22 17:30:24 2013 -0700
+
+    ceph_rest_api.py: create own default for log_file
+    
+    common/config thinks the default log_file for non-daemons should be "".
+    Override that so that the default is
+        /var/log/ceph/{cluster}-{name}.{pid}.log
+    since ceph-rest-api is more of a daemon than a client.
+    
+    Fixes: #6099
+    Backport: dumpling
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 2031f391c3df68e0d9e381a1ef3fe58d8939f0a8)
+
+commit 8a1da62d9564a32f7b8963fe298e1ac3ad0ea3d9
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 16 17:59:11 2013 -0700
+
+    ceph-post-file: single command to upload a file to cephdrop
+    
+    Use sftp to upload to a directory that only this user and ceph devs can
+    access.
+    
+    Distribute an ssh key to connect to the account.  This will let us revoke
+    the key in the future if we feel the need.  Also distribute a known_hosts
+    file so that users have some confidence that they are connecting to the
+    real ceph drop account and not some third party.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit d08e05e463f1f7106a1f719d81b849435790a3b9)
+
+commit 3f8663477b585dcb528fdd7047c50d9a52d24b95
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Thu Aug 22 13:29:32 2013 -0700
+
+    ceph.spec.in:  remove trailing paren in previous commit
+    
+    Signed-off-by: Gary Lowell  <gary.lowell@inktank.com>
+
+commit 23fb908cb3ac969c874ac12755d20ed2f636e1b9
+Author: Gary Lowell <glowell@inktank.com>
+Date:   Thu Aug 22 11:07:16 2013 -0700
+
+    ceph.spec.in:  Don't invoke debug_package macro on centos.
+    
+    If the redhat-rpm-config package is installed, the debuginfo rpms will
+    be built by default.   The build will fail when the package installed
+    and the specfile also invokes the macro.
+    
+    Signed-off-by: Gary Lowell  <gary.lowell@inktank.com>
+
+commit 11f5853d8178ab60ab948d373c1a1f67324ce3bd
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Aug 24 14:04:09 2013 -0700
+
+    osd: install admin socket commands after signals
+    
+    This lets us tell by the presence of the admin socket commands whether
+    a signal will make us shut down cleanly.  See #5924.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit c5b5ce120a8ce9116be52874dbbcc39adec48b5c)
+
+commit 39adc0195e6016ce36828885515be1bffbc10ae1
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 20 22:39:09 2013 -0700
+
+    ceph-disk: partprobe after creating journal partition
+    
+    At least one user reports that a partprobe is needed after creating the
+    journal partition.  It is not clear why sgdisk is not doing it, but this
+    fixes ceph-disk for them, and should be harmless for other users.
+    
+    Fixes: #5599
+    Tested-by: lurbs in #ceph
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2af59d5e81c5e3e3d7cfc50d9330d7364659c5eb)
+    (cherry picked from commit 3e42df221315679605d68b2875aab6c7eb6b3cc4)
+
+commit 6a4fe7b9b068ae990d6404921a46631fe9ebcd31
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 20 11:27:23 2013 -0700
+
+    mon/Paxos: always refresh after any store_state
+    
+    If we store any new state, we need to refresh the services, even if we
+    are still in the midst of Paxos recovery.  This is because the
+    subscription path will share any committed state even when paxos is
+    still recovering.  This prevents a race like:
+    
+     - we have maps 10..20
+     - we drop out of quorum
+     - we are elected leader, paxos recovery starts
+     - we get one LAST with committed states that trim maps 10..15
+     - we get a subscribe for map 10..20
+       - we crash because 10 is no longer on disk because the PaxosService
+         is out of sync with the on-disk state.
+    
+    Fixes: #6045
+    Backport: dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 981eda9f7787c83dc457f061452685f499e7dd27)
+
+commit 13d396e46ed9200e4b9f21db2f0a8efbc5998d82
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 20 11:27:09 2013 -0700
+
+    mon/Paxos: return whether store_state stored anything
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 7e0848d8f88f156a05eef47a9f730b772b64fbf2)
+
+commit f248383bacff76203fa94716cfdf6cf766da24a7
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 20 11:26:57 2013 -0700
+
+    mon/Paxos: cleanup: use do_refresh from handle_commit
+    
+    This avoid duplicated code by using the helper created exactly for this
+    purpose.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit b9dee2285d9fe8533fa98c940d5af7b0b81f3d33)
+
+commit 02608a12d4e7592784148a62a47d568efc24079d
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Aug 15 21:48:06 2013 -0700
+
+    osdc/ObjectCacher: do not merge rx buffers
+    
+    We do not try to merge rx buffers currently.  Make that explicit and
+    documented in the code that it is not supported.  (Otherwise the
+    last_read_tid values will get lost and read results won't get applied
+    to the cache properly.)
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1c50c446152ab0e571ae5508edb4ad7c7614c310)
+
+commit 0e2bfe71965eeef29b47e8032637ea820a7ce49c
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Aug 15 21:47:18 2013 -0700
+
+    osdc/ObjectCacher: match reads with their original rx buffers
+    
+    Consider a sequence like:
+    
+     1- start read on 100~200
+           100~200 state rx
+     2- truncate to 200
+           100~100 state rx
+     3- start read on 200~200
+           100~100 state rx
+           200~200 state rx
+     4- get 100~200 read result
+    
+    Currently this makes us crash on
+    
+    osdc/ObjectCacher.cc: 738: FAILED assert(bh->length() <= start+(loff_t)length-opos)
+    
+    when processing the second 200~200 bufferhead (it is too big).  The
+    larger issue, though, is that we should not be looking at this data at
+    all; it has been truncated away.
+    
+    Fix this by marking each rx buffer with the read request that is sent to
+    fill it, and only fill it from that read request.  Then the first reply
+    will fill the first 100~100 extend but not touch the other extent; the
+    second read will do that.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit b59f930ae147767eb4c9ff18c3821f6936a83227)
+
+commit 6b51c960715971a0351e8203d4896cb0c4138a3f
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Aug 22 15:54:48 2013 -0700
+
+    mon/Paxos: fix another uncommitted value corner case
+    
+    It is possible that we begin the paxos recovery with an uncommitted
+    value for, say, commit 100.  During last/collect we discover 100 has been
+    committed already.  But also, another node provides an uncommitted value
+    for 101 with the same pn.  Currently, we refuse to learn it, because the
+    pn is not strictly > than our current uncommitted pn... even though it is
+    the next last_committed+1 value that we need.
+    
+    There are two possible fixes here:
+    
+     - make this a >= as we can accept newer values from the same pn.
+     - discard our uncommitted value metadata when we commit the value.
+    
+    Let's do both!
+    
+    Fixes: #6090
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit fe5010380a3a18ca85f39403e8032de1dddbe905)
+
+commit b3a280d5af9d06783d2698bd434940de94ab0fda
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 23 11:45:35 2013 -0700
+
+    os: make readdir_r buffers larger
+    
+    PATH_MAX isn't quite big enough.
+    
+    Backport: dumpling, cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 99a2ff7da99f8cf70976f05d4fe7aa28dd7afae5)
+
+commit 989a664ef0d1c716cab967f249112f595cf98c43
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 23 11:45:08 2013 -0700
+
+    os: fix readdir_r buffer size
+    
+    The buffer needs to be big or else we're walk all over the stack.
+    
+    Backport: dumpling, cuttlefish, bobtail
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2df66d9fa214e90eb5141df4d5755b57e8ba9413)
+    
+    Conflicts:
+    
+    	src/os/BtrfsFileStoreBackend.cc
+
+commit a4cca31c82bf0e84272e01eb1b3188dfdb5b5615
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Aug 22 10:53:12 2013 -0700
+
+    rgw: fix crash when creating new zone on init
+    
+    Moving the watch/notify init before the zone init,
+    as we might need to send a notification.
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 3d55534268de7124d29bd365ea65da8d2f63e501)
+
+commit 4cf6996803ef66f2b6083f73593259d45e2740a3
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Aug 19 08:40:16 2013 -0700
+
+    rgw: change cache / watch-notify init sequence
+    
+    Fixes: #6046
+    We were initializing the watch-notify (through the cache
+    init) before reading the zone info which was much too
+    early, as we didn't have the control pool name yet. Now
+    simplifying init/cleanup a bit, cache doesn't call watch/notify
+    init and cleanup directly, but rather states its need
+    through a virtual callback.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d26ba3ab0374e77847c742dd00cb3bc9301214c2)
+
+commit aea6de532b0b843c3a8bb76d10bab8476f0d7c09
+Author: Alexandre Oliva <oliva@gnu.org>
+Date:   Thu Aug 22 03:40:22 2013 -0300
+
+    enable mds rejoin with active inodes' old parent xattrs
+    
+    When the parent xattrs of active inodes that the mds attempts to open
+    during rejoin lack pool info (struct_v < 5), this field will be filled
+    in with -1, causing the mds to retry fetching a backtrace with a pool
+    number that matches the expected value, which fails and causes the
+    err==-ENOENT branch to be taken and retry pool 1, which succeeds, but
+    with pool -1, and so keeps on bouncing between the two retry cases
+    forever.
+    
+    This patch arranges for the mds to go along with pool -1 instead of
+    insisting that it be refetched, enabling it to complete recovery
+    instead of eating cpu, network bandwidth and metadata osd's resources
+    like there's no tomorrow, in what AFAICT is an infinite and very busy
+    loop.
+    
+    This is not a new problem: I've had it even before upgrading from
+    Cuttlefish to Dumpling, I'd just never managed to track it down, and
+    force-unmounting the filesystem and then restarting the mds was an
+    easier (if inconvenient) work-around, particularly because it always
+    hit when the filesystem was under active, heavy-ish use (or there
+    wouldn't be much reason for caps recovery ;-)
+    
+    There are two issues not addressed in this patch, however.  One is
+    that nothing seems to proactively update the parent xattr when it is
+    found to be outdated, so it remains out of date forever.  Not even
+    renaming top-level directories causes the xattrs to be recursively
+    rewritten.  AFAICT that's a bug.
+    
+    The other is that inodes that don't have a parent xattr (created by
+    even older versions of ceph) are reported as non-existing in the mds
+    rejoin message, because the absence of the parent xattr is signaled as
+    a missing inode (?failed to reconnect caps for missing inodes?).  I
+    suppose this may cause more serious recovery problems.
+    
+    I suppose a global pass over the filesystem tree updating parent
+    xattrs that are out-of-date would be desirable, if we find any parent
+    xattrs still lacking current information; it might make sense to
+    activate it as a background thread from the backtrace decoding
+    function, when it finds a parent xattr that's too out-of-date, or as a
+    separate client (ceph-fsck?).
+    
+    Backport: dumpling, cuttlefish
+    Signed-off-by: Alexandre Oliva <oliva@gnu.org>
+    Reviewed-by: Zheng, Yan <zheng.z.yan@intel.com>
+    (cherry picked from commit 617dc36d477fd83b2d45034fe6311413aa1866df)
+
+commit 0738bdf92f5e5eb93add152a4135310ac7ea1c91
+Author: David Disseldorp <ddiss@suse.de>
+Date:   Mon Jul 29 17:05:44 2013 +0200
+
+    mds: remove waiting lock before merging with neighbours
+    
+    CephFS currently deadlocks under CTDB's ping_pong POSIX locking test
+    when run concurrently on multiple nodes.
+    The deadlock is caused by failed removal of a waiting_locks entry when
+    the waiting lock is merged with an existing lock, e.g:
+    
+    Initial MDS state (two clients, same file):
+    held_locks -- start: 0, length: 1, client: 4116, pid: 7899, type: 2
+    	      start: 2, length: 1, client: 4110, pid: 40767, type: 2
+    waiting_locks -- start: 1, length: 1, client: 4116, pid: 7899, type: 2
+    
+    Waiting lock entry 4116@1:1 fires:
+    handle_client_file_setlock: start: 1, length: 1,
+    			    client: 4116, pid: 7899, type: 2
+    
+    MDS state after lock is obtained:
+    held_locks -- start: 0, length: 2, client: 4116, pid: 7899, type: 2
+    	      start: 2, length: 1, client: 4110, pid: 40767, type: 2
+    waiting_locks -- start: 1, length: 1, client: 4116, pid: 7899, type: 2
+    
+    Note that the waiting 4116@1:1 lock entry is merged with the existing
+    4116@0:1 held lock to become a 4116@0:2 held lock. However, the now
+    handled 4116@1:1 waiting_locks entry remains.
+    
+    When handling a lock request, the MDS calls adjust_locks() to merge
+    the new lock with available neighbours. If the new lock is merged,
+    then the waiting_locks entry is not located in the subsequent
+    remove_waiting() call because adjust_locks changed the new lock to
+    include the old locks.
+    This fix ensures that the waiting_locks entry is removed prior to
+    modification during merge.
+    
+    Signed-off-by: David Disseldorp <ddiss@suse.de>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 476e4902907dfadb3709ba820453299ececf990b)
+
+commit a0ac88272511d670b5c3756dda2d02c93c2e9776
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Tue Aug 20 11:10:42 2013 -0700
+
+    mon/PGMap: OSD byte counts 4x too large (conversion to bytes overzealous)
+    
+    Fixes: #6049
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit eca53bbf583027397f0d5e050a76498585ecb059)
+
+commit 87b19c33ce29e2ca4fc49a2adeb12d3f14ca90a9
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Fri Aug 23 08:56:07 2013 -0400
+
+    ceph-disk: specify the filetype when mounting
+    
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f040020fb2a7801ebbed23439159755ff8a3edbd)
diff --git a/doc/changelog/v0.67.4.txt b/doc/changelog/v0.67.4.txt
new file mode 100644
index 000000000..73b997ea3
--- /dev/null
+++ b/doc/changelog/v0.67.4.txt
@@ -0,0 +1,550 @@
+commit ad85b8bfafea6232d64cb7ba76a8b6e8252fa0c7
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Thu Oct 3 22:41:31 2013 +0000
+
+    v0.67.4
+
+commit 5cd66d3b4bca92b402c95ab256fbc3f0329c446f
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Sep 20 14:04:47 2013 -0700
+
+    rgw: fix keystone token expiration test
+    
+    Fixes: #6360
+    The test was inverted, need expiration to be greater than
+    current time in order for token to be valid.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit e0203c61a3f45fdd6d3d3ece26fef6152bdc036d
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed Sep 11 16:55:06 2013 -0700
+
+    osd/OSD.cc: Use MIN() so that we don't exceed osd_recovery_max_active
+    
+    Caused by 944f3b73531af791c90f0f061280160003545c63
+    
+    Fixes: #6291
+    
+    Backport: dumpling
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 139a714e13aa3c7f42091270b55dde8a17b3c4b8)
+    
+    Conflicts:
+    
+    	src/osd/OSD.cc
+
+commit c376708358cedb5561fbb43e9b9e622df3ea7a58
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Sep 25 22:08:24 2013 +0100
+
+    mon: OSDMonitor: do not write full_latest during trim
+    
+    On commit 81983bab we patched OSDMonitor::update_from_paxos() such that we
+    write the latest full map version to 'full_latest' each time the latest
+    full map was built from the incremental versions.
+    
+    This change however clashed with OSDMonitor::encode_trim_extra(), which
+    also wrote to 'full_latest' on each trim, writing instead the version of
+    the *oldest* full map.  This duality of behaviors could lead the store
+    to an inconsistent state across the monitors (although there's no sign of
+    it actually imposing any issues besides rebuilding already existing full
+    maps on some monitors).
+    
+    We now stop OSDMonitor::encode_trim_extra() from writing to 'full_latest'.
+    This function will still write out the oldest full map it has in the store,
+    but it will no longer write to full_latest, instead leaving it up to
+    OSDMonitor::update_from_paxos() to figure it out -- and it already does.
+    
+    Fixes: #6378
+    
+    Backport: dumpling
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit bd0f29a2c28cca496ec830eac932477ebf3182ba)
+
+commit de40d0b3e35ab0124cd3c4ebfcaa435ab8abfab9
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Oct 1 15:53:42 2013 -0700
+
+    crush: invalidate rmap on create (and thus decode)
+    
+    If we have an existing CrushWrapper object and decode from a bufferlist,
+    reset build_rmaps so that they get rebuilt.
+    
+    Remove the build_rmaps() all in decode that was useless on a redecode
+    (because have_rmaps == true in that case and it did nothing).
+    
+    Fixes: #6442
+    Backport: dumpling, maybe cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 9b7a2ae329b6a511064dd3d6e549ba61f52cfd21)
+
+commit 32f5233288c47d95b87c0a9cab5f9c2ffcf15417
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Mon Sep 30 14:58:11 2013 -0700
+
+    Invoke python with /usr/bin/env python instead of directly
+    
+    Fixes: #6311
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit b9000b314b9166845ff302d4a827a996775d9a14)
+
+commit 66aeca5a9079be398403bbff67bd5bf68c6fb111
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Sep 25 10:10:21 2013 -0700
+
+    qa/workunits/mon/crush_ops.sh: fix test
+    
+    Fix root.
+    
+    Fixes: #6392
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c8cae87e9e08468cc86145e0fd60c05d12826239)
+
+commit beb366302a125dd422c4f092b12eb541cb3bc788
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Sep 23 09:04:34 2013 -0700
+
+    Revert "ceph: parse CEPH_ARGS environment variable"
+    
+    This reverts commit 67a95b9880c9bc6e858150352318d68d64ed74ad.
+    
+    We now put CEPH_ARGS in the actual args we parse in python, which are passed
+    to rados piecemeal later.  This lets you put things like --id ... in there
+    that need to be parsed before librados is initialized.
+    (cherry picked from commit 97f462be4829f0167ed3d65e6694dfc16f1f3243)
+
+commit b475ff9576f145d31c053213c699e13df76d2bcb
+Author: Benoît Knecht <benoit.knecht@fsfe.org>
+Date:   Mon Sep 23 15:58:42 2013 +0200
+
+    Add CEPH_ARGS at the end of sys.argv
+    
+    This allows, for instance, to pass a different client name to ceph by
+    exporting CEPH_ARGS="--id client_id".
+    
+    Signed-off-by: Benoît Knecht <benoit.knecht@fsfe.org>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 30abe3244c86cbbe1f5b005850c29c9c0eafcad4)
+
+commit 94548b4b67cca37366c7d8719209a6d2e7956811
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Sep 24 15:26:03 2013 -0700
+
+    mon/OSDMonitor: fix 'ceph osd crush reweight ...'
+    
+    The adjust method returns a count of adjusted items.
+    
+    Add a test.
+    
+    Fixes: #6382
+    Backport: dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 3de32562b55c6ece3a6ed783c36f8b9f21460339)
+
+commit 00ff7f5c20e13869d0694379739ba4e61d44b97c
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Sep 10 00:20:41 2013 +0100
+
+    qa: workunits: mon: crush_ops: test 'ceph osd crush move'
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 3bc618b7b46496c5110edde0da9cae5d3e68e0e1)
+
+commit 0ff5b4a96833681e92cc41f019a569134474f4cf
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Sep 24 19:04:23 2013 +0200
+
+    osd: change warn_interval_multiplier to uint32_t
+    
+    to prevent overflow in OpTracker::check_ops_in_flight when
+    multiplying warn_interval_multiplier *= 2
+    
+    Backport: cuttlefish, dumpling
+    
+    http://tracker.ceph.com/issues/6370 fixes #6370
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 1bce1f009bffd3e28025a08775fec189907a81db)
+
+commit fb15040b6cec6221baa550ddfffade823f784c4a
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Mon Sep 9 13:01:12 2013 -0700
+
+    crushtool: do not dump core with non-unique bucket IDs
+    
+    Return -EEXIST on duplicate ID
+    BUG FIX: crush_add_bucket() mixes error returns and IDs
+      Add optional argument to return generated ID
+    
+    Fixes: #6246
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 8c76f3a0f9cf100ea2c941dc2b61c470aa5033d7)
+
+commit 410db3f30c6eb54b807908c1f251ad4026e7d446
+Author: Joao Eduardo Luis <jecluis@gmail.com>
+Date:   Fri Sep 20 17:06:30 2013 +0100
+
+    qa: workunits: cephtool: check if 'heap' commands are parseable
+    
+    Signed-off-by: Joao Eduardo Luis <jecluis@gmail.com>
+    (cherry picked from commit b1eeaddd5f214c1b0883b44fc8cae07c649be7c4)
+
+commit 062060a38bb26ff260cc51accc534413d726de49
+Author: Joao Eduardo Luis <jecluis@gmail.com>
+Date:   Fri Sep 20 17:50:27 2013 +0100
+
+    osd: OSD: add 'heap' command to known osd commands array
+    
+    Must have been forgotten during the cli rework.
+    
+    Backport: dumpling
+    
+    Signed-off-by: Joao Eduardo Luis <jecluis@gmail.com>
+    (cherry picked from commit 296f2d0db31e9f5a59a3a62a1e95b6c440430fa3)
+
+commit 3f32f57b98e0224a1d30b2a81d7d260be0f53800
+Author: Joao Eduardo Luis <jecluis@gmail.com>
+Date:   Fri Sep 20 16:43:27 2013 +0100
+
+    mds: MDS: pass only heap profiler commands instead of the whole cmd vector
+    
+    The heap profiler doesn't care, nor should it, what our command name is.
+    It only cares about the commands it handles.
+    
+    Backport: dumpling
+    
+    Signed-off-by: Joao Eduardo Luis <jecluis@gmail.com>
+    (cherry picked from commit 238fe272c6bdb62d4e57fd8555c0136de99c8129)
+
+commit 46dcc46617d8f35ab8433540b22343ddcbcc3716
+Author: Joao Eduardo Luis <jecluis@gmail.com>
+Date:   Fri Sep 20 16:41:14 2013 +0100
+
+    perfglue/heap_profiler.cc: expect args as first element on cmd vector
+    
+    We used to pass 'heap' as the first element of the cmd vector when
+    handling commands.  We haven't been doing so for a while now, so we
+    needed to fix this.
+    
+    Not expecting 'heap' also makes sense, considering that what we need to
+    know when we reach this function is what command we should handle, and
+    we should not care what the caller calls us when handling his business.
+    
+    Fixes: #6361
+    Backport: dumpling
+    
+    Signed-off-by: Joao Eduardo Luis <jecluis@gmail.com>
+    (cherry picked from commit c98b910d49bd2b46ceafdc430044a31524c29f5b)
+
+commit 9dc5f15fbae22244ad1f62925e17c9d81e856e55
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Sep 16 14:35:25 2013 -0700
+
+    rgw: destroy get_obj handle in copy_obj()
+    
+    Fixes: #6176
+    Backport: dumpling
+    We take different code paths in copy_obj, make sure we close the handle
+    when we exit the function. Move the call to finish_get_obj() out of
+    copy_obj_data() as we don't create the handle there, so that should
+    makes code less confusing and less prone to errors.
+    Also, note that RGWRados::get_obj() also calls finish_get_obj(). For
+    everything to work in concert we need to pass a pointer to the handle
+    and not the handle itself. Therefore we needed to also change the call
+    to copy_obj_data().
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 9e98620e4325d15c88440a890b267131613e1aa1)
+
+commit 471233e98a9f64ad513a4a196b7661b80534cb00
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Mon Sep 9 23:14:11 2013 +0100
+
+    mon: MonCommands: expect a CephString as 1st arg for 'osd crush move'
+    
+    Fixes: #6230
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 7d3799fde19138f957f26ec6be10a8a0000fc1f0)
+
+commit 2908225092bd2aa1b8afcb7848c1cdac5bd9e638
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Sep 23 16:23:33 2013 -0700
+
+    osd: revert 'osd max xattr size' limit
+    
+    Set it to 0 (unlimited) for now.
+    
+    Backport: dumpling
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit abb88d70643c3a76435b7a9d5b04ff29f7502361)
+
+commit b3d3b3747c1eef695138dac828e5fcb435309c7b
+Author: Greg Farnum <greg@inktank.com>
+Date:   Wed Sep 11 16:24:32 2013 -0700
+
+    mds: be more careful about decoding LogEvents
+    
+    We need to wrap the full decode section or we can abort the process
+    if there's an issue (which we may want to just skip by).
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 73289b34b0be5b6612e38944794d59b5e789f841)
+
+commit 06c58132199ed22413b509dfa751321ccdb24225
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Sep 17 17:58:20 2013 +0100
+
+    mon: OSDMonitor: multiple rebuilt full maps per transaction
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 0d20cae0be701c5b6151a26ee5e4fe24d89aa20a)
+
+commit 65bbcaf4b68790dae4506c1f5db237077e1ff0ae
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Sun Sep 15 21:03:50 2013 +0100
+
+    mon: OSDMonitor: update latest_full while rebuilding full maps
+    
+    Not doing so will make the monitor rebuild the osdmap full versions, even
+    though they may have been rebuilt before, every time the monitor starts.
+    
+    This mostly happens when the cluster is left in an unhealthy state for
+    a long period of time and incremental versions build up.  Even though we
+    build the full maps on update_from_paxos(), not updating 'full_latest'
+    leads to the situation initially described.
+    
+    Fixes: #6322
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 81983bab3630520d6c7ee9b7e4a747bc17b8c5c3)
+
+commit 9b9edb04581cca15e67c567332529f5b3f426743
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Sun Sep 15 21:00:55 2013 +0100
+
+    mon: OSDMonitor: smaller transactions when rebuilding full versions
+    
+    Otherwise, for considerably sized rebuilds, the monitor will not only
+    consume vast amounts of memory, but it will also have troubles committing
+    the transaction.  Anyway, it's also a good idea to adjust transactions to
+    the granularity we want, and to be fair we care that each rebuilt full map
+    gets to disk, even if subsequent full maps don't (those can be rebuilt
+    later).
+    
+    Fixes: #6323
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 4ac1570c5cdcd6556dc291cc6d7878fd92d343ae)
+
+commit 298811f7a15541b9ec1015c416ad2aa075be5691
+Author: Joao Eduardo Luis <jecluis@gmail.com>
+Date:   Wed Aug 28 15:51:01 2013 +0100
+
+    mon: OSDMonitor: check if pool is on unmanaged snaps mode on mk/rmsnap
+    
+    Backport: dumpling
+    Fixes: #6047
+    
+    Signed-off-by: Joao Eduardo Luis <jecluis@gmail.com>
+    (cherry picked from commit fab79543c54c2e446d3f76520d7906645c6b0075)
+
+commit a992664435db9dde3745eb7f354cce3fc5400a47
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Sep 12 14:32:17 2013 -0700
+
+    lru_map: don't use list::size()
+    
+    replace list::size() with map::size(), which should have
+    a constant time complexity.
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 7c1d2ded8fa8061bf3f14932800998b963745dd1)
+
+commit 788546ea71c994ff35323747294ed9c177fe7020
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Sep 12 14:30:19 2013 -0700
+
+    common/lru_map: rename tokens to entries
+    
+    This code was originally used in a token cache, now
+    as a generic infrastructure rename token fields.
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 532e41a9985a16b35a6e49cdcba38af0ad166fa8)
+
+commit babeb00c42af760b3e7575166479e95365cfcc0a
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Sep 18 10:37:21 2013 -0700
+
+    rgw: use bufferlist::append() instead of bufferlist::push_back()
+    
+    push_back() expects char *, whereas append can append a single char.
+    Appending a NULL char to push_back is cast as a NULL pointer which is
+    bad.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 08fe028bad13096d482454a2f303158727c363ff)
+
+commit daf85c45dd4d158bc7c33a2fb784857bc7db35cd
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Sep 11 13:46:31 2013 -0700
+
+    rgw: NULL terminate buffer before parsing it
+    
+    Fixes: #6175
+    Backport: dumpling
+    We get a buffer off the remote gateway which might
+    not be NULL terminated. The JSON parser needs the
+    buffer to be NULL terminated even though we provide
+    a buffer length as it calls strlen().
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit e7f7483192cddca1159aba439ce62b1e78669d51)
+
+commit c73040a5518971813b9ebaae1624c5bacef315d0
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Sep 11 22:30:12 2013 -0700
+
+    rgw: don't call list::size() in ObjectCache
+    
+    Fixes: #6286
+    Use an external counter instead of calling list::size()
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 31e3a51e933429d286104fe077e98ea883437ad6)
+
+commit a855aba9d18936e9a060119e041518790cd4b831
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Sep 10 12:18:55 2013 -0700
+
+    rgw: drain pending requests before completing write
+    
+    Fixes: #6268
+    When doing aio write of objects (either regular or multipart parts) we
+    need to drain pending aio requests. Otherwise if gateway goes down then
+    object might end up corrupted.
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 626669afaa333d73707553a85f5c874e99e9cbd8)
+
+commit 670db7e80ddc9c26c43a4f66907a5996ce207c4d
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Sep 6 22:33:38 2013 -0700
+
+    rgw: fix get cors, delete cors
+    
+    Remove a couple of variables that overrode class member. Not
+    really clear how it was working before, might have been a bad
+    merge / rebase.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 13872785aeeddbe1b8dd97e49fd6a2d879514f8d)
+
+commit a304016fa01b02efd500135c00b9bf3407a9999c
+Merge: 408cd61 ac0a30f
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Sep 11 09:47:10 2013 -0700
+
+    Merge branch 'wip-6078-dumpling' into dumpling
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit ac0a30feb8c64a3b80d9c519a7b561213403afab
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Aug 28 21:25:20 2013 -0700
+
+    rgw: fix certain return status cases in CORS
+    
+    Change return values in certain cases, reorder
+    checks, etc.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 13b28cc3f1eb8ef42875b630c485ee0105cd244a
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Aug 28 21:24:36 2013 -0700
+
+    rgw: add COPY method to be handled by CORS
+    
+    Was missing this http method.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit d45c87ea738807487e72c0719b0d3d459cbe19e9
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Aug 27 19:38:45 2013 -0700
+
+    rgw: fix CORS rule check
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 986fa92a7a1d88111ba28457160adfcfdaabc5d2
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Aug 27 19:38:18 2013 -0700
+
+    rgw: don't handle CORS if rule not found (is NULL)
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 71873aba6553492d3ad71596cefd7c841030a277
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Aug 22 13:38:55 2013 -0700
+
+    rgw: tie CORS header response to all relevant operations
+    
+    Have the CORS responses on all relevant operations. Also add headers
+    on failure cases.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 94e7b594d85dbd26e58d823b41f418032e9f163f
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Aug 22 10:00:53 2013 -0700
+
+    rgw: add a generic CORS response handling
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit c3385d8a102faf5379559bb98cf89637ceda1579
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Aug 21 17:22:46 2013 -0700
+
+    rgw: OPTIONS request doesn't need to read object info
+    
+    This is a bucket-only operation, so we shouldn't look at the
+    object. Object may not exist and we might respond with Not
+    Exists response which is not what we want.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit a5fdd44e5d8ce4b8d82273d83e27aea19e63aa7c
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Aug 21 14:43:28 2013 -0700
+
+    rgw: remove use of s->bucket_cors
+    
+    Some old code still tried to use s->bucket_cors, which was
+    abandoned in a cleanup work.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
diff --git a/doc/changelog/v0.67.5.txt b/doc/changelog/v0.67.5.txt
new file mode 100644
index 000000000..2767fb85e
--- /dev/null
+++ b/doc/changelog/v0.67.5.txt
@@ -0,0 +1,711 @@
+commit a60ac9194718083a4b6a225fc17cad6096c69bd1
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Thu Dec 19 20:53:10 2013 +0000
+
+    v0.67.5
+
+commit 8745ba63533ba77b2ec9d40780cad822a577089c
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Dec 18 13:11:01 2013 -0800
+
+    rgw: fix use-after-free when releasing completion handle
+    
+    Backport: emperor, dumpling
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c8890ab2d46fe8e12200a0d2f9eab31c461fb871)
+
+commit 7587ee52161d74a41bf89dc741e8ece9c964fb67
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Dec 18 13:10:21 2013 -0800
+
+    rgw: don't return data within the librados cb
+    
+    Fixes: #7030
+    The callback is running within a single Finisher thread, thus we
+    shouldn't block there. Append read data to a list and flush it within
+    the iterate context.
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit d6a4f6adfaa75c3140d07d6df7be03586cc16183)
+
+commit 9bd1ebe960aae69f6528a33809a58c13481f8df3
+Author: Christophe Courtaut <christophe.courtaut@gmail.com>
+Date:   Mon Jul 22 15:15:38 2013 +0200
+
+    rgw: Allow wildcard in supported keystone roles.
+    
+    http://tracker.ceph.com/issues/4365 fixes #4365
+    
+    Signed-off-by: Christophe Courtaut <christophe.courtaut@gmail.com>
+    (cherry picked from commit 60195581273aee259e8c83d0b471af859d928342)
+
+commit e52b9b76844f935daf2dc060905f869248bba60b
+Author: Roald J. van Loon <roaldvanloon@gmail.com>
+Date:   Fri Aug 9 13:31:10 2013 +0200
+
+    Validate S3 tokens against Keystone
+    
+    - Added config option to allow S3 to use Keystone auth
+    - Implemented JSONDecoder for KeystoneToken
+    - RGW_Auth_S3::authorize now uses rgw_store_user_info on keystone auth
+    - Minor fix in get_canon_resource; dout is now after the assignment
+    
+    Reviewed-by: Yehuda Sadeh<yehuda@inktank.com>
+    Signed-off-by: Roald J. van Loon <roaldvanloon@gmail.com>
+    (cherry picked from commit a200e184b15a03a4ca382e94caf01efb41cb9db7)
+    
+    Conflicts:
+    	src/rgw/rgw_swift.h
+
+commit 5df2f35ee5797ffe85e8398b1ecf75f2eefb5031
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Oct 21 18:02:32 2013 -0700
+
+    rgw: init src_bucket_name, src_object in the S3 handler
+    
+    Be consistent and initialize these fields also in the S3 case.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit ccaab2abf9e49ce3e50c15df793a3a3ca6b29bb8)
+
+commit 0edee8e9c9b140a280dc782b491c330c7f86edfc
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Oct 21 14:45:59 2013 -0700
+
+    rgw: get rid of req_state.bucket_name
+    
+    No need for this field, as we already have req_state.bucket_name_str.
+    This saves us some memory allocation / freeing through every request
+    processing.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit db7eb771226beaa2372ef5860ce742cb3457cd89)
+    
+    Conflicts:
+    	src/rgw/rgw_rest_s3.cc
+
+commit 0eae05f31e4a03e923d51521ecd0078657eda9ba
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Oct 21 14:17:12 2013 -0700
+
+    rgw: turn swift COPY into PUT
+    
+    Fixes: #6606
+    The swift COPY operation is unique in a sense that it's a write
+    operation that has its destination not set by the URI target, but by a
+    different HTTP header. This is problematic as there are some hidden
+    assumptions in the code that the specified bucket/object in the URI is
+    the operation target. E.g., certain initialization functions, quota,
+    etc. Instead of creating a specialized code everywhere for this case
+    just turn it into a regular copy operation, that is, a PUT with
+    a specified copy source.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit cbf8f9a51737e7d89fb0ec0832b50fd47c35b08d)
+
+commit 455f28a78ee22b6a535831ff70a330fa1379567c
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Aug 13 20:32:14 2013 -0700
+
+    OSDMonitor: add 'osd perf' command to dump recent osd perf information
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 94c3f29a32cbf169d896015da6765febd3c724e0)
+
+commit ec086df1beccda6d5a6c12c727c84c2555b0b2cd
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Aug 13 17:36:57 2013 -0700
+
+    ObjectStore: add ObjectStore::get_cur_stats and impl in FileStore
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit ebde89d5602536b4bc651737e4280fdfb6634c32)
+
+commit 5bf7e7be1fa39ec39726635581e98a0e3a1df20f
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Aug 13 17:21:39 2013 -0700
+
+    osd_types: add osd_perf_stat_t type
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 95f3353a6ab3a4dd2bf28eaca7122487942e913e)
+
+commit b37c5d208778ee52292757e96265d59d75f19654
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Dec 3 11:22:56 2013 -0800
+
+    rgw: do not use UNPRIVILEGED_DAEMON option
+    
+    This is sort of a backport for 4f403c26dc0048ad63e20d20369fa86bfb31c50e
+    in that we get the same behavior change, but only for rgw.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit a447fb7d04fbad84f9ecb57726396bb6ca29d8f6
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Dec 3 21:51:26 2013 -0800
+
+    osd/OSDMonitor: accept 'osd pool set ...' value as string
+    
+    Newer monitors take this as a CephString.  Accept that so that if we are
+    mid-upgrade and get a forwarded message using the alternate schema from
+    a future mon we will handle it properly.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit 9875c8b1992c59cc0c40901a44573676cdff2669
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Nov 25 13:43:43 2013 -0800
+
+    init, upstart: prevent daemons being started by both
+    
+    There can be only one init system starting a daemon. If there is a
+    host entry in ceph.conf for a daemon, sysvinit would try to start it
+    even if the daemon's directory did not include a sysvinit file. This
+    preserves backwards compatibility with older installs using sysvinit,
+    but if an upstart file is present in the daemon's directory, upstart
+    will try to start them, regardless of host entries in ceph.conf.
+    
+    If there's an upstart file in a daemon's directory and a host entry
+    for that daemon in ceph.conf, both sysvinit and upstart would attempt
+    to manage it.
+    
+    Fix this by only starting daemons if the marker file for the other
+    init system is not present. This maintains backwards compatibility
+    with older installs using neither sysvinit or upstart marker files,
+    and does not break any valid configurations. The only configuration
+    that would break is one with both sysvinit and upstart files present
+    for the same daemon.
+    
+    Backport: emperor, dumpling
+    Reported-by: Tim Spriggs <tims@uahirise.org>
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 5e34beb61b3f5a1ed4afd8ee2fe976de40f95ace)
+
+commit bd08d332f5d23b6097a7e8344a6183896b2303a9
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Sun Nov 3 21:02:36 2013 -0800
+
+    OSD: allow project_pg_history to handle a missing map
+    
+    If we get a peering message for an old map we don't have, we
+    can throwit out: the sending OSD will learn about the newer
+    maps and update itself accordingly, and we don't have the
+    information to know if the message is valid. This situation
+    can only happen if the sender was down for a long enough time
+    to create a map gap and its PGs have not yet advanced from
+    their boot-up maps to the current ones, so we can rely on it
+    
+    Fixes: #6712
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit cd0d612e1abdf5c87082eeeccd4ca09dd14fd737)
+
+commit 8cd33e3a8ebf7c2aa796ec9f92d6b554c39ff705
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Aug 19 16:56:27 2013 -0700
+
+    rgw: bucket meta remove don't overwrite entry point first
+    
+    Fixes: #6056
+    
+    When removing a bucket metadata entry we first unlink the bucket
+    and then we remove the bucket entrypoint object. Originally
+    when unlinking the bucket we first overwrote the bucket entrypoint
+    entry marking it as 'unlinked'. However, this is not really needed
+    as we're just about to remove it. The original version triggered
+    a bug, as we needed to propagate the new header version first (which
+    we didn't do, so the subsequent bucket removal failed).
+    
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 0373d749cea7d9b532069ba8ebca2f005b2c9f59)
+
+commit a24f1326adcdda1d387e0f4e3c18dda5ec4aa055
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Nov 22 07:04:01 2013 -0800
+
+    rgw: lower some debug message
+    
+    Fixes: #6084
+    Backport: dumpling, emperor
+    
+    Reported-by: Ron Allred <rallred@itrefined.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit b35fc1bc2ec8c8376ec173eb1c3e538e02c1694e)
+
+commit 9e2d0d311437c1ffdd43fc8c961c60aacfea339c
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Nov 18 14:39:12 2013 -0800
+
+    osd: fix bench block size
+    
+    The command was declared to take 'size' in dumpling, but was trying to
+    read 'bsize' instead, so it always used the default of 4MiB. Change
+    the bench command to read 'size', so it matches what existing clients
+    are sending.
+    
+    Fixes: #6795
+    Backport: emperor, dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 40a76ef0d09f8ecbea13712410d9d34f25b91935)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+
+commit 35c91cac9a2b414ba3d52ea06cf6f71a0a1d4ba3
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed Sep 25 09:19:16 2013 -0700
+
+    os, osd, tools: Add backportable compatibility checking for sharded objects
+    
+    OSD
+      New CEPH_OSD_FEATURE_INCOMPAT_SHARDS
+    FileStore
+      NEW CEPH_FS_FEATURE_INCOMPAT_SHARDS
+      Add FSSuperblock with feature CompatSet in it
+      Store sharded_objects state using CompatSet
+      Add set_allow_sharded_objects() and get_allow_sharded_objects() to FileStore/ObjectStore
+      Add read_superblock()/write_superblock() internal filestore functions
+    ceph_filestore_dump
+      Add OSDsuperblock to export format
+      Use CompatSet from OSD code itself in filestore-dump tool
+      Always check compatibility of OSD features with on-disk features
+      On import verify compatibility of on-disk features with export data
+      Bump super_ver due to export format change
+    
+    Backport: dumpling, cuttlefish
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit c6b83180f9f769de27ca7890f5f8ec507ee743ca)
+    
+    Conflicts:
+    
+    	src/os/FileStore.cc
+    	src/os/FileStore.h
+    	src/osd/OSD.cc
+    
+    Excluded from cherry-pick:
+      Didn't add set_allow_sharded_objects() and get_allow_sharded_objects() to FileStore/ObjectStore
+      Didn't add code to check for incomplete transition to sharded objects in ceph-filestore-dump
+
+commit 372f62717c56d9ab883ae2942e13d6d8d37c2925
+Author: Ray Lv <xiangyulv@gmail.com>
+Date:   Tue Oct 29 19:34:51 2013 +0800
+
+    rgw: Wrap hex_to_num table into class HexTable
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    Signed-off-by: Ray Lv <xiangyulv@gmail.com>
+    (cherry picked from commit 588ed60a8ec1d8db5e096fd8f7b762f2afcf7dd3)
+
+commit 6917b02530103b8c86ed75592da33144b0dea168
+Merge: 7ef210a f1fa811
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Nov 5 12:27:42 2013 -0800
+
+    Merge pull request #805 from ceph/wip-rgw-replica-log-dumpling
+    
+    backport a bunch of rgw stuff to dumpling
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit f1fa8116d441924d44c99624829f3daa090c821c
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Fri Nov 1 16:12:52 2013 -0700
+
+    rgw: don't turn 404 into 400 for the replicalog api
+    
+    404 is not actually a problem to clients like radosgw-agent, but 400
+    implies something about the request was incorrect.
+    
+    Backport: dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 306ec7142d699c26ce874b11dc02ccdb3cf296c7)
+
+commit 739e7861f6d5fde5c4a2b59170377b6147fecfa3
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Sep 3 13:27:21 2013 -0700
+
+    rgw: when failing read from client, return correct error
+    
+    Fixes: #6214
+    When getting a failed read from client when putting an object
+    we returned the wrong value (always 0), which in the chunked-
+    upload case ended up in assuming that the write was done
+    successfully.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 0499948aad64c6c66668b43963403fb0cb1a2737)
+
+commit 465087e64ab2bd85d7f5e183565e3619b3aa76fc
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Sep 30 14:15:19 2013 -0700
+
+    rgw: fix leak in RGWMetadataManager::remove()
+    
+    Backport: dumpling
+    Fixes: #6445
+    handler->get() returns a newly allocated obj, need to delete it when
+    done.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 8dd7ea3fadc06b5ebb330af41d686d3da155fb5d)
+
+commit a4dc2694db6a7e45109f111dcdfd077da67b6030
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Aug 26 11:16:08 2013 -0700
+
+    rgw: quiet down warning message
+    
+    Fixes: #6123
+    We don't want to know about failing to read region map info
+    if it's not found, only if failed on some other error. In
+    any case it's just a warning.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 055e31359ac977943f014270cc56a1b7786abee3)
+
+commit 7f57d9dda74a1feb2da85be3c77d61c6f81ec1b2
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Sep 13 14:43:54 2013 -0700
+
+    rgw: try to create log pool if doesn't exist
+    
+    When using replica log, if the log pool doesn't exist all operations are
+    going to fail. Try to create it if doesn't exist.
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 4216eac0f59af60f60d4ce909b9ace87a7b64ccc)
+
+commit 4fb233cf702957e782083d760877afa48d4f6965
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Oct 11 16:56:46 2013 -0700
+
+    formatter: dump_bool dumps unquoted strings
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit ad409f8a6d230e9b1199226a333bb54159c2c910)
+
+commit de7ede925acc47a5aee84b9fe82b9cf0368aa1b2
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Sep 27 13:15:35 2013 -0700
+
+    Formatter: add dump_bool()
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 28949d5d43beba7cef37cb2f83e3399d978061a6)
+
+commit c5991f26febd86878aeb78baab33c071660fdee6
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Oct 24 08:42:48 2013 -0700
+
+    rgw: escape bucket and object names in StreamReadRequests
+    
+    This fixes copy operations for objects that contain unsafe characters,
+    like a newline, which would return a 403 otherwise, since the GET to
+    the source rgw would be unable to verify the signature on a partially
+    valid bucket name.
+    
+    Fixes: #6604
+    Backport: dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit ec45b3b88c485140781b23d2c4f582f2cc26ea43)
+
+commit 672679d26110dd93393bc18a681ae41175e4a299
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Oct 24 08:37:25 2013 -0700
+
+    rgw: move url escaping to a common place
+    
+    This is useful outside of the s3 interface. Rename url_escape()
+    url_encode() for consistency with the exsting common url_decode()
+    function. This is in preparation for the next commit, which needs
+    to escape url-unsafe characters in another place.
+    
+    Backport: dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit dd308cd481b368f90a64220847b91fc233d92a59)
+
+commit 91997fca6ec6c06beecd63c3e2ae521319fca7c2
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Oct 24 08:34:24 2013 -0700
+
+    rgw: update metadata log list to match data log list
+    
+    Send the last marker whether the log is truncated in the same format
+    as data log list, so clients don't have more needless complexity
+    handling the difference.  Keep bucket index logs the same, since they
+    contain the marker already, and are not used in exactly the same way
+    metadata and data logs are.
+    
+    Backport: dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit e0e8fb1b2b4a308b2a9317e10c6fd53ad48dbfaf)
+
+commit 8916648521e2362a54eed0f5ebe078a269e4d703
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Oct 24 08:26:19 2013 -0700
+
+    rgw: include marker and truncated flag in data log list api
+    
+    Consumers of this api need to know their position in the log. It's
+    readily available when fetching the log, so return it.  Without the
+    marker in this call, a client could not easily or efficiently figure
+    out its position in the log, since it would require getting the global
+    last marker in the log, and then reading all the log entries.
+    
+    This would be slow for large logs, and would be subject to races that
+    would cause potentially very expensive duplicate work.
+    
+    Returning this atomically while fetching the log entries simplifies
+    all of this.
+    
+    Fixes: #6615
+    Backport: dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit c275912509255f8bb4c854e181318b45ab0f8564)
+
+commit a49c064b7a1b4083818b6f950d7e0aade71c6afe
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Oct 24 08:18:19 2013 -0700
+
+    cls_log: always return final marker from log_list
+    
+    There's no reason to restrict returning the marker to the case where
+    less than the whole log is returned, since there's already a truncated
+    flag to tell the client what happened.
+    
+    Giving the client the last marker makes it easy to consume when the
+    log entries do not contain their own marker. If the last marker is not
+    returned, the client cannot get the last marker without racing with
+    updates to the log.
+    
+    Backport: dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit e74776f4176470122485a79a4c07e9c12c9fc036)
+
+commit 90101aea6818c2f43bf31243d912ec9143f68882
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Oct 10 15:50:39 2013 -0700
+
+    rgw: skip read_policy checks for system_users
+    
+    A system user should still be able to examine suspended buckets, and
+    get -ENOENT instead of -EACCESS for a deleted object.
+    
+    Fixes: #6616
+    Backport: dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit ea816c1c2fd47eab647d6fab96c9ca4bfeecd5bb)
+
+commit 7ef210adbd4645bd45e5387c9b03a122a7ca82df
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Oct 29 08:39:27 2013 -0700
+
+    common/crc32c: fix #ifdef to be x86_64 specific
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 29f4eebf63bcba9ea86b9a2e4d093b782b5fbf1e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Sep 24 08:05:38 2013 +0200
+
+    rgw: fix rgw test to reflect usage change
+    
+    otherwise src/test/cli/radosgw-admin/help.t fails when running make
+    check when run after a configure --with-radosgw
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit e50343e4423e20130035c860ba47a0edea876f7c)
+
+commit 0106569a88a92aa94fbc5efc6a0a142ad0de325d
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Oct 24 17:25:16 2013 -0700
+
+    rbd.py: increase parent name size limit
+    
+    64 characters isn't all that long. 4096 ought to be enough for anyone.
+    
+    Fixes: #6072
+    Backport: dumpling, cuttlefish
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 3c0042cde5a12de0f554a16b227ab437c6254ddd)
+
+commit 352d8c966c323a008362130eb979ae5dfa47494f
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Sep 23 14:10:36 2013 -0700
+
+    common/config: include --cluster in default usage message
+    
+    Clean it up a bit too.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 157754b3a0191c5ff534a84adbeed88025615898)
+
+commit 299ddd31b29e332dc5e76bc4f871e4769698665d
+Author: Yan, Zheng <zheng.z.yan@intel.com>
+Date:   Thu Oct 10 10:35:48 2013 +0800
+
+    mds: fix infinite loop of MDCache::populate_mydir().
+    
+    make MDCache::populate_mydir() only fetch bare-bone stray dirs.
+    After all stray dirs are populated, call MDCache::scan_stray_dir(),
+    it fetches incomplete stray dirs.
+    
+    Fixes: #4405
+    Signed-off-by: Yan, Zheng <zheng.z.yan@intel.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 007f06ec174d4ee5cfb578c8b3f1c96b2bb0c238)
+    
+    Conflicts:
+    
+    	src/mds/MDCache.h
+    
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit 7d99940467019bda74da83f3f1ebc2da46bd6967
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Oct 15 10:20:48 2013 -0700
+
+    rgw: fix authenticated users acl group check
+    
+    Fixes: #6553
+    Backport: bobtail, cuttlefish, dumpling
+    Authenticated users group acl bit was not working correctly. Check to
+    test whether user is anonymous was wrong.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit bebbd6cb7b71697b34b8f27652cabdc40c97a33b)
+
+commit e5736de834027a281d29afcf6743347a8db97d26
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Oct 15 10:55:07 2013 -0700
+
+    rgw: change default log level
+    
+    Fixes: #6554
+    Backport: cuttlefish, dumpling
+    Default log level was just too high, bring it down a bit.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 8d7dbf85472cfca9268d81ecf057ea078cf345b3)
+
+commit e6e9ee7145bcfc1774af738af607a653fe8dfe46
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Oct 4 13:00:26 2013 -0700
+
+    rgw: swift update obj metadata also add generic attrs
+    
+    Fixes: #6462
+    We were missing the generic attributes when we updated the object
+    metadata (operation that only exists in the swift api).
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit f2645e1c6d7383a0ace3b239f4304e353249c4bb)
+
+commit cd1c3c9e00e90b19e83c1f11a48e516a7de93665
+Author: Yan, Zheng <zheng.z.yan@intel.com>
+Date:   Wed Oct 2 11:43:12 2013 +0800
+
+    mds: return -EAGAIN if standby replay falls behind
+    
+    standby replay may fall behind and get -ENOENT when reading the
+    journal. return -EAGAIN in this case, it makes the MDS respawn itself.
+    
+    fixes: #5458
+    
+    Signed-off-by: Yan, Zheng <zheng.z.yan@intel.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d2cb2bf6bac83ac6db9df6cb876317d30e7493cc)
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit bd073eeac28d8cba969e5746c5e6adcb95820fdf
+Author: Yan, Zheng <zheng.z.yan@intel.com>
+Date:   Thu Sep 12 09:50:51 2013 +0800
+
+    mon/MDSMonitor: don't reset incarnation when creating newfs
+    
+    Fixes: #6279
+    
+    Signed-off-by: Yan, Zheng <zheng.z.yan@intel.com>
+    (cherry picked from commit 40613b700b87b495d67793101ae05d6ba58b2a9a)
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit 3a3724b782c53c92c45624ad02df9f0de77f1545
+Author: Yan, Zheng <zheng.z.yan@intel.com>
+Date:   Thu Sep 12 10:36:39 2013 +0800
+
+    osdc/ObjectCacher: finish contexts after dropping object reference
+    
+    The context to finish can be class C_Client_PutInode, which may drop
+    inode's last reference. So we should first drop object's reference,
+    then finish contexts.
+    
+    Signed-off-by: Yan, Zheng <zheng.z.yan@intel.com>
+    (cherry picked from commit b66ac77fa7aa3ff37804918c4308a348f239af09)
+
+commit 102f63993cacf3b3404256e62a3096c5836f2282
+Author: Sandon Van Ness <sandon@inktank.com>
+Date:   Tue Oct 8 12:08:08 2013 -0700
+
+    Go back to $PWD in fsstress.sh if compiling from source.
+    
+    Although fsstress was being called with a static path the directory
+    it was writing to was in the current directory so doing a cd to the
+    source directory that is made in /tmp and then removing it later
+    caused it to be unable to write the files in a non-existent dir.
+    
+    This change gets the current path first and cd's back into it after
+    it is done compiling fsstress.
+    
+    Issue #6479.
+    
+    Signed-off-by: Sandon Van Ness <sandon@inktank.com>
+    Reviewed-by: Alfredo Deza <alfredo.deza@inktank.com>
+
+commit 62db186a19f902b0a23d14034552c83db8de7ece
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Tue Aug 27 09:53:12 2013 -0700
+
+    ceph.spec.in:  radosgw package doesn't require mod_fcgi
+    
+    Fixes #5702
+    
+    Signed-off-by: Gary Lowell  <gary.lowell@inktank.com>
+
+commit 42fc9b49a7bbc242096946b381aafd435ebdd530
+Merge: ad85b8b 67ac3cf
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Fri Oct 4 19:51:32 2013 +0000
+
+    Merge branch 'dumpling' of jenkins:ceph/ceph into dumpling
+
+commit 67ac3cffa1066030ad840937958bd534e8b26272
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Oct 3 21:47:26 2013 -0700
+
+    ceph_test_rados: do not let rollback race with snap delete
+    
+    Note that the OSD behaves in a weird way when you rollback to a non-
+    existent snap, so the test probably isn't the only party at fault here.
+    
+    Fixes (test half of): #6254
+    Backport: dumpling, cuttlefish
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 55d279b98553ba4542219b126fc7159b20b18b1f)
+    
+    Conflicts:
+    
+    	src/test/osd/RadosModel.h
+    	src/test/osd/TestRados.cc
diff --git a/doc/changelog/v0.67.6.txt b/doc/changelog/v0.67.6.txt
new file mode 100644
index 000000000..510893a19
--- /dev/null
+++ b/doc/changelog/v0.67.6.txt
@@ -0,0 +1,1258 @@
+commit c71f8a60cf49fb3d4efb5ea75b2366cd7f167497 (tag: refs/tags/v0.67.6)
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Thu Feb 13 20:20:42 2014 +0000
+
+    release build 67.6
+    
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+
+commit 1c42a6e34bb8e318e001384270cda003add946a7
+Merge: 3e473d4 c5e12f9
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Feb 13 06:55:37 2014 -0800
+
+    Merge pull request #1232 from ceph/dumpling-7334
+    
+    backport ceph-disk improvements to dumpling
+    
+    http://pulpito.ceph.com/ubuntu-2014-02-12_16:52:33-ceph-deploy-dumpling-7334-testing-basic-plana/
+
+commit 3e473d4237697872498094b41fed19b414f8d3c4
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Fri Sep 27 16:23:09 2013 -0700
+
+    common,os: Remove filestore_xattr_use_omap option
+    
+    Now we operate just like when this was set to true
+    
+    Fixes: #6143
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit dc0dfb9e01d593afdd430ca776cf4da2c2240a20)
+
+commit c5e12f94efc07f0ff174c0f19494ee2e06d88dfa
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Wed Feb 12 16:43:59 2014 -0500
+
+    add support for absence of PATH
+    
+    Note that this commit is actually bisecting the changes from
+    Loic Dachary that touch ceph-disk only (ad515bf). As that changeset
+    also touches other files it causes conflicts that are not resolvable
+    for backporting it to dumpling.
+    
+    Signed-off-by: Alfredo Deza <alfredo@deza.pe>
+
+commit c531d75f7e86cee619e44c247298b50c37b81539
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Sep 9 22:27:23 2013 -0700
+
+    ceph-disk: make initial journal files 0 bytes
+    
+    The ceph-osd will resize journal files up and properly fallocate() them
+    so that the blocks are preallocated and (hopefully) contiguous.  We
+    don't need to do it here too, and getting fallocate() to work from
+    python is a pain in the butt.
+    
+    Fixes: #5981
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a786ad773cd33880075f1deb3691528d1afd03ec)
+
+commit 836ead3e42491597c192c63d3171021c5daecbc0
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Mon Feb 10 15:07:55 2014 -0500
+
+    alert the user about error messages from partx
+    
+    Signed-off-by: Alfredo Deza <alfredo@deza.pe>
+    (cherry picked from commit 9bcc42a3e6b08521694b5c0228b2c6ed7b3d312e)
+
+commit cff3547c60dc6d164f9a95eaa2706633e32756d0
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Fri Feb 7 11:55:01 2014 -0500
+
+    use partx for red hat or centos instead of partprobe
+    
+    Signed-off-by: Alfredo Deza <alfredo@deza.pe>
+    (cherry picked from commit 42900ff9da9f5adcac239a84ebf4d2e407c29699)
+
+commit 5ebd7c4520a9805f985cab1a0ba9311e19efa041
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Jan 28 17:26:58 2014 -0800
+
+    ceph-disk: run the right executables from udev
+    
+    When run by the udev rules, PATH is not defined. Thus,
+    ceph-disk-activate relies on its which() function to locate the
+    correct executable.  The which() function used os.defpath if none was
+    set, and this worked for anything using it.
+    
+    ad6b4b4b08b6ef7ae8086f2be3a9ef521adaa88c added a new default value to
+    PATH, so only /usr/bin was checked by callers that did not use
+    which(). This resulted in the mount command not being found when
+    ceph-disk-activate was run by udev, and thus osds failing to start
+    after being prepared by ceph-deploy.
+    
+    Make ceph-disk consistently use the existing helpers (command() and
+    command_check_call()) that use which(), so lack of PATH does not
+    matter. Simplify _check_output() to use command(),
+    another wrapper around subprocess.Popen.
+    
+    Fixes: #7258
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit d7b0c7faafd37e4ae8a1680edfa60c22b419cbd8)
+
+commit 4a7c4e0a81753dfb811396e7bfa1df72135ee12b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jan 3 16:06:55 2014 +0100
+
+    ceph-disk: implement --sysconfdir as /etc/ceph
+    
+    Replace hardcoded /etc/ceph with the SYSCONFDIR global variable and
+    implement the --sysconfdir option to override the default value.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit a71025d33621257b6fd6632516cfed2849ff1637)
+
+commit c8b9798a69bd1655a7131c7bce8a14c2c74aba0c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jan 3 16:03:09 2014 +0100
+
+    ceph-disk: implement --statedir as /var/lib/ceph
+    
+    Replace hardcoded /var/lib/ceph with the STATEDIR global variable and
+    implement the --statedir option to override the default value.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit ca713f48ae7a1fece2869f1a1c97d23ab33fb441)
+
+commit 79564874be35d4414092fbb80df3214d7f282b1c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jan 1 22:05:03 2014 +0100
+
+    ceph-disk: add copyright notice
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 51ee3c04777aaf6b9609dde9bc318b5c66c70787)
+
+commit c4b66bbdf09a8bb85ed14fbb643f40f87345463b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jan 1 22:11:30 2014 +0100
+
+    ceph-disk: create the data directory if it does not exist
+    
+    Instead of failing if the OSD data directory does not exist, create
+    it. Only do so if the data directory is not enforced to be a device via
+    the use of the --data-dev flag. The directory is not recursively created.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 306b099ab093bfac466d68fe1cb87367bc01e577)
+
+commit c473c65d4b6235fe20a97bd1d4741c12f86858e8
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Dec 31 00:03:42 2013 +0100
+
+    ceph-disk: run ceph-osd when --mark-init=none
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 0fcc081858fae4febbb6a613a93cfbbcedd5a320)
+
+commit 6ea59a3a0c38c9629036442aff652489eecce51a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Dec 30 23:57:39 2013 +0100
+
+    ceph-disk: implement --mark-init=none
+    
+    It is meant to be used when preparing and activating a directory that is
+    not to be used with init. No file is created to identify the init
+    system, no symbolic link is made to the directory in /var/lib/ceph
+    and the init scripts are not called.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit e773b68f4c89ac56b425c710d7dcdc3d74a92926)
+
+commit 3aae5b764c5698c665a708a3f6f2243dfee1a28d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jan 1 22:07:57 2014 +0100
+
+    ceph-disk: fsid is a known configuration option
+    
+    Use get_conf_with_default instead of get_conf because fsid is a known
+    ceph configuration option. It allows overriding via CEPH_ARGS which is
+    convenient for testing. Only options that are not found in config_opts.h
+    are fetch via get_conf.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit b65eb377f5e93ea85644e4c0939365fd7ac36072)
+
+commit a33ad6d150206232806ee466f301e41f15c631e9
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jan 1 22:06:29 2014 +0100
+
+    ceph-disk: use CalledProcessError.returncode
+    
+    CalledProcessError has no errno data member
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 97f516a1ddfb2d014e1f7e762c4155e4b9bcb90b)
+
+commit 6ae96d3ce757b05daeb7c1437778149b07248d47
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jan 1 22:13:15 2014 +0100
+
+    ceph-disk: display the command output on OSD creation failure
+    
+    The string form of a CalledProcessError instance does not include the
+    output datamember. Add it to the Error exception for debug purposes.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit d09af0fa50f322c9e59765f3badd497f5ca184d4)
+
+commit deefb3a05788b308631812c652e9eccaeb999d7f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Dec 30 23:07:27 2013 +0100
+
+    ceph-disk: which() uses PATH first
+    
+    Instead of relying on a hardcoded set of if paths. Although this has the
+    potential of changing the location of the binary being used by ceph-disk
+    on an existing installation, it is currently only used for sgdisk. It
+    could be disruptive for someone using a modified version of sgdisk but
+    the odds of this happening are very low.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 2b935bbf60bafb6dd488c0eb30f156fce1b9d197)
+
+commit 06bbea187293ac52aefbcf4e98c162bb3286341d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Dec 30 22:48:46 2013 +0100
+
+    ceph-disk: add --prepend-to-path to control execution
+    
+    /usr/bin is hardcoded in front of some ceph programs which makes it
+    impossible to control where they are located via the PATH.
+    
+    The hardcoded path cannot be removed altogether because it will most
+    likely lead to unexpected and difficult to diagnose problems for
+    existing installations where the PATH finds the program elsewhere.
+    
+    The --prepend-to-path flag is added and defaults to /usr/bin : it prepends
+    to the PATH environment variable. The hardcoded path is removed
+    and the PATH will be used: since /usr/bin is searched first, the
+    legacy behavior will not change.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit ad6b4b4b08b6ef7ae8086f2be3a9ef521adaa88c)
+
+commit ccf182241dc04c79e2377b6ba2f060dbfe6dddae
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Dec 30 17:15:29 2013 +0100
+
+    ceph-disk: make exception handling terse in main_activate_journal
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 908348b8047e8577ecf9133f2683f91423694416)
+
+commit d60e952f3c00d0e551b04c0d6486b11d9761be35
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Dec 30 17:13:47 2013 +0100
+
+    ceph-disk: do not hide main_activate() exceptions
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 55ca7bb2da73f1be1293710a635cfea42abd7682)
+
+commit 827edb93e3971fc7549fe831894776c6bf163346
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Dec 30 17:11:30 2013 +0100
+
+    ceph-disk: fix activate() indent
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 324804a81c37ff89f2488e2ba106033c0e6e119e)
+
+commit 27988cf9ffedf3289099e51f1e7c3f5b8d79d667
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Dec 30 17:09:35 2013 +0100
+
+    ceph-disk: remove noop try:
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit de0050596b5f56863c3486c1cd5e7ffea62e3d00)
+
+commit 75bdb5430d9403bed8bd2ccd6a9c5a30d63a189e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Dec 30 16:57:36 2013 +0100
+
+    ceph-disk: fix Error() messages formatting
+    
+    Mainly using % instead of ,
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit b82ccfbfa786cd5436b48ec38276c5a48028ce1d)
+
+commit 5df1eaa55bc7ebe68473831d06cf9819fa39baa6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Dec 30 12:26:20 2013 +0100
+
+    ceph-disk: prepare --data-dir must not override files
+    
+    ceph-disk does nothing when given a device that is already prepared. If
+    given a directory that already contains a successfully prepared OSD, it
+    will however override it.
+    
+    Instead of overriding the files in the osd data directory, return
+    immediately if the magic file exists. Make it so the magic file is
+    created last to accurately reflect the success of the OSD preparation.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 7dfe550ce18623cde4ae43a2416e31ef81381ab9)
+
+commit 2e8496a6e0b74ff351bc2a4f138ddc665d8c30ca
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Dec 15 15:34:17 2013 +0100
+
+    ceph-disk: zap needs at least one device
+    
+    If given no argument, ceph-disk zap should display the usage instead of
+    silently doing nothing. Silence can be confused with "I zapped all the
+    disks".
+    
+    http://tracker.ceph.com/issues/6981 fixes #6981
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 07888ef3fd4440332c8287d0faa9f23a32cf141c)
+
+commit 0fe2ac47a82d3caa9dca2a9009b68a14b577e35c
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Fri Dec 13 12:06:25 2013 -0500
+
+    use the new get_command helper in check_call
+    
+    Signed-off-by: Alfredo Deza <alfredo@deza.pe>
+    (cherry picked from commit 897dfc113fe3b86f3dda53172933bfd4f8089869)
+
+commit 8fc9f1b791248f08a311d619addc4465cac298f9
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Thu Dec 12 11:16:38 2013 -0500
+
+    use the absolute path for executables if found
+    
+    Signed-off-by: Alfredo Deza <alfredo@deza.pe>
+    (cherry picked from commit a9334a1c8c6681305e76b361377864d0dd1e3d34)
+
+commit a2de2f718c9e341c0b6ec2ebffdf03d0f9da0fbd
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Thu Dec 12 10:26:05 2013 -0500
+
+    remove trailing semicolon
+    
+    Signed-off-by: Alfredo Deza <alfredo@deza.pe>
+    (cherry picked from commit 43561f791607f5fd6f03d5421e1f30a29fb4194e)
+
+commit 2cd22b5d39760ac4b5baad0b05c676b833ae9213
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Wed Dec 11 15:41:45 2013 -0500
+
+    replace sgdisk subprocess calls with a helper
+    
+    Signed-off-by: Alfredo Deza <alfredo@deza.pe>
+    (cherry picked from commit e19e38012bc4579054f63865e682c8c3a7829c7b)
+
+commit e8cc27cc8eb981c71b8cafb58964317687ca5787
+Author: Jonathan Davies <jonathan.davies@canonical.com>
+Date:   Tue Dec 3 21:26:43 2013 +0000
+
+    Call --mbrtogpt on journal run of sgdisk should the drive require a GPT table.
+    
+    Signed-off-by: Jonathan Davies <jonathan.davies@canonical.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 35011e0b01d65e4c001876882d597451f2028345)
+
+commit 3942a93fae3331b2d08f93181eb2c9cf90b50440
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Nov 21 13:21:22 2013 +0100
+
+    ceph-disk: blacklist /dev/fd0
+    
+    blkid -s TYPE /dev/fd0 has been verified to hang forever on a
+    H8DMR-82 supermicro motherboard running
+    
+    3.8.0-33-generic #48~precise1-Ubuntu SMP Thu Oct 24 16:28:06 UTC 2013
+    x86_64
+    
+    It is unlikely that ceph will ever be used on floppy disks, they
+    can be blacklisted.
+    
+    http://tracker.ceph.com/issues/6827 fixes: #6827
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 65701978715237ff5a4c68573c0696fd9d438e4f)
+
+commit ccc6b0c5e6d47eefef5cf1fa17bf69c47c9459c4
+Author: git-harry <git-harry@live.co.uk>
+Date:   Thu Oct 3 10:32:50 2013 +0100
+
+    Make fsid comparison case-insensitive
+    
+    get_fsid and find_cluster_by_uuid are modified so ceph-disk activate and
+    ceph-disk activate-all will work if the fsid uses uppercase characters.
+    
+    Signed-off-by: Harry Harrington <git-harry@live.co.uk>
+    (cherry picked from commit 22f8325dbfce7ef2e97bf015c0f8bba53e75dfe9)
+
+commit 367cf1bbf86233eb20ff2304e7d6caab77b84fcc
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Feb 11 10:14:36 2014 -0800
+
+    librbd: remove limit on number of objects in the cache
+    
+    The number of objects is not a significant indicated of when data
+    should be written out for rbd. Use the highest possible value for
+    number of objects and just rely on the dirty data limits to trigger
+    flushing. When the number of objects is low, and many start being
+    flushed before they accumulate many requests, it hurts average request
+    size and performance for many concurrent sequential writes.
+    
+    Fixes: #7385
+    Backport: emperor, dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 0559d31db29ea83bdb6cec72b830d16b44e3cd35)
+
+commit 4856fc2950442fafc208c967af5131f7874f5441
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Feb 11 11:53:00 2014 -0800
+
+    ObjectCacher: use uint64_t for target and max values
+    
+    All the options are uint64_t, but the ObjectCacher was converting them
+    to int64_t. There's never any reason for these to be negative, so
+    change the type.
+    
+    Adjust a few conditionals so that they only convert known-positive
+    signed values to uint64_t before comparing with the target and max
+    values. Leave the actual stats accounting as loff_t for now, since
+    bugs in accounting will have bad effects if negative values wrap
+    around.
+    
+    Backport: emperor, dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit db034acf546a72739ff6543241543f3bd651f3ae)
+
+commit a572aafe178dfa3ab4d82aa5c7f4a72a63590ccf
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Feb 11 10:35:14 2014 -0800
+
+    ObjectCacher: remove max_bytes and max_ob arguments to trim()
+    
+    These are never passed, so replace them with the defaults.
+    
+    Backport: emperor, dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit bf8cf2d6d21a204a099347f3dcd5b48100b8c445)
+
+commit 4889234f80dbd95e5b2012ff9bfc2d87960034e2
+Merge: 6af657b 30a604d
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Feb 11 08:32:37 2014 -0800
+
+    Merge pull request #1210 from dachary/dumpling
+    
+    common: admin socket fallback to json-pretty format (dumpling)
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit 30a604d2d6a5de7bcbfa0136e9a766cb34f96c0a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 10 23:42:38 2014 +0100
+
+    common: admin socket fallback to json-pretty format
+    
+    If the format argument to a command sent to the admin socket is not
+    among the supported formats ( json, json-pretty, xml, xml-pretty ) the
+    new_formatter function will return null and the AdminSocketHook::call
+    function must fall back to a sensible default.
+    
+    The CephContextHook::call and HelpHook::call failed to do that and a
+    malformed format argument would cause the mon to crash. A check is added
+    to each of them and fallback to json-pretty if the format is not
+    recognized.
+    
+    To further protect AdminSocketHook::call implementations from similar
+    problems the format argument is checked immediately after accepting the
+    command in AdminSocket::do_accept and replaced with json-pretty if it is
+    not known.
+    
+    A test case is added for both CephContextHook::call and HelpHook::call
+    to demonstrate the problem exists and is fixed by the patch.
+    
+    Three other instances of unsafe calls to new_formatter were found and
+    a fallback to json-pretty was added. All other calls have been audited
+    and appear to be safe.
+    
+    http://tracker.ceph.com/issues/7378 fixes #7378
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 165e76d4d03ffcc490fd3c2ba60fb37372990d0a)
+
+commit 6af657bd4cf87f6f7afe84257941e7e2366183d9
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Feb 5 17:26:02 2014 -0800
+
+    qa: add script for testing rados client timeout options
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 9e62beb80b6c92a97ec36c0db5ea39e417661b35)
+
+commit 21a0b67440eaa07fa051d26321a88654ee9e8ae7
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Feb 5 17:25:24 2014 -0800
+
+    rados: check return values for commands that can now fail
+    
+    A few places were not checking the return values of commands, since
+    they could not fail before timeouts were added.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 79c1874346ff55e2dc74ef860db16ce70242fd00)
+
+commit 2c42dd26b49d01922bd3df3ba26af70da0387a99
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Feb 5 17:24:16 2014 -0800
+
+    librados: check and return on error so timeouts work
+    
+    Some functions could not previously return errors, but they had an
+    int return value, which can now receive ETIMEDOUT.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 8e9459e897b1bc2f97d52ee07701fd22069efcf3)
+
+commit 28fe564cc298b96c063e6cca44304046a86942cd
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Feb 5 17:22:14 2014 -0800
+
+    msg/Pipe: add option to restrict delay injection to specific msg type
+    
+    This makes it possible to test timeouts reliably by delaying certain
+    messages effectively forever, but still being able to e.g. connect and
+    authenticate to the monitors.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit d389e617c1019e44848330bf9570138ac7b0e5d4)
+
+commit 1d2ea9efb844cb902d282c3a752acc45fefeb04c
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Feb 3 18:30:00 2014 -0800
+
+    MonClient: add a timeout on commands for librados
+    
+    Just use the conf option directly, since librados is the only caller.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 671a76d64bc50e4f15f4c2804d99887e22dcdb69)
+
+commit 30dafacd0b54bb98b01284851e0d5abf76324e95
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Feb 3 17:59:21 2014 -0800
+
+    Objecter: implement mon and osd operation timeouts
+    
+    This captures almost all operations from librados other than mon_commands().
+    
+    Get the values for the timeouts from the Objecter constructor, so only
+    librados uses them.
+    
+    Add C_Cancel_*_Op, finish_*_op(), and *_op_cancel() for each type of
+    operation, to mirror those for Op. Create a callback and schedule it
+    in the existing timer thread if the timeouts are specified.
+    
+    Fixes: #6507
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 3e1f7bbb4217d322f4e0ece16e676cd30ee42a20)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+    	src/osd/ReplicatedPG.cc
+    	src/osdc/Objecter.cc
+    	src/osdc/Objecter.h
+
+commit 2aa0d7589283fe5879a966601fbfe690816cc4a1
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Feb 3 12:53:15 2014 -0800
+
+    librados: add timeout to wait_for_osdmap()
+    
+    This is used by several pool operations independent of the objecter,
+    including rados_ioctx_create() to look up the pool id in the first
+    osdmap.
+    
+    Unfortunately we can't just rely on WaitInterval returning ETIMEDOUT,
+    since it may also get interrupted by a signal, so we can't avoid
+    keeping track of time explicitly here.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 1829d2c9fd13f2cbae4e192c9feb553047dad42c)
+    
+    Conflicts:
+    	src/librados/RadosClient.cc
+
+commit 55d254d5ea113d8be29afe18dc9940f14d93ebb4
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Feb 3 12:09:34 2014 -0800
+
+    conf: add options for librados timeouts
+    
+    These will be implemented in subsequent patches.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 0dcceff1378d85ca6d81d102d201890b8a71af6b)
+
+commit ccd9b5eb9bcfe2360e874b306d4decd8815ecf46
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Feb 9 20:53:00 2014 -0800
+
+    test_striper: fix warning
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit bb44c0d5e06c2b699c81a76373a0edde1bf90445
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Feb 8 12:21:26 2014 -0800
+
+    crushtool: add cli test for off-by-one tries vs retries bug
+    
+    See bug #7370.  This passes on dumpling and breaks prior to the #7370 fix.
+    
+    Backport: emperor, dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ed32c4002fb5cb1dd546331651eaf7de1a017471)
+
+commit c301ec689caac5bbcd21e02226045d3454cc82fd
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Feb 3 08:54:14 2014 -0800
+
+    client: use 64-bit value in sync read eof logic
+    
+    The file size can jump to a value that is very much larger than our current
+    position (for example, it could be a disk image file that gets a sparse
+    write at a large offset).  Use a 64-bit value so that 'some' doesn't
+    overflow.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: John Spray <john.spray@inktank.com>
+    (cherry picked from commit 7ff2b541c24d1c81c3bcfbcb347694c2097993d7)
+
+commit 183deb899bc6b1b7b2a1ec639425e45786e56b01
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 4 22:39:35 2014 -0800
+
+    osd: do not send peering messages during init
+    
+    Do not send any peering messages while we are still working our way
+    through init().
+    
+    Fixes: #7093
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 35da8f9d80e0c6c33fb6c6e00f0bf38f1eb87d0e)
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+
+commit 3a1fb17401897b0717cf1d264c6e6fdf99fc703f
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jan 28 18:46:37 2014 -0800
+
+    OSDMap: fix deepish_copy_from
+    
+    Start with a shallow copy!
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d0f13f54146694a197535795da15b8832ef4b56f)
+    
+    Conflicts:
+    
+    	src/osd/OSDMap.h
+
+commit f01bd8a94adc4ce8e7131f74282de5b434a7316a
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jan 23 13:48:28 2014 -0800
+
+    rgw: fix listing of multipart upload parts
+    
+    Fixes: #7169
+    
+    There are two issues here. One is that we may return more entries than
+    we should (as specified by max_parts). Second issue is that the
+    NextPartNumberMarker is set incorrectly. Both of these issues mainly
+    affect uploads with > 1000 parts, although can be triggered with less
+    than that.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 6b657992d2397e399eacf554357ed807fda399fa
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Nov 21 06:52:48 2013 -0800
+
+    rgw: initialize RGWUserAdminOpState::system_specified
+    
+    Fixes: #6829
+    Backport: dumpling, emperor
+    We didn't init this member variable, which might cause that when
+    modifying user info that has this flag set the 'system' flag might
+    inadvertently reset.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 561e7b0b287e65e90b80699e45a52ae44e94684f)
+
+commit 585e0e7eec1bbee60fe352166b593d53476003f8
+Author: Robin H. Johnson <robbat2@gentoo.org>
+Date:   Sun Dec 15 12:26:19 2013 -0800
+
+    rgw: Fix CORS allow-headers validation
+    
+    This fix is needed because Ceph presently validates CORS headers in a
+    case-sensitive manner. Keeps a local cache of lowercased allowed headers
+    to avoid converting the allowed headers to lowercase each time.
+    
+    CORS 6.2.6: If any of the header field-names is not a ASCII
+    case-insensitive match for any of the values in list of headers do not
+    set any additional headers and terminate this set of steps.
+    
+    Signed-off-by: Robin H. Johnson <robbat2@gentoo.org>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 31b60bfd9347a386ff12b4e4f1812d664bcfff01)
+
+commit da2267a87e37afa28385e915a566de26fd784b70
+Author: Robin H. Johnson <robbat2@gentoo.org>
+Date:   Sun Dec 15 11:40:31 2013 -0800
+
+    rgw: Clarify naming of case-change functions
+    
+    It is not clear that the lowercase_http_attr & uppercase_http_attr
+    functions replace dashes with underscores. Rename them to match the
+    pattern established by the camelcase_dash_http_attr function in
+    preperation for more case-change functions as needed by later fixes.
+    
+    Signed-off-by: Robin H. Johnson <robbat2@gentoo.org>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 6a7edab2673423c53c6a422a10cb65fe07f9b235)
+
+commit fdea76ef6f7d7c4ee89c30d430f5495072023957
+Author: Robin H. Johnson <robbat2@gentoo.org>
+Date:   Sun Dec 15 11:27:49 2013 -0800
+
+    rgw: Look at correct header about headers for CORS
+    
+    The CORS standard dictates that preflight requests are made with the
+    Access-Control-Request-Headers header containing the headers of the
+    author request. The Access-Control-Allow-Headers header is sent in the
+    response.
+    
+    The present code looks for Access-Control-Allow-Headers in request, so
+    fix it to look at Access-Control-Request-Headers instead.
+    
+    Signed-off-by: Robin H. Johnson <robbat2@gentoo.org>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 2abacd9678ae04cefac457882ba718a454948915)
+
+commit a5739e3903f8c30c8168199413818922b4a29bad
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Dec 6 11:07:09 2013 -0800
+
+    rgw: fix reading bucket policy in RGWBucket::get_policy()
+    
+    Fixes: 6940
+    Backport: dumpling, emperor
+    
+    We changed the way we keep the bucket policy, and we shouldn't try to
+    access the bucket object directly. This had changed when we added the
+    bucket instance object around dumpling.
+    
+    Reported-by: Gao, Wei M <wei.m.gao@intel.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 7a9a088d82d04f6105d72f6347673724ac16c9f8)
+
+commit ba792cdc8d0c006e260e6a02a33ca1491b35901c
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jan 16 11:45:27 2014 -0800
+
+    rgw: handle racing object puts when object doesn't exist
+    
+    If the object didn't exist before and now we have multiple puts coming
+    in concurrently, we need to make sure that we behave correctly. Only one
+    needs to win, the other one can fail silently. We do that by setting
+    exclusive flag on the object creation and handling the error correctly.
+    Note that we still want to return -EEXIST in some cases (when the
+    exclusive flag is passed to put_obj_meta(), e.g., on bucket creation).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 86c15480fc3e33b9a3b84d0af68d8398fc732bae)
+
+commit 1e2984ac4a4ed0508e1b5efc8b69ae4da0cac824
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jan 16 11:33:49 2014 -0800
+
+    rgw: don't return -ENOENT in put_obj_meta()
+    
+    Fixes: #7168
+    
+    An object put may race with the same object's delete. In this case just
+    ignore the error, same behavior as if object was created and then
+    removed.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 5c24a7ea905587fd4077e3b0cfc0f5ad2b178c29)
+
+commit 5f1eea2ccf1ee4296530fc754bfd5dfc1571cd62
+Author: Robin H. Johnson <robbat2@gentoo.org>
+Date:   Sat Jan 18 18:01:20 2014 -0800
+
+    rgw: Use correct secret key for POST authn
+    
+    The POST authentication by signature validation looked up a user based
+    on the access key, then used the first secret key for the user. If the
+    access key used was not the first access key, then the expected
+    signature would be wrong, and the POST would be rejected.
+    
+    Signed-off-by: Robin H. Johnson <robbat2@gentoo.org>
+    (cherry picked from commit 586ad1f69e6260ef298805647911b3b49f8be7cf)
+
+commit 14e933eb2e8adf3e7df2aef823d863c593067928
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jan 28 15:54:33 2014 +0000
+
+    osd: OSDMonitor: ignore pgtemps from removed pool
+    
+    There's a window in-between receiving an MOSDPGTemp message from an OSD
+    and actually handling it that may lead to the pool the pg temps refer to
+    no longer existing. This may happen if the MOSDPGTemp message is queued
+    pending dispatching due to an on-going proposal (maybe even the pool
+    removal).
+    
+    This patch fixes such behavior in two steps:
+    
+    1. Check if the pool exists in the osdmap upon preprocessing
+     - if pool does not exist in the osdmap, then the pool must have been
+       removed prior to handling the message, but after the osd sent it.
+     - safe to ignore the pg update
+    2. If all pg updates in the message have been ignored, ignore the whole
+       message.  Otherwise, let prepare handle the rest.
+    
+    3. Recheck if pool exists in the osdmap upon prepare
+     - We may have ignored this pg back in preprocess, but other pgs in the
+       message may have led the message to be passed on to prepare; ignore
+       pg update once more.
+    4. Check if pool is pending removal and ignore pg update if so.
+    
+    We delegate checking the pending value to prepare_pgtemp() because in this
+    case we should only ignore the update IFF the pending value is in fact
+    committed.  Otherwise we should retry the message.  prepare_pgtemp() is
+    the appropriate place to do so.
+    
+    Fixes: 7116
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit f513f66f48383a07c70ca18a4dba6c2449ea9860)
+
+commit 3672fc49bea5efcef2069673c88fd8e526eeb7e5
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jan 28 11:00:34 2014 -0800
+
+    OSDMonitor: use deepish_copy_from for remove_down_pg_temp
+    
+    This is a backport of 368852f6c0a884b8fdc80a5cd6f9ab72e814d412.
+    
+    Make a deep copy of the OSDMap to avoid clobbering the in-memory copy with
+    the call to apply_incremental.
+    
+    Fixes: #7060
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit e6fef68ee51a2e3be6537086246ce860819788eb
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 24 11:03:26 2014 -0800
+
+    OSDMap: deepish_copy_from()
+    
+    Make a deep(ish) copy of another OSDMap.  Unfortunatley we can't make the
+    compiler-generated copy operator/constructors private until c++11.  :(
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit bd54b9841b9255406e56cdc7269bddb419453304)
+
+commit 2528186c0dd327607b1e76a51581df72aeea52f2
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jan 28 10:26:12 2014 -0800
+
+    buffer: make 0-length splice() a no-op
+    
+    This was causing a problem in the Striper, but fixing it here will avoid
+    corner cases all over the tree.  Note that we have to bail out before
+    the end-of-buffer check to avoid hitting that check when the bufferlist is
+    also empty.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit ff5abfbdae07ae8a56fa83ebaa92000896f793c2)
+
+commit fe9539e118a830aa71bc5846ad2693543213fc59
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jan 28 10:09:17 2014 -0800
+
+    osdc/Striper: test zero-length add_partial_result
+    
+    If we add a partial result that is 0-length, we used to hit an assert in
+    buffer::list::splice().  Add a unit test to verify the fix.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 28c7388d320a47657c2e12c46907f1bf40672b08)
+
+commit 9801718e6e7a335d1aeffb7f3d958b012d8dd79c
+Author: Derek Yarnell <derek@umiacs.umd.edu>
+Date:   Mon Jan 27 12:27:51 2014 -0700
+
+    packaging: apply udev hack rule to RHEL
+    
+    In the RPM spec file there is a test to deploy the uuid hack udev rules
+    for older udev operating systems. This includes CentOS and RHEL, but the
+    check currently only is for CentOS, causing RHEL clients to get a bogus
+    osd rules file.
+    
+    Adjust the conditional to apply to RHEL as well as CentOS. (The %{rhel}
+    macro is defined in both platforms' redhat-rpm-config package.)
+    
+    Fixes http://tracker.ceph.com/issues/7245
+    
+    Signed-off-by: Ken Dreyer <ken.dreyer@inktank.com>
+    (cherry picked from commit 64a0b4fa563795bc22753940aa3a4a2946113109)
+
+commit 5817078ba9b2aa38f39e1f62d8d08e943646c0bb
+Merge: 9fa8d2d abf6441
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jan 23 07:12:37 2014 -0800
+
+    Merge pull request #1129 from ceph/wip-dumpling-backport-6620
+    
+    mds: MDSMap: adjust buffer size for uint64 values with more than 5 chars
+
+commit abf64417bbc63b006c53a07262a5774d0d40550e
+Author: Joao Eduardo Luis <jecluis@gmail.com>
+Date:   Thu Jan 23 11:36:16 2014 +0000
+
+    mds: MDSMap: adjust buffer size for uint64 values with more than 5 chars
+    
+    Fixes: #6620
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 0e8182edd850f061421777988974efbaa3575b9f)
+
+commit 9fa8d2d1bd4a8cf56fdebae484bee35c3930fb62
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jan 21 11:29:56 2014 -0800
+
+    mon/MDSMonitor: do not generate mdsmaps from already-laggy mds
+    
+    There is one path where a mds that is not sending its beacon (e.g.,
+    because it is not running at all) will lead to proposal of new mdsmaps.
+    Fix it.
+    
+    Backport: emperor, dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 584c2dd6bea3fe1a3c7f306874c054ce0cf0d2b5)
+
+commit a2e34b8d322e571600d6c0a3ff363f9910196f28
+Author: John Spray <john.spray@inktank.com>
+Date:   Mon Jan 20 11:08:27 2014 +0000
+
+    Fix #7187: Include all summary items in JSON health output
+    
+    Signed-off-by: John Spray <john.spray@inktank.com>
+    (cherry picked from commit fdf3b5520d150f14d90bdfc569b70c07b0579b38)
+
+commit ee9fc590a556354b54a174a4790c4142949d7409
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Jan 6 18:32:42 2014 -0800
+
+    rgw: convert bucket info if needed
+    
+    Fixes: #7110
+    In dumpling, the bucket info was separated into bucket entry point and
+    bucket instance objects. When setting bucket attrs we only ended up
+    updating the bucket instance object. However, pre-dumpling buckets still
+    keep everything at the entry-point object, so acl changes didn't affect
+    anything (because we never updated the entry point). This change just
+    converts the bucket info into the new format.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit a5f8cc7ec9ec8bef4fbc656066b4d3a08e5b215b)
+
+commit 69a99e6edd67aadaea77fb5ebcf46190dbfcdbc6
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 4 22:40:43 2014 -0800
+
+    osd: ignore OSDMap messages while we are initializing
+    
+    The mon may occasionally send OSDMap messages to random OSDs, but is not
+    very descriminating in that we may not have authenticated yet.  Ignore any
+    messages if that is the case; we will reqeust whatever we need during the
+    BOOTING state.
+    
+    Fixes: #7093
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f68de9f352d53e431b1108774e4a23adb003fe3f)
+
+commit 1545ddbd3b6dfac897d483bd58a4335d68b9b62a
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jan 4 22:43:26 2014 -0800
+
+    mon: only send messages to current OSDs
+    
+    When choosing a random OSD to send a message to, verify not only that
+    the OSD id is up but that the session is for the same instance of that OSD
+    by checking that the address matches.
+    
+    Fixes: #7093
+    Backport: emperor, dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 98ed9ac5fed6eddf68f163086df72faabd9edcde)
+
+commit 4d88dd10bfab4e5fb45632245be5f79eeba73a30
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Aug 26 13:58:47 2013 -0700
+
+    osd: discriminate based on connection messenger, not peer type
+    
+    Replace ->get_source().is_osd() checks and instead see if it is the
+    cluster_messenger so that we do not confuse ourselves when we get
+    legit requests from other OSDs on our public interface.
+    
+    NOTE: backporting this because a mixed cluster may send OSD requests
+    via the client interface, even though dumpling doesn't do this.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a6b04c5d8ba043727a2e39a62e9d4126485bcfeb)
+    
+    Conflicts:
+    
+    	src/osd/OSD.cc
+
+commit 2354b957b7924af3e81f559d5c975262848c3bf6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Dec 15 22:59:51 2013 +0100
+
+    qa: test for error when ceph osd rm is EBUSY
+    
+    http://tracker.ceph.com/issues/6824 fixes #6824
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 31507c90f0161c4569a2cc634c0b5f671179440a)
+
+commit 10ad2e01f832ca9293a1bda1c82f7d318e8c3589
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Dec 15 16:27:02 2013 +0100
+
+    mon: set ceph osd (down|out|in|rm) error code on failure
+    
+    Instead of always returning true, the error code is set if at least one
+    operation fails.
+    
+    EINVAL if the OSD id is invalid (osd.foobar for instance).
+    EBUSY if trying to remove and OSD that is up.
+    
+    When used with the ceph command line, it looks like this:
+    
+        ceph -c ceph.conf osd rm osd.0
+        Error EBUSY: osd.0 is still up; must be down before removal.
+        kill PID_OF_osd.0
+        ceph -c ceph.conf osd down osd.0
+        marked down osd.0.
+        ceph -c ceph.conf osd rm osd.0 osd.1
+        Error EBUSY: removed osd.0, osd.1 is still up; must be down before removal.
+    
+    http://tracker.ceph.com/issues/6824 fixes #6824
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 15b8616b13a327701c5d48c6cb7aeab8fcc4cafc)
+
+commit db82bfec50349783c3bb708c65cf89bceff7b2ca
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Oct 29 20:30:37 2013 +0000
+
+    mon: OSDMonitor: fix some annoying whitespace
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 42c4137cbfacad5654f02c6608cc0e81b45c06be)
+
+commit 1abb16988727a21dff61edca48bfcb05c60d26f2
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Dec 26 17:38:52 2013 -0800
+
+    librbd: call user completion after incrementing perfcounters
+    
+    The perfcounters (and the ictx) are only valid while the image is
+    still open.  If the librbd user gets the callback for its last I/O,
+    then closes the image, the ictx and its perfcounters will be
+    invalid. If the AioCompletion object is has not run the rest of its
+    complete() method yet, it will access these now-invalid addresses,
+    possibly leading to a crash.
+    
+    The AioCompletion object is independent of the ictx and does not
+    access it again after incrementing perfcounters, so avoid this race by
+    calling the user's callback after this step. The AioCompletion object
+    will be cleaned up by the rest of complete_request(), independent of
+    the ImageCtx.
+    
+    Fixes: #5426
+    Backport: dumpling, emperor
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 4cea7895da7331b84d8c6079851fdc0ff2f4afb1)
+
+commit 4dae50d8a0ffe3f3491eae62950fe46229aaa0e6
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Fri Dec 6 16:03:20 2013 -0800
+
+    objecter: don't take extra throttle budget for resent ops
+    
+    These ops have already taken their budget in the original op_submit().
+    It will be returned via put_op_budget() when they complete.
+    If there were many localized reads of missing objects from replicas,
+    or cache pool redirects, this would cause the objecter to use up all
+    of its op throttle budget and hang.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 8d0180b1b7b48662daef199931efc7f2a6a1c431)
+    
+    Conflicts:
+    
+    	src/osdc/Objecter.cc
+
+commit 4944618fe74e1875e37a5de9a5bfe3922f906942
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Dec 5 17:44:37 2013 -0800
+
+    rbd: check write return code during bench-write
+    
+    This is allows rbd-bench to detect http://tracker.ceph.com/issues/6938
+    when combined with rapidly changing the mon osd full ratio.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 3caf3effcb113f843b54e06099099909eb335453)
+
+commit 3a7a0353d7c8521a2c3c25778801d45f0ca10957
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Dec 5 17:36:33 2013 -0800
+
+    objecter: resend all writes after osdmap loses the full flag
+    
+    Now that the osd does not respond if it gets a map with the full flag
+    set first, clients need to resend all writes.
+    
+    Clients talking to old osds are still subject to the race condition,
+    so both sides must be upgraded to avoid it.
+    
+    Refs: #6938
+    Backport: dumpling, emperor
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit e32874fc5aa6f59494766b7bbeb2b6ec3d8f190e)
+    
+    Conflicts:
+    
+    	src/osdc/Objecter.h
+
+commit f4d0fc8afac1bcbea76f46744397a8462025f373
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Dec 5 17:34:38 2013 -0800
+
+    osd: drop writes when full instead of returning an error
+    
+    There's a race between the client and osd with a newly marked full
+    osdmap.  If the client gets the new map first, it blocks writes and
+    everything works as expected, with no errors from the osd.
+    
+    If the osd gets the map first, however, it will respond to any writes
+    with -ENOSPC. Clients will pass this up the stack, and not retry these
+    writes later.  -ENOSPC isn't handled well by all clients. RBD, for
+    example, may pass it on to qemu or kernel rbd which will both
+    interpret it as EIO.  Filesystems on top of rbd will not behave well
+    when they receive EIOs like this, especially if the cluster oscillates
+    between full and not full, so some writes succeed.
+    
+    To fix this, never return ENOSPC from the osd because of a map marked
+    full, and rely on the client to retry all writes when the map is no
+    longer marked full.
+    
+    Old clients talking to osds with this fix will hang instead of
+    propagating an error, but only if they run into this race
+    condition. ceph-fuse and rbd with caching enabled are not affected,
+    since the ObjectCacher will retry writes that return errors.
+    
+    Refs: #6938
+    Backport: dumpling, emperor
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 4111729dda7437c23f59e7100b3c4a9ec4101dd0)
+
+commit 2b109803b9434d2d495e65d6a045838ee3ebf169
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Nov 6 16:55:52 2013 -0800
+
+    objecter: clean pause / unpause logic
+    
+    op->paused holds now whether operation should be paused or not, and it's
+    being updated when scanning requests. No need to do a second scan.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 5fe3dc647bf936df8e1eb2892b53f44f68f19821)
+
+commit 7b229fca346b7ab60131b229591d35f63f64a6ad
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Nov 6 16:15:47 2013 -0800
+
+    objecter: set op->paused in recalc_op_target(), resend in not paused
+    
+    When going through scan_requests() in handle_osd_map() we need to make
+    sure that if an op should not be paused anymore then set it on the op
+    itself, and return NEED_RESEND. Otherwise we're going to miss reset of
+    the full flag.
+    Also in handle_osd_map(), make sure that op shouldn't be paused before
+    sending it. There's a lot of cleanup around that area that we should
+    probably be doing, make the code much more tight.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 98ab7d64a191371fa39d840c5f8e91cbaaa1d7b7)
+
+commit a3fdfd77c953a8f9ace4991630599e60f56abc4a
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Nov 6 10:46:37 2013 +0800
+
+    objecter: don't resend paused ops
+    
+    Paused ops are meant to block on the client side until a new map that
+    unpauses them is recieved. If we send paused writes when the FULL flag
+    is set, we'll get -ENOSPC from the osds, which is not what Objecter
+    users expect. This may cause rbd without caching to produce an I/O
+    error instead of waiting for the cluster to have capacity.
+    
+    Fixes: #6725
+    Backport: dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit c5c399d327cfc0d232d9ec7d49ababa914d0b21a)
+
+commit 78037886ae6ae278d8fb2db52d7767b8612313fe
+Merge: a60ac91 4e47a00
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Dec 29 21:10:30 2013 -0800
+
+    Merge pull request #808 from ceph/wip-6152-dumpling
+    
+    rgw: Fix S3 auth when using response-* query string params
+
+commit 4e47a003bb3adcee690067d4037c248a93023c20
+Author: Sylvain Munaut <s.munaut@whatever-company.com>
+Date:   Thu Aug 29 16:17:30 2013 +0200
+
+    rgw: Fix S3 auth when using response-* query string params
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    Signed-off-by: Sylvain Munaut <s.munaut@whatever-company.com>
+    (cherry picked from commit 7a7361d7e798d94796d4985e2e4b35af22463ae2)
diff --git a/doc/changelog/v0.67.7.txt b/doc/changelog/v0.67.7.txt
new file mode 100644
index 000000000..71442f66b
--- /dev/null
+++ b/doc/changelog/v0.67.7.txt
@@ -0,0 +1,110 @@
+commit d7ab4244396b57aac8b7e80812115bbd079e6b73 (tag: refs/tags/v0.67.7, refs/remotes/gh/dumpling)
+Author: Ken Dreyer <ken.dreyer@inktank.com>
+Date:   Wed Feb 19 18:09:39 2014 +0000
+
+    v0.67.7
+    
+    Signed-off-by: Ken Dreyer <ken.dreyer@inktank.com>
+
+commit c1f42770f45205c8a3eae774d09e747408130c50
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Jan 6 12:53:58 2014 -0800
+
+    radosgw-admin: fix object policy read op
+    
+    Fixes: #7083
+    This was broken when we fixed #6940. We use the same function to both
+    read the bucket policy and the object policy. However, each needed to be
+    treated differently. Restore old behavior for objects.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit b1976dd00f5b29c01791272f63a18250319f2edb)
+
+commit 404500811adc74c79583bbf3122dc333e0b41b51
+Merge: 3e7ea33 fec55a2
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Feb 15 22:20:45 2014 -0800
+
+    Merge pull request #1243 from dachary/wip-ceph-disk-dumpling
+    
+    ceph-disk: unit tests (dumpling)
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit 3e7ea33cfc80031aed8d6c61e1bdca947b36d69b
+Author: Ken Dreyer <ken.dreyer@inktank.com>
+Date:   Fri Jan 3 16:38:25 2014 -0700
+
+    add autotools-generated files to .gitignore
+    
+    When running "make check", Automake generates test-suite.log, along with
+    various *.log and *.trs files in the tree. Add these files to
+    .gitignore.
+    
+    (It looks like this feature arrived in Automake 1.13.)
+    
+    Signed-off-by: Ken Dreyer <ken.dreyer@inktank.com>
+    (cherry picked from commit bb8b7503b03fac5830fb71b9723963fdc803ca90)
+
+commit fec55a2174b49ef68c816755c1218f9cd69033b6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sat Feb 15 18:34:04 2014 +0100
+
+    ceph-disk: unit tests
+    
+    src/test/ceph-disk.sh replaces src/test/cli/ceph-disk/data-dir.t
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+
+commit 0679b11b4d2f7cb7bd9c1cd86854a064aaa83f8a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Jan 2 22:42:17 2014 +0100
+
+    ceph-disk: cannot run unit tests
+    
+    Because ceph-disk relies on hardcoded paths. The corresponding test will
+    be added back when ceph-disk can run from sources.
+    
+    Fixes: #7085
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 2ba6930d62263a39f150ab43bf8cd860b9245188)
+
+commit 9dae9993e9b36e741e05804807bdb738fbe52e6f
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Feb 14 23:52:06 2014 -0800
+
+    Revert "librbd: remove limit on number of objects in the cache"
+    
+    This reverts commit 367cf1bbf86233eb20ff2304e7d6caab77b84fcc.
+    
+    Removing the limit on objects means we leak memory, since Objects without
+    any buffers can exist in the cache.
+
+commit 2795339d3632df975486d51e69762f9f567fb43d
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Oct 7 05:22:20 2013 -0700
+
+    os/FileStore: fix ENOENT error code for getattrs()
+    
+    In commit dc0dfb9e01d593afdd430ca776cf4da2c2240a20 the omap xattrs code
+    moved up a block and r was no longer local to the block.  Translate
+    ENOENT -> 0 to compensate.
+    
+    Fix the same error in _rmattrs().
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 6da4b91c07878e07f23eee563cf1d2422f348c2f)
+
+commit c158de232fcfb0e39f4b7eaac6dacbaaaee84b5f
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Oct 15 16:00:26 2013 -0700
+
+    test/filestore/run_seed_to.sh: avoid obsolete --filestore-xattr-use-omap
+    
+    This option no longer exists.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1d4f501a015727a7ff4b2f9b20dc91f2bbd9707b)
diff --git a/doc/changelog/v0.67.8.txt b/doc/changelog/v0.67.8.txt
new file mode 100644
index 000000000..2f8e0c944
--- /dev/null
+++ b/doc/changelog/v0.67.8.txt
@@ -0,0 +1,1220 @@
+commit ffc58b4ea6a3ea10e91050aaa28d14d87dea5656 (tag: refs/tags/v0.67.8, refs/remotes/gh/dumpling)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Thu May 1 11:18:24 2014 +0000
+
+    0.67.8
+
+commit 4b16b70c53be83481efefcf394eca99c73bb9805
+Merge: 5a6b351 fb0944e
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Apr 30 15:15:48 2014 -0700
+
+    Merge pull request #1743 from ceph/wip-mon-backports.dumpling
+    
+    mon: OSDMonitor: HEALTH_WARN on 'mon osd down out interval == 0'
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit fb0944e22acf6f8b6cefb59cc4c41dc48087bfd7
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Apr 30 17:13:30 2014 +0100
+
+    mon: OSDMonitor: HEALTH_WARN on 'mon osd down out interval == 0'
+    
+    A 'status' or 'health' request will return a HEALTH_WARN whenever the
+    monitor handling the request has the option set to zero.
+    
+    Fixes: 7784
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit b2112d5087b449d3b019678cb266ff6fa897897e)
+
+commit 5a6b35160417423db7c6ff892627f084ab610dfe
+Author: Sandon Van Ness <sandon@inktank.com>
+Date:   Tue Mar 4 16:15:15 2014 -0800
+
+    Make symlink  of librbd to qemu's folder so it can detect it.
+    
+    Per issue #7293.
+    
+    Signed-off-by: Sandon Van Ness <sandon@inktank.com>
+    (cherry picked from commit 65f3354903fdbdb81468a84b8049ff19c00f91ba)
+
+commit 735a90a95eea01dbcce5026758895117c2842627
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Apr 25 14:11:27 2014 -0700
+
+    rgw: fix url escaping
+    
+    Fixes: #8202
+    This fixes the radosgw side of issue #8202. Needed to cast value
+    to unsigned char, otherwise it'd get padded.
+    
+    Backport: dumpling
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit bcf92c496aba0dfde432290fc2df5620a2767313)
+
+commit 438b57890dfce04226d769389a601d35b74e11fe
+Merge: c049967 476b929
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Apr 25 16:00:24 2014 -0700
+
+    Merge pull request #1700 from xanpeng/patch-1
+    
+    Fix error in mkcephfs.rst
+    
+    Signed-off-by: Xan Peng <xanpeng@gmail.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit 476b929ecc5b7351a5be3024817b900976a90a3e
+Author: xanpeng <xanpeng@gmail.com>
+Date:   Mon Apr 21 11:30:42 2014 +0800
+
+    Update mkcephfs.rst
+    
+    There should be no blank between mount options.
+
+commit c049967af829497f8a62e0cbbd6031f85ead8a59
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Apr 1 17:27:01 2014 -0700
+
+    auth: add rwlock to AuthClientHandler to prevent races
+    
+    For cephx, build_authorizer reads a bunch of state (especially the
+    current session_key) which can be updated by the MonClient. With no
+    locks held, Pipe::connect() calls SimpleMessenger::get_authorizer()
+    which ends up calling RadosClient::get_authorizer() and then
+    AuthClientHandler::bulid_authorizer(). This unsafe usage can lead to
+    crashes like:
+    
+    Program terminated with signal 11, Segmentation fault.
+    0x00007fa0d2ddb7cb in ceph::buffer::ptr::release (this=0x7f987a5e3070) at common/buffer.cc:370
+    370 common/buffer.cc: No such file or directory.
+    in common/buffer.cc
+    (gdb) bt
+    0x00007fa0d2ddb7cb in ceph::buffer::ptr::release (this=0x7f987a5e3070) at common/buffer.cc:370
+    0x00007fa0d2ddec00 in ~ptr (this=0x7f989c03b830) at ./include/buffer.h:171
+    ceph::buffer::list::rebuild (this=0x7f989c03b830) at common/buffer.cc:817
+    0x00007fa0d2ddecb9 in ceph::buffer::list::c_str (this=0x7f989c03b830) at common/buffer.cc:1045
+    0x00007fa0d2ea4dc2 in Pipe::connect (this=0x7fa0c4307340) at msg/Pipe.cc:907
+    0x00007fa0d2ea7d73 in Pipe::writer (this=0x7fa0c4307340) at msg/Pipe.cc:1518
+    0x00007fa0d2eb44dd in Pipe::Writer::entry (this=<value optimized out>) at msg/Pipe.h:59
+    0x00007fa0e0f5f9d1 in start_thread (arg=0x7f987a5e4700) at pthread_create.c:301
+    0x00007fa0de560b6d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
+    
+    and
+    
+    Error in `qemu-system-x86_64': invalid fastbin entry (free): 0x00007ff12887ff20
+    *** ======= Backtrace: =========
+    /lib/x86_64-linux-gnu/libc.so.6(+0x80a46)[0x7ff3dea1fa46]
+    /usr/lib/librados.so.2(+0x29eb03)[0x7ff3e3d43b03]
+    /usr/lib/librados.so.2(_ZNK9CryptoKey7encryptEP11CephContextRKN4ceph6buffer4listERS4_RSs+0x71)[0x7ff3e3d42661]
+    /usr/lib/librados.so.2(_Z21encode_encrypt_enc_blIN4ceph6buffer4listEEvP11CephContextRKT_RK9CryptoKeyRS2_RSs+0xfe)[0x7ff3e3d417de]
+    /usr/lib/librados.so.2(_Z14encode_encryptIN4ceph6buffer4listEEiP11CephContextRKT_RK9CryptoKeyRS2_RSs+0xa2)[0x7ff3e3d41912]
+    /usr/lib/librados.so.2(_ZN19CephxSessionHandler12sign_messageEP7Message+0x242)[0x7ff3e3d40de2]
+    /usr/lib/librados.so.2(_ZN4Pipe6writerEv+0x92b)[0x7ff3e3e61b2b]
+    /usr/lib/librados.so.2(_ZN4Pipe6Writer5entryEv+0xd)[0x7ff3e3e6c7fd]
+    /lib/x86_64-linux-gnu/libpthread.so.0(+0x7f8e)[0x7ff3ded6ff8e]
+    /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d)[0x7ff3dea99a0d]
+    
+    Fix this by adding an rwlock to AuthClientHandler. A simpler fix would
+    be to move RadosClient::get_authorizer() into the MonClient() under
+    the MonClient lock, but this would not catch all uses of other
+    Authorizer, e.g. for verify_authorizer() and it would serialize
+    independent connection attempts.
+    
+    This mainly matters for cephx, but none and unknown can have the
+    global_id reset as well.
+    
+    Partially-fixes: #6480
+    Backport: dumpling, emperor
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 2cc76bcd12d803160e98fa73810de2cb916ef1ff)
+
+commit 2b4b00b76b245b1ac6f95e4537b1d1a4656715d5
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Apr 1 11:37:29 2014 -0700
+
+    pipe: only read AuthSessionHandler under pipe_lock
+    
+    session_security, the AuthSessionHandler for a Pipe, is deleted and
+    recreated while the pipe_lock is held. read_message() is called
+    without pipe_lock held, and examines session_security. To make this
+    safe, make session_security a shared_ptr and take a reference to it
+    while the pipe_lock is still held, and use that shared_ptr in
+    read_message().
+    
+    This may have caused crashes like:
+    
+    *** Error in `qemu-system-x86_64': invalid fastbin entry (free): 0x00007f42a4002de0 ***
+    ======= Backtrace: =========
+    /lib/x86_64-linux-gnu/libc.so.6(+0x80a46)[0x7f452f1f3a46]
+    /usr/lib/x86_64-linux-gnu/libnss3.so(PK11_FreeSymKey+0xa8)[0x7f452e72ff98]
+    /usr/lib/librados.so.2(+0x2a18cd)[0x7f453451a8cd]
+    /usr/lib/librados.so.2(_ZNK9CryptoKey7encryptEP11CephContextRKN4ceph6buffer4listERS4_RSs+0x71)[0x7f4534519421]
+    /usr/lib/librados.so.2(_Z21encode_encrypt_enc_blIN4ceph6buffer4listEEvP11CephContextRKT_RK9CryptoKeyRS2_RSs+0xfe)[0x7f453451859e]
+    /usr/lib/librados.so.2(_Z14encode_encryptIN4ceph6buffer4listEEiP11CephContextRKT_RK9CryptoKeyRS2_RSs+0xa2)[0x7f45345186d2]
+    /usr/lib/librados.so.2(_ZN19CephxSessionHandler23check_message_signatureEP7Message+0x246)[0x7f4534516866]
+    /usr/lib/librados.so.2(_ZN4Pipe12read_messageEPP7Message+0xdcc)[0x7f453462ecbc]
+    /usr/lib/librados.so.2(_ZN4Pipe6readerEv+0xa5c)[0x7f453464059c]
+    /usr/lib/librados.so.2(_ZN4Pipe6Reader5entryEv+0xd)[0x7f4534643ecd]
+    /lib/x86_64-linux-gnu/libpthread.so.0(+0x7f8e)[0x7f452f543f8e]
+    /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d)[0x7f452f26da0d]
+    
+    Partially-fixes: #6480
+    Backport: dumpling, emperor
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 1d74170a4c252f35968ccfbec8e432582e92f638)
+
+commit 48895a46015c9d6d67543816f5a400c21aa206b1 (refs/remotes/gh/wip-objectcacher-flusher-dumpling)
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jan 3 12:51:15 2014 -0800
+
+    osdc/ObjectCacher: back off less during flush
+    
+    In cce990efc8f2a58c8d0fa11c234ddf2242b1b856 we added a limit to avoid
+    holding the lock for too long.  However, if we back off, we currently
+    wait for a full second, which is probably a bit much--we really just want
+    to give other threads a chance.
+    
+    Backport: emperor
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e2ee52879e9de260abbf5eacbdabbd71973a6a83)
+
+commit f3b5ba6f25010291a2918bdd286f1b39570bb907
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Oct 1 09:28:29 2013 -0700
+
+    osdc/ObjectCacher: limit writeback IOs generated while holding lock
+    
+    While analyzing a log from Mike Dawson I saw a long stall while librbd's
+    objectcacher was starting lots (many hundreds) of IOs.  Limit the amount of
+    time we spend doing this at a time to allow IO replies to be processed so
+    that the cache remains responsive.
+    
+    I'm not sure this warrants a tunable (which we would need to add for both
+    libcephfs and librbd).
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cce990efc8f2a58c8d0fa11c234ddf2242b1b856)
+
+commit 06f27fc6446d47b853208357ec4277c5dc10d9fe
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Apr 8 10:52:43 2014 -0700
+
+    os/FileStore: reset journal state on umount
+    
+    We observed a sequence like:
+    
+     - replay journal
+       - sets JournalingObjectStore applied_op_seq
+     - umount
+     - mount
+       - initiate commit with prevous applied_op_seq
+     - replay journal
+       - commit finishes
+       - on replay commit, we fail assert op > committed_seq
+    
+    Although strictly speaking the assert failure is harmless here, in general
+    we should not let state leak through from a previous mount into this
+    mount or else assertions are in general more difficult to reason about.
+    
+    Fixes: #8019
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4de49e8676748b6ab4716ff24fd0a465548594fc)
+
+commit b29238729f87c73dfdcf16dddcf293577678dea2
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Nov 5 14:54:20 2013 -0800
+
+    rgw: deny writes to a secondary zone by non-system users
+    
+    Fixes: #6678
+    We don't want to allow regular users to write to secondary zones,
+    otherwise we'd end up with data inconsistencies.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 6961b5254f16ac3362c3a51f5490328d23640dbf)
+    
+    Conflicts:
+    	src/rgw/rgw_rados.h
+
+commit 051a17eb008d75aa6b0737873318a2e7273501ab
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Apr 5 16:58:55 2014 -0700
+
+    mon: wait for quorum for MMonGetVersion
+    
+    We should not respond to checks for map versions when we are in the
+    probing or electing states or else clients will get incorrect results when
+    they ask what the latest map version is.
+    
+    Fixes: #7997
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 67fd4218d306c0d2c8f0a855a2e5bf18fa1d659e)
+
+commit 0716516da05eee967796fb71eb2f85c86afc40f1
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Feb 19 08:59:07 2014 -0800
+
+    rgw: fix swift range response
+    
+    Fixes: #7099
+    Backport: dumpling
+    The range response header was broken in swift.
+    
+    Reported-by: Julien Calvet <julien.calvet@neurea.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 0427f61544529ab4e0792b6afbb23379fe722de1)
+
+commit 94a1deefcfe525a7e698a1ae70a3bb561b6157de
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Nov 22 15:41:49 2013 -0800
+
+    rgw: don't log system requests in usage log
+    
+    Fixes: 6889
+    System requets should not be logged in the usage log.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 42ef8ba543c7bf13c5aa3b6b4deaaf8a0f9c58b6)
+
+commit 23fed8fc427e7077c61f86168a42f61a5f73867d
+Author: Greg Farnum <greg@inktank.com>
+Date:   Fri Apr 4 16:06:05 2014 -0700
+
+    OSD: _share_map_outgoing whenever sending a message to a peer
+    
+    This ensures that they get new maps before an op which requires them (that
+    they would then request from the monitor).
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 232ac1a52a322d163d8d8dbc4a7da4b6a9acb709)
+
+commit c45e15fd5cbe57a34c743b2835ecc30ee5a43963
+Author: Xihui He <xihuihe@gmail.com>
+Date:   Mon Dec 30 12:04:10 2013 +0800
+
+    msgr: fix rebind() race
+    stop the accepter and mark all pipes down before rebind to avoid race
+    
+    Fixes: #6992
+    
+    Signed-off-by: Xihui He xihuihe@gmail.com
+    (cherry picked from commit f8e413f9c79a3a2a12801f5f64a2f612de3f06a0)
+
+commit 3d31cf012a59e1fea8080b13bdc06c9021ba0656
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Nov 26 13:20:21 2013 -0800
+
+    PG: retry GetLog() each time we get a notify in Incomplete
+    
+    If for some reason there are no up OSDs in the history which
+    happen to have usable copies of the pg, it's possible that
+    there is a usable copy elsewhere on the cluster which will
+    become known to the primary if it waits.
+    
+    Fixes: #6909
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 964c8e978f86713e37a13b4884a6c0b9b41b5bae)
+
+commit 1f80bbdf45439c7224ed52e4956973fc6d007848
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Mar 17 15:37:44 2014 -0700
+
+    os/FileJournal: return errors on make_writeable() if reopen fails
+    
+    This is why #7738 is resulting in a crash instead of an error.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit aed074401d2834a5b04edd1b7f6b4f36336f6293)
+
+commit 62d942294a54208cdc82aebf8b536d164cae5dc6
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Mar 17 16:21:17 2014 -0700
+
+    mon/Paxos: commit only after entire quorum acks
+    
+    If a subset of the quorum accepts the proposal and we commit, we will start
+    sharing the new state.  However, the mon that didn't yet reply with the
+    accept may still be sharing the old and stale value.
+    
+    The simplest way to prevent this is not to commit until the entire quorum
+    replies.  In the general case, there are no failures and this is just fine.
+    In the failure case, we will call a new election and have a smaller quorum
+    of (live) nodes and will recommit the same value.
+    
+    A more performant solution would be to have a separate message invalidate
+    the old state and commit once we have all invalidations and a majority of
+    accepts.  This will lower latency a bit in the non-failure case, but not
+    change the failure case significantly.  Later!
+    
+    Fixes: #7736
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit fa1d957c115a440e162dba1b1002bc41fc1eac43)
+
+commit 2160c72b393893896be581e89a42c4e37b79cb23
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Mar 13 14:04:19 2014 -0700
+
+    PrioritizedQueue: cap costs at max_tokens_per_subqueue
+    
+    Otherwise, you can get a recovery op in the queue which has a cost
+    higher than the max token value.  It won't get serviced until all other
+    queues also do not have enough tokens and higher priority queues are
+    empty.
+    
+    Fixes: #7706
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 2722a0a487e77ea2aa0d18caec0bdac50cb6a264)
+
+commit 1dd42e0f75fe1f5363f92bd5a4506812e54b8fb5
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Thu Apr 3 13:59:59 2014 -0700
+
+    Fix byte-order dependency in calculation of initial challenge
+    
+    Fixes: #7977
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4dc62669ecd679bc4d0ef2b996b2f0b45b8b4dc7)
+
+commit c66b61f9dcad217429e4876d27881d9fb2e7666f
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Dec 31 17:00:06 2013 -0800
+
+    rbd: return 0 and an empty list when pool is entirely empty
+    
+    rbd_list will return -ENOENT when no rbd_directory object
+    exists. Handle this in the cli tool and interpret it as success with
+    an empty list.
+    
+    Add this to the release notes since it changes command line behavior.
+    
+    Fixes: #6693
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit ac547a5b7dc94282f079aef78e66348d99d9d5e9)
+    
+    Conflicts:
+    	PendingReleaseNotes
+    	src/rbd.cc
+
+commit 60b7aa96abfe09f7e9a263fa3f9b72c556dee8cb
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Nov 20 18:35:34 2013 -0800
+
+    test: use older names for module setup/teardown
+    
+    setUp and tearDown require nosetests 0.11, but 0.10.4 is the latest on
+    centos. Rename to use the older aliases, which still work with newer
+    versions of nosetests as well.
+    
+    Fixes: #6368
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit f753d56a9edba6ce441520ac9b52b93bd8f1b5b4)
+
+commit b405bfa49ec31f0c6d8636c4bdde17ee1f81deb7
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Sun Nov 3 11:06:10 2013 -0800
+
+    OSD: don't clear peering_wait_for_split in advance_map()
+    
+    I really don't know why I added this...  Ops can be discarded from the
+    waiting_for_pg queue if we aren't primary simply because there must have
+    been an exchange of peering events before subops will be sent within a
+    particular epoch.  Thus, any events in the waiting_for_pg queue must be
+    client ops which should only be seen by the primary.  Peering events, on
+    the other hand, should only be discarded if we are in a new interval,
+    and that check might as well be performed in the peering wq.
+    
+    Fixes: #6681
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 9ab513334c7ff9544bac07bd420c6d5d200cf535)
+
+commit a498c940bd630cb103d17ad8532a11122439411d
+Merge: 225fc97 80e0a0a
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Apr 2 12:57:30 2014 -0700
+
+    Merge remote-tracking branch 'gh/wip-7888-dumpling' into dumpling
+
+commit 225fc97f228490dfc13c2e4deed8fecffdb28c5e
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Nov 5 21:48:53 2013 -0800
+
+    PG: fix operator<<,log_wierdness log bound warning
+    
+    Split may cause holes such that head != tail and yet
+    log.empty().
+    
+    Fixes: #6722
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit c6826c1e8a301b2306530c6e5d0f4a3160c4e691)
+
+commit 26eeab43f3f703a25e7ba62c75d0382d15e38263
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Nov 5 17:47:48 2013 -0800
+
+    PGLog::rewind_divergent_log: log may not contain newhead
+    
+    Due to split, there may be a hole at newhead.
+    
+    Fixes: #6722
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit f4648bc6fec89c870e0c47b38b2f13496742b10f)
+
+commit 040abd75ad45bbcc05b24c9dddbd2026dd35e659
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Mar 29 14:23:21 2014 -0700
+
+    qa/workunits/fs/misc/layout_vxattrs: ceph.file.layout is not listed
+    
+    As of 08a3d6bd428c5e78dd4a10e6ee97540f66f9729c.  A similar change was made
+    in the kernel.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4f9f7f878953b29cd5f56a8e0834832d6e3a9cec)
+
+commit f5aa492a338ff711f3a45e4ebbf0d6b187b5f78e
+Merge: fef70cb 84cb345
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 28 18:01:08 2014 -0700
+
+    Merge pull request #1519 from ceph/wip-6951-dumpling
+    
+    rgw: reset objv tracker on bucket recreation
+
+commit fef70cbb52cf1ad12db45998b38858d9bbc3360d
+Merge: 9bfbce3 f443ff3
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 28 17:02:39 2014 -0700
+
+    Merge pull request #1559 from ceph/wip-7881-dumpling
+    
+    Wip 7881 dumpling
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit 80e0a0a8fee2f6f903f612734b2cc72eae703eae (refs/remotes/gh/wip-7888-dumpling)
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Mar 27 21:33:21 2014 -0700
+
+    mon/MonClient: use keepalive2 to verify the mon session is live
+    
+    Verify that the mon is responding by checking the keepalive2 reply
+    timestamp.  We cannot rely solely on TCP timing out and returning an
+    error.
+    
+    Fixes: #7888
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 056151a6334c054505c54e59af40f203a0721f28)
+
+commit 8723218379e80725a449b0594a4b15eb1c236b05
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Mar 27 21:09:13 2014 -0700
+
+    msgr: add KEEPALIVE2 feature
+    
+    This is similar to KEEPALIVE, except a timestamp is also exchanged.  It is
+    sent with the KEEPALIVE, and then returned with the ACK.  The last
+    received stamp is stored in the Connection so that it can be queried for
+    liveness.  Since all of the users of keepalive are already regularly
+    triggering a keepalive, they can check the liveness at the same time.
+    
+    See #7888.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d747d79fd5ea8662a809c5636dfd2eaaa9bf8f5d)
+    
+    Conflicts:
+    
+    	src/include/ceph_features.h
+
+commit a2f0974f8b3567c0385494a0b2c828ade6ca8e1c
+Author: Greg Farnum <greg@inktank.com>
+Date:   Wed Mar 26 15:58:10 2014 -0700
+
+    Pipe: rename keepalive->send_keepalive
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 38d4c71a456c1cc9a5044dbcae5378836a34484d)
+
+commit 9bfbce30678742515025ca235c4443bb3a69199f
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Mar 26 21:52:00 2014 -0700
+
+    client: pin Inode during readahead
+    
+    Make sure the Inode does not go away while a readahead is in progress.  In
+    particular:
+    
+     - read_async
+       - start a readahead
+       - get actual read from cache, return
+     - close/release
+       - call ObjectCacher::release_set() and get unclean > 0, assert
+    
+    Fixes: #7867
+    Backport: emperor, dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f1c7b4ef0cd064a9cb86757f17118d17913850db)
+
+commit 232445578a2c6d0fb974e55378057fce473095f7
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 28 12:34:07 2014 -0700
+
+    osdc/ObjectCacher: call read completion even when no target buffer
+    
+    If we do no assemble a target bl, we still want to return a valid return
+    code with the number of bytes read-ahead so that the C_RetryRead completion
+    will see this as a finish and call the caller's provided Context.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 032d4ec53e125ad91ad27ce58da6f38dcf1da92e)
+
+commit f443ff3006d41a7b0a2d7b649e8def0ffef6df12
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Oct 30 16:54:39 2013 -0700
+
+    PGLog: remove obsolete assert in merge_log
+    
+    This assert assumes that if olog.head != log.head, olog contains
+    a log entry at log.head, which may not be true since pg splitting
+    might have left the log with arbitrary holes.
+    
+    Related: 0c2769d3321bff6e85ec57c85a08ee0b8e751bcb
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 353813b2e1a98901b876790c7c531f8a202c661d)
+
+commit 636e53c0f4fc43e9bfc1c8e7214cab9e0b46a359
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Sep 30 15:54:27 2013 -0700
+
+    PGLog: on split, leave log head alone
+    
+    This way last_update doesn't go backwards.
+    
+    Fixes: 6447
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 0c2769d3321bff6e85ec57c85a08ee0b8e751bcb)
+
+commit a0a560a9f04311306a9784fa3c6ea2586d637f56
+Merge: 466cd53 41d5e9a
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Mar 26 17:18:24 2014 -0700
+
+    Merge pull request #1539 from ceph/wip-6910-dumpling
+    
+    PG: don't query unfound on empty pgs
+
+commit 41d5e9ab39e69c80bec1cb0627004c3fae6dc81d
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Nov 26 19:17:59 2013 -0800
+
+    PG: don't query unfound on empty pgs
+    
+    When the replica responds, it responds with a notify
+    rather than a log, which the primary then ignores since
+    it is already in the peer_info map.  Rather than fix that
+    we'll simply not send queries to peers we already know to
+    have no unfound objects.
+    
+    Fixes: #6910
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 838b6c8387087543ce50837277f7f6b52ae87d00)
+
+commit 466cd536ed7e541a36d88bce43683a2d9e2ca283
+Merge: 2ef0d6a c188949
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 21 14:53:23 2014 -0700
+
+    Merge pull request #1313 from ceph/dumpling-osd-subscribe
+    
+    Dumpling backport: clean up osd subscriptions
+
+commit 2ef0d6a25bf8c0cfb38768c157c29ba52295f3ca
+Merge: 77e46d0 bdd96c6
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Mar 21 14:52:20 2014 -0700
+
+    Merge pull request #1485 from ceph/wip-7212.dumpling
+    
+    backport 7212 fixes to dumpling
+
+commit 84cb345e4f12a9b1db5e384411492a9d88f17dd8
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Feb 19 08:11:56 2014 -0800
+
+    rgw: reset objv tracker on bucket recreation
+    
+    Fixes: #6951
+    If we cannot create a new bucket (as it already existed), we need to
+    read the old bucket's info. However, this was failing as we were holding
+    the objv tracker that we created for the bucket creation. We need to
+    clear it, as subsequent read using it will fail.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 859ed33ed7f9a96f4783dfb3e130d5eb60c622dd)
+
+commit 77e46d0d7984f2d3ee0e15f27d2961a637c20b45
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Nov 6 14:33:03 2013 -0800
+
+    ReplicatedPG: don't skip missing if sentries is empty on pgls
+    
+    Formerly, if sentries is empty, we skip missing.  In general,
+    we need to continue adding items from missing until we get
+    to next (returned from collection_list_partial) to avoid
+    missing any objects.
+    
+    Fixes: #6633
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit c7a30b881151e08b37339bb025789921e7115288)
+
+commit bdd96c620f33fb8f48f30f8d543af3290e6c934a
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Feb 15 08:59:51 2014 -0800
+
+    mon/Elector: bootstrap on timeout
+    
+    Currently if an election times out we call a new
+    election.  If we have never joined a quorum, bootstrap
+    instead. This is heavier weight, but captures the case
+    where, during bootstrap:
+    
+     - a and b have learned each others' addresses
+     - everybody calls an election
+     - a and b form a quorum
+     - c loops trying to call an election, but is ignored
+       because a and b don't see its address in the monmap
+    
+    See logs:
+      ubuntu@teuthology:/var/lib/teuthworker/archive/sage-2014-02-14_13:50:04-ceph-deploy-wip-7212-sage-b-testing-basic-plana/83194
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a4bcb1f8129a4ece97bd3419abf1ff45d260ad8e)
+    (cherry picked from commit 143ec0281aa8b640617a3fe19a430248ce3b514c)
+
+commit 68fcc63c0423a2071b7b944ea6c3448282a78a09
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Feb 14 11:25:52 2014 -0800
+
+    mon: tell MonmapMonitor first about winning an election
+    
+    It is important in the bootstrap case that the very first paxos round
+    also codify the contents of the monmap itself in order to avoid any manner
+    of confusing scenarios where subsequent elections are called and people
+    try to recover and modify paxos without agreeing on who the quorum
+    participants are.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ad7f5dd481a7f45dfe6b50d27ad45abc40950510)
+    (cherry picked from commit e073a062d56099b5fb4311be2a418f7570e1ffd9)
+
+commit a3e57b7231cb28c2e0a896f747537ebdbe3a4e96
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Feb 14 11:13:26 2014 -0800
+
+    mon: only learn peer addresses when monmap == 0
+    
+    It is only safe to dynamically update the address for a peer mon in our
+    monmap if we are in the midst of the initial quorum formation (i.e.,
+    monmap.epoch == 0).  If it is a later epoch, we have formed our initial
+    quorum and any and all monmap changes need to be agreed upon by the quorum
+    and committed via paxos.
+    
+    Fixes: #7212
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 7bd2104acfeff0c9aa5e648d82ed372f901f767f)
+    (cherry picked from commit 1996fd89fb3165a63449b135e05841579695aabd)
+
+commit 21ed54201bd4b0f02c07f6f96a63a5720057f011
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Mon Mar 17 14:37:09 2014 +0000
+
+    ceph.in: do not allow using 'tell' with interactive mode
+    
+    This avoids a lot of hassle when dealing with to whom tell each command
+    on interactive mode, and even more so if multiple targets are specified.
+    
+    As so, 'tell' commands should be used while on interactive mode instead.
+    
+    Backport: dumpling,emperor
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit e39c213c1d230271d23b74086664c2082caecdb9)
+
+commit be0205c33ccbab3b6f105bdf4da114658a981557
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Mar 12 22:56:44 2014 +0100
+
+    RGWListBucketMultiparts: init max_uploads/default_max with 0
+    
+    CID 717377 (#1 of 1): Uninitialized scalar field (UNINIT_CTOR)
+     2. uninit_member: Non-static class member "max_uploads" is not initialized
+        in this constructor nor in any functions that it calls.
+     4. uninit_member: Non-static class member "default_max" is not initialized
+        in this constructor nor in any functions that it calls.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit b23a141d54ffb39958aba9da7f87544674fa0e50)
+
+commit 6c3d4fbeb9cc50eca6eba12cfe7fed64b34eec3d
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Mar 13 14:49:30 2014 -0700
+
+    ceph_test_rados: wait for commit, not ack
+    
+    First, this is what we wanted in the first place
+    
+    Second, if we wait for ACK, we may look at a user_version value that is
+    not stable.
+    
+    Fixes: #7705
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f2124c5846f1e9cb44e66eb2e957b8c7df3e19f4)
+    
+    Conflicts:
+    
+    	src/test/osd/RadosModel.h
+
+commit 2daed5ff99dab238b696da5aba3816c4f5d763e8
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Mar 13 09:50:16 2014 -0700
+
+    test-upgrade-firefly: skip watch-notify system test
+    
+    This also fails on mixed version clusters due to watch on a
+    non-existent object returning ENOENT in firefly and 0 in dumpling.
+    
+    Reviewed-by: Sage Weil <sage.weil@inktank.com>
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 90a21d8cf6df5fe14b2dc9b2c175983b6bb017ce
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Mar 12 21:30:12 2014 -0700
+
+    qa/workunit/rados/test-upgrade-firefly: skip watch-notify test
+    
+    A watch on a non-existent object now returns ENOENT in firefly; skip this
+    test as it will fail on a hybrid or upgraded cluster.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit 32fdca6d9c2e3b923db7f21568bd315ab2c1c4ad
+Merge: 6700dd0 cd7986c
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Mar 11 21:33:40 2014 -0700
+
+    Merge pull request #1411 from ceph/wip-7076-dumpling
+    
+    dumpling backport of watchers check for rbd_remove()
+
+commit 6700dd068e236473343d15eee6307d44156958a3
+Author: Ray Lv <xiangyulv@gmail.com>
+Date:   Wed Feb 26 21:17:32 2014 +0800
+
+    rgw: off-by-one in rgw_trim_whitespace()
+    
+    Fixes: #7543
+    Backport: dumpling
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    Signed-off-by: Ray Lv <raylv@yahoo-inc.com>
+    (cherry picked from commit 195d53a7fc695ed954c85022fef6d2a18f68fe20)
+
+commit cd7986caf6baee5f9d6498b113b3382e66dd6f77
+Author: Ilya Dryomov <ilya.dryomov@inktank.com>
+Date:   Wed Jan 29 16:12:01 2014 +0200
+
+    rbd: check for watchers before trimming an image on 'rbd rm'
+    
+    Check for watchers before trimming image data to try to avoid getting
+    into the following situation:
+    
+      - user does 'rbd rm' on a mapped image with an fs mounted from it
+      - 'rbd rm' trims (removes) all image data, only header is left
+      - 'rbd rm' tries to remove a header and fails because krbd has a
+        watcher registered on the header
+      - at this point image cannot be unmapped because of the mounted fs
+      - fs cannot be unmounted because all its data and metadata is gone
+    
+    Unfortunately, this fix doesn't make it impossible to happen (the
+    required atomicity isn't there), but it's a big improvement over the
+    status quo.
+    
+    Fixes: http://tracker.ceph.com/issues/7076
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Signed-off-by: Ilya Dryomov <ilya.dryomov@inktank.com>
+    (cherry picked from commit 0a553cfa81b06e75585ab3c39927e307ec0f4cb6)
+
+commit a931aaa6cc104d63b20c0cbe9e3af4006c3abfaf
+Merge: b476d32 f5668b3
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Mar 9 10:56:31 2014 -0700
+
+    Merge pull request #1407 from dachary/wip-7188-dumpling
+    
+    common: ping existing admin socket before unlink (dumpling)
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit f5668b363b0724f385bebded3cbc7f363893f985
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sat Feb 15 11:43:13 2014 +0100
+
+    common: ping existing admin socket before unlink
+    
+    When a daemon initializes it tries to create an admin socket and unlinks
+    any pre-existing file, regardless. If such a file is in use, it causes
+    the existing daemon to loose its admin socket.
+    
+    The AdminSocketClient::ping is implemented to probe an existing socket,
+    using the "0" message. The AdminSocket::bind_and_listen function is
+    modified to call ping() on when it finds existing file. It unlinks the
+    file only if the ping fails.
+    
+    http://tracker.ceph.com/issues/7188 fixes: #7188
+    
+    Backport: emperor, dumpling
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 45600789f1ca399dddc5870254e5db883fb29b38)
+
+commit b476d324c69d4e6018191a7ffea8c9d6c1dfa008
+Merge: d3e13a7 5133dd6
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Mar 5 14:19:31 2014 -0800
+
+    Merge pull request #1366 from ceph/wip-6820.dumpling
+    
+    mon: OSDMonitor: don't crash if formatter is invalid during osd crush dump
+
+commit d3e13a7cdab42fa33182680f45fe21b4f9dc4b20
+Merge: c218c99 9c626e0
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Mar 5 12:45:57 2014 -0800
+
+    Merge pull request #1377 from ceph/wip-7584
+    
+    qa/workunit/rados/test-upgrade-firely.sh
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 9c626e0b18f538eb60883da01713ba629220e35e
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Mar 5 12:37:10 2014 -0800
+
+    qa/workunit/rados/test-upgrade-firely.sh
+    
+    Skip the tests that don't pass when run against firefly OSDs.
+    
+    Fixes: #7584
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit c218c999ecebe41e6de6fde76e85cc765cad8257
+Merge: 0eabbf1 24711cd
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Mar 4 07:28:44 2014 -0800
+
+    Merge pull request #1357 from ceph/wip-dumpling-removewq
+    
+    OSD: ping tphandle during pg removal
+    
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit 5133dd60e272d3fcbaacd5662a708ee4cf0db46d
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri Nov 22 02:17:16 2013 +0000
+
+    mon: OSDMonitor: don't crash if formatter is invalid during osd crush dump
+    
+    Code would assume a formatter would always be defined. If a 'plain'
+    formatter or even an invalid formatter were to be supplied, the monitor
+    would crash and burn in poor style.
+    
+    Fixes: 6820
+    Backport: emperor
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 49d2fb71422fe4edfe5795c001104fb5bc8c98c3)
+
+commit 24711cd49f85dbe827d41c4bcad2700cd6c42ad7
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Oct 15 13:11:29 2013 -0700
+
+    OSD: ping tphandle during pg removal
+    
+    Fixes: #6528
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    
+    (cherry picked from commit c658258d9e2f590054a30c0dee14a579a51bda8c)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+
+commit 0eabbf145e1c44f4d128b192cc77b708f180c968
+Merge: fe8915a 9d5d931
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Feb 25 15:47:05 2014 -0800
+
+    Merge pull request #1316 from ceph/dumpling-6922
+    
+    Dumpling: Prevent extreme PG split multipliers
+    
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+
+commit fe8915ae7e182340d1e22154e852895742c7da51
+Merge: 87822cc 5667566
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Feb 25 15:45:45 2014 -0800
+
+    Merge pull request #1315 from ceph/dumpling-hashpspool
+    
+    mon: OSDMonitor: allow (un)setting 'hashpspool' flag via 'osd pool set'
+    
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+
+commit 87822ccc862b533132c1fe232dfe4b7b17b816ad
+Merge: 0ae3352 37fbcb9
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Feb 25 15:44:39 2014 -0800
+
+    Merge pull request #1314 from ceph/dumpling-osd-pgstatsack
+    
+    Dumpling osd pgstatsack
+    
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+
+commit 5667566313b69dca011e897b2fa752356ad8901b
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Thu Oct 10 17:43:48 2013 -0700
+
+    mon: OSDMonitor: allow (un)setting 'hashpspool' flag via 'osd pool set'
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1c2886964a0c005545abab0cf8feae7e06ac02a8)
+    
+    Conflicts:
+    
+    	src/mon/MonCommands.h
+    	src/mon/OSDMonitor.cc
+    
+    mon: ceph hashpspool false clears the flag
+    instead of toggling it.
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    Reviewed-by: Christophe Courtaut <christophe.courtaut@gmail.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 589e2fa485b94244c79079f249428d4d545fca18
+    
+    Replace some of the infrastructure required by this command that
+    was not present in Dumpling with single-use code.
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+
+commit 9d5d931c60104823b3b20dcfb09480d65ffaa5ed
+Author: Greg Farnum <greg@inktank.com>
+Date:   Tue Dec 3 10:57:09 2013 -0800
+
+    OSDMonitor: use a different approach to prevent extreme multipliers on PG splits
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d8ccd73968fbd0753ca08916ebf1062cdb4d5ac1)
+    
+    Conflicts:
+    
+    	src/mon/OSDMonitor.cc
+
+commit c0c4448dc7df7900a564a6745903398cd39be7f1
+Author: Greg Farnum <greg@inktank.com>
+Date:   Mon Dec 2 15:13:40 2013 -0800
+
+    OSDMonitor: prevent extreme multipliers on PG splits
+    
+    Fixes: #6922
+    Backport: emperor
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f57dad6461171c903e8b5255eaed300374b00e74)
+    
+    Conflicts:
+    
+    	src/mon/OSDMonitor.cc
+
+commit c1889497b93ae9f0a946b11d9f5f6fcc7427e934
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Feb 22 08:08:37 2014 -0800
+
+    osd: fix off-by-one is boot subscription
+    
+    If we have osdmap N, we want to onetime subscribe
+    starting at N+1.  Among other things, it means we
+    hear when the NOUP flag is cleared.
+    
+    This appears to have broken somewhere around
+    3c76b81f2f96b790b72f2088164ed8e9d5efbba1.
+    
+    Fixes: #7511
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Sam Just <sam.just@inktank.com>
+    (cherry picked from commit 70d23b9a0ad9af5ca35a627a7f93c7e610e17549)
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit 4584f60653bee0305e85418323d80332ceecd0cf
+Author: Greg Farnum <greg@inktank.com>
+Date:   Tue Feb 11 12:51:19 2014 -0800
+
+    OSD: use the osdmap_subscribe helper
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3c76b81f2f96b790b72f2088164ed8e9d5efbba1)
+
+commit 61b2aeee7c37e03d5f6691c08c7760c48a85a2e1
+Author: Greg Farnum <greg@inktank.com>
+Date:   Tue Feb 11 13:34:39 2014 -0800
+
+    OSD: create a helper for handling OSDMap subscriptions, and clean them up
+    
+    We've had some trouble with not clearing out subscription requests and
+    overloading the monitors (though only because of other bugs). Write a
+    helper for handling subscription requests that we can use to centralize
+    safety logic. Clear out the subscription whenever we get a map that covers
+    it; if there are more maps available than we received, we will issue another
+    subscription request based on "m->newest_map" at the end of handle_osd_map().
+    
+    Notice that the helper will no longer request old maps which we already have,
+    and that unless forced it will not dispatch multiple subscribe requests
+    to a single monitor.
+    Skipping old maps is safe:
+    1) we only trim old maps when the monitor tells us to,
+    2) we do not send messages to our peers until we have updated our maps
+    from the monitor.
+    That means only old and broken OSDs will send us messages based on maps
+    in our past, and we can (and should) ignore any directives from them anyway.
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6db3ae851d1c936de045390d18b1c6ae95f2a209)
+    
+    Conflicts:
+    
+    	src/osd/OSD.h
+
+commit d93d67d1a315d8abe8d1cd9d7ea83417a19e2406
+Author: Greg Farnum <greg@inktank.com>
+Date:   Tue Feb 11 13:31:26 2014 -0800
+
+    monc: new fsub_want_increment( function to make handling subscriptions easier
+    
+    Provide a subscription-modifying function which will not decrement
+    the start version.
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 5b9c187caf6f7847aaa4a1003d200158dd32bf63)
+
+commit 37fbcb958f79bbfcba57c516b4862a14c52be398
+Author: Greg Farnum <greg@inktank.com>
+Date:   Wed Feb 12 11:30:15 2014 -0800
+
+    OSD: disable the PGStatsAck timeout when we are reconnecting to a monitor
+    
+    Previously, the timeout counter started as soon as we issued the reopen,
+    but if the reconnect process itself took a while, we might time out and
+    issue another reopen just as we get to the point where it's possible to
+    get work done. Since the mon client has its own reconnect timeouts (that is,
+    the OSD doesn't need to trigger those), we instead disable our timeouts
+    while the reconnect is happening, and then turn them back on again starting
+    from when we get the reconnect callback.
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 64cedf6fa3ee309cc96554286bfb805e4ca89439)
+    
+    Conflicts:
+    
+    	src/osd/OSD.cc
+
+commit 9be395e870c50d97604ef41f17667cc566fd84e1
+Author: Greg Farnum <greg@inktank.com>
+Date:   Wed Feb 12 13:51:48 2014 -0800
+
+    monc: backoff the timeout period when reconnecting
+    
+    If the monitors are systematically slowing down, we don't want to spam
+    them with reconnect attempts every three seconds. Instead, every time
+    we issue a reconnect, multiply our timeout period by a configurable; when
+    we complete the connection, reduce that multipler by 50%. This should let
+    us respond to monitor load.
+    Of course, we don't want to do that for initial startup in the case of a
+    couple down monitors, so don't apply the backoff until we've successfully
+    connected to a monitor at least once.
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 794c86fd289bd62a35ed14368fa096c46736e9a2)
+
+commit 8f4c20bdab153d1603cc99186d8d3e3970aa8976
+Author: Greg Farnum <greg@inktank.com>
+Date:   Wed Feb 12 13:37:50 2014 -0800
+
+    monc: set "hunting" to true when we reopen the mon session
+    
+    If we don't have a connecton to a monitor, we want to retry to another
+    monitor regardless of whether it's the first time or not.
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 60da8abe0ebf17ce818d6fcc6391401878123bb7)
+
+commit c6317558e0d3c8c62aecee0d95a839f93303f681
+Author: Greg Farnum <greg@inktank.com>
+Date:   Tue Feb 11 17:53:56 2014 -0800
+
+    monc: let users specify a callback when they reopen their monitor session
+    
+    Then the callback is triggered when a new session is established, and the
+    daemon can do whatever it likes. There are no guarantees about how long it
+    might take to trigger, though. In particular we call the provided callback
+    while not holding our own lock in order to avoid deadlock. This could lead
+    to some funny ordering from the user's perspective if they call
+    reopen_session() again before getting the callback, but there's no way around
+    that, so they just have to use it appropriately.
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1a8c43474bf36bfcf2a94bf9b7e756a2a99f33fd)
+
+commit 0ae335298b85daba5125a3da4ad26d598c76ecab (refs/remotes/gh/multi-object-delete)
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Feb 11 16:54:05 2014 -0800
+
+    rgw: multi object delete should be idempotent
+    
+    Fixes: #7346
+    When doing a multi object delete, if an object does not exist then we
+    should return a success code for that object.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 8ca3d95bf633ea9616852cec74f02285a03071d5)
+    
+    Conflicts:
+    	src/rgw/rgw_op.cc
diff --git a/doc/changelog/v0.67.9.txt b/doc/changelog/v0.67.9.txt
new file mode 100644
index 000000000..8ae98cd89
--- /dev/null
+++ b/doc/changelog/v0.67.9.txt
@@ -0,0 +1,233 @@
+commit ba340a97c3dafc9155023da8d515eecc675c619a (tag: refs/tags/v0.67.9, refs/remotes/gh/dumpling)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Wed May 21 16:57:02 2014 +0000
+
+    0.67.9
+
+commit b638d19d126646d2a8f6da11067c5f392a62525e
+Author: Guang Yang <yguang@yahoo-inc.com>
+Date:   Fri May 9 09:21:23 2014 +0000
+
+    msg: Fix inconsistent message sequence negotiation during connection reset
+    
+    Backport: firefly, emperor, dumpling
+    
+    Signed-off-by: Guang Yang (yguang@yahoo-inc.com)
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit bdee119076dd0eb65334840d141ccdf06091e3c9)
+
+commit 55b1112819ed2e6fd509c09c2c1a2b434d22aac6
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue May 20 10:46:34 2014 -0700
+
+    OSD::handle_pg_query: on dne pg, send lb=hobject_t() if deleting
+    
+    We will set lb=hobject_t() if we resurrect the pg.  In that case,
+    we need to have sent that to the primary before hand.  If we
+    finish the removal before the pg is recreated, we'll just end
+    up backfilling it, which is ok since the pg doesn't exist anyway.
+    
+    Fixes: #7740
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 04de781765dd5ac0e28dd1a43cfe85020c0854f8)
+    
+    Conflicts:
+    
+    	src/osd/OSD.cc
+
+commit 3b8ab41e1ec86f2ab5c6b4bee3fb4030077e2c21
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 2 14:48:35 2014 -0700
+
+    mon/MonClient: remove stray _finish_hunting() calls
+    
+    Callig _finish_hunting() clears out the bool hunting flag, which means we
+    don't retry by connection to another mon periodically.  Instead, we send
+    keepalives every 10s.  But, since we aren't yet in state HAVE_SESSION, we
+    don't check that the keepalives are getting responses.  This means that an
+    ill-timed connection reset (say, after we get a MonMap, but before we
+    finish authenticating) can drop the monc into a black hole that does not
+    retry.
+    
+    Instead, we should *only* call _finish_hunting() when we complete the
+    authentication handshake.
+    
+    Fixes: #8278
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 77a6f0aefebebf057f02bfb95c088a30ed93c53f)
+
+commit a915350491092abcd2584d0bbb5f5d8610ed89fc
+Merge: bd5d6f1 d4e67ff
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue May 20 10:19:00 2014 -0700
+
+    Merge pull request #1826 from ceph/wip-8162-dumpling
+    
+    Wip 8162 dumpling
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit bd5d6f116416d1b410d57ce00cb3e2abf6de102b
+Author: Greg Farnum <greg@inktank.com>
+Date:   Thu May 15 16:50:43 2014 -0700
+
+    OSD: fix an osdmap_subscribe interface misuse
+    
+    When calling osdmap_subscribe, you have to pass an epoch newer than the
+    current map's. _maybe_boot() was not doing this correctly -- we would
+    fail a check for being *in* the monitor's existing map range, and then
+    pass along the map prior to the monitor's range. But if we were exactly
+    one behind, that value would be our current epoch, and the request would
+    get dropped. So instead, make sure we are not *in contact* with the monitor's
+    existing map range.
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 290ac818696414758978b78517b137c226110bb4)
+
+commit c5ae8ffc281ac538dd0966a552a6cf729728e925
+Merge: fced056 8d991fa
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon May 19 13:45:50 2014 -0700
+
+    Merge pull request #1827 from ceph/wip-6565-dumpling
+    
+    Wip 6565 dumpling
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit 8d991fa2c3fdbfd9178d93e4ca33d47504a37195
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Oct 16 10:07:37 2013 -0700
+
+    OSD: check for splitting when processing recover/backfill reservations
+    
+    Fixes: 6565
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 15ec5332ba4154930a0447e2bcf1acec02691e97)
+
+commit d4e67ff3037a3cc7ae2ecc9e1d8d086c45ae515a
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu May 8 13:25:32 2014 -0700
+
+    ReplicatedPG::recover_backfill: do not update last_backfill prematurely
+    
+    Previously, we would update last_backfill on the backfill peer to
+    
+    backfills_in_flight.empty() ? backfill_pos :
+      backfills_in_flight.begin()->first
+    
+    which is actually the next backfill to complete.  We want to update
+    last_backfill to the largest completed backfill instead.
+    
+    We use the pending_backfill_updates mapping to identify the most
+    recently completed backfill.  Due to the previous patch, deletes
+    will also be included in that mapping.
+    
+    Related sha1s from master:
+    4139e75d63b0503dbb7fea8036044eda5e8b7cf1
+    7a06a71e0f2023f66d003dfb0168f4fe51eaa058
+    
+    We don't really want to backport those due to the changes in:
+    9ec35d5ccf6a86c380865c7fc96017a1f502560a
+    
+    This patch does essentially the same thing, but using backfill_pos.
+    
+    Fixse: #8162
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit d620b13c51fc69b509824177af32de55c89c2259
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Oct 28 16:03:25 2013 -0700
+
+    ReplicatedPG: add empty stat when we remove an object in recover_backfill
+    
+    Subsequent updates to that object need to have their stats added
+    to the backfill info stats atomically with the last_backfill
+    update.
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit ecddd12b01be120fba87f5ac60539f98f2c69a28)
+
+commit fced0562f6b80035cd522cc81bdc844ccc92d9bc
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Nov 27 13:34:00 2013 -0800
+
+    rgw: don't error out on empty owner when setting acls
+    
+    Fixes: #6892
+    Backport: dumpling, emperor
+    s3cmd specifies empty owner field when trying to set acls on object
+    / bucket. We errored out as it didn't match the current owner name, but
+    with this change we ignore it.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 14cf4caff58cc2c535101d48c53afd54d8632104)
+
+commit 0e685c68aa6a500aa7fa433cd9b8246f70c5383e
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Apr 21 15:34:04 2014 -0700
+
+    rgw: send user manifest header field
+    
+    Fixes: #8170
+    Backport: firefly
+    If user manifest header exists (swift) send it as part of the object
+    header data.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 5cc5686039a882ad345681133c9c5a4a2c2fd86b)
+
+commit 20383e359cdfbf146838742fa050ec4b6073271d
+Author: Yan, Zheng <zheng.z.yan@intel.com>
+Date:   Fri Apr 11 15:03:37 2014 +0800
+
+    client: add asok command to kick sessions that were remote reset
+    
+    Fixes: #8021
+    Signed-off-by: Yan, Zheng <zheng.z.yan@intel.com>
+    (cherry picked from commit 09a1bc5a4601d356b9cc69be8541e6515d763861)
+
+commit 4e5e41deeaf91c885773d90e6f94da60f6d4efd3
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Apr 18 13:50:11 2014 -0700
+
+    osd: throttle snap trimmming with simple delay
+    
+    This is not particularly smart, but it is *a* knob that lets you make
+    the snap trimmer slow down.  It's a flow and a simple delay, so it is
+    adjustable at runtime.  Default is 0 (no change in behavior).
+    
+    Partial solution for #6278.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4413670d784efc2392359f0f22bca7c9056188f4)
+
+commit 14650b282ecd344025714a4e743b255ae01b3ce0
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Apr 1 16:01:28 2014 -0700
+
+    PG: only complete replicas should count toward min_size
+    
+    Backport: emperor,dumpling,cuttlefish
+    Fixes: #7805
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 0d5d3d1a30685e7c47173b974caa12076c43a9c4)
+
+commit 3f1d7f5e0a67ad646de465335fb7ee00eb07e220
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 2 15:10:43 2014 -0700
+
+    mon/PGMonitor: set tid on no-op PGStatsAck
+    
+    The OSD needs to know the tid.  Both generally, and specifically because
+    the flush_pg_stats may be blocking on it.
+    
+    Fixes: #8280
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 5a6ae2a978dcaf96ef89de3aaa74fe951a64def6)
diff --git a/doc/changelog/v0.72.2.txt b/doc/changelog/v0.72.2.txt
new file mode 100644
index 000000000..66fd667e0
--- /dev/null
+++ b/doc/changelog/v0.72.2.txt
@@ -0,0 +1,189 @@
+commit a913ded2ff138aefb8cb84d347d72164099cfd60
+Author: Gary Lowell <gary.lowell@inktank.com>
+Date:   Fri Dec 20 19:28:37 2013 +0000
+
+    v0.72.2
+
+commit b570c8014325fef590d16e6157cb33026155932c
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Dec 18 13:11:01 2013 -0800
+
+    rgw: fix use-after-free when releasing completion handle
+
+    Backport: emperor, dumpling
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c8890ab2d46fe8e12200a0d2f9eab31c461fb871)
+
+commit 451381b4dba9d50fbee2accf5d75562615001ef3
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Dec 18 13:10:21 2013 -0800
+
+    rgw: don't return data within the librados cb
+
+    Fixes: #7030
+    The callback is running within a single Finisher thread, thus we
+    shouldn't block there. Append read data to a list and flush it within
+    the iterate context.
+
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit d6a4f6adfaa75c3140d07d6df7be03586cc16183)
+
+commit c044b218cb32d7ef2f2aa5db81ebf38ad0dc1049
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Dec 1 22:21:31 2013 -0800
+
+    Partial revert "mon: osd pool set syntax relaxed, modify unit tests"
+
+    This reverts commit 08327fed8213a5d24cd642e12b38a171b98924cb, except
+    for the hashpspool bit.  We switched back to an integer argument in
+    commit 337195f04653eed8e8f153a5b074f3bd48408998.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e80ab94bf44e102fcd87d16dc11e38ca4c0eeadb)
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit 1983ed8013637e9b80ca4a0e8d2800d6a36b120e
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Nov 19 23:21:11 2013 +0000
+
+    mon: OSDMonitor: drop cmdval_get() for unused variable
+
+    We don't ever use any value as a float, so just drop obtaining it.  This
+    makes it easier to partially revert 2fe0d0d9 in an upcoming patch.
+
+    Backport: emperor
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 7191bb2b2485c7819ca7b9d9434d803d0c94db7a)
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit 1c4759cebb11ae1aa84aa3cbdb2b31e6d111f28e
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri Nov 22 02:10:35 2013 +0000
+
+    mon: OSDMonitor: receive CephInt on 'osd pool set' instead on CephString
+
+    This partially reverts 2fe0d0d9 in order to allow Emperor monitors to
+    forward mon command messages to Dumpling monitors without breaking a
+    cluster.
+
+    The need for this patch became obvious after issue #6796 was triggered.
+    Basically, in a mixed cluster of Emperor/Dumpling monitors, if a client
+    happens to obtain the command descriptions from an Emperor monitor and
+    then issue an 'osd pool set' this can turn out in one of two ways:
+
+    1. client msg gets forwarded to an Emperor leader and everything's a-okay;
+    2. client msg gets forwarded to a Dumpling leader and the string fails to
+    be interpreted without the monitor noticing, thus leaving the monitor with
+    an uninitialized variable leading to trouble.
+
+    If 2 is triggered, a multitude of bad things can happen, such as thousands
+    of pg splits, due to a simple 'osd set pool foo pg_num 128' turning out
+    to be interpreted as 109120394 or some other random number.
+
+    This patch is such that we make sure the client sends an integer instead
+    of a string. We also make sure to interpret anything the client sends as
+    possibly being a string, or an integer.
+
+    Fixes: 6796
+    Backport: emperor
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 337195f04653eed8e8f153a5b074f3bd48408998)
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit c2e467b42dfaecf082a3838b10e873f9f8bd5633
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Nov 25 13:43:43 2013 -0800
+
+    init, upstart: prevent daemons being started by both
+
+    There can be only one init system starting a daemon. If there is a
+    host entry in ceph.conf for a daemon, sysvinit would try to start it
+    even if the daemon's directory did not include a sysvinit file. This
+    preserves backwards compatibility with older installs using sysvinit,
+    but if an upstart file is present in the daemon's directory, upstart
+    will try to start them, regardless of host entries in ceph.conf.
+
+    If there's an upstart file in a daemon's directory and a host entry
+    for that daemon in ceph.conf, both sysvinit and upstart would attempt
+    to manage it.
+
+    Fix this by only starting daemons if the marker file for the other
+    init system is not present. This maintains backwards compatibility
+    with older installs using neither sysvinit or upstart marker files,
+    and does not break any valid configurations. The only configuration
+    that would break is one with both sysvinit and upstart files present
+    for the same daemon.
+
+    Backport: emperor, dumpling
+    Reported-by: Tim Spriggs <tims@uahirise.org>
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 5e34beb61b3f5a1ed4afd8ee2fe976de40f95ace)
+
+commit 8812c4e958b154ca401c7257fd48ccaffe013639
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Nov 27 13:34:00 2013 -0800
+
+    rgw: don't error out on empty owner when setting acls
+
+    Fixes: #6892
+    Backport: dumpling, emperor
+    s3cmd specifies empty owner field when trying to set acls on object
+    / bucket. We errored out as it didn't match the current owner name, but
+    with this change we ignore it.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 14cf4caff58cc2c535101d48c53afd54d8632104)
+
+commit 69e055f367ae37dd771c050f158d684e2a3cb246
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Nov 22 07:04:01 2013 -0800
+
+    rgw: lower some debug message
+
+    Fixes: #6084
+    Backport: dumpling, emperor
+
+    Reported-by: Ron Allred <rallred@itrefined.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit b35fc1bc2ec8c8376ec173eb1c3e538e02c1694e)
+
+commit 53951fd5459d6396b6ab59b52dc33df0ebcf4414
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Nov 12 15:15:26 2013 -0800
+
+    ReplicatedPG: test for missing head before find_object_context
+
+    find_object_context doesn't return EAGAIN for a missing head.
+    I chose not to change that behavior since it might hide bugs
+    in the future.  All other callers check for missing on head
+    before calling into find_object_context because we potentially
+    need head or snapdir to map a snapid onto a clone.
+
+    Backport: emperor
+    Fixes: 6758
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Reviewed-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit dd9d8b020286d5e3a69455023c3724a7b436d687)
+
+commit 83ee6843de6432278dc2891526691006c51eb1fa
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Nov 18 14:39:12 2013 -0800
+
+    osd: fix bench block size
+
+    The command was declared to take 'size' in dumpling, but was trying to
+    read 'bsize' instead, so it always used the default of 4MiB. Change
+    the bench command to read 'size', so it matches what existing clients
+    are sending.
+
+    Fixes: #6795
+    Backport: emperor, dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 40a76ef0d09f8ecbea13712410d9d34f25b91935)
diff --git a/doc/changelog/v0.80.1.txt b/doc/changelog/v0.80.1.txt
new file mode 100644
index 000000000..70d6fd6af
--- /dev/null
+++ b/doc/changelog/v0.80.1.txt
@@ -0,0 +1,216 @@
+commit a38fe1169b6d2ac98b427334c12d7cf81f809b74 (tag: refs/tags/v0.80.1)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Mon May 12 15:12:54 2014 +0000
+
+    0.80.1
+
+commit ab873622c6d41bc8053a1d205da2e4b55e77351d
+Author: Jenkins <jenkins@inktank.com>
+Date:   Mon May 12 15:11:33 2014 +0000
+
+    0.80.1
+
+commit 03e7d1d550132a55e87972123d5bc3c0bb0cc981
+Author: Jenkins <jenkins@inktank.com>
+Date:   Mon May 12 15:10:56 2014 +0000
+
+    0.80.1
+
+commit 7a4919c7fa10020643df09e012376e4b563ca17d
+Author: Jenkins <jenkins@inktank.com>
+Date:   Mon May 12 15:09:01 2014 +0000
+
+    0.80.1
+
+commit ce852b29dc5cf9857a6bfeef7839293cfadf1877 (refs/remotes/gh/firefly)
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri May 2 16:21:26 2014 -0700
+
+    Revert "ReplicatedPG: block scrub on blocked object contexts"
+
+    This reverts commit e66f2e36c06ca00c1147f922d3513f56b122a5c0.
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+    0f3235d46c8fd6c537bd4aa8a3faec6c00f311a8 is the firefly commit
+    corresponding to e66f2e36c06ca00c1147f922d3513f56b122a5c0.
+
+    (cherry picked from commit 84728058dbb91b8ed062240b3373b18078f0c9ca)
+
+commit db8873b69c73b40110bf1512c114e4a0395671ab
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue May 6 16:55:27 2014 -0700
+
+    rgw: fix stripe_size calculation
+
+    Fixes: #8299
+    Backport: firefly
+    The stripe size calculation was broken, specifically affected cases
+    where we had manifest that described multiple parts.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 9968b938b5d47fdf3a67db134bd2ea6bf3a28086)
+
+commit e0fb2e63074d5f71679bb68656336f68e8190f39
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue May 6 11:06:29 2014 -0700
+
+    rgw: cut short object read if a chunk returns error
+
+    Fixes: #8289
+    Backport: firefly, dumpling
+    When reading an object, if we hit an error when trying to read one of
+    the rados objects then we should just stop. Otherwise we're just going
+    to continue reading the rest of the object, and since it can't be sent
+    back to the client (as we have a hole in the middle), we end up
+    accumulating everything in memory.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 03b0d1cfb7bd30a77fedcf75eb06476b21b14e95)
+
+commit 328665db17bf87d36a3dcbf5f0d0d1a9410e2f96
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Apr 21 15:34:04 2014 -0700
+
+    rgw: send user manifest header field
+
+    Fixes: #8170
+    Backport: firefly
+    If user manifest header exists (swift) send it as part of the object
+    header data.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 5cc5686039a882ad345681133c9c5a4a2c2fd86b)
+
+commit cdb0fac29693fb0250b515891804c4c7fd3045c7
+Author: Yan, Zheng <zheng.z.yan@intel.com>
+Date:   Fri Apr 11 15:03:37 2014 +0800
+
+    client: add asok command to kick sessions that were remote reset
+
+    Fixes: #8021
+    Signed-off-by: Yan, Zheng <zheng.z.yan@intel.com>
+    (cherry picked from commit 09a1bc5a4601d356b9cc69be8541e6515d763861)
+
+commit 4a91196bf49936471a731f9316ef904fda982b26
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Apr 18 13:50:11 2014 -0700
+
+    osd: throttle snap trimmming with simple delay
+
+    This is not particularly smart, but it is *a* knob that lets you make
+    the snap trimmer slow down.  It's a flow and a simple delay, so it is
+    adjustable at runtime.  Default is 0 (no change in behavior).
+
+    Partial solution for #6278.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4413670d784efc2392359f0f22bca7c9056188f4)
+
+commit 3ba2e228db1829a02f9cd31225672604fdc8adf7
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 2 14:48:35 2014 -0700
+
+    mon/MonClient: remove stray _finish_hunting() calls
+
+    Callig _finish_hunting() clears out the bool hunting flag, which means we
+    don't retry by connection to another mon periodically.  Instead, we send
+    keepalives every 10s.  But, since we aren't yet in state HAVE_SESSION, we
+    don't check that the keepalives are getting responses.  This means that an
+    ill-timed connection reset (say, after we get a MonMap, but before we
+    finish authenticating) can drop the monc into a black hole that does not
+    retry.
+
+    Instead, we should *only* call _finish_hunting() when we complete the
+    authentication handshake.
+
+    Fixes: #8278
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 77a6f0aefebebf057f02bfb95c088a30ed93c53f)
+
+commit 383f6440ce34b0d84c0dfa06f561896dad6862f4
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 2 16:41:26 2014 -0700
+
+    osd/ReplicatedPG: fix trim of in-flight hit_sets
+
+    We normally need to stat the hit_set to know how many bytes to adjust the
+    stats by.  If the hit_set was just written, we will get ENOENT.
+
+    Get the obc instead, which will either get the in-memory copy (because the
+    repop is still in flight) or load it off of disk.
+
+    Fixes: #8283
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 72fdd557c35cb721d4b502c5a8f68c878f11a19c)
+
+commit ef35448e63916102261ff6198f2c0ab4e4a57e9b
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue May 6 11:01:27 2014 -0700
+
+    osd/ReplicatedPG: fix whiteouts for other cache mode
+
+    We were special casing WRITEBACK mode for handling whiteouts; this needs to
+    also include the FORWARD and READONLY modes.  To avoid having to list
+    specific cache modes, though, just check != NONE.
+
+    Fixes: #8296
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3e387d62ed95898db8a7d7163c2bacc363b8f617)
+
+commit e7df73dd7aaf5a0b1171f73d6695d26cd25b7b35
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 1 16:53:17 2014 -0700
+
+    osd: Prevent divide by zero in agent_choose_mode()
+
+    Fixes: #8175
+    Backport: firefly
+
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f47f867952e6b2a16a296c82bb9b585b21cde6c8)
+
+commit 022d467b5d6b77c17b6fdaeec8369cae61e9e5a4
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Mon Apr 21 23:52:04 2014 -0700
+
+    osd, common: If agent_work() finds no objs to work on delay 5 (default) secs
+
+    Add config osd_agent_delay_time of 5 seconds
+    Honor delay by ignoring agent_choose_mode() calls
+    Add tier_delay to logger
+    Treat restart after delay like we were previously idle
+
+    Fixes: #8113
+    Backport: firefly
+
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit b7d31e5f5952c631dd4172bcb825e77a13fc60bc)
+
+commit 6a55c3bc3caf46652e962fa9434900fb494d1e6c
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Thu May 1 18:54:30 2014 -0700
+
+    osd/ReplicatedPG: agent_work() fix next if finished early due to start_max
+
+    Backport: firefly
+
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 9cf470cac8dd4d8f769e768f2de6b9eb67a3c3af)
+
+commit 1b899148a729235ab2835d368077f18e62a36a93
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Sat May 3 12:53:06 2014 +0800
+
+    Fix clone problem
+
+    When clone happened, the origin header also will be updated in GenericObjectMap,
+    so the new header wraper(StripObjectHeader) should be updated too.
+
+    Fix #8282
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit 3aee1e0ffe0583f74c02d9c9e86c7fb267f3515c)
diff --git a/doc/changelog/v0.80.10.txt b/doc/changelog/v0.80.10.txt
new file mode 100644
index 000000000..be82bcc9b
--- /dev/null
+++ b/doc/changelog/v0.80.10.txt
@@ -0,0 +1,3308 @@
+commit ea6c958c38df1216bf95c927f143d8b13c4a9e70 (tag: refs/tags/v0.80.10)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Thu Jun 18 09:48:56 2015 -0700
+
+    0.80.10
+
+commit 9b7f09e280b4610b9f85ac34ec014018f0e2e1d3
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 17 09:35:28 2015 -0700
+
+    qa/workunits/rados/test-upgarde-v9.0.1: fix exclude syntax
+    
+    It's -, then a list of all exclusions separated by :.  There are just 2.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 78d894a634d727a9367f809a1f57234e5e6935be)
+
+commit 4e28fbc52db7d808aeaede884f342beafb7fc581
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jun 16 21:05:29 2015 -0700
+
+    qa/workunits/rados/test-upgrade-v9.0.1: skip one more evict test
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3e8d60a80ce31860eac76a1f6489a35e1795a0c0)
+
+commit 34ba3719997fed6d0b8148f21e59b446bdf42962
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Jun 15 15:12:43 2015 -0700
+
+    qa: add compatibility filtered rados api tests for upgrades
+    
+    Post-9.0.1, the evict op returns success when an object doesn't exist
+    in the cache tier. Skip the tests that are incompatible across
+    versions.
+    
+    Fixes: #11548
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 348a3d3c9880e7d022e71a2faafe51c8f771406e)
+
+commit d1f478200342f422ee6e563bf7aad54ba38b3ed3
+Merge: d0f9c5f 7f1abd9
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 10 15:03:48 2015 -0700
+
+    Merge pull request #4924 from ceph/wip-11955-firefly
+    
+    11955: ceph.spec.in: package mkcephfs on EL6
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 7f1abd9053088ded6613aeca4e7c9489c44910e4
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Wed Jun 10 15:43:41 2015 -0600
+
+    ceph.spec.in: package mkcephfs on EL6
+    
+    Commit efbca0465c2946e113771966df08cf7cf37b1196 added mkcephfs to the
+    RPM %files listing, but this /usr/sbin path is only correct for CentOS
+    7. In CentOS 6, the utility is present at /sbin/mkcephfs instead. This
+    causes rpmbuild to fail to build the tip of the firefly branch on EL6.
+    
+    Adjust the RPM %files list so we properly package mkcephfs on both EL7
+    and EL6.
+    
+    http://tracker.ceph.com/issues/11955 Refs: #11955
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit d0f9c5f47024f53b4eccea2e0fde9b7844746362
+Merge: efbca04 9930138
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Jun 4 19:07:03 2015 +0200
+
+    Merge pull request #4851 from ceph/wip-10873-firefly
+    
+    rgw: generate the "Date" HTTP header for civetweb.
+
+commit 99301384be96997203682679c0430ca0a53be5d3 (refs/remotes/gh/wip-10873-firefly)
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed Feb 18 15:48:43 2015 +0100
+
+    rgw: generate the "Date" HTTP header for civetweb.
+    
+    Fixes: #10873
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit ea384f83b601f60e135c3d3f960fdb75a919dd84)
+
+commit efbca0465c2946e113771966df08cf7cf37b1196
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 3 13:03:50 2015 -0400
+
+    ceph.spec: add mkcephfs to ceph.rpm
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit e785f1a2bb4cd77b0aaaccd78060183157ad842e
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Dec 2 13:58:10 2014 -0700
+
+    packaging: package ceph-disk(8)
+    
+    The ceph-disk man page was added in
+    a450cab2b8148cb8a9b043d629feccf89e5aabac, but this was not added to the
+    RPM or DEB packaging. Add it here.
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit b743a951114b00bbb6e14fb88f1928b504bc0f8b)
+
+commit 819cf6ddb986b37c5d1229c4ea330010e88bb615
+Merge: e93711a 665a857
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Jun 2 19:53:03 2015 +0300
+
+    Merge pull request #4780 from oritwas/wip-10295-firefly
+    
+    rgw: civetweb should use unique request id
+    
+    Reviewd-by: Yehuda Sadeh <yehuda@rehat.com>
+
+commit e93711a40d680230b03fe722a4da49a6f9a9b667
+Merge: 071c943 50e8579
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Jun 2 19:02:07 2015 +0300
+
+    Merge pull request #4829 from oritwas/wip-negative_content_length-firefly
+    
+    rgw: don't allow negative / invalid content length
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 50e85797507a3ba13193f368cff461c08e44a9b3
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Aug 1 16:15:36 2014 -0700
+
+    rgw: don't allow negative / invalid content length
+    
+    Certain frontends (e.g., civetweb) don't filter such requests.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 0e74b7a1d56733358e2f1d3df4386125a94c2966)
+
+commit 071c94385ee71b86c5ed8363d56cf299da1aa7b3
+Merge: 68211f6 c632ef1
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed May 27 15:36:15 2015 -0700
+
+    Merge pull request #4766 from SUSE/wip-11673-firefly
+    
+    Debian: ceph-test and rest-bench debug packages should require their respective binary packages
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 665a85701177230365e43a351d7722cb2adbde93
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Apr 23 17:36:47 2015 +0200
+
+    rgw: civetweb should use unique request id
+    
+    max_req_id was moved to RGWRados and changed to atomic64_t.
+    
+    The same request id resulted in gc giving the same idtag to all objects
+    resulting in a leakage of rados objects. It only kept the last deleted object in
+    it's queue, the previous objects were never freed.
+    
+    Fixes: 10295
+    Backport: Hammer, Firefly
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit c262259)
+    
+    Conflicts:
+    	src/rgw/rgw_main.cc
+    	src/rgw/rgw_rados.h
+
+commit c632ef13e05654d0e5ddc79fc7e9553cad8cbed0
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Mon May 18 10:50:58 2015 -0600
+
+    debian: set rest-bench-dbg ceph-test-dbg dependencies
+    
+    Debian's debug packages ought to depend on their respective binary
+    packages. This was the case for many of our ceph packages, but it was
+    not the case for ceph-test-dbg or rest-bench-dbg.
+    
+    Add the dependencies on the relevant binary packages, pinned to
+    "= ${binary:Version}" per convention.
+    
+    http://tracker.ceph.com/issues/11673 Fixes: #11673
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit f898ec1e4e3472b0202280f09653a769fc62c8d3)
+
+commit 68211f695941ee128eb9a7fd0d80b615c0ded6cf
+Merge: 7d11b19 cd8f183
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 18 14:25:59 2015 +0200
+
+    Merge pull request #4697 from ceph/wip-11622-firefly
+    
+    Wip 11622 firefly
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit cd8f1830eb5d7ff75b17d7f0915ee4b3b834b149 (refs/remotes/gh/wip-11622-firefly)
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed May 13 17:05:22 2015 -0700
+
+    rgw: merge manifests correctly when there's prefix override
+    
+    Fixes: #11622
+    Backport: hammer, firefly
+    
+    Prefix override happens in a manifest when a rados object does not
+    conform to the generic prefix set on the manifest. When merging
+    manifests (specifically being used in multipart objects upload), we need
+    to check if the rule that we try to merge has a prefix that is the same
+    as the previous rule. Beforehand we checked if both had the same
+    override_prefix setting, but that might not apply as both manifests
+    might have different prefixes.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 389ae6739ddc6239a4dd7c5f7f9bfc9b645b8577)
+
+commit eef3d2f1c400573db90936fd417769183950b6ee
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 12 16:40:10 2015 -0700
+
+    rgw: restore buffer of multipart upload after EEXIST
+    
+    Fixes #11604
+    Backport: hammer, firefly
+    
+    When we need to restart a write of part data, we need to revert to
+    buffer to before the write, otherwise we're going to skip some data.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 580ccaec12daae64c38a1616d0be907bdd70a888)
+
+commit 7d11b19003503a9db1572d01f7a170e35b29017d
+Merge: 114f2e9 9b33965
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 15 10:27:42 2015 -0700
+
+    Merge pull request #4414 from xinxinsh/wip-11125-firefly
+    
+    rgw: keystone token cache does not work correctly
+
+commit 114f2e9bb5665760a5be9816785508f1c97662d5
+Merge: ac7d28a f33effc
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri May 15 17:03:04 2015 +0200
+
+    Merge pull request #4415 from xinxinsh/wip-11244-firefly
+    
+    cancel_pull: requeue waiters
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit ac7d28a6977084da0b70a3f2d0a54c8fa55b16fa
+Merge: f273792 1f6b1bb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri May 15 17:02:27 2015 +0200
+
+    Merge pull request #4416 from xinxinsh/wip-10976-firefly
+    
+    fix PG::all_unfound_are_queried_or_lost for non-existent osds
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit f273792c675aca95694cd36794d8a854731bf308
+Merge: 8a6632b 019b28b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri May 15 17:02:05 2015 +0200
+
+    Merge pull request #4556 from xinxinsh/wip-11429-firefly
+    
+    OSD::load_pgs: we need to handle the case where an upgrade from earlier versions which ignored non-existent pgs resurrects a pg with a prehistoric osdmap
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 8a6632b4e5ffad305f92c89656b161a521529e45
+Merge: 156c385 a71f309
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu May 14 09:50:28 2015 -0700
+
+    Merge pull request #4638 from SUSE/wip-11453-firefly
+    
+    run RGW as root
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 019b28b6397dbc3111faa6a8eb57349a48f408d8
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Apr 20 23:45:57 2015 -0700
+
+    OSD: handle the case where we resurrected an old, deleted pg
+    
+    Prior to giant, we would skip pgs in load_pgs which were not present in
+    the current osdmap.  Those pgs would eventually refer to very old
+    osdmaps, which we no longer have causing the assertion failure in 11429
+    once the osd is finally upgraded to a version which does not skip the
+    pgs.  Instead, if we do not have the map for the pg epoch, complain to
+    the osd log and skip the pg.
+    
+    Fixes: 11429
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit fbfd50de5b9b40d71d2e768418a8eca28b1afaca)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+            resolved by add a new comment line
+
+commit 156c385ef726e56d219c3383a4015c50aec2758a
+Merge: fcd0ea3 8cc4bc1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue May 12 13:57:36 2015 +0200
+
+    Merge pull request #4379 from ceph/wip-11416
+    
+    rgw: use correct objv_tracker for bucket instance
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit a71f3091a4ea810c02517642fb4ab9ce5516b452
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Wed Apr 22 16:36:42 2015 -0600
+
+    init-radosgw: run RGW as root
+    
+    The ceph-radosgw service fails to start if the httpd package is not
+    installed. This is because the init.d file attempts to start the RGW
+    process with the "apache" UID. If a user is running civetweb, there is
+    no reason for the httpd or apache2 package to be present on the system.
+    
+    Switch the init scripts to use "root" as is done on Ubuntu.
+    
+    http://tracker.ceph.com/issues/11453 Refs: #11453
+    
+    Reported-by: Vickey Singh <vickey.singh22693@gmail.com>
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 47339c5ac352d305e68a58f3d744c3ce0fd3a2ac)
+
+commit fcd0ea3383582e7a1dc8091e7a48e1d4bbaa76ee
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Fri May 8 11:29:44 2015 -0700
+
+    workunits: remove defunct cephfs set_layout test
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit df053b86a89edf8f390400dad8c5e654e14df327
+Merge: aef0272 62645d3
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed May 6 13:07:11 2015 -0700
+
+    Merge pull request #4571 from ceph/wip-11256-firefly
+    
+    Wip 11256 firefly
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit aef0272d72afaef849b5d4acbf55626033369ee8
+Author: Noah Watkins <noahwatkins@gmail.com>
+Date:   Fri Mar 27 19:34:12 2015 -0700
+
+    java: libcephfs_jni.so is in /usr/lib64 on rhel
+    
+    Signed-off-by: Noah Watkins <noahwatkins@gmail.com>
+    (cherry picked from commit aed3434dc7c5161c72c7d5655faa3bc693fc9777)
+
+commit a551a23d36e3f30ff5b0679a98ee760166ae47ae
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Mar 2 21:04:25 2015 +0800
+
+    qa/workunits/fs/misc: fix filelock_interrupt.py
+    
+    Handle the case that kernel does not support fcntl.F_OFD_SETLK.
+    Also fix the code that checks if fnctl fails with errno == EINTR.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 4ececa3dc4a21b98f61a592da9e2be60a0d71625)
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 07031b151b4ac6677b1f663ac200d09088deff64
+Merge: 83f8d43 3fb97e2
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Apr 30 00:32:21 2015 +0200
+
+    Merge pull request #4385 from xinxinsh/wip-11199-firefly
+    
+    osd: ENOENT on clone
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 83f8d434a5c245711922ea43a962160177aa5a40
+Merge: f1425e0 29bc9e5
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Apr 30 00:31:48 2015 +0200
+
+    Merge pull request #4384 from xinxinsh/wip-11197-firefly
+    
+    pg stuck stale after create with activation delay
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit f1425e093e00343a5ae4b9ff56911bf654a5c152
+Merge: e980e00 69d9339
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Apr 30 00:31:32 2015 +0200
+
+    Merge pull request #4382 from xinxinsh/wip-10718-firefly
+    
+    osd/PGLog.h: 279: FAILED assert(log.log.size() == log_keys_debug.size())
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit e980e0036909afc392f9bdf5532ce500af602c2f
+Merge: b36229b 110c354
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Apr 30 00:23:04 2015 +0200
+
+    Merge pull request #4185 from ldachary/wip-11156-firefly
+    
+    FAILED assert(soid < scrubber.start || soid >= scrubber.end)
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit b36229b5aeec669cd1494f47120ae207a393846f
+Merge: da75978 20c2175
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Apr 28 08:11:18 2015 -0700
+
+    Merge pull request #4475 from ceph/wip-9538.firefly
+    
+    mon: OSDMonitor: fallback to json-pretty in case of invalid formatter
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 20c2175352c9d4e4d0e709d294fe6fe0695761cc
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Fri Sep 19 15:28:36 2014 +0200
+
+    mon: osd find / metadata --format plain fallback
+    
+    ceph --format plain osd find 1 (and metadata) are not implemented and
+    must fallback to the default (json-pretty).
+    
+    http://tracker.ceph.com/issues/9538 Fixes: #9538
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 13780d755115387591888f94ea6c58ac0db3ecc4)
+
+commit da759784ebda11275106c3b280f1d32b64ade00a
+Merge: 1a10a7e e1d5773
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Apr 27 10:30:49 2015 -0700
+
+    Merge pull request #4453 from ceph/wip-11454
+    
+    PG::actingset should be used when checking the number of acting OSDs for...
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 1a10a7eac190249a41b5200f0cc5e3863e76c31d
+Merge: a860e2b b9da6f1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Apr 27 10:14:00 2015 +0200
+
+    Merge pull request #4245 from ceph/wip-11113-firefly
+    
+    librbd: snap_remove should ignore -ENOENT errors
+    
+    Reviewed-by: Shu, Xinxin <xinxin.shu@intel.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit a860e2b3c02ef4f1402b54b08a6bacd4b02cc07f
+Merge: dd15e54 364563a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Apr 27 10:12:40 2015 +0200
+
+    Merge pull request #4206 from ceph/wip-5488-firefly
+    
+    librbd: acquire cache_lock before refreshing parent
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit dd15e54b2fae134126e9795a8a973e391c628eb7
+Merge: 51ff2b6 5404fbf
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Apr 27 10:09:23 2015 +0200
+
+    Merge pull request #3963 from dachary/wip-10153-firefly
+    
+    Rados.shutdown() dies with Illegal instruction (core dumped)
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 51ff2b60dc576fb444bcfea8f6cd3e12043ed5b8
+Merge: 12143ff c0cfd6e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Apr 27 10:08:15 2015 +0200
+
+    Merge pull request #4383 from xinxinsh/wip-11144-firefly
+    
+    erasure-code-profile set races with erasure-code-profile rm
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 1f6b1bb51d0b40899155313165ac5e3a21dde11a
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Mar 3 08:45:58 2015 +0200
+
+    osd: fix PG::all_unfound_are_queried_or_lost for non-existent osds
+    
+    A common mistake upon osd loss is to remove the osd from the crush map
+    before marking the osd lost. This tends to make it so that the user
+    can no longer mark the osd lost to satisfy all_unfound_are_queried_or_lost.
+    
+    The simple solution is for all_unfound_are_queried_or_lost to ignore
+    the osd if it does not exist.
+    
+    Fixes: #10976
+    Backports: firefly,giant
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 5bb51320138ff714806d24312149f6275d546608)
+
+commit e1d57730ddfc92b17562080b10bc22fceb86539a
+Author: Guang Yang <yguang@yahoo-inc.com>
+Date:   Mon Sep 29 08:21:10 2014 +0000
+
+    PG::actingset should be used when checking the number of acting OSDs for a given PG.
+    Signed-off-by: Guang Yang <yguang@yahoo-inc.com>
+    
+    (cherry picked from commit 19be358322be48fafa17b28054619a8b5e7d403b)
+    
+    Conflicts:
+    	src/osd/PG.cc	PG::get_backfill_priority() doesn't exist in firefly
+    			Variation in code related to no "undersized" state in firefly
+    
+    Fixes: #11454
+
+commit f33effccc0592fddfcd9c5c2f5c3385462aa9b84
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Mar 26 10:26:48 2015 -0700
+
+    ReplicatedPG::cancel_pull: requeue waiters as well
+    
+    If we are in recovery_wait, we might not recover that object as part of
+    recover_primary for some time.  Worse, if we are waiting on a backfill
+    which is blocked waiting on a copy_from on the missing object in
+    question, it can become a dead lock.
+    
+    Fixes: 11244
+    Backport: firefly
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 0b2e272430dd7433e6763be99b8a4cb127d9be19)
+
+commit 9b3396516c00ef931bab2d4aa6288ad974ec579d
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Mar 3 11:03:35 2015 -0800
+
+    rgw: update keystone cache with token info
+    
+    Fixes: #11125
+    Backport: hammer, firefly
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 6616294aa140ceb83cc61c6ab6f9947636f5e67d)
+
+commit 8cc4bc162c8c1c240e62840d968b967f5f47b682 (refs/remotes/gh/wip-11416)
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Dec 17 17:12:43 2014 -0800
+
+    rgw: use correct objv_tracker for bucket instance
+    
+    When trying to create a bucket that already existed, use the
+    objv_tracker of the newly created instance, and not of the original
+    bucket.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit fe158ecc25feefcea8aea4133118e4a84900a8ec)
+
+commit c0cfd6e56ca9d17241da159295bcee7cf44c9ba3
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Mar 18 14:17:00 2015 +0100
+
+    osd: erasure-code-profile incremental rm before set
+    
+    It is possible for an incremental change to have both a rm and a set for
+    a given erasure code profile. It only happens when a rm is followed by a
+    set. When a set is followed by a rm, the rm will remove the pending set
+    in the incremental change.
+    
+    The logic is the same for pool create and pool delete.
+    
+    We must apply the incremental erasure-code-profile removal before the
+    creation otherwise rm and set in the same proposal will ignore the set.
+    
+    This fix is minimal. A better change would be that erasure-code-profile
+    set checks if there is a pending removal and wait_for_finished_proposal
+    before creating.
+    
+    http://tracker.ceph.com/issues/11144 Fixes: #11144
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 0d52aca0d0c302983d03b0f5213ffed187e4ed63)
+    
+    Conflicts:
+    	src/osd/OSDMap.cc
+            resolved by replacing i++ with ++i
+
+commit 3fb97e25b194e92112077385b10381801e02ddb9
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Mar 24 15:14:34 2015 -0700
+
+    ReplicatedPG: trim backfill intervals based on peer's last_backfill_started
+    
+    Otherwise, we fail to trim the peer's last_backfill_started and get bug 11199.
+    
+    1) osd 4 backfills up to 31bccdb2/mira01213209-286/head (henceforth: foo)
+    
+    2) Interval change happens
+    
+    3) osd 0 now finds itself backfilling to 4 (lb=foo) and osd.5
+    (lb=b6670ba2/mira01213209-160/snapdir//1, henceforth: bar)
+    
+    4) recover_backfill causes both 4 and 5 to scan forward, so 4 has an interval
+    starting at foo, 5 has an interval starting at bar.
+    
+    5) Once those have come back, recover_backfill attempts to trim off the
+    last_backfill_started, but 4's interval starts after that, so foo remains in
+    osd 4's interval (this is the bug)
+    
+    7) We serve a copyfrom on foo (sent to 4 as well).
+    
+    8) We eventually get to foo in the backfilling. Normally, they would have the
+    same version, but of course we don't update osd.4's interval from the log since
+    it should not have received writes in that interval. Thus, we end up trying to
+    recover foo on osd.4 anyway.
+    
+    9) But, an interval change happens between removing foo from osd.4 and
+    completing the recovery, leaving osd.4 without foo, but with lb >= foo
+    
+    Fixes: #11199
+    Backport: firefly
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 1388d6bd949a18e8ac0aecb0eb79ffb93d316879)
+
+commit 29bc9e56a978d4e18a507e71858baa21037964c4
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Mar 24 10:48:02 2015 -0700
+
+    PG: set/clear CREATING in Primary state entry/exit
+    
+    Previously, we did not actually set it when we got a pg creation message from
+    the mon.  It would actually get set on the first start_peering_interval after
+    that point.  If we don't get that far, but do send a stat update to the mon, we
+    can end up with 11197.  Instead, let's just set it and clear it upon entry into
+    and exit from the Primary state.
+    
+    Fixes: 11197
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit ddf0292250dfb1040d3cad96fa2cf886fd65933c)
+
+commit 12143ff9b25fdd96f8d1a9cecb1329c7f354d414
+Merge: 53eff01 bf1d8e8
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Apr 6 10:24:06 2015 -0700
+
+    Merge pull request #4275 from jeanchlopez/wip-11160-firefly
+    
+    rgw: shouldn't need to disable rgw_socket_path if frontend is configured
+
+commit bf1d8e888439df75feadde76bafe7d07d0e8481a
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Mar 18 20:49:13 2015 -0700
+
+    rgw: don't use rgw_socket_path if frontend is configured
+    
+    Fixes: #11160
+    Backport: hammer, firefly
+    
+    Previously if we wanted to use the tcp fcgi socket, we needed to clear
+    rgw_socket_path.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 53eff01f1e7ecd3783bb05c47588a134d4fededc
+Merge: 899738e 80afc5e
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Apr 1 16:30:23 2015 -0700
+
+    Merge pull request #4247 from ceph/wip-11303
+    
+    Fix do_autogen.sh so that -L is allowed
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 80afc5eca293e5e2f168f219931a2f554040cdd8
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Apr 1 19:10:33 2015 -0400
+
+    Fix do_autogen.sh so that -L is allowed
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+
+commit b9da6f1025683d240baa73a9fcccbf2780035231
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Mar 16 18:40:49 2015 -0400
+
+    librbd: snap_remove should ignore -ENOENT errors
+    
+    If the attempt to deregister the snapshot from the parent
+    image fails with -ENOENT, ignore the error as it is safe
+    to assume that the child is not associated with the parent.
+    
+    Fixes: #11113
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit cf8094942ccdba831e03e5a79451cfa5d78a135f)
+
+commit b250b3cfae69f4e8354027fae26fd85a792da0df
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Mar 16 18:35:07 2015 -0400
+
+    librbd: get_parent_info should protect against invalid parent
+    
+    get_parent_info should return -ENOENT if the image does not
+    have an associated parent image.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 21afd0ef8e1fb81c5ace5fd403513c542e2413e3)
+
+commit 69d9339fb352342a87e1148e9b5161246b27776a
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Feb 2 09:07:27 2015 -0800
+
+    PGLog: improve PGLog::check() debugging
+    
+    Related: 10718
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit c656bcec2441c90b084ca50a17c37625d69942a1)
+
+commit 73cbb6f34a5a556dbbd4db2e6f81391d83447efd
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Mar 17 10:07:03 2015 -0700
+
+    PGLog::merge_log: in tail extend case, log.log might be empty
+    
+    Even if the tail != last_update, the log might be empty due to split
+    moving all entries into other logs.
+    
+    Fixes: 10718
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit f5a2aef3db29b99634250fd388a0c2d9b9a23d38)
+
+commit 62645d30b0ceee075a56bc692fd65cee85b09aa5 (refs/remotes/gh/wip-11256-firefly)
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Mar 30 17:34:57 2015 -0700
+
+    cls_rgw: use multimap to keep pending operations in bucket index
+    
+    Fixes: #11256
+    Multiple concurrent requests might be sent using the same tag, need the
+    entry map to be able to hold multiple entries.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 7538319dd7aa80a3318c108d345dee8044cf20a8
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Mar 27 16:32:48 2015 -0700
+
+    rgw: generate new tag for object when setting object attrs
+    
+    Fixes: #11256
+    Backport: firefly, hammer
+    
+    Beforehand we were reusing the object's tag, which is problematic as
+    this tag is used for bucket index updates, and we might be clobbering a
+    racing update (like object removal).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 364563aac979fdf5ccbb6c588051d097a26bc594
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Mar 16 11:04:22 2015 -0400
+
+    librbd: acquire cache_lock before refreshing parent
+    
+    cache_lock needs to be acquired before snap_lock to avoid
+    the potential for deadlock.
+    
+    Fixes: #5488
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 110c35498942ea0feec395b6e7992f802dd740ce
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Mar 20 15:28:15 2015 -0700
+
+    ReplicatedPG::promote_object: check scrubber and block if necessary
+    
+    Otherwise, we might attempt to promote into an in-progress scrub
+    interval causing 11156.  I would have added a return value to
+    promote_object(), but could not find an existing user which
+    cared to distinguish the cases, even with a null op passed.
+    All existing users are in maybe_handle_cache.  The ones which
+    pass a null op are for promoting the object in parallel
+    with a proxy -- a case where not actually performing the promote
+    does not really matter.
+    
+    Fixes: #11156
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 65bb4df599541cd2e0f195b905f24f529e255c00)
+
+commit 899738e10e82b50dcf7dfffe5cc83937179bf323
+Merge: 15acfe7 9437cb1
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 23 20:38:43 2015 +0100
+
+    Merge pull request #4126 from dzafman/wip-11176-firefly
+    
+    ceph-objectstore-tool: Output only unsupported features when incomatible
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 15acfe7f226ca7bc2c942d5fbcd3a40bd41e5930
+Merge: 6c95cd2 cddbff9
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sat Mar 21 11:38:28 2015 +0100
+
+    Merge pull request #4079 from dachary/wip-11157-firefly
+    
+    doc,tests: force checkout of submodules
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 6c95cd24f6f4a1a933f8799754831e7a5c0a725d
+Merge: ea79a3b 43053fc
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Mar 20 21:06:17 2015 -0700
+
+    Merge pull request #4129 from dzafman/wip-11139-firefly
+    
+    ceph-objectstore-tool: Use exit status 11 for incompatible import attemp...
+
+commit 43053fcd8969c406969fef67613aa37ad1cc86bf
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Mar 20 17:48:01 2015 -0700
+
+    ceph-objectstore-tool: Use exit status 11 for incompatible import attempt
+    
+    This is used so upgrade testing doesn't generate false failure.
+    Fixes: #11139
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 175aff8afe8215547ab57f8d8017ce8fdc0ff543)
+
+commit 9437cb1b284ec9366b51d660396c2c8a9366b31f
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Mar 20 16:57:40 2015 -0700
+
+    ceph-objectstore-tool: Output only unsupported features when incomatible
+    
+    Fixes: #11176
+    Backport: firefly, giant
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 5b23f5b5892b36fb7d06efc0d77e64a24ef6e8c9)
+
+commit ea79a3b122917ddf5dc1972bb9dc5b20f78e2f18
+Merge: c1c20d8 07fc9f6
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Mar 20 08:44:39 2015 +0100
+
+    Merge pull request #3970 from ceph/firefly-11053
+    
+    mds: fix assertion caused by system clock backwards
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit c1c20d89d511499d678fdba0667581e88b9b5d95
+Merge: 2fbb9a6 63b39db
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Mar 20 08:42:58 2015 +0100
+
+    Merge pull request #4021 from ceph/wip-7737-firefly
+    
+    osd: fix negative degraded objects during backfilling
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 2fbb9a63a82ac5136b033799aec303fc151b25e5
+Merge: 3160e8d 2c7eda6
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Mar 20 08:38:26 2015 +0100
+
+    Merge pull request #3952 from dachary/wip-9986-firefly
+    
+    objecter: fix map skipping
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 3160e8d6918a04d1fec9fdccbc30bf007c7940c6
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Mar 18 20:55:24 2015 -0700
+
+    init-radosgw*: don't require rgw_socket_path to be defined
+    
+    Fixes: #11159
+    Backport: hammer, firefly
+    
+    Scripts required rgw_socket_path to exist in order to start radosgw.
+    This is not needed.
+    
+    Reported-by: Dan Mick <dmick@redhat.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 6823bcdcd0ce72cd223e809291f46d82da76115c)
+
+commit cddbff9a3653d6257d13e0ac411cd6e4cd71feef
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Mar 19 00:32:39 2015 +0100
+
+    doc,tests: force checkout of submodules
+    
+    When updating submodules, always checkout even if the HEAD is the
+    desired commit hash (update --force) to avoid the following:
+    
+        * a directory gmock exists in hammer
+        * a submodule gmock replaces the directory gmock in master
+        * checkout master + submodule update : gmock/.git is created
+        * checkout hammer : the gmock directory still contains the .git from
+          master because it did not exist at the time and checkout won't
+          remove untracked directories
+        * checkout master + submodule update : git rev-parse HEAD is
+          at the desired commit although the content of the gmock directory
+          is from hammer
+    
+    http://tracker.ceph.com/issues/11157 Fixes: #11157
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+
+commit 63b39dbd529936e60d0fd08dffd35f82b3d1729c (refs/remotes/gh/wip-7737-firefly)
+Author: Guang Yang <yguang@yahoo-inc.com>
+Date:   Thu Feb 26 08:13:12 2015 +0000
+
+    osd: fix negative degraded objects during backfilling
+    
+    When there is deleting requests during backfilling, the reported number of degraded
+    objects could be negative, as the primary's num_objects is the latest (locally) but
+    the number for replicas might not reflect the deletings. A simple fix is to ignore
+    the negative subtracted value.
+    
+    Signed-off-by: Guang Yang <yguang@yahoo-inc.com>
+    (cherry picked from commit 14d7e36d3c978844da73d0e1c8a3a1ec863bac15)
+    
+    Conflicts:
+    	src/osd/PG.cc
+
+commit 493d285508914769cba3639b601ae6c20303af0d
+Merge: 9839726 8a25a51
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 23:20:36 2015 +0100
+
+    Merge pull request #3966 from theanalyst/wip-10698-backport
+    
+    rgw: fail s3 POST auth if keystone not configured
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 9839726f009a3189013a23e8226d2f7618f1d56b
+Merge: dab85cb 651dc55
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 23:19:22 2015 +0100
+
+    Merge pull request #3934 from dachary/wip-10665-firefly
+    
+    rgw: Swift API. Dump container's custom metadata.
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit dab85cb7fcc5770fe4d6f075cf583a33ea335aa3
+Merge: 0e55046 d0fd417
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 23:18:59 2015 +0100
+
+    Merge pull request #3929 from dachary/wip-10475-firefly
+    
+    rgw: Swift API. Support for X-Remove-Container-Meta-{key} header.
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 0e550461666d6e3d32981f99c8327931a6bd6d62
+Merge: 1ca6dd9 963439f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 23:18:24 2015 +0100
+
+    Merge pull request #3938 from dachary/wip-10770-firefly
+    
+    rgw: send appropriate op to cancel bucket index pending operation
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 1ca6dd9676014a81983bd397e3154bf53243c7f2
+Merge: 66327d6 34d19e1
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 23:17:56 2015 +0100
+
+    Merge pull request #3961 from dachary/wip-10106-firefly
+    
+    rgw: flush xml header on get acl request
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 66327d69a9bfa875d58b063c180a11b2769a093e
+Merge: de45d9e 9074eb7
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 23:16:53 2015 +0100
+
+    Merge pull request #3958 from dachary/wip-10062-firefly
+    
+    rgw: check for timestamp for s3 keystone auth
+    
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+    Reviewed-by: Valery Tschopp <valery.tschopp@switch.ch>
+
+commit de45d9e2606d8fb6ea5533521163669e748e7d01
+Merge: 308f758 8ef14fc
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 17:00:27 2015 +0100
+
+    Merge pull request #3942 from dachary/wip-10844-firefly
+    
+    mon: MonCap: take EntityName instead when expanding profiles
+    
+    Reviewed-by: Joao Eduardo Luis <joao@redhat.com>
+
+commit 308f758f49d28c012f3ba765519144e5ea5794e6
+Merge: c59a081 ca42905
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:53:48 2015 +0100
+
+    Merge pull request #3936 from dachary/wip-10724-firefly
+    
+     backport ceph-disk man page to Firefly
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit c59a08135c35072fa05104e26b500553e800cdd1
+Merge: 331acae 8ec8f11
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:50:39 2015 +0100
+
+    Merge pull request #3940 from dachary/wip-8753-firefly
+    
+    ReplicatedPG::on_change: clean up callbacks_for_degraded_object
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 331acae4d6b34bb047c914945a2d9c0d9c7b9562
+Merge: fc364a7 4e32ff2
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:49:56 2015 +0100
+
+    Merge pull request #3827 from dachary/wip-10259-firefly
+    
+    osd_types: op_queue_age_hist and fs_perf_stat should be in osd_stat_t::o...
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit fc364a76f85eb5016ea460e5fd9c1603df374bcf
+Merge: 31d99d2 b0d0d44
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:48:02 2015 +0100
+
+    Merge pull request #3962 from dachary/wip-10150-firefly
+    
+    ReplicatedPG::scan_range: an object can disappear between the list and t...
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 31d99d2de714eb4a8eeb4a431cbc0d40ca749f15
+Merge: 45f0870 5865411
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:45:32 2015 +0100
+
+    Merge pull request #3948 from dachary/wip-9891-firefly
+    
+    DBObjectMap: lock header_lock on sync()
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 45f087032e27c63dc459318717b05fe5f9888664
+Merge: 3050262 6207333
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:43:48 2015 +0100
+
+    Merge pull request #3949 from dachary/wip-9915-firefly
+    
+    osd: cache tiering: fix the atime logic of the eviction
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 3050262d7dcb8ac20b9f86544461bb59f4b87cf5
+Merge: c0abc4d 0f31388
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:40:12 2015 +0100
+
+    Merge pull request #3944 from dachary/wip-9193-firefly
+    
+    messages/MWatchNotify: include an error code in the message
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit c0abc4d769afd00773a9f466ffd3feced9cdb17d
+Merge: f9acd3a f856739
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:39:41 2015 +0100
+
+    Merge pull request #3943 from dachary/wip-8011-firefly
+    
+    ReplicatedPG: fail a non-blocking flush if the object is being scrubbed
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit f9acd3ad7397ac1e745beeeaf21b55ecd95484a1
+Merge: f95d327 ca96b59
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:39:16 2015 +0100
+
+    Merge pull request #3941 from dachary/wip-10817-firefly
+    
+    WorkQueue: make wait timeout on empty queue configurable
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit f95d327cbc0750cfb77114c66082ddd5fc458b94
+Merge: 32de8ab 3782b8b
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:38:42 2015 +0100
+
+    Merge pull request #3937 from dachary/wip-10762-firefly
+    
+    mon: ignore osd failures from before up_from
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 32de8ab146f242f7b73aca211f059bc39d38f85c
+Merge: 24a8c10 6fd3dfa
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:38:22 2015 +0100
+
+    Merge pull request #3933 from dachary/wip-10617-firefly
+    
+    osd: do not ignore deleted pgs on startup
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 24a8c10a5a215c60b525f4170e45565390e02231
+Merge: f4e76c3 368a5a8
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:38:01 2015 +0100
+
+    Merge pull request #3932 from dachary/wip-10546-firefly
+    
+    mon: Monitor: fix timecheck rounds period
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit f4e76c3db90bf383c02cdb8cca19a37cd187095b
+Merge: fa518ff 66b13f2
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:37:36 2015 +0100
+
+    Merge pull request #3931 from dachary/wip-10512-firefly
+    
+    osd: requeue blocked op before flush it was blocked on
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit fa518ffc5961785f01f0f91980c38a7e02686901
+Merge: ac9980e a22aa8f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:37:05 2015 +0100
+
+    Merge pull request #3930 from dachary/wip-10497-firefly
+    
+    librados: Translate operation flags from C APIs
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit ac9980e074adf587b8b16af0bd08b3fc3233804d
+Merge: ccebb5f d3de8a5
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:31:40 2015 +0100
+
+    Merge pull request #3955 from dachary/wip-10059-firefly
+    
+    PG: always clear_primary_state
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit ccebb5f90bea0fd3bac1bd7ab93e5e07700f1301
+Merge: a30379d f4bab86
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:31:16 2015 +0100
+
+    Merge pull request #3954 from dachary/wip-10014-firefly
+    
+    ObjectStore: Don't use largest_data_off to calc data_align.
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit a30379d9af89cffa4b3083ae14a94e3758c2b01c
+Merge: e575ca8 eb03e79
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:25:59 2015 +0100
+
+    Merge pull request #3947 from dachary/wip-9555-firefly
+    
+    osd: check that source OSD is valid for MOSDRepScrub
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit e575ca84419f8be5c7fae7e2ffac549956a74f82
+Merge: cd675bb c60da2f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:25:30 2015 +0100
+
+    Merge pull request #3964 from dachary/wip-10157-firefly
+    
+    PGLog: include rollback_info_trimmed_to in (read|write)_log
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit cd675bba5fdfb85689880ca7ecbd284181984a63
+Merge: 2b8e476 1a0f770
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:24:55 2015 +0100
+
+    Merge pull request #3960 from dachary/wip-6003-firefly
+    
+    FileJournal: fix journalq population in do_read_entry()
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 2b8e4762edb7c35950f7e0d69bba2b5e1d83133a
+Merge: d434ead a746f7e
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:24:03 2015 +0100
+
+    Merge pull request #3950 from dachary/wip-9985-firefly
+    
+    Get the currently atime of the object in cache pool for eviction
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Xinze Chi <xmdxcxz@gmail.com>
+
+commit d434eadadd1df6e4df18a6914015736bc09722d1
+Merge: e79e5f8 5f1245e
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 16:20:12 2015 +0100
+
+    Merge pull request #3935 from dachary/wip-10723-firefly
+    
+    Fix memory leak in python rados bindings
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit e79e5f8fbbfba6984cf21784e20d1c46cb60a397
+Merge: a9a36de 91b2aca
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 15:02:37 2015 +0100
+
+    Merge pull request #3866 from ceph/wip-cot-firefly
+    
+    Backport ceph-objectstore-tool changes to firefly
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit a9a36deb89ac5719787e905469fe0b1bde5d58ca
+Merge: 83c571e 7e85722
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 15:00:56 2015 +0100
+
+    Merge pull request #3996 from dzafman/wip-10676
+    
+    Fix ceph command manpage to match ceph -h
+    
+    Reviewed-by: Xinxin Shu <xinxin.shu5040@gmail.com>
+
+commit 83c571e3067b4cad3e4567522c797d09a82d87db
+Merge: baa74b8 d5c3a14
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 16 14:55:37 2015 +0100
+
+    Merge pull request #3927 from dachary/wip-10351-firefly
+    
+    mount.ceph: avoid spurious error message
+    
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+
+commit 7e85722fd4c89715fc2ed79697c82d65d7ebf287
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Mar 13 16:50:13 2015 -0700
+
+    doc: Minor fixes to ceph command manpage
+    
+    Fixes: #10676
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+
+commit 9ac488c1eb0e30511079ba05aaf11c79615b3940
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Mar 12 11:39:52 2015 -0700
+
+    doc: Fix ceph command manpage to match ceph -h (firefly)
+    
+    Improve synopsis section
+    Fixes: #10676
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+
+commit 5f1245e131e33a98572408c8223deed2c7cf7b75
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Feb 9 20:50:23 2015 -0800
+
+    rados.py: keep reference to python callbacks
+    
+    If we don't keep a reference to these, the librados aio calls will
+    segfault since the python-level callbacks will have been garbage
+    collected. Passing them to aio_create_completion() does not take a
+    reference to them. Keep a reference in the python Completion object
+    associated with the request, since they need the same lifetime.
+    
+    This fixes a regression from 60b019f69aa0e39d276c669698c92fc890599f50.
+    
+    Fixes: #10775
+    Backport: dumpling, firefly, giant
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 36d37aadbbbece28d70e827511f1a473d851463d)
+
+commit cf366fc3b21ff6f98530dbadb75a430c25672d56
+Author: Nilamdyuti Goswami <ngoswami@redhat.com>
+Date:   Thu Dec 18 17:11:22 2014 +0530
+
+    doc: Changes format style in ceph to improve readability as html.
+    
+    Signed-off-by: Nilamdyuti Goswami <ngoswami@redhat.com>
+    (cherry picked from commit 8b796173063ac9af8c21364521fc5ee23d901196)
+
+commit 07fc9f66a69aa31d2cf8bf7a277d3e14ad6209be
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Mar 12 11:01:46 2015 +0800
+
+    mds: fix assertion caused by system clock backwards
+    
+    Fixes: #11053
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+
+commit 8a25a51e42cdaed2c66dc25a6c6d0245441123a3
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jan 30 07:03:30 2015 -0800
+
+    rgw: fail s3 POST auth if keystone not configured
+    
+    Fixes: #10698
+    This fixes issue introduced in 8b3dfc9472022ea45ad24e02e0aa21dfdad798f8,
+    where if user does not exist, we try keystone authentication. However,
+    if keystone is not configured we justt fall through without failing.
+    This would have failed later on due to bucket permissions, unless bucket
+    had a public write permissions.
+    
+    Backports: Firefly
+    Reported-by: Valery Tschopp <valery.tschopp@switch.ch>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    
+    Conflicts:
+    	src/rgw/rgw_rest_s3.cc
+
+commit eb03e792040bd22c1ae8b7dd73d94fbfd6208eec
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Dec 13 08:06:31 2014 -0800
+
+    osd: check that source OSD is valid for MOSDRepScrub
+    
+    Make sure the message we got from the peer OSD is valid.  Specifically,
+    this avoids a race like this:
+    
+    - A marks down B
+    - B sends MOSDRepScrub
+    - A accepts connection from B as new
+    - A replies to scrub
+    - B crashes because msgr seq 1 < expected seq 1000+
+    
+    See #8880 for the most recent fix for requests.
+    
+    Fixes: #9555
+    Backport: giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 847e5e102522d651aa9687a54aaafcebf3afc596)
+    
+    Conflicts:
+        src/osd/OSD.cc: require functions first argument is now a reference
+
+commit ae18707b3caa115dc510aff38b77f8afe8555c61
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Dec 13 07:56:39 2014 -0800
+
+    osd: pass Message* to most require_* helpers
+    
+    These do nothing but op->get_req(); pass the Message* explicitly so that
+    non-OpRequest callers can use them.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 707a111d53efb09b3471dd3788b86d2bfee4e96f)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+    	src/osd/OSD.h
+            significantly changes had to be made but in a simple way and
+    	stays within the scope of the original commit
+
+commit c60da2f3c34e7325c748d2d6e55140a0a30013fd
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Nov 20 15:15:08 2014 -0800
+
+    PGLog: include rollback_info_trimmed_to in (read|write)_log
+    
+    Fixes: #10157
+    Backport: firefly, giant
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 1fe8b846641486cc294fe7e1d2450132c38d2dba)
+    
+    Conflicts:
+    	src/osd/PGLog.cc
+            in the context coll_t::META_COLL was replaced with META_COLL
+
+commit 1a0f770eea18af6b276a31157f201a93166eb038
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Feb 6 09:52:29 2015 -0800
+
+    FileJournal: fix journalq population in do_read_entry()
+    
+    Fixes: 6003
+    Backport: dumpling, firefly, giant
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit bae1f3eaa09c4747b8bfc6fb5dc673aa6989b695)
+    
+    Conflicts:
+    	src/os/FileJournal.cc
+            because reinterpret_cast was added near two hunks after firefly
+
+commit a746f7e5985198024067cb6e123569c09169b356
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 31 19:33:59 2014 -0700
+
+    osd/ReplicatedPG: fix compile error
+    
+    From 1fef4c3d541cba360738437420ebfa2447d5802e.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4a9ad7dc2da6f4fa6a64235776a3f1d2799aef60)
+
+commit 5404fbfdd9b18cdb3fe5bed67146c769ec3acfa0
+Author: Federico Simoncelli <fsimonce@redhat.com>
+Date:   Sat Nov 15 14:14:04 2014 +0000
+
+    common: do not unlock rwlock on destruction
+    
+    According to pthread_rwlock_unlock(3p):
+    
+     Results are undefined if the read-write lock rwlock is not held
+     by the calling thread.
+    
+    and:
+    
+     https://sourceware.org/bugzilla/show_bug.cgi?id=17561
+    
+     Calling pthread_rwlock_unlock on an rwlock which is not locked
+     is undefined.
+    
+    calling pthread_rwlock_unlock on RWLock destruction could cause
+    an unknown behavior for two reasons:
+    
+    - the lock is acquired by another thread (undefined)
+    - the lock is not acquired (undefined)
+    
+    Moreover since glibc-2.20 calling pthread_rwlock_unlock on a
+    rwlock that is not locked results in a SIGILL that kills the
+    application.
+    
+    This patch removes the pthread_rwlock_unlock call on destruction
+    and replaces it with an assertion to check that the RWLock is
+    not in use.
+    
+    Any code that relied on the implicit release is now going to
+    break the assertion, e.g.:
+    
+     {
+       RWLock l;
+       l.get(for_write);
+     } // implicit release, wrong.
+    
+    Signed-off-by: Federico Simoncelli <fsimonce@redhat.com>
+    (cherry picked from commit cf2104d4d991361c53f6e2fea93b69de10cd654b)
+
+commit a73a4cb3889a6da21c3cfa4ddfa16d1a7059d20c
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Sat May 3 08:32:19 2014 -0700
+
+    common/RWLock: track read/write locks via atomics for assertion checks
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 92615ea95a31d9fd22c3d11c860e0f502dc52c26)
+
+commit b0d0d44018de6289b64b6334edd4959ca4a0cc48
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Dec 11 13:05:54 2014 -0800
+
+    ReplicatedPG::scan_range: an object can disappear between the list and the attr get
+    
+    The first item in the range is often last_backfill, upon which writes
+    can be occuring.  It's trimmed off on the primary side anyway.
+    
+    Fixes: 10150
+    Backport: dumpling, firefly, giant
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit dce6f288ad541fe7f0ef8374301cd712dd3bfa39)
+
+commit 34d19e1501b242fd8fc7cc95656592b5982f29a6
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jan 30 18:42:40 2015 -0800
+
+    rgw: flush xml header on get acl request
+    
+    Fixes: #10106
+    Backport: firefly, giant
+    
+    dump_start() updates the formatter with the appropriate prefix, however,
+    we never flushed the formatter.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit eb45f861343162e018968b8c56693a8c6f5b2cab)
+
+commit 9074eb7c46589aa1e5d10a2b9a8534f22dff2154
+Author: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+Date:   Mon Nov 17 17:37:00 2014 +0530
+
+    rgw: check for timestamp for s3 keystone auth
+    
+    This commit ensures that we check for timestamp of s3 request is within
+    acceptable grace time of radosgw
+    Addresses some failures in #10062
+    Fixes: #10062
+    Signed-off-by: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+    
+    (cherry picked from commit 4b35ae067fef9f97b886afe112d662c61c564365)
+
+commit d3de8a5722a68d69023cf60c9076c2fb19058bf9
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Nov 19 08:20:16 2014 -0800
+
+    PG: always clear_primary_state on new interval, but only clear pg temp if not primary
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit f692bfe076b8ddb679c6d1a6ea78cc47f0876326)
+
+commit 8b07236cd799b4752cbe620b737343a02a703a17
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Nov 14 15:44:20 2014 -0800
+
+    PG: always clear_primary_state when leaving Primary
+    
+    Otherwise, entries from the log collection process might leak into the next
+    epoch, where we might end up choosing a different authoritative log.  In this
+    case, it resulted in us not rolling back to log entries on one of the replicas
+    prior to trying to recover from an affected object due to the peer_missing not
+    being cleared.
+    
+    Fixes: #10059
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit c87bde64dfccb5d6ee2877cc74c66fc064b1bcd7)
+
+commit f4bab86fe3b218d66c14d06883c297836d9ca19d
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Mon Oct 27 11:22:13 2014 +0800
+
+    ObjectStore: Don't use largest_data_off to calc data_align.
+    
+    If largest_data_off % CEPH_PAGE_SIZE != 0, the get_data_aligment return
+    a erro value. This make the FileJouranl::align_bl to memcopy much data.
+    
+    Tested-by: Sage Weil <sage@redhat.com>
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit a1aa70f2f21339feabfe9c1b3c9c9f97fbd53c9d)
+
+commit 2c7eda690ae2f71c8e8e189b8f7330196c1f385f
+Author: Ding Dinghua <dingdinghua85@gmail.com>
+Date:   Thu Oct 30 14:58:42 2014 +0800
+
+    osdc/Objecter: Fix a bug of dead looping in Objecter::handle_osd_map
+    
+    If current map epoch is less than oldest epoch, current map epoch
+    should step up to oldest epoch.
+    
+    Fixes: #9986
+    Signed-off-by: Ding Dinghua <dingdinghua85@gmail.com>
+    (cherry picked from commit e0166a23c2cf655bfb4cf873be021a14d9b9be27)
+    
+    Conflicts:
+    	src/osdc/Objecter.cc
+            the debug line that follows was re-indented
+
+commit 11f8017cbdf94d4a8083412b96c251ee39286541
+Author: Ding Dinghua <dingdinghua85@gmail.com>
+Date:   Thu Oct 30 14:58:05 2014 +0800
+
+    osdc/Objecter: e shouldn't be zero in Objecter::handle_osd_map
+    
+    Signed-off-by: Ding Dinghua <dingdinghua85@gmail.com>
+    (cherry picked from commit 31c584c8ba022cd44fe2872d221f3026618cefab)
+    
+    Conflicts:
+    	src/osdc/Objecter.cc
+            the debug line that follows was re-indented
+
+commit 7831582026441abbd6066dd951db4b63ffb45402
+Author: Xinze Chi <xmdxcxz@gmail.com>
+Date:   Wed Oct 29 07:11:11 2014 +0000
+
+    Get the currently atime of the object in cache pool for eviction
+    
+    Because if there are mutiple atime in agent_state for the same object, we should use the recently one.
+    
+    Signed-off-by: Xinze Chi <xmdxcxz@gmail.com>
+    (cherry picked from commit 1fef4c3d541cba360738437420ebfa2447d5802e)
+
+commit 620733386bd0694960cecac8f32bd1538382d5bb
+Author: Zhiqiang Wang <zhiqiang.wang@intel.com>
+Date:   Tue Oct 28 09:37:11 2014 +0800
+
+    osd: cache tiering: fix the atime logic of the eviction
+    
+    Reported-by: Xinze Chi <xmdxcxz@gmail.com>
+    Signed-off-by: Zhiqiang Wang <zhiqiang.wang@intel.com>
+    (cherry picked from commit 622c5ac41707069ef8db92cb67c9185acf125d40)
+
+commit 5865411360f722ec511f2df6656d4ba975bef8eb
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Feb 20 13:43:46 2015 -0800
+
+    DBObjectMap: lock header_lock on sync()
+    
+    Otherwise, we can race with another thread updating state.seq
+    resulting in the old, smaller value getting persisted.  If there
+    is a crash at that time, we will reuse a sequence number, resulting
+    in an inconsistent node tree and bug #9891.
+    
+    Fixes: 9891
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 2b63dd25fc1c73fa42e52e9ea4ab5a45dd9422a0)
+    
+    Conflicts:
+    	src/os/DBObjectMap.cc
+            because we have state.v = 1; instead of state.v = 2;
+
+commit baa74b88a4f4b6073b99b7a9774692b37405b59e
+Merge: be6559a 420ea03
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Mar 11 09:09:23 2015 +0100
+
+    Merge pull request #3828 from dachary/wip-10425-firefly
+    
+    librados: fix resources leakage in RadosClient::connect()
+    
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit be6559a423badde3b573b0c9055056999baae104
+Merge: f91d355 6e58732
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Mar 11 09:06:27 2015 +0100
+
+    Merge pull request #3826 from dachary/wip-10257-firefly
+    
+    mon: PGMonitor: several stats output error fixes
+    
+    Reviewed-by: Joao Eduardo Luis <joao@redhat.com>
+
+commit f91d355306620cc543113ed21fddf84f4c170d6e
+Merge: f5525a1 1e58bb4
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Mar 11 09:05:13 2015 +0100
+
+    Merge pull request #3824 from dachary/wip-10353-firefly
+    
+    crush: set_choose_tries = 100 for erasure code rulesets
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f5525a1f41c9154b48f3ad0ccd899b5203c897bd
+Merge: b7e3614 27dbbb3
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Mar 11 09:04:33 2015 +0100
+
+    Merge pull request #3823 from dachary/wip-10787-firefly
+    
+    mon/OSDMonitor: do not trust small values in osd epoch cache
+    
+    Reviewed-by: Joao Eduardo Luis <joao@redhat.com>
+
+commit b7e3614ff38379aeea66ba00c64bc88ffa080963
+Merge: b5a67f0 7ed92f7
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Mar 11 09:02:16 2015 +0100
+
+    Merge pull request #3915 from dachary/wip-10080-firefly
+    
+    SimpleMessenger: allow RESETSESSION whenever we forget an endpoint
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 8ef14fcca715a33be8c611a37628c90d7fafca43
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Wed Feb 11 23:36:01 2015 +0000
+
+    mon: MonCap: take EntityName instead when expanding profiles
+    
+    entity_name_t is tightly coupled to the messenger, while EntityName is
+    tied to auth.  When expanding profiles we want to tie the profile
+    expansion to the entity that was authenticated.  Otherwise we may incur
+    in weird behavior such as having caps validation failing because a given
+    client messenger inst does not match the auth entity it used.
+    
+    e.g., running
+    
+    ceph --name osd.0 config-key exists foo daemon-private/osd.X/foo
+    
+    has entity_name_t 'client.12345' and EntityName 'osd.0'.  Using
+    entity_name_t during profile expansion would not allow the client access
+    to daemon-private/osd.X/foo (client.12345 != osd.X).
+    
+    Fixes: #10844
+    Backport: firefly,giant
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 87544f68b88fb3dd17c519de3119a9ad9ab21dfb)
+
+commit 370f37f452aff3b48f9ae8a33b7ef26b572b41c8
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Fri Nov 14 21:03:54 2014 +0000
+
+    mon: Monitor: stash auth entity name in session
+    
+    Backport: giant
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit ca8e1efc0be9bffcfbdce5593526d257aa498062)
+
+commit 0f31388eb2bb68c09ab270f871b421a9c368af97
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Aug 10 17:51:08 2014 -0700
+
+    messages/MWatchNotify: include an error code in the message
+    
+    Document the fields, while we are here.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 7c7bf5fee7be397ef141b947f532a2a0b3567b42)
+
+commit f856739824bc271405a6fa35bdefc2bdc42c2f02
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Nov 20 14:27:39 2014 -0800
+
+    ReplicatedPG: fail a non-blocking flush if the object is being scrubbed
+    
+    Fixes: #8011
+    Backport: firefly, giant
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 9b26de3f3653d38dcdfc5b97874089f19d2a59d7)
+
+commit ca96b59db529ffbba0c834795800b6e90a7e4fce
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Feb 9 17:11:38 2015 -0800
+
+    WorkQueue: make wait timeout on empty queue configurable
+    
+    Fixes: 10817
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 5aa6f910843e98a05bfcabe6f29d612cf335edbf)
+
+commit 8ec8f1175a129624dffb511782664d74966d3c42
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Feb 9 10:37:15 2015 -0800
+
+    ReplicatedPG::on_change: clean up callbacks_for_degraded_object
+    
+    Backport: dumpling, firefly, giant
+    Fixes: 8753
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit a5ecaa189d47a69466a6cd55fa4180e5c3092dc2)
+
+commit 963439f1038000c3f28c728350a2e9d351341e0b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Feb 5 09:33:26 2015 -0800
+
+    rgw: send appropriate op to cancel bucket index pending operation
+    
+    Fixes: #10770
+    Backport: firefly, giant
+    
+    Reported-by: baijiaruo <baijiaruo@126.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit dfee96e3aebcaeef18c721ab73f0460eba69f1c7)
+    
+    Conflicts:
+    	src/rgw/rgw_rados.cc
+
+commit 3782b8b3652c0eb3fb8f65193ecfe09e92925170
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Feb 5 03:07:50 2015 -0800
+
+    mon: ignore osd failures from before up_from
+    
+    If the failure was generated for an instance of the OSD prior to when
+    it came up, ignore it.
+    
+    This probably causes a fair bit of unnecessary flapping in the wild...
+    
+    Backport: giant, firefly
+    Fixes: #10762
+    Reported-by: Dan van der Ster <dan@vanderster.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 400ac237d35d0d1d53f240fea87e8483c0e2a7f5)
+
+commit ca42905a6fca8b2b404500a6f74951ae20c8a488
+Author: Nilamdyuti Goswami <ngoswami@redhat.com>
+Date:   Wed Nov 26 22:21:32 2014 +0530
+
+    doc: Adds the updated man page for ceph-disk utility.
+    
+    Signed-off-by: Nilamdyuti Goswami <ngoswami@redhat.com>
+    (cherry picked from commit 016080d2c39919d73956e5e138ba5f079327aa44)
+
+commit 6602ab4581f27e066484b4c419d8206fcc20e045
+Author: Nilamdyuti Goswami <ngoswami@redhat.com>
+Date:   Wed Nov 26 22:19:01 2014 +0530
+
+    doc: Updates the man page for ceph-disk utility with some changes.
+    
+    Signed-off-by: Nilamdyuti Goswami <ngoswami@redhat.com>
+    (cherry picked from commit 8a48847cd46d4807ca1dbd13d3a561a693cdc877)
+
+commit 459807021dd3bf86ce31d30a3b576e783da67e3f
+Author: Nilamdyuti Goswami <ngoswami@redhat.com>
+Date:   Tue Nov 25 21:23:08 2014 +0530
+
+    doc: Adds man page for ceph-disk utility.
+    
+    Signed-off-by: Nilamdyuti Goswami <ngoswami@redhat.com>
+    (cherry picked from commit 9a118d56d4a5b0a0456e9f092f5ae9293f7bf3f3)
+
+commit 430d821b7592148ad9b2656bb3031a0484763f33
+Author: Nilamdyuti Goswami <ngoswami@redhat.com>
+Date:   Tue Nov 25 21:16:16 2014 +0530
+
+    doc: Removes ceph-deploy usage instances from ceph-disk man page.
+    
+    Signed-off-by: Nilamdyuti Goswami <ngoswami@redhat.com>
+    (cherry picked from commit 242dd1c0bbb728475a94f47740790b8a196d9804)
+
+commit 21c3256c851065521e34a179dc05d48fcc0a6e0f
+Author: Nilamdyuti Goswami <ngoswami@redhat.com>
+Date:   Tue Nov 25 02:06:39 2014 +0530
+
+    doc: Updates man page for ceph-disk utility.
+    
+    Signed-off-by: Nilamdyuti Goswami <ngoswami@redhat.com>
+    (cherry picked from commit 7dcc85042b0c0a26e495f7574ce144d1083d15f8)
+
+commit 1a6490e38d7f5fd3ff640a810c3b911699cd4884
+Author: Nilamdyuti Goswami <ngoswami@redhat.com>
+Date:   Mon Nov 24 22:05:11 2014 +0530
+
+    doc: Adds man page for ceph-disk utility.
+    
+    Signed-off-by: Nilamdyuti Goswami <ngoswami@redhat.com>
+    (cherry picked from commit a450cab2b8148cb8a9b043d629feccf89e5aabac)
+
+commit 3bab47054dc77b9a00d3f47fa73f458ede7d4ab4
+Author: Billy Olsen <billy.olsen@gmail.com>
+Date:   Mon Feb 2 16:24:59 2015 -0700
+
+    Fix memory leak in python rados bindings
+    
+    A circular reference was inadvertently created when using the
+    CFUNCTYPE binding for callbacks for the asynchronous i/o callbacks.
+    This commit refactors the usage of the callbacks such that the
+    Ioctx object does not have a class reference to the callbacks.
+    
+    Fixes: #10723
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Billy Olsen <billy.olsen@gmail.com>
+    Reviewed-by: Dan Mick <dmick@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 60b019f69aa0e39d276c669698c92fc890599f50)
+
+commit d0fd417e872a73033903fb36144fe7a39e90fc9a
+Author: Dmytro Iurchenko <diurchenko@mirantis.com>
+Date:   Tue Feb 3 17:54:38 2015 +0200
+
+    rgw: Swift API. Support for X-Remove-Container-Meta-{key} header.
+    
+    Fixes: #10475
+    Backport: hammer, firefly
+    Reported-by: Josh Durgin <jdurgin@redhat.com>
+    Signed-off-by: Dmytro Iurchenko <diurchenko@mirantis.com>
+    (cherry picked from commit f67bfa24fd6f69c2fcc0987eba8b6b426dd78320)
+    
+    Conflicts:
+    	src/rgw/rgw_rest.h
+            trivial merge: prototype of an unrelated function changed
+            s/is_object_op/!(s->object == NULL)/
+
+commit 651dc556047aa4ee9e95fe9fc7bcd11488973872
+Author: Dmytro Iurchenko <diurchenko@mirantis.com>
+Date:   Mon Feb 2 11:27:00 2015 +0200
+
+    rgw: Swift API. Dump container's custom metadata.
+    
+    Fixes: #10665
+    Backport: hammer, firefly
+    Reported-by: Ahmad Faheem <ahmad.faheem@ril.com>
+    Signed-off-by: Dmytro Iurchenko <diurchenko@mirantis.com>
+    (cherry picked from commit 2f8d31e9b1c4b09506bd1b0dad635c6e820783b2)
+    
+    Conflicts:
+    	src/rgw/rgw_rest_swift.cc
+            The first hunk conflicts because X-Storage-Policy was added
+    	after firefly. The second hunk conflicts because the type of the
+    	s->object data member changed after firefly but it is irrelevant
+    	because the patch does not need to access s->object anymore.
+
+commit 6fd3dfa7224f0af3101fd1614506e8cb2fc7f6a0
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jan 23 10:47:44 2015 -0800
+
+    osd: do not ignore deleted pgs on startup
+    
+    These need to get instantiated so that we can complete the removal process.
+    
+    Fixes: #10617
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 879fd0c192f5d3c6afd36c2df359806ea95827b8)
+
+commit 368a5a8a8739e98ffdb8ac1210d111092e31be9e
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Fri Jan 30 11:37:28 2015 +0000
+
+    mon: Monitor: fix timecheck rounds period
+    
+    Fixes: #10546
+    Backports: dumpling?,firefly,giant
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 2e749599ac6e1060cf553b521761a93fafbf65bb)
+
+commit 66b13f2beff702c3b0bcb0aaa8da4e12d594eddf
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Jan 11 17:28:04 2015 -0800
+
+    osd: requeue blocked op before flush it was blocked on
+    
+    If we have request A (say, cache-flush) that blocks things, and then
+    request B that gets blocked on it, and we have an interval change, then we
+    need to requeue B first, then A, so that the resulting queue will keep
+    A before B and preserve the order.
+    
+    This was observed on this firefly run:
+    
+      ubuntu@teuthology:/a/sage-2015-01-09_21:43:43-rados-firefly-distro-basic-multi/694675
+    
+    Backport: giant, firefly
+    Fixes: #10512
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 11bdfb4131ecac16d4a364d651c6cf5d1d28c702)
+
+commit a22aa8f3228e0baf2ba08d9e79ee4924cd341a07
+Author: Matt Richards <mattjrichards@gmail.com>
+Date:   Thu Jan 8 13:16:17 2015 -0800
+
+    librados: Translate operation flags from C APIs
+    
+    The operation flags in the public C API are a distinct enum
+    and need to be translated to Ceph OSD flags, like as happens in
+    the C++ API. It seems like the C enum and the C++ enum consciously
+    use the same values, so I reused the C++ translation function.
+    
+    Signed-off-by: Matthew Richards <mattjrichards@gmail.com>
+    (cherry picked from commit 49d114f1fff90e5c0f206725a5eb82c0ba329376)
+    
+    Conflicts:
+    	src/librados/librados.cc
+            comes from lttng tracepoints introduced after firefly
+
+commit d5c3a14390f6bb2af45a1a4ad842777302dd0553
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat Jan 3 15:29:29 2015 +0800
+
+    mount.ceph: avoid spurious error message
+    
+    /etc/mtab in most modern distributions is a symbol link to
+    /proc/self/mounts.
+    
+    Fixes: #10351
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit bdd0e3c4bda97fe18487a58dd173a7dff752e1a2)
+
+commit 7ed92f7dc5f0f327b77b6f3835e3f821fc810708
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Tue Dec 2 15:17:57 2014 -0800
+
+    SimpleMessenger: allow RESETSESSION whenever we forget an endpoint
+    
+    In the past (e229f8451d37913225c49481b2ce2896ca6788a2) we decided to disable
+    reset of lossless Pipes, because lossless peers resetting caused trouble and
+    they can't forget about each other. But they actually can: if mark_down()
+    is called.
+    
+    I can't figure out how else we could forget about a remote endpoint, so I think
+    it's okay if we tell them we reset in order to clean up state. That's desirable
+    so that we don't get into strange situations with out-of-whack counters.
+    
+    Fixes: #10080
+    Backport: giant, firefly, dumpling
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 8cd1fdd7a778eb84cb4d7161f73bc621cc394261)
+
+commit 91b2acaadee1b62c1fcac73147908ec4477840f3
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Oct 9 11:20:13 2014 -0700
+
+    osd: Get pgid ancestor from last_map when building past intervals
+    
+    Fixed OSD::build_past_intervals_parallel() and PG::generate_past_intervals()
+    
+    Fixes: #10430
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0c5b66da7a9ba516340d06d9e806beb9d1040d0e)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+
+commit c09b6d9d64fdcdc3842c4f89acf10080125a4adc
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Dec 23 12:04:26 2014 -0800
+
+    osd: Pass oldest_map when generating past intervals
+    
+    From load_pgs() the superblock hasn't been published yet
+    so we need to retrieve the value of oldest_map depending on the context.
+    
+    Fixes: #10427
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 7fb721c1ceb39b38ca2e653299bcf51e109411d7)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+
+commit 9235d781bf860c1ecef5af600431f1619b56fbc1
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Dec 17 16:59:09 2014 -0800
+
+    osd: build_push_op() handle short reads so recipient doesn't crash
+    
+    Fixes: #8121
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b9a04846d18e1c6621d7f2794ec1fae02875bed2)
+
+commit 7ce0cb8e33a281d5f675273d7bcbc570a32e5497
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Feb 5 16:33:08 2015 +0800
+
+    ceph_objectstore_tool: fix check_output on python2.6
+    
+    * backported the subprocess.check_output from python2.7
+    
+    Fixes: #10756
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 15350a088d84bc6fc664f0d3f5d09b35f58b2144)
+    
+    Conflicts:
+    	src/test/ceph_objectstore_tool.py
+
+commit d5d34ddd1eac688d9422ca02c859d61be8e3e56e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Jan 6 15:49:50 2015 -0800
+
+    ceph_objectstore_tool: For import get object_info_t available for each object
+    
+    Add object_info_t to object_begin so we have at object create time
+    This will be useful for importing from multiple erasure coded exports
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 05d916ed12f361da48ef73953bcc0cef465fcc2a)
+
+commit f70590b211ba28f350921b2f0c92712ed779858e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Dec 12 15:16:03 2014 -0800
+
+    ceph_objectstore_tool: Handle import of pg which OSD has split
+    
+    Fail import if import data doesn't include OSDMap and can't find it locally
+    See if local map can be read for import's epoch
+    Jump to current epoch like a split would if local map not present
+    
+    Fixes: #9781
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit afda6e4f3b98cc1773fd014583dfb5e1f214a939)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc
+
+commit aedd324f8fbb031d5126ae158f03066c7342f4b0
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Dec 12 15:38:33 2014 -0800
+
+    ceph_objectstore_tool: On import following a split skip objects no longer in pg
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 68b27e25a3729566c3a22c0b71f70f7f3aca29a3)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc
+
+commit c3de607c2ce16a26af7a78a4e557f321ffbcb44d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Dec 5 15:12:21 2014 -0800
+
+    ceph_objectstore_tool: Verify that object pool and pgid pool match
+    
+    Also, earlier check for invalid --pgid with import op
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit da3be80497a1b1878ee5d2015f8840d202a83aa2)
+
+commit 5ec38e54f40431a5303cafe202c5097cc400fdb6
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Nov 19 11:47:36 2014 -0800
+
+    ceph_objectstore_tool: Check for splits and fail import if there were splits
+    
+    Add osdmap into metadata_section
+    On export put metadata_section before file data
+    
+    Fixes: #9780
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 19fdeea8b67091ed044ebce25799d3237b4d734a)
+
+commit aa472fa4df19d826a5af72e286365fa5ce4c71fb
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Dec 15 10:03:53 2014 -0800
+
+    ceph_objectstore_tool: Add special exit code for test infrastructure
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b1f12f09c0211b608178f5ca2e292ab1765ce620)
+
+commit 22b7c2faee8cfad1b40cef019984f4a2d112b268
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Nov 19 11:41:39 2014 -0800
+
+    ceph_objectstore_tool: Check that pool exists before allowing import
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit f24f646d870703b7b79563cdbc20920248be6142)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc
+
+commit f65db436f6eb83d7c59fdddced4d35dd9eeeb078
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 15 15:21:11 2014 -0700
+
+    ceph_objectstore_tool: Check cluster_fsid before allowing an import
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 196c8112dc9a6e8780b05d6c579cd7fbd5b07589)
+
+commit 360f68ec69b36b172d15c6206698340a5c00aafa
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Oct 16 12:27:56 2014 -0700
+
+    ceph_objectstore_tool: Allow the metadata_section to be anywhere in the export
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 62dd912f1192b28700a15b02507a8c9efd710cb5)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc
+
+commit c3fcbe636ae1a936b9180628cff939b2b5dddf7c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Dec 12 15:01:24 2014 -0800
+
+    ceph_objectstore_tool: import-rados shouldn't import internal namespace objects
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit f727d2eaf50b0351feb9f12dcd65d50fb6eff7e9)
+
+commit 2d69076943b9724ce1d5c9f03f2f7594a77b92d2
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Dec 12 14:58:54 2014 -0800
+
+    ceph_objectstore_tool: Get g_ceph_context available to import-rados
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit ddc4613ec71752e5dccbbfe6dc078b86f0f186a9)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc
+
+commit bbed3a728471292de625d922abeae8b39d290045
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Dec 9 18:09:04 2014 -0800
+
+    ceph_objectstore_tool: Fix import-rados skipping of snapshots
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit fe936026ed87c9f95f3b7ad235b24c22e8de5f55)
+
+commit d962c79dc1f0f2189e25743c6d253fac412c004d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Nov 20 13:00:10 2014 -0800
+
+    ceph_objectstore_tool: read_fd() doesn't handle ^D from tty stdin, don't allow
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 5cb692528e3ac0ebea3f1714b3ac43f69d176888)
+
+commit 8b7f4cb232c2f1bde6f50c1f092cb622fe2c41f1
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Dec 19 13:47:32 2014 -0800
+
+    ceph-objectstore-tool: Remove --pretty-format and use new --format options
+    
+    Call new_formatter() with --format specified argument
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 22b71744bb0cb56434d5f6214ccea7d81f771860)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc
+
+commit ee183b8e56c6f8a88ac781cf1fedb6a7a93f1005
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 15 15:20:03 2014 -0700
+
+    ceph_objectstore_tool: Describe super_ver values
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0aeba0f216a54390288b5e3d6147deb31877f744)
+
+commit 5f82f824c7b1bb246bdb54b11a694133a49da70a
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Fri Jan 2 18:36:54 2015 +0100
+
+    ceph_objectstore_tool.cc: reduce scope of variable
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 78542f9a901c05e627b53b5306ea604be3bc82e8)
+
+commit 2f97c51b5ee5de7c1657bc8fee6ccaa474f6f478
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Fri Jan 2 15:48:08 2015 +0100
+
+    ceph_objectstore_tool.cc: prefer ++operator for non-primitive iterators
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 252fc03ba21c7b09922598a8d40997fc639bb994)
+
+commit 2a22bfedc4fa9f461dc8cfd42c659b9c55ddafca
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Nov 25 16:56:19 2014 -0800
+
+    ceph_objectstore_tool: Prevent tool from transitioning struct_v on rm-past-intervals
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit f056bdf93980a0a8e6157dbb124a79389a8f1a3c)
+
+commit 53aa04f95c43795da81a7d9f3117d7e5816aedcb
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Dec 4 18:53:08 2014 -0800
+
+    ceph_objectstore_tool: Accept json object with --pgid instead of array
+    
+    It isn't anticipated that anyone would use this but keeps backward compatible
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 59b423e2e8846b098326fdec440de46b8e3d2769)
+
+commit 60e9a8120b292628ee4e5ef33fe933222609b861
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Dec 4 18:27:50 2014 -0800
+
+    ceph_objectstore_tool: Improve object spec parsing error messages
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit de6384fda183801c16af1b61ed36eaed289bb4f6)
+
+commit 4f95409c0dadeed18334c00630ddc6d7c99d2819
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Dec 4 17:48:28 2014 -0800
+
+    ceph_objectstore_tool: Fix errors messages in newer code
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit eae7c02fded460f6c8aaf18aa83d2730b89e0eb1)
+
+commit 1703867735c8f8ab1c83aa526c84b278436f38d5
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Dec 4 16:00:40 2014 -0800
+
+    ceph_objectstore_tool: Remove extraneous endl on error throw messages
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 32c832f0c62259a492d1c934c56ac165496763a0)
+
+commit f2d2110a8cb1a1b1216c1083b54ea6212138dc93
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Dec 4 14:01:39 2014 -0800
+
+    ceph_objectstore_tool: Add --format and --pretty-format support
+    
+    --pretty-format defaults true
+    Add --format so xml output can be requested
+    --op list defaults to single line of json per object
+    To override this more human readable output use --pretty-format=false
+    Add testing of --op list special handling
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit cca85a534fe488ae314400e8faad55a758609467)
+
+commit a5544c12c3e982edf66f55a8edfb7fc69300520b
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Dec 3 17:53:11 2014 -0800
+
+    ceph_objectstore_tool: Strip _* (always _head) from pgid in list entry output
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 5eacd3c5f39766cb8be6b3251d139d16431cf6b6)
+
+commit 4a0a42f3e6b9a1b7ed4df3d4b6c417acfc00050a
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Dec 3 16:39:04 2014 -0800
+
+    ceph_objectstore_tool: BUG: --op list wasn't including snapshots
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b617ee2d45886ec6b3dc0db0edbf814ea5748083)
+
+commit 06579b9f269dc9864e920368c5bea9bcb9dd8be9
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Dec 3 16:38:22 2014 -0800
+
+    ceph_objectstore_tool: For terminal output of list one object per line
+    
+    Instead of a parsable array make it easier to cut and paste listed objects
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 5a66db9418aeed31fec98999c5053dab357d9c1e)
+
+commit 22c6bf410de1b0c81f131aa4d5682f5162dd1a56
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Dec 3 16:35:09 2014 -0800
+
+    ceph_objectstore_tool: In error case umount objectstore
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit bc6ea9cb8a2b86c73f7f15bc46145177ccf91f4b)
+
+commit d802ab69a4a0f37ed9328ec385746f59643e1420
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Nov 27 01:24:03 2014 +0100
+
+    objectstore_tool: test --op list variants
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit ddba2676c4c48c2a556f5d4ffd817bfe134a9153)
+
+commit f30e053fe7b3e5efc679b20cf1b3e2f7f8ed7e54
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Nov 27 00:11:45 2014 +0100
+
+    objectstore_tool: parse new object description format
+    
+    The object format changed from
+    
+        {json object}
+    
+    to
+    
+        [pgid,{json object}]
+    
+    The parser is updated accordingly. If the --pgid is present, check that
+    it equals the pgid from the object description.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit df9d5c5cfd8b0ff793647a592c7661965cef5c92)
+
+commit fce87c9d7dbb51989826d068d6b6657a2f3d129d
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Nov 26 23:35:21 2014 +0100
+
+    objectstore_tool: filter --op list and explore all PGs
+    
+    The positional object name is used to filter the output of --op list and
+    only show the objects with a matching name. If both the object name and
+    the pgid are omitted, all objects from all PGs are displayed.
+    
+    The output format is changed from
+    
+        {"oid":"GROUP","key":"","snapid":-2,
+         "hash":2659194943,"max":0,"pool":0,"namespace":""}
+    
+    to
+    
+        [["0.7_head",{"oid":"GROUP","key":"","snapid":-2,
+                      "hash":2659194943,"max":0,"pool":0,
+                      "namespace":""}]]
+    
+    where the first member is the pgid where the object is found.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit c69aaceac7f370e5369d511bf17898adc338ae43)
+
+commit 2f874fd3715d216a68658e94a5c741e665c76632
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Nov 26 23:34:22 2014 +0100
+
+    objectstore_tool: lookup objects by name
+    
+    If the object is not a parsable JSON string, assume an object name and
+    look it up in all the PGs. If multiple objects have the same name, only
+    apply the command to one of them. It is primarily useful in a test
+    environment where the names of the tests objects are known and only a
+    small number of objects exists. It replaces the following:
+    
+        path='--data-path dev/osd0 --journal-path dev/osd0.journal'
+        for pgid in $(./ceph_objectstore_tool $path --op list-pgs) ; do
+          object=$(./ceph_objectstore_tool $path --pgid $pgid --op list |
+                   grep '"oid":"NAME"')
+          test -n "$object" && break
+        done
+        ./ceph_objectstore_tool $path --pgid $pgid "$object" remove
+    
+    with:
+    
+        ./ceph_objectstore_tool $path NAME remove
+    
+    http://tracker.ceph.com/issues/10192 Fixes: #10192
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 7c1165f96391821c00cca1ac04b3433dbec6bb6e)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc
+
+commit eb48aba2119959c53ea8a103b53f0c2e07c52acb
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Nov 26 17:30:30 2014 +0100
+
+    objectstore_tool: refactor list-lost and fix-lost
+    
+    Abstract out the PG exploration loops and encapsulate the list-lost and
+    fix-lost semantic in a callable object.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit d9e747b1bdb53d1fe543ef311e3db35fb78d8051)
+
+commit eab3226edc8ecc6d2d93f463e019ef4eacb9f468
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Nov 26 17:26:54 2014 +0100
+
+    objectstore_tool: update usage strings
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit a90233c8b79ae6c035b5169c7f8809f853631689)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc
+
+commit 5bbe41cb1e7faf40a9c53a9bdc01db8c35670a36
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Nov 18 13:00:15 2014 -0800
+
+    ceph_objectstore_tool: Add feature called set-allow-sharded-objects
+    
+    Uses --op set-allow-sharded-objects option
+    This operation will be rejected if on the target OSD's osdmap there is
+        at least one OSD which does not support ERASURE CODES.
+    Prompt the user that they could import if sharded state allowed
+    Prompt the user to use new feature if sharded state found inconsistent
+    
+    Fixes: #10077
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit f3dab446fc8e58b3b3d9334b8c38722e73881b9e)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc
+
+commit 4e30d8c53c0a4952f81cc3b5680c7e92fffb1355
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Nov 18 11:59:18 2014 -0800
+
+    ceph_objectstore_tool: Add utility routine get_osdmap()
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b3021b0d3362000b5938d406ed2e032a8eb38405)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc
+
+commit f997b126afaeadebe6b6d76954fbe2008bd2a7d8
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Nov 12 21:14:11 2014 -0800
+
+    ceph_objectstore_tool: Clear ...INCOMPAT_SHARDS from feature if exporting replicated pg
+    
+    Don't require importing OSD to have shards feature for replicated pg
+    
+    http://tracker.ceph.com/issues/10077 Fixes: #10077
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 86baf2d38170ef19de2dd5e9ce3f280237d8474d)
+
+commit 3f862da57945f821ed459f5a77f8032331c9cb20
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Nov 18 00:10:41 2014 -0800
+
+    tests: ceph_objectstore_tool.py test all variants of export/import
+    
+    Handle change of error message text
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 9e53c3554c71121f2e2dd0234b5232da37ad5a1b)
+
+commit 2a58bdee19eb8f02a2800d728d402e76566d7f58
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Nov 17 23:23:40 2014 -0800
+
+    ceph_objectstore_tool: Make --file option consistent by treating "-" as stdout/stdin
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 8c87f3284f87d1121218cb7f41edc81b74c9df29)
+
+commit 70329be7be5d80e52d5424958cecd7f4d939add6
+Author: David Zafman <dzafman@redhat.com>
+Date:   Sat Nov 15 11:43:10 2014 -0800
+
+    tests: ceph_objectstore_tool.py fix list-attr for erasure code
+    
+    Adding testing of xattr for erasure coded shards
+    Fix error message when finding an unexpected xattr key
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit cbecab477a70782f2f69258f035e78fb5c829810)
+    
+    Conflicts:
+    	src/test/ceph_objectstore_tool.py
+
+commit bc921c7ecbc061ccef4847d967986c9fa902111f
+Author: David Zafman <dzafman@redhat.com>
+Date:   Sat Nov 15 11:46:15 2014 -0800
+
+    tests: ceph_objectstore_tool.py check for malformed JSON for erasure code objs
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 40717aa4c399e87d2c3e32038f78788eb213f87d)
+
+commit 99ffd137f17c438e3ee9dbbc0ab73cdcd3a45a5c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Sat Nov 15 11:44:54 2014 -0800
+
+    tests: ceph_objectstore_tool.py fix off by 1 ATTR_OBJS handling
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit eaf1d1e35243566c46b478788e79e0ebf7583015)
+
+commit 951c951a0f164271c5e9b96ecbd510cf5c6663b6
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Nov 14 11:00:17 2014 +0100
+
+    tests: ceph_objectstore_tool.py skip if /dev/tty fails
+    
+    Some environments do not have a /dev/tty. When opening /dev/tty fails,
+    skip the test instead of returning an error.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 4c94f1778fdf483e9e0b62f89f7e46e78aeeebf3)
+
+commit 8dc263555d7aca2befa912c78c585a43c8e7592c
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Nov 13 19:15:50 2014 +0100
+
+    tests: ceph_objectstore_tool.py encapsulate init-ceph stop
+    
+    Call init-ceph in kill_daemons and add a call to kill_daemon when main
+    returns on error so that it never leaves daemons hanging.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit fc435ff3623d196dd7cc375302acd8dfc9eb59fd)
+
+commit b78d802f3e082b0762203ae37b3c3a44b7608907
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Nov 13 19:14:49 2014 +0100
+
+    tests: ceph_objectstore_tool.py main returns
+    
+    Instead of calling sys.exit() the main function returns the desired exit
+    code.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 0f3d7b1315f2b5595047d8bd13949ed0d9194bfa)
+
+commit 04e480b09b59f5318e9b206e6c3e529d8bb22328
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Nov 13 17:32:14 2014 +0100
+
+    tests: ceph_objectstore_tool.py replace stop.sh with init-ceph
+    
+    The stop.sh will stop all ceph-* processes. Use the init-ceph script
+    instead to selectively kill the daemons run by the vstart.sh cluster
+    used for ceph_objectstore_tool.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit e8f34bd62bf282144b8851fb9764cf4429a49c25)
+
+commit 51855584eb970b28dfa266ee3653963fb77a2b1e
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Nov 13 17:30:29 2014 +0100
+
+    tests: ceph_objectstore_tool.py use a dedicated directory
+    
+    Set CEPH_DIR to a directory that is specific to ceph_objectstore_tool so
+    that it can run in parallel with other vstart.sh clusters.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit e303d1afde58e68c1f587962010da9e1f1278bc3)
+    
+    Conflicts:
+    	src/test/ceph_objectstore_tool.py
+
+commit 454ec85ae449cb20c2ddecade421262d9d9de615
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Nov 13 17:27:01 2014 +0100
+
+    tests: ceph_objectstore_tool.py run faster by default
+    
+    By default use only a small number of objects to speed up the tests. If
+    the argument "big" is given, use a large number of objects as it may
+    help find some problems.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 235257c257aea98b770d9637957818c8aeb6c745)
+
+commit 427fe9177f90db091685b937839dcc4dfe4c8a01
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Nov 13 17:21:48 2014 +0100
+
+    tests: ceph_objectstore_tool.py run mon and osd on specific port
+    
+    By default vstart.sh runs MDS but they are not needed for the tests,
+    only run mon and osd instead. Instead of using the default vstart.sh
+    port which may conflict with a already running vstart.sh, set the
+    CEPH_PORT=7400 which is not used by any other test run with make check.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit f04d4e7539bc8c1b6cf94db815f9dcdecc52faa2)
+
+commit 28ed34a7dc32033be61cd61178eb59471de45aac
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Nov 13 17:16:41 2014 +0100
+
+    tests: ceph_objectstore_tool.py can use a WARN cluster
+    
+    The tests do not need HEALTH_OK exclusively, a HEALTH_WARN cluster can
+    also run them successfully.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 783378c019aaac36d542e1b12c0d64196ea21012)
+
+commit 8548d7c65e8743d4e743c77981acd702efab3fb2
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Nov 13 17:12:35 2014 +0100
+
+    tests: ceph_objectstore_tool.py use env python
+    
+    Using #/usr/bin/env python instead of a hard coded path is more flexible
+    and can also be used to run from virtualenv.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 74506d2506d03d05935cbe342fef9dc1d9022a13)
+
+commit 5b4c2ee0902d94e6fa5bee67188fddcf0e0c38a2
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Nov 12 15:22:04 2014 -0800
+
+    ceph_objectstore_tool: Fixes to make import work again
+    
+    The is_pg() call is now true even for pgs pending removal, fix broken
+        finish_remove_pgs() by removing is_pg() check.
+    Need to add create_collection() to the initial transaction on import
+    
+    Fixes: #10090
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5ce09198bf475e5c3a2df26232fa04ba9912b103)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc
+
+commit e0dab883d29bd3a54b5707f8d3ea830f8a1ce516
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Oct 6 18:26:44 2014 -0700
+
+    ceph_objectstore_tool: Accept CEPH_ARGS environment arguments
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 10fe7cfe561f91717f2ac2e13aeecc06a903704e)
+
+commit 5f788eaa61054c6cb27960b5544b321078706343
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Oct 3 15:12:28 2014 -0700
+
+    ceph_objectstore_tool: MemStore needs a CephContext
+    
+    Pass g_ceph_context to ObjectStore::create() needed by MemStore
+    
+    Fixes: #9661
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0b155d00c542f0d8b8b5b0324dac4b3cf7ff37b5)
+
+commit 7470c7ca02ef6313b925be3ce4b27437a0c2e1e0
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Mar 3 10:41:28 2015 -0800
+
+    ceph_objectstore_tool: Rename generated binary to ceph-objectstore-tool
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 4f72ba545e204a24a55adead43c61cb1d4394381)
+    
+    Conflicts:
+    	debian/ceph-test.install
+    	src/.gitignore
+    	src/test/ceph-helpers.sh (doesn't exist in firefly)
+    	src/test/ceph_objectstore_tool.py
+    	src/tools/ceph_objectstore_tool.cc
+
+commit 4b51645fdfe8a761f7ebc0faee1a87187d498fee
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 12 13:35:43 2014 -0800
+
+    vstart.sh: warn less
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a69b8450f642af91a352d0de4378d93828291933)
+
+commit 3075919c0dcee1f33a0beb299e98d8a88836c5f8
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Nov 17 23:02:50 2014 -0800
+
+    ceph_objectstore_tool: When exporting to stdout, don't cout messages
+    
+    Fixes: #10128
+    Caused by a2bd2aa7
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0d5262ac2f69ed3996af76a72894b1722a27b37d)
+    (cherry picked from commit 6cb9a2499cac2645e2cc6903ab29dfd95aac26c7)
+
+commit ca9df2803f3200431f5f7ea99a713005f15b7f5a
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Sep 2 14:56:10 2014 +0200
+
+    ceph_objectstore_tool.cc: prefer ++operator for non-primitive iterators
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit a5468abe4459159e8a9f7a4f21d082bb414e1cdd)
+
+commit ac396f5df3c51d09b9dbf7e6be7bed1d755d2317
+Author: Zhiqiang Wang <wonzhq@hotmail.com>
+Date:   Fri Aug 29 16:39:40 2014 +0800
+
+    Test: fixing a compile warning in ceph_objectstore_tool.cc
+    
+    For the compiler's sake:
+    tools/ceph_objectstore_tool.cc:2547:15: warning: ‘r’ may be used
+    uninitialized in this function [-Wmaybe-uninitialized]
+    
+    Signed-off-by: Zhiqiang Wang <wonzhq@hotmail.com>
+    (cherry picked from commit c3e1466b46076f133b62f98e2c0b712bdde0e119)
+
+commit b863970110e7f8e835e77864ea59b81f0b026158
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Aug 20 01:33:45 2014 -0700
+
+    ceph_objectstore_tool: Bug fixes and test improvements
+    
+    ceph_objectgstore_tool:
+    Fix bugs in the way collection_list_partial() was being called
+    which caused objects to be seen over and over again.
+    
+    Unit test:
+    Fix get_objs() to walk pg tree for pg with sub-directories
+    Create more objects to test object listing code
+    Limit number of larger objects
+    Limit number of objects which get attributes and omaps
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit a03f719eb3a46f410550afce313e6720e0c27946)
+
+commit 4f83005bb8a615df370de7b6dfe5d926c7cfef7f
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Tue Aug 5 18:26:11 2014 -0700
+
+    ceph_objectstore_tool, test: Implement import-rados feature and unit test code
+    
+    Fixes: #8276
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 23ec93a86140c4b271b45d87c62682288079cbba)
+
+commit df334617705fe862b820ef5de674ae54efad0cea
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed Aug 6 19:53:43 2014 -0700
+
+    test: ceph_objectstore_tool unit test improvements
+    
+    Add namespaces to testing
+    Increase filesize so export will have multiple chunks
+    Put json for each object into the db dict
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+
+commit ecd25cf6ce1a1a34e536c7fd313225b3bdedd2e0
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Thu Aug 7 13:31:48 2014 -0700
+
+    ceph_objectstore_tool: Add operation "rm-past-intervals" for testing purposes
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 3694068b67fd625495c4511390bc5bcbfbbd28f5)
+
+commit 6b50b384efc1f0735f8635a59663b50e3155de1a
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Thu Aug 7 11:46:08 2014 -0700
+
+    ceph_objectstore_tool: Add past_intervals to export/import code
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 9325ec15d4b89c5537cbcbda4c7594e25dc6e7df)
+
+commit 63529079b97c99cbaa863e1be865e2693e90c556
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Thu Aug 7 14:11:21 2014 -0700
+
+    ceph_objectstore_tool: Minor improvements
+    
+    Make all non-error non-debug output to stdout
+    Fix a message
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit a2bd2aa7babb4ad45ba21c70f9d179fda27742aa)
+
+commit 5e3f89ece7bdd09ed06ca4208cfa0a0b3104f109
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Tue Aug 5 12:26:42 2014 -0700
+
+    ceph_objectstore_tool, test: Add list-pgs operations and unit test case
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit f01e334c697057158354f0ce5ecff6d6ba8e2704)
+
+commit edc9f71efeebe5241004c669cc58089905907634
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed Jul 30 12:39:49 2014 -0700
+
+    Complete replacement of ceph_filestore_tool and ceph_filestore_dump
+    with unified ceph_objectstore_tool
+    
+    Move list-lost-objects and fix-lost-objects features from
+      ceph_filestore_tool to ceph_objectstore_tool as list-lost, fix-lost
+    Change --type to --op for info, log, export...operations
+    Add --type for the ObjectStore type (defaults to filestore)
+    Change --filestore-path to --data-path
+    Update installation, Makefile.am, and .gitignore
+    Fix and rename test case to match
+      Add some additional invalid option checks
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 83fbc91e5c4e52cc1513f34908f99d2ac3b930df)
+
+commit a42273ecd955470105cba1cc4ac7eb782ac46833
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed Jul 30 11:22:29 2014 -0700
+
+    Renames and removal towards a unified ceph_objectstore_tool
+    
+    Rename ceph_filestore_dump.cc and ceph_filestore_dump.py
+    Remove ceph_filestore_tool.cc
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 77864193a1162393ade783480aee39a232934377)
+
+commit 9ee2c27096784efceb02b06a0df4325979385f44
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Tue May 20 11:19:19 2014 -0700
+
+    ceph_filestore_dump: Add set-omaphdr object command
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit b4d95cc85af9af64d33d541cd69c5f28fd45423b)
+
+commit c7d261d40367ab01a577bf255da776a622f8984a
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Tue May 20 10:44:37 2014 -0700
+
+    ceph_filestore_dump: Add get-omaphdr object command
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 30c0f3114e665acdd99e64bf0d2a7399b33e8d61)
+
+commit 271e3ca19b9e878c6c3c26a9ec461faf06e7a19d
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Mon May 19 20:55:47 2014 -0700
+
+    ceph_filestore_dump: Add rm-omap object command
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 0fc6bd2777edf24a044f454beacf1647cc52f9fe)
+
+commit 20165d101a30c6beb591ca56b56bdf5505f70cf3
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Mon May 19 20:47:14 2014 -0700
+
+    ceph_filestore_dump: Add set-omap object command
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 50cd57e902fe508f98f63fea30626780b07561d9)
+
+commit 7547f3d17bc89437c529aa96413b0bebb808da5e
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Mon May 19 20:37:01 2014 -0700
+
+    ceph_filestore_dump: Add get-omap object command
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit b50c43ce5e52f5bbcb3684f6793d50f34ed741d1)
+
+commit f391feb1f50632adbe94c3e8cdc4b75091d6b8e1
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Mon May 19 18:33:24 2014 -0700
+
+    ceph_filestore_dump: Add rm-attr object command
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 465d77733c7499fbd65bebe7141895714c625e0d)
+
+commit 7bd2dd3a1d022df6d3f886ad12a191d0cfcef1d6
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Mon May 19 18:17:27 2014 -0700
+
+    ceph_filestore_dump: Add set-attr object command
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 95554e03dcb74b6d74b2f1b2891b3570abb187b8)
+
+commit 6724da821158ddee6ef6ee7b5bac9e97dcfc2292
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Thu May 15 15:50:48 2014 -0700
+
+    ceph_filestore_dump: Add get-attr object command
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 55d43c0e20fc853daec134449b9954248fd7ef31)
+
+commit 55c21b898834d77234227d3fc14c8580ef698663
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed May 14 17:52:09 2014 -0700
+
+    ceph_filestore_dump: Add set-bytes object command
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 3c24d1f46a624d0a053ad234997a1f8c8b036db5)
+
+commit 2f1926d2f57082666350d8223b09f61da5f95b6f
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed May 14 17:51:29 2014 -0700
+
+    ceph_filestore_dump: Add get-bytes object command
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 869dd92cc8ec29a3a684f88c335d359f225bba24)
+
+commit fcabb8133af3b90d5d9e976ce658ceccfc5b89c5
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed May 14 17:50:16 2014 -0700
+
+    ceph_filestore_dump: Add list-omap object command
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 48890c7741d76cf92b5f589f49378ca57292e88b)
+
+commit 303e4cedd91ca3553e956eec495a05e3136b3c56
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed May 14 18:32:42 2014 -0700
+
+    ceph_filestore_dump: Add list-attrs object command
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 00c6b0673288ca76fe144575b7af76eaa36f5857)
+
+commit aff2c995f67bdde1a592a9b24e4e96e85735d500
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed May 14 17:39:17 2014 -0700
+
+    ceph_filestore_dump: Add --type list to output objects in a pg in json
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 844dabb7f311e68eba0293ae9ca4c68521745877)
+
+commit 7cda8b7a0b43d709b73d875088ecd169f47d59ab
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed May 14 17:44:31 2014 -0700
+
+    ceph_filestore_dump: Add remove object command and usage for new commands
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 605caec64b036f8ab5ae451d7e9e7515d414f28e)
+
+commit a4694643ae4503746d3fac8a0feac706ddc13a16
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Fri Jun 6 17:05:53 2014 -0700
+
+    ceph_filestore_dump: Add utility function get_fd_data()
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit d4a9dafe442f139562497d746f80ba49faa954e8)
+
+commit 62dc823fb3b969c0ad52594419e8a86a3ff1e6ef
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Mon May 19 18:16:52 2014 -0700
+
+    ceph_filestore_dump: Fail import/export with a tty for stdin/stdout
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 7520e504cf2cdd3de2f236acb2cbf8a5016e6698)
+
+commit 9816f872ad59bcaa1a125b297f3991b333aad39c
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Tue May 20 11:56:20 2014 -0700
+
+    ceph_filstore_dump: Save if stdout is a tty and add routine to clean binary strings
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 3a574cc78b0e3ec6d8dd0c39ee20e7a54ad64056)
+
+commit d4aedeb833f23bf4ce6187cb82910ab2e71d48e5
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed May 14 15:30:11 2014 -0700
+
+    common: Add missing ghobject_t::decode() for json
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit c05f895d15a1d0e78ff5e9ae1a83f0a5424103d0)
+    
+    Changes:
+    	Adjusted for older shard_t
+
+commit dadecb1e05e528093642ba356fa7a70a0b546727
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed May 14 15:37:17 2014 -0700
+
+    ceph_filestore_dump: Add --skip-journal-replay and --skip-mount-omap
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 2e9dcb256509e7c921556202052f0cc1dcd91398)
+
+commit c6369987b1e1e55a9d0ab0bc328f61f52fc608d0
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed May 14 15:41:15 2014 -0700
+
+    os: Add optional flags to generic ObjectStore creation (SKIP_JOURNAL_REPLAY
+    and SKIP_MOUNT_OMAP)
+    
+    Only FileStore cares about these flags, so passed on during create()
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 3d9fde9d92322cd8ac3e3d8bcbf5b0a01ef0528b)
+    
+    Conflicts:
+    	src/os/FileStore.cc
+
+commit 3381aebb113d14249f6998a86ebf6b4ec6adc42d
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Fri May 16 18:20:11 2014 -0700
+
+    ceph_filestore_dump: Improve debug output by showing actual offset
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 44b261d5d1b36528bfbcb37dbd866b615e14be99)
+
+commit 1164b2e1610028a40cddbed09f9da5649bd2023a
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed May 14 12:36:37 2014 -0700
+
+    ceph_filestore_dump: Use cerr now that we aren't closing stderr
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 087c0f9d31e0f3d5bae7eac6231978105a71677e)
+
+commit f224429aa4fcba897be5e438bbb49d1025e2c482
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Wed May 14 12:42:21 2014 -0700
+
+    common,ceph_filestore_dump: Add ability for utilities to suppress library dout output
+    
+    Suppress dout output with CODE_ENVIRONMENT_UTILITY_NODOUT
+    ceph_filestore_dump turns on dout output if --debug specified
+    When used it can still be enable with --log-to-stderr --err-to-stderr
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit f7f9b251fc377651d8da4cbfd1942c3b86f3247e)
+
+commit 3f4cabdb84e58fcec0c3f508f980881c59fba948
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Tue May 13 18:27:30 2014 -0700
+
+    ceph_filestore_dump: Export omap in batches for large omap case
+    
+    New function get_omap_batch()
+    Create a TYPE_OMAP section for each batch
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 501dd3c05b8983159a289b021943cb828b908f53)
+
+commit 0b757af5be338b65fd9124ac5158bfe02ad5f899
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Mon May 12 15:50:34 2014 -0700
+
+    ceph_filestore_dump: Remove unused bufferlist databl
+    
+    Signed-off-by: David Zafman <david.zafman@inktank.com>
+    (cherry picked from commit 398b418e2b9f8260bcfacac8bcebea5beffcceca)
+
+commit 4a742fe29b6d959912a38d132344c695f89dd34f
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed May 7 14:12:15 2014 +0200
+
+    ceph_filestore_dump.cc: cleanup includes
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 8620609884243596d35b69c571d2da751e63cf2b)
+
+commit 420ea03aa3cd52bd035d31ba111c3d6d0745352d
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Thu Jan 29 18:19:16 2015 +0100
+
+    librados: rectify the guard in RadosClient::wait_for_osdmap().
+    
+    RadosClient::wait_for_osdmap() did not signalize lack of connection
+    via -ENOTCONN error code when the Objecter instance was allocated.
+    The proper way is to check the connection state explicitly.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 34473f78f101d87d2606e0e7112682a47ff24830)
+    
+    Conflicts:
+    	src/librados/RadosClient.cc
+            the modified guard was not present: add the new guard instead
+            of modifying it
+
+commit 1b2667211f90a1b630d2ddffe99b0fb00bb3c07c
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Mon Jan 19 15:07:21 2015 +0100
+
+    librados: fix resources leakage in RadosClient::connect().
+    
+    If RadosClient::connect was called a second time (which could
+    happen as a part of recovery from failure), the instances
+    of Objecter and Messenger allocated by the first call were leaked.
+    
+    Additionally, the implementation of the method wrongly reported
+    memory allocation problems -- it throwed std::bad_alloc exception
+    instead of returning -ENOMEM error code.
+    
+    Fixes: #10425
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 624c056da093c8741242892413438a291c03c7d5)
+    
+    Conflicts:
+    	src/librados/RadosClient.cc
+            resolve adding (std::nothrow) that failed because the
+            prototype of the constructor is not the same
+
+commit 4e32ff2b60549742d01b18429810c89f5f707548
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Dec 5 15:29:52 2014 -0800
+
+    osd_types: op_queue_age_hist and fs_perf_stat should be in osd_stat_t::operator==
+    
+    Fixes: 10259
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 1ac17c0a662e6079c2c57edde2b4dc947f547f57)
+
+commit 6e58732f546ec6241b198d2473902d66327cdc36
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Mon Jan 19 18:49:15 2015 +0000
+
+    mon: PGMonitor: skip zeroed osd stats on get_rule_avail()
+    
+    Fixes: #10257
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit b311e7c36273efae39aa2602c1f8bd90d39e5975)
+    
+    Conflicts:
+    	src/mon/PGMonitor.cc
+            ceph::unordered_map changed the context, simple resolution
+
+commit bcc8cfb24a96a7874a24760771755088a231a8d7
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Fri Jan 16 18:13:05 2015 +0000
+
+    mon: PGMonitor: available size 0 if no osds on pool's ruleset
+    
+    get_rule_avail() may return < 0, which we were using blindly assuming it
+    would always return an unsigned value.  We would end up with weird
+    values if the ruleset had no osds.
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 8be6a6ab2aa5a000a39c73a98b11a0ab32fffa1c)
+
+commit 894c8ad26fd2da203dcbf8eb0ad5e2af0223d5a9
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Fri Jan 16 18:12:42 2015 +0000
+
+    mon: PGMonitor: fix division by zero on stats dump
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 50547dc3c00b7556e26b9a44ec68640c5c3a2384)
+
+commit 1e58bb49b99118d064c1ca92e42cbfb2786fdaff
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Dec 17 16:06:55 2014 +0100
+
+    crush: set_choose_tries = 100 for erasure code rulesets
+    
+    It is common for people to try to map 9 OSDs out of a 9 OSDs total ceph
+    cluster. The default tries (50) will frequently lead to bad mappings for
+    this use case. Changing it to 100 makes no significant CPU performance
+    difference, as tested manually by running crushtool on one million
+    mappings.
+    
+    http://tracker.ceph.com/issues/10353 Fixes: #10353
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 2f87ac807f3cc7ac55d9677d2051645bf5396a62)
+    
+    Conflicts:
+    	src/erasure-code/lrc/ErasureCodeLrc.cc
+            safely ignored because the file does not exist
+
+commit 27dbbb3c312ea68a96b011ccb12394c75c0fb0f3
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Feb 12 13:49:50 2015 -0800
+
+    mon/OSDMonitor: do not trust small values in osd epoch cache
+    
+    If the epoch cache says the osd has epoch 100 and the osd is asking for
+    epoch 200+, do not send it 100+.
+    
+    Fixes: #10787
+    Backport: giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a5759e9b97107488a8508f36adf9ca1aba3fae07)
diff --git a/doc/changelog/v0.80.11.txt b/doc/changelog/v0.80.11.txt
new file mode 100644
index 000000000..beb1ea96e
--- /dev/null
+++ b/doc/changelog/v0.80.11.txt
@@ -0,0 +1,3184 @@
+commit 8424145d49264624a3b0a204aedb127835161070 (tag: refs/tags/v0.80.11, refs/remotes/gh/firefly)
+Author: Jenkins Build Slave User <jenkins-build@jenkins-slave-wheezy.localdomain>
+Date:   Tue Nov 17 18:57:48 2015 +0000
+
+    0.80.11
+
+commit 2a14d16712fbe2ee9d7d2aef389e18fcb2809a47
+Merge: c551622 a06179a
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Nov 2 14:34:28 2015 -0800
+
+    Merge pull request #6451 from jcsp/wip-firefly-urls
+    
+    qa: http://ceph.com/qa -> http://download.ceph.com/qa
+
+commit a06179acb268b073fcf67fbbf19152c1a5a275af
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 24 18:10:46 2015 -0400
+
+    qa: http://ceph.com/qa -> http://download.ceph.com/qa
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d4d65fbd6ad15320339751c604613ac63511e645)
+
+commit c551622ca21fe044bc1083614c45d888a2a34aeb
+Merge: 241edfa bff2f47
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Oct 23 21:47:27 2015 +0200
+
+    Merge pull request #5823 from SUSE/wip-11700-firefly
+    
+    make the all osd/filestore thread pool suicide timeouts separately configurable
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 241edfabf6a6c767afc1e283dd7387377ece36c7
+Merge: 07e90f5 f39c791
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Oct 23 21:01:54 2015 +0200
+
+    Merge pull request #5815 from SUSE/wip-12838-firefly
+    
+    Mutex Assert from PipeConnection::try_get_pipe
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 07e90f54318c0a6202add4de16f965c6e331d513
+Merge: eaf7dc3 d0d6727
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 22 18:19:39 2015 +0200
+
+    Merge pull request #5529 from SUSE/wip-12586-firefly
+    
+    FileStore calls syncfs(2) even it is not supported
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit eaf7dc343eb78f09d9a4b1815f6bd6ceb4217ad7
+Merge: a960327 98769f7
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Oct 22 07:44:12 2015 -0700
+
+    Merge pull request #5532 from SUSE/wip-12635-firefly
+    
+    swift smoke test fails on TestAccountUTF8
+
+commit a96032738ba1e1cc35f21607eba06e7e241abd97
+Merge: 15f3628 86f9e55
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Oct 22 07:33:02 2015 -0700
+
+    Merge pull request #5997 from SUSE/wip-13178-firefly
+    
+    rgw: content length parsing calls strtol() instead of strtoll()
+
+commit 15f362808b7100a5bade24cc78e718b04d391b54
+Merge: e010899 ebba1d5
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 22 10:45:48 2015 +0200
+
+    Merge pull request #5526 from SUSE/wip-12584-firefly
+    
+    OSD crash creating/deleting pools
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit e010899ac9cd4060c09f0f1c754f0840b5149f6b
+Merge: 24ffe4c 8c02376
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 22 10:43:29 2015 +0200
+
+    Merge pull request #5287 from SUSE/wip-12389-firefly
+    
+    PGLog::proc_replica_log: correctly handle case where entries between olog.head and log.tail were split out
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 24ffe4c5156843ce69cae30087fb767063fd489a
+Merge: f48450b c155df2
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 21 10:03:13 2015 +0200
+
+    Merge pull request #6328 from dachary/wip-13551-firefly
+    
+    qemu workunit refers to apt-mirror.front.sepia.ceph.com
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit c155df217287b980e176c3c98ee49753d6378740
+Author: Yuan Zhou <yuan.zhou@intel.com>
+Date:   Fri Jun 19 00:02:20 2015 +0800
+
+    qa: Use public qemu repo
+    
+    This would allow some external tests outside of sepia lab
+    
+    Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>
+    (cherry picked from commit 4731c1e35539c9506ff3fe3141553fad099d0eee)
+
+commit d9ae7d80850b9ea4a84549c3afb4603cf9585b6a
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Apr 16 08:29:41 2015 -0700
+
+    use git://git.ceph.com
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 34c467ba06345eacd2fd0e79c162dfd0b22f0486)
+    
+    Conflicts:
+    	qa/run_xfstests-obsolete.sh : does not exist in firefly
+    	qa/workunits/suites/fsx.sh  : trivial resolution
+
+commit f48450bdc70eef7cef8f76bc30f678a6d8d52b78
+Merge: f275ad2 b894b36
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 21 01:02:08 2015 +0200
+
+    Merge pull request #5822 from SUSE/wip-12837-firefly
+    
+    WBThrottle::clear_object: signal on cond when we reduce throttle values
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit f275ad21a48cc4f2812f3ab9e3e9b8b05469c3c7
+Merge: 143e267 81bb6dd
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:48:39 2015 +0200
+
+    Merge pull request #6325 from dachary/wip-13041-firefly
+    
+    common/Thread:pthread_attr_destroy(thread_attr) when done with it
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 143e267a30ef7aac04fdfa530babab529cf30c1a
+Merge: 725401e 6d396a3
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:46:28 2015 +0200
+
+    Merge pull request #6091 from SUSE/wip-13252-firefly
+    
+    mon: does not check for IO errors on every transaction
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 725401e05d0fbe41d93b200fe39d73d9dddae80c
+Merge: 8fd4dd7 c85d063
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:45:19 2015 +0200
+
+    Merge pull request #6087 from SUSE/wip-13244-firefly
+    
+    client nonce collision due to unshared pid namespaces
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 8fd4dd7480148c664d2c99956a7c987b8707c3c3
+Merge: 77a1b5f 6d82eb1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:44:32 2015 +0200
+
+    Merge pull request #6010 from liewegas/wip-globalid-firefly
+    
+    increase globalid default for firefly
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 77a1b5f1292d8410a1c29cebb0da449b09a230d3
+Merge: a3823fe 8c14cad
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:43:33 2015 +0200
+
+    Merge pull request #5991 from SUSE/wip-13043-firefly
+    
+    ThreadPool add/remove work queue methods not thread safe
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit a3823fe76171479e4c8f071f6e8eee3f13a02441
+Merge: e771054 48c929e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:42:28 2015 +0200
+
+    Merge pull request #5988 from SUSE/wip-13038-firefly
+    
+    PG::handle_advance_map: on_pool_change after handling the map change
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit e771054241fcba536a478f1d014feb20a434fa3f
+Merge: e43d049 cd1396c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:41:19 2015 +0200
+
+    Merge pull request #5820 from SUSE/wip-12845-firefly
+    
+    osd/PGLog.cc: 732: FAILED assert(log.log.size() == log_keys_debug.size())
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit e43d0492a415be5a0feb361efbfbdda6e00a6234
+Merge: 11ed9f9 2c2ffa1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:40:28 2015 +0200
+
+    Merge pull request #5814 from SUSE/wip-12840-firefly
+    
+    recursive lock of md_config_t (0)
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 11ed9f9b263a1d3a8df7996d5644f0f6de7a9b1a
+Merge: aead425 2c0d7fe
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:39:32 2015 +0200
+
+    Merge pull request #5813 from SUSE/wip-12822-firefly
+    
+    ceph-fuse crash in test_client_recovery
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit aead4255528e60a2ccc6f4d47a6cb482bfdccc82
+Merge: 1c65d09 d3c9469
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:37:58 2015 +0200
+
+    Merge pull request #5726 from ceph/wip-12752
+    
+    is_new_interval() fixes
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 1c65d09ef8de2732cfa6f18d22de2c7b3fac0f0f
+Merge: 1f0f4bb 411769c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:37:01 2015 +0200
+
+    Merge pull request #5698 from tchaikov/wip-12638-firefly
+    
+    mon: add a cache layer over MonitorDBStore
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 1f0f4bb5347cb935095dd374d6dcdec44ff175e2
+Merge: 4a1e54f 278d732
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:35:56 2015 +0200
+
+    Merge pull request #5619 from liewegas/wip-journal-firefly
+    
+    os/FileJournal: Fix journal write fail, align for direct io
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.cz>
+
+commit 4a1e54fc88e43885c57049d1ad4c5641621b6c29
+Merge: f272435 de53add
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:31:52 2015 +0200
+
+    Merge pull request #5360 from ceph/wip-11786.firefly
+    
+    mon: MonitorDBStore: get_next_key() only if prefix matches
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f272435c1303327e7dae248179387d12e28ea6bc
+Merge: 21452ed e7d08b1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:30:04 2015 +0200
+
+    Merge pull request #5236 from SUSE/wip-12312-firefly
+    
+    mon: adding exsting pool as tier with --force-nonempty clobbers removed_snaps
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 21452ed93b707984d5301a51040ba8c311678c9c
+Merge: 14c8528 971eb3f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 21:28:31 2015 +0200
+
+    Merge pull request #5050 from SUSE/wip-12015-firefly
+    
+    LibRadosTwoPools[EC]PP.PromoteSnap failure
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 14c85282399aca29a31566e0f068376b10cb95e6
+Merge: b6d5768 7a8b6fb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 13:09:38 2015 +0200
+
+    Merge pull request #6207 from SUSE/wip-13417-firefly
+    
+    Fix -Wno-format and -Werror=format-security options clash
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit b6d57686e1f482bf510e6c6c0ae56cff575f3eca
+Merge: fca8a17 415335d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 13:08:01 2015 +0200
+
+    Merge pull request #6203 from SUSE/wip-13413-firefly
+    
+    fedora21 has junit, not junit4
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit fca8a17bbb91ce5b3cb0c6a282cded03fe37956f
+Merge: 6240b61 20ad17d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 13:06:14 2015 +0200
+
+    Merge pull request #5992 from SUSE/wip-13168-firefly
+    
+    upstart: configuration is too generous on restarts
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 6240b61fc48e4f7b306ec18eca51f40081b5160d
+Merge: d2459d8 04b2a87
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 13:04:40 2015 +0200
+
+    Merge pull request #5541 from dachary/wip-12418-firefly
+    
+    ceph-disk: sometimes the journal symlink is not created
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit d2459d8b9c189e557f81bb7d10dd17cc53cf9af1
+Merge: 3e0e3c5 644f213
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 12:36:01 2015 +0200
+
+    Merge pull request #5729 from ceph/wip-11455-firefly
+    
+    rgw: init some manifest fields when handling explicit objs
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 3e0e3c518bf68e9d0ad7a79d804602634b9c6466
+Merge: e44a777 e7931a7
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 12:35:22 2015 +0200
+
+    Merge pull request #5730 from ceph/wip-12537-firefly
+    
+    RGW returns requested bucket name raw in "Bucket" response header
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit e44a7775fb34ccdf7ff36e5e56aba9a309521be1
+Merge: 028da25 3be204f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 20 12:34:01 2015 +0200
+
+    Merge pull request #5831 from gnuoy/wip-11140-firefly
+    
+    /etc/init.d/radosgw restart does not work correctly
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 81bb6dd7de8a191c11c16b3cdc07355ef3434435
+Author: zqkkqz <zheng.qiankun@h3c.com>
+Date:   Fri Aug 7 10:49:45 2015 +0800
+
+    Common/Thread: pthread_attr_destroy(thread_attr) when done with it
+    When a thread attributes object is no longer required, it should be destroyed using the
+    pthread_attr_destroy() function. Destroying a thread attributes object has no effect on threads that were created using that object.
+    
+    Fixes: #12570
+    Signed-off-by: zqkqkz zheng.qiankun@h3c.com
+    (cherry picked from commit 9471bb838a420ef5f439191d87e5388fecccb9e6)
+
+commit 77d6426a8323db91a60f246f385826c35f31521c
+Author: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+Date:   Fri Jul 17 12:43:52 2015 +0200
+
+    Thread.cc: remove malloc/free pair
+    
+    There's no need for mallocing pthread_attr_t in Thread::try_create(),
+    it can be located on stack as it is freed in same function. This reduces
+    pressure put on memory manager.
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+    (cherry picked from commit 614ac0f4e375f3dd008c8937d3f89f6005d6e0a0)
+
+commit 7a8b6fb4c7b80c33dd3565fc4140d45fe70f0438
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri Aug 15 19:34:27 2014 +0200
+
+    Fix -Wno-format and -Werror=format-security options clash
+    
+    This causes build failure in latest fedora builds, ceph_test_librbd_fsx adds -Wno-format cflag but the default AM_CFLAGS already contain -Werror=format-security, in previous releases, this was tolerated but in the latest fedora rawhide it no longer is, ceph_test_librbd_fsx builds fine without -Wno-format on x86_64 so there is likely no need for the flag anymore
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 7df67a544f6401fd31157b6e2a6ca590d87d7cbc)
+    
+    Conflicts:
+    	src/test/Makefile.am
+
+commit 415335df53bf0d6b42f745b038c1c0c9c4aaa810
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 3 16:14:23 2015 +0100
+
+    ceph.spec.in: junit always except for EPEL 6
+    
+    The package was renamed a long time ago (around the Fedora 15
+    timeframe). The "junit4" name is only relevant for EPEL 6. For EPEL 7
+    and Fedora 20, the "junit" package has "Provides: junit4". And most
+    recently, in the junit package that ships in Fedora 21 and 22, the
+    package maintainer dropped the old Provides: line.
+    
+    http://tracker.ceph.com/issues/10728 Fixes: #10728
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 4f36eaee9ea71ed48018ccb2d7eac3e809c72d82)
+
+commit 98769f7ccf97d6a0c98438f1eab7a5705fb9920b
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Fri Sep 4 01:07:48 2015 +0000
+
+    Fix casing of Content-Type header
+    
+    It turns out, despite the HTTP spec declaring that header field names
+    should be case-insensitive, some clients treat them wrongly, and
+    consider "Content-type" to not match "Content-Type".
+    
+    CyberDuck was one of those clients, now fixed upstream in
+    https://trac.cyberduck.io/ticket/8999
+    
+    To reduce future occurances of this bug, fix the casing of the
+    Content-Type header, to strictly comply with the HTTP specification (be
+    strict about what you send, and generous about what you receive).
+    
+    Fixes: #12939
+    Backport: hammer, firefly
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit 1b9fbffdc24160251b96cec820d62fb2a12b6eab)
+
+commit ccf268502b0592f5ff58cd1a5920ec02a117acfb
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Aug 13 15:02:03 2015 +0200
+
+    rgw: we should not overide Swift sent content type
+    
+    Fixes: #12363
+    backport: hammer
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 423cf136f15df3099c9266f55932542d303c2713)
+
+commit 6829ee29a484a3b7055122addcda06a233d3ce53
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Thu Jun 25 18:28:50 2015 +0200
+
+    rgw: send Content-Length in response for GET on Swift account.
+    
+    Fixes: #12158
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 5f47b11791cd46ce2771046e4888575869cc9685)
+    
+    Conflicts:
+    	src/rgw/rgw_rest_swift.cc
+                omit dump_account_metadata() call
+
+commit c9226afd7b9e54a8998bf88dfbd6753b262fdf9d
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue Jun 23 11:54:09 2015 +0200
+
+    rgw: enforce Content-Type in Swift responses.
+    
+    Swift sends Content-Type HTTP header even if the response
+    doesn't contain body. We have this behaviour implemented
+    until applying some changes in end_header() function.
+    Unfortunately, lack of Content-Type causes early exits in
+    many Tempest's tests for Swift API verification.
+    
+    Fixes: #12157
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 106aeba206736d4080326f9bc191876bed63370b)
+
+commit 9139fbbc1ab3ae455d2e36b3a768601ccb216801
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Jun 22 13:51:49 2015 +0200
+
+    rgw: force content_type for swift bucket stats request
+    
+    Fixes: 12095
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit da259e4da1329e1e448d75c48c6ee3d1318e19b2)
+
+commit 767f1f74c9f015d6bb3376266ec77a10a2d2c59a
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Tue Apr 21 12:31:19 2015 +0200
+
+    rgw: force content-type header for swift account responses without body
+    
+    This regression was introduced in commit 4cfeca03ade21861ab70ca759f94eb244c16cb39
+    
+    Fixes: #11438
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 009664e560ee37ef6dc170a0155d6c749e151b06)
+
+commit e3da03dd6be04e782c3e5786767bba4b4964fbec
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Apr 9 20:04:59 2015 +0200
+
+    rgw: shouldn't return content-type: application/xml if content length is 0
+    
+    Fixed: #11091
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 5cc92bb27cfbaee4af24dc312c9b98331f899794)
+
+commit e7d08b1bca951f9a963eb0aeb4cd88abbe5723ba
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri May 8 10:26:48 2015 -0700
+
+    test/librados/tier.cc: destroy and recreate cache pool on every test
+    
+    Namespaces are not sufficient with the checks for 11493 in the mon.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit bef09e0cdb274cb1c87335a2af9ee532d14a4596)
+
+commit d7f53442260dcfc92e7afa8bba2dd116c15e030d
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Apr 29 12:34:25 2015 -0700
+
+    mon: prevent pool with snapshot state from being used as a tier
+    
+    If we add a pool with snap state as a tier the snap state gets clobbered
+    by OSDMap::Incremental::propogate_snaps_to_tiers(), and may prevent OSDs
+    from starting.  Disallow this.
+    
+    Include a test.
+    
+    Fixes: #11493
+    Backport: hammer, giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit bbec53edf9e585af4e20bbc9ba9057d6fdfda342)
+    
+    Conflicts:
+    	qa/workunits/cephtool/test.sh
+               properly co-exist with "# make sure we can't create an ec pool tier"
+    	src/mon/OSDMonitor.cc
+               properly co-exist with preceding "if (tp->ec_pool())"
+            (The changes to both files would have applied cleanly if
+            https://github.com/ceph/ceph/pull/5389 had not been merged first.)
+
+commit 028da252d85733fcc7d8495750b9ae97043cb0dd
+Merge: ecabc67 f71a6eb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sat Oct 3 09:40:33 2015 +0200
+
+    Merge pull request #5993 from SUSE/wip-13169-firefly
+    
+    update docs to point to download.ceph.com
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 971eb3f6dddb509d15e5e4bcfd5e34d42ae2e75b
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Nov 13 22:33:10 2014 -0800
+
+    ceph_test_rados_api_tier: fix cleanup of whiteouts
+    
+    Doing a normal DELETE on a whiteout returns ENOENT.  Use the IGNORE_CACHE
+    flag to make sure it removes the whiteout, too.
+    
+    Fixes: #10052
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8dde6a61921798aa188bd69406c8fa7afeb96b45)
+    
+    Conflicts:
+    	src/test/librados/TestCase.cc
+                for it of type ObjectIterator:
+                - use it->first instead of it->get_oid()
+                - use it->second instead of it->get_locator()
+
+commit b134a629863b4601892db03d7c4377040d556bdd
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Nov 13 22:32:20 2014 -0800
+
+    osd/ReplicatedPG: allow whiteout deletion with IGNORE_CACHE flag
+    
+    If the client specifies IGNORE_CACHE, allow a regular DELETE to zap a
+    whiteout.  Expand test case to verify this works.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 34e4d24)
+    
+        Conflicts:
+            src/test/librados/tier.cc
+                replaced NObjectIterator -> ObjectIterator
+                replaced cache_ioctx.nobjects_begin -> cache_ioctx.objects_begin
+                replaced cache_ioctx.nobjects_end -> cache_ioctx.objects_end
+                replace it->get_oid() with it->first for it of type ObjectIterator
+
+commit 6d396a322e5ad61a3ca262b1c78a670481ff740c
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 23 10:58:01 2015 -0400
+
+    mon/Elector: do a trivial write on every election cycle
+    
+    Currently we already do a small write when the *first* election in
+    a round happens (to update the election epoch).  If the backend
+    happens to fail while we are already in the midst of elections,
+    however, we may continue to call elections without verifying we
+    are still writeable.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ef909ccbdc303cce8a39edef255325127832ff16)
+    
+        Conflicts:
+    	src/mon/Elector.cc
+                no MonitorDBStore::TransactionRef in firefly
+
+commit d2bfa19077422eea37205fee9ed6fd5f2692f222
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 23 10:58:13 2015 -0400
+
+    mon/MonitorDBStore: assert/crash if there is a write error
+    
+    Do this globally intead of relying on teh zillion mon callers to
+    check the error code.  There are no cases where we want to
+    tolerate a commit failure.
+    
+    Fixes: #13089
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2fb7b1f0e33ada7c9a1be3de2f7708eb0760fcef)
+
+commit c85d06384e795ab2717868ce45f01ae93181a6be
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Sep 21 15:46:42 2015 -0700
+
+    use simplifed messenger constructor for clients
+    
+    This is all mechanical except the calculation of the nonce, which is
+    now always randomized for clients.
+    
+    Fixes: #13032
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit a3a8c85b79afef67681c32c57b591c0e0a87a349)
+    
+    Conflicts:
+    	src/ceph_fuse.cc
+    	src/ceph_syn.cc
+    	src/libcephfs.cc
+    	src/librados/RadosClient.cc
+    	src/mds/MDSUtility.cc
+    	src/mon/MonClient.cc
+    	src/test/mon/test_mon_workloadgen.cc
+                - different arguments to Messenger::create() in firefly
+
+commit ef24b4ec7245959b1960a415617439bc3baac9a3
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 6 21:20:08 2014 -0700
+
+    librados: use Messenger factory method
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5a860bec48348ed4c09256ccd394b33ace6b4f52)
+
+commit 3a52acde3133aab2a717fefa967b0d404e50e2d8
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 6 08:50:48 2014 -0700
+
+    mon/MonClient: use generic Messenger factory
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e55cb1f68ae1a04cc5d281bea57ee79ef4a8af38)
+
+commit de8da652b6c969aa0446f0d3351f63b012063c71
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Sep 21 15:20:53 2015 -0700
+
+    msg: add simplified messenger constructor
+    
+    Several callers create messengers using exactly the same parameters:
+    
+    - reading the ms type from cct that is also passed in
+    - a default entity_name_t::CLIENT
+    - the default features
+    
+    Additionally, the nonce should be randomized and not depend on
+    e.g. pid, as it does in several callers now. Clients running in
+    containers can easily have pid collisions, leading to hangs, so
+    randomize the nonce in this simplified constructor rather than
+    duplicating that logic in every caller.
+    
+    Daemons have meaningful entity_name_ts, and monitors currently depend
+    on using 0 as a nonce, so make this simple constructer
+    client-specific.
+    
+    Related: #13032
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit e3785b0b5fbff870adbd5f121021b671912386cf)
+    
+    Conflicts:
+    	src/msg/Messenger.cc
+                - different #includes in firefly
+                - different Messenger::create() arguments in firefly
+
+commit ecabc6796f19c03947bb5b14da4e1b761ff8847f
+Merge: 741dd5b 2e09a2c
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 23 17:09:18 2015 -0400
+
+    Merge pull request #6038 from liewegas/wip-13088
+    
+    common/HeartBeat: include types
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 2e09a2c22ab885f8ec81dbc55f2c8fc0f2984543
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 23 17:06:52 2015 -0400
+
+    common/RWLock: include assert
+    
+    Fixes atomic_t build issue without libatomic-ops.
+    
+    Fixes: #13088
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 741dd5b9433d1592ce91aea70ebec106a1354973
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 22 13:03:49 2015 -0400
+
+    .gitmodules: fix ceph-object-corpus source url
+    
+    Fixes: #13196
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit a1997879aa26299519ed157017f8f5715c626dcc
+Merge: ee81812 01dc221
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 22 12:29:08 2015 -0400
+
+    Merge pull request #6000 from SUSE/wip-12097-firefly
+    
+    kernel_untar_build fails on EL7
+
+commit ee8181206726712729235e4a5f8d2094a6e72f71
+Merge: bb8b422 4e7de5b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 21 13:07:49 2015 -0700
+
+    Merge pull request #5234 from SUSE/wip-12244-firefly
+    
+    Civetweb RGW appears to report full size of object as downloaded when only partially downloaded
+
+commit bb8b422beac22bd8c2e9c1ecd83e265a3de1a3e6
+Merge: 4b08895 f602263
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 21 12:35:52 2015 -0700
+
+    Merge pull request #4535 from xinxinsh/wip-11416-firefly
+    
+    rgw: use correct objv_tracker for bucket instance
+
+commit 6d82eb165fdc91851f702a463022b26c50f5094b (refs/remotes/me/wip-globalid-firefly)
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Jan 18 16:49:20 2015 -0800
+
+    mon: handle case where mon_globalid_prealloc > max_global_id
+    
+    This triggers with the new larger mon_globalid_prealloc value.  It didn't
+    trigger on the existing cluster I tested on because it already had a very
+    large max.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit bbaf582e5bb54ae35327ffd234e4ccf0f07317db)
+
+commit f545a0f430bf6f1e26983fc0ff20a645697f017c
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Jan 18 10:39:25 2015 -0800
+
+    mon: change mon_globalid_prealloc to 10000 (from 100)
+    
+    100 ids (session 100 authentications) can be consumed quite quickly if
+    the monitor is being queried by the CLI via scripts or on a large cluster,
+    especially if the propose interval is long (many seconds).  These live in
+    a 64-bit value and are only "lost" if we have a mon election before they
+    are consumed, so there's no real risk here.
+    
+    Backport: giant, firefly
+    Reviewed-by: Joao Eduardo Luis <joao@redhat.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1d1215fe5f95c2bafee5b670cdae1353104636a0)
+
+commit 01dc221f280ad25aa556f246414e3c3c42762956
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Tue Jun 16 08:13:41 2015 -0700
+
+    qa: update to newer Linux tarball
+    
+    This should make newer gcc releases happier in their default configuration.
+    kernel.org is now distributing tarballs as .xz files so we change to that
+    as well when decompressing (it is supported by Ubuntu Precise so we should
+    be all good).
+    
+    Fixes: #11758
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 1ea3f47ab806d48ca7b045c2731d344eae3900e1)
+    
+    Conflicts:
+    	qa/workunits/kernel_untar_build.sh
+                trivial resolution
+
+commit 86f9e55f0c151c0b9a289b475f87b6a11329e6e1
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jan 30 10:51:52 2015 -0800
+
+    rgw: use strict_strtoll() for content length
+    
+    instead of strict_strtol().
+    
+    Backport: giant, firefly
+    Fixes: #10701
+    
+    Reported-by: Axel Dunkel <ad@dunkel.de>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 1c25dbafb45caf1b261cfcec15b868a2ba6b5fef)
+
+commit f71a6ebf1b371f9389865a0a33652841726ff77b
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Sep 18 14:13:02 2015 -0400
+
+    doc: correct links to download.ceph.com
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 387d7800359154950431d0984c756f43f21dd9b4)
+    
+    Conflicts:
+    	doc/install/get-packages.rst
+                preferred the new version in all cases
+
+commit 20ad17d271fb443f6c40591e205e880b5014a4f3
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 3 14:57:34 2015 -0400
+
+    upstart: limit respawn to 3 in 30 mins (instead of 5 in 30s)
+    
+    It may take tens of seconds to restart each time, so 5 in 30s does not stop
+    the crash on startup respawn loop in many cases.  In particular, we'd like
+    to catch the case where the internal heartbeats fail.
+    
+    This should be enough for all but the most sluggish of OSDs and capture
+    many cases of failure shortly after startup.
+    
+    Fixes: #11798
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit eaff6cb24ef052c54dfa2131811758e335f19939)
+
+commit 8c14cad0895590f19a6640c945b52213f30a9671
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 10 19:10:19 2015 -0400
+
+    WorkQueue: add/remove_work_queue methods now thread safe
+    
+    These methods were not acquiring the ThreadPool lock when
+    manipulating the work_queue collection.  This was causing
+    occasional crashes within librbd when opening and closing
+    images.
+    
+    Fixes: #12662
+    Backport: hammer, firefly
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3e18449b01c1ab78d1bbfc1cf111aa9bdbef7b1f)
+
+commit 48c929e689b0fa5138922fcb959be5d05296e59a
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Aug 27 11:08:33 2015 -0700
+
+    PG::handle_advance_map: on_pool_change after handling the map change
+    
+    Otherwise, the is_active() checks in the hitset code can erroneously
+    return true firing off repops stamped with the new epoch which then get
+    cleared in the map change code.  The filestore callbacks then pass the
+    interval check and call into a destroyed repop structure.
+    
+    Fixes: 12809
+    Backport: hammer,firefly
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 14e02bc90a463805f4c3e2de210892067a52514b)
+
+commit 4b0889567d7fe28ef6976f8dbb0d329b03200986
+Merge: 0a740d7 5426421
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 10 11:46:41 2015 +0200
+
+    Merge pull request #5200 from SUSE/wip-12289-firefly
+    
+    mon: scrub error (osdmap encoding mismatch?) upgrading from 0.80 to ~0.80.2
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit de53addac8234037a66cdd45cf8007deba7a0530 (refs/remotes/gh/wip-11786.firefly)
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Mon Sep 7 14:12:19 2015 +0100
+
+    mon: MonitorDBStore: make get_next_key() work properly
+    
+    We introduced a significant bug with 2cc7aee, when we fixed issue #11786.
+    Although that patch would fix the problem described in #11786, we
+    managed to not increment the iterator upon returning the current key.
+    This would have the iterator iterating over the same key, forever and
+    ever.
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit 70d31082fd3dc8c7857994104577f1a3631c678c)
+
+commit 0a740d7b2b62e55056fb96d6423203c47e0f3cf6
+Merge: aeea956 37d16a9
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Sep 9 15:45:37 2015 +0200
+
+    Merge pull request #5388 from SUSE/wip-12490-firefly
+    
+    buffer: critical bufferlist::zero bug
+    
+    Reviewed-by: Gregory Farnum <gfarnum@redhat.com>
+
+commit aeea9567e8b047b3a67badfcc4e43c880f0efa46
+Merge: 973b74d 6b42b06
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Sep 9 06:10:09 2015 +0200
+
+    Merge pull request #5408 from SUSE/wip-12492-firefly
+    
+    the output is wrong when runing ceph osd reweight
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 973b74d5b266f4e9e7f992a85b22f42c9fb543b1
+Merge: 79403ba 5445461
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Sep 9 06:07:51 2015 +0200
+
+    Merge pull request #5404 from SUSE/wip-12395-firefly
+    
+    register_new_pgs() should check ruleno instead of its index
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 79403ba8ce526d65521e6cd2ca2bc06f5fe261b9
+Merge: db44bb2 dc12875
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Sep 9 06:05:39 2015 +0200
+
+    Merge pull request #5199 from SUSE/wip-11980-firefly
+    
+    MDSMonitor: handle MDSBeacon messages properly
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit db44bb29b8b036642c50f5d322b5edd9d51699bb
+Merge: fc699f4 31ff7d1
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Sep 9 06:04:15 2015 +0200
+
+    Merge pull request #5410 from SUSE/wip-12497-firefly
+    
+    get pools health'info have error
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit fc699f42459ab3723f3e63b28a4aa5f481c78fc4
+Merge: 6762295 ea0d67c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Sep 9 06:01:50 2015 +0200
+
+    Merge pull request #5409 from SUSE/wip-12495-firefly
+    
+    pgmonitor: wrong at/near target max“ reporting
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 67622955b1cfd0b97ac8d3d96b223995440af0b1
+Merge: c4fa5ef 2f35a41
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Sep 7 14:22:52 2015 +0200
+
+    Merge pull request #5358 from ceph/wip-11470.firefly
+    
+    mon: PaxosService: call post_refresh() instead of post_paxos_update()
+    
+    Reviewed-by: Joao Eduardo Luis <joao@suse.de>
+
+commit 3be204f6a22be109d2aa8cfd5cee09ec3381d9b2
+Author: Dmitry Yatsushkevich <dyatsushkevich@mirantis.com>
+Date:   Tue Mar 17 15:44:52 2015 -0700
+
+    rgw: init script waits until the radosgw stops
+    
+    Fixes: #11140
+    Init script waits in stop action until the radowgw daemons stop.
+    
+    Signed-off-by: Dmitry Yatsushkevich <dyatsushkevich@mirantis.com>
+    (cherry picked from commit 1cca0c1e91a1e18f82f4d22855e96b2fc947f5ea)
+
+commit c4fa5ef989247404ded7adf5dad4e5316b8af04a
+Merge: beff99b c6f3780
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Sep 7 10:20:27 2015 +0200
+
+    Merge pull request #5403 from SUSE/wip-12393-firefly
+    
+    Memory leak in Mutex.cc, pthread_mutexattr_init without pthread_mutexattr_destroy
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit beff99bc7358ee0625079ca60adbf109986f2285
+Merge: 4a3c178 68508e2
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sun Sep 6 21:19:55 2015 +0200
+
+    Merge pull request #5225 from SUSE/wip-12266-firefly
+    
+    ceph.spec.in: 50-rbd.rules conditional is wrong
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 4a3c17834890b69e0467b65f83b1979328f3d27c
+Merge: 6571cd9 2544a6f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sun Sep 6 21:19:42 2015 +0200
+
+    Merge pull request #5217 from SUSE/wip-12268-firefly
+    
+    ceph-common needs python-argparse on older distros, but doesn't require it
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit bff2f477c4ad86b4bd6e3ca3e637a6168c5c8053
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Sep 6 16:30:35 2015 +0200
+
+    OSD: add scrub_finalize_wq suicide timeout
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 91d4c217e32b8b76fcac49f37879a3f78088694d
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 6 10:52:40 2015 -0700
+
+    OSD: add scrub_wq suicide timeout
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 547a7041edc833f3cc8e04d388574809e30a8af6)
+    
+    Conflicts:
+            src/osd/OSD.cc
+                firefly still has scrub_finalize_wq
+
+commit 7f6ec65b7c2ca0174142c1c48f18998d8c586b02
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 6 11:02:19 2015 -0700
+
+    OSD: add op_wq suicide timeout
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 3f2946aa9f9b4b432e8c018283503153a2b1eddc)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+              adapt op_wq() and peering_wq() calls in OSD::OSD
+    	src/osd/OSD.h
+              adapt OpWQ instead of ShardedOpWQ
+
+commit 9ce8ce689009cf8ef749edd320d1c2a73ecc2f90
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 6 10:54:31 2015 -0700
+
+    OSD: add remove_wq suicide timeout
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit f2fbfa32a16666be46359f0eab7b04ca80a753f5)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+               No next_removal_seq(0) in master
+
+commit 6926a64fbd4718b8a5df8e04545bc93c4981e413
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 6 10:51:28 2015 -0700
+
+    OSD: add snap_trim_wq suicide timeout
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit e1073a4a577211672148a4112bd633831552d66f)
+
+commit d31d1f6f5b08362779fa6af72690e898d2407b90
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 6 10:50:19 2015 -0700
+
+    OSD: add recovery_wq suicide timeout
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 85311b656852af75bfbbc6699f92fc6aa233c316)
+    
+    Conflicts:
+    	src/common/config_opts.h
+               place OPTION(osd_command_thread_suicide_timeout, ...)
+               directly after OPTION(osd_command_thread_timeout, ...)
+
+commit f85ec2a52e969f9a7927d0cfacda6a1cc6f2898c
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 6 10:49:00 2015 -0700
+
+    OSD: add command_wq suicide timeout
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit df4e5de819c30003cfbe50a071c49039cf534419)
+    
+    Conflicts:
+    	src/common/config_opts.h
+               place OPTION(osd_command_thread_suicide_timeout, ...)
+               directly after OPTION(osd_command_thread_timeout, ...)
+
+commit b894b368790de3383295372250888ba674502fb1
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Jul 7 08:49:54 2015 -0700
+
+    WBThrottle::clear_object: signal if we cleared an object
+    
+    Fixes: #12223
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 8447b08d60d5cf7dce20f8b839cc2cb5a93782de)
+
+commit cd1396cd62c79b177e46cfb57ab6b3b6fdcd227b
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Aug 9 10:46:10 2015 -0400
+
+    osd/PGLog: dirty_to is inclusive
+    
+    There are only two callers of mark_dirty_to who do not pass max,
+    and they are both in the merge_log extending tail path.  In that
+    case, we want to include the last version specified in the log
+    writeout.  Fix the tail extending code to always specify the
+    last entry added, inclusive.
+    
+    Fixes: #12652
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f0ca14df0641daa04eee39d98d8bd0faf46e4e6d)
+
+commit f39c7917a39e445efa8d73178657fc5960772275
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Sep 4 17:38:32 2015 +0200
+
+    osd: Keep a reference count on Connection while calling send_message()
+    
+    Fixes: #12437
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (manual backport of commit a140085)
+    
+    Conflicts:      src/osd/OSD.cc
+                       master has share_map_peer; firefly has osd->_share_map_outgoing
+
+commit 6571cd9a9616637bf32f9ac97a9bd864c0bff856
+Merge: 8edc091 bd25136
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Fri Sep 4 11:50:22 2015 -0600
+
+    Merge pull request #5224 from SUSE/wip-12304-firefly
+    
+    ceph.spec.in running fdupes unnecessarily
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 8edc091df90be6935fcfe59c18d7060c88501556
+Merge: 6053bff 601eadb
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Fri Sep 4 11:47:49 2015 -0600
+
+    Merge pull request #5394 from SUSE/wip-12447-firefly
+    
+    ceph.spec.in: useless %py_requires breaks SLE11-SP3 build
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 2c2ffa1d6d1112dbf52cbbe36f4a5376e17da56a
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Aug 24 15:40:39 2015 -0700
+
+    config: skip lockdep for intentionally recursive md_config_t lock
+    
+    lockdep can't handle recursive locks, resulting in false positive
+    reports for certain set_val_or_die() calls, like via
+    md_config_t::parse_argv() passed "-m".
+    
+    Fixes: #12614
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 415226995ebe841cc9e8639a3233150a75aecaed)
+
+commit 6053bffa8955953cf40a05f2b22684e77b6003a3
+Merge: 49d416e 5559a5f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Sep 4 17:18:30 2015 +0200
+
+    Merge pull request #5043 from SUSE/wip-12007-firefly
+    
+    Objecter: pg listing can deadlock when throttling is in use
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 2c0d7feeb1c7592887e0408fe4fadaa9b4f659e9
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Aug 13 19:41:47 2015 +0200
+
+    tests: tiering agent and proxy read
+    
+    Verify that an object promoted to a cache tier because of a proxy read
+    is evicted as expected.
+    
+    http://tracker.ceph.com/issues/12673 Refs: #12673
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 7924231930732bd297d3bd034c8295e96cb81088)
+    
+    Conflicts:
+    	qa/workunits/cephtool/test.sh
+                resolved by manually adding the new test to TESTS
+
+commit aa911767d9326c8aa37671883892b7d383596960
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Aug 13 13:47:24 2015 +0200
+
+    osd: trigger the cache agent after a promotion
+    
+    When a proxy read happens, the object promotion is done in parallel. The
+    agent_choose_mode function must be called to reconsider the situation
+    to protect against the following scenario:
+    
+      * proxy read
+      * agent_choose_mode finds no object exists and the agent
+        goes idle
+      * object promotion happens
+      * the agent does not reconsider and eviction does not happen
+        although it should
+    
+    http://tracker.ceph.com/issues/12673 Fixes: #12673
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit e1f58feb9b1d20b72f2eb2eefdea5982e0cddccd)
+
+commit 49d416e6e2e2f3e1f317978371089f9d92358642
+Merge: 7744518 f93eadd
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Sep 4 17:08:41 2015 +0200
+
+    Merge pull request #4769 from SUSE/wip-11741-firefly
+    
+    crush: take crashes due to invalid arg
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 774451810ff055984b79e1d1875bd6a3468dfb7a
+Merge: fc27b0d f122788
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Sep 4 16:50:45 2015 +0200
+
+    Merge pull request #4788 from SUSE/wip-11801-firefly
+    
+    Make it more difficult to delete pools in firefly
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit fc27b0dbe074dc15450d7669ea722aa3b605b3b2
+Merge: 055624b da78889
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Sep 4 16:44:57 2015 +0200
+
+    Merge pull request #5235 from SUSE/wip-12310-firefly
+    
+    read on chunk-aligned xattr not handled
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 055624bf9a6480fffefb3c7cf5c3a4d4d73985c1
+Merge: 045c056 46e5dea
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Sep 4 16:42:28 2015 +0200
+
+    Merge pull request #5389 from SUSE/wip-12391-firefly
+    
+    EC pools are not allowed as cache pools, disallow in the mon
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 045c0562f97e7d2c68278fbddb9ca1707bb8d5fd
+Merge: 99612e7 b8e3f6e
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 4 09:42:03 2015 -0400
+
+    Merge pull request #5406 from ceph/wip-12465-firefly
+    
+    log: take mutex while opening fd
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 99612e7ed9573f2af36604f7d258535f6feadff3
+Merge: 7f8a397 4a709a4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 3 12:34:32 2015 +0200
+
+    Merge pull request #4854 from ceph/wip-11769-firefly
+    
+    librbd: aio calls may block
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 7f8a397e38101017ef9783d8ed08bb393a8d8825
+Merge: ea601ef f0d2429
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Sep 2 07:55:15 2015 +0200
+
+    Merge pull request #5233 from SUSE/wip-12074-firefly
+    
+    RGW logfile does not get chowned properly
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 644f213c672f6fe2786e041043fdd55f8399871e (refs/remotes/gh/wip-11455-firefly)
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Aug 26 14:34:30 2015 -0700
+
+    rgw: init some manifest fields when handling explicit objs
+    
+    Fixes: #11455
+    When dealing with old manifest that has explicit objs, we also
+    need to set the head size and head object correctly so that
+    code that relies on this info doesn't break.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d7bf8cb594e5276d1c80544f5ec954d52b159750)
+
+commit e7931a73df1ab77feb1c2ece13e3de3989ef7a0e
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jul 30 14:47:15 2015 -0700
+
+    rgw: url encode exposed bucket
+    
+    Fixes: #12537
+    Don't send the bucket name back without url encoding it.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit ad5507fe0bf72ed5bdf8353e315cc9092c740144)
+
+commit 558c52955d464827630e0aa2fed970df987bb036
+Author: Wido den Hollander <wido@42on.com>
+Date:   Wed Jun 3 13:13:33 2015 +0200
+
+    rgw: Do not enclose the Bucket header in quotes
+    
+    Fixes: #11860
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit 8af25faed93fe02d3dad585b8579ce8b041cc4e6)
+
+commit d3c94698e4e852bef3e65fbf439f5f209fbc0b25 (refs/remotes/gh/wip-12752)
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Aug 21 11:32:39 2015 -0400
+
+    Objecter: pg_interval_t::is_new_interval needs pgid from previous pool
+    
+    When increasing the pg_num of a pool, an assert would fail since the
+    calculated pgid seed would be for the pool's new pg_num value instead
+    of the previous pg_num value.
+    
+    Fixes: #10399
+    Backport: infernalis, hammer, firefly
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f20f7a23e913d09cc7fc22fb3df07f9938ddc144)
+    
+    Conflicts: (hobject_t sort order not backported, trivial resolution)
+    	src/osdc/Objecter.cc
+    	src/osdc/Objecter.h
+
+commit 56d267b7ae02070a7d7ed247990b84124fd62411
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue May 26 12:35:11 2015 -0700
+
+    osd_types::is_new_interval: size change triggers new interval
+    
+    Fixes: 11771
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit ff79959c037a7145f7104b06d9e6a64492fdb95f)
+
+commit 411769c1461c11611b479bd826c72c56b3ce47c5
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Aug 10 04:25:03 2015 -0700
+
+    mon: add a cache layer over MonitorDBStore
+    
+    the cache of of leveldb does not perform well under some condition,
+    so we need a cache in our own stack.
+    
+    * add an option "mon_osd_cache_size" to control the size of cache size
+      of MonitorDBStore.
+    
+    Fixes: #12638
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5e99a578050976ca22b549812ac80d494fe7041d)
+    
+    Conflicts:
+    	src/mon/OSDMonitor.h
+    minor differences in OSDMonitor ctor
+
+commit 278d732ecd3594cd76e172d78ce3ec84e58e178b (refs/remotes/me/wip-journal-firefly)
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 15 11:20:18 2015 -0800
+
+    os/FileJournal: Fix journal write fail, align for direct io
+    
+    when config journal_zero_on_create true, osd mkfs will fail when zeroing journal.
+    journal open with O_DIRECT, buf should align with blocksize.
+    
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Xie Rui <875016668@qq.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 80473f63853593e53d72f17c098152caf17f5e9e)
+
+commit d0d6727762ebda858065101635935df3d44a18ad
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Jul 29 15:52:19 2015 +0800
+
+    common/syncfs: fall back to sync(2) if syncfs(2) not available
+    
+    Fixes: #12512
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 08210d6a85527b0759f6dc4411f742bc13a4939f)
+
+commit 40494c6e479c2ec4dfe5f6c2d6aef3b6fa841620
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Thu Mar 12 17:38:27 2015 +0100
+
+    sync_filesystem.h: fix unreachable code
+    
+    Fix for:
+    
+    CID 1264460 (#1 of 1): Structurally dead code (UNREACHABLE)
+    unreachable: This code cannot be reached: sync();
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 9921836e982818b92e20280eecb74e701523045c)
+
+commit 3dbbc86ad6d1e7131bbe49a4eff1557d7da9822f
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Thu Jan 8 10:29:37 2015 +0800
+
+    mon, os: check the result of sync_filesystem.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 27cb78bd3b2dde40c70de245769d1c0ec252533b)
+
+commit 7fa6fdc6c5e52f11456e4bea4ae32fd62248c80b
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Wed Jan 7 15:28:37 2015 +0800
+
+    common: Don't call ioctl(BTRFS_IOC_SYNC) in sync_filesystem.
+    
+    From the manual of syncfs, it first appeared in Linux 2.6.39. At this
+    point, btrfs didn't as a stable production. So remove this.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 397b261630dccc656f6eca509f4c70173c8d8d8f)
+
+commit 9f15ed5bb5a837727cb3bef70508e056c125a518
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Wed Jan 7 15:26:08 2015 +0800
+
+    common: Directly return the result of syncfs().
+    
+    In commit 808c644248e4, it will try sync() if syncfs() return error.
+    No evidence prove this way can work. And sync() don't return result
+    so make this function always return zero which cause filestore omit the
+    error.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 3f7faa4d8d3af3d2e3889c23290b016f27d36386)
+
+commit ea601efc6c693ba8041509c98756fa8c49b9c13a
+Merge: 50fa963 a128ce3
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Aug 12 09:43:19 2015 +0200
+
+    Merge pull request #5390 from SUSE/wip-12503-firefly
+    
+    rgw/logrotate.conf calls service with wrong init script name
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 50fa9636d0c5e8946fa410891623e2c8363e2067
+Merge: abe6c03 d4ef556
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Aug 11 11:27:16 2015 -0700
+
+    Merge pull request #4765 from SUSE/wip-11721-firefly
+    
+    rgw: Keystone PKI token expiration is not enforced
+
+commit 04b2a878b1329202758465cf8e9b0f874cbeeef5
+Author: Dan van der Ster <daniel.vanderster@cern.ch>
+Date:   Tue Nov 18 15:51:46 2014 +0100
+
+    ceph-disk: don't change the journal partition uuid
+    
+    We observe that the new /dev/disk/by-partuuid/<journal_uuid>
+    symlink is not always created by udev when reusing a journal
+    partition. Fix by not changing the uuid of a journal partition
+    in this case -- instead we can reuse the existing uuid (and
+    journal_symlink) instead. We also now assert that the symlink
+    exists before further preparing the OSD.
+    
+    Fixes: #10146
+    Signed-off-by: Dan van der Ster <daniel.vanderster@cern.ch>
+    Tested-by: Dan van der Ster <daniel.vanderster@cern.ch>
+    (cherry picked from commit 29eb1350b4acaeabfe1d2b19efedbce22641d8cc)
+
+commit e57e6f5da10a62f2f4d7b1a6a734a095ed494ebe
+Author: Dan van der Ster <daniel.vanderster@cern.ch>
+Date:   Mon Sep 29 13:20:10 2014 +0200
+
+    ceph-disk: set guid if reusing a journal partition
+    
+    When reusing a journal partition (e.g. /dev/sda2) we should set a
+    new partition guid and link it correctly with the OSD. This way
+    the journal is symlinked by its persistent name and ceph-disk list
+    works correctly.
+    
+    Signed-off-by: Dan van der Ster <daniel.vanderster@cern.ch>
+    (cherry picked from commit beff616f506b96eb52285f0d2e268e10f8edaa2c)
+
+commit ebba1d59b5e4bc11cbdfcda4e480639f7d9e1498
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jul 30 14:20:56 2015 +0100
+
+    osd/OSDMap: handle incrementals that modify+del pool
+    
+    Because new_pools was processed after old_pools, if something
+    was modified and then deleted in the same incremental
+    map, then the resulting state would be a pool in the map
+    that had no entry in pool_name.
+    
+    Fixes: #12429
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 8d4932e721e32a33c1bada54e11d4ecccd868618)
+
+commit 31ff7d14e801ef7b7dff3b050094f1eceacf75dd
+Author: renhwztetecs <rhwlyw@163.com>
+Date:   Tue Jul 21 18:55:00 2015 +0800
+
+    Update OSDMonitor.cc
+    
+    OSDMonitor::get_pools_health(), s/objects/bytes/
+    
+    Fixes: #12402
+    Signed-off-by: ren.huanwen@zte.com.cn
+    (cherry picked from commit 7fc13c9d6b9a4962d7640240416105d8f558d600)
+
+commit ea0d67c482659dfb8c08055a77f400d0689866e0
+Author: huangjun <hjwsm1989@gmail.com>
+Date:   Mon Jul 20 10:50:20 2015 +0800
+
+      mon/PGMonitor: avoid uint64_t overflow when checking pool 'target/max' status.
+      Fixes: #12401
+    
+      Signed-off-by: huangjun <hjwsm1989@gmail.com>
+    
+    (cherry picked from commit f84e6b8157e78fd05cd1c8ac957c92df17e6ba06)
+
+commit 6b42b06239426e5ec54dfaa82f48abca56f824ca
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Tue Jul 21 11:20:53 2015 +0100
+
+    mon: OSDMonitor: fix hex output on 'osd reweight'
+    
+    Fixes: #12251
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit 787fa80c2746fde44ac0583ff7995ec5be9a672d)
+
+commit 5445461ba08fa5b2e12a951a1432916224d3f668
+Author: Xinze Chi <xmdxcxz@gmail.com>
+Date:   Fri Jul 3 18:27:13 2015 +0800
+
+    mon/PGMonitor: bug fix pg monitor get crush rule
+    
+    when some rules have been deleted before, the index in array of crush->rules
+    is not always equals to crush_ruleset of pool.
+    
+    Fixes: #12210
+    Reported-by: Ning Yao <zay11022@gmail.com>
+    Signed-off-by: Xinze Chi <xmdxcxz@gmail.com>
+    (cherry picked from commit 498793393c81c0a8e37911237969fba495a3a183)
+
+commit c6f37807d04af66fbed24e75776aed1b44241150
+Author: Ketor Meng <d.ketor@gmail.com>
+Date:   Tue May 26 18:50:17 2015 +0800
+
+    Mutex: fix leak of pthread_mutexattr
+    
+    Need pthread_mutexattr_destroy after pthread_mutexattr_init
+    
+    Fixes: #111762
+    Signed-off-by: Ketor Meng <d.ketor@gmail.com>
+    (cherry picked from commit 2b23327b3aa8d96341d501a5555195ca1bc0de8f)
+
+commit 601eadbf707597d9a1572efbdf86b22ff6813602
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jul 9 11:51:13 2015 +0200
+
+    ceph.spec.in: drop SUSE-specific %py_requires macro
+    
+    %py_requires expands to
+    
+    BuildRequires: /usr/bin/python
+    PreReq: python = 2.7
+    
+    The BuildRequires: is already provided, and the PreReq is wrong because
+    e.g. SLE11-SP3 (a platform we are trying to support) has Python 2.6.
+    
+    http://tracker.ceph.com/issues/12351 Fixes: #12351
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit cbf6c7b46b42e163950bf7956e5b7565fca961c9)
+    
+    Conflicts:
+    	ceph.spec.in
+                in master, the %py_requires is in the python-rados package,
+                while in firefly it is in the python-ceph package
+
+commit a128ce37038ef6e44c8d7bd082ba196583974ef7
+Author: wuxingyi <wuxingyi2015@outlook.com>
+Date:   Wed Mar 11 17:34:40 2015 +0800
+
+    rgw/logrotate.conf: Rename service name
+    
+    The service name for ceph rados gateway was changed to "ceph-radosgw",
+    the previous version of service name "radosgw" would cause a failed reload,
+    and finally make it impossible to write any log data to the log file.
+    
+    Signed-off-by: wuxingyi <wuxingyi2015@outlook.com>
+    (cherry picked from commit 9df3f798179481fe8ae6ae873dcb793de7d8f367)
+
+commit 46e5deabebeeb047783cfc980aa28a3df46eeeca
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri May 15 13:05:40 2015 -0700
+
+    OSDMonitor: disallow ec pools as tiers
+    
+    Fixes: 11650
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 11b7801bb57cb25cd2d26d58722d49691747725b)
+    
+    Conflicts:
+    	qa/workunits/cephtool/test.sh
+               no "# make sure we can't clobber snapshot state" tests in firefly
+    	src/mon/OSDMonitor.cc
+               no tp->removed_snaps.empty() in firefly
+
+commit 37d16a9e572580eeae86a2bae6d4ddd0299fb833
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Thu Jul 9 13:32:03 2015 +0800
+
+    UnittestBuffer: Add bufferlist zero test case
+    
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit 577acf665948e531e50d36f5780ea19351a5bf65)
+
+commit 4443acdbef1148e0261bce25f7d7a3433e09cecc
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Thu Jul 9 13:42:42 2015 +0800
+
+    buffer: Fix bufferlist::zero bug with special case
+    
+    Fixes: #12252
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit 43f583d160ccaf879eaf0f3020e77860cf8d1df0)
+
+commit d52187019d321fe8a2dc54fe8a67a5139c310db1
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Wed May 27 22:28:49 2015 +0100
+
+    mon: MonitorDBStore: get_next_key() only if prefix matches
+    
+    get_next_key() had a bug in which we would always return the first key
+    from the iterator, regardless of whether its prefix had been specified
+    to the iterator.
+    
+    Fixes: #11786
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit 2cc7aee1abe40453093306c8fef2312b650dff5d)
+
+commit 2f35a415dcd31683151defb0ddb1a6d5696c7051
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Fri Jun 12 19:21:10 2015 +0100
+
+    mon: PaxosService: call post_refresh() instead of post_paxos_update()
+    
+    Whenever the monitor finishes committing a proposal, we call
+    Monitor::refresh_from_paxos() to nudge the services to refresh.  Once
+    all services have refreshed, we would then call each services
+    post_paxos_update().
+    
+    However, due to an unfortunate, non-critical bug, some services (mainly
+    the LogMonitor) could have messages pending in their
+    'waiting_for_finished_proposal' callback queue [1], and we need to nudge
+    those callbacks.
+    
+    This patch adds a new step during the refresh phase: instead of calling
+    directly the service's post_paxos_update(), we introduce a
+    PaxosService::post_refresh() which will call the services
+    post_paxos_update() function first and then nudge those callbacks when
+    appropriate.
+    
+    [1] - Given the monitor will send MLog messages to itself, and given the
+    service is not readable before its initial state is proposed and
+    committed, some of the initial MLog's would be stuck waiting for the
+    proposal to finish.  However, by design, we only nudge those message's
+    callbacks when an election finishes or, if the leader, when the proposal
+    finishes.  On peons, however, we would only nudge those callbacks if an
+    election happened to be triggered, hence the need for an alternate path
+    to retry any message waiting for the initial proposal to finish.
+    
+    Fixes: #11470
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit 1551ebb63238073d2fd30201e6b656a8988e958c)
+
+commit b8e3f6e190809febf80af66415862e7c7e415214
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Jul 24 15:38:18 2015 -0700
+
+    Log::reopen_log_file: take m_flush_mutex
+    
+    Otherwise, _flush() might continue to write to m_fd after it's closed.
+    This might cause log data to go to a data object if the filestore then
+    reuses the fd during that time.
+    
+    Fixes: #12465
+    Backport: firefly, hammer
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 8778ab3a1ced7fab07662248af0c773df759653d)
+    
+      - modified to drop m_flush_mutex_holder, which isn't present in firefly
+
+commit abe6c03d411f5f1d277f910ae87123309fa95d4a
+Merge: 5df7273 9614e4d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Jul 25 13:51:43 2015 +0200
+
+    Merge pull request #4583 from SUSE/wip-11542-firefly
+    
+    rgw: Change variable length array of std::strings (not legal in C++) to std::vector<std::string>
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 5df727379b63abb25421a32c95fdda9a0b62eeac
+Merge: 2a77154 8c573c8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Jul 25 13:50:06 2015 +0200
+
+    Merge pull request #4584 from SUSE/wip-8911-firefly
+    
+    rgw: doesn't return 'x-timestamp' in header which is used by 'View Details' of OpenStack
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 2a77154b4abcd4eebc8d1a315f7beeeabaf2213b
+Merge: 64c3a64 d78e31f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Jul 25 13:48:31 2015 +0200
+
+    Merge pull request #4642 from SUSE/wip-11323-firefly
+    
+    rgw: quota not respected in POST object
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 64c3a645c4dba1846532997758c5d7c3a42760b2
+Merge: f11df3c 177ef55
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Jul 25 13:24:39 2015 +0200
+
+    Merge pull request #4641 from SUSE/wip-11149-firefly
+    
+    rgw: delete bucket does not remove .bucket.meta file
+    
+    Reviewed-by: Yahuda Sadeh <yehuda@redhat.com>
+
+commit f11df3c7819a5f8dd5336be7dbe43e0f21dbb394
+Merge: 090bd12 2ee6e9a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Jul 25 13:21:31 2015 +0200
+
+    Merge pull request #5170 from SUSE/wip-12039-firefly
+    
+    rgw: Swift API: response for PUT on /container does not contain the mandatory Content-Length header when FCGI is used
+    
+    Reviewed-by: Yahuda Sadeh <yehuda@redhat.com>
+
+commit 090bd126e15acacc46bce3271d2aa0cfa889b499
+Merge: 94060b7 2d5d8da
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Jul 25 13:20:15 2015 +0200
+
+    Merge pull request #4762 from SUSE/wip-11754-firefly
+    
+    rgw: Object copy bug
+    
+    Reviewed-by: Yahuda Sadeh <yehuda@redhat.com>
+
+commit 94060b78a2e9b38422a951664dd4f37a6f52316f
+Merge: b9f45fe 3223bd6
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri Jul 24 11:42:58 2015 -0700
+
+    Merge pull request #5171 from SUSE/wip-12240-firefly
+    
+    librbd: internal.cc: 1967: FAILED assert(watchers.size() == 1)
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit b9f45fe7816da1ce1fcbfd2a60fe370abe0f1848
+Merge: 57bbc8e 9845258
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri Jul 24 11:26:23 2015 -0700
+
+    Merge pull request #4639 from SUSE/wip-7385-firefly
+    
+    librbd: Objectcacher setting max object counts too low
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 8c02376bd58d463f742966b67fa075a59b5f4269
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jul 24 18:50:03 2015 +0200
+
+    TestPGLog: fix invalid proc_replica_log test case
+    
+    If 1'3 is non-divergent, no osd could correctly have created 2'3.  Also,
+    proc_replica_log does not add the extra entries from the auth log to the
+    missing set, that happens later on in activate.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 1333b676dba4ed4b5db854fe935bdadbf9f21d08)
+    
+    Conflicts:
+        src/test/osd/TestPGLog.cc
+           "sjustm: set_hash() method just does hash = and then rebuilds a cache,
+            but the cache does not exist in firefly so hash = will do fine"
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 4a709a450987cc90126b11a20c1c762d4fbfcea9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 2 10:33:35 2015 -0400
+
+    tests: verify librbd blocking aio code path
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4cf41486e9c9e1efcb863960a8f3e0326ffca7e5)
+
+commit 4501e90d9b667310a222c3616a7b1f1fb382f997
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 1 22:56:11 2015 -0400
+
+    librbd: new rbd_non_blocking_aio config option
+    
+    Setting this option to false reverts librbd to legacy behavior
+    where AIO operations could potentially block.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 769cad12716b85d87eacc1069dd9f5c21cad3915)
+    
+    Conflicts:
+    	src/common/config_opts.h: trivial resolution
+    	src/librbd/librbd.cc: trivial resolution
+
+commit 68863968dd734fc45557dca100c8a0a5060cbd8b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 9 20:34:28 2015 -0400
+
+    PendingReleaseNotes: document changes to librbd's aio_read methods
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 861c20bb2cc05a13a45b0e2368c591212e4eda72
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 21:55:36 2015 -0400
+
+    tests: update librbd AIO tests to remove result code
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    
+    Conflicts:
+    	src/test/librbd/test_librbd.cc: trivial resolution
+
+commit 4e7cf6115798dd339efe41a528aac148be640b29
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 9 13:33:09 2015 -0400
+
+    librbd: AioRequest::send no longer returns a result
+    
+    The librados calls used by AioRequest::send should always return
+    zero unless there is a bug.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c77bce3311ab62892eb8c1d883263ba7ed663b20)
+    
+    Conflicts:
+    	src/librbd/AioRequest.cc: trivial resolution
+    	src/librbd/AioRequest.h: trivial resolution
+    	src/librbd/internal.cc: trivial resolution
+
+commit 0ccf47bf9f070ff7fe4e1dd3d3c3e3daa1f621c4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 21:37:50 2015 -0400
+
+    librbd: internal AIO methods no longer return result
+    
+    All failures should be returned via the AioCompletion.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9ab42d613128ab08c688ddbea93df4c95068b9cd)
+    
+    Conflicts:
+    	src/librbd/AioRequest.cc: trivial resolution
+    	src/librbd/internal.cc: trivial resolution
+    	src/librbd/internal.h: trivial resolution
+
+commit e48040413849c5d3a577addff644502858ac5a66
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 21:48:21 2015 -0400
+
+    Throttle: added pending_error method to SimpleThrottle
+    
+    Allow the client of SimpleThrottle to detect an async error
+    so that it can exit early.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b88b88c5df91325fb713c2031a56bffe421268e0)
+
+commit 8ec341ee8f256e8ab91a931f6005e1ebd4b1db60
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 20:18:50 2015 -0400
+
+    librbd: add new fail method to AioCompletion
+    
+    Helper method to handle passing fatal errors generated within
+    librbd (not from the OSDs) back to the client.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6d1d0c867855a96bee4c13a0c0a39a0e002ccd12)
+    
+    Conflicts:
+    	src/librbd/AioCompletion.cc: trivial resolution
+    	src/librbd/AioCompletion.h: trivial resolution
+
+commit eed8ce258d116e837d8777f246f31c21a03ceb74
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 19:06:52 2015 -0400
+
+    librbd: avoid blocking AIO API methods
+    
+    Enqueue all AIO API methods within the new librbd thread pool to
+    reduce the possibility of any blocking operations. To maintain
+    backwards compatibility with the legacy return codes of the API's
+    AIO methods, it's still possible to block attempting to acquire
+    the snap_lock.
+    
+    Fixes: #11056
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3a7b5e30efdb21aa1a0aeb68a5d02a1ac2a5faf3)
+    
+    Conflicts:
+    	src/librbd/librbd.cc: trivial resolution
+
+commit 220a23e8d99882838626d5647f9c055eefeb28c6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 17:24:08 2015 -0400
+
+    librbd: add task pool / work queue for AIO requests
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit afb896d91f886b647baf38f7ec94cc3739f6d2a9)
+    
+    Conflicts:
+    	src/common/config_opts.h: trivial resolution
+    	src/librbd/ImageCtx.cc: trivial resolution
+    	src/librbd/ImageCtx.h: trivial resolution
+    	src/librbd/internal.cc: trivial resolution
+
+commit 3292bd26127f5beb69ac9c5fbcdea7efe002fd40
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 11 17:05:49 2015 -0400
+
+    WorkQueue: added virtual destructor
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b3f5a75332c058816dc39b71e9d2b36e752159f4)
+
+commit 9bf970c1d956c94bc420d874eb83db1f16cf5d44
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 16:46:34 2015 -0400
+
+    WorkQueue: add new ContextWQ work queue
+    
+    The queue holds a collection of Context pointers that will
+    be completed by the thread pool.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 24a33e977f7b71962adeeb48f75d488a76e70fa9)
+    
+    Conflicts:
+    	src/common/WorkQueue.h: trivial resolution
+
+commit 94aa947b33083bcfe3cb61f41bfdaea7df605cbb
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 5 14:21:08 2014 -0800
+
+    common/ceph_context: don't import std namespace
+    
+    This was broken by 7fed5dee4f96a83d1d6914f6fc0895bba2d15b99
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9029813b2784fe71879b1b7684855415b4c7566b)
+
+commit b91e594401ab75987a370f6dee0ee8d212189c9f
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Mon Dec 1 23:54:16 2014 +0800
+
+    CephContext: Add AssociatedSingletonObject to allow CephContext's singleton
+    
+    If some objects associated to CephContext want to create a singleton object,
+    it can inherit AssociatedSingletonObject and implement destruction to get notified.
+    
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit 7fed5dee4f96a83d1d6914f6fc0895bba2d15b99)
+
+commit f122788b3592da516cdb9f6abdc3383a2e7424a2
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jan 16 07:54:22 2015 -0800
+
+    mon/OSDMonitor: require mon_allow_pool_delete = true to remove pools
+    
+    This is a simple safety check.  Since we default to true it is currently
+    opt-in.
+    
+    Backport: giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6b982e4cc00f9f201d7fbffa0282f8f3295f2309)
+    
+    Conflicts:
+         src/test/mon/osd-pool-create.sh
+    
+         Changed $CEPH_MON to 127.0.0.1 -- the CEPH_MON was introduced after
+         firefly to allow tests to run in parallel. Back in firefly all tests
+         use the same port because 127.0.0.1 was hardcoded. We can't
+         conveniently backport all that's necessary for tests to run in
+         parallel, therefore we keep the 127.0.0.1 hardcoded.
+    
+    Conflicts:
+    	src/test/mon/osd-pool-create.sh
+                TEST_no_pool_delete() follows a different test than in master
+
+commit 57bbc8e630e6611d1ab74b9756c96daf0e91be6b
+Merge: c0a601f 3f2fb8a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 24 10:49:50 2015 +0200
+
+    Merge pull request #5051 from SUSE/wip-12016-firefly
+    
+    osd: cache full mode still skips young objects
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit c0a601ff5be45c0630a8659994107672558a982a
+Merge: 71174a5 ef648e5
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jul 22 11:36:47 2015 -0400
+
+    Merge pull request #5307 from dachary/wip-release-notes-firefly
+    
+    backport v0.80.10 release notes to firefly
+
+commit 5559a5fe0f593c0e7af47c6d8a8359dbcbfd1f08
+Author: Guang Yang <yguang@yahoo-inc.com>
+Date:   Mon Sep 15 11:41:06 2014 +0000
+
+    For pgls OP, get/put budget on per list session basis, instead of per OP basis, which could lead to deadlock.
+    
+    Signed-off-by: Guang Yang (yguang@yahoo-inc.com)
+    (cherry picked from commit 0f884fdb31a26f241401ce2a9329dc0f2c4eccc7)
+    
+    Conflicts:
+    	src/librados/IoCtxImpl.cc
+                In firefly, return value of objecter->pg_read() is not assigned to c->tid.
+    	src/osdc/Objecter.cc
+    	src/osdc/Objecter.h
+                There is no _op_submit_with_budget() function in firefly.
+                There is no Objecter::_finish_op() function in firefly.
+                In firefly, _take_op_budget() is called take_op_budget().
+
+commit 71174a53c9a3a2b54b947da7e1425ff36fd44e37
+Merge: f7a085f 989e047
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jul 22 10:20:32 2015 +0200
+
+    Merge pull request #4867 from SUSE/wip-11878-firefly
+    
+    mon: Clock skew causes missing summary and confuses Calamari
+    
+    Reviewed-by: Joao Eduardo Luis <joao@suse.de>
+
+commit 989e04799be567ee93369a87b90405833c3fe8d2
+Author: Thorsten Behrens <tbehrens@suse.com>
+Date:   Fri Feb 6 01:26:40 2015 +0100
+
+    Always provide summary for non-healthy cluster.
+    
+    This fixes a problem, wherein calamari does not provide
+    popup drill-downs for warnings or errors, should the summary
+    be missing.
+    
+    Calamari gets health info from /api/v1/cluster/$FSID/health.
+    If the data here has a summary field, this summary is provided
+    in a popup window:
+    
+      /api/v1/cluster/$FSID/health is populated (ultimately) with
+      status obtained via librados python bindings from the ceph
+      cluster. In the case where there's clock skew, the summary
+      field supplied by the ceph cluster is empty.
+    
+    No summary field, no popup window with more health details.
+    
+    Signed-off-by: Thorsten Behrens <tbehrens@suse.com>
+    (cherry picked from commit eaf6e0cf48488fe604d0ef0db164d44948d4e8d4)
+    
+    Conflicts:
+    	src/mon/Monitor.cc
+                firefly needs ss
+
+commit f7a085f04c544895bab3d79569276574a96636eb
+Merge: 2f1dd10 d296120
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Tue Jul 21 13:31:33 2015 -0700
+
+    Merge pull request #5062 from SUSE/wip-11699-firefly
+    
+    objecter: resend linger ops on split
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit ef648e592404e8677df91f218c4400caad4034b6
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jul 21 12:54:19 2015 -0400
+
+    doc: v0.80.10 changelog
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b3a3e0e509eb9530994723ed9d68f85b8434a22d)
+
+commit df837e5cc30cf08ade26df6e42ca402fc628a9bb
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jul 21 10:55:18 2015 -0400
+
+    doc/release-notes: v0.80.10
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2a61b611987d169d5a501d77fbd545068550c28c)
+    
+    Conflicts:
+    	doc/release-notes.rst
+
+commit 90cddd425ca8f1731fff0b8fb4b7e02a96048f92
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Jul 14 16:15:44 2015 +0200
+
+    doc/release-notes: update notes for v0.80.10
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 1231ae09d675f2ee9e76a04b91f771d73a2fb346)
+    
+    Conflicts:
+    	doc/release-notes.rst
+
+commit 2f1dd10b71d29a647aa2bab4a51fecc0501d6741
+Merge: 0caf6b3 1d2af55
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 21 13:03:07 2015 +0200
+
+    Merge pull request #5129 from SUSE/wip-12202-firefly
+    
+    mon: ceph fails to compile with boost 1.58
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 0caf6b32849bd673d83e96f0eae19bb6d7e423ae
+Merge: 3010f23 6793d50
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 21 12:51:10 2015 +0200
+
+    Merge pull request #5044 from SUSE/wip-12011-firefly
+    
+    osd: PG stuck with remapped
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 3010f23fae10d37ff78b8d7d5049ccba798bbe6b
+Merge: ad58a32 fcac7ab
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 21 12:45:22 2015 +0200
+
+    Merge pull request #5039 from SUSE/wip-11961-firefly
+    
+    osd: Cleanup boost optionals for boost 1.56
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit ad58a32292eee39905ae7bacc8fc52a52c2d08fc
+Merge: 13946e9 a62c720
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 21 12:39:34 2015 +0200
+
+    Merge pull request #5037 from SUSE/wip-11875-firefly
+    
+    tools: ceph-post-file fails on rhel7
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 13946e9c18bbeb209458a3faac6e687ff380ac6e
+Merge: bde35c1 90a1a75
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 21 12:34:51 2015 +0200
+
+    Merge pull request #4771 from SUSE/wip-11759-firefly
+    
+    tools: ceph-disk: support NVMe device partitions
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit bde35c11acd34c76d704b41292d167c3fc4cea53
+Merge: 9e11564 10d85e1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 21 12:22:52 2015 +0200
+
+    Merge pull request #4636 from SUSE/wip-11143-firefly
+    
+    tools:  src/ceph-disk : disk zap sgdisk invocation
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 9e11564b36a42b605f256943c7236b91a736fceb
+Merge: 8638443 424289d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 21 12:20:50 2015 +0200
+
+    Merge pull request #4635 from SUSE/wip-7387-firefly
+    
+    common: Malformed JSON command output when non-ASCII strings are present
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 8638443eac92ac80b4c57271f7350d94e445abf5
+Merge: 4c37d3d 8372f1d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 21 12:17:43 2015 +0200
+
+    Merge pull request #4632 from SUSE/wip-11507-firefly
+    
+    osd: object creation by write cannot use an offset on an erasure coded pool
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 4c37d3d99f660c835d2f73e8f08b6ef170c0c7b7
+Merge: 8f11ab7 9090967
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 21 12:12:36 2015 +0200
+
+    Merge pull request #4630 from SUSE/wip-10983-firefly
+    
+    tools: use a new disk as journal disk,ceph-disk prepare fail
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 8f11ab7ce27cab43145a2821f3996d3103cf7870
+Merge: 64a21b7 b1ff739
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 21 12:08:26 2015 +0200
+
+    Merge pull request #4597 from xinxinsh/wip-11026-firefly
+    
+    objecter: can get stuck in redirect loop if osdmap epoch == last_force_op_resend
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 64a21b71526c51f8e787b365039de2b0cb0a285b
+Merge: 82e8359 2cb2333
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 21 12:04:39 2015 +0200
+
+    Merge pull request #4582 from SUSE/wip-11543-firefly
+    
+    tools: ceph_argparse_flag has no regular 3rd parameter
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit bba50ce8f227af29d559b486274871bb3999fb24
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Apr 9 16:28:23 2015 -0700
+
+    TestPGLog: fix noop log proc_replica_log test case
+    
+    Need to fill in log.head, olog.head.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 1e5b22053381580708af29a1536f1e106c9b479c)
+
+commit fdff8ce6c996cda7b3966d20c24b20ff545e468a
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Apr 9 15:01:41 2015 -0700
+
+    TestPGLog: add test for 11358
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit b61e5aef15d7448d24b713d13dce77697bc0af74)
+    
+    Conflicts:
+    	src/test/osd/TestPGLog.cc
+                leave out all the other tests that have been
+                added to master since firefly
+
+commit 65028b6304235ba5fa54d14805028db1a032e5a0
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Apr 9 15:01:29 2015 -0700
+
+    PGLog::proc_replica_log: handle split out overlapping entries
+    
+    See the comment, if the overlapping entries between olog.head
+    and log.tail were split out, last_update should be log.tail.
+    
+    Fixes: 11358
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 6561e0d955c4563a8505b717de41f5bd89119f63)
+
+commit 82e8359783f056cce5f7ede2dabd0cc61d278eab
+Merge: 741f0c2 b061140
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri Jul 17 16:10:43 2015 -0700
+
+    Merge pull request #4633 from SUSE/wip-11535-firefly
+    
+    Client admin socket leaks file descriptors
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit da78889d76f641dfcbb80164be32b85e880bb97f
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Mar 18 13:49:20 2015 -0700
+
+    os/chain_xattr: handle read on chnk-aligned xattr
+    
+    If we wrote an xattr that was a multiple of a chunk, we will try to read
+    the next chunk and get ENODATA.  If that happens bail out of the loop and
+    assume we've read the whole thing.
+    
+    Backport: hammer, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8614dcebf384b74b283cd352001a65fa26ba925c)
+    
+    Conflicts:
+        src/test/objectstore/chain_xattr.cc
+            added #define FILENAME
+
+commit 4e7de5b5f0e32d1183e2a0490d65e4e01490d942
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jun 25 14:31:03 2015 -0700
+
+    rgw: error out if frontend did not send all data
+    
+    Fixes: #11851
+    The civetweb mg_write() doesn't return error when it can't flush all data
+    to the user, it just sends the total number of bytes written. Modified the
+    client io to return total number of bytes and return an error if didn't
+    send anything.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit daa679c3dd3770a6d5421e2cc9a36924f4550439)
+
+commit f0d2429f764aaa756fc8002309545b02dc45c199
+Author: Thorsten Behrens <tbehrens@suse.com>
+Date:   Wed Dec 10 11:53:43 2014 +0100
+
+    Unconditionally chown rados log file.
+    
+    This fixes bnc#905047 (in a somewhat ad-hoc way). Sadly the log
+    file gets created from several places, so its existence does not
+    mean init-radosgw had actually run.
+    
+    Signed-off-by: Thorsten Behrens <tbehrens@suse.com>
+    (cherry picked from commit 41611ea27a42d6221c418613d24dce34d988f2e6)
+
+commit 68508e28adffae64a52ea4c0046fbe8537e0d093
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jun 25 22:37:52 2015 +0200
+
+    ceph.spec.in: use _udevrulesdir to eliminate conditionals
+    
+    The conditionals governing where 50-rbd.rules is installed were not doing the
+    right thing on SUSE distros.
+    
+    Start using the %_udevrulesdir RPM macro, while taking care that it is defined
+    and set to the right value. Use it to eliminate some conditionals around other
+    udev rules files as well.
+    
+    http://tracker.ceph.com/issues/12166 Fixes: #12166
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 8aa758ee7a14e93ded1d55b4aca7a54aa47d7174)
+    
+    Conflicts:
+    	ceph.spec.in
+                master spec file has diverged significantly from firefly
+
+commit bd25136d45d32b8d419fbe60e39572038e33cc59
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Jul 13 18:12:01 2015 +0200
+
+    ceph.spec.in: do not run fdupes, even on SLE/openSUSE
+    
+    In openSUSE there is a policy to use %fdupes in the spec file if RPMLINT
+    complains about duplicate files wasting space in the filesystem.
+    
+    However, RPMLINT is not so complaining, so drop fdupes.
+    
+    http://tracker.ceph.com/issues/12301 Fixes: #12301
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 53072b9019caf72e0313b2804ea174237ed7da33)
+
+commit 2544a6fc59a86d3f02dcd67d3c84a72d7a1100a0
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Jun 16 18:27:20 2015 +0200
+
+    ceph.spec.in: python-argparse only in Python 2.6
+    
+    argparse is a widely-used Python module for parsing command-line arguments.
+    Ceph makes heavy use of Python scripts, both in the build environment and on
+    cluster nodes and clients.
+    
+    Until Python 2.6, argparse was distributed separately from Python proper.
+    As of 2.7 it is part of the Python standard library.
+    
+    Although the python package in a given distro may or may not Provide:
+    python-argparse, this cannot be relied upon.
+    
+    Therefore, this commit puts appropriate conditionals around Requires:
+    python-argparse and BuildRequires: python-argparse. It does so for Red
+    Hat/CentOS and SUSE only, because the last Fedora version with Python 2.6
+    was Fedora 13, which is EOL.
+    
+    argparse is required by both the ceph and ceph-common packages, but since ceph
+    requires ceph-common, the argparse Requires and BuildRequires need only appear
+    once, under ceph-common.
+    
+    http://tracker.ceph.com/issues/12034 Fixes: #12034
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 23171c952997d104cfa6b5700ec0bb658a8e0915)
+    
+    Conflicts:
+    	ceph.spec.in
+
+commit 54264210f4ebec23b08dd6712e09aea49543b52b
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Aug 12 15:24:26 2014 -0700
+
+    ReplicatedPG::maybe_handle_cache: do not skip promote for write_ordered
+    
+    We cannot redirect a RW ordered read.
+    
+    Fixes: #9064
+    Introduced: 0ed3adc1e0a74bf9548d1d956aece11f019afee0
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 24aeca9c439474b079d23600d8d42c0af5d9ce5c)
+
+commit 7e2526784203b0f1bce08869aa7b1fda9c5eedd9
+Author: Zhiqiang Wang <wonzhq@hotmail.com>
+Date:   Mon Jul 28 14:06:06 2014 +0800
+
+    osd: promotion on 2nd read for cache tiering
+    
+    http://tracker.ceph.com/issues/8674 Fixes: #8674
+    
+    Signed-off-by: Zhiqiang Wang <wonzhq@hotmail.com>
+    (cherry picked from commit 0ed3adc1e0a74bf9548d1d956aece11f019afee0)
+
+commit 66f61cd9ae105948f653fd888812df270ff1e832
+Author: Signed-off-by: Zhiqiang Wang <wonzhq@hotmail.com>
+Date:   Thu Jul 31 15:49:44 2014 -0700
+
+    ceph_test_rados_api_tier: test promote-on-second-read behavior
+    
+    http://tracker.ceph.com/issues/8674 References: #8674
+    
+    Signed-off-by: Zhiqiang Wang <wonzhq@hotmail.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1417eded65f40bdb2a49c8252fcfffa383a7b965)
+
+commit a8f3d6e1f1f186cbe2299566a575bf5a40500227
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 11 11:31:22 2014 -0700
+
+    osd/osd_types: be pedantic about encoding last_force_op_resend without feature bit
+    
+    The addition of the value is completely backward compatible, but if the
+    mon feature bits don't match it can cause monitor scrub noice (due to the
+    parallel OSDMap encoding).  Avoid that by only adding the new field if the
+    feature (which was added 2 patches after the encoding, see
+    3152faf79f498a723ae0fe44301ccb21b15a96ab and
+    45e79a17a932192995f8328ae9f6e8a2a6348d10.
+    
+    Fixes: #8815
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c93da05bd3dd6b73e8cfa182c6349359b042717c)
+    
+    Conflicts:
+        src/osd/osd_types.cc
+            drop encode_compat variable which is otherwise unused in this context
+
+commit dc128758bb68a50992819f759b5684ff656406ba
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri May 15 22:50:36 2015 +0800
+
+    mon: always reply mdsbeacon
+    
+    the MDS (Beacon) is always expecting the reply for the mdsbeacon messages from
+    the lead mon, and it uses the delay as a metric for the laggy-ness of the
+    Beacon. when it comes to the MDSMonitor on a peon, it will remove the route
+    session at seeing a reply (route message) from leader, so a reply to
+    mdsbeacon will stop the peon from resending the mdsbeacon request to the
+    leader.
+    
+    if the MDSMonitor re-forwards the unreplied requests after they are
+    outdated, there are chances that the requests reflecting old and even wrong
+    state of the MDSs mislead the lead monitor. for example, the MDSs which sent
+    the outdated messages could be dead.
+    
+    Fixes: #11590
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b3555e9c328633c9e1fbc27d652c004b30535e5b)
+
+commit da720d7d870a53faeacc169796083b4eba4b9406
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jun 2 23:20:21 2015 -0700
+
+    mon/MDSMonitor: rename labels to a better name
+    
+    * s/ignore/reply/
+    * s/out/ignore/
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit f00ecb8b3df73ce6337985bc6d43bce5143ee537)
+    
+    Conflicts:
+        src/mon/MDSMonitor.cc
+            Do not compare known daemon health with m->get_health()
+
+commit 329da091d6e8444722b312ff94c7093f81939f13
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jun 2 12:55:06 2015 +0800
+
+    mon: send no_reply() to peon to drop ignored mdsbeacon
+    
+    so the peon can remove the ignored mdsbeacon request from the
+    routed_requets at seeing this reply, and hence no longer resend the
+    request.
+    
+    Fixes: #11590
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 72a37b3a8e145d8522ea67fc14ce2c5510b6852b)
+
+commit 508332f53075a99295b9edb683ed343d8f59ca4f
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jun 2 12:22:26 2015 +0800
+
+    mon: remove unnecessary error handling
+    
+    msg.get_session() should always return a non-zero pointer in
+    Monitor.dispatch()
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 16e8e2cc82a90c49cd8aa3d0e3acc4694ba659a0)
+    
+    Conflicts:
+        src/mon/Monitor.cc
+            Monitor::_ms_dispatch(Message *m) is bool in firefly
+
+commit 741f0c235228b371486574ba0c3fbf55729e03bd
+Merge: 1623e6c 0a7f2b2
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Jul 9 17:24:04 2015 -0700
+
+    Merge pull request #4631 from SUSE/wip-11069-firefly
+    
+    PGLog: split divergent priors as well
+    
+    Reviewed-by: Samuel Just
+
+commit 3223bd604117b0b302b1e3e235ff9bced8e986bc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 26 09:59:36 2015 -0400
+
+    librbd: assertion failure race condition if watch disconnected
+    
+    It's possible for librbd's watch of the header object to be reset by
+    connection issues just prior to the image being removed.  This will
+    causes an assertion failure which assumes at least one watcher on the
+    image.
+    
+    Fixes: #12176
+    Backport: hammer, firefly
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit af276de4f299960e43761904c043924cec5fef11)
+
+commit 2ee6e9a0cc6513097d6c9f9e40787a1c60c09796
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Sun Mar 8 22:45:34 2015 +0100
+
+    rgw: send Content-Length in response for HEAD on Swift account.
+    
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 29073d84640b61d0221b2d2ab73c271d60fd13ba)
+
+commit 1b4536aa47fd6d2440fbd30b64b5b46eaa21215a
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Mar 6 13:18:19 2015 +0100
+
+    rgw: send Content-Length in response for DELETE on Swift container.
+    
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit d260a93405a7a34a899f716bd9c4e731baa9ffd0)
+
+commit 177324ced968b254458a450f8e348c96cbd10931
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Mar 6 13:14:26 2015 +0100
+
+    rgw: send Content-Length in response for PUT on Swift container.
+    
+    Fixes: #11036
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 10c1f1aa1e09df5edf5d2f9f9b0273ddbcb384fa)
+
+commit 0a580c4a23b985ede28cdfc751eed2a368473e09
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Feb 27 15:23:16 2015 +0100
+
+    rgw: send Content-Length in response for GET on Swift container.
+    
+    Fixes: #10971
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 5a64fb5430b6b42174bf53a5910d9435043c1380)
+
+commit 84525806a057054dca295f860e0a31d6936d4985
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Mar 6 12:42:52 2015 +0100
+
+    rgw: enable end_header() to handle proposal of Content-Length.
+    
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit f98fd4dfda7a105d621f99c2b48eb9ab9b45d979)
+
+commit 1d2af55f8c69b4a382eb516d8f90a8316fd3bc02
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri May 8 15:21:20 2015 +0800
+
+    mon: remove unused variable
+    
+    * as a side effect, this change silences
+      http://tracker.ceph.com/issues/11576
+    
+    Fixes: #11576
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit e7b196a4a091c0ea258866559ba06e7ed0cc4247)
+
+commit d296120c144a62794483338fd696ca8089778e84
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Oct 20 13:29:13 2014 -0700
+
+    Objecter: resend linger ops on any interval change
+    
+    Watch/notify ops need to be resent after a pg split occurs, as well as
+    a few other circumstances that the existing objecter checks did not
+    catch.
+    
+    Refactor the check the OSD uses for this to add a version taking the
+    more basic types instead of the whole OSD map, and stash the needed
+    info when an op is sent.
+    
+    Fixes: #9806
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit cb9262abd7fd5f0a9f583bd34e4c425a049e56ce)
+    
+    Conflicts:
+    	src/osd/osd_types.cc
+    	src/osdc/Objecter.cc
+                Minor differences.
+
+commit a44b7b4bd870cc72e50a49900cd81fde6d40845b
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed May 7 11:02:16 2014 -0700
+
+    osd_types: factor out is_new_interval from check_new_interval
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 2ee355113ed4374c664909a8cd7dc5dfdc4c9ed7)
+
+commit 3f2fb8a996820843d0287a9337d38d80c7cb616b
+Author: Xinze Chi <xmdxcxz@gmail.com>
+Date:   Wed Nov 5 01:00:01 2014 +0000
+
+    osd: cache pool: flush object ignoring cache min flush age when cache pool is full
+    Signed-off-by: Xinze Chi <xmdxcxz@gmail.com>
+    
+    (cherry picked from commit f4ee9496ed15e7734abd659f5a353a3cde0dcb23)
+
+commit 9969ae5710bc66e1e96125f22593ea1e0ba381af
+Author: Zhiqiang Wang <wonzhq@hotmail.com>
+Date:   Fri Aug 1 16:09:50 2014 +0800
+
+    osd: add local_mtime to struct object_info_t
+    
+    This fixes a bug when the time of the OSDs and clients are not
+    synchronized (especially when client is ahead of OSD), and the cache
+    tier dirty ratio reaches the threshold, the agent skips the flush work
+    because it thinks the object is too young.
+    
+    Signed-off-by: Zhiqiang Wang <wonzhq@hotmail.com>
+    (cherry picked from commit 13b9dc708426c35436ba58388fc54f04f066f03a)
+
+commit 6793d504b5b622a7a6f69ffb30d6822167b557a3
+Author: Guang Yang <yguang@yahoo-inc.com>
+Date:   Mon Oct 13 04:18:45 2014 +0000
+
+    The fix for issue 9614 was not completed, as a result, for those erasure coded PGs with one OSD down, the state was wrongly marked as active+clean+degraded. This patch makes sure the clean flag is not set for such PG.
+    Signed-off-by: Guang Yang <yguang@yahoo-inc.com>
+    
+    (cherry picked from commit 35bbb2d6002b1b435ff0b1df653cc7ab271f9258)
+
+commit fcac7ab6305dc3215fdcb46951fa9ee4cbf68ce3
+Author: William A. Kennington III <william@wkennington.com>
+Date:   Sat Sep 20 22:52:31 2014 -0700
+
+    osd: Cleanup boost optionals
+    
+    Signed-off-by: William A. Kennington III <william@wkennington.com>
+    (cherry picked from commit a53ead14c113047567177630b4906136a2109b65)
+
+commit a62c720dc783b40377dcd34d4e53aa778ac1a84b
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jun 1 15:10:14 2015 -0700
+
+    Makefile: install ceph-post-file keys with mode 600
+    
+    Otherwise ssh (may) prompt for a password.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 106a1c3081d02446aa3d8e13865da0c3393bae90)
+
+commit fc59ca391f639b2ef1b5c9dda202a3d7d718cf25
+Author: Joseph McDonald <joseph.mcdonald@alcatel-lucent.com>
+Date:   Mon Jun 1 15:05:32 2015 -0700
+
+    ceph-post-file: improve check for a source install
+    
+    Signed-off-by: Joseph McDonald <joseph.mcdonald@alcatel-lucent.com>
+    (cherry picked from commit ee170eadcdcb4b54d36a7d474558484de9d917eb)
+
+commit 50bfb354d0648162f94c2659e8b915d69f27bf73
+Author: Joseph McDonald <joseph.mcdonald@alcatel-lucent.com>
+Date:   Mon Jun 1 15:00:39 2015 -0700
+
+    ceph-post-file: behave when sftp doesn't take -i
+    
+    Fixes: #11836
+    Signed-off-by: Joseph McDonald <joseph.mcdonald@alcatel-lucent.com>
+    (cherry picked from commit b84031ed5eaace1222e14d3c4076a3ab1155da96)
+
+commit 1623e6c6517ff6958b343493fce6a8e50a52a042
+Merge: ea6c958 254676b
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Jun 19 15:57:46 2015 -0700
+
+    Merge pull request #4960 from ceph/wip-10911-firefly
+    
+    osd,mon: explicitly specify OSD features in MOSDBoot
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 254676b63b559fa0117520edb5dcbfe067a76747 (refs/remotes/jashan/wip-10911-firefly)
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Feb 18 14:53:04 2015 -0800
+
+    osd,mon: explicitly specify OSD features in MOSDBoot
+    
+    We are using the connection features to populate the features field in the
+    OSDMap, but this is the *intersection* of mon and osd features, not the
+    osd features.  Fix this by explicitly specifying the features in
+    MOSDBoot.
+    
+    Fixes: #10911
+    Backport: giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit bee91548b07c6a28314dddc50fef8b6a2677e774)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+    Minor difference in the MOSDBoot constructor
+
+commit b1ff7396c8e9668c0be1a4023e0ffa292067cf64
+Author: xinxin shu <xinxin.shu@intel.com>
+Date:   Thu May 7 05:49:18 2015 +0800
+
+    osdc: add epoch_t last_force_resend in Op/LingerOp.
+    
+    Using this field record the pg_poo_t::last_force_op_resend to avoid op
+    endless when osd reply with redirect.
+    
+    Fixes: #11026
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit def4fc4a)
+
+commit 90a1a7587fcb567e2cc6e9188fdd56ec1255a881
+Author: islepnev <islepnev@gmail.com>
+Date:   Fri Apr 17 22:33:01 2015 +0300
+
+    ceph-disk: support NVMe device partitions
+    
+    Linux nvme kernel module v0.9 enumerate devices as following:
+    
+    /dev/nvme0 - characted revice
+    /dev/nvme0n1 - whole block device
+    /dev/nvme0n1p1 - first partition
+    /dev/nvme0n1p2 - second partition
+    
+    http://tracker.ceph.com/issues/11612 Fixes: #11612
+    
+    Signed-off-by: Ilja Slepnev <islepnev@gmail.com>
+    (cherry picked from commit 9b62cf254d02d30609793be8b1cb8a94f38891f1)
+    
+    Conflicts:
+    	src/ceph-disk
+
+commit f93eadd793f9f4fded30df5589f98ccfc0e1839f
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue May 12 16:37:56 2015 -0700
+
+    mon: prevent bucket deletion when referenced by a rule
+    
+    If a rule references a bucket with 'take', prevent deletion.
+    
+    Fixes: #11602
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3d591afef90b0601572c748f13faac029d05f5a0)
+
+commit d4ef5566e6b1b2ac73925660a1d627dbb6cc7632
+Author: Anton Aksola <anton.aksola@nebula.fi>
+Date:   Fri Apr 10 13:25:21 2015 +0300
+
+    rgw: always check if token is expired
+    
+    Fixes: #11367
+    
+    Currently token expiration is only checked by the token cache. With PKI
+    tokens no expiration check is done after decoding the token. This causes
+    PKI tokens to be valid indefinitely. UUID tokens are validated by
+    keystone after cache miss so they are not affected by this bug.
+    
+    This commit adds explicit token expiration check to
+    RGWSwift::validate_keystone_token()
+    
+    Signed-off-by: Anton Aksola <anton.aksola@nebula.fi>
+    Reported-by: Riku Lehto <riku.lehto@nexetic.com>
+    (cherry picked from commit 2df069390ea3bbcfbab5022750e89f51d197cc11)
+
+commit 2d5d8da35f627840ee57a8fa071147943b890d0c
+Author: Javier M. Mellid <jmunhoz@igalia.com>
+Date:   Fri May 15 14:22:29 2015 +0200
+
+    rgw: Use attrs from source bucket on copy
+    
+    On copy objects, when bucket source is the same as the destination, use attrs
+    from source bucket.
+    
+    Fixes: #11639
+    
+    Signed-off-by: Javier M. Mellid <jmunhoz@igalia.com>
+    (cherry picked from commit 1dac80df1d4a2364154ed8b404d13609936c257b)
+
+commit d78e31f005756700e15d26d187c6cef6d5d24a7a
+Author: 9seconds <nineseconds@yandex.ru>
+Date:   Wed Apr 1 09:12:06 2015 +0300
+
+    rgw: quota not respected in POST object
+    
+    Signed-off-by: Sergey Arkhipov <nineseconds@yandex.ru>
+    Backport: hammer, firefly
+    Fixes: #11323
+    (cherry picked from commit e76f84e179d2ba8bfc0dc5abf4e620fef14bc8a0)
+    
+    Conflicts:
+    	src/rgw/rgw_op.cc
+            discard the whitespace modification hunks that were creating
+            conflicts and ignore the conflict due to an unrelated cast
+            modification in the context
+
+commit 177ef55d61c9b08783ae746aa521d2535324d6ba
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Fri Apr 3 13:43:31 2015 +0200
+
+    rgw: remove meta file after deleting bucket
+    The meta file is deleted only if the bucket meta data is not synced
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    
+    Fixes: #11149
+    Backport: hammer, firefly
+    (cherry picked from commit dfdc7afb59cc8e32cf8bff55faa09076c853de06)
+
+commit 9845258f184c823d1e48da1dcb97bdde97220bb6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 22 11:27:35 2015 -0400
+
+    librbd: updated cache max objects calculation
+    
+    The previous calculation was based upon the image's object size.
+    Since the cache stores smaller bufferheads, the object size is not
+    a good indicator of cache usage and was resulting in objects being
+    evicted from the cache too often.  Instead, base the max number of
+    objects on the memory load required to store the extra metadata
+    for the objects.
+    
+    Fixes: #7385
+    Backport: firefly, hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0b378942c4f1b79cb65967f2d3466728ca1c8d5b)
+    
+    Conflicts:
+    	src/librbd/ImageCtx.cc
+            because firefly has cct->_conf->rbd_cache_size
+            instead of cache_size
+
+commit 0a7f2b2a859aefebc995ad5c7b699eb117d6fa12
+Author: Nathan Cutler <ncutler@suse.cz>
+Date:   Sun May 10 17:13:22 2015 +0200
+
+    PGLog: split divergent priors as well
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit bbe231a61fc06a1a668e6b831fb8224526721f93)
+    
+    Conflicts:
+        src/osd/PGLog.h
+    
+        The cherry-picked commit did not compile as-is, because the hobject_t
+        class in firefly lacks a get_hash() method, which was added in 6de83d4.
+        To get the patch to compile, I replaced i->second.get_hash() with
+        i->second.hash.
+
+commit 424289dc4ee8fb09dd00d507b5a9d1fecc786937
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri May 8 12:01:16 2015 +0800
+
+    json_sprit: fix the FTBFS on old gcc
+    
+    Fixes: #11574
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 6b68b27146852f057a3373cd04b08cd6917f3eea)
+
+commit 84b00f189554fc37413c990ac4011079bb5cdb60
+Author: Tim Serong <tserong@suse.com>
+Date:   Sat May 2 01:59:53 2015 +1000
+
+    json_spirit: use utf8 intenally when parsing \uHHHH
+    
+    When the python CLI is given non-ASCII characters, it converts them to
+    \uHHHH escapes in JSON.  json_spirit parses these internally into 16 bit
+    characters, which could only work if json_spirit were built to use
+    std::wstring, which it isn't; it's using std::string, so the high byte
+    ends up being zero'd, leaving the low byte which is effectively garbage.
+    
+    This hack^H^H^H^H change makes json_spirit convert to utf8 internally
+    instead, which can be stored just fine inside a std::string.
+    
+    Note that this implementation still assumes \uHHHH escapes are four hex
+    digits, so it'll only cope with characters in the Basic Multilingual
+    Plane.  Still, that's rather a lot more characters than it could cope
+    with before ;)
+    
+    (For characters outside the BMP, Python seems to generate escapes in the
+    form \uHHHHHHHH, i.e. 8 hex digits, which the current implementation
+    doesn't expect to see)
+    
+    Fixes: #7387
+    
+    Signed-off-by: Tim Serong <tserong@suse.com>
+    (cherry picked from commit 8add15b86e7aaef41397ab8fa9e77ee7957eb607)
+    
+    Conflicts:
+    	src/test/mon/osd-pool-create.sh
+    
+            Changed $CEPH_MON to 127.0.0.1 -- the CEPH_MON was introduced after
+            firefly to allow tests to run in parallel. Back in firefly all tests
+            use the same port because 127.0.0.1 was hardcoded. We can't
+            conveniently backport all that's necessary for tests to run in
+            parallel, therefore we keep the 127.0.0.1 hardcoded.
+
+commit 10d85e1126768f6c72f7d2a301ea7ca9ba59f8e0
+Author: Owen Synge <osynge@suse.com>
+Date:   Tue Mar 17 15:41:33 2015 +0100
+
+    Fix "disk zap" sgdisk invocation
+    
+    Fixes #11143
+    
+    If the metadata on the disk is truly invalid, sgdisk would fail to zero
+    it in one go, because --mbrtogpt apparently tried to operate on the
+    metadata it read before executing --zap-all.
+    
+    Splitting this up into two separate invocations to first zap everything
+    and then clear it properly fixes this issue.
+    
+    Based on patch by Lars Marowsky-Bree <lmb@suse.com> in ceph-deploy.
+    Created by Vincent Untz <vuntz@suse.com>
+    
+    Signed-off-by: Owen Synge <osynge@suse.com>
+    Signed-off-by: Thorsten Behrens <tbehrens@suse.com>
+    (cherry picked from commit fdd7f8d83afa25c4e09aaedd90ab93f3b64a677b)
+
+commit b06114088a354f9dbcfc9182643733399c31f2a8
+Author: Jon Bernard <jbernard@tuxion.com>
+Date:   Fri May 8 11:54:06 2015 -0400
+
+    common/admin_socket: close socket descriptor in destructor
+    
+    Long-running processes that do not reuse a single client connection will
+    see accumulating file descriptors as a result of not closing the
+    listening socket.  In this case, eventually the system will reach
+    file-max and subsequent connections will fail.
+    
+    Fixes: #11535
+    
+    Signed-off-by: Jon Bernard <jbernard@tuxion.com>
+    (cherry picked from commit 88fabb1ee6cd3c7d211ca46919055195e32245db)
+
+commit 8372f1d53b847224bf4c61894d18851e84ccd357
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Fri Mar 6 15:54:07 2015 +0800
+
+    osd: refuse to write a new erasure coded object with an offset > 0
+    
+    Even if the offset is properly aligned.
+    
+    http://tracker.ceph.com/issues/11507 Fixes: #11507
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit a4f1256c214ee0e7ebb91ac4ea8655f5d9642dc8)
+
+commit 909096713dcef5dc4fdbb22a9971609554688e23
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 8 15:15:35 2015 +0100
+
+    ceph-disk: more robust parted output parser
+    
+    In some cases, depending on the implementation or the operating system,
+    
+       parted --machine -- /dev/sdh print
+    
+    may contain empty lines. The current parsing code is fragile and highly
+    depends on output details. Replace it with code that basically does the
+    same sanity checks (output not empty, existence of units, existence of
+    the dev entry) but handles the entire output instead of checking line by
+    line.
+    
+    http://tracker.ceph.com/issues/10983 Fixes: #10983
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit f5acf6bb6a342b05897605703d7d9cb7c09714fd)
+
+commit 8c573c8826096d90dc7dfb9fd0126b9983bc15eb
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 30 14:15:47 2014 -0700
+
+    rgw: swift GET / HEAD object returns X-Timestamp field
+    
+    Fixes: #8911
+    Backport: giant, firefly, dumpling
+    Swift clients expect X-Timestamp header, dump it.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5b41d80b7fb9ed96c26801fc42c044191bb18d84)
+
+commit 9614e4dc5be745dab367d17f29a07377e348b2ca
+Author: Daniel J. Hofmann <daniel@trvx.org>
+Date:   Mon May 5 16:06:45 2014 +0200
+
+    Variable length array of std::strings (not legal in C++) changed to std::vector<std::string>
+    
+    Signed-off-by: Daniel J. Hofmann <daniel@trvx.org>
+    (cherry picked from commit 08a4e8889783673436972a979c2fc780466893c8)
+
+commit 2cb2333ecdf2106925d74e024697f73ad947f378
+Author: Thorsten Behrens <tbehrens@suse.com>
+Date:   Thu Jul 3 00:15:03 2014 +0200
+
+    ceph_argparse_flag has no regular 3rd parameter.
+    
+    With clang warning: 'va_start' has undefined behavior with reference
+    types, noticing copy-paste mistake on ceph_argparse_flag.
+    
+    Signed-off-by: Thorsten Behrens <tbehrens@suse.com>
+    (cherry picked from commit 0c7c7223a2beb4ffb953d89d316f87d350677063)
+
+commit f6022639758ec13b9a25b03cd831882db0b517b3
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Dec 17 17:12:43 2014 -0800
+
+    rgw: use correct objv_tracker for bucket instance
+    
+    When trying to create a bucket that already existed, use the
+    objv_tracker of the newly created instance, and not of the original
+    bucket.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit fe158ecc25feefcea8aea4133118e4a84900a8ec)
diff --git a/doc/changelog/v0.80.2.txt b/doc/changelog/v0.80.2.txt
new file mode 100644
index 000000000..1b4a857d3
--- /dev/null
+++ b/doc/changelog/v0.80.2.txt
@@ -0,0 +1,1187 @@
+commit 198236d68deceb16b6b590d807786a140209bfea (tag: refs/tags/v0.80.2, refs/remotes/gh/last)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Wed Jul 9 19:32:26 2014 +0000
+
+    0.80.2
+
+commit 0b6bd2545925b5e8a80d41de1fda13ffe9d30e2b
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Jul 9 12:04:49 2014 -0700
+
+    Revert "rgw: if extra data pool name is empty, use data pool name instead"
+    
+    This breaks multipart uploads on firefly, though not on master.
+    Revert for now so we can release 0.80.2.
+    
+    This reverts commit 5fd8b0d1639c67e355f0fc0d7e6d7036618d87a1.
+
+commit 9426c4faa9ca327f18f20a57ba772b08d6f069e7
+Merge: b81a277 4733178
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 8 10:06:01 2014 -0700
+
+    Merge pull request #2083 from ceph/wip-8727.firefly
+    
+    messages: MForward: fix compat version
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 4733178c3a8f57c54196f176b20154415783dba0
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jul 8 16:46:27 2014 +0100
+
+    messages: MForward: fix compat version
+    
+    A while ago we bumped the head version and reset the compat version to 0.
+    Doing this so happens to make the messenger assume that the message does
+    not support the compat versioning and sets the compat version to the head
+    version -- thus making compat = 2 when it should have been 1.
+    
+    The nasty side-effect of this is that upgrading from emperor to firefly
+    will have emperor-leaders being unable to decode forwarded messages from
+    firefly-peons.
+    
+    Fixes: #8727
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit c92feebb81f90344a085f8256f0ed03cdc2f14e1)
+
+commit b81a2777aea636ffe544e737cd8e17dc892ba328
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Thu Jul 3 16:11:24 2014 -0700
+
+    ceph.spec.in: add bash completion file for radosgw-admin
+    
+    Signed-off-by: Sandon Van Ness <sandon@inktank.com>
+    (cherry picked from commit b70096307130bcbac176704493a63c5d039d3edc)
+
+commit 4649d7e90ffec89b452aef69cf833522bdb81f88
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Thu Jul 3 16:10:55 2014 -0700
+
+    ceph.spec.in: rhel7-related changes:
+    
+    udev rules: /lib -> /usr/lib
+    /sbin binaries move to /usr/sbin or %{_sbindir}
+    
+    Signed-off-by: Sandon Van Ness <sandon@inktank.com>
+    (cherry picked from commit 235e4c7de8f8efe491edefbdde8e5da4dfc44034)
+    
+    Conflicts:
+    	ceph.spec.in
+
+commit 38a9566e70d3a2de80c7547fd4e84c8372639f80
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Thu Jul 3 16:08:44 2014 -0700
+
+    Fix/add missing dependencies:
+    
+    - rbd-fuse depends on librados2/librbd1
+    - ceph-devel depends on specific releases of libs and libcephfs_jni1
+    - librbd1 depends on librados2
+    - python-ceph does not depend on libcephfs1
+    
+    Signed-off-by: Sandon Van Ness <sandon@inktank.com>
+    (cherry picked from commit 7cf81322391b629b241da90181800ca1f138ce78)
+    
+    Conflicts:
+    	ceph.spec.in
+
+commit 733c89dc6af89321ee58ad78b30bc6a0c4e78291
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Thu Jul 3 16:05:00 2014 -0700
+
+    ceph.spec.in: whitespace fixes
+    
+    Signed-off-by: Sandon Van Ness <sandon@inktank.com>
+    (cherry picked from commit ec8af52a5ede78511423a1455a496d46d580c644)
+    
+    Conflicts:
+    	ceph.spec.in
+
+commit 21b03bc96ab1a0877235f1e3e4a96899cba2b4fa
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Thu Jul 3 16:04:10 2014 -0700
+
+    ceph.spec.in: split out ceph-common as in Debian
+    
+    Move files, postun scriptlet, and add dependencies on ceph-common
+    where appropriate
+    
+    Signed-off-by: Sandon Van Ness <sandon@inktank.com>
+    (cherry picked from commit e131b9d5a5e90e87d8a8346cb96cb5a26135c144)
+
+commit dbee797d1344aec47fedc8d3fdd6c9460e0ffa16
+Merge: 6d6039a 9e81fc1
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 7 09:40:51 2014 -0700
+
+    Merge pull request #2057 from ceph/wip-8593-firefly
+    
+    mon: backport health check improvements
+    
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+
+commit 6d6039a5a56743c006a0d081157cb6ee9e3b7af6
+Author: Greg Farnum <greg@inktank.com>
+Date:   Thu Jul 3 16:30:49 2014 -0700
+
+    mon: check changes to the whole CRUSH map and to tunables against cluster features
+    
+    When we change the tunables, or set a new CRUSH map, we need to make sure it's
+    supported by all the monitors and OSDs currently participating in the cluster.
+    
+    Fixes: #8738
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 54af8104816ccc76aad251aa47a6efc1bf879e7d)
+
+commit 0a8d194f2c44a73c31ae8c52fabb430d66d4a1b1
+Author: Greg Farnum <greg@inktank.com>
+Date:   Thu Jul 3 15:24:30 2014 -0700
+
+    OSDMonitor: fix quorum_features comparison in check_cluster_features
+    
+    We need to see if there's a feature which is not in the quorum_features,
+    not if there are no features in common!
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 2280c0eaa9f3f733062085a5e7f7dc23c3bbe291)
+
+commit a49c61222f8014db533d8f505ec44d10dc5478e6
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 16 12:57:36 2014 -0700
+
+    ceph_test_rados_api_tier: disable LibRadosTierECPP::HitSetWrite
+    
+    Disable this test until hitget-get reliably works on EC pools (currently
+    it does not, and this test usually passes only because we get the in-memory
+    HitSet).
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c5b5ed672c039cc4fa04772e065a59d3b1df803a)
+
+commit 828ce4055c8580e42175159c02a9dfa76bc414ff
+Merge: b9b498e b1a4a7c
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 4 21:18:32 2014 -0700
+
+    Merge pull request #2073 from ceph/wip-rgw-firefly-2
+    
+    rgw: more firefly backports
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit b1a4a7cb91e164d1f8af8ce9319e3b3c1949858d
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Jun 11 16:50:41 2014 -0700
+
+    rgw: set a default data extra pool name
+    
+    Fixes: #8585
+    Have a default name for the data extra pool, otherwise it would be empty
+    which means that it'd default to the data pool name (which is a problem
+    with ec backends).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit b2542f8e880e45bb6d052f13eb3ef6a7c61e4cd4)
+
+commit b9b498e6a7794205af8aa99ffe5266e478c0e711
+Merge: f285266 053c261
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 3 07:00:52 2014 -0700
+
+    Merge pull request #2028 from ceph/wip-rgw-firefly
+    
+    rgw: a couple backports for firefly
+    
+    Passed the rgw suite, modulo a python bootstrap issue.
+    
+    sage-2014-07-01_09:50:22-rgw-wip-rgw-firefly-testing-basic-plana/337393
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 9f06e2c16e409a9a8284af9964f0cfff6e222164
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 30 12:55:45 2014 -0700
+
+    rgw: fix uninit ofs in RGWObjManifect::obj_iterator
+    
+    Valgrind picked this up:
+    
+      <kind>UninitCondition</kind>
+      <what>Conditional jump or move depends on uninitialised value(s)</what>
+      <stack>
+        <frame>
+          <ip>0x5145B8</ip>
+          <obj>/usr/bin/radosgw</obj>
+          <fn>RGWObjManifest::obj_iterator::seek(unsigned long)</fn>
+          <dir>/srv/autobuild-ceph/gitbuilder.git/build/out~/ceph-0.82-354-g62027ec/src/rgw</dir>
+          <file>rgw_rados.cc</file>
+          <line>562</line>
+        </frame>
+        <frame>
+          <ip>0x5672A4</ip>
+          <obj>/usr/bin/radosgw</obj>
+          <fn>list_multipart_parts(RGWRados*, req_state*, std::string const&amp;, std::string&amp;, int, int, std::map&lt;unsigned int, RGWUploadPartInfo, std::less&lt;unsigned int&gt;, std::allocator&lt;std::pair&lt;unsigned int const, RGWUploadPartInfo&gt; &gt; &gt;&amp;, int*, bool*, bool)</fn>
+          <dir>/srv/autobuild-ceph/gitbuilder.git/build/out~/ceph-0.82-354-g62027ec/src/rgw</dir>
+          <file>rgw_rados.h</file>
+          <line>217</line>
+        </frame>
+        <frame>
+          <ip>0x5688EE</ip>
+          <obj>/usr/bin/radosgw</obj>
+          <fn>RGWListMultipart::execute()</fn>
+          <dir>/srv/autobuild-ceph/gitbuilder.git/build/out~/ceph-0.82-354-g62027ec/src/rgw</dir>
+          <file>rgw_op.cc</file>
+          <line>2956</line>
+        </frame>
+    ...
+    
+    Fixes: #8699
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e0d3b78b6af523ae77b0cee385cd40f67f7a0ab6)
+
+commit 5fd8b0d1639c67e355f0fc0d7e6d7036618d87a1
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri May 30 14:47:25 2014 -0700
+
+    rgw: if extra data pool name is empty, use data pool name instead
+    
+    Fixes: #8311
+    An empty pool name could be used as the extra data pool.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit b300318113b162522759d4794b1cfa3d1d9398e4)
+
+commit cbad09e1907ceac4feb069f9cf257e580f36da91
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue May 6 15:35:20 2014 -0700
+
+    rgw: extend manifest to avoid old style manifest
+    
+    In case we hit issue #8269 we'd like to avoid creating an old style
+    manifest. Since we need to have parts that use different prefix we add a
+    new rule param that overrides the manifest prefix.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 545d8ad1d2dfe53b967ab1bd17dfeb1780abbb5e)
+
+commit ea68b9372319fd0bab40856db26528d36359102e
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri May 2 17:06:05 2014 -0700
+
+    rgw: don't allow multiple writers to same multiobject part
+    
+    Fixes: #8269
+    Backport: firefly, dumpling
+    
+    A client might need to retry a multipart part write. The original thread
+    might race with the new one, trying to clean up after it, clobbering the
+    part's data.
+    The fix is to detect whether an original part already existed, and if so
+    use a different part name for it.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit bd8e026f88b812cc70caf6232c247844df5d99bf)
+
+commit f285266a320bf22df8b09a669d1c1fd400b05765
+Merge: d43e711 6a0dd1b
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 1 16:16:09 2014 -0700
+
+    Merge pull request #2056 from ceph/wip-8010
+    
+    Revert "mon: OSDMonitor: Refuse to delete CephFS pools"
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 9e81fc180a3999ebec95949a2cb31fea970573fb
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 16 17:00:51 2014 -0700
+
+    mon: ensure HealthService warning(s) include a summary
+    
+    The low disk space check would change our status to HEALTH_WARN and include
+    a detail message, but no summary.  We need both.
+    
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3ed7f2dd4345633ff34017b201082f4c261ef387)
+
+commit e96637c2e33ec092f28af0ce96bfe2e4d7748ff8
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 16 16:58:14 2014 -0700
+
+    mon: refactor check_health()
+    
+    Refactor the get_health() methods to always take both a summary and detail.
+    Eliminate the return value and pull that directly from the summary, as we
+    already do with the PaxosServices.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 82e47db8073b622183a5e33f6e0b999a3a144804)
+
+commit 28584200df36dda5d434d4d7f58585933905de30
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 16 16:40:05 2014 -0700
+
+    mon: fix typos, punctuation for mon disk space warning(s)
+    
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 98883f6308ce72f69a71feab29ef00e13f319cdb)
+    
+    Conflicts:
+    
+    	src/mon/DataHealthService.cc
+
+commit 4f0666085e2fd553ccc05cef8510aae820849aca
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 16 16:27:05 2014 -0700
+
+    mon/OSDMonitor: make down osd count sensible
+    
+    We currently log something like
+    
+     1/10 in osds are down
+    
+    in the health warning when there are down OSDs, but this is based on a
+    comparison of the number of up vs the number of in osds, and makes no sense
+    when there are up osds that are not in.
+    
+    Instead, count only the number OSDs that are both down and in (relative to
+    the total number of OSDs in) and warn about that.  This means that, if a
+    disk fails, and we mark it out, and the cluster fully repairs itself, it
+    will go back to a HEALTH_OK state.
+    
+    I think that is a good thing, and certainly preferable to the current
+    nonsense.  If we want to distinguish between down+out OSDs that were failed
+    vs those that have been "acknowledged" by an admin to be dead, we will
+    need to add some additional state (possibly reusing the AUTOOUT flag?), but
+    that will require more discussion.
+    
+    Backport: firefly (maybe)
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 55a97787088b79356c678a909b2410b3924e7f5b)
+
+commit 6a0dd1be909b423f52e2c7f11531b278dc8838ae
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Jul 1 22:04:25 2014 +0100
+
+    Revert "mon: OSDMonitor: Refuse to delete CephFS pools"
+    
+    This reverts commit c7d265a9b276aad5caf9b35f3ce8bc7bdd169f0f.
+    
+    Because this pre-dates the `fs rm` command, this change was
+    preventing firefly users from ever deleting their filesystem pools.
+    
+    Fixes: #8010
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+
+commit 053c261e5bfcb4d39770048775515eb6a667f987
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Jun 4 09:24:33 2014 -0700
+
+    rgw: set meta object in extra flag when initializing it
+    
+    As part of the fix for 8452 we moved the meta object initialization.
+    Missed moving the extra flag initialization that is needed. This breaks
+    setups where there's a separate extra pool (needed in ec backends).
+    
+    Reported-by: Sylvain Munaut <s.munaut@whatever-company.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 703d0eb9bffdf6c553b659728a3a07f068fb15ee)
+
+commit d43e7113dd501aea1db33fdae30d56e96e9c3897
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 30 07:05:04 2014 -0700
+
+    qa/workunits/suites/fsx.sh: don't use zero range
+    
+    Zero range is not supported by cephfs.
+    
+    Fixes: #8542
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2dec8a810060f65d022c06e82090b4aa5ccec0cb)
+
+commit ee0d322c0b1ebaba2f225b8bed9af2948fdcf4c3
+Merge: 3ceafcf a7119c3
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Jun 30 17:01:03 2014 +0200
+
+    Merge pull request #1991 from dachary/wip-8307-erasure-code-profile-implicit-creation
+    
+    erasure code profile implicit creation (firefly backport)
+    
+    Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>
+
+commit a7119c31e9a1b415f56426d2b4cd362357ad8b17
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jun 11 22:44:57 2014 +0200
+
+    erasure-code: pool create must not create profiles
+    
+    If a non existent profile is provided as an argument to osd pool create,
+    it must exit on error and not create the profile as a side effect.
+    
+    http://tracker.ceph.com/issues/8307 refs: #8307
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit ff2eb234e63cd281b40405cb3397da5babda943f)
+
+commit a67c1c1c51b111f40819643530e07f8026227caa
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jun 11 22:47:49 2014 +0200
+
+    erasure-code: OSDMonitor::get_erasure_code is a const
+    
+    If it is not, the non const version of OSDMap::get_erasure_code_profile
+    is called and a profile is created as a side effect, which is not
+    intended.
+    
+    http://tracker.ceph.com/issues/8307 refs: #8307
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 3c638111a4943758b6089c63a42aabbf281ac257)
+
+commit 3ceafcf3576efaef58dffbe70654baf1db4f3a1d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue May 27 10:06:46 2014 +0200
+
+    mon: fix set cache_target_full_ratio
+    
+    It was a noop because it was incorrectly using the variable n. Add a
+    test to protect against regression.
+    
+    http://tracker.ceph.com/issues/8440 Fixes: #8440
+    
+    Reported-by: Geoffrey Hartz <hartz.geoffrey@gmail.com>
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit c2225f874dcf37222d831b65b5a319d598d2fcd9)
+
+commit 5f66a2c99fdef2c8d609126740ca790ce10e67a4
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Jun 16 11:48:24 2014 -0700
+
+    rgw: allocate enough space for bucket instance id
+    
+    Fixes: #8608
+    Backport: dumpling, firefly
+    Bucket instance id is a concatenation of zone name, rados instance id,
+    and a running counter. We need to allocate enough space to account zone
+    name length.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit d2e86a66ca55685e04ffbfaa58452af59f381277)
+
+commit 84d7971c8e50e7d35d7a5fac9353e687f168a1a1
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Fri Jun 20 11:14:25 2014 -0400
+
+    log the command that is being run with subprocess
+    
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit e189a668285f9ab73116bc19f9df1cc515473541)
+
+commit 291d448478360a1be0a9826ada58c94ecd643ad0
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri May 30 16:17:47 2014 -0700
+
+    rgw: fetch object attrs on multipart completion
+    
+    Fixes: #8452
+    Backport: firefly
+    This fixes a regression following a code cleanup.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 70afaaabf979d4cc1a14dbc2b772224cbafcac2f)
+
+commit 2fd808fe93fe0cbd8754eb81d281c83254411039
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu May 22 18:27:58 2014 -0700
+
+    rgw: check appropriate entity permission on put_metadata
+    
+    Fixes: #8428
+    Backport: firefly
+    
+    Cannot use verify_object_permission() to test acls, as the operation
+    here might either be on object or on bucket.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 2ceb13af34bc38b418bb88d81131d770a71159bf)
+
+commit d1afeb38b522baae1b15a7a3548002c830bb7567
+Author: Ilya Dryomov <ilya.dryomov@inktank.com>
+Date:   Thu Jun 5 10:08:42 2014 +0400
+
+    XfsFileStoreBackend: call ioctl(XFS_IOC_FSSETXATTR) less often
+    
+    No need to call ioctl(XFS_IOC_FSSETXATTR) if extsize is already set to
+    the value we want or if any extents are allocated - XFS will refuse to
+    change extsize in that's the case.
+    
+    Fixes: #8241
+    Signed-off-by: Ilya Dryomov <ilya.dryomov@inktank.com>
+    (cherry picked from commit bc3b30ed09b8f3eb86b61e3a05ccacfd928faa95)
+
+commit 7b8e5c99a4a40ae788ad29e36b0d714f529b12eb
+Author: John Spray <john.spray@inktank.com>
+Date:   Tue May 20 16:25:19 2014 +0100
+
+    mon: Fix default replicated pool ruleset choice
+    
+    Specifically, in the case where the configured
+    default ruleset is CEPH_DEFAULT_CRUSH_REPLICATED_RULESET,
+    instead of assuming ruleset 0 exists, choose the lowest
+    numbered ruleset.
+    
+    In the case where an explicit ruleset is passed to
+    OSDMonitor::prepare_pool_crush_ruleset, verify
+    that it really exists.
+    
+    The idea is to eliminate cases where a pool could
+    exist with its crush ruleset set to something
+    other than a value ruleset ID.
+    
+    Fixes: #8373
+    
+    Signed-off-by: John Spray <john.spray@inktank.com>
+    (cherry picked from commit 1d9e4ac2e2bedfd40ee2d91a4a6098150af9b5df)
+    
+    Conflicts:
+    
+    	src/crush/CrushWrapper.h
+
+commit 25a4a874c8e53cd644d6705478fbea4fdb232c4d
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri May 23 14:58:54 2014 -0700
+
+    rgw: calc md5 and compare if user provided appropriate header
+    
+    Fixes: #8436
+    Backport: firefly
+    
+    This was broken in ddc2e1a8e39a5c6b9b224c3eebd1c0e762ca5782. The fix
+    resurrects and old check that was dropped.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    Reviewed-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 9c56c86bdac6bcb8e76c3f04e7d393e4eaadd721)
+
+commit ad18a45a71be34899d9400c8bdef39d0548f9aaf
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Apr 21 15:07:12 2014 -0700
+
+    rgw: calculate user manifest
+    
+    Fixes: #8169
+    Backport: firefly
+    We didn't calculate the user manifest's object etag at all. The etag
+    needs to be the md5 of the contantenation of all the parts' etags.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit ddc2e1a8e39a5c6b9b224c3eebd1c0e762ca5782)
+
+commit 6aa7f7ee4e082819decc48da1f0eaffc4e616302
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 3 11:45:20 2014 -0700
+
+    librados: simplify/fix rados_pool_list bounds checks
+    
+    We were not breaking out of the loop when we filled up the buffer unless
+    we happened to do so on a pool name boundary.  This means that len would
+    roll over (it was unsigned).  In my case, I was not able to reproduce
+    anything particularly bad since (I think) the strncpy was interpreting the
+    large unsigned value as signed, but in any case this fixes it, simplifies
+    the arithmetic, and adds a simple test.
+    
+    - use a single 'rl' value for the amount of buffer space we want to
+      consume
+    - use this to check that there is room and also as the strncat length
+    - rely on the initial memset to ensure that the trailing 0 is in place.
+    
+    Fixes: #8447
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3ec32a6bb11d92e36a0e6381b40ce2fd1fbb016a)
+
+commit 2ba03e08f1bce0a8af87014583d635eb5c0aa376
+Merge: 50dc246 3ad451e
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jun 25 12:42:11 2014 -0700
+
+    Merge pull request #1982 from accelazh/firefly-fix-issue-8256
+    
+    Make <poolname> in "ceph osd tier --help" clearer (fix issue 8256).
+    
+    Reviewed-by: Loic Dachary <loic@dachary.org>
+
+commit 50dc2462ab96c3457b08c50b4cc07676b50dbf72
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Jun 3 16:14:15 2014 -0700
+
+    OSD::calc_priors_during: handle CRUSH_ITEM_NONE correctly
+    
+    Fixes: #8507
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 0bd6f6797c69af9aff851033c57c42121671c684)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+
+commit a53bcf2357c7f351ad520039774d895b8c27987f
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Jun 3 16:11:32 2014 -0700
+
+    OSD::calc_priors_during: fix confusing for loop bracing (cosmetic)
+    
+    Confusing lack of braces is confusing.
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit d76936b92300be5cc403fd5a36616a2424c7877d)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+
+commit 178e6f0f422c9f1b8a51d0e9c364773397a4555f
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Jun 24 10:11:21 2014 -0700
+
+    rados.cc: fix pool alignment check
+    
+    Only check pool alignment if io_ctx is initialized.
+    
+    Introduced in 304b08a23a3db57010078046955a786fe3589ef8
+    Fixes: #8652
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    
+    (cherry picked from commit d7350a3741bf4cdb270c6361e68090fe280cf36d)
+    
+    Conflicts:
+    	src/tools/rados/rados.cc
+
+commit a9f97f8cae5b9d0245e0b0842c5c581cdacbb2da
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 17 13:33:14 2014 -0700
+
+    osd: fix filestore perf stats update
+    
+    Update the struct we are about to send, not the (unlocked!) one we will
+    send the next time around.
+    
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 4afffb4a10a0bbf7f2018ef3ed6b167c7921e46b)
+
+commit b758fd5f0ebcccdb1641a852d40ee880bad01107
+Author: Greg Farnum <greg@inktank.com>
+Date:   Thu Apr 24 15:34:24 2014 -0700
+
+    FileStore: set XATTR_NO_SPILL_OUT when creating new files.
+    
+    Fixes: #8205
+    Backport: firefly
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit e3b995e1567f3ccc6d00ae27ab2aa99ca157228a)
+
+commit d6f302314fc3580dfc746c9d82bd2f7f0bc87d42
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Sat Jun 7 14:32:23 2014 +0800
+
+    FileStore: make _clone() copy spill out marker
+    
+    Previously we were not doing so, and that resulted in unpredictable loss
+    of xattrs from the client's perspective.
+    
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 239476a92849159d2a8966d90ca055c116bee91e)
+
+commit bb2ed23d8129c8db628475d744656b9b43f839c0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jun 18 17:01:54 2014 +0200
+
+    erasure-code: verify that rados put enforces alignment
+    
+    http://tracker.ceph.com/issues/8622 refs: #8622
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit b46c4056014dd6de5e3bd736f2c41f096ea708b4)
+
+commit dece65064d949b5afcc359cd408615883b5e002a
+Author: Lluis Pamies-Juarez <lluis@pamies.cat>
+Date:   Wed Jun 18 10:00:09 2014 -0700
+
+    enforce rados put aligment
+    
+    Signed-off-by: Lluis Pamies-Juarez <lluis.pamies-juarez@hgst.com>
+    (cherry picked from commit 304b08a23a3db57010078046955a786fe3589ef8)
+
+commit 203318b8eb3753723277ec7a459239c81097f090
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jun 6 13:31:29 2014 -0700
+
+    osd/OSDMap: do not require ERASURE_CODE feature of clients
+    
+    Just because an EC pool exists in the cluster does not mean tha tthe client
+    has to support the feature:
+    
+    1) The way client IO is initiated is no different for EC pools than for
+       replicated pools.
+    2) People may add an EC pool to an existing cluster with old clients and
+       locking those old clients out is very rude when they are not using the
+       new pool.
+    3) The only direct client user of EC pools right now is rgw, and the new
+       versions already need to support various other features like CRUSH_V2
+       in order to work.  These features are present in new kernels.
+    
+    Fixes: #8556
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3fe1699f9620280d0070cfe6f01cfeb2332e7470)
+
+commit 6340b14b9c9a9c786b5448f2ea2a5556055a02e9
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 12 16:44:53 2014 -0700
+
+    osd/OSDMap: make get_features() take an entity type
+    
+    Make the helper that returns what features are required of the OSDMap take
+    an entity type argument, as the required features may vary between
+    components in the cluster.
+    
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 250677c965365edf3ecd24ef73700fc6d992ea42)
+
+commit 400fa88f6cf8b1e5b4b15d9e36b1de8c3d1c3970
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Wed May 21 18:12:22 2014 +0800
+
+    Avoid extra check for clean object
+    
+    We needn't to check clean object via buffer state, skip the clean object.
+    
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit f51e33bd9c5a8e1cfc7065b30785696dc45918bc)
+
+commit 3ad451ef57e2ab3fc644c94d44c2de7a17cced45
+Author: Accela Zhao <accelazh@gmail.com>
+Date:   Wed Jun 18 17:17:03 2014 +0800
+
+    Make <poolname> in "ceph osd tier --help" clearer.
+    
+    The ceph osd tier --help info on the left always says <poolname>.
+    It is unclear which one to put <tierpool> on the right.
+    
+    $ceph osd tier --help
+    osd tier add <poolname> <poolname> {--   add the tier <tierpool> to base pool
+    force-nonempty}                          <pool>
+    osd tier add-cache <poolname>            add a cache <tierpool> of size <size>
+    <poolname> <int[0-]>                     to existing pool <pool>
+    ...
+    
+    This patch modifies description on the right to tell which <poolname>:
+    
+    osd tier add <poolname> <poolname> {--   add the tier <tierpool> (the second
+     force-nonempty}                          one) to base pool <pool> (the first
+                                               one)
+    ...
+    
+    Fix: http://tracker.ceph.com/issues/8256
+    
+    Signed-off-by: Yilong Zhao <accelazh@gmail.com>
+
+commit f290a021bc8ec31c18b91cb37323345692f2f198
+Merge: 5722419 d02d46e
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 16 09:25:32 2014 -0700
+
+    Merge pull request #1962 from dachary/wip-8599-ruleset-firefly
+    
+    mon: pool set <pool> crush_ruleset must not use rule_exists (firefly)
+    
+    Reviewed-by: Sage Weil <sage@inktank.com>
+
+commit d02d46e25080d5f7bb8ddd4874d9019a078b816b
+Author: John Spray <john.spray@inktank.com>
+Date:   Tue May 20 16:50:18 2014 +0100
+
+    mon: pool set <pool> crush_ruleset must not use rule_exists
+    
+    Implement CrushWrapper::ruleset_exists that iterates over the existing
+    rulesets to find the one matching the ruleset argument.
+    
+    ceph osd pool set <pool> crush_ruleset must not use
+    CrushWrapper::rule_exists, which checks for a *rule* existing, whereas
+    the value being set is a *ruleset*. (cherry picked from commit
+    fb504baed98d57dca8ec141bcc3fd021f99d82b0)
+    
+    A test via ceph osd pool set data crush_ruleset verifies the ruleset
+    argument is accepted.
+    
+    http://tracker.ceph.com/issues/8599 fixes: #8599
+    
+    Backport: firefly, emperor, dumpling
+    Signed-off-by: John Spray <john.spray@inktank.com>
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+
+commit 572241942706e2f845a147f19717bf6c487f23f4
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jun 8 20:18:49 2014 -0700
+
+    init-ceph: continue after failure doing osd data mount
+    
+    If we are starting many daemons and hit an error, we normally note it and
+    move on.  Do the same when doing the pre-mount step.
+    
+    Fixes: #8554
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6a7e20147cc39ed4689809ca7d674d3d408f2a17)
+
+commit e5a291ef69316519b105b57193753f0bf8ac8e62
+Author: Steve Taylor <steve@DES-U1404-STETAY.stc.local>
+Date:   Tue Jun 10 12:42:55 2014 -0600
+
+    Fix for bug #6700
+    
+    When preparing OSD disks with colocated journals, the intialization process
+    fails when using dmcrypt. The kernel fails to re-read the partition table after
+    the storage partition is created because the journal partition is already in use
+    by dmcrypt. This fix unmaps the journal partition from dmcrypt and allows the
+    partition table to be read.
+    
+    Signed-off-by: Stephen F Taylor <steveftaylor@gmail.com>
+    (cherry picked from commit 673394702b725ff3f26d13b54d909208daa56d89)
+
+commit 34d9b2bc659c4107b30e1ebe7a55315512a0222a
+Author: John Wilkins <john.wilkins@inktank.com>
+Date:   Thu Jun 5 11:29:20 2014 -0700
+
+    doc: Added Disable requiretty commentary.
+    
+    Signed-off-by: John Wilkins <john.wilkins@inktank.com>
+
+commit 54a89bab15a945d47b3f44448910521a2ebf41c6
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri May 16 16:56:33 2014 -0700
+
+    ReplicatedPG::start_flush: fix clone deletion case
+    
+    dsnapc.snaps will be non-empty most of the time if there
+    have been snaps before prev_snapc.  What we really want to
+    know is whether there are any snaps between oi.snaps.back()
+    and prev_snapc.
+    
+    Fixes: 8334
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 29f615b7ac9e92f77cdef9927070727fee9d5e33)
+
+commit 308e2ef5c107bf04d3c5623f3c6bdffa10cf8c21
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon May 12 15:08:07 2014 -0700
+
+    ReplicatedPG::start_flush: send delete even if there are no snaps
+    
+    Even if all snaps for the clone have been removed, we still have to
+    send the delete to ensure that when the object is recreated the
+    new snaps aren't included in the wrong clone.
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 2ec2182745fa7c22526a7cf3dedb25bc314c9db4)
+
+commit 8d04a7ece364c0a95b087990d6365401819b3efa
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu May 15 20:53:27 2014 -0700
+
+    HashIndex: in cleanup, interpret missing dir as completed merge
+    
+    If we stop between unlinking the empty subdir and removing the root
+    merge marker, we get ENOENT on the get_info.  That's actually fine.
+    
+    Backport: firefly
+    Fixes: 8332
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 5ff95dbdd2dbb533d344f37fea722ca4f140e670)
+
+commit b2bf8cb21761d88557fb2d5c0935c4a3678ee01f
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Wed May 28 11:48:12 2014 -0400
+
+    add backport of collections.Counter for python2.6
+    
+    Using Raymond Hettinger's MIT backport
+    
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit 23b75b550507438c79b3aa75e06721e5f7b134a4)
+
+commit 0b00443eec4c372f2ef245f85dc7c1f497ba3778
+Author: Ailing <ailzhang@users.noreply.github.com>
+Date:   Wed May 28 12:37:48 2014 -0700
+
+    rest-api: key missing for per "rx" and "rwx"
+    
+    commit 85a1cf31e6 and db266a3fb2 introduce new per "rx" and "rwx", but key missing for per "rx" and "rwx" in permmap
+    
+    Signed-off-by: Ailing Zhang <ailzhang@cisco.com>
+    (cherry picked from commit 0b5a67410793ec28cac47e6e44cbbcf5684d77e7)
+
+commit a898ee56e01f971c12c0f7bebc22e59f850f27c0
+Author: Greg Farnum <greg@inktank.com>
+Date:   Wed May 21 21:41:23 2014 -0700
+
+    cephfs-java: build against older jni headers
+    
+    Older versions of the JNI interface expected non-const parameters
+    to their memory move functions. It's unpleasant, but won't actually
+    change the memory in question, to do a cast_const in order to satisfy
+    those older headers. (And even if it *did* modify the memory, that
+    would be okay given our single user.)
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 4d4b77e5b6b923507ec4a0ad9d5c7018e4542a3c)
+
+commit 8a12a940817d957063c21ac71a6128f991801573
+Author: Ilya Dryomov <ilya.dryomov@inktank.com>
+Date:   Fri May 16 19:03:13 2014 +0400
+
+    OSDMonitor: set next commit in mon primary-affinity reply
+    
+    Commit 8c5c55c8b47e ("mon: set next commit in mon command replies")
+    fixed MMonCommand replies to include the right version, but the
+    primary-affinity handler was authored before that.  Fix it.
+    
+    Backport: firefly
+    Signed-off-by: Ilya Dryomov <ilya.dryomov@inktank.com>
+    (cherry picked from commit a78b14ec1769ef37bef82bfda6faabb581b4cd7d)
+
+commit 6ba57da94a6e00040677d6e43abf10a36ed9a931
+Author: Dmitry Smirnov <onlyjob@member.fsf.org>
+Date:   Mon May 12 14:08:44 2014 +1000
+
+    prioritise use of `javac` executable (gcj provides it through alternatives).
+    
+     On Debian this fixes FTBFS when gcj-jdk and openjdk-7-jdk are installed at
+     the same time because build system will use default `javac` executable
+     provided by current JDK through `update-alternatives` instead of blindly
+     calling GCJ when it is present.
+    
+    Signed-off-by: Dmitry Smirnov <onlyjob@member.fsf.org>
+    (cherry picked from commit 8b682d167e4535df582f1c77542e2b1ea0981228)
+
+commit 9e335a494c6eff209cff64bd663b5aa2a7bbc242
+Author: Dmitry Smirnov <onlyjob@member.fsf.org>
+Date:   Mon May 12 14:02:53 2014 +1000
+
+    pass '-classpath' option (gcj/javah ignores CLASSPATH environment variable).
+    
+     This should not affect OpenJDK which understands '-classpath' as well.
+    
+     With gcj-jdk we still get FTBFS later:
+    
+     ~~~~
+        java/native/libcephfs_jni.cc:2878:55: error: invalid conversion from 'const jbyte* {aka const signed char*}' to 'jbyte* {aka signed char*}' [-fpermissive]
+                     reinterpret_cast<const jbyte*>(rawAddress));
+                                                               ^
+        In file included from java/native/libcephfs_jni.cc:27:0:
+        /usr/lib/gcc/x86_64-linux-gnu/4.8/include/jni.h:1471:8: error:   initializing argument 4 of 'void _Jv_JNIEnv::SetByteArrayRegion(jbyteArray, jsize, jsize, jbyte*)' [-fpermissive]
+           void SetByteArrayRegion (jbyteArray val0, jsize val1, jsize val2, jbyte * val3)
+                ^
+        make[5] *** [java/native/libcephfs_jni_la-libcephfs_jni.lo] Error 1
+     ~~~~
+    
+    Signed-off-by: Dmitry Smirnov <onlyjob@member.fsf.org>
+    (cherry picked from commit 89fe0353582bde7e2fbf32f1626d430a20002dd0)
+
+commit fcd30500790eac0266ffb70d83ee17b23aeaf209
+Author: Dmitry Smirnov <onlyjob@member.fsf.org>
+Date:   Mon May 12 13:57:20 2014 +1000
+
+    look for "jni.h" in gcj-jdk path, needed to find "jni.h" with gcj-jdk_4.9.0
+    
+    Signed-off-by: Dmitry Smirnov <onlyjob@member.fsf.org>
+    (cherry picked from commit 0f4120c0115e7977ae7c03458addcc2b2916db07)
+
+commit de43f6d2d813d2c7e5cf915c1b9037fe662d6afc
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 8 08:52:51 2014 -0700
+
+    ceph-disk: partprobe before settle when preparing dev
+    
+    Two users have reported this fixes a problem with using --dmcrypt.
+    
+    Fixes: #6966
+    Tested-by: Eric Eastman <eric0e@aol.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 0f196265f049d432e399197a3af3f90d2e916275)
+
+commit fe1fbfb7a43d6590efce99711e47f986249f0956
+Author: Greg Farnum <greg@inktank.com>
+Date:   Tue May 13 13:15:28 2014 -0700
+
+    test: fix some templates to match new output code
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 00225d739cefa1415524a3de45fb9a5a2db53018)
+
+commit 5c8afaa8861345efbcc5488e0336327a8a38d3bc
+Author: Greg Farnum <greg@inktank.com>
+Date:   Thu May 15 16:50:43 2014 -0700
+
+    OSD: fix an osdmap_subscribe interface misuse
+    
+    When calling osdmap_subscribe, you have to pass an epoch newer than the
+    current map's. _maybe_boot() was not doing this correctly -- we would
+    fail a check for being *in* the monitor's existing map range, and then
+    pass along the map prior to the monitor's range. But if we were exactly
+    one behind, that value would be our current epoch, and the request would
+    get dropped. So instead, make sure we are not *in contact* with the monitor's
+    existing map range.
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 290ac818696414758978b78517b137c226110bb4)
+
+commit eb98f4ab3a27aa8e00e0eb7e504ca7bec7c48a06
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon May 19 10:32:12 2014 -0700
+
+    osd: skip out of order op checks on tiered pools
+    
+    When we send redirected ops, we do not assign a new tid, which means that
+    a given client's ops for a pool may not have strictly ordered tids.  Skip
+    this check if the pool is tiered to avoid false positives.
+    
+    Fixes: #8380
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cf2b172c843da0599164901956b66c306a59e570)
+
+commit 29ee6faecb9e16c63acae8318a7c8f6b14367af7
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue May 6 11:50:14 2014 -0700
+
+    ReplicatedPG: block scrub on blocked object contexts
+    
+    Fixes: #8011
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 7411477153219d66625a74c5886530029c516036)
+
+commit 6078726e0b89d0202c3a4bc594be11fc346bed9d
+Author: Guang Yang <yguang@yahoo-inc.com>
+Date:   Fri May 9 09:21:23 2014 +0000
+
+    msg: Fix inconsistent message sequence negotiation during connection reset
+    
+    Backport: firefly, emperor, dumpling
+    
+    Signed-off-by: Guang Yang (yguang@yahoo-inc.com)
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit bdee119076dd0eb65334840d141ccdf06091e3c9)
+
+commit 02557f8bc05ff28013e241611cb046b28f97b4c6
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Apr 15 12:55:47 2014 -0700
+
+    Objecter::_op_submit: only replace the tid if it's 0
+    
+    Otherwise, redirected ops will suddenly have a different tid
+    and will become uncancelable.
+    
+    Fixes: #7588
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 76568aa0db4e16ac1af8fe6405edade1e61cbc81)
+
+commit 788b5fad515c48d4028f5a67d63052e373c81e66
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 8 10:42:42 2014 -0700
+
+    mon/OSDMonitor: force op resend when pool overlay changes
+    
+    If a client is sending a sequence of ops (say, a, b, c, d) and partway
+    through that sequence it receives an OSDMap update that changes the
+    overlay, the ops will get send to different pools, and the replies will
+    come back completely out of order.
+    
+    To fix this, force a resend of all outstanding ops any time the overlay
+    changes.
+    
+    Fixes: #8305
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 63d92ab0969357f78fdade749785136a509bc81b)
+
+commit 268df4862d95bf506ef6f594fdfa8294671babc3
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 8 10:50:51 2014 -0700
+
+    osd: discard client ops sent before last_force_op_resend
+    
+    If an op is sent before last_force_op_resend, and the client's feature is
+    present, drop the op because we know they will resend.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 45e79a17a932192995f8328ae9f6e8a2a6348d10)
+
+commit 8bc4992f434ed0b180d5e4a293d8191f4846358c
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 8 10:52:11 2014 -0700
+
+    osdc/Objecter: resend ops in the last_force_op_resend epoch
+    
+    If we are a client, and process a map that sets last_force_op_resend to
+    the current epoch, force a resend of this op.
+    
+    If the OSD expects us to do this, it will discard our previous op.  If the
+    OSD is old, it will process the old one, this will appear as a dup, and we
+    are no worse off than before.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit dd700bdf7115223cb3e517b851f462d75dd76a2b)
+
+commit 83d03f0e1c65385dcf465108d544da84f8bfad97
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 8 10:40:10 2014 -0700
+
+    osd/osd_types: add last_force_op_resend to pg_pool_t
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 3152faf79f498a723ae0fe44301ccb21b15a96ab)
+
+commit c94ac44ab41fc62add3fc93c5c7ce06950eed3f1
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 9 09:20:34 2014 -0700
+
+    osd: handle race between osdmap and prepare_to_stop
+    
+    If we get a MOSDMarkMeDown message and set service.state == STOPPING, we
+    kick the prepare_to_stop() thread.  Normally, it will wake up and then
+    set osd.state == STOPPING, and when we process the map message next we
+    will not warn.  However, if dispatch() takes the lock instead and processes
+    the map, it will fail the preparing_to_stop check and issue a spurious
+    warning.
+    
+    Fix by checking for either preparing_to_stop or stopping.
+    
+    Fixes: #8319
+    Backport: firefly, emperor, dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 6b858be0676f937a99dbd51321497f30c3a0097f)
+
+commit 22e2e7cff42970b00dc352513f1f138132f78d04
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat May 10 10:29:11 2014 -0700
+
+    osd/ReplicatedPG: do not queue NULL dup_op
+    
+    We call start_flush() with a NULL op in a couple different places.  Do not
+    put a NULL pointer on the dup_ops list or we will crash later.
+    
+    Fixes: #8328
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 0d67f9b0695765824bdc4a65fbed88edf8ea232e)
diff --git a/doc/changelog/v0.80.3.txt b/doc/changelog/v0.80.3.txt
new file mode 100644
index 000000000..d44f62834
--- /dev/null
+++ b/doc/changelog/v0.80.3.txt
@@ -0,0 +1,19 @@
+commit a129e85cdc3446a427fffffe4c31617d49024946 (tag: refs/tags/v0.80.3)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Fri Jul 11 17:14:12 2014 +0000
+
+    0.80.3
+
+commit 8589c108dd5f00471986deae72d88811ccf0627f (refs/remotes/gh/firefly)
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jul 10 10:36:16 2014 -0700
+
+    rgw: fix RGWObjManifestRule decoder
+    
+    Only decode the new field if it is a new struct.
+    
+    Fixes: #8804
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit c4afaf9dabd261853a44b2e08f0911c075c1cd3a)
diff --git a/doc/changelog/v0.80.4.txt b/doc/changelog/v0.80.4.txt
new file mode 100644
index 000000000..4b44638b1
--- /dev/null
+++ b/doc/changelog/v0.80.4.txt
@@ -0,0 +1,36 @@
+commit 7c241cfaa6c8c068bc9da8578ca00b9f4fc7567f (tag: refs/tags/v0.80.4)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Tue Jul 15 12:13:55 2014 -0700
+
+    0.80.4
+
+commit 7557a8139425d1705b481d7f010683169fd5e49b (refs/remotes/gh/firefly)
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Jul 14 13:29:37 2014 -0700
+
+    XfsFileStoreBackend: default to disabling extsize on xfs
+    
+    This appears to be responsible for the deep scrub mismatches on some rbd
+    workloads.
+    
+    Fixes: 8830
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 01cd3cdc726a3e838bce05b355a021778b4e5db1)
+
+commit ee7016a046c09be674808efb093b6ff80d0df18c
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jul 14 15:00:30 2014 -0700
+
+    Revert "Revert "rgw: if extra data pool name is empty, use data pool name instead""
+    
+    This reverts commit 0b6bd2545925b5e8a80d41de1fda13ffe9d30e2b.
+    
+    We confused commit 5fd8b0d1639c67e355f0fc0d7e6d7036618d87a1 with commit
+    b1a4a7cb91e164d1f8af8ce9319e3b3c1949858d in our tests.  We tested without
+    the latter, saw a failure, applied it and then reverted the former, and it
+    passed, but didn't actually resolve the problem.
+    
+    This puts them both back in place and all should be well.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
diff --git a/doc/changelog/v0.80.5.txt b/doc/changelog/v0.80.5.txt
new file mode 100644
index 000000000..4c7a231d6
--- /dev/null
+++ b/doc/changelog/v0.80.5.txt
@@ -0,0 +1,354 @@
+commit 38b73c67d375a2552d8ed67843c8a65c2c0feba6 (tag: refs/tags/v0.80.5, refs/remotes/gh/last, refs/remotes/gh/firefly)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Tue Jul 29 05:24:39 2014 -0700
+
+    0.80.5
+
+commit b576d5a242c16bc9e38ba283a9784f838614882a
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri May 9 08:41:33 2014 -0700
+
+    osd: cancel agent_timer events on shutdown
+    
+    We need to cancel all agent timer events on shutdown.  This also needs to
+    happen early so that any in-progress events will execute before we start
+    flushing and cleaning up PGs.
+    
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c0dc245b662f1f9c640d7dd15fdf4cf26e729782)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+
+commit 6213130f76b1ee89947bae578a1fccf16b845956
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 8 16:11:44 2014 -0700
+
+    osd: s/applying repop/canceling repop/
+    
+    The 'applying' language dates back to when we would wait for acks from
+    replicas before applying writes locally.  We don't do any of that any more;
+    now, this loop just cancels the repops with remove_repop() and some other
+    cleanup.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ef40737eee4389faa7792661a0f9d15b3d0440f2)
+
+commit 2e7c91587d99c9eba85ab7bd407a5b722b25a81e
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 8 16:11:27 2014 -0700
+
+    osd: clear PGBackend state on shutdown
+    
+    This was leaking state on shutdown whenever there were in-flight repops
+    that were canceled.
+    
+    Fixes: #7891
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cafceae6c243369d3616af8217884e7e029896b9)
+
+commit f03110f3889ce7340c64f19358840b2425c93f68
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jul 8 16:10:58 2014 -0700
+
+    osd: separate cleanup from PGBackend::on_change()
+    
+    The generic portion of on_change() cleaned up temporary on-disk objects
+    and requires a Transaction.  The rest is clearing out in-memory state and
+    does not.  Separate the two.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e299357e2a24474ab7a02e65ca0279336fd04b76)
+
+commit 42ad241efab751a1d4f50d5a748d91d0fe967844
+Merge: 5f4ceb2 5049fc4
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Jul 23 10:09:19 2014 -0700
+
+    Merge remote-tracking branch 'origin/wip-8858-firefly' into firefly
+    
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 5f4ceb208b55fab7fd1530dcf8fe03198b044148
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jul 22 00:25:37 2014 +0100
+
+    mon: AuthMonitor: always encode full regardless of keyserver having keys
+    
+    On clusters without cephx, assuming an admin never added a key to the
+    cluster, the monitors have empty key servers.  A previous patch had the
+    AuthMonitor not encoding an empty keyserver as a full version.
+    
+    As such, whenever the monitor restarts we will have to read the whole
+    state from disk in the form of incrementals.  This poses a problem upon
+    trimming, as we do every now and then: whenever we start the monitor, it
+    will start with an empty keyserver, waiting to be populated from whatever
+    we have on disk.  This is performed in update_from_paxos(), and the
+    AuthMonitor's will rely on the keyserver version to decide which
+    incrementals we care about -- basically, all versions > keyserver version.
+    
+    Although we started with an empty keyserver (version 0) and are expecting
+    to read state from disk, in this case it means we will attempt to read
+    version 1 first.  If the cluster has been running for a while now, and
+    even if no keys have been added, it's fair to assume that version is
+    greater than 0 (or even 1), as the AuthMonitor also deals and keeps track
+    of auth global ids.  As such, we expect to read version 1, then version 2,
+    and so on.  If we trim at some point however this will not be possible,
+    as version 1 will not exist -- and we will assert because of that.
+    
+    This is fixed by ensuring the AuthMonitor keeps track of full versions
+    of the key server, even if it's of an empty key server -- it will still
+    keep track of the key server's version, which is incremented each time
+    we update from paxos even if it is empty.
+    
+    Fixes: #8851
+    Backport: dumpling, firefly
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit b551ae2bcea2dd17b37f5f5ab34251cc78de0e26)
+
+commit 5c1d74848863b45aa9e98d134cbe0cd1c8199557
+Merge: 442c3fd c390ec4
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 21 13:10:28 2014 -0700
+
+    Merge pull request #2122 from ceph/wip-dencoder-firefly
+    
+    fix linking deps, cpeh-dencoder, etc., firefly backport
+
+commit 5049fc4c9d6513bc58c3c4464f3de18949ac0aa5
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jul 18 14:52:48 2014 -0700
+
+    rgw: dump prefix unconditionally
+    
+    As part of issue #8858, and to be more in line with S3, dump the Prefix
+    field when listing bucket even if bucket is empty.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d7209c11251d42227608bc54cc69232ef62ffe80)
+
+commit 7fdd6d94262d8e6c81d168f5b997b3a5f030e6fa
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jul 17 15:48:26 2014 -0700
+
+    rgw: list extra objects to set truncation flag correctly
+    
+    Otherwise we end up returning wrong truncated value, and no data on the
+    next iteration.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit dc417e477d4ad262885c6b5f5987cf06d63b159d)
+
+commit 1f8774e01a616739ab70a78c348ab986b64b8fce
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jul 17 11:45:44 2014 -0700
+
+    rgw: account common prefixes for MaxKeys in bucket listing
+    
+    To be more in line with the S3 api. Beforehand we didn't account the
+    common prefixes towards the MaxKeys (a single common prefix counts as a
+    single key). Also need to adjust the marker now if it is pointing at a
+    common prefix.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 82d2d612e700f94a0bb2d9fb7555abf327be379b)
+
+commit cb0a7ab8eb1d2fc8799b97c5a387b5f4f93b515c
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jul 17 11:24:51 2014 -0700
+
+    rgw: add NextMarker param for bucket listing
+    
+    Partially fixes #8858.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 924686f0b6593deffcd1d4e80ab06b1e7af00dcb)
+
+commit 34aa7086ba5293ee4d008016558dcdb1ada02e83
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Jul 16 15:21:09 2014 -0700
+
+    rgw: improve delmited listing of bucket
+    
+    If found a prefix, calculate a string greater than that so that next
+    request we can skip to that. This is still not the most efficient way to
+    do it. It'll be better to push it down to the objclass, but that'll
+    require a much bigger change.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit e6cf618c257f26f97f60a4c1df1d23a14496cab0)
+
+commit d027ff1b432a265ad49ddfeafab45093bcd03e6c
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Jul 16 16:05:58 2014 -0700
+
+    utf8: export encode_utf8() and decode_utf8()
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit 49fc68cf8c3122c878ea9503c9c74d7046bc9c6f)
+
+commit c390ec4fa919e0a17e646e595b30cfa878c3454f
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 18 22:44:51 2014 -0700
+
+    ceph-dencoder: don't link librgw.la (and rados, etc.)
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 34671108ce0b7597dba4f619732ecdb8933cda6e)
+
+commit 3d89380c90ab632eb76d3e429788fd81630e7a57
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 18 22:27:25 2014 -0700
+
+    rgw: move a bunch of stuff into rgw_dencoder
+    
+    This will help out ceph-dencoder ...
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b1a641f307942cbf43036f75ef67fb30441dfe95)
+
+commit dc6c9bf2b92d28bbd2bc822c267b0555f74c87fc
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 18 21:58:29 2014 -0700
+
+    libosd_types, libos_types, libmon_types
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1c170776cb8c0266f0f54f049ed07bbdb9c9ab5e)
+    
+    Conflicts:
+    
+    	src/os/Makefile.am
+    	src/os/ObjectStore.cc
+    	src/osd/Makefile.am
+
+commit 7be91e8c5d935f7094033842bde1da2486905c70
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 18 20:55:39 2014 -0700
+
+    Revert "ceph.spec: move ceph-dencoder to ceph from ceph-common"
+    
+    This reverts commit 95f5a448b52db545a2b9bbad47fdb287254f93ea.
+    (cherry picked from commit 58cc894b3252a848ebc2169bcc4980a0ae6cc375)
+
+commit 04ad08a18a773e104a826b2a77ab0b2ffaac1e5f
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 18 20:55:35 2014 -0700
+
+    Revert "debian: move ceph-dencoder to ceph from ceph-common"
+    
+    This reverts commit b37e3bde3bd31287b11c069062280258666df7c5.
+    (cherry picked from commit f181f78b7473260a717bc8ab4fc4d73a80e3b5ba)
+
+commit 442c3fd5a3b6a6c6a85d9bd7755923f14aa598e7
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Thu Jul 17 21:44:06 2014 -0700
+
+    configure: do not link leveldb with everything
+    
+    Detect leveldb, but do not let autoconf blindly link it with everything on the
+    planet.
+    
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    Sighed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 06a8f7b99c5533f397b34f448138220384df60ac)
+
+commit c1073e430eafeb1f213f80636a3a24966b292698
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jul 14 17:02:05 2014 -0700
+
+    ceph.spec: move ceph-dencoder to ceph from ceph-common
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 95f5a448b52db545a2b9bbad47fdb287254f93ea)
+
+commit f4e9e0ef2eab3174584f1695122b114549feff57
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jul 14 14:16:43 2014 -0700
+
+    debian: move ceph-dencoder to ceph from ceph-common
+    
+    It links against the world currently (notably leveldb).  Not nice for the
+    client-side lib.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b37e3bde3bd31287b11c069062280258666df7c5)
+
+commit 681c399e83df4c6ebd7e0836203eb2466455f548
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jul 17 16:40:06 2014 -0700
+
+    logrotate.conf: fix osd log rotation under upstart
+    
+    In commit 7411c3c6a42bef5987bdd76b1812b01686303502 we generalized this
+    enumeration code by copying what was in the upstart scripts.  However,
+    while the mon and mds directories get a 'done' file, the OSDs get a 'ready'
+    file.  Bah!  Trigger off of either one.
+    
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 14a9ca602fa9573b0b6b94709b384bca02d12fac)
+
+commit 794ec7fb43da4db930d28571a84c1c1cec1eb066
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Jul 16 12:23:31 2014 -0700
+
+    rgw: don't try to wait for pending if list is empty
+    
+    Fixes: #8846
+    Backport: firefly, dumpling
+    
+    This was broken at ea68b9372319fd0bab40856db26528d36359102e. We ended
+    up calling wait_pending_front() when pending list was empty.
+    This commit also moves the need_to_wait check to a different place,
+    where we actually throttle (and not just drain completed IOs).
+    
+    Reported-by: Sylvain Munaut <s.munaut@whatever-company.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit f9f2417d7db01ecf2425039539997901615816a9)
+
+commit e75dd2e4b7adb65c2de84e633efcd6c19a6e457b
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Thu Jul 17 10:06:37 2014 -0400
+
+    remove suse service restarts
+    
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit 0f11aaeefd7e8b49f88607937aade6113ebda52c)
+
+commit 361c1f8554ce1fedfd0020cd306c41b0ba25f53e
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Wed Jul 16 16:12:12 2014 -0400
+
+    remove ceph restarts on upgrades for RPMs
+    
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit e3a5756f58ef5d07badf18ab08a26f47f7d232cb)
+
+commit 88f694dcd3d3342f764f3790572988b7808028ae
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Wed Jul 16 11:07:13 2014 -0400
+
+    set the default log level to WARNING
+    
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit 420f0a471a31d1e56359ea642ed831e8a38b1e90)
+
+commit 67b5193f73a2c9ec9e503ad3431473998217375d
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jul 15 18:11:41 2014 -0700
+
+    init-ceph: wrap daemon startup with systemd-run when running under systemd
+    
+    We want to make sure the daemon runs in its own systemd environment.  Check
+    for systemd as pid 1 and, when present, use systemd-run -r <cmd> to do
+    this.
+    
+    Probably fixes #7627
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    Tested-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 3e0d9800767018625f0e7d797c812aa44c426dab)
diff --git a/doc/changelog/v0.80.6.txt b/doc/changelog/v0.80.6.txt
new file mode 100644
index 000000000..2e1435e5f
--- /dev/null
+++ b/doc/changelog/v0.80.6.txt
@@ -0,0 +1,3266 @@
+commit f93610a4421cb670b08e974c6550ee715ac528ae (tag: refs/tags/v0.80.6, refs/remotes/gh/firefly)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Wed Oct 1 09:37:12 2014 -0700
+
+    0.80.6
+
+commit 2ecf362e15301ee07b511b2a84585ad691543569
+Merge: 1fafd6b 76341b0
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 29 13:55:02 2014 -0700
+
+    Merge pull request #2603 from dachary/wip-9620-test-mon-thrash-firefly
+
+    qa/workunits/cephtool/test.sh: fix thrash (ultimate)
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 76341b0b7581194273ac787df2b940221825d083
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Mon Sep 29 13:47:06 2014 +0200
+
+    qa/workunits/cephtool/test.sh: fix thrash (ultimate)
+
+    Keep the osd trash test to ensure it is a valid command but make it a
+    noop by giving it a zero argument (meaning thrash 0 OSD maps).
+
+    Remove the loops that were added after the command in an attempt to wait
+    for the cluster to recover and not pollute the rest of the tests. Actual
+    testing of osd thrash would require a dedicated cluster because it the
+    side effects are random and it is unnecessarily difficult to ensure they
+    are finished.
+
+    http://tracker.ceph.com/issues/9620 Fixes: #9620
+
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit beade63a17db2e6fc68d1f55332d602f8f7cb93a)
+
+    Conflicts:
+	qa/workunits/cephtool/test.sh
+
+commit 1fafd6bf2ef03672dfa27ec7a201a274927040b7
+Merge: ebb5995 71005c1
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Sep 25 10:08:03 2014 -0700
+
+    Merge pull request #2576 from ceph/wip-9593
+
+    osd/ReplicatedPG: fix objecter locking in start_flush
+
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 71005c1bdf38212ea2fde12de08c45598c815d37
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 25 09:59:29 2014 -0700
+
+    osd/ReplicatedPG: fix objecter locking in start_flush
+
+    Broken in backport fd96eb62ece27f5c660429584c2ff2e058bc6e94.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit ebb599545d70e4289e385f88b4f0594b4f23f3b7
+Merge: 2abd7a3 2e60c2f
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Sep 23 10:37:27 2014 -0700
+
+    Merge pull request #2548 from dachary/wip-9547-python-rados-truncate-firefly
+
+    python radio aio_read must not truncate on \000 (firefly)
+
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+
+commit 2abd7a3fd018b52f45f9d6b7c9d847242aa97fe9
+Merge: 2675b0d 80f2f9c
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Sep 23 10:27:15 2014 -0700
+
+    Merge branch 'wip-sam-testing-firefly' into firefly
+
+commit 80f2f9cf687e25d6f2c4cee34fef55215ac6b027
+Merge: bb9e65b e6da732
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Sun Sep 21 10:04:30 2014 -0700
+
+    Merge remote-tracking branch 'origin/wip-9240' into wip-sam-testing-firefly
+
+commit bb9e65bd9292e2ee03a260cc98aa6715008e6682
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Aug 28 15:32:22 2014 -0700
+
+    PG: wait until we've build the missing set to discover_all_missing
+
+    Fixes: #9179
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 970d9830a3a6e8568337c660fb8b4c4a60a2b3bf)
+
+    Conflicts:
+	src/osd/PG.cc
+
+commit fa645b21b587b6a1d9adbb9cedce1af3af1e5a62
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Aug 26 16:53:02 2014 -0700
+
+    PG: mark_log_for_rewrite on resurrection
+
+    Fixes: #8777
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 8346e10755027e982f26bab4642334fd91cc31aa)
+
+commit fd96eb62ece27f5c660429584c2ff2e058bc6e94
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Sun Sep 7 20:13:41 2014 -0700
+
+    ReplicatedPG:start_flush send a second delete
+
+    Suppose we start with the following in the cache pool:
+
+    30:[29,21,20,15,10,4]:[22(21), 15(15,10), 4(4)]+head
+
+    The object doesn't exist at 29 or 20.
+
+    First, we flush 4 leaving the backing pool with:
+
+    3:[]+head
+
+    Then, we begin to flush 15 with a delete with snapc 4:[4] leaving the
+    backing pool with:
+
+    4:[4]:[4(4)]
+
+    Then, we finish flushing 15 with snapc 9:[4] with leaving the backing
+    pool with:
+
+    9:[4]:[4(4)]+head
+
+    Next, snaps 10 and 15 are removed causing clone 10 to be removed leaving
+    the cache with:
+
+    30:[29,21,20,4]:[22(21),4(4)]+head
+
+    We next begin to flush 22 by sending a delete with snapc 4(4) since
+    prev_snapc is 4 <---------- here is the bug
+
+    The backing pool ignores this request since 4 < 9 (ORDERSNAP) leaving it
+    with:
+
+    9:[4]:[4(4)]
+
+    Then, we complete flushing 22 with snapc 19:[4] leaving the backing pool
+    with:
+
+    19:[4]:[4(4)]+head
+
+    Then, we begin to flush head by deleting with snapc 22:[21,20,4] leaving
+    the backing pool with:
+
+    22[21,20,4]:[22(21,20), 4(4)]
+
+    Finally, we flush head leaving the backing pool with:
+
+    30:[29,21,20,4]:[22(21*,20*),4(4)]+head
+
+    When we go to flush clone 22, all we know is that 22 is dirty, has snaps
+    [21], and 4 is clean. As part of flushing 22, we need to do two things:
+    1) Ensure that the current head is cloned as cloneid 4 with snaps [4] by
+    sending a delete at snapc 4:[4].
+    2) Flush the data at snap sequence < 21 by sending a copyfrom with snapc
+    20:[20,4].
+
+    Unfortunately, it is possible that 1, 1&2, or 1 and part of the flush
+    process for some other now non-existent clone have already been
+    performed.  Because of that, between 1) and 2), we need to send
+    a second delete ensuring that the object does not exist at 20.
+
+    Fixes: #9054
+    Backport: firefly
+    Related: 66c7439ea0888777b5cfc08bcb0fbd7bfd8653c3
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 4843fd510b33a71999cdf9c2cfa2b4c318fa80fd)
+
+commit 5aa35ac262c6e0910f99086f46dd792775ab6790
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Aug 11 12:59:16 2014 -0700
+
+    ReplicatedPG::start_flush: remove superfluous loop
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 66c7439ea0888777b5cfc08bcb0fbd7bfd8653c3)
+
+commit 14fb643bbda51cc31919197541973c332b0c37bc
+Merge: c468507 3578b11
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Sun Sep 21 10:03:53 2014 -0700
+
+    Merge remote-tracking branch 'origin/wip-9339' into wip-sam-testing-firefly
+
+commit 2e60c2f1ec8c2aaf56b73a64f55becd679dd2825
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Sat Sep 20 12:41:30 2014 +0200
+
+    test: check python rados aio_read with buffers containing null
+
+    http://tracker.ceph.com/issues/9547 Refs: #9547
+
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 226c0c7ac6ee95ff2c1665d4e7164e2962c0346e)
+
+commit 72a46dec211d5c08d1ca9f969013d939c4361ba9
+Author: Mohammad Salehe <salehe+dev@gmail.com>
+Date:   Sun May 25 10:42:11 2014 +0430
+
+    pybind: Fix aio_read handling of string buffer
+
+    Read data may contain \0, and buf.value interprerts them as string terminator.
+
+    Signed-off-by: Mohammad Salehe <salehe+dev@gmail.com>
+    (cherry picked from commit 8bda44ff37fd04a0fc9498fbbc22f0daf515d721)
+
+commit 2675b0d159ff0608bc6e607be3727ed23e7e2976
+Merge: a67f915 d535fab
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 19 11:30:17 2014 -0700
+
+    Merge pull request #2535 from dachary/wip-9470-pidfile-firefly
+
+    daemons: write pid file even when told not to daemonize (firefly)
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit d535fab6ac4a025541340ded0d3568f565acf1cd
+Author: Alexandre Oliva <oliva@gnu.org>
+Date:   Wed Jul 30 23:08:43 2014 -0300
+
+    daemons: write pid file even when told not to daemonize
+
+    systemd wants to run daemons in foreground, but daemons wouldn't write
+    out the pid file with -f.  Fixed.
+
+    Signed-off-by: Alexandre Oliva <oliva@gnu.org>
+    (cherry picked from commit bccb0eb64891f65fd475e96b6386494044cae8c1)
+
+commit c4685075f583ff58cc05220a3044a10b8392033c (refs/remotes/gh/wip-log-crash-firefly)
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Sep 15 15:44:11 2014 -0700
+
+    PGLog::claim_log_and_clear_rollback_info: fix rollback_info_trimmed_to
+
+    We have been setting it to the old head value.  This is usually
+    harmless since the new head will virtually always be ahead of the
+    old head for claim_log_and_clear_rollback_info, but can cause trouble
+    in some edge cases.
+
+    Fixes: #9481
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 0769310ccd4e0dceebd8ea601e8eb5c0928e0603)
+
+commit 112f6f021bd063e2647169ba42729a447e5848e1
+Merge: a67f915 92cfd37 07f54f2
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Sep 18 09:46:38 2014 -0700
+
+    Merge remote-tracking branches 'origin/wip-9497' and 'origin/wip-9482' into wip-log-crash-firefly
+
+commit 07f54f28df7429a88dd9243800281ecf23f10036
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Sep 15 16:53:21 2014 -0700
+
+    PG::find_best_info: let history.last_epoch_started provide a lower bound
+
+    If we find a info.history.last_epoch_started above any
+    info.last_epoch_started, we must be missing updates and
+    min_last_update_acceptable should provisionally be max().
+
+    Fixes: #9482
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 92cfd370395385ca5537b5bc72220934c9f09026
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Sep 16 20:36:51 2014 -0700
+
+    PG::choose_acting: let the pg go down if acting is smaller than min_size
+
+    Even if the backfill peer would bring us up to min_size, we can't go
+    active since build_prior will not consider the interval maybe_went_rw.
+
+    Fixes: #9497
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit a67f9152f128d693d160e5b08f2ac8b8bc83e8e0
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Jul 24 15:29:40 2014 -0700
+
+    librbd: fix crash using clone of flattened image
+
+    The crash occurs due to ImageCtx->parent->parent being uninitialized,
+    since the inital open_parent() -> open_image(parent) ->
+    ictx_refresh(parent) occurs before ImageCtx->parent->snap_id is set,
+    so refresh_parent() is not called to open an ImageCtx for the parent
+    of the parent. This leaves the ImageCtx->parent->parent NULL, but the
+    rest of ImageCtx->parent updated to point at the correct parent snapshot.
+
+    Setting the parent->snap_id earlier has some unintended side effects
+    currently, so for now just call refresh_parent() during
+    open_parent(). This is the easily backportable version of the
+    fix. Further patches can clean up this whole initialization process.
+
+    Fixes: #8845
+    Backport: firefly, dumpling
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 2545e80d274b23b6715f4d8b1f4c6b96182996fb)
+
+commit 2422f9fd634c239a1159c99aa4a49cfb5c5d097f
+Author: JuanJose 'JJ' Galvez <jgalvez@redhat.com>
+Date:   Sun Sep 14 20:38:20 2014 -0700
+
+    init-radosgw.sysv: Support systemd for starting the gateway
+
+    When using RHEL7 the radosgw daemon needs to start under systemd.
+
+    Check for systemd running on PID 1. If it is then start
+    the daemon using: systemd-run -r <cmd>. pidof returns null
+    as it is executed too quickly, adding one second of sleep and
+    script reports startup correctly.
+
+    Signed-off-by: JuanJose 'JJ' Galvez <jgalvez@redhat.com>
+    (cherry picked from commit ddd52e87b25a6861d3b758a40d8b3693a751dc4d)
+
+commit 6070383bc2c433e36f7ab5dc1ec09a3e7439d523
+Merge: 668cd13 7b25512
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 12 17:31:03 2014 -0700
+
+    Merge pull request #2479 from ceph/wip-9444
+
+    mds: fix root and mdsdir inodes' rsubdirs
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 7b25512d65c6337cbddc7d6c6f55747ba1591be5
+Author: Yan, Zheng <zheng.z.yan@intel.com>
+Date:   Fri May 2 23:08:41 2014 +0800
+
+    mds: fix root and mdsdir inodes' rsubdirs
+
+    inode rstat accounts inode itself.
+
+    Signed-off-by: Yan, Zheng <zheng.z.yan@intel.com>
+    (cherry picked from commit da17394941386dab88ddbfed4af2c8cb6b5eb72f)
+
+commit 668cd1359c94698bbb4f7b841e2f30264d4af937
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Sep 9 14:03:50 2014 -0700
+
+    FileStore: report l_os_j_lat as commit latency
+
+    l_os_commit_lat is actually the commit cycle latency.
+
+    Fixes: #9269
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit d165238b4ee7e925e06ca22890c1e9dac101a7da)
+
+commit 7686966677867d098141aa08f45364eaa97121dd
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Sep 9 12:58:07 2014 -0700
+
+    Objecter::_recalc_linger_op: resend for any acting set change
+
+    Fixes: #9220
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 1349383ac416673cb6df2438729fd2182876a7d1)
+
+    Conflicts:
+
+	src/osdc/Objecter.cc
+	src/osdc/Objecter.h
+
+commit 3abf95456783346e6686919b803c4819d5e82ab7
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 8 13:44:57 2014 -0700
+
+    osdc/Objecter: revoke rx_buffer on op_cancel
+
+    If we cancel a read, revoke the rx buffers to avoid a use-after-free and/or
+    other undefined badness by using user buffers that may no longer be
+    present.
+
+    Fixes: #9362
+    Backport: firefly, dumpling
+    Reported-by: Matthias Kiefer <matthias.kiefer@1und1.de>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2305b2897acba38384358c33ca3bbfcae6f1c74e)
+
+    (adjusted for op->con instead of s->con)
+
+commit bc4108443a17de47b7c582e50fb884396a287152
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 8 13:45:52 2014 -0700
+
+    ceph_test_rados_api_io: add read timeout test
+
+    Verify we don't receive data after a timeout.
+
+    Based on reproducer for #9362 written by
+    Matthias Kiefer <matthias.kiefer@1und1.de>.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f295c1fee4afb9447cdf46f05a44234274d23b6c)
+
+commit 19e7606545f829e5b9ea0255049d9130c51e42fd
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 8 13:42:43 2014 -0700
+
+    ceph_test_rados_api_*: expose nspace
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 977d289055d69ab8a7baaf7ef68c013019225833)
+
+commit 881c267fa4f8257a9205ec897c631183ef5c6388
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Sep 9 12:40:51 2014 -0700
+
+    Revert "PG: mark_log_for_rewrite on resurrection"
+
+    Actually, we don't want to backport this one without the fix
+    for #9293.
+
+    This reverts commit 7ddf0a252bb887553b29fd93e58d01cac38835e6.
+
+commit 3578b1193d497373db14d040ff7f0fc9e06e5ac2
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Sep 3 15:49:47 2014 -0700
+
+    ReplicatedPG: create max hitset size
+
+    Otherwise, hit_set_create could create an unbounded size hitset
+    object.
+
+    Fixes: #9339
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit d73eb895cdc792ddef2626643e61c1521cf53434
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Aug 27 16:21:41 2014 -0700
+
+    PG::can_discard_op: do discard old subopreplies
+
+    Otherwise, a sub_op_reply from a previous interval can stick around
+    until we either one day go active again and get rid of it or delete the
+    pg which is holding it on its waiting_for_active list.  While it sticks
+    around futily waiting for the pg to once more go active, it will cause
+    harmless slow request warnings.
+
+    Fixes: #9259
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit ae3d87348ca4e2dde809c9593b0d54ce0469f7a0)
+
+commit 7ddf0a252bb887553b29fd93e58d01cac38835e6
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Aug 26 16:53:02 2014 -0700
+
+    PG: mark_log_for_rewrite on resurrection
+
+    Fixes: #8777
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 8346e10755027e982f26bab4642334fd91cc31aa)
+
+commit 49a08eb46ef277225fff0d9a0920c37427175f54
+Author: Thorsten Glaser <tg@mirbsd.de>
+Date:   Mon Sep 8 12:49:50 2014 -0700
+
+    debian: only B-R yasm on amd64
+
+    Make yasm dependency amd64 only, it isn?t used elsewhere
+    but breaks x32 (which is mis-detected as amd64)
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9ab46dc5b49219aa6194861c393c938f23001c52)
+
+commit f19162adc241e627ce20140360352ac96f809a6e
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 26 17:43:10 2014 -0700
+
+    osd: fix osd_tp shutdown
+
+    We need to clear the queue, not just drain the currently executing jobs.
+
+    Fixes: #9218
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c2f21c04207b9a2a65e514994a775632b36d6874)
+
+    Conflicts:
+
+	src/osd/OSD.cc
+
+commit 598cde42e7f57fd0b294f70c0a930f94fd361fb3
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 27 06:19:12 2014 -0700
+
+    osd/PG: fix crash from second backfill reservation rejection
+
+    If we get more than one reservation rejection we should ignore them; when
+    we got the first we already sent out cancellations.  More importantly, we
+    should not crash.
+
+    Fixes: #8863
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2b13de16c522754e30a0a55fb9d072082dac455e)
+
+commit 0a9d49e5b5867c58ca1f5c6b9a4c5106639a8d01
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 8 06:58:45 2014 -0700
+
+    mon/Paxos: don't spam log with is_readable at dout level 1
+
+    Backport: firefly, dumpling
+    Reported-by: Aanchal Agrawal <Aanchal.Agrawal@sandisk.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 62ca27d0b119b597ebad40dde64c4d86599e466d)
+
+commit 1660503ffafd69fac3722aea9915008113906e05
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Thu Sep 4 13:58:14 2014 -0400
+
+    doc: add note on soft JS dependency for navigating docs
+
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit 657be818375bea2d8b5998ea1e5505eedc2f294d)
+
+commit 2f075d442f58e9cfefdbc421b9b20f2034a56314
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Wed Sep 3 21:21:45 2014 -0400
+
+    doc: fix missing bracket
+
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit 69638dfaeb0dcd96dac4b5f5c00ed08042432487)
+
+commit 4dacb593e830f42cd0394486333315f1709b502f
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Wed Sep 3 20:47:54 2014 -0400
+
+    doc: attempt to get the ayni JS into all head tags
+
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit 35663fa55ac1579a3b0c8b67028a3a8dfea87b48)
+
+commit 275df4407b37923cb47510890a686ed370e2f39e
+Author: Dmitry Smirnov <onlyjob@member.fsf.org>
+Date:   Sat Aug 23 22:41:30 2014 +1000
+
+    Fix FTBFS on alpha due to incorrect check on BLKGETSIZE
+
+    Ceph FTBFS on Alpha with:
+
+    ~~~~
+    libtool: compile:  g++ -DHAVE_CONFIG_H -I. -D__CEPH__ -D_FILE_OFFSET_BITS=64 -D_REENTRANT -D_THREAD_SAFE -D__STDC_FORMAT_MACROS -D_GNU_SOURCE -DCEPH_LIBDIR=\"/usr/lib/alpha-linux-gnu\" -DCEPH_PKGLIBDIR=\"/usr/lib/alpha-linux-gnu/ceph\" -DGTEST_HAS_TR1_TUPLE=0 -D_FORTIFY_SOURCE=2 -I/usr/include/nss -I/usr/include/nspr -Wall -Wtype-limits -Wignored-qualifiers -Winit-self -Wpointer-arith -Werror=format-security -fno-strict-aliasing -fsigned-char -rdynamic -ftemplate-depth-1024 -Wnon-virtual-dtor -Wno-invalid-offsetof -Wstrict-null-sentinel -g -O2 -Wformat -Werror=format-security -c common/blkdev.cc  -fPIC -DPIC -o common/.libs/blkdev.o
+    In file included from /usr/include/alpha-linux-gnu/asm/ioctls.h:4:0,
+                     from /usr/include/alpha-linux-gnu/bits/ioctls.h:23,
+                     from /usr/include/alpha-linux-gnu/sys/ioctl.h:26,
+                     from common/blkdev.cc:3:
+    common/blkdev.cc:13:7: error: missing binary operator before token "int"
+     #elif BLKGETSIZE
+           ^
+    ~~~~
+
+    This error occurs because the value of BLKGETSIZE is tested in a
+    c-preprocessor conditional compilation test whereas the test should
+    be for existence.
+
+    From: Michael Cree <mcree@orcon.net.nz>
+    Bug-Debian: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=756892
+    Signed-off-by: Dmitry Smirnov <onlyjob@member.fsf.org>
+    (cherry picked from commit 6ad8e61a428cfc9fc60ccdb9bce812e1f49822ac)
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit fbbdc31a8cd4c6457e06570ddeb9b2021563498e
+Merge: 46d9611 895bddc
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Aug 29 17:31:29 2014 -0700
+
+    Merge pull request #2356 from dachary/wip-9273-mon-preload-erasure-code-firefly
+
+    erasure-code: preload the default plugins in the mon (firefly)
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 46d9611aedd3194cfd55a338e5f94011e6e70668
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri Aug 29 20:21:25 2014 +0100
+
+    osd: OSDMap: ordered blacklist on non-classic encode function
+
+    Fixes: #9211
+    Backport: firefly
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 81102044f417bd99ca570d9234b1df5195e9a8c9)
+
+commit 994a9e35e96984c0806261f7eb349d95c0fb4873
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 26 08:16:29 2014 -0700
+
+    osd/OSDMap: encode blacklist in deterministic order
+
+    When we use an unordered_map the encoding order is non-deterministic,
+    which is problematic for OSDMap.  Construct an ordered map<> on encode
+    and use that.  This lets us keep the hash table for lookups in the general
+    case.
+
+    Fixes: #9211
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4672e50922b75d642056020b9745a3a5844424d3)
+
+commit 895bddc5d4dabd2f05c13c343eb0d7457d6f132f
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Fri Aug 29 18:13:08 2014 +0200
+
+    erasure-code: preload the default plugins in the mon
+
+    The commit 164f1a1959a863848319585fa752250c7b261381 preloads the
+    jerasure plugin in the OSD. They must also be preloaded in the mon for
+    the same reasons.
+
+    http://tracker.ceph.com/issues/9273 Fixes: #9273
+
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+
+commit db8d6e6e190de964ec3d5d1bbc87da2dcb3213f5
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Aug 26 17:36:16 2014 +0100
+
+    mds: fix FP error in ROUND_UP_TO
+
+    Explicitly handle case where denominator is 0, instead of
+    passing into ROUND_UP_TO.
+
+    Regression from 9449520b121fc6ce0c64948386d4ff77f46f4f5f
+
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit bf3e4835dabc057982def1b5c9a6499c04ac5312)
+
+commit 8e3120fcb379a00d370e4c04d34af35e596e2de9
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 21 11:14:39 2014 -0700
+
+    mon: generate cluster_fingerprint if null
+
+    This triggers after an upgrade of a legacy cluster that has no fingerprint.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b245d600163f6337af15aedd1fea68f4e2a668a8)
+
+commit ebcdeb4cfe201dd0c630386226f9970650689ccf
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 20 08:59:46 2014 -0700
+
+    mon: add a cluster fingerprint
+
+    Generate it on cluster creations with the initial monmap.  Include it in
+    the report.  Provide no way for this uuid to be fed in to the cluster
+    (intentionally or not) so that it can be assumed to be a truly unique
+    identifier for the cluster.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 675b0042eff0ad5e1453838410210b1206c39004)
+
+commit b3b029165d4ac0af3f2143fad205f4e8ab08a30f
+Merge: 21e1faa 07e1ceb
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 26 13:13:08 2014 -0700
+
+    Merge pull request #2244 from dachary/wip-9044-use-ruleset-firefly
+
+    erasure-code: OSDMonitor::crush_ruleset_create_erasure needs ruleset (firefly)
+
+commit 21e1faa75da4e9cab8e139a8e2a1c4fc0ee30867
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Aug 12 16:41:38 2014 -0700
+
+    ReplicatedPG::cancel_copy: clear cop->obc
+
+    Otherwise, an objecter callback might still be hanging
+    onto this reference until after the flush.
+
+    Fixes: #8894
+    Introduced: 589b639af7c8834a1e6293d58d77a9c440107bc3
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 5040413054e923d6d5a2b4928162dba140d980e0)
+
+commit e6da7323655f3bfa4f669fb8361fcaa96f633456
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Aug 26 12:02:52 2014 -0700
+
+    PG: recover from each osd at most once
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 8a1723f67f995253c9e6da95ab433743ef23f61c
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Aug 26 11:38:53 2014 -0700
+
+    PG: make the reservation sets more descriptively named
+
+    These sets won't precisely be the backfill_targets or actingbackfill
+    shortly.
+
+    %s/sorted_backfill_set/remote_shards_to_reserve_backfill/g
+    %s/acting_osd_it/remote_recovery_reservation_it/g
+    %s/sorted_actingbackfill_set/remote_shards_to_reserve_recovery/g
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 938d735697e2af0b01dd2613a9152b3caef5db57
+Merge: 2f11631 9ca451a
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Aug 26 10:30:14 2014 -0700
+
+    Merge pull request #2203 from ceph/wip-scrub-firefly
+
+    backport scrub throttling to firefly
+
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+
+commit 2f11631f3144f2cc0e04d718e40e716540c8af19
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Aug 16 12:42:33 2014 -0700
+
+    os/FileStore: fix mount/remount force_sync race
+
+    Consider:
+
+     - mount
+     - sync_entry is doing some work
+     - umount
+       - set force_sync = true
+       - set done = true
+     - sync_entry exits (due to done)
+       - ..but does not set force_sync = false
+     - mount
+     - journal replay starts
+     - sync_entry sees force_sync and does a commit while op_seq == 0
+     ...crash...
+
+    Fixes: #9144
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit dd11042f969b94f7a461d02e1475794031c79f61)
+
+    Conflicts:
+	src/os/FileStore.cc
+
+commit a38cf1bd503a09843d439f65937e7d60aed163e2
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Thu Jul 10 10:32:17 2014 +0800
+
+    Add random_cache.hpp to Makefile.am
+
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit a3e5c6d632119febd2150944a6f2cbce33cfda3a)
+
+commit a2c1532b5610207eccd24b8253643eec6bb05294
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 26 06:42:12 2014 -0700
+
+    os/KeyValueStore, MemStore: fix warning
+
+    os/MemStore.cc: In member function 'void MemStore::_do_transaction(ObjectStore::Transaction&)':
+    os/MemStore.cc:956:18: warning: unused variable 'expected_object_size' [-Wunused-variable]
+    os/MemStore.cc:957:18: warning: unused variable 'expected_write_size' [-Wunused-variable]
+    os/KeyValueStore.cc: In member function 'unsigned int KeyValueStore::_do_transaction(ObjectStore::Transaction&, KeyValueStore::BufferTransaction&, ThreadPool::TPHandle*)':
+    os/KeyValueStore.cc:1426:18: warning: unused variable 'expected_object_size' [-Wunused-variable]
+    os/KeyValueStore.cc:1427:18: warning: unused variable 'expected_write_size' [-Wunused-variable]
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 2b86bf4f4c1be2603368d5031ee7d706056e2d1e
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Apr 29 11:23:58 2014 -0700
+
+    osd: automatically scrub PGs with invalid stats
+
+    If a PG has recnetly split and has invalid stats, scrub it now, even if
+    it has scrubbed recently.  This helps the stats become valid again soon.
+
+    Fixes: #8147
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 68b440d66539e820c9ce86a6942c3188be4ee1ec)
+
+commit 271ca7d9bc41e3a2ac387b05ca9f30d8a250fb62
+Merge: 54333a4 87cd3a8
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 26 06:38:34 2014 -0700
+
+    Merge pull request #2328 from dachary/wip-9209-round-up-to-firefly
+
+    common: ROUND_UP_TO accepts any rounding factor (firefly)
+
+commit 54333a49c9e99a398bf3695845b248ba76ee3930
+Merge: 4bff6f3 58d0871
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 26 06:09:17 2014 -0700
+
+    Merge pull request #2326 from yuyuyu101/wip-kvstore-firefly
+
+    Backport from master to Firefly(KeyValueStore)
+
+commit 87cd3a8f6e190aed06e361595afc4ec83148da98
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Mon Aug 25 17:05:04 2014 +0200
+
+    common: ROUND_UP_TO accepts any rounding factor
+
+    The ROUND_UP_TO function was limited to rounding factors that are powers
+    of two. This saves a modulo but it is not used where it would make a
+    difference. The implementation is changed so it is generic.
+
+    http://tracker.ceph.com/issues/9209 Fixes: #9209
+
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 9449520b121fc6ce0c64948386d4ff77f46f4f5f)
+
+commit 58d08714340049a5165ad682ec5b54292525b45d
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Thu Mar 20 14:09:49 2014 +0800
+
+    Remove exclusive lock on GenericObjectMap
+
+    Now most of GenericObjectMap interfaces use header as argument not the union of
+    coll_t and ghobject_t. So caller should be responsible for maintain the
+    exclusive header.
+
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+
+commit 98df982a6f830154ff6880f5ee00350ba3375b6b
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Wed Jul 23 11:26:18 2014 +0800
+
+    common/RandomCache: Fix inconsistence between contents and count
+
+    The add/clear method may cause count inconsistent with the real size of
+    contents.
+
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+
+commit c8ec7721526ccf4bfd91a220fee0dd75bef8fb21
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Tue Aug 26 04:41:28 2014 +0000
+
+    Add random cache and replace SharedLRU in KeyValueStore
+
+    SharedLRU plays pool performance in KeyValueStore with large header cache size,
+    so a performance optimized RandomCache could improve it.
+
+    RandomCache will record the lookup frequency of key. When evictint element,
+    it will randomly compare several elements's frequency and evict the least
+    one.
+
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+
+    Conflicts:
+
+	src/common/config_opts.h
+	src/os/KeyValueStore.cc
+
+commit 33ca26a103a0d99627fddc30f2916c5aeb37c691
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Tue Aug 26 04:40:16 2014 +0000
+
+    Add Header cache to KeyValueStore
+
+    In the performance statistic recently, the header lookup becomes the main time
+    consuming for the read/write operations. Most of time it occur 50% to deal with
+    header lookup, decode/encode logics.
+
+    Now adding header cache using SharedLRU structure which will maintain the header
+    cache and caller will get the pointer to the real header. It also avoid too much
+    header copy operations overhead.
+
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+
+    Conflicts:
+
+	src/os/KeyValueStore.cc
+	src/os/KeyValueStore.h
+
+commit e8dcb8cc603cff64bafc11ee42c5c04a9cf85d12
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Wed Feb 26 17:46:07 2014 +0800
+
+    Fix write operation on a deleted object in the same transaction
+
+    If the following op happened:
+    touch obj
+    delete obj
+    write obj
+
+    KeyValueStore will fail at "write" operation.
+
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+
+commit e0ae6f9de685b60eee5efa5ca960c7a15481e41a
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Tue Aug 26 04:35:57 2014 +0000
+
+    Remove SequencerPosition from KeyValueStore
+
+    Now KeyValueStore expects kv backend to ensure consistency and there is unusable
+    for KeyValueStore to store a SequencerPosition.
+
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+
+    Conflicts:
+
+	src/os/KeyValueStore.cc
+	src/os/KeyValueStore.h
+
+commit 44721f71266e3098169ed00f1ca7e86496b76b07
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Wed Jun 4 12:58:07 2014 +0800
+
+    Fix keyvaluestore fiemap bug
+
+    The result of fiemap is wrong and the offset get from
+    "StripObjectMap::file_to_extents" need to multiply by sequence number
+
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+
+commit 4bff6f3476e5df400db3dc1f0ad7b35a2a41830e
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 21 13:05:35 2014 -0700
+
+    mon: fix occasional message leak after session reset
+
+    Consider:
+
+     - we get a message, put it on a wait list
+     - the client session resets
+     - we go back to process the message later and discard
+       - _ms_dispatch returns false, but nobody drops the msg ref
+
+    Since we call _ms_dispatch() a lot internally, we need to always return
+    true when we are an internal caller.
+
+    Fixes: #9176
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 19df386b2d36d716be2e6d02de0386fac9e7bc1f)
+
+commit ca3ac907aa73c5d77760e12da6f1509ea860e894
+Merge: fdbab46 8d7e77b
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 21 10:14:18 2014 -0700
+
+    Merge pull request #2298 from dachary/wip-9153-jerasure-upgrade-firefly
+
+    erasure-code: preload the jerasure plugin variant (sse4,sse3,generic)
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 8d7e77b9747f1b3dc3c10d1b0877a4b9c899ba86
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Thu Aug 21 14:41:55 2014 +0200
+
+    erasure-code: preload the jerasure plugin variant (sse4,sse3,generic)
+
+    The preloading of the jerasure plugin ldopen the plugin that is in
+    charge of selecting the variant optimized for the
+    CPU (sse4,sse3,generic). The variant plugin itself is not loaded because
+    it does not happen at load() but when the factory() method is called.
+
+    The JerasurePlugin::preload method is modified to call the factory()
+    method to load jerasure_sse4 or jerasure_sse3 or jerasure_generic as a
+    side effect.
+
+    Indirectly loading another plugin in the factory() method is error prone
+    and should be moved to the load() method instead. This change should be
+    done in a separate commit.
+
+    http://tracker.ceph.com/issues/9153 Fixes: #9153
+
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+
+commit fdbab46852e74d405b5c747da98564a5866ec8a7
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Tue May 20 14:32:18 2014 +0800
+
+    Fix set_alloc_hint op cause KeyValueStore crash problem
+
+    Now KeyValueStore doesn't support set_alloc_hit op, the implementation of
+    _do_transaction need to consider decoding the arguments. Otherwise, the
+    arguments will be regarded as the next op.
+
+    Fix the same problem for MemStore.
+
+    Fix #8381
+
+    Reported-by: Xinxin Shu <xinxin.shu5040@gmail.com>
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit c08adbc98ff5f380ecd215f8bd9cf3cab214913c)
+
+commit 922e24e658f09a8a772309f18a8d54c560fad653
+Merge: ae787cf 164f1a1
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 20 10:10:08 2014 -0700
+
+    Merge pull request #2286 from dachary/wip-9153-jerasure-upgrade-firefly
+
+    erasure-code: preload the jerasure plugin (firefly)
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 164f1a1959a863848319585fa752250c7b261381
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Tue Aug 19 01:30:15 2014 +0200
+
+    erasure-code: preload the jerasure plugin
+
+    Load the jerasure plugin when ceph-osd starts to avoid the following
+    scenario:
+
+    * ceph-osd-v1 is running but did not load jerasure
+
+    * ceph-osd-v2 is installed being installed but takes time : the files
+      are installed before ceph-osd is restarted
+
+    * ceph-osd-v1 is required to handle an erasure coded placement group and
+      loads jerasure (the v2 version which is not API compatible)
+
+    * ceph-osd-v1 calls the v2 jerasure plugin and does not reference the
+      expected part of the code and crashes
+
+    Although this problem shows in the context of teuthology, it is unlikely
+    to happen on a real cluster because it involves upgrading immediately
+    after installing and running an OSD. Once it is backported to firefly,
+    it will not even happen in teuthology tests because the upgrade from
+    firefly to master will use the firefly version including this fix.
+
+    While it would be possible to walk the plugin directory and preload
+    whatever it contains, that would not work for plugins such as jerasure
+    that load other plugins depending on the CPU features, or even plugins
+    such as isa which only work on specific CPU.
+
+    http://tracker.ceph.com/issues/9153 Fixes: #9153
+
+    Backport: firefly
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 9b802701f78288ba4f706c65b853415c69002d27)
+
+    Conflicts:
+	src/test/erasure-code/test-erasure-code.sh
+	src/common/config_opts.h
+
+commit ae787cfa88dfd0f5add5932b297258c46af4e333
+Author: Matt Benjamin <matt@linuxbox.com>
+Date:   Thu May 29 10:34:20 2014 -0400
+
+    Work around an apparent binding bug (GCC 4.8).
+
+    A reference to h->seq passed to std::pair ostensibly could not bind
+    because the header structure is packed.  At first this looked like
+    a more general unaligned access problem, but the only location the
+    compiler rejects is a false positive.
+
+    Signed-off-by: Matt Benjamin <matt@linuxbox.com>
+    (cherry picked from commit c930a1f119069a424af28a618b0abff4947c221f)
+
+commit 486deefdc1496b72cc680e432a61d71e5fa265c9
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Aug 17 20:54:28 2014 -0700
+
+    qa/workunits/rbd/qemu-iotests: touch common.env
+
+    This seems to be necessary on trusty.
+
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 055be68cf8e1b84287ab3631a02e89a9f3ae6cca)
+
+commit cc4e6258d67fb16d4a92c25078a0822a9849cd77
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri May 23 16:52:08 2014 +0100
+
+    unittest_strtol: fix compilation warning
+
+    Was fixed in master by a4923f5bc373d530d1ffdf6c58a4d88139daedd2
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 7f9fe22a1c73d5f2783c3303fb9f3a7cfcea61c5
+Author: huangjun <hjwsm1989@gmail.com>
+Date:   Tue Jun 17 13:12:58 2014 +0800
+
+      Fix EINVAL err when use "ceph tell osd.* bench"
+
+      Signed-off-by: huangjun <hjwsm1989@gmail.com>
+    (cherry picked from commit 7dc93a9651f602d9c46311524fc6b54c2f1ac595)
+
+commit 14607fc045aa169f1e8fa6872b412fba8f09d645
+Author: Ma Jianpeng <jianpeng.ma@intel.com>
+Date:   Wed Jul 16 17:48:34 2014 -0700
+
+    qa/workunits/cephtool/test.sh: fix get erasure_code_profile test
+
+    Manual backport of 4d6899c7560e990650959b442980a7249f0ba4c1
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 068b906ee396f1ad717968f214610ea86dd8d6b4
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri Jun 27 21:41:18 2014 +0100
+
+    mon: OSDMonitor: add 'osd pool get-quota' command
+
+    Enables us to obtain current quotas for a given pool.
+
+    Fixes: #8523
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 714a9bb5a058b2553f3be3e4cfb7e7f30150e75a)
+
+commit 2ed0fa034aa8e49ab703bdb798bd6211a83e3a9d
+Author: John Spray <jspray@redhat.com>
+Date:   Tue Jun 3 10:12:41 2014 +0100
+
+    mon: name instead of id in "has tiers" message
+
+    Instead of "Pool foo has tiers 1 2" print
+    "Pool foo has tiers bar baz".
+
+    Signed-off-by: John Spray <jspray@redhat.com>
+    (cherry picked from commit 97772c2f53f726bd71710d0d3e34159d2679390a)
+
+commit 7e509b1b8b5a3679094f794cd9334598aef4441e
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri May 23 17:01:38 2014 +0100
+
+    common/config.cc: allow integer values to be parsed as SI units
+
+    We are allowing this for all and any integer values; that is, OPT_INT,
+    OPT_LONGLONG, OPT_U32 and OPT_U64.
+
+    It's on the user to use appropriate units.  For instance, the user should
+    not use 'E(xabyte)' when setting a signed int, and use his best judgment
+    when setting options that, for instance, ought to receive seconds.
+
+    Fixes: 8265
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 5500437e064cd6b4b45d63ee9396193df87f4d44)
+
+commit 4e34d83e337d0febf20c86b051721878feef49a8
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri May 23 16:52:08 2014 +0100
+
+    test/strtol.cc: Test 'strict_strtosi()'
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 40587d4792fd55db72d33870aae8b6a806c9baaf)
+
+commit 3b06b8f4893bd18440ecb9705414831a827cef90
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri May 23 16:51:37 2014 +0100
+
+    common/strtol.cc: strict_strtosi() converts str with SI units to uint64_t
+
+    Accepts values with a suffix (B, K, M, G, T, P, E) and returns the
+    appropriate byte value.
+
+    E.g., 10B = 10, while 10K = 10240.
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 67dc5751ba9a4e527ff12ea65000d1ba45d956f6)
+
+commit 92ab6ecf321ab68ea8fe3e8c2b310303a5726af3
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Wed Aug 13 15:50:20 2014 -0400
+
+    ceph-disk: linter cleanup
+
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit d74ed9d53fab95f27a9ad8e9f5dab7192993f6a3)
+
+commit 08772fd888fb0eca4570bd6b4bb25f8122691cb4
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 13 12:00:50 2014 -0700
+
+    ceph-disk: warn about falling back to sgdisk (once)
+
+    This way the user knows something funny might be up if dmcrypt is in use.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6f7798e37e098de38fbc73f86c4c6ee705abbe38)
+
+commit 50166efd205f46fa325dec9636d817387e5d4d3b
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 13 11:40:34 2014 -0700
+
+    ceph-disk: only fall back to sgdisk for 'list' if blkid seems old
+
+    If the blkid doesn't show us any ID_PART_ENTRY_* fields but we know it is
+    a GPT partition, *then* fallback.  Otherwise, don't bother.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b1651afb34d9d2c324db3bf5f54ac9ce001c6af9)
+
+commit a45e815cb00f1a807691cbf687990839995f7629
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 13 11:39:47 2014 -0700
+
+    ceph-disk: add get_partition_base() helper
+
+    Return the base devices/disk for a partition device.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b75e8a340c49cbc067baa19790b994a5f904bb4f)
+
+commit 8c04e475efb4471c605e3c921ec8a03b67934899
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 12 17:26:07 2014 -0700
+
+    ceph-disk: display information about dmcrypted data and journal volumes
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c7a1ceba441fa99a82e19ed2cd3c6782a5d77636)
+
+commit b09e659226b26002b58c7697f8c600d06fe78c58
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 12 17:25:42 2014 -0700
+
+    ceph-disk: move fs mount probe into a helper
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f80ed26d2403ba12e80da6459fc45c22584f72de)
+
+commit 6775f46e4a5862ddbf81bef11016bc036222fda9
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 12 17:25:10 2014 -0700
+
+    ceph-disk: use partition type UUIDs, and blkid
+
+    Use blkid to give us the GPT partition type.  This lets us distinguish
+    between dmcrypt and non-dmcrypt partitions.  Fake it if blkid doesn't
+    give us what we want and try with sgdisk.  This isn't perfect (it can't
+    tell between dmcrypt and not dmcrypt), but such is life, and we are better
+    off than before.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6c77f5f2f994c881232d76ce9c69af80d10772bd)
+
+commit 1804a83400b7c3dc07650e09da5224bb999d0940
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 12 13:53:16 2014 -0700
+
+    ceph-disk: fix log syntax error
+
+      File "/usr/sbin/ceph-disk", line 303, in command_check_call
+        LOG.info('Running command: %s' % ' '.join(arguments))
+    TypeError: sequence item 2: expected string, NoneType found
+
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1088d6cd11b476cd67ed30e07edd363c4057a003)
+
+commit 8a8a9c35349d3f58d899afa1a9e219b99586bc9f
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Aug 11 15:58:15 2014 -0700
+
+    Revert "Fix for bug #6700"
+
+    This reverts commit 673394702b725ff3f26d13b54d909208daa56d89.
+
+    This appears to break things when the journal and data disk are *not* the same.
+    And I can't seem to reproduce the original failure...
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2edf01ffa4a7425af2691b4e94bc5fd0bfab1e5b)
+
+commit 50b700fadf6ce6e0bf14b5daa2288786426d7359
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Aug 11 15:57:52 2014 -0700
+
+    ceph-disk: fix verify_no_in_use check
+
+    We only need to verify that partitions aren't in use when we want to
+    consume the whole device (osd data), not when we want to create an
+    additional partition for ourselves (osd journal).
+
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d6e6ba198efc4b3afff0c70af53497a70c6b3f19)
+
+commit c9847ef059f182ad15ef027c8bdfae6c99d91867
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Thu May 22 17:04:28 2014 -0400
+
+    better error reporting on incompatible device requirements
+
+    Signed-off-by: Alfredo Deza <alfredo@deza.pe>
+    (cherry picked from commit 1ac3a503a15ddf7f7c1a33310a468fac10a1b7b6)
+
+commit 40649902891244242d5861ccd9538aec026cae3a
+Author: Stuart Longland <stuartl@vrt.com.au>
+Date:   Tue May 6 14:06:36 2014 -0700
+
+    ceph-disk: fix list for encrypted or corrupt volume
+
+    Continue gracefully if an fs type is not detected, either because it is
+    encrypted or because it is corrupted.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 09beebe3f1fd1b179547743648049b891cb8bc56)
+
+commit 80896dcf2c3ccba0b9b0b1bd9f6e075585e3d502
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Fri Jun 13 09:37:33 2014 -0400
+
+    support dmcrypt partitions when activating
+
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit ef8a1281512c4ee70a3764b28891da691a183804)
+
+commit 1b0da81d733a36de4a9dc80263058a6cdf05c745
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Aug 15 16:41:43 2014 -0700
+
+    init-ceph: don't use bashism
+
+           -z STRING
+                  the length of STRING is zero
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 0d6d1aa7e0c5e0b5f99c9b548a1f890c511b4299)
+
+commit 9005f33d0d80d6d2b93493b38866b7f74f87d1eb
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Aug 15 14:28:57 2014 -0700
+
+    osd: fix feature requirement for mons
+
+    These features should be set on the client_messenger, not
+    cluster_messenger.
+
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ae0b9f17760eda9a7e09a6babac50bfe8ebb4b36)
+
+commit 9e46c5ad7dadd1c342c82629577b5a1bf037fe75
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Aug 15 13:54:11 2014 -0700
+
+    unittest_osdmap: test EC rule and pool features
+
+    TODO: tiering feature bits.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2f0e2951d773b6acce781b4b991d6d8e817ee2f9)
+
+commit dbbe0c627bcd4a60c1346bb120698b5a0c2192c2
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Aug 15 14:04:05 2014 -0700
+
+    unittest_osdmap: create an ec pool in test osdmap
+
+    This is part of 7294e8c4df6df9d0898f82bb6e0839ed98149310.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit c2aa74a22718d9a59d60201d71213aaff2492ff8
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Aug 15 08:55:10 2014 -0700
+
+    osd: only require crush features for rules that are actually used
+
+    Often there will be a CRUSH rule present for erasure coding that uses the
+    new CRUSH steps or indep mode.  If these rules are not referenced by any
+    pool, we do not need clients to support the mapping behavior.  This is true
+    because the encoding has not changed; only the expected CRUSH output.
+
+    Fixes: #8963
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 16dadb86e02108e11a970252411855d84ab0a4a2)
+
+commit 12430fed85b2cfeaeb33a443e6a4c6538d0833f8
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Aug 15 08:52:37 2014 -0700
+
+    crush: add is_v[23]_rule(ruleid) methods
+
+    Add methods to check if a *specific* rule uses v2 or v3 features.  Refactor
+    the existing checks to use these.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1d95486780a54c85a5c88936a4da4bdc3576a7b8)
+
+commit 2427ea8f153caf55b3bce77cd2fc88935563c7b3
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Jun 30 13:40:07 2014 -0700
+
+    PGLog: fix clear() to avoid the IndexLog::zero() asserts
+
+    Introduced in:
+      c5b8d8105d965da852c79add607b69d5ae79a4d4
+      ac11ca40b4f4525cbe9b1778b1c5d9472ecb9efa
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+    (cherry picked from commit 959f2b25910360b930183fbf469ce984a48542dd)
+
+commit 9ca451a6bf2f29e75f72cbd3dbcff0bf8986e462
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jun 19 12:34:36 2014 -0700
+
+    osd: allow io priority to be set for the disk_tp
+
+    The disk_tp covers scrubbing, pg deletion, and snap trimming
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit d9073f486527ca13cdb2774745c4c63c218333ad)
+
+commit 11858d7e7a493c9493f7039b7e45dc03fc4feb90
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 18 11:02:09 2014 -0700
+
+    common/WorkQueue: allow io priority to be set for wq
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit dd6badcb5eedfec6748b3e6ca4d46e3b266038f6)
+
+    Conflicts:
+
+	src/common/WorkQueue.cc
+
+commit b75f85a2c4dd9807947862f7b89a5f25dfa1defe
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 18 11:01:42 2014 -0700
+
+    common/Thread: allow io priority to be set for a Thread
+
+    Ideally, set this before starting the thread.  If you set it after, we
+    could potentially race with create() itself.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1b8741022c5a2ebae38905215dadee696433e931)
+
+commit 9f29788fbc19aea341b4fe997a567aa3054b9d1c
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 18 11:01:09 2014 -0700
+
+    common/io_priority: wrap ioprio_set() and gettid()
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit a2b49110ef65efd526c3430ad03c988ca9dde768)
+
+commit fac5bfafef1eb82f8cf23c369fb5f50502e2e6f3
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Jun 17 10:47:24 2014 -0700
+
+    osd: introduce simple sleep during scrub
+
+    This option is similar to osd_snap_trim_sleep: simply inject an optional
+    sleep in the thread that is doing scrub work.  This is a very kludgey and
+    coarse knob for limiting the impact of scrub on the cluster, but can help
+    until we have a more robust and elegant solution.
+
+    Only sleep if we are in the NEW_CHUNK state to avoid delaying processing of
+    an in-progress chunk.  In this state nothing is blocked on anything.
+    Conveniently, chunky_scrub() requeues itself for each new chunk.
+
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit c4e8451cc5b4ec5ed07e09c08fb13221e31a7ac6)
+
+commit ad82f28122dc41095d5e703279a8a5c494d35913
+Author: Sage Weil <sage@inktank.com>
+Date:   Sat Jun 14 10:30:50 2014 -0700
+
+    osd: add sanity check/warning on a few key configs
+
+    Warn when certain config values are set to bad values.
+
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit f3ec7d0b23fdee39a34bda7595cd2a79c08daf8a)
+
+commit 289360c5dc57a4788289472f90d63781143539be
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 1 17:24:48 2014 -0700
+
+    osd: prevent pgs from getting too far ahead of the min pg epoch
+
+    Bound the range of PG epochs between the slowest and fastest pg
+    (epoch-wise) with 'osd map max advance'.  This value should be set to
+    something less than 'osd map cache size' so that the maps we are
+    processing will be in memory as many PGs advance forward in time in
+    loose synchrony.
+
+    This is part of the solution to #7576.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cf25bdf6b0090379903981fe8cee5ea75efd7ba0)
+
+commit 662af7ac83b67af8c9edc554bbd505de717f9709
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 7 17:42:06 2014 -0700
+
+    osd: fix pg epoch floor tracking
+
+    If you call erase() on a multiset it will delete all instances of a value;
+    we only want to delete one of them.  Fix this by passing an iterator.
+
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a52a855f6c92b03dd84cd0cc1759084f070a98c2)
+
+commit 229a346d824d84c361d7963a356af5251adc9e2e
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Apr 2 14:29:08 2014 -0700
+
+    osd: track per-pg epochs, min
+
+    Add some simple tracking so that we can quickly determine what the min
+    pg osdmap epoch is.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 81e4c47722255ac3d46f701a80e104cc390e766c)
+
+commit 38c3a3c0b0d1903aea2fdc6224fc4f70bbc39329
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 13 13:31:10 2014 -0700
+
+    mon: fix divide by zero when pg_num adjusted and no osds
+
+    Fixes: #9052
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 239401db7b51541a57c59a261b89e0f05347c32d)
+
+commit 87bf00a2cb93ce2dc904a42c45cc9f43023725c3
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Aug 10 12:48:29 2014 -0700
+
+    ceph_test_rados_api_tier: fix cache cleanup (ec too)
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d7fb7bf5f2059f411633751e376c2270e6040fba)
+
+commit 3be0d731cbe158d837d05c907aefe16aa95977be
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Aug 10 12:15:38 2014 -0700
+
+    ceph_test_rados_api: fix cleanup of cache pool
+
+    We can't simply try to delete everything in there because some items may
+    be whiteouts.  Instead, flush+evict everything, then remove overlay, and
+    *then* delete what remains.
+
+    Fixes: #9055
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ebbe8aab17cfa2281902f167a706639535da4010)
+
+commit 7f511ef7fd55cca8f1babaf696ca0faa0e1e411d
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Aug 10 11:41:23 2014 -0700
+
+    librados/TestCase: inheret cleanup_default_namespace
+
+    No need to duplicate this code.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1d199fb1173dc9685dba96c22b83e4e9edf51a11)
+
+commit 4a5c93a90986de4510c50f0fba72ae7d5aed8a15
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 13 10:34:53 2014 -0700
+
+    osd/ReplicatedPG: only do agent mode calculations for positive values
+
+    After a split we can get negative values here.  Only do the arithmetic if
+    we have a valid (positive) value that won't through the floating point
+    unit for a loop.
+
+    Fixes: #9082
+    Tested-by: Karan Singh <karan.singh@csc.fi>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5be56ff86d9f3ab2407a258a5285d0b8f52f041e)
+
+commit 39bcafa6f2b14e4e11e1541cf01f24525e2b6449
+Merge: 7da121d cb48cd4
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Aug 12 21:15:26 2014 -0700
+
+    Merge pull request #2231 from ceph/wip-8944-firefly
+
+    Wip 8944 firefly
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 7da121d2aa1ea5c5c8accef92d7304912d4b2eb3
+Author: Greg Farnum <greg@inktank.com>
+Date:   Mon Jul 28 18:33:56 2014 -0700
+
+    OSD: add require_same_peer_inst(OpRequestRef&,OSDMap&) helper
+
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit e99acf9810976b1fc74b84ad289773af43be973f)
+
+    Conflicts:
+
+	src/osd/OSD.cc
+
+commit 8595e9bed3689933c03a8f3443052a36ff1d62f5
+Author: Greg Farnum <greg@inktank.com>
+Date:   Mon Jul 28 14:19:59 2014 -0700
+
+    OSD: introduce require_self_aliveness(OpRequestRef&,epoch_t) function
+
+    Take the self-aliveness checks out of require_same_or_newer_map() and use
+    the new function for that and for require_up_osd_peer().
+
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit e179e9227b4a4482d8359682092fd7f426b9a919)
+
+    Conflicts:
+
+	src/osd/OSD.cc
+
+commit d0f2c4891bbcc1938bee15b3fc0bfb796ce4c7de
+Author: Greg Farnum <greg@inktank.com>
+Date:   Mon Jul 28 14:08:30 2014 -0700
+
+    OSD: use OpRequestRef& for a few require_* functions
+
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit eb2f1ea2c33647934af878b504383829f5a198ce)
+
+commit 8d395f1cbbc63f4cd5d166fd0a667938c83f303c
+Author: Greg Farnum <greg@inktank.com>
+Date:   Tue Jul 22 16:57:00 2014 -0700
+
+    OSD: introduce require_up_osd_peer() function for gating replica ops
+
+    This checks both that a Message originates from an OSD, and that the OSD
+    is up in the given map epoch.
+    We use it in handle_replica_op so that we don't inadvertently add operations
+    from down peers, who might or might not know it.
+
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit ccd0eec50103b919b3eb6eea96f7dc6438520ed3)
+
+commit 72c2474e05b918535047181d876678a6a077aa1d
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Fri Aug 8 10:16:20 2014 -0400
+
+    init-ceph: conditionally update  after argparsing
+
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit 44ed1885fb588aad7e4729373e06c5933178b5e5)
+
+commit 8d7fb538e0163f828289d7da719692bd36a66f83
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Thu Aug 7 11:11:57 2014 -0400
+
+    make ceph-disk use the new init flag for cluster
+
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit d0ccb1cbef86b8ccd8967af892e3582ee80edd55)
+
+commit 260a9c84630ca6956eb772d87abdc3bd4c4bb8fc
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Thu Aug 7 10:48:09 2014 -0400
+
+    allow passing a --cluster flag to the init script
+
+    It will fallback to looking at /etc/ceph/$cluster.conf
+    otherwise.
+
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit 23b4915fa252852cf02cc42a1b6143e2cd61b445)
+
+commit cb5ecdce68dcd4e4bc32b5c7c029cfd7ffe7c80e
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Wed Aug 6 15:30:30 2014 -0400
+
+    use cluster name when checking the host
+
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit bdf5f30010117c71a227855a47fe22ab55b3e0b8)
+
+commit 79452540826d0aed7b94d374cc61b5002e89e4a4
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Wed Aug 6 15:30:09 2014 -0400
+
+    use  name in /var/lib/ceph paths
+
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit 3fb25c28e8e3cc62bc541538cc11f5ed41471611)
+
+commit 92755d1060ea06f13577cd2fffda6c7984a610b2
+Author: Alfredo Deza <alfredo.deza@inktank.com>
+Date:   Wed Aug 6 15:29:42 2014 -0400
+
+    when starting an OSD, pass in the cluster name
+
+    Signed-off-by: Alfredo Deza <alfredo.deza@inktank.com>
+    (cherry picked from commit 9e08e3a7794ce007c80c8fdb56e96900eb5240a3)
+
+commit 272b26f5aab71c22a1954ff92548b535a1f018d8
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Aug 1 14:04:35 2014 -0700
+
+    osd_types: s/stashed/rollback_info_completed and set on create
+
+    Originally, this flag indicated that the object had already been stashed and
+    that therefore recording subsequent changes is unecessary.  We want to set it
+    on create() as well since operations like [create, writefull] should not need
+    to stash the object.
+
+    Fixes: #8625
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 4260767973d96978e808cb87ef6ae991104b4f8d)
+
+commit 7b5bcb943578b99f5a2aeeae0bbc67f371c30233
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Thu Jun 19 18:27:49 2014 +0100
+
+    osd: use appropriate json types instead of stream when dumping info
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 363496ad878c685008ccd87f26f121d62a48c24c)
+
+commit daec3226e073b8d6047eb9111f3deb39b1001647
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Thu Jun 19 16:52:06 2014 +0100
+
+    osd: have 'tid' dumped as a json unsigned int where appropriate
+
+    Most of the places dumping it would dump it as a string. Others as an int.
+    Just a couple would dump it as an unsigned int.
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 3f0ea9512be78d10ac0bc605a38e83cfbab7ffee)
+
+commit 439bce34bc25cd7c8eba7ffc124185d810d3711f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat Aug 2 13:01:05 2014 -0700
+
+    rgw: need to pass need_to_wait for throttle_data()
+
+    need_to_wait wasn't passed into processor->throttle_data(). This was
+    broken in fix for #8937.
+
+    CID 1229541:    (PW.PARAM_SET_BUT_NOT_USED)
+
+    Backport: firefly
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit e93818df33286a2a7f73b593dc20da412db4e0a6)
+    (cherry picked from commit 8519e9ab06caf6999e1243514a93eaffb155ab2e)
+
+commit fc1a4b5fdce7b92ba1b321dec37c4c75501e3521
+Author: Sylvain Munaut <s.munaut@whatever-company.com>
+Date:   Thu Jun 5 11:28:27 2014 +0200
+
+    rgw: Don't send error body when it's a HEAD request
+
+    The main 'operation' know not to send any body for HEAD requests.
+
+    However for errors, this was not the case, the formatter would be
+    flushed and would send the error 'message' in the body in all cases.
+
+    For the FastCGI case it doesn't seem to be an issue, it's possible
+    that the webserver (apache/lighttpd/...) cleans up the response
+    into shape.
+
+    But when using the new civetweb frontend this cause invalid HTTP.
+
+    Backport: firefly
+    Fixes #8539
+
+    Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
+    Signed-off-by: Sylvain Munaut <s.munaut@whatever-company.com>
+    (cherry picked from commit 0a2b4c25541bbd15776d3d35986518e37166910f)
+
+commit e3bc15343720d0455e32e429e7cf865037bd90d7
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jul 30 13:57:34 2014 -0700
+
+    osd: prevent old clients from using tiered pools
+
+    If the client is old and doesn't understand tiering, don't let them use a
+    tiered pool.  Reply with EOPNOTSUPP.
+
+    Fixes: #8714
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 0190df53056834f219e33ada2af3a79e8c4dfb77)
+
+commit f3113e9adde54fa50a64010381f5b6c113f5cda0
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jul 31 11:02:55 2014 -0700
+
+    mon/OSDMonitor: warn when cache pools do not have hit_sets configured
+
+    Give users a clue when cache pools are enabled but the hit_set is not
+    configured.  Note that technically this will work, but not well, so for
+    now let's just steer them away.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 383536a2ef2506c5591b25c0fd61dc8c181fdab9)
+
+    NOTE: removed readforward state for backport
+
+commit aaf732f7475c075f8bf8e79bc08b11a020e5fadf
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jul 31 09:28:12 2014 -0700
+
+    osd/ReplicatedPG: improve agent_choose_mode args
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit caf554bf2f752b50670d118c99843b7c4a2967b4)
+
+commit 76a519b215e591e0948d627fb1de826ed0bf415d
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jul 31 09:26:03 2014 -0700
+
+    osd/ReplicatedPG: evict blindly if there is no hit_set
+
+    If there is no hit set for a PG, blindly evict objects.  This avoids an
+    assert(hit_set) in agent_estimate_atime_temp().
+
+    Fixes: #8982
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ea4996d3ef15c5d5a776d162b08d9fad6859c976)
+
+commit 8ff48d79697e98dca04799ee5982a5308a4efc92
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jul 31 09:13:11 2014 -0700
+
+    osd/ReplicatedPG: check agent_mode if agent is enabled but hit_sets aren't
+
+    It is probably not a good idea to try to run the tiering agent without a
+    hit_set to inform its actions, but it is technically possible.  For
+    example, one could simply blindly evict when we reach the full point.
+    However, this doesn't work because the agent mode is guarded by a hit_set
+    check, even though agent_setup() is not.  Fix that.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5d1c76f641310f5f65600f70ae76945b2aa472d7)
+
+commit f98a6194a9a9f5794c38814c1c33d13b7fcea538
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 12 07:05:34 2014 -0700
+
+    common/LogClient: fix sending dup log items
+
+    We need to skip even the most recently sent item in order to get to the
+    ones we haven't sent yet.
+
+    Fixes: #9080
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 057c6808be5bc61c3f1ac2b956c1522f18411245)
+
+commit bbb1e6a3a84b2d853f935687f01016e7c6a67951
+Author: Pavan Rallabhandi <pavan.rallabhandi@sandisk.com>
+Date:   Wed Aug 6 15:10:14 2014 +0530
+
+    RadosClient: Fixing potential lock leaks.
+
+    In lookup_pool and pool_delete, a lock is taken
+    before invoking wait_for_osdmap, but is not
+    released for the failure case of the call. Fixing the same.
+
+    Fixes: #9022
+    Signed-off-by: Pavan Rallabhandi <pavan.rallabhandi@sandisk.com>
+    (cherry picked from commit f1aad8bcfc53f982130dbb3243660c3c546c3523)
+
+commit e767254ce9a1610f3d640d8157fcb0d1fdf8edf2
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Aug 11 16:41:26 2014 -0700
+
+    librbd: fix error path cleanup for opening an image
+
+    If the image doesn't exist and caching is enabled, the ObjectCacher
+    was not being shutdown, and the ImageCtx was leaked. The IoCtx could
+    later be closed while the ObjectCacher was still running, resulting in
+    a segfault. Simply use the usual cleanup path in open_image(), which
+    works fine here.
+
+    Fixes: #8912
+    Backport: dumpling, firefly
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 3dfa72d5b9a1f54934dc8289592556d30430959d)
+
+commit dec028d113ab4bcc7106fea0bf413b54829f67e7
+Merge: e087862 c7fbf27
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 12 14:38:33 2014 -0700
+
+    Merge remote-tracking branch 'gh/firefly-next' into firefly
+
+commit 07e1ceb55ac3cf0bea58cec27737e45391de9106
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 10 17:10:04 2014 +0200
+
+    erasure-code: ErasureCodeJerasure::create_ruleset must return a ruleset
+
+    CrushWrapper::add_simple_ruleset does not return a ruleset, it returns
+    a ruleid that must be converted into a ruleset before being returned.
+
+    http://tracker.ceph.com/issues/9044 Fixes: #9044
+
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 0029a35872d3fc15f9a0d60d095b2e111d6e98a6)
+
+commit 96047f1de4e0591a294e31ed6af6edfb3b2357e4
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 10 17:06:33 2014 +0200
+
+    erasure-code: OSDMonitor::crush_ruleset_create_erasure needs ruleset
+
+    When OSDMonitor::crush_ruleset_create_erasure checks the ruleset for
+    existence, it must convert the ruleid into a ruleset before assigning it
+    back to the *ruleset parameter.
+
+    http://tracker.ceph.com/issues/9044 Fixes: #9044
+
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 04a484a4d5800c9bcf9805d3e87ed0c8dee01c9a)
+
+commit e087862d5bd894572a6c223b45f4275cadd4b480
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Mon Jul 14 14:27:17 2014 +0800
+
+    Add rbdcache max dirty object option
+
+    Librbd will calculate max dirty object according to rbd_cache_max_size, it
+    doesn't suitable for every case. If user set image order 24, the calculating
+    result is too small for reality. It will increase the overhead of trim call
+    which is called each read/write op.
+
+    Now we make it as option for tunning, by default this value is calculated.
+
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit 3c7229a2fea98b30627878c86b1410c8eef2b5d7)
+
+commit 5f74fb294a0cd9b973aae45de17f38436bac6d61
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Jun 4 23:22:18 2014 +0200
+
+    librbd/internal.cc: check earlier for null pointer
+
+    Fix potential null ponter deref, move check for 'order != NULL'
+    to the beginning of the function to prevent a) deref in ldout() call
+    and b) to leave function as early as possible if check fails.
+
+    [src/librbd/internal.cc:843] -> [src/librbd/internal.cc:865]: (warning)
+     Possible null pointer dereference: order - otherwise it is redundant
+     to check it against null.
+
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 3ee3e66a9520a5fcafa7d8c632586642f7bdbd29)
+
+commit b2189465f36fc375b864f88a47d25c7808c08901
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Apr 24 14:47:24 2014 -0700
+
+    librbd: add an interface to invalidate cached data
+
+    This is useful for qemu to guarantee live migration with caching is
+    safe, by invalidating the cache on the destination before starting it.
+
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 5d340d26dd70192eb0e4f3f240e3433fb9a24154)
+
+commit f7dc58552a9ee228b7fa2a10094e962ed66fe8fd
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Thu Apr 24 14:43:35 2014 -0700
+
+    librbd: check return code and error out if invalidate_cache fails
+
+    This will only happen when shrinking or rolling back an image is done
+    while other I/O is in flight to the same ImageCtx. This is unsafe, so
+    return an error before performing the resize or rollback.
+
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit e08b8b66c77be3a3d7f79d91c20b1619571149ee)
+
+commit c7fbf272a202cd8a72d91b7a55e53fbb5cbbbadd
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Aug 9 13:23:06 2014 -0700
+
+    os/FileStore: dump open fds before asserting
+
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4e8de1792731cf30f2744ab0659d036adc0565a3)
+
+commit 978f5749f3d08bb4c96529ca2d1b10760ee2606d
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 25 13:51:45 2014 -0700
+
+    ceph_test_rados_api_tier: do fewer writes in HitSetWrite
+
+    We don't need to do quite so many writes.  It can be slow when we are
+    thrashing and aren't doing anything in parallel.
+
+    Fixes: #8932
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c5f766bb16c0ab3c3554e73791ad0b74077ad35c)
+
+commit 22aa81413e7db43418e613aea037e978031f5b8a
+Merge: b65cef6 7025075
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Aug 8 19:05:37 2014 -0700
+
+    Merge remote-tracking branch 'gh/firefly-next' into firefly
+
+commit cb48cd47a26d27869884af83012d6a8a684abf25
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Wed Jul 30 14:50:37 2014 -0700
+
+    qa/workunits/cephtool/test_daemon.sh: verify ceph -c works with daemon
+
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit aa9ae1f270293778aa937e7f7e4bcaee3099b9b2)
+
+commit 1ac0e5436da68e441f42a1016b12481046147850
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Wed Jul 30 14:49:02 2014 -0700
+
+    qa/workunits/cephtool/test_daemon.sh: typo
+
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 22d20f39b7355966554319d5a1aa888967607569)
+
+commit a144950624208d8ed2901fd7421060a706ac2016
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Wed Jul 30 14:48:28 2014 -0700
+
+    qa/workunits/cephtool/test_daemon.sh: allow local ceph command
+
+    (cherry picked from commit 97a8d5a9fdbd3a25cc922c242ee57da58c57d0bc)
+
+commit 76b593f6268d77e6cf5a9931f6f3524b3f574c09
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Mon Jul 28 21:28:42 2014 -0700
+
+    ceph.in: Pass global args to ceph-conf for proper lookup
+
+    Fixes: #8944
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 6d89a99648630f81b85ad115fe7662dba6b08a55)
+
+commit b65cef678777c1b87d25385595bf0df96168703e
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jul 14 08:04:16 2014 -0700
+
+    ceph_test_rados_api_tier: fix [EC] HitSet{Read,Write,Trim} tests
+
+    The hit_set_ fields can only be set on tier pools as of
+    f131dfbaedf6f451572e7aa3a83f653912122953.
+
+    Fixes: #8823
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e17e9d857722ee478abda10adb32e15b11fff2ff)
+
+commit 70250755e4b2c114f1f0f463b42930dd99751d81
+Merge: 6f1a544 f8494c8
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Sun Aug 3 12:47:23 2014 -0700
+
+    Merge branch 'wip-8701-firefly' into firefly-next
+
+commit f8494c87e5276f0587c3341ddef9d7a933dfea11
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jul 22 06:53:41 2014 -0700
+
+    ceph_test_objectstore: clean up on finish of MoveRename
+
+    Otherwise, we leave collections around, and the next test fails.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d4faf747b73e70dff9cb5c98ee6aaa4ecec215fc)
+
+commit 94fee997204d439abfd642be20a7e33df5979342
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jul 21 13:45:21 2014 -0700
+
+    os/LFNIndex: use FDCloser for fsync_dir
+
+    This prevents an fd leak when maybe_inject_failure() throws an exception.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3ec9a42b470422b1fe72b6294d82d9efcaca7f53)
+
+commit b213866eac422a6ac6aeacd3ab4741a6421ce20c
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 18 23:16:09 2014 -0700
+
+    os/LFNIndex: only consider alt xattr if nlink > 1
+
+    If we are doing a lookup, the main xattr fails, we'll check if there is an
+    alt xattr.  If it exists, but the nlink on the inode is only 1, we will
+    kill the xattr.  This cleans up the mess left over by an incomplete
+    lfn_unlink operation.
+
+    This resolves the problem with an lfn_link to a second long name that
+    hashes to the same short_name: we will ignore the old name the moment the
+    old link goes away.
+
+    Fixes: #8701
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6fb3260d59faab1e20ebf1e44f850f85f6b8342a)
+
+commit 4fc72d8d936ac499fbb53a1d73a3f7c05ebe596a
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 18 17:28:18 2014 -0700
+
+    os/LFNIndex: remove alt xattr after unlink
+
+    After we unlink, if the nlink on the inode is still non-zero, remove the
+    alt xattr.  We can *only* do this after the rename or unlink operation
+    because we don't want to leave a file system link in place without the
+    matching xattr; hence the fsync_dir() call.
+
+    Note that this might leak an alt xattr if we happen to fail after the
+    rename/unlink but before the removexattr is committed.  We'll fix that
+    next.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ec36f0a130d67df6cbeefcc9c2d83eb703b6b28c)
+
+commit 816ed7e951b32bb9ac57466c0affaec9781ac7f4
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jul 21 13:43:42 2014 -0700
+
+    os/LFNIndex: FDCloser helper
+
+    Add a helper to close fd's when we leave scope.  This is important when
+    injecting failures by throwing exceptions.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a320c260a9e088ab0a4ea3d5298c06a2d077de37)
+
+commit cbfbe637851c7ebe4a9ec1fd6e429cdf85aef608
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 18 17:09:07 2014 -0700
+
+    os/LFNIndex: handle long object names with multiple links (i.e., rename)
+
+    When we rename an object (collection_move_rename) to a different name, and
+    the name is long, we run into problems because the lfn xattr can only track
+    a single long name linking to the inode.  For example, suppose we have
+
+    foobar -> foo_123_0 (attr: foobar) where foobar hashes to 123.
+
+    At first, collection_add could only link a file to another file in a
+    different collection with the same name. Allowing collection_move_rename
+    to rename the file, however, means that we have to convert:
+
+    col1/foobar -> foo_123_0 (attr: foobar)
+
+    to
+
+    col1/foobaz -> foo_234_0 (attr: foobaz)
+
+    This is a problem because if we link, reset xattr, unlink we end up with
+
+    col1/foobar -> foo_123_0 (attr: foobaz)
+
+    if we restart after we reset the attr.  This will cause the initial foobar
+    lookup to since the attr doesn't match, and the file won't be able to be
+    looked up.
+
+    Fix this by allow *two* (long) names to link to the same inode.  If we
+    lfn_link a second (different) name, move the previous name to the "alt"
+    xattr and set the new name.  (This works because link is always followed
+    by unlink.)  On lookup, check either xattr.
+
+    Don't even bother to remove the alt xattr on unlink.  This works as long
+    as the old name and new name don't hash to the same shortname and end up
+    in the same LFN chain.  (Don't worry, we'll fix that next.)
+
+    Fixes part of #8701
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+    (cherry picked from commit b2cdfce6461b81f4926602a8c63b54aa92684e6c)
+
+commit 5db6c12b61e00b0bc8084ead5976a912ece0fc65
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 18 15:46:58 2014 -0700
+
+    ceph_test_objectstore: fix warning
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit cf98805c09a38cce78ac08317899dc4152ae55a5)
+
+commit e18777ed6eb492e8421d53090b3c9af6e75bd792
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Jul 15 14:50:33 2014 -0700
+
+    store_test: add long name collection_move_rename tests
+
+    Currently fails.
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 6aa48a485e03ca100f3d9ebec77cc06f99756cd7)
+
+    Conflicts:
+	src/test/objectstore/store_test.cc
+
+commit 4e03d5b512c8d2f7fa51dda95c6132e676529f9b
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Aug 3 08:40:57 2014 -0700
+
+    Revert "enforce rados put aligment"
+
+    This reverts commit 7a58da53ebfcaaf385c21403b654d1d2f1508e1a.
+
+    This was alread backported in dece65064d949b5afcc359cd408615883b5e002a.
+
+    Fixes: #8996
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 6f1a54436c63a4cb63712936fccade3267d03db8
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Jul 29 15:25:47 2014 -0700
+
+    rgw: fix crash in swift CORS preflight request
+
+    Fixes: #8586
+
+    This fixes error handling, in accordance with commit 6af5a537 that fixed
+    the same issue for the S3 case.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 18ea2a869791b4894f93fdafde140285f2e4fb65)
+
+commit dfcd99567e925bd6565ef4676c560f6c8c89e683
+Author: devicenull <dn@devicenull.org>
+Date:   Fri Jul 18 10:25:51 2014 -0400
+
+    rgw: fix decoding + characters in URL
+
+    Fixes: #8702
+    Backport: firefly
+
+    Only decode + characters to spaces if we're in a query argument. The +
+    query argument.  The + => ' ' translation is not correct for
+    file/directory names.
+
+    Resolves http://tracker.ceph.com/issues/8702
+
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+    Signed-off-by: Brian Rak <dn@devicenull.org>
+    (cherry picked from commit 4a63396ba1611ed36cccc8c6d0f5e6e3e13d83ee)
+
+commit f9716214a35e3cddd3006f3839f9688c41f0c7ca
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jul 25 20:33:52 2014 -0700
+
+    rgw: call processor->handle_data() again if needed
+
+    Fixes: #8937
+
+    Following the fix to #8928 we end up accumulating pending data that
+    needs to be written. Beforehand it was working fine because we were
+    feeding it with the exact amount of bytes we were writing.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 0553890e79b43414cc0ef97ceb694c1cb5f06bbb)
+
+    Conflicts:
+	src/rgw/rgw_rados.h
+
+commit 4cd1b60c043089358ea8e70a51310005d5b68383
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jul 24 15:30:27 2014 -0700
+
+    rgw: object write should not exceed part size
+
+    Fixes: #8928
+
+    This can happen if the stripe size is not a multiple of the chunk size.
+
+    Backport: firefly
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 14cad5ece7d1de9d93e72acca6d4c3b4a9cfcfa2)
+
+commit f68720a6a61364f72f4e0c62aabea65eabe33f98
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Jul 22 15:30:11 2014 -0700
+
+    rgw: align object chunk size with pool alignment
+
+    Fixes: #8442
+    Backport: firefly
+    Data pools might have strict write alignment requirements. Use pool
+    alignment info when setting the max_chunk_size for the write.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit fc83e197ab85355e385c13f2a64957cad7481298)
+
+    Conflicts:
+	src/rgw/rgw_rados.cc
+
+commit 67d73451c893b58634f06ed141792da1bf0fa77c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jul 30 11:53:16 2014 -0700
+
+    cls_rgw: fix object name of objects removed on object creation
+
+    Fixes: #8972
+    Backport: firefly, dumpling
+
+    Reported-by: Patrycja Szabłowska <szablowska.patrycja@gmail.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 0f8929a68aed9bc3e50cf15765143a9c55826cd2)
+
+commit d90148da293b4a3e2a44d4486a6b48e27d35b79b
+Merge: 223a0a9 d48a737
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Sat Aug 2 11:28:14 2014 -0700
+
+    Merge remote-tracking branch 'origin/wip-8438' into firefly-next
+
+    Backport of c5b8d8105d965da852c79add607b69d5ae79a4d4
+
+commit 223a0a9e7ed76fbc114c2f058d3aff93556244e9
+Merge: 84bd3cd d72eec0
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Sat Aug 2 11:22:47 2014 -0700
+
+    Merge remote-tracking branch 'origin/wip-7999' into firefly-next
+
+    Backport of 830940bf242a73403ec1882a489e31f7694b7f7e
+
+commit 84bd3cd79961d1ddce12048f9ae980fc197e6be4
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jul 28 17:17:23 2014 -0700
+
+    unittest_crush_wrapper: fix build
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f36cffc986c973014c89aa37ca73740b2fc194ca)
+
+commit 4983aa32fe8c64c78957b921f5d31ab07be9f850
+Merge: a2681c9 b6d8fea
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Aug 1 19:26:44 2014 -0700
+
+    Merge pull request #2178 from dachary/wip-erasure-code-profile-default-firefly
+
+    erasure-code: create default profile if necessary (firefly)
+
+commit d89a5a37f2515cc9af1322fd18357e445b94256f
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jul 30 12:26:44 2014 -0700
+
+    mon: s/%%/%/
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d700076a42a5a5ebe769a8311fd3b52bf2e98cd2)
+
+commit da0de4ee973d9a9c8ee6f9a5c12eb37c83b4d453
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jul 30 14:52:06 2014 -0700
+
+    atomic: fix read() on i386, clean up types
+
+    Among other things, fixes #8969
+
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 96863128e6668257f435c6962263caae0d7d10dd)
+
+commit 1e3b927731104629cae9682b55b8d540ef8af12f
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Jun 5 10:43:16 2014 -0700
+
+    include/atomic: make 32-bit atomic64_t unsigned
+
+    This fixes
+
+    In file included from test/perf_counters.cc:19:0:
+    ./common/perf_counters.h: In member function ‘std::pair PerfCounters::perf_counter_data_any_d::read_avg() const’:
+    warning: ./common/perf_counters.h:156:36: comparison between signed and unsigned integer expressions [-Wsign-compare]
+    } while (avgcount2.read() != count);
+    ^
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2081c992bbe3a83d711f465634d19c011d28ea3e)
+
+commit 34ac3060d408604c7082c583641b90f330f12663
+Author: Dmitry Smirnov <onlyjob@member.fsf.org>
+Date:   Wed May 21 07:56:43 2014 +1000
+
+    Define AO_REQUIRE_CAS (fixes FTBFS on 'hppa')
+
+     to fix FTBFS due to undeclared atomic functions.
+
+    As reported
+
+        https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=748571
+
+    by John David Anglin <dave.anglin@bell.net>
+
+    ~~~~
+    ./include/atomic.h: In member function 'size_t ceph::atomic_t::inc()':
+    ./include/atomic.h:42:36: error: 'AO_fetch_and_add1' was not declared in this scope
+           return AO_fetch_and_add1(&val) + 1;
+                                        ^
+    ./include/atomic.h: In member function 'size_t ceph::atomic_t::dec()':
+    ./include/atomic.h:45:42: error: 'AO_fetch_and_sub1_write' was not declared in this scope
+           return AO_fetch_and_sub1_write(&val) - 1;
+                                              ^
+    ./include/atomic.h: In member function 'void ceph::atomic_t::add(size_t)':
+    ./include/atomic.h:48:36: error: 'AO_fetch_and_add' was not declared in this scope
+           AO_fetch_and_add(&val, add_me);
+                                        ^
+    ./include/atomic.h: In member function 'void ceph::atomic_t::sub(int)':
+    ./include/atomic.h:52:48: error: 'AO_fetch_and_add_write' was not declared in this scope
+           AO_fetch_and_add_write(&val, (AO_t)negsub);
+                                                    ^
+    ./include/atomic.h: In member function 'size_t ceph::atomic_t::dec()':
+    ./include/atomic.h:46:5: warning: control reaches end of non-void function [-Wreturn-type]
+         }
+         ^
+    make[5]: *** [cls/user/cls_user_client.o] Error 1
+    ~~~~
+
+    Signed-off-by: Dmitry Smirnov <onlyjob@member.fsf.org>
+    (cherry picked from commit 74218f3d6ca8ca9943ff9d08b7926e38fb13b329)
+
+commit a0c12b70d93604cde59fc21b4d42cae6bd0f4f26
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Mar 31 14:49:50 2014 -0700
+
+    atomic_t: add atomic64_t
+
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+    (cherry picked from commit bf3ba6001c7b4cf37edfe6551d3ef298ebcbf421)
+
+commit 5a8c8c01cac7f8ef8b898a88cac96af973350a3d
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jul 30 13:40:33 2014 -0700
+
+    test/cli-integration/rbd: fix trailing space
+
+    Newer versions of json.tool remove the trailing ' ' after the comma.  Add
+    it back in with sed so that the .t works on both old and new versions, and
+    so that we don't have to remove the trailing spaces from all of the test
+    cases.
+
+    Backport: firefly
+    Fixes: #8920
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 605064dc685aa25cc7d58ec18b6449a3ce476d01)
+
+    Conflicts:
+	src/test/cli-integration/rbd/defaults.t
+
+commit 234b2d1933f7f403c094a3c65bf912fcee688181
+Author: John Spray <jspray@redhat.com>
+Date:   Mon Jul 7 15:45:54 2014 +0100
+
+    tests: don't depend on 'data' pool in rbd test
+
+    Since we removed the default 'data' and 'metadata' pools,
+    tests which need a pool should create it themselves.
+
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit a7a631d1e284f151e305f770cef2042a1b9f86c0)
+
+commit 751258db76d9f5288ff42e8e961a9ffea1d226de
+Author: Xiaoxi Chen <xiaoxi.chen@intel.com>
+Date:   Mon Jul 28 16:54:48 2014 +0800
+
+    PGMonitor: fix bug in caculating pool avail space
+
+    Currently for pools with different rules, "ceph df" cannot report
+    right available space for them, respectively. For detail assisment
+    of the bug ,pls refer to bug report #8943
+
+    This patch fix this bug and make ceph df works correctlly.
+
+    Fixes Bug #8943
+
+    Signed-off-by: Xiaoxi Chen <xiaoxi.chen@intel.com>
+    (cherry picked from commit 04d0526718ccfc220b4fe0c9046ac58899d9dafc)
+
+commit 7b1546cc5c361a1af7f724e3c3b7ca9436bad76f
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 28 13:59:47 2014 -0700
+
+    mon: set min_size to data chunk count for erasure pools
+
+    Make the min_size value meaningful for erasure pools.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e06c58c9b8f585d2fe7c97d010aa0aa61c09d609)
+
+commit e859587911edddd3a5b7fd96ee2f6a7ed77478dc
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun May 11 13:36:03 2014 -0700
+
+    mon: include 'max avail' in df output
+
+    Include an estimate of the maximum writeable space for each pool.  Note
+    that this value is a conservative estimate for that pool based on the
+    most-full OSD.  It is also potentially misleading as it is the available
+    space if *all* new data were written to this pool; one cannot (generally)
+    add up the available space for all pools.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 7a9652b58ea70f9a484a135bde20d872616c5947)
+
+commit a5e0ebc36ccbdf36e733e5eab926e4446a48fbd8
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun May 11 13:31:14 2014 -0700
+
+    mon: right justify df values
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 2f63a309df4b7086725949bc0a532595cf927edf)
+
+commit 3d578479420f4b0c153dffe96605bf413070ad45
+Author: John Spray <john.spray@inktank.com>
+Date:   Thu May 8 11:04:46 2014 +0100
+
+    mon: Fix % escaping (\% should be %%)
+
+    Clang's -Wpedantic points this out.
+
+    Signed-off-by: John Spray <john.spray@inktank.com>
+    (cherry picked from commit f0231ef364d531eb60351598c4a0f5fa6efad23c)
+
+    Conflicts:
+	src/mon/DataHealthService.cc
+
+commit 6164c3ef8d9ced38d878c191e4de3d84a8bda45e
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed May 28 13:49:52 2014 -0700
+
+    crush: add get_rule_weight_map
+
+    Calculate a weight map of OSDs for a given rule.
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 297f6169feecd20e121d102e1b63a505c8b3e74a)
+
+commit 2f90bfdb854750990881f2f53492687a07b19546
+Author: Guang Yang <yguang@yahoo-inc.com>
+Date:   Wed Jul 9 11:20:36 2014 +0000
+
+    Fix the PG listing issue which could miss objects for EC pool (where there is object shard and generation).
+    Backport: firefly
+    Signed-off-by: Guang Yang (yguang@yahoo-inc.com)
+
+    (cherry picked from commit 228760ce3a7109f50fc0f8e3c4a5697a423cb08f)
+
+commit 61c90252c3a496e09962cfda2eb11e6bd112af8a
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 25 14:48:10 2014 -0700
+
+    osd/ReplicatedPG: requeue cache full waiters if no longer writeback
+
+    If the cache is full, we block some requests, and then we change the
+    cache_mode to something else (say, forward), the full waiters don't get
+    requeued until the cache becomes un-full.  In the meantime, however, later
+    requests will get processed and redirected, breaking the op ordering.
+
+    Fix this by requeueing any full waiters if we see that the cache_mode is
+    not writeback.
+
+    Fixes: #8931
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8fb761b660c268e2264d375a4db2f659a5c3a107)
+
+commit f701688a360fd977b5e1f59518622ab6d2299a08
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 25 14:43:48 2014 -0700
+
+    osd/ReplicatedPG: fix cache full -> not full requeueing when !active
+
+    We only want to do this if is_active().  Otherwise, the normal
+    requeueing code will do its thing, taking care to get the queue orders
+    correct.
+
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 36aaab9eee7ed41a46a4ac27376d630a29de5eb9)
+
+commit a1ddce36ebf1ce479129def364328486fbcc5c86
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Wed Jul 30 14:50:37 2014 -0700
+
+    qa/workunits/cephtool/test_daemon.sh: verify ceph -c works with daemon
+
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit aa9ae1f270293778aa937e7f7e4bcaee3099b9b2)
+
+commit 50a1c08e46eaedd3d81a52f64923228bc8a81c26
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Wed Jul 30 14:49:02 2014 -0700
+
+    qa/workunits/cephtool/test_daemon.sh: typo
+
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 22d20f39b7355966554319d5a1aa888967607569)
+
+commit 4cfb7dbc5c8fe1a28fb945fa12d9795943db1db5
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Wed Jul 30 14:48:28 2014 -0700
+
+    qa/workunits/cephtool/test_daemon.sh: allow local ceph command
+
+    (cherry picked from commit 97a8d5a9fdbd3a25cc922c242ee57da58c57d0bc)
+
+commit 120b5d5ad51fa47a2bb0a144821635f5f81b5a2e
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Mon Jul 28 21:28:42 2014 -0700
+
+    ceph.in: Pass global args to ceph-conf for proper lookup
+
+    Fixes: #8944
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 6d89a99648630f81b85ad115fe7662dba6b08a55)
+
+commit e0c27cd7fde84944fd350f7df89e79d9a39ace98
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 16 17:55:36 2014 -0700
+
+    qa/workunits/cephtool/test.sh: test osd pool get erasure_code_profile
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit ce9f12d7a2202948532fed9da4d763ed03f6b8fa)
+
+    Conflicts:
+	qa/workunits/cephtool/test.sh
+
+commit cf22759cdf1c0a6d258e5c0a2d32268ce051c742
+Author: Ma Jianpeng <jianpeng.ma@intel.com>
+Date:   Wed Jul 16 17:48:34 2014 -0700
+
+    mon: OSDMonitor: add "osd pool get <pool> erasure_code_profile" command
+
+    Enable us to obtain the erasure-code-profile for a given erasure-pool.
+
+    Signed-off-by: Ma Jianpeng <jianpeng.ma@intel.com>
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit e8ebcb79a462de29bcbabe40ac855634753bb2be)
+
+commit b94d2f142db63bdf395d8288f8dcef1558d30eaf
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jul 23 18:25:53 2014 -0700
+
+    osd/ReplicatedPG: observe INCOMPLETE_CLONES in is_present_clone()
+
+    We cannot assume that just because cache_mode is NONE that we will have
+    all clones present; check for the absense of the INCOMPLETE_CLONES flag
+    here too.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 63abf11390bb9b8dd604aae2b3e90596f9ab65ac)
+
+commit 22cc1b643f83edbfc0475f6da89cf4bd4c89aae1
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jul 23 18:24:51 2014 -0700
+
+    osd/ReplicatedPG: observed INCOMPLETE_CLONES when doing clone subsets
+
+    During recovery, we can clone subsets if we know that all clones will be
+    present.  We skip this on caching pools because they may not be; do the
+    same when INCOMPLETE_CLONES is set.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 41364711a66c89ce2e94435fe0d54eeda6092614)
+
+commit c42232e3539fc4682e8fec3e064b21acbcc996ed
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jul 23 18:23:56 2014 -0700
+
+    osd/ReplicatedPG: do not complain about missing clones when INCOMPLETE_CLONES is set
+
+    When scrubbing, do not complain about missing cloens when we are in a
+    caching mode *or* when the INCOMPLETE_CLONES flag is set.  Both are
+    indicators that we may be missing clones and that that is okay.
+
+    Fixes: #8882
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 956f28721dd98c5fb9eb410f4fe9e320b3f3eed3)
+
+commit bde3a9f0f610e6af30066dd77949a1249cd658bf
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jul 23 18:21:38 2014 -0700
+
+    osd/osd_types: add pg_pool_t FLAG_COMPLETE_CLONES
+
+    Set a flag on the pg_pool_t when we change cache_mode NONE.  This
+    is because object promotion may promote heads without all of the clones,
+    and when we switch the cache_mode back those objects may remain.  Do
+    this on any cache_mode change (to or from NONE) to capture legacy
+    pools that were set up before this flag existed.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 54bf055c5dadc55acf5731e08712d529b180ffc5)
+
+commit f19aa8ef0a2cbad7af3135defc1fe3485d377409
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri Jul 11 00:40:47 2014 +0100
+
+    qa/workunits: cephtool: adjust pool name where missing as it has changed
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 50e93c2138978f7f7c2fbafacc1611c8705a8eab)
+
+commit 163440a764a6fac8f2efbee6a8aee5cbc460dfe6
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri Jul 11 00:41:01 2014 +0100
+
+    qa/workunits: cephtool: cleanup after pool creation
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 6cd345732b15e84de17d743e06bc4d85569b79d4)
+
+commit c71c99ff21063e26ebfef431f916463f5606aae2
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jul 8 19:22:49 2014 +0100
+
+    qa/workunits: cephtool: pool needs to be a tier to be used as such
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 704b0a33f2071eabeb8c5b000a6805ef6d498961)
+
+commit ba0a9bf013e8d056a4126b6840621e5a3a8b37fc
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jul 8 19:22:01 2014 +0100
+
+    qa/workunits: cephtool: test erroneous 'tier remove'
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 49db6767152092d503ccf8ead6f7cb069e152a22)
+
+commit ffec081a551817288303cf3fa00c016bbffe114a
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Thu Jul 3 15:32:46 2014 +0100
+
+    qa/workunits: cephtool: test get/set on both tier and non-tier pools
+
+    Make sure gets and sets of tiering-specific variables succeed on tier
+    pools and fail on non-tier pools.
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 9fea033f30aec44a3273c623ec6c93eb1d7dd26b)
+
+commit 55166c3a098d8751ed6dafb3541e45a6439c5491
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Thu Jul 3 15:24:51 2014 +0100
+
+    qa/workunits: cephtool: split get/set on tier pools from get/set tests
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit df5944955d96c041e65964a13b802028e9700904)
+
+commit 0425eb1dbd156d03a51e91c6f3079b8a687d62f3
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri Jun 27 22:20:51 2014 +0100
+
+    qa/workunits: cephtool: test for 'osd pool {get,set}-quota'
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit b927c0de7d5c7a78bf3c133be52cbc1d769974bb)
+
+commit 68197bc28c8960db67c4246d50eb5146659593bd
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Mon Jun 30 17:51:47 2014 +0100
+
+    mon: OSDMonitor: 'osd pool' - if we can set it, we must be able to get it
+
+    Add support to get the values for the following variables:
+     - target_max_objects
+     - target_max_bytes
+     - cache_target_dirty_ratio
+     - cache_target_full_ratio
+     - cache_min_flush_age
+     - cache_min_evict_age
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit ddc04c83ff6842ca0b2f804b46099ea729b9fb6b)
+
+commit 01498ce8c897f7e3b66c8054a8b144a505a33010
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 1 08:42:58 2014 +0200
+
+    qa: support running under non privileged user
+
+    If the test is run against a cluster started with vstart.sh (which is
+    the case for make check), the --asok-does-not-need-root disables the use
+    of sudo and allows the test to run without requiring privileged user
+    permissions.
+
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 522174b066044e51a7019bd6cad81117e83c394e)
+
+commit 99259dff5bad93dce928c1ec07a16b34e3dac3dc
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 30 16:38:10 2014 -0700
+
+    qa/workunits/cephtool/test.sh: sudo ceph daemon
+
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit bcc09f93761d46202742ca85cce498a352edd494)
+
+commit 4f911d45632fd23c4124bf6b29287ece473d33b4
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Jun 25 21:25:29 2014 +0100
+
+    qa/workunits: cephtool: fix 'osd bench' test
+
+    Commit 7dc93a9651f602d9c46311524fc6b54c2f1ac595 fixed an incorrect
+    behavior with the OSD's 'osd bench' value hard-caps.  The test wasn't
+    appropriately modified unfortunately.
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 48e38ac6323f4a0e06b0dedd37ecd10dc339b1e3)
+
+commit bad2f513c5bed4602005eaedff9384012103aaee
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Jun 25 01:48:24 2014 +0100
+
+    qa/workunits: cephtool: only run heap profiler test if tcmalloc enabled
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 4b0809a13eb363a15e52a6a57372a0a31a64cef8)
+
+commit 26f9860d19c38595bfced76fce864fe9ad9f3a27
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jun 24 23:16:17 2014 +0100
+
+    qa/workunits: cephtool: set +e for the tcmalloc tests
+
+    Avoids failing the tests when tcmalloc is not present
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 5c4616e100297ba8639919aca7a9cb59e4bda54a)
+
+commit a0720b608db208ac766df8eea53bdbba31d79492
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jun 24 22:47:31 2014 +0100
+
+    qa/workunits: cephtool: delete unnecessary function calls
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 67255435151627314cc2fc38732d4fb5efddc3cc)
+
+commit 373224729e9ab2a46afb43b1388d37ff951cc42e
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jun 24 22:46:44 2014 +0100
+
+    qa/workunits: cephtool: disable bash debug when not running tests
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 946bd0dad4b027326b03c13405782b99ef0f91b2)
+
+commit 52d733c7202b393b3b7da48e3aa6db275946757a
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jun 24 22:39:34 2014 +0100
+
+    qa/workunits: cephtool: allow running individual tests
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 5d26575ef2d31d745ec4aa69ca1501cd76e5e8db)
+
+commit 96e58d6d38969ba8b47b1924205eb34850a24ed6
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Jun 24 20:55:27 2014 +0100
+
+    qa/workunits: cephtool: cleanup state after erasure-code-profile test
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit f4184086d0d647e064e34308f678ef889e13c373)
+
+commit 4cd5d179375c06a25f04ef8150e2a53aabf7427e
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Jun 25 01:17:58 2014 +0100
+
+    qa/workunits: cephtool: add/remove comments
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 780424df3a107c7da57fc28d64f9e7a4bb47f8e8)
+
+commit 5bcf690b1dbf5a2e4388661f9d280b5934db1c9c
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Thu Jun 26 00:55:46 2014 +0100
+
+    qa/workunits: cephtool: split into properly indented functions
+
+    The test was a big sequence of commands being run and it has been growing
+    organically for a while, even though it has maintained a sense of
+    locality with regard to the portions being tested.
+
+    This patch intends to split the commands into functions, allowing for a
+    better semantic context and easier expansion.  On the other hand, this
+    will also allow us to implement mechanisms to run specific portions of
+    the test instead of always having to run the whole thing just to test a
+    couple of lines down at the bottom (or have to creatively edit the test).
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 3d14a96f4b2b7094d05ead1dec7547d165857e31)
+
+    Conflicts:
+	qa/workunits/cephtool/test.sh
+
+commit 2af751807d1b51348302734c46d506db294303fd
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Jun 25 01:12:55 2014 +0100
+
+    qa/workunits: cephtool: move test line to where it's more appropriate
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 04658b7b2b5f98ae81ffb3f77303745e6d46eb81)
+
+commit 5bf9e3c7b59d7ce5996d4d05312b4a92f34bbcb6
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Jun 25 01:11:52 2014 +0100
+
+    qa/workunits: cephtool: split into functions
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit db6cc133ba4bb38b3c11eb835fd3983dc2f6b00e)
+
+    Conflicts:
+	qa/workunits/cephtool/test.sh
+
+commit c898c9736950437256b8c261106f2827fc160e01
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jun 20 09:38:39 2014 +0200
+
+    mon: test that pools used in tiers cannot be removed
+
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 39a4b78177cb9896ff35ab05bcf8774bfc934f3a)
+
+commit 5b72674238db80e62cf3d5cec1d94be5c0102388
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Fri May 23 17:02:23 2014 +0100
+
+    qa/workunits/cephtool: test setting options using SI units
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 38405d3554dfb0caf2b0a2c010b95a61bdb99349)
+
+    Conflicts:
+	qa/workunits/cephtool/test.sh
+
+commit aacf67d13f81fcc88881091d175fee9ecb0f8954
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Jul 2 20:19:19 2014 +0100
+
+    mon: OSDMonitor: be scary about inconsistent pool tier ids
+
+    We may not crash your cluster, but you'll know that this is not something
+    that should have happened.  Big letters makes it obvious.  We'd make them
+    red too if we bothered to look for the ANSI code.
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 8e5a8daf98052954f3880d2d3516841b5062466b)
+
+commit af59cbf5854bd0fabeb0ef394d7d5b1207bbd6d3
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Jul 2 19:47:52 2014 +0100
+
+    osd: pg_pool_t: clear tunables on clear_tier()
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 64bdf6c92bc44adad5a49b9dc4f674789cee80b0)
+
+commit 8f788e14d22c913479630bce9f9c502ac900328c
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Wed Jul 2 19:46:48 2014 +0100
+
+    mon: OSDMonitor: limit tier-specific pool set/get on non-tier pools
+
+    Fixes: 8696
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit f131dfbaedf6f451572e7aa3a83f653912122953)
+
+commit 4b768a8fe9146e6ec033d4f6780e0ef373fcfd7a
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jul 24 10:06:31 2014 -0700
+
+    mon/OSDMonitor: improve no-op cache_mode set check
+
+    If we have a pending pool value but the cache_mode hasn't changed, this is
+    still a no-op (and we don't need to block).
+
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 67d13d76f5692fa20649ea877f254c34094c11f6)
+
+commit c3f403293c7f8d946f66a871aa015a558120ce78
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Sat May 3 03:42:19 2014 +0100
+
+    mon: OSDMonitor: disallow nonsensical cache-mode transitions
+
+    Fixes: 8155
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit fd970bbc95d89bf66c9551feca17ac0afbf4d1e2)
+
+commit d9aacd96acfe24b15c4e3b2441838945de1ab720
+Author: Joao Eduardo Luis <joao.luis@inktank.com>
+Date:   Tue Apr 29 18:03:56 2014 +0100
+
+    mon: OSDMonitor: return immediately if 'osd tier cache-mode' is a no-op
+
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit d01aa5bff30441eec1ffaa3e59a21187f8478475)
+
+commit 981add9cad5f2db06a4932be228eb0ff30254a9b
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jul 22 18:01:14 2014 -0700
+
+    osd/ReplicatedPG: debug obc locks
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 356af4bf46d6387e2f1a59646548f9a77e49e5f8)
+
+commit e32e2559c8ff408fff81cce0f15d6601dcd4a2f7
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jul 22 13:16:11 2014 -0700
+
+    osd/ReplicatedPG: greedily take write_lock for copyfrom finish, snapdir
+
+    In the cases where we are taking a write lock and are careful
+    enough that we know we should succeed (i.e, we assert(got)),
+    use the get_write_greedy() variant that skips the checks for
+    waiters (be they ops or backfill) that are normally necessary
+    to avoid starvation.  We don't care about staration here
+    because our op is already in-progress and can't easily be
+    aborted, and new ops won't start because they do make those
+    checks.
+
+    Fixes: #8889
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6fe27823b8459271bf0c0e807493bb7cf1e4559b)
+
+commit 68651e9137df2b1034a5b470bbb2fc1d19d8debb
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jul 22 13:11:42 2014 -0700
+
+    osd: allow greedy get_write() for ObjectContext locks
+
+    There are several lockers that need to take a write lock
+    because there is an operation that is already in progress and
+    know it is safe to do so.  In particular, they need to skip
+    the starvation checks (op waiters, backfill waiting).
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 09626501d7a0ff964027fd7a534465b76bad23cb)
+
+commit a2681c93fb30b912f653b596e27eb8b1bc5c6b78
+Author: Sage Weil <sage@inktank.com>
+Date:   Wed Jul 2 10:38:43 2014 -0700
+
+    qa/workunits/rest/test.py: make osd create test idempotent
+
+    Avoid possibility that we create multiple OSDs do to retries by passing in
+    the optional uuid arg.  (A stray osd id will make the osd tell tests a
+    few lines down fail.)
+
+    Fixes: #8728
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit bb3e1c92b6682ed39968dc5085b69c117f43cbb0)
+
+commit 7a58da53ebfcaaf385c21403b654d1d2f1508e1a
+Author: Lluis Pamies-Juarez <lluis@pamies.cat>
+Date:   Wed Jun 18 10:00:09 2014 -0700
+
+    enforce rados put aligment
+
+    Signed-off-by: Lluis Pamies-Juarez <lluis.pamies-juarez@hgst.com>
+    (cherry picked from commit 304b08a23a3db57010078046955a786fe3589ef8)
+    (cherry picked from commit dece65064d949b5afcc359cd408615883b5e002a)
+
+commit 7ae15549a795ffd9ffcd34ea1d5c101b90eee26d
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jul 31 15:39:40 2014 -0700
+
+    use llrintl when converting double to micro
+
+    This avoids rounding error (noticeable on i386).
+
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 80911736bd61b6b88eac0974d24f21c15c5385a4)
+
+    Conflicts:
+	src/mon/OSDMonitor.cc
+
+commit f5fc24368756c794d186a0990241e0dc0adfc03c
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jul 24 18:22:22 2014 -0700
+
+    msg/SimpleMessenger: drop local_conneciton priv link on shutdwon
+
+    This breaks ref cycles between the local_connection and session, and let's
+    us drop the explicit set_priv() calls in OSD::shutdown().
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 63c1711a9e237c14d137131f56751970ad1567b1)
+
+commit b6d8feab85fda97bf3a68980249a71846778a157
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Jun 19 00:49:13 2014 +0200
+
+    erasure-code: create default profile if necessary
+
+    After an upgrade to firefly, the existing Ceph clusters do not have the
+    default erasure code profile. Although it may be created with
+
+       ceph osd erasure-code-profile set default
+
+    it was not included in the release notes and is confusing for the
+    administrator.
+
+    The *osd pool create* and *osd crush rule create-erasure* commands are
+    modified to implicitly create the default erasure code profile if it is
+    not found.
+
+    In order to avoid code duplication, the default erasure code profile
+    code creation that happens when a new firefly ceph cluster is created is
+    encapsulated in the OSDMap::get_erasure_code_profile_default method.
+
+    Conversely, handling the pending change in OSDMonitor is not
+    encapsulated in a function but duplicated instead. If it was a function
+    the caller would need a switch to distinguish between the case when goto
+    wait is needed, or goto reply or proceed because nothing needs to be
+    done. It is unclear if having a function would lead to smaller or more
+    maintainable code.
+
+    http://tracker.ceph.com/issues/8601 Fixes: #8601
+
+    Backport: firefly
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 4e1405e7720eda71a872c991045ac8ead6f3e7d8)
+
+commit 2fff5f031b2cc00b55bd1ada4f83e4838fe59ddd
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Jan 13 17:54:08 2014 +0100
+
+    common: s/stringstream/ostream/ in str_map
+
+    There is no need to specialize more than ostream : it only makes it
+    impossible to use cerr or cout as a parameter to str_map.
+
+    Reviewed-By: Christophe Courtaut <christophe.courtaut@gmail.com>
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 6aa45b133956b974a992b372496b90c908d94f12)
+
+commit d48a737a6f8daa463ddf82106455b2efe7715a33
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Jun 2 13:44:49 2014 -0700
+
+    osd/: in scrub, check for and remove obsolete rollback objs
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 953c33edb640f6eb0c3f3a0d07c061c0d324242b
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri May 30 16:53:37 2014 -0700
+
+    osd/: plumb rollback_trim_to through interfaces and messages
+
+    We simply trim the roll back info up to min_last_complete_ondisk
+    since we cannot correctly rollback past that point anyway!
+
+    ReplicatedPG
+    -> PGBackend::submit_transaction
+    -> Backend messages
+    -> PGBackend::Listener::log_operation
+    -> append_log
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit ac11ca40b4f4525cbe9b1778b1c5d9472ecb9efa
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri May 30 16:27:32 2014 -0700
+
+    PGLog: add rollback_info_trimmed_to_riter to IndexedLog and use in PG
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 2207ed12fa544afc48d6d28f199381ccf0367292
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri May 30 15:40:15 2014 -0700
+
+    PGLog: fix logging whitespace in trim
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit d72eec0341e02881229d0e4ffeafcae92eb33fb6
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Jun 27 10:51:13 2014 -0700
+
+    PG: block outgoing peering messages until the last interval is durable
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit f9f89be38333d3817a0eafc52a961365e21e1f3a
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed May 21 13:09:33 2014 -0700
+
+    PG: add machinery to temporarily buffer outgoing recovery messages
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit f7498124325fe2269c33e45fae4c8181c8f7be14
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Jun 9 11:00:32 2014 -0700
+
+    os/: add async flush_commit() method to Sequencer and implementations
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 669b60571d6db9a9f74020e6a43b12b82edee41e
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri May 30 15:18:53 2014 -0700
+
+    PGLog: initialize complete_to
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 8679cdbeea6350fd860a9419f54578a90482895c
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu May 29 12:10:31 2014 -0700
+
+    osd_types: add pg_log_t::rollback_info_trimmed_to
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 3bbe29eb5cca7cc42266a6c5561ce68feb4072de
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue May 20 16:05:01 2014 -0700
+
+    common/Finisher: add queue(list<Context*>&)
+
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
diff --git a/doc/changelog/v0.80.7.txt b/doc/changelog/v0.80.7.txt
new file mode 100644
index 000000000..fd49a5bfb
--- /dev/null
+++ b/doc/changelog/v0.80.7.txt
@@ -0,0 +1,70 @@
+commit 6c0127fcb58008793d3c8b62d925bc91963672a3 (tag: refs/tags/v0.80.7, refs/remotes/gh/firefly)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Tue Oct 14 12:10:38 2014 -0700
+
+    0.80.7
+
+commit fa599aad2d79e0bbc19e4f6a2ac6771b37bf6beb
+Merge: 805a4ff b8d2fc7
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Oct 13 10:34:47 2014 -0700
+
+    Merge remote-tracking branch 'origin/wip-7588-firefly' into firefly
+
+commit 805a4ff3f32c60798d01beef80e0702a4e9d82ee
+Merge: 4a3bc78 c5fd2d0
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Oct 13 10:34:34 2014 -0700
+
+    Merge remote-tracking branch 'upstream/wip-9696-firefly' into firefly
+
+commit b8d2fc72ea54eb17611d7ac90be4da6c4e4e7485
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 8 14:19:22 2014 -0700
+
+    osd/ReplicatedPG: carry CopyOpRef in copy_from completion
+
+    There is a race with copy_from cancellation.  The internal Objecter
+    completion decodes a bunch of data and copies it into pointers provided
+    when the op is queued.  When we cancel, we need to ensure that we can cope
+    until control passes back to our provided completion.
+
+    Once we *do* get into the (ReplicatedPG) callbacks, we will bail out
+    because the tid in the CopyOp or FlushOp no longer matches.
+
+    Fix this by carrying a ref to keep the copy-from targets alive, and
+    clearing out the tids that we cancel.
+
+    Note that previously, the trigger for this was that the tid changes when
+    we handle a redirect, which made the op_cancel() call fail.  With the
+    coming Objecter changes, this will no longer be the case.  However, there
+    are also locking and threading changes that will make cancellation racy,
+    so we will not be able to rely on it always preventing the callback.
+    Either way, this will avoid the problem.
+
+    Fixes: #7588
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 589b639af7c8834a1e6293d58d77a9c440107bc3)
+
+commit c5fd2d043ed4aa4fdb60fc19a284f51a86cef408 (refs/remotes/gh/wip-9696-firefly)
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Oct 9 16:21:18 2014 -0700
+
+    PG::choose_acting: in mixed cluster case, acting may include backfill
+
+    Fixes: 9696
+    Backport: firefly, giant
+    Introduced: 92cfd370395385ca5537b5bc72220934c9f09026
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 9b18d99817c8b54e30dff45047dfe1b29871d659)
+
+commit 4a3bc78c033ed68183f91824c7942dc54afb8797 (refs/remotes/gh/wip-9731-firefly)
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Oct 10 13:53:29 2014 -0700
+
+    PGLog::IndexedLog::trim(): rollback_info_trimmed_to_riter may be log.rend()
+
+    Fixes: #9731
+    Backport: giant, firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit d458b4f0f31161f561ff98e58ed979cf20c6f588)
diff --git a/doc/changelog/v0.80.8.txt b/doc/changelog/v0.80.8.txt
new file mode 100644
index 000000000..849e933b1
--- /dev/null
+++ b/doc/changelog/v0.80.8.txt
@@ -0,0 +1,2547 @@
+commit 69eaad7f8308f21573c604f121956e64679a52a7 (tag: refs/tags/v0.80.8)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Tue Jan 13 06:28:08 2015 -0800
+
+    0.80.8
+
+commit 3c7cacf00b66a0e17272cf67fe0823ee863dfa7c
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 19 16:43:02 2014 -0700
+
+    mon/OSDMonitor: fix double-free on old MOSDBoot
+    
+    send_latest() does an m->put().
+    
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 97f9b6df668315fba6a5924b79024c7a986f4110)
+
+commit 852d7b5b3c019c02c042b767fc88916088e1a94d
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 8 11:17:03 2015 -0800
+
+    osd: requeue PG when we skip handling a peering event
+    
+    If we don't handle the event, we need to put the PG back into the peering
+    queue or else the event won't get processed until the next event is
+    queued, at which point we'll be processing events with a delay.
+    
+    The queue_null is not necessary (and is a waste of effort) because the
+    event is still in pg->peering_queue and the PG is queued.
+    
+    This is a firefly-specific patch; a (very) similar one will appear in master
+    in 492ccc900c3358f36b6b14a207beec071eb06707.
+    
+    Backport: giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit c26ebd38335bb361aade5aacd05ba3217e602b9c
+Merge: b67b7e7 96ba529
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 8 18:31:26 2015 -0800
+
+    Merge pull request #3217 from boydc2014/firefly
+    
+    clear data and payload after removed from ops_in_flight
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit b67b7e7ad72a1af96f9fb26ade815e65f72b4cb0
+Merge: db92718 7faae89
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Jan 9 01:32:17 2015 +0100
+
+    Merge pull request #3127 from ktdreyer/firefly-no-epoch
+    
+    Revert "ceph.spec.: add epoch"
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit db927186288cd4c63a3483b42e9eb9e016c96156
+Merge: 0d4abda 820dbfd
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Jan 9 01:31:29 2015 +0100
+
+    Merge pull request #3128 from dachary/wip-10281-make-check-fedora-20
+    
+    tests: fixes to run make check on fedora 20
+    
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 0d4abdaf80f1fedff7975d595abaac0a620c8035
+Merge: 1fdcb52 11995b3
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Jan 9 01:30:59 2015 +0100
+
+    Merge pull request #3169 from ceph/wip-8797-firefly
+    
+    Wip 8797 firefly
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 1fdcb524411a02b5627be66d9fd821a473223e9d
+Merge: 4b7b1b0 465eede
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Jan 9 01:28:49 2015 +0100
+
+    Merge pull request #3179 from dachary/wip-9998-crush-underfloat-firefly
+    
+    crush: fix weight underfloat issue (firefly)
+    
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 4b7b1b03bfcb7cb056783555884f211009ea5d46
+Merge: 4897ba4 f55b097
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Jan 9 01:28:11 2015 +0100
+
+    Merge pull request #3220 from ceph/wip-mon-backports.firefly
+    
+    mon: backports for #9987 against firefly
+    
+    Reviewed-by: Joao Eduardo Luis <joao@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 4897ba4a304bcac548b2121312cd7235c34dd5aa
+Merge: efe801f f20225c
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Jan 9 01:26:30 2015 +0100
+
+    Merge pull request #3258 from ceph/wip-10372-firefly
+    
+    osd: fix librados pool deletion race on firefly
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit efe801ff3a0d25243da18937c07b89227edbaac4
+Author: Warren Usui <warren.usui@inktank.com>
+Date:   Thu Dec 18 20:00:28 2014 -0800
+
+    If trusty, use older version of qemu
+    
+    Fixes #10319
+    Signed-off-by: Warren Usui <warren.usui@inktank.com>
+    (cherry-picked from 46a1a4cb670d30397979cd89808a2e420cef2c11)
+
+commit 96ba529ef8ce76e07b8444c94883afb3468d6762
+Author: Lei Dong <leidong@yahoo-inc.com>
+Date:   Tue Dec 30 21:02:45 2014 +0800
+
+    clear data and payload inside ops_in_flight_lock
+    
+    http://tracker.ceph.com/issues/9916 Fixes: #9916
+    Signed-off-by: Dong Lei <leidong@yahoo-inc.com>
+
+commit e0648e3d30de504b096c4ae3bbe7d9c17652bdb5
+Merge: 455f940 3624006
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 29 14:31:23 2014 -0800
+
+    Merge pull request #3264 from dachary/wip-jerasure-firefly
+    
+    erasure-code: update links to jerasure upstream
+
+commit 455f940908f242b1e34983af61351fd3045ce8ab
+Merge: b1ab685 aa95a2d
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 29 10:55:39 2014 -0800
+
+    Merge pull request #3268 from ceph/firefly-10415
+    
+    libcephfs/test.cc: close fd before umount
+
+commit 362400667aad0b5098fbe8dbec1b0bde059f84a6
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Dec 28 10:29:54 2014 +0100
+
+    erasure-code: update links to jerasure upstream
+    
+    It moved from bitbucket to jerasure.org
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 8e86f901939f16cc9c8ad7a4108ac4bcf3916d2c)
+
+commit aa95a2d20dbba2f3a775f709493c987d0d001e9c
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Dec 23 10:22:00 2014 +0800
+
+    libcephfs/test.cc: close fd before umount
+    
+    Fixes: #10415
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit d3fb563cee4c4cf08ff4ee01782e52a100462429)
+
+commit f20225cb99a0d2d08fccfdf88dc89d758ecba077 (refs/remotes/gh/wip-10372-firefly)
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 23 15:49:26 2014 -0800
+
+    osdc/Objecter: handle reply race with pool deletion
+    
+    We need to handle this scenario:
+    
+     - send request in epoch X
+     - osd replies
+     - pool is deleted in epoch X+1
+     - client gets map X+1, sends a map check
+     - client handles reply
+       -> asserts that no map checks are in flight
+    
+    This isn't the best solution.  We could infer that a map check isn't needed
+    since the pool existed earlier and doesn't now.  But this is firefly and
+    the fix is no more expensive than the old assert.
+    
+    Fixes: #10372
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 820dbfd9947455d07426981b7152861c3c216080
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 5 15:11:18 2014 -0700
+
+    test/ceph-disk.sh: mkdir -p
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c2f58e6694a2457200ab3d59e037ad17b9c82028)
+
+commit 77d393024f9d867b574b8ec8e15ec48a1a291511
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Thu Sep 4 12:23:27 2014 +0200
+
+    test/ceph-disk.sh: resolve symlinks before check
+    
+    Make sure symlinks are resolved in command_fixture()
+    before compare result of which command and the current
+    path.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 8ea86dfa7c4a3d7e089cf9d4e49586657875f851)
+
+commit ed6ec2936513d7dd6c45bccd8edf69a12c71dc7b
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Jun 24 19:54:17 2014 +0200
+
+    test/ceph-disk.sh: fix for SUSE
+    
+    On SUSE 'which' returns always the full path of (shell) commands and
+    not e.g. './ceph-conf' as on Debian. Add check also for full
+    path returned by which.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 39530536ff923b91899f6303507c283b78040a20)
+
+commit 754363f4563e7dbda1ef23fadc8d6ef1a3fdd0af
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jun 13 14:41:39 2014 +0200
+
+    tests: prevent kill race condition
+    
+    When trying to kill a daemon, keep its pid in a variable instead of
+    retrieving it from the pidfile multiple times. It prevents the following
+    race condition:
+    
+      * try to kill ceph-mon
+      * ceph-mon is in the process of dying and removed its pidfile
+      * try to kill ceph-mon fails because the pidfile is not found
+      * another ceph-mon is spawned and fails to bind the port
+        because the previous ceph-mon is still holding it
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit a1c13c57ba20fc329d943ea57523913e11067dc7)
+
+commit 5be6f2f60e3225bf3d214432044721fe474d55d7
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Jun 25 00:31:48 2014 +0200
+
+    osd/OSD.cc: parse lsb release data via lsb_release
+    
+    Use lsb_release tool to be portable since parsing /etc/lsb-release
+    is not the same between different distributions. The old code failed
+    e.g. for SUSE products to parse LSB information.
+    
+    Fixes: #8654
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 0b3a3987d382ff33fdf892f189b30df22be80e59)
+
+commit b62187c52324a4489c2fc1385b1d6574a058f7e8
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Tue Aug 26 21:59:39 2014 +0200
+
+    tests: histogram prevent re-use of local variables
+    
+    By moving the tests to separate functions.
+    
+    http://tracker.ceph.com/issues/9235 Fixes: #9235
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 4b8b25ecd128c34a386ad7c4cc89f323c4d384e1)
+
+commit e2741c8f1d42cfe91b18201a6a49005d90d85d98
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Tue Aug 26 21:59:39 2014 +0200
+
+    tests: histogram prevent re-use of local variables
+    
+    By moving the test to a separate function.
+    
+    http://tracker.ceph.com/issues/9235 Fixes: #9235
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit ee02293ad2ef050672fa8c164ba17b10e8d4ceeb)
+
+commit cf102df7a3f3b38824c26b9e44a21664fcf979a7
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Dec 10 00:08:57 2014 +0100
+
+    tests: avoid tab interpretation problem on fedora 20
+    
+    Use . instead of tab in echo to avoid variations in how escape sequences
+    are interpreted by the shell.
+    
+    http://tracker.ceph.com/issues/10281 Fixes: #10281
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+
+commit f55b097764beb973c12866f2d7161c6bd870aa07 (refs/remotes/gh/wip-mon-backports.firefly)
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Nov 2 08:50:59 2014 -0800
+
+    mon/PGMap and PGMonitor: update last_epoch_clean cache from new osd keys
+    
+    We were only invalidating the cached value from apply_incremental, which
+    is no longer called on modern clusters.
+    
+    Fix this by storing the update epoch in the key as well (it is not part
+    of osd_stat_t).
+    
+    Backport: giant, firefly, dumpling(?)
+    Fixes: #9987
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 093c5f0cabeb552b90d944da2c50de48fcf6f564)
+
+commit 1d314e7a9ab7af5b693583cf2faa5db54f6beb69
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Nov 2 08:49:48 2014 -0800
+
+    mon/PGMap: invalidate cached min_last_epoch_clean from new-style pg keys
+    
+    We were only invalidating the cache from the legacy apply_incremental(),
+    which is no longer called on modern clusters.
+    
+    Fixes: #9987
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3fb731b722c50672a5a9de0c86a621f5f50f2d06)
+
+commit 465eedea9f7411b1e352dc3ccee60a3f1221541d
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Nov 23 18:50:51 2014 -0800
+
+    crush/CrushWrapper: fix create_or_move_item when name exists but item does not
+    
+    We were using item_exists(), which simply checks if we have a name defined
+    for the item.  Instead, use _search_item_exists(), which looks for an
+    instance of the item somewhere in the hierarchy.  This matches what
+    get_item_weightf() is doing, which ensures we get a non-negative weight
+    that converts properly to floating point.
+    
+    Backport: giant, firefly
+    Fixes: #9998
+    Reported-by: Pawel Sadowski <ceph@sadziu.pl>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9902383c690dca9ed5ba667800413daa8332157e)
+
+commit cee51af9c1dbde550177c95caf6c93f612442300
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Nov 21 17:47:56 2014 -0800
+
+    crush/builder: prevent bucket weight underflow on item removal
+    
+    It is possible to set a bucket weight that is not the sum of the item
+    weights if you manually modify/build the CRUSH map.  Protect against any
+    underflow on the bucket weight when removing items.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8c87e9502142d5b4a282b94f929ae776a49be1dc)
+
+commit 7ccd5eec11e8cd945d24bf9f6390d6c3fa4a06de
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Nov 21 17:37:03 2014 -0800
+
+    crush/CrushWrapper: fix _search_item_exists
+    
+    Reported-by: Pawel Sadowski <ceph@sadziu.pl>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit eeadd60714d908a3a033aeb7fd542c511e63122b)
+
+commit b1ab685e00034751a161a3d5e0325c6581999c75
+Merge: dd7c8c2 ef3773a
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 12 06:19:50 2014 -0800
+
+    Merge pull request #3124 from ceph/wip-10194-firefly
+    
+    rgw: optionally call FCGX_Free() on the fcgi connection
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 11995b329045341c17553269267cfd3688a51b0f
+Author: Dan Mick <dan.mick@redhat.com>
+Date:   Wed Dec 10 13:19:53 2014 -0800
+
+    Call Rados.shutdown() explicitly before exit
+    
+    This is mostly a demonstration of good behavior, as the resources will
+    be reclaimed on exit anyway.
+    
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    (cherry picked from commit b038e8fbf9103cc42a4cde734b3ee601af6019ea)
+
+commit e00270b51896f168d5013b7dc92ec7f8b9e19da3
+Author: Dan Mick <dan.mick@redhat.com>
+Date:   Wed Dec 10 13:19:16 2014 -0800
+
+    rados.py: remove Rados.__del__(); it just causes problems
+    
+    Recent versions of Python contain a change to thread shutdown that
+    causes ceph to hang on exit; see http://bugs.python.org/issue21963.
+    As it turns out, this is relatively easy to avoid by not spawning
+    threads on exit, as Rados.__del__() will certainly do by calling
+    shutdown(); I suspect, but haven't proven, that the problem is
+    that shutdown() tries to start() a threading.Thread() that never
+    makes it all the way back to signal start().
+    
+    Also add a PendingReleaseNote and extra doc comments to clarify.
+    
+    Fixes: #8797
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    (cherry picked from commit 5ba9b8f21f8010c59dd84a0ef2acfec99e4b048f)
+    
+    Conflicts:
+    	PendingReleaseNotes
+
+commit 7faae891aefa4c21c50430fa03d9204a86d082f8
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Dec 9 14:52:19 2014 -0700
+
+    Revert "ceph.spec.: add epoch"
+    
+    If ICE ships 0.80.8, then it will be newer than what RHCEPH ships
+    (0.80.7), and users won't be able to seamlessly upgrade via Yum.
+    
+    We have three options:
+    A) Revert the "Epoch: 1" change on the Firefly branch.
+    B) Revert the "Epoch: 1" change in the ICE packages.
+    C) Bump the Epoch to "2" in Red Hat's packages.
+    
+    This commit does Option A.
+    
+    Option B may or may not be feasible - it would require a "downstream"
+    change in ICE, and we haven't done that sort of thing before.
+    
+    Due to the RHEL release schedule, Option C is not available to us at
+    this point.
+    
+    This reverts commit b890c1e4706d7cfef7ed24c9df65b439b4f7ff1d.
+
+commit ef3773ac93413c644e056babce1971f846bbc276
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Nov 26 15:18:07 2014 -0800
+
+    rgw: optionally call FCGX_Free() on the fcgi connection
+    
+    Fixes: #10194
+    
+    A new configurable controls this behavior. This forces disconnection of
+    the fcgi connection when done with the request.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit dd7c8c2bc6b2810ff7f483af940fa09dbe74e83a
+Merge: d759467 a597096
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Mon Dec 8 15:02:52 2014 -0800
+
+    Merge pull request #3109 from ceph/firefly-10263
+    
+    mds: store backtrace for straydir
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit d7594672b673796901961cc684c9e7de8dc2c95d
+Merge: adf9a75 73b47db
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Dec 8 13:19:44 2014 -0800
+
+    Merge pull request #3009 from dachary/wip-10018-primary-erasure-code-hinfo-firefly
+    
+    osd: deep scrub must not abort if hinfo is missing (firefly)
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit a5970963a2148697fc6da64facfbf6ab6686b9cd
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Nov 7 11:38:37 2014 +0800
+
+    mds: store backtrace for straydir
+    
+    Backport: giant, firefly, emperor, dumpling
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 0d89db5d3e5ae5d552d4058a88a4e186748ab1d2)
+
+commit adf9a758000182d27a6582d516356730d02e4099
+Merge: dea38a7 b4a4b75
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Dec 6 11:06:02 2014 -0800
+
+    Merge pull request #3089 from dachary/wip-10063-hobject-shard-firefly
+    
+    common: do not omit shard when ghobject NO_GEN is set (firefly)
+
+commit dea38a7af638c833304272c324ed2bc386a40f8f
+Merge: ccc8b46 5138091
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Dec 6 10:59:44 2014 -0800
+
+    Merge pull request #2480 from dachary/wip-9420-erasure-code-non-regression-firefly
+    
+    erasure-code: store and compare encoded contents (firefly)
+
+commit ccc8b46b2cdc55c1a861f092259ef36a1296f073
+Merge: bef363c cd3447d
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 5 17:33:05 2014 -0800
+
+    Merge pull request #3096 from dachary/wip-9785-dmcrypt-keys-permissions-firefly
+    
+    ceph-disk: dmcrypt file permissions (firefly)
+
+commit cd3447d04cabf6745001afeef69f25a92400cd0e
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Dec 4 22:21:32 2014 +0100
+
+    ceph-disk: dmcrypt file permissions
+    
+    The directory in which key files are stored for dmcrypt must be 700 and
+    the file 600.
+    
+    http://tracker.ceph.com/issues/9785 Fixes: #9785
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 58682d1776ab1fd4daddd887d921ca9cc312bf50)
+
+commit bef363c1d1fc06fbb315024145a97a809a2471cd
+Merge: cb2c83b 9f3b21d
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 5 09:04:00 2014 -0800
+
+    Merge pull request #3086 from dachary/wip-10125-radosgw-init-firefly
+    
+    rgw: run radosgw as apache with systemd (firefly)
+
+commit b4a4b75e6d4deb6818681902f85baa9f63acdb4f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Nov 14 01:16:10 2014 +0100
+
+    common: do not omit shard when ghobject NO_GEN is set
+    
+    Do not silence the display of shard_id when generation is NO_GEN.
+    Erasure coded objects JSON representation used by ceph_objectstore_tool
+    need the shard_id to find the file containing the chunk.
+    
+    Minimal testing is added to ceph_objectstore_tool.py
+    
+    http://tracker.ceph.com/issues/10063 Fixes: #10063
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit dcf09aed121f566221f539106d10283a09f15cf5)
+    
+    Conflicts:
+    	src/test/ceph_objectstore_tool.py
+
+commit 9f3b21d1b70be591d68bfa57c3393e8f9af8e7df
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Dec 2 18:10:48 2014 +0100
+
+    rgw: run radosgw as apache with systemd
+    
+    Same as sysv.
+    
+    http://tracker.ceph.com/issues/10125 Fixes: #10125
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 7b621f4abf63456272dec3449aa108c89504a7a5)
+    
+    Conflicts:
+    	src/init-radosgw.sysv
+
+commit cb2c83b2f216e503f7a52115f775bda1dbfe0c6a
+Merge: e2ec37b 02d4685
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Dec 4 11:32:18 2014 -0800
+
+    Merge pull request #3078 from ceph/wip-10030-firefly
+    
+    librbd: don't close an already closed parent image upon failure
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit e2ec37bf45fa5e7f5e787db9b67dbb2a98f2fbb7
+Merge: c4c63e8 af12194
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 3 23:01:44 2014 -0800
+
+    Merge pull request #3063 from ceph/wip-10123-firefly
+    
+    librbd: protect list_children from invalid child pool IoCtxs
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit c4c63e82cd7e4716315ca81208293a2567026c72
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Sep 23 15:52:08 2014 -0700
+
+    ReplicatedPG: don't move on to the next snap immediately
+    
+    If we have a bunch of trimmed snaps for which we have no
+    objects, we'll spin for a long time.  Instead, requeue.
+    
+    Fixes: #9487
+    Backport: dumpling, firefly, giant
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit c17ac03a50da523f250eb6394c89cc7e93cb4659)
+
+commit 1b656450ca75b12fb98dee82bace914ef5f45c44
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 23 16:21:33 2014 -0700
+
+    osd: initialize purged_snap on backfill start; restart backfill if change
+    
+    If we backfill a PG to a new OSD, we currently neglect to initialize
+    purged_snaps.  As a result, the first time the snaptrimmer runs it has to
+    churn through every deleted snap for all time, and to make matters worse
+    does so in one go with the PG lock held.  This leads to badness on any
+    cluster with a significant number of removed snaps that experiences
+    backfill.
+    
+    Resolve this by initializing purged_snaps when we finish backfill.  The
+    backfill itself will clear out any stray snaps and ensure the object set
+    is in sync with purged_snaps.  Note that purged_snaps on the primary
+    that is driving backfill will not change during this period as the
+    snaptrimmer is not scheduled unless the PG is clean (which it won't be
+    during backfill).
+    
+    If we by chance to interrupt backfill, go clean with other OSDs,
+    purge snaps, and then let this OSD rejoin, we will either restart
+    backfill (non-contiguous log) or the log will include the result of
+    the snap trim (the events that remove the trimmed snap).
+    
+    Fixes: #9487
+    Backfill: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 255b430a87201c7d0cf8f10a3c1e62cbe8dd2d93)
+
+commit 02d4685c56e129cb179a5ddfb8e87aefc2fce0b5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Nov 6 05:01:38 2014 -0500
+
+    librbd: don't close an already closed parent image upon failure
+    
+    If librbd is not able to open a child's parent image, it will
+    incorrectly close the parent image twice, resulting in a crash.
+    
+    Fixes: #10030
+    Backport: firefly, giant
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 61ebfebd59b61ffdc203dfeca01ee1a02315133e)
+
+commit af121942d7bdfc59fcfae0429ffb12993e7e019d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 17 21:49:26 2014 -0500
+
+    librbd: protect list_children from invalid child pool IoCtxs
+    
+    While listing child images, don't ignore error codes returned
+    from librados when creating an IoCtx. This will prevent seg
+    faults from occurring when an invalid IoCtx is used.
+    
+    Fixes: #10123
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0d350b6817d7905908a4e432cd359ca1d36bab50)
+
+commit c982da44e0e9e0be3c3d4e8f5e0a186fb2fcebb3
+Merge: 4a148df c4e4a31
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Nov 30 10:12:04 2014 -0800
+
+    Merge pull request #3014 from dachary/wip-9665-ceph-disk-partprobe-firefly
+    
+    ceph disk zap must call partprobe
+
+commit c4e4a310f14ca3049ac90422aea95051fe0d4b15
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Fri Oct 10 10:26:31 2014 +0200
+
+    ceph-disk: use update_partition in prepare_dev and main_prepare
+    
+    In the case of prepare_dev the partx alternative was missing and is not
+    added because update_partition does it.
+    
+    http://tracker.ceph.com/issues/9721 Fixes: #9721
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 23e71b1ee816c0ec8bd65891998657c46e364fbe)
+    
+    Conflicts:
+    	src/ceph-disk
+
+commit e70a81464b906b9a304c29f474e6726762b63a7c
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Thu Oct 9 18:52:17 2014 +0200
+
+    ceph-disk: run partprobe after zap
+    
+    Not running partprobe after zapping a device can lead to the following:
+    
+    * ceph-disk prepare /dev/loop2
+    * links are created in /dev/disk/by-partuuid
+    * ceph-disk zap /dev/loop2
+    * links are not removed from /dev/disk/by-partuuid
+    * ceph-disk prepare /dev/loop2
+    * some links are not created in /dev/disk/by-partuuid
+    
+    This is assuming there is a bug in the way udev events are handled by
+    the operating system.
+    
+    http://tracker.ceph.com/issues/9665 Fixes: #9665
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit fed3b06c47a5ef22cb3514c7647544120086d1e7)
+
+commit 5a5f427bc09076ef3fb13a710dede1b47bb232e0
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Fri Oct 10 10:23:34 2014 +0200
+
+    ceph-disk: encapsulate partprobe / partx calls
+    
+    Add the update_partition function to reduce code duplication.
+    The action is made an argument although it always is -a because it will
+    be -d when deleting a partition.
+    
+    Use the update_partition function in prepare_journal_dev
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 922a15ea6865ef915bbdec2597433da6792c1cb2)
+    
+    Conflicts:
+    	src/ceph-disk
+
+commit 73b47dbee8858f182fd2b4fd8eb5f3c786877bf4
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Thu Nov 6 17:11:20 2014 +0100
+
+    osd: deep scrub must not abort if hinfo is missing
+    
+    Instead it should set read_error.
+    
+    http://tracker.ceph.com/issues/10018 Fixes: #10018
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 9d84d2e8309d26e39ca849a75166d2d7f2dec9ea)
+
+commit 5138091a4073d966a65f537280f89372e801d019
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Tue Sep 23 11:38:09 2014 +0200
+
+    erasure-code: add corpus verification to make check
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+
+commit 8d3d6bf59aec3877c0231e637270e371d9ed3b8c
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Sat Sep 13 13:36:09 2014 +0200
+
+    erasure-code: workunit to check for encoding regression
+    
+    Clone the archive of encoded objects and decode all archived objects, up
+    to and including the current ceph version.
+    
+    http://tracker.ceph.com/issues/9420 Refs: #9420
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+
+commit 4f4358708ed3c261ca4027cc9c3dc3f952a99470
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Sat Sep 13 10:16:31 2014 +0200
+
+    erasure-code: store and compare encoded contents
+    
+    Introduce ceph_erasure_code_non_regression to check and compare how an
+    erasure code plugin encodes and decodes content with a given set of
+    parameters. For instance:
+    
+    ./ceph_erasure_code_non_regression \
+          --plugin jerasure \
+          --parameter technique=reed_sol_van \
+          --parameter k=2 \
+          --parameter m=2 \
+          --stripe-width 3181 \
+          --create \
+          --check
+    
+    Will create an encoded object (--create) and store it into a directory
+    along with the chunks, one chunk per file. The directory name is derived
+    from the parameters. The content of the object is a random pattern of 31
+    bytes repeated to fill the object size specified with --stripe-width.
+    
+    The check function (--check) reads the object back from the file,
+    encodes it and compares the result with the content of the chunks read
+    from the files. It also attempts recover from one or two erasures.
+    
+    Chunks encoded by a given version of Ceph are expected to be encoded
+    exactly in the same way by all Ceph versions going forward.
+    
+    http://tracker.ceph.com/issues/9420 Refs: #9420
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+
+commit 4a148df544978383c1ed7cd8b90f590adb563f3d
+Merge: c069bce 01faf13
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Wed Nov 19 02:45:26 2014 +0100
+
+    Merge pull request #2961 from ceph/wip-10114-firefly
+    
+    Add annotation to all assembly files to turn off stack-execute bit
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 01faf1356f648ded9acda02e7cc67c1adb9e9ee3
+Author: Dan Mick <dan.mick@redhat.com>
+Date:   Fri Nov 14 17:59:57 2014 -0800
+
+    Add annotation to all assembly files to turn off stack-execute bit
+    
+    See discussion in http://tracker.ceph.com/issues/10114
+    
+    Building with these changes allows output from readelf like this:
+    
+     $ readelf -lW src/.libs/librados.so.2 | grep GNU_STACK
+      GNU_STACK      0x000000 0x0000000000000000 0x0000000000000000 0x000000
+    0x000000 RW  0x8
+    
+    (note the absence of 'X' in 'RW')
+    
+    Fixes: #10114
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    (cherry picked from commit 06a245a9845c0c126fb3106b41b2fd2bc4bc4df3)
+    (not-yet-present-in-firefly files in isa-l manually removed)
+
+commit c069bce4e8180da3c0ca4951365032a45df76468
+Merge: 0d8ad6a fac1654
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Nov 13 10:36:12 2014 -0800
+
+    Merge pull request #2760 from ceph/wip-9835-firefly
+    
+    osd: fix erasure hung op bug (9835)
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit fac165475031efdebbb88898ca5c12cd307a5bc3
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Nov 5 12:12:14 2014 -0800
+
+    osd: use OSDMap helper to tell if ops are misdirected
+    
+    calc_pg_role doesn't actually take into account primary affinity.
+    
+    Fixes: #9835
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 588602bf0095de5b59064123ca01345f1364bdde
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 20 13:55:33 2014 -0700
+
+    osd: discard rank > 0 ops on erasure pools
+    
+    Erasure pools do not support read from replica, so we should drop
+    any rank > 0 requests.
+    
+    This fixes a bug where an erasure pool maps to [1,2,3], temporarily maps
+    to [-1,2,3], sends a request to osd.2, and then remaps back to [1,2,3].
+    Because the 0 shard never appears on osd.2, the request sits in the
+    waiting_for_pg map indefinitely and cases slow request warnings.
+    This problem does not come up on replicated pools because all instances of
+    the PG are created equal.
+    
+    Fix by only considering role == 0 for erasure pools as a correct mapping.
+    
+    Fixes: #9835
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 0c1c4152e6f402af7612c8c8d4719ab0f4cc6ad9
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 12 17:04:35 2014 -0800
+
+    osd/OSDMap: add osd_is_valid_op_target()
+    
+    Helper to check whether an osd is a given op target for a pg.  This
+    assumes that for EC we always send ops to the primary, while for
+    replicated we may target any replica.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 89c02637914ac7332e9dbdbfefc2049b2b6c127d)
+
+commit 0d8ad6ad3c376bcab981bea9a49e1924d7eddb68
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Tue Nov 11 18:16:02 2014 -0800
+
+    qa: allow small allocation diffs for exported rbds
+    
+    The local filesytem may behave slightly differently. This isn't
+    foolproof, but seems to be reliable enough on rhel7 rootfs, where
+    exact comparison was failing.
+    
+    Fixes: #10002
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit e94d3c11edb9c9cbcf108463fdff8404df79be33)
+
+commit 0804deeab293e09123d1b58825051ccc4dddbc0e
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun May 25 08:38:38 2014 -0700
+
+    osd: fix map advance limit to handle map gaps
+    
+    The recent change in cf25bdf6b0090379903981fe8cee5ea75efd7ba0 would stop
+    advancing after some number of epochs, but did not take into consideration
+    the possibilty that there are missing maps.  In that case, it is impossible
+    to advance past the gap.
+    
+    Fix this by increasing the max epoch as we go so that we can always get
+    beyond the gap.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 1e0a82fd55dede473c0af32924f4bcb5bb697a2b)
+
+commit d30d6b986433eaef920b3703cf5af3c030f8dcf4
+Merge: d241aa7 d548431
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Fri Nov 7 14:10:18 2014 -0800
+
+    Merge pull request #2880 from ceph/wip-10025-firefly
+    
+    #10025/firefly -- tools: fix MDS journal import
+    
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+
+commit d548431a388da1130564d710e1f006772934224b
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Nov 7 11:34:43 2014 +0000
+
+    tools: fix MDS journal import
+    
+    Previously it only worked on fresh filesystems which
+    hadn't been trimmed yet, and resulted in an invalid
+    trimmed_pos when expire_pos wasn't on an object
+    boundary.
+    
+    Fixes: #10025
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit fb29e71f9a97c12354045ad2e128156e503be696)
+
+commit d241aa7a347655242cc71b8fa3d778df6948c494
+Merge: 2c85b5d 4afb542
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Nov 6 10:37:42 2014 -0800
+
+    Merge remote-tracking branch 'origin/wip-sam-firefly-backports' into firefly
+
+commit 2c85b5d72953d01296213185382707122e06415c
+Merge: 23cbffa 1228658
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Nov 6 10:30:20 2014 -0800
+
+    Merge pull request #2737 from ceph/wip-9629-firefly
+    
+    osd: do not clone/preserve snapdir on cache-evict (firefly backport)
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 23cbffaa2936dc2707b5b42f8c0e7ce804324ae2
+Merge: 3bba938 e296685
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Nov 6 10:26:21 2014 -0800
+
+    Merge pull request #2657 from ceph/wip-9053-9301-firefly
+    
+    mon: backport two paxos fixes to firefly
+    
+    Reviewed-by: Joao Luis <joao@redhat.com>
+
+commit 3bba9387eb123c6cf055e874db2925b998dc406c
+Merge: 3f9bf73 835f8c6
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Nov 6 10:21:12 2014 -0800
+
+    Merge pull request #2656 from ceph/wip-9502-firefly
+    
+    mon: backport mon disk full check to firefly
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 3f9bf738daf47ff4ff56c9f76d1487a5afc5e30a
+Merge: a340e6d 3e17a08
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Nov 6 10:18:27 2014 -0800
+
+    Merge pull request #2764 from ceph/wip-9851
+    
+    osd: bring FileJournal in sync with giant
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit a340e6d0b166019f58dca0703faf30dd3178c14f
+Merge: b7d5f99 b9450b5
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Nov 6 10:12:17 2014 -0800
+
+    Merge pull request #2776 from ceph/wip-9675.firefly
+    
+    CrushWrapper: pick a ruleset same as rule_id
+    
+    Reviewed-by: Samuel Just <sam.just@inktank.com>
+
+commit b7d5f99c8f4d751e83dc29305649d7a465c657b1
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 15 15:29:08 2014 -0700
+
+    ceph-disk: mount xfs with inode64 by default
+    
+    We did this forever ago with mkcephfs, but ceph-disk didn't.  Note that for
+    modern XFS this option is obsolete, but for older kernels it was not the
+    default.
+    
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 11496399ef318498c11e551f139d96db52d3309c)
+
+commit 1a9d000bb679a7392b9dd115373c3827c9626694
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Oct 9 10:20:27 2014 -0700
+
+    rgw: set length for keystone token validation request
+    
+    Fixes: #7796
+    Backport: giany, firefly
+    Need to set content length to this request, as the server might not
+    handle a chunked request (even though we don't send anything).
+    
+    Tested-by: Mark Kirkwood <mark.kirkwood@catalyst.net.nz>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 3dd4ccad7fe97fc16a3ee4130549b48600bc485c)
+
+commit 49d27efde2ce5d282c9ee6ca9c8ea9db8f609392
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Aug 19 13:15:46 2014 -0700
+
+    rgw: subuser creation fixes
+    
+    Fixes: #8587
+    There were a couple of issues, one when trying to identify whether swift
+    user exists, we weren't using the correct swift id. The second problem
+    is that we relied on the gen_access flag in the swift case, where it
+    doesn't really need to apply.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 1441ffe8103f03c6b2f625f37adbb2e1cfec66bb)
+
+commit 8db2f0969e1715f57088c311a33e7e3499933afb
+Merge: 9a15592 2a5d7f0
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 31 08:35:50 2014 -0700
+
+    Merge pull request #2847 from dachary/wip-9752-past-intervals-firefly
+    
+    osd: past_interval display bug on acting
+
+commit 2a5d7f08303eb8b1687c5b58426443b3d40e415c
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Fri Oct 31 00:49:21 2014 +0100
+
+    osd: past_interval display bug on acting
+    
+    The acting array was incorrectly including the primary and up_primary.
+    
+    http://tracker.ceph.com/issues/9752 Fixes: #9752
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit c5f8d6eded52da451fdd1d807bd4700221e4c41c)
+
+commit 9a15592eae6739183049f8376392c5577145871e
+Merge: 6fd8879 c20a242
+Author: Yan, Zheng <ukernel@gmail.com>
+Date:   Thu Oct 30 17:01:24 2014 -0700
+
+    Merge pull request #2840 from ceph/firefly-9869
+    
+    Backport "client: cast m->get_client_tid() to compare to 16-bit Inode::flushing_cap_tid"
+
+commit c20a2421f4a5b33407d7f31806dd4587b4e9077c
+Author: Greg Farnum <greg@inktank.com>
+Date:   Wed Oct 22 17:16:31 2014 -0700
+
+    client: cast m->get_client_tid() to compare to 16-bit Inode::flushing_cap_tid
+    
+    m->get_client_tid() is 64 bits (as it should be), but Inode::flushing_cap_tid
+    is only 16 bits. 16 bits should be plenty to let the cap flush updates
+    pipeline appropriately, but we need to cast in the proper direction when
+    comparing these differently-sized versions. So downcast the 64-bit one
+    to 16 bits.
+    
+    Fixes: #9869
+    Backport: giant, firefly, dumpling
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit a5184cf46a6e867287e24aeb731634828467cd98)
+
+commit 4afb54274bb2087da348103d0a7641b3a018d777
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Sep 11 13:46:51 2014 -0700
+
+    ReplicatedPG: cancel cb on blacklisted watcher
+    
+    Fixes: #8315
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 16bd45777166c29c433af3b59254a7169e512d98)
+
+commit 037aa342fa277351fc605da41489a6ceba81ab05
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Sun Sep 21 10:19:43 2014 -0700
+
+    ReplicatedPG::on_removal: clear rollback info
+    
+    Fixes: #9293
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 544b8c7ffb4af01765b87239f2d7ab88479ee779)
+
+commit 8978c8dc1f153a9052b5d967ceff11d4f08d51ea
+Merge: 8401e7f 4be53d5
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Oct 30 13:48:18 2014 -0700
+
+    Merge remote-tracking branch 'origin/wip-9574' into wip-sam-firefly-backports
+
+commit 8401e7ffa1768770f451143b3c110d1deae1bd40
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Sep 29 15:01:25 2014 -0700
+
+    PG: release backfill reservations if a backfill peer rejects
+    
+    Also, the full peer will wait until the rejection from the primary
+    to do a state transition.
+    
+    Fixes: #9626
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 624aaf2a4ea9950153a89ff921e2adce683a6f51)
+
+commit 5df09fa1b5a42992f9e41aca09e86db0d03d9fbd
+Merge: a0937ef a1aa06b
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Oct 30 13:47:22 2014 -0700
+
+    Merge remote-tracking branch 'origin/wip-9113' into wip-sam-firefly-backports
+
+commit a0937ef214b9221e7a5e69e7b0f0697471d56293
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Oct 12 10:05:51 2014 -0700
+
+    osd/osd_types: consider CRUSH_ITEM_NONE in check_new_interval() min_size check
+    
+    Fixes: #9718
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d947050c82a511f91c98e1c76e48ffa9e187eee7)
+    
+    Conflicts:
+    	src/osd/osd_types.cc
+
+commit 35e8e6b2c1733cd421bf6c3916553eea3786e76e
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Oct 20 14:10:58 2014 -0700
+
+    PG:: reset_interval_flush and in set_last_peering_reset
+    
+    If we have a change in the prior set, but not in the up/acting set, we go back
+    through Reset in order to reset peering state.  Previously, we would reset
+    last_peering_reset in the Reset constructor.  This did not, however, reset the
+    flush_interval, which caused the eventual flush event to be ignored and the
+    peering messages to not be sent.
+    
+    Instead, we will always reset_interval_flush if we are actually changing the
+    last_peering_reset value.
+    
+    Fixes: #9821
+    Backport: firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit d9ff3a6b789c5b9c77aefa3751bd808f5d7b8ca7)
+
+commit 3f35db42977704a12ac4b5bcad6261aaf6b6a88d
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Oct 23 09:11:28 2014 -0700
+
+    ReplicatedPG: writeout hit_set object with correct prior_version
+    
+    Fixes: #9875
+    Backport: giant, firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+    (cherry picked from commit 1a3ad307f1a4c0a956d6fd31d13f01ffe411a09d)
+
+commit 6fd88792e77cdc7ad33ff0acf9b3189a7c525430
+Merge: 0975ec9 afe6bd8
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Oct 26 20:37:52 2014 -0700
+
+    Merge pull request #2717 from dachary/wip-9747-ceph-spec-firefly
+    
+    rpm: 95-ceph-osd-alt.rules is not needed for centos7 / rhel7 (firefly)
+
+commit b9450b532ab7ad23ec6e2c22ed7cf55e6e1cc4c0
+Author: Xiaoxi Chen <xiaoxi.chen@intel.com>
+Date:   Wed Aug 20 15:35:44 2014 +0800
+
+    CrushWrapper: pick a ruleset same as rule_id
+    
+    Originally in the add_simple_ruleset funtion, the ruleset_id
+    is not reused but rule_id is reused. So after some add/remove
+    against rules, the newly created rule likely to have
+    ruleset!=rule_id.
+    
+    We dont want this happen because we are trying to hold the constraint
+    that ruleset == rule_id.
+    
+    Signed-off-by: Xiaoxi Chen <xiaoxi.chen@intel.com>
+    (cherry picked from commit 78e84f34da83abf5a62ae97bb84ab70774b164a6)
+    
+    Conflicts:
+    	src/test/erasure-code/TestErasureCodeIsa.cc
+    
+    Fixes: #9675
+
+commit 3e17a0872a3864cb6aee46959afd955ef0cbafeb
+Author: Ma Jianpeng <jianpeng.ma@intel.com>
+Date:   Mon Jul 21 15:08:55 2014 +0800
+
+    os/FileJournal: When dump journal, using correctly seq avoid misjudging joural corrupt.
+    
+    In func FileJournal::dump, it always using seq=0 as last-seq and it can
+    misjudge the journal corrupt.
+    
+    Signed-off-by: Ma Jianpeng <jianpeng.ma@intel.com>
+    (cherry picked from commit 5f65b4db6d1dad7c2c5a09eab42af63a82ea9e9b)
+
+commit 350da8c98bc1e90cd392992aba290c7478280d88
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Fri Sep 26 01:15:53 2014 +0200
+
+    os: io_event.res is the size written
+    
+    And not an error code to be converted with cpp_strerror()
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 7827e0035e3350ad2d9230f27a1629545f53af5c)
+
+commit ecff3761f3d15061a2cbf0a595ca249a4c424f4c
+Author: Ma Jianpeng <jianpeng.ma@intel.com>
+Date:   Thu Aug 21 15:10:46 2014 +0800
+
+    os/FileJournal: For journal-aio-mode, don't use aio when closing journal.
+    
+    For jouranl-aio-mode when closing journal, the write_finish_thread_entry may exit before
+    write_thread_entry. This cause no one wait last aios to complete.
+    On some platform, after that the journal-header on journal corrupted.
+    To avoid this, when closing jouranl we don't use aio.
+    
+    Fixes: 9073
+    Reported-by: Mark Kirkwood <mark.kirkwood@catalyst.net.nz>
+    Tested-by: Mark Kirkwood <mark.kirkwood@catalyst.net.nz>
+    Signed-off-by: Ma Jianpeng <jianpeng.ma@intel.com>
+    (cherry picked from commit e870fd09ce846e5642db268c33bbe8e2e17ffef2)
+
+commit dbc33fbab4b35e2ce1e46a881f6714262502c243
+Author: Ma Jianpeng <jianpeng.ma@intel.com>
+Date:   Thu Aug 21 21:07:51 2014 +0800
+
+    os/FileJournal: Only using aio then alloc the related resources.
+    
+    If define HAVE_LIBAIO, it alloc related resouces. But itt don't check whether
+    using aio mode. Only using aio it alloc the related resources.
+    
+    Signed-off-by: Ma Jianpeng <jianpeng.ma@intel.com>
+    (cherry picked from commit a66a4931d5be9ee26c0983b3154fdbe37261a51c)
+
+commit 3312c6eeca21dcd566df9bdd8de7b3fe33356b57
+Author: Ma Jianpeng <jianpeng.ma@intel.com>
+Date:   Thu Aug 21 15:49:44 2014 +0800
+
+    os/FileJournal: Tune the judge logic for read_header.
+    
+    When reading journal-header, it should firstly check the result of
+    pread and then do decoce operation.
+    
+    Signed-off-by: Ma Jianpeng <jianpeng.ma@intel.com>
+    (cherry picked from commit c8e2b89cf6bc36a0ff29887b9e76cbbeceef9f8f)
+
+commit b42107584449e1f85cbee97bfa486ebeb310e6a1
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 19 20:50:13 2014 -0700
+
+    os/FileJournal: signal aio_cond even if seq is 0
+    
+    This can happen if we write a journal but no events.
+    
+    Reported-by: Somnath Roy <somnath.roy@sandisk.com>
+    Reported-by: Ma, Jianpeng <jianpeng.ma@intel.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 57778e2c577c1e1bbf9525232720a2994fa36abc)
+
+commit 4a43ea171d84a9f3a13064030eb386fcfbe3bbb8
+Author: Ma Jianpeng <jianpeng.ma@intel.com>
+Date:   Wed Jul 23 10:10:38 2014 -0700
+
+    os/FileJournal: Update the journal header when closing journal
+    
+    When closing journal, it should check must_write_header and update
+    journal header if must_write_header alreay set.
+    It can reduce the nosense journal-replay after restarting osd.
+    
+    Signed-off-by: Ma Jianpeng <jianpeng.ma@intel.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5bf472aefb7360a1fe17601b42e551df120badfb)
+
+commit 0975ec9cec1c466f7b15f5173541a7eab02dae18
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 21 06:53:36 2014 -0700
+
+    Revert "os/FileJournal: stop aio completion thread *after* writer thread"
+    
+    This reverts commit 334631ae4641824b3df49245f36a8fd4b143bf3f.
+
+commit 5a10b95f7968ecac1f2af4abf9fb91347a290544
+Merge: cc69c16 d5bac46
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Oct 17 10:47:22 2014 -0700
+
+    Merge pull request #2716 from ceph/wip-firefly-9419
+    
+    Backport fix for bug #9419
+
+commit cc69c16c2108cb231b2434a53e3eef51b597756b
+Merge: f9cdaab 334631a
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Oct 17 10:44:30 2014 -0700
+
+    Merge pull request #2724 from dachary/wip-9073-journal-aio-mode-firefly
+    
+    os/FileJournal: stop aio completion thread *after* writer thread
+
+commit f9cdaabe078415d7927e2618030996f2f24be2f1
+Merge: 0b4b34a 412c277
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 17 08:20:53 2014 -0700
+
+    Merge pull request #2742 from ceph/firefly-unknown-locktype
+    
+    mds: reply -EOPNOTSUPP for unknown lock type
+
+commit 412c2770c74abea73a94e10df7b83ebe11ac82ee
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Oct 14 22:02:41 2014 +0800
+
+    mds: reply -EOPNOTSUPP for unknown lock type
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 675392335c53ff7879031fb9184e4f35bcc90fe2)
+
+commit 1228658871e53e350bdab3e72cdefd1caf33c291
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Sep 21 15:56:18 2014 -0700
+
+    osd/ReplicatedPG: do not clone or preserve snapdir on cache_evict
+    
+    If we cache_evict a head in a cache pool, we need to prevent
+    make_writeable() from cloning the head and finish_ctx() from
+    preserving the snapdir object.
+    
+    Fixes: #8629
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ce8eefca13008a9cce3aedd67b11537145e1fd77)
+
+commit 88e6014463e86e48d78ac419226644209f83f2a0
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Sep 21 15:54:15 2014 -0700
+
+    ceph_test_rados_api_tier: add EvictSnap2 test case
+    
+    Verify an evict doesn't create a snapdir object.  Reproduces #8629
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 398c74eacb1ce4e573aef0d24718a5925d90272b)
+
+commit 0b4b34aac497d17a6474c35891aab2bde962524b
+Merge: 322958a 0a72235
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 16 06:09:51 2014 -0700
+
+    Merge pull request #2734 from ceph/wip-firefly-undump
+    
+    mds: fix --undump-journal
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 0a72235b0556752fadebc3e155ad41b13a0a15e9
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Oct 16 11:17:40 2014 +0100
+
+    mds: fix --undump-journal
+    
+    This hadn't worked for a long time.  This is a fix
+    for firefly only, as this code was refactored in giant.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+
+commit 835f8c6f6121f3ebdec3a0d2d5cb1376301dc03a
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Tue Sep 23 14:02:55 2014 +0100
+
+    ceph-mon: check fs stats just before preforking
+    
+    Otherwise statfs may fail if mkfs hasn't been run yet or if the monitor
+    data directory does not exist.  There are checks to account for the mon
+    data dir not existing and we should wait for them to clear before we go
+    ahead and check the fs stats.
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 7f71c11666b25e91dd612c58b4eda9ac0d4752f8)
+    
+    Conflicts:
+    	src/ceph_mon.cc
+
+commit 1ddf435464562f70f63cdb0032da3187f34ce853
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Thu Sep 18 16:53:43 2014 +0100
+
+    ceph_mon: check available storage space for mon data dir on start
+    
+    error out if available storage space is below 'mon data avail crit'
+    
+    Fixes: #9502
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 2da1a2914ac7df18ce842b0aac728fffb5bed2b6)
+    
+    Conflicts:
+    	src/ceph_mon.cc
+
+commit 112317791b744d9890a65adcc13554c85e90f3af
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Thu Sep 18 16:52:34 2014 +0100
+
+    mon: DataHealthService: use get_fs_stats() instead
+    
+    and relieve the DataStats struct from clutter by using
+    ceph_data_stats_t instead of multiple fields.
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 9996d446988768658db751a7843b13cf3d194213)
+    
+    Conflicts:
+    	src/mon/DataHealthService.cc
+
+commit f0a92d72fd44542619338db7d6da98e147b6a9fc
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Thu Sep 18 16:32:20 2014 +0100
+
+    common: util: add get_fs_stats() function
+    
+    simplifies the task of obtaining available/used disk space, as well as
+    used available percentage.
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 3d74230d1c0fbfa15487e2a90ac60b883476e840)
+
+commit a8fa009fbe5d5d4d9cfa134d5ecd05c92290a8eb
+Author: Joao Eduardo Luis <jeclui@gmail.com>
+Date:   Thu Sep 18 16:25:44 2014 +0100
+
+    include/util.h: prevent multiple inclusion of header
+    
+    Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>
+    (cherry picked from commit 76eff9503493312cb97e4a2f9236f4dbcbf931df)
+
+commit e296685e8f3f5158238216eefb76482bd6d55134
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 18 14:23:36 2014 -0700
+
+    mon: re-bootstrap if we get probed by a mon that is way ahead
+    
+    During bootstrap we verify that our paxos commits overlap with the other
+    mons we will form a quorum with.  If they do not, we do a sync.
+    
+    However, it is possible we pass those checks, then fail to join a quorum
+    before the quorum moves ahead in time such that we no longer overlap.
+    Currently nothing kicks up back into a probing state to discover we need
+    to sync... we will just keep trying to call or join an election instead.
+    
+    Fix this by jumping back to bootstrap if we get a probe that is ahead of
+    us.  Only do this from non probe or sync states as these will be common;
+    it is only the active and electing states that matter (and probably just
+    electing!).
+    
+    Fixes: #9301
+    Backport: giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c421b55e8e15ef04ca8aeb47f7d090375eaa8573)
+
+commit 0e57767d5fc524939e8968b506ce2fb3f4f80656
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 18 14:11:24 2014 -0700
+
+    mon/Paxos: fix off-by-one in last_ vs first_committed check
+    
+    peon last_committed + 1 == leader first_committed is okay.  Note that the
+    other check (where I clean up whitespace) gets this correct.
+    
+    Fixes: #9301 (partly)
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d81cd7f86695185dce31df76c33c9a02123f0e4a)
+
+commit 1f4aaf648f4aa6f6056d0e8ce629eeea05c5424d
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 13 16:17:02 2014 -0700
+
+    mon/Paxos: share state and verify contiguity early in collect phase
+    
+    We verify peons are contiguous and share new paxos states to catch peons
+    up at the end of the round.  Do this each time we (potentially) get new
+    states via a collect message.  This will allow peons to be pulled forward
+    and remain contiguous when they otherwise would not have been able to.
+    For example, if
+    
+      mon.0 (leader)  20..30
+      mon.1 (peon)    15..25
+      mon.2 (peon)    28..40
+    
+    If we got mon.1 first and then mon.2 second, we would store the new txns
+    and then boot mon.1 out at the end because 15..25 is not contiguous with
+    28..40.  However, with this change, we share 26..30 to mon.1 when we get
+    the collect, and then 31..40 when we get mon.2's collect, pulling them
+    both into the final quorum.
+    
+    It also breaks the 'catch-up' work into smaller pieces, which ought to
+    smooth out latency a bit.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c54f1e4d66b22bad715ac17e9baa72ab93e48c46)
+
+commit 6c5b9a666fcd94e175a8b9771368b55246957efe
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 14 16:55:58 2014 -0700
+
+    mon/Paxos: verify all new peons are still contiguous at end of round
+    
+    During the collect phase we verify that each peon has overlapping or
+    contiguous versions as us (and can therefore be caught up with some
+    series of transactions).  However, we *also* assimilate any new states we
+    get from those peers, and that may move our own first_committed forward
+    in time.  This means that an early responder might have originally been
+    contiguous, but a later one moved us forward, and when the round finished
+    they were not contiguous any more.  This leads to a crash on the peon
+    when they get our first begin message.
+    
+    For example:
+    
+     - we have 10..20
+     - first peon has 5..15
+       - ok!
+     - second peon has 18..30
+       - we apply this state
+     - we are now 18..30
+     - we finish the round
+       - send commit to first peon (empty.. we aren't contiguous)
+       - send no commit to second peon (we match)
+     - we send a begin for state 31
+       - first peon crashes (it's lc is still 15)
+    
+    Prevent this by checking at the end of the round if we are still
+    contiguous.  If not, bootstrap.  This is similar to the check we do above,
+    but reverse to make sure *we* aren't too far ahead of *them*.
+    
+    Fixes: #9053
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3e5ce5f0dcec9bbe9ed4a6b41758ab7802614810)
+
+commit 11d2c9dd4aeb835ca73bfb41fb15b1038547adf6
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 13 16:01:01 2014 -0700
+
+    mon/Paxos: put source mon id in a temp variable
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit bb046ed01ecf58b8c87eeeee2e00a476e6fba467)
+
+commit 322958a5aad82c031b54592b372aa053e8993be4
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 15 12:26:00 2014 -0700
+
+    qa/workunits/rbd/import_export.sh: be case insensitive
+    
+    Stop tripping over this change (from dumpling).
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5558afa03dbd1b20766b76e9410ef5bc3e73784f)
+
+commit 334631ae4641824b3df49245f36a8fd4b143bf3f
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Aug 29 19:40:29 2014 -0700
+
+    os/FileJournal: stop aio completion thread *after* writer thread
+    
+    The writer thread may submit a new aio to update the header in its
+    final moments before shutting down.  Do not stop the aio thread until after
+    that has happened or else we may not wait for those aio completions.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c776a89880fdac270e6334ad8e49fa616d05d0d4)
+    
+    Conflicts:
+    	src/os/FileJournal.cc
+
+commit 111eec9ff6325a12fcbf066ae08f27919aeae5d8
+Merge: cf4e300 6c0127f
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 14 14:57:42 2014 -0700
+
+    Merge remote-tracking branch 'gh/firefly' into firefly-next
+
+commit cf4e30095e8149d1df0f2c9b4c93c9df0779ec84
+Author: Xiaoxi Chen <xiaoxi.chen@intel.com>
+Date:   Tue Aug 5 16:12:22 2014 +0800
+
+    mon/OSDMonitor : Use user provided ruleset for replicated pool
+    
+    When creating a replicated pool, currently ceph ignore the ruleset
+    name provided by user but use a global default ruleset.
+    
+    This patch fix this bug, so the rulset specified by
+    ceph osd pool create replicated
+    can be properly set.
+    
+    Signed-off-by: Xiaoxi Chen <xiaoxi.chen@intel.com>
+    (cherry picked from commit bf9726a294abd32b429170284ac328a592802648)
+
+commit bfd7da10e708a6eefc6d992b2b6337b7f06fd5ed
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jun 3 13:05:19 2014 +0200
+
+    documentation: update osd pool create erasure
+    
+    The properties are replaced with erasure code profiles. Remove the
+    reference to properties and the documentation of each erasure-code
+    related property.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 8ff4edda73abb920c91e1226a330e3659def1fbe)
+
+commit afe6bd89f8a1588fb67063d1a08a4be8c1ab2ce6
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Sat Oct 11 18:20:36 2014 +0200
+
+    rpm: 95-ceph-osd-alt.rules is not needed for centos7 / rhel7
+    
+    The || instead of && had it always installed. That was fixed in EPEL
+    already.
+    
+    http://tracker.ceph.com/issues/9747 Fixes: #9747
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 5ff4a850a0d809b3f25988c6cceb82c35095ef84)
+
+commit d5bac46e06c5420f29a021b294e391b2c6694cbd
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Sep 24 16:02:21 2014 -0700
+
+    osd: Return EOPNOTSUPP if a set-alloc-hint occurs with OSDs that don't support
+    
+    Add CEPH_FEATURE_OSD_SET_ALLOC_HINT feature bit
+    Collect the intersection of all peer feature bits during peering
+    When handling CEPH_OSD_OP_SETALLOCHINT check that all OSDs support it
+    by checking for CEPH_FEATURE_OSD_SET_ALLOC_HINT feature bit.
+    
+    Fixes: #9419
+    Backport: firefly
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 9b39033f2b2bcdd2be0f6da4dff06023d0f77499)
+    
+    Conflicts:
+    
+    	src/include/ceph_features.h
+    	src/osd/PG.cc
+    	src/osd/PG.h
+    	src/osd/ReplicatedPG.cc
+
+commit de08802dcf35aea516d013d3d6116aaa7707b923
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Sep 19 15:12:55 2014 -0700
+
+    osd: Remove unused PG functions queue_notify(), queue_info(), queue_log()
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 70ef4c11cbae669799c30c7592073ad7aa11dcd6)
+
+commit 5b5aba73031e901457ca27cf15600ce1ca90e258
+Merge: 345714b a1ae7f4
+Author: Gregory Farnum <greg@inktank.com>
+Date:   Fri Oct 10 06:57:06 2014 -0700
+
+    Merge pull request #2691 from ceph/firefly-unused-variable
+    
+    Firefly unused variable
+
+commit a1ae7f471c809e69d363b9145e70160533bfa48c
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Oct 10 21:36:39 2014 +0800
+
+    mds: Locker: remove unused variable
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+
+commit 345714b6b4d004ad03cc7952dc56c6db87664ee4
+Merge: fd20a1d 2afb6fe
+Author: Yan, Zheng <ukernel@gmail.com>
+Date:   Fri Oct 10 09:37:53 2014 +0800
+
+    Merge pull request #2681 from ceph/firefly-locker-null
+    
+    mds: Locker: fix a NULL deref in _update_cap_fields
+
+commit 2afb6febdd8482b8fec5890d79944d656faf1382
+Author: Greg Farnum <greg@inktank.com>
+Date:   Thu Oct 9 15:12:19 2014 -0700
+
+    mds: Locker: fix a NULL deref in _update_cap_fields
+    
+    The MClientCaps* is allowed to be NULL, so we can't deref it unless
+    the dirty param is non-zero. So don't do the ahead-of-time lookup;
+    just call it explicitly in the if block.
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit 3cd8a7fb9683577a7d6e934f18c29b7e84415be6)
+
+commit fd20a1d01bde67fb1edc6058e38435af9d5d6abc
+Merge: e1bd1b2 86926c6
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Wed Oct 8 08:44:46 2014 +0200
+
+    Merge pull request #2662 from dachary/wip-9677-ioprio-class-firefly
+    
+    common: ceph_ioprio_string_to_class always returns -EINVAL
+
+commit 86926c6089d63014dd770b4bb61fc7aca3998542
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Tue Oct 7 14:06:38 2014 +0200
+
+    common: ceph_ioprio_string_to_class always returns -EINVAL
+    
+    The l string is always empty because std::transform needs a
+    pre-allocated string. Replace with the in-place version. Add unit tests.
+    
+    http://tracker.ceph.com/issues/9677 Fixes: #9677
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 3535b7aba3df8b54fa5117b8a9c2f52b8f0f118b)
+    
+    Conflicts:
+    	src/test/Makefile.am
+
+commit 5f2eec5036a2910aca1e8ce2d94444d3ed0477df
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Tue Oct 7 14:05:08 2014 +0200
+
+    osd: log error if set_ioprio fails to parse class
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 5088e0d49332d579ba7e33c2c9baee3d5f701a3e)
+
+commit 2796d5151df4dcde324a4d09a83c9a779cece00e
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Tue Oct 7 14:03:39 2014 +0200
+
+    common: set_ioprio debug message including pid
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 33339c7754875eb7e513345ee6b26a9b2b4d2707)
+
+commit d5ed6b0587b9999b2fd41377b0426e3b09ef8ab9
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Tue Oct 7 14:02:09 2014 +0200
+
+    common: do not set ioprio if pid is not set
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit c7e4c0bfe70bf29d3b8fe4df4e4b934853e33d26)
+
+commit e1bd1b2774f4eae5444b5f7b984193cb91a2dd98
+Merge: 726c6a1 459dca1
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 7 09:58:03 2014 -0700
+
+    Merge pull request #2632 from ceph/wip-9039-firefly
+    
+    rgw: copy object data if target bucket is in a different pool
+
+commit 726c6a147a14c00cf12eb6c6561655475282419f
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 6 15:50:51 2014 -0700
+
+    debian/control: fix python-ceph -> ceph file move to allow upgrades
+    
+    This is a backport of 5c6c366d2abe771c581690270c2d176ebb30c571 with the
+    version numbers changed, to compensate for the change in
+    fe3434f41cd09433975d7d0f9dbb2fae662e4a1b (backported in
+    bf1933e5c184476a354664c42fec834e9f59067c).
+    
+    Tested-by: Tamil Muthamizhan <tamil.muthamizhan@inktank.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 884f7c40c4a28d519847d3995c8d98e5837ceaf0
+Merge: 31d57c9 c8a8e47
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 6 07:01:50 2014 -0700
+
+    Merge remote-tracking branch 'gh/wip-rpm-epoch-firefly' into firefly
+    
+    Reviewed-by: Boris Ranto <branto@redhat.com>
+
+commit 31d57c9a28502a4a72f8aa141f7ed63ffe1e0192
+Merge: 9a3bac0 548be0b
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Mon Oct 6 09:50:33 2014 +0200
+
+    Merge pull request #2643 from johnugeorge/wip-9492-crush-firefly
+    
+    Crush: Backporting fixes for #9492 to firefly
+    
+    Reviewed-by: Loic Dachary <loic-201408@dachary.org>
+
+commit 548be0b2aea18ed3196ef8f0ab5f58a66e3a9af4
+Author: Johnu George <johnugeo@cisco.com>
+Date:   Mon Sep 29 10:07:44 2014 -0700
+
+    Crush: Ensuring at most num-rep osds are selected
+    
+    Crush temporary buffers are allocated as per replica size configured
+    by the user.When there are more final osds (to be selected as per
+    rule) than the replicas, buffer overlaps and it causes crash.Now, it
+    ensures that at most num-rep osds are selected even if more number of
+    osds are allowed by indep rule. The fix for firstn rules is already
+    merged as part of bug #9492. Required test files are added.
+    
+    Fixes: #9492
+    
+    Signed-off-by: Johnu George johnugeo@cisco.com
+    (cherry picked from commit 234b066ba04976783d15ff2abc3e81b6cc06fb10)
+
+commit e30c570ce691a994898b4a933c57e7ae014cdc30
+Author: Johnu George <johnugeo@cisco.com>
+Date:   Wed Sep 24 09:32:50 2014 -0700
+
+    Crush: Ensuring at most num-rep osds are selected
+    
+    Crush temporary buffers are allocated as per replica size configured
+    by the user.When there are more final osds (to be selected as per
+    rule) than the replicas, buffer overlaps and it causes crash.Now, it
+    ensures that at most num-rep osds are selected even if more number of
+    osds are allowed by the rule.
+    
+    Fixes: #9492
+    
+    Signed-off-by: Johnu George <johnugeo@cisco.com>
+    (cherry picked from commit 6b4d1aa99718e3b367496326c1e64551330fabc0)
+
+commit 9a3bac0c1a7a42cdf7ba846c9ad0a3ae0f15b4bb
+Merge: 0b978fb 5a5e7e7
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 3 11:58:41 2014 -0700
+
+    Merge pull request #2634 from dachary/wip-9653-ceph-disk-bootstrap-osd-firefly
+    
+    ceph-disk: bootstrap-osd keyring ignores --statedir (firefly)
+
+commit c8a8e4763a55dec44836bc679254ee7dcc448567
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 1 12:33:38 2014 -0700
+
+    ceph.spec: fix typo
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit da9ae5c92ecb4059e0ec6be5ce03af46430e76a5)
+
+commit b890c1e4706d7cfef7ed24c9df65b439b4f7ff1d
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 1 06:02:02 2014 -0700
+
+    ceph.spec.: add epoch
+    
+    This is done in fedora packaging.  Do it here too so that you can move
+    between upstream packages (from ceph.com) and fedora and other derivatives
+    will builds.
+    
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 83888362089346e473d6fd6e1d366b826d7bd739)
+    
+    Conflicts:
+    
+    	ceph.spec.in
+
+commit d01db8080d97bfae34dcee3d153bf10e6f5327dd
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue May 20 13:41:35 2014 -0700
+
+    ceph.spec.in: remove BuildRoot
+    
+    Deprecated
+    
+    Fixes: #8143
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit 401319a1527dd9cb5398916105d31e7ec065763d)
+
+commit 3c2b5c440863df548afc2bd8aa5440f15a44ac02
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Tue Aug 12 16:31:22 2014 -0700
+
+    ceph.spec.in: tests for rhel or centos need to not include _version
+    
+    rhel_version and centos_version are apparently the OpenSUSE Build
+    names; the native macros are just "rhel" and "centos" (and contain
+    a version number, should it be necessary).
+    
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 7474f720c2418cf3d52b755f2b60c524e413570a)
+
+commit c82c29d6174022be45929fe9ba8a84993eef974a
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Tue Aug 12 14:46:52 2014 -0700
+
+    ceph.spec.in: Add a small comment on the empty %files section
+    
+    as suggested by Dan Mick.
+    
+    Signed-off-by: Erik Logtenberg <erik@logtenberg.eu>
+    (cherry picked from commit e37b262c7928934530c5bb09fe56f83eb61f4244)
+
+commit a4f748aa906fc65b14f65515721bc3a815c18fb8
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Tue Aug 12 14:39:18 2014 -0700
+
+    ceph.spec.in: Obsolete all older versions.
+    
+    Now this changeset can be used on all current ceph releases that already
+    have the package split.
+    
+    Signed-off-by: Erik Logtenberg <erik@logtenberg.eu>
+    (cherry picked from commit 875a99e25f0ad2cb47149a3b5a28b4771a09125c)
+
+commit 74c7f3caec1cc7a5da89ef33de36a8b59249cfcd
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Tue Aug 12 14:09:43 2014 -0700
+
+    ceph.spec.in: No version on ceph-libs Obsoletes.
+    
+    If we are installing with the new package structure we don't ever want the
+    new package to co-exist with the old one; this includes the mistakenly-
+    released v0.81 on Fedora, which should be removed in favor of this
+    version.
+    
+    Signed-off-by: Sandon Van Ness <sandon@inktank.com>
+    Reviewed-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 8f95daf66b5fdb2a8141988480f984c1249599c5)
+
+commit 561261b6efb7ea442686bb8fce387c4de2482067
+Author: Sandon Van Ness <sandon@inktank.com>
+Date:   Fri Aug 8 18:01:30 2014 -0700
+
+    ceph.spec.in: Obselete ceph-libcephfs (not libcephfs)
+    
+    I am guessing that because it was a sub-package libcephfs was mistakenly
+    used instead of ceph-libcephfs.
+    
+    Signed-off-by: Sandon Van Ness <sandon@inktank.com>
+    (cherry picked from commit 75985024bd30ca6fbe4c61aa7f7cbe5306c9a988)
+
+commit 107bfd9ee7dbf360561187b9e0946964d40b9b1c
+Author: Erik Logtenberg <erik@logtenberg.eu>
+Date:   Fri Aug 1 14:20:18 2014 +0200
+
+    ceph.spec.in: We need those nice recent changes for rhel7 in Fedora too.
+    
+    Signed-off-by: Erik Logtenberg <erik@logtenberg.eu>
+    (cherry picked from commit 00877ae502ac52613bcd5c5c834d72787d668dca)
+
+commit 7946c5e5de8d6cd25d20beee15f3489113e51539
+Author: Dan Mick <dan.mick@inktank.com>
+Date:   Wed Aug 27 12:56:43 2014 -0700
+
+    Move fedora patch file (used by ceph.spec.in) to rpm/ subdir
+    
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>
+    (cherry picked from commit 06b92cee621cbe33a6f17e8c64169db4453a5160)
+
+commit cb2ae9afa611175226efb5544f7d2aa705d55ece
+Author: Erik Logtenberg <erik@logtenberg.eu>
+Date:   Fri Aug 1 00:13:50 2014 +0200
+
+    ceph.spec.in, init-ceph.in: Don't autostart ceph service on Fedora.
+    
+    This patch is taken from the current Fedora package and makes the upstream
+    ceph.spec compliant with Fedora policy. The goal is to be fully compliant
+    upstream so that we can replace current Fedora package with upstream
+    package to fix many bugs in Fedora.
+    
+    Addition from Dan Mick <dan.mick@inktank.com>:
+    Do this for RHEL and Centos as well, since they surely will benefit
+    from the same policy.   Note: this requires changes to
+    autobuild-ceph and ceph-build scripts, which currently copy
+    only the dist tarball to the rpmbuild/SOURCES dir.
+    
+    Signed-off-by: Erik Logtenberg <erik@logtenberg.eu>
+    Signed-off-by: Dan Mick <dan.mick@inktank.com>:
+    (cherry picked from commit 461523b06cdf93e32f1d8b354ac3799e73162d33)
+
+commit 2b11376f1ee8925ab16065ebda912b11d3d7be59
+Author: Erik Logtenberg <erik@logtenberg.eu>
+Date:   Thu Jul 31 23:54:03 2014 +0200
+
+    ceph.spec.in: Add obsoletes for libcephfs
+    
+    This fixes a bug for Fedora:
+    https://bugzilla.redhat.com/show_bug.cgi?id=1116614
+    
+    Signed-off-by: Erik Logtenberg <erik@logtenberg.eu>
+    (cherry picked from commit e9da2d8f2142771f206ef67f19e7f194855275d0)
+
+commit eefc62e3b85f402a7e1ae31c272c8a432d979379
+Author: Erik Logtenberg <erik@logtenberg.eu>
+Date:   Thu Jul 31 23:49:56 2014 +0200
+
+    ceph.spec.in: add ceph-libs-compat
+    
+    Added a ceph-libs-compat package in accordance with Fedora packaging
+    guidelines [1], to handle the recent package split more gracefully.
+    In Fedora this is necessary because there are already other packages
+    depending on ceph-libs, that need to be adjusted to depend on the new
+    split packages instead. In the mean time, ceph-libs-compat prevents
+    breakage.
+    
+    [1] http://fedoraproject.org/wiki/Upgrade_paths_%E2%80%94_renaming_or_splitting_packages
+    
+    Signed-off-by: Erik Logtenberg <erik@logtenberg.eu>
+    (cherry picked from commit 6c264f2204cbd54d90b02101e40ac9aa5aa72d7c)
+    
+    Conflicts:
+    
+    	ceph.spec.in
+
+commit 0b978fb15a1307644aba3119419bb7386f98ee04
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Aug 10 14:41:19 2014 -0700
+
+    mon/Paxos: add perfcounters for most paxos operations
+    
+    I'm focusing primarily on the ones that result in IO here.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b09b8563d35dda23faed43afef2a983e93a879c5)
+
+commit 74aa7afc719e517dbed300f802c1bc2dafe43ee0
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Aug 10 14:00:11 2014 -0700
+
+    mon/MonitorDBStore: add get_{keys,bytes}() accounting to Transaction
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit fd421b26748e872ddf8e0f068dda2106853edff1)
+
+commit 5a5e7e7bcedbccbe4ae8aab159af6d8615eb3887
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Fri Oct 3 14:08:57 2014 +0200
+
+    ceph-disk: bootstrap-osd keyring ignores --statedir
+    
+    The STATEDIR variable is used to initialize the bootstrap-osd keyring
+    before it gets a chance to be overriden by --statedir. Replace it with
+    {statedir} so that it can be substituted after all options have been
+    parsed.
+    
+    http://tracker.ceph.com/issues/9653 Fixes: #9653
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit fa0bd06b4657e5b84e237b76033ac3d3478b6a1f)
+
+commit 459dca1613a14cfad8d3afd7e3c783d825573a42
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Aug 12 13:36:11 2014 -0700
+
+    rgw: copy object data if target bucket is in a different pool
+    
+    Fixes: #9039
+    Backport: firefly
+    
+    The new manifest does not provide a way to put the head and the tail in
+    separate pools. In any case, if an object is copied between buckets in
+    different pools, we may really just want the object to be copied, rather
+    than reference counted.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5d3a7e595f47455896304bf358e5251915d0f16f)
+
+commit 711a7e6f81983ff2091caa0f232af914a04a041c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Sep 15 00:53:50 2014 -0400
+
+    rbd: ObjectCacher reads can hang when reading sparse files
+    
+    The pending read list was not properly flushed when empty objects
+    were read from a space file.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit cdb7675a21c9107e3596c90c2b1598def3c6899f)
+
+commit b7784dc1baa47560a733fe9dcd2acec51bc93165
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sat Sep 6 22:59:40 2014 -0400
+
+    Enforce cache size on read requests
+    
+    In-flight cache reads were not previously counted against
+    new cache read requests, which could result in very large
+    cache usage.  This effect is most noticeable when writing
+    small chunks to a cloned image since each write requires
+    a full object read from the parent.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4fc9fffc494abedac0a9b1ce44706343f18466f1)
+
+commit ac4fca065a12f3b00a47bf5ec37983696255174b
+Author: Alexandre Marangone <alexandre.marangone@inktank.com>
+Date:   Fri Sep 5 10:36:24 2014 -0700
+
+    rgw: add .log to default log path
+    
+    Fixes: #9353
+    Signed-off-by: Alexandre Marangone <alexandre.marangone@inktank.com>
+    (cherry picked from commit 46732420897a2619059050044f4980a4737df43e)
+
+commit f03ae7b00e5694f1670493396a1cee195fcc6b35
+Merge: 78c3ef9 ee02cfd
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Oct 2 15:28:40 2014 -0700
+
+    Merge pull request #2565 from ceph/wip-rgw-firefly-backports
+    
+    Wip rgw firefly backports
+
+commit 78c3ef90604ca117255cefe232771a7564fed8b1
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 25 13:16:52 2014 -0700
+
+    osdc/Objecter: only post_rx_buffer if no op timeout
+    
+    If we post an rx buffer and there is a timeout, the revocation can happen
+    while the reader has consumed the buffers but before it has decoded and
+    constructed the message.  In particular, we calculate a crc32c over the
+    data portion of the message after we've taken the buffers and dropped the
+    lock.
+    
+    Instead of fixing this race (for example, by reverifying rx_buffers under
+    the lock while calculating the crc.. bleh), just skip the rx buffer
+    optimization entirely when a timeout is present.
+    
+    Note that this doesn't cover the op_cancel() paths, but none of those users
+    provide static buffers to read into.
+    
+    Fixes: #9582
+    Backport: firefly, dumpling
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    
+    backport of 126d0b30e990519b8f845f99ba893fdcd56de447
+
+commit a261b4952056aab7b067453930342960bbe55089
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 29 14:28:32 2014 -0700
+
+    debian: move ceph_rest_api.py into ceph
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit fe3434f41cd09433975d7d0f9dbb2fae662e4a1b)
+
+commit eb0f6e347969b40c0655d3165a6c4531c6b595a3
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 29 14:24:01 2014 -0700
+
+    ceph.spec.in: move ceph_rest_api.py into ceph
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8cda623e0ba34a48a70e9ea988d619b15605c4fd)
+
+commit c4188e31f7bc8f3c337e637cd99c41d5ee4b6787
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 29 13:44:03 2014 -0700
+
+    ceph.spec: fix python-flask dependency
+    
+    This is needed by ceph-rest-api, which is in ceph.rpm; it's not related to
+    python-ceph (except that ceph-rest-api happens to require that too).
+    
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b2416240b88b2e067dfc79a2723335f1584562d0)
+
+commit bf1933e5c184476a354664c42fec834e9f59067c
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 29 13:40:18 2014 -0700
+
+    debian: python-flask is needed by ceph, not python-ceph
+    
+    It's used by ceph-rest-api which is in the 'ceph' (server) package.
+    
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e42424e777e4f7d8b03650482253734c1fa8709d)
+    
+    Conflicts:
+    
+    	debian/control
+
+commit 94a7fbaa11c51db294dce0dc1df728f69aef5bf8
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Fri Sep 19 12:25:07 2014 +0200
+
+    rgw_main.cc: add missing virtual destructor for RGWRequest
+    
+    CID 1160858 (#1 of 1): Non-virtual destructor (VIRTUAL_DTOR)
+     nonvirtual_dtor: Class RGWLoadGenRequest has a destructor
+     and a pointer to it is upcast to class RGWRequest which doesn't
+     have a virtual destructor.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit b82ceda7775ff85943d9143b73789eb37b09bfa9)
+
+commit 9fee8de25ab5c155cd6a3d32a71e45630a5ded15
+Author: Greg Farnum <greg@inktank.com>
+Date:   Mon Sep 29 16:10:36 2014 -0700
+
+    Locker: accept ctime updates from clients without dirty write caps
+    
+    The ctime changes any time the inode does. That can happen even without
+    the file itself having changed, so we'd better accept the update whenever
+    the auth caps have dirtied, without worrying about the file caps!
+    
+    Fixes: #9514
+    Backport: firefly
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 0ea20a668cf859881c49b33d1b6db4e636eda18a)
+
+commit 461ece5e9fb1d4994a6214a3b6bdae136773629d
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 1 18:01:51 2014 -0700
+
+    doc/release-notes: fix attributions for 8702 fix
+    
+    Oops!
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 188370a94353e29fcb8981699022803e23f3fedd)
+
+commit 917529a78e5046f621df5c48fe5d50d2f7e56560
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 1 17:48:12 2014 -0700
+
+    doc/release-notes: v0.80.6
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c0dc3a56974a469b61523b67cc032cc5726a3a5f)
+    
+    Conflicts:
+    
+    	doc/release-notes.rst
+
+commit 060a5b1422fcdfb8e84636579a2f0c2f1ec14300
+Author: Adam Crume <adamcrume@gmail.com>
+Date:   Thu Sep 18 16:57:27 2014 -0700
+
+    common: Add cctid meta variable
+    
+    Fixes: #6228
+    Signed-off-by: Adam Crume <adamcrume@gmail.com>
+    (cherry picked from commit bb45621cb117131707a85154292a3b3cdd1c662a)
+
+commit a1aa06b7fb30e509193e1b1bb7355b5f21aedc82
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Oct 1 14:30:59 2014 -0700
+
+    ReplicatedPG: dump snap_trimq on pg query
+    
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit 34f38b68d89baf1dcbb4571d4f4d3076dc354538
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Sep 29 16:26:54 2014 -0700
+
+    ReplicatedPG: do not queue the snap trimmer constantly
+    
+    Previously, we continuously requeued the snap trimmer while in
+    TrimmingObjects.  This is not a good idea now that we try to
+    limit the number of snap trimming repops in flight and requeue
+    the snap trimmer directly as those repops complete.
+    
+    Fixes: #9113
+    Backport: giant, dumpling, firefly
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit b29bf00f68cf133151c98db06e9498b3e8be22ed
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Wed Sep 24 13:55:47 2014 -0700
+
+    ReplicatedPG: clean out completed trimmed objects as we go
+    
+    Also, explicitely maintain a max number of concurrently trimming
+    objects.
+    
+    Fixes: 9113
+    Backport: dumpling, firefly, giant
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
+
+commit ee02cfd23facb3404fc377f643b213c2f498474d
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 18 20:53:10 2014 -0700
+
+    rgw: calculate hash after writing data
+    
+    Since data is written asynchronously, we should do the hash calculation
+    while it's pending.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5bb94ede19a50543a02a8019ed6c9680b3852d4e)
+
+commit 216730221575d88a72b06ed3d71c9a54cffc5719
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 18 20:51:02 2014 -0700
+
+    crypto: don't hash zero sized buffer
+    
+    libnss returns an error and we assert in that case.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 7b137246b49a9f0b4d8b8d5cebfa78cc1ebd14e7)
+
+commit dab7a4f6ad054b53cedca76ee329a6395918b1ab
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Sep 12 14:07:44 2014 -0700
+
+    rgw: push hash calculater deeper
+    
+    This might have been the culprit for #9307. Before we were calculating
+    the hash after the call to processor->handle_data(), however, that
+    method might have spliced the bufferlist, so we can't be sure that the
+    pointer that we were holding originally is still invalid. Instead, push
+    the hash calculation down. Added a new explicit complete_hash() call to
+    the processor, since when we're at complete() it's too late (we need to
+    have the hash at that point already).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d41c3e858c6f215792c67b8c2a42312cae07ece9)
+    
+    Conflicts:
+    	src/rgw/rgw_rados.h
+
+commit bd0a91343ce70d71acced753688a502b7e8b552e
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Aug 21 16:30:10 2014 -0700
+
+    rgw: separate civetweb log from rgw log
+    
+    The civetweb log now is independent from the rgw log.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 850242cad749e33e1e6bc008baa75c8ea7eda0c1)
+    
+    Conflicts:
+    	src/civetweb
+    	src/rgw/rgw_main.cc
+
+commit a777562b780e7ab312f881c38b1db26983a1ac47
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 23 13:40:39 2014 -0700
+
+    civetweb: update submodule
+    
+    Update submodule to include multiple fixes.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 1bdcc079d79d8211b44e2a46511cd2240f71744b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Aug 22 15:12:16 2014 -0700
+
+    rgw: convert header field underscores into dashes
+    
+    Fixes: 9206
+    Backport: firefly
+    
+    Certain web servers filter out underscores in the header field name.
+    Convert them into dashes.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 11acb7097ce21c6218dd48d0c21e0e04a361eb9a)
+
+commit b8fa2ed60b6cce51701df972dbb6f5e02e0d84ba
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Aug 25 10:38:42 2014 -0700
+
+    rgw: fix test to identify whether object has tail
+    
+    Fixes: #9226
+    Reported-by: Sylvain Munaut <s.munaut@whatever-company.com>
+    Backport: firefly
+    
+    We need to identify whether an object is just composed of a head, or
+    also has a tail. Test for pre-firefly objects ("explicit objs") was
+    broken as it was just looking at the number of explicit objs in the
+    manifest. However, this is insufficient, as we might have empty head,
+    and in this case it wouldn't appear, so we need to check whether the
+    sole object is actually pointing at the head.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 751b3e26532932a42ca34f9c062a0a3e29a58cff)
+
+commit 6fee71154d838868807fd9824d829c8250d9d2eb
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Aug 27 17:44:18 2014 -0700
+
+    rgw: don't try to authenticate a CORS preflight request
+    
+    Fixes: #8718
+    Backport: firefly
+    
+    CORS preflight requests don't need to be authenticated. Treat them as
+    coming from anonymous user.
+    
+    Reported-by: Robert Hubbard <bobby.hubbard@garmin.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 848fcf7871e07fc689bdcd18943ace36b2f4906e)
+
+commit c75a79cbac30cd14d37d89217113824d98693572
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Aug 14 13:35:12 2014 -0700
+
+    rgw: fix compilation
+    
+    RGWRadosPutObj couldn't refer to the ceph context.
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 6a555434ee3edaf742ee7e5910bcba8dd0de46dd)
+
+commit b0d08aab837808f18708a4f8ced0503c0fce2fec
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Aug 12 11:17:47 2014 -0700
+
+    rgw: call throttle_data() even if renew_state() failed
+    
+    Otherwise we're going to leak the aio callback handle.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 7998c322179dd69a1250937321c3c2bb023e0e57)
+
+commit a953b313f1e2f884be6ee2ce356780f4f70849dd
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jul 30 21:32:48 2014 -0700
+
+    rgw: disable civetweb url decoding
+    
+    Fixes: #8621
+    
+    We want to have the raw request uri, as we do the decoding ourselves.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit ffac52b316e7022796d44ae58804d9c20b9c3df9)
+
+commit ba5357714a19b8af989fef1c75ef775837c6a9d6
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Aug 12 14:23:46 2014 -0700
+
+    rgw: copy_obj_data() uses atomic processor
+    
+    Fixes: #9089
+    
+    copy_obj_data was not using the current object write infrastructure,
+    which means that the end objects weren't striped.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 800eff24824c0083b8e2441fc34e0bdca5da36dc)
+
+commit d73dbc3a39117eddaaabb2c25d9238cd7c51711b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Aug 21 21:53:38 2014 -0700
+
+    rgw: clear bufferlist if write_data() successful
+    
+    Fixes: #9201
+    Backport: firefly
+    
+    We sometimes need to call RGWPutObjProcessor::handle_data() again,
+    so that we send the pending data. However, we failed to clear the buffer
+    that was already sent, thus it was resent. This triggers when using non
+    default pool alignments.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 9181114d6f6062c55ee4b351fc3495345e545c36)
+
+commit 4be53d5eebbc294878ba38050d841359b6c8e19e
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Tue Sep 23 12:16:55 2014 -0700
+
+    PG: check full ratio again post-reservation
+    
+    Otherwise, we might queue 30 pgs for backfill at 0.80 fullness
+    and then never check again filling the osd after pg 11.
+    
+    Fixes: #9574
+    Backport: dumpling, firefly, giant
+    Signed-off-by: Samuel Just <sam.just@inktank.com>
diff --git a/doc/changelog/v0.80.9.txt b/doc/changelog/v0.80.9.txt
new file mode 100644
index 000000000..aa274d137
--- /dev/null
+++ b/doc/changelog/v0.80.9.txt
@@ -0,0 +1,1148 @@
+commit b5a67f0e1d15385bc0d60a6da6e7fc810bde6047 (tag: refs/tags/v0.80.9, refs/remotes/gh/firefly)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Mon Mar 9 10:42:08 2015 -0700
+
+    0.80.9
+
+commit 37901afd1556257151c029395caa1143e84860f2
+Merge: 00e5947 3dac68a
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Mar 3 15:58:53 2015 +0100
+
+    Merge pull request #3852 from dachary/wip-10965-rgw-firefly
+    
+    rgw: enable ipv6 in civetweb
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 00e5947a28f7810a28329d2f4901aed09a289421
+Merge: 293222a b13f483
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Mar 3 15:58:39 2015 +0100
+
+    Merge pull request #3853 from dachary/wip-10907-rgw-firefly
+    
+    rgw: pass civetweb configurables to civetweb
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 293222a284c072a13950831205c106fec7a400df
+Merge: 6512b06 d57b38f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Mar 3 15:58:25 2015 +0100
+
+    Merge pull request #3851 from dachary/wip-10978-rgw-firefly
+    
+    rgw: don't overwrite bucket / object owner when setting acls
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit d57b38f85b683dfb365c3cb98362d486594f9eb3
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Feb 27 15:32:50 2015 -0800
+
+    rgw: don't overwrite bucket / object owner when setting acls
+    
+    Fixes: #10978
+    Backport: hammer, firefly
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit eb13f2d4b60c031f16139f7cc4237c012644dd78)
+    
+    Conflicts:
+    	src/rgw/rgw_op.cc : trivial s/.empty()/== NULL/
+
+commit b13f483e51b37a768c7f4313b6933bf648950c7d
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Feb 17 15:05:40 2015 -0800
+
+    rgw: pass civetweb configurables to civetweb
+    
+    Fixes: #10907
+    Backport: firefly
+    
+    Pass any configurables in the rgw frontends config line to civetweb.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 986d7554426764a149621ba733c5c075b94e0431)
+
+commit 3dac68a17a909b212a36b0a3c0ae2c47d323deee
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Feb 27 08:14:27 2015 -0800
+
+    civetweb: update submodule
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f28fe744285117a1715eac0d08911cdb37285103)
+
+commit 14aca3af18119a76c1cdfa6d71e6085d360e45e2
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Feb 27 08:14:41 2015 -0800
+
+    rgw: update makefile to enable civetweb config
+    
+    Fixes: #10965
+    Backport: hammer, firefly
+    
+    Civetweb compilation now includes conf header to enable ipv6.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit caa90225bad9fe5e9c275e6189b3396b4d396e3f)
+    
+    Conflicts:
+    	src/rgw/Makefile.am
+               radosgw_CFLAGS = -Icivetweb/include
+            was not yet changed into
+               radosgw_CFLAGS = -I$(srcdir)/civetweb/include
+            in firefly
+
+commit 6512b06fc8a80e3befbe15b543f8850991f74e8a
+Merge: 7b748c6 555cc42
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sat Feb 28 16:47:55 2015 +0100
+
+    Merge pull request #3820 from tchaikov/firefly-pg-leak-10421
+    
+    osd: fix PG leak in SnapTrimWQ._clear()
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 7b748c62764311572342593820ce3f28f74fe9ca
+Merge: d971c95 da95149
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sat Feb 28 16:46:42 2015 +0100
+
+    Merge pull request #3772 from ceph/wip-10883-firefly
+    
+    osd: Fix FileJournal wrap to get header out first
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit d971c95c3d8b48494d05b68f40c4c9a4cc6f87a8
+Merge: 0820041 e539971
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sat Feb 28 16:45:48 2015 +0100
+
+    Merge pull request #3521 from dzafman/wip-10676
+    
+    Backport doc fixes that appear to apply to firefly
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 555cc42fc826fd801f0d45187429079d4072d129
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Feb 10 16:29:45 2015 +0800
+
+    osd: fix PG leak in SnapTrimWQ._clear()
+    
+    Fixes: #10421
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 01e154d592d6cdbf3f859cf1b4357e803536a6b4)
+
+commit 0820041e6515c82c41b81b9e6825e2dd5fcb8165
+Merge: 6565774 702dbc0
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Thu Feb 26 15:59:24 2015 -0800
+
+    Merge pull request #3730 from ceph/wip-firefly-flock
+    
+    backport ceph-fuse file locking patches to Firefly
+    
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+
+commit 6565774d0356efc6225ad561bb13e7cf11da7b1e
+Merge: 7ab02ee 08c2fda
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Feb 26 14:18:10 2015 -0800
+
+    Merge pull request #3406 from ceph/wip-10299-firefly
+    
+    librbd: complete all pending aio ops prior to closing image
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 08c2fda12cf46937a09a59bb032379c3c5321292
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Dec 15 10:53:53 2014 -0500
+
+    librbd: complete all pending aio ops prior to closing image
+    
+    It was possible for an image to be closed while aio operations
+    were still outstanding.  Now all aio operations are tracked and
+    completed before the image is closed.
+    
+    Fixes: #10299
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 7ab02ee5afb3d017b94d58c3dfc7731f7a3866d9
+Merge: 0c0a552 c23e42e
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Feb 26 14:12:29 2015 -0800
+
+    Merge pull request #3404 from ceph/wip-10270-firefly
+    
+    librbd: gracefully handle deleted/renamed pools
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 0c0a5520b1b883bcdd5b865b217ba61e471e3ca2
+Merge: 9ef7743 836ab86
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Feb 26 13:31:37 2015 -0800
+
+    Merge pull request #3410 from ceph/wip-9854-firefly
+    
+    osdc: Constrain max number of in-flight read requests
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit da951497b7e5ec227aa6a5e459b0d04d8b88ef13 (refs/remotes/gh/wip-10883-firefly)
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 18 16:21:12 2015 -0800
+
+    osd: Fix FileJournal wrap to get header out first
+    
+    Correct and restore assert that was removed
+    
+    Cause by f46b1b473fce0322a672b16c7739e569a45054b6
+    Fixes: #10883
+    Backport: dumpling, firefly, giant
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 970bb4901f93575709421b5b25c3eff213de61b8)
+
+commit 702dbc0a247c149d53b52d1929f9880bc99d0522
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Oct 15 12:00:58 2014 +0800
+
+    qa/workunits/fs/misc: Add a workunit for file lock interruption
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit ac92c455a9aa19e4288acdf0c9a746e03a640efb)
+
+commit d2523b82c5c7b29293d70a66ba95493a1564a840
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Oct 15 12:03:46 2014 +0800
+
+    mds: fix neighbor lock check
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit b0e6e85aa08ea74cd209aad04f3f0bf991761e12)
+
+commit f9b6b66b05ddadef043d81676728bf40730ea16c
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Oct 13 11:34:18 2014 +0800
+
+    client: use finisher to abort MDS request
+    
+    When a request is interrupted, libfuse first locks an internal mutex,
+    then calls the interrupt callback. libfuse need to lock the same mutex
+    when unregistering interrupt callback. We unregister interrupt callback
+    while client_lock is locked, so we can't acquiring the client_lock in
+    the interrupt callback.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 09699454e729592d426aeff5b578697e850af12e)
+    
+    Conflicts:
+    	src/client/Client.cc
+    	src/client/Client.h
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit ea355e9ca5203b77e6f74ca4a3e39ce23cc86f67
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Oct 9 13:16:18 2014 +0800
+
+    client: use atomic variable to track reference of MetaRequeset
+    
+    this allow us to increase reference count of MetaRequest while not holding
+    the client_lock
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit e464a7765230c504b3e3b88bcb1106f67b7c3eb9)
+
+commit ccbdf514717cb0e48d67b57197d6e538faeea415
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Oct 13 10:44:46 2014 +0800
+
+    client: allow interrupting blocked file lock operation
+    
+    This commit introduce two new types of setfilelock request. Unlike
+    setfilelock (UNLOCK) request, these two new types of setfilelock request
+    do not drop locks that have alread been acquired, they only interrupt
+    blocked setfilelock request.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 4134c149d3759dd6a3aaa1a353b77bbfe8e9491b)
+
+commit 875e2fcb060554941d94714ad48ebbc0cbbf8077
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Oct 9 09:42:08 2014 +0800
+
+    client: register callback for fuse interrupt
+    
+    libfuse allows program to reigster a callback for interrupt. When a file
+    system operation is interrupted, the fuse kernel driver sends interupt
+    request to libfuse. libfuse calls the interrupt callback when receiving
+    interrupt request.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 289e8b4a7efa1ae6427115af9bbe541c9e1f0e90)
+
+commit c96aabbd54b24a0273af21a450cd6f517fe4ada0
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat Oct 4 09:14:44 2014 +0800
+
+    client: add helper function that updates lock state
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 6a2303a6b6d97f2a6d1422e42d3d88991857618f)
+    
+    Conflicts:
+    	src/client/Client.h
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit ebbd3ca66722cdc61b56d243baf8e63f7b8c1c1b
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Oct 2 20:21:36 2014 +0800
+
+    fuse: enable fuse_multithreaded by default
+    
+    GETFILELOCK MDS request may block for a long time, so we need to
+    use multithread event loop,
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit b17b43a8660ed0db29fbecf44798265e47712f85)
+
+commit 924e6f2b6b03456a8b18d8b158b8f325051f0519
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Oct 2 19:07:41 2014 +0800
+
+    client: posix file lock support
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit a1b2c8ff955b30807ac53ce6bdc97cf61a7262ca)
+    
+    Conflicts:
+    	src/client/Client.cc
+    	src/client/Client.h
+    	src/client/Inode.h
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 82994946bdcb95867a61e0acf443e30b13925a34
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Oct 2 18:02:50 2014 +0800
+
+    common: link mds/flock.o to libcommon
+    
+    later commit will use this code to track file locks held by cephfs
+    client.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit e075c27c3554380c59dce0cc17ef0944eb415025)
+    
+    Conflicts:
+    	src/mds/Makefile.am
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 9ef77430f3d46789b0ba1a2afa42729627734500
+Merge: 1205867 b668566
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Feb 13 18:34:11 2015 +0100
+
+    Merge pull request #3684 from ceph/wip-crush-straw-firefly
+    
+    osd: backport straw bucket fixes and all other recent crush goodness to firefly
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit b668566fd8148414b8074f096b85b22c42ed3af9 (refs/remotes/gh/wip-crush-straw-firefly)
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jan 16 09:02:28 2015 -0800
+
+    crush/builder: fix warnings
+    
+    crush/builder.c: In function 'crush_remove_list_bucket_item':
+    crush/builder.c:977:13: warning: comparison between signed and unsigned integer expressions [-Wsign-compare]
+      if (weight < bucket->h.weight)
+                 ^
+    crush/builder.c: In function 'crush_remove_tree_bucket_item':
+    crush/builder.c:1031:14: warning: comparison between signed and unsigned integer expressions [-Wsign-compare]
+       if (weight < bucket->h.weight)
+                  ^
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 14eb1a73c71d81b7f193fce27c59cb3babf3e74a)
+
+commit 247afa4efe7d11b5ef06e094680b50ea30d7d5e3
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Wed Oct 15 17:02:58 2014 -0700
+
+    crush: improve constness of CrushWrapper methods
+    
+    A number of CrushWrapper get methods or predicates were not const
+    because they need to maintain transparently the rmaps. Make the rmaps
+    mutable and update the constness of the methods to match what the caller
+    would expect.
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 236895eea65f8706baa5fdef96fb00ad5b82218c)
+
+commit c59279a25b6c53e0ab3988b0f00ae3cce94f33d7
+Author: Xiaoxi Chen <xiaoxi.chen@intel.com>
+Date:   Fri Sep 5 10:56:36 2014 +0800
+
+    Change CrushWrapper::crush to private
+    
+    Currently in CrushWrapper, the member "struct crush_map *crush"  is a public member,
+    so people can break the encapsulation and manipulate directly to the crush structure.
+    
+    This is not a good practice for encapsulation and will lead to inconsistent if code
+    mix use the CrushWrapper API and crush C API.A simple example could be:
+    1.some code use crush_add_rule(C-API) to add a rule, which will not set the have_rmap flag to false in CrushWrapper
+    2.another code using CrushWrapper trying to look up the newly added rule by name will get a -ENOENT.
+    
+    This patch move CrushWrapper::crush to private, together with three reverse map(type_rmap, name_rmap, rule_name_rmap)
+    and also change codes accessing the CrushWrapper::crush to make it compile.
+    
+    Signed-off-by: Xiaoxi Chen <xiaoxi.chen@intel.com>
+    (cherry picked from commit d734600f9251b52f525faa35441e2b5dd660161b)
+
+commit 9fc2fd50aec94233528870aaa371347238a6ea75
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Jul 11 06:58:57 2014 -0700
+
+    crush: include CRUSH_V3, v2/v3 rules checks in dump_tunables()
+    
+    Backport: firefly
+    Signed-off-by: Sage Weil <sage@inktank.com>
+    (cherry picked from commit cf94cf3531a349bbd1fc6ee56c3fc260110a252a)
+
+commit f6009a614a3689cc9a96f55b35e70362be4ec64b
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 5 15:58:03 2014 -0800
+
+    mon: 'osd crush reweight-all'
+    
+    This corresponds to the crushtool --reweight command.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 89b2feea8d53b9dc15ab5ae7f5920ad19c8bba18)
+
+commit d5d7495432dd023ed8c9ba2ac08222f06afee48f
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 5 15:55:24 2014 -0800
+
+    crush: set straw_calc_version=1 for default+optimal; do not touch for presets
+    
+    When using the presets for compatibility (i.e., based on version), do not
+    touch the straw behavior, as it does not affect mapping or compatibility.
+    However, make a point of setting it by default and for optimal.
+    
+    For most users, this means that they will not see any change unless they
+    explicitly enable the new behavior, or switch to default or optimal
+    tunables.  The idea is that if they touched it, they shouldn't be
+    too surprised by the subsequent data movement.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit dd7b58f3b1aa1febfc6dc227937df93ee6e284eb)
+    
+    (Note: differs from original in that the hammer tunable profile is not
+    present)
+
+commit 5b7b7599123a9d7a837ded52946bd553b09b1ac7
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 3 22:30:00 2014 -0800
+
+    crush/builder: a note about the original crush_calc_straw()
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit adf5c6de0aca24a53d3c7b4e7eeb0a5dce9db0f1)
+
+commit 3ab835b059fd74a525cc2a8ebe8b6a1453e0cc87
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 2 16:43:16 2014 -0800
+
+    mon: add 'osd crush {get,set}-tunable <name> [value]' commands
+    
+    For now, just add the straw_calc_version tunable.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9000068ae45a8b89315c152b7d5509ac873f2957)
+    
+    Conflicts:
+    	src/mon/OSDMonitor.cc
+
+commit 884414c5788bac9a269f01b26cbc0c55850c34f6
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 2 16:33:11 2014 -0800
+
+    crush: fix crush_calc_straw() scalers when there are duplicate weights
+    
+    The straw bucket was originally tested with uniform weights and with a
+    few more complicated patterns, like a stair step (1,2,3,4,5,6,7,8,9).  And
+    it worked!
+    
+    However, it does not behave with a pattern like
+     1, 2, 2, 3, 3, 4, 4
+    
+    Strangely, it does behave with
+     1, 1, 2, 2, 3, 3, 4, 4
+    
+    and more usefully it does behave with
+     1, 2, 2.001, 3, 3.001, 4, 4.001
+    
+    That is, the logic that explicitly copes with weights that are duplicates
+    is broken.
+    
+    The fix is to simply remove the special handling for duplicate weights --
+    it isn't necessary and doesn't work correctly anyway.
+    
+    Add a test that compares the mapping result of  [1, 2, 2, 3, 3, ...] with
+    [1, 2, 2.001, 3, 3.001, ...] and verifies that the difference is small.
+    With the fix, we get .00012, whereas the original implementation gets
+    .015.
+    
+    Note that this changes the straw bucket scalar *precalculated* values that
+    are encoded with the map, and only when the admin opts into the new behavior.
+    
+    Backport: giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 43d5c7caa7ce478477bde1bbd4f0649b5159cdcf)
+
+commit bf677093302f475a23de541471ddde6a7c7153b6
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 2 14:50:21 2014 -0800
+
+    crush: fix distortion of straw scalers by 0-weight items
+    
+    The presence of a 0-weight item in a straw bucket should have no effect
+    on the placement of other items.  Add a test validating that and fix
+    crush_calc_straw() to fix the distortion.
+    
+    Note that this effects the *precalculation* of the straw bucket inputs and
+    does not effect the actually mapping process given a compiled or encoded
+    CRUSH map, and only when straw_calc_version == 1 (i.e., the admin opted in
+    to the new behavior).
+    
+    Backport: giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 85498bc8f62ca56506b33f3c5ec4fc4b111ed73d)
+
+commit e02574ef5d3b66e73424a16460366de2d36eded3
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 2 14:49:42 2014 -0800
+
+    crush/builder: break out new version 1 of crush_calc_straw
+    
+    No change, yet.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 7c1203635f2b5911f281ce3a441905df6e9bd103)
+
+commit c7312a47865c758e67852999803d8aa90ff809c1
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 2 14:45:04 2014 -0800
+
+    crush: pass crush_map * to various builder methods
+    
+    In particular, we will need it for crush_calc_straw().
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f35a3d88cb944c292e966f679ac7a8d7a1cd3093)
+    
+    Conflicts:
+    	src/crush/CrushWrapper.cc
+
+commit 5137cc656548d942d0f21ba3ef28a5b7d4d21831
+Author: Rongze Zhu <zrzhit@gmail.com>
+Date:   Fri Oct 10 19:18:00 2014 +0800
+
+    crush: fix incorrect use of adjust_item_weight method
+    
+    adjust_item_weight method will adjust all buckets which the item
+    inside. If the osd.0 in host=fake01 and host=fake02, we execute
+    "ceph osd crush osd.0 10 host=fake01", it not only will adjust fake01's
+    weight, but also will adjust fake02's weight.
+    
+    the patch add adjust_item_weightf_in_loc method and fix remove_item,
+    _remove_item_under, update_item, insert_item, detach_bucket methods.
+    
+    Signed-off-by: Rongze Zhu <zrzhit@gmail.com>
+    (cherry picked from commit 9850227d2f0ca2f692a154de2c14a0a08e751f08)
+    
+    Conflicts:
+    	src/crush/CrushWrapper.cc
+
+commit 80ec22014deb8536b9c34e8d57b286052898d05a
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Nov 13 10:59:22 2014 -0800
+
+    crush/CrushWrapper: fix detach_bucket
+    
+    In commit 9850227d2f0ca2f692a154de2c14a0a08e751f08 we changed the call that
+    changed the weight of all instances of item to one that explicitly
+    changes it in the parent bucket, but parent_id may not be valid at the
+    call site.  Move this into the conditional block to fix.
+    
+    Fixes: #10095
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6f8b96a9fe4793906c74a571109a457aca7ca220)
+
+commit 5bc554ac5a7a4cd86b873bfc4f4313c91287b52c
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 2 14:10:49 2014 -0800
+
+    crush: default to straw_calc_version 1
+    
+    Stick with bobtail tunables where it affects compatibility.  Use v1 of
+    straw_calc, though, since that does not, and we want the best for new
+    clusters.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9565621d6007302fdd68ba27b6aef22e487f0985)
+
+commit cfc718a5a1fec937cf00e6c2b55f66d4390088bb
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Feb 13 08:30:35 2015 -0800
+
+    crush: add straw_calc_version tunable
+    
+    It doesn't do anything, yet.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 50e2ceefee6a5dfbecbe54890139b1fa80a313c8)
+    
+    (Modified from original to not create the 'hammer' tunable profile, which
+    we will not backport in its entirety.)
+
+commit 1205867d3afe6d457483f9f51aaee79cca74e040
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Feb 2 16:43:35 2015 +0100
+
+    qa: use correct binary path on rpm-based systems
+    
+    Fixes: #10715
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 05ce2aa1bf030ea225300b48e7914577a412b38c)
+
+commit da4146a5e03503083e6bc5c12984c06f41a3b4be
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Thu Feb 5 21:12:17 2015 -0800
+
+    fsync-tester: print info about PATH and locations of lsof lookup
+    
+    We're seeing the lsof invocation fail (as not found) in testing and nobody can
+    identify why. Since attempting to reproduce the issue has not worked, this
+    patch will gather data from a genuinely in-vitro location.
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit a85051483874ff5b8b0fb50426a3577040457596)
+
+commit 61b4f09848796faeacf7eb78dd6dc06513beb737
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 2 14:04:34 2014 -0800
+
+    crush/CrushWrapper: dump chooseleaf_vary_r with other tunables
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c133a83fe7b77e2a7e7c711cb8ab943c59ff0885)
+
+commit e4939ed5535cc4678cf1d1ae80906290448e6590
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 2 10:12:25 2014 -0800
+
+    crush/CrushTester: add new --show-mappings option
+    
+    This makes --show-utilization and --show-statistics usable.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3877f00877f53e9f86630c2d89e81ab9cff729ec)
+
+commit 8c48ebbf13e6bdb689b4b8ae58ac811653ad2acc
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 2 10:11:57 2014 -0800
+
+    crushtool/CrushTester: output utilization even with 1 batch
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 294b06c6424f4cb69394976add826d9725073b50)
+
+commit e9faab9f296af9cf26b9475afd3562c3f3b9236b
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 2 10:08:18 2014 -0800
+
+    crush: recalculate straw scalers during a reweight
+    
+    The crushtool --reweight function triggers a fresh calculation of bucket
+    weights so that they are always the sum of the item weights.  In the
+    straw bucket case, the weights were updated but the corresponding straw
+    scalers were not being recalculated.  The result is that there was not
+    effect on placement in adjusted buckets until the next time a bucket item's
+    weight was adjusted.
+    
+    Backport: giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 35062937f174a45fb13d9c177eaa1fe4ed5ff4c2)
+
+commit 4c951e9dc1ca31429119de15755caf3d73f6ffce
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 2 08:36:41 2014 -0800
+
+    osdmaptool: --test-map-pgs-dump
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 7a99b489909b368bade36d8cc9722ad33d03a2e8)
+
+commit 3cb5d08feee4811624dd495f74912824203edf00
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 2 06:53:20 2014 -0800
+
+    crush: add dprintk's for crush_calc_straw
+    
+    These are compiled out by default.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 946bebd94b109642f95710802e892c59eb4f0b76)
+
+commit ca8471d65fb2cf7d2247d823c5436faa273efda5
+Author: Rongze Zhu <zrzhit@gmail.com>
+Date:   Tue Nov 11 00:13:42 2014 +0800
+
+    crush: fix tree bucket functions
+    
+    There are incorrect nodes' weight in tree bucket when construct tree
+    bucket. The tree bucket don't store item id in items array, so the tree
+    bucket will not work correctly. The patch fix above bugs and add a
+    simple test for tree bucket.
+    
+    Signed-off-by: Rongze Zhu <zrzhit@gmail.com>
+    (cherry picked from commit 13425488882d360fa740613dfcfd0d098c1b7616)
+
+commit af502f25b04cd0758b753941ecf0b1b59d33ea9e
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Nov 11 11:16:58 2014 -0800
+
+    crush/builder: replace printf with an empty dprintk macro
+    
+    This mirrors mapper.c.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e444b221733360cdea4f4bbdbbbfbf2b6ee30ff5)
+
+commit 26966c4aa9cd79cb53db052553a5fc5653f2591b
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Thu Nov 6 17:48:01 2014 -0800
+
+    qa: use sudo even more when rsyncing /usr
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 3aa7797741f9cff06053a2f31550fe6929039692)
+
+commit 6af48421c0ba6195e9a1607053c42eeb62b14ccb
+Author: Greg Farnum <greg@inktank.com>
+Date:   Tue Oct 21 10:55:06 2014 -0700
+
+    qa: use sudo when rsyncing /usr so we can read everything
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit fa07c04231db2d130de54647957ffab4a7a53733)
+
+commit 861a18b1f7dd5bdcbb85bc3fa15d1719bb4b2813
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jan 15 16:31:22 2015 -0800
+
+    rgw: fix partial GET in swift
+    
+    Fixes: #10553
+    backport: firefly, giant
+    
+    Don't set the ret code to reflect partial download, just set the
+    response status when needed.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 7e1553cedff90fa0fefded65cde87ad068eb5f0c)
+
+commit e539971e2d528b4de6009ea44565f037acb2be66 (refs/remotes/gh/wip-10676)
+Author: Nilamdyuti Goswami <ngoswami@redhat.com>
+Date:   Thu Dec 18 17:13:27 2014 +0530
+
+    doc: Adds updated man page for ceph under man/
+    
+    Signed-off-by: Nilamdyuti Goswami <ngoswami@redhat.com>
+    (cherry picked from commit 8de9a0f437822c770600c19a9f61977745b7e530)
+
+commit 15596ffe388147b7984457041a38cbb9f472556c
+Author: Nilamdyuti Goswami <ngoswami@redhat.com>
+Date:   Sat Dec 13 02:27:45 2014 +0530
+
+    doc: Adds man page for ceph under man/.
+    
+    Signed-off-by: Nilamdyuti Goswami <ngoswami@redhat.com>
+    (cherry picked from commit ffd6c7e49686f8f92ddb400ffdec62520708e64b)
+
+commit 893e5cd1f4fd5ea77d65f0e78cc8cff88eb19c1a
+Author: Nilamdyuti Goswami <ngoswami@redhat.com>
+Date:   Sat Dec 13 02:24:41 2014 +0530
+
+    doc: Adds man page for ceph.
+    
+    Signed-off-by: Nilamdyuti Goswami <ngoswami@redhat.com>
+    (cherry picked from commit 76da87a64ca6b3cc0ceeaf63e19a9f440d6f4161)
+
+commit 8d29a4a231666830914903b95599d80da7b97def
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 15 17:04:32 2014 -0800
+
+    osd: handle no-op write with snapshot case
+    
+    If we have a transaction that does something to the object but it !exists
+    both before and after, we will continue through the write path.  If the
+    snapdir object already exists, and we try to create it again, we will
+    leak a snapdir obc and lock and later crash on an assert when the obc
+    is destroyed:
+    
+    0> 2014-12-06 01:49:51.750163 7f08d6ade700 -1 osd/osd_types.h: In function 'ObjectContext::~ObjectContext()' thread 7f08d6ade700 time 2014-12-06 01:49:51.605411
+    osd/osd_types.h: 2944: FAILED assert(rwstate.empty())
+    
+    Fix is to not recreated the snapdir if it already exists.
+    
+    Fixes: #10262
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 02fae9fc54c10b5a932102bac43f32199d4cb612)
+
+commit 8ba48d10c252b28cde5b4da1286421db12b57cc2
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jan 19 18:28:20 2015 -0800
+
+    ceph_test_rados_api_misc: do not assert rbd feature match
+    
+    This test fails on upgrades when we (or the server) have new
+    features.  Make it less fragile.
+    
+    Fixes: #10576
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9147c62989871cea8b3a85b02c53017825efb55b)
+
+commit 836ab86d89ae15ed5b228ff656bc81c7cc3495aa
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Oct 27 14:47:19 2014 -0400
+
+    osdc: Constrain max number of in-flight read requests
+    
+    Constrain the number of in-flight RADOS read requests to the
+    cache size.  This reduces the chance of the cache memory
+    ballooning during certain scenarios like copy-up which can
+    invoke many concurrent read requests.
+    
+    Fixes: #9854
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 068d68850d09dfcaccc5a3ce85c80b2f6d808ea9)
+
+commit 2964efaf327cf798f892a6722eb4e24f2ffa0fde
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jan 19 12:46:00 2015 -0500
+
+    Revert "Enforce cache size on read requests"
+    
+    This reverts commit b7784dc1baa47560a733fe9dcd2acec51bc93165.
+
+commit e4c5b153735aba3f84554a50243f779f36568373
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jan 19 12:45:25 2015 -0500
+
+    Revert "rbd: ObjectCacher reads can hang when reading sparse files"
+    
+    This reverts commit 711a7e6f81983ff2091caa0f232af914a04a041c.
+
+commit c23e42e78ea3ba44706951a728e2ccb74cac7b33
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jan 19 10:28:56 2015 -0500
+
+    librbd: gracefully handle deleted/renamed pools
+    
+    snap_unprotect and list_children both attempt to scan all
+    pools. If a pool is deleted or renamed during the scan,
+    the methods would previously return -ENOENT. Both methods
+    have been modified to more gracefully handle this condition.
+    
+    Fixes: #10270
+    Backport: giant, firefly
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 436923c68b77c900b7774fbef918c0d6e1614a36)
+
+commit 24c13d87039d4f61df0bcabdb8862e0e94fe575d
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 12 05:24:01 2014 -0800
+
+    rgw: change multipart upload id magic
+    
+    Fixes: #10271
+    Backport: firefly, giant
+    
+    Some clients can't sign requests correctly with the original magic
+    prefix.
+    
+    Reported-by: Georgios Dimitrakakis <giorgis@acmac.uoc.gr>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5fc7a0be67a03ed63fcc8408f8d71a31a1841076)
+
+commit 617002d3ff469ef409a83e35d4f4fd6a0b5b1278
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Dec 11 09:07:10 2014 -0800
+
+    rgw: url decode http query params correctly
+    
+    Fixes: #10271
+    Backport: firefly
+    
+    This got broken by the fix for #8702. Since we now only url_decode if
+    we're in query, we need to specify that we're in query when decoding
+    these args.
+    
+    Reported-by: Georgios Dimitrakakis <giorgis@acmac.uoc.gr>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 21e07eb6abacb085f81b65acd706b46af29ffc03)
+
+commit d7ccf71d962cec1571f53c9392f9b58350569062
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Jan 14 15:01:38 2015 -0800
+
+    qa: ignore duplicates in rados ls
+    
+    These can happen with split or with state changes due to reordering
+    results within the hash range requested. It's easy enough to filter
+    them out at this stage.
+    
+    Backport: giant, firefly
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit e7cc6117adf653a4915fb7a75fac68f8fa0239ec)
+
+commit aef69572588a0dfad58df94cb0d0980d0590d8e4
+Merge: 69eaad7 534624b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jan 15 16:40:48 2015 -0800
+
+    Merge branch 'wip-firefly-rgw-backports' into firefly
+
+commit 534624b7e9decc880e88496355a6fbbe008ede5f
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 21 17:59:30 2014 -0700
+
+    init-radosgw.sysv: set ulimit -n before starting daemon
+    
+    If we do the ulimit inside the daemon command we will have already
+    dropped privs and will fail.
+    
+    Fixes: #9587
+    Backport: giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9803cedf54a7baff45ccd0e0f65d2bc220958a46)
+
+commit fd49cbc535d0d7fa64ebfa458386b47c8ec8616e
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 12 17:07:30 2014 -0800
+
+    rgw: use s->bucket_attrs instead of trying to read obj attrs
+    
+    Fixes: #10307
+    Backport: firefly, giant
+    
+    This is needed, since we can't really read the bucket attrs by trying to
+    read the bucket entry point attrs. We already have the bucket attrs
+    anyway, use these.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5cf193c8686196d5235889e68cb5ea8f1fc8e556)
+
+commit 79bfffb55a0b3a302368e34417d62f74b82dc224
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Nov 5 13:40:55 2014 -0800
+
+    rgw: remove swift user manifest (DLO) hash calculation
+    
+    Fixes: #9973
+    Backport: firefly, giant
+    
+    Previously we were iterating through the parts, creating hash of the
+    parts etags (as S3 does for multipart uploads). However, swift just
+    calculates the etag for the empty manifest object.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit ef6d3ad964d34bc526dc4435486bd5c8cdc3b230)
+    
+    Conflicts:
+    	src/rgw/rgw_op.cc
+
+commit ac799f0834783590cbb6eb91784c8e0753cb1e03
+Author: Lei Dong <leidong@yahoo-inc.com>
+Date:   Mon Oct 27 10:29:48 2014 +0800
+
+    fix can not disable max_size quota
+    
+    Currently if we enable quota and set max_size = -1, it doesn’t
+    mean max_size is unlimited as expected. Instead, it means object
+    with any size is not allowed to upload because of “QuotaExceeded�.
+    The root cause is the function rgw_rounded_kb which convert max_size
+    to max_size_kb returns 0 for -1 because it takes an unsigned int
+    but we pass an int to it. A simple fix is check max_size before
+    it’s rounded to max_size_kb.
+    
+    Test case:
+    1 enable and set quota:
+    radosgw-admin quota enable --uid={user_id} --quota-scope=user
+    radosgw-admin quota set --quota-scope=user --uid={user_id}\
+     --max-objects=100 --max-size=-1
+    2 upload any object with non-zero length
+    it will return 403 with “QuotaExceeded� and return 200 if you apply the fix.
+    
+    Fixes: #9907
+    Backport: giant, firefly
+    Signed-off-by: Dong Lei leidong@yahoo-inc.com
+    (cherry picked from commit abd3fd3ef9ee9999b99811937af60b7a5e673e35)
+
+commit 30963fdc98f5650a68b5737729920d43459b5899
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 23 12:43:55 2014 -0700
+
+    rgw: rados->set_attrs() updates bucket index
+    
+    Fixes: #5595
+    Backport: dumpling, firefly
+    We need to update the bucket index when updating object attrs, otherwise
+    we're missing meta changes that need to be registered. It also
+    solves issue of bucket index not knowing about object acl changes,
+    although this one still requires some more work.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f833f12a200ecc2c4f74ddb443d6fa61b7ad14db)
+
+commit 31f5e332c616dbb7f7338af3ab37ac65ff66f733
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Nov 4 22:05:03 2014 -0800
+
+    rgw: RGWRados::get_obj() returns wrong len if len == 0
+    
+    Fixes: #9877
+    We only updated if len was > 0, should update it if r >= 0. This was the
+    culprit for issue #9877.
+    Backport: giant, firefly
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit fe7bf06366adaf787816d1e68f5e3f68e8c91134)
+
+commit 8682d1b15e097ececd927dfbd096dbb46403afca
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Nov 5 13:28:02 2014 -0800
+
+    rgw: send back ETag on S3 object copy
+    
+    Fixes: #9479
+    Backport: firefly, giant
+    We didn't send the etag back correctly. Original code assumed the etag
+    resided in the attrs, but attrs only contained request attrs.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit b1bfc3a7e0c9088f01f8ff770ae14f569fbc570d)
+    
+    Conflicts:
+    	src/rgw/rgw_rados.cc
+
+commit 2d61fc76e97e5e28cf24cc6e341d49a8fe47059c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Nov 5 12:35:42 2014 -0800
+
+    rgw: S3 object copy content type fix
+    
+    Fixes: #9478
+    Backport: firefly, giant
+    Content type for S3 object copy response should be set to
+    application/xml.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 31963459a0a869c4d32f55baa629f36df33eaa90)
+
+commit 011a8c3bd621337e5a1746f18882be20d3854b14
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 12 11:20:26 2014 -0800
+
+    rgw: http headers need to end with \r\n
+    
+    Fixes: #9254
+    Backport: firefly, giant
+    
+    Reported-by: Benedikt Fraunhofer <fraunhofer@traced.net>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 7409ab3df18fb312dd6c9f79084f889c523afdce)
+    
+    Conflicts:
+    	src/rgw/rgw_civetweb.cc
+    	src/rgw/rgw_fcgi.cc
+
+commit 71d119f2952716aa1f75817e1daaf5fb67ecde94
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jan 7 13:56:14 2015 -0800
+
+    rgw: index swift keys appropriately
+    
+    Fixes: #10471
+    Backport: firefly, giant
+    
+    We need to index the swift keys by the full uid:subuser when decoding
+    the json representation, to keep it in line with how we store it when
+    creating it through other mechanism.
+    
+    Reported-by: hemant burman <hemant.burman@gmail.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 478629bd2f3f32afbe6e93eaebb8a8fa01af356f)
+
+commit 7b89a827a063acf78a848a34c1c3298db213950f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Nov 20 10:36:05 2014 -0800
+
+    rgw-admin: create subuser if needed when creating user
+    
+    Fixes: #10103
+    Backport: firefly, giant
+    This turned up after fixing #9973. Earlier we also didn't create the
+    subuser in this case, but we didn't really read the subuser info when it
+    was authenticating. Now we do that as required, so we end up failing the
+    authentication. This only applies to cases where a subuser was created
+    using 'user create', and not the 'subuser create' command.
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 9ba17a321db06d3d76c9295e411c76842194b25c)
+
+commit 9ee29b5355e9ffeac76707e8d4070bfff5dc99d5
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Nov 5 14:38:46 2014 -0800
+
+    rgw: update swift subuser perm masks when authenticating
+    
+    Fixes: #9918
+    Backport: firefly, giant
+    It seems that we weren't setting the swift perm mask correctly.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5d9f36f757a7272c24d2c9adc31db1ed5e712992)
+
+commit d298fdda2aa177e1a4b43ca94a2292435a50dea4
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Oct 23 17:39:42 2014 -0700
+
+    rgw: send http status reason explicitly in fastcgi
+    
+    There are issues in certain versions of apache 2.4, where the reason is
+    not sent back. Instead, just provide the reason explicitly.
+    
+    Backport: firefly, giant
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit a9dd4af401328e8f9071dee52470a0685ceb296b)
diff --git a/doc/changelog/v0.87.1.txt b/doc/changelog/v0.87.1.txt
new file mode 100644
index 000000000..cf0eecac6
--- /dev/null
+++ b/doc/changelog/v0.87.1.txt
@@ -0,0 +1,2151 @@
+commit 283c2e7cfa2457799f534744d7d549f83ea1335e (tag: refs/tags/v0.87.1, refs/remotes/gh/giant)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Mon Feb 23 12:02:04 2015 -0800
+
+    0.87.1
+
+commit 4178e32dd085adeead84fb168ab8a8a121256259
+Merge: ccb0914 734e9af
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 17 01:09:54 2015 +0100
+
+    Merge pull request #3731 from liewegas/wip-10834-giant
+    
+    osd: tolerate sessionless con in fast dispatch path
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 734e9af5df4ae419ded108f5036bee068a9bc2b2
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 1 18:15:59 2014 -0800
+
+    osd: tolerate sessionless con in fast dispatch path
+    
+    We can now get a session cleared from a Connection at any time.  Change
+    the assert to an if in ms_fast_dispatch to cope.  It's pretty rare, but it
+    can happen, especially with delay injection.  In particular, a racing
+    thread can call mark_down() on us.
+    
+    Fixes: #10209
+    Backport: giant
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 01df2227125abf94571b4b0c7bccca57098ed2dc)
+
+commit ccb0914f76da23acdd7374233cd1939ab80ef3c8
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Feb 2 16:43:35 2015 +0100
+
+    qa: use correct binary path on rpm-based systems
+    
+    Fixes: #10715
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 05ce2aa1bf030ea225300b48e7914577a412b38c)
+
+commit 78c71b9200da5e7d832ec58765478404d31ae6b5
+Merge: 222aa22 91515e7
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Feb 11 00:11:57 2015 +0100
+
+    Merge pull request #3407 from ceph/wip-9854-giant
+    
+    osdc: Constrain max number of in-flight read requests
+
+commit 222aa22ebc0ccb1b04156e0c9d05f4e4733ec290
+Merge: b9ff170 a5cb39c
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 22:01:21 2015 +0100
+
+    Merge pull request #3568 from dachary/wip-10471-rgw-giant
+    
+    rgw: use s->bucket_attrs instead of trying to read obj attrs
+
+commit b9ff1708ad85ca5aeb10b4202bcbe197251e3bd8
+Merge: 34103b6 b1e4882
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:59:40 2015 +0100
+
+    Merge pull request #3263 from dachary/wip-jerasure-giant
+    
+    erasure-code: update links to jerasure upstream (giant)
+
+commit 34103b6355881820aa10b354c2427654bf229e8f
+Merge: 94889cf d125743
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:59:17 2015 +0100
+
+    Merge pull request #3191 from ceph/giant-10277
+    
+    Giant 10277
+
+commit 94889cf6bef5a542e51bf8434dbe7c68f64604ce
+Merge: d7b10d8 d28c8e0
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:58:52 2015 +0100
+
+    Merge pull request #3186 from ceph/wip-giant-mon-backports
+    
+    mon: backports for #9987 against giant
+
+commit d7b10d897e17bc3fa690c8484ad2d6f233896237
+Merge: 11f7d06 16c6d0d
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:58:29 2015 +0100
+
+    Merge pull request #3185 from dachary/wip-10325-cauchy-giant
+    
+    erasure-code: relax cauchy w restrictions (giant)
+
+commit 11f7d064e5d93bc0ed8896750344c6cf6b37aeab
+Merge: 975be75 636b98f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:58:05 2015 +0100
+
+    Merge pull request #3178 from dachary/wip-9998-crush-underfloat-giant
+    
+    crush: fix weight underfloat issue (giant)
+
+commit 975be75f4bcea88b232ea76087b49e288d7c29f7
+Merge: 51fe79d d759e71
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:57:50 2015 +0100
+
+    Merge pull request #3579 from dachary/wip-9877-rgw-giant
+    
+    rgw: RGWRados::get_obj() returns wrong len if len == 0
+
+commit 51fe79d9e63c7df4da547a0ba7a12aa9c6cd7ab2
+Merge: fca9ead 319f9c9
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:57:18 2015 +0100
+
+    Merge pull request #3168 from ceph/wip-8797-giant
+    
+    Wip 8797 giant
+
+commit fca9eadaf3fcef77e269d4936d4eea86ab6c3faf
+Merge: 317532b 9886620
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:56:53 2015 +0100
+
+    Merge pull request #3582 from dachary/wip-10062-rgw-giant
+    
+    rgw: s3 keystone auth fixes
+
+commit 317532b70383762f473a910e043c889574eb6087
+Merge: 3e8f3e3 debc0c5
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:56:37 2015 +0100
+
+    Merge pull request #3581 from dachary/wip-9918-rgw-giant
+    
+    rgw: update swift subuser perm masks when authenticating
+
+commit 3e8f3e38af76fa2cba86aedf962d3230d7979f63
+Merge: 1d77591 76f9de3
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:56:17 2015 +0100
+
+    Merge pull request #3580 from dachary/wip-9907-rgw-giant
+    
+    fix can not disable max_size quota
+
+commit 1d7759149697242192be05decf7ffafb17b24cbe
+Merge: b016863 ad04a67
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:55:56 2015 +0100
+
+    Merge pull request #3083 from dachary/wip-10211-erasure-code-buffer-alignement-giant
+    
+    erasure-code: enforce chunk size alignment (giant)
+
+commit b016863ad243388e7571da9ffca3013c8f99237a
+Merge: bdcc9dc d21f4e3
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:55:23 2015 +0100
+
+    Merge pull request #3577 from dachary/wip-9587-rgw-giant
+    
+    init-radosgw.sysv: set ulimit -n before starting daemon
+
+commit bdcc9dcb8586c91b432c7087e33a2b52ef467b54
+Merge: df475f9 7b5f746
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:54:58 2015 +0100
+
+    Merge pull request #3576 from dachary/wip-5595-9576-rgw-giant
+    
+    update object content-length doesn't work correctly
+
+commit df475f92a41e3bd5a022335b2c9023ad40c3b47b
+Merge: db7adf8 b2f6f7f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:54:13 2015 +0100
+
+    Merge pull request #3575 from dachary/wip-9479-rgw-giant
+    
+    rgw: send back ETag on S3 object copy
+
+commit db7adf8d8ca225fea2d0277ced614e936df086c9
+Merge: 9b50db9 67ba4d3
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:53:55 2015 +0100
+
+    Merge pull request #3574 from dachary/wip-9478-rgw-giant
+    
+    rgw: S3 object copy content type fix
+
+commit 9b50db97a9552841ed143588e2f63bab56d0aecb
+Merge: 583fe31 84e9b6c
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:53:33 2015 +0100
+
+    Merge pull request #3573 from dachary/wip-9254-rgw-giant
+    
+    rgw: http headers need to end with \r\n
+
+commit 583fe31681c4eea8b85b413674074445e2b424a6
+Merge: 5240db5 1cb0955
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:53:15 2015 +0100
+
+    Merge pull request #3572 from dachary/wip-9973-rgw-giant
+    
+    rgw: remove swift user manifest (DLO) hash calculation
+
+commit 5240db588e9017dd8a487b7a9ee16f171fdda1ff
+Merge: 62e1552 e230fab
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:52:54 2015 +0100
+
+    Merge pull request #3571 from dachary/wip-8911-rgw-giant
+    
+    rgw: swift GET / HEAD object returns X-Timestamp field
+
+commit 62e15528dda20b5419e39744fa9e0c9c4cae053c
+Merge: 16cd892 c24fab3
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:52:36 2015 +0100
+
+    Merge pull request #3570 from dachary/wip-10701-rgw-giant
+    
+    rgw: use strict_strtoll() for content length
+
+commit 16cd892aab4ffb1dc15b93a4101d9bc209591c94
+Merge: 028904c 6aef29e
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:52:00 2015 +0100
+
+    Merge pull request #3569 from dachary/wip-10103-rgw-giant
+    
+    rgw-admin: create subuser if needed when creating user
+
+commit 028904cf7c36a1d5342cf29c115bc0437e9b2d74
+Merge: 520dcf8 425ee8a
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:51:06 2015 +0100
+
+    Merge pull request #3567 from dachary/wip-10307-rgw-giant
+    
+    rgw: use s->bucket_attrs instead of trying to read obj attrs
+
+commit 520dcf8624eeafd694115b382616be83f9b344d3
+Merge: cae1de2 14cdb9b
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:50:34 2015 +0100
+
+    Merge pull request #3443 from ceph/wip-10590-giant
+    
+    rbd: ensure aio_write buffer isn't invalidated during image import
+
+commit cae1de29922c2183eff021c6fe2b921a87b5f5b2
+Merge: b346ad3 83a0a2e
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:50:14 2015 +0100
+
+    Merge pull request #3557 from dachary/wip-10688-boost-157-giant
+    
+    support Boost 1.57.0
+
+commit b346ad37a84b7cfd0bae84528f2533a8cc4a8e3d
+Merge: aacd51c 13bb880
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:49:47 2015 +0100
+
+    Merge pull request #2954 from sponce/giant
+    
+    Fixed trivial locking issue in the trunc method of libradosstriper - Giant branch
+
+commit aacd51c74c102d44982421b9bc384d12fc160e3c
+Merge: 1d97c7c 081f49b
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:49:20 2015 +0100
+
+    Merge pull request #3405 from ceph/wip-10299-giant
+    
+    librbd: complete all pending aio ops prior to closing image
+
+commit 1d97c7c9a3087e7bc98774d9fe2882bdc4a84531
+Merge: 53dec0e 436923c
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:48:49 2015 +0100
+
+    Merge pull request #3403 from ceph/wip-10270-giant
+    
+    librbd: gracefully handle deleted/renamed pools
+
+commit 53dec0eeee60b315e88acb4ba05666857ae3e0eb
+Merge: df8285c 1261bf2
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 10 21:48:20 2015 +0100
+
+    Merge pull request #3356 from liewegas/wip-msgr-giant
+    
+    msgr: fast dispatch backports for giant
+
+commit df8285c5e5b14d9a4cd42fb9be8e18fe6cdf6f83
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Thu Feb 5 21:12:17 2015 -0800
+
+    fsync-tester: print info about PATH and locations of lsof lookup
+    
+    We're seeing the lsof invocation fail (as not found) in testing and nobody can
+    identify why. Since attempting to reproduce the issue has not worked, this
+    patch will gather data from a genuinely in-vitro location.
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit a85051483874ff5b8b0fb50426a3577040457596)
+
+commit 91515e750bfe2453ce8ac9ec568b0e314823dd82
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Oct 27 14:47:19 2014 -0400
+
+    osdc: Constrain max number of in-flight read requests
+    
+    Constrain the number of in-flight RADOS read requests to the
+    cache size.  This reduces the chance of the cache memory
+    ballooning during certain scenarios like copy-up which can
+    invoke many concurrent read requests.
+    
+    Fixes: #9854
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 98866208c64348ca885335d95a1c737071a17004
+Author: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+Date:   Mon Dec 22 19:57:19 2014 +0530
+
+    rgw: check keystone auth also for s3 post requests
+    
+    This patch adds keystone auth for s3 post requests, once a user fails in
+    cephx authentication, they are checked for keystone if configured.
+    
+    Fixes #10062
+    Signed-off-by: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+    
+    (cherry picked from commit 8b3dfc9472022ea45ad24e02e0aa21dfdad798f8)
+
+commit 4e4372b8e551bb1b974f08dc69f5b27bdd22bb4b
+Author: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+Date:   Mon Nov 17 17:37:00 2014 +0530
+
+    rgw: check for timestamp for s3 keystone auth
+    
+    This commit ensures that we check for timestamp of s3 request is within
+    acceptable grace time of radosgw
+    Addresses some failures in #10062
+    Fixes: #10062
+    Signed-off-by: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+    
+    (cherry picked from commit 4b35ae067fef9f97b886afe112d662c61c564365)
+
+commit debc0c593fb7401d07a34f7916380092ad7285f9
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Nov 5 14:38:46 2014 -0800
+
+    rgw: update swift subuser perm masks when authenticating
+    
+    Fixes: #9918
+    Backport: firefly, giant
+    It seems that we weren't setting the swift perm mask correctly.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5d9f36f757a7272c24d2c9adc31db1ed5e712992)
+
+commit 76f9de330deaa4fafb86d3f6f2754f0d419306b1
+Author: Lei Dong <leidong@yahoo-inc.com>
+Date:   Mon Oct 27 10:29:48 2014 +0800
+
+    fix can not disable max_size quota
+    
+    Currently if we enable quota and set max_size = -1, it doesn’t
+    mean max_size is unlimited as expected. Instead, it means object
+    with any size is not allowed to upload because of “QuotaExceeded�.
+    The root cause is the function rgw_rounded_kb which convert max_size
+    to max_size_kb returns 0 for -1 because it takes an unsigned int
+    but we pass an int to it. A simple fix is check max_size before
+    it’s rounded to max_size_kb.
+    
+    Test case:
+    1 enable and set quota:
+    radosgw-admin quota enable --uid={user_id} --quota-scope=user
+    radosgw-admin quota set --quota-scope=user --uid={user_id}\
+     --max-objects=100 --max-size=-1
+    2 upload any object with non-zero length
+    it will return 403 with “QuotaExceeded� and return 200 if you apply the fix.
+    
+    Fixes: #9907
+    Backport: giant, firefly
+    Signed-off-by: Dong Lei leidong@yahoo-inc.com
+    (cherry picked from commit abd3fd3ef9ee9999b99811937af60b7a5e673e35)
+
+commit d759e71c8167ea29c8fda9483039a3e491083da5
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Nov 4 22:05:03 2014 -0800
+
+    rgw: RGWRados::get_obj() returns wrong len if len == 0
+    
+    Fixes: #9877
+    We only updated if len was > 0, should update it if r >= 0. This was the
+    culprit for issue #9877.
+    Backport: giant, firefly
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit fe7bf06366adaf787816d1e68f5e3f68e8c91134)
+
+commit d21f4e326eb4821cc9bd38a1b62a0210272277d4
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 21 17:59:30 2014 -0700
+
+    init-radosgw.sysv: set ulimit -n before starting daemon
+    
+    If we do the ulimit inside the daemon command we will have already
+    dropped privs and will fail.
+    
+    Fixes: #9587
+    Backport: giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9803cedf54a7baff45ccd0e0f65d2bc220958a46)
+
+commit 7b5f746d2f97c7139f9c31962c107a074bfd1863
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 23 12:42:10 2014 -0700
+
+    rgw: PutObjMetadata, clear bufferlist before appending into it
+    
+    Fixes: #9576
+    Backport: firefly, dumpling
+    
+    We need to completely rewrite the bufferlist, not append into it.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 44cfd88dc65d30f4295743c5834768bb13f7b805)
+
+commit e24f27b7b2e2aeb84b14788e8bf2757ecdf8f0c0
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 23 12:43:55 2014 -0700
+
+    rgw: rados->set_attrs() updates bucket index
+    
+    Fixes: #5595
+    Backport: dumpling, firefly
+    We need to update the bucket index when updating object attrs, otherwise
+    we're missing meta changes that need to be registered. It also
+    solves issue of bucket index not knowing about object acl changes,
+    although this one still requires some more work.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f833f12a200ecc2c4f74ddb443d6fa61b7ad14db)
+
+commit b2f6f7f6205682aeb09c0785e373ddf5d89c6d04
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Nov 5 13:28:02 2014 -0800
+
+    rgw: send back ETag on S3 object copy
+    
+    Fixes: #9479
+    Backport: firefly, giant
+    We didn't send the etag back correctly. Original code assumed the etag
+    resided in the attrs, but attrs only contained request attrs.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit b1bfc3a7e0c9088f01f8ff770ae14f569fbc570d)
+
+commit 67ba4d3444f0f64dae6286be28276ba85376ecf6
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Nov 5 12:35:42 2014 -0800
+
+    rgw: S3 object copy content type fix
+    
+    Fixes: #9478
+    Backport: firefly, giant
+    Content type for S3 object copy response should be set to
+    application/xml.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 31963459a0a869c4d32f55baa629f36df33eaa90)
+
+commit 84e9b6c32c6fdc38e2c64f3360c185332e691bf4
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 12 11:20:26 2014 -0800
+
+    rgw: http headers need to end with \r\n
+    
+    Fixes: #9254
+    Backport: firefly, giant
+    
+    Reported-by: Benedikt Fraunhofer <fraunhofer@traced.net>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 7409ab3df18fb312dd6c9f79084f889c523afdce)
+
+commit 1cb09555d3fdb568296797cd83eb5557552f056c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Nov 5 13:40:55 2014 -0800
+
+    rgw: remove swift user manifest (DLO) hash calculation
+    
+    Fixes: #9973
+    Backport: firefly, giant
+    
+    Previously we were iterating through the parts, creating hash of the
+    parts etags (as S3 does for multipart uploads). However, swift just
+    calculates the etag for the empty manifest object.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit ef6d3ad964d34bc526dc4435486bd5c8cdc3b230)
+
+commit e230fabf29c4660594d19027af49810e57b82e35
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 30 14:15:47 2014 -0700
+
+    rgw: swift GET / HEAD object returns X-Timestamp field
+    
+    Fixes: #8911
+    Backport: giant, firefly, dumpling
+    Swift clients expect X-Timestamp header, dump it.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5b41d80b7fb9ed96c26801fc42c044191bb18d84)
+
+commit c24fab3065ba6d81435981b609f2b69c3d98d21d
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jan 30 10:51:52 2015 -0800
+
+    rgw: use strict_strtoll() for content length
+    
+    instead of strict_strtol().
+    
+    Backport: giant, firefly
+    Fixes: #10701
+    
+    Reported-by: Axel Dunkel <ad@dunkel.de>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 1c25dbafb45caf1b261cfcec15b868a2ba6b5fef)
+
+commit 6aef29e31e9c7c7ccf8e95d573700c08218b2b45
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Nov 20 10:36:05 2014 -0800
+
+    rgw-admin: create subuser if needed when creating user
+    
+    Fixes: #10103
+    Backport: firefly, giant
+    This turned up after fixing #9973. Earlier we also didn't create the
+    subuser in this case, but we didn't really read the subuser info when it
+    was authenticating. Now we do that as required, so we end up failing the
+    authentication. This only applies to cases where a subuser was created
+    using 'user create', and not the 'subuser create' command.
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 9ba17a321db06d3d76c9295e411c76842194b25c)
+
+commit a5cb39cbb6aee869b92ac20975b5c80a01210b63
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 12 17:07:30 2014 -0800
+
+    rgw: use s->bucket_attrs instead of trying to read obj attrs
+    
+    Fixes: #10307
+    Backport: firefly, giant
+    
+    This is needed, since we can't really read the bucket attrs by trying to
+    read the bucket entry point attrs. We already have the bucket attrs
+    anyway, use these.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5cf193c8686196d5235889e68cb5ea8f1fc8e556)
+
+commit 425ee8a07bb8ce12eee124b3c374031f644aa32b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 12 17:07:30 2014 -0800
+
+    rgw: use s->bucket_attrs instead of trying to read obj attrs
+    
+    Fixes: #10307
+    Backport: firefly, giant
+    
+    This is needed, since we can't really read the bucket attrs by trying to
+    read the bucket entry point attrs. We already have the bucket attrs
+    anyway, use these.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5cf193c8686196d5235889e68cb5ea8f1fc8e556)
+
+commit 83a0a2e5b52b18a25009faaf09fa4f48af3c52ee
+Author: William A. Kennington III <william@wkennington.com>
+Date:   Sat Sep 20 22:52:31 2014 -0700
+
+    osd: Cleanup boost optionals
+    
+    Signed-off-by: William A. Kennington III <william@wkennington.com>
+    (cherry picked from commit a53ead14c113047567177630b4906136a2109b65)
+
+commit eb30631ec3d081fd1bc2cdbd4812a334de9e1282
+Author: Petr Machata <pmachata@redhat.com>
+Date:   Thu Jan 29 10:15:02 2015 -0700
+
+    support Boost 1.57.0
+    
+    Sometime after 1.55, boost introduced a forward declaration of
+    operator<< in optional.hpp. In 1.55 and earlier, when << was used
+    without the _io having been included, what got dumped was an implicit
+    bool conversion.
+    
+    http://tracker.ceph.com/issues/10688 Refs: #10688
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 85717394c33137eb703a7b88608ec9cf3287f67a)
+    
+    Conflicts:
+    	src/include/encoding.h
+            trivial conflict
+
+commit 1ccd73a16e1829b5519ec5b83b2554af173ad052
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Oct 23 17:39:42 2014 -0700
+
+    rgw: send http status reason explicitly in fastcgi
+    
+    There are issues in certain versions of apache 2.4, where the reason is
+    not sent back. Instead, just provide the reason explicitly.
+    
+    Backport: firefly, giant
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit a9dd4af401328e8f9071dee52470a0685ceb296b)
+
+commit 3bf42af2e932a473b19cb54637e8543a666a4a28
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jan 15 16:31:22 2015 -0800
+
+    rgw: fix partial GET in swift
+    
+    Fixes: #10553
+    backport: firefly, giant
+    
+    Don't set the ret code to reflect partial download, just set the
+    response status when needed.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 7e1553cedff90fa0fefded65cde87ad068eb5f0c)
+
+commit aa038684dce1964d5d23802d23f2bd772458ea11
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 15 17:04:32 2014 -0800
+
+    osd: handle no-op write with snapshot case
+    
+    If we have a transaction that does something to the object but it !exists
+    both before and after, we will continue through the write path.  If the
+    snapdir object already exists, and we try to create it again, we will
+    leak a snapdir obc and lock and later crash on an assert when the obc
+    is destroyed:
+    
+    0> 2014-12-06 01:49:51.750163 7f08d6ade700 -1 osd/osd_types.h: In function 'ObjectContext::~ObjectContext()' thread 7f08d6ade700 time 2014-12-06 01:49:51.605411
+    osd/osd_types.h: 2944: FAILED assert(rwstate.empty())
+    
+    Fix is to not recreated the snapdir if it already exists.
+    
+    Fixes: #10262
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 02fae9fc54c10b5a932102bac43f32199d4cb612)
+
+commit e045ad4a39076547209ac1dc298df5ebffb76669
+Merge: a463b92 9f865fa
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Tue Jan 27 09:40:16 2015 -0800
+
+    Merge pull request #3502 from ceph/wip-10382-giant
+    
+    [giant backport] mds: handle heartbeat_reset during shutdown
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 9f865fae095a1fe8a26acb50667f1d774d6020b6
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Jan 14 10:35:53 2015 +0000
+
+    mds: handle heartbeat_reset during shutdown
+    
+    Because any thread might grab mds_lock and call heartbeat_reset
+    immediately after a call to suicide() completes, this needs
+    to be handled as a special case where we tolerate MDS::hb having
+    already been destroyed.
+    
+    Fixes: #10382
+    Signed-off-by: John Spray <john.spray@redhat.com>
+
+commit a463b92e475cd1f4cdb963e402033ebc9d37dbdc
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jan 19 18:28:20 2015 -0800
+
+    ceph_test_rados_api_misc: do not assert rbd feature match
+    
+    This test fails on upgrades when we (or the server) have new
+    features.  Make it less fragile.
+    
+    Fixes: #10576
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9147c62989871cea8b3a85b02c53017825efb55b)
+
+commit 14cdb9bb6d27f2017a3a8e6c1f274b9f40fb7456
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jan 21 14:55:02 2015 -0500
+
+    rbd: ensure aio_write buffer isn't invalidated during image import
+    
+    The buffer provided to aio_write shouldn't be invalidated until
+    after aio_write has indicated that the operation has completed.
+    
+    Fixes: #10590
+    Backport: giant
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 4d3b49e9d62bc1040356ca3ebe7f90c181734eb6)
+
+commit 081f49b47ca8d7583211f546ab5699b14f773bfc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Dec 15 10:53:53 2014 -0500
+
+    librbd: complete all pending aio ops prior to closing image
+    
+    It was possible for an image to be closed while aio operations
+    were still outstanding.  Now all aio operations are tracked and
+    completed before the image is closed.
+    
+    Fixes: #10299
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 436923c68b77c900b7774fbef918c0d6e1614a36
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jan 19 10:28:56 2015 -0500
+
+    librbd: gracefully handle deleted/renamed pools
+    
+    snap_unprotect and list_children both attempt to scan all
+    pools. If a pool is deleted or renamed during the scan,
+    the methods would previously return -ENOENT. Both methods
+    have been modified to more gracefully handle this condition.
+    
+    Fixes: #10270
+    Backport: giant, firefly
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 4c8a5cedcb7942e1e01ab4cedfbf03e4c56cc1e4
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 12 05:24:01 2014 -0800
+
+    rgw: change multipart upload id magic
+    
+    Fixes: #10271
+    Backport: firefly, giant
+    
+    Some clients can't sign requests correctly with the original magic
+    prefix.
+    
+    Reported-by: Georgios Dimitrakakis <giorgis@acmac.uoc.gr>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5fc7a0be67a03ed63fcc8408f8d71a31a1841076)
+
+commit b10c0d5110547586b2edac53c267391d3d42f974
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Dec 11 09:07:10 2014 -0800
+
+    rgw: url decode http query params correctly
+    
+    Fixes: #10271
+    Backport: firefly
+    
+    This got broken by the fix for #8702. Since we now only url_decode if
+    we're in query, we need to specify that we're in query when decoding
+    these args.
+    
+    Reported-by: Georgios Dimitrakakis <giorgis@acmac.uoc.gr>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 21e07eb6abacb085f81b65acd706b46af29ffc03)
+
+commit 65bf3b08b572b9b25ad064fb784742e5d6456f06
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Jan 14 15:01:38 2015 -0800
+
+    qa: ignore duplicates in rados ls
+    
+    These can happen with split or with state changes due to reordering
+    results within the hash range requested. It's easy enough to filter
+    them out at this stage.
+    
+    Backport: giant, firefly
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit e7cc6117adf653a4915fb7a75fac68f8fa0239ec)
+
+commit 1261bf24624f871672002ab0915e23f1c95b0aa5
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 14 12:42:40 2014 -0700
+
+    Revert "Objecter: disable fast dispatch of CEPH_MSG_OSD_OPREPLY messages"
+    
+    This reverts commit 3f23709c474292f9239f77a6cce26309fc86ce29.
+    
+    We have changed mark_down() behavior so that it no longer blocks on
+    fast dispatch.  This makes the objecter reply handler safe again.
+    
+    Fixes: #9598
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c9f9e72e558521cb90f90538bc27f995f82d76c2)
+
+commit 300d4c6ff7e998dba0c67f6dde746dc23d681397
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 14 12:41:48 2014 -0700
+
+    msg/simple: do not stop_and_wait on mark_down
+    
+    We originally blocked in mark_down for fast dispatch threads
+    to complete to avoid various races in the code.  Most of these
+    were in the OSD itself, where we were not prepared to get
+    messges on connections that had no attached session.  Since
+    then, the OSD checks have been cleaned up to handle this.
+    There were other races we were worried about too, but the
+    details have been lost in the depths of time.
+    
+    Instead, take the other route: make mark_down never block on
+    dispatch.  This lets us remove the special case that
+    was added in order to cope with fast dispatch calling
+    mark_down on itself.
+    
+    Now, the only stop_and_wait() user is the shutdown sequence.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 00907e032011b9d2acd16ea588555cf379830814)
+
+commit c3335c7aa6c1e6c3f1879c0cd3cd2f13091221be
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 31 16:25:09 2014 -0700
+
+    msg/Pipe: inject delay in stop_and_wait
+    
+    Inject a delay in stop_and_wait.  This will mostly affect the connection
+    race Pipe takeover code which currently calls stop_and_wait while holding
+    the msgr->lock.  This should make it easier for a racing fast_dispatch
+    method to get stuck on a call that (indirectly) needs the msgr lock.
+    See #9921.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2fe5c4c305218fdb1771857e4e0ef7c98a8d0fb6)
+
+commit 1dbe8f5a6f7bf2b7c86d24f27d569d71e0076ee9
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Tue Oct 28 16:45:43 2014 -0700
+
+    SimpleMessenger: Pipe: do not block on takeover while holding global lock
+    
+    We previously were able to cause deadlocks:
+    1) Existing pipe is fast_dispatching
+    2) Replacement incoming pipe is accepted
+      *) blocks on stop_and_wait() of existing Pipe
+    3) External things are blocked on SimpleMessenger::lock() while
+      blocking completion of the fast dispatch.
+    
+    To resolve this, if we detect that an existing Pipe we want to take over is
+    in the process of fast dispatching, we unlock our locks and wait on it to
+    finish. Then we go back to the lookup step and retry.
+    
+    The effect of this should be safe:
+    1) We are not making any changes to the existing Pipe in new ways
+    2) We have not registered the new Pipe anywhere
+    3) We have not sent back any replies based on Messenger state to
+       the remote endpoint.
+    
+    Backport: giant
+    Fixes: #9921
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 2d6980570af2226fdee0edfcfe5a8e7f60fae615)
+
+commit 16c023d8fa5575d4dd138aeee4d4fd9b8f32c0f6
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 8 13:34:52 2015 -0800
+
+    osd: requeue PG when we skip handling a peering event
+    
+    If we don't handle the event, we need to put the PG back into the peering
+    queue or else the event won't get processed until the next event is
+    queued, at which point we'll be processing events with a delay.
+    
+    The queue_null is not necessary (and is a waste of effort) because the
+    event is still in pg->peering_queue and the PG is queued.
+    
+    Note that this only triggers when we exceeed osd_map_max_advance, usually
+    when there is a lot of peering and recovery activity going on.  A
+    workaround is to increase that value, but if you exceed osd_map_cache_size
+    you expose yourself to crache thrashing by the peering work queue, which
+    can cause serious problems with heavily degraded clusters and bit lots of
+    people on dumpling.
+    
+    Backport: giant, firefly
+    Fixes: #10431
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 492ccc900c3358f36b6b14a207beec071eb06707)
+
+commit 16c6d0d589d53aad7bb2cd0e104300fb920d5caf
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Dec 16 13:31:30 2014 +0100
+
+    erasure-code: relax cauchy w restrictions
+    
+    A restriction that the w parameter of the cauchy technique is limited to
+    8, 16 or 32 was added incorrectly while refactoring parameter parsing in
+    the jerasure plugin and must be relaxed.
+    
+    http://tracker.ceph.com/issues/10325 Fixes: #10325
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit bb80437f247345502203ad87a7e7bbb5b5602b9a)
+
+commit 636b98faa6b1c9fd6de1b8653d1d282577b54684
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Nov 23 18:50:51 2014 -0800
+
+    crush/CrushWrapper: fix create_or_move_item when name exists but item does not
+    
+    We were using item_exists(), which simply checks if we have a name defined
+    for the item.  Instead, use _search_item_exists(), which looks for an
+    instance of the item somewhere in the hierarchy.  This matches what
+    get_item_weightf() is doing, which ensures we get a non-negative weight
+    that converts properly to floating point.
+    
+    Backport: giant, firefly
+    Fixes: #9998
+    Reported-by: Pawel Sadowski <ceph@sadziu.pl>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9902383c690dca9ed5ba667800413daa8332157e)
+
+commit ced2472664fab06d03de03d7b23325f9319163b7
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Nov 21 17:47:56 2014 -0800
+
+    crush/builder: prevent bucket weight underflow on item removal
+    
+    It is possible to set a bucket weight that is not the sum of the item
+    weights if you manually modify/build the CRUSH map.  Protect against any
+    underflow on the bucket weight when removing items.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8c87e9502142d5b4a282b94f929ae776a49be1dc)
+
+commit adf8798dabb679110c6815af5d73ab6ff20a1af8
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Nov 21 17:37:03 2014 -0800
+
+    crush/CrushWrapper: fix _search_item_exists
+    
+    Reported-by: Pawel Sadowski <ceph@sadziu.pl>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit eeadd60714d908a3a033aeb7fd542c511e63122b)
+
+commit 6caa4fa42c6eaa76b3b9caf37e4ee09844f017a7
+Author: Warren Usui <warren.usui@inktank.com>
+Date:   Thu Dec 18 20:00:28 2014 -0800
+
+    If trusty, use older version of qemu
+    
+    Fixes #10319
+    Signed-off-by: Warren Usui <warren.usui@inktank.com>
+    (cherry-picked from 46a1a4cb670d30397979cd89808a2e420cef2c11)
+
+commit 44c944e96440bd338d22533779e0650b99115a16
+Merge: abdbbd6 910ec62
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 29 10:55:22 2014 -0800
+
+    Merge pull request #3266 from ceph/giant-10415
+    
+    libcephfs/test.cc: close fd before umount
+
+commit b1e48820785a1d3153fc926ad21355b3927b44e9
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Dec 28 10:29:54 2014 +0100
+
+    erasure-code: update links to jerasure upstream
+    
+    It moved from bitbucket to jerasure.org
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 8e86f901939f16cc9c8ad7a4108ac4bcf3916d2c)
+
+commit 910ec624156d26a1830078161f47328a950a4eee
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Dec 23 10:22:00 2014 +0800
+
+    libcephfs/test.cc: close fd before umount
+    
+    Fixes: #10415
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit d3fb563cee4c4cf08ff4ee01782e52a100462429)
+
+commit abdbbd6e846727385cf0a1412393bc9759dc0244
+Author: Warren Usui <warren.usui@inktank.com>
+Date:   Tue Dec 16 22:01:26 2014 -0800
+
+    Remove sepia dependency (use fqdn)
+    
+    Fixes: #10255
+    Signed-off-by: Warren Usui <warren.usui@inktank.com>
+    (cherry picked from commit 19dafe164833705225e168a686696fb4e170aba7)
+
+commit d1257436fdf79bad5fe0719a6be71e2abb2d2462 (refs/remotes/gh/giant-10277)
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Dec 17 15:59:44 2014 +0800
+
+    client: use remount to trim kernel dcache
+    
+    when remounting a file system, linux kernel trims all unused dentry
+    in the file system.
+    
+    Fixes: #10277
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+
+commit 9de9901cacd2ed2c8c5f65a938fb6a996efab4cd
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Dec 17 15:46:49 2014 +0800
+
+    client: cleanup client callback registration
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+
+commit d28c8e0fb924fbf36e4e15e19554ad30da3ff8f2 (refs/remotes/gh/wip-giant-mon-backports)
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Nov 2 08:50:59 2014 -0800
+
+    mon/PGMap and PGMonitor: update last_epoch_clean cache from new osd keys
+    
+    We were only invalidating the cached value from apply_incremental, which
+    is no longer called on modern clusters.
+    
+    Fix this by storing the update epoch in the key as well (it is not part
+    of osd_stat_t).
+    
+    Backport: giant, firefly, dumpling(?)
+    Fixes: #9987
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 093c5f0cabeb552b90d944da2c50de48fcf6f564)
+
+commit 7646f239476609c96b6baf94dfd5f727fff49502
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Nov 2 08:49:48 2014 -0800
+
+    mon/PGMap: invalidate cached min_last_epoch_clean from new-style pg keys
+    
+    We were only invalidating the cache from the legacy apply_incremental(),
+    which is no longer called on modern clusters.
+    
+    Fixes: #9987
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3fb731b722c50672a5a9de0c86a621f5f50f2d06)
+
+commit 6ec14b07940ff64d6a121e21a730f691a1a71546
+Merge: 758d9cf 7bbf80f
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Thu Dec 11 17:03:07 2014 -0800
+
+    Merge pull request #3159 from ceph/wip-10229-giant
+    
+    osdc/Filer: use finisher to execute C_Probe and C_PurgeRange [giant backport]
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 7bbf80ff7388f104cf318dd5ac61ca7d35274694
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Dec 4 12:18:47 2014 +0800
+
+    osdc/Filer: use finisher to execute C_Probe and C_PurgeRange
+    
+    Currently contexts C_Probe/C_PurgeRange are executed while holding
+    OSDSession::completion_lock. C_Probe and C_PurgeRange may call
+    Objecter::stat() and Objecter::remove() respectively, which acquire
+    Objecter::rwlock. This can cause deadlock because there is intermediate
+    dependency between Objecter::rwlock and OSDSession::completion_lock:
+    
+     Objecter::rwlock -> OSDSession::lock -> OSDSession::completion_lock
+    
+    The fix is exexcute C_Probe/C_PurgeRange in finisher thread.
+    
+    Fixes: #10229
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit d3ee89ace660161df7796affbf9a70f3d0dedce1)
+
+commit 758d9cf30bfc7736cc297ba3b047756f7eb8183e
+Merge: a8e5638 994dcbb
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Thu Dec 11 10:47:38 2014 -0800
+
+    Merge pull request #3151 from ceph/wip-10288-giant
+    
+    mon: fix `fs ls` on peons [giant backport]
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 994dcbbef4bea532aea4143c3ac1372ca14d2aea
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Dec 11 14:00:57 2014 +0000
+
+    mon: fix `fs ls` on peons
+    
+    This was incorrectly using pending_mdsmap instead
+    of mdsmap.  We didn't notice in test because of
+    single-mon configurations.
+    
+    Fixes: #10288
+    Backport: giant
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 5559e6aea9e9374ecdac0351777dfd6f5f5d1e67)
+
+commit 319f9c9352bfd1b95bd685500922e6cee2199b34 (refs/remotes/gh/wip-8797-giant)
+Author: Dan Mick <dan.mick@redhat.com>
+Date:   Wed Dec 10 13:19:53 2014 -0800
+
+    Call Rados.shutdown() explicitly before exit
+    
+    This is mostly a demonstration of good behavior, as the resources will
+    be reclaimed on exit anyway.
+    
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    (cherry picked from commit b038e8fbf9103cc42a4cde734b3ee601af6019ea)
+
+commit ed8c9af3376aeb6f245cbab694fdbc0ce95634a8
+Author: Dan Mick <dan.mick@redhat.com>
+Date:   Wed Dec 10 13:19:16 2014 -0800
+
+    rados.py: remove Rados.__del__(); it just causes problems
+    
+    Recent versions of Python contain a change to thread shutdown that
+    causes ceph to hang on exit; see http://bugs.python.org/issue21963.
+    As it turns out, this is relatively easy to avoid by not spawning
+    threads on exit, as Rados.__del__() will certainly do by calling
+    shutdown(); I suspect, but haven't proven, that the problem is
+    that shutdown() tries to start() a threading.Thread() that never
+    makes it all the way back to signal start().
+    
+    Also add a PendingReleaseNote and extra doc comments to clarify.
+    
+    Fixes: #8797
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    (cherry picked from commit 5ba9b8f21f8010c59dd84a0ef2acfec99e4b048f)
+    
+    Conflicts:
+    	PendingReleaseNotes
+
+commit a8e56380f08cd5940def4cc47cadba699a8ba45d
+Merge: 247a6fa e7faed5
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Mon Dec 8 13:19:20 2014 -0800
+
+    Merge pull request #3010 from dachary/wip-10018-primary-erasure-code-hinfo-giant
+    
+    osd: deep scrub must not abort if hinfo is missing (giant)
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 247a6fac54854e92a7df0e651e248a262d3efa05
+Merge: 3372060 309fd5f
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Mon Dec 8 12:36:48 2014 -0800
+
+    Merge pull request #3110 from ceph/giant-10263
+    
+    mds: store backtrace for straydir
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 309fd5f56ef5ea76ffd525fdde6e6fbbc9ef6ef1
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Nov 7 11:38:37 2014 +0800
+
+    mds: store backtrace for straydir
+    
+    Backport: giant, firefly, emperor, dumpling
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 0d89db5d3e5ae5d552d4058a88a4e186748ab1d2)
+
+commit 3372060894a1da0adef6d36380a131902ca05c5f
+Merge: 1f00420 bff6747
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Dec 6 11:06:20 2014 -0800
+
+    Merge pull request #3088 from dachary/wip-10063-hobject-shard-giant
+    
+    common: do not omit shard when ghobject NO_GEN is set (giant)
+
+commit 1f004209434570337a3f90d7f89741f80dcc7075
+Merge: 3b65226 1ec557c
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 5 17:33:12 2014 -0800
+
+    Merge pull request #3095 from dachary/wip-9785-dmcrypt-keys-permissions-giant
+    
+    ceph-disk: dmcrypt file permissions (giant)
+
+commit 3b65226df806958f6a2f24df6099ee3a86d2a71f
+Merge: 691f011 36c7484
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 5 17:30:31 2014 -0800
+
+    Merge pull request #3006 from dachary/wip-9420-erasure-code-non-regression-giant
+    
+     erasure-code: store and compare encoded contents (giant)
+
+commit 1ec557c0eab94cb898ad3f5448482bd7afc53e09
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Dec 4 22:21:32 2014 +0100
+
+    ceph-disk: dmcrypt file permissions
+    
+    The directory in which key files are stored for dmcrypt must be 700 and
+    the file 600.
+    
+    http://tracker.ceph.com/issues/9785 Fixes: #9785
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 58682d1776ab1fd4daddd887d921ca9cc312bf50)
+
+commit 691f0118ecd051d5f3f61fc696280e3c482de3de
+Merge: 81295c5 dabf6f5
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 5 09:03:54 2014 -0800
+
+    Merge pull request #3085 from dachary/wip-10125-radosgw-init-giant
+    
+    rgw: run radosgw as apache with systemd (giant)
+
+commit bff67475c775914237604ed3374c8ccfe74d0ffd
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Nov 14 01:16:10 2014 +0100
+
+    common: do not omit shard when ghobject NO_GEN is set
+    
+    Do not silence the display of shard_id when generation is NO_GEN.
+    Erasure coded objects JSON representation used by ceph_objectstore_tool
+    need the shard_id to find the file containing the chunk.
+    
+    Minimal testing is added to ceph_objectstore_tool.py
+    
+    http://tracker.ceph.com/issues/10063 Fixes: #10063
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit dcf09aed121f566221f539106d10283a09f15cf5)
+
+commit dabf6f5f43b53a588bd9fa0cc5aa617ae8128735
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Dec 2 18:10:48 2014 +0100
+
+    rgw: run radosgw as apache with systemd
+    
+    Same as sysv.
+    
+    http://tracker.ceph.com/issues/10125 Fixes: #10125
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 7b621f4abf63456272dec3449aa108c89504a7a5)
+    
+    Conflicts:
+    	src/init-radosgw.sysv
+
+commit 81295c5ad2befced2e308c1cfb4e036cd5a825a9
+Merge: 8046359 3ff94ed
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Dec 4 11:32:01 2014 -0800
+
+    Merge pull request #3077 from ceph/wip-10030-giant
+    
+    librbd: don't close an already closed parent image upon failure
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit ad04a677cefc1f0a02fbff0c68409fda6874fdc7
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Dec 2 00:59:08 2014 +0100
+
+    common: add bufferlist::rebuild_aligned_size_and_memory
+    
+    The function bufferlist::rebuild_aligned checks memory and size
+    alignment with the same variable. It is however useful to separate
+    memory alignment constraints from size alignment constraints. For
+    instance rebuild_aligned could be called to allocate an erasure coded
+    buffer where each 2048 bytes chunk needs to start on a memory address
+    aligned on 32 bytes.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 9ade88e8dacc9b96c042bb668f4452447139a544)
+
+commit cc469b238f42ce989d0efa49154b95612e3d4111
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Dec 2 01:07:34 2014 +0100
+
+    erasure-code: enforce chunk size alignment
+    
+    Let say the ErasureCode::encode function is given a 4096 bytes
+    bufferlist made of a 1249 bytes bufferptr followed by a 2847 bytes
+    bufferptr, both properly starting on SIMD_ALIGN address. As a result the
+    second 2048 had to be reallocated when bufferlist::substr_of gets the
+    second 2048 buffer, the address starts at 799 bytes after the beginning
+    of the 2847 buffer ptr and is not SIMD_ALIGN'ed.
+    
+    The ErasureCode::encode must enforce a size alignment based on the chunk
+    size in addition to the memory alignment required by SIMD operations,
+    using the bufferlist::rebuild_aligned_size_and_memory function instead of
+    bufferlist::rebuild_aligned.
+    
+    http://tracker.ceph.com/issues/10211 Fixes: #10211
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 4e955f41297283798236c505c3d21bdcabb5caa0)
+
+commit 5205d4dacf7ebe2e42d2294bc30cb27f226c8d22
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Dec 2 02:04:14 2014 +0100
+
+    common: allow size alignment that is not a power of two
+    
+    Do not assume the alignment is a power of two in the is_n_align_sized()
+    predicate. When used in the context of erasure code it is common
+    for chunks to not be powers of two.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 73ad2d63d479b09037d50246106bbd4075fbce80)
+
+commit 80463596919d28f58010d16ad017b3c5ae6e558c
+Merge: 26e8cf1 3dc6298
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 3 23:02:43 2014 -0800
+
+    Merge pull request #3062 from ceph/wip-10123-giant
+    
+    librbd: protect list_children from invalid child pool IoCtxs
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 26e8cf174b8e76b4282ce9d9c1af6ff12f5565a9
+Merge: aac7946 7cd8c3f
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Wed Dec 3 06:44:56 2014 -0800
+
+    Merge pull request #3055 from ceph/wip-10135-giant
+    
+    mon: OSDMonitor: allow adding tiers to FS pools
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 7cd8c3f8a5afa9481b6f6a78d5fb8c04784ef4ca (refs/remotes/gh/wip-10135-giant)
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Nov 25 16:54:42 2014 +0000
+
+    mon: OSDMonitor: allow adding tiers to FS pools
+    
+    This was an overly-strict check.  In fact it is perfectly
+    fine to set an overlay on a pool that is already in use
+    as a filesystem data or metadata pool.
+    
+    Fixes: #10135
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 17b5fc9a40440e76dd1fa64f7fc19577ae3b58ce)
+
+commit 3ff94ed73ff27af2c8ea215ab693d815e285a27f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Nov 6 05:01:38 2014 -0500
+
+    librbd: don't close an already closed parent image upon failure
+    
+    If librbd is not able to open a child's parent image, it will
+    incorrectly close the parent image twice, resulting in a crash.
+    
+    Fixes: #10030
+    Backport: firefly, giant
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 61ebfebd59b61ffdc203dfeca01ee1a02315133e)
+
+commit aac794616580ed0bb00608c5867199b12d4e1d94
+Merge: 65f6814 c8b46d6
+Author: John Spray <jcspray@gmail.com>
+Date:   Tue Dec 2 11:35:59 2014 +0000
+
+    Merge pull request #2990 from ceph/wip-10151-giant
+    
+    mon: fix MDS health status from peons
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 3dc629822adeee961d78208b46b9bd7ef1200890
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 17 21:49:26 2014 -0500
+
+    librbd: protect list_children from invalid child pool IoCtxs
+    
+    While listing child images, don't ignore error codes returned
+    from librados when creating an IoCtx. This will prevent seg
+    faults from occurring when an invalid IoCtx is used.
+    
+    Fixes: #10123
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0d350b6817d7905908a4e432cd359ca1d36bab50)
+
+commit 65f6814847fe8644f5d77a9021fbf13043b76dbe
+Merge: 28e2708 9158326
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Mon Dec 1 17:59:19 2014 -0800
+
+    Merge pull request #3047 from ceph/wip-10011-giant
+    
+    osdc: fix Journaler write error handling [giant backport]
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 9158326eeb69312283a6e8174352f36ea30d0cbf
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Nov 6 11:46:29 2014 +0000
+
+    osdc: fix Journaler write error handling
+    
+    Since we started wrapping the write error
+    handler in a finisher, multiple calls to
+    handle_write_error would hit the assert()
+    on the second call before the actual
+    handler had been called (at the other end
+    of the finisher) from the first call.
+    
+    The symptom was that the MDS was intermittently
+    failing to respawn on blacklist, seen in #10011.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 762eda88a18ba707bd5410f38e21e95c4a6b3a46)
+
+commit 28e27080e25f95851039a0cc4e1c1d06b2cd597d
+Merge: 37ffccb cb1d681
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Nov 25 21:18:59 2014 -0800
+
+    Merge pull request #3005 from dachary/wip-9665-ceph-disk-partprobe-giant
+    
+    ceph disk zap must call partprobe
+
+commit e7faed5d903cf7681d77a6af53cf8137eeb2fc69
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Thu Nov 6 17:11:20 2014 +0100
+
+    osd: deep scrub must not abort if hinfo is missing
+    
+    Instead it should set read_error.
+    
+    http://tracker.ceph.com/issues/10018 Fixes: #10018
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 9d84d2e8309d26e39ca849a75166d2d7f2dec9ea)
+
+commit 36c7484c18fd072ba2f7b176403414dd32fbe92b
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Thu Sep 25 14:46:07 2014 +0200
+
+    erasure-code: erasure_code_benchmark exhaustive erasure exploration
+    
+    Add the --erasure-generation exhaustive flag to try all combinations of
+    erasures, not just one at random.
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 2d7adb23bc52e7c0753f4571fecd8eefa209ef02)
+    
+    Conflicts:
+    	src/test/erasure-code/ceph_erasure_code_benchmark.h
+
+commit 32daa9b0f4d39f8a49512b18d5c19437aca5fec6
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Mon Sep 29 11:17:13 2014 +0200
+
+    erasure-code: add erasure_code_benchmark --verbose
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 3ff2816b3eecfb7277295583387549dac5429628)
+    
+    Conflicts:
+    	src/test/erasure-code/ceph_erasure_code_benchmark.cc
+    	src/test/erasure-code/ceph_erasure_code_benchmark.h
+
+commit da9a7f07787d7f8c20b0c3e7a53fcaf95ed7ca20
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Tue Sep 23 14:37:57 2014 +0200
+
+    erasure_code: implement ceph_erasure_code to assert the existence of a plugin
+    
+    This is handy when scripting in the context of teuthology and only
+    conditionally run tests for the isa plugin, for instance.
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit efe121d9f2028c312eef2650d32ccf0cbc828edb)
+
+commit c855f3958fb8c10bd824075c1739f40799f6d74b
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Tue Sep 23 14:36:08 2014 +0200
+
+    erasure-code: ceph_erasure_code does not need to avoid dlclose
+    
+    The only reason for not dlclosing plugins at exit is for callgrind but
+    ceph_erasure_code has no workload that would require callgrind.
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 49613cb2aab6e73e3ea50fa164735b55e80121cd)
+
+commit ba8ceb1f067e0f9f6419358435ed0008b61fa438
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Tue Sep 23 11:38:09 2014 +0200
+
+    erasure-code: add corpus verification to make check
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 6fdbdff2ad1b55d4a37dcb95cfbb06c4454cdaf2)
+
+commit ca4c2702139cc7fd8f2e3fa2ee5cda4094ecad79
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Sat Sep 13 12:58:27 2014 +0200
+
+    erasure-code: Makefile.am cosmetics
+    
+    Cluster benchmark related lines together.
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 10c88c8f27080a8e25f128b7065cee5c2f68e91b)
+
+commit 208a5ee1c9975adaa8b09b1bf541aff0d8551c63
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Sat Sep 13 12:55:26 2014 +0200
+
+    erasure-code: s/alignement/alignment/ typos in jerasure
+    
+    The jerasure-per-chunk-alignment prameter was mispelled and while
+    useable that would lead to confusion.
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 2c84d0b1db57d918840e669a17bbd8c5ddca9747)
+
+commit 1def82d530965bd0441e4f7f6aa032666984f17d
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Sat Sep 13 13:36:09 2014 +0200
+
+    erasure-code: workunit to check for encoding regression
+    
+    Clone the archive of encoded objects and decode all archived objects, up
+    to and including the current ceph version.
+    
+    http://tracker.ceph.com/issues/9420 Refs: #9420
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 7638b15f23976c3265cf766e16cf93af1a7e0091)
+
+commit 1b7fc7e49e6edf0d0f7d1d6d9f9447c42067d8b8
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Sat Sep 13 10:16:31 2014 +0200
+
+    erasure-code: store and compare encoded contents
+    
+    Introduce ceph_erasure_code_non_regression to check and compare how an
+    erasure code plugin encodes and decodes content with a given set of
+    parameters. For instance:
+    
+    ./ceph_erasure_code_non_regression \
+          --plugin jerasure \
+          --parameter technique=reed_sol_van \
+          --parameter k=2 \
+          --parameter m=2 \
+          --stripe-width 3181 \
+          --create \
+          --check
+    
+    Will create an encoded object (--create) and store it into a directory
+    along with the chunks, one chunk per file. The directory name is derived
+    from the parameters. The content of the object is a random pattern of 31
+    bytes repeated to fill the object size specified with --stripe-width.
+    
+    The check function (--check) reads the object back from the file,
+    encodes it and compares the result with the content of the chunks read
+    from the files. It also attempts recover from one or two erasures.
+    
+    Chunks encoded by a given version of Ceph are expected to be encoded
+    exactly in the same way by all Ceph versions going forward.
+    
+    http://tracker.ceph.com/issues/9420 Refs: #9420
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit f5901303dbf50e9d08f2f1e510a1936a20037909)
+
+commit cb1d68113477cf9c2028a65372d2d4a3e6a8bdc1
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Thu Oct 9 18:52:17 2014 +0200
+
+    ceph-disk: run partprobe after zap
+    
+    Not running partprobe after zapping a device can lead to the following:
+    
+    * ceph-disk prepare /dev/loop2
+    * links are created in /dev/disk/by-partuuid
+    * ceph-disk zap /dev/loop2
+    * links are not removed from /dev/disk/by-partuuid
+    * ceph-disk prepare /dev/loop2
+    * some links are not created in /dev/disk/by-partuuid
+    
+    This is assuming there is a bug in the way udev events are handled by
+    the operating system.
+    
+    http://tracker.ceph.com/issues/9665 Fixes: #9665
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit fed3b06c47a5ef22cb3514c7647544120086d1e7)
+
+commit d9c04b880d8bf867aa454132117119be5bd550ad
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Fri Oct 10 10:26:31 2014 +0200
+
+    ceph-disk: use update_partition in prepare_dev and main_prepare
+    
+    In the case of prepare_dev the partx alternative was missing and is not
+    added because update_partition does it.
+    
+    http://tracker.ceph.com/issues/9721 Fixes: #9721
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 23e71b1ee816c0ec8bd65891998657c46e364fbe)
+
+commit 5c9cd3c2a292ae540fd0a487defaf4d712e41e62
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Fri Oct 10 10:23:34 2014 +0200
+
+    ceph-disk: encapsulate partprobe / partx calls
+    
+    Add the update_partition function to reduce code duplication.
+    The action is made an argument although it always is -a because it will
+    be -d when deleting a partition.
+    
+    Use the update_partition function in prepare_journal_dev
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 922a15ea6865ef915bbdec2597433da6792c1cb2)
+
+commit c8b46d68c71f66d4abbda1230741cc4c7284193b
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Nov 24 11:00:25 2014 +0000
+
+    mon: fix MDS health status from peons
+    
+    The health data was there, but we were attempting
+    to enumerate MDS GIDs from pending_mdsmap (empty on
+    peons) instead of mdsmap (populated from paxos updates)
+    
+    Fixes: #10151
+    Backport: giant
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 0c33930e3a90f3873b7c7b18ff70dec2894fce29)
+    
+    Conflicts:
+    	src/mon/MDSMonitor.cc
+
+commit 37ffccbd57e7e441e0eb1499e5c173aa9c375d35
+Merge: b13a56a 65c5657
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Nov 20 13:13:33 2014 -0800
+
+    Merge pull request #2975 from ceph/wip-9936-giant
+    
+    rbd: Fix the rbd export when image size more than 2G
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit b13a56afe99c091600392a2fc15befa9438d59c9
+Merge: 828c1a2 46bd344
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Wed Nov 19 02:40:47 2014 +0100
+
+    Merge pull request #2963 from ceph/wip-10114-giant
+    
+    Wip 10114 giant
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 828c1a2bcd81a49264f8a81ca7f1070169037820
+Merge: ccfd241 6cb9a24
+Author: David Zafman <david.zafman@inktank.com>
+Date:   Tue Nov 18 15:48:16 2014 -0800
+
+    Merge pull request #2958 from ceph/wip-10128-giant
+    
+    ceph_objectstore_tool: When exporting to stdout, don't cout messages
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 46bd3441b00c22ba78f04617fd77f0231ccc698d
+Author: Dan Mick <dan.mick@redhat.com>
+Date:   Tue Nov 18 15:21:30 2014 -0800
+
+    erasure-code isa-l: remove duplicated lines (fix warning)
+    
+    06a245a added a section def to assembly files; I added it twice to
+    this file.  There's no damage, but a compiler warning (on machines with
+    yasm installed)
+    
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    (cherry picked from commit 10f6ef185a9d09e396e94036ec90bfe8a0738ce9)
+
+commit 1bba3887038aade137a808d751561cc02002f4bf
+Author: Dan Mick <dan.mick@redhat.com>
+Date:   Fri Nov 14 17:59:57 2014 -0800
+
+    Add annotation to all assembly files to turn off stack-execute bit
+    
+    See discussion in http://tracker.ceph.com/issues/10114
+    
+    Building with these changes allows output from readelf like this:
+    
+     $ readelf -lW src/.libs/librados.so.2 | grep GNU_STACK
+      GNU_STACK      0x000000 0x0000000000000000 0x0000000000000000 0x000000
+    0x000000 RW  0x8
+    
+    (note the absence of 'X' in 'RW')
+    
+    Fixes: #10114
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    (cherry picked from commit 06a245a9845c0c126fb3106b41b2fd2bc4bc4df3)
+
+commit 6cb9a2499cac2645e2cc6903ab29dfd95aac26c7
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Nov 17 23:02:50 2014 -0800
+
+    ceph_objectstore_tool: When exporting to stdout, don't cout messages
+    
+    Fixes: #10128
+    Caused by a2bd2aa7
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0d5262ac2f69ed3996af76a72894b1722a27b37d)
+
+commit 13bb880b2a04ab354506eb183d2907b9054bf937
+Author: Sebastien Ponce <sebastien.ponce@cern.ch>
+Date:   Tue Nov 18 10:30:36 2014 +0100
+
+    Fixed locking issue in the trun method of libradosstriper leading to potential race conditions - Fixes: #10129
+    
+    Signed-off-by: Sebastien Ponce <sebastien.ponce@cern.ch>
+    (cherry picked from commit 8613984373de946e1815cc84d50bbd4437a3f7a7)
+
+commit 65c565701eb6851f4ed4d2dbc1c7136dfaad6bcb
+Author: Vicente Cheng <freeze.bilsted@gmail.com>
+Date:   Wed Oct 29 12:21:11 2014 +0800
+
+    rbd: Fix the rbd export when image size more than 2G
+    
+    When using export <image-name> <path> and the size of image is more
+    than 2G, the previous version about finish() could not handle in
+    seeking the offset in image and return error.
+    
+    This is caused by the incorrect variable type. Try to use the correct
+    variable type to fixed it.
+    
+    I use another variable which type is uint64_t for confirming seeking
+    and still use the previous r for return error.
+    
+    uint64_t is more better than type int for handle lseek64().
+    
+    Signed-off-by: Vicente Cheng <freeze.bilsted@gmail.com>
+    (cherry picked from commit 4b87a81c86db06f6fe2bee440c65fc05cd4c23ce)
+
+commit ccfd2414c68afda55bf4cefa2441ea6d53d87cc6
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 12 17:11:10 2014 -0800
+
+    osd/OSD: use OSDMap helper to determine if we are correct op target
+    
+    Use the new helper.  This fixes our behavior for EC pools where targetting
+    a different shard is not correct, while for replicated pools it may be. In
+    the EC case, it leaves the op hanging indefinitely in the OpTracker because
+    the pgid exists but as a different shard.
+    
+    Fixes: #9835
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9e05ba086a36ae9a04b347153b685c2b8adac2c3)
+
+commit 963947718a954f63f351ce4034bf97380421ab7c
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 12 17:04:35 2014 -0800
+
+    osd/OSDMap: add osd_is_valid_op_target()
+    
+    Helper to check whether an osd is a given op target for a pg.  This
+    assumes that for EC we always send ops to the primary, while for
+    replicated we may target any replica.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 89c02637914ac7332e9dbdbfefc2049b2b6c127d)
+
+commit 0cb32c157c6c11b26607521a20c6f320c5170516
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Tue Nov 11 18:16:02 2014 -0800
+
+    qa: allow small allocation diffs for exported rbds
+    
+    The local filesytem may behave slightly differently. This isn't
+    foolproof, but seems to be reliable enough on rhel7 rootfs, where
+    exact comparison was failing.
+    
+    Fixes: #10002
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit e94d3c11edb9c9cbcf108463fdff8404df79be33)
+
+commit fe705c8fdef2371d3f5b11eb73f87a0cf6ef0f9e
+Author: Adam Crume <adamcrume@gmail.com>
+Date:   Thu Sep 18 16:57:27 2014 -0700
+
+    common: Add cctid meta variable
+    
+    Fixes: #6228
+    Signed-off-by: Adam Crume <adamcrume@gmail.com>
+    (cherry picked from commit bb45621cb117131707a85154292a3b3cdd1c662a)
+
+commit 5fc659a0d52094a4c595ca8b33b407ecdefc180a
+Merge: b27f5db a6c02a1
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Nov 11 08:28:19 2014 -0800
+
+    Merge pull request #2804 from ceph/wip-9301-giant
+    
+    mon: backport paxos off-by-one bug (9301) to giant
+
+commit b27f5dba8677ca48c9819980e3c90b76f5f04267
+Merge: 97e423f fc5354d
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Mon Nov 10 22:41:19 2014 -0800
+
+    Merge pull request #2887 from ceph/wip-9977-backport
+    
+    tools: skip up to expire_pos in journal-tool
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 97e423f52155e2902bf265bac0b1b9ed137f8aa0
+Author: Yan, Zheng <ukernel@gmail.com>
+Date:   Thu Sep 11 09:36:44 2014 +0800
+
+    client: trim unused inodes before reconnecting to recovering MDS
+    
+    So the recovering MDS does not need to fetch these ununsed inodes during
+    cache rejoin. This may reduce MDS recovery time.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 2bd7ceeff53ad0f49d5825b6e7f378683616dffb)
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 387efc5fe1fb148ec135a6d8585a3b8f8d97dbf8
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Oct 27 12:02:17 2014 +0000
+
+    client: allow xattr caps in inject_release_failure
+    
+    Because some test environments generate spurious
+    rmxattr operations, allow the client to release
+    'X' caps.  Allows xattr operations to proceed
+    while still preventing client releasing other caps.
+    
+    Fixes: #9800
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 5691c68a0a44eb2cdf0afb3f39a540f5d42a5c0c)
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit fc5354dec55248724f8f6b795e3a96882c33b490 (refs/remotes/gh/wip-9977-backport)
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Nov 3 19:19:45 2014 +0000
+
+    tools: skip up to expire_pos in journal-tool
+    
+    Previously worked for journals starting from an
+    object boundary (i.e. freshly created filesystems)
+    
+    Fixes: #9977
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 65c33503c83ff8d88781c5c3ae81d88d84c8b3e4)
+    
+    Conflicts:
+    	src/tools/cephfs/JournalScanner.cc
+
+commit 9680613141b3eef62f35a6728e654efa5f6ba8e8
+Merge: fd4363d a5984ba
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Fri Nov 7 16:26:54 2014 -0800
+
+    Merge pull request #2876 from ceph/giant-readdir-fix
+    
+    Giant readdir fix
+
+commit fd4363d1bd49f73e1b3c22516686c7b7e1745b57
+Merge: f66bf31 7166ff8
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Fri Nov 7 14:10:40 2014 -0800
+
+    Merge pull request #2879 from ceph/wip-10025-giant
+    
+    #10025/giant -- tools: fix MDS journal import
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 7166ff83f6343d31d52a58363e2767434554505c
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Nov 7 11:34:43 2014 +0000
+
+    tools: fix MDS journal import
+    
+    Previously it only worked on fresh filesystems which
+    hadn't been trimmed yet, and resulted in an invalid
+    trimmed_pos when expire_pos wasn't on an object
+    boundary.
+    
+    Fixes: #10025
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit fb29e71f9a97c12354045ad2e128156e503be696)
+
+commit a5984ba34cb684dae623df22e338f350c8765ba5
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Oct 27 13:57:16 2014 -0700
+
+    client: fix I_COMPLETE_ORDERED checking
+    
+    Current code marks a directory inode as complete and ordered when readdir
+    finishes, but it does not check if the directory was modified in the middle
+    of readdir. This is wrong, directory inode should not be marked as ordered
+    if it was modified during readddir
+    
+    The fix is introduce a new counter to the inode data struct, we increase
+    the counter each time the directory is modified. When readdir finishes, we
+    check the counter to decide if the directory should be marked as ordered.
+    
+    Fixes: #9894
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit a4caed8a53d011b214ab516090676641f7c4699d)
+
+commit b5ff4e99c87958211e4b7716b59084fc3417ec17
+Author: Yan, Zheng <ukernel@gmail.com>
+Date:   Tue Sep 9 17:34:46 2014 +0800
+
+    client: preserve ordering of readdir result in cache
+    
+    Preserve ordering of readdir result in a list, so that the result of cached
+    readdir is consistant with uncached readdir.
+    
+    As a side effect, this commit also removes the code that removes stale dentries.
+    This is OK because stale dentries does not have valid lease, they will be
+    filter out by the shared gen check in Client::_readdir_cache_cb()
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 346c06c1647658768e927a47768a0bc74de17b53)
+
+commit 0671c1127015311d9894f15e2493805f93432910
+Author: Yan, Zheng <ukernel@gmail.com>
+Date:   Tue Sep 9 14:06:06 2014 +0800
+
+    client: introduce a new flag indicating if dentries in directory are sorted
+    
+    When creating a file, Client::insert_dentry_inode() set the dentry's offset
+    based on directory's max offset. The offset does not reflect the real
+    postion of the dentry in directory. Later readdir reply from real postion
+    of the dentry in directory. Later readdir reply from MDS may change the
+    dentry's position/offset. This inconsistency can cause missing/duplicate
+    entries in readdir result if readdir is partly satisfied by dcache_readdir().
+    
+    The fix is introduce a new flag indicating if dentries in directory are
+    sorted. We use _readdir_cache_cb() to handle readdir only when the flag is
+    set, clear the flag after creating/deleting/renaming file.
+    
+    Fixes: #9178
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 600af25493947871c38214aa370e2544a7fea399)
+
+commit f66bf31b6743246fb1c88238cf18101238dee3a4
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Thu Nov 6 17:48:01 2014 -0800
+
+    qa: use sudo even more when rsyncing /usr
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 3aa7797741f9cff06053a2f31550fe6929039692)
+
+commit f7ec448d6579f965eec184416a97b47ae27ab47a
+Merge: f410d76 f111bc8
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Wed Nov 5 08:51:18 2014 +0100
+
+    Merge pull request #2858 from ceph/wip-9909
+    
+    tools: rados put /dev/null should write() and not create()
+    
+    Reviewed-by: Loic Dachary <loic-201408@dachary.org>
+
+commit f111bc8eac8a521b13340f4a75418d839725e010
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Thu Oct 2 09:23:55 2014 +0200
+
+    tools: rados put /dev/null should write() and not create()
+    
+    In the rados.cc special case to handle put an empty objects, use
+    write_full() instead of create().
+    
+    A special case was introduced 6843a0b81f10125842c90bc63eccc4fd873b58f2
+    to create() an object if the rados put file is empty. Prior to this fix
+    an attempt to rados put an empty file was a noop. The problem with this
+    fix is that it is not idempotent. rados put an empty file twice would
+    fail the second time and rados put a file with one byte would succeed as
+    expected.
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit 50e80407f3c2f74d77ba876d01e7313c3544ea4d)
+
+commit f410d764d2e6795389cb320b4436cff3607927bd
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Oct 9 10:20:27 2014 -0700
+
+    rgw: set length for keystone token validation request
+    
+    Fixes: #7796
+    Backport: giany, firefly
+    Need to set content length to this request, as the server might not
+    handle a chunked request (even though we don't send anything).
+    
+    Tested-by: Mark Kirkwood <mark.kirkwood@catalyst.net.nz>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 3dd4ccad7fe97fc16a3ee4130549b48600bc485c)
+
+commit dba7defc623474ad17263c9fccfec60fe7a439f0
+Merge: 6a201f8 e0b0441
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 31 08:35:42 2014 -0700
+
+    Merge pull request #2846 from dachary/wip-9752-past-intervals-giant
+    
+    osd: past_interval display bug on acting
+
+commit e0b04414b92018277a0d3b9d82e72ea7529f4ef5
+Author: Loic Dachary <loic-201408@dachary.org>
+Date:   Fri Oct 31 00:49:21 2014 +0100
+
+    osd: past_interval display bug on acting
+    
+    The acting array was incorrectly including the primary and up_primary.
+    
+    http://tracker.ceph.com/issues/9752 Fixes: #9752
+    
+    Signed-off-by: Loic Dachary <loic-201408@dachary.org>
+    (cherry picked from commit c5f8d6eded52da451fdd1d807bd4700221e4c41c)
+
+commit 6a201f89b1aa6c2197383c29919cdeb4a8353d1b
+Merge: ebe1637 905aba2
+Author: Yan, Zheng <ukernel@gmail.com>
+Date:   Thu Oct 30 17:01:12 2014 -0700
+
+    Merge pull request #2841 from ceph/giant-9869
+    
+    Backport "client: cast m->get_client_tid() to compare to 16-bit Inode::flushing_cap_tid"
+
+commit 905aba2f3d847933f98124f3ea8d1d76d644edb4
+Author: Greg Farnum <greg@inktank.com>
+Date:   Wed Oct 22 17:16:31 2014 -0700
+
+    client: cast m->get_client_tid() to compare to 16-bit Inode::flushing_cap_tid
+    
+    m->get_client_tid() is 64 bits (as it should be), but Inode::flushing_cap_tid
+    is only 16 bits. 16 bits should be plenty to let the cap flush updates
+    pipeline appropriately, but we need to cast in the proper direction when
+    comparing these differently-sized versions. So downcast the 64-bit one
+    to 16 bits.
+    
+    Fixes: #9869
+    Backport: giant, firefly, dumpling
+    
+    Signed-off-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit a5184cf46a6e867287e24aeb731634828467cd98)
+
+commit ebe16373e966917ca8cb03ebeac974bdff7b7685
+Merge: c51c8f9 b704f0d
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 30 10:05:22 2014 -0700
+
+    Merge pull request #2838 from ceph/wip-9945-giant
+    
+    messages: fix COMPAT_VERSION on MClientSession
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit b704f0dd888aacb10c32cdb63cdbf9f06296fc18
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Oct 30 16:43:21 2014 +0000
+
+    messages: fix COMPAT_VERSION on MClientSession
+    
+    This was incorrectly incremented to 2 by omission
+    of an explicit COMPAT_VERSION value.
+    
+    Fixes: #9945
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 1eb9bcb1d36014293efc687b4331be8c4d208d8e)
+
+commit a6c02a18d6249ea62cf7a74710c8d0192b6eecaa
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 18 14:23:36 2014 -0700
+
+    mon: re-bootstrap if we get probed by a mon that is way ahead
+    
+    During bootstrap we verify that our paxos commits overlap with the other
+    mons we will form a quorum with.  If they do not, we do a sync.
+    
+    However, it is possible we pass those checks, then fail to join a quorum
+    before the quorum moves ahead in time such that we no longer overlap.
+    Currently nothing kicks up back into a probing state to discover we need
+    to sync... we will just keep trying to call or join an election instead.
+    
+    Fix this by jumping back to bootstrap if we get a probe that is ahead of
+    us.  Only do this from non probe or sync states as these will be common;
+    it is only the active and electing states that matter (and probably just
+    electing!).
+    
+    Fixes: #9301
+    Backport: giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c421b55e8e15ef04ca8aeb47f7d090375eaa8573)
+
+commit 92d2a38efd458f9e8f4da228ea1e94df08dc8222
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 18 14:11:24 2014 -0700
+
+    mon/Paxos: fix off-by-one in last_ vs first_committed check
+    
+    peon last_committed + 1 == leader first_committed is okay.  Note that the
+    other check (where I clean up whitespace) gets this correct.
+    
+    Fixes: #9301 (partly)
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d81cd7f86695185dce31df76c33c9a02123f0e4a)
diff --git a/doc/changelog/v0.87.2.txt b/doc/changelog/v0.87.2.txt
new file mode 100644
index 000000000..98cf11732
--- /dev/null
+++ b/doc/changelog/v0.87.2.txt
@@ -0,0 +1,1413 @@
+commit 87a7cec9ab11c677de2ab23a7668a77d2f5b955e (tag: refs/tags/v0.87.2, refs/remotes/gh/giant)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Fri Apr 24 12:31:27 2015 -0700
+
+    0.87.2
+
+commit c1301e84aee0f399db85e2d37818a66147a0ce78
+Merge: 1a13e10 9e9c3c6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Apr 7 21:08:24 2015 +0200
+
+    Merge pull request #4214 from dachary/wip-10430-giant
+    
+    osd/osd_types.cc: 456: FAILED assert(m_seed < old_pg_num)
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 1a13e1065829c59987c2f57a13eaa03de31df4ed
+Merge: 1fb08d3 5f4e62f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Apr 7 16:39:28 2015 +0200
+
+    Merge pull request #4258 from ceph/wip-10643-v2
+    
+    mon: MDSMonitor: additional backports for #10643
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 1fb08d3066b14b178a8912ffb3c9f50d2333738c
+Merge: 90b37d9 7684ee2
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Apr 2 08:55:48 2015 -0700
+
+    Merge pull request #4261 from ceph/wip-11303-giant
+    
+    allow -L to disable lttng. Enable it by default
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 7684ee25ac21810153a44bdc4fc00b36e39eb12f
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Apr 2 10:34:02 2015 -0400
+
+    allow -L to disable lttng. Enable it by default
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+
+commit 5f4e62f382767ee69e5b0c701b1a01d9e4132237
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Fri Oct 17 19:08:20 2014 +0100
+
+    mon: MDSMonitor: wait for osdmon to be writable when requesting proposal
+    
+    Otherwise we may end up requesting the osdmon to propose while it is
+    mid-proposal.  We can't simply return EAGAIN to the user either because
+    then we would have to expect the user to be able to successfully race
+    with the whole cluster in finding a window in which 'mds fs new' command
+    would succeed -- which is not a realistic expectation.  Having the
+    command to osdmon()->wait_for_writable() guarantees that the command
+    will be added to a queue and that we will, eventually, tend to it.
+    
+    Fixes: #9794
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 2ae1cba595d9c56a0a4c534b34fe25250e7eb2d5)
+
+commit 257bd17db6470ca050403b1c8ff8daa94a4b80b5
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Fri Oct 17 18:59:51 2014 +0100
+
+    mon: MDSMonitor: have management_command() returning int instead of bool
+    
+    We can more easily differentiate between errors we get out of this
+    function, which makes the code a bit more versatile and readable.
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 0dd473cbad4f9ea403fe60badffdc6da4dd3aa3c)
+
+commit 9e9c3c652339d85863af01cac621228f04eb4f18
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Oct 9 11:20:13 2014 -0700
+
+    osd: Get pgid ancestor from last_map when building past intervals
+    
+    Fixed OSD::build_past_intervals_parallel() and PG::generate_past_intervals()
+    
+    Fixes: #10430
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0c5b66da7a9ba516340d06d9e806beb9d1040d0e)
+
+commit 90b37d9bdcc044e26f978632cd68f19ece82d19a
+Merge: 2ccbc14 9f1f355
+Author: Loic Dachary <loic-redhat@dachary.org>
+Date:   Thu Mar 26 07:58:14 2015 +0100
+
+    Merge pull request #4175 from wonzhq/objecter-timer-2
+    
+    Objecter: failed assert(tick_event==NULL) at osdc/Objecter.cc
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 9f1f35546e00e8f1ecbce0697d59b64f3537facf
+Author: Zhiqiang Wang <zhiqiang.wang@intel.com>
+Date:   Wed Mar 25 16:32:44 2015 +0800
+
+    Objecter: failed assert(tick_event==NULL) at osdc/Objecter.cc
+    
+    When the Objecter timer erases the tick_event from its events queue and
+    calls tick() to dispatch it, if the Objecter::rwlock is held by shutdown(),
+    it waits there to get the rwlock. However, inside the shutdown function,
+    it checks the tick_event and tries to cancel it. The cancel_event function
+    returns false since tick_event is already removed from the events queue. Thus
+    tick_event is not set to NULL in shutdown(). Later the tick function return
+    ealier and doesn't set tick_event to NULL as well. This leads to the assertion
+    failure.
+    
+    This is a regression introduced by an incorrect conflict resolution when
+    d790833 was backported.
+    
+    Fixes: #11183
+    
+    Signed-off-by: Zhiqiang Wang <zhiqiang.wang@intel.com>
+
+commit 2ccbc14d17b54ea4fd4126cb04a7b83cd64c7f1e
+Merge: 02f9cdb de4b087
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 23 20:39:26 2015 +0100
+
+    Merge pull request #4127 from dzafman/wip-11176-giant
+    
+    ceph-objectstore-tool: Output only unsupported features when incomatible
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 02f9cdbf889071ca6fe3811d9b9a92a0b630fa55
+Merge: 83bcc51 fc43d8c
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:11:46 2015 +0100
+
+    Merge pull request #4097 from dachary/wip-10497-giant
+    
+    librados: c api does not translate op flag
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 83bcc516743e426c7a8c6f6401721bffbbec4fc0
+Merge: ebab2bd d790833
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:11:26 2015 +0100
+
+    Merge pull request #4096 from dachary/wip-9617-giant
+    
+    objecter shutdown races with msg dispatch
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit ebab2bd5f36205d666673600624aaa3e5e06c405
+Merge: e31c92d 970a797
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:11:03 2015 +0100
+
+    Merge pull request #4095 from dachary/wip-9675-giant
+    
+    splitting a pool doesn't start when rule_id != ruleset_id
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit e31c92d8dd814d276357e431ed87b93d75933f77
+Merge: f0ec5e3 7653511
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:10:42 2015 +0100
+
+    Merge pull request #4094 from dachary/wip-9891-giant
+    
+    Assertion: os/DBObjectMap.cc: 1214: FAILED assert(0)
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit f0ec5e3b3ed58af65323bcc494e589935147aa45
+Merge: dd7c15b 13b0147
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:10:25 2015 +0100
+
+    Merge pull request #4093 from dachary/wip-9915-giant
+    
+    osd: eviction logic reversed
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit dd7c15b2b24027a7cc5fa4bff21222c5a4606e60
+Merge: 33b09e1 13b8364
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:09:28 2015 +0100
+
+    Merge pull request #4092 from dachary/wip-9985-giant
+    
+    osd: incorrect atime calculation
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 33b09e18aa78252d94cbec8bf94ec97ed5bb1573
+Merge: 950123e 5550cdd
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:08:41 2015 +0100
+
+    Merge pull request #4091 from dachary/wip-9986-giant
+    
+    objecter: map epoch skipping broken
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 950123e4daa85562b2f52e0e12e0bae07f444095
+Merge: abdc065 21f81b8
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:08:16 2015 +0100
+
+    Merge pull request #4090 from dachary/wip-10059-giant
+    
+    osd/ECBackend.cc: 876: FAILED assert(0)
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit abdc065acd6cafa6439e9c1724cc87de02352bd7
+Merge: aee2825 1ccf583
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:07:52 2015 +0100
+
+    Merge pull request #4089 from dachary/wip-10080-giant
+    
+    Pipe::connect() cause osd crash when osd reconnect to its peer
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit aee28250fee66bca08d91e56ce47ec46c2e9fc24
+Merge: 6582253 3e875ab
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:07:20 2015 +0100
+
+    Merge pull request #4088 from dachary/wip-6003-giant
+    
+    journal Unable to read past sequence 406 ...
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 65822530ebfdea2feb9192c6eb6e3b8b9d60fe33
+Merge: 4b20f2d 96a5c67
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:06:51 2015 +0100
+
+    Merge pull request #4082 from dachary/wip-10106-giant
+    
+    rgw acl response should start with <?xml version=1.0 ?>
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 4b20f2d2d1ee52deed33617f000fa342ebce2e49
+Merge: 7ff3a67 c7b02f5
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:06:23 2015 +0100
+
+    Merge pull request #4078 from dachary/wip-11157-giant
+    
+    doc,tests: force checkout of submodules
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 7ff3a67c44ba3dc20a663a7dc6ba28c25714f063
+Merge: 440e706 4d4eb9f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:05:20 2015 +0100
+
+    Merge pull request #4077 from dachary/wip-10150-giant
+    
+    osd/ReplicatedPG.cc: 10853: FAILED assert(r >= 0) (in _scan_range)
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 440e70607e7e3cd1d8ca33843c626109431caf8d
+Merge: 66f639b 499d94f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:04:51 2015 +0100
+
+    Merge pull request #4076 from dachary/wip-10153-giant
+    
+    Rados.shutdown() dies with Illegal instruction (core dumped)
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 66f639b30ce6c74caae9397f20053761203f8e87
+Merge: 43b45df b79852f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:04:25 2015 +0100
+
+    Merge pull request #4074 from dachary/wip-10220-giant
+    
+    mon/Paxos.cc: 1033: FAILED assert(mon->is_leader())
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 43b45dfacd598bab51fa06c5d0e2d0605d6e83d6
+Merge: d282cfd b318e2f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 23:03:35 2015 +0100
+
+    Merge pull request #3548 from ceph/wip-10643
+    
+    mon: MDSMonitor: missing backports for giant
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit d282cfd5ae3e65b74801cd27480ce8c0bd72d9a7
+Merge: 3f3b981 681c99f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 22:12:58 2015 +0100
+
+    Merge pull request #4053 from dachary/wip-8011-giant
+    
+    osd/ReplicatedPG.cc: 5244: FAILED assert(soid < scrubber.start || soid >= scrubber.end)
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 3f3b98123cb46080068c1c73f4be41acbe18bd0d
+Merge: 6919eb1 4427358
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 22:12:40 2015 +0100
+
+    Merge pull request #4052 from dachary/wip-10844-giant
+    
+    mon: caps validation should rely on EntityName instead of entity_name_t
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 6919eb1684e34a6395963be6cc65215a51f5ba13
+Merge: 8876585 1d4ffbe
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 22:12:15 2015 +0100
+
+    Merge pull request #4050 from dachary/wip-10817-giant
+    
+    WorkQueue: make timeout when calling WaitInterval configurable
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 88765851bd56a6a526f6ab724920c8858ec4956c
+Merge: 6da3171 92c352d
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 22:11:43 2015 +0100
+
+    Merge pull request #4049 from dachary/wip-10787-giant
+    
+    mon: OSDMonitor::map_cache is buggy, send_incremental is not conservative
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 6da3171f21b8e1a56fe941a5028f2ccfdccee18a
+Merge: 5a6eefc 25fcaca
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 22:09:28 2015 +0100
+
+    Merge pull request #4048 from dachary/wip-10770-giant
+    
+    rgw: pending bucket index operations are not cancelled correctly
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 5a6eefcb40c94da9a900d9893ecb6eaaf8fd0cea
+Merge: c67a7a5 2858327
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 22:09:06 2015 +0100
+
+    Merge pull request #4046 from dachary/wip-10723-giant
+    
+    rados python binding leaks Ioctx objects
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit c67a7a52dc96f176431125921e36e4a2b8a30f1c
+Merge: 41dcd2d d5b1b7e
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 22:08:45 2015 +0100
+
+    Merge pull request #4044 from dachary/wip-10617-giant
+    
+    osd: pgs for deleted pools don't finish getting removed if osd restarts
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 41dcd2d9c307dd0b25d27bd6673943f3fdaaa28b
+Merge: 42e7413 c3d998e
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 22:08:20 2015 +0100
+
+    Merge pull request #4034 from dachary/wip-10475-giant
+    
+    rgw: Swift API. Support for X-Remove-Container-Meta-{key} header.
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit 42e741339913990521c4509f4e266921f8dbe007
+Merge: ffb76d1 fa8d454
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 22 22:07:53 2015 +0100
+
+    Merge pull request #4033 from dachary/wip-10471-giant
+    
+    rgw: index swift keys appropriately
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+
+commit de4b08704172ac31b511dde50e5c11d58d811ca2
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Mar 20 16:57:40 2015 -0700
+
+    ceph-objectstore-tool: Output only unsupported features when incomatible
+    
+    Fixes: #11176
+    Backport: firefly, giant
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 5b23f5b5892b36fb7d06efc0d77e64a24ef6e8c9)
+
+commit ffb76d16c5d9d65ac94d21b4e8fc1fdf86441977
+Merge: fea29b1 61d6006
+Author: John Spray <jcspray@gmail.com>
+Date:   Thu Mar 19 22:07:58 2015 +0000
+
+    Merge pull request #3971 from ceph/giant-11053
+    
+    mds: fix assertion caused by system clock backwards
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit c3d998e4c3c9d90b50c3f6c0af7d48785616e032
+Author: Dmytro Iurchenko <diurchenko@mirantis.com>
+Date:   Tue Feb 3 17:54:38 2015 +0200
+
+    rgw: Swift API. Support for X-Remove-Container-Meta-{key} header.
+    
+    Fixes: #10475
+    Backport: hammer, firefly
+    Reported-by: Josh Durgin <jdurgin@redhat.com>
+    Signed-off-by: Dmytro Iurchenko <diurchenko@mirantis.com>
+    (cherry picked from commit f67bfa24fd6f69c2fcc0987eba8b6b426dd78320)
+    
+    Conflicts:
+    	src/rgw/rgw_rest.h
+            trivial merge: prototype of an unrelated function changed
+            src/rgw/rgw_op.cc
+            s/is_object_op/!(s->object == NULL)/
+
+commit fc43d8c2c54ec9e9cb6ef4d19cca695eb2fb3aab
+Author: Matt Richards <mattjrichards@gmail.com>
+Date:   Thu Jan 8 13:16:17 2015 -0800
+
+    librados: Translate operation flags from C APIs
+    
+    The operation flags in the public C API are a distinct enum
+    and need to be translated to Ceph OSD flags, like as happens in
+    the C++ API. It seems like the C enum and the C++ enum consciously
+    use the same values, so I reused the C++ translation function.
+    
+    Signed-off-by: Matthew Richards <mattjrichards@gmail.com>
+    (cherry picked from commit 49d114f1fff90e5c0f206725a5eb82c0ba329376)
+
+commit d790833cb84d6f6349146e4f9abdcdffb4db2ee0
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Sep 29 18:17:29 2014 -0700
+
+    Objecter: check the 'initialized' atomic_t safely
+    
+    shutdown() resets initialized to 0, but we can still receive messages
+    after this point, so fix message handlers to skip messages in this
+    case instead of asserting.
+    
+    Also read initialized while holding Objecter::rwlock to avoid races
+    where e.g. handle_osd_map() checks initialized -> 1, continues,
+    shutdown() is called, sets initialized to 0, then handle_osd_map()
+    goes about its business and calls op_submit(), which would fail the
+    assert(initialized.read()) check. Similar races existed in other
+    message handlers which change Objecter state.
+    
+    The Objecter is not destroyed until after its Messenger in
+    the MDS, OSD, and librados, so this should be safe.
+    
+    Fixes: #9617
+    Backport: giant
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit e506f896a9217324ab7a7865989f4454562aed5f)
+    
+    Conflicts:
+    	src/osdc/Objecter.cc
+            context changed: Objecter::tick() did not have
+            assert(initialized.read())
+
+commit ce436a33e0f720ea4b8cf0363bcac1126be3c28b
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Mon Sep 29 18:12:50 2014 -0700
+
+    Objecter: init with a constant of the correct type
+    
+    Just a tiny cleanup.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+    (cherry picked from commit 1feba200aae7d9a042cda705c3de8fba2fc82331)
+
+commit 970a79753dc52d82d2abf29ffe6f88adac678eb0
+Author: Xiaoxi Chen <xiaoxi.chen@intel.com>
+Date:   Wed Aug 20 15:35:44 2014 +0800
+
+    CrushWrapper: pick a ruleset same as rule_id
+    
+    Originally in the add_simple_ruleset funtion, the ruleset_id
+    is not reused but rule_id is reused. So after some add/remove
+    against rules, the newly created rule likely to have
+    ruleset!=rule_id.
+    
+    We dont want this happen because we are trying to hold the constraint
+    that ruleset == rule_id.
+    
+    Signed-off-by: Xiaoxi Chen <xiaoxi.chen@intel.com>
+    (cherry picked from commit 78e84f34da83abf5a62ae97bb84ab70774b164a6)
+
+commit 76535116823f02f0392226e5725fbfef14c277ba
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Feb 20 13:43:46 2015 -0800
+
+    DBObjectMap: lock header_lock on sync()
+    
+    Otherwise, we can race with another thread updating state.seq
+    resulting in the old, smaller value getting persisted.  If there
+    is a crash at that time, we will reuse a sequence number, resulting
+    in an inconsistent node tree and bug #9891.
+    
+    Fixes: 9891
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 2b63dd25fc1c73fa42e52e9ea4ab5a45dd9422a0)
+    
+    Conflicts:
+    	src/os/DBObjectMap.cc
+            because we have state.v = 1; instead of state.v = 2;
+
+commit 13b01473c5c1a116d6cd6acbbb6cbe08ee6e9433
+Author: Zhiqiang Wang <zhiqiang.wang@intel.com>
+Date:   Tue Oct 28 09:37:11 2014 +0800
+
+    osd: cache tiering: fix the atime logic of the eviction
+    
+    Reported-by: Xinze Chi <xmdxcxz@gmail.com>
+    Signed-off-by: Zhiqiang Wang <zhiqiang.wang@intel.com>
+    (cherry picked from commit 622c5ac41707069ef8db92cb67c9185acf125d40)
+
+commit 13b8364dc869b2eefcb6646cff1e18c31126ce7d
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 31 19:33:59 2014 -0700
+
+    osd/ReplicatedPG: fix compile error
+    
+    From 1fef4c3d541cba360738437420ebfa2447d5802e.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4a9ad7dc2da6f4fa6a64235776a3f1d2799aef60)
+
+commit 55541b90db0a6d6c26c1fae6a4958fa5a320f82c
+Author: Xinze Chi <xmdxcxz@gmail.com>
+Date:   Wed Oct 29 07:11:11 2014 +0000
+
+    Get the currently atime of the object in cache pool for eviction
+    
+    Because if there are mutiple atime in agent_state for the same object, we should use the recently one.
+    
+    Signed-off-by: Xinze Chi <xmdxcxz@gmail.com>
+    (cherry picked from commit 1fef4c3d541cba360738437420ebfa2447d5802e)
+
+commit 5550cdd876db913b152edad17a4de16bded31627
+Author: Ding Dinghua <dingdinghua85@gmail.com>
+Date:   Thu Oct 30 14:58:42 2014 +0800
+
+    osdc/Objecter: Fix a bug of dead looping in Objecter::handle_osd_map
+    
+    If current map epoch is less than oldest epoch, current map epoch
+    should step up to oldest epoch.
+    
+    Fixes: #9986
+    Signed-off-by: Ding Dinghua <dingdinghua85@gmail.com>
+    (cherry picked from commit e0166a23c2cf655bfb4cf873be021a14d9b9be27)
+
+commit 7c5056f5cf77065e0e5a920f525f0a3be6b4b299
+Author: Ding Dinghua <dingdinghua85@gmail.com>
+Date:   Thu Oct 30 14:58:05 2014 +0800
+
+    osdc/Objecter: e shouldn't be zero in Objecter::handle_osd_map
+    
+    Signed-off-by: Ding Dinghua <dingdinghua85@gmail.com>
+    (cherry picked from commit 31c584c8ba022cd44fe2872d221f3026618cefab)
+
+commit 21f81b85de652aede51e88b87bdff71f2e411da3
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Nov 19 08:20:16 2014 -0800
+
+    PG: always clear_primary_state on new interval, but only clear pg temp if not primary
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit f692bfe076b8ddb679c6d1a6ea78cc47f0876326)
+
+commit 0c3f7637d8cf2146a4268330d8c0506ad38c354d
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Nov 14 15:44:20 2014 -0800
+
+    PG: always clear_primary_state when leaving Primary
+    
+    Otherwise, entries from the log collection process might leak into the next
+    epoch, where we might end up choosing a different authoritative log.  In this
+    case, it resulted in us not rolling back to log entries on one of the replicas
+    prior to trying to recover from an affected object due to the peer_missing not
+    being cleared.
+    
+    Fixes: #10059
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit c87bde64dfccb5d6ee2877cc74c66fc064b1bcd7)
+
+commit 1ccf58355d0796172935938572cf68ceb31a6801
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Tue Dec 2 15:17:57 2014 -0800
+
+    SimpleMessenger: allow RESETSESSION whenever we forget an endpoint
+    
+    In the past (e229f8451d37913225c49481b2ce2896ca6788a2) we decided to disable
+    reset of lossless Pipes, because lossless peers resetting caused trouble and
+    they can't forget about each other. But they actually can: if mark_down()
+    is called.
+    
+    I can't figure out how else we could forget about a remote endpoint, so I think
+    it's okay if we tell them we reset in order to clean up state. That's desirable
+    so that we don't get into strange situations with out-of-whack counters.
+    
+    Fixes: #10080
+    Backport: giant, firefly, dumpling
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 8cd1fdd7a778eb84cb4d7161f73bc621cc394261)
+
+commit 3e875ab108de8d2aa3717f76a3fe48ede286abb7
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Feb 6 09:52:29 2015 -0800
+
+    FileJournal: fix journalq population in do_read_entry()
+    
+    Fixes: 6003
+    Backport: dumpling, firefly, giant
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit bae1f3eaa09c4747b8bfc6fb5dc673aa6989b695)
+    
+    Conflicts:
+    	src/os/FileJournal.cc
+            because reinterpret_cast was added near two hunks after firefly
+
+commit 96a5c67121dde0d4d4cd13793bb131414b64cc28
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jan 30 18:42:40 2015 -0800
+
+    rgw: flush xml header on get acl request
+    
+    Fixes: #10106
+    Backport: firefly, giant
+    
+    dump_start() updates the formatter with the appropriate prefix, however,
+    we never flushed the formatter.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit eb45f861343162e018968b8c56693a8c6f5b2cab)
+
+commit c7b02f5a7347ea8688c5214f85bb9f612925a586
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Mar 19 00:32:39 2015 +0100
+
+    doc,tests: force checkout of submodules
+    
+    When updating submodules, always checkout even if the HEAD is the
+    desired commit hash (update --force) to avoid the following:
+    
+        * a directory gmock exists in hammer
+        * a submodule gmock replaces the directory gmock in master
+        * checkout master + submodule update : gmock/.git is created
+        * checkout hammer : the gmock directory still contains the .git from
+          master because it did not exist at the time and checkout won't
+          remove untracked directories
+        * checkout master + submodule update : git rev-parse HEAD is
+          at the desired commit although the content of the gmock directory
+          is from hammer
+    
+    http://tracker.ceph.com/issues/11157 Fixes: #11157
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+
+commit 4d4eb9faf1871c4469b78a7ee75d527ce5cc67ad
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Dec 11 13:05:54 2014 -0800
+
+    ReplicatedPG::scan_range: an object can disappear between the list and the attr get
+    
+    The first item in the range is often last_backfill, upon which writes
+    can be occuring.  It's trimmed off on the primary side anyway.
+    
+    Fixes: 10150
+    Backport: dumpling, firefly, giant
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit dce6f288ad541fe7f0ef8374301cd712dd3bfa39)
+
+commit 499d94f573e78f6545bd5a12ade6b5ba5a75bbca
+Author: Federico Simoncelli <fsimonce@redhat.com>
+Date:   Sat Nov 15 14:14:04 2014 +0000
+
+    common: do not unlock rwlock on destruction
+    
+    According to pthread_rwlock_unlock(3p):
+    
+     Results are undefined if the read-write lock rwlock is not held
+     by the calling thread.
+    
+    and:
+    
+     https://sourceware.org/bugzilla/show_bug.cgi?id=17561
+    
+     Calling pthread_rwlock_unlock on an rwlock which is not locked
+     is undefined.
+    
+    calling pthread_rwlock_unlock on RWLock destruction could cause
+    an unknown behavior for two reasons:
+    
+    - the lock is acquired by another thread (undefined)
+    - the lock is not acquired (undefined)
+    
+    Moreover since glibc-2.20 calling pthread_rwlock_unlock on a
+    rwlock that is not locked results in a SIGILL that kills the
+    application.
+    
+    This patch removes the pthread_rwlock_unlock call on destruction
+    and replaces it with an assertion to check that the RWLock is
+    not in use.
+    
+    Any code that relied on the implicit release is now going to
+    break the assertion, e.g.:
+    
+     {
+       RWLock l;
+       l.get(for_write);
+     } // implicit release, wrong.
+    
+    Signed-off-by: Federico Simoncelli <fsimonce@redhat.com>
+    (cherry picked from commit cf2104d4d991361c53f6e2fea93b69de10cd654b)
+
+commit b79852f3ab0643bbb3f6b71a56b15e5a4b5fc1f5
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Wed Dec 10 17:46:35 2014 +0000
+
+    mon: Paxos: reset accept timeout before submiting work to the store
+    
+    Otherwise we may trigger the timeout while waiting for the work to be
+    committed to the store -- and it would only take the write to take a bit
+    longer than 10 seconds (default accept timeout).
+    
+    We do wait for the work to be properly committed to the store before
+    extending the lease though.
+    
+    Fixes: #10220
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 18534615f184ba56b441fd1d4242eb06debdfe13)
+
+commit e997c9fed5feb3e877dfe07ffac1327b85d09ea2
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Tue Dec 9 17:35:47 2014 +0000
+
+    mon: MonitorDBStore: allow randomly injecting random delays on writes
+    
+    Adds two new config options:
+    
+    mon_inject_transaction_delay_probability : DOUBLE (0.0-1.0, default: 0.0)
+    mon_inject_transaction_delay_max : DOUBLE (seconds, default: 10.0)
+    
+    If probability is set to a value greater than 0, just before applying
+    the transaction, the store will decide whether to inject a delay,
+    randomly choosing a value between 0 and the max.
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit beaa04e4119765d5775a6c48fd072dd95c984e3b)
+
+commit 1d4ffbe1d233de555c1ecb0a33eebe2391b29f33
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Feb 9 17:41:19 2015 -0800
+
+    ShardedThreadPool: make wait timeout on empty queue configurable
+    
+    Fixes: 10818
+    Backport: giant
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 7002f934e6664daa995ca0629c0ea3bae1c6bddf)
+
+commit 292c4339c064968c2aa05eec701fbc2a8d82dab0
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Feb 9 17:11:38 2015 -0800
+
+    WorkQueue: make wait timeout on empty queue configurable
+    
+    Fixes: 10817
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 5aa6f910843e98a05bfcabe6f29d612cf335edbf)
+
+commit fea29b1bcbd17b3d1f642398ec70dbe258bbc98f
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Nov 20 15:15:08 2014 -0800
+
+    PGLog: include rollback_info_trimmed_to in (read|write)_log
+    
+    Fixes: #10157
+    Backport: firefly, giant
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 1fe8b846641486cc294fe7e1d2450132c38d2dba)
+
+commit 4427358bb556d902b01df27fd097cc3eefa561da
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Wed Feb 11 23:36:01 2015 +0000
+
+    mon: MonCap: take EntityName instead when expanding profiles
+    
+    entity_name_t is tightly coupled to the messenger, while EntityName is
+    tied to auth.  When expanding profiles we want to tie the profile
+    expansion to the entity that was authenticated.  Otherwise we may incur
+    in weird behavior such as having caps validation failing because a given
+    client messenger inst does not match the auth entity it used.
+    
+    e.g., running
+    
+    ceph --name osd.0 config-key exists foo daemon-private/osd.X/foo
+    
+    has entity_name_t 'client.12345' and EntityName 'osd.0'.  Using
+    entity_name_t during profile expansion would not allow the client access
+    to daemon-private/osd.X/foo (client.12345 != osd.X).
+    
+    Fixes: #10844
+    Backport: firefly,giant
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 87544f68b88fb3dd17c519de3119a9ad9ab21dfb)
+
+commit 8902279dd4b52516d59db712bd59e8d9372611a6
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Fri Nov 14 21:03:54 2014 +0000
+
+    mon: Monitor: stash auth entity name in session
+    
+    Backport: giant
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit ca8e1efc0be9bffcfbdce5593526d257aa498062)
+
+commit 681c99fe12cfa6318f8cf06f4e825805635bcc07
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Nov 20 14:27:39 2014 -0800
+
+    ReplicatedPG: fail a non-blocking flush if the object is being scrubbed
+    
+    Fixes: #8011
+    Backport: firefly, giant
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 9b26de3f3653d38dcdfc5b97874089f19d2a59d7)
+
+commit 1f58a0adc3b23785fac00083d721b62f6a4c44a1
+Merge: 9fee7ba fe7d4ca
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 17 10:52:01 2015 -0700
+
+    Merge pull request #4042 from dachary/wip-10546-giant
+    
+    ceph time check start round bug in monitor.cc
+
+commit 9fee7ba9d7343bda29bc5113f8db5cdd9c09d71a
+Merge: ca635ce e7af52a
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 17 10:50:26 2015 -0700
+
+    Merge pull request #4047 from dachary/wip-10762-giant
+    
+    mon: osd gets marked down twice
+
+commit ca635ce3eb21135b08613b97ce0c56977de44351
+Merge: 9094185 5771f57
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 17 10:49:53 2015 -0700
+
+    Merge pull request #4041 from dachary/wip-10512-giant
+    
+    osd: cancel_flush requeues blocked events after blocking event
+
+commit 90941850ceb2c080ac72c67fed991c951b13f449
+Merge: bc1b9d2 5fca232
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 17 10:47:26 2015 -0700
+
+    Merge pull request #4031 from dachary/wip-10353-giant
+    
+    crush: set_choose_tries = 100 for erasure code rulesets
+
+commit bc1b9d22d29a74e6853834fb0430e8803710bbbf
+Merge: bd69cfc 30a1744
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 17 10:47:08 2015 -0700
+
+    Merge pull request #4029 from dachary/wip-9910-giant
+    
+    msg/Pipe: discard delay queue before incoming queue
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit bd69cfcbd58103843fbc42c4e55d43c2bafe54ff
+Merge: 30f7df2 4bd5237
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 17 10:44:53 2015 -0700
+
+    Merge pull request #4030 from dachary/wip-10351-giant
+    
+    mount.ceph: avoid spurious error message
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 30f7df2b110f73477354fbfa607b4777317dddfa
+Merge: e5a50c3 8307318
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 17 10:44:00 2015 -0700
+
+    Merge pull request #4028 from dachary/wip-10259-giant
+    
+    osd_types: op_queue_age_hist and fs_perf_stat should be in osd_stat_t::o...
+
+commit e5a50c3ae42c8b9d16a19a13e10fc35b91149e73
+Merge: ba1d55d a5a76f8
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 17 10:42:10 2015 -0700
+
+    Merge pull request #4027 from dachary/wip-10257-giant
+    
+      mon: PGMonitor: several stats output error fixes
+
+commit ba1d55de1e92a0143702e9e3fbbb1820db17d6c3
+Merge: 530fce4 640986f
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 17 10:41:56 2015 -0700
+
+    Merge pull request #3998 from dzafman/wip-10677
+    
+    Fix ceph command manpage to match ceph -h (giant)
+    
+    Reviewed-by: Xinxin Shu <xinxin.shu@intel.com>
+
+commit 530fce4d7cf51f4a4ac1e70ebe20b2795a30010e
+Merge: a2a3dad 8a5a3c7
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 17 10:40:02 2015 -0700
+
+    Merge pull request #3921 from sponce/wip-11078-giant
+    
+    Fix libstriprados::stat, use strtoll insdead of strtol
+
+commit a2a3dad8ec00434b99d6d538615bf8ff9eada217
+Merge: 0f57188 984df3b
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 17 10:36:28 2015 -0700
+
+    Merge pull request #3819 from tchaikov/giant-pg-leak-10421
+    
+    osd: fix PG leak in SnapTrimWQ._clear()
+
+commit 0f5718822793074ffab45124afb3bf3160e92004
+Merge: 776c82c c5c6fcd
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 17 10:35:37 2015 -0700
+
+    Merge pull request #3771 from ceph/wip-10883-giant
+    
+    osd: Fix FileJournal wrap to get header out first
+
+commit 776c82caac36de14a5f895688f81c88a883fca3e
+Merge: 938e036 37f196e
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 17 10:35:29 2015 -0700
+
+    Merge pull request #3637 from sponce/wip-10758-giant
+    
+    Backport of pull request 3633 to giant : Fixed write_full behavior in libradosstriper
+
+commit 92c352d047c84035478e71828e32f554d9f507fc
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Feb 12 13:49:50 2015 -0800
+
+    mon/OSDMonitor: do not trust small values in osd epoch cache
+    
+    If the epoch cache says the osd has epoch 100 and the osd is asking for
+    epoch 200+, do not send it 100+.
+    
+    Fixes: #10787
+    Backport: giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a5759e9b97107488a8508f36adf9ca1aba3fae07)
+
+commit 25fcaca19c76765284787c61584d25efd105a0ca
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Feb 5 09:33:26 2015 -0800
+
+    rgw: send appropriate op to cancel bucket index pending operation
+    
+    Fixes: #10770
+    Backport: firefly, giant
+    
+    Reported-by: baijiaruo <baijiaruo@126.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit dfee96e3aebcaeef18c721ab73f0460eba69f1c7)
+    
+    Conflicts:
+    	src/rgw/rgw_rados.cc
+            resolved by manual s/ADD/CANCEL/
+
+commit e7af52a147cc6e1a578d51193a2d5a425cee8a20
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Feb 5 03:07:50 2015 -0800
+
+    mon: ignore osd failures from before up_from
+    
+    If the failure was generated for an instance of the OSD prior to when
+    it came up, ignore it.
+    
+    This probably causes a fair bit of unnecessary flapping in the wild...
+    
+    Backport: giant, firefly
+    Fixes: #10762
+    Reported-by: Dan van der Ster <dan@vanderster.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 400ac237d35d0d1d53f240fea87e8483c0e2a7f5)
+
+commit 28583276254ba6121b93688a6c6d8fab6588c864
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Feb 9 20:50:23 2015 -0800
+
+    rados.py: keep reference to python callbacks
+    
+    If we don't keep a reference to these, the librados aio calls will
+    segfault since the python-level callbacks will have been garbage
+    collected. Passing them to aio_create_completion() does not take a
+    reference to them. Keep a reference in the python Completion object
+    associated with the request, since they need the same lifetime.
+    
+    This fixes a regression from 60b019f69aa0e39d276c669698c92fc890599f50.
+    
+    Fixes: #10775
+    Backport: dumpling, firefly, giant
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 36d37aadbbbece28d70e827511f1a473d851463d)
+    (cherry picked from commit 5f1245e131e33a98572408c8223deed2c7cf7b75)
+
+commit 1b5fb51a995d5d672a46176240d5887354c12967
+Author: Billy Olsen <billy.olsen@gmail.com>
+Date:   Mon Feb 2 16:24:59 2015 -0700
+
+    Fix memory leak in python rados bindings
+    
+    A circular reference was inadvertently created when using the
+    CFUNCTYPE binding for callbacks for the asynchronous i/o callbacks.
+    This commit refactors the usage of the callbacks such that the
+    Ioctx object does not have a class reference to the callbacks.
+    
+    Fixes: #10723
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Billy Olsen <billy.olsen@gmail.com>
+    Reviewed-by: Dan Mick <dmick@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 60b019f69aa0e39d276c669698c92fc890599f50)
+
+commit d5b1b7e52242e082820e0cdd76dc59e9aea7f284
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jan 23 10:47:44 2015 -0800
+
+    osd: do not ignore deleted pgs on startup
+    
+    These need to get instantiated so that we can complete the removal process.
+    
+    Fixes: #10617
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 879fd0c192f5d3c6afd36c2df359806ea95827b8)
+
+commit fe7d4ca17dacf4419be7bdc50010115cee36517b
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Fri Jan 30 11:37:28 2015 +0000
+
+    mon: Monitor: fix timecheck rounds period
+    
+    Fixes: #10546
+    Backports: dumpling?,firefly,giant
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 2e749599ac6e1060cf553b521761a93fafbf65bb)
+
+commit 5771f57af731e61bf70f630cf85f5b94dcfd1edb
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Jan 11 17:28:04 2015 -0800
+
+    osd: requeue blocked op before flush it was blocked on
+    
+    If we have request A (say, cache-flush) that blocks things, and then
+    request B that gets blocked on it, and we have an interval change, then we
+    need to requeue B first, then A, so that the resulting queue will keep
+    A before B and preserve the order.
+    
+    This was observed on this firefly run:
+    
+      ubuntu@teuthology:/a/sage-2015-01-09_21:43:43-rados-firefly-distro-basic-multi/694675
+    
+    Backport: giant, firefly
+    Fixes: #10512
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 11bdfb4131ecac16d4a364d651c6cf5d1d28c702)
+
+commit fa8d45423e22f7b916a345024a9f70283fc3097d
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jan 7 13:56:14 2015 -0800
+
+    rgw: index swift keys appropriately
+    
+    Fixes: #10471
+    Backport: firefly, giant
+    
+    We need to index the swift keys by the full uid:subuser when decoding
+    the json representation, to keep it in line with how we store it when
+    creating it through other mechanism.
+    
+    Reported-by: hemant burman <hemant.burman@gmail.com>
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 478629bd2f3f32afbe6e93eaebb8a8fa01af356f)
+
+commit 5fca232080c0bde50b5c65c60c133bf81311e8d3
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Dec 17 16:06:55 2014 +0100
+
+    crush: set_choose_tries = 100 for erasure code rulesets
+    
+    It is common for people to try to map 9 OSDs out of a 9 OSDs total ceph
+    cluster. The default tries (50) will frequently lead to bad mappings for
+    this use case. Changing it to 100 makes no significant CPU performance
+    difference, as tested manually by running crushtool on one million
+    mappings.
+    
+    http://tracker.ceph.com/issues/10353 Fixes: #10353
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 2f87ac807f3cc7ac55d9677d2051645bf5396a62)
+
+commit 4bd523744cefb4a5e6b0d4440b9829b0e19dc012
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat Jan 3 15:29:29 2015 +0800
+
+    mount.ceph: avoid spurious error message
+    
+    /etc/mtab in most modern distributions is a symbol link to
+    /proc/self/mounts.
+    
+    Fixes: #10351
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit bdd0e3c4bda97fe18487a58dd173a7dff752e1a2)
+
+commit 30a17441f5030ac5c1227a9672811f88e580b4b2
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 29 14:45:11 2014 -0700
+
+    msg/Pipe: discard delay queue before incoming queue
+    
+    Shutdown the delayed delivery before the incoming queue in case the
+    DelayedDelivery thread is busy queuing messages.
+    
+    Fixes: #9910
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Greg Farnum <greg@inktank.com>
+    (cherry picked from commit f7431cc3c25878057482007beb874c9d4473883e)
+
+commit 8307318cdf9487dde3231aa4a00fca6079379738
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Dec 5 15:29:52 2014 -0800
+
+    osd_types: op_queue_age_hist and fs_perf_stat should be in osd_stat_t::operator==
+    
+    Fixes: 10259
+    Backport: giant, firefly, dumpling
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 1ac17c0a662e6079c2c57edde2b4dc947f547f57)
+
+commit a5a76f81ae9242a54d6ef150fa0ac32b31e90c25
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Mon Jan 19 18:49:15 2015 +0000
+
+    mon: PGMonitor: skip zeroed osd stats on get_rule_avail()
+    
+    Fixes: #10257
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit b311e7c36273efae39aa2602c1f8bd90d39e5975)
+
+commit 95ec68505ce4a26ea707716791cfb1e46a75ed25
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Fri Jan 16 18:13:05 2015 +0000
+
+    mon: PGMonitor: available size 0 if no osds on pool's ruleset
+    
+    get_rule_avail() may return < 0, which we were using blindly assuming it
+    would always return an unsigned value.  We would end up with weird
+    values if the ruleset had no osds.
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 8be6a6ab2aa5a000a39c73a98b11a0ab32fffa1c)
+
+commit 1c480b39acc184018d976beddc55187f86dfff1a
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Fri Jan 16 18:12:42 2015 +0000
+
+    mon: PGMonitor: fix division by zero on stats dump
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 50547dc3c00b7556e26b9a44ec68640c5c3a2384)
+
+commit 640986fdef3543c7469e43853900b63d7b2f6f36
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Mar 13 19:16:47 2015 -0700
+
+    doc: Fix ceph command manpage to match ceph -h (giant)
+    
+    Fixes: #10677
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+
+commit ec4c8997a67397e3ba5f335e444586f6f1a7864c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Mar 13 16:50:13 2015 -0700
+
+    doc: Minor fixes to ceph command manpage
+    
+    Fixes: #10676
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 7e85722fd4c89715fc2ed79697c82d65d7ebf287)
+
+commit 15e1c6fb3a0e2e191025f96df33feaf115c1411d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Mar 12 11:39:52 2015 -0700
+
+    doc: Fix ceph command manpage to match ceph -h (firefly)
+    
+    Improve synopsis section
+    Fixes: #10676
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 9ac488c1eb0e30511079ba05aaf11c79615b3940)
+    
+    Conflicts:
+    
+    	man/ceph.8 (took incoming version)
+
+commit 17d03e8e8b055ed50b1abb679a0dd1edf4ec3f30
+Author: Nilamdyuti Goswami <ngoswami@redhat.com>
+Date:   Thu Dec 18 17:11:22 2014 +0530
+
+    doc: Changes format style in ceph to improve readability as html.
+    
+    Signed-off-by: Nilamdyuti Goswami <ngoswami@redhat.com>
+    (cherry picked from commit 8b796173063ac9af8c21364521fc5ee23d901196)
+
+commit 61d600687ae6887a2edea0e79d582b1353558a83
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Mar 10 19:55:57 2015 +0800
+
+    mds: fix assertion caused by system clock backwards
+    
+    Fixes: #11053
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+
+commit f1c9c71fd6d566687db05e1d8f15b04a9f793005
+Author: Nilamdyuti Goswami <ngoswami@redhat.com>
+Date:   Sat Dec 13 02:24:41 2014 +0530
+
+    doc: Adds man page for ceph.
+    
+    Signed-off-by: Nilamdyuti Goswami <ngoswami@redhat.com>
+    (cherry picked from commit 76da87a64ca6b3cc0ceeaf63e19a9f440d6f4161)
+
+commit 8a5a3c7a20517c7482dc9f5f2783a3da54bd4f23
+Author: Dongmao Zhang <deanraccoon@gmail.com>
+Date:   Fri Nov 14 18:48:58 2014 +0800
+
+    Fix libstriprados::stat, use strtoll insdead of strtol
+    
+    The return value(long int) of strict_strtol is too small for unstriped
+    object.
+    
+    Signed-off-by: Dongmao Zhang <deanraccoon@gmail.com>
+    (cherry picked from commit fe6679dca479fc24806d7e57ab0108a516cd6d55)
+
+commit 5d423a5188c62650eaa39077eb99a84085f5f3e2
+Author: Dongmao Zhang <deanraccoon@gmail.com>
+Date:   Wed Dec 10 18:55:28 2014 +0800
+
+    Fix libstriprados::remove, use strtoll insdead of strtol
+    
+    Signed-off-by: Dongmao Zhang <deanraccoon@gmail.com>
+    (cherry picked from commit 78a15ee4c61fdadccb1921e861748400cc651862)
+
+commit 938e03630e075af03780da139ae879b5b0377734
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Feb 2 13:57:00 2015 -0800
+
+    Objecter::_op_submit_with_budget: add timeout before call
+    
+    Objecter::_send_op depends on the ontimeout field being filled in
+    to avoid 10340 and 9582.
+    
+    Fixes: 10340
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit cfcfafcb0f33994dbda1efe478ef3ab822ff50d4)
+
+commit 984df3b865f295ecf77b041a69d1d59384d80671
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Feb 10 16:29:45 2015 +0800
+
+    osd: fix PG leak in SnapTrimWQ._clear()
+    
+    Fixes: #10421
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 01e154d592d6cdbf3f859cf1b4357e803536a6b4)
+
+commit c5c6fcd6ad7a0317301970bc25419727ee3192fa
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 18 16:21:12 2015 -0800
+
+    osd: Fix FileJournal wrap to get header out first
+    
+    Correct and restore assert that was removed
+    
+    Cause by f46b1b473fce0322a672b16c7739e569a45054b6
+    Fixes: #10883
+    Backport: dumpling, firefly, giant
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 970bb4901f93575709421b5b25c3eff213de61b8)
+
+commit 37f196e80906d6b855ca0a7ce7344d1f8f50ae7f
+Author: Sebastien Ponce <Sebastien.Ponce@cern.ch>
+Date:   Thu Feb 5 10:46:37 2015 +0100
+
+    libradosstriper: fixed write_full when ENOENT
+    
+    Fixes: #10758
+    write_full was returning ENOENT when the file did not exists, while it should just have created it without complaining.
+    
+    Signed-off-by: Sebastien Ponce <sebastien.ponce@cern.ch>
+    (cherry picked from commit 6a91f2bb82035b3c8021a7dc7b23548ce3d61eb6)
+
+commit b318e2ffc3faedf6b02908429eb4ce79147d3471 (refs/remotes/gh/wip-10643)
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Fri Oct 17 18:01:05 2014 +0100
+
+    mon: MDSMonitor: don't return -EINVAL if function is bool
+    
+    Returning -EINVAL on a function that expects bool and the error code to
+    be in a variable 'r' can only achieve one thing: if this path is ever
+    touched, instead of returning an error as it was supposed to, we're
+    returning 'true' with 'r = 0' and, for no apparent reason, the user will
+    think everything went smoothly but with no new fs created.
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 91b2a2b69b198567d42f45f75b7f7ea559f89150)
+
+commit a1a7d856f5409280b6345afbbd961bcd8ee8d046
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Fri Oct 17 17:43:33 2014 +0100
+
+    mon: MDSMonitor: check all conditions are met *before* osdmon proposal
+    
+    We should not allow ourselves to request the osdmon to propose before we
+    know for sure that we meet the required conditions to go through with
+    our own state change.  Even if we still can't guarantee that our
+    proposal is going to be committed, we shouldn't just change the osdmon's
+    state just because we can.  This way, at least, we make sure that our
+    checks hold up before doing anything with side-effects.
+    
+    Fixes: #10643
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 1d1ae413e9f0aafa314af903bcf6b815f6558082)
+
+commit 0df7cf1e58bc4d77c69354f2a7160e3d9e6d1fc1
+Author: Joao Eduardo Luis <joao@redhat.com>
+Date:   Fri Oct 17 17:37:03 2014 +0100
+
+    mon: MDSMonitor: return if fs exists on 'fs new'
+    
+    We were just setting return code to -EINVAL, while allowing the logic to
+    continue regardless.  If we are to return error, then we should abort
+    the operation as well and let the user know it went wrong instead of
+    continuing as if nothing had happened.
+    
+    Signed-off-by: Joao Eduardo Luis <joao@redhat.com>
+    (cherry picked from commit 07b7f101057a80d1356bd20040180e94cd20c211)
diff --git a/doc/changelog/v0.94.1.txt b/doc/changelog/v0.94.1.txt
new file mode 100644
index 000000000..227267ddb
--- /dev/null
+++ b/doc/changelog/v0.94.1.txt
@@ -0,0 +1,74 @@
+commit e4bfad3a3c51054df7e537a724c8d0bf9be972ff (tag: refs/tags/v0.94.1, refs/remotes/gh/hammer)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Fri Apr 10 12:04:50 2015 -0700
+
+    0.94.1
+
+commit 733864738fa93979727e480e403293e079bb51e9
+Merge: b5921d5 5ca771a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Apr 10 17:51:38 2015 +0200
+
+    Merge pull request #4328 from dachary/wip-11364-hammer
+    
+    v4 bucket feature detection
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 5ca771a7d1df8e78ee503a7063068cf744d5efcc
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Apr 10 08:43:45 2015 -0700
+
+    crush: fix has_v4_buckets()
+    
+    alg, not type!
+    
+    This bug made us incorrectly think we were using v4 features when user type
+    5 was being used.  That's currently 'rack' with recent crush maps, but
+    was other types for clusters that were created with older versions.  This
+    is clearly problematic as it will lock out non-hammer clients incorrectly,
+    breaking deployments on upgrade.
+    
+    Fixes: #11364
+    Backport: hammer
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 38b35ab9d17eb84ac178c4cd3ebcf2ec0f66d8b6)
+
+commit 33e79ab7aa0b5428e8fb82a90eea17d31d363a88
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Apr 9 17:17:59 2015 -0700
+
+    crush: fix dump of has_v4_buckets
+    
+    Backport: hammer
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d6e23413017fb8f5d7f18d74e993ceeedb82d8bc)
+
+commit b5921d55d16796e12d66ad2c4add7305f9ce2353
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Apr 9 14:42:34 2015 -0700
+
+    crush/mapper: fix divide-by-0 in straw2
+    
+    If the item weight is 0 we don't want to divide; instead draw a minimal
+    value.
+    
+    Fixes: #11357
+    Reported-by: Yann Dupont <yd@objoo.org>
+    Tested-by: Yann Dupont <yd@objoo.org>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 64d1e900ec4f5831972ec80e8d0129604669f5bb)
+
+commit 93c8f436a4f84ac8088e1a1de82350dd33c68d64
+Author: Guang Yang <yguang@yahoo-inc.com>
+Date:   Thu Feb 26 08:13:12 2015 +0000
+
+    osd: fix negative degraded objects during backfilling
+    
+    When there is deleting requests during backfilling, the reported number of degraded
+    objects could be negative, as the primary's num_objects is the latest (locally) but
+    the number for replicas might not reflect the deletings. A simple fix is to ignore
+    the negative subtracted value.
+    
+    Signed-off-by: Guang Yang <yguang@yahoo-inc.com>
+    (cherry picked from commit 14d7e36d3c978844da73d0e1c8a3a1ec863bac15)
diff --git a/doc/changelog/v0.94.10.txt b/doc/changelog/v0.94.10.txt
new file mode 100644
index 000000000..342e6a676
--- /dev/null
+++ b/doc/changelog/v0.94.10.txt
@@ -0,0 +1,2728 @@
+commit b1e0532418e4631af01acbc0cedd426f1905f4af
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Tue Feb 21 15:12:34 2017 +0000
+
+    0.94.10
+
+commit 83af8cdaaa6d94404e6146b68e532a784e3cc99c
+Merge: 5855083 5485b49
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jan 23 17:49:54 2017 +0100
+
+    Merge pull request #13041 from SUSE/wip-18448-hammer
+
+    hammer: osd: filestore: FALLOC_FL_PUNCH_HOLE must be used with FALLOC_FL_KEEP_SIZE
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 585508358f6412b179175a7812b76330ec5dda0d
+Merge: 4e4d3f0 eabe906
+Author: John Spray <jspray@redhat.com>
+Date:   Mon Jan 23 12:26:27 2017 +0100
+
+    Merge pull request #13022 from SUSE/wip-18602-hammer
+
+    hammer: cephfs test failures (ceph.com/qa is broken, should be download.ceph.com/qa)
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 4e4d3f0fd86840609940f6b336f6aaf54a8695bc
+Merge: 9036fee cf9678c
+Author: John Spray <jspray@redhat.com>
+Date:   Mon Jan 23 12:25:43 2017 +0100
+
+    Merge pull request #13039 from SUSE/wip-fs-thrash-xfs
+
+    tests: run fs/thrash on xfs instead of btrfs
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit eabe9064aed3693599e3b6053b57eafb835d35da
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Jan 17 17:12:46 2017 +0100
+
+    qa: update remaining ceph.com to download.ceph.com
+
+    Fixes: http://tracker.ceph.com/issues/18574
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 549d993d3fd8ffffa280ed4a64aca41d1c6f2da1)
+
+commit 5485b49da39b9f07a4216ff26f8ec9e04590ddf8
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Jan 21 14:54:52 2017 +0100
+
+    os/filestore: FALLOC_FL_PUNCH_HOLE must be used with FALLOC_FL_KEEP_SIZE
+
+    Signed-off-by: xinxin shu <xinxin.shu@intel.com>
+    (cherry picked from commit 7bd95b595fddb8a4e618a2c7df1ba04eccf0829d)
+
+    Conflicts:
+	src/os/FileStore.cc (in master, this file is in a different location)
+
+commit cf9678c998600e76e94777b2904ba3047d051bb8
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Jan 21 14:24:25 2017 +0100
+
+    tests: run fs/thrash on xfs instead of btrfs
+
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 9036feed9bd9f386dc013dee8326071ef0b4cbee
+Merge: 33c7ab7 fa5c625
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Jan 21 10:00:04 2017 +0100
+
+    Merge pull request #12824 from SUSE/wip-18449-hammer
+
+    hammer: [teuthology] update rbd/singleton/all/formatted-output.yaml to support ceph-ci
+
+commit 33c7ab7480aa8d0ee80d6a47f91f4488515e07f8
+Merge: ef8ae1f ef513a7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jan 20 23:10:10 2017 +0100
+
+    Merge pull request #12906 from SUSE/wip-15943-hammer
+
+    hammer: osd/PG: fix cached_removed_snaps bug in PGPool::update after map gap
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit ef8ae1ff13b297c68ed4f1d0e01aba8d20dc5007
+Merge: c51602f 8bab745
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jan 20 23:02:05 2017 +0100
+
+    Merge pull request #12805 from SUSE/wip-18432-hammer
+
+    hammer: ceph-create-keys loops forever
+
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit c51602f0106e3681c143a2cfd94a08428e966caf
+Merge: 48687c7 83e7f17
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jan 17 08:39:14 2017 -0500
+
+    Merge pull request #12957 from SUSE/wip-18544-hammer
+
+    hammer: [teuthology] update Ubuntu image url after ceph.com refactor
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 83e7f178c9a0b70305644ca30598c1b1ff4d0474
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jan 16 22:12:51 2017 -0500
+
+    qa/tasks/qemu: update default image url after ceph.com redesign
+
+    Fixes: http://tracker.ceph.com/issues/18542
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6d17befb3bbc3d83c9d23d763ad95e1e7b2e4be0)
+
+commit ef513a78062569de081311b50fffdf59d4031ea7
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Dec 12 10:35:38 2016 -0800
+
+    PG: fix cached_removed_snaps bug in PGPool::update after map gap
+
+    5798fb3bf6d726d14a9c5cb99dc5902eba5b878a actually made 15943 worse
+    by always creating an out-of-date cached_removed_snaps value after
+    a map gap rather than only in the case where the the first map after
+    the gap did not remove any snapshots.
+
+    Introduced: 5798fb3bf6d726d14a9c5cb99dc5902eba5b878a
+    Fixes: http://tracker.ceph.com/issues/15943
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 5642e7e1b3bb6ffceddacd2f4030eb13a17fcccc)
+
+commit fa5c6250125a51e61e34aa9586cc917df7458138
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Jan 7 13:48:51 2017 +0100
+
+    tests: subst repo and branch in qemu test urls
+
+    References: http://tracker.ceph.com/issues/18440
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 2484a5546160560a4050d35400a6edce37ae48ad)
+
+    Conflicts: (hammer doesn't have these files)
+	qa/suites/rbd/maintenance/qemu/xfstests.yaml
+	qa/suites/rbd/qemu/workloads/#qemu_xfstests.yaml#
+
+commit 572cd2660f5bf1cc7f980831d12bb3075816b96f
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Jan 7 13:45:10 2017 +0100
+
+    tests: subst branch and repo in qa/tasks/qemu.py
+
+    References: http://tracker.ceph.com/issues/18440
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 74689df754561e11a3377998840efcea9e780755)
+
+commit 237fb80f2b2874329aed25ba655a6789e1707114
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jan 6 21:06:11 2017 +0100
+
+    tests: subst repo name in qa/tasks/cram.py
+
+    Inspired by bcbe45d948f1c4da02e27b3be5f29a8b52745e28
+
+    Fixes: http://tracker.ceph.com/issues/18440
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 56e37e41f4dddd289dd3c1886b192cd328ed311b)
+
+commit 6045a83a438918f445504224b80809dfe715c9b0
+Author: Venky Shankar <vshankar@redhat.com>
+Date:   Fri Dec 9 15:11:49 2016 +0530
+
+    cram: support fetching from sha1 branch, tag, commit hash
+
+    Signed-off-by: Venky Shankar <vshankar@redhat.com>
+    (cherry picked from commit d2f0d745987a2e2eee4e1822146aad8da5d42708)
+
+    Conflicts:
+	suites/krbd/unmap/tasks/unmap.yaml (hammer doesn't have this)
+	suites/rbd/singleton/all/formatted-output.yaml (moved to qa/,
+		trivial resolution)
+
+commit 48687c79adf489b27af15bb44205913acc89af7a
+Merge: 61917cb ac62d79
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 8 09:26:03 2017 +0200
+
+    Merge pull request #12227 from ceph/wip-orit-resharding-hammer
+
+    Wip orit resharding hammer
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 8bab74537f5891f2b97f0fb6722e2e00416b723a
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 2 12:28:49 2016 -0400
+
+    ceph-create-keys: wait 10 minutes to get or create the bootstrap key, not forever
+
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit bea802bd13df883d06b7b67a83ac6540e44ab6c1)
+
+commit a7d919100e9e78b7f821da76b338bca24f374a0a
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 2 12:25:32 2016 -0400
+
+    ceph-create-keys: wait 10 minutes to get or create a key, not forever
+
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 8f84681a4d97a7ea6bc04e759eccfc40204a2fdb)
+
+    Conflicts:
+	src/ceph-create-keys ( hammer does not have
+		52e978e4b3660baa9f50b1bb8247909b672142e7 )
+
+commit 7b3bc5fbc55da3cb2ce036ef3df367832d7a59e2
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 2 12:19:10 2016 -0400
+
+    ceph-create-keys: wait for quorum for ten minutes, not forever
+
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 32cedd2c18e48d5f9a1547e5fd69c24c7eac1638)
+
+commit 61917cbcb7a4e35aa0ae8718377526ae55b5b7cf
+Merge: 7b1f8fe 7106f42
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 5 09:15:13 2017 -0800
+
+    Merge pull request #11615 from mslovy/wip-17610-hammer
+
+    hammer: osd: fix fiemap issue in xfs when #extents > 1364
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 7b1f8fe4c97b6c8ef81d899f0a24235946cc585a
+Merge: 5ba3c08 5f00e1d
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 5 09:14:10 2017 -0800
+
+    Merge pull request #11936 from SUSE/wip-16432-hammer
+
+    hammer: librados: bad flags can crash the osd
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 5ba3c0873d10b247a45bfbc1cb483cfc2189a9fd
+Merge: b44e1db 8312dc6
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 5 09:12:55 2017 -0800
+
+    Merge pull request #12121 from dzafman/wip-16225-min
+
+    hammer: common: SIGABRT in TrackedOp::dump() via dump_ops_in_flight()
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit b44e1db11b0d7310f2563fa7acfb1464db4b04fb
+Merge: 7bfb05b 92de2fd
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 5 09:11:57 2017 -0800
+
+    Merge pull request #12266 from SUSE/wip-18109-hammer
+
+    hammer: msgr: simple/Pipe: error decoding addr
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 7bfb05b00157fa40c053eeefb59cd3bb211fc2b3
+Merge: 715012b fe77c9a
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 5 09:11:34 2017 -0800
+
+    Merge pull request #12417 from SUSE/wip-18132-hammer
+
+    hammer: osd: ReplicatedBackend::build_push_op: add a second config to limit omap entries/chunk independently of object data
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 715012bebe6e7838c702aae0283f1c3d37428762
+Merge: a20ad42 e9311dd
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 5 09:11:13 2017 -0800
+
+    Merge pull request #12759 from SUSE/wip-18399-hammer
+
+    hammer: tests: objecter_requests workunit fails on wip branches
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit a20ad42ecab39541107e01ac4516a0a71e379846
+Merge: 3e62a1d 67e2101
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 5 09:10:54 2017 -0800
+
+    Merge pull request #12762 from SUSE/wip-18405-hammer
+
+    hammer: Cannot reserve CentOS 7.2 smithi machines
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 3e62a1d5dbd2ee6672388121be26afd5d0374f5f
+Merge: 77fa49f aa91bc7
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 5 09:10:28 2017 -0800
+
+    Merge pull request #12685 from SUSE/wip-18273-hammer
+
+    hammer: tests: Workunits needlessly wget from git.ceph.com
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 77fa49f67d1c8c59d9131e6f52e595661ceda8f2
+Merge: 2c164b7 c28d5ea
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 5 09:09:59 2017 -0800
+
+    Merge pull request #12687 from SUSE/wip-18281-hammer
+
+    hammer: mon: osd flag health message is misleading
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 2c164b73af9de4196821d9214fb7598ac88ef509
+Merge: dfefeda 996fc25
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jan 5 15:20:28 2017 +0100
+
+    Merge pull request #12423 from SUSE/wip-18222-hammer
+
+    hammer: tests: qemu/tests/qemu-iotests/077 fails in dumpling, hammer, and jewel
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit dfefeda86e3e6761bc4a59cee960d807609344e4
+Merge: b0648f8 0c5ebf4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jan 4 20:15:21 2017 +0100
+
+    Merge pull request #12744 from SUSE/wip-18385-hammer
+
+    hammer: tests: Cannot clone ceph/s3-tests.git (missing branch)
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit b0648f85ee67da82ac8fb7e8f4c1546aa560bfda
+Merge: c45896a e97ceb1
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jan 4 11:43:29 2017 +0100
+
+    Merge pull request #12418 from SUSE/wip-18218-hammer
+
+    hammer: rgw sends omap_getvals with (u64)-1 limit
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit c45896a86810f02b86b3c355855eb70a998a3c9a
+Merge: 811213e db2d112
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jan 4 11:37:29 2017 +0100
+
+    Merge pull request #12619 from Werkov/wip-18317-hammer
+
+    hammer: rgw: TempURL in radosgw behaves now like its Swift's counterpart.
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit db2d112f22a525bfffb62e85747769a4b30f7375
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Mar 4 16:30:50 2016 +0100
+
+    rgw: TempURL in radosgw behaves now like its Swift's counterpart.
+
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit a1ea7180969dcbaf88dfcc1bfafb17e82639df64)
+
+    Conflicts:
+	src/rgw/rgw_swift.cc
+	  - replace C++11 constructs,
+	    - s->info.method is actually unconditional member of
+	      allowed_methods, which was opaqued by C++11 ctors
+	  - drop temp_url_make_content_disp() which has no use with
+	    hammer's req_state
+
+commit 811213ef9df188c8cf21dec5aeab08093af483d7
+Merge: 9a8aaca 47e7202
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 3 15:31:27 2017 +0100
+
+    Merge pull request #12312 from YankunLi/fix_parameter_check
+
+    hammer: rbd: fix parameter check
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 9a8aaca56e48aae78dff27a969e2398c332f2b09
+Merge: 78e3451 c4f1b42
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 3 15:31:03 2017 +0100
+
+    Merge pull request #12446 from dillaman/wip-18111
+
+    hammer: librbd: diffs to clone's first snapshot should include parent diffs
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 78e34519f9e3c72aa91587593dcb23dac4ade3a3
+Merge: a9de35a 0f83bb7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 3 15:23:07 2017 +0100
+
+    Merge pull request #12398 from linuxbox2/hammer
+
+    hammer: rgw: do not abort when accept a CORS request with short origin
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit a9de35a27cb7d57f8094de78f550291b1b45b944
+Merge: 3ef8d8c a602e2b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jan 3 09:17:35 2017 -0500
+
+    Merge pull request #12758 from SUSE/wip-18397-hammer
+
+    hammer: tests: OSDs commit suicide in rbd suite when testing on btrfs
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 67e2101ffa2b4defe07dec30b12cc37c9f2ed488
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Jan 3 14:53:27 2017 +0100
+
+    qa/distros: add centos yaml; use that instead
+
+    Modeled after https://github.com/ceph/ceph/pull/12632
+
+    Fixes: http://tracker.ceph.com/issues/18405
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit e9311ddb60638c1dd39325e279b97c9a7d09b8a4
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 15 13:35:02 2016 -0500
+
+    qa/tasks/admin_socket: subst in repo name
+
+    It is either ceph.git or ceph-ci.git.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit bcbe45d948f1c4da02e27b3be5f29a8b52745e28)
+
+commit a602e2bf2e1cae8d21ae215c35a611cc99b6c7c9
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Jan 3 11:26:26 2017 +0100
+
+    rbd: only test on xfs
+
+    btrfs is not recommended for production
+
+    See 5864d56592ef3ffe765aa1ef0865610dbc7e8725 for precendent.
+
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit aa91bc7df963728e6a4d4db1086214e7873fc2e7
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Jan 2 22:49:13 2017 +0100
+
+    tests: rbd/test_lock_fence.sh: fix rbdrw.py relative path
+
+    This commit fixes a regression introduced by
+    cf294777ea92f0911813a7132068584d4f73a65a
+
+    Fixes: http://tracker.ceph.com/issues/18388
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 91231de16dbe4d0e493ec617165a2b38078d122b)
+
+commit 0c5ebf4adf26274802ec4cff229b2272dc4a4266
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Jan 4 10:03:08 2016 +0100
+
+    use ceph-master branch for s3tests
+
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 21160124028ab123b5c547499d911b0a1397214f)
+
+    Conflicts: (trivial resolution)
+	    qa/suites/rgw/multifs/tasks/rgw_s3tests.yaml
+	qa/suites/rgw/verify/tasks/rgw_s3tests.yaml
+	qa/suites/rgw/verify/tasks/rgw_s3tests_multiregion.yaml
+
+commit 3ef8d8c6f0ef712dcff207fb6ef17887bd8719d7
+Merge: e038662 d18240f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jan 2 20:34:42 2017 +0100
+
+    Merge pull request #12743 from SUSE/wip-18383
+
+    hammer: tests: populate mnt_point in qa/tasks/ceph.py
+
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit d18240f679840bdaeec1e3f7cdd9164b4406673c
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Jan 2 14:22:43 2017 +0100
+
+    qa/tasks/ceph.py: populate mnt_point in hammer
+
+    In hammer, there is no "mnt_point" variable, yet 53225d5 was cherry-picked to
+    hammer - see 87e0299448908775600c62130617bad503027768 and https://github.com/ceph/ceph/pull/12455
+
+    This caused lots of "NameError: global name 'mnt_point' is not defined" errors
+    in hammer integration testing.
+
+    This is not a cherry-pick, because the mnt_point variable was introduced by
+    a54ff597faa379c8fd369352fb924651be269f95 ("tasks/ceph: convert cluster creation
+    to work with multiple clusters") which was not backported to hammer.
+
+    Fixes: http://tracker.ceph.com/issues/18383
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit ac62d7996b95d8fdd5665e490fc09506c8fe9e68
+Author: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+Date:   Mon Oct 31 13:59:55 2016 +0800
+
+    rgw: fix osd crashes when execute "radosgw-admin bi list --max-entries=1" command
+
+    Fixes: http://tracker.ceph.com/issues/17745
+    Signed-off-by: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+    (cherry picked from commit 51a4405a1ef59671cbd51c530a84333359d3f3dd)
+
+commit 530d1e22920bd843d34ceffccad346fd5003c075
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Tue Nov 29 13:36:37 2016 +0100
+
+    rgw: use hammer rgw_obj_key api
+
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+
+commit e396b37af147770434147c5bd0377e8624254cd9
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Tue Nov 29 13:15:05 2016 +0100
+
+    Revert "rgw: rgw_obj encoding fixes"
+
+    This reverts commit c1b64c8f9c23473b758300958a94a172f1d39b64.
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+
+commit a3e9865058e12fff18ee7674fcfd8435a1a37bf4
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 22 13:05:22 2016 -0500
+
+    qa/tasks/workunit: clear clone dir before retrying checkout
+
+    If we checkout ceph-ci.git, and don't find a branch,
+    we'll try again from ceph.git. But the checkout will
+    already exist and the clone will fail, so we'll still
+    fail to find the branch.
+
+    The same can happen if a previous workunit task already
+    checked out the repo.
+
+    Fix by removing the repo before checkout (the first and
+    second times).  Note that this may break if there are
+    multiple workunit tasks running in parallel on the same
+    role.  That is already racy, so if it's happening, we'll
+    want to switch to using a truly unique clonedir for each
+    instantiation.
+
+    Fixes: http://tracker.ceph.com/issues/18336
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2a7013cd5a033c5be43350505d75f088e831e201)
+
+commit a70e0d58e10184b0903ba36c047a237be3c7cc19
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 16 15:06:16 2016 -0500
+
+    qa/tasks/workunit: retry on ceph.git if checkout fails
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 72d73b8c8836ae35c518fa09f44805a74038f02a)
+
+commit c28d5ea7107427175ef3f6089aaa36c97c2919bc
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 7 09:50:26 2016 -0500
+
+    mon/OSDMonitor: only show interesting flags in health warning
+
+    Also add PAUSEREC flag into the list.
+
+    Fixes: http://tracker.ceph.com/issues/18175
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 26220f0608f5ed4a7d97fb8d10d0d12a0fcf5583)
+
+    Conflicts:
+	src/mon/OSDMonitor.cc (hammer lacks CEPH_OSDMAP_FULL and order of flags
+				   is slightly different)
+
+commit d1c654ea3f3b8c194f3fdc7295cb379c6270d6b3
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 19 14:08:11 2016 -0500
+
+    qa/workunits: include extension for nose tests
+
+    When you have a relative path you have to include the extension.
+    Weird.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5666fd61d6dbd40be1d79354227cabd562e829ea)
+
+    Conflicts:
+	qa/workunits/fs/test_python.sh (not in hammer)
+	qa/workunits/rbd/test_librbd_python.sh (no valgrind in hammer)
+
+commit eaa51f6f907019727eb18fc4174fe4f2d04f396a
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 15 15:10:28 2016 -0500
+
+    qa/workunits: use relative path instead of wget from git
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit cf294777ea92f0911813a7132068584d4f73a65a)
+
+    Conflicts: (all trivial)
+	qa/workunits/fs/test_python.sh
+	qa/workunits/rados/test_python.sh
+	qa/workunits/rbd/rbd_mirror.sh
+	qa/workunits/rbd/rbd_mirror_stress.sh
+	qa/workunits/rbd/test_librbd_python.sh
+	qa/workunits/rbd/test_lock_fence.sh
+
+commit 8eb4e8b6fd6c803fa9bea781db07c54d901e34cc
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 15 13:26:14 2016 -0500
+
+    qa/tasks/workunit.py: add CEPH_BASE env var
+
+    Root of git checkout
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 27b8eac24922f8b4bd065e6e7f0bc8e2ba37b5d5)
+
+commit dccf904241f2a00ba0317135bd3cbe56c6dfdb80
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 15 13:25:23 2016 -0500
+
+    qa/tasks/workunit: leave workunits inside git checkout
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4602884ab8f5a256d13091f7239d938990482d95)
+
+commit e038662d80ec1c59edb3d6a1c63197ca387d26b2
+Merge: 12d6781 59878b3
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 14 11:40:08 2016 -0600
+
+    Merge pull request #12455 from liewegas/qa-suite-hammer
+
+    hammer: merge ceph-qa-suite
+
+commit 59878b3bf775fabf0f4c168e67d6f473e69c15a1
+Merge: 12d6781 7ed0dce
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 14 11:30:00 2016 -0600
+
+    merge ceph-qa-suite
+
+commit 7ed0dce3cf7ded278e59a9b36d3fd713e3c42b8d
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 14 11:29:55 2016 -0600
+
+    move ceph-qa-suite dirs into qa/
+
+commit 08ce5441dbad8754205717a20d6273c0a099777a
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 14 12:27:47 2016 -0500
+
+    Revert "tasks/workunit.py: depth 1 clone"
+
+    This reverts commit 12a707cb0f7b162e15c31915eb677bbd49b5e2cf.
+
+commit 12a707cb0f7b162e15c31915eb677bbd49b5e2cf
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 14 12:19:44 2016 -0500
+
+    tasks/workunit.py: depth 1 clone
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4faf77a649cb3f8ddf497ca81937b3dbf63a18dc)
+
+commit 6bdc3bb6a00480b721b9795bd5e132b1dbf04ac0
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 14 12:18:29 2016 -0500
+
+    tasks/workunit: remove kludge to use git.ceph.com
+
+    This was hard-coded to ceph.git (almost) and breaks when
+    you specify --ceph-repo.  Remove it entirely.  We'll see if
+    github.com is better at handling our load than it used to
+    be!
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 159c455a0326eef2c017b3e3cf510f918b5ec76c)
+
+commit 87e0299448908775600c62130617bad503027768
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Dec 10 02:36:52 2016 +0800
+
+    tasks/ceph: restore context of osd mount path before mkfs
+
+    all newly created files and directories under the mount dir inherit the
+    SELinux type of their parent directory. so we need to set it before
+    mkfs.
+
+    Fixes: http://tracker.ceph.com/issues/16800
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 53225d5272a1d35d4183fcfa55a139f55f77e122)
+
+commit ca933160bec3532a5b2846fc7e684f11509c8298
+Merge: bbb2615 107e9c3
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 12 16:42:55 2016 -0600
+
+    Merge pull request #1278 from ceph/wip-16225-min
+
+    hammer: SIGABRT in TrackedOp::dump() via dump_ops_in_flight()
+
+commit c4f1b42810339b5b45b48f74c3ff0ae661a8807a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Dec 12 11:53:00 2016 -0500
+
+    librbd: diffs to clone's first snapshot should include parent diffs
+
+    If the clone has a backing object created after the snapshot that
+    overwrites an extent of the parent, the parent diffs within that
+    extent are not included in the result.
+
+    Hammer-specific implementation due to librbd refactoring during the
+    Infernalis release.
+
+    Fixes: http://tracker.ceph.com/issues/18111
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 996fc2575ef988e6b20fda9ca771b2f6f141cbd3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Dec 7 09:59:39 2016 -0500
+
+    qa/workunits/rbd: removed qemu-iotest case 077
+
+    The test case is not stable due to racing console output. This
+    results in spurious failures.
+
+    Fixes: http://tracker.ceph.com/issues/10773
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2c70df978d605a45ff81971b86f5afbefbdaabb6)
+
+commit e97ceb13242fd73b05a20b60208fee7f032e3ef3
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Nov 28 14:20:03 2016 -0800
+
+    rgw: omap_get_all() fixes
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 99e866feb3f8aa0ff768d6253979b6b4fc0fa452)
+
+commit 38ab892201ce9ca95010e5f33277be2b0213e96d
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Nov 21 18:37:39 2016 -0500
+
+    rgw/rgw_rados: do not omap_getvals with (u64)-1 max
+
+    Fixes: http://tracker.ceph.com/issues/17985
+
+    This will make the OSD fall over if the object is big
+    and the OSD doesn't impose it's own fail-safe limit.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 7cd123ec0dd5951628e4f8ddd4e044a0fbb82f2f)
+
+commit fe77c9a906fb0fbd85762c8c079d85ca106613ce
+Author: Wanlong Gao <wanlong.gao@easystack.cn>
+Date:   Thu Jun 23 20:04:25 2016 +0800
+
+    osd: limit omap data in push op
+
+    We already have the config osd_recovery_max_chunk to limit the total
+    size of omap entries and omap data. But we need an individual config
+    to limit the number of omap entries independently. We call this config
+    osd_recovery_max_omap_entries_per_chunk here with the default number
+    of 64000.
+
+    Signed-off-by: Wanlong Gao <wanlong.gao@easystack.cn>
+    (cherry picked from commit 56064c5cb645254574dbad00e6c16b783115bb93)
+
+commit 0f83bb7dbc5f81ecef0108b746f5bca58e76ff31
+Author: LiuYang <yippeetry@gmail.com>
+Date:   Thu Dec 8 14:21:43 2016 +0800
+
+    rgw: do not abort when accept a CORS request with short origin
+
+    Fixed: #18187
+
+    when accept a CROS request, the request http origin shorter than the bucket's corsrule
+    (eg. origin: http://s.com corsrule: <AllowedOrigin>*.verylongdomain.com</AllowedOrigin>).
+    the rgw_cors.cc::is_string_in_set() will have a wrong index, the radosrgw server will
+    abort.
+
+    $ curl http://test.localhost:8000/app.data -H "Origin:http://s.com"
+
+     0> 2016-12-05 03:22:29.548138 7f6add05d700 -1 *** Caught signal (Aborted) **
+     in thread 7f6add05d700 thread_name:civetweb-worker
+
+     ceph version 11.0.2-2168-gd2f8fb4 (d2f8fb4a6ba75af7e6da0f5a7f1b49ec998b1631)
+     1: (()+0x50720a) [0x7f6b147c420a]
+     2: (()+0xf370) [0x7f6b09a33370]
+     3: (gsignal()+0x37) [0x7f6b081ca1d7]
+     4: (abort()+0x148) [0x7f6b081cb8c8]
+     5: (__gnu_cxx::__verbose_terminate_handler()+0x165) [0x7f6b08ace9d5]
+     6: (()+0x5e946) [0x7f6b08acc946]
+     7: (()+0x5e973) [0x7f6b08acc973]
+     8: (()+0x5eb93) [0x7f6b08accb93]
+     9: (std::__throw_out_of_range(char const*)+0x77) 0x7f6b08b21a17]
+     10: (()+0xbd97a) [0x7f6b08b2b97a]
+     11: (()+0x449c1e) [0x7f6b14706c1e]
+     12: (RGWCORSRule::is_origin_present(char const*)+0x48) [0x7f6b147073b8]
+     13: (RGWCORSConfiguration::host_name_rule(char const*)+0x37) [0x7f6b147074e7]
+     14: (RGWOp::generate_cors_headers(std::string&, std::string&, std::string&, std::string&, unsigned int*)+0xa3) [0x7f6b14593e63]
+     15: (dump_access_control(req_state*, RGWOp*)+0x61) [0x7f6b14653f91]
+
+    Signed-off-by: LiuYang <yippeetry@gmail.com>
+    (cherry picked from commit 67d4d9e64bc224e047cf333e673bb22cd6290789)
+
+commit 47e72024e091a55a5fcd31853e143c94d850b00a
+Author: YankunLi <lioveni99@gmail.com>
+Date:   Mon Dec 5 14:39:20 2016 +0800
+
+    rbd: fix parameter check
+
+    using import-diff command, the imagename must be specified.
+
+    Signed-off-by: Yankun Li <lioveni99@gmail.com>
+
+commit bbb261533455d37b4ef1e5bfc3c8f7765ab74e5a
+Merge: 8e5872b 4533623
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 2 15:09:32 2016 -0500
+
+    Merge pull request #1276 from ceph/no-btrfs-on-hammer
+
+    rados: remove btrfs tests; use xfs instead
+
+commit 7106f426c96978a0a41e9d1482239a4da505157e
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Nov 23 14:48:11 2016 +0800
+
+    FileStore::_do_fiemap: do not reference fiemap after it is freed
+
+    `last` points to fiemap::fm_extends[n], and if fiemap gets freed, we can
+    not reference any of its fieldis. so we could remember the check result before
+    freeing it.
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit c3748fa7737b6c43de2dae5d957fcff01322515c)
+
+    Conflicts:
+	src/os/FileStore.cc
+	put the parameter is_last in the right place
+
+commit 92de2fd9a8b19c3d83d2567720f4e7ad21e43a63
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Nov 29 21:50:52 2016 -0500
+
+    msg/simple/Pipe: handle addr decode error
+
+    Fixes: http://tracker.ceph.com/issues/18072
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2f5cfbc0de5464ab80581c416413a86c3d38dfb7)
+
+commit 12d67814912514bb34ba61ceb45a744299d65ff2
+Merge: 8406ee5 e1694d6
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Nov 29 22:16:34 2016 +0100
+
+    Merge pull request #11457 from tchaikov/wip-17558-hammer
+
+    hammer: mon: send updated monmap to its subscribers
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 8406ee58ca5c4abbe962ba5a4025aabe220e3981
+Merge: bfda134 03af4aa
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Nov 29 22:12:46 2016 +0100
+
+    Merge pull request #11628 from ceph/wip-hammer-11567
+
+    hammer: rgw: Don't loop forever when reading data from 0 sized segment.
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 5f00e1d4c04698c68248844aad31f4b36eff5735
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Wed Feb 11 11:18:41 2015 +0800
+
+    osd: Add func has_flag in MOSDOp.
+
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (manually cherry picked just one line from d9a2ca5c5ae50dc34e184d577c7b24c56b6cc6ad)
+
+commit be7184589de9208986a2054491b86782dac938e8
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue May 24 14:02:32 2016 -0400
+
+    osd: reject PARALLELEXEC ops with EINVAL
+
+    Errors are better than crashing.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 407495197dd878b62f1356f6d939e33ab50d78c6)
+
+    Conflicts:
+	src/osd/ReplicatedPG.cc (trivial resolution)
+
+commit 8d352358edc156f70b5eb68c37bb56cb2392fce2
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue May 24 14:02:14 2016 -0400
+
+    ceph_test_rados_api_misc: test rados op with bad flas
+
+    Pass the bad PARALLELEXEC flag to remove(), which takes a flags arg.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1aa807f4f29b4b016cc737ffa443e8f30c3b7693)
+
+    Conflicts:
+	src/test/librados/misc.cc (dropped ioctx.remove() test because there is no
+				  remove() in hammer librados)
+
+commit bfda134f6844d6e238052e4f1900f3d2e7497d5e
+Merge: 9bf08dd 0d2f431
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Nov 24 09:15:50 2016 +0100
+
+    Merge pull request #10437 from odivlad/fix-doc
+
+    hammer: docs: Bucket object versions
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 03af4aae8103b52b19737c951e432efc7135b7ca
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Thu Oct 13 21:12:36 2016 -0400
+
+    Don't loop forever when reading data from 0 sized segment.
+
+    The 0 sized segment can arise depending on how
+    the client uploads the object in the first place.
+    The cpu loop then happens during a swift `GET'.
+
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    (cherry picked from commit 46c5f9773246522e66bb2cca49345d0b62a16c42)
+
+commit 8312dc61ceaa39dca675fda2cb7c88ec3c5ca9f7
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Mar 14 13:28:39 2016 -0700
+
+    common/TrackedOp: Move tracking_enabled check into register_inflight_op()
+
+    Make tracking_enabled (and the lock private.)*
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (Partial cherry picked from commit 4e8e6172e8f6cc2adc63a429c32f5edd03982f5e)
+
+    * Backport: Exclude locking since tracking_enabled can't change after launch
+
+commit c7220ccc0ec80bb6788befefe71f676c75457b70
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Mar 10 21:24:25 2016 -0800
+
+    common/TrackedOp: Handle dump racing with constructor
+
+    Use is_tracked to prevent TrackedOp::dump() from trying to call
+    virtual function while still in OpRequest constructor.
+
+    Fixes: #8885
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit ad13e05499669a79bde9a219ba1089f929e0388e)
+
+    Conflicts:
+	src/common/TrackedOp.cc
+	src/common/TrackedOp.h
+    Hammer can't use atomic<bool> so use atomic_t instead
+
+commit e8e7664247c7112f1ba03ce4cab915fe70ea5b8e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Mar 4 18:43:43 2016 -0800
+
+    common/TrackedOp: Missed locking when examining "events"
+
+    Another crash caused by a dump_ops_in_flight similar to #8885
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit be098c83143226e502a0587c9215cff941dbace1)
+
+    Conflicts:
+	src/common/TrackedOp.h (trivial)
+
+commit ce58688537d41e0662949f34b3595459581e74fc
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Jul 11 17:58:22 2016 -0700
+
+    CLEANUP: Move locking into dump_ops_in_flight()/dump_historic_ops()
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 17c645f70708134f50374cadb5505b0babe2265a)
+
+    Conflicts:
+	src/mds/MDSRank.cc (variation of code exists in mds/MDS.cc)
+	src/mon/Monitor.cc (no equivalent code here)
+
+commit 5717eb001c6202837fe2e22039e3f855ea4aa065
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Mar 14 15:47:02 2016 -0700
+
+    mds, osd: Fix missing locking for "dump_blocked_ops"
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit be5bed840f21da998be3b8374a8e33f208258e1b)
+
+    Conflicts:
+	src/mds/MDSRank.cc (missing dump_blocked_ops in mds)
+
+commit 6a7fbf42fdb253b7143cb9b3933990145358e06c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Mar 10 21:29:13 2016 -0800
+
+    osd: cleanup: Specify both template types for create_request()
+
+    Code consistency
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit cecdc49e6d2f38d5eb9f85218484f7a02bd2e1b7)
+
+commit 3e00ee5cfb0769e9b1c0e0a7540d0b14be880fed
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Fri Jul 17 10:44:06 2015 +0800
+
+    osd: add dump_blocked_ops asok command.
+
+    From ceph health we can see there are blocked ops and from 'ceph health
+    detai' we can see which osd has blocked ops. Later we only in log of osd
+    to see the detail message of blocked ops.
+    Now via asok dump blocked ops.
+
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit cf4e8c655c4a0e6c367c88338cd6e51a3474fa2f)
+
+    Conflicts:
+	src/common/TrackedOp.h (trivial)
+
+commit 656d0c9f307c1b3db072282be46decd72bc88810
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Wed Jul 8 13:12:49 2015 +0800
+
+    common/TrackedOp: Should lock ops_history_lock when access shutdown.
+
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 7fe72a2e7d116093ccff6683666e6289f5f12421)
+
+commit 9bf08ddc9422484f5cf0f05c8ccdb4b440e9b641
+Merge: 9655228 2581953
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 22:50:58 2016 +0100
+
+    Merge pull request #9873 from odivlad/fix-init-el7-hammer
+
+    hammer: build/ops: remove SYSTEMD_RUN from initscript
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 9655228d1900d56f4b439d41b2c4a532158f6c51
+Merge: e2c3e21 cb9cc4e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 22:49:29 2016 +0100
+
+    Merge pull request #10238 from aiicore/hammer
+
+    hammer: mon: fix memory leak in prepare_beacon
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit e2c3e21aa4cae160c0136d8d1652b7e4a2c000f0
+Merge: 637d348 c5639e4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 22:44:19 2016 +0100
+
+    Merge pull request #10569 from jschmid1/wip-16918-hammer
+
+    hammer: build/ops: make stop.sh more portable
+
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 637d348a98676cc9bf13346d96e7f83219e914c2
+Merge: 84df525 2b87a55
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 22:40:28 2016 +0100
+
+    Merge pull request #10724 from Vicente-Cheng/wip-16584-hammer
+
+    hammer: crush: reset bucket->h.items[i] when removing tree item
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 84df525e60ea66a4a1808641a12eccd95e16dd2d
+Merge: a7c8c51 8568e57
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 22:20:29 2016 +0100
+
+    Merge pull request #10904 from tchaikov/wip-14470-hammer
+
+    hammer: mon: return size_t from MonitorDBStore::Transaction::size()
+
+    Reviewed-by: Joao Eduardo Luis <joao@suse.de>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit a7c8c511ed061102e1b91858ca9772743d7e094e
+Merge: c8f54c8 a3ba985
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 22:18:42 2016 +0100
+
+    Merge pull request #10905 from SUSE/wip-11101-hammer
+
+    hammer: build/ops: improve ceph.in error message
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit c8f54c84c33354ca27d277a25a2193a3fbd80880
+Merge: 48b6054 6c0b2b4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 22:16:48 2016 +0100
+
+    Merge pull request #10987 from Vicente-Cheng/wip-16592-hammer
+
+    hammer: librbd: ceph 10.2.2 rbd status on image format 2 returns "(2) No such file or directory"
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 48b6054adf4a2d0c58a87bb2a9e8e8719febdc10
+Merge: a97c81c 10472f0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 22:13:05 2016 +0100
+
+    Merge pull request #11125 from tchaikov/wip-17179-hammer
+
+    hammer: tools: add a tool to rebuild mon store from OSD
+
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit a97c81c3f00abbb79cef0ae069a6f151a089e30e
+Merge: c4cee49 4a36933
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 22:10:54 2016 +0100
+
+    Merge pull request #11273 from aiicore/wip-17403-hammer
+
+    hammer: mon: OSDMonitor: Missing nearfull flag set
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit c4cee499622c3c151dcb363193af21844a2f24f5
+Merge: 9495584 ece0ec6
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 22:06:32 2016 +0100
+
+    Merge pull request #11618 from asheplyakov/hammer-16546
+
+    hammer: rbd: fix possible rbd data corruption
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 949558462317e16e1f517bda3ce3465fe30b931b
+Merge: 1fda4a0 5fb8fb4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:56:22 2016 +0100
+
+    Merge pull request #11676 from asheplyakov/hammer-16151
+
+    hammer: osd: update PGPool to detect map gaps and reset cached_removed_snaps
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 1fda4a0fcfe6a9d8f96fe0e054c5b9e8a6e2ecd8
+Merge: 11886bd 31c2594
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:54:39 2016 +0100
+
+    Merge pull request #11899 from SUSE/wip-17840-hammer
+
+    hammer: rgw: the value of total_time is wrong in the result of 'radosgw-admin log show' opt
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 11886bd5d20c8b2bf09dbabda1e49f14d2283260
+Merge: 21dc0ec f99a1d3
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:53:40 2016 +0100
+
+    Merge pull request #11927 from SUSE/wip-17764-hammer
+
+    hammer: osd: fix collection_list shadow return value
+
+    Reviewed-by: Haomai Wang <haomai@xsky.com>
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 21dc0ec953adeeabe934cd1090f959db7d11cefc
+Merge: 8f19eca 5711e79
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:50:04 2016 +0100
+
+    Merge pull request #12071 from SUSE/wip-17957-hammer
+
+    hammer: RWLock.h: 124: FAILED assert(r == 0) in rados-jewel-distro-basic-smithi
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 8f19eca51863ca1c1dfde068479bca5606e4550d
+Merge: af6e208 76eed80
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:47:24 2016 +0100
+
+    Merge pull request #11929 from SUSE/wip-17359-hammer
+
+    hammer: tools: ceph-objectstore-tool crashes if --journal-path <a-directory>
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit af6e2080f1ca56f1715ccb76d2372f86bdc272e9
+Merge: 655e904 fbf51ba
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:46:08 2016 +0100
+
+    Merge pull request #11930 from SUSE/wip-17346-hammer
+
+    hammer: cephx: Fix multiple segfaults due to attempts to encrypt or decrypt
+
+    Reviewed-by: Brad Hubbard <bhubbard@redhat.com>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 655e904142c295ef66c376a3aca148cf2bce60c0
+Merge: c118c0a 7ad7de6
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:44:39 2016 +0100
+
+    Merge pull request #11931 from SUSE/wip-17333-hammer
+
+    hammer: tools: crushtool --compile generates output despite missing item
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit c118c0af9da259330f5e5c3f3babdfca647a566b
+Merge: e4ea945 ad42da4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:43:19 2016 +0100
+
+    Merge pull request #11932 from SUSE/wip-17146-hammer
+
+    hammer: PG::choose_acting valgrind error or ./common/hobject.h: 182: FAILED assert(!max || (*this == hobject_t(hobject_t::get_max())))
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit e4ea9452cf78e01413714b597f49d32d1d11a6dd
+Merge: 92e0a47 e302e63
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:42:24 2016 +0100
+
+    Merge pull request #11933 from SUSE/wip-17142-hammer
+
+    hammer: osd: PG::_update_calc_stats wrong for CRUSH_ITEM_NONE up set items
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 92e0a471adffff1fa2027714116b3d0b5120b6bb
+Merge: d7fc569 64eeb8d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:40:47 2016 +0100
+
+    Merge pull request #11934 from SUSE/wip-17120-hammer
+
+    hammer: mon: %USED of ceph df is wrong
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit d7fc569f70b98cf2fa5a842c753f43adc801cdf7
+Merge: 8a15ac7 fc46591
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:39:12 2016 +0100
+
+    Merge pull request #11935 from SUSE/wip-16870-hammer
+
+    hammer: osd: crash on EIO during deep-scrubbing
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 8a15ac77c39cc3cc571c62bab68d327894f2db7a
+Merge: 44588f2 4c780bd
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:33:27 2016 +0100
+
+    Merge pull request #11937 from SUSE/wip-16428-hammer
+
+    hammer: mon: prepare_pgtemp needs to only update up_thru if newer than the existing one
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 44588f2755429e65960eebf51b21ddb14f245bb8
+Merge: 8c90170 a56b93f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:31:15 2016 +0100
+
+    Merge pull request #11938 from SUSE/wip-14323-hammer
+
+    hammer: msg: OpTracker needs to release the message throttle in _unregistered
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 8c90170685fa74fa338b4f22b349e62a01920220
+Merge: 49964b0 2332cf2
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 21:29:45 2016 +0100
+
+    Merge pull request #11939 from SUSE/wip-13927-hammer
+
+    hammer: mds: fix cephfs-java ftruncate unit test failure
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 0d2f431b2ecad03a6cda6ffbc9f9cbaac0956af9
+Author: Yuan Zhou <yuan.zhou@intel.com>
+Date:   Mon Jul 20 15:37:09 2015 +0800
+
+    rgw: doc: Mark S3 object version API as supported
+
+    S3 object version is already in since Hammer.
+    Ref: http://marc.info/?l=ceph-devel&m=143715290627682
+
+    Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>
+    (cherry picked from commit b62c3b9537e8ab33416a7fb0dd7b801a790d42ee)
+
+commit 8ca16052138bebe9f009104e62b147750dd6584d
+Author: shawn <chen.xiaowei@h3c.com>
+Date:   Thu Jul 30 10:09:12 2015 +0800
+
+    doc: add bucket object version description.
+    bucket object version has been supported, but do not have description in the docs,
+    so add this part.
+
+    Signed-off-by: shawn chen <cxwshawn@gmail.com>
+    (cherry picked from commit 5c395ff85627f0f4154af27aff03dd0a9c409fb4)
+
+commit 49964b01508c5f2f9b5edd620051c18fbea7dfd7
+Merge: 6e1ae86 d91c62d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 10:52:43 2016 +0100
+
+    Merge pull request #11946 from SUSE/wip-17883-hammer
+
+    hammer: mon: OSDs marked OUT wrongly after monitor failover
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 6e1ae86e88661ce9de5aefb38390b4bfbc677a0a
+Merge: 2f9869c ab0786d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 10:51:51 2016 +0100
+
+    Merge pull request #11948 from SUSE/wip-17671-hammer
+
+    hammer: rbd: export diff should open image as read-only
+
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 2f9869c1be5e03ca1d9daac9c541cfb226b2ae77
+Merge: 2dac884 815541f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 10:48:06 2016 +0100
+
+    Merge pull request #11949 from SUSE/wip-17374-hammer
+
+    hammer: librbd: image.stat() call in librbdpy fails sometimes
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 2dac8848e94b9dcf1a217c6c70164edf669a8eee
+Merge: 1503c2b da653fc
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 10:41:03 2016 +0100
+
+    Merge pull request #11950 from SUSE/wip-17123-hammer
+
+    hammer: rgw: COPY broke multipart files uploaded under dumpling
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 1503c2ba753b8ee31e7c632d0463691a83a22b8b
+Merge: 118beb9 009c844
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 10:39:46 2016 +0100
+
+    Merge pull request #11951 from SUSE/wip-16594-hammer
+
+    hammer: rgw: swift: ranged request on a DLO provides wrong values in Content-Range HTTP header
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 118beb9a390bf481e520d6888453eecc805bb0d7
+Merge: 3b226b8 3404da8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 10:17:00 2016 +0100
+
+    Merge pull request #11952 from SUSE/wip-16318-hammer
+
+    hammer: rgw: fix inconsistent uid/email handling in radosgw-admin
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 3b226b8282f1c994cf3fbb747131121ec9b0c7a0
+Merge: 9ad7369 d89b1f8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 10:14:10 2016 +0100
+
+    Merge pull request #12018 from trociny/wip-17068-hammer
+
+    hammer: librbd: request exclusive lock if current owner cannot execute op
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 9ad73698f57598ae1302aaf175cb96082eb64961
+Merge: 3ca7317 a7b182e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 06:42:06 2016 +0100
+
+    Merge pull request #11045 from SUSE/wip-17150-hammer
+
+    hammer: rgw: Anonymous user is able to read bucket with authenticated read ACL
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 3ca73177fc28327960f0a8c1612de0358051d515
+Merge: 708c549 d50a501
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Nov 23 06:32:09 2016 +0100
+
+    Merge pull request #11809 from SUSE/wip-17631-hammer
+
+    hammer: rgw: fix crash when client posts object with null condition
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 10472f0521f6aef516660a470be8c5fe453b1a76
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Nov 22 11:53:57 2016 +0800
+
+    PendingReleaseNotes: document the feature to rebuild monstore
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+
+commit 9473135aa65282499f67847edb52b7dfd8b374fb
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Oct 10 18:43:39 2016 +0800
+
+    doc: fill keyring with caps before passing it to ceph-monstore-tool
+
+    to make sure the recovered monitor store is ready for use.
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit af8e21163735377071b6832d8a81b035bb835257)
+
+commit 9c00ba17627c2b3526c558564954b8bb1d8472ef
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Oct 10 16:32:27 2016 +0800
+
+    tools/ceph_monstore_tool: bail out if no caps found for a key
+
+    we take it as an error if no caps is granted to an entity in the
+    specified keyring file when rebuilding the monitor db.
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b4bd4004a836121c11b0bb97d8123df54c271f04)
+
+commit 48696e9d6d89f6b6e99fb76ffa61665c026d7a0e
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Sep 30 17:58:14 2016 +0800
+
+    tools/ceph_monstore_tool: update pgmap_meta also when rebuilding store.db
+
+    we should rebuild pgmap_meta table from the collected osdmaps
+
+    Fixes: http://tracker.ceph.com/issues/17400
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit cdfa7a69f63d047205dcfccd63b5d58ab0d4695b)
+    Conflicts:
+	src/tools/ceph_monstore_tool.cc: remove C++11-ism
+
+commit ca67e8e88c06a9a37e5dc9e8fe5a1dd1821f8452
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Sun Sep 18 11:40:56 2016 +0800
+
+    tools/rebuild_mondb: kill compiling warning
+
+    As follow:
+
+    [ 72%] Building CXX object src/tools/CMakeFiles/ceph-objectstore-tool.dir/RadosDump.cc.o
+    /home/jenkins-build/build/workspace/ceph-pull-requests/src/tools/rebuild_mondb.cc: In function ‘int update_mon_db(ObjectStore&, OSDSuperblock&, const string&, const string&)’:
+    /home/jenkins-build/build/workspace/ceph-pull-requests/src/tools/rebuild_mondb.cc:289:22: warning: ‘crc’ may be used uninitialized in this function [-Wmaybe-uninitialized]
+	     if (have_crc && osdmap.get_crc() != crc) {
+			  ^
+    /home/jenkins-build/build/workspace/ceph-pull-requests/src/tools/rebuild_mondb.cc:238:14: note: ‘crc’ was declared here
+	 uint32_t crc;
+
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit f16a31476a3f9b44a7c3dabf0dfd2a0d015b11b9)
+
+commit 94390a3296483323d1b2c57856ea2e8ecca99b64
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Sun Sep 18 10:43:10 2016 +0800
+
+    tools/rebuild_mondb: return error if ondisk version of pg_info is incompatible
+
+    In this case "r" will be >= 0 and caller will be able to catch
+    this kind of error.
+
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit ac7c1dca0a4f6e92e50d362784122e2d3200e6b5)
+
+commit eaeb52057509b1fa45592ecc147ad694f57d33f3
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Sun Sep 18 10:33:56 2016 +0800
+
+    tools/rebuild_mondb: avoid unnecessary result code cast
+
+    In general we return negative codes for error cases, so there is
+    no need perform the cast here.
+
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 6a1c01d334fe65124043aa68a6e0cfaea43836b5)
+
+commit 828a4d7a07d178f2dcde30dfe96ca9f054bc1c0d
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Aug 31 00:59:58 2016 +0800
+
+    doc: add rados/operations/disaster-recovery.rst
+
+    document the process to recover from leveldb corruption.
+
+    Fixes: http://tracker.ceph.com/issues/17179
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 79a9f299253e24d20547131b3c9c9e0667e3b869)
+
+commit 3ef5dcd2b37055994794f9cf6139dc2d8807d637
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Aug 29 19:53:11 2016 +0800
+
+    tools/ceph_monstore_tool: add "rebuild" command
+
+    Fixes: http://tracker.ceph.com/issues/17179
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d909fa035c8fbbded786b2ca072acc10ea6b6052)
+
+    Conflicts:
+	ceph_monstore_tool.cc: remove C++11ish
+
+commit 6695d1792df9ca7cf5afd5a8c25a04f1eb9ff0fa
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Aug 29 19:52:19 2016 +0800
+
+    tools/ceph-objectstore-tool: add "update-mon-db" command
+
+    Fixes: http://tracker.ceph.com/issues/17179
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 24faea7ce446bbf09cbd4a9d3434dd5444a6c295)
+
+    Conflicts:
+	src/tools/Makefile-server.am: add rebuild_mondb.{cc,h}, as autotools
+	      support is dropped in master
+	ceph_objectstore_tool.cc, rebuild_mondb.cc: remove C++11ish
+	      adapt to hammer
+
+commit e7d9c15871efcd17b9b1064a5ee49edfe4a159e9
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Aug 31 13:11:24 2016 +0800
+
+    mon/AuthMonitor: make AuthMonitor::IncType public
+
+    so ceph-objectstore-tool is able to use it when rebuilding monitor
+    db.
+
+    Fixes: http://tracker.ceph.com/issues/17179
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 19ef4f16b3aba04119ac647cd6261c74a57ce829)
+
+commit 107e9c358acbc75f212f0bc6fcf83075933035b3
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Mar 15 11:29:17 2016 -0700
+
+    thrasher: Add dump_ops_enable options
+
+    Add dump_ops_enable which continuously dumps ops using 3 commands
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (Partial cherry picked from commit 7a528763d126eaca90e40b02aad02ab9c3c92a32)
+
+    Excluded dynamic optracker testing with optrack_toggle_delay option
+
+    Conflicts:
+	tasks/ceph_manager.py (trivial)
+	tasks/thrashosds.py (trivial)
+
+commit b7c87ae74a7f3f3c0096fe6cfe36e1077f96ea1f
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Apr 12 09:27:33 2016 -0700
+
+    ceph_manager: Add timeout to admin_socket/osd_admin_socket
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 4ad3b866041a3a5391f46cd3d96297834b95956b)
+
+commit f79027a229b2cbb16eaf877b06bf35ecff647bea
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Feb 25 16:10:51 2015 +0800
+
+    Thrasher: log backtrace of thrown exception
+
+    * add a wrapper to log uncaught exception to self.logger, greenlet also
+      prints the backtrace and exception to stderr, but teuthology.log does
+      not capture stderr. so we need to catch them by ourselves to reveal
+      more info to root-cause this issue.
+    * log uncaught exception thrown by Thrasher.do_thrash() to self.log.
+
+    See: #10630
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 64de3cd574b200d90cb3aea0d646a6236d8fec5e)
+
+commit 4533623b3df1e48b821a509013a91e8c3a7ec4a0
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Nov 21 16:58:48 2016 -0500
+
+    rados: remove btrfs tests; use xfs instead
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit fb7f49658815d5427f6454b20560991a785a8399
+Author: Zhiqiang Wang <zhiqiang.wang@intel.com>
+Date:   Mon Mar 30 15:25:34 2015 +0800
+
+    common/TrackedOp: checking in flight ops fix
+
+    Return earlier if tracking is not enabled when checking in flight ops.
+
+    Signed-off-by: Zhiqiang Wang <zhiqiang.wang@intel.com>
+    (cherry picked from commit 7509a6caa4964d71625c43f2ddc10e6274b6a21b)
+
+commit b58873efafabdc9791f500cff77be9a0a7002227
+Author: Zhiqiang Wang <zhiqiang.wang@intel.com>
+Date:   Mon Mar 30 14:41:21 2015 +0800
+
+    common/OpTracker: don't dump ops if tracking is not enabled
+
+    If tracking is disabled, there is nothing to dump. No need to go through
+    each sharded list. This is true for both in flight and historical ops.
+
+    Signed-off-by: Zhiqiang Wang <zhiqiang.wang@intel.com>
+    (cherry picked from commit ec3c40952bbc4b495875e08b57d764a7487c6c09)
+
+commit 24c92fea4bc68a27a240155f48ee5fa4ee3bbfa0
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Mon Jul 6 13:49:42 2015 +0800
+
+    common/TrackedOp: check tracking_enabled for event "initiated/done".
+
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit d7585c020de41f25e6a598661844169f2bdbb2b4)
+
+commit 0f37a397e35f105b5ea57c7a1a0463c95a2384e1
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Mon Jul 6 10:48:36 2015 +0800
+
+    common/TrackedOp: clean up code make look good.
+
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit edefd05edce6728774bb396ea4e6641ea53463cd)
+
+commit ece0ec60154d22bca7002cc308f3a0f4e36a053e
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Mon May 23 15:14:21 2016 -0700
+
+    hammer: ObjectCacher: fix bh_read_finish offset logic
+
+    If we have an incoming read split across multiple BufferHeads, we want to
+    line up the BufferHead's bl with the incoming OSDOp's bl at the right offset. We
+    were erroneously using this nonsense calculation (always equal to zero!) when
+    a much simpler comparison of the BufferHead's logical object offset to the
+    incoming OSDOp's logical offset will do the trick nicely.
+
+    Fixes: http://tracker.ceph.com/issues/16002
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 9ec6e7f608608088d51e449c9d375844631dcdde)
+
+commit 33f277e601199003fc226c84f67d1bbebb36ad2c
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Fri Jun 10 17:01:09 2016 -0700
+
+    hammer: test: build a correctness test for the ObjectCacher
+
+    For now it's very specifically designed to reproduce
+    http://tracker.ceph.com/issues/16002, but it can
+    be extended to other patterns in the future.
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 0fd55a9886dd8da344c23a5e9898ee5c5061e8f9)
+
+    Hammer specific changes:
+
+    * use utime_t instead of ceph::real_time
+    * ObjectCacher::prepare_write() has no journal_tid argument
+    * rewrite the code in C++98
+
+commit 439ce945dde196a4439e4468681e35eadd4fca5f
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Tue May 31 16:18:19 2016 -0700
+
+    hammer: test: split objectcacher test into 'stress' and 'correctness'
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit cc9aab1b0a22c3f7320046b97f75dccf2b86cc6d)
+
+commit ff3635a5457ac89ea0fb02f40f0ca723fe34e87c
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Thu Jun 23 13:41:46 2016 -0700
+
+    hammer: test: add a data-storing MemWriteback for testing ObjectCacher
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit ed5801492bf2850e80a328ce5a61cb1ff2709215)
+
+    Hammer specific changes:
+
+    * use utime_t instead of ceph::real_time
+    * ObjectCacher::prepare_write() has no journal_tid argument
+    * rewrite code in C++98
+
+commit 7a2abfac759acaf668c6ace25df94619f27e829d
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Oct 28 17:06:59 2015 +0800
+
+    hammer: objectcacher: introduce ObjectCacher::flush_all()
+
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit bd86c301a54a6e23cf2ce257b8024b6b92dfbe11)
+
+    Conflicts:
+	src/osdc/ObjectCacher.h - whitespace only changes
+
+    Semantic conflicts:
+	    ObjectCacher::flush_all(): dirty_or_tx_bh is a set<BufferHead *>
+	in Hammer
+
+commit 9df96606edfb9ff9932297b8253a297cf75c02f6
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Thu Jun 23 14:23:51 2016 -0700
+
+    hammer: osd: provide some contents on ObjectExtent usage in testing
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit a555d9a0c7d3b6c2206520cf78445234b0834639)
+
+commit c5639e4e276a8d5f6457ce9f63b3e77a230a669c
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Dec 14 09:41:10 2015 +0000
+
+    stop.sh: make more portable
+
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 27f2d806f905df1a51b11bd79f1af4c26471407a)
+
+commit 708c549b8d77b9a9ea4d46640287848a74898a40
+Merge: f776699 0ac7f2f
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Nov 21 09:03:44 2016 -0600
+
+    Merge pull request #12006 from SUSE/wip-17905-hammer
+
+    hammer: mon: MonmapMonitor should return success when MON will be removed
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit d1e1207275eeaf2409b435236c579116900eb6b7
+Author: Ning Yao <yaoning@unitedstack.com>
+Date:   Thu Oct 13 16:53:18 2016 +0800
+
+    FileStore:: fix fiemap issue in xfs when #extents > 1364
+
+    Fixes: #17610
+    Backport: jewel, hammer
+    Signed-off-by: Ning Yao <yaoning@unitedstack.com>
+    (cherry picked from commit 1a1c126d80b427d5230347fbc71a4edea5c0b6c8)
+
+    Conflicts:
+	    src/os/FileStore.cc
+		in hammer, there is no _do_seek_hole_data() function so remove it
+		in hammer, the logic is in FileStore::fiemap not in _do_fiemap()
+		so port the logic to the else branch in FileStore::fiemap
+
+commit a3ba985c38c07f85056f90d125e32ea277c65932
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jan 5 15:00:59 2016 +0800
+
+    ceph.in: improve the error message
+
+    Fixes: #11101
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit e19e8f1553fdf8b6b642d3222d28f242fab696f0)
+
+commit 5711e794218cc83db023e1d6600ed75ef3ac3fcc
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 29 15:24:34 2016 -0400
+
+    os/ObjectStore: fix _update_op for split dest_cid
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e3dc7c772f563f97bc68ebc6dc6e0d408e7c11f3)
+
+commit cb9cc4e57a2abd5c13f61599a7fe594c5f21c9c3
+Author: Igor Podoski <igor.podoski@ts.fujitsu.com>
+Date:   Mon Jul 11 12:37:25 2016 +0200
+
+    mon/MDSMonitor: fix memory leak in prepare_beacon
+
+    prepare_beacon() case of prepare_update() should put()
+    message in two more cases, because is the last step
+    of dispatch()
+
+    Fixes: http://tracker.ceph.com/issues/17285
+    Signed-off-by: Igor Podoski <igor.podoski@ts.fujitsu.com>
+
+commit 4a36933df8b9338c0a32d582e005894442092d0f
+Author: Igor Podoski <igor.podoski@ts.fujitsu.com>
+Date:   Wed Sep 14 07:36:43 2016 -0400
+
+    mon: OSDMonitor: Missing nearfull flag set
+
+    Output from 'ceph -s -f json-pretty' wans't showing nearfull flag set.
+
+    Signed-off-by: Igor Podoski <igor.podoski@ts.fujitsu.com>
+    (cherry picked from commit 754887b575795ef96cda095f7f0d8c898e20f00f)
+
+commit e1694d6e85694ee8750b2af211505b89b3d6f0c8
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Oct 13 13:19:27 2016 +0800
+
+    mon: send updated monmap to its subscribers
+
+    prior to this change, we sends monmap when serving the subscription
+    requests, but the updates are not sent to the subscribers anymore.
+    so we need to send latest monmap to subscribes and update the
+    subscription status accordingly when the monmap is updated.
+
+    http://tracker.ceph.com/issues/17558
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit c2b348b12c47ba505f72228ae0ea9b1a1b674719)
+    Conflicts:
+	src/mon/Monitor.cc: resolve conflicts
+	src/mon/MonmapMonitor.cc: remove C++11-ism
+
+commit ab0786d6a023e391e61119bd309985dcf6e05a4d
+Author: YankunLi <lioveni99@gmail.com>
+Date:   Tue Sep 6 16:16:58 2016 +0800
+
+    rbd: this command should be EXPORT_DIFF
+
+    Fixes: http://tracker.ceph.com/issues/17671
+
+    Signed-off-by: liyankun <lioveni99@gmail.com>
+
+commit d91c62dcf5b1c98b4cfed4eebf29ff9a9d64289b
+Author: Dong Wu <archer.wudong@gmail.com>
+Date:   Thu Oct 27 11:02:36 2016 +0800
+
+    mon: update mon(peon)'s down_pending_out when osd up
+
+    Fixes: http://tracker.ceph.com/issues/17719
+    Signed-off-by: Dong Wu <archer.wudong@gmail.com>
+    (cherry picked from commit 664254a17afbad86983ea5b5b8d385662d89e65e)
+
+    Conflicts:
+	src/mon/OSDMonitor.cc (auto declaration for variable "found" is a C++11ism,
+	    replace with the full verbose C++98 type)
+
+commit d89b1f848b8c942d150076cef99491e4c5031bda
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Nov 16 12:01:00 2016 +0200
+
+    librbd: request exclusive lock if current owner cannot execute op
+
+    This is a direct commit to hammer due to librbd code has evolved
+    significantly in the master.
+
+    Fixes: http://tracker.ceph.com/issues/17068
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 2b87a551c603291d6c018ed72b0c58e8e45bc334
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Jul 1 20:44:35 2016 +0800
+
+    crush: reset bucket->h.items[i] when removing tree item
+
+    * crush: so we don't see the reference after the removing, this keeps
+      check_item_loc() happy, and move_bucket() use check_item_loc() to see if
+      the removed bucket disappears after the removal.
+    * test: also add unittest_crush_wrapper::CrushWrapper.insert_item
+
+    Fixes: http://tracker.ceph.com/issues/16525
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit a7069c7aab6b3f605f3d8f909af96f87977e8698)
+
+    Backport change: modify some codes that only works on c++ 11
+
+commit 0ac7f2f6c87e0d454466927b8528b3f3dae812a2
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Wed Nov 2 15:33:52 2016 +0000
+
+    mon: MonmapMonitor: return success when monitor will be removed
+
+    Fixes: http://tracker.ceph.com/issues/17725
+
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit c9d46cfbf2512bc3495c6901de2b8f711bef9bae)
+
+    Conflicts:
+	src/mon/MonmapMonitor.cc (just inserted the "err = 0" assignment
+	    immediately after the error message line)
+
+commit 6c0b2b4ceba188397d5877a602b0e1996d29a4eb
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 4 20:40:10 2016 -0400
+
+    librbd: prevent creation of v2 image ids that are too large
+
+    The librbd API is capped at 24 characters for expressing the
+    object prefix for data blocks (including trailing null byte).
+
+    Fixes: http://tracker.ceph.com/issues/16887
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4de7c8d0a7222c83268d03b99015c6b9d25f124d)
+
+commit 5fb8fb4e450949ebc4c724c102a5b2ae6f1d06c8
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu May 19 16:00:35 2016 -0700
+
+    PG: update PGPool to detect map gaps and reset cached_removed_snaps
+
+    Fixes: http://tracker.ceph.com/issues/15943
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 5798fb3bf6d726d14a9c5cb99dc5902eba5b878a)
+
+    Conflicts:
+	src/osd/OSD.cc: trivial, fuzz is too big so git prefers to be safe
+
+commit 3404da8f1e491df86c8a1f12606546cd5a7d952c
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Jan 18 15:58:07 2016 -0500
+
+    rgw-admin: return error on email address conflict (add user)
+
+    Fixes the error return inconsistency documented in upstream
+    Ceph issue #13598
+
+    Revised after upstream review, permits changing email address as
+    originally (but use case-insensitive comp).
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 90c143efa2156e55dc017ebe81005315cacaf2c5)
+
+    Conflicts:
+	src/rgw/rgw_user.cc (there is no op_state.found_by_email
+	    conditional in hammer)
+
+commit f5bf5c0f9e9db788fd556f9e92f805fc215138f6
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Jan 18 15:06:19 2016 -0500
+
+    rgw-admin: convert user email addresses to lower case
+
+    Fixes the email address inconsistency documented in upstream
+    Ceph BUG #13598
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 4c438dbbc0e6eda6b9a3018d60019a1a780d6f65)
+
+commit 009c844e34bcd3e4e34eace90f90a140484e2794
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Mon Oct 12 18:43:27 2015 +0200
+
+    rgw: fix wrong length in Content-Range HTTP header of Swift's DLO.
+
+    Fixes: #13452
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit b48f3d774129e0b37531fccda513626bd8ef07e5)
+
+commit 2175b688358455472ac6320bb82ac6bef7133e59
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Mon Oct 12 17:32:21 2015 +0200
+
+    rgw: fix wrong first byte pos in Content-Range HTTP header of Swift's DLO.
+
+    Fixes: #13452
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 66f47f35bbb4966cb289811672938b90ae6d44ae)
+
+commit da653fc3594d49bdac5dc00129c8aa5fd227d6c0
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jul 7 18:01:54 2016 -0700
+
+    rgw: adjust manifest head object
+
+    adjust the manifest head object:
+     - when reading manifest, set the head object to the object
+       we read the manifest from (and its size). Some manifests are
+       broken and point at a different object
+     - when copying multipart object, set the manifest head object to
+       point at the new head object
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 419264586fc46337322f0d60b0ffcdcce3bb5c5a)
+
+commit 8f1c915fa47a889cb50cf84e2d1230df7443752c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jul 7 15:49:07 2016 -0700
+
+    rgw: adjust objs when copying obj with explicit_objs set
+
+    If the head object contains data, need to set it in the list of
+    objs (in addition to being pointed at by head_obj).
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 1e012112b99004745952accc6fb11ada5e8e0045)
+
+commit 43a5d5eb2835e1750ef4fbe6d8d3d85af02c6b30
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jul 7 15:36:33 2016 -0700
+
+    rgw: patch manifest to handle explicit objs copy issue
+
+    Fixes: http://tracker.ceph.com/issues/16435
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit b51476466e5bb03bbaa6e428bb6bb189a259b9fd)
+
+commit 815541feb989442bdf4d30faf603ef2795bc1ade
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 20 07:25:36 2016 -0400
+
+    librbd: block name prefix might overflow fixed size C-string
+
+    The issue which resulted in too large v2 image ids was fixed
+    under #16887.
+
+    Fixes: http://tracker.ceph.com/issues/17310
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 61734d266c6ee476c2f5fcfbbaefc7d0c7939617)
+
+commit a56b93faf2ae8db2ebe55aaf6003a48ac2bfb8b2
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Jan 5 16:24:15 2016 -0800
+
+    OpRequest: release the message throttle when unregistered
+
+    We don't want messages in the OpTracker history hanging on to
+    message throttle.
+
+    Fixes: #14248
+    Backport: hammer, firefly
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 67be35cba7c384353b0b6d49284a4ead94c4152e)
+
+    Conflicts:
+	src/msg/Message.h (s/nullptr/0/ because C++98)
+
+commit 4c780bd7fc6e8a98334ddd049a4969d88a9de49f
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Jun 7 17:15:05 2016 -0700
+
+    OSDMonitor::prepare_pgtemp: only update up_thru if newer
+
+    Fixes: http://tracker.ceph.com/issues/16185
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 5f2bd7b6b28aad96d68444b22c04b8b24564616b)
+
+    Conflicts:
+	src/mon/OSDMonitor.cc (master uses C++11 "auto" for new_up_thru
+	    iterator, replace with explicit type)
+	src/mon/OSDMonitor.h (trivial resolution)
+
+commit 7ad7de69631372ef364d19bf023e7dba3c88092c
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 20 14:10:16 2016 +0800
+
+    crush/CrushCompiler: error out as long as parse fails
+
+    do not output compiled crush map if anything goes wrong when parsing
+    crush map.
+
+    Fixes: http://tracker.ceph.com/issues/17306
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b2c0a079dc074c907e1bc429861230096534f597)
+
+    Conflicts:
+	src/test/cli/crushtool/compile-decompile-recompile.t (hammer needs $TESTDIR/ prefix)
+
+commit 2332cf22d23fd40ae238b98798e3290094548914
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Mar 30 17:35:59 2015 +0800
+
+    mds: fix out-of-order messages
+
+    When MDS is no longer laggy, it should process deferred messages
+    first, then process newly received messages.
+
+    Fix: #11258
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit ccdeaf87df8b66e09f6b20950b57ac61bf213086)
+
+commit fc4659155406432404d6e7afb63de545e4abf1a6
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Nov 12 12:37:00 2016 +0100
+
+    pg: restore correct behavior of read() callers
+
+    This patch is inspired by fabd6357e42e526d2704d7cb80375c12d731df8d but is not a
+    cherry-pick.
+
+    Fixes: http://tracker.ceph.com/issues/16870
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 64eeb8d96cf295c8a3739d5d993b51f77c35e466
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Aug 5 14:27:43 2016 +0800
+
+    mon/PGMonitor: calc the %USED of pool using used/(used+avail)
+
+    we were using
+    "the raw space used by this pool" / "the raw space of the whole cluster"
+    as the %USED. but it's wrong:
+
+    - there is chance that not all OSDs are assigned to the pool in question
+    - we can not utilize all the raw space for the pool: there is overhead.
+      and the size of available space for a pool is capped by the assigned
+      OSD with minimal free space.
+
+    so we should use
+     USED / (USED + AVAIL)
+    as the %USED. so once we have no AVAIL space left, %USED will be 100%.
+
+    Fixes: http://tracker.ceph.com/issues/16933
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 08532ea166dfd97c548d9f1fb478e163021cdda3)
+
+    Conflicts:
+	src/test/mon/CMakeLists.txt (file does not exist in hammer)
+
+commit d62258c03492fca5fca99650cc7514348251cb7b
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sun Aug 7 00:58:04 2016 +0800
+
+    mon/PGMonitor: mark dump_object_stat_sum() as static
+
+    so we can test this method without create an instance of PGMonitor.
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit af3d3e25823ca9b40ddf09835edb82795ac68f33)
+
+    Conflicts:
+	src/mon/PGMonitor.cc ("const" suffix of function definition was removed
+		in master, but in hammer it wasn't there, so no change necessary)
+	src/mon/PGMonitor.h (added static prefix to function declaration; did
+		not remove "const" suffix because it wasn't there in hammer)
+
+commit e302e636b8fd8f3106a3971dfc44f2c91995e3f5
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Aug 11 08:57:51 2016 -0700
+
+    PG: use upset rather than up for _update_calc_stats
+
+    Fixes: http://tracker.ceph.com/issues/16998
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 200cae8c9db99b3dede377f74413cc0b15033e1f)
+
+commit cfa2b2d3bcad8ebf0a94b89b3a27ce72445fc30a
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Aug 11 08:57:35 2016 -0700
+
+    PG: introduce and maintain upset
+
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 80a5c21d97116e7a66351a0e1f5d9b561f8663ac)
+
+commit ad42da47909f2ce6902c9f34e2e9e0c445bb1cc7
+Author: Tao Chang <changtao@hihuron.com>
+Date:   Mon May 9 09:50:17 2016 -0400
+
+    Cleanup: delete find_best_info again
+
+    After called find_best_info find a auth_pg_shard, it must be not incomplete,
+    so it will not enter find_best_info again.
+
+    Signed-off-by: Tao Chang <changtao@hihuron.com>
+    (cherry picked from commit a25cfc4d46c03d8d78e0254c728ea5c29e2246e1)
+
+    Conflicts:
+	src/osd/PG.cc (trivial resolution)
+
+commit fbf51ba878c7a00c045fc97bf309c39b0d9c4fcc
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Tue Jun 14 17:34:44 2016 +1000
+
+    cephx: Fix multiple segfaults due to attempts to encrypt or decrypt
+    an empty secret and a null CryptoKeyHandler
+
+    Fixes: http://tracker.ceph.com/issues/16266
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 009e777fbd18602e5fd66f97bdad95e977e6fecc)
+
+commit 76eed80e6649fdd53a2a3ee3d7242a8e48e1168a
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 20 17:39:24 2016 +0800
+
+    os/filestore/FileJournal: fail out if FileJournal is not block device or regular file
+
+    otherwise JournalingFileStore will assert when deleting FileJournal
+    which still has the non block/regular file opened.
+
+    Fixes: http://tracker.ceph.com/issues/17307
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 7431eec6fd24cd08ca6c76a9893e3f6e8c63a916)
+
+commit f99a1d364ae8ec0f7bad220fdd6d665ecdc8029c
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Sun Oct 9 18:31:54 2016 +0800
+
+    osd/PGBackend: fix collection_list shadow return value
+
+    Fixes: http://tracker.ceph.com/issues/17713
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 8a53a604f375eb568a6fc88a402a6afd83fa151a)
+
+    Conflicts:
+	src/osd/PGBackend.cc (master is using store->collection_list and ch;
+		hammer is using store->collection_list_partial and coll)
+
+commit a7b182e00639e635092fa40c39073859720215af
+Author: root <rahul.1aggarwal@gmail.com>
+Date:   Thu Sep 24 00:21:13 2015 +0530
+
+    13207: Rados Gateway: Anonymous user is able to read bucket with authenticated read ACL
+
+    Signed-off-by: root <rahul.1aggarwal@gmail.com>
+    (cherry picked from commit 99ba6610a8f437604cadf68cbe9969def893e870)
+
+commit 31c25941d551c61df4eabdb6c3e2226ef4b3cf02
+Author: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+Date:   Tue Oct 11 19:32:43 2016 +0800
+
+    rgw: fix the field 'total_time'  of log entry in log show opt
+
+    Fixes: http://tracker.ceph.com/issues/17598
+
+    Signed-off-by: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+    (cherry picked from commit 8f4825c5f012c77a3a91e22706dcab9c6067d77e)
+
+commit d50a5018c43d200e4cd36daa7826b2b4c17434ed
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Oct 20 10:17:36 2016 -0700
+
+    rgw: handle empty POST condition
+
+    Fixes: http://tracker.ceph.com/issues/17635
+
+    Before accessing json entity, need to check that iterator is valid.
+    If there is no entry return appropriate error code.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 23cb642243e09ca4a8e104f62a3bb7b2cbb6ea12)
+
+commit 258195373a6aaabe9a74805268d8094bf48fae0a
+Author: Vladislav Odintsov <odivlad@gmail.com>
+Date:   Mon Jul 4 15:29:22 2016 +0300
+
+    init-radosgw: do not use systemd-run in sysvinit
+
+    `systemd-run` logic in initscripts was introduced because of ticket
+    http://tracker.ceph.com/issues/7627.
+
+    If we have systemd-based operating system, we should use systemd unit files
+    from systemd directory to start/stop ceph daemons.
+    Otherwise, `daemon()` from `/etc/init.d/functions` on systemd distro starts service
+    in `system.slice` and everything works well.
+
+    `systemd-run` can not be run on non-systemd distros, so it's not needed
+    on SysV systems.
+
+    also, ceph-disk is now able to run the "systemctl"
+    to enable and start the ceph-osd, and ceph-deploy is also now
+    playing well with systemd when it comes to ceph-mon and ceph-mds
+
+    Fixes: http://tracker.ceph.com/issues/16440
+
+    Signed-off-by: Vladislav Odintsov <odivlad@gmail.com>
+    (cherry picked from commit 1fd4f92a025e80092b8d08d9b7da2e0b73a52f0f)
+
+commit f776699fd00ff5f0deea15ec4f2efb991831b055
+Merge: 8e13e7c bfa7c14
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sat Oct 29 10:24:18 2016 +0200
+
+    Merge pull request #11372 from ceph/wip-17386-hammer
+
+    hammer: doc: add "Upgrading to Hammer" section
+
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit bfa7c14894ebab3ab58282782d8217438a3c4047
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Oct 27 15:44:09 2016 +0800
+
+    doc: add "to Hammer" upgrade section
+
+    Fixes: http://tracker.ceph.com/issues/17534
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+
+commit 8e5872bf6c18b410136fd76974ae11e645c09928
+Merge: 995d3a9 2c27626
+Author: Zack Cerza <zack@cerza.org>
+Date:   Wed Oct 19 11:36:07 2016 -0600
+
+    Merge pull request #1215 from jcsp/wip-generate-caps-hammer
+
+    tasks/ceph: move generate_caps from teuthology
+
+commit 2c27626c1e5d9989ac6942a94b6d9198ccbd28fb
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Sep 27 12:17:22 2016 +0100
+
+    tasks/ceph: move generate_caps from teuthology
+
+    This was only used in this task, and it is much too
+    ceph-specific to belong in teuthology.
+
+    Fixes: http://tracker.ceph.com/issues/17614
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 298cc8f932a2a327e07f5b73708404588a611322)
+
+commit e7253b8ecb49bf894dbc21d260a767ecd8d3eb19
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 3 16:43:44 2016 -0700
+
+    rgw_admin: add bi purge command
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+    Conflicts:
+	src/rgw/rgw_admin.cc
+
+commit 4f605ebf3f415943e8c151364dadfdce3b69cde5
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Sep 28 12:54:47 2016 -0700
+
+    rgw: bucket resharding, adjust logging
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 6503430d1b47630b12164f46702bbb42dd7bb453
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Sep 28 10:41:45 2016 -0700
+
+    cls/rgw: bi_list() fix is_truncated returned param
+
+    is_truncated was never set. Also, make sure that we don't return
+    more entries than requested.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit f07e22ae7beab634068cc37a975356393c8b60bf
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 27 15:13:37 2016 -0700
+
+    rgw_admin: require --yes-i-really-mean-it for bucket reshard
+
+    in the case where num shards are less or equal to current bucket
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 55e3e7742b22fd0f015f7439e7aeabdce1845caa
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 27 14:35:31 2016 -0700
+
+    rgw_admin: better bucket reshard logging
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+    Conflicts:
+	src/rgw/rgw_admin.cc
+
+commit 75f69da3ecba55992453f0a901f997e8a1737884
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 27 14:11:41 2016 -0700
+
+    rgw: limit bucket reshard num shards to max possible
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+    Conflicts:
+	src/rgw/rgw_admin.cc
+	src/rgw/rgw_rados.h
+
+commit b6a15514e7cad6c8fd84f589fd5e4c7b505045e7
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 27 11:55:32 2016 -0700
+
+    rgw_admin: fix bi list command
+
+    Changes scoping of json section, and push the --object param in so that
+    an object can be specified as a filter.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+    Conflicts:
+	src/rgw/rgw_admin.cc
+
+commit 6e91760e045b61a144d669f1caa5a55e10b13e58
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 15:49:37 2016 -0700
+
+    rgw_admin: use aio operations for bucket resharding
+
+    also created shards manager to make things slightly cleaner
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+    Conflicts:
+	src/rgw/rgw_admin.cc
+
+commit a68d56febde154b3b92d3178d2e877e86b3e2dd9
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 10:45:17 2016 -0700
+
+    rgw: bucket reshard updates stats
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+    Conflicts:
+	src/cls/rgw/cls_rgw_types.h
+	src/rgw/rgw_admin.cc
+
+commit 858bcf189afb9fd042c370bb1f53864447dcdc96
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 09:52:29 2016 -0700
+
+    cls/rgw: add bucket_update_stats method
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 37cda4ec1fee1d6e413d1ffe8c5f4f6558634740
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat Sep 24 10:46:36 2016 -0700
+
+    rgw_admin: reshard also links to new bucket instance
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 6c3e734cd87138b55a8a65951d76f7f91c9f0d93
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat Sep 24 10:33:57 2016 -0700
+
+    rgw: rgw_link_bucket, use correct bucket structure for entry point
+
+    The bucket structure might be different than the one that we were using
+    before.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+    Conflicts:
+	src/rgw/rgw_bucket.cc
+
+commit 7cc1de860dce16e3956fe604b3bcb8ed1a2fe9ea
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat Sep 24 10:01:00 2016 -0700
+
+    radosgw-admin: bucket reshard needs --num-shards to be specified
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+    Conflicts:
+	src/rgw/rgw_admin.cc
+
+commit b2cc000bbdb7e02e3fa9bdc7d7d71d3e979fee2b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat Sep 24 05:49:37 2016 -0700
+
+    cls/rgw: fix bi_list objclass command
+
+    was filtering entries, even if filter was not specified, and need to
+    set boundary for plain entries. Also, list_instance_entries() was not
+    working correctly, and added list_olh_entries().
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 53982a2e61614398de2e37705da12aa6874360a0
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 16:09:34 2016 -0700
+
+    rgw_admin: bucket rehsrading, initial work
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+    Conflicts:
+	src/rgw/rgw_admin.cc
+
+commit 995d3a9c49c04d6ded62aa34cfaacc650b7d13a6
+Merge: c2a8f05 e4038a0
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Sep 29 21:34:49 2016 -0700
+
+    Merge pull request #1192 from ceph/wip-offline-split-hammer2
+
+    ceph_manager: test offline split via ceph-objectstore-tool
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 8e13e7ce98e99d3ef1b2942f06b4c9ab35c12066
+Merge: e2d4f0a 99bb9f6
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Sep 29 21:34:26 2016 -0700
+
+    Merge pull request #11253 from ceph/wip-offline-split-hammer2
+
+    ceph-objectstore-tool: add a way to split filestore directories offline
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit e4038a0c0068569cc1c7bcaf1b590fba31291610
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri Aug 5 18:16:20 2016 -0700
+
+    ceph_manager: test offline split via ceph-objectstore-tool
+
+    When killing an osd, split all pools with a low threshold.
+    This will slow down tests, but should not impact correctness.
+
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit f28f881bda0ad426bfa9c0e22cc4658b2ce2ec5d)
+
+commit 99bb9f603f2123e81470a819c1cd64df13a22425
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri Aug 5 11:45:00 2016 -0700
+
+    ceph-objectstore-tool: add a way to split filestore directories offline
+
+    Use the usual split code, and split each dir that meets the
+    usual split criteria.
+
+    This can be run with lower than usual split settings, to avoid costly
+    online splits. To make sure the directories are not merged again, use
+    a load merge threshold (e.g. filestore merge threshold = 1), and
+    adjust the split multiplier accordingly.
+
+    Fixes: http://tracker.ceph.com/issues/17220
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit e7b0428e0e8d8f5459311dc698d94a3ac4f04684)
+
+    Conflicts:
+	src/tools/ceph_objectstore_tool.cc
+    * remove c++11 auto usage
+    * change HashIndex::list_subdirs() call to use set instead of vector
+    * adjust to hammer signature of coll_t::is_pg()
+
+commit c1b64c8f9c23473b758300958a94a172f1d39b64
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 10 12:52:43 2015 -0700
+
+    rgw: rgw_obj encoding fixes
+
+    Fix a few issues:
+     - conversion to and from rgw_obj_key
+     - strip_namespace_from_object() in the case where object starts
+       with underscore
+     - encoding if object has either namespace or instance and starts
+       with underscore
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit c381cc5d69ebec4370ee18e350b9d003e15b2770
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 16:09:15 2016 -0700
+
+    rgw: utilities to support raw bucket index operations
+
+    and other related changes.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+    Conflicts:
+	src/rgw/rgw_rados.cc
+	src/rgw/rgw_rados.h
+
+commit d41cbb44c55cf2dca5ab25d5a1ec76e30db26150
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 16:06:17 2016 -0700
+
+    rgw: use bucket_info.bucket_id instead of marker where needed
+
+    We used to use these interchangeably, but they actually have diffent meaning.
+    The marker is the prefix we assign to the objects in this bucket, whereas
+    the bucket_id is the bucket instance's id. These used to hold the same
+    value, but with bucket resharding it's not going to be true anymore.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 228f52cca1866c40a3840e06bd260c1d92d59caa
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 16:01:39 2016 -0700
+
+    cls/rgw: utilities to support raw bucket index operations
+
+    New flavour of bi_put() call, and a function to extract key off
+    a raw bi entry.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit e2d4f0af387869569a8c143fa2d2873e6788d2a2
+Merge: 8f66abd 0b42082
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Tue Sep 20 10:33:02 2016 -0400
+
+    Merge pull request #11140 from cbodley/wip-radosgw-admin-man-hammer
+
+    hammer: add orphan options to radosgw-admin --help and man page
+
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 0b420821fd3ed33d077ce76beef7a6f1a0e5edb7
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Sep 20 09:27:52 2016 -0400
+
+    man: add orphan options to radosgw-admin.8
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+
+commit 054e0d45c351525454517472df2c0b1dcecb5c39
+Author: tserlin <tserlin@redhat.com>
+Date:   Mon Sep 19 14:40:12 2016 -0400
+
+    Add two options to radosgw-admin.rst manpage
+
+    Add '--job-id' and '--max-concurrent-ios' to Orphan Search Options
+
+    Fixes: http://tracker.ceph.com/issues/17281
+    Signed-off-by: Thomas Serlin <tserlin@redhat.com>
+    (cherry picked from commit 697f30d86f49b73c981c06375ab2937570b1db01)
+
+commit 2920334210376bb34e59d6ecf08557732bcc3861
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Thu Sep 15 12:56:39 2016 -0600
+
+    radosgw-admin: add "--orphan-stale-secs" to --help
+
+    The radosgw-admin --help did not include the description of the
+    `--orphan-stale-secs` option of the `orphans find` command. The option
+    sets the number of seconds to wait before declaring an object to be an
+    orphan.
+
+    Fixes: http://tracker.ceph.com/issues/17280
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 354059ae43b4f4cc797da1669715399cd96a4738)
+
+commit 34ecb3584230db8458f193ac01deff9264cd119a
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Fri Apr 1 13:46:30 2016 +0200
+
+    rgw_admin: improve the orphans find help
+
+    Adding  a few missing commandline switches and improve the help message
+
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit 27e4c46f741a5f908d5c8ca1af227b3581c697f6)
+
+commit 3575fa8112ac39b1c6edf2aaee5b02aa67fa5766
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Thu Sep 15 11:40:14 2016 -0600
+
+    doc: add "--orphan-stale-secs" to radosgw-admin(8)
+
+    The radosgw-admin(8) manual page did not include the description of the
+    `--orphan-stale-secs` option of the `orphans find` command. The option sets
+    the number of seconds to wait before declaring an object to be an
+    orphan.
+
+    Fixes: http://tracker.ceph.com/issues/17280
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit a676c516069cc448591018ecf4f7d1f7f7bc3bfd)
+
+commit 8f66abd02ed5f0bc697b80b3071f0c70886aab6e
+Merge: fe6d859 c7f6b47
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Wed Sep 7 13:06:33 2016 -0400
+
+    Merge pull request #10839 from oritwas/hammer-default-quota
+
+    Hammer default quota
+
+    Reviewed-by: Daniel Gryniewicz <dang@redhat.com>
diff --git a/doc/changelog/v0.94.2.txt b/doc/changelog/v0.94.2.txt
new file mode 100644
index 000000000..f8113af7f
--- /dev/null
+++ b/doc/changelog/v0.94.2.txt
@@ -0,0 +1,1563 @@
+commit 5fb85614ca8f354284c713a2f9c610860720bbf3 (tag: refs/tags/v0.94.2, refs/remotes/gh/hammer)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Tue Jun 9 12:32:34 2015 -0700
+
+    0.94.2
+
+commit d967cecf0a5d7fbf992a0195341cbd893a358264
+Merge: eb69cf7 968573b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri May 29 09:49:43 2015 +0200
+
+    Merge pull request #4795 from dachary/wip-11806-hammer
+    
+    ceph / ceph-dbg steal ceph-objecstore-tool from ceph-test / ceph-test-dbg
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 968573b8930a7c8485bf53e3a989ce2f7d0a2fff
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu May 28 10:35:51 2015 +0200
+
+    debian: ceph-dbg steals ceph-objectstore-tool from ceph-test-dbg
+    
+    When ceph-objectstore-tool was moved from ceph-test to
+    ceph by 61cf5da0b51e2d9578c7b4bca85184317e30f4ca, the ceph package in
+    debian/control was updated accordingly, as recommended by
+    https://www.debian.org/doc/debian-policy/ch-relationships.html#s-replaces
+    
+    The same must be done for the ceph-dbg package because
+    /usr/lib/debug/usr/bin/ceph-objectstore-too is no longer in
+    ceph-test-dbg.
+    
+    Although the change was merged may 6th, 2015
+    8f23382064c189b657564d58c3f9d17720e891ed, teuthology jobs were not
+    always failing because packages were not systematically upgraded during
+    the installation. The missing dependencies that were responsible for
+    this upgrade problem were fixed by
+    f898ec1e4e3472b0202280f09653a769fc62c8d3 on may 18th, 2015 and all
+    upgrade tests relying on ceph-*-dbg packages started to fail
+    systematically after this date.
+    
+    http://tracker.ceph.com/issues/11546 Fixes: #11546
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 6f11fbf41fab10924b1e0e41fcf27864779d4073)
+
+commit eb69cf758eb25e7ac71e36c754b9b959edb67cee
+Merge: 63832d4 344328d
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue May 26 19:52:59 2015 -0600
+
+    Merge pull request #4773 from dachary/wip-11733-hammer
+    
+    Debian: ceph-test and rest-bench debug packages should require their respective binary packages
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 344328de584ac707b59ab857f1f3dd4165adfcf5
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Mon May 18 10:50:58 2015 -0600
+
+    debian: set rest-bench-dbg ceph-test-dbg dependencies
+    
+    Debian's debug packages ought to depend on their respective binary
+    packages. This was the case for many of our ceph packages, but it was
+    not the case for ceph-test-dbg or rest-bench-dbg.
+    
+    Add the dependencies on the relevant binary packages, pinned to
+    "= ${binary:Version}" per convention.
+    
+    http://tracker.ceph.com/issues/11673 Fixes: #11673
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit f898ec1e4e3472b0202280f09653a769fc62c8d3)
+
+commit 63832d4039889b6b704b88b86eaba4aadcfceb2e
+Merge: 195884e 293affe
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 18 14:26:16 2015 +0200
+
+    Merge pull request #4696 from ceph/wip-11622-hammer
+    
+    Wip 11622 hammer
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 195884e21760f4948f7d1df8b65788514f918054
+Merge: b69fb89 95818da
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri May 15 13:47:04 2015 -0700
+
+    Merge pull request #4649 from ceph/wip-hammer-package-perf-objectstore
+    
+    packaging: include ceph_perf_objectstore
+
+commit 293affe992118ed6e04f685030b2d83a794ca624 (refs/remotes/gh/wip-11622-hammer)
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed May 13 17:05:22 2015 -0700
+
+    rgw: merge manifests correctly when there's prefix override
+    
+    Fixes: #11622
+    Backport: hammer, firefly
+    
+    Prefix override happens in a manifest when a rados object does not
+    conform to the generic prefix set on the manifest. When merging
+    manifests (specifically being used in multipart objects upload), we need
+    to check if the rule that we try to merge has a prefix that is the same
+    as the previous rule. Beforehand we checked if both had the same
+    override_prefix setting, but that might not apply as both manifests
+    might have different prefixes.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 389ae6739ddc6239a4dd7c5f7f9bfc9b645b8577)
+
+commit a43d24861089a02f3b42061e482e05016a0021f6
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 12 16:40:10 2015 -0700
+
+    rgw: restore buffer of multipart upload after EEXIST
+    
+    Fixes #11604
+    Backport: hammer, firefly
+    
+    When we need to restart a write of part data, we need to revert to
+    buffer to before the write, otherwise we're going to skip some data.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 580ccaec12daae64c38a1616d0be907bdd70a888)
+
+commit 95818dac1522c218662ec12bd42c470d8394b3b9
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Fri Mar 13 16:08:35 2015 -0600
+
+    packaging: include ceph_perf_objectstore
+    
+    The /usr/bin/ceph_perf_objectstore file is installed by default. Prior
+    to this commit it was missing from the packaging. This caused the RPM to
+    fail to build in mock.
+    
+    Add ceph_perf_objectstore to the "ceph-test" RPM and Debian package.
+    
+    If we end up developing further ceph_perf_* utilities, it would make
+    sense to glob them all with a wildcard, similar to what we are doing
+    with all the ceph_test_* utilities in ceph-test.
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 338b44bc74bc4597939c5c58f2a6f2cb08de7d9c)
+
+commit b69fb89122d6f989152a29124cc7ed54b5e4d43b
+Merge: 0f02512 efbfe6f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon May 11 10:27:02 2015 -0700
+
+    Merge pull request #4568 from dachary/wip-10662-10663-hammer
+    
+    RGW swift API: Response header of COPY request for object does not
+    
+    contain x-copied-from, custom metadata, x-copied-from-last-modified, X-Copied-From-Account headers
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 0f025122fe574b99504a630e3d489c3449cbbe46
+Merge: 7df3eb5 6e2dd40
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 11 14:08:53 2015 +0200
+
+    Merge pull request #4629 from ceph/hammer-uclient-checking
+    
+    Hammer uclient checking
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 6e2dd408be95644ee5bceb556a90483f882fe51c
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu May 7 18:42:01 2015 +0100
+
+    client: fix error handling in check_pool_perm
+    
+    Previously, on an error such as a pool not existing,
+    the caller doing the check would error out, but
+    anyone waiting on waiting_for_pool_perm would
+    block indefinitely (symptom was that reads on a
+    file with a bogus layout would block forever).
+    
+    Fix by triggering the wait list on errors and
+    clear the CHECKING state so that the other callers
+    also perform the check and find the error.
+    
+    Additionally, don't return the RADOS error code
+    up to filesystem users, because it can be
+    misleading.  For example, nonexistent pool is
+    ENOENT, but we shouldn't give ENOENT on IO
+    to a file which does exist, we should give EIO.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit e08cf25cafef5752877439c18cc584b0a75eca08)
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 3c2e6ae97d6129cb8f5befb3e7bf4be16373f6a5
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu May 7 18:17:37 2015 +0100
+
+    client: use SaferCond in check_pool_perm
+    
+    Just because it's easier to read.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 289ee3b80ccce6bab2966f513a37332280d04a06)
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 79b2ac215187402a99594424944db4169f2b2cdf
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Apr 24 15:23:21 2015 +0800
+
+    client: check OSD caps before read/write
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 3c4028ec21e3ef9e8801c4570420c88722651cc7)
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 7df3eb5e548f7b95ec53d3b9d0e43a863d6fe682
+Merge: 6a7fa83 2f86995
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 8 11:07:07 2015 -0700
+
+    Merge pull request #4567 from dachary/wip-10938-hammer
+    
+    RGW Swift API: response for GET/HEAD on container does not contain the X-Timestamp header
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 6a7fa83b3e72b85f92d003a5bbb03a301354a657
+Merge: 553f0db 3edb196
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 8 11:05:59 2015 -0700
+
+    Merge pull request #4570 from dachary/wip-10243-hammer
+    
+    civetweb is hitting a limit (number of threads 1024)
+
+commit 553f0db9a1fcff2601a8791af1d2bb6975d2821d
+Merge: 3fe1f2b 3aef0f2
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 8 11:05:41 2015 -0700
+
+    Merge pull request #4573 from dachary/wip-11125-hammer
+    
+    rgw: keystone token cache does not work correctly
+
+commit 3fe1f2b8ab3d0d3943a312e90f6a3de99c36beb4
+Merge: e0ed459 4d1f3f0
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 8 11:05:22 2015 -0700
+
+    Merge pull request #4574 from dachary/wip-11160-hammer
+    
+    rgw: shouldn't need to disable rgw_socket_path if frontend is configured
+
+commit e0ed459442b1e9053e29e345cd0f30d1b4b4b994
+Merge: d6de3fa d2043a5
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 8 11:04:36 2015 -0700
+
+    Merge pull request #4575 from dachary/wip-10650-hammer
+    
+    Response header of swift API PUT /container/object returned by RGW
+    
+    does not contain last-modified, content-length, x-trans-id headers. But Swift returns these headers.
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit d6de3fa0b55918bc2ac2d65ee8308f04d3605dfd
+Merge: 96dc624 2cb5d60
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 8 11:03:37 2015 -0700
+
+    Merge pull request #4576 from dachary/wip-10661-hammer
+    
+    RGW swift API: Response header of POST request for object does not contain content-length and x-trans-id headers
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 96dc624ee1f593f817055d3426054ef2e05cbf92
+Merge: ae61aee f4a0dab
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 8 10:59:15 2015 -0700
+
+    Merge pull request #4579 from dachary/wip-11036-hammer
+    
+    RGW Swift API: response for PUT on /container does not contain the mandatory Content-Length header when FCGI is used
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit ae61aee99bee9185af22424ec8019e0308828bf5
+Merge: 593d07f 7f2a9ed
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 8 10:57:47 2015 -0700
+
+    Merge pull request #4580 from dachary/wip-11088-hammer
+    
+    RGW Swift API: wrong handling of empty metadata on Swift container
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 593d07f29df8584629936051be4458b00f8a8f1f
+Merge: 4f2b41c d164d80
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 8 10:57:12 2015 -0700
+
+    Merge pull request #4581 from dachary/wip-11323-hammer
+    
+    rgw: quota not respected in POST object
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 4f2b41c4c803786c49cd2d2806d82e9223ab96a9
+Merge: 92e7a7f 893ffd3
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri May 8 09:15:41 2015 -0700
+
+    Merge pull request #4566 from dachary/wip-11478-hammer
+    
+    Queued AIO reference counters not properly updated
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 92e7a7f057dd9aabb4f66965c412135d05f6812f
+Merge: 2fbf171 0944051
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri May 8 09:15:19 2015 -0700
+
+    Merge pull request #4564 from dachary/wip-11369-hammer
+    
+    Periodic failure of TestLibRBD.DiffIterateStress
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 2fbf1712d1e190774ada0af5094134369effb3ac
+Merge: 9d97946 02a3813
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri May 8 17:32:35 2015 +0200
+
+    Merge pull request #4585 from ceph/wip-11370-hammer
+    
+    A retransmit of proxied flatten request can result in -EINVAL
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 9d97946a8908e48849355a410415e09914ef3948
+Merge: fb10594 c548d8d
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Fri May 8 20:39:29 2015 +0800
+
+    Merge pull request #4618 from dachary/wip-11398-kill-daemons-hammer
+    
+    tests: ceph-helpers kill_daemons fails when kill fails
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit c548d8d44baae78b868391e3c6fb7294f024b082
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed May 6 20:14:37 2015 +0200
+
+    tests: ceph-helpers kill_daemons fails when kill fails
+    
+    Instead of silently leaving the daemons running, it returns failure so
+    the caller can decide what to do with this situation. The timeout is
+    also extended to minutes instead of seconds to gracefully handle the
+    rare situations when a machine is extra slow for some reason.
+    
+    http://tracker.ceph.com/issues/11398 Fixes: #11398
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 0e26e9f72bc14417266b87ac1159607e1b109303)
+
+commit fb10594f2ab2427a2bf1d2f2b164a3a0928e3335
+Author: Yuri Weinstein <yuri.weinstein@inktank.com>
+Date:   Thu May 7 17:12:35 2015 -0700
+
+    Added a "ceph hello world" for a simple check for ceph-deploy qa suite
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+    (cherry picked from commit 13abae186357f4e9bb40990a7a212f93ec2e1e79)
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 6cfae7f074462498e82cfeeddbc2fe8d302c7aa0
+Merge: 736cdf4 8d9f4d4
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu May 7 13:29:57 2015 -0700
+
+    Merge pull request #4502 from dachary/wip-11026-hammer
+    
+    objecter: can get stuck in redirect loop if osdmap epoch == last_force_op_resend
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 736cdf450caa0b22cbfc54f4497717cf63d5bda7
+Merge: 3bd8e4f 46a4e8a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu May 7 21:45:53 2015 +0200
+
+    Merge pull request #4562 from dachary/wip-11376-hammer
+    
+    ceph-objectstore-tool should be in the ceph server package
+    
+    
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 46a4e8af5be54d8348a920c4a3a58e24dbf35988
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Thu Apr 30 15:53:22 2015 -0600
+
+    packaging: mv ceph-objectstore-tool to main ceph pkg
+    
+    This change ensures that the ceph-objectstore-tool utility is present on
+    all OSDs.  This makes it easier for users to run this tool to do manual
+    debugging/recovery in some scenarios.
+    
+    http://tracker.ceph.com/issues/11376 Refs: #11376
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 61cf5da0b51e2d9578c7b4bca85184317e30f4ca)
+        Conflicts:
+                debian/control
+                because file layout changes from ceph-test and ceph << 0.94.1-46
+
+commit 3bd8e4fa17d8acfd645b8a553bf58de48d59d648
+Merge: 76f6db2 6a04b55
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu May 7 19:05:48 2015 +0200
+
+    Merge pull request #4561 from dachary/wip-11143-hammer
+    
+    src/ceph-disk : disk zap sgdisk invocation
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 76f6db24e09a4bfc55cbfb7075104f20653263cc
+Merge: e5a20f8 8996907
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu May 7 19:03:53 2015 +0200
+
+    Merge pull request #4560 from dachary/wip-11507-hammer
+    
+    object creation by write cannot use an offset on an erasure coded pool
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit e5a20f83ed6d8d3d09827343d757318026f6a690
+Merge: e7671a5 8a6e6e4
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu May 7 19:03:16 2015 +0200
+
+    Merge pull request #4559 from dachary/wip-11429-hammer
+    
+    OSD::load_pgs: we need to handle the case where an upgrade from earlier versions which ignored non-existent pgs resurrects a pg with a prehistoric osdmap
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit e7671a58939f6e2e37f9f6df0f039f485ad4ca16
+Merge: 126a372 113f3b1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu May 7 18:48:56 2015 +0200
+
+    Merge pull request #4563 from tchaikov/wip-hammer-11534
+    
+    mon: Total size of OSDs is a maginitude less than it is supposed to be.
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 126a37212550a4a59beaa80e0579098198f74db5
+Merge: 58b30d5 c87aa11
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu May 7 18:18:17 2015 +0200
+
+    Merge pull request #4577 from dachary/wip-10666-hammer
+    
+    RGW swift API: Response header of GET request for container does not contain X-Container-Object-Count, X-Container-Bytes-Used and x-trans-id headers
+    
+    Reviewed-by: Dmytro Iurchenko <diurchenko@mirantis.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit f4a0dabfe8debc7b54afa5da179d51db891f5bc0
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Sun Mar 8 22:45:34 2015 +0100
+
+    rgw: send Content-Length in response for HEAD on Swift account.
+    
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 29073d84640b61d0221b2d2ab73c271d60fd13ba)
+
+commit 32f4a7439d1e0ce6aab1c1e39407b14e124d03bf
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Mar 6 13:18:19 2015 +0100
+
+    rgw: send Content-Length in response for DELETE on Swift container.
+    
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit d260a93405a7a34a899f716bd9c4e731baa9ffd0)
+
+commit d39a66007fa7211c00a67f9cd898e55551f5ae62
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Mar 6 13:14:26 2015 +0100
+
+    rgw: send Content-Length in response for PUT on Swift container.
+    
+    Fixes: #11036
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 10c1f1aa1e09df5edf5d2f9f9b0273ddbcb384fa)
+
+commit 02a3813a6a4f9c8ce14f64fc7f378030e7ea6f93
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Apr 27 01:03:52 2015 -0400
+
+    librbd: flatten should return -EROFS if image is read-only
+    
+    Fixes: #11475
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f141e02ab719df830648318f4c1d9ca286071ed3)
+
+commit af8939be80310e234745fe81e67244ab52c6add5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Apr 27 01:00:38 2015 -0400
+
+    librbd: allow snapshots to be created when snapshot is active
+    
+    The librbd API previously permitted the creation of snapshots while
+    the image context was associated to another snapshot.  A recent code
+    cleanup broke that ability, so this re-introduces it.  The code change
+    also allows minor cleanup with rebuild_object_map.
+    
+    Fixes: #11475
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 594a6610802f2cadb62200815bd8b9860809e759)
+    
+    Conflicts:
+    	src/librbd/internal.cc
+
+commit d21c0c00d2bed282677d2063a3fb6f5346641286
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Apr 21 12:59:33 2015 -0400
+
+    librbd: better handling for duplicate flatten requests
+    
+    A proxied flatten request could be replayed, resulting in a
+    -EINVAL error code being generated on the second attempt. Filter
+    out that error if it is known the parent did exist before the
+    op started.
+    
+    Fixes: #11370
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ef7e210c3f747bc4c3c8768c7b6407cc91c5c319)
+
+commit ec0bd1dea526e04333d8059421666dcd2a59044e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Mar 18 11:51:47 2015 -0400
+
+    librbd: use generic helper for issuing async requests
+    
+    resize, flatten, and rebuild object map now use the same
+    bootstrap code for sending the request to the remote lock owner
+    or executing the request locally.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 18fd6ca7f59d5545f0bb0b0e899d0739639ce104)
+    
+    Conflicts:
+    	src/librbd/internal.cc
+
+commit 8a6e6e4c107b03563b2e38aa24cc2067ce6a7350
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Apr 20 23:45:57 2015 -0700
+
+    OSD: handle the case where we resurrected an old, deleted pg
+    
+    Prior to giant, we would skip pgs in load_pgs which were not present in
+    the current osdmap.  Those pgs would eventually refer to very old
+    osdmaps, which we no longer have causing the assertion failure in 11429
+    once the osd is finally upgraded to a version which does not skip the
+    pgs.  Instead, if we do not have the map for the pg epoch, complain to
+    the osd log and skip the pg.
+    
+    Fixes: 11429
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit fbfd50de5b9b40d71d2e768418a8eca28b1afaca)
+
+commit efbfe6fced72d07309ccf1f1a219c037b7f535fa
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Sat Mar 21 15:53:08 2015 +0100
+
+    rgw: improve metadata handling on copy operation of Swift API.
+    
+    Fixes: #10645
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit a00cb31cc52e91bfacdd15e0af60be74e66c1996)
+
+commit d164d8004feddb3e2238b26e3360a905e209d117
+Author: 9seconds <nineseconds@yandex.ru>
+Date:   Wed Apr 1 09:12:06 2015 +0300
+
+    rgw: quota not respected in POST object
+    
+    Signed-off-by: Sergey Arkhipov <nineseconds@yandex.ru>
+    Backport: hammer, firefly
+    Fixes: #11323
+    (cherry picked from commit e76f84e179d2ba8bfc0dc5abf4e620fef14bc8a0)
+    
+    Conflicts:
+    	src/rgw/rgw_op.cc
+            discard the whitespace modification hunk that were creating
+    	conflict and ignore the conflict due to an unrelated cast
+            modification in the context
+
+commit 7f2a9ed7a986145d4b34517a1a1bb44799ebf621
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue Mar 10 19:36:59 2015 +0100
+
+    rgw: fix handling empty metadata items on Swift container.
+    
+    Fixes: #11088
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 1660d8625212f7b5fb30568ed640f467297e36cb)
+
+commit 8e6efdbcb0f820b2ab3728662efbfb4bc45495af
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Feb 27 15:23:16 2015 +0100
+
+    rgw: send Content-Length in response for GET on Swift container.
+    
+    Fixes: #10971
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 5a64fb5430b6b42174bf53a5910d9435043c1380)
+
+commit 54b62904a4cc3913be23803734fa68741a3c33cc
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Mar 6 12:42:52 2015 +0100
+
+    rgw: enable end_header() to handle proposal of Content-Length.
+    
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit f98fd4dfda7a105d621f99c2b48eb9ab9b45d979)
+
+commit c87aa110bd224ae3220c5486fbd2486ebdfb8b8e
+Author: Dmytro Iurchenko <diurchenko@mirantis.com>
+Date:   Mon Feb 16 18:47:59 2015 +0200
+
+    rgw: Swift API. Complement the response to "show container details"
+    
+    OpenStack Object Storage API v1 states that X-Container-Object-Count, X-Container-Bytes-Used and user-defined metadata headers should be included in a response.
+    
+    Fixes: #10666
+    Backport: hammer
+    Reported-by: Ahmad Faheem <ahmad.faheem@ril.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+    Signed-off-by: Dmytro Iurchenko <diurchenko@mirantis.com>
+    (cherry picked from commit d3a3d5ae5859cd7f2eed307e8f942f9a9fd75f35)
+
+commit 2cb5d600699085fe0e996b91cf85603be3da230e
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Thu Feb 26 19:21:03 2015 +0100
+
+    rgw: enforce Content-Length in response for POST on Swift cont/obj.
+    
+    Fixes: #10661
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit fd0c612cf867d2e99e74820130357e63305970fb)
+
+commit d2043a5f3b8d7a3ce50c4e84aa88481c8912ae25
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Mar 13 21:41:45 2015 +0100
+
+    rgw: send Last-Modified header in response for PUT on Swift object.
+    
+    Fixes: #10650
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 9a22acc57684534a245f25848c23d1db21f16653)
+
+commit 4d1f3f03ef8809693c8ea2bcbc4d24fd41ae1842
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Mar 18 20:49:13 2015 -0700
+
+    rgw: don't use rgw_socket_path if frontend is configured
+    
+    Fixes: #11160
+    Backport: hammer, firefly
+    
+    Previously if we wanted to use the tcp fcgi socket, we needed to clear
+    rgw_socket_path.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 36d6eea3caa79fcb4e08bdd340ccda2474b9e5ea)
+
+commit 3aef0f2bb6f88bb17c460a3cef0d3503550f716c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Mar 3 11:03:35 2015 -0800
+
+    rgw: update keystone cache with token info
+    
+    Fixes: #11125
+    Backport: hammer, firefly
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 6616294aa140ceb83cc61c6ab6f9947636f5e67d)
+
+commit 3edb1964fe9e8574aafcb758d170007f0e43a324
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Apr 21 11:08:45 2015 -0700
+
+    civetweb: update max num of threads
+    
+    Fixes: #10243
+    cherry-picked upstream fixed into submodule
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 7a432f7bdfbbc51518fb63d7f2ecab401e8a8d4f)
+
+commit bc6eb8d5f0a66aec3fbda0b794d008a3157a8154
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue Apr 7 14:09:57 2015 +0200
+
+    rgw: improve code formatting ONLY.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 214c8b32b1b04885005e929a7ed2d4354b3ea20b)
+
+commit 7aa1ae60cea17e0bd140c0cf2313d82f2f64554f
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue Mar 17 14:07:34 2015 +0100
+
+    rgw: send X-Copied-From-Last-Modified header of Swift API.
+    
+    Fixes: #10663
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit e7724a1d8c0872362c19f578fe30ac2cf3dada90)
+
+commit 150b9e2b85a72dc247da4ba1ab770e6af053acb7
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue Mar 17 14:06:37 2015 +0100
+
+    rgw: dump object metadata in response for COPY request of Swift API.
+    
+    Fixes: #10663
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit c52b75edeaeef19471b9aca772bf08055bf04031)
+
+commit e749701be5368a22cad1630f8202e48f5d980409
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Mar 13 17:59:01 2015 +0100
+
+    rgw: refactor dumping metadata of Swift objects.
+    
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit ccf6eaac6f7068289c4a4ffd3f0481d497ba7c87)
+
+commit b034511fa79996415640b4aca3e8747340f2a127
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Mar 13 18:37:39 2015 +0100
+
+    rgw: add support for X-Copied-From{-Account} headers of Swift API.
+    
+    Fixes: #10663
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 94f1375ccb9df02cdd8f6828153ae2e3a7ad36b1)
+
+commit c9e6a0bf8c601c4fd9065c1f3a8ea445bd652a52
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed Mar 11 14:08:03 2015 +0100
+
+    rgw: send ETag, Last-Modified in response for copying Swift cobject.
+    
+    Fixes: #11087
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 385fe4b4bbbd4a9aab92abf2a813090deeaa037e)
+
+commit 7f41ff0a6577b0784c6719b9d705f32921b1c40e
+Author: Dmytro Iurchenko <diurchenko@mirantis.com>
+Date:   Fri Feb 20 18:31:03 2015 +0200
+
+    rgw: Swift API. Allows setting attributes with COPY object operation.
+    
+    http://developer.openstack.org/api-ref-objectstorage-v1.html says: "With COPY, you can add additional metadata to the object."
+    
+    Fixes: #10662
+    Backport: hammer
+    Reported-by: Ahmad Faheem <ahmad.faheem@ril.com>
+    Signed-off-by: Dmytro Iurchenko <diurchenko@mirantis.com>
+    (cherry picked from commit 1b722bbcd691e0a4a39ea77cd28e309fd723ec88)
+
+commit 2f869959ffe1adbcfef7d26ae2d022d23d982673
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed Feb 25 16:12:58 2015 +0100
+
+    rgw: improve format of X-Timestamp on Swift objects.
+    
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit e54fef9542bc2b5db192308728a99df139a4b6cf)
+
+commit 48b19810a9860f6fccbf8d9b8a2fadfb37f598dd
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed Feb 25 14:11:40 2015 +0100
+
+    rgw: add support for X-Timestamp on Swift containers.
+    
+    Fixes: #10938
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit b7e9bf6b98ee48d1977d907a9e5130c0ce073c54)
+
+commit 893ffd3767678ab881c4bc44ecfe1801cb9f9704
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Apr 27 03:43:10 2015 -0400
+
+    tests: AioCompletion incorrectly freed
+    
+    The AioCompletion should be released instead of directly
+    deleted.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 387a09eeeaf0b66b3a2ddc36388da27d5804a4c7)
+
+commit 96b0db5decfad452964750cff92a63007433e519
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Apr 27 03:42:24 2015 -0400
+
+    librbd: update ref count when queueing AioCompletion
+    
+    If the client releases the AioCompletion while librbd is waiting
+    to acquire the exclusive lock, the memory associated with the
+    completion will be freed too early.
+    
+    Fixes: #11478
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit fd7723a1e62e682ac5a5279231a9fd6f5682bf94)
+
+commit 094405171bdead0ce09055d4acc6445274992a01
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Apr 10 12:37:05 2015 -0400
+
+    librbd: failure to update the object map should always return success
+    
+    If an object map update fails, the object map will be flagged as
+    invalid.  However, if a subsequent update failure occurs, the error
+    code will propagate back to the caller.
+    
+    Fixes: #11369
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 99f5a7d595c653447b351898192410c9cb773770)
+
+commit 7ee7dcfd609731d3c7f51b74c1d99fb3fa51c413
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Mar 6 15:40:48 2015 -0500
+
+    tests: librados_test_stub reads should deep-copy
+    
+    If a client of librados_test_stub modified a bufferlist
+    retrieved via a read call, the client will actually be
+    changing the contents of the file.  Therefore, read calls
+    should deep-copy the contents of the buffer::ptrs.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 76fe8d73ff79da2d734f70680208a2c188b58671)
+
+commit 113f3b14ec8631d57bbbc2931e242ac96cf9e6fc
+Author: Zhe Zhang <zzxuanyuan@gmail.com>
+Date:   Tue May 5 18:08:48 2015 -0500
+
+    mon: Total size of OSDs is a maginitude less than it is supposed to be.
+    
+    When dumping statistics of OSDs such as running command "ceph osd df",
+    the sum of OSDs' size is 2^10 times less than their real size.
+    
+    Fixes: #11534
+    Signed-off-by: Zhe Zhang <zzxuanyuan@gmail.com>
+    (cherry picked from commit 73d16f69d6f58fe8be262b0fb8db28c94605ea7d)
+
+commit 6a04b55df76faad9b1b0770fbe6038b3b8c7d645
+Author: Owen Synge <osynge@suse.com>
+Date:   Tue Mar 17 15:41:33 2015 +0100
+
+    Fix "disk zap" sgdisk invocation
+    
+    Fixes #11143
+    
+    If the metadata on the disk is truly invalid, sgdisk would fail to zero
+    it in one go, because --mbrtogpt apparently tried to operate on the
+    metadata it read before executing --zap-all.
+    
+    Splitting this up into two separate invocations to first zap everything
+    and then clear it properly fixes this issue.
+    
+    Based on patch by Lars Marowsky-Bree <lmb@suse.com> in ceph-deploy.
+    Created by Vincent Untz <vuntz@suse.com>
+    
+    Signed-off-by: Owen Synge <osynge@suse.com>
+    Signed-off-by: Thorsten Behrens <tbehrens@suse.com>
+    (cherry picked from commit fdd7f8d83afa25c4e09aaedd90ab93f3b64a677b)
+
+commit 8996907e0a777320b505e74754f48a1a82308166
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Fri Mar 6 15:54:07 2015 +0800
+
+    osd: refuse to write a new erasure coded object with an offset > 0
+    
+    Even if the offset is properly aligned.
+    
+    http://tracker.ceph.com/issues/11507 Fixes: #11507
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit a4f1256c214ee0e7ebb91ac4ea8655f5d9642dc8)
+
+commit 58b30d5426998bf7eab4a773f1a04e5bcfbf9b93
+Merge: bc51476 3a58e30
+Author: Yan, Zheng <ukernel@gmail.com>
+Date:   Wed May 6 16:01:57 2015 +0800
+
+    Merge pull request #4481 from ceph/hammer-11482
+    
+    mds: remove caps from revoking list when caps are voluntarily released
+
+commit bc51476181429d7d95d2bba5f774d8b60c47fb1f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Apr 13 16:33:17 2015 -0700
+
+    ceph_json: add decode / encoder for multimap
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 55594623e2a478c3c023336b924bfdef0017d97f)
+
+commit 7c7e651a0ae8bbcebe136da74b7dbe3a3e9edcc8
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Mar 30 17:34:57 2015 -0700
+
+    cls_rgw: use multimap to keep pending operations in bucket index
+    
+    Fixes: #11256
+    Multiple concurrent requests might be sent using the same tag, need the
+    entry map to be able to hold multiple entries.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 4e6a66b55e73c01347fc3330faa5c1307d29e9d3)
+
+commit cb7571375377295d0aff791a03b22da6eb26109d
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Mar 27 16:32:48 2015 -0700
+
+    rgw: generate new tag for object when setting object attrs
+    
+    Fixes: #11256
+    Backport: firefly, hammer
+    
+    Beforehand we were reusing the object's tag, which is problematic as
+    this tag is used for bucket index updates, and we might be clobbering a
+    racing update (like object removal).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit edc0627a1dbeb66ea2f5f177f6ceca64559ff3d8)
+
+commit 7387c43382e7f114c43db3cc26ca77d081749d8e
+Author: Noah Watkins <noahwatkins@gmail.com>
+Date:   Fri Mar 27 19:34:12 2015 -0700
+
+    java: libcephfs_jni.so is in /usr/lib64 on rhel
+    
+    Signed-off-by: Noah Watkins <noahwatkins@gmail.com>
+    (cherry picked from commit aed3434dc7c5161c72c7d5655faa3bc693fc9777)
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 5cc0f20ba00080881aff460fab2110cb7eaba279
+Merge: f7bcb2d 0e6a032
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 4 23:58:37 2015 +0200
+
+    Merge pull request #4498 from dachary/wip-11342-hammer
+    
+    librbd notification race condition on snap_create
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit f7bcb2dc6043501d478c9d7664bf39f34d5ad6d8
+Merge: 4d95929 2864da8
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 4 23:58:18 2015 +0200
+
+    Merge pull request #4497 from dachary/wip-7385-hammer
+    
+    Objectcacher setting max object counts too low
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 4d95929c3129028de2d48efdef71fba2b949edd9
+Merge: 16c2f4c c615972
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 4 23:57:48 2015 +0200
+
+    Merge pull request #4499 from dachary/wip-11363-hammer
+    
+    ImageWatcher should cancel in-flight ops on watch error
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 16c2f4c9e0428bda5d784c6c82929f748ec6cb4a
+Merge: addb0ec 0cdc93f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 4 23:57:17 2015 +0200
+
+    Merge pull request #4496 from dachary/wip-5488-hammer
+    
+    librbd: deadlock in image refresh
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit addb0ec950e3a3b1150f927f028d2260c1cd0615
+Merge: c0782ed 379ef71
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 4 16:31:24 2015 +0200
+
+    Merge pull request #4505 from dachary/wip-11322-hammer
+    
+    rgw - improve performance for large object (multiple chunks) GET
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit c0782ed9ab1e59f39d3a30496214971d4cc509d9
+Merge: e074695 2f34d2e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 4 16:24:53 2015 +0200
+
+    Merge pull request #4501 from dachary/wip-11001-hammer
+    
+    Improve rgw HEAD request by avoiding read the body of the first chunk
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit e074695e003f891e9c52e19a1679809155364d32
+Merge: 7f9d78d c6edc16
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 4 16:23:28 2015 +0200
+
+    Merge pull request #4500 from dachary/wip-11047-hammer
+    
+    rgw : make quota/gc thread configurable for starting
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 7f9d78d4f12ff3feaee519dd319426650ca0ff88
+Merge: 11b1ccd 4789686
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 4 11:46:13 2015 +0200
+
+    Merge pull request #4504 from dachary/wip-10691-hammer
+    
+    ceph-dencoder links to libtcmalloc, and shouldn't
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 11b1ccdb6502406456905d05624f0600ef9df893
+Merge: 7c050ec 8709e34
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 4 11:44:11 2015 +0200
+
+    Merge pull request #4503 from dachary/wip-10983-hammer
+    
+    use a new disk as journal disk,ceph-disk prepare fail
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 7c050ecab2ccc29b03bab2c4ad67c22e9736bb9c
+Merge: c5e0b61 da7f683
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 4 11:43:23 2015 +0200
+
+    Merge pull request #4507 from dachary/wip-11432-hammer
+    
+    compilation error: No high-precision counter available (armhf, powerpc..)
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit c5e0b615d7c98ef700ebe8048ed038f1ff036ff4
+Merge: ee61a61 856b2fa
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Apr 30 17:10:29 2015 -0700
+
+    Merge pull request #4515 from ceph/hammer-next
+    
+    rgw: critical fixes for hammer
+
+commit ee61a61face479d9895a5cd08ebc8aa93c8bb6ce
+Merge: abc0741 0ee022b
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Apr 30 09:23:53 2015 -0700
+
+    Merge pull request #4462 from liewegas/wip-11211-hammer
+    
+    osd/ReplicatedPG: don't check order in finish_proxy_read
+
+commit 856b2fa1fc72916349e484bf3615860392b74100
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Apr 22 15:39:05 2015 -0700
+
+    rgw-admin: a tool to fix object locator issue
+    
+    Objects that start with underscore need to have an object locator,
+    this is due to an old behavior that we need to retain. Some objects
+    might have been created without the locator. This tool creates a new
+    rados object with the appropriate locator.
+    
+    Syntax:
+    
+    $ ./radosgw-admin bucket check --check-head-obj-locator \
+                                   --bucket=<bucket> [--fix]
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit be4355ad8ed622734172fdce77ca71fb2635b36c)
+
+commit 512ae4cb3e182ce79aca7354c66d2f2a662555da
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Apr 21 17:31:41 2015 -0700
+
+    rgw: set a special object locator if object starts with underscore
+    
+    Fixes: #11442
+    Backport: hammer
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 3d4a1d20b86a5a00556df3d6a8dba096509274b7)
+
+commit da4d2274b5d83a116e767f3063752624d1719c32
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Apr 23 15:33:03 2015 -0700
+
+    rgw: use correct oid when creating gc chains
+    
+    Fixes: #11447
+    Backport: hammer
+    
+    When creating gc chain, use the appropriate oid, otherwise objects will
+    leak.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit cc5d34678c6d4bdcd552e6334a383c4df9797f46)
+
+commit 4e84f318de97e592e16493c67491ba0d7f8103a8
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Apr 23 17:36:47 2015 +0200
+
+    rgw: civetweb should use unique request id
+    
+    max_req_id was moved to RGWRados and changed to atomic64_t.
+    
+    The same request id resulted in gc giving the same idtag to all objects
+    resulting in a leakage of rados objects. It only kept the last deleted object in
+    it's queue, the previous objects were never freed.
+    
+    Fixes: 10295
+    Backport: Hammer, Firefly
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit c26225980c2fd018e70033a453d635533fcdefec)
+
+commit abc0741d57f30a39a18106bf03576e980ad89177
+Merge: 74c2dc1 3001fad
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Wed Apr 29 15:02:56 2015 -0700
+
+    Merge pull request #4506 from dachary/wip-11381-hammer
+    
+    messenger: double clear of pipe in reaper
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 478968670caecd67f7995a09b60f6208729e3de3
+Author: Boris Ranto <branto@redhat.com>
+Date:   Mon Apr 13 12:38:58 2015 +0200
+
+    Move ceph-dencoder build to client
+    
+    The patch simply moves the ceph-dencoder build from server part of the
+    Makefiles to client part of the Makefiles.
+    
+    Refs: #10691
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit ef2164725f0b55ffa4b609d407eed5f3f3048b46)
+
+commit 7eabb70b906e50901551ab96453f05fe662a8876
+Author: Boris Ranto <branto@redhat.com>
+Date:   Mon Apr 13 15:07:03 2015 +0200
+
+    Rework mds/Makefile.am to support a dencoder client build
+    
+    The patch adds all the mds sources to DENCODER_SOURCES to allow a
+    dencoder client build. The patch also splits the Makefile.am file to
+    better accomodate the change.
+    
+    Refs: #10691
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit c2b3a35fb3cbf28d46a5427f32fbaff142c85f2a)
+    
+    Conflicts:
+            src/mds/Makefile-server.am
+    	src/mds/Makefile.am
+            because the mds/StrayManager.h file was added after hammer
+
+commit da7f6835b15370ce0120a64f7ac3359f3ba4729b
+Author: James Page <james.page@ubuntu.com>
+Date:   Fri Mar 13 19:46:04 2015 +0000
+
+    Add support for PPC architecture, provide fallback
+    
+    Add high precision cpu cycles support for powerpc and powerpc64.
+    
+    Provide a fallback for other architectures and warn during
+    compilation.
+    
+    Signed-off-by: James Page <james.page@ubuntu.com>
+    (cherry picked from commit b2781fb5638afae7438b983a912ede126a8c5b85)
+
+commit 3001fad4b6d7e692f6070ef166ed4a3e4849760f
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Fri Apr 17 22:07:00 2015 +0800
+
+    Fix clear_pipe after reaping progress
+    
+    In pipe.cc:1353 we stop this connection and we will let reader and write threads stop. If now reader and writer quit ASAP and we call queue_reap to trigger the reap progress. Now we haven't call "connection_state->clear_pipe(this)" in pipe.cc:1379, so we may assert failure  here.
+    
+    Fixes: #11381
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit 0ea0e011a6a6c6d6b40f5d97328bbad0e4568dd7)
+
+commit 379ef714f7149a748891dafd41db80c247d35975
+Author: Guang Yang <yguang@yahoo-inc.com>
+Date:   Fri Apr 3 12:27:04 2015 +0000
+
+    rgw : Issue AIO for next chunk first before flush the (cached) data.
+    
+    When handling GET request for large object (with multiple chunks), currently it will first flush the
+    cached data, and then issue AIO request for next chunk, this has the potential issue to make the retriving
+    from OSD and sending to client serialized. This patch switch the two operations.
+    
+    Fixes: 11322
+    Signed-off-by: Guang Yang <yguang@yahoo-inc.com>
+    (cherry picked from commit 366e8a85c0e9f00eed364eaebbfb6b672852eae9)
+
+commit b903ad28a68772fa0b7a88b4db2724f4d07565d5
+Author: Boris Ranto <branto@redhat.com>
+Date:   Mon Apr 13 12:33:00 2015 +0200
+
+    rgw/Makefile.am: Populate DENCODER_SOURCES properly
+    
+    Dencoder is built if ENABLE_CLIENT is set. However, the rgw/Makefile.am
+    populated DENCODER_SOURCES only if WITH_RADOSGW was set. The patch fixes
+    this and populates DENCODER_SOURES if ENABLE_CLIENT is set.
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 0b264331f57b64880ce05fe3bd752e8df226d00c)
+
+commit f994483c0e3a60226c8fb6983380ef8400b0160e
+Author: Boris Ranto <branto@redhat.com>
+Date:   Mon Apr 13 12:32:30 2015 +0200
+
+    Dencoder should never be built with tcmalloc
+    
+    The patch adds disabled perfglue stubs to DENCODER sources in order to
+    avoid tcmalloc-enabled ceph-dencoder builds.
+    
+    Refs: #10691
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit fb11c74d1dc3843f2f5b6dca9c76278c5ceeca1c)
+
+commit 8709e34f931809f7129cdac1203ec4e774e3eb4c
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Mar 8 15:15:35 2015 +0100
+
+    ceph-disk: more robust parted output parser
+    
+    In some cases, depending on the implementation or the operating system,
+    
+       parted --machine -- /dev/sdh print
+    
+    may contain empty lines. The current parsing code is fragile and highly
+    depends on output details. Replace it with code that basically does the
+    same sanity checks (output not empty, existence of units, existence of
+    the dev entry) but handles the entire output instead of checking line by
+    line.
+    
+    http://tracker.ceph.com/issues/10983 Fixes: #10983
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit f5acf6bb6a342b05897605703d7d9cb7c09714fd)
+
+commit 8d9f4d4eb546e26eeb3911811bdeb166d06cb1d1
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Fri Mar 6 11:26:31 2015 +0800
+
+    osdc: add epoch_t last_force_resend in Op/LingerOp.
+    
+    Using this field record the pg_poo_t::last_force_op_resend to avoid op
+    endless when osd reply with redirect.
+    
+    Fixes: #11026
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit def4fc4ae51174ae92ac1fb606427f4f6f00743e)
+
+commit 2f34d2e73bb4bded4779af15a337c75eb2d1497f
+Author: Guang Yang <yguang@yahoo-inc.com>
+Date:   Tue Mar 3 09:46:52 2015 +0000
+
+    rgw: do not pre-fetch data for HEAD requests
+    
+    Backport: hammer
+    
+    Signed-off-by: Guang Yang <yguang@yahoo-inc.com>
+    (cherry picked from commit e32da3e7c880eaf7cb84d1c078447b28e1d8052b)
+
+commit c6edc16fee027f6cdefacba08f1edc436a7406c5
+Author: Guang Yang <yguang@yahoo-inc.com>
+Date:   Wed Feb 4 03:21:52 2015 +0000
+
+    rgw - make starting quota/gc threads configurable
+    
+    Fixes: 11047
+    Backport: hammer
+    
+    Signed-off-by: Guang Yang <yguang@yahoo-inc.com>
+    (cherry picked from commit a88712aeb4e7cd4208b9a707aa3bd4d03340c3ff)
+
+commit c6159724f065731c41b2d29a48d0f0a3dc82340b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Mar 20 11:56:55 2015 -0400
+
+    librbd: ImageWatcher should cancel in-flight ops on watch error
+    
+    Upon an watch error notification from librados, pending requests
+    should be canceled and in-flight IO should be flushed prior to
+    unlocking the image.
+    
+    Fixes: #11363
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit fc2e511b2aed4d40eff5101a4c9e513b34e5e58e)
+
+commit 0e6a032c9e6ddae20be82df7500a0758d2cd8e74
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Apr 7 15:39:13 2015 -0400
+
+    librbd: moved snap_create header update notification to initiator
+    
+    When handling a proxied snap_create operation, the client which
+    invoked the snap_create should send the header update notification
+    to avoid a possible race condition where snap_create completes but
+    the client doesn't see the new snapshot (since it didn't yet receive
+    the notification).
+    
+    Fixes: #11342
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6e20ed6834912ccd979d16e3f3b340c239e05288)
+
+commit 2864da86ca689472341e8a80f7ad6fcc5eb8321a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 22 11:27:35 2015 -0400
+
+    librbd: updated cache max objects calculation
+    
+    The previous calculation was based upon the image's object size.
+    Since the cache stores smaller bufferheads, the object size is not
+    a good indicator of cache usage and was resulting in objects being
+    evicted from the cache too often.  Instead, base the max number of
+    objects on the memory load required to store the extra metadata
+    for the objects.
+    
+    Fixes: #7385
+    Backport: firefly, hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0b378942c4f1b79cb65967f2d3466728ca1c8d5b)
+    
+    Conflicts:
+    	src/librbd/ImageCtx.cc
+            because hammer has cct->_conf->rbd_cache_size
+            instead of cache_size
+
+commit 0cdc93fbdcf68a31e6aada38b0cb9d66efdc512d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Mar 16 11:04:22 2015 -0400
+
+    librbd: acquire cache_lock before refreshing parent
+    
+    cache_lock needs to be acquired before snap_lock to avoid
+    the potential for deadlock.
+    
+    Fixes: #5488
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 703ba377e3de4007920f2ed7d8a0780f68676fe2)
+    
+    Conflicts:
+    	src/librbd/internal.cc
+            resolved by moving int r; in the scope of the block
+
+commit a1b4aeb8e8c3a8d5c8284dcee8e03f501a77928c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Mar 13 18:08:47 2015 -0400
+
+    librados_test_stub: AIO operation callbacks should be via Finisher
+    
+    librados will execute all AIO callbacks via a single finisher to
+    prevent blocking the Objecter.  Reproduce this behavior to avoid
+    deadlocks that only exist when using the test stub.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b38c96f2c1747a6d864e7aaa2e9858139ce9d1fd)
+
+commit 3a58e30dc6563197f0effeabbd2fbf804403ad34
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Apr 28 15:45:32 2015 +0800
+
+    mds: remove caps from revoking list when caps are voluntarily released
+    
+    Fixes: #11482
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 86788c4ea4155f0388b7ebaf475a3d3c37d39331)
+
+commit 74c2dc1f3924fa05e2c40f4cceb2ab060493bdfb
+Merge: 8a58d83 f30fa4a
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Apr 24 16:13:56 2015 -0700
+
+    Merge pull request #4463 from ceph/wip-11453-hammer-rgw-init-as-root
+    
+    rgw: init-radosgw: run RGW as root
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit f30fa4a364602fb9412babf7319140eca4c64995
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Wed Apr 22 16:36:42 2015 -0600
+
+    init-radosgw: run RGW as root
+    
+    The ceph-radosgw service fails to start if the httpd package is not
+    installed. This is because the init.d file attempts to start the RGW
+    process with the "apache" UID. If a user is running civetweb, there is
+    no reason for the httpd or apache2 package to be present on the system.
+    
+    Switch the init scripts to use "root" as is done on Ubuntu.
+    
+    http://tracker.ceph.com/issues/11453 Refs: #11453
+    
+    Reported-by: Vickey Singh <vickey.singh22693@gmail.com>
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 47339c5ac352d305e68a58f3d744c3ce0fd3a2ac)
+
+commit 0ee022b1ae832c70a80e9d2cdf32403039f3f125 (refs/remotes/me/wip-11211-hammer)
+Author: Zhiqiang Wang <zhiqiang.wang@intel.com>
+Date:   Tue Mar 24 16:00:16 2015 +0800
+
+    osd/ReplicatedPG: don't check order in finish_proxy_read
+    
+    Read doesn't need to be ordered. So when proxy read comes back from base
+    tier, it's not necessarily at the front of the in progress list.
+    
+    Fixes: #11211
+    
+    Signed-off-by: Zhiqiang Wang <zhiqiang.wang@intel.com>
+    (cherry picked from commit 560a5839c0d1852b5816937b845b60390777636c)
+
+commit 8a58d83b0d039d2c2be353fee9c57c4e6181b662
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Sun Mar 22 23:59:19 2015 +0800
+
+    Fix ceph_test_async_driver failed
+    
+    This test will create 10000 sockets which will failed because of limited system fd. Actually we only need to create several hundreds sockets and it's enough to get the test's goal.
+    
+    Fix bug #11198(cherry picked from commit cd11daa2d21b7b059df9877cad38432678bb6161)
+
+commit 85a68f9a8237f7e74f44a1d1fbbd6cb4ac50f8e8
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Fri Apr 3 13:43:31 2015 +0200
+
+    rgw: remove meta file after deleting bucket
+    The meta file is deleted only if the bucket meta data is not synced
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    
+    Fixes: #11149
+    Backport: hammer, firefly
+    (cherry picked from commit dfdc7afb59cc8e32cf8bff55faa09076c853de06)
+
+commit b01e68fae2f8235b7a813188b664d3ec7bd1fa09
+Merge: addc7e2 bd0ec49
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Apr 14 20:58:40 2015 +0200
+
+    Merge pull request #4332 from dachary/wip-11217-hammer
+    
+    tests: TestFlatIndex.cc races with TestLFNIndex.cc
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit addc7e2c6a4973cbcb5f02635be01a446cb223f0
+Merge: e4bfad3 51f5763
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Apr 14 09:05:53 2015 -0700
+
+    Merge pull request #4357 from dachary/wip-release-notes-hammer
+    
+    release-notes: backport Hammer release notes
+
+commit 51f57632f6d463e5f702bdb4e12c1914ec76d2b5
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Apr 14 11:30:43 2015 +0200
+
+    release-notes: backport Hammer release notes
+    
+    Include the Hammer release notes in Hammer.
+    
+    Cherry picking the Hammer release notes cannot be done cleanly, they are
+    copy/pasted instead. This will allow cherry-picking the release notes
+    for the next point releases. It should be undisturbed by the release
+    notes for other point releases because they modify parts of the file
+    that will not generate cherry-pick conflicts.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+
+commit bd0ec494739308dc33e6b042ae3e1aafd53c634c
+Author: Xinze Chi <xmdxcxz@gmail.com>
+Date:   Tue Mar 24 18:42:04 2015 +0800
+
+    bug fix: test case for lfn index
+    
+    tests: TestFlatIndex.cc races with TestLFNIndex.cc
+    Both use the same PATH and when run in parallel they sometime conflict.
+    
+    Fixes: #11217
+    Signed-off-by: Xinze Chi <xmdxcxz@gmail.com>
+    (cherry picked from commit 791c3879d62b848616972f9c6d921aac30ac4925)
diff --git a/doc/changelog/v0.94.3.txt b/doc/changelog/v0.94.3.txt
new file mode 100644
index 000000000..7c0a60d75
--- /dev/null
+++ b/doc/changelog/v0.94.3.txt
@@ -0,0 +1,2660 @@
+commit 95cefea9fd9ab740263bf8bb4796fd864d9afe2b (tag: refs/tags/v0.94.3, refs/remotes/gh/hammer)
+Author: Jenkins <jenkins@inktank.com>
+Date:   Wed Aug 26 10:39:37 2015 -0700
+
+    0.94.3
+
+commit 697101e4dfd9822050ce401b5f6212bfd81fea89
+Merge: 88e7ee7 81a311a
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Tue Aug 18 12:43:57 2015 +0100
+
+    Merge pull request #5589 from ceph/hammer-12709
+    
+    Workunits : fs/misc/chmod.sh : Include ACL characters in permission check
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 81a311a744987564b70852fdacfd915523c73b5d
+Author: Yazen Ghannam <yazen.ghannam@linaro.org>
+Date:   Mon May 4 12:33:16 2015 -0400
+
+    Workunits : fs/misc/chmod.sh : Include ACL characters in permission check.
+    
+    Signed-off-by: Yazen Ghannam <yazen.ghannam@linaro.org>
+    (cherry picked from commit d3dbfffefb0ae53583350f53258dc902670da659)
+
+commit 88e7ee716fdd7bcf81845087021a677de5a50da8
+Merge: bb12f92 1a32379
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 4 13:02:17 2015 +0200
+
+    Merge pull request #5160 from theanalyst/wip-11910-hammer
+    
+    mon: pg ls is broken
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit bb12f925cf0d78f97eefc2e271e73596050b9919
+Merge: e801d4c e19f928
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 4 12:34:18 2015 +0200
+
+    Merge pull request #5384 from dachary/wip-12502-hammer
+    
+    rgw: need conversion tool to handle fixes following #11974
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit e801d4c943b8004ef613345505df91057913cd39
+Merge: 78a4024 154f18c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 4 11:22:08 2015 +0200
+
+    Merge pull request #5405 from ceph/wip-12465-hammer
+    
+    Log::reopen_log_file: take m_flush_mutex
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 78a4024c14253503b770070aa36a090c6b8f1eaf
+Merge: a451e88 7034720
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 31 15:18:52 2015 -0400
+
+    Merge pull request #5121 from theanalyst/wip-11983-hammer
+    
+    FAILED assert(!old_value.deleted()) in upgrade:giant-x-hammer-distro-basic-multi run
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit a451e882ab9a929d240747b4b09786ca4b4ce377
+Merge: 218f537 1063f52
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 31 15:17:00 2015 -0400
+
+    Merge pull request #5269 from dachary/wip-12362-hammer
+    
+    stuck incomplete
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 218f537491a46a0251bef7690a7f5a86b988ee63
+Merge: 07fa83a 8abc46a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 31 20:55:48 2015 +0200
+
+    Merge pull request #5117 from theanalyst/wip-12099-hammer
+    
+    rgw: rados objects wronly deleted
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 07fa83aefc9bd4ea1495fa1e117a438b2c460e46
+Merge: a69d431 56c2688
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 31 20:55:29 2015 +0200
+
+    Merge pull request #5118 from theanalyst/wip-12042-hammer
+    
+    DragonDisk fails to create directories via S3: MissingContentLength
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit a69d431ea52b5de7fc5cfe142ff3b69ff64e8048
+Merge: 5353480 c78cc00
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 31 20:55:14 2015 +0200
+
+    Merge pull request #5214 from SUSE/wip-12299-hammer
+    
+    RGW Swift API: support for 202 Accepted response code on container creation
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 5353480f1df297ad9dd1bd3154887ed2564f0280
+Merge: fb9156f a5dbcbb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 31 20:54:57 2015 +0200
+
+    Merge pull request #5226 from SUSE/wip-12322-hammer
+    
+    rgw: keystone does not support chunked input
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit fb9156f412f83c0ce1fe4c5c9ccd57fd79f0c992
+Merge: 7193c16 b1618a9
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 31 20:54:41 2015 +0200
+
+    Merge pull request #5227 from SUSE/wip-12323-hammer
+    
+    RGW Swift API: XML document generated in response for GET on account does not contain account name
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 7193c16b65fdc1694b968899d23eae0638d89f11
+Merge: ac86490 e39dce7
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 31 20:54:26 2015 +0200
+
+    Merge pull request #5228 from theanalyst/wip-11872-hammer
+    
+    RGW does not send Date HTTP header when civetweb frontend is used
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit ac86490821336ce024940d48d82f7a5ff7a302b1
+Merge: 33dbfc6 557865c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 31 20:54:02 2015 +0200
+
+    Merge pull request #5229 from theanalyst/wip-12242-hammer
+    
+    Fix tool for #11442 does not correctly fix objects created via multipart uploads
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 33dbfc6919840882c6cbc10dad2fc24cf0720bf9
+Merge: 99ca62f e50caab
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 31 20:53:41 2015 +0200
+
+    Merge pull request #5237 from theanalyst/wip-12245-hammer
+    
+    rgw: empty json response when getting user quota
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 99ca62f2bf1e21a41cb7b6ecdb8a8731a18de195
+Merge: 1f5f319 2357b6c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 31 20:53:18 2015 +0200
+
+    Merge pull request #5284 from SUSE/wip-12398-hammer
+    
+    rgw: Properly respond to the Connection header with Civetweb
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 1f5f31905bb5f499a2db4a02993dbc6efa1c4251
+Merge: 5cbb6cf 9458b84
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 31 20:53:04 2015 +0200
+
+    Merge pull request #5285 from SUSE/wip-12399-hammer
+    
+    rgw: multipart list part response returns incorrect field
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 5cbb6cfb69aad0db470f99e39e33f4b4b1abfb95
+Merge: 1df93e1 e4b55b3
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 31 20:52:43 2015 +0200
+
+    Merge pull request #5286 from SUSE/wip-12400-hammer
+    
+    rgw: radosgw-admin dumps user info twice
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 1df93e19a0275ed218c8f83bc674f16d1856f241
+Merge: a48cbc0 2ecb3b7
+Author: Yan, Zheng <ukernel@gmail.com>
+Date:   Fri Jul 31 09:50:22 2015 +0800
+
+    Merge pull request #5427 from dachary/wip-12088-hammer-part-2
+    
+    Fh ref count will leak if readahead does not need to do read from osd
+
+commit 2ecb3b7f4a49c574bc178a106c6bf0d8247f2a5e
+Author: Zhi Zhang <zhangz.david@outlook.com>
+Date:   Wed Jul 22 10:54:53 2015 +0800
+
+    Fh ref count will leak if readahead does not need to do read from osd
+    
+    The 3c8cdeacf46ae4031189d2ef6948aa3b6ab4ae43 backport introduced a leak.
+    
+    http://tracker.ceph.com/issues/12319 Fixes: #12319
+    
+    Signed-off-by: Zhi Zhang <zhangz.david@outlook.com>
+
+commit a48cbc0a847f19ea613b76a479acc831e9316c62
+Merge: 06c27cd 5ef0846
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Jul 30 21:43:48 2015 +0200
+
+    Merge pull request #5120 from theanalyst/wip-11999-hammer
+    
+    cephfs Dumper tries to load whole journal into memory at once
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 06c27cdd420598c497766ee5879335942a0acc09
+Merge: 19abe5e 408880b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Jul 30 21:43:21 2015 +0200
+
+    Merge pull request #5119 from theanalyst/wip-12098-hammer
+    
+    kernel_untar_build fails on EL7
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 19abe5ee35c099c67b56ac268710fcd20bec60d3
+Merge: e3d17e4 4c199bf
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Jul 30 17:00:14 2015 +0200
+
+    Merge pull request #5417 from dachary/wip-11998-hammer
+    
+    debian/control: ceph-common (>> 0.94.2) must be >= 0.94.2-2
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 4c199bf57dc54dc5e5f45cd9b34878a8459d434e
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Jul 30 09:43:20 2015 +0200
+
+    debian/control: ceph-common (>> 0.94.2) must be >= 0.94.2-2
+    
+    The d8733be2ef8874b9a858a7ffddfb81b9b656e9a6 backport introduced a
+    regression by adding an incorrect Depends / Break combo supposed to
+    reflect the fact that ceph_argparse moved from ceph to ceph-common after
+    v0.94.2. It assumed the package is released under the 0.94.2 version
+    where in reality it is released under the 0.94.2-1xxx version (where xxx
+    is trusty, jessie etc.).
+    
+    The Depends / Break combo is changed to use 0.94.2-2 instead.
+    
+    See also http://tracker.ceph.com/issues/12529 for a larger discussion.
+    
+    http://tracker.ceph.com/issues/11998 Fixes: #11998
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+
+commit e3d17e49731569ea92917f574d42d93258c77189
+Merge: cbba706 89aa8ff
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jul 29 16:22:36 2015 +0200
+
+    Merge pull request #5248 from ceph/wip-11833-hammer
+    
+    mon: add an "osd crush tree" command
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit cbba7064c6cc4cde3e8a49c25ce671e91d31b9c7
+Merge: 8355bda 3c8cdea
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jul 29 10:49:08 2015 +0200
+
+    Merge pull request #5222 from ceph/hammer-12088
+    
+    client: reference counting 'struct Fh'
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 8355bdab56bc4e5ce4d20ba3486c082f06d8dcd1
+Merge: 52d0e5d ec70533
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Tue Jul 28 23:33:18 2015 +0200
+
+    Merge pull request #5231 from theanalyst/wip-12243-hammer
+    
+    Civetweb RGW appears to report full size of object as downloaded when only partially downloaded
+
+commit 52d0e5da5ebad7fe42c2e469cea9773c7714c2b5
+Merge: 7fd31b1 03c07d7
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 28 22:40:23 2015 +0200
+
+    Merge pull request #5243 from theanalyst/wip-12239-hammer
+    
+    librbd/internal.cc: 1967: FAILED assert(watchers.size() == 1)
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 7fd31b1b3c2c8e9dd3d9e5464775422215f7a4bc
+Merge: 7230de3 5c812c1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 28 22:40:03 2015 +0200
+
+    Merge pull request #5241 from theanalyst/wip-12238-hammer
+    
+    [  FAILED  ] TestLibRBD.ExclusiveLockTransition
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 7230de317736a71a5764cf224bd1309da1c7b3c6
+Merge: 6b6228f 7132277
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 28 22:30:23 2015 +0200
+
+    Merge pull request #5265 from SUSE/wip-12368-hammer
+    
+    linking ceph to tcmalloc causes segfault on SUSE SLE11-SP3
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 6b6228f8949e975cac763513898ea9704cb8baf1
+Merge: d62c3ea f99f312
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 28 22:27:40 2015 +0200
+
+    Merge pull request #5280 from ceph/wip-12384-hammer
+    
+    librbd: add valgrind memory checks for unit tests
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit d62c3ea344d9e49e9586867e872e8d5b3f019948
+Merge: 7b57ff8 b872882
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 28 22:26:25 2015 +0200
+
+    Merge pull request #5279 from ceph/wip-12237-hammer
+    
+    A client opening an image mid-resize can result in the object map being invalidated
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 7b57ff8a9ced6c2f22456ed034cc83d07f82fbb3
+Merge: 481728a f819332
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 28 22:10:03 2015 +0200
+
+    Merge pull request #5283 from SUSE/wip-12397-hammer
+    
+    ceph.spec.in: 95-ceph-osd.rules, mount.ceph, and mount.fuse.ceph not installed properly on SUSE
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 481728a04dd2c85096c3bc01cc37da9642b038ca
+Merge: 54bb924 d8733be
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 28 21:54:33 2015 +0200
+
+    Merge pull request #5206 from SUSE/wip-11998-hammer
+    
+    /usr/bin/ceph from ceph-common is broken without installing ceph
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 54bb924e68ae2b4df65576a5d788d593b9d9e722
+Merge: e099058 c5c627f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 28 21:47:29 2015 +0200
+
+    Merge pull request #5055 from SUSE/wip-12044-hammer
+    
+    rgw/logrotate.conf calls service with wrong init script name
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit e0990583298277f1c631f7c2d2260d6c3fa64c9f
+Merge: 8b93978 e149916
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 28 21:46:11 2015 +0200
+
+    Merge pull request #5040 from SUSE/wip-11964-hammer
+    
+    systemd: Increase max files open limit for OSD daemon
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 8b93978881375d063fe2df8f40406ea650dda766
+Merge: 5a7cab2 22f58ce
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 28 21:45:44 2015 +0200
+
+    Merge pull request #5038 from SUSE/wip-11876-hammer
+    
+    ceph-post-file fails on rhel7
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 5a7cab205bb1b3fdbf49a852cb978fc28eba8212
+Merge: 5218eff 38d36b1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 28 21:45:25 2015 +0200
+
+    Merge pull request #5030 from SUSE/wip-12092-hammer
+    
+    packaging: add SuSEfirewall2 service files
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 5218eff07c303fb2762ea9f38b9a9c23c24efcae
+Merge: 0b54d50 8acfb99
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 28 21:38:14 2015 +0200
+
+    Merge pull request #5028 from SUSE/wip-12090-hammer
+    
+    rcceph script is buggy
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 0b54d50ecd1445dfc1a46552adb83b9dae9210d9
+Merge: 45beb86 37d77d3
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 28 21:37:44 2015 +0200
+
+    Merge pull request #5026 from SUSE/wip-12087-hammer
+    
+    max files open limit for OSD daemon is too low
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit e19f928bd770a37f2f631c4cd796e2e30a494234
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jun 26 16:56:28 2015 -0700
+
+    rgw: conversion tool to fix broken multipart objects
+    
+    Fixes: #12079
+    
+    Broken multipart objects: multipart objects that created on 0.94.2
+    and that start with underscore have a bad locator on their tail objects.
+    This extends the tool that was needed for older issue we've had with
+    hammer multipart objects (that start with underscore). The same usage
+    applies:
+    
+    $ radosgw-admin bucket check --check-head-obj-locator \
+                                 --bucket=<bucket> [--fix]
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f02ca6107172cecd80a490df9f0d66204e62326c)
+
+commit 28d32f6090724d62b6168d64031454f44eb4cc88
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jun 26 13:49:55 2015 -0700
+
+    rgw: only scan for objects not in namespace
+    
+    Fixes: #11984
+    The tool should only work on the head objects, and these are not inside
+    any namespace.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 8103908548bf7d6c9fa47fb181cd450670bae8d6)
+
+commit e22e2b43b4039a44f5f8fbbe59edc21fbe118bdc
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Apr 22 16:04:35 2015 -0700
+
+    rgw_admin: add --remove-bad flag to bucket check
+    
+    Add this flag so that the bad object will be removed (should be called
+    only after user has verified that objects content is correct).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 06d67d9139a95b704b80de527381fd1bbf7981ce)
+
+commit 154f18ce3e52094fe84b058565a865ed97b079d6 (refs/remotes/gh/wip-12465-hammer)
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Jul 24 15:38:18 2015 -0700
+
+    Log::reopen_log_file: take m_flush_mutex
+    
+    Otherwise, _flush() might continue to write to m_fd after it's closed.
+    This might cause log data to go to a data object if the filestore then
+    reuses the fd during that time.
+    
+    Fixes: #12465
+    Backport: firefly, hammer
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 8778ab3a1ced7fab07662248af0c773df759653d)
+
+commit b8728823493b9dfde0333fb41725002fc50e4d9b (refs/remotes/gh/wip-12237-hammer)
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun Jul 5 10:47:38 2015 -0400
+
+    librados_test_stub: read op should return number of bytes read
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f8a7b507983e31399831e802e99429b95386ed41)
+
+commit 7d9fce3aa3832a1b8bd7f18abd4745dbc0033582
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun Jul 5 10:35:28 2015 -0400
+
+    tests: fixed TestObjectMap.InvalidateFlagInMemoryOnly
+    
+    librados and librados_test_stub return different result codes
+    for a read full object operation.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2ace2b77f8ed83e753fe4a48bcc997f5d1dd465f)
+
+commit 4a77be0a65c8b4ec3dc437721f8c321737b260de
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun Jul 5 11:09:09 2015 -0400
+
+    librbd: don't attempt to invalidate an object map in R/O mode
+    
+    The ImageWatcher is not initialized when in R/O mode, which
+    resulted in a NULL pointer dereference.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 64d740f8fa10ba872e324ec2580a4d8c3f99a9ce)
+
+commit 0aea70f68b299441e692efdce6d5e7ff18b78c39
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 23 11:17:12 2015 -0400
+
+    tests: add new unit tests for object map invalidation
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0215e9753c09460f6fc84ded9397e36a209f2e32)
+
+commit c732cb889b4a61254d06703bf032082e56b196de
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Mar 25 09:41:13 2015 -0400
+
+    librbd: move object map codes to common location
+    
+    These codes will need to be accessible from cls_rbd and librbd.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4ac584c34d576b489ed4c4862703b8fb427b3bc2)
+
+commit 27c99ea972a7b218ea591b208d0d1dd51eef6f95
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 23 11:14:51 2015 -0400
+
+    librbd: only update image flags when holding exclusive lock
+    
+    It was possible for a client to open an image while another client
+    was shrinking an image.  This would result in the former invalidating
+    the object map on-disk if it openned the image between updating the
+    image header and resizing the object map.
+    
+    Fixes: #11791
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit eb81a6a7e391327ac993fd406443b206a7f7bffc)
+
+commit ef453630200ab72373f08357ca6b5ac5c5bbb397
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jul 17 12:43:46 2015 -0400
+
+    librbd: new ImageWatcher::is_lock_supported method
+    
+    The new version does not attempt to acquire the snap_lock, to avoid
+    cases where a recursive lock would result.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit e4b55b398e68e870a7cf21276e63da2c4c6e3faa
+Author: guce <guce@h3c.com>
+Date:   Sat Jul 11 14:08:33 2015 +0800
+
+    Fixes: #12286 radosgw-admin: after subuser modify print only once user info.
+    
+    remove rgw_admin.cc OPT_SUBUSER_MODIFY, show_user_info code block.
+    
+    switch (opt_cmd) {
+    ...
+    case OPT_SUBUSER_MODIFY:
+    show_user_info(info, formatter);       //show first time (remove this)
+    break;
+    ...
+    }
+    
+    // output the result of a user operation
+    if (output_user_info) {
+    ...
+    show_user_info(info, formatter);       //show second time
+    }
+    
+    test fix:
+    before: after subuser modify print twice user info.
+    after changes, do the same procedure, print only once user info.
+    
+    Signed-off-by: guce guce@h3c.com
+    (cherry picked from commit c604dd97fc179e5c2f640818c0f6e7cf99701947)
+
+commit 9458b845bf863ccf878873c4f0b089ddf84c7203
+Author: Henry Chang <henry@bigtera.com>
+Date:   Wed Apr 22 18:26:45 2015 +0800
+
+    rgw: fix ListParts response
+    
+    The response XML element name should be 'ListPartsResult'.
+    
+    Fixes: #11494
+    
+    Signed-off-by: Henry Chang <henry@bigtera.com>
+    (cherry picked from commit caa9f0e461f1eed526fc43ee74699a7243aef9b8)
+
+commit 2357b6c808f4f7c5997af48149585a6051c04b8f
+Author: Wido den Hollander <wido@42on.com>
+Date:   Sat Jul 11 00:01:52 2015 +0200
+
+    rgw: If the client sends a Connection: close header respond accordingly.
+    
+    HTTP/1.1 assumes Keep-Alive by default, but if a Connection: close header is send
+    the server should respond with it as well.
+    
+    This makes the client close the connection after the request.
+    
+    Fixes: #12298
+    (cherry picked from commit 79197d3711edc4b04a7ea4335b6e1b65754996d5)
+
+commit f819332e2826eae14849c5e68a380d1d87039d22
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jul 9 21:38:46 2015 +0200
+
+    ceph.spec.in: install 95-ceph-osd.rules, mount.ceph, and mount.fuse.ceph properly on SUSE
+    
+    http://tracker.ceph.com/issues/12261 Fixes: #12261
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 5ce38b9536efabf99a236c7a9d15c149fa4c16a6)
+
+commit d8733be2ef8874b9a858a7ffddfb81b9b656e9a6
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Apr 14 07:58:17 2015 -0600
+
+    debian: move ceph_argparse into ceph-common
+    
+    Prior to this commit, if a user installed the "ceph-common" Debian
+    package without installing "ceph", then /usr/bin/ceph would crash
+    because it was missing the ceph_argparse library.
+    
+    Ship the ceph_argparse library in "ceph-common" instead of "ceph". (This
+    was the intention of the original commit that moved argparse to "ceph",
+    2a23eac54957e596d99985bb9e187a668251a9ec)
+    
+    http://tracker.ceph.com/issues/11388 Refs: #11388
+    
+    Reported-by: Jens Rosenboom <j.rosenboom@x-ion.de>
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 110608e5bdd9e2f03020ad41f0c2d756684d4417)
+    
+    Conflicts:
+    	debian/ceph.install
+                There is no ceph_daemon.py in hammer
+            debian/control
+                Depends/Replaces/Breaks version adapted (from 9.0.0 to 0.94.2)
+                also adapted ceph-dbg Replaces/Breaks
+
+commit f99f3125ff76628e2525dca00bb7b983f941a08b (refs/remotes/gh/wip-12384-hammer)
+Author: Zhiqiang Wang <zhiqiang.wang@intel.com>
+Date:   Fri Mar 20 16:15:42 2015 +0800
+
+    test: potential memory leak in FlushAioPP
+    
+    Should call the release function instead of deleting it to free
+    librbd::RBD::AioCompletion and librbd::AioCompletion. Otherwise there is
+    a potential memory leak.
+    
+    Signed-off-by: Zhiqiang Wang <zhiqiang.wang@intel.com>
+    (cherry picked from commit ada7ec860cb7901c560c12a5af36dc7c23051b76)
+
+commit a4fc63af630e77586e3ba2f17df3b6be4a1e2055
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Apr 28 15:25:49 2015 -0400
+
+    pybind: fix valgrind warning on rbd_get_parent_info call
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2586e3ba1e20603a87c833513e09dae9281beb4d)
+
+commit aa3eb28f6be62991bc790de5c19cb7b6e30fa189
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Apr 28 11:12:00 2015 -0400
+
+    osdc: invalid read of freed memory
+    
+    The bytes not in cache stat was potentially reading the bh length
+    from a deleted bufferhead.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5ccc4422d6172376bd6f1be8d3a99c0a54eab807)
+
+commit 18ede754388372cf210d7db87fa46f3536cf0e44
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Apr 28 10:56:15 2015 -0400
+
+    krbd: fix incorrect types in the krbd API
+    
+    The C API functions were referencing the C++ CephContext
+    instead of the C rados_config_t.  Additionally, the ceph
+    namespace was missing on the Formatter class.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 740fd275a60630e60b3bcf41637a2ca486885d9c)
+
+commit 488578c1d557ebec7e50d53e45ed46f42984f4f8
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Apr 28 10:54:47 2015 -0400
+
+    fsx: cleanup crypto library at exit
+    
+    Also made small tweaks so that it can be compiled under
+    a C++ compiler.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c44f8e7fbc19924a9453d8c032c624ebb6c0296f)
+
+commit 97ff6cb2f8fdd4d946eeab338ec225450e3ad8f3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Apr 24 14:29:59 2015 -0400
+
+    tests: add run-rbd-valgrind-unit-tests.sh
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5534faaa469b8a6a4c9687aad1a6723f3e859353)
+
+commit e690907cbb3b229f84f1e996d58636d00f823e8f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Apr 24 00:23:03 2015 -0400
+
+    valgrind: update valgrind suppressions for lttng-ust
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8d87bdf597aad3d6be47aedd216a673bd9093a24)
+
+commit fe013e0a813c5697e917da642143388de60e8528
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Apr 24 00:21:15 2015 -0400
+
+    librbd: TaskFinisher should finish all queued tasks
+    
+    The destructor wasn't waiting for all Finisher tasks
+    to complete before stopping the thread.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8e20240e4155e2f0398e79f4c0095d2d6ba1d4cb)
+
+commit 43cd3ac923c9accfb81acf41f5bd12b8a05322c7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 23 23:10:23 2015 -0400
+
+    tests: fix valgrind errors with librbd unit test
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ed5472a10eb515e2a177a640c3f6ed929db9ee4f)
+
+commit 5d8d6a1a776f833847edc80d2a9b31ecb440ade5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 23 23:09:45 2015 -0400
+
+    tests: librbd should release global data before exit
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6ab1bb5614a5d257a82cf8ea280eef5c90cf765b)
+
+commit 13f926e4e96d0b7178a9762bbbf589961dba47b7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 23 23:08:51 2015 -0400
+
+    librados_test_stub: cleanup singleton memory allocation
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 54c88255b74741d882b88f791497862635357634)
+
+commit 45beb86423c3bd74dbafd36c6822e71ad9680e17
+Merge: 5e399b0 582cf73
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 17 19:48:05 2015 +0200
+
+    Merge pull request #5046 from ceph/wip-12109-hammer
+    
+    librbd: new QA client upgrade tests
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 1063f5275d1031812d564a1bd8ada64bed561026
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 20 12:08:15 2015 -0700
+
+    PG::find_best_info: ignore info.les for incomplete peer
+    
+    See included update to doc/dev/osd_internals/last_epoch_started.rst
+    
+    Fixes: 11687
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 371d9baa120dc0302e9e61d3bc0e309dfaa773a0)
+
+commit 5e399b035d7cf861cf66a8ead00b388c4857cbb6
+Merge: 706b1c7 ad5745b
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Thu Jul 16 14:58:49 2015 -0700
+
+    Merge pull request #5159 from theanalyst/wip-11701-hammer
+    
+    make the all osd/filestore thread pool suicide timeouts separately configurable
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 713227791ab28c5e09073acb7b2c3c83ca0f0d6a
+Author: Thorsten Behrens <tbehrens@suse.com>
+Date:   Mon Mar 16 00:13:38 2015 +0100
+
+    Conditional-compile against minimal tcmalloc.
+    
+    Certain older systems (SLE11 in this case) do not provide the full
+    tcmalloc functionality, due to e.g. incomplete libunwind
+    pieces. Use --with-tcmalloc-minimal to enable the cut-down
+    version.
+    
+    Here's how the various mem allocator switches interact now:
+    
+    --with-jemalloc: overrides --with-tcmalloc & --with-tcmalloc-minimal
+    --with-tcmalloc-minimal: overrides --with-tcmalloc
+    --with-tcmalloc: the default. use --without-tcmalloc to disable
+    
+    Signed-off-by: Thorsten Behrens <tbehrens@suse.com>
+    (cherry picked from commit c6f1c07113ca19547fdac10cd9b817a60142aee2)
+
+commit 706b1c7c5bcaaff96aa6950302b7aef097918d30
+Merge: daf5450 5e72479
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jul 16 11:04:52 2015 -0400
+
+    Merge pull request #5252 from ceph/wip-12021-hammer
+    
+    OSDMonitor: allow addition of cache pool with non-empty snaps with co…
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit daf5450765684b0b2ed049320d7463b637321e5a
+Merge: d20f513 bd91fb0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Jul 16 01:15:51 2015 +0200
+
+    Merge pull request #4891 from theanalyst/wip-11740-hammer
+    
+    crush: take crashes due to invalid arg
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit ad5745bfd768b52ae6a766391232becad8587641
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 6 10:49:00 2015 -0700
+
+    OSD: add command_wq suicide timeout
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit df4e5de819c30003cfbe50a071c49039cf534419)
+    
+    Conflicts:
+    	src/common/config_opts.h
+    Trivial merge conflict
+
+commit 059a579c02b312bbd32fa41485c361ae3847a3ba
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 6 10:54:31 2015 -0700
+
+    OSD: add remove_wq suicide timeout
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit f2fbfa32a16666be46359f0eab7b04ca80a753f5)
+
+commit b8826bc4e3da6fcb9338ad6c01af1a88e6585a4d
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 6 10:52:40 2015 -0700
+
+    OSD: add scrub_wq suicide timeout
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 547a7041edc833f3cc8e04d388574809e30a8af6)
+
+commit 878dd403930a2058656a99c14b465358e134843c
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 6 10:51:28 2015 -0700
+
+    OSD: add snap_trim_wq suicide timeout
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit e1073a4a577211672148a4112bd633831552d66f)
+
+commit 11575832a37ea247a8febe912b3058f51a464ab6
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 6 10:50:19 2015 -0700
+
+    OSD: add recovery_wq suicide timeout
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 85311b656852af75bfbbc6699f92fc6aa233c316)
+    
+    Conflicts: src/common/config_opts.h
+    
+    There was a merge conflict due to introduction of `osd_recovery_sleep`
+    which was introduced in #3829
+
+commit a82b4505848c09ad0094768c886f2015bdaa1148
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 6 11:02:19 2015 -0700
+
+    OSD: add op_wq suicide timeout
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+
+commit 89aa8ff9855ae868d59bd10fe3a3aab8517e90fc
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jun 2 23:52:22 2015 +0800
+
+    mon: add an "osd crush tree" command
+    
+    * to print crush buckets/items in a tree
+    
+    Fixes: #11833
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5436c290f3622feb8d4b279ed6552b2510e0cee9)
+    
+    Conflicts:
+    	src/test/mon/osd-crush.sh:
+    		do not start mon as run() takes care of it already
+
+commit d20f513d9b185eff82bee2ca719b5453358e740b
+Merge: 8753b2b 3d74164
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 14 20:43:02 2015 +0200
+
+    Merge pull request #4899 from theanalyst/wip-11911-hammer
+    
+    start_flush: filter out removed snaps before determining snapc's
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 8753b2b14536c34a7b6dec927c7a5b8100de7f68
+Merge: 3d72652 ecac1a4
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 14 20:42:45 2015 +0200
+
+    Merge pull request #4868 from SUSE/wip-11879-hammer
+    
+    Clock skew causes missing summary and confuses Calamari
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 3d72652d7ba6b2fff3d39ea7965c3c61d5fa0a04
+Merge: 9a79e8e fdb43eb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jul 14 16:42:12 2015 +0200
+
+    Merge pull request #4883 from SUSE/wip-11638-hammer
+    
+    ceph.spec.in: ceph-common subpackage def needs tweaking for SUSE/openSUSE
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 03c07d76ac8361ddd302f5bc0575aee7fb5edc99
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 26 09:59:36 2015 -0400
+
+    librbd: assertion failure race condition if watch disconnected
+    
+    It's possible for librbd's watch of the header object to be reset by
+    connection issues just prior to the image being removed.  This will
+    causes an assertion failure which assumes at least one watcher on the
+    image.
+    
+    Fixes: #12176
+    Backport: hammer, firefly
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit af276de4f299960e43761904c043924cec5fef11)
+
+commit 5c812c1552d954f2c91c000332ddc74c9e91825a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 25 16:51:31 2015 -0400
+
+    librbd: prevent object map updates from being interrupted
+    
+    Object map updates were being canceled in-flight when the exclusive lock
+    is released.  This resulted in an ERESTART error code bubbling up to
+    AioRequest.
+    
+    Fixes: 12165
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 590cdc90edaf4f4ff06c97eb2f43b92ab9b60084)
+    
+    Conflicts:
+    	src/librbd/ObjectMap.h
+    conflict due to a variable `m_snap_id' which was introduced in PR #4140
+    which is dropped as we are not backporting that feature
+
+commit e50caab2251bb68fea1adbd17acc43aa98ab1206
+Author: wuxingyi <wuxingyi@letv.com>
+Date:   Tue Jun 23 01:46:48 2015 +0000
+
+    rgw: fix empty json response when getting user quota
+    
+    Fixes: #12117
+    Signed-off-by: wuxingyi <wuxingyi@letv.com>
+    (cherry picked from commit 64fceed2202c94edf28b8315fe14c9affa8c0116)
+
+commit ec705336551436517c16bffdc6bf5467899ae4bb
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jun 25 14:31:03 2015 -0700
+
+    rgw: error out if frontend did not send all data
+    
+    Fixes: #11851
+    The civetweb mg_write() doesn't return error when it can't flush all data
+    to the user, it just sends the total number of bytes written. Modified the
+    client io to return total number of bytes and return an error if didn't
+    send anything.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit daa679c3dd3770a6d5421e2cc9a36924f4550439)
+
+commit 557865c85bb907fe69248c4f1acb88320a7c1bb5
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jun 17 15:11:28 2015 -0700
+
+    rgw: fix reset_loc()
+    
+    Fixes: #11974
+    
+    Only need to set locator for underscore if namespace is empty
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d3bd27f4855df6bb207b656527138026af1a36a2)
+
+commit b1618a97fef644dc3dced502d600de6a5d55d085
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Mon Apr 20 14:55:00 2015 +0200
+
+    rgw: fix lack of account name in XML listing of Swift account.
+    
+    Fixes: #11431
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 837388bbc39a1bf9019302c3a4d3a3fe22caeeb4)
+
+commit e39dce7935dd513b77ce34bc79d70a2c23437cbb
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed Feb 18 15:48:43 2015 +0100
+
+    rgw: generate the "Date" HTTP header for civetweb.
+    
+    Fixes: #10873
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit ea384f83b601f60e135c3d3f960fdb75a919dd84)
+
+commit a5dbcbbdddce6cdeccb1e6f5641601d673cd1896
+Author: Hervé Rousseau <hroussea@cern.ch>
+Date:   Mon Apr 27 17:54:30 2015 +0200
+
+    Swift: Set Content-Length when requesting/checking Keystone tokens
+    
+    Running Keystone with WSGIChunkedRequest=On is not supported.
+    
+    We have to make sure that we set the Content-Length header when getting
+    an admin token and checking revoked tokens, otherwise Keystone returns
+    a HTTP 411 error.
+    
+    Same applies when checking revoked tickets.
+    
+    Fixes: #11473
+    Backport: Hammer, Firefly
+    Signed-off-by: Hervé Rousseau <hroussea@cern.ch>
+    (cherry picked from commit 24f477417fdac9d68902fa211c8edf92a2e8729f)
+
+commit 3c8cdeacf46ae4031189d2ef6948aa3b6ab4ae43
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jul 8 10:11:43 2015 +0800
+
+    client: reference counting 'struct Fh'
+    
+    The async readahead finisher needs to reference 'struct Fh'. But
+    it's possible user closes FD and free the corresponding 'struct Fh'
+    before async readahead finishes.
+    
+    Fixes: #12088
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 34b939a81d38173b882c429b28dedce778504ba8)
+
+commit c78cc00afb6deb8022db60dbe8649335f61bd345
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Thu Mar 19 14:52:18 2015 +0100
+
+    rgw: rectify 202 Accepted in response for PUT on existing bucket.
+    
+    Fixes: #11148
+    Backport: hammer
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 3998fe7e02a6c25a3302c80a9c9907357fd3a23e)
+
+commit 9a79e8e7da5f34f1adaf6137e01bcd42766ae677
+Merge: 5527720 7f1c0cc
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Sun Jul 12 02:19:57 2015 +0800
+
+    Merge pull request #5208 from tchaikov/wip-11975-hammer
+    
+    tests: TEST_crush_reject_empty must not run a mon
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 7f1c0cc9cd3deab925440b56d82c3e61a8ba5ab1
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Jul 11 23:04:33 2015 +0800
+
+    crush/CrushTester: return EINVAL if crushtool returns non-zero
+    
+    this backports a tiny part of ec02441, otherwise
+    CrushTester will return 1, and "ceph" cli will take it
+    as EPERM, which is miss leading, and fails
+    osd-crush.sh:TEST_crush_reject_empty.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+
+commit 2aaeea145b24b972a0b98549c3527ccf98f4c96f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Jul 10 16:23:47 2015 +0200
+
+    tests: TEST_crush_reject_empty must not run a mon
+    
+    * Back in Hammer, the osd-crush.sh individual tests did not run the
+      monitor, it was taken care of by the run() function. An attempt to run
+      another mon fails with:
+    
+      error: IO lock testdir/osd-crush/a/store.db/LOCK: Resource temporarily
+      unavailable
+    
+      This problem was introduced by cc1cc033930e8690a57674e842a003f6bbc7a242
+      from https://github.com/ceph/ceph/pull/4936
+    * replace test/mon/mon-test-helpers.sh with test/ceph-helpers.sh as
+      we need run_osd() in this newly added test
+    * update the run-dir of commands: ceph-helpers.sh use the different
+      convention for the run-dir of daemons.
+    
+    http://tracker.ceph.com/issues/11975 Refs: #11975
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+
+commit 80afb81124a0d2ef25a23a12c86617ab1da3a4bd
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Apr 21 16:11:33 2015 +0200
+
+    ceph-helpers: implement test_expect_failure
+    
+    To display the output in case the command did not fail with the expected
+    output.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 5871781b10ff0b26c731b70d1898c474006cbee3)
+
+commit 6b5e9a1df7dfb3a971e40aec35119ec019515b69
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Jun 10 23:16:01 2015 +0200
+
+    tests: display the output of failed make check runs
+    
+    After a make check fails, it shows a summary but not the output of the
+    failed tests although they contain information to diagnose the problem.
+    
+    Set the VERBOSE=true automake variable which is documented to collect
+    and display the failed script output at the end of a run (the content of
+    the test-suite.log file (valid from automake-1.11 up).
+    
+    http://www.gnu.org/software/automake/manual/automake.html#index-VERBOSE
+    
+    Also remove the run-make-check.sh that did the same in a way that is not
+    compatible with automake-1.11.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 3a55cb029bb7db9542d2b14f2deda90feb0ae0f6)
+
+commit 552772025cb8d5f51ffb3a069d1bd93bc73f1123
+Merge: f4d77c2 1440122
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 10 10:17:24 2015 +0200
+
+    Merge pull request #4889 from theanalyst/wip-11484-hammer
+    
+    OPT_INT option interprets 3221225472 as -1073741824, and crashes in Throttle::Throttle()
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit f4d77c22aa51edb45211e080f3fdf28a7a0cfdd4
+Merge: 5088105 a62c3aa
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 10 10:16:25 2015 +0200
+
+    Merge pull request #4776 from tchaikov/wip-11279-hammer
+    
+    ceph: cli interactive mode does not understand quotes
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 5088105300c013b1b804c938a30ac63ba710556d
+Merge: e3b1f7b 0b6d442
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 10 10:14:42 2015 +0200
+
+    Merge pull request #4657 from ceph/wip-hammer-11535-admin-socket
+    
+    common/admin_socket: close socket descriptor in destructor
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit e3b1f7be9e8474fbec98076790ff683bccd44ce9
+Merge: dd29a86 558d639
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Fri Jul 10 16:07:48 2015 +0800
+
+    Merge pull request #4687 from SUSE/wip-7387-hammer
+    
+    utf8 and old gcc breakage on RHEL6.5
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit dd29a869db5503fc9e2c6d1d44ee4311d95af20c
+Merge: 7f1fb57 0e5e7e1
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Fri Jul 10 16:00:00 2015 +0800
+
+    Merge pull request #5122 from theanalyst/wip-11982-hammer
+    
+    ceph fails to compile with boost 1.58
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 7f1fb574608800c3e6aa12df6c7888acbf397a52
+Merge: adc7016 5141301
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Fri Jul 10 15:59:35 2015 +0800
+
+    Merge pull request #4936 from ceph/wip-11975-hammer
+    
+    mon crashes when "ceph osd tree 85 --format json"
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit adc70161d14fc2b51e6c6f38580f76ff0067717a
+Merge: 2d68db8 82988d6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 10 09:48:44 2015 +0200
+
+    Merge pull request #4892 from theanalyst/wip-11760-hammer
+    
+    ceph-disk: get_partition_type fails on /dev/cciss...
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 2d68db8371263645642cf28473deea4456ca7021
+Merge: 1cffe8c ba1a016
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 10 09:43:57 2015 +0200
+
+    Merge pull request #4877 from SUSE/wip-11902-hammer
+    
+    admin/build-doc: script fails silently under certain circumstances
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 51413011417b76f5ad2830d9f93fbfe78c77e467
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue May 26 18:11:59 2015 +0800
+
+    mon: add "--check" to CrushTester::test_with_crushtool()
+    
+    so we don't need to call CrushTester::check_name_maps() in OSDMonitor.cc
+    anymore.
+    
+    Fixes: #11680
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit c6e634875316cf17368d497e6dc4f6f4b5dd65d2)
+
+commit 5ec27cf589b4535f07e28a86bd304f7a46427ac4
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue May 26 17:51:50 2015 +0800
+
+    crushtool: rename "--check-names" to "--check"
+    
+    * because "--check" also checks for the max_id
+    
+    Note: edited since we do not have the fix introduced in 46103b2 in
+          hammer.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 9381d53acdce85fcbff828926b911e050ba36e51)
+
+commit 2a8fe8862a15342cc5716c146487d0b42af0fbf6
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue May 26 16:58:23 2015 +0800
+
+    mon: check the new crush map against osdmap.max_osd
+    
+    Fixes: #11680
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 22e6bd6e01d5df3f3e897562597e22ca1737f8c8)
+
+commit c0b0f52ddbd4e22998a36addfb32f27614183e19
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue May 26 15:35:10 2015 +0800
+
+    crushtool: enable check against max_id
+    
+    add an argument "max_id" for "--check-names" to check if any item
+    has an id greater or equal to given "max_id" in crush map.
+    
+    Note: edited since we do not have the fix introduced in 46103b2 in
+          hammer.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d0658dd3cdf072b2a7c2a1986f8785a697c591ee)
+
+commit f041bbebf98a2aff2ad542e8d0c12c46af427573
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue May 26 15:34:33 2015 +0800
+
+    crush/CrushTester: check if any item id is too large
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit e640d89240017956b8c7411babb86be0f1e2b172)
+
+commit cc1cc033930e8690a57674e842a003f6bbc7a242
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon May 25 20:14:32 2015 +0800
+
+    mon: validate new crush for unknown names
+    
+    * the "osd tree dump" command enumerates all buckets/osds found in either the
+      crush map or the osd map. but the newly set crushmap is not validated for
+      the dangling references, so we need to check to see if any item in new crush
+      map is referencing unknown type/name when a new crush map is sent to
+      monitor, reject it if any.
+    
+    Fixes: #11680
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit a955f36a509e5412b1f72632a1a956d99e768e35)
+
+commit ff29a7f9dd21505c681881e609183aed9ac3250e
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue May 26 12:08:36 2015 +0800
+
+    crushtool: add the "--check-names" option
+    
+    * so one is able to verify that the "ceph osd tree" won't chock on the
+      new crush map because of dangling name/type references
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d6b46d4c7b722945ce24ac2930381a109b1e3dda)
+
+commit 960ea49699f421ceb89c9e0c9430378a35f09a9a
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue May 26 12:08:09 2015 +0800
+
+    crush/CrushTester: add check_name_maps() method
+    
+    * check for dangling bucket name or type names referenced by the
+      buckets/items in the crush map.
+    * also check for the references from Item(0, 0, 0) which does not
+      necessarily exist in the crush map under testing. the rationale
+      behind this is: the "ceph osd tree" will also print stray OSDs
+      whose id is greater or equal to 0. so it would be useful to
+      check if the crush map offers the type name indexed by "0"
+      (the name of OSDs is always "OSD.{id}", so we don't need to
+      look up the name of an OSD item in the crushmap).
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b75384d73958faf81d45847a7dfa56f4fa347e6f)
+
+commit 1cffe8c46a39142ee0da4e2279eda7276df262e1
+Merge: 6ffb1c4 ef6641c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jul 10 09:39:15 2015 +0200
+
+    Merge pull request #4667 from SUSE/wip-11611-hammer
+    
+    ceph.spec: update OpenSUSE BuildRequires
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 5e72479b0882ac13597d7a613698e583dcb2b932
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Jul 7 11:43:01 2015 -0700
+
+    OSDMonitor: allow addition of cache pool with non-empty snaps with config
+    
+    We need to be able to allow the version of ceph_test_* from earlier
+    versions of ceph to continue to work.  This patch also adjusts the
+    work unit to use a single rados snap to test the condition without
+    --force-nonempty to ensure that we don't need to be careful about
+    the config value when running that script.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+
+commit 6ffb1c4ae43bcde9f5fde40dd97959399135ed86
+Merge: c7ebf96 524f4a5
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Wed Jul 8 16:52:12 2015 +0100
+
+    Merge pull request #5123 from theanalyst/wip-11979-hammer
+    
+    MDSMonitor: handle MDSBeacon messages properly
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit ecac1a458bc7331ed8d667f20ba31995d74892d3
+Author: Thorsten Behrens <tbehrens@suse.com>
+Date:   Fri Feb 6 01:26:40 2015 +0100
+
+    Always provide summary for non-healthy cluster
+    
+    This fixes a problem, wherein calamari does not provide
+    popup drill-downs for warnings or errors, should the summary
+    be missing.
+    
+    Calamari gets health info from /api/v1/cluster/$FSID/health.
+    If the data here has a summary field, this summary is provided
+    in a popup window:
+    
+      /api/v1/cluster/$FSID/health is populated (ultimately) with
+      status obtained via librados python bindings from the ceph
+      cluster. In the case where there's clock skew, the summary
+      field supplied by the ceph cluster is empty.
+    
+    No summary field, no popup window with more health details.
+    
+    Signed-off-by: Thorsten Behrens <tbehrens@suse.com>
+    (cherry picked from commit eaf6e0cf48488fe604d0ef0db164d44948d4e8d4)
+
+commit c7ebf96a9a4a6143b112c8606d5ee346fb800cec
+Merge: b163728 1a321e4
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jul 8 15:36:38 2015 +0200
+
+    Merge pull request #4862 from SUSE/wip-11874-hammer
+    
+    Bucket header is enclosed by quotes
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit b1637289ec4ff99d923457577893b4c4a8d2e9fe
+Merge: e33af22 54f4e7d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jul 8 15:35:53 2015 +0200
+
+    Merge pull request #4885 from theanalyst/wip-11755-hammer
+    
+    Object copy bug
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit e33af22dec32467f229ca2dc1dc9668702a44ce8
+Merge: 5696b0f 9dfef60
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jul 8 15:34:45 2015 +0200
+
+    Merge pull request #4884 from theanalyst/wip-11722-hammer
+    
+    Keystone PKI token expiration is not enforced
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 5696b0ff2a981ae8afe72df796ba7d7da47bb7d1
+Merge: 72ecd52 ed5442b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jul 8 15:30:48 2015 +0200
+
+    Merge pull request #4875 from ceph/wip-11770-hammer
+    
+    librbd: aio calls may block
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 1a32379dd6cb56ed69b8f448d3612506c8131fbe
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon May 11 19:30:30 2015 +0800
+
+    mon/PGMap: add more constness
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit e1f1c56ce88ef3ad05e881d650fac637931ce3fe)
+
+commit 84ebc3d320bdf871ccf85e555951cea421b56021
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon May 11 19:29:13 2015 +0800
+
+    mon/PGMap: sort pg states by the states in "pg ls" spec
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 990dfb6cd45438bc8293ac37882daa413860a2f8)
+
+commit e310461aecc667cf26806bc5a3bbabb05696bdfc
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon May 11 17:02:41 2015 +0800
+
+    mon: s/recovery/recoverying/ in "pg ls*" commands' spec
+    
+    * also translate "repair" if specified as "states"
+    * update test_mon_pg in cephtool-test-mon.sh
+    
+    Fixes: #11569
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 89f89ca3477eddcae11a05fbd58a8f3658eb1fc1)
+
+commit 524f4a52d115ecda8cd7793d0f8bea148eff92af
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri May 15 22:50:36 2015 +0800
+
+    mon: always reply mdsbeacon
+    
+    the MDS (Beacon) is always expecting the reply for the mdsbeacon messages from
+    the lead mon, and it uses the delay as a metric for the laggy-ness of the
+    Beacon. when it comes to the MDSMonitor on a peon, it will remove the route
+    session at seeing a reply (route message) from leader, so a reply to
+    mdsbeacon will stop the peon from resending the mdsbeacon request to the
+    leader.
+    
+    if the MDSMonitor re-forwards the unreplied requests after they are
+    outdated, there are chances that the requests reflecting old and even wrong
+    state of the MDSs mislead the lead monitor. for example, the MDSs which sent
+    the outdated messages could be dead.
+    
+    Fixes: #11590
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b3555e9c328633c9e1fbc27d652c004b30535e5b)
+
+commit 413e407dea446bebb9c36abb3025ada450dd4fe9
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jun 2 23:20:21 2015 -0700
+
+    mon/MDSMonitor: rename labels to a better name
+    
+    * s/ignore/reply/
+    * s/out/ignore/
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit f00ecb8b3df73ce6337985bc6d43bce5143ee537)
+
+commit a03968ad584a3ff8e351cc5dce053e535fcdc454
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jun 2 12:55:06 2015 +0800
+
+    mon: send no_reply() to peon to drop ignored mdsbeacon
+    
+    so the peon can remove the ignored mdsbeacon request from the
+    routed_requets at seeing this reply, and hence no longer resend the
+    request.
+    
+    Fixes: #11590
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 72a37b3a8e145d8522ea67fc14ce2c5510b6852b)
+
+commit 39f34596b0ec6f769f507e2b372204f8551f7ee0
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jun 2 12:22:26 2015 +0800
+
+    mon: remove unnecessary error handling
+    
+    msg.get_session() should always return a non-zero pointer in
+    Monitor.dispatch()
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 16e8e2cc82a90c49cd8aa3d0e3acc4694ba659a0)
+
+commit 0e5e7e1d259579571c1fc05660f6af8e295e733b
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri May 8 15:21:20 2015 +0800
+
+    mon: remove unused variable
+    
+    * as a side effect, this change silences
+      http://tracker.ceph.com/issues/11576
+    
+    Fixes: #11576
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit e7b196a4a091c0ea258866559ba06e7ed0cc4247)
+
+commit 70347209260688f1a067354c744569499adb6920
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Apr 1 16:37:51 2015 -0700
+
+    ReplicatedPG::finish_promote: handle results->snaps is empty case
+    
+    If results->snaps winds up empty after filtering removed snaps,
+    we need to treat the object as if we had gotten an ENOENT.
+    
+    PartialFix: #11296
+    Backport: firefly, hammer
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 6150757dbe0fa11cceb14460865b859a7c8164c7)
+
+commit 3e44dc16ed3fbda053996e44826aa3d90042a234
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Apr 1 16:25:22 2015 -0700
+
+    ReplicatedPG::finish_promote: fix snap promote head snaps
+    
+    If the snaps vector is: 10=[9,5,2]:[4]+head, the backing pool's snaps
+    vector is 3=[2]:[]+head, and we request clone 4 from the backing pool,
+    the backing pool will send us head with an empty results->snaps vector.
+    Actually, clone 4 should be trimmed, but the cache pool does not know
+    that.  Thus, we should construct an empty snaps vector for that clone.
+    
+    PartialFix: #11296
+    Backport: firefly, hammer
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit a45a698372def1623323470c6a1c4eb70e0bb79f)
+
+commit 5ef08466abf1b3934fcad0a0ca46f3a4380d6dbd
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Jun 3 10:04:26 2015 +0100
+
+    tools: chunk reads in Dumper
+    
+    Previously tried to read entire journal
+    into memory in one go, which was problematic
+    for large journals.
+    
+    Fixes: #11746
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit e3ddcb894ad09326698999d42de0ce3feb69f28e)
+
+commit 408880bed296e5cbf05864fa6744a5b00a245272
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Tue Jun 16 08:13:41 2015 -0700
+
+    qa: update to newer Linux tarball
+    
+    This should make newer gcc releases happier in their default configuration.
+    kernel.org is now distributing tarballs as .xz files so we change to that
+    as well when decompressing (it is supported by Ubuntu Precise so we should
+    be all good).
+    
+    Fixes: #11758
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 1ea3f47ab806d48ca7b045c2731d344eae3900e1)
+
+commit 56c2688b87d7d78831f8e147fc67cc0651ab644c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Apr 24 14:45:40 2015 -0700
+
+    rgw: simplify content length handling
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit e97fd5052cab83c5f699531a8c960b93437a8f9f)
+
+commit d9bbef3e470c6406bb65dc40e7e9c08c5d599f73
+Author: Robin H. Johnson <robbat2@gentoo.org>
+Date:   Fri Apr 24 10:49:16 2015 -0700
+
+    rgw: make compatability deconfliction optional.
+    
+    Per request from Yehuda, the deconfliction for having both
+    HTTP_CONTENT_LENGTH and CONTENT_LENGTH set is now optional, and
+    controlled by new configuration boolean, which defaults to false.
+    rgw content length compat
+    
+    X-URL: https://github.com/ceph/ceph/pull/4436#issuecomment-95994887
+    Signed-off-by: Robin H. Johnson <robbat2@gentoo.org>
+    (cherry picked from commit 79d17af1a1ec0659884f768945a7bac6282b5e0b)
+
+commit 0260abd5d265bd63ea9c89f4082c31ba1c5ae8fa
+Author: Robin H. Johnson <robbat2@gentoo.org>
+Date:   Wed Apr 22 12:53:06 2015 -0700
+
+    rgw: improve content-length env var handling
+    
+    The FastCGI specification, section 6.3 on Authorizers, describes a case
+    where HTTP_CONTENT_LENGTH will be set in the environment and
+    CONTENT_LENGTH will NOT be set.
+    
+    Further documention in the code.
+    
+    Fixes: #11419
+    Signed-off-by: Robin H. Johnson <robbat2@gentoo.org>
+    (cherry picked from commit 3e38eab44bfb082fdd2b6f29b8b0357f8f5c11bb)
+
+commit 8abc46a157e4c1431a92a1e52ab694dccff5d514
+Author: wuxingyi <wuxingyi@letv.com>
+Date:   Wed Jun 10 06:57:57 2015 +0000
+
+    rgw: fix data corruption when race condition
+    
+    We should delete the object in the multipart namespace lastly to prevent a previous upload
+    wrongly deleting objects belong to the following upload.
+    
+    Fixes: #11749
+    Signed-off-by: wuxingyi <wuxingyi@letv.com>
+    (cherry picked from commit ac1e729a75b5d995028bbc223bcf5ecce0d112cc)
+
+commit 72ecd522941156c8a7e5303531944b0735dcbeb8
+Merge: 59f37a9 d723e11
+Author: Abhishek L <abhishekl.2006@gmail.com>
+Date:   Wed Jul 1 18:09:46 2015 +0530
+
+    Merge pull request #4886 from theanalyst/wip-11737-hammer
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 59f37a9bafc095181b3f41ec5d93ac58e2cda604
+Merge: 53a2143 89d0266
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Sat Jun 27 17:11:11 2015 +0800
+
+    Merge pull request #5095 from ceph/wip-fix-doc-hammer
+    
+    doc: fix doc build
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 89d0266a60729d5d9747867e8c30abf71a891231
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Jun 27 14:44:55 2015 +0800
+
+    doc: add the corresponding @endcond command for @cond
+    
+    * they are used to applease asphyxiate, as it
+      is not able to handle "enum" sections
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+
+commit 2aa77b33a97e5a3ca134c9c555aa6e7a69ef50f7
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Jun 27 14:43:01 2015 +0800
+
+    doc: remove orphan: directive in ceph-create-keys.rst
+    
+    * it is used to silence the sphinx warning, but conf.py
+      does not like it.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+
+commit ad66e40e8bd598da7c9738cb44abb543008c90a3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Jun 27 14:41:59 2015 +0800
+
+    doc: let doxygen ignore src/tracing
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+
+commit 53a2143eb7e549de1185b01ed0bde841ffa5235a
+Merge: 6f7cd04 bfb1442
+Author: Samuel Just <sam.just@inktank.com>
+Date:   Fri Jun 26 14:19:40 2015 -0700
+
+    Merge pull request #4902 from theanalyst/wip-11908-hammer
+    
+    Fixes for rados ops with snaps
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 6f7cd0460d5729c15966119e0177ddc56a361d8e
+Merge: 78d894a 356bd2c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Jun 25 20:03:42 2015 -0400
+
+    Merge pull request #5069 from dachary/wip-11806-hammer
+    
+    ceph / ceph-dbg steal ceph-objecstore-tool from ceph-test / ceph-test-dbg
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 356bd2c68ca730e766d06c46a0364784f5d72275
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Jun 24 14:58:47 2015 -0400
+
+    debian: ceph-dbg steals ceph-objectstore-tool from ceph-test-dbg (take 2)
+    
+    968573b8930a7c8485bf53e3a989ce2f7d0a2fff incorrectly backported
+    6f11fbf41fab10924b1e0e41fcf27864779d4073. It should instead reflect that
+    ceph-dbg in 0.94.2 and above will break ceph-test-dbg in all versions
+    prior to 0.94.2.
+    
+    In other words, 0.94-XXXX being lower than 0.94.1, upgrading from
+    0.94.1 to 0.94.2 will not notice that ceph-dbg breaks ceph-test-dbg.
+    
+    $ dpkg --compare-versions 0.94-XXXX lt 0.94.1 && echo yes || echo no
+    yes
+    $ dpkg --compare-versions 0.94.2 lt 0.94.1-xxx && echo yes || echo no
+    no
+    
+    http://tracker.ceph.com/issues/11806 Fixes: #11806
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+
+commit c5c627fdd3de669ee94ab7ecd6939c29a4a68011
+Author: wuxingyi <wuxingyi2015@outlook.com>
+Date:   Wed Mar 11 17:34:40 2015 +0800
+
+    rgw/logrotate.conf: Rename service name
+    
+    The service name for ceph rados gateway was changed to "ceph-radosgw",
+    the previous version of service name "radosgw" would cause a failed reload,
+    and finally make it impossible to write any log data to the log file.
+    
+    Signed-off-by: wuxingyi <wuxingyi2015@outlook.com>
+    (cherry picked from commit 9df3f798179481fe8ae6ae873dcb793de7d8f367)
+
+commit 582cf731e05baabd2cd79567af89d7f005c24925 (refs/remotes/gh/wip-12109-hammer)
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 5 13:08:21 2015 -0400
+
+    tests: add librbd watch/notify version compatibility test
+    
+    Fixes: #11405
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 41e4cbe032e32762e3a9e8bc3eff8ece19f91a54)
+
+commit 43b9aef11c1281d8b83b659a523dba365d79add4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 5 11:22:55 2015 -0400
+
+    qa/workunits/rbd: add notify_master/slave bootstrap scripts
+    
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 124b1d3d2d0dbd685bbd717856d29b316e62e660)
+
+commit f995fb50d3d7734161fa498db9e204eaded651b6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 5 10:34:48 2015 -0400
+
+    qa/workunits/rbd: add new test_librbd_api workunit
+    
+    This only tests the public librbd API for use during upgrade tests.
+    
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9039955f42d01044cfcf20c56ca2181e51c317ee)
+
+commit a09da2a2c6c12c6bf2f8e17f2096b3ab24587007
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 5 10:27:38 2015 -0400
+
+    tests: create librbd API-only integration test suite
+    
+    The QA client-upgrade test suite requires a librbd test
+    that is dynamically linked to librbd.  Since the current
+    ceph_test_librbd includes tests against the non-public API,
+    it is statically linked against librbd and thus cannot be
+    used to test a client upgrade scenario.
+    
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6fe94c8cbb924c31c1007e2d0c76f28ce9dbca57)
+
+commit e149916cd40a58ce5db5f63a2b4efd82a8c4e1ba
+Author: Owen Synge <osynge@suse.com>
+Date:   Tue Apr 21 11:31:24 2015 +0200
+
+    Increase max files open limit for OSD daemon.
+    
+    Under heavy load the number of file descriptors opened
+    by the OSD can go beyond the 64K file limit. This patch
+    increases the default to 128K.
+    
+    Signed-off-by: Owen Synge <osynge@suse.com>
+    (cherry picked from commit ebda4ba1c67172852587e47a8e6fb538809a1b1c)
+
+commit 22f58ce2665f1ea6b38f6016dc585202a4595322
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jun 1 15:10:14 2015 -0700
+
+    Makefile: install ceph-post-file keys with mode 600
+    
+    Otherwise ssh (may) prompt for a password.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 106a1c3081d02446aa3d8e13865da0c3393bae90)
+
+commit 3e65a10bd2351744da199a4b076659191b4378a2
+Author: Joseph McDonald <joseph.mcdonald@alcatel-lucent.com>
+Date:   Mon Jun 1 15:05:32 2015 -0700
+
+    ceph-post-file: improve check for a source install
+    
+    Signed-off-by: Joseph McDonald <joseph.mcdonald@alcatel-lucent.com>
+    (cherry picked from commit ee170eadcdcb4b54d36a7d474558484de9d917eb)
+
+commit c1f6743940250b04ae6dbea30d8805571add39b6
+Author: Joseph McDonald <joseph.mcdonald@alcatel-lucent.com>
+Date:   Mon Jun 1 15:00:39 2015 -0700
+
+    ceph-post-file: behave when sftp doesn't take -i
+    
+    Fixes: #11836
+    Signed-off-by: Joseph McDonald <joseph.mcdonald@alcatel-lucent.com>
+    (cherry picked from commit b84031ed5eaace1222e14d3c4076a3ab1155da96)
+
+commit 38d36b1174160ad104704aaa7ff5290d1ec3e782
+Author: Tim Serong <tserong@suse.com>
+Date:   Thu Apr 30 10:55:38 2015 +1000
+
+    packaging: move SuSEfirewall2 templates out of src
+    
+    Better to have static config like this that needs to be installed in
+    /etc in a separate subdirectory of the project.
+    
+    Signed-off-by: Tim Serong <tserong@suse.com>
+    (cherry picked from commit 70292658d5febb6c5f94af4df9c3e93551772d12)
+
+commit 24bc9f2c58ca78c8023935b8b69f8b01adbc11af
+Author: Tim Serong <tserong@suse.com>
+Date:   Wed Apr 29 13:12:38 2015 +1000
+
+    packaging: add SuSEfirewall2 service files
+    
+    This adds SuSEfirewall2 service files for Ceph MON, OSD and MDS, for use
+    on SLES and openSUSE.  The MON template opens port 6789 and the OSD/MDS
+    template opens the range 6800-7300 as per
+    http://docs.ceph.com/docs/master/rados/configuration/network-config-ref/
+    
+    Signed-off-by: Tim Serong <tserong@suse.com>
+    (cherry picked from commit 77685f5b787c56bcb1c4d9f1e058e25312fa62fe)
+
+commit 8acfb994f22efa07beeecccda300cbd50d683566
+Author: Owen Synge <osynge@suse.com>
+Date:   Thu May 7 12:02:41 2015 +0200
+
+    Bug fix to ceph systemV compatability script.
+    
+    Was failing with more than one OSD / MON deamon on a single node.
+    Fixes suse bugzilla #927862
+    
+    Signed-off-by: Owen Synge <osynge@suse.com>
+    (cherry picked from commit dfda3ff8741fcdbac3150456ca7614cf75ef1776)
+
+commit bd3fd929e6f95e6d9840317aa3ac02f52711c94b
+Author: Owen Synge <osynge@suse.com>
+Date:   Thu Jun 18 14:16:03 2015 +0200
+
+    Fixes to rcceph script
+    
+     - only start OSDs if mon daemons are also present
+     - adds support for mask and unmask
+     - removes support for cluster with non default cluster name,
+       as this was very limited and inconsistent
+     - Reapplied from a patch as could not cherry-pick
+       66cb46c411d874be009c225450eea5021cf1219b from Mon Jan 12
+       as this produced issues with src/gmock
+    
+    Signed-off-by: Owen Synge <osynge@suse.com>
+    (cherry picked from commit bfa0c4a626fdbb2bf978ccfab783ac428156144b)
+
+commit 37d77d3680d2c8adda35eddf9625a331a45ece11
+Author: Owen Synge <osynge@suse.com>
+Date:   Tue Apr 21 11:31:24 2015 +0200
+
+    Increase max files open limit for OSD daemon.
+    
+    Under heavy load the number of file descriptors opened
+    by the OSD can go beyond the 64K file limit. This patch
+    increases the default to 128K.
+    
+    Signed-off-by: Owen Synge <osynge@suse.com>
+    (cherry picked from commit ebda4ba1c67172852587e47a8e6fb538809a1b1c)
+
+commit 78d894a634d727a9367f809a1f57234e5e6935be
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 17 09:35:28 2015 -0700
+
+    qa/workunits/rados/test-upgarde-v9.0.1: fix exclude syntax
+    
+    It's -, then a list of all exclusions separated by :.  There are just 2.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 3e8d60a80ce31860eac76a1f6489a35e1795a0c0
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jun 16 21:05:29 2015 -0700
+
+    qa/workunits/rados/test-upgrade-v9.0.1: skip one more evict test
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 348a3d3c9880e7d022e71a2faafe51c8f771406e
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Jun 15 15:12:43 2015 -0700
+
+    qa: add compatibility filtered rados api tests for upgrades
+    
+    Post-9.0.1, the evict op returns success when an object doesn't exist
+    in the cache tier. Skip the tests that are incompatible across
+    versions.
+    
+    Fixes: #11548
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+
+commit d2b80966b8f74de818a671c90b4c821a4b0782db
+Merge: 70bba62 f68bf94
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jun 15 12:38:57 2015 -0700
+
+    Merge pull request #4961 from ceph/wip-11493-hammer
+    
+    backport 11493 fixes, and test, prevetning ec cache pools
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit f68bf94e370fb11a3047ec2762a972a0b7a7c0bb
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri May 15 13:05:40 2015 -0700
+
+    OSDMonitor: disallow ec pools as tiers
+    
+    Fixes: 11650
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 11b7801bb57cb25cd2d26d58722d49691747725b)
+
+commit 13c8d58da1303cc68d99da19f79d625f91f99d43
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Apr 29 12:34:25 2015 -0700
+
+    mon: prevent pool with snapshot state from being used as a tier
+    
+    If we add a pool with snap state as a tier the snap state gets clobbered
+    by OSDMap::Incremental::propogate_snaps_to_tiers(), and may prevent OSDs
+    from starting.  Disallow this.
+    
+    Include a test.
+    
+    Fixes: #11493
+    Backport: hammer, giant, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit bbec53edf9e585af4e20bbc9ba9057d6fdfda342)
+
+commit 58e62662f6ef04ac76470090d1d958467e34194a
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri May 8 10:26:48 2015 -0700
+
+    test/librados/tier.cc: destroy and recreate cache pool on every test
+    
+    Namespaces are not sufficient with the checks for 11493 in the mon.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit bef09e0cdb274cb1c87335a2af9ee532d14a4596)
+
+commit 70bba6226a64090dcf41cd90b23fdf5aed8cd0ca (refs/remotes/jashan/hammer)
+Merge: 3b6977b 5a60a03
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Jun 15 17:26:07 2015 +0200
+
+    Merge pull request #4846 from SUSE/wip-11862-hammer
+    
+    missing man pages for ceph-create-keys, ceph-disk-*
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 3b6977b706dbc99cac25bec1b71a628c398c6ff1
+Merge: 5fb8561 3db1026
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jun 11 13:54:35 2015 -0700
+
+    Merge pull request #4934 from dachary/wip-releases-hammer
+    
+    doc/release-notes: v0.94.2
+
+commit 3db1026f3706e6f5a5c25013cb6646a0298057d8
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 10 12:48:41 2015 -0700
+
+    doc/release-notes: v0.94.2
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 306345b29c259ab04a58ed5d40f801645485b29d)
+    
+    Conflicts:
+    	doc/release-notes.rst
+
+commit ed5442b1057dcc4fb1f9404c805dabe2bbde2252 (refs/remotes/jashan/wip-11770-hammer, refs/remotes/gh/wip-11770-hammer)
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 2 10:33:35 2015 -0400
+
+    tests: verify librbd blocking aio code path
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4cf41486e9c9e1efcb863960a8f3e0326ffca7e5)
+    
+    Conflicts:
+    	src/test/librbd/test_librbd.cc: trival resolution
+
+commit 20e104869f3d17ce672438144700a4d984d487b4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 1 22:56:11 2015 -0400
+
+    librbd: new rbd_non_blocking_aio config option
+    
+    Setting this option to false reverts librbd to legacy behavior
+    where AIO operations could potentially block.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 769cad12716b85d87eacc1069dd9f5c21cad3915)
+
+commit b4571b3e238efc39767f753e0ec1622c8bd6d6e6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 9 20:34:28 2015 -0400
+
+    PendingReleaseNotes: document changes to librbd's aio_read methods
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 9ea1edd0ca9e385f823ad04b05bc887d77aa5136
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 9 13:33:09 2015 -0400
+
+    librbd: AioRequest::send no longer returns a result
+    
+    The librados calls used by AioRequest::send should always return
+    zero unless there is a bug.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c77bce3311ab62892eb8c1d883263ba7ed663b20)
+    
+    Conflicts:
+    	src/librbd/AioRequest.cc: trivial resolution
+    	src/librbd/AsyncFlattenRequest.cc: trivial resolution
+
+commit 272df2aed79a95dd9c45db4e0953e9b321f7b0f5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 21:55:36 2015 -0400
+
+    tests: update librbd AIO tests to remove result code
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 948b15eb52fd5d9ce842fa12ee0cecda17353b01)
+    
+    Conflicts:
+    	src/test/librbd/test_internal.cc: trivial resolution
+    	src/test/librbd/test_librbd.cc: trivial resolution
+
+commit dd2e4c13ff6d88edb25f90af62af16ba825c15c9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 21:37:50 2015 -0400
+
+    librbd: internal AIO methods no longer return result
+    
+    All failures should be returned via the AioCompletion.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9ab42d613128ab08c688ddbea93df4c95068b9cd)
+    
+    Conflicts:
+    	src/librbd/AioRequest.cc: trivial resolution
+    	src/librbd/internal.cc: trivial resolution
+
+commit dbd4e293d7124c89a22148e8fa5f425a995c900c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 21:48:21 2015 -0400
+
+    Throttle: added pending_error method to SimpleThrottle
+    
+    Allow the client of SimpleThrottle to detect an async error
+    so that it can exit early.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b88b88c5df91325fb713c2031a56bffe421268e0)
+
+commit 7df6091a30b1b94d764240262195e971175554b3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 20:18:50 2015 -0400
+
+    librbd: add new fail method to AioCompletion
+    
+    Helper method to handle passing fatal errors generated within
+    librbd (not from the OSDs) back to the client.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6d1d0c867855a96bee4c13a0c0a39a0e002ccd12)
+
+commit cf6e1f50ea7b5c2fd6298be77c06ed4765d66611
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 19:06:52 2015 -0400
+
+    librbd: avoid blocking AIO API methods
+    
+    Enqueue all AIO API methods within the new librbd thread pool to
+    reduce the possibility of any blocking operations. To maintain
+    backwards compatibility with the legacy return codes of the API's
+    AIO methods, it's still possible to block attempting to acquire
+    the snap_lock.
+    
+    Fixes: #11056
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3a7b5e30efdb21aa1a0aeb68a5d02a1ac2a5faf3)
+
+commit e61974aed09a3f81e1f65a4bbaed43e3f22b27b4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 17:24:08 2015 -0400
+
+    librbd: add task pool / work queue for requests
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit afb896d91f886b647baf38f7ec94cc3739f6d2a9)
+    
+    Conflicts:
+    	src/librbd/ImageCtx.cc: trivial resolution
+    	src/librbd/ImageCtx.h: trivial resolution
+
+commit bfb144268b803340efad29cd6c627b170ea32402
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 20 16:10:02 2015 -0700
+
+    ReplicatedPG::release_op_ctx_locks: requeue in scrub queue if blocked
+    
+    Otherwise we can reorder an op around another op which got blocked by a
+    scrub which started after the first blocked on an obc.
+    
+    Fixes: #11691
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit be873eb8da7b29ecefaa5a99b88de7ddcca711ee)
+
+commit c7b6a6370a69149ea94f9e35d536aa90f06e7659
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue May 19 10:56:11 2015 -0700
+
+    ReplicatedPG::finish_ctx: take excl lock if operation is rw
+    
+    Fixes: #11677
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 5c2b795724423ed484ab451de855ddcfc085342b)
+
+commit 1550a569dab120ce28396fe365565e8e4acd9801
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu May 21 12:13:43 2015 -0700
+
+    RadosModel: randomly prefix delete with assert_exists
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 4fe7d2abdff2fce359e5e992206644cc03825ee0)
+
+commit 4cdc5f7d6b3ec488c79c09cb44a43d4d9398b74c
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu May 21 11:36:42 2015 -0700
+
+    RadosModel: assert exists on subsequent writes
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 121aa3bc612b86281535ac3bcfe98bc99bc99ace)
+
+commit 25c730bda74b94f2c894c508ab09988dbd528c4e
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue May 19 10:23:01 2015 -0700
+
+    test/librados/snapshots.cc: add test for 11677
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit c2d17b927f8a222164b3bf2922a4ff337696f566)
+
+commit 3d74164d3d6caaa5099abd9a1d1920482d3e05c2
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 27 11:14:15 2015 -0700
+
+    ReplicatedPG::trim_object: write filtered snapset while we're at it
+    
+    If we trimmed an object, we might as well remove the obsolete snaps
+    as well.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 90eb7768f99ea249952df195a844a3a7c9a59b78)
+
+commit a1161540bc0094a951021d4ca651b95ec045213e
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 27 11:00:54 2015 -0700
+
+    ReplicatedPG: start_flush: use filtered snapset
+    
+    Otherwise, we might send our deletes based on deleted snaps.  This is
+    problematic since we may have trimmed the clones to which those snaps
+    belong, causing us to send them at an earlier snap than we used before.
+    
+    The specific situation was
+    
+    78:[78, 70, 63, 5a, 58, 57]:[64(63), 58(58, 57)]
+    
+    with 58 already clean.  To flush 64, we send:
+    
+    delete@58
+    delete@59
+    copyfrom@62
+    
+    Then, snap 63 is trimmed leaving us with a snapset of:
+    
+    78:[78, 70, 63, 5a, 58, 57]:[58(58, 57)]
+    
+    since trim_object doesn't filter the head object snapset snaps.  This
+    isn't really a bug since in general all snapset users must be aware
+    that there may be trimmed snaps in snapset::snaps.  However, here
+    it becomes a problem when we go to flush head:
+    
+    delete@58 -- ignored due to snapc
+    delete@59 -- ignored due to snapc
+    copyfrom@78 -- not ignored
+    
+    The base pool head is at snap seq 62, so it clones that value into
+    clone 78(78, 70) instead of forgetting it.  What should have happened
+    is that we should have based our flushes on filtered snapset:
+    
+    78:[78, 70, 58, 57]:[58(58, 57)]
+    
+    Causing us to instead send:
+    
+    delete@58 -- ignored due to snapc
+    delete@69 -- not ignored, causes no clone to be made
+    copyfrom@78 -- not ignored, updates head such that a subsequent clone
+    will leave 70 out of the clone snaps vector.
+    
+    Fixes: 11787
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 6051e255ac062985ada1989edb7f23cd750915e2)
+
+commit 82988d611bad6226138b94590275faadbca3554a
+Author: islepnev <islepnev@gmail.com>
+Date:   Fri Apr 17 22:33:01 2015 +0300
+
+    ceph-disk: support NVMe device partitions
+    
+    Linux nvme kernel module v0.9 enumerate devices as following:
+    
+    /dev/nvme0 - characted revice
+    /dev/nvme0n1 - whole block device
+    /dev/nvme0n1p1 - first partition
+    /dev/nvme0n1p2 - second partition
+    
+    http://tracker.ceph.com/issues/11612 Fixes: #11612
+    
+    Signed-off-by: Ilja Slepnev <islepnev@gmail.com>
+    (cherry picked from commit 9b62cf254d02d30609793be8b1cb8a94f38891f1)
+
+commit bd91fb027ab91d487b1d61d25516c13590735d89
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue May 12 16:37:56 2015 -0700
+
+    mon: prevent bucket deletion when referenced by a rule
+    
+    If a rule references a bucket with 'take', prevent deletion.
+    
+    Fixes: #11602
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3d591afef90b0601572c748f13faac029d05f5a0)
+
+commit 56565ee1cdb06a7705d1c3f26f5592b10399324a
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue May 12 14:03:49 2015 -0700
+
+    crush: fix crash from invalid 'take' argument
+    
+    Verify that the 'take' argument is a valid device or bucket.  Otherwise,
+    ignore it (do not add the value to the working vector).
+    
+    Backport: hammer, firefly
+    Fixes: #11602
+    Reported-by: shiva rkreddy <shiva.rkreddy@gmail.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9324d0a1af61e1c234cc48e2175b4e6320fff8f4)
+
+commit 1440122d61d5b0a3f8360f4e2101db1018109799
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Apr 29 03:28:18 2015 -0700
+
+    common/config: detect overflow of float values
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 1ff409ef8d022a1a84d034bd3db976c4d769e993)
+
+commit 9b947fa320b77e0055a581005353c2561a12a198
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Apr 29 15:41:08 2015 +0800
+
+    common/config: detect overflow of int values
+    
+    * #include "strtol.h" in strtol.cc, to ensure the function defintions
+      are consistent.
+    * add a test accordingly
+    * fix the testcase of converting 1024E.
+    * do not accept integers overflow after adding SI suffix
+    * do not accept integers underflow (i.e. negative values)
+    
+    Fixes: #11484
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d62f80dc7b25d312ff05b65b7be854aae15b66a8)
+
+commit d723e1156e70a492d633e43b86e7c373e5750065
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue May 12 14:52:30 2015 +0800
+
+    mds: clear CDir::STATE_REJOINUNDEF after fetching dirfrag
+    
+    Fixes: #11541
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit ab1e5394dc778f6799472bd79a4d9ba7197107c2)
+
+commit 54f4e7d4a534448293c74612f8140cf34b9cf9f8
+Author: Javier M. Mellid <jmunhoz@igalia.com>
+Date:   Fri May 15 14:22:29 2015 +0200
+
+    rgw: Use attrs from source bucket on copy
+    
+    On copy objects, when bucket source is the same as the destination, use attrs
+    from source bucket.
+    
+    Fixes: #11639
+    
+    Signed-off-by: Javier M. Mellid <jmunhoz@igalia.com>
+    (cherry picked from commit 1dac80df1d4a2364154ed8b404d13609936c257b)
+
+commit 9dfef6004d208af3730634796aad199391707826
+Author: Anton Aksola <anton.aksola@nebula.fi>
+Date:   Fri Apr 10 13:25:21 2015 +0300
+
+    rgw: always check if token is expired
+    
+    Fixes: #11367
+    
+    Currently token expiration is only checked by the token cache. With PKI
+    tokens no expiration check is done after decoding the token. This causes
+    PKI tokens to be valid indefinitely. UUID tokens are validated by
+    keystone after cache miss so they are not affected by this bug.
+    
+    This commit adds explicit token expiration check to
+    RGWSwift::validate_keystone_token()
+    
+    Signed-off-by: Anton Aksola <anton.aksola@nebula.fi>
+    Reported-by: Riku Lehto <riku.lehto@nexetic.com>
+    (cherry picked from commit 2df069390ea3bbcfbab5022750e89f51d197cc11)
+
+commit fdb43ebe2b1211acdb454836a64dbd589feeef45
+Author: Nathan Cutler <ncutler@suse.cz>
+Date:   Fri May 15 21:43:34 2015 +0200
+
+    ceph.spec.in: tweak ceph-common for SUSE/openSUSE
+    
+    ceph-common needs python-argparse in SUSE/openSUSE and
+    needs redhat-lsb-core only in RHEL/CentOS/Fedora.
+    
+    http://tracker.ceph.com/issues/11638 Fixes: #11638
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.cz>
+    (cherry picked from commit 363d957d8fdd15a1674befbd8e485fd89b76d716)
+
+commit ba1a0167cc809081eda7e6cc9ecfb971e439d696
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Jun 3 10:09:09 2015 +0100
+
+    admin/build-doc: fix dependency checks
+    
+    http://tracker.ceph.com/issues/11857 Fixes: #11857
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 539c1ba7211f579bad4f59ae824f1e68e620ecbd)
+    
+    Conflicts:
+    	admin/build-doc
+                Insert lines at appropriate spot
+
+commit 5a60a034bf3015eaf468e5e3f9d8feb08b8fdd95
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Jun 3 10:41:27 2015 +0200
+
+    man/ceph-create-keys.8: add missing file
+    
+    This is the generated manpage, taken from a local build of master
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit bcda61fcbe07ee36cd5172d80018f287591660ec)
+
+commit 19305b897ac147602049752a2dfbe1fd39ba562c
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Apr 21 14:59:32 2015 +0800
+
+    doc: add ceph-create-keys.8
+    
+    Fixes: #10725
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 27cee2f1f46a9f47cda9dfeb56ff1259e982960c)
+    
+    Conflicts:
+        doc/man/8/ceph-create-keys.rst
+        Includes fixes from https://github.com/ceph/ceph/pull/4855
+
+commit ffd0933dcc790d7cedc1048b664bf4e8c40464a3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 11 17:05:49 2015 -0400
+
+    WorkQueue: added virtual destructor
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b3f5a75332c058816dc39b71e9d2b36e752159f4)
+
+commit a28adfbdd8abc86e7766c303bc610c0c252910f7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 16:46:34 2015 -0400
+
+    WorkQueue: add new ContextWQ work queue
+    
+    The queue holds a collection of Context pointers that will
+    be completed by the thread pool.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 24a33e977f7b71962adeeb48f75d488a76e70fa9)
+
+commit 1a321e477effa23bffbb6cb057aa955586197345
+Author: Wido den Hollander <wido@42on.com>
+Date:   Wed Jun 3 13:13:33 2015 +0200
+
+    rgw: Do not enclose the Bucket header in quotes
+    
+    Fixes: #11860
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit 8af25faed93fe02d3dad585b8579ce8b041cc4e6)
+
+commit a62c3aa1df9e0f79ac75d94083d10b902dbba382
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Apr 24 14:04:30 2015 +0800
+
+    ceph.in: handle unknown Exception correctly
+    
+    * in case parse_cmdargs() throws
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 7789eefc01deb9ca7fe90f5521aece3e36c3c350)
+
+commit cc7f7441c1d0457e840bfdc3413f9bc6d5f5134a
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Apr 24 01:27:44 2015 +0800
+
+    ceph.in: improve the interactive mode
+    
+    * if ceph is not reading from a tty, expect EOF instead of "quit"
+      as the end of input.
+    * do not panic at seeing the EOF
+    * update the test case test_mon_injectargs_SI(). since we disables
+      "ceph injectargs <args,...>" in a458bd83, in which the arguments
+      of "injectargs" are supposed to be consumed by "tell" instead.
+      so "ceph injectargs ..." is taken as an incomplete command, and
+      this command will bring ceph cli into the interactive mode,
+      redirecting its stdin to /dev/null helps ceph cli quit the loop,
+      but in a way of throwing EOFError exception. this change handles
+      the EOF, so the "ceph injectargs ..." does not throws anymore.
+      but the side effect is that the test fails since it expects a
+      non-zero return code. so replace it with an equivalent "tell"
+      command which also fails but due to the non-SI postfix.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit da9d2b4077ab1dceeed979ab71f0d9ed59b14266)
+
+commit eb26388cb985456e8f6e07a52a20fa912cf4efaa
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Apr 24 00:50:37 2015 +0800
+
+    ceph.in: parse quote correctly in interactive mode
+    
+    Fixes: #11279
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit bc7d8c99d2e16a141a8b575281ba12c67628dac3)
+
+commit 558d6391ecdfd5d716558341ce5d84ce1f7ec9c3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri May 8 12:01:16 2015 +0800
+
+    json_sprit: fix the FTBFS on old gcc
+    
+    Fixes: #11574
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 6b68b27146852f057a3373cd04b08cd6917f3eea)
+
+commit 678b3e6082729698ce3575ba70313dd8399b1aed
+Author: Tim Serong <tserong@suse.com>
+Date:   Sat May 2 01:59:53 2015 +1000
+
+    json_spirit: use utf8 intenally when parsing \uHHHH
+    
+    When the python CLI is given non-ASCII characters, it converts them to
+    \uHHHH escapes in JSON.  json_spirit parses these internally into 16 bit
+    characters, which could only work if json_spirit were built to use
+    std::wstring, which it isn't; it's using std::string, so the high byte
+    ends up being zero'd, leaving the low byte which is effectively garbage.
+    
+    This hack^H^H^H^H change makes json_spirit convert to utf8 internally
+    instead, which can be stored just fine inside a std::string.
+    
+    Note that this implementation still assumes \uHHHH escapes are four hex
+    digits, so it'll only cope with characters in the Basic Multilingual
+    Plane.  Still, that's rather a lot more characters than it could cope
+    with before ;)
+    
+    (For characters outside the BMP, Python seems to generate escapes in the
+    form \uHHHHHHHH, i.e. 8 hex digits, which the current implementation
+    doesn't expect to see)
+    
+    Fixes: #7387
+    
+    Signed-off-by: Tim Serong <tserong@suse.com>
+    (cherry picked from commit 8add15b86e7aaef41397ab8fa9e77ee7957eb607)
+
+commit ef6641c0aa47c7f559aa56d7c35a5815afc2ba49
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Mar 23 21:26:23 2015 +0100
+
+    ceph.spec: update OpenSUSE BuildRequires
+    
+    OpenSUSE 13.2 needs libsnappy-devel but not bzip2-devel.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit bdac3dc3fb189688af52f60b0b34339dd8fea251)
+
+commit 0b6d4427ac7234fd509a9541adf45e303e40cb16 (refs/remotes/jashan/wip-hammer-11535-admin-socket, refs/remotes/gh/wip-hammer-11535-admin-socket)
+Author: Jon Bernard <jbernard@tuxion.com>
+Date:   Fri May 8 11:54:06 2015 -0400
+
+    common/admin_socket: close socket descriptor in destructor
+    
+    Long-running processes that do not reuse a single client connection will
+    see accumulating file descriptors as a result of not closing the
+    listening socket.  In this case, eventually the system will reach
+    file-max and subsequent connections will fail.
+    
+    Fixes: #11535
+    
+    Signed-off-by: Jon Bernard <jbernard@tuxion.com>
+    (cherry picked from commit 88fabb1ee6cd3c7d211ca46919055195e32245db)
diff --git a/doc/changelog/v0.94.4.txt b/doc/changelog/v0.94.4.txt
new file mode 100644
index 000000000..a99f8713e
--- /dev/null
+++ b/doc/changelog/v0.94.4.txt
@@ -0,0 +1,3576 @@
+commit 95292699291242794510b39ffde3f4df67898d3a (tag: refs/tags/v0.94.4, refs/remotes/gh/hammer)
+Author: Jenkins <jenkins@ceph.com>
+Date:   Mon Oct 19 07:43:50 2015 -0700
+
+    0.94.4
+
+commit 5764e233e56be08a59ffe6292f6fba9a76288aee
+Merge: 7f485ed b203979
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 15 16:35:10 2015 -0400
+
+    Merge pull request #6287 from jcsp/wip-test-fixes
+    
+    git path backports for hammer
+
+commit b2039797638057dd74a8a47c99091c2c892b042e
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Apr 16 08:29:41 2015 -0700
+
+    use git://git.ceph.com
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 34c467ba06345eacd2fd0e79c162dfd0b22f0486)
+
+commit 0f4ef19047c58aacfb359bdd9d104f059aa2f5b4
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 24 18:10:46 2015 -0400
+
+    qa: http://ceph.com/qa -> http://download.ceph.com/qa
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d4d65fbd6ad15320339751c604613ac63511e645)
+
+commit 7f485ed5aa620fe982561663bf64356b7e2c38f2
+Merge: 3dc1de2 294f016
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 13 00:28:52 2015 +0200
+
+    Merge pull request #6161 from dachary/wip-13227-hammer
+    
+    With root as default user, unable to have multiple RGW instances running
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 294f016ff84c7efef36ca94daef39fe2686df690
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu May 7 15:40:05 2015 -0700
+
+    init-radosgw.sysv: remove
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9d8c115b0ab05cfc1db6670d1d367737e61fe0ea)
+
+commit 3dc1de22552065bd930cc3e42a486e29639927ba
+Merge: aa74de9 7de65e7
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 12 08:16:17 2015 -0700
+
+    Merge pull request #6166 from dachary/wip-13046-hammer
+    
+    RGW : setting max number of buckets for user via ceph.conf option
+
+commit aa74de9d34801552d7d3e0aced07b6bc5bd129b9
+Merge: d3e49eb 77cb503
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 12 08:15:21 2015 -0700
+
+    Merge pull request #6039 from ceph/wip-13015-hammer
+    
+    rgw: don't preserve acls when copying object
+
+commit d3e49ebce00d749c36fe8dc46a5c432b1b907126
+Merge: 417e948 424fc1c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 12 08:14:47 2015 -0700
+
+    Merge pull request #6042 from dreamhost/wip-hammer-crossftp
+    
+    rgw: remove trailing :port from host for purposes of subdomain matching
+
+commit 417e948898d260a1ad6354244f9967ccbd75d768
+Merge: b5badcb 9ab9c44
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 12 08:14:09 2015 -0700
+
+    Merge pull request #6160 from dachary/wip-13228-hammer
+    
+    rgw: segments are read during HEAD on Swift DLO
+
+commit b5badcb47e565cd17145adca49abf07e167963e0
+Merge: e3f8df9 6119b15
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 12 08:13:25 2015 -0700
+
+    Merge pull request #6162 from dachary/wip-13226-hammer
+    
+    Keystone Fernet tokens break auth
+
+commit e3f8df9e12b434714b67036838f37b75ed1f9a25
+Merge: 8078233 ad83304
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 12 08:12:26 2015 -0700
+
+    Merge pull request #6163 from dachary/wip-13054-hammer
+    
+    rgw: region data still exist in region-map after region-map update
+
+commit 8078233ca1dda8eef3c59d6a92760365634cf41a
+Merge: dd64c78 4b0686f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 12 08:12:07 2015 -0700
+
+    Merge pull request #6164 from dachary/wip-13053-hammer
+    
+    rgw: GWWatcher::handle_error -> common/Mutex.cc: 95: FAILED assert(r == 0)
+
+commit dd64c7813bbdd2a6a4ffde6767c5d060ee1b9290
+Merge: 135c112 e80bd0a
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 12 08:11:16 2015 -0700
+
+    Merge pull request #5718 from dachary/wip-12859-hammer
+    
+    testGetContentType and testHead failed
+
+commit 135c112789d4925a735229394c0ec9e2b728d3ca
+Merge: fd17f3c 3b2affc
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 12 08:10:55 2015 -0700
+
+    Merge pull request #5860 from lebauce/wip-12960-hammer
+    
+    rgw: add delimiter to prefix only when path is specified
+
+commit fd17f3ccbf98832098bd1cd35fe2a5d567b51f4c
+Merge: 036c718 297c04d
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 12 08:08:59 2015 -0700
+
+    Merge pull request #6165 from dachary/wip-13052-hammer
+    
+    rgw: init_rados failed leads to repeated delete
+
+commit 036c7186f9304e4e48ecf50a4299cf74d55875fe
+Merge: 629b631 698d75c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Oct 9 09:17:19 2015 +0200
+
+    Merge pull request #6201 from dachary/wip-13410-hammer
+    
+    TEST_crush_rule_create_erasure consistently fails on i386 builder
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 629b631488f044150422371ac77dfc005f3de1bc
+Merge: 70e612a 0a5b856
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 8 12:18:49 2015 -0400
+
+    Merge pull request #5885 from Abhishekvrshny/wip-13034-hammer
+    
+    osd: copy-from doesn't preserve truncate_{seq,size}
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 698d75c11fa116d6546b09d14484d094fae8f6d7
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Jul 22 10:42:32 2015 +0200
+
+    tests: robust test for the pool create crushmap test
+    
+    The test that goes with f1e86be589803596e86acc964ac5c5c03b4038d8 to
+    verify that a bugous crush ruleset will prevent the creation of a pool
+    trying to use it was fragile. I depends on the implementation of the
+    erasure code lrc plugin and turns out to not work on i386.
+    
+    The test is modified to use a fake crushtool that always returns false
+    and validate that it prevents the creation of a pool, which demonstrate
+    it is used for crushmap validation prior to the pool creation.
+    
+    http://tracker.ceph.com/issues/12419 Fixes: #12419
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit e47903727b5d53f795014bf29b417d357f2bd65a)
+
+commit 70e612a395f7f5daf2fa289d8ae522824ef7a38f
+Merge: 0650862 2a28114
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 8 00:09:24 2015 +0200
+
+    Merge pull request #6192 from dachary/wip-13401-hammer
+    
+    mon: fix crush testing for new pools
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 2a28114053fbf4ceee09558790ad3cf755270d5f
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Aug 21 16:40:34 2015 -0400
+
+    crush/CrushTester: test fewer inputs when running crushtool
+    
+    If there are a lot of crush rules (say, 100) then the test can
+    take a long time.  100 values per rule should be enough to catch
+    most issues.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 0f82f461b33d93d868e185912a2c7e4074d06900)
+    
+    Conflicts:
+    	src/crush/CrushTester.cc
+    	  in hammer the crushtool validation is via a shell
+        	  and not via an internal subprocess utility
+
+commit abc5b5f139b05da0be0fbaa99282be16386980db
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun Sep 20 23:42:45 2015 +0200
+
+    tests: update to match crushmap validation message
+    
+    http://tracker.ceph.com/issues/13182 Fixes: #13182
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 21a1e75d8a7bad89a48cd9d36902c5d609be5015)
+    
+    Conflicts:
+    	qa/workunits/cephtool/test.sh
+    	  the test structure in hammer is different, adapt
+    	  the relevant test.
+
+commit 25bd2778f8b7f5eb5245efebca56a7348dd064eb
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 18 09:42:47 2015 -0400
+
+    mon/OSDMonitor: fix crush injection error message
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1b3090d50e5bd5ca3e6e396b23d2d9826896c718)
+    
+    Conflicts:
+    	src/mon/OSDMonitor.cc
+    	  hammer has a separate error message for EINTR
+    	  which is in the context of the fix for error message
+
+commit 6635530aa94b1f21c3052e64d4dfe3952f98fe83
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 18 09:41:25 2015 -0400
+
+    mon/OSDMonitor: only test crush ruleset for the newly created pool
+    
+    Otherwise, we test *all* crush rules.. which might be a lot, and
+    which is a big waste of time and effort.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 524b0bdcc45c2f4b95f2239c988e93250f337f3d)
+    
+    Conflicts:
+    	src/mon/OSDMonitor.cc : trivial resolution
+
+commit cc1fedda02fbf3c95d7a00f0b395db5439e96f90
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 18 09:40:13 2015 -0400
+
+    crush/CrushTester: allow testing by ruleset
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit aa238e5ed50f44a94caf84567267e4f6be8732a2)
+    
+    Conflicts:
+    	src/crush/CrushTester.cc
+              in hammer the crushtool validation is via a shell
+    	  and not via an internal subprocess utility
+    	src/tools/crushtool.cc
+    	  ceph_argparse_withint is preferred to ceph_argparse_witharg
+
+commit 065086246290cfa02f67c37c95a2ccdb05872f21
+Merge: 79385a8 3a50b90
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 6 11:11:14 2015 -0400
+
+    Merge pull request #5887 from Abhishekvrshny/wip-13044-hammer
+    
+    LibCephFS.GetPoolId failure
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 79385a85beea9bccd82c99b6bda653f0224c4fcd
+Merge: 5b25fc9 3228161
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Oct 4 15:33:49 2015 +0200
+
+    Merge pull request #6172 from dachary/wip-13354-hammer
+    
+    qa/workunits/cephtool/test.sh: don't assume crash_replay_interval=45
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 322816116ea14a8fd79616b1c55545ae7c62e7f6
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 28 13:13:40 2015 -0400
+
+    qa/workunits/cephtool/test.sh: don't assume crash_replay_interval=45
+    
+    e.g., it's 5 in teuthology's ceph.conf.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 80b7237e5e74d12b9d8b1f96ea535c96bdff9c6f)
+
+commit ad833044dee4208625213bc83c7847ce9d1a73aa
+Author: dwj192 <duanweijun@h3c.com>
+Date:   Wed Sep 9 09:48:16 2015 +0800
+
+    rgw:add --reset-regions for regionmap update
+    
+    Fixes: #12964
+    
+    Signed-off-by: Weijun Duan <duanweijun@h3c.com>
+    (cherry picked from commit 95685c19d6f1eab50b903e61273b5351bedc2980)
+    
+    Conflicts:
+    	src/rgw/rgw_admin.cc: usage is on cerr in hammer, not cout
+            src/test/cli/radosgw-admin/help.t: remove extra empty line
+
+commit 5b25fc9c8431423c6239da2eafcf7e8863d2262f
+Merge: e72bdc3 9026c4a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sat Oct 3 09:39:12 2015 +0200
+
+    Merge pull request #6156 from dachary/wip-13170-hammer
+    
+    doc: update docs to point to download.ceph.com
+    
+    Reviewed-by: Abhishek Varshney <abhishek.varshney@flipkart.com>
+
+commit 7de65e7ede52f3e417c354f180cfd80f1407c6cb
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Thu Sep 3 12:02:05 2015 +0530
+
+    rgw : setting max number of buckets for users via ceph.conf option
+    
+    This patch adds a new option "rgw_user_max_buckets" for setting
+    max number of buckets for users via ceph.conf.
+    
+    Fixes #12714
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit f65267c96cbd4cd25036b6bf399692e77bbb9436)
+    
+    Conflicts:
+    	src/rgw/rgw_admin.cc: usage is on cerr in hammer, not cout
+
+commit 297c04d856c4864d21b2241ad01560afa61dde40
+Author: Xiaowei Chen <cxwshawn@gmail.com>
+Date:   Tue Sep 8 06:58:57 2015 -0400
+
+    rgw: init_rados failed leads to repeated delete
+    
+    Fixes: #12978
+    
+    Signed-off-by: Xiaowei Chen <chen.xiaowei@h3c.com>
+    (cherry picked from commit ab4232baa7bcc86e90746e13312ac9bda1772872)
+
+commit 4b0686f18f323b95adb1fc5cffe7c1e55e8f1c63
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Aug 26 15:08:18 2015 -0700
+
+    rgw: delete finisher only after finalizing watches
+    
+    Fixes: #12208
+    
+    The watch error path might try to schedule a finisher work, delete finisher
+    only after watch destruction.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 97aed59fb9e865d30d31d2b7f4e93fc9727c96fa)
+
+commit 6119b152412ddfa2a614c1862aad52a123a4fe8e
+Author: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+Date:   Mon Aug 24 23:11:35 2015 +0530
+
+    rgw: be more flexible with iso8601 timestamps
+    
+    make parsing 8601 more flexible by not restricting the length of seconds
+    to 5, this allows timestamp to be specified both as ms or us. Newer
+    keystone backends such as fernet token backend default to microseconds
+    when publishing iso8601 timestamps, so this allows these timestamps to
+    be allowed when specifying the token expiry time.
+    
+    Fixes: #12761
+    Reported-by: Ian Unruh <ianunruh@gmail.com>
+    Signed-off-by: Abhishek Lekshmanan <abhishek.lekshmanan@ril.com>
+    (cherry picked from commit 136242b5612b8bbf260910b1678389361e86d22a)
+
+commit 607904e8d5616bd7df20e1e387a14d55e4ca43b6
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 22 14:12:02 2015 -0400
+
+    init-radosgw: specify pid file to start-stop-daemon
+    
+    This fixes restart when multiple instances are running.
+    
+    Fixes: #12407
+    Tested-by: Pavan Rallabhandi <pavan.rallabhandi@sandisk.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e17c8e1a7a06db0e2680dedc7e954021610320d1)
+
+commit f51ab26d8721bde6852e5c169869d7ae16bd0574
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Thu Jul 23 18:10:06 2015 +0530
+
+    rgw: fix radosgw start-up script.
+    
+    radosgw init script is unable to start radosgw daemon.
+    as it is relies on requiretty being disabled.
+    once init script start this daemon with sudo it fails
+    to start the daemon.
+    
+    changing 'sudo' to 'su', it will fix this issue and
+    will also help running radosgw daemon with our new
+    "ceph" UID project.
+    
+    Fixes: #10927
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit c83542296aeb8f5dd96f5d9e09327e7a89370742)
+
+commit 544a98fa9dd47d02778f622de5f7ebe1acccc12f
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu May 7 15:39:22 2015 -0700
+
+    init-radosgw: unify init-radosgw[.sysv]
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1c45f512a91ee49e9253ba6fd3857af9bd2ae57a)
+
+commit 2a733e9e720bcdb8ea342a96be679d6f9efa3b2b
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu May 7 15:29:11 2015 -0700
+
+    init-radosgw: look in /var/lib/ceph/radosgw
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a4bb992426b39188e750fce9198052ca0d510117)
+
+commit d00c52b874ddc74aa5f4c2e07b70b881ce963114
+Author: Nathan Cutler <ncutler@suse.cz>
+Date:   Sun May 10 18:17:53 2015 +0200
+
+    doc: rgw: fix typo in comments
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.cz>
+    (cherry picked from commit 05cf0db22dc3a9cb2bf4b86b8f240683c23f41ac)
+
+commit eb001d3a7275cc040bea716b8bdb0cf8b97efbc9
+Author: Dmitry Yatsushkevich <dyatsushkevich@mirantis.com>
+Date:   Tue Mar 17 15:44:52 2015 -0700
+
+    rgw: init script waits until the radosgw stops
+    
+    Fixes: #11140
+    Init script waits in stop action until the radowgw daemons stop.
+    
+    Signed-off-by: Dmitry Yatsushkevich <dyatsushkevich@mirantis.com>
+    (cherry picked from commit 1cca0c1e91a1e18f82f4d22855e96b2fc947f5ea)
+
+commit 9ab9c44583c96f1679db3dbcef322a706548a7dd
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 17 09:10:39 2015 -0700
+
+    rgw: don't read actual data on user manifest HEAD
+    
+    Fixes: #12780
+    We unconditionally read all the data, which is not needed
+    when doing HEAD operation on user manifest objects.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 4a8b08d1dd2511e28e999f1625103fb919fb5465)
+
+commit 9026c4a7c7699a9a013d4fe207e6681403f934a6
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Sep 18 14:20:12 2015 -0400
+
+    doc: remove mention of ceph-extra as a requirement
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a6f07e9fa3bd5e50a2d8cfa387b44a39d383fa65)
+
+commit 45ed24da879fe12b921cf03e4a921ccc0582b8d2
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Sep 18 14:13:21 2015 -0400
+
+    doc: remove ceph-extras
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit c0ef84fd22aac58b6ec8e58eb4f8dffed306d39e)
+
+commit faccdce79e003318c751dd9203773af7ed31d052
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Sep 18 14:13:02 2015 -0400
+
+    doc: correct links to download.ceph.com
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 387d7800359154950431d0984c756f43f21dd9b4)
+
+commit e9f4aecf3f406b42b74f840edb398be9243da85a
+Author: ritz303 <ritz_303@yahoo.com>
+Date:   Tue Aug 25 10:02:49 2015 -0500
+
+    doc: Added "Hammer" in the list of major releases.
+    
+    Fixes: #12771
+    
+    Signed-off-by: ritz303 <ritz_303@yahoo.com>
+    (cherry picked from commit 582f0f64455079290ad3b4ae7338b716e170911f)
+
+commit 424fc1ccb244f5084f94904cb20cbf88a1bd4638
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Sep 25 10:44:40 2015 -0700
+
+    rgw: set default value for env->get() call
+    
+    Fixes: #13239
+    
+    This fixes a regression introduced at commit abe4ec293d08b0314bf5c081ace2456073f3a22c.
+    The host var is a string, env->get() returns a char pointer, shouldn't
+    pass in NULL.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 0d5730efc0054334897315dc23ba04f30548e36e)
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit e72bdc3bf1541e445f74def70b5aa185f1757468
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 21 11:41:03 2015 -0400
+
+    osd/ReplicatedPG: tolerate promotion completion with stopped agent
+    
+    We may start a promotion, then get a pool update that disables the
+    agent, and then complete it.  We should not segfault in this case.
+    
+    Fixes: #13190
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit fd9ce66e8b76787f69c5e337332d57aa72c8b8d5)
+
+commit a3afb3f59435050efa711436134b4abe63a8f5cf
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jun 9 14:15:10 2015 -0400
+
+    rgw: remove trailing :port from host for purposes of subdomain matching
+    
+    Some clients (ahem, CrossFTP) include the :port in the HTTP_HOST header.
+    Strip it out.
+    
+    Switch req_info field to a std::string and avoid copying it in preprocess.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit abe4ec293d08b0314bf5c081ace2456073f3a22c)
+
+commit fe0fc4f18f6f275b34e8bbfa868a7310ecc3842f
+Merge: e26f8bc 8ba6b2f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 24 14:01:26 2015 -0700
+
+    Merge pull request #5715 from dachary/wip-12592-hammer
+    
+    RGW returns requested bucket name raw in Bucket response header
+
+commit e26f8bc2dbc82b6ff31d2bd2d5c890aa12da9f4a
+Merge: 19e549e 6acf36f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 24 14:00:55 2015 -0700
+
+    Merge pull request #5719 from dachary/wip-12855-hammer
+    
+    segmentation fault when rgw_gc_max_objs > HASH_PRIME
+
+commit 19e549e9c2fbacdbf747344c1dafcc6b66902339
+Merge: 2b11a4c a13c7fd
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 24 14:00:09 2015 -0700
+
+    Merge pull request #5720 from dachary/wip-12854-hammer
+    
+    the arguments 'domain' should not be assigned when return false
+
+commit 77cb503ba057f033825aa40d57dee42e838c67ac (refs/remotes/gh/wip-13015-hammer)
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Sep 9 15:41:02 2015 -0700
+
+    rgw: preserve all attrs if intra-zone copy
+    
+    Fixes: #13015
+    Intra zone copy requires that all objects' attributes are preserved.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit b9f2ed303eedfa0b747884f8e66fbe97cfeeb0d5
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Aug 13 10:59:32 2015 -0700
+
+    rgw: don't preserve acls when copying object
+    
+    Fixes: #12370
+    
+    When copying an object we need to use the acls we calculated earlier,
+    and not the source acls.
+    This was broken at e41d97c8e38bb60d7e09e9801c0179efe7af1734.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit fb020247449d3daf033885bab5b3a10c0e5df4a5)
+
+commit 2b11a4c07a9416a2f75c8ce62a923994977d799c
+Merge: 0022c03 b3822f1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Sep 15 10:23:49 2015 +0200
+
+    Merge pull request #5930 from liewegas/wip-11798-hammer
+    
+    upstart: limit respawn to 3 in 30 mins (instead of 5 in 30s)
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 0022c035476eff471a777b23a619d024f632a1bf
+Merge: 23fb811 0d6a8c6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Sep 14 23:57:13 2015 +0200
+
+    Merge pull request #5908 from liewegas/wip-simple-connect-hammer
+    
+    Pipe: Drop connect_seq increase line
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit b3822f113e07547194b844f647bcb7d45513b25f (refs/remotes/me/wip-11798-hammer)
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jun 3 14:57:34 2015 -0400
+
+    upstart: limit respawn to 3 in 30 mins (instead of 5 in 30s)
+    
+    It may take tens of seconds to restart each time, so 5 in 30s does not stop
+    the crash on startup respawn loop in many cases.  In particular, we'd like
+    to catch the case where the internal heartbeats fail.
+    
+    This should be enough for all but the most sluggish of OSDs and capture
+    many cases of failure shortly after startup.
+    
+    Fixes: #11798
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit eaff6cb24ef052c54dfa2131811758e335f19939)
+
+commit 0d6a8c631f55367ac4f6f792a10ba62cd97e9fed (refs/remotes/me/wip-simple-connect-hammer)
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Wed Feb 25 23:38:54 2015 +0800
+
+    Pipe: Drop connect_seq increase line
+    
+    Revert commit 0fc47e267b6f8dcd4511d887d5ad37d460374c25.
+    
+    When accepting and "connect.connect_seq == existing->connect_seq",
+    existing->state maybe STATE_OPEN, STATE_STANDBY or STANDY_CONNECTING.
+    This commit only fix partial problem and want to assert
+    "(existing->state == STATE_CONNECTING)".
+    
+    So later we added codes to catch
+    "(existing->state == STATE_OPEN || existing->state == STATE_STANDBY)"
+    before asserting.
+    
+    Backport: dumpling, firefly, giant
+    
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit 67225cb3ee1f6d274a02293724942bdb25cec8ca)
+
+commit 23fb811303971152f1c348e2a02de4e2bc6ed53d
+Merge: c1849ec 6c4ccc8
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Sep 13 14:10:23 2015 +0200
+
+    Merge pull request #5767 from dachary/wip-12597-hammer
+    
+    Crash during shutdown after writeback blocked by IO errors
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit c1849ecfb2ae55b99b6db75ee49af19886c246b2
+Merge: f35c53d f028389
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Sep 13 07:42:36 2015 -0400
+
+    Merge pull request #5687 from liewegas/wip-hammer-feature-hammer
+    
+    include/ceph_features: define HAMMER_0_94_4 feature
+
+commit f35c53df7c155d7ff4a316a0817b11bc018c57ca
+Merge: 6e22620 4be8a28
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Sep 12 09:23:13 2015 -0400
+
+    Merge pull request #5892 from ceph/wip-13060-hammer
+    
+    osd: allow peek_map_epoch to return an error
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 4be8a28d5b5ba868d100300ab64ebb82b188222b
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 11 11:00:38 2015 -0400
+
+    osd/PG: peek_map_epoch: skip legacy PGs if infos object is missing
+    
+    - pg is removed
+    - osd is stopped before pg is fully removed
+    - on restart, we ignore/skip the pg because its epoch is too old
+    - we upgrade to hammer and convert other pgs, skipping this one, and then
+      remove the legacy infos object
+    - hammer starts, tries to parse the legacy pg, and fails because the infos
+      object is gone, crashing.
+    
+    The fix is to continue ignoring the zombie pg.
+    
+    Fixes: #16030
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit f237ed97228839a1b412ad213945f0343df05bf5
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 11 17:41:01 2015 -0400
+
+    osd: allow peek_map_epoch to return an error
+    
+    Allow PG::peek_map_epoch to return an error indicating the PG
+    should be skipped.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f15d9585edc5a12ac2d076951076247253b897c2)
+    
+    [fixed *pepoch default of 0]
+    [fixed other return paths in peek_map_epoch]
+
+commit 3a50b904a2ffd613b695ead1b26c93278044d7df
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Aug 5 15:19:13 2015 +0800
+
+    crypto: fix unbalanced ceph::crypto::init/ceph::crypto:shutdown
+    
+    we may create a CephContext without calling common_init_finish(), then
+    delete the CephContext. In this case, ceph::crypto:init() is not called,
+    so CephContext::~CephContext() should not call ceph::crypto::shutdown().
+    
+    Fixes: #12598
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 98a85ec6464d8ec3fc7f0c647ac97c8cf002ebe2)
+
+commit 0a5b8569ecc6efa42b3d4fa7a58c155b3c2dbbe8
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jul 30 16:59:32 2015 -0700
+
+    ReplicatedPG,Objecter: copy_get should include truncate_seq and size
+    
+    Otherwise, we break CephFS over cache tiers.
+    
+    Fixes: #12551
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 6f9ee7961eee9ee3a61a07cbe0d8d289ee98fa9a)
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.cc
+
+commit 6e22620165872a88ab4b75ed2fb93f61beb65d12
+Merge: 508733f 00e73ad
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Sep 9 19:44:09 2015 -0400
+
+    Merge pull request #5769 from dachary/wip-12850-hammer
+    
+    Crash during TestInternal.MultipleResize
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 508733fd48f2c704b79ad36841346965f4d0429b
+Merge: 69a320e dc693fc
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Sep 9 19:44:01 2015 -0400
+
+    Merge pull request #5768 from dachary/wip-12849-hammer
+    
+    [  FAILED  ] TestLibRBD.BlockingAIO
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 82ea02ab8d886c16c511c4bd7d03d826158f954b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jun 17 11:35:18 2015 -0700
+
+    rgw: fix assignment of copy obj attributes
+    
+    Fixes: #11563
+    Clarify the confusing usage of set_copy_attrs() by switching the source and
+    destinatiion params (attrs, src_attrs). Switch to use attrs instead of
+    src_attrs afterwards. In one of the cases we originally used the wrong
+    variable.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit e41d97c8e38bb60d7e09e9801c0179efe7af1734)
+
+commit 3b2affce1a229cead3e0ffacfdfde69258a267cb
+Author: Sylvain Baubeau <sbaubeau@redhat.com>
+Date:   Fri Sep 4 22:51:44 2015 +0200
+
+    rgw: add delimiter to prefix only when path is specified
+    
+    http://tracker.ceph.com/issues/12960
+    Fixes: #12960
+    
+    Signed-off-by: Sylvain Baubeau <sbaubeau@redhat.com>
+    (cherry picked from commit 27cf257248ea55f8f4bc7851c3956611828bcae2)
+
+commit 69a320e185f20da0824d0a62d3de77578c431a7a
+Merge: 8039375 0bc909e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Sep 9 16:58:56 2015 +0200
+
+    Merge pull request #5697 from tchaikov/wip-12638-hammer
+    
+    mon: add a cache layer over MonitorDBStore
+    
+    Reviewed-by: Joao Eduardo Luis <joao@suse.de>
+
+commit 8039375327b5074a700193aae2d8b1922ca85398
+Merge: 19ff928 256620e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Sep 9 14:52:54 2015 +0200
+
+    Merge pull request #5381 from dachary/wip-12499-hammer
+    
+    ceph-fuse 0.94.2-1trusty segfaults / aborts
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 19ff92806fd1e0fb866737f58e379aa8078b8017
+Merge: 7614bf7 059bf98
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Sep 8 14:58:16 2015 -0700
+
+    Merge pull request #5757 from dachary/wip-12836-hammer
+    
+    WBThrottle::clear_object: signal on cond when we reduce throttle values
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 7614bf7d117fdd476931cdd8825265321cef3c4c
+Merge: 2100631 bf72785
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Sep 7 10:06:30 2015 +0200
+
+    Merge pull request #5759 from dachary/wip-12841-hammer
+    
+    recursive lock of md_config_t (0)
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 210063120708a7d6502c2b8d3e8c61731e37de5b
+Merge: a2e1fe2 836f763
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Sep 7 10:05:41 2015 +0200
+
+    Merge pull request #5761 from dachary/wip-12843-hammer
+    
+    long standing slow requests: connection->session->waiting_for_map->connection ref cycle
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit a2e1fe26f602d0165999dfb72b97985f9bfe38ce
+Merge: 8ccb771 2348a5b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Sep 7 10:04:51 2015 +0200
+
+    Merge pull request #5762 from dachary/wip-12844-hammer
+    
+    osd suicide timeout during peering - search for missing objects
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 8ccb7711d68dd3b95d2522c6e46f5021362c9088
+Merge: e9db807 77624af
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Sep 7 10:04:03 2015 +0200
+
+    Merge pull request #5763 from dachary/wip-12846-hammer
+    
+    osd/PGLog.cc: 732: FAILED assert(log.log.size() == log_keys_debug.size())
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit e9db8075034dc4ac0ffe5d8dd9dd2153509237fa
+Merge: 5ef999e aa00373
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Sep 7 10:03:10 2015 +0200
+
+    Merge pull request #5764 from dachary/wip-12847-hammer
+    
+    common: do not insert emtpy ptr when rebuild emtpy bufferlist
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 5ef999e4a1dc04d7b7e58b1a61f85511099fd6d4
+Merge: c243f7a cd11b88
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Sep 7 10:02:14 2015 +0200
+
+    Merge pull request #5373 from dachary/wip-12489-hammer
+    
+    pg_interval_t::check_new_interval - for ec pool, should not rely on min_size to determine if the PG was active at the interval
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit c243f7a33b0fff261bb92595a3cc7abf12b7e56d
+Merge: e61fdc6 7bddf5d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Sep 7 10:00:32 2015 +0200
+
+    Merge pull request #5383 from dachary/wip-12504-hammer
+    
+    rest-bench common/WorkQueue.cc: 54: FAILED assert(_threads.empty())
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit e61fdc67031b0358800335b638c96d22ecb85bd6
+Merge: 62f543b 9f69660
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Sep 7 09:54:07 2015 +0200
+
+    Merge pull request #5765 from dachary/wip-12883-hammer
+    
+    cache agent is idle although one object is left in the cache
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 62f543bbb092554746d3b89aab4b1ea75a5cdbf1
+Merge: f25315c e471c5d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Sep 7 09:53:14 2015 +0200
+
+    Merge pull request #5754 from dachary/wip-12588-hammer
+    
+    Change radosgw pools default crush ruleset
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit f25315c1e7e6e4700c2f39ba849dbcb25b0f1f89
+Merge: 021abe7 43a72e4
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Sep 7 09:51:50 2015 +0200
+
+    Merge pull request #5377 from dachary/wip-12396-hammer
+    
+    register_new_pgs() should check ruleno instead of its index
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 021abe742ccec876497468ac556b9dfaf6c9503d
+Merge: 4125196 c94fd92
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Sep 6 21:07:38 2015 -0400
+
+    Merge pull request #5758 from dachary/wip-12839-hammer
+    
+    Mutex Assert from PipeConnection::try_get_pipe
+
+commit 9f69660133f9534722589801a6fc827b74bead44
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Aug 13 19:41:47 2015 +0200
+
+    tests: tiering agent and proxy read
+    
+    Verify that an object promoted to a cache tier because of a proxy read
+    is evicted as expected.
+    
+    http://tracker.ceph.com/issues/12673 Refs: #12673
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 7924231930732bd297d3bd034c8295e96cb81088)
+
+commit 5656eec0000e95ea790c796a9f572c7261163cfa
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Aug 13 13:47:24 2015 +0200
+
+    osd: trigger the cache agent after a promotion
+    
+    When a proxy read happens, the object promotion is done in parallel. The
+    agent_choose_mode function must be called to reconsider the situation
+    to protect against the following scenario:
+    
+      * proxy read
+      * agent_choose_mode finds no object exists and the agent
+        goes idle
+      * object promotion happens
+      * the agent does not reconsider and eviction does not happen
+        although it should
+    
+    http://tracker.ceph.com/issues/12673 Fixes: #12673
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit e1f58feb9b1d20b72f2eb2eefdea5982e0cddccd)
+
+commit 4125196d31de8933656561b2f860e16a546ec4b9
+Merge: 47af509 f47ba4b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Sep 6 23:17:22 2015 +0200
+
+    Merge pull request #5276 from dachary/wip-11824-hammer
+    
+    implicit erasure code crush ruleset is not validated
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 47af509003e551238702858ccd845b930e5ef1ed
+Merge: 2fd8ec8 1e05578
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Sep 6 17:24:43 2015 +0200
+
+    Merge pull request #5382 from dachary/wip-12500-hammer
+    
+    segfault launching ceph-fuse with bad --name
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 2fd8ec8385a90daaa1bdb9de189ec73c7ba49c6c
+Merge: e6bdae9 fa19474
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Sep 6 17:23:19 2015 +0200
+
+    Merge pull request #5367 from dachary/wip-12311-hammer
+    
+    read on chunk-aligned xattr not handled
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit dc693fcaf2c596087b434f9e90fbc553394b9a14
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jul 29 12:46:24 2015 -0400
+
+    lockdep: allow lockdep to be dynamically enabled/disabled
+    
+    librbd test cases attempt to enable lockdep coverage via the librados
+    API.  Use a configuration observer to register/unregister lockdep
+    support.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit af0cade0293be22e74a3293765676c7b227b06c4)
+
+commit 805732b143fd28603238969ba784adb57eff1a12
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 28 14:23:14 2015 -0400
+
+    tests: librbd API test cannot use private md_config_t struct
+    
+    Remove all depencencies on md_config_t and instead use librados API
+    methods to get/set configuration values.
+    
+    Fixes: #12479
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2c51aadb447d3447b5dace19c5cb6268e94247c8)
+
+commit 7ac01731bcb0f1d46cf98ecf623c7e8e6cb1a38e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 9 15:06:27 2015 -0400
+
+    tests: ensure old-format RBD tests still work
+    
+    Override the RBD default image format back to version 1
+    to ensure tests properly cover the old format.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3b95edb5bd373bd05f83516eccfd30a6b5cf4693)
+
+commit b68d7574d12944589b246281474bcad01f5347be
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jul 30 09:00:57 2015 -0400
+
+    librados_test_stub: implement conf get/set API methods
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4d03c664f2a9362009c7e3077ed65e2989f0f21b)
+
+commit f0fa637e4a91a93a326ba303bc22101fadcc787d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 28 13:14:29 2015 -0400
+
+    crypto: use NSS_InitContext/NSS_ShutdownContex to avoid memory leak
+    
+    Switched to context-aware NSS init/shutdown functions to avoid conflicts
+    with parent application.  Use a reference counter to properly shutdown the
+    NSS crypto library when the last CephContext is destroyed.  This avoids
+    memory leaks with the NSS library from users of librados.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9fa0112dd9286178da1d76020158503b2062d252)
+
+commit 3f542aa2410139bece0f4192079e4e34260da782
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat Mar 21 07:13:51 2015 +0800
+
+    auth: use crypto_init_mutex to protect NSS_Shutdown()
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit cf058174fe6be26129eb52f4fb24d0f7146ef7af)
+
+commit e487e8e3d84c7845ce7824ace3e375c6c389ba20
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat Mar 21 01:02:42 2015 +0800
+
+    auth: reinitialize NSS modules after fork()
+    
+    Fixes: #11128
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 4c24d0cc074462ae258b5cf901cd884bb4f50a53)
+
+commit 00e73adb03ef4453599268779fe354e4e8513e54
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Aug 11 09:26:33 2015 -0400
+
+    librbd: prevent race condition between resize requests
+    
+    It was possible that the same resize request could be sent twice
+    if a completed resize op started a newly created resize op while
+    it was also being concurrently started by another thread.
+    
+    Fixes: #12664
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6ec431bac1cf9446f3af90209d193dfcf003f2c5)
+
+commit 6c4ccc854fa8a8403b03785b06cb35a7174f4f42
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Mon Mar 9 14:23:23 2015 +0800
+
+    librbd: Add a paramter:purge_on_error in ImageCtx::invalidate_cache().
+    
+    If bh_write met error, it will try again. For closing image, if met this
+    issue, it will trigger a assert:
+    >>2015-02-03 15:22:49.198292 7ff62d537800 -1 osdc/ObjectCacher.cc: In
+    function 'ObjectCacher::~ObjectCacher()' thread 7ff62d537800 time
+    >>2015-02-03 15:22:49.195927osdc/ObjectCacher.cc: 551: FAILED
+    >>assert(i->empty())
+    
+    Now add purge_on_error, when shutdown_cache it set true.
+    In ImageCtx::invalidate_cache, if met error and purge_on_error is true,
+    purge the dirty bh.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 35def5c81f7fc83d55d18320e4860c6a63d4c7f5)
+    
+    Conflicts:
+    	src/librbd/ImageCtx.cc : trivial resolution
+
+commit 05734916ab119c6d3879c2ce2bc9f9581907861a
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Thu Feb 5 11:28:50 2015 +0800
+
+    librbd: Remvoe unused func ImageCtx::read_from_cache.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 101440a41253680770f94bc380af7072c7adaebf)
+
+commit 28838f20a722b72a6e926e53f104342d3b9f4791
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Thu Feb 5 11:03:10 2015 +0800
+
+    osdc: clean up code in ObjectCacher::Object::map_write
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 9f80c2909ace09cd51c24b49c98a093e0e864dca)
+
+commit 5c4f152efa8e8f57c59ea7decc05ae1a34f2a9ee
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Thu Feb 5 10:13:06 2015 +0800
+
+    osdc: Don't pass mutex into ObjectCacher::_wait_for_write.
+    
+    Because the mutex is the same as ObjectCacher::lock.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit d7cf7aeea5cba1ffa8e51ff1ad2424b1ec161a12)
+
+commit 86e7698a19745c26b9d3e7a12a16c87ea9a5d565
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Mon Mar 2 11:23:44 2015 +0800
+
+    osdc: After write try merge bh.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 1a48a8a2b222e41236341cb1241f0885a1b0b9d8)
+
+commit c96541ad19fb142ed31ff3006f1dc25e0c1de86c
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Mon Mar 2 11:36:24 2015 +0800
+
+    osdc: Make last missing bh to wake up the reader.
+    
+    Avoid wakeup early and wait again.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit d582bda090b3339d03e25b0e6d0971ad0267f476)
+
+commit 4135b9a2d199583685a1bae3713347dcc1b872e6
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Fri Feb 6 09:53:36 2015 +0800
+
+    osdc: For trust_enoent is true, there is only one extent.
+    
+    Now the judgement only in conditon which will return -ENOENT.
+    But o->exists don't depend on the extent size. It only depend on
+    trust_enoent. So move this judgement at the first of _readx().
+    Make this bug ASAP occur.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 2449fddc13b5ce8bbf1bb4ecaa5d6937f54e54d1)
+
+commit 81376b6a1a5db9f8b45c58ea318ec924e932d990
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Tue Feb 3 14:46:39 2015 +0800
+
+    osdc: In _readx() only no error can tidy read result.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 540346d4a901d8041c9fd74641c98cdfd2e1ab32)
+
+commit e6bdae903cdd2f702829adc31fa9a31c239a3b87
+Merge: 317770d cdde626
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Fri Sep 4 15:38:43 2015 -0600
+
+    Merge pull request #5223 from SUSE/wip-12305-hammer
+    
+    ceph.spec.in: running fdupes unnecessarily
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit e80bd0a2a39afdabcb134100e0cc8fa74cbd14c2
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Thu Jun 25 18:28:50 2015 +0200
+
+    rgw: send Content-Length in response for GET on Swift account.
+    
+    Fixes: #12158
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 5f47b11791cd46ce2771046e4888575869cc9685)
+    Conflicts:
+    	src/rgw/rgw_rest_swift.cc
+
+commit 2e542456dee6c30b3200e9830d4ccae4b0325388
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Jun 22 13:51:49 2015 +0200
+
+    rgw: force content_type for swift bucket stats request
+    
+    Fixes: 12095
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit da259e4da1329e1e448d75c48c6ee3d1318e19b2)
+
+commit 5d57b63c32e43000ff95c2d8ce34347ed0579a8a
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Aug 13 15:02:03 2015 +0200
+
+    rgw: we should not overide Swift sent content type
+    
+    Fixes: #12363
+    backport: hammer
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 423cf136f15df3099c9266f55932542d303c2713)
+
+commit b8aafbc6f9f80f029606a577d73f8685a80225b8
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue Jun 23 11:54:09 2015 +0200
+
+    rgw: enforce Content-Type in Swift responses.
+    
+    Swift sends Content-Type HTTP header even if the response
+    doesn't contain body. We have this behaviour implemented
+    until applying some changes in end_header() function.
+    Unfortunately, lack of Content-Type causes early exits in
+    many Tempest's tests for Swift API verification.
+    
+    Fixes: #12157
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 106aeba206736d4080326f9bc191876bed63370b)
+
+commit 143cfc331efcc7bed8b1cf90d7e3188d0ac7604b
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Tue Apr 21 12:31:19 2015 +0200
+
+    rgw: force content-type header for swift account responses without body
+    
+    This regression was introduced in commit 4cfeca03ade21861ab70ca759f94eb244c16cb39
+    
+    Fixes: #11438
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 009664e560ee37ef6dc170a0155d6c749e151b06)
+
+commit b5420d6f1a6a1da34c7bba339e6763c4b369d349
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Apr 9 20:04:59 2015 +0200
+
+    rgw: shouldn't return content-type: application/xml if content length is 0
+    
+    Fixed: #11091
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 5cc92bb27cfbaee4af24dc312c9b98331f899794)
+
+commit 317770dd39510221cbc0fc7ee50960992137eaaf
+Merge: 6b02be7 6b36514
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Sep 3 12:20:38 2015 +0200
+
+    Merge pull request #5716 from dachary/wip-12851-hammer
+    
+    Ensure that swift keys don't include backslashes
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 6b02be7395ad8f41f70458e7bd11630ddd498cc5
+Merge: fa74ea1 f1c7c62
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Sep 3 12:11:24 2015 +0200
+
+    Merge pull request #5717 from dachary/wip-12591-hammer
+    
+    rgw: create a tool for orphaned objects cleanup
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit fa74ea10ac25fe93f359ccf0fd036d567e9610fe
+Merge: 011e4c5 a478385
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Sep 2 23:35:58 2015 +0200
+
+    Merge pull request #5755 from dachary/wip-12589-hammer
+    
+    ceph-disk zap should ensure block device
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 011e4c58ffb41aa5ae3240a40cbeba8266b89001
+Merge: e7c2269 b80859e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Sep 2 23:22:59 2015 +0200
+
+    Merge pull request #5732 from ceph/wip-11455-hammer
+    
+    rgw: init some manifest fields when handling explicit objs
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit e7c2269dbebd5337ae5aad127d6ddaa4260a8f53
+Merge: 92b641f da00bed
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Sep 2 23:19:02 2015 +0200
+
+    Merge pull request #5721 from dachary/wip-12853-hammer
+    
+    RGW Swift API: X-Trans-Id header is wrongly formatted
+    
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit 92b641f301df33a38ccfcf018c5f7657db8548ae
+Merge: 845d2ae 3ab5d82
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Sep 2 23:08:24 2015 +0200
+
+    Merge pull request #5498 from ceph/wip-12432-hammer
+    
+    rgw: HTTP return code is not being logged by CivetWeb
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 836f7636f76b2778812e4fa055206a887fd27dad
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Jul 21 11:31:12 2015 -0700
+
+    OSD: break connection->session->waiting message->connection cycle
+    
+    Fixes: #12338
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 7f15e22be291df139c85909b96b2366f6dbadf37)
+
+commit 77624aff03a8df7805ac0c1c6a7a2085f41fb6a3
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Aug 9 10:46:10 2015 -0400
+
+    osd/PGLog: dirty_to is inclusive
+    
+    There are only two callers of mark_dirty_to who do not pass max,
+    and they are both in the merge_log extending tail path.  In that
+    case, we want to include the last version specified in the log
+    writeout.  Fix the tail extending code to always specify the
+    last entry added, inclusive.
+    
+    Fixes: #12652
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f0ca14df0641daa04eee39d98d8bd0faf46e4e6d)
+
+commit aa003736d2aeea937ce0d8663b159cf06b2dda3e
+Author: Xinze Chi <xmdxcxz@gmail.com>
+Date:   Mon Aug 24 23:02:10 2015 +0800
+
+    common: fix code format
+    
+    Signed-off-by: Xinze Chi <xmdxcxz@gmail.com>
+    (cherry picked from commit 15e5ebe14787bbbc6b906d72ccd2d418d762cfad)
+
+commit aab35da90d7d50750fda6355d5a124a45d66f1c5
+Author: Xinze Chi <xmdxcxz@gmail.com>
+Date:   Mon Aug 24 22:59:40 2015 +0800
+
+    test: add test case for insert empty ptr when buffer rebuild
+    
+    Signed-off-by: Xinze Chi <xmdxcxz@gmail.com>
+    (cherry picked from commit 2d2f0eb338906742d516a2f8e6d5d05505be8f47)
+
+commit 2b0b7ae6443eeca271813415a91ab749c886fdf0
+Author: Xinze Chi <xmdxcxz@gmail.com>
+Date:   Mon Aug 24 23:01:27 2015 +0800
+
+    common: fix insert empty ptr when bufferlist rebuild
+    
+    Fixes: #12775
+    Signed-off-by: Xinze Chi <xmdxcxz@gmail.com>
+    (cherry picked from commit fb1b6ddd7bfe40d4a38d5ec59b1c3f59f2a83f23)
+    
+    Conflicts:
+    	src/common/buffer.cc : because invalidate_crc did not exist
+            in hammer
+
+commit 845d2aed9e9a9893678226b8661fbd8ff87c090f
+Merge: a6517d2 3396a96
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Sep 2 12:00:11 2015 +0200
+
+    Merge pull request #5527 from SUSE/wip-12585-hammer
+    
+    OSD crash creating/deleting pools
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 2348a5b5c96d065fbc2188bd9e8e7ff20c75ecaa
+Author: Guang G Yang <yguang@renownedground.corp.gq1.yahoo.com>
+Date:   Wed Jul 29 21:47:17 2015 +0000
+
+    osd: copy the RecoveryCtx::handle when creating a new RecoveryCtx instance from another one
+    
+    Fixes: 12523
+    
+    Signed-off-by: Guang Yang <yguang@yahoo-inc.com>
+    (cherry picked from commit 6ab9efe799b947f2ef6113df8cc542c73760107d)
+
+commit bf72785e1e11b18df9a03ebfa7c284fe4f43eaa0
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Aug 24 15:40:39 2015 -0700
+
+    config: skip lockdep for intentionally recursive md_config_t lock
+    
+    lockdep can't handle recursive locks, resulting in false positive
+    reports for certain set_val_or_die() calls, like via
+    md_config_t::parse_argv() passed "-m".
+    
+    Fixes: #12614
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 415226995ebe841cc9e8639a3233150a75aecaed)
+
+commit c94fd926239a31dea1389a760f55f332a9b6814a
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Jul 23 16:36:19 2015 -0700
+
+    osd: Keep a reference count on Connection while calling send_message()
+    
+    Fixes: #12437
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit a140085f467889f2743294a3c150f13b62fcdf51)
+
+commit 059bf98d40fa7e1eddb7aef2ecfdfb6bc96c14fb
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Jul 7 08:49:54 2015 -0700
+
+    WBThrottle::clear_object: signal if we cleared an object
+    
+    Fixes: #12223
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 8447b08d60d5cf7dce20f8b839cc2cb5a93782de)
+
+commit a478385334323071b5cfa4d546eef5b0ef43dce7
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sun May 31 19:42:45 2015 +0200
+
+    ceph-disk: always check zap is applied on a full device
+    
+    zap only makes sense on a full device and this is verified for
+    
+        ceph-disk prepare --zap-disk
+    
+    but not for
+    
+        ceph-disk zap
+    
+    Unify the two and add a test to check that it fails as expected when
+    trying to zap a directory.
+    
+    http://tracker.ceph.com/issues/11272 Fixes: #11272
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 8e1bcf21f090fc98ac9b495d76614e3012f5872a)
+    
+    Conflicts:
+    	src/test/ceph-disk.sh : trivial resolution
+
+commit e471c5dc16847ba35d3c3c8c74da1bf7c4c946e0
+Author: Yuan Zhou <yuan.zhou@intel.com>
+Date:   Wed Jul 8 10:35:49 2015 +0800
+
+    librados: Make librados pool_create respect default_crush_ruleset
+    
+    Make pool_create in librados use the osd_pool_default_crush_replicated_ruleset
+    
+    Fixes: #11640
+    
+    Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>
+    (cherry picked from commit da96a89033590277460aef1c80f385bd93d625e1)
+
+commit a6517d286b59eac283d7bfd04b620f5b859657b6
+Merge: a89c173 35fa47a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Sep 2 02:19:26 2015 +0200
+
+    Merge pull request #5551 from ceph/wip-corpus-hammer
+    
+    ceph-object-corpus: add 0.94.2-207-g88e7ee7 hammer objects
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 35fa47a117a3e8d185f7ebe02eea5671f8229427 (refs/remotes/me/wip-corpus-hammer, refs/remotes/gh/wip-corpus-hammer)
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 1 17:44:06 2015 -0400
+
+    ceph-object-corpus: add 0.94.2-207-g88e7ee7 hammer objects
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit b80859e2b2efc2402dc62aa319e40319863586f7 (refs/remotes/gh/wip-11455-hammer)
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Aug 26 14:34:30 2015 -0700
+
+    rgw: init some manifest fields when handling explicit objs
+    
+    Fixes: #11455
+    When dealing with old manifest that has explicit objs, we also
+    need to set the head size and head object correctly so that
+    code that relies on this info doesn't break.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d7bf8cb594e5276d1c80544f5ec954d52b159750)
+
+commit f47ba4b1a1029a55f8bc4ab393a7fa3712cd4e00
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Jul 16 18:02:02 2015 +0200
+
+    mon: test the crush ruleset when creating a pool
+    
+    We want to fix the following scenario:
+    
+     * an erasure code plugin (or another part of the code) creates a
+       ruleset
+     * the ruleset crashes during mapping (for whatever reason)
+     * ceph osd pool create uses the bugous ruleset
+     * the monitors try to do mapping a crash
+    
+    Having a bugous ruleset in the crush map is very difficult prevent. The
+    catastrophic event of using it with a newly created pool can however be
+    prevented by calling the CrushTester just before creating the pool and
+    after all implicit or explicit crush ruleset creation happened.
+    
+    http://tracker.ceph.com/issues/11814 Fixes: #11814
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit f1e86be589803596e86acc964ac5c5c03b4038d8)
+    
+    Conflicts:
+    	src/test/mon/osd-crush.sh
+              removed the run_mon because in hammer it is shared
+              between all tests
+            src/mon/OSDMonitor.cc
+              prepare_new_pool changed stringstream to *ostream
+
+commit b58cbbab4f74e352c3d4a61190cea2731057b3c9
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sat May 30 12:40:26 2015 +0200
+
+    erasure-code: set max_size to chunk_count() instead of 20 for shec
+    
+    See 8b64fe93b088a3a33d357869c47e6bf928c3f0e4 for a detailed explanation.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit a44499f7e6609d9a32254dc6f4303122de74db31)
+
+commit 6f0af185ad7cf9640557efb7f61a7ea521871b5b
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Feb 26 21:22:31 2015 +0200
+
+    vstart.sh: set PATH to include pwd
+    
+    ceph-mon needs crushtool to be in PATH. Don't set if it is run
+    from ceph_vstart_wrapper, which already sets it as it needs.
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit e08bccf377292d72463796ae1dff5d8be40d88e9)
+
+commit a89c1731e0e327acdb876d39df574e27379aaa1e
+Merge: c3a2bf3 0fde3a2
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Mon Aug 31 09:41:33 2015 +0100
+
+    Merge pull request #5365 from dachary/wip-12491-hammer
+    
+    buffer: critical bufferlist::zero bug
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit c3a2bf3f6b1d1bf416fa04df1629630abe2fe563
+Merge: 0a44eb1 4457d3e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 19:58:25 2015 +0200
+
+    Merge pull request #5369 from dachary/wip-12498-hammer
+    
+    get pools health'info have error
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 0a44eb12fc7d2ef902f1999e37abae0f11511d66
+Merge: dfcdba7 9916d37
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 19:52:52 2015 +0200
+
+    Merge pull request #5370 from dachary/wip-12496-hammer
+    
+    pgmonitor: wrong at/near target max“ reporting
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit dfcdba75f310eaa2cfc5ae97ba3093afe1198737
+Merge: 69a9a4c b8176d0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 19:44:56 2015 +0200
+
+    Merge pull request #5378 from dachary/wip-12394-hammer
+    
+    Memory leak in Mutex.cc, pthread_mutexattr_init without pthread_mutexattr_destroy
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 69a9a4cbf28c87fcd68f6c6c0c09f7a6d95ab003
+Merge: d3b160e 42bff0b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 19:44:26 2015 +0200
+
+    Merge pull request #5372 from dachary/wip-12493-hammer
+    
+    the output is wrong when runing ceph osd reweight
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit da00bedbc2f5d2facb9472078fd6cdd7260492ac
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Thu Aug 6 15:52:58 2015 +0200
+
+    rgw: rework X-Trans-Id header to be conform with Swift API.
+    
+    Fixes: #12108
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit d1735a4e8ea2056d333a5001a7a410fc03fe1b9e)
+
+commit 9937c818f1073bceffcf5811c7cc543b9043e646
+Author: Abhishek Dixit <dixitabhi@gmail.com>
+Date:   Mon Jun 8 22:59:54 2015 +0530
+
+    Transaction Id added in response
+    
+    Signed-off-by: Abhishek Dixit dixitabhi@gmail.com
+    
+    (cherry picked from commit b711e3124f8f73c17ebd19b38807a1b77f201e44)
+    
+    Conflicts:
+    	src/rgw/rgw_rest.cc
+                 the prototype of the end_header( function
+                 in the context of the diff changed
+
+commit f1c7c629f2a7baf87a3d25d68b7a40fb3b2f6c49
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Jun 29 15:35:04 2015 -0700
+
+    rgw: api adjustment following a rebase
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 7cf1f376e7dbc98a44f4a8e78bfa11f84291a941)
+
+commit 85911df5226dccfd2f1806a204a23eae3966eee7
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Jun 29 15:34:44 2015 -0700
+
+    rgw: orphans, fix check on number of shards
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 47edec359be2a4d230a4a54c0c670d0aa711fa1e)
+
+commit c1cf7dfb672032012889840fb4462b1a076f34c0
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Jun 29 15:34:11 2015 -0700
+
+    rgw: orphans, change default number of shards
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5528f21d0861d42378a8eb10759a1e55ddd66c09)
+
+commit bb1d4cc3a3e9af3bf8cd8fd80085fbbbaeb8b89f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 5 14:43:05 2015 -0700
+
+    rgw: change error output related to orphans
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit cac57caf639236adc9a641a85352eb729b58725a)
+
+commit 2e0f6fe98787e1f3ea7d107a814caadce7ee5f23
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon May 4 17:02:29 2015 -0700
+
+    rgw: orphan, fix truncated detection
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d8ce04db8542dae0027423f3c50355e5de74dc9a)
+
+commit 1bfebefba2e09448d28853b34415d3f65e323c76
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon May 4 16:32:57 2015 -0700
+
+    radosgw-admin: simplify orphan command
+    
+    No need for --init-search
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 4980cbd04ff650d79a033bc5298e6c1abe985133)
+
+commit f244b159ac8793c11a5707ea0b104b3180efb58f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon May 4 15:24:00 2015 -0700
+
+    radosgw-admin: stat orphan objects before reporting leakage
+    
+    We don't want to report new objects as leaked, because they might just
+    be written, and waiting to be linked to a new object. The number of
+    seconds prior to the job initialization can be cofigured through
+    --orphan-stale-secs (by default it's 24h).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 66b0090023859d97da2bbaca698b6a50c225abd0)
+
+commit f80e2b2bc8879d1cc40338816196f51378aaa785
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon May 4 14:39:20 2015 -0700
+
+    radosgw-admin: orphans finish command
+    
+    A command to remove orphans data
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 55d6f5ecf18f532c7f056f8b60c101843594b00c)
+
+commit 88d32c641a06823df93f0e3fae640a880778e1e4
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat May 2 17:28:30 2015 -0700
+
+    rgw: cannot re-init an orphan scan job
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f8993102731518153bb18eb74c3e6e4943a5dbac)
+
+commit 80a40342b6a0d9e7a47ca4b672901315246385df
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat May 2 16:38:08 2015 -0700
+
+    rgw: stat_async() sets the object locator appropriately
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d7d117121858155acc8548908e9459a48c676dd1)
+
+commit 0082036348e0463e42520504466961eb2bd77602
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat May 2 16:34:09 2015 -0700
+
+    rgw: list_objects() sets namespace appropriately
+    
+    list_objects() now uses parse_raw_oid(), so that it can set the correct
+    namespace. It only affects users of the function that want to get all
+    objects in bucket, regardless to the namespace associated with it. This
+    makes it so that the orphan code actually works now with namespaced
+    objects, and with special named objects (namely, start with underscore).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 20bd490bebf9be3553d4e25322c9544b0b090086)
+
+commit 1c3707291e83771ca1d5cd24ce445f723006b744
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 1 17:23:44 2015 -0700
+
+    rgw: modify orphan search fingerprints
+    
+    Now works with multipart uploads.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit fa61ea2ccc94f04afb39868ac34abc264806d8d4)
+
+commit ef81367a1eaeb7778dc60a668b9fe2f6a77e06ff
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 1 15:17:10 2015 -0700
+
+    rgw: compare oids and dump leaked objects
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 1bc63d98ff8a3aa180065153690b4f8a73658b79)
+
+commit f4d0544e63f68fa94b21fda71c0b78edd05038a1
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Apr 30 16:17:54 2015 -0700
+
+    rgw: keep accurate state for linked objects orphan scan
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f19b2f087cfb5a89edf3b19487c0df3a214e350f)
+
+commit 748ea57cfeca7b295afca5b8126d454eb85550c7
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Apr 29 17:12:34 2015 -0700
+
+    rgw: iterate over linked objects, store them
+    
+    only keep part of the oid name if it is in a namespace.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 75902fdb12ce0413d3beb455df47134f3d7386e7)
+
+commit 6c6aa5ddf38102012e61f79f5234b9647d42b461
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Apr 29 17:12:00 2015 -0700
+
+    rgw: add rgw_obj::parse_raw_oid()
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 7c1aa8373a090d5c1b2aa121582547723120a079)
+
+commit 62d562d76e0456cc3c735c6708531c7deb2874da
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Apr 29 14:50:15 2015 -0700
+
+    rgw: iterate asynchronously over linked objects
+    
+    Read objects manifest. So that we could keep the relevant info later.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 05a953d965a0a6a0e9feddaa62f7429e154e828e)
+
+commit 00ecf2d0a895407bc8ec0e55e1b1b0f47a1aa96f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Apr 29 14:15:33 2015 -0700
+
+    rgw: async object stat functionality
+    
+    An async functionality that stats object.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 13adf3cf4f5fd8a0feb7cc1d7a4ee4ded9f573db)
+
+commit 7d1cc48ba32e8bc732c43ecb539312c6bfcc0f80
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Apr 28 16:45:49 2015 -0700
+
+    rgw-admin: build index of bucket indexes
+    
+    for the orphan search tool
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 16a2dbd838b69fb445c06122a68fc65ec3a7b3de)
+
+commit c1b0e7a985cc9e742dcef4c8c427c57a2f58fa43
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat Apr 25 09:37:53 2015 -0700
+
+    rgw: initial work of orphan detection tool implementation
+    
+    So far doesn't do much, iterate through all objects in a specific pool
+    data, store it in a sharded index.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 767fc29aa62a0a4e4dd8a78785576e8f1b44ee5b)
+
+commit b16129c3393d6e301b4af5203ffa1c6d8dfdda38
+Author: Pavan Rallabhandi <pavan.rallabhandi@sandisk.com>
+Date:   Wed Apr 29 13:35:29 2015 +0530
+
+    Avoid an extra read on the atomic variable
+    
+    Signed-off-by: Pavan Rallabhandi <pavan.rallabhandi@sandisk.com>
+    (cherry picked from commit 7476e944af049aefdb8871cce62883b86f6b6f8d)
+
+commit 1f6916d5a9b0ebe5181b801c4c381a699883bb09
+Author: Pavan Rallabhandi <pavan.rallabhandi@sandisk.com>
+Date:   Wed Apr 8 18:53:14 2015 +0530
+
+    RGW: Make RADOS handles in RGW to be a configurable option
+    
+    Having a configurable number of RADOS handles is resulting
+    in significant performance boost for all types of workloads.
+    Each RGW worker thread would now get to pick a RADOS handle
+    for its lifetime, from the available bunch.
+    
+    Signed-off-by: Pavan Rallabhandi <pavan.rallabhandi@sandisk.com>
+    (cherry picked from commit b3c3a24fe9c0fbb2bdbbccb08b40efc1dca207cc)
+
+commit d3b160ecbe181c2b969a14825cf314a056630188
+Merge: 7357659 695f782
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 15:50:03 2015 +0200
+
+    Merge pull request #5374 from dachary/wip-12487-hammer
+    
+    ceph osd crush reweight-subtree does not reweight parent node
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 7357659b59d9bf3522bf664f1a3da7475a7c8a49
+Merge: 8f559d5 8a2ad05
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 15:49:41 2015 +0200
+
+    Merge pull request #5380 from dachary/wip-12390-hammer
+    
+    PGLog::proc_replica_log: correctly handle case where entries between olog.head and log.tail were split out
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit a13c7fd7c39e83072f2975477e1cdcfe079fb041
+Author: Ruifeng Yang <149233652@qq.com>
+Date:   Sat Aug 1 17:39:17 2015 +0800
+
+    rgw:the arguments 'domain' should not be assigned when return false
+    
+        Hostnames: [B.A]
+        Inputs: [X.BB.A]
+        Return: [false]
+        Output: [B.A] it is wrong.
+    
+    Fixes: #12629
+    Signed-off-by: Ruifeng Yang <149233652@qq.com>
+    (cherry picked from commit 9420d24c21b4e06d62f356637599c6ec8e7e3e4d)
+
+commit 6acf36f8851b8a369949f0fad9d0156af7dde20f
+Author: Ruifeng Yang <149233652@qq.com>
+Date:   Sun Aug 2 16:16:05 2015 +0800
+
+    rgw:segmentation fault when rgw_gc_max_objs > HASH_PRIME
+    
+    obj_names[] is index out of bounds when rgw_gc_max_objs > HASH_PRIME
+    
+    Fixes: #12630
+    Signed-off-by: Ruifeng Yang <149233652@qq.com>
+    (cherry picked from commit e3147b8ddcaafafde9ce2029f7af5062e22076d3)
+
+commit 6b365144862cbd387d8b931939a3dc2cfe17bbbd
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Jul 20 20:27:33 2015 -0700
+
+    rgw: avoid using slashes for generated secret keys
+    
+    Just use plain alphanumeric characterset.
+    
+    Fixes: #7647
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 3f04a6126fdbfb93304f798da3775c0eec9b7d44)
+
+commit 8f559d58b42b91f070bfa590d9c33e9a156e0066
+Merge: d94e764 931ffe3
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Aug 30 08:36:37 2015 -0400
+
+    Merge pull request #5366 from dachary/wip-12303-hammer
+    
+    arm: all programs that link to librados2 hang forever on startup
+
+commit 8ba6b2f628a0f80964afbf4a83b61c4eee9c78ce
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jul 30 14:47:15 2015 -0700
+
+    rgw: url encode exposed bucket
+    
+    Fixes: #12537
+    Don't send the bucket name back without url encoding it.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit ad5507fe0bf72ed5bdf8353e315cc9092c740144)
+
+commit d94e7642d0b0213a4b650a5acffa050d042087eb
+Merge: 11a80d8 6417e8e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:41:01 2015 +0200
+
+    Merge pull request #5202 from SUSE/wip-12293-hammer
+    
+    ceph.spec.in: rpm: cephfs_java not fully conditionalized
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 11a80d8bec6ff704e794a2995d653cd5d5c1c979
+Merge: a792ae6 3728477
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:40:48 2015 +0200
+
+    Merge pull request #5203 from SUSE/wip-11997-hammer
+    
+     ceph.spec.in: rpm: not possible to turn off Java
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit a792ae65e2e03e1ca8804380cdbbfa1aec7a2123
+Merge: f7e76d2 8f78001
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:40:30 2015 +0200
+
+    Merge pull request #5204 from SUSE/wip-11629-hammer
+    
+    ceph.spec.in: SUSE/openSUSE builds need libbz2-devel
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f7e76d2c9e681b3c5775f74c582830f546359703
+Merge: d1feb04 4eb58ad
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:40:09 2015 +0200
+
+    Merge pull request #5207 from SUSE/wip-12267-hammer
+    
+    ceph.spec.in: 50-rbd.rules conditional is wrong
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit d1feb0415b3b3dc9c5b003d8528079ccccd0729f
+Merge: 2dca2ec 8b576bd
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:39:57 2015 +0200
+
+    Merge pull request #5216 from SUSE/wip-12269-hammer
+    
+    ceph.spec.in: ceph-common needs python-argparse on older distros, but doesn't require it
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 2dca2ece0478972e89ed6611a15892ad9e449cc8
+Merge: 64fafc9 0818e9f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:39:31 2015 +0200
+
+    Merge pull request #5264 from SUSE/wip-12361-hammer
+    
+    ceph.spec.in: snappy-devel for all supported distros
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 64fafc95785024421cbb7ccae41ed4276eeb8fe9
+Merge: 4675971 add0f1e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:39:14 2015 +0200
+
+    Merge pull request #5368 from dachary/wip-12331-hammer
+    
+    ceph: cli throws exception on unrecognized errno
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 4675971ce3331f1285398d1556a594f671e489c0
+Merge: 53cdcf2 e004941
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:39:00 2015 +0200
+
+    Merge pull request #5371 from dachary/wip-12494-hammer
+    
+    ceph tell: broken error message / misleading hinting
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 53cdcf20482823cdb8682dde89706d6f21d8d0d2
+Merge: 362874c eccf369
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:38:43 2015 +0200
+
+    Merge pull request #5385 from dachary/wip-12501-hammer
+    
+    error in ext_mime_map_init() when /etc/mime.types is missing
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 362874ca39e32c3bc5ee13b13f905fd7693d9449
+Merge: 2044f3c 8804b3f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:38:29 2015 +0200
+
+    Merge pull request #5411 from SUSE/wip-12446-hammer
+    
+    ceph.spec.in: radosgw requires apache for SUSE only -- makes no sense
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 2044f3cd7b220cc642245e7ca93c21a17cc8c536
+Merge: 260e24b a785193
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:38:13 2015 +0200
+
+    Merge pull request #5412 from SUSE/wip-12448-hammer
+    
+    ceph.spec.in: useless %py_requires breaks SLE11-SP3 build
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 260e24bba0e83dfa61d25ac0489f656244211db4
+Merge: dc944fb b575ecc
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:30:18 2015 +0200
+
+    Merge pull request #5318 from ceph/wip-12236-hammer
+    
+    Possible crash while concurrently writing and shrinking an image
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit dc944fbbedcee56dfbd9d4f5ac3c38d15a6621ea
+Merge: a8c1b4e 92272dd
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:30:01 2015 +0200
+
+    Merge pull request #5319 from ceph/wip-12235-hammer
+    
+    librbd: deadlock during cooperative exclusive lock transition
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit a8c1b4e7436832ed3040d5a90df713ef6029cb69
+Merge: a39aeee e971820
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 13:29:26 2015 +0200
+
+    Merge pull request #5296 from ceph/wip-12345-hammer
+    
+    librbd: correct issues discovered via lockdep / helgrind
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit a39aeeedf07518d65c9944476b98a0788585f5c1
+Merge: 6a949e1 153744d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 12:14:22 2015 +0200
+
+    Merge pull request #5559 from ceph/wip-12682-hammer
+    
+    object_map_update fails with -EINVAL return code
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 6a949e10198a1787f2008b6c537b7060d191d236
+Merge: 1e423af 4faa8e0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 12:11:10 2015 +0200
+
+    Merge pull request #5468 from ceph/wip-12577-hammer
+    
+    osd: fix repair when recorded digest is wrong
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 1e423afd3c7e534e302dbea0798a47f82acec141
+Merge: 574932b 0ca93db
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 12:06:59 2015 +0200
+
+    Merge pull request #5376 from dachary/wip-12433-hammer
+    
+    Show osd as NONE in ceph osd map <pool> <object>  output
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 574932b2f438787f1a330a271644d9787eecbb63
+Merge: ed162d4 d08db7a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 30 11:54:55 2015 +0200
+
+    Merge pull request #5359 from ceph/wip-11470.hammer
+    
+    mon: PaxosService: call post_refresh() instead of post_paxos_update()
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit ed162d498051ac237be33ac6a6764ae0c61d57a6
+Merge: 41a245a bee8666
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri Aug 28 14:43:31 2015 -0400
+
+    Merge pull request #5691 from jdurgin/wip-10399-hammer
+    
+    is_new_interval() fixes
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 0bc909e54b0edd8a32a3e7db16877f1d875c3186
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Aug 10 04:25:03 2015 -0700
+
+    mon: add a cache layer over MonitorDBStore
+    
+    the cache of of leveldb does not perform well under some condition,
+    so we need a cache in our own stack.
+    
+    * add an option "mon_osd_cache_size" to control the size of cache size
+      of MonitorDBStore.
+    
+    Fixes: #12638
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5e99a578050976ca22b549812ac80d494fe7041d)
+    
+    Conflicts:
+    	src/mon/OSDMonitor.h
+    minor differences in OSDMonitor ctor
+
+commit bee86660377cfaa74f7ed668dd02492f25553ff9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Aug 21 11:32:39 2015 -0400
+
+    Objecter: pg_interval_t::is_new_interval needs pgid from previous pool
+    
+    When increasing the pg_num of a pool, an assert would fail since the
+    calculated pgid seed would be for the pool's new pg_num value instead
+    of the previous pg_num value.
+    
+    Fixes: #10399
+    Backport: infernalis, hammer, firefly
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f20f7a23e913d09cc7fc22fb3df07f9938ddc144)
+    
+    Conflicts: (hobject_t sort order not backported, trivial resolution)
+    	src/osdc/Objecter.cc
+    	src/osdc/Objecter.h
+
+commit b5418b9a7138b45b5fe4f24cfb679c2abb30ab71
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue May 26 12:35:11 2015 -0700
+
+    osd_types::is_new_interval: size change triggers new interval
+    
+    Fixes: 11771
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit ff79959c037a7145f7104b06d9e6a64492fdb95f)
+
+commit f0283890d22e8d5c2210d565077ad3af1921f098 (refs/remotes/gh/liewegas-wip-hammer-feature-hammer)
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 27 14:09:50 2015 -0400
+
+    include/ceph_features: define HAMMER_0_94_4 feature
+    
+    This is to constrain upgrades past hammer to version that include
+    the appropriate compatibility fixes (e.g., hobject_t encoding).
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2868b49c0e39fdc7ae72af81962370c4f95a859e)
+    
+    # Conflicts:
+    #	src/include/ceph_features.h
+
+commit 41a245aff77dde8a2ab212f9c91a73a23a77c40d
+Merge: 95cefea 65b380f
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 27 13:07:29 2015 -0400
+
+    Merge remote-tracking branch 'gh/wip-12536-hammer' into hammer
+    
+    This includes wip-osd-compat-hammer.
+
+commit 65b380f7f28a36bb51a6771a2fd9b657ea2848ba (refs/remotes/gh/wip-12536-hammer)
+Merge: 00d802d 9b91adc
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 19 09:52:02 2015 -0400
+
+    Merge remote-tracking branch 'gh/wip-osd-compat-hammer' into wip-12536-hammer
+
+commit 153744d7c596705c4f92bee5e827846b46c80141
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 10 09:39:50 2015 -0400
+
+    tests: increase test coverage for partial encodes/decodes
+    
+    Multiple combinations of offsets/lengths are now tested when
+    performing partial encodes/decodes of the bit vector.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3e145f714ac9b2d599b45a058c6b93595e38f424)
+
+commit fca78765366f3ac365cfa98224aca5fb79b2a7fe
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 10 09:34:42 2015 -0400
+
+    common: bit_vector extent calculation incorrect for last page
+    
+    It's highly probable that the last page in the bit vector will not
+    be a full page size.  As a result, the computed extents will extend
+    beyond the data portion of the bit vector, resulting in a end_of_buffer
+    exception.
+    
+    Fixes: #12611
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c6d98992691683524d3d96def83a90a6f5fe7f93)
+
+commit 3396a969741f74e5b1f55d8d7af258493ca26478
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jul 30 14:20:56 2015 +0100
+
+    osd/OSDMap: handle incrementals that modify+del pool
+    
+    Because new_pools was processed after old_pools, if something
+    was modified and then deleted in the same incremental
+    map, then the resulting state would be a pool in the map
+    that had no entry in pool_name.
+    
+    Fixes: #12429
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 8d4932e721e32a33c1bada54e11d4ecccd868618)
+
+commit 3ab5d82a1c09142267c27156b69a3117d30c5b7d (refs/remotes/gh/wip-12432-hammer)
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jul 22 10:01:00 2015 -0700
+
+    rgw: set http status in civetweb
+    
+    Need to set the http status in civetweb so that we report it correctly.
+    Fixes: #12432
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit b8e28ab9f914bf48c9ba4f0def9a0deb9dbb93bc)
+
+commit 10a03839853246c535e639b6dc6cea45b8673642
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jul 31 11:03:29 2015 -0700
+
+    civetweb: update submodule to support setting of http status
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 00d802d97ba5e1a8bde2414f8710494f15807fcd
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 23 11:03:17 2014 -0800
+
+    hobject_t: fix get_boundary to work with new sorting regime
+    
+    The hash is no longer the most significant field; set everything that is
+    more significant, too.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 45281efad1ae6b2b5ddd5549c56dbdd46671e5c5)
+
+commit 9b91adc560c8dffc1f5f7470d28c25f448b2b619 (refs/remotes/gh/wip-osd-compat-hammer)
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 4 13:21:50 2015 -0400
+
+    mon: disallow post-hammer OSDs if there are up pre-hammer OSDs
+    
+    Force *all* OSDs to upgrade to hammer before allowing post-hammer
+    OSDs to join.  This prevents any pre-hammer OSDs from running at
+    the same time as a post-hammer OSD.
+    
+    This commit, as well as the definition of the sentinal post-hammer
+    feature, should get backported to hammer stable series.
+    
+    Backport: hammer
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e38d60e94503c7e8511a60a8dceceb3cd39c2bd8)
+    
+    # Conflicts:
+    #	src/mon/OSDMonitor.cc
+
+commit 8a559c1e80bb7c70e13c46f1900896c9b8fcd9a9
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 4 13:16:11 2015 -0400
+
+    include/ceph_features: define MON_METADATA feature
+    
+    This was added in master in 40307748b9b8f84c98354a7141da645a3c556ba9.  We
+    define it but notably do not support it!
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 4faa8e04069417a6e2a6c4ea5c10b54d76aa00c4 (refs/remotes/gh/wip-12577-hammer)
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Mar 27 10:17:12 2015 -0700
+
+    osd: include newlines in scrub errors
+    
+    We may log more than one of these in a scrubmap; make sure they are
+    not concatenated.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2b3dd1bb7ab360e8f467a47b7ce3a0e4d3a979b2)
+
+commit 455eb2a89197e9609c5bb510112d661d388b4c85
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Mar 27 10:04:00 2015 -0700
+
+    osd: fix condition for loggin scrub errors
+    
+    We may have an error to log even though the authoritative map is empty:
+    when we have a digest that doesn't match all replicas.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d23766be902080ffb29528acb5a00d70beb97a9f)
+
+commit 67e7946ce8e28f00db9588bed670ef5141268f41
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Mar 27 10:03:22 2015 -0700
+
+    osd: fix fallback logic; move into be_select_auth_object
+    
+    The fallback behavior was printing a promising message but then doing
+    a 'continue' and not actually scrubbing the object.  Instead, fall back to
+    a less-bad auth inside be_select_auth_object.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b597db59fedc9a96dbe4b37f03b819dcf8fdb1bb)
+
+commit 0f57c7017a98f7e416a160375b8b80355b8e0fc3
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Mar 27 09:34:44 2015 -0700
+
+    osd: log a scrub error when we can't pick an auth object
+    
+    If we really can't find a suitable auth, log an error.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a476d8f3903ca78b5cf36d63059f13cb5268efe5)
+
+commit d4f4c5c4334add6acfc66407c2383d01094d384d
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Mar 27 10:20:23 2015 -0700
+
+    osd: repair record digest if all replicas match but do not match
+    
+    If the recorded digest does not match the replicas, and all replicas
+    match, then (on repair) fix the recorded digest.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f581fecd8a16b9250fa96beec5aa26074f401028)
+
+commit acfed6b5eae764d381c88ed7d3f4942c61191127
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 26 13:52:20 2015 -0700
+
+    osd: move recorded vs on disk digest warning into be_compare_scrubmaps
+    
+    This is a better place for it.  While we are here, change the wording to
+    clearly call out the 'recorded' digest (from object_info_t) vs the 'on
+    disk' digest (what we observed during scrub).
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit cf349ff35e9111ca52db1b0aad3753d0680b486d)
+
+commit 674029bc6c519e82011d1429ab5eddff9720c2dc
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 26 13:46:22 2015 -0700
+
+    osd: be slightly paranoid about value of okseed
+    
+    The scrubber.seed value is set based on the peer feature; check for that
+    too explicitly, and assert the scrubber.seed value matches.
+    
+    No change in behavior here.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e34d31b7a9a71132a17b95cf5ad60255ece60243)
+
+commit f2002b7fa2af2db52037da4c1cf66d405a4941e7
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 26 13:44:25 2015 -0700
+
+    osd: be precise about "known" vs "best guess"
+    
+    We cannot assume that the auth info has the "known" digest; all replicas
+    may have a matching digest that does not match the oi, or we may choose
+    a different auth info for some other reason.  Verify that the digest
+    matches the oi before calling it "known".
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 546d1c7b8224d96d0cc93dc3c96f52787cd69413)
+
+commit 4e5d146ad36ef0380eb72e45dca651494658aeeb
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 26 13:24:11 2015 -0700
+
+    osd: record digest if object is clean (vs entire scrub chunk)
+    
+    If an individual object is clean, record it in the missing_digest map.
+    If not, leave it out.  This lets us drop the old condition that only
+    recorded any digests if the entire scrub chunk was clean.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c39e0e5f126028f552d410ab6924cac6e1f714fb)
+
+commit 1357ed1bd0e250b942bcba0346c97c24bb79a5d1
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Aug 3 13:05:45 2015 -0400
+
+    hobject_t: decode future hobject_t::get_min() properly
+    
+    The post-hammer wip-temp branch changed hobject_t::get_min() so that pool
+    is INT64_MIN instead of -1 and neglected to deal with the encoding compat
+    with older versions.  Compensate on hammer by mapping INT64_MIN to -1
+    locally.  See commit ff99af38df830e215359bfb8837bf310a2023a4d.
+    
+    Note that this means upgrades from hammer to post-hammer *must* include
+    this fix prior to the upgrade.  This will need to be well-documented in the
+    release notes.
+    
+    Master gets a similar fix so that they map our min value to the new
+    INT64_MIN one on decode.
+    
+    Fixes: #12536 (for hammer)
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 6d01d6b700300841f49f288a89d514f26a4f94b1
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Jul 29 11:49:57 2015 -0700
+
+    OSDMonitor::preprocess_get_osdmap: send the last map as well
+    
+    Fixes: #12410
+    Backport: hammer
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 1320e29dfaee9995409a6d99b9ccaa748dc67b90)
+
+commit a7851933033473d11e69fa1f237c0ad8bbbd4a13
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jul 9 11:51:13 2015 +0200
+
+    ceph.spec.in: drop SUSE-specific %py_requires macro
+    
+    %py_requires expands to
+    
+    BuildRequires: /usr/bin/python
+    PreReq: python = 2.7
+    
+    The BuildRequires: is already provided, and the PreReq is wrong because
+    e.g. SLE11-SP3 (a platform we are trying to support) has Python 2.6.
+    
+    http://tracker.ceph.com/issues/12351 Fixes: #12351
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit cbf6c7b46b42e163950bf7956e5b7565fca961c9)
+
+commit 8804b3f1ab5180c8a85b25b9b46b986fe7022868
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jul 16 15:16:22 2015 +0200
+
+    ceph.spec.in: remove SUSE-specific apache2-mod_fcgid dependency
+    
+    This package is no longer required for RGW to work in SUSE.
+    
+    http://tracker.ceph.com/issues/12358 Fixes: #12358
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit f3f8d3660d0ece3139e5f3dc7881ccceadcf648d)
+    
+    Conflicts:
+    	ceph.spec.in
+                In master, the "%if %else %endif" has been changed
+                into two "%if %endif"s
+
+commit b575ecca9743ab08439bfe0b5283e483e2a799ce
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 23 14:20:16 2015 -0400
+
+    tests: verify that image shrink properly handles flush op
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8dfcc72071d63c7867a5a59abe871cf5db673a0a)
+
+commit d4eb7bd63a5e46b790bad7001d2873c5c238cc90
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 23 14:18:20 2015 -0400
+
+    librbd: invalidate cache outside cache callback context
+    
+    When shrinking an image, it's possible that the op flush callback
+    will be from within the cache callback context.  This would result
+    in a deadlock when attempting to re-lock the cache lock in order to
+    invalidate the cache.
+    
+    Fixes: #11743
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 726d699b7790c7e371279281ab32cd3aeb8ece7b)
+
+commit 92272dd676ebb67a3095ce3933cac24809b054da
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 15 11:18:29 2015 -0400
+
+    librbd: don't cancel request lock early
+    
+    It's possible that a stale notice is received and will
+    be discarded after the request lock has been canceled.
+    This will stale the client.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d9dd5c5890029107df40ed438f23fb9865d7de29)
+
+commit 58ae92f1a716e1ffac6408793e19d47f99d63520
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 14 16:13:38 2015 -0400
+
+    tests: new test for transitioning exclusive lock
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f97ce46ea4c9c7cdc6e37e3759871d5b41b25769)
+
+commit 7b21ccba7c2885027b1d345baaad08a978fbc72f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 7 13:31:50 2015 -0400
+
+    tests: verify that librbd will periodically resend lock request
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d2a1c226ab3f7e202f62896f0c80c4cf3607cdaf)
+
+commit c95b37f48c129ef6780f67b326e97957f3771472
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 15 10:49:36 2015 -0400
+
+    common: Mutex shouldn't register w/ lockdep if disabled
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 879b8a73e2452332b26b8f3428ff5e3e0af8ddad)
+
+commit 117205af89398457e197793505381622705488b2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 15 10:47:04 2015 -0400
+
+    librbd: improve debugging output for ImageWatcher
+    
+    Include the instance pointer so that different images
+    can be differentiated in the logs.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b951a7398e3e749cb64a3f6a604212a5627e82a0)
+
+commit 08ae01231754d3010c1ede762579cf75c1c7460d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 15 10:45:04 2015 -0400
+
+    librados_test_stub: watcher id should be the instance id (gid)
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3e1e561cc29043eb64dff8afb3a2c1ed77788e93)
+
+commit 704c0e09479d435ecbb84693a786c60694732b19
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 7 12:51:49 2015 -0400
+
+    librbd: retry lock requests periodically until acquired
+    
+    If the exclusive lock owner acks the lock release request but crashes
+    before it actually releases the lock, the requestor will wait forever.
+    Therefore, after a certain timeout, retry the request again until it
+    succeeds.
+    
+    Fixes: #11537
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 37c74e6e5274208d1b6efaf315afec03ea7eaa82)
+
+commit dbaaed9cf99121d0c97c22a695a7a6e872a11f48
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 7 12:35:36 2015 -0400
+
+    librbd: don't hold owner_lock for write during flush
+    
+    The various IO callback codepaths will attempt to take
+    the lock, which will result in deadlock since the flush
+    cannot complete.
+    
+    Backport: hammer
+    Fixes: #11537
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    
+    (cherry picked from commit 2b6d0633d5c89de3a557cdd72621dfc19f0540de)
+
+commit e9718201c6ed785c61077797a74ffda7e0438fb9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 22 15:30:02 2015 -0400
+
+    lockdep: do not automatically collect all backtraces
+    
+    It is expensive to collect backtraces every time a lock is
+    checked in order to provide cycle backtraces.  The backtraces
+    can be forced on for specific locks or globally via the new
+    config option "lockdep_force_backtrace".
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7354d25f56c5e004f288febdee2b6961c069163b)
+
+commit 27f7042e923366c31748c4cc9992c1a8cb37b457
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 9 13:20:54 2015 -0400
+
+    librbd: flush operations need to acquire owner lock
+    
+    Cache writeback operations will expect the owner lock to be held.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit adfa2e0070ccca7b6556d3bfc5fac7ce4d43a4d0)
+
+commit 5b39983025f510c3119ebb375870e7669be7ddd0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 21 00:13:31 2015 -0400
+
+    librbd: avoid infinite loop if copyup fails
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 43e0e3cd63f0067217ed0811d73f6c546f3027be)
+
+commit 88b583b249a74a71bc87409015f7a8d85200e1bf
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 12 10:19:48 2015 -0400
+
+    librbd: flush pending ops while not holding lock
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3d5cef38c37e5dda6b23751ad560851f1304d86d)
+
+commit a88b180571cb481a31cb94c249a3b486220232fa
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 12 10:07:21 2015 -0400
+
+    tests: fix possible deadlock in librbd ImageWatcher tests
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 742a85d667b26b3490d96270b5c500b08f2a5283)
+
+commit 321eb8d1c3f7d553addb780928e201acf5091132
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 11 13:59:49 2015 -0400
+
+    tests: enable lockdep for librbd unit tests
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 45cb9cb5980e90235b8244d0d61ece40bde4784a)
+
+commit bfe5b90921a3faaad0533388cb150cea20f47ae9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 7 14:17:37 2015 -0400
+
+    librbd: owner_lock should be held during flush request
+    
+    Flush might result in the cache writing out dirty objects, which
+    would require that the owner_lock be held.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c9142fe35372cf69b7a56f334622a775a6b7c43f)
+
+commit 1e84fb05f48c146cc9d2d090be8c2d355326938b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 7 14:06:16 2015 -0400
+
+    osdc: ObjectCacher flusher might needs additional locks
+    
+    librbd requires the ObjectCacher flusher thread to acquire
+    an additional lock in order to maintain lock ordering
+    constraints.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a38f9e5104a6e08e130dc4f15ad19a06d9e63719)
+
+commit 506a45a906024d4bb5d3d4d6cc6cbb9eec39c5f2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 16:11:03 2015 -0400
+
+    librbd: fix recursive locking issues
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1b57cc1da7a51e6f8ffea689b94ef843732c20a4)
+
+commit acf5125fe27fb3b9de8b97c4e44fa1f5c61147fd
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 16:04:28 2015 -0400
+
+    librbd: simplify state machine handling of exclusive lock
+    
+    It is expected that all IO is flushed and all async ops are cancelled
+    prior to releasing the exclusive lock.  Therefore, replace handling of
+    lost exclusive locks in state machines with an assertion.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d6b733dbdd0aeb5d1e136dcbf30c58c80952651e)
+
+commit 9454f04e62ece39fdcdbb4eb5a83945f76bcc0a5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 15:32:38 2015 -0400
+
+    librbd: ObjectMap::aio_update can acquire snap_lock out-of-order
+    
+    Detected during an fsx run where a refresh and CoR were occurring
+    concurrently.  The refresh held the snap_lock and was waiting on
+    the object_map_lock, while the CoR held object_map_lock and was
+    waiting for snap_lock.
+    
+    Fixes: #11577
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8cbd92b1fe835b1eb3a898976f9507f51cc115b2)
+
+commit 3e0358e17e7ddb488109bce04bb08cd16fd5bb2f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 16 14:15:10 2015 -0400
+
+    librbd: move copyup class method call to CopyupRequest
+    
+    Move AbstractWrite's invocation of copyup to the CopyupRequest
+    class.  The AioRequest write path will now always create a
+    CopyupRequest, which will now append the actual write ops to the
+    copyup.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7be3df67809925164237cc185f9f29e145f45768)
+
+commit 2ee64a80794297d4fc5aeafbb185818f7f6a77b8
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Mar 31 12:28:11 2015 -0400
+
+    librbd: simplify AioRequest constructor parameters
+    
+    Moved all parent overlap computation to within AioRequest so that
+    callers don't need to independently compute the overlap.  Also
+    removed the need to pass the snap_id for write operations since
+    it can only be CEPH_NOSNAP.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 46515971edab8888284b1a8307ffca88a9c75f5c)
+
+commit 3e71a7554da05aee06d5a2227808c321a3e3f0f1
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Sat Mar 14 01:16:31 2015 +0100
+
+    librbd/AioRequest.h: fix UNINIT_CTOR
+    
+    Fix for:
+    
+    CID 1274319: Uninitialized scalar field (UNINIT_CTOR)
+     uninit_member: Non-static class member m_object_state is not
+     initialized in this constructor nor in any functions that it calls.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 48f18ea0e1c4c6de1921ea2359448deb761461e7)
+
+commit cb57fe581df7c14d174e6eaff382d6a28ab658d6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jul 17 15:04:10 2015 -0400
+
+    librbd: add object state accessor to ObjectMap
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 9249ab7dacd43e7a546656b99013a595381fd5fd
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 15:41:59 2015 -0400
+
+    librbd: AsyncObjectThrottle should always hold owner_lock
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c352bcdc0f63eea55677fe3c3b5f0c61347c0db3)
+
+commit 26902b94f97edb189ae620c86a8dda8166df471a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 15:34:43 2015 -0400
+
+    librbd: execute flush completion outside of cache_lock
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5f157f20980de7e9a05fb933fb57efdc759da78b)
+
+commit 571220d6f4642dd3cd78988882645fdf647c150e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 15:17:54 2015 -0400
+
+    librbd: add AsyncRequest task enqueue helper method
+    
+    In order to support the invariant that all state machine
+    callbacks occur without holding locks, transitions that
+    don't always involve a librados call should queue their
+    callback.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 218bc2d0f8d90c9c64408cb22e26680e88138844)
+
+commit 8e280f4cfef0d2fdc706d43bbee0c377d288a457
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 7 15:32:27 2015 -0400
+
+    librbd: disable lockdep on AioCompletion
+    
+    It is only used by clients and it causes a large slowdown
+    in performance due to the rate at which the lock is constructed/
+    destructed for each IO request.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 66e74641baeed9aadb7e474d6e6b142b7725722d)
+
+commit b38da48059792e9182fe2877786d9159007683d3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 13:42:19 2015 -0400
+
+    librbd: AioCompletion shouldn't hold its lock during callback
+    
+    The callback routine most likely will attempt to retrieve the result
+    code, which will result in a recursive lock attempt.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3ad19ae349ebb27ff6027e40883735240fa97a3c)
+
+commit 6fdd3f1ce69fa2e00c6f1bedd5f72352953e1e44
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 13:51:03 2015 -0400
+
+    librbd: give locks unique names to prevent false lockdep failures
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c474ee42b87975c04eeb2b40b976deb5a5e2d518)
+
+commit 70041499f99cad551b3aab16f086f725b33b25e2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 13:40:16 2015 -0400
+
+    librbd: complete cache read in a new thread context
+    
+    The ObjectCacher complete the read callback while still holding
+    the cache lock.  This introduces lock ordering issues which are
+    resolved by queuing the completion to execute in a clean (unlocked)
+    context.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0024677dc8adfd610418ca47599dd95d3a5a6612)
+
+commit 65ef695eba0f5b5d87347ffb3407bb5d6d75b402
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Mar 19 15:35:59 2015 -0400
+
+    librbd: require callers to ObjectMap::aio_update to acquire lock
+    
+    This is needed to allow an atomic compare and update operation
+    from the rebuild object map utility.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2db758cb4cb6d88cbaf9842b1e5d0872a185d8df)
+
+commit 58b8fafb87b18f2a4d8c38f77ecf81ff58452409
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 13:38:29 2015 -0400
+
+    log: fix helgrind warnings regarding possible data race
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c1e14451775049bbd95bcb19a0b62ab5e2c0a7bb)
+
+commit a5203d32544d40cb51eb60fc906cc3ceaacbfe96
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 13:37:56 2015 -0400
+
+    librados_test_stub: fix helgrind warnings
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b65ae4bc26f2dcaaa9518d5cce0e8b83ea310de8)
+
+commit b73e87eb307137786e0f0d89362ef0e92e2670b8
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 13:36:26 2015 -0400
+
+    librados_test_stub: add support for flushing watches
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6e400b9049ede5870e40e4dd2cb41874550eac25)
+
+commit 2fa35b1c5ca8e33959fff8c84eaa4feca0f67df3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 13:29:12 2015 -0400
+
+    common: lockdep now support unregistering once destructed
+    
+    librbd use of an image hierarchy resulted in lock names being
+    re-used and incorrectly analyzed.  librbd now uses unique lock
+    names per instance, but to prevent an unbounded growth of
+    tracked locks, we now remove lock tracking once a lock is
+    destructed.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7c7df2ce9f837628535d21df61ae4f13d809c4fa)
+
+commit 7b85c7bfe599644eb29e6b1e03733da4774c2eac
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 30 13:26:41 2015 -0400
+
+    common: add valgrind.h convenience wrapper
+    
+    Conditionally support helgrind annotations if valgrind support is
+    enabled during the build.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 650ad32d74a24d248dd8e19bff1bbd31f0bb224b)
+
+commit 6d3db5ffbef71c8927a9ffc68c5955dca0f5612c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 17:24:08 2015 -0400
+
+    librbd: add work queue for op completions
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 21f990efe6c6d710376d5b1a49fc8cd53aa020e6)
+
+commit 64425e861400fde4703fb06529777c55650b6676
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 8 16:46:34 2015 -0400
+
+    WorkQueue: ContextWQ can now accept a return code
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e5ffae578e83c1e4fca3f328c937e7c9be8cc03c)
+
+commit eccf36980511de7ed122a38d426170496ffdea64
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Jun 23 13:41:53 2015 -0600
+
+    packaging: RGW depends on /etc/mime.types
+    
+    If the mimecap RPM or mime-support DEB is not installed, then the
+    /etc/mime.types file is not present on the system. RGW attempts to read
+    this file during startup, and if the file is not present, RGW logs an
+    error:
+    
+      ext_mime_map_init(): failed to open file=/etc/mime.types ret=-2
+    
+    Make the radosgw package depend on the mailcap/mime-support packages so
+    that /etc/mime.types is always available on RGW systems.
+    
+    http://tracker.ceph.com/issues/11864 Fixes: #11864
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit c74a2f86a1f9b15aae0e723b410bedfc1d01a79b)
+
+commit 7bddf5d4dad09b1f6d3b4e80e4bdd895191560b0
+Author: huangjun <hjwsm1989@gmail.com>
+Date:   Thu Jul 16 09:42:55 2015 +0800
+
+      rest_bench: bucketname is not mandatory as we have a default name
+    
+      Signed-off-by: huangjun <hjwsm1989@gmail.com>
+    
+    (cherry picked from commit bc56a87ea3b69b4856a2b9f9fb2a63dbfb1211c3)
+
+commit 6e7358b3958b7ee11ef612302b300d07f3982d54
+Author: huangjun <hjwsm1989@gmail.com>
+Date:   Thu Jul 16 09:17:59 2015 +0800
+
+      rest_bench: drain the work queue to fix a crash
+      Fixes: #3896
+      Signed-off-by: huangjun <hjwsm1989@gmail.com>
+    
+    (cherry picked from commit f3d34d8ff921dbd2ff21f6b72af7c73bb9c6940e)
+
+commit 1e055782f5ade94c19c2614be0d61d797490701e
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Jul 21 16:09:32 2015 +0100
+
+    auth: check return value of keyring->get_secret
+    
+    get_secret can fail to populate the passed CryptoKey, for
+    example if the entity name is not found in the keyring.  In
+    this case, attempts to use the CryptoKey will lead to
+    segfaults.
+    
+    Fixes: #12417
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 64e5041008744362fdbb16e16bc3e049a2d426aa)
+
+commit 256620e37fd94ee4b3af338ea6955be55529d0d6
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Thu Jul 16 04:45:05 2015 -0700
+
+    Client: check dir is still complete after dropping locks in _readdir_cache_cb
+    
+    We drop the lock when invoking the callback, which means the directory
+    we're looking at might get dentries trimmed out of memory. Make sure that
+    hasn't happened after we get the lock back. If it *has* happened, fall back
+    to requesting the directory contents from the MDS. Update the dirp location
+    pointers after each entry to facilitate this.
+    Because this requires we update the dirp->at_cache_name value on every loop,
+    we rework the updating scheme a bit: to dereference the dn->name before
+    unlocking, so we know it's filled in; and since we update it on every loop
+    we don't need to refer to the previous dentry explicitly like we did before.
+    
+    This should also handle racing file deletes: we get back a trace on
+    the removed dentry and that will clear the COMPLETE|ORDERED flags.
+    
+    Fixes #12297
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 62dd63761701a7e0f7ce39f4071dcabc19bb1cf4)
+
+commit 8a2ad0540994efe530a1775ebd949aeaf6609e57
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Apr 9 16:29:45 2015 -0700
+
+    TestPGLog: fix invalid proc_replica_log test caes
+    
+    If 1'3 is non-divergent, no osd could correctly have created 2'3.  Also,
+    proc_replica_log does not add the extra entries from the auth log to the
+    missing set, that happens later on in activate.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 1333b676dba4ed4b5db854fe935bdadbf9f21d08)
+
+commit df71e6be7ccbe18d99200b2a5844a9f19ac70141
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Apr 9 16:28:23 2015 -0700
+
+    TestPGLog: fix noop log proc_replica_log test case
+    
+    Need to fill in log.head, olog.head.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 1e5b22053381580708af29a1536f1e106c9b479c)
+
+commit 549ff9a938c5126d4dcb15535845d5354cb44bb2
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Apr 9 15:01:41 2015 -0700
+
+    TestPGLog: add test for 11358
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit b61e5aef15d7448d24b713d13dce77697bc0af74)
+
+commit c224fc70e14bd76494bc437595c8b0b0d275694e
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Apr 9 15:01:29 2015 -0700
+
+    PGLog::proc_replica_log: handle split out overlapping entries
+    
+    See the comment, if the overlapping entries between olog.head
+    and log.tail were split out, last_update should be log.tail.
+    
+    Fixes: 11358
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 6561e0d955c4563a8505b717de41f5bd89119f63)
+
+commit b8176d0f21ce80d2e5a92057ab85f7618504cd9f
+Author: Ketor Meng <d.ketor@gmail.com>
+Date:   Tue May 26 18:50:17 2015 +0800
+
+    Mutex: fix leak of pthread_mutexattr
+    
+    Need pthread_mutexattr_destroy after pthread_mutexattr_init
+    
+    Fixes: #111762
+    Signed-off-by: Ketor Meng <d.ketor@gmail.com>
+    (cherry picked from commit 2b23327b3aa8d96341d501a5555195ca1bc0de8f)
+
+commit 43a72e47c4adae674e02bb262645d88ac528e2be
+Author: Xinze Chi <xmdxcxz@gmail.com>
+Date:   Fri Jul 3 18:27:13 2015 +0800
+
+    mon/PGMonitor: bug fix pg monitor get crush rule
+    
+    when some rules have been deleted before, the index in array of crush->rules
+    is not always equals to crush_ruleset of pool.
+    
+    Fixes: #12210
+    Reported-by: Ning Yao <zay11022@gmail.com>
+    Signed-off-by: Xinze Chi <xmdxcxz@gmail.com>
+    (cherry picked from commit 498793393c81c0a8e37911237969fba495a3a183)
+
+commit 0ca93db7d79f11e51af75e322941419b860bb2f8
+Author: Shylesh Kumar <shmohan@redhat.com>
+Date:   Wed Jul 22 18:47:20 2015 +0530
+
+    mon: ceph osd map shows NONE when an osd is missing
+    
+    Signed-off-by: Shylesh Kumar <shmohan@redhat.com>
+    (cherry picked from commit 4d030302ea3709eb021710e873f1ebe62f951cec)
+
+commit 695f782e59baffbe8429f9daa47a4f1476a02976
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Jun 7 20:05:58 2015 -0400
+
+    crush/CrushWrapper: fix adjust_subtree_weight debug
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 26be86f649a1e9846280204fe1e3f8344eb975ce)
+
+commit 0bd4c81ce41a22b22ca76e539de87260b140bbc4
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jun 5 12:15:45 2015 -0700
+
+    crush/CrushWrapper: return changed from adjust_subtree_weight
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 551ab2d69dd50a078d62340740d357d85ab8dcbb)
+
+commit 05fc59bee9138d04227255a2f31bd82efe94b1e9
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jun 5 12:15:22 2015 -0700
+
+    crush/CrushWrapper: adjust subtree base in adjust_subtree_weight
+    
+    We were correctly adjusting all the children, but the subtree root was
+    not being updated.
+    
+    Fixes: #11855
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1da96ab8442522bda4fc39e05597f17b8ebddf89)
+
+commit d2f31adc49a7c6b8276dbf8773575e91ba5e18fa
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jun 5 12:12:41 2015 -0700
+
+    unittest_crush_wrapper: test adjust_subtree_weight
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b461dc2e970f139dcb25d93b32be6b4bb9f1c8eb)
+
+commit 0ccdf342e7cf0072d3c38c6e1a6cf3a87f154a28
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jun 5 12:17:26 2015 -0700
+
+    unittest_crush_wrapper: attach buckets to root in adjust_item_weight test
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a8351eb3de2b4069ffec9c4f561eb1519ea11aa2)
+
+commit 1e737532cfe1daf43bea3d9bd4e55faed560f0f2
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jun 5 12:12:32 2015 -0700
+
+    unittest_crush_wrapper: parse env
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 14c60b288b33e4cec9c9819d4668a5ff9855ccc1)
+
+commit cd11b887c6c586085a7014ba44123b115370a462
+Author: Guang G Yang <yguang@renownedground.corp.gq1.yahoo.com>
+Date:   Thu Jul 2 05:29:47 2015 +0000
+
+    osd: pg_interval_t::check_new_interval should not rely on pool.min_size to determine if the PG was active
+    
+    If the pool's min_size is set improperly, during peering, pg_interval_t::check_new_interval
+    might wrongly determine the PG's state and cause the PG to stuck at down+peering forever
+    
+    Fixes: #12162
+    Signed-off-by: Guang Yang yguang@yahoo-inc.com
+    (cherry picked from commit 684927442d81ea08f95878a8af69d08d3a14d973)
+    
+    Conflicts:
+    	src/osd/PG.cc
+               because PG::start_peering_interval has an assert
+               that is not found in hammer in the context
+    	src/test/osd/types.cc
+               because include/stringify.h is not included by
+               types.cc in hammer
+
+commit c5f0e2222cc55a92ebc46d12b618fadb3d2fb577
+Author: Guang G Yang <yguang@renownedground.corp.gq1.yahoo.com>
+Date:   Wed Jul 1 20:26:54 2015 +0000
+
+    osd: Move IsRecoverablePredicate/IsReadablePredicate to osd_types.h
+    
+    Signed-off-by: Guang Yang <yguang@yahoo-inc.com>
+    (cherry picked from commit 466b083166231ec7e4c069fef8c9e07d38accab9)
+
+commit 42bff0b9e2f80e32001082a608d63719332677e3
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Tue Jul 21 11:20:53 2015 +0100
+
+    mon: OSDMonitor: fix hex output on 'osd reweight'
+    
+    Fixes: #12251
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit 787fa80c2746fde44ac0583ff7995ec5be9a672d)
+
+commit e00494118ee42322784988fe56623cbc7aac4cc9
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Jul 15 19:03:33 2015 +0800
+
+    ceph.in: print more detailed warning for 'ceph <type> tell'
+    
+    put the full signature of "tell <target> <command> [options...]"
+    instead of "tell {0}.<id>", which could be misleading somehow.
+    
+    Fixes: 11101
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit aa1a5222eebd0256d4ecffb0eefeb0f78ecf0a1f)
+
+commit f18900f2eb724acd898e8b32fe75c0850d7cf94c
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Jul 15 18:01:52 2015 +0800
+
+    ceph.in: print more detailed error message for 'tell' command
+    
+    * we do not allow user specify a certain daemon when starting an
+      interactive session. the existing error message could lead to
+      some confusion. so put more details in it.
+    
+    Fixes: #11101
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 972dc9145235ff0f60663369ebcab8c16628b89f)
+
+commit 9916d37ead705d467b1d730adb15bb6974b3bf1b
+Author: huangjun <hjwsm1989@gmail.com>
+Date:   Mon Jul 20 10:50:20 2015 +0800
+
+      mon/PGMonitor: avoid uint64_t overflow when checking pool 'target/max' status.
+      Fixes: #12401
+    
+      Signed-off-by: huangjun <hjwsm1989@gmail.com>
+    
+    (cherry picked from commit f84e6b8157e78fd05cd1c8ac957c92df17e6ba06)
+
+commit 4457d3eb70712ec3b7d8e98af973e099c8c7d258
+Author: renhwztetecs <rhwlyw@163.com>
+Date:   Tue Jul 21 18:55:00 2015 +0800
+
+    Update OSDMonitor.cc
+    
+    OSDMonitor::get_pools_health(), s/objects/bytes/
+    
+    Fixes: #12402
+    Signed-off-by: ren.huanwen@zte.com.cn
+    (cherry picked from commit 7fc13c9d6b9a4962d7640240416105d8f558d600)
+
+commit add0f1e5e229c9ad66e8ef77ad59e0e390e20db6
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Apr 16 18:29:00 2015 +0800
+
+    ceph.in: do not throw on unknown errno
+    
+    some of the errnos are not listed in errno.errorcode. if we happen
+    to run into them, print 'Unknown' instead.
+    
+    Fixes: #11354
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 53514ad97f6f75746613f09d464f69a7c31bed55)
+
+commit fa194744fdaed13949bffb6f2b8a0fc420006e0b
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Mar 18 13:49:20 2015 -0700
+
+    os/chain_xattr: handle read on chnk-aligned xattr
+    
+    If we wrote an xattr that was a multiple of a chunk, we will try to read
+    the next chunk and get ENODATA.  If that happens bail out of the loop and
+    assume we've read the whole thing.
+    
+    Backport: hammer, firefly
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8614dcebf384b74b283cd352001a65fa26ba925c)
+
+commit 931ffe3b3a43157798717e64de8f46fadbcd9e9b
+Author: Boris Ranto <branto@redhat.com>
+Date:   Wed Jun 3 12:24:48 2015 +0200
+
+    common/Cycles.cc: skip initialization if rdtsc is not implemented
+    
+    The Cycles initialization gets stuck in infinite loop if rdtsc is not
+    implemented. This patch fixes the issue by quitting the initialization
+    if rtdsc fails.
+    
+    The patch was cherry-picked from ubuntu patch by James Page, see
+    
+    https://bugzilla.redhat.com/show_bug.cgi?id=1222286
+    
+    for more details on the patch.
+    
+    Signed-off-by: James Page <james.page@ubuntu.com>
+    (cherry picked from commit 35c5fd0091fc4d63b21207fb94e46b343519fd56)
+
+commit 0fde3a2465e156ebf9a5cdc6adc45d66d6d647fc
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Thu Jul 9 13:42:42 2015 +0800
+
+    buffer: Fix bufferlist::zero bug with special case
+    
+    Fixes: #12252
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit 43f583d160ccaf879eaf0f3020e77860cf8d1df0)
+
+commit dabc61110a5773262614fc77d3653674aac421f7
+Author: Haomai Wang <haomaiwang@gmail.com>
+Date:   Thu Jul 9 13:32:03 2015 +0800
+
+    UnittestBuffer: Add bufferlist zero test case
+    
+    Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
+    (cherry picked from commit 577acf665948e531e50d36f5780ea19351a5bf65)
+
+commit d08db7a0677412dbe590c850976fb4fe2f503a79 (refs/remotes/gh/wip-11470.hammer)
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Fri Jun 12 19:21:10 2015 +0100
+
+    mon: PaxosService: call post_refresh() instead of post_paxos_update()
+    
+    Whenever the monitor finishes committing a proposal, we call
+    Monitor::refresh_from_paxos() to nudge the services to refresh.  Once
+    all services have refreshed, we would then call each services
+    post_paxos_update().
+    
+    However, due to an unfortunate, non-critical bug, some services (mainly
+    the LogMonitor) could have messages pending in their
+    'waiting_for_finished_proposal' callback queue [1], and we need to nudge
+    those callbacks.
+    
+    This patch adds a new step during the refresh phase: instead of calling
+    directly the service's post_paxos_update(), we introduce a
+    PaxosService::post_refresh() which will call the services
+    post_paxos_update() function first and then nudge those callbacks when
+    appropriate.
+    
+    [1] - Given the monitor will send MLog messages to itself, and given the
+    service is not readable before its initial state is proposed and
+    committed, some of the initial MLog's would be stuck waiting for the
+    proposal to finish.  However, by design, we only nudge those message's
+    callbacks when an election finishes or, if the leader, when the proposal
+    finishes.  On peons, however, we would only nudge those callbacks if an
+    election happened to be triggered, hence the need for an alternate path
+    to retry any message waiting for the initial proposal to finish.
+    
+    Fixes: #11470
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit 1551ebb63238073d2fd30201e6b656a8988e958c)
+
+commit 0818e9f78b11801aa619b9ec04eeffb2795c3f62
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jul 9 13:28:22 2015 +0200
+
+    ceph.spec.in: snappy-devel for all supported distros
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit e4634ddcc1c0a2abe51679425d15e6f693811e17)
+    
+    Conflicts:
+    	ceph.spec.in
+                minor difference
+
+commit 8b576bd529d7c1604f0a4cdf97ece64fc5d69447
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Jun 16 18:27:20 2015 +0200
+
+    ceph.spec.in: python-argparse only in Python 2.6
+    
+    argparse is a widely-used Python module for parsing command-line arguments.
+    Ceph makes heavy use of Python scripts, both in the build environment and on
+    cluster nodes and clients.
+    
+    Until Python 2.6, argparse was distributed separately from Python proper.
+    As of 2.7 it is part of the Python standard library.
+    
+    Although the python package in a given distro may or may not Provide:
+    python-argparse, this cannot be relied upon.
+    
+    Therefore, this commit puts appropriate conditionals around Requires:
+    python-argparse and BuildRequires: python-argparse. It does so for Red
+    Hat/CentOS and SUSE only, because the last Fedora version with Python 2.6
+    was Fedora 13, which is EOL.
+    
+    argparse is required by both the ceph and ceph-common packages, but since ceph
+    requires ceph-common, the argparse Requires and BuildRequires need only appear
+    once, under ceph-common.
+    
+    http://tracker.ceph.com/issues/12034 Fixes: #12034
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 23171c952997d104cfa6b5700ec0bb658a8e0915)
+
+commit cdde626f6371e1da501306ed16bc450ee1ec0b91
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Jul 13 18:12:01 2015 +0200
+
+    ceph.spec.in: do not run fdupes, even on SLE/openSUSE
+    
+    In openSUSE there is a policy to use %fdupes in the spec file if RPMLINT
+    complains about duplicate files wasting space in the filesystem.
+    
+    However, RPMLINT is not so complaining, so drop fdupes.
+    
+    http://tracker.ceph.com/issues/12301 Fixes: #12301
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 53072b9019caf72e0313b2804ea174237ed7da33)
+
+commit 6417e8eff942dbbf6cd231ffb73136b7bdcd837f
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jun 12 17:50:58 2015 +0200
+
+    rpm: add missing Java conditionals
+    
+    http://tracker.ceph.com/issues/11991 Fixes: #11991
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 04e91bbd929801de7f1d6a05086d47a781c2e361)
+
+commit 37284773a754ef4b91d2b5436d46a5fd2be4f67e
+Author: Owen Synge <osynge@suse.com>
+Date:   Fri May 29 13:28:24 2015 +0200
+
+    Add rpm conditionals : cephfs_java
+    
+    Extracted conditionals from SUSE rpm spec file to forward port.
+    Original work done by Thorsten Behrens <tbehrens@suse.com>
+    
+    Signed-off-by: Owen Synge <osynge@suse.com>
+    (cherry picked from commit c65538f489897821877421043300d9c008224f02)
+
+commit 8f78001c2e09d483dcbcd33c24e2ef8043119773
+Author: Nathan Cutler <ncutler@suse.cz>
+Date:   Thu May 14 20:07:37 2015 +0200
+
+    ceph.spec.in: SUSE/openSUSE builds need libbz2-devel
+    
+    http://tracker.ceph.com/issues/11629 Fixes: #11629
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.cz>
+    (cherry picked from commit 584ed2e28b0a31821f29a99cf6078cfd76ee31fc)
+
+commit 4eb58ad2027148561d94bb43346b464b55d041a6
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jun 25 22:37:52 2015 +0200
+
+    ceph.spec.in: use _udevrulesdir to eliminate conditionals
+    
+    The conditionals governing where 50-rbd.rules is installed were not doing the
+    right thing on SUSE distros.
+    
+    Start using the %_udevrulesdir RPM macro, while taking care that it is defined
+    and set to the right value. Use it to eliminate some conditionals around other
+    udev rules files as well.
+    
+    http://tracker.ceph.com/issues/12166 Fixes: #12166
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 8aa758ee7a14e93ded1d55b4aca7a54aa47d7174)
+    
+    Conflicts:
+    	ceph.spec.in
+                hammer does not yet have _with_systemd or ceph_daemon.py
diff --git a/doc/changelog/v0.94.5.txt b/doc/changelog/v0.94.5.txt
new file mode 100644
index 000000000..56c9e50bd
--- /dev/null
+++ b/doc/changelog/v0.94.5.txt
@@ -0,0 +1,86 @@
+commit 9764da52395923e0b32908d83a9f7304401fee43 (tag: refs/tags/v0.94.5, refs/remotes/gh/hammer)
+Author: Jenkins <jenkins@ceph.com>
+Date:   Mon Oct 26 07:05:32 2015 -0700
+
+    0.94.5
+
+commit 1107f29224e24211860b719b82f9b85a022c25e3
+Merge: d86eab5 d3abcbe
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Oct 23 20:41:33 2015 +0200
+
+    Merge pull request #6348 from dillaman/wip-13567-hammer
+    
+    librbd: potential assertion failure during cache read
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit d86eab5b860186066c888208d92a42a748569d78
+Merge: 4ebfa1b 250dc07
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Oct 23 15:20:32 2015 +0200
+
+    Merge pull request #6362 from liewegas/wip-shut-up-osd-hammer
+    
+    osd/ReplicatedPG: remove stray debug line
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 250dc0722b9156fe1b8be81e09fb4ead699065f3 (refs/remotes/me/wip-shut-up-osd-hammer)
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Sep 12 08:33:44 2015 -0400
+
+    osd/ReplicatedPG: remove stray debug line
+    
+    This snuck in
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ff9600a6cef613d40e875597b6392778df1bb04c)
+
+commit d3abcbea1fdb04f0994f19584b93f6f1b1ff37ca
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Oct 21 13:12:48 2015 -0400
+
+    librbd: potential assertion failure during cache read
+    
+    It's possible for a cache read from a clone to trigger a writeback if a
+    previous read op determined the object doesn't exist in the clone,
+    followed by a cached write to the non-existent clone object, followed
+    by another read request to the same object.  This causes the cache to
+    flush the pending writeback ops while not holding the owner lock.
+    
+    Fixes: #13559
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4692c330bd992a06b97b5b8975ab71952b22477a)
+
+commit 991d0f0575411e2f2b53df35e36ff6170bcc9d8b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Oct 21 13:09:54 2015 -0400
+
+    tests: reproduce crash during read-induced CoW
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 382219b6bba09156f8cf3d420c6348738e7ed4c6)
+
+commit 4ebfa1bcc7e3b57c3c12000633654d36a005d512
+Merge: 9529269 51f3d6a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 22 12:14:42 2015 -0400
+
+    Merge pull request #6330 from dachary/wip-13550-hammer
+    
+    qemu workunit refers to apt-mirror.front.sepia.ceph.com
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 51f3d6a489760dbcb931676396eea92159ca2ad9
+Author: Yuan Zhou <yuan.zhou@intel.com>
+Date:   Fri Jun 19 00:02:20 2015 +0800
+
+    qa: Use public qemu repo
+    
+    This would allow some external tests outside of sepia lab
+    
+    Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>
+    (cherry picked from commit 4731c1e35539c9506ff3fe3141553fad099d0eee)
diff --git a/doc/changelog/v0.94.6.txt b/doc/changelog/v0.94.6.txt
new file mode 100644
index 000000000..7fc155b7d
--- /dev/null
+++ b/doc/changelog/v0.94.6.txt
@@ -0,0 +1,4027 @@
+commit e832001feaf8c176593e0325c8298e3f16dfb403 (tag: refs/tags/v0.94.6, refs/remotes/gh/hammer)
+Author: Jenkins Build Slave User <jenkins-build@jenkins-slave-wheezy.localdomain>
+Date:   Mon Feb 22 21:10:17 2016 +0000
+
+    0.94.6
+
+commit 7abb6ae8f3cba67009bd022aaeee0a87cdfc6477
+Merge: ceb6fcc a8fc6a9
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 8 14:19:57 2016 +0700
+
+    Merge pull request #7501 from SUSE/wip-14624-hammer
+    
+    hammer: fsx failed to compile
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit ceb6fcc5f2828995bfe59bfc4e206010a4fe3230
+Merge: 2f4e1d1 f2ca42b
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Feb 5 21:10:46 2016 -0500
+
+    Merge pull request #7524 from ktdreyer/wip-14637-hammer-man-radosgw-admin-orphans
+    
+    hammer: doc: regenerate man pages, add orphans commands to radosgw-admin(8)
+
+commit 2f4e1d1ff8e91fc2ee9c23d5a17c3174d15a7103
+Merge: 2ca3c3e 9ab5fd9
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Fri Feb 5 10:30:22 2016 +0100
+
+    Merge pull request #7526 from ceph/wip-14516-hammer
+    
+    hammer: rgw-admin: document orphans commands in usage
+
+commit 2ca3c3e5683ef97902d0969e49980d69c81b4034
+Merge: 02353f6 5c8d1d7
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Feb 5 12:47:33 2016 +0700
+
+    Merge pull request #7441 from odivlad/backport-pr-14569
+    
+    [backport] hammer: rgw: Make RGW_MAX_PUT_SIZE configurable
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 02353f6baa460949d7dd2738346d8d748401bc5b
+Merge: f3bab8c 0e1378e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Feb 5 12:46:54 2016 +0700
+
+    Merge pull request #7442 from odivlad/backport-pr-14570
+    
+    [backport] rgw: fix wrong etag calculation during POST on S3 bucket.
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 9ab5fd9d67da43e986489e4f580a597dd8cb551e
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Feb 1 16:33:55 2016 -0800
+
+    rgw-admin: document orphans commands in usage
+    
+    Fixes: #14516
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 105a76bf542e05b739d5a03ca8ae55432350f107)
+    
+    Conflicts:
+    	src/rgw/rgw_admin.cc (trivial resolution)
+
+commit 0e1378effdd1d0d70d3de05c79b208e9f8b8e328
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Mon Sep 21 20:32:29 2015 +0200
+
+    [backport] rgw: fix wrong etag calculation during POST on S3 bucket.
+    
+    Closes: #14570
+    (cherry picked from commit 742906a)
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    Signed-off-by: Vladislav Odintsov <odivlad@gmail.com>
+    Tested-by: Vladislav Odintsov <odivlad@gmail.com>
+
+commit 5c8d1d74069f70b85bc4286e6d1136bce1dc593f
+Author: Yuan Zhou <yuan.zhou@intel.com>
+Date:   Thu Jul 9 16:56:07 2015 +0800
+
+    [backport] rgw: Make RGW_MAX_PUT_SIZE configurable
+    
+    The 5GB limit of a single operation uploading was part of S3 spec.
+    However some private setups may have some special requirements
+    on this limit. It's more convinent to have a configurable value.
+    
+    Closes: http://tracker.ceph.com/issues/14569
+    (cherry picked from commit df97f28)
+    
+    Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>
+    Signed-off-by: Vladislav Odintsov <odivlad@gmail.com>
+    Tested-by: Vladislav Odintsov <odivlad@gmail.com>
+
+commit f2ca42b1218182f4bfa27718c9606705d8b9941f
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Wed Feb 3 19:51:58 2016 -0700
+
+    doc: add orphans commands to radosgw-admin(8)
+    
+    The --help text was added in 105a76bf542e05b739d5a03ca8ae55432350f107.
+    Add the corresponding entries to the man page.
+    
+    Fixes: #14637
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    
+    (cherry picked from commit ec162f068b40f594c321df5caa9fe2541551b89e)
+      Cherry-pick to hammer includes nroff source change (in master the
+      nroff sources are no longer present in Git.)
+
+commit e42ed6d6414ad55d671dd0f406b1dababd643af8
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Thu Feb 4 11:04:39 2016 -0700
+
+    man: rebuild manpages
+    
+    following the procedure in admin/manpage-howto.txt.
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit a8fc6a9ffb2f8021657ad412fd0aaaaf7f98bd53
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Wed Jan 13 13:17:53 2016 -0800
+
+    fsx: checkout old version until it compiles properly on miras
+    
+    I sent a patch to xfstests upstream at
+    http://article.gmane.org/gmane.comp.file-systems.fstests/1665, but
+    until that's fixed we need a version that works in our test lab.
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 7d52372ae74878ebd001036ff0a7aad525eb15b6)
+
+commit f3bab8c7bc6bba3c79329d7106684596634e17a6
+Merge: 31d86b1 1b02859
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 3 12:41:56 2016 +0700
+
+    Merge pull request #7454 from dachary/wip-14584-hammer
+    
+    hammer: fsstress.sh fails
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 31d86b1580d59581d130e7a5e84905a5b4d67666
+Merge: 2c8e579 2817ffc
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 3 11:38:57 2016 +0700
+
+    Merge pull request #6918 from asheplyakov/hammer-bug-12449
+    
+    osd: check for full before changing the cached obc (hammer)
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 1b02859480677abbd8708650764ed6815917e0cd
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Aug 18 15:22:55 2015 +0800
+
+    qa/fsstress.sh: fix 'cp not writing through dangling symlink'
+    
+    On some test machines, /usr/lib/ltp/testcases/bin/fsstress is
+    dangling symlink. 'cp -f' is impotent in this case.
+    
+    Fixes: #12710
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 479f2a760baf6af4714d3391a366a6b3acf1bcaf)
+
+commit 2c8e57934284dae0ae92d1aa0839a87092ec7c51
+Merge: 1cab151 700be56
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jan 30 21:42:29 2016 -0500
+
+    Merge pull request #7236 from athanatos/wip-14376
+    
+    config_opts: increase suicide timeout to 300 to match recovery
+    
+    http://pulpito.ceph.com/sage-2016-01-30_09:58:32-rados-wip-sage-testing-hammer---basic-mira/
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 1cab151a6cdc8a1b9b2f11518b77ea149c19d067
+Merge: a134c44 5105d50
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jan 30 21:42:12 2016 -0500
+
+    Merge pull request #6450 from dachary/wip-13672-hammer
+    
+    tests: testprofile must be removed before it is re-created
+    
+    http://pulpito.ceph.com/sage-2016-01-30_09:58:32-rados-wip-sage-testing-hammer---basic-mira/
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit a134c44d626ead3308464474d67604363bac6f5d
+Merge: 520792d 3f1292e
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jan 30 21:41:39 2016 -0500
+
+    Merge pull request #6680 from SUSE/wip-13859-hammer
+    
+    hammer: ceph.spec.in License line does not reflect COPYING
+    
+    http://pulpito.ceph.com/sage-2016-01-30_09:58:32-rados-wip-sage-testing-hammer---basic-mira/
+
+commit 520792d51e949c451767d116e267d86fee812ada
+Merge: 4d0fafb c2c6d02
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jan 30 21:41:18 2016 -0500
+
+    Merge pull request #6791 from branch-predictor/bp-5812-backport
+    
+    Race condition in rados bench
+    
+    http://pulpito.ceph.com/sage-2016-01-30_09:58:32-rados-wip-sage-testing-hammer---basic-mira/
+
+commit 4d0fafb289fc35f44e6e74bb974c402ba147d4d4
+Merge: 211a093 6379ff1
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jan 30 21:40:38 2016 -0500
+
+    Merge pull request #6973 from dreamhost/wip-configure-hammer
+    
+    configure.ac: no use to add "+" before ac_ext=c
+    
+    http://pulpito.ceph.com/sage-2016-01-30_09:58:32-rados-wip-sage-testing-hammer---basic-mira/
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 211a093d4107c1806ad7e4876bc5c550a5b5b7d4
+Merge: 0c13656 53742bd
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jan 30 21:40:13 2016 -0500
+
+    Merge pull request #7206 from dzafman/wip-14292
+    
+    osd/PG.cc: 3837: FAILED assert(0 == "Running incompatible OSD")
+    
+    http://pulpito.ceph.com/sage-2016-01-30_09:58:32-rados-wip-sage-testing-hammer---basic-mira/
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 0c136561600e295ec48dcf29a77aa2cd293a7236
+Merge: 1ea14ba ae56de0
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jan 30 21:39:42 2016 -0500
+
+    Merge pull request #7207 from rldleblanc/recency_fix_for_hammer
+    
+    hammer: osd/ReplicatedPG: Recency fix for Hammer
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 1ea14bae10930ed3e66b7b0140e359009a44275e
+Merge: 1740d8c 8d9e08c
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jan 30 21:39:11 2016 -0500
+
+    Merge pull request #7347 from tchaikov/wip-hammer-10093
+    
+    tools: ceph-monstore-tool must do out_store.close()
+    
+    http://pulpito.ceph.com/sage-2016-01-30_09:58:32-rados-wip-sage-testing-hammer---basic-mira/
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 1740d8ce8eb7e7debce1684a19736783489628c3
+Merge: 7848cdc c7252a3
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jan 30 21:38:35 2016 -0500
+
+    Merge pull request #7411 from dachary/wip-14467-hammer
+    
+    hammer: disable filestore_xfs_extsize by default
+    
+    http://pulpito.ceph.com/sage-2016-01-30_09:58:32-rados-wip-sage-testing-hammer---basic-mira/
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 7848cdc4c8c3598cd291d26a2dd4d76abc4bcda9
+Merge: 57abeab 70f1ba3
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jan 30 21:38:13 2016 -0500
+
+    Merge pull request #7412 from dachary/wip-14470-hammer
+    
+    tools: tool for artificially inflate the leveldb of the mon store for testing purposes
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 57abeab1f2610d89f0ab2a028c8e093afde5e403
+Merge: 4d7d7c3 9109304
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jan 30 21:37:46 2016 -0500
+
+    Merge pull request #7446 from liewegas/wip-14537-hammer
+    
+    mon: compact full epochs also
+    
+    http://pulpito.ceph.com/sage-2016-01-30_09:58:32-rados-wip-sage-testing-hammer---basic-mira/
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 4d7d7c33ff58ffeb27b47d379bf7dd4af90785b0
+Merge: 8360486 8c28f2f
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Sat Jan 30 11:45:31 2016 -0800
+
+    Merge pull request #7182 from dachary/wip-14143-hammer
+    
+    hammer: Verify self-managed snapshot functionality on image create
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 8360486764c3ab4e10a34f5bdf7555e4c3385977
+Merge: 501e01a b2961ce
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Sat Jan 30 11:45:20 2016 -0800
+
+    Merge pull request #7183 from dachary/wip-14283-hammer
+    
+    hammer: rbd: fix bench-write
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 501e01acc55f846cd3ad85e8294a9598c1c90907
+Merge: 97d4f6a 24c0b27
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Sat Jan 30 11:45:05 2016 -0800
+
+    Merge pull request #7416 from dachary/wip-14466-hammer
+    
+    hammer: rbd-replay does not check for EOF and goes to endless loop
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 97d4f6a7bed5a0d47f18b3a223a916aef355eaf9
+Merge: 51cc015 46d626d
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Sat Jan 30 11:44:50 2016 -0800
+
+    Merge pull request #7417 from dachary/wip-14553-hammer
+    
+    hammer: rbd: TaskFinisher::cancel should remove event from SafeTimer
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 51cc015c7e20d2ea9b9517238481fb80687be17c
+Merge: 23c24fc 73e03de
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Sat Jan 30 11:44:32 2016 -0800
+
+    Merge pull request #7407 from dillaman/wip-14543-hammer
+    
+    librbd: ImageWatcher shouldn't block the notification thread
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 23c24fc50d07a07db12239ac372bfce5a9fe2255
+Merge: 9513391 26e832e
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Sat Jan 30 11:44:12 2016 -0800
+
+    Merge pull request #6980 from dillaman/wip-14063-hammer
+    
+    librbd: fix merge-diff for >2GB diff-files
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 91093041a86fda40de08a366b5118e5e3ae275f0 (refs/remotes/me/wip-14537-hammer)
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Jan 28 02:09:53 2016 -0800
+
+    mon: compact full epochs also
+    
+    by compacting the ${prefix}.${start}..${prefix}..${end} does not
+    necessary compact the range of ${prefix}."full_"${start}..
+    ${prefix}."full_"${end}. so when more and more epochs get trimmed
+    with out a full range compaction, the size of monitor store could
+    be very large.
+    
+    Fixes: #14537
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 93d633a25ad8c36c972bb766c38187e2612041e1)
+
+commit 2817ffcf4e57f92551b86388681fc0fe70c386ec
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Wed Dec 16 15:31:52 2015 +0300
+
+    Check for full before changing the cached obc
+    
+    ReplicatedPG::prepare_transaction(): check if the pool is full before
+    updating the cached ObjectContext to avoid the discrepancy between
+    the cached and the actual object size (and other metadata).
+    While at it improve the check itself: consider cluster full flag,
+    not just the pool full flag, also consider object count changes too,
+    not just bytes.
+    
+    Based on commit a1eb380c3d5254f9f1fe34b4629e51d77fe010c1
+    
+    Fixes: #13335
+    
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+
+commit 951339103d35bc8ee2de880f77aada40d15b592a (refs/remotes/gh/wip-test-14716-4)
+Merge: e43aca5 5e5b512
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jan 29 23:31:47 2016 +0700
+
+    Merge pull request #6353 from theanalyst/wip-13513-hammer
+    
+    rgw: value of Swift API's X-Object-Manifest header is not url_decoded during segment look up
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit e43aca51d630274a57237b0f91a56df79ce8874a
+Merge: 7e20e6e 4420929
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jan 29 23:31:16 2016 +0700
+
+    Merge pull request #6620 from SUSE/wip-13820-hammer
+    
+    hammer: rgw: Setting ACL on Object removes ETag
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 7e20e6e9d6626c5ac8b1f479011ab77a9e87da23
+Merge: f1d5570 cbb5c1f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jan 29 23:30:57 2016 +0700
+
+    Merge pull request #7186 from dachary/wip-13888-hammer
+    
+    hammer: rgw: orphans finish segfaults
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit ae56de09fe1385958b5b600d1f0c91383989926f
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 25 14:40:26 2015 -0500
+
+    osd: recency should look at newest (not oldest) hitsets
+    
+    Reported-by: xinxin shu <xinxin.shu@intel.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 528eae92d010ea34ae8cf0e8b2290aaa5e058d24)
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.cc
+                        code section was moved to ReplicatedPG::maybe_promote
+                        in master.
+    Signed-off-by: Robert LeBlanc <robert.leblanc@endurance.com>
+
+commit 5cefcb975771f0c2efb7dfc77ce14a93a4ee7f1b
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 25 14:39:08 2015 -0500
+
+    osd/ReplicatedPG: fix promotion recency logic
+    
+    Recency is defined as how many of the last N hitsets an object
+    must appear in in order to be promoted.  The previous logic did
+    nothing of the sort... it checked for the object in any one of
+    the last N hitsets, which led to way to many promotions and killed
+    any chance of the cache performing properly.
+    
+    While we are here, we can simplify the code to drop the max_in_*
+    fields (no longer necessary).
+    
+    Note that we may still want a notion of 'temperature' that does
+    tolerate the object missing in one of the recent hitsets.. but
+    that would be different than recency, and should probably be
+    modeled after the eviction temperature model.
+    
+    Backport: infernalis, hammer
+    Reported-by: Nick Fisk <nick@fisk.me.uk>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 180c8743addc5ae2f1db9c58cd2996ca6e7ac18b)
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.cc
+                        code section was moved to ReplicatedPG::maybe_promote
+                        in master.
+    Signed-off-by: Robert LeBlanc <robert.leblanc@endurance.com>
+
+commit f1d5570beab0769b925b917e402d441ff053794c
+Merge: c4bb343 50c82f2
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jan 29 08:52:51 2016 -0500
+
+    Merge pull request #5789 from SUSE/wip-12928-hammer
+    
+    rpm: libcephfs_jni1 has no %post and %postun
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit c4bb34320df8bfa734512a400fe8664c131363ff
+Merge: 86ba6ca a5e4f70
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jan 29 08:50:56 2016 -0500
+
+    Merge pull request #7434 from tchaikov/wip-14441-hammer
+    
+    man: document listwatchers cmd in "rados" manpage
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit a5e4f70af422b05ece53f245bc15491bb1dd540d
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Dec 23 11:23:38 2015 +0800
+
+    man: document listwatchers cmd in "rados" manpage
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit c2e391301efc43f0b431e89737246b2c43bf10a9)
+    
+    man/rados.8: also added the rendered man.8 man page, as we don't
+                 put the generated man pages in master anymore. but
+                 they are still in the hammer's source repo.
+
+commit 46d626d92d7af218816d831cfcc1265b3ea31b72
+Author: Douglas Fuller <dfuller@redhat.com>
+Date:   Fri Jan 22 11:18:40 2016 -0800
+
+    rbd: remove canceled tasks from timer thread
+    
+    When canceling scheduled tasks using the timer thread, TaskFinisher::cancel
+    does not call SafeTimer::cancel_event, so events fire anyway. Add this call.
+    
+    Fixes: #14476
+    Signed-off-by: Douglas Fuller <dfuller@redhat.com>
+    (cherry picked from commit 2aa0f318c862dbe3027d74d345671506605778eb)
+
+commit 24c0b27c6f6a26c2b7bab5bcbc421a18592d026f
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Jan 21 13:45:42 2016 +0200
+
+    rbd-replay: handle EOF gracefully
+    
+    Fixes: #14452
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit c59b84c3e2c9bbda68219e4d2288a889dd9ca6cb)
+    
+    Conflicts:
+    	src/rbd_replay/BufferReader.cc: trivial resolution
+            caused by s/CEPH_PAGE_SIZE/CEPH_BUFFER_APPEND_SIZE/
+            in the context of one hunk.
+
+commit 70f1ba33fe1a81d2631d54429749433a6cbfca44
+Author: Cilang Zhao <zhao.cilang@h3c.com>
+Date:   Tue Jan 5 14:34:05 2016 +0800
+
+    tools: monstore: add 'show-versions' command.
+    
+    Using this tool, the first/last committed version of maps will be shown.
+    
+    Signed-off-by: Cilang Zhao <zhao.cilang@h3c.com>
+    (cherry picked from commit 21e6ba0c18428caff45733e6b43d197be38af8bb)
+
+commit 926017187910c9e6a3fb8babf9b498cf07941819
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Sep 16 18:28:52 2015 +0800
+
+    tools: ceph_monstore_tool: add inflate-pgmap command
+    
+    this command repeatly add the latest pgmap to the monstore in order
+    to inflate it. the command helps with the testing of some monstore
+    related performance issue of monitor
+    
+    Fixes: #14217
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b0f6af814a58dd374ca95e84a4e381f8ef1368de)
+
+commit a1d5728c928eb7e6b8c655741a0db3398ced7d56
+Author: Bo Cai <cai.bo@h3c.com>
+Date:   Tue Oct 20 15:23:49 2015 +0800
+
+    tools:support printing the crushmap in readable fashion.
+    
+    Signed-off-by: Bo Cai <cai.bo@h3c.com>
+    (cherry picked from commit b7faf678ed0241abc7eb86b9baaa6db669a22d53)
+
+commit bd9484750f546974047dacd9176a8072be846596
+Author: Bo Cai <cai.bo@h3c.com>
+Date:   Mon Sep 14 19:50:47 2015 +0800
+
+    tools:print the map infomation in human readable format.
+    
+    Signed-off-by: Bo Cai <cai.bo@h3c.com>
+    (cherry picked from commit 5ed8cdc19150382c946a373ec940d76f98e6ecb7)
+
+commit fba65832aad8a46d94a9256a56997e9df9e62297
+Author: Bo Cai <cai.bo@h3c.com>
+Date:   Mon Sep 14 19:19:05 2015 +0800
+
+    tools:remove the local file when get map failed.
+    
+    Signed-off-by: Bo Cai <cai.bo@h3c.com>
+    (cherry picked from commit 0b03b32d8ba76fe9f6f1158e68eb440e3670393a)
+
+commit 1bb899a290b77188b44a53ef7c7a40910c9248b2
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Mon Jul 13 12:35:13 2015 +0100
+
+    tools: ceph_monstore_tool: describe behavior of rewrite command
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit a881f9385feb0f5a61fa22357984d6f291c08177)
+
+commit 9035c69481f4aa4786414ac41cbc36fb4a3ca51d
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Jun 19 22:57:57 2015 +0800
+
+    tools/ceph-monstore-tools: add rewrite command
+    
+    "rewrite" command will
+     - add a new osdmap version to update current osdmap held by OSDMonitor
+     - add a new paxos version, as a proposal it will
+       * rewrite all osdmap epochs from specified epoch to  the last_committed
+         one with the specified crush map.
+       * add the new osdmap which is added just now
+    so the leader monitor can trigger a recovery process to apply the transaction
+    to all monitors in quorum, and hence bring them back to normal after being
+    injected with a faulty crushmap.
+    
+    Fixes: #11815
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 306373427836ca0c2418dbe6caab26d74d94d12e)
+
+commit 90aaed744243dfc7c620f91e19fc0bfa233f711f
+Author: huangjun <hjwsm1989@gmail.com>
+Date:   Tue Apr 21 14:04:40 2015 +0800
+
+    tools: ceph-monstore-tool must do out_store.close()
+    
+    this change fixes the "store-copy" command.
+    Like the bug reported in http://tracker.ceph.com/issues/10093.
+    
+    Signed-off-by: huangjun <hjwsm1989@gmail.com>
+    (cherry picked from commit d85e0f8c50fce62be012506f7ffcb1cdeb0c819c)
+
+commit c7252a3d54bae88f91c2b4e63fc9c27cfbb2423e
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Mon Jan 18 08:24:46 2016 -0700
+
+    osd: disable filestore_xfs_extsize by default
+    
+    This option involves a tradeoff: When disabled, fragmentation is worse,
+    but large sequential writes are faster. When enabled, large sequential
+    writes are slower, but fragmentation is reduced.
+    
+    Fixes: #14397
+    (cherry picked from commit aed85775bf53c273786ce4999320134822722af5)
+
+commit 86ba6caf992d2544cdd174e3b3f26a6099c91fc5
+Merge: b6b8ee4 0325f8a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jan 29 10:36:05 2016 +0700
+
+    Merge pull request #7316 from ceph/wip-deb-lttng-hammer
+    
+    deb: strip tracepoint libraries from Wheezy/Precise builds
+    
+    All other "modern" Debian-based OSes have a functional LTTng-UST. Since only hammer needs to build on these older distros, this fix only affects the deb building process for those two releases(since autoconf detects that LTTng is broken).
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit b6b8ee41fc77679e82304e79da6dbd0c35f5c067
+Merge: d54840b f96c812
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jan 29 10:21:43 2016 +0700
+
+    Merge pull request #7187 from dachary/wip-13831-hammer
+    
+    hammer: init script reload doesn't work on EL7
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 73e03def9271fb5d1739b195e428c3ebfcebd59b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 28 14:38:20 2016 -0500
+
+    librbd: ImageWatcher shouldn't block the notification thread
+    
+    Blocking the notification thread will also result in librados async
+    callbacks becoming blocked (since they use the same thread).
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7e2019a72733dff43e55c9b22df12939d584f87d)
+    
+     Conflicts:
+    	src/librbd/ImageWatcher.[cc|h]: fewer RPC messages
+
+commit 5617166f78c1995436b4e0794dab2d8254331815
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 28 14:35:54 2016 -0500
+
+    librados_test_stub: watch/notify now behaves similar to librados
+    
+    Notifications are executed via the same librados AIO callback
+    thread, so it's now possible to catch deadlock.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 43722571838a2d78ca8583b5a1ea381cd988de0b)
+
+commit 8fc82b23304ef327933723373cd4d1090d04bfbc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 28 12:40:18 2016 -0500
+
+    tests: simulate writeback flush during snap create
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e9570fcf4be9cd5c06937769d074fa52ffb32e4b)
+
+commit d54840bf4a70fc65285bbfdff0c7bf8f579643b1
+Merge: 4051bc2 9f30fe1
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jan 26 10:12:29 2016 -0500
+
+    Merge pull request #7365 from liewegas/wip-tcmalloc-hammer
+    
+    osd: pass tcmalloc env through to ceph-osd
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 9f30fe18e839f5846c90e3b5995171a0132d7f3a
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jan 26 08:43:15 2016 -0500
+
+    upstart/ceph-osd.conf: pass TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES through
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a95a3d34cccb9f9f0782a96ca05e5fe9c2b01772)
+
+commit a58873e213d7423d89c95db4e1710dc9631e3313
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jan 20 18:36:08 2016 -0500
+
+    init-ceph: pass TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES through
+    
+    ..when set in the sysconfig/default file.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 03e01ae263ef207046933890821ae75d5e60d9b8)
+    
+    [source default/sysconfig file too]
+
+commit 8d9e08c44150a4f3ad06ec1abc130b9aeaf479d9
+Author: huangjun <hjwsm1989@gmail.com>
+Date:   Tue Apr 21 14:04:40 2015 +0800
+
+    tools: ceph-monstore-tool must do out_store.close()
+    
+    this change fixes the "store-copy" command.
+    Like the bug reported in http://tracker.ceph.com/issues/10093.
+    
+    Signed-off-by: huangjun <hjwsm1989@gmail.com>
+    (cherry picked from commit d85e0f8c50fce62be012506f7ffcb1cdeb0c819c)
+
+commit 0325f8af5cbee3d74e9f363f61c2e2ababf501d9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 21 15:36:37 2016 -0500
+
+    deb: strip tracepoint libraries from Wheezy/Precise builds
+    
+    These releases do not (by default) have a sane LTTng-UST environment,
+    which results in autoconf disabling support for tracing.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 4051bc2a5e4313ac0f6236d7a34ed5fb4a1d9ea2
+Merge: 4e67418 e4d3e9b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jan 18 20:46:07 2016 -0500
+
+    Merge pull request #7252 from ceph/wip-13483-hammer
+    
+    qa: remove legacy OS support from rbd/qemu-iotests
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit e4d3e9b29aa6a5f9efff8b787949a5bbf48ad7de (refs/remotes/gh/wip-13483-hammer)
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jan 18 15:57:43 2016 -0500
+
+    qa: disable rbd/qemu-iotests test case 055 on RHEL/CentOS
+    
+    Fixes: #14385
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit f7acd44c26310242b69ee50322bd6b43fdc774b9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 15 12:44:06 2015 -0400
+
+    qa: remove legacy OS support from rbd/qemu-iotests
+    
+    RHEL7 derivatives were failing test 002 since they were using
+    legacy test cases for now unsupported OSes.
+    
+    Fixes: #13483
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+
+commit 8c28f2f28d960d823ffd632671edaf029c30fb0f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Dec 14 17:41:49 2015 -0500
+
+    librbd: optionally validate RBD pool configuration (snapshot support)
+    
+    Fixes: #13633
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1fea4dadc60e13518e9ee55d136fbc4e9d3a621e)
+    
+    Conflicts:
+    	src/common/config_opts.h: trivial resolution
+
+commit 700be56c530879a72a628c62265d18f0a5d8fb3b
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 14 08:35:23 2016 -0800
+
+    config_opts: increase suicide timeout to 300 to match recovery
+    
+    Fixes: 14376
+    Backport: hammer, firefly
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+
+commit 4e67418958e5caf5e4f81c4ed566e8c7269930fa
+Merge: 28e99a9 1ab2b48
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 14 09:21:50 2016 -0500
+
+    Merge pull request #7179 from dachary/wip-14287-hammer
+    
+    hammer: ReplicatedPG: wrong result code checking logic during sparse_read
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 28e99a90b38ce8c0285c1aec9c5524365385be00
+Merge: 17a2965 86f5cf6
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 14 09:20:53 2016 -0500
+
+    Merge pull request #7178 from dachary/wip-14285-hammer
+    
+    hammer: osd/OSD.cc: 2469: FAILED assert(pg_stat_queue.empty()) on shutdown
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 17a2965825494765249c244aef59ebd821711d42
+Merge: a1459ea b0856ee
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 14 09:20:34 2016 -0500
+
+    Merge pull request #7177 from dachary/wip-14043-hammer
+    
+    hammer: osd/PG.cc: 288: FAILED assert(info.last_epoch_started >= info.history.last_epoch_started)
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit a1459eacd7cc0d064493d01f963b8c1231cd3518
+Merge: c9a13a1 9cee89b
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 14 09:20:10 2016 -0500
+
+    Merge pull request #7180 from dachary/wip-14288-hammer
+    
+    hammer: ceph osd pool stats broken in hammer
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit c9a13a12e88c9a18bfb3cc99d4a7c103b4330426
+Merge: 174a1a9 4d0b9a1
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 14 09:19:18 2016 -0500
+
+    Merge pull request #6994 from badone/wip-13993-hammer
+    
+    log: Log.cc: Assign LOG_DEBUG priority to syslog calls
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 174a1a974725ce4025be4dfdf3b13df766dfac76
+Merge: cb167e9 6cf261c
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 14 09:18:55 2016 -0500
+
+    Merge pull request #6839 from SUSE/wip-13789-hammer
+    
+    Objecter: potential null pointer access when do pool_snap_list.
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit cb167e998bbbdf1b9c273f6bc4f118796d396d37
+Merge: 1c4ba85 66ff0aa
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 14 09:18:23 2016 -0500
+
+    Merge pull request #6835 from SUSE/wip-13892-hammer
+    
+    hammer: auth/cephx: large amounts of log are produced by osd
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 1c4ba855b34290069c7ce0fa990fe72b7a1b381e
+Merge: 72b6b68 61da13b
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 14 09:17:53 2016 -0500
+
+    Merge pull request #6834 from SUSE/wip-13930-hammer
+    
+    hammer: Ceph Pools' MAX AVAIL is 0 if some OSDs' weight is 0
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 72b6b68f43ffc953ca7f3056574bd68428a5abe8
+Merge: d064f9f 53e81aa
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 14 09:17:25 2016 -0500
+
+    Merge pull request #6832 from SUSE/wip-13936-hammer
+    
+    hammer: Ceph daemon failed to start, because the service name was already used.
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit d064f9f6b3bc93fa5887304a9b9ab92d503ebd15
+Merge: 0ca6285 ac05617
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 14 09:16:54 2016 -0500
+
+    Merge pull request #6755 from SUSE/wip-13870-hammer
+    
+    hammer: OSD: race condition detected during send_failures
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 0ca62855f22136c7042a5988366d46e59629f206
+Merge: e9f545b a81bcf7
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Jan 14 08:22:35 2016 +0100
+
+    Merge pull request #6415 from dillaman/wip-13541-hammer
+    
+    LTTng-UST tracing should be dynamically enabled
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit e9f545b4a44c91faf91a8ae3824b1b0d46909244
+Merge: 8521916 b2f1e76
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Jan 14 08:15:02 2016 +0100
+
+    Merge pull request #7176 from dachary/wip-13440-hammer
+    
+    ceph-disk prepare fails if device is a symlink
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 85219163f9b78faa3c1fe96cad7129c6813550c0
+Merge: ebfb3fb 5264bc6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Jan 14 08:14:31 2016 +0100
+
+    Merge pull request #7150 from jecluis/wip-14236
+    
+    mon: OSDMonitor: do not assume a session exists in send_incremental()
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit ebfb3fbe8a1920474941d4a95540cc4a53a747d4
+Merge: e1b9208 447ab1d
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Jan 13 17:15:32 2016 -0800
+
+    Merge pull request #7226 from dillaman/wip-13810-hammer
+    
+    tests: notification slave needs to wait for master
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 447ab1d70a63fceed39518d5714c138c54f6525e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jan 13 12:44:01 2016 -0500
+
+    tests: notification slave needs to wait for master
+    
+    If the slave instance starts before the master, race
+    conditions are possible.
+    
+    Fixes: #13810
+    Backport: infernalis, hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3992d6fe67bbf82322cedc1582406caaf6d4de60)
+
+commit 53742bdbc587747c93413f9db38f3eeb71487872
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Jun 15 17:55:41 2015 -0700
+
+    ceph_osd: Add required feature bits related to this branch to osd_required mask
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0b2bab460c013ea5cece10ea028d26da3485eaf5)
+
+commit 3066231865e2fe56344de9db26024ac65e03053d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Jun 4 18:47:42 2015 -0700
+
+    osd: CEPH_FEATURE_CHUNKY_SCRUB feature now required
+    
+    Feature present since at least the Dumpling release.
+    A later commit will add it to the osd_required mask
+    
+    Fixes: #11661
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 7a10a7e196efd0f59fa7adb87d0a120655b227d8)
+
+commit e1b92081c9e4b21eb30cc873c239083a08fce12f
+Merge: 9708e59 3d3595f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jan 12 14:28:26 2016 +0100
+
+    Merge pull request #6530 from SUSE/wip-13760-hammer
+    
+    unknown argument --quiet in udevadm settle
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 6379ff16dbc7fdecccaa8874d978d1ab58ce44cf
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue May 5 15:07:33 2015 +0800
+
+    configure.ac: no use to add "+" before ac_ext=c
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 899dd23766c5ae6fef39bf24ef0692127d17deb3)
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit f96c8125d423f24e52df0f8420b0b2748e34ca90
+Author: Herve Rousseau <hroussea@cern.ch>
+Date:   Fri Nov 6 09:52:28 2015 +0100
+
+    rgw: fix reload on non Debian systems.
+    
+    When using reload in non-debian systems, /bin/sh's kill is used to send the HUP signal to the radosgw process.
+    This kill version doesn't understand -SIGHUP as a valid signal, using -HUP does work.
+    
+    Fix: #13709
+    Backport: hammer
+    Signed-off-by: Hervé Rousseau <hroussea@cern.ch>
+    (cherry picked from commit 1b000abac3a02d1e788bf25eead4b6873133f5d2)
+
+commit cbb5c1fc2386205277ad22474c1f696a07fcb972
+Author: Igor Fedotov <ifedotov@mirantis.com>
+Date:   Thu Nov 19 13:38:40 2015 +0300
+
+    Fixing NULL pointer dereference
+    
+    Signed-off-by: Igor Fedotov <ifedotov@mirantis.com>
+    (cherry picked from commit 93d3dfe0441be50a6990d458ee0ee3289af39b20)
+
+commit b2961cef3d8e84d2302815e33eb7dc9033d2bb78
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 18 16:05:29 2015 -0400
+
+    rbd: fix bench-write
+    
+    Make each IO get a unique offset!
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 333f3a01a9916c781f266078391c580efb81a0fc)
+
+commit 9cee89bbdd1d89d2fe72c8d73fc0c18db515198a
+Author: BJ Lougee <almightybeeij@gmail.com>
+Date:   Mon Apr 13 11:24:38 2015 -0500
+
+    Check that delta_sum.stats.sum.num_object_copies and delta_sum.stats.sum.num_object are greater than zero
+    
+    This fixes division by zero.
+    
+    Signed-off-by BJ Lougee <almightybeeij@gmail.com>
+    
+    (cherry picked from commit 27ed729c1088133400aa072eeca9e125942f2d94)
+
+commit 1ab2b481596ad5296e168bf8027e59d41190176e
+Author: xiexingguo <258156334@qq.com>
+Date:   Tue Dec 22 17:05:06 2015 +0800
+
+    ReplicatedPG: fix sparse-read result code checking logic
+    
+    Move ahead the result code checking logic before we continue to verify the trailing hole, otherwise
+    the real result of non-hole reading may be overwritten and thus confuse caller.
+    
+    Fixes: #14151
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit b3aa29e324cf0a96a1f2f5dcf1ba998219457bcd)
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.cc: trivial resolution
+
+commit 86f5cf6caa36760802775df19cdabe55bcafa33e
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jan 1 09:32:04 2016 -0500
+
+    osd: clear pg_stat_queue after stopping pgs
+    
+    Fixes: #14212
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c25ff9927febe77cda31b534971b62f75c196ca2)
+
+commit b0856ee67c0e7cf6ab6095d3f657c18014859526
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Dec 3 14:52:24 2015 -0800
+
+    osd: Test osd_find_best_info_ignore_history_les config in another assert
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 02a9a41f151a3d968bf8066749658659dc6e3ac4)
+
+commit b2f1e76d8e95b6e2f88fa1c122ea8cd24532cd10
+Author: Joe Julian <jjulian@io.com>
+Date:   Fri Oct 9 12:57:06 2015 -0700
+
+    Compare parted output with the dereferenced path
+    
+    Compare parted output with the dereferenced path of the device as parted
+    prints that instead of the symlink we called it with.
+    
+    http://tracker.ceph.com/issues/13438 Fixes: #13438
+    
+    Signed-off-by: Joe Julian <jjulian@io.com>
+    (cherry picked from commit b3c7cb098195111b9c642e5a9b726b63717f2e0d)
+
+commit 9708e59a1fc2ae52cfae848ce585751bc9fbe572
+Merge: 9739d4d b62cac6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Jan 11 09:01:12 2016 +0100
+
+    Merge pull request #6512 from SUSE/wip-13734-hammer
+    
+    rgw: swift API returns more than real object count and bytes used when retrieving account metadata
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 5264bc677f0f612165987bf51fe7d7b4af32fa77
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Thu Jan 7 19:20:47 2016 +0000
+
+    mon: OSDMonitor: do not assume a session exists in send_incremental()
+    
+    We may not have an open session for a given osd. If we blindly assume we
+    do, we may end up trying to send incrementals we do not have to the osd.
+    
+    And then we will crash.
+    
+    This fixes a regression introduced by
+    
+      171fee1b82d2675e364da7f96dfb9dd286d9b6e6
+    
+    which is meant as a backport of
+    
+      de43a02e06650a552f048dc8acd17f255126fed9
+    
+    but so happens to intruduce a line that wasn't on the original patch. We
+    imagine it was meant to make the 's->osd_epoch' assignment work without
+    checking the session, as per the original patch, but the backporter must
+    have forgotten to also backport the assertion on the not-null session.
+    The unfortunate introduction of the check for a not-null session
+    triggered this regression.
+    
+    The regression itself is due to enforcing that a session exists for the
+    osd we are sending the incrementals to. However, if we come via the
+    OSDMonitor::process_failures() path, that may very well not be the case,
+    as we are handling potentially-old MOSDFailure messages that may no
+    longer have an associated session. By enforcing the not-null session, we
+    don't check whether we have the requested versions (i.e., if
+    our_earliest_version <= requested_version), and thus we end up on the
+    path that assumes that we DO HAVE all the necessary versions -- when we
+    may not, thus finally asserting because we are reading blank
+    incremental versions.
+    
+    Fixes: #14236
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+
+commit 4d0b9a1bbbabe7b27279a7b6e0a45f5b0d920c66
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Mon Dec 7 11:31:28 2015 +1000
+
+    log: Log.cc: Assign LOG_DEBUG priority to syslog calls
+    
+    Fixes: #13993
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 8e93f3f45db681f82633ca695a7dc4e7bd030584)
+
+commit 26e832e76de90e0a751868b044ea745a97a5af82
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Dec 18 15:22:13 2015 -0500
+
+    librbd: fix merge-diff for >2GB diff-files
+    
+    Fixes: #14063
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 9739d4de49f8167866eda556b2f1581c068ec8a7
+Merge: d064636 a9d3f07
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 17 10:16:29 2015 -0500
+
+    Merge pull request #6544 from liewegas/wip-smaller-object-info
+    
+    osd: make encoded object_info_t smaller to fit inside the XFS inode
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit a9d3f078cb6cdaf7f8778ba1a07d333548a9d1a2
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 14 13:13:33 2015 -0500
+
+    osd/osd_types: skip encoding newer object_info_t fields if they are unused
+    
+    This reduces the size of the encoded object_info_t in most cases,
+    enough to get us under the 255 byte limit for a single inline
+    xattr in XFS.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 1548a3fda7dd63e28792140e7e8ad1ac9b706e49
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 14 13:00:27 2015 -0500
+
+    osd/ReplicatedPG: do not set local_mtime on non-tiered pool
+    
+    If a pool isn't tiered, don't bother with setting local_mtime.  The only
+    users are the tiering agent (which isn't needed if there is not tiering)
+    and scrub for deciding if an object should get its digest recorded (we can
+    use mtime instead).
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 98bdb09bb8d632b2df0b252b3d9676fd9511a1c8
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 14 12:59:29 2015 -0500
+
+    osd/PGBackend: use mtime for digest decision if local_mtime is empty
+    
+    If we don't have a local_mtime value, use mtime instead, for the purposes
+    of deciding if we should record a digest after scrub.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit fe1c28dea4e5607a0c72eab1f046074616cd55a7
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 14 11:35:46 2015 -0500
+
+    osd/ReplicatedPG: do not set omap digest if there is no omap
+    
+    We want to avoid encoding it if we can.  And if the FLAG_OMAP is not set
+    we don't need to *also* store an empty crc.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 6cf261cd2e80adbaf2898cb354c8159a57749e9d
+Author: xiexingguo <258156334@qq.com>
+Date:   Mon Nov 2 21:46:11 2015 +0800
+
+    Objecter: remove redundant result-check of _calc_target in _map_session.
+    
+    Result-code check is currently redundant since _calc_target never returns a negative value.
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    
+    (cherry picked from commit 5a6117e667024f51e65847f73f7589467b6cb762)
+
+commit 5d0b5c1389d8c0baddb40cc5ff8f1d9cc6a4d315
+Author: xiexingguo <258156334@qq.com>
+Date:   Thu Oct 29 17:32:50 2015 +0800
+
+    Objecter: potential null pointer access when do pool_snap_list.
+    
+    Objecter: potential null pointer access when do pool_snap_list. Shall check pool existence first.
+    Fixes: #13639
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 865541605b6c32f03e188ec33d079b44be42fa4a)
+
+commit 66ff0aa0130973aa2bea3a1a4ec8f4e3e6c19da8
+Author: qiankunzheng <zheng.qiankun@h3c.com>
+Date:   Thu Nov 5 07:29:49 2015 -0500
+
+    auth/cephx: large amounts of log are produced by osd
+    if the auth of osd is deleted when the osd is running, the osd will produce large amounts of log.
+    
+    Fixes:#13610
+    Signed-off-by: Qiankun Zheng <zheng.qiankun@h3c.com>
+    (cherry picked from commit 102f0b19326836e3b0754b4d32da89eb2bc0b03c)
+
+commit 61da13bbe271964c56116fb2bed51045290f951a
+Author: Chengyuan Li <chengyli@ebay.com>
+Date:   Thu Nov 19 22:29:39 2015 -0700
+
+    mon/PGMonitor: MAX AVAIL is 0 if some OSDs' weight is 0
+    
+    In get_rule_avail(), even p->second is 0, it's possible to be used
+    as divisor and quotient is infinity, then is converted to an integer
+    which is negative value.
+    So we should check p->second value before calculation.
+    
+    It fixes BUG #13840.
+    
+    Signed-off-by: Chengyuan Li <chengyli@ebay.com>
+    (cherry picked from commit 18713e60edd1fe16ab571f7c83e6de026db483ca)
+
+commit 53e81aab442855b34ee6e922f181bae3bf33e292
+Author: wangchaunhong <root@A22832429.(none)>
+Date:   Tue Oct 20 18:40:23 2015 +0800
+
+    init-ceph: fix systemd-run cant't start ceph daemon sometimes
+    
+    Fixes: #13474
+    Signed-off-by: Chuanhong Wang <wang.chuanhong@zte.com.cn>
+    (cherry picked from commit 2f36909e1e08bac993e77d1781a777b386335669)
+    
+    Conflicts:
+    	src/init-ceph.in
+                different content of cmd variable
+
+commit c2c6d02591519dfd15ddcb397ac440322a964deb
+Author: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+Date:   Mon Jul 6 09:56:11 2015 +0200
+
+    tools: fix race condition in seq/rand bench
+    
+    Under certain conditions (like bench seq/rand -b 1024 -t 128) it is
+    possible that aio_read reads data into destination buffers before or
+    during memcmp execution, resulting in "[..] is not correct!" errors
+    even if actual objects are perfectly fine.
+    Also, moved latencty calculation around, so it is no longer affeted
+    by memcmp.
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+    
+    Conflicts:
+    	src/common/obj_bencher.cc
+
+commit a619b621b0a7c670eeaf163d9e2b742d13c9f517
+Author: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+Date:   Wed May 20 12:41:22 2015 +0200
+
+    tools: add --no-verify option to rados bench
+    
+    When doing seq and rand read benchmarks using rados bench, a quite large
+    portion of cpu time is consumed by doing object verification. This patch
+    adds an option to disable this verification when it's not needed, in turn
+    giving better cluster utilization. rados -p storage bench 600 rand scores
+    without --no-verification:
+    
+    Total time run:       600.228901
+    Total reads made:     144982
+    Read size:            4194304
+    Bandwidth (MB/sec):   966
+    Average IOPS:         241
+    Stddev IOPS:          38
+    Max IOPS:             909522486
+    Min IOPS:             0
+    Average Latency:      0.0662
+    Max latency:          1.51
+    Min latency:          0.004
+    
+    real    10m1.173s
+    user    5m41.162s
+    sys     11m42.961s
+    
+    Same command, but with --no-verify:
+    
+    Total time run:       600.161379
+    Total reads made:     174142
+    Read size:            4194304
+    Bandwidth (MB/sec):   1.16e+03
+    Average IOPS:         290
+    Stddev IOPS:          20
+    Max IOPS:             909522486
+    Min IOPS:             0
+    Average Latency:      0.0551
+    Max latency:          1.12
+    Min latency:          0.00343
+    
+    real    10m1.172s
+    user    4m13.792s
+    sys     13m38.556s
+    
+    Note the decreased latencies, increased bandwidth and more reads performed.
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+    (cherry picked from commit ca6abca63de813c83a6960f83624be8e1a86a1f8)
+    
+    Conflicts:
+    	src/common/obj_bencher.cc
+    	src/common/obj_bencher.h
+
+commit d06463604cb8daeda288e824e8812352c0d6a7d9
+Merge: ec35347 609f256
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Dec 2 20:48:00 2015 +0100
+
+    Merge pull request #6527 from theanalyst/wip-12856-hammer
+    
+    rgw: missing handling of encoding-type=url when listing keys in bucket
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit ec35347d8fd7a0eb175710e6111a9196746e278c
+Merge: 487dc34 1b06f03
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Dec 2 20:47:43 2015 +0100
+
+    Merge pull request #6491 from SUSE/wip-13716-hammer
+    
+    rgw:swift use Civetweb ssl can not get right url
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 487dc3436a8fabd1b1a31563071c907bdf18f525
+Merge: 18d3ba3 99b4d1d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Dec 2 20:47:27 2015 +0100
+
+    Merge pull request #6351 from theanalyst/wip-13538-hammer
+    
+    rgw: orphan tool should be careful about removing head objects
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 18d3ba30323a44c1cc45bf4e870b8d9aeb5c3b0b
+Merge: 36e4db3 6d89f4b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Dec 2 20:47:09 2015 +0100
+
+    Merge pull request #6349 from theanalyst/wip-13540-hammer
+    
+    rgw: get bucket location returns region name, not region api name
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 36e4db36f5ec9418672bbfe0dce6ae757f697efe
+Merge: 8c4145e db1cbe7
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Dec 2 20:46:49 2015 +0100
+
+    Merge pull request #5910 from kmroz/wip-hammer-backport-content-type
+    
+    rgw: backport content-type casing
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit ac05617b246d3c2a329b1b99b0371e3f6b8541e9
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 17 21:42:53 2015 -0400
+
+    osd: fix send_failures() locking
+    
+    It is unsafe to check failure_queue.empty() without the lock.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b3ca828ae8ebc9068073494c46faf3e8e1443ada)
+
+commit 8c4145ecc4a68accdb2120889fd933e8f6630dba
+Merge: 4804eec 112c686
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Nov 25 08:47:10 2015 -0800
+
+    Merge pull request #6587 from theanalyst/wip-13758-hammer
+    
+    common: pure virtual method called
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 4804eec095959c3747b13d29224b2cfa5b7d198d
+Merge: 9b97e4f e693755
+Author: Daniel Gryniewicz <dang@fprintf.net>
+Date:   Wed Nov 25 10:49:15 2015 -0500
+
+    Merge pull request #6352 from theanalyst/wip-13536-hammer
+    
+    rgw: bucket listing hangs on versioned buckets
+
+commit 9b97e4f6a41bb3fe7bae92b71ae266361022cf5c
+Merge: 5a9e0a7 0378445
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 25 16:48:09 2015 +0100
+
+    Merge pull request #6589 from theanalyst/wip-13693-hammer
+    
+    osd: bug with cache/tiering and snapshot reads
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 5a9e0a722c654b83fc483e083ddb3035281f5397
+Merge: 3047f2b a322317
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 25 16:22:10 2015 +0100
+
+    Merge pull request #6585 from theanalyst/wip-13753-hammer
+    
+    rbd: avoid re-writing old-format image header on resize
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 3047f2bc50d5d26e70e832bd427f15073bf7733c
+Merge: 407bd02 9c33dcc
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Nov 23 19:57:03 2015 +0100
+
+    Merge pull request #6586 from theanalyst/wip-13755-hammer
+    
+    rbd: QEMU hangs after creating snapshot and stopping VM
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 407bd0281582ebb3b50ce51bdb48244e22463c8f
+Merge: c3c400f bddbda1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Nov 23 17:32:21 2015 +0100
+
+    Merge pull request #6588 from theanalyst/wip-13770-hammer
+    
+    Objecter: pool op callback may hang forever.
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit c3c400f68d9c2593acd9485d1214d14af4e930d0
+Merge: d116959 a52f7cb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Nov 23 15:13:24 2015 +0100
+
+    Merge pull request #6430 from SUSE/wip-13654-hammer
+    
+    crush: crash if we see CRUSH_ITEM_NONE in early rule step
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 3f1292e117ab1cdcaef3b24ee33854f7be142795
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Sep 3 20:30:50 2015 +0200
+
+    ceph.spec.in: fix License line
+    
+    This is closer to my reading of
+    https://github.com/ceph/ceph/blob/master/COPYING than the previous version.
+    
+    http://tracker.ceph.com/issues/12935 Fixes: #12935
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit ef7418421b3748c712019c8aedd02b8005c1e1ea)
+
+commit d116959442f67c8f36898ca989b490ca84a609c6
+Merge: 12be099 74203b8
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Nov 23 09:28:09 2015 +0100
+
+    Merge pull request #6420 from SUSE/wip-13637-hammer
+    
+    FileStore: potential memory leak if getattrs fails.
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 12be099a7b0f2692c167af96928e939b2227160c
+Merge: 39c70d4 2052187
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sat Nov 21 09:10:27 2015 +0100
+
+    Merge pull request #6528 from theanalyst/wip-13695-hammer
+    
+    init-rbdmap uses distro-specific functions
+    
+    Reviewed-by: Boris Ranto <branto@redhat.com>
+
+commit 39c70d4364a1be39f7c393847417f44279b4364c
+Merge: 65aeba0 9643ee6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sat Nov 21 09:07:13 2015 +0100
+
+    Merge pull request #6499 from SUSE/wip-13692-hammer
+    
+    osd: do not cache unused memory in attrs
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 65aeba0b2f165d893d06df7e9bbe25f989edafe9
+Merge: 24d6698 394fbfc
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sat Nov 21 08:59:25 2015 +0100
+
+    Merge pull request #6143 from dachary/wip-13340-hammer
+    
+    small probability sigabrt when setting rados_osd_op_timeout
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 24d66981d351848c44ae9e0395dad392f852be2b
+Merge: 5d204db 171fee1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sat Nov 21 08:58:11 2015 +0100
+
+    Merge pull request #5773 from tchaikov/wip-12835-hammer
+    
+    mon: map_cache can become inaccurate if osd does not receive the osdmaps
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit a81bcf723c1099f2bea5daf8b01b7d9853de323a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 15 00:15:54 2015 -0400
+
+    ceph_context: remove unsafe cast for singletons
+    
+    It was previously assumed that a CephContext singleton would
+    inherit from CephContext::AssociatedSingletonObject, but it was
+    not enforced.  This could result in unknown behavior when the
+    singleton is destroyed due to the implied virtual destructor.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit fb62c78637d7092f48871d943282f45029bd6d29)
+
+commit d50d7b2fa3751e8520694ee75eefe5ae56e57267
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Sep 30 22:53:28 2015 -0400
+
+    osd: conditionally initialize the tracepoint provider
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0009f343a5daf28358a669836d95660c0d9068d8)
+
+commit bf34b3657339dc40c7939fcdddaf2b7ae78c82ad
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Sep 30 22:37:05 2015 -0400
+
+    librados: conditionally initialize the tracepoint provider
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6368c281b53d6175a564725b85516de4b6ae54de)
+
+commit afc4f176f8371f0e61ce3463602f8a355b9283b9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Sep 30 22:10:10 2015 -0400
+
+    librbd: conditionally initialize the tracepoint provider
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1a6eea95bd1ccef53f01cf53e493e652faa203ee)
+    
+    Conflicts:
+    	src/librbd/librbd.cc: trivial resolution
+
+commit c82f93915875875b21ed779e7aaf297b8fbcdca1
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Sep 30 19:17:08 2015 -0400
+
+    common: new tracing config options
+    
+    Config options to enable LTTng-UST tracepoint providers for
+    the OSD, OSD objectstore, librados, and librbd.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2ccef89badb1f7a63721a0bf48d05eb4c2d0a97a)
+    
+    Conflicts:
+    	src/common/config_opts.h: trivial resolution
+
+commit 2168c151066e0cea86284ffdf947a353f3b323d5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Sep 30 18:49:32 2015 -0400
+
+    tracing: merge tracepoint provider makefiles into main makefile
+    
+    This results in the tracepoint provider shared libraries being
+    placed in the library path for unittests.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e11d8d68da27a1c224aa55fc40a72ac6aaa0441c)
+    
+    Conflicts:
+    	configure.ac: trivial resolution
+    	src/Makefile.am: trivial resolution
+
+commit d02beff1cf7650fe6e57cdafe64dcbee2631ed52
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Sep 30 15:13:24 2015 -0400
+
+    tracing: dynamic tracepoint provider helper
+    
+    The TracepointProvider class is a configuration observer.  When
+    tracing is enabled, it will dynamically load the associated
+    tracepoint provider.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b3d02cc21b4cc40a194f84a9bbbf82cf9e7956d1)
+    
+    Conflicts:
+    	src/common/Makefile.am: trivial resolution
+
+commit e53d66e42b1c3aea47832f7e8983284ec45d9efa
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 29 14:33:48 2015 -0400
+
+    packaging: add new tracepoint probe shared libraries
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a7ed8e161b27c7852bf0f3a23a977ebd39137943)
+
+commit bb7c0f501918b75dee97052faeafb1d6db61bd0a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 29 14:30:43 2015 -0400
+
+    ceph.spec.in: add new tracepoint probe shared libraries
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f4feee2f9dcd7b94476701b66d1f0bdf6fb6e0c2)
+    
+     Conflicts:
+    	ceph.spec.in: trivial resolution
+
+commit e1da27134e4c9b4a2881aca664818598e5b2125b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 29 14:13:46 2015 -0400
+
+    lttng: move tracepoint probes to dynamic libraries
+    
+    LTTng-UST initializes itself at program load, which means it is
+    currently always enabled.  This can lead to issues with SElinux
+    and AppArmor which might restrict access to the necessary device
+    files.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4a5305e41677885d45b2b74299113a3d00189c66)
+    
+     Conflicts:
+    	src/test/Makefile-client.am: trivial resolution
+    	src/tracing/Makefile.am: trivial resolution
+
+commit 5d204db85b2ed84411cfabea9eb73aa04c6ce05a
+Merge: 1dbd4c6 8378aaf
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Nov 19 19:45:25 2015 +0100
+
+    Merge pull request #6580 from dachary/wip-13786-hammer
+    
+    rbd-replay-* moved from ceph-test-dbg to ceph-common-dbg as well
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 1dbd4c66307e615670c7569f107418876f8b977f
+Merge: 24a5dc0 634d7f6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Nov 19 19:45:03 2015 +0100
+
+    Merge pull request #6286 from dillaman/wip-13460-hammer
+    
+    rbd-replay-prep and rbd-replay improvements
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 24a5dc0a6b4f73c92c82e1933a1924db3f8e1090
+Merge: d27da2e 6a40e4f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Nov 19 19:41:20 2015 +0100
+
+    Merge pull request #5790 from SUSE/wip-12932-hammer
+    
+    Miscellaneous spec file fixes
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit d27da2ee8fa3e074fdd91d414df2d59978db9497
+Merge: 7d21127 8358fb8
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Thu Nov 19 19:13:11 2015 +0800
+
+    Merge pull request #6644 from dachary/wip-13812-upgrade
+    
+    revert: osd: use GMT time for hitsets
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 8358fb8946d8809d695092baa4a6abf5d5b5e265
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 18 18:08:58 2015 +0100
+
+    revert: osd: use GMT time for hitsets
+    
+    "Merge pull request #5825 from tchaikov/wip-12848-hammer"
+    
+    This reverts commit 39544718dc2f09bcfdc632ac72fd2a3cda87687e, reversing
+    changes made to 4ad97162026e1eb6e6e948ddf3eb39f711431e45.
+    
+    http://tracker.ceph.com/issues/13812 Fixes: #13812
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+
+commit 4420929eacb67d17635fed53f15471a28942f31d
+Author: brian <bjfelton@gmail.com>
+Date:   Wed Sep 23 09:49:36 2015 -0500
+
+    rgw: fix modification to index attrs when setting acls
+    
+    Fixes: #12955
+    
+     - add ACL change after initial population
+     - populate bufferlist with object attributes
+    
+    Signed-off-by: Brian Felton <bjfelton@gmail.com>
+    (cherry picked from commit 7496741ebbd75b74d5ffeca5341cccb2318176e6)
+
+commit 7d21127f433afa2d9172954e7b8ff47c40d2d62b
+Merge: 6930601 1448915
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Nov 17 10:24:45 2015 +0100
+
+    Merge pull request #6402 from SUSE/wip-13621-hammer
+    
+    CephFS restriction on removing cache tiers is overly strict
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 693060102594a2c505bed1e7e274731b554e9179
+Merge: c2daf09 3f33ce6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Nov 16 22:06:51 2015 +0100
+
+    Merge pull request #6354 from theanalyst/wip-13387-hammer
+    
+    librbd: reads larger than cache size hang
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit c2daf09fc6b8802240e13367a0a2f3349a7cca56
+Merge: 1ca72fb af734e6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Nov 16 19:48:07 2015 +0100
+
+    Merge pull request #6289 from dillaman/wip-13461-hammer
+    
+    librbd: invalidate object map on error even w/o holding lock
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 1ca72fbe6be0d664a3cbbd4356a8547c3b749a03
+Merge: ca573fe 3e65730
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Nov 16 18:06:25 2015 +0100
+
+    Merge pull request #6153 from dachary/wip-13205-hammer
+    
+    ReplicatedBackend: populate recovery_info.size for clone (bug symptom is size mismatch on replicated backend on a clone in scrub)
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit ca573fe7ac5a699b77cacb8fedfa50fffc77dd52
+Merge: fdb3446 7161a2c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Nov 16 18:06:12 2015 +0100
+
+    Merge pull request #6158 from dachary/wip-13336-hammer
+    
+    osd: we do not ignore notify from down osds
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit fdb3446194743135b78a65c30a8285d001e563b6
+Merge: f017566 0ad9521
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Nov 16 18:06:01 2015 +0100
+
+    Merge pull request #6335 from Abhishekvrshny/wip-13488-hammer
+    
+    object_info_t::decode() has wrong version
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit f017566823af70323b7d4cbe1f7b3b761378d07e
+Merge: a79acd4 9f3aebe
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Nov 16 18:05:39 2015 +0100
+
+    Merge pull request #6401 from SUSE/wip-13620-hammer
+    
+    osd: pg stuck in replay
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 8378aaf3f9d5463fb61a19d601201dd63a884419
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Nov 13 19:10:28 2015 +0100
+
+    build/ops: rbd-replay moved from ceph-test-dbg to ceph-common-dbg
+    
+    http://tracker.ceph.com/issues/13785 Fixes: #13785
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit a0204dc47acc0c50223e449fd23a2fc97bfd111a)
+    
+        Conflicts:
+        	debian/control: << 0.94.5-2 is appropriate for hammer
+
+commit 03784457ab77617ddafc048b01044858b1e65bd3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 29 22:26:48 2015 +0800
+
+    osd: fix the snapshot reads of evicted tiering pool
+    
+    reset ssc->exsits in finish_ctx() if the ctx->cache_evict is true, and
+    the head is removed.
+    
+    Fixes: #12748
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b61f3e43f1c9d43daa0dedd7cbd0fe5787cbdfbb)
+
+commit bddbda121c8d52ba8e50abd0341f14e332c6af5a
+Author: xiexingguo <258156334@qq.com>
+Date:   Thu Oct 29 20:04:11 2015 +0800
+
+    Objecter: pool_op callback may hang forever.
+    
+    pool_op callback may hang forever due to osdmap update during reply handling.
+    Fixes: #13642
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    
+    (cherry picked from commit 00c6fa9e31975a935ed2bb33a099e2b4f02ad7f2)
+
+commit 112c686ffc98e3739c9944635f4044e2b34f210a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 7 12:11:13 2015 -0400
+
+    WorkQueue: new PointerWQ base class for ContextWQ
+    
+    The existing work queues do not properly function if added to a running
+    thread pool.  librbd uses a singleton thread pool which requires
+    dynamically adding/removing work queues as images are opened and closed.
+    
+    Fixes: #13636
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3e78b18b09d75626ca2599bac3b9f9c9889507a5)
+    
+    Conflicts:
+    	src/common/WorkQueue.h
+    Trivial merge conflict at class `ContextWQ` initialization
+
+commit 9c33dccaad9a4cdd46e9ecfa1e3ba6c03d95885a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 9 11:22:24 2015 -0500
+
+    librbd: fixed deadlock while attempting to flush AIO requests
+    
+    In-flight AIO requests might force a flush if a snapshot was created
+    out-of-band.  The flush completion was previously invoked asynchronously,
+    potentially via the same thread worker handling the AIO request. This
+    resulted in the flush operation deadlocking since it can't complete.
+    
+    Fixes: #13726
+    Backport: infernalis, hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit bfeb90e5fe24347648c72345881fd3d932243c98)
+
+commit b3b7877f9b4b3f43acab09d0dd6ee971b6aa1c29
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 9 10:48:10 2015 -0500
+
+    tests: new test case to catch deadlock on RBD image refresh
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a9729d9553e7fb925509cad8d388cf52a9fede9c)
+
+commit a3223173f85c42147ff4ced730beffe85146a4ed
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 2 16:50:19 2015 -0500
+
+    librbd: resize should only update image size within header
+    
+    Previously, the whole RBD image format 1 header struct was
+    re-written to disk on a resize operation.
+    
+    Fixes: #13674
+    Backport: infernalis, hammer, firefly
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d5be20b6d4646284571568ab28cbf45b0729390b)
+
+commit 6a40e4f19d7b4cd45a25161303c7363e96fe799e
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Sep 29 10:39:16 2015 +0200
+
+    ceph.spec.in: lttng in SLES12 only
+    
+    Over in the SUSE sector, we are trying to enable the SLE_12 and openSUSE_13.2
+    build targets. The lttng/babeltrace stuff is currently available only in
+    SLE_12.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit bf9ca1eb107c4462d8768faf3ff4db7972384ffd)
+    
+    Conflicts:
+    	ceph.spec.in
+                trivial resolution
+
+commit e508a44485366557ac8a280de35584f3b5edf720
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Jul 15 12:54:20 2015 +0200
+
+    ceph.spec.in: fix lttng/babeltrace conditionals
+    
+    lttng and babeltrace are build dependencies for rbd-replay-prep. Make
+    sure the right package names are used. Enable for SLE12, as well as
+    for openSUSE 13.1 and higher.
+    
+    Move the BuildRequires out of the ceph-test subpackage and into the
+    distro-conditional dependencies section.
+    
+    Make ordering of BuildRequires a little more alphabetical.
+    
+    http://tracker.ceph.com/issues/12360 Fixes: #12360
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit e267128ce22d8b8cd83f6d6d82f24f496600e678)
+    
+    Conflicts:
+    	ceph.spec.in
+                Dropped Requires: xmlstarlet in ceph-test that had inadvertently
+                been grabbed from d2cc2b1
+
+commit 19c9546b79f506d0b4ee005a138a77f120c629e8
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Sep 8 10:41:02 2015 -0600
+
+    packaging: move rbd-replay* to ceph-common
+    
+    The rbd-replay* utilities are useful for Ceph users with RBD clients.
+    Currently the rbd-replay* utilities ship in the "ceph-test" package, and
+    we intend this ceph-test package for Ceph developers and contributors,
+    not normal users.
+    
+    Move the rbd-replay* utilities to "ceph-common".
+    
+    http://tracker.ceph.com/issues/12994 Fixes: #12994
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 96486fd6f284ca3ab67f4f94631896d41c529e85)
+    
+    Conflicts:
+    	ceph.spec.in
+                50a33dea has not been backported to hammer
+    	debian/ceph-test.install
+                50a33dea has not been backported to hammer
+    	debian/control
+                different ceph-test Replaces: and Breaks: version
+
+commit a79acd41187e6b049432bdc314f192e3fbb560a3
+Merge: 3d61493 31b7864
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 11 11:59:36 2015 +0100
+
+    Merge pull request #6213 from SUSE/wip-13425-hammer
+    
+    wrong conditional for boolean function KeyServer::get_auth()
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 3d61493337d481dec5139ccb6172710a118b2009
+Merge: 147f437 7ffd072
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 11 06:59:55 2015 +0100
+
+    Merge pull request #6336 from Abhishekvrshny/wip-13535-hammer
+    
+    LibRadosWatchNotify.WatchNotify2Timeout
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 147f437279e72c0766a126653e3db53abb73a4ac
+Merge: 1970e61 9085c82
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 11 06:59:42 2015 +0100
+
+    Merge pull request #6391 from SUSE/wip-13590-hammer
+    
+    mon: should not set isvalid = true when cephx_verify_authorizer return false
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 1970e618209a242fc0bd3c707dfdfb1a886d6a2c
+Merge: fb83704 ade0f1a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 11 06:58:55 2015 +0100
+
+    Merge pull request #6398 from SUSE/wip-13588-hammer
+    
+    OSD::build_past_intervals_parallel() shall reset primary and up_primary when begin a new past_interval.
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 3d3595f86083a0f9847585eae4cffb8a82c816d4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 27 10:13:27 2015 -0400
+
+    krbd: remove deprecated --quiet param from udevadm
+    
+    This parameter has been removed since systemd 213, so this
+    effects Fedora 21+, Debian Jessie, and potentially future
+    releases of RHEL 7.
+    
+    Fixes: #13560
+    Backport: hammer, infernalis
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4300f2a9fe29627eea580564ff2d576de3647467)
+
+commit 4d81cd19087e049bf3c2fe0d10de9215852ab51d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 27 10:12:34 2015 -0400
+
+    run_cmd: close parent process console file descriptors
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f46f7dc94139c0bafe10361622416d7dc343d31f)
+
+commit fb83704d5db81aad95c61a61be52d07ce2d4d2b5
+Merge: 3954471 0742177
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Nov 10 20:24:35 2015 +0100
+
+    Merge pull request #6199 from liewegas/wip-randomize-scrub-hammer
+    
+    osd: randomize scrub times
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 39544718dc2f09bcfdc632ac72fd2a3cda87687e
+Merge: 4ad9716 6a4734a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Nov 10 20:23:23 2015 +0100
+
+    Merge pull request #5825 from tchaikov/wip-12848-hammer
+    
+    ReplicatedPG::hit_set_trim osd/ReplicatedPG.cc: 11006: FAILED assert(obc)
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 2052187929e059a25f6a3baf67329f7ce0bf6d8a
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri Oct 23 15:31:27 2015 +0200
+
+    init-rbdmap: Rewrite to use logger + clean-up
+    
+    This patch rewrites the init-rbdmap init script so that it uses logger
+    instead of the log_* functions. The patch also fixes various smaller
+    bugs like:
+    * MAP_RV was undefined if mapping already existed
+    * UMNT_RV and UMAP_RV were almost always empty (if they succeeded) ->
+      removed them
+    * use of continue instead RET_OP in various places (RET_OP was not being
+      checked after the switch to logger messages)
+    * removed use of DESC (used only twice and only one occurrence actually
+      made sense)
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit c567341e98fffbe39177f951511a7698f88abf5f)
+
+commit f972f38445bc7a6efe122a5f9fc2ba7658625e26
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri Oct 23 16:39:16 2015 +0200
+
+    ceph.spec.in: We no longer need redhat-lsb-core
+    
+    Drop the redhat-lsb-core dependency as it is no longer necessary on
+    fedora/rhel.
+    
+    The other two init scripts do not use redhat-lsb-core either. The
+    init-ceph.in conditionally requires /lib/lsb/init-functions and does not
+    use any of the functions defined in that file (at least not directly).
+    The init-radosgw file includes /etc/rc.d/init.d/functions on non-debian
+    platforms instead of /lib/lsb/init-functions file so it does not require
+    redhat-lsb-core either.
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 15600572265bed397fbd80bdd2b7d83a0e9bd918)
+    
+    Conflicts:
+    	ceph.spec.in
+    Merge conflict because there were other commits that introduced systemd
+    & SUSE specific bits in the master which havent yet reached
+    hammer. Since this commit only removed redhat-lsb-core from *el distros
+    removing the specific lines
+
+commit 609f256a00f6ee18564ee055dd79f8293217940c
+Author: Jeff Weber <jweber@cofront.net>
+Date:   Wed Aug 5 19:06:46 2015 -0400
+
+    rgw: implement s3 encoding-type for get bucket
+    
+    This change introduces handling for the encoding-type request
+    parameter on the get bucket operation. An object key may contain
+    characters which are not supported in XML. Passing the value "url" for
+    the encoding-type parameter will cause the key to be urlencoded in the
+    response.
+    
+    Fixes: #12735
+    Signed-off-by: Jeff Weber <jweber@cofront.net>
+    (cherry picked from commit 180ca7b56ba91a3995c76dac698aa4ee31d9a2ce)
+
+commit 394fbfcc40d77ad6c8b3301c5ff2630c052107cd
+Author: Ruifeng Yang <yangruifeng.09209@h3c.com>
+Date:   Fri Sep 25 10:18:11 2015 +0800
+
+    Objecter: maybe access wild pointer(op) in _op_submit_with_budget.
+    
+    look at "after giving up session lock it can be freed at any time by response handler" in _op_submit,
+    so the _op_submit_with_budget::op maybe is wild after call _op_submit.
+    
+    Fixes: #13208
+    Signed-off-by: Ruifeng Yang <yangruifeng.09209@h3c.com>
+    (cherry picked from commit 0635b1358354b19ae44105576f730381f3b5b963)
+
+commit 84068f8cea0452333f00f8b65230c00caa2e52ff
+Author: Ruifeng Yang <yangruifeng.09209@h3c.com>
+Date:   Fri Sep 25 12:42:28 2015 +0800
+
+    Objecter: repeated free op->ontimeout.
+    
+    repeated free op->ontimeout in SafeTimer::timer_thread::callback->complete
+    
+    Fixes: #13208
+    Signed-off-by: Ruifeng Yang <yangruifeng.09209@h3c.com>
+    (cherry picked from commit f1d8a8f577cee6d66f4dcffac667675f18145ebb)
+
+commit 3e657304dc41facd40b8cab7531180083d14d22a
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Aug 28 12:46:57 2015 -0700
+
+    ReplicatedBackend::prepare_pull: set recover_info.size for clones
+    
+    Fixes: #12828
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 1d3e46bedb9a8c0d2200e39f62f4e2c3337619f3)
+
+commit 4ad97162026e1eb6e6e948ddf3eb39f711431e45
+Merge: 92b2153 f1271ea
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Nov 10 10:34:01 2015 +0100
+
+    Merge pull request #6157 from dachary/wip-13040-hammer
+    
+    common/Thread:pthread_attr_destroy(thread_attr) when done with it
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 92b21532f64792da023188d9aa620d900c0245ca
+Merge: 1f02886 a17f2a9
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Nov 10 10:33:41 2015 +0100
+
+    Merge pull request #6155 from dachary/wip-13171-hammer
+    
+    objecter: cancellation bugs
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 1f02886d7edc10ff93074afc3d2bf4cf77c709d5
+Merge: 77df385 7d4b303
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Nov 10 10:33:19 2015 +0100
+
+    Merge pull request #6152 from dachary/wip-13233-hammer
+    
+    mon: include min_last_epoch_clean as part of PGMap::print_summary and PGMap::dump
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 77df385e23452f401f3c0c269e220446d288fb8e
+Merge: 469398f d8ca88d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Nov 10 10:33:01 2015 +0100
+
+    Merge pull request #6146 from dachary/wip-13337-hammer
+    
+    segfault in agent_work
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 469398f14b62af01068e65bcc5cc7c585a04e878
+Merge: d87d136 397042a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Nov 10 10:32:53 2015 +0100
+
+    Merge pull request #6145 from dachary/wip-13338-hammer
+    
+    filestore: fix peek_queue for OpSequencer
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit d87d1364fb8e345a1f85b65c2d4c7c80596ba5f9
+Merge: 717eff9 d4e4d85
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Nov 10 10:32:42 2015 +0100
+
+    Merge pull request #6144 from dachary/wip-13339-hammer
+    
+    mon: check for store writeablility before participating in election
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 717eff9a61818fde71ec5220998e9b5c14bbb3a9
+Merge: e133ddc e8cce08
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Nov 10 10:30:54 2015 +0100
+
+    Merge pull request #5891 from Abhishekvrshny/wip-13035-hammer
+    
+    requeue_scrub when kick_object_context_blocked
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit e133ddc622d3532b0182c208f0ebb9956daab44f
+Merge: c72306a 8ee93d5
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Nov 10 10:30:38 2015 +0100
+
+    Merge pull request #5890 from Abhishekvrshny/wip-13037-hammer
+    
+    hit set clear repops fired in same epoch as map change -- segfault since they fall into the new interval even though the repops are cleared
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit c72306a1dd3b193948d5df3371471797d4131cfe
+Merge: 5e8c9d9 2bd5d0b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Nov 10 10:21:07 2015 +0100
+
+    Merge pull request #6322 from sponce/WIP-13210-hammer
+    
+    tests : BACKPORT #13210 Fixed broken Makefiles after integration of ttng into rados
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit b62cac6c345df53bbda1c42a5336cffc8f0f9652
+Author: Sangdi Xu <xu.sangdi@h3c.com>
+Date:   Wed Sep 23 19:31:32 2015 +0800
+
+    rgw: fix swift API returning incorrect account metadata
+    
+    Fixes: #13140
+    
+    Fix the bug that swift account stat command returns doubled object count and bytes used
+    
+    Signed-off-by: Sangdi Xu <xu.sangdi@h3c.com>
+    (cherry picked from commit 66d19c78ba75b79190c81c95f94e7ef3084fda9e)
+
+commit 9643ee65a6e968e479ca33a102f5f575924a7ff0
+Author: Xinze Chi <xmdxcxz@gmail.com>
+Date:   Sun Aug 2 18:36:40 2015 +0800
+
+    bug fix: osd: do not cache unused buffer in attrs
+    
+    attrs only reference the origin bufferlist (decode from MOSDPGPush or
+    ECSubReadReply message) whose size is much greater than attrs in recovery.
+    If obc cache it (get_obc maybe cache the attr), this causes the whole origin
+    bufferlist would not be free until obc is evicted from obc cache. So rebuild
+    the bufferlist before cache it.
+    
+    Fixes: #12565
+    Signed-off-by: Ning Yao <zay11022@gmail.com>
+    Signed-off-by: Xinze Chi <xmdxcxz@gmail.com>
+    (cherry picked from commit c5895d3fad9da0ab7f05f134c49e22795d5c61f3)
+
+commit 5e8c9d967e0ade393e36893965ca8ddfaa317b48
+Merge: db0366d dba8b5b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Nov 9 08:38:31 2015 +0100
+
+    Merge pull request #5810 from Abhishekvrshny/wip-12948-hammer
+    
+    Heavy memory shuffling in rados bench
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 1b06f03b541983461469f8f53ef2a271b08d2f78
+Author: Weijun Duan <duanweijun@h3c.com>
+Date:   Thu Oct 29 21:46:02 2015 -0400
+
+    rgw:swift use Civetweb ssl can not get right url
+    
+    Fixes: #13628
+    
+    Signed-off-by: Weijun Duan <duanweijun@h3c.com>
+    (cherry picked from commit e0fd540bf441e2f8276cbd96c601a0539892efe2)
+
+commit db0366dd979cc0d401b1974c233e38dfe5b1b5d1
+Merge: 22dce75 a65c398
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Nov 5 13:12:11 2015 +0100
+
+    Merge pull request #5530 from SUSE/wip-12587-hammer
+    
+    FileStore calls syncfs(2) even it is not supported
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 22dce75181b254548226fc09decafac14dcf84a6
+Merge: ed153c1 d8ac510
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 4 19:38:53 2015 +0100
+
+    Merge pull request #5889 from Abhishekvrshny/wip-13042-hammer
+    
+    ThreadPool add/remove work queue methods not thread safe
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit ed153c13226138eec71e472d1e3c8481db9b9009
+Merge: ca8802f 8610de8
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 4 19:14:28 2015 +0100
+
+    Merge pull request #6151 from dachary/wip-13245-hammer
+    
+    client nonce collision due to unshared pid namespaces
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit ca8802fb0cb8603267ecfc7320fee7a679617b8b
+Merge: 19450b9 6e29e90
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 4 11:36:39 2015 +0100
+
+    Merge pull request #6159 from dachary/wip-13341-hammer
+    
+    ceph upstart script rbdmap.conf incorrectly processes parameters
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 19450b90e4c20556580d276bab7dafb23db1758f
+Merge: 18882c8 6849288
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 4 11:34:07 2015 +0100
+
+    Merge pull request #6154 from dachary/wip-13195-hammer
+    
+    should recalc the min_last_epoch_clean when decode PGMap
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 18882c81a71c4b517cb0eaa5a884637ea5cbe7a6
+Merge: de4f37b 3c1f7cb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 4 11:28:03 2015 +0100
+
+    Merge pull request #6132 from SUSE/wip-13307-hammer
+    
+    dumpling incrementals do not work properly on hammer and newer
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit de4f37bb0abc932c813983792e170de1bb40241b
+Merge: 1c632a7 24268cf
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 4 11:23:14 2015 +0100
+
+    Merge pull request #6097 from dillaman/wip-13045-hammer
+    
+    librbd: diff_iterate needs to handle holes in parent images
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 1c632a7ea7057522fb9df61994b5b3cf65416689
+Merge: bedb334 71a42f7
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 4 11:22:25 2015 +0100
+
+    Merge pull request #6060 from XinzeChi/wip-hammer-outdata-set
+    
+    bug fix: osd: avoid multi set osd_op.outdata in tier pool
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit bedb3344f6c802ea04a171dabd4fb596284af34a
+Merge: 0d3d819 cecd68d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 4 11:20:50 2015 +0100
+
+    Merge pull request #5897 from tchaikov/wip-12940-hammer
+    
+    IO error on kvm/rbd with an erasure coded pool tier
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 0d3d81951db61a97f6cb26d9f75e80a3400a2a37
+Merge: 381cfcc 7475a8f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 4 11:11:00 2015 +0100
+
+    Merge pull request #5787 from SUSE/wip-12923-hammer
+    
+    logrotate reload error on Ubuntu 14.04
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 381cfccf38bade28a001f3f24392c61896b3c711
+Merge: cad1cfb 86f88df
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 4 11:10:26 2015 +0100
+
+    Merge pull request #5766 from dachary/wip-12590-hammer
+    
+    ceph mds add_data_pool check for EC pool is wrong
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit cad1cfb088b095e8333c4a7d79ccefc2b2c4d782
+Merge: 9764da5 e8d6d5a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 4 11:06:18 2015 +0100
+
+    Merge pull request #5361 from ceph/wip-11786.hammer
+    
+    mon: MonitorDBStore: get_next_key() only if prefix matches
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 5105d50d1c20d465229189bbb24cdfb5d16cf7bc
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Nov 3 00:21:51 2015 +0100
+
+    tests: test/librados/test.cc must create profile
+    
+    Now that the create_one_ec_pool function removes the testprofile each
+    time it is called, it must create the testprofile erasure code profile
+    again for the test to use.
+    
+    http://tracker.ceph.com/issues/13664 Refs: #13664
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit a60342942b5a42ee04d59af77a6b904ce62eefc4)
+
+commit 302375068a59b86c50bda73ecefd99831ab52ea4
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Nov 2 20:24:51 2015 +0100
+
+    tests: destroy testprofile before creating one
+    
+    The testprofile erasure code profile is destroyed before creating a new
+    one so that it does not fail when another testprofile erasure code
+    profile already exists with different parameters.
+    
+    This must be done when creating erasure coded pools with the C++
+    interface, in the same way it's done with the C interface.
+    
+    http://tracker.ceph.com/issues/13664 Fixes: #13664
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 47abab9a6f182aa0abe5047c04402850379bcd6d)
+
+commit 3a7423dbe0a6af831d4c47acc98e6ced5ed5fe0c
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Nov 2 20:23:52 2015 +0100
+
+    tests: add destroy_ec_profile{,_pp} helpers
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit ab46d79bc09fc711fa35302f49eecac81a98519b)
+
+commit a52f7cb372339dffbeed7dae8ce2680586760754
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 13 09:55:01 2015 -0400
+
+    crush/mapper: ensure take bucket value is valid
+    
+    Ensure that the take argument is a valid bucket ID before indexing the
+    buckets array.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 93ec538e8a667699876b72459b8ad78966d89c61)
+
+commit 81d8aa14f3f2b7bf4bdd0b4e53e3a653a600ef38
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 27 20:55:26 2015 -0400
+
+    crush/mapper: ensure bucket id is valid before indexing buckets array
+    
+    We were indexing the buckets array without verifying the index was within
+    the [0,max_buckets) range.  This could happen because a multistep rule
+    does not have enough buckets and has CRUSH_ITEM_NONE
+    for an intermediate result, which would feed in CRUSH_ITEM_NONE and
+    make us crash.
+    
+    Fixes: #13477
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 976a24a326da8931e689ee22fce35feab5b67b76)
+
+commit 74203b806e8c268aa3edac8cfc7c4b393f7b1f27
+Author: xiexingguo <258156334@qq.com>
+Date:   Mon Oct 26 18:38:01 2015 +0800
+
+    FileStore: potential memory leak if _fgetattrs fails
+    
+    Memory leak happens if _fgetattrs encounters some error and simply returns.
+    Fixes: #13597
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    
+    (cherry picked from commit ace7dd096b58a88e25ce16f011aed09269f2a2b4)
+
+commit db1cbe700e41ee7a2db5ef10173c429283bc5a2a
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Fri Sep 4 01:07:48 2015 +0000
+
+    Fix casing of Content-Type header
+    
+    It turns out, despite the HTTP spec declaring that header field names
+    should be case-insensitive, some clients treat them wrongly, and
+    consider "Content-type" to not match "Content-Type".
+    
+    CyberDuck was one of those clients, now fixed upstream in
+    https://trac.cyberduck.io/ticket/8999
+    
+    To reduce future occurances of this bug, fix the casing of the
+    Content-Type header, to strictly comply with the HTTP specification (be
+    strict about what you send, and generous about what you receive).
+    
+    Fixes: #12939
+    Backport: infernalis, hammer, firefly
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit 1b9fbffdc24160251b96cec820d62fb2a12b6eab)
+
+commit 1448915e49bb6c8abc192053e6f2e8a380c4e92a
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Jun 1 13:55:22 2015 +0100
+
+    qa: update cephtool test for CephFS tier cases
+    
+    1. Creating a filesystem using a
+       readonly tier on an EC pool (should be forbidden)
+    2. Removing a tier from a replicated base pool (should
+       be permitted)
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit f9b11b0abe7c2b0dcdfc5802336be8a0426ccb3a)
+
+commit 14c36f0ccf103dea39b63e0e1b03082b99c42cb1
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Jun 1 13:54:25 2015 +0100
+
+    mon: forbid readonly cache tiers for CephFS
+    
+    For EC pools, we need a tier that is not just forwarding
+    writes, in order to provide the ops required
+    by CephFS.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 11e5faf3c1d4fa28a8b41e3a92f0f684a844e5f5)
+
+commit 31e59fbca0ba636fc57c208e612d7488a48a6287
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu May 7 14:23:37 2015 +0100
+
+    mon: refine check_remove_tier checks
+    
+    Fixes: #11504
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit a50c8f1f2ad8845c7f77110868f9376f2d1ff883)
+
+commit 9f3aebee16e256888b149fa770df845787b06b6e
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 22 13:57:37 2015 -0400
+
+    osd: fix requeue of replay requests during activating
+    
+    If the replay period expires while we are still in the activating
+    state, we can simply insert our list of requests at the front of
+    the waiting_for_active list.
+    
+    Fixes: #13116
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d18cf51d9419819cdda3782b188b010969288911)
+
+commit ade0f1a4285348c20fbae8e2a366fc332a7b7fbe
+Author: xiexingguo <258156334@qq.com>
+Date:   Tue Oct 13 14:04:20 2015 +0800
+
+    OSD:shall reset primary and up_primary fields when beginning a new past_interval.
+    
+    Shall reset primary and up_primary fields when we start over a new past_interval in OSD::build_past_intervals_parallel().
+    Fixes: #13471
+    Signed-off-by: xie.xingguo@zte.com.cn
+    (cherry picked from commit 65064ca05bc7f8b6ef424806d1fd14b87add62a4)
+
+commit 9085c820491f25a04ec02accc5098c1ab9b57311
+Author: yangruifeng <yangruifeng.09209@h3c.com>
+Date:   Mon Oct 19 08:08:12 2015 -0400
+
+    mon: should not set isvalid = true when cephx_verify_authorizer return false
+    
+    Fixes: #13525
+    Signed-off-by: Ruifeng Yang <yangruifeng.09209@h3c.com>
+    (cherry picked from commit c7f75b8f7c0a773148ec16141941efd00ee76626)
+
+commit 4875d05a3e265e80463a2a33a2c922affb0dad66
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Mon Mar 16 10:02:35 2015 -0600
+
+    ceph.spec.in: rm EOL Fedoras; add OBS RHEL5 instead
+    
+    Fedora 12 has been EOL for a long time. Remove the reference in the
+    RPM .spec file.
+    
+    Since RHEL 5 support for Ceph is a work in progress, we won't remove
+    this entire python_sitelib / python_sitearch conditional for now, since
+    those are still needed on RHEL 5.
+    
+    Add the rhel_version macro to make the conditional compatible with
+    SUSE's OBS.
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 353a3258968f76deaea4542d2165e9ab32b88de8)
+
+commit dfba6acce75eea02251e98e2d57e6fbb344241fa
+Author: Travis Rhoden <trhoden@redhat.com>
+Date:   Thu May 7 20:13:53 2015 -0400
+
+    packaging: Add rgw placeholder dirs in proper packages
+    
+    Automatically create /var/lib/ceph/radosgw with the
+    ceph-radosgw package, and /var/lib/ceph/bootstrap-rgw
+    via the ceph package.
+    
+    Signed-off-by: Travis Rhoden <trhoden@redhat.com>
+    (cherry picked from commit bf93128f664ac74383806530a1f974edde8df5b6)
+    
+    Conflicts:
+        ceph.spec.in
+            hammer uses %ghost
+
+commit 2387c8e0caef00e7181241e16549457d58a12192
+Author: Nathan Cutler <ncutler@suse.cz>
+Date:   Fri May 15 21:39:39 2015 +0200
+
+    ceph.spec.in: consolidate centos/rhel macros
+    
+    0%{?rhel} || 0%{?centos} can be simplified to just 0%{?rhel}, since
+    CentOS defines that macro for compatibility with RHEL.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.cz>
+    (cherry picked from commit 46404dd381e240634d224e77adc69f0ba4807045)
+    
+    Conflicts:
+    	ceph.spec.in
+                https://github.com/ceph/ceph/pull/4068 has not been backported
+
+commit 7c58d312db2aa42088f419dbfa7e841d897fdad6
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jun 25 16:55:10 2015 +0200
+
+    packaging: make logrotate a hard dependency across all distros
+    
+    Given ceph's reputation for voluminous logs, it makes sense for
+    logrotate to be a hard dependency for all distros.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 0734cd1ec3ca3bd012686dc1bdda675909224a49)
+
+commit 61acc4e20ca58e11b9aba1efcdca78d1d7ddd2f7
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jul 9 12:05:24 2015 +0200
+
+    ceph.spec.in: add missing -%{release}
+    
+    We have it everywhere else and I can't think of any reason why
+    it should be omitted here.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit d952d59ae95d5f66aa4b98fd7aa4f4ecb36a4848)
+
+commit 20590d367226ea01c9d4d95de15cfd1aa354d02b
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jun 26 20:21:59 2015 +0200
+
+    ceph.spec.in: remove duplicate BuildRequires: sharutils
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit c418bd9615e6f23f3c14ae9e71003c66d31d4b78)
+
+commit 6b306eba829b1ec9822b24243ee7c021aab3d738
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Jun 15 12:58:00 2015 +0200
+
+    ceph.spec.in: fix python-flask dependency for SUSE
+    
+    In SLE and openSUSE, the package is called python-Flask with an upper-case F.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 39c28b9faf0fb069d91ac069a4d8df7bf03fb490)
+    
+    Conflicts:
+    	ceph.spec.in
+                add missing SUSE-specific build deps
+
+commit 7a2cbcb0a4efa365b6d92ec58d7acf4a66e9d2b1
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jun 25 18:55:39 2015 +0200
+
+    ceph.spec.in: clarify two important comments
+    
+    First, the terms "common" and "specific" are vague. Second,
+    "common" can easily be confused with the ceph-common subpackage.
+    
+    Fix this by rephrasing to "distro-unconditional dependencies" and
+    "distro-conditional dependencies", respectively.
+    
+    Third, move the "distro-unconditional dependencies" header so it
+    is above the part where these dependencies are actually defined.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit dd212fd1d367f70c61f65a97599959668b44aa53)
+
+commit dfe27c6ba05d49bb73854754b9d5c02cf1c7b1ef
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Wed Jun 24 16:37:58 2015 -0600
+
+    ceph.spec.in: rm reference to EOL Fedoras (< 20)
+    
+    Fedora 19 is EOL, so there's no need to refer to Fedoras less than 20.
+    "%{fedora}" effectively implies "Fedora 20 and above".
+    
+    Reported-by: Ira Cooper <ira@samba.org>
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit f3e4a91c458ca7b11706fe4608cb739499f5d3cc)
+    
+    Conflicts:
+    	ceph.spec.in
+               added xmlstarlet dependency to ceph-test package
+
+commit 371b64ef4aa786db3fd29bc84ff98e63000a859b
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Wed Jun 24 16:39:30 2015 -0600
+
+    ceph.spec.in: package rbd-replay-prep on all Fedoras
+    
+    This reverts the change in commit
+    85517d611b7bf4cb6cbffcd2c65303be0d038264. Since we BuildRequire:
+    libbabeltrace-devel, autoconf will see that babeltrace is available
+    during the build, and make will build/install the rbd-replay-prep
+    utility.
+    
+    This change also simplifies Fedora selection logic, because Fedora 19 is
+    EOL, so "%{fedora}" implies "Fedora 20 and above".
+    
+    Reported-by: Ira Cooper <ira@samba.org>
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 2db9480fa2646de776f0076d82747e202e574ddd)
+
+commit 82425b33f712047442d9d29c5a4349818b44b40a
+Author: Boris Ranto <branto@redhat.com>
+Date:   Tue Jun 16 23:07:04 2015 +0200
+
+    ceph.spec.in: rbd-replay-prep is not being built on f21+
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 85517d611b7bf4cb6cbffcd2c65303be0d038264)
+
+commit 53e2f702681dc08ddd0414b116e51de6488f42ca
+Author: Owen Synge <osynge@suse.com>
+Date:   Mon Jun 8 17:48:55 2015 +0200
+
+    ceph.spec.in:BuildRequires sharutils
+    
+    The uudecode binary is used to build Java-related components, and
+    uudecode is provided by the sharutils package on all supported
+    RPM platforms. When building with "--without=cephfs_java",
+    sharutils is not needed.
+    
+    Thanks to Nathan Cutler <ncutler@suse.cz> for going into the
+    details with me.
+    
+    On OBS without this patch we get the error message:
+    
+    [  170s] -----------------------------------------------------------------
+    [  170s] ----- building ceph.spec (user abuild)
+    [  170s] -----------------------------------------------------------------
+    [  170s] -----------------------------------------------------------------
+    [  170s] + exec rpmbuild -ba --define '_srcdefattr (-,root,root)' --nosignature --define 'disturl obs://build.opensuse.org/home:osynge:ceph:wip:wip_obs_fedora/Fedora_20/c0bbbc1e62228ca956ac3d367edc4fba-master' /home/abuild/rpmbuild/SOURCES/ceph.spec
+    [  170s] error: Failed build dependencies:
+    [  170s]    sharutils is needed by ceph-1:2+git.1435043747.c1bd02c-1.1.x86_64
+    
+    With this patch we can build fedora 22 and fedora 20 rpms fine.
+    
+    Signed-off-by: Owen Synge <osynge@suse.com>
+    (cherry picked from commit 43c1784640538b813f0575c50f721ac911a30b3e)
+    
+    Conflicts:
+    	ceph.spec.in
+                _with_systemd -> trivial resolution
+
+commit a84376c68499845de01f9e3b194b16a698125b6a
+Author: Owen Synge <osynge@suse.com>
+Date:   Wed Jun 10 12:11:50 2015 +0200
+
+    ceph.spec.in: fix:Add missing directories breaking build
+    
+    SUSE builds on OBS are failing with the missing dir entries:
+    
+        /usr/share/ceph
+        /usr/lib/ceph
+    
+    On suse these correspond to:
+    
+        %dir %{_datarootdir}/ceph/
+        %dir %{_libexecdir}/ceph/
+    
+    Signed-off-by: Owen Synge <osynge@suse.com>
+    (cherry picked from commit f1ff3e4dcb5dd7eddd2c346a05cfa78d860a5ce6)
+
+commit 171fee1b82d2675e364da7f96dfb9dd286d9b6e6
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jul 21 00:33:19 2015 +0800
+
+    mon: track osd_epoch of all MonSessions
+    
+    previously, we only track the osd_epoch for OSD peers. but other
+    MonClients who receives osdmaps can also benefit from osd_epoch.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit de43a02e06650a552f048dc8acd17f255126fed9)
+    
+    Conflicts:
+    	src/mon/OSDMonitor.cc:
+    		do not assume that the MonSession is always available when
+            OSDMonitor::send_incremental() is called. as when the
+            connection to the mon client is dropped, the Monitor is
+            aware of this, but not the OSDMonitor.
+
+commit cc7da674965c3f758469e18788dc0e106e54d5bc
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jul 21 00:24:52 2015 +0800
+
+    mon: track osd_epoch in MonSession
+    
+    * remove osd_epoch<osd, epoch> from OSDMonitor
+    * add osd_epoch to MonSession to track the latest osdmap epoch
+      OSDMonitor sends to a mon client
+    * do not remove osd_epoch entries if an OSD is down, or
+      max_osd > osd_id
+    
+    Fixes: #10930
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit c05753eacc26e90b2e3b56e641a71bffd5b39bd0)
+
+commit 3f33ce6132d5ce8b3454f30fbe14471ece39951d
+Author: Lu Shi <shi.lu@h3c.com>
+Date:   Thu Sep 24 16:02:41 2015 +0800
+
+    librbd:reads larger than cache size hang.
+    
+    Fixes:#13164
+    
+    Signed-off-by: Lu Shi <shi.lu@h3c.com>
+    (cherry picked from commit 9c8200bb5d1ac9359803a182df03298b565b8479)
+
+commit 5e5b512189b809bef503c34b0a19e30ed81cfbfa
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed Aug 19 20:32:39 2015 +0200
+
+    rgw: url_decode values from X-Object-Manifest during GET on Swift DLO.
+    
+    Fixes: #12728
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 69cf089596a0e5d4da2826b5c276306316a00690)
+
+commit e693755a55e4da681a1947b1e1eb3bb537436cc0
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 22 14:53:42 2015 -0700
+
+    cls_rgw: fix bucket listing when dealing with invisible entries
+    
+    Fixes: #12913
+    
+    The bucket listing logic was broken when dealing with invisible entries.
+    Need to read keys until we can fetch all the requested entries, or
+    determine that there are no more entries. Anything else requires
+    a change to the protocol.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit ef9730515e001d77637a77eb89950f70fd5e7f5a)
+
+commit 99b4d1df3f6867e5ff0dd74691203737f6de6974
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 22 15:49:27 2015 -0700
+
+    rgw: orphan tool shouldn't clean up head objects
+    
+    Fixes: #12958
+    
+    Head objects are mutable, so removing them can race with object removal
+    and a later recreation, so we might end up cleaning them up when we don't
+    need to.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 030f697d196df9963ecf4f05f727da4798c583dc)
+
+commit 6d89f4ba85063debe96734e9fdfcba247e770206
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 12 11:55:40 2015 -0700
+
+    rgw: location constraints should return api name
+    
+    Fixes: #13458
+    The S3 location constraints request should return the api name,
+    not the region name.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 03078bab6ee8c617025902dbe52ec19e64653c33)
+
+commit 2bd5d0b835b546c80b5af503d2b9538835c768fd
+Author: Sebastien Ponce <sebastien.ponce@cern.ch>
+Date:   Tue Oct 20 09:22:16 2015 +0200
+
+    radosstriper : Fixed broken Makefiles after integration of lttng
+    
+    Fixes: #13210, backport of #6322 into hammer
+    
+    Signed-off-by: Sebastien Ponce <sebastien.ponce@cern.ch>
+
+commit 7ffd072a8cc7e114d4d674d30333fea4db04dcdd
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 1 14:50:34 2015 -0400
+
+    osdc/Objecter: distinguish between multiple notify completions
+    
+    We may send a notify to the cluster multiple times due to OSDMap
+    changes.  In some cases, earlier notify attempts may complete with
+    an error, while later attempts succeed.  We need to only pay
+    attention to the most-recently send notify's completion.
+    
+    Do this by making note of the notify_id in the initial ACK (only
+    present when talking to newer OSDs).  When we get a notify
+    completion, match it against our expected notify_id (if we have
+    one) or else discard it.
+    
+    This is important because in some cases an early notify completion
+    may be an error while a later one succeeds.
+    
+    Note that if we are talking to an old cluster we will simply not record a
+    notify_id and our behavior will be the same as before (we will trust any
+    notify completion we get).
+    
+    Fixes: #13114
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e86d033854c76f344c678e92016c4e5c5e0385e2)
+    
+    Conflicts:
+    	src/osdc/Objecter.cc
+    		In Objecter::handle_watch_notify, a conflict was there due to a modified comment by commit 47277c51db7bb2725ea117e4e8834869ae93e006, which was not backported
+
+commit 647c91098e8517912c73e3f7d27954e6f340472c
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 1 14:50:00 2015 -0400
+
+    osd: reply to notify request with our unique notify_id
+    
+    The OSD assigns a unique ID to each notify it queues for
+    processing.  Include this in the reply to the notifier so that
+    they can match it up with the eventual completions they receive.
+    
+    This is necessary to distinguish between multiple completions
+    they may receive if there is PG peering and the notify is resent.
+    In particular, an earlier notify may return an error when a later
+    attempt succeeds.
+    
+    This is forwards and backwards compatible: new clients will make use of
+    this reply payload but older clients ignore it.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 049ea702b9bf4a8a62ae2770d9ba55c0730b3eef)
+
+commit 0ad9521944e0de5c135a066b0cda07f8976bdae1
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Oct 12 13:24:31 2015 -0700
+
+    osd: Correct the object_info_t::decode() version
+    
+    Caused by: 13b9dc70
+    
+    Fixes: #13462
+    Backport: Hammer, Firefly
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 2b7ddde685213f3908f2dd9545b35f1d97f35a79)
+
+commit af734e63f28ac5af414b6507c945ffd9e736d5cf
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 6 12:31:59 2015 -0400
+
+    librbd: invalidate object map on error even w/o holding lock
+    
+    If there is a deep bug that prevents the object map from owning
+    the lock before objects are updated, flag the object map as invalid
+    regardless.
+    
+    Fixes: #13372
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e675400df7f8ba796d60e9bac4234857f0cb1392)
+    
+     Conflicts:
+    	src/librbd/ObjectMap.h/cc: hammer branch doesn't support rebuild
+
+commit 634d7f62486847846e3862091b8cb587d095cb52
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 8 13:28:04 2015 -0400
+
+    tests: removed obsolete rbd_replay test cases
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e0d8cb1283e4bdbfc770684b173e477c22558524)
+
+commit 5c63d3c614189381cc169c64a3138c7abcb368c6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 8 13:26:42 2015 -0400
+
+    ceph-dencoder: new rbd_replay trace file types
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c2a83d0b518497dc8bcbbdea77b88febe52a3034)
+
+commit a145a1cd8e245505abae66f0497be0df0078cecb
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 8 13:21:29 2015 -0400
+
+    rbd-replay: added version control to trace output file
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3ecdae8388d69123b937a40ce614a0b795a757f1)
+
+commit a30ed76c34ecca6278112908871126e1730a22bc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Oct 7 15:39:18 2015 -0400
+
+    rbd-replay-prep: added --verbose command line option
+    
+    This new command-line will dump all the processed events as they
+    are extracted from the trace file.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 646e50a771c3089121ec2f4369d1a8188001a3eb)
+
+commit adf31c88d3ed95d0c29c649e2642ee3ae758743b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Oct 7 15:22:25 2015 -0400
+
+    rbd-replay-prep: stream events to the prep file
+    
+    Avoid building the entire prep event history in memory before
+    attempting to write the prepped trace file.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 98f513a113f4e8aff17c83991d2e6f0f3738a9c9)
+
+commit 8c9e67ad0b21629565a09f6b9b92723b98594f84
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Oct 7 14:56:22 2015 -0400
+
+    rbd-replay-prep: simplify IO dependency calculation
+    
+    Only track read-after-write and write-after-write IO dependencies
+    via the associated write completions.  All IO events after a write
+    completion are considered to be dependent and can be pruned down
+    to at most the number of concurrent IOs.  This reduces the prep
+    time from a simple 'rbd bench-write' from over 4 hrs down to seconds.
+    
+    Fixes: #13378, #13384
+    Backport: hammer
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 65fb1b86cbab65023f6207798c9e189bce55dcf6)
+
+commit e7406fcba86646a4c3fbac66ee54b52ce3d5a33d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 6 13:16:51 2015 -0400
+
+    rbd-replay: improve error messages
+    
+    Fixes: #13221
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4f436e10802d7b3c3949b592fec238c189a02c50)
+
+commit 9f81d28107bd82b7118d1521ade982ab179c5a7a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 6 12:48:50 2015 -0400
+
+    rbd-replay-prep: support new read/write APIs
+    
+    Added support for read2, write2, aio_read2, and aio_write2 APIs.
+    
+    Fixes: 13325
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b5b4a9dd6bfe10844327a457849a0dbf7ef597ea)
+
+commit a856bee2aa7e22f2bedf54b684c4d55452766b01
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Oct 5 17:30:54 2015 -0400
+
+    rbd-replay-prep: avoid using assert statements on user inputs
+    
+    Fixes: #13220
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a1e99f06a78d7cf2ecc0c8e9fbc3331e4e4f18f7)
+
+commit 6a4734a7afecb359b8bc9c114332cd7321097c38
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Sep 21 11:16:49 2015 -0700
+
+    ReplicatedPG::hit_set_setup: fix hit_set_remove_all call
+    
+    We only want to do it if the pool config changed AND we are
+    primary && active.
+    
+    fd38902dd4693b9b72f7532833e78f5df2c9efa5 partially fixed a related
+    bug.  This should be backported along with it.
+    
+    Fixes: 13192
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 36e4a80c39f7daab2a35efc04650ae8b7b11c4e0)
+
+commit ab3877dd566f3951e76abaa6b7deeb5835a512b1
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Sep 22 11:13:44 2015 -0700
+
+    osd/: assert in HitSet constructor if type is TYPE_NONE as well
+    
+    Otherwise impl is unpopulated and we risk segfaults in several
+    methods.  It also seems like it would always indicate a bug.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 8e5a801b43babf751b619ed9606d7be7e5b12837)
+
+commit 4fe07b7f69321535a985958fc761a54cb71136f6
+Author: Zhiqiang Wang <zhiqiang.wang@intel.com>
+Date:   Thu Jul 23 09:26:44 2015 +0800
+
+    osd: only remove all the hit set objects on primary osd
+    
+    Only the primary PG is allowed to remove all the hit set objects. And
+    the PG should be in the active or peered states.
+    
+    Signed-off-by: Zhiqiang Wang <zhiqiang.wang@intel.com>
+    (cherry picked from commit fd38902dd4693b9b72f7532833e78f5df2c9efa5)
+
+commit 313a09b58b6aa6fe8999f4d5da07a04f00c64cd7
+Author: Zhiqiang Wang <zhiqiang.wang@intel.com>
+Date:   Thu Jun 18 09:05:28 2015 +0800
+
+    osd: implement hit_set_remove_all
+    
+    When hit set is not configured on startup or on a change, remove all
+    previous hitsets.
+    
+    Signed-off-by: Zhiqiang Wang <zhiqiang.wang@intel.com>
+    (cherry picked from commit be28319bf3dc54b4b6f400d83405a29facfe3bd4)
+    Conflicts:
+            src/osd/ReplicatedPG.cc
+              pass the new argument of `p->using_gmt` to get_hit_set_archive_object()
+
+commit b632254252f56d3b2c0c32803593ba765b45b222
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 8 16:52:32 2015 +0800
+
+    mon: disable gmt_hitset if not supported
+    
+    the gmt_hitset is enabled by default in the ctor of pg_pool_t, this
+    is intentional. because we want to remove this setting and make
+    gmt_hitset=true as a default in future. but this forces us to
+    disable it explicitly when preparing a new pool if any OSD does
+    not support gmt hitset.
+    
+    Fixes: #12968
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit e48cec3dc93b3988dcd8924933deb1b3a43e1d0f)
+    
+    Conflicts:
+    	src/mon/OSDMonitor.cc
+    		remove unrelated hunk
+
+commit 27fc9a3b2bd34bec8d5bb3954b873ce6344771f4
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Aug 28 00:32:38 2015 +0800
+
+    osd: do not let OSD_HITSET_GMT reuse the feature bit
+    
+    * to ease the backport to hammer
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5a4f6a866bb675195ad83c38fd23238d078c78ed)
+    
+    Conflicts:
+    	src/include/ceph_features.h
+    		minor changes to resolve conflicts
+
+commit 7fcc61d26028eb417a0c224e8c9cd3cc95bc13d9
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Aug 27 11:24:25 2015 -0700
+
+    osd: Decode use_gmt_hitset with a unique version
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 2bc5a48f4c5d3667213be3a7b5a0e0f5ef9daf4f)
+    
+    Conflicts:
+    	src/osd/osd_types.cc
+    		minor changes to resolve conflicts
+
+commit 26c7e968523ac09f608c4ed7c50681fbb741ca1f
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Aug 18 16:53:50 2015 +0800
+
+    mon: print use_gmt_hitset in "ceph osd pool get"
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit cc2bcf760f2d2f20fc4a2fa78ba52475a23e55bf)
+    
+    Conflicts:
+    	src/mon/OSDMonitor.cc
+    		use the old way to dump pool info
+
+commit e8e00dab1b064e77a997504f385b5d60ee8aca25
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Aug 18 00:04:23 2015 +0800
+
+    mon: add "ceph osd pool set $pool use_gmt_hitset true" cmd
+    
+    allow "ceph osd pool set $pool use_gmt_hitset <true|1>" as long as
+    the cluster supports gmt hitset.
+    
+    Fixes: #9732
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 03a1a3cf023a9aeb2fa26820e49e5efe3f3b3789)
+
+commit 040e390d29fc68dcefe48c59cccacf6258c6f690
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Jun 5 21:06:48 2015 +0800
+
+    osd: use GMT time for the object name of hitsets
+    
+    * bump the encoding version of pg_hit_set_info_t to 2, so we can
+      tell if the corresponding hit_set is named using localtime or
+      GMT
+    * bump the encoding version of pg_pool_t to 20, so we can know
+      if a pool is using GMT to name the hit_set archive or not. and
+      we can tell if current cluster allows OSDs not support GMT
+      mode or not.
+    * add an option named `osd_pool_use_gmt_hitset`. if enabled,
+      the cluster will try to use GMT mode when creating a new pool
+      if all the the up OSDs support GMT mode. if any of the
+      pools in the cluster is using GMT mode, then only OSDs
+      supporting GMT mode are allowed to join the cluster.
+    
+    Fixes: #9732
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 42f8c5daad16aa849a0b99871d50161673c0c370)
+    
+     Conflicts:
+    	src/include/ceph_features.h
+    	src/osd/ReplicatedPG.cc
+    	src/osd/osd_types.cc
+    	src/osd/osd_types.h
+    		fill pg_pool_t with default settings in master branch.
+
+commit 31b7864b4161221f6027c6cbe1e42ed598d42b6a
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Sep 27 21:55:00 2015 +0200
+
+    CephxServiceHandler.cc: fix get_auth conditional
+    
+    The KeyServer class has a public method get_auth() that returns a boolean
+    value. This value is being checked here - fix the conditional so it triggers
+    when get_auth() returns false.
+    
+    http://tracker.ceph.com/issues/9756 References: #9756
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 7250db62cb7ac1e9c0b1f5956d60b9d874496702)
+
+commit 0742177c767613d41bda0d260eebdd0087e3b392 (refs/remotes/me/wip-randomize-scrub-hammer)
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat May 16 00:07:27 2015 +0800
+
+    osd: use another name for randomize scrub option
+    
+    s/osd_scrub_interval_limit/osd_scrub_interval_randomize_ratio/
+    
+    Fixes: #10973
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+
+commit fad33861a87f2feb944ad5454775c95d09364b56
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Mar 9 16:42:34 2015 +0800
+
+    osd: randomize scrub times to avoid scrub wave
+    
+    - to avoid the scrub wave when the osd_scrub_max_interval reaches in a
+      high-load OSD, the scrub time is randomized.
+    - extract scrub_load_below_threshold() out of scrub_should_schedule()
+    - schedule an automatic scrub job at a time which is uniformly distributed
+      over [now+osd_scrub_min_interval,
+            now+osd_scrub_min_interval*(1+osd_scrub_time_limit]. before
+      this change this sort of scrubs will be performed once the hard interval
+      is end or system load is below the threshold, but with this change, the
+      jobs will be performed as long as the load is low or the interval of
+      the scheduled scrubs is longer than conf.osd_scrub_max_interval. all
+      automatic jobs should be performed in the configured time period, otherwise
+      they are postponed.
+    - the requested scrub job will be scheduled right away, before this change
+      it is queued with the timestamp of `now` and postponed after
+      osd_scrub_min_interval.
+    
+    Fixes: #10973
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5e44040e8528bff06cc0a5a3f3293ab146e0e4e1)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+
+commit 9c130336a1a1dcf2338d202914aeacbc194af0b0
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Mar 9 16:47:30 2015 +0800
+
+    osd: use __func__ in log messages
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 0f7f35670f03f3f58329c00d323963710e7e495d)
+
+commit 4a3dedff8693953944af3eeb40994f38befebf48
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Apr 15 12:04:05 2015 +0800
+
+    osd: simplify OSD::scrub_load_below_threshold() a little bit
+    
+    avoid unnecessary comparison
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 2ab0e606dfd2847e1918be9f21f572d91befbf7b)
+
+commit 24268cf4309267a0a1e32e28379cf7eafac00bca
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 1 14:21:35 2015 -0400
+
+    librbd: invoking RBD::open twice will leak memory
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a5e88fcf11ebb16d4a1405b21cf6f895b37f185e)
+
+commit 8610de81c8eeae763354a4133e160ccb3faa8dad
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Sep 21 15:46:42 2015 -0700
+
+    use simplifed messenger constructor for clients
+    
+    This is all mechanical except the calculation of the nonce, which is
+    now always randomized for clients.
+    
+    Fixes: #13032
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit a3a8c85b79afef67681c32c57b591c0e0a87a349)
+
+commit 9d059c3f1b3aa0b1830b6f84694c850d36e8a77a
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Sep 21 15:20:53 2015 -0700
+
+    msg: add simplified messenger constructor
+    
+    Several callers create messengers using exactly the same parameters:
+    
+    - reading the ms type from cct that is also passed in
+    - a default entity_name_t::CLIENT
+    - the default features
+    
+    Additionally, the nonce should be randomized and not depend on
+    e.g. pid, as it does in several callers now. Clients running in
+    containers can easily have pid collisions, leading to hangs, so
+    randomize the nonce in this simplified constructor rather than
+    duplicating that logic in every caller.
+    
+    Daemons have meaningful entity_name_ts, and monitors currently depend
+    on using 0 as a nonce, so make this simple constructer
+    client-specific.
+    
+    Related: #13032
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit e3785b0b5fbff870adbd5f121021b671912386cf)
+    
+    Conflicts:
+       src/msg/Messenger.cc: Messenger::create only has 5 arguments in
+       hammer. Since the last one is zero, it does not change the
+       semantic.
+
+commit 6e29e904ce8e7b3481fd3b318b070d326cd4a71a
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 30 08:29:05 2015 -0400
+
+    init-rbdmap: fix CMDPARAMS
+    
+    Fixes: #13214
+    Reported-by: Wyllys Ingersoll <wyllys.ingersoll@keepertech.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 02113ac449cd7631f1c9a3840c94bbf253c052bd)
+
+commit 7161a2c927a6ded0d6ffe7d7621b7abd7f18acc3
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Sep 21 12:00:49 2015 -0700
+
+    PG: ignore info from down osd
+    
+    Fixes: #12990
+    Backport: firefly, hammer
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 930d8eb1fb0daacd692d2e80b5ff473448bd4e8d)
+
+commit 9753551f9f4ea9a1638dd469ef536d895ca9bc3b
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Sep 21 12:01:03 2015 -0700
+
+    OSDMap.h: add has_been_up_since
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 4b00350870e3a43aee5fff7982b3785fe1876634)
+
+commit f1271eae0decf4d1fd9cfd83e5e54d3be6abfb17
+Author: zqkkqz <zheng.qiankun@h3c.com>
+Date:   Fri Aug 7 10:49:45 2015 +0800
+
+    Common/Thread: pthread_attr_destroy(thread_attr) when done with it
+    When a thread attributes object is no longer required, it should be destroyed using the
+    pthread_attr_destroy() function. Destroying a thread attributes object has no effect on threads that were created using that object.
+    
+    Fixes: #12570
+    Signed-off-by: zqkqkz zheng.qiankun@h3c.com
+    (cherry picked from commit 9471bb838a420ef5f439191d87e5388fecccb9e6)
+
+commit 9581f9f4446ed5838bb3de6e1d7cf5faca85df90
+Author: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+Date:   Fri Jul 17 12:43:52 2015 +0200
+
+    Thread.cc: remove malloc/free pair
+    
+    There's no need for mallocing pthread_attr_t in Thread::try_create(),
+    it can be located on stack as it is freed in same function. This reduces
+    pressure put on memory manager.
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+    (cherry picked from commit 614ac0f4e375f3dd008c8937d3f89f6005d6e0a0)
+
+commit a17f2a9a6600803a8b7f6dd306f7bafb6acc1af5
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Tue Sep 1 21:35:19 2015 +0800
+
+    osdc/Objecter: optimize Objecter::tick.
+    
+    Set bool value is better than insert same value into set<>.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit ee204044d9e69ea44b533c05cec154974039264c)
+
+commit f80d237decac775d82f1e276a1ea600a9f352ffe
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Tue Sep 1 19:32:32 2015 +0800
+
+    osdc/Objecter: remove the unuseful code.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 80f10e3e59dadda2dca4eb62c68af972b701b316)
+
+commit 8f76f4c467dc69cd2af75d9cbb134c26f2d04a84
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Tue Sep 1 15:39:29 2015 +0800
+
+    osdc/Objecter: Don't forget call _op_cancel_map_check when cancel linger op.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 7cc963b1c6ab37bf33638dc6eca7848d93f7908f)
+
+commit ba36caba5226a44c41fdd2b19074916b38af5fdf
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Tue Sep 1 15:12:02 2015 +0800
+
+    osdc/Objecter: In _cancel_linger_op, it should make num_unacked/num_committed decrease.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 36b62710ddef0ffaee25837a92ca1ac9b353ff05)
+
+commit 6ff7b02dc23b42a99c7136d1a55ead05c12e2b97
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Thu Aug 20 17:00:23 2015 +0800
+
+    osdc/Objeter: When cancel op, decrease num_unacked/num_uncommitted.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 064e8585a04edb3d87b38db6bed03e965cfcb359)
+
+commit d815e0aacb4a516f4bbbb59a2b2073b38759d9e8
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Thu Aug 20 15:38:58 2015 +0800
+
+    osdc/Objecter: For func op_cancel_writes it can directly call op_cancel.
+    
+    Becasue we get write-lock of rwlock, so it is safe to call op_cancel
+    rather than _op_canchel(homeless_session for this case don't met).
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit e4ce619fe17a7a9dfc18e6af0b84928aa2d88c00)
+    
+    Conflicts:
+          src/osdc/Objecter.cc: the last if (found) was incorrectly
+          implemented as if (to_cancel.size()) after hammer. The
+          conflict is because the patch tries to revert an incorrect
+          if (to_cancel.size()) which is not in hammer.
+
+commit 53316f3694c2a219541dcbdb0269ce97e2d2e9d7
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Wed Aug 19 14:54:21 2015 +0800
+
+    Objecter: Take RLocker when call is_active.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 89f0112e001a2561f9a5cd705898d43c8909501f)
+
+commit 684928883f5eaaec99a8cc8c43e81fae69f9fbfa
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Sep 16 15:08:17 2015 +0800
+
+    mon/PGMap: calc min_last_epoch_clean when decode
+    
+    Fixes: #13112
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d0ac68bf1785b330f3202d924c2203ace9393fe6)
+
+commit 7d4b303b2d9649e2722a9b8f0be5c505776124df
+Author: Guang Yang <yguang@yahoo-inc.com>
+Date:   Wed Sep 23 15:58:02 2015 +0000
+
+    osd: print min_last_epoch_clean along with pg dump
+    
+    Fixes: 13198
+    Signed-off-by: Guang Yang <yguang@yahoo-inc.com>
+    (cherry picked from commit f5359f2314ce3e76ed446c01889dd23550c3ac36)
+
+commit d8ca88dd4b47b4956211d206b0d42112e5842d3f
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Sep 22 11:44:18 2015 -0700
+
+    OSDService::agent_entry: don't use PG::operator<< without pg lock
+    
+    Fixes: #13199
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit d422f2820d5beee15117e166546a6386120e0349)
+    
+    Conflicts:
+    	src/osd/OSD.cc: trivial resolution
+
+commit 397042a0cb2f4c628dd722192fee284359d2cbb3
+Author: Xinze Chi <xinze@xksy.com>
+Date:   Wed Sep 23 19:26:00 2015 +0800
+
+    filestore: fix peek_queue for OpSequencer
+    
+    Fixes: #13209
+    Signed-off-by: Xinze Chi <xinze@xsky.com>
+    (cherry picked from commit 5f7b3f5cde779f1300a35a04df2cdb1299f55472)
+
+commit d4e4d85c005ff68cb0461e9d7f0cbe5e6cf2f073
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 23 10:58:01 2015 -0400
+
+    mon/Elector: do a trivial write on every election cycle
+    
+    Currently we already do a small write when the *first* election in
+    a round happens (to update the election epoch).  If the backend
+    happens to fail while we are already in the midst of elections,
+    however, we may continue to call elections without verifying we
+    are still writeable.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ef909ccbdc303cce8a39edef255325127832ff16)
+
+commit 20a4c0c4831a9dcd49d8ad34b99369effbda9ab8
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 23 10:58:13 2015 -0400
+
+    mon/MonitorDBStore: assert/crash if there is a write error
+    
+    Do this globally intead of relying on teh zillion mon callers to
+    check the error code.  There are no cases where we want to
+    tolerate a commit failure.
+    
+    Fixes: #13089
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2fb7b1f0e33ada7c9a1be3de2f7708eb0760fcef)
+
+commit 3c1f7cbc45c4744eea3c821f0343596756e05ab1
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Sep 24 18:35:39 2015 -0700
+
+    OSDMap: fill in known encode_features where possible
+    
+    Otherwise, if we get an incremental from hammer (struct_v = 6) we will
+    encode the full map as if it were before CEPH_FEATURE_PGID64, which
+    was actually pre-argonaut.  Similarly, if struct_v >= 7, we know it
+    was encoded with CEPH_FEATURE_OSDMAP_ENC.
+    
+    Fixes: #13234
+    Backport: hammer
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 04679c5451e353c966f6ed00b33fa97be8072a79)
+
+commit 93c523c65c1f0b6f9a76b1d71acf9e8df57c0fc9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 8 15:47:37 2015 -0400
+
+    librbd: diff_iterate needs to handle holes in parent images
+    
+    If a clone's parent image snapshot includes a discarded extent,
+    this was previously causing an assert failure.  Instead, ignore
+    any discard holes in the parent image.
+    
+    Fixes: #12885
+    Backport: hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3ccc3bb4bd35e57209852d460633e371b4d004e2)
+    
+     Conflicts:
+    	src/librbd/DiffIterate.cc : file doesn't exist in hammer, logic is in internal.cc
+
+commit 359b68447393980bb7cdd4acc23582e8046b186c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Sep 28 10:25:07 2015 -0400
+
+    tests: new test case for librbd diff_iterate over discard extents
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d5650c9cf85188efa73b279c8f4e4723fa475308)
+    
+    Conflicts:
+        src/test/librbd/test_librbd.cc : diff_iterate2 doesn't exist in hammer
+
+commit 71a42f7c3aa55de688ef16ea7e392573fa5a7e62
+Author: Xinze Chi <xmdxcxz@gmail.com>
+Date:   Mon Jun 8 16:54:08 2015 +0800
+
+    bug fix: osd: avoid multi set osd_op.outdata in tier pool
+    
+    There are two read op on the same object for ec pool. First op read
+    miss happend, calling do_proxy_read and promote_object, The second op only
+    do_proxy_read. but before first op process_copy_chunk finish, the second op
+    finish_proxy_read. first op receive reply from base pool
+    first and then second received. so the second op set the field "outdata"
+    in m->ops first. And then first op requeue_ops in process_copy_chunk,
+    At last in do_osd_ops, it append outdata field.
+    
+    Fixes: 12540
+    Signed-off-by: Xinze Chi <xmdxcxz@gmail.com>
+    (cherry picked from commit 855ae1fd6e4557adba1cbd8ab532488b867cee2a)
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.cc
+
+commit e8cce089102abd8ec1b8e916c6f2c4a5b339cbd3
+Author: Xinze Chi <xmdxcxz@gmail.com>
+Date:   Wed Jul 29 16:28:33 2015 +0800
+
+    bug fix: osd: requeue_scrub when kick_object_context_blocked
+    
+    when read miss in writeback cache pool, osd do_proxy_read first
+    and maybe promote it. but in this case, the op is not added to
+    waiting_for_blocked_object. pg scrub maybe block by this object
+    (_range_available_for_scrub). so after promote it, we should
+    requeue_scrub.
+    
+    Fixes: 12515
+    Signed-off-by: Xinze Chi <xmdxcxz@gmail.com>
+    (cherry picked from commit f77949fedce3449befd74efeb5270579f5085b16)
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.cc
+    		in ReplicatedPG::kick_object_context_blocked, removed hobject_t::BitwiseComparator from map p
+
+commit 8ee93d5f661b303cafcb0134a0e9a1fce7b8e6b2
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Aug 27 11:08:33 2015 -0700
+
+    PG::handle_advance_map: on_pool_change after handling the map change
+    
+    Otherwise, the is_active() checks in the hitset code can erroneously
+    return true firing off repops stamped with the new epoch which then get
+    cleared in the map change code.  The filestore callbacks then pass the
+    interval check and call into a destroyed repop structure.
+    
+    Fixes: 12809
+    Backport: hammer,firefly
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 14e02bc90a463805f4c3e2de210892067a52514b)
+
+commit cecd68d06d4d114b0b735155da5c2bac736714c3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Aug 28 11:36:49 2015 +0800
+
+    ceph_test_rados_api_aio: add a test for aio_sparse_read
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 076bad955d374cbb37b77e2b0429f3c85f32abc0)
+
+commit 8849f9933fe0fc315da0981f9cd88ea108f686ed
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Aug 27 22:57:16 2015 +0800
+
+    ceph_test_rados_api_io: add tests for sparse_read
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 4d4920610ebfcb516630ed15678979c9e9292f5a)
+    
+     Conflicts:
+    	src/test/librados/test.cc
+    		minor changes and remove C++11 stuff
+
+commit 64b22dcba316bf9ac5598c249e4fba47c157f588
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Aug 26 15:41:13 2015 +0800
+
+    ceph_test_rados: also send sparse_read in ReadOp
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5ae2e7a185b5f95753a09a89d7110fc38848a083)
+    
+    Conflicts:
+    	src/test/osd/RadosModel.h
+    		remove the piped-read test
+
+commit 6ca593026f81580ecbd02ad6140347f31086a87d
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Aug 28 14:27:53 2015 +0800
+
+    osd: should use ec_pool() when checking for an ecpool
+    
+    we were using pool.info.require_rollback() in do_osd_ops() when
+    handling OP_SPARSE_READ to tell if a pool is an ecpool. should
+    use pool.info.ec_pool() instead.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit a5bfde69a9d14de67da1e3354173ec70ba089b37)
+
+commit 318049911445a3cc257db2b3beb3ab53e6f1d64e
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Aug 27 22:57:49 2015 +0800
+
+    osd: translate sparse_read to read for ecpool
+    
+    Fixes: #12012
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 700d42ef1c82f5602249b96690ae881c1d259d54)
+
+commit d8ac5103e433f6649e28392b365d2e02b4ba1ffe
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 10 19:10:19 2015 -0400
+
+    WorkQueue: add/remove_work_queue methods now thread safe
+    
+    These methods were not acquiring the ThreadPool lock when
+    manipulating the work_queue collection.  This was causing
+    occasional crashes within librbd when opening and closing
+    images.
+    
+    Fixes: #12662
+    Backport: hammer, firefly
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3e18449b01c1ab78d1bbfc1cf111aa9bdbef7b1f)
+
+commit e8d6d5a5f43b75f6aaec4c977221ad97d11e314c (refs/remotes/gh/wip-11786.hammer)
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Mon Sep 7 14:12:19 2015 +0100
+
+    mon: MonitorDBStore: make get_next_key() work properly
+    
+    We introduced a significant bug with 2cc7aee, when we fixed issue #11786.
+    Although that patch would fix the problem described in #11786, we
+    managed to not increment the iterator upon returning the current key.
+    This would have the iterator iterating over the same key, forever and
+    ever.
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit 70d31082fd3dc8c7857994104577f1a3631c678c)
+
+commit dba8b5b113f585586fcbbc0acf05543262486328
+Author: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+Date:   Tue May 19 13:44:21 2015 +0200
+
+    tools: Don't delete, recreate and re-fill buffers in rados bench.
+    
+    Fixes the high CPU usage and corrects rados bench scores on fast SSDs
+    and ramdisks/memstore.
+    For bench run on SSD, on Intel(R) Xeon(R) CPU E5-2640 v2 @ 2.00GHz
+    before this patch, times are:
+    write: real 5m0.169s, user 2m33.565s, sys 4m39.791s
+    seq: real 4m28.642s, user 1m35.250s, sys 6m42.948s
+    rand: real 5m0.258s, user 1m19.656s, sys 6m47.145s
+    
+    After this patch:
+    write: real 5m1.162s, user 0m27.788s, sys 3m11.707s
+    seq: real 5m1.149s, user 2m23.278s, sys 4m14.427s
+    rand: real 5m1.021s, user 2m30.514s, sys 4m20.347s
+    
+    Bench run: rados -p ssd bench 300 write|seq|read --no-cleanup
+    
+    Note the increase in user time cpu on seq/read tests,
+    along with decreased sys cpu time; this is because there's
+    additional memcmp() that compares read objects with expected
+    contents. With less time spent memory juggling, more time is
+    spent performing more reads per second, increasing memcmp call
+    count and increasing amount of user cpu time used.
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+    (cherry picked from commit b894fc7acf7dee7f7ec8c5c280e7a6be41133328)
+    
+    Conflicts:
+    	src/common/obj_bencher.cc
+    		return -5 was replaced with return r
+
+commit fa72ecb509fa4ddd5f227b1e279eae390284a72c
+Author: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+Date:   Fri May 15 15:44:18 2015 +0200
+
+    bufferlist: implement bufferlist::invalidate_crc()
+    
+    This function iterates over all bufferlist internal buffers and calls
+    their invalidate_crc() method. Required for rados bench to work
+    correctly, because it modifies buffers outside buffer api, invalidating
+    internal CRC cache in the process - this method clears that cache, so
+    another call for buffer::get_crc() to return correct checksum.
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+    (cherry picked from commit 55a6f9efbee041d041742810ca4fa8874b6191a7)
+
+commit 50c82f2095add5ad75200a62081fd2f5fea8d8d9
+Author: Owen Synge <osynge@suse.com>
+Date:   Mon Jun 1 15:57:03 2015 +0200
+
+    ceph.spec.in summary-ended-with-dot
+    
+    RPM spec files summary should not end with a dot.
+    This was inconsistent across the ceph rpm packages
+    and creates errors with rpm lint.
+    
+    Signed-off-by: Owen Synge <osynge@suse.com>
+    (cherry picked from commit 042e8ff4dc71795d957bbfac12fade2bc0fc4674)
+
+commit 94699e724b45a9a6de84a943c5b834d2f20f85ef
+Author: Owen Synge <osynge@suse.com>
+Date:   Mon Jun 1 14:58:31 2015 +0200
+
+    ceph.spec.in libcephfs_jni1 has no %post and %postun
+    
+    /usr/lib64/libcephfs_jni.so.1.0.0 requires /sbin/ldconfig to be
+    run after installing and after removing.
+    
+    Signed-off-by: Owen Synge <osynge@suse.com>
+    (cherry picked from commit 0742d82b5561634a5f4ed73fc6cbe4d4f26b0d42)
+
+commit 7475a8fe3f31a46a949f051ded383cd6f396a0fa
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Apr 21 16:08:09 2015 -0700
+
+    logrotate.conf: prefer service over invoke-rc.d
+    
+    On trusty 14.04, service works but invoke-rc.d does not (but is present).
+    
+    Fixes: #11330
+    Reported-by: Wim Heirman <wim@heirman.net>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6063a21f9edbbf80e44d45ff52a9f4ba1a1cb2c9)
+
+commit 86f88dfce3c0e785d0ff48ec8892e53e6a53b080
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jul 23 10:15:56 2015 +0100
+
+    mon: fix checks on mds add_data_pool
+    
+    Use same _check_pool helper as during fs creation.
+    
+    Fixes: #12426
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 329741ec238d6af4b69bc838ba667be232741b3d)
+
+commit a65c39852775b17323fa4efee54bde12ce22a512
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Jul 29 15:52:19 2015 +0800
+
+    common/syncfs: fall back to sync(2) if syncfs(2) not available
+    
+    Fixes: #12512
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 08210d6a85527b0759f6dc4411f742bc13a4939f)
+    
+    Conflicts:
+    	src/common/sync_filesystem.h
+              - subtle difference in #if/#else/#endif structure
+
+commit e6eaa914d685f2a8f1f8cf3b01112a1b720e1358
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Wed May 27 22:28:49 2015 +0100
+
+    mon: MonitorDBStore: get_next_key() only if prefix matches
+    
+    get_next_key() had a bug in which we would always return the first key
+    from the iterator, regardless of whether its prefix had been specified
+    to the iterator.
+    
+    Fixes: #11786
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit 2cc7aee1abe40453093306c8fef2312b650dff5d)
diff --git a/doc/changelog/v0.94.7.txt b/doc/changelog/v0.94.7.txt
new file mode 100644
index 000000000..b639752f6
--- /dev/null
+++ b/doc/changelog/v0.94.7.txt
@@ -0,0 +1,3179 @@
+commit d56bdf93ced6b80b07397d57e3fa68fe68304432 (tag: refs/tags/v0.94.7)
+Author: Jenkins Build Slave User <jenkins-build@trusty-small--296081f3-e7b1-46b1-96c2-d6b655cf71d3.localdomain>
+Date:   Tue May 10 14:47:39 2016 +0000
+
+    0.94.7
+
+commit 6e6c85a0d71df7cd6207febc3e22d30617648abc
+Merge: 934ad41 76c33de
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon May 9 13:55:24 2016 -0400
+
+    Merge pull request #8979 from athanatos/wip-15655-hammer
+    
+    OSDMonitor: avoid underflow in reweight-by-utilization if max_change=1
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 934ad41dbf3f7004c93aa8f7660353329b7f3fad
+Merge: e219e85 3c03eee
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed May 4 11:37:33 2016 +0200
+
+    Merge pull request #8583 from Vicente-Cheng/wip-15089-hammer
+    
+    hammer: Bucket owner isn't changed after unlink/link
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 76c33decdeba83d4d977a9556e76e68fa964465f (refs/remotes/athanatos/wip-15655-hammer)
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Apr 28 13:47:48 2016 -0700
+
+    OSDMonitor: avoid underflow in reweight-by-utilization if max_change=1
+    
+    Fixes: http://tracker.ceph.com/issues/15655
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit c121bc51a5a877a49094cfbd3bd16af1f3ae5090)
+
+commit 3c03eee07e02b6b4d31f6a58a03253eff2d41324
+Author: Aran85 <zhangzengran@h3c.com>
+Date:   Mon Sep 28 08:23:22 2015 +0800
+
+    rgw:bucket link now set the bucket.instance acl
+    
+    Fixes: #11076
+    
+    Signed-off-by: Zengran Zhang <zhangzengran@h3c.com>
+    (cherry picked from commit 4d59b1d36f8924290c3ecb5b7608747191470188)
+    
+    Conflicts:
+    
+    	src/rgw/rgw_bucket.cc
+                1. Do not use the rgw_user structure and remove the tenant parameter that describes as below
+                2. user_id is not used so just remove the line
+                3. instead of system_obj_set_attr you can use the method set_attr
+    
+    Backport Change:
+    We do not use the rgw_user structure and remove the `tenant` parameter
+    because this feature is not introduced on hammer version.
+    The rgw multi-tenant feature is introduced on pr#6784 (https://github.com/ceph/ceph/pull/6784)
+    This feature is supported from v10.0.2 and later version.
+
+commit e219e85be00088eecde7b1f29d7699493a79bc4d
+Merge: eea2a77 f794ada
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Apr 6 10:43:16 2016 -0400
+
+    Merge pull request #7922 from badone/wip-14961-hammer
+    
+    hammer: PK11_DestroyContext() is called twice if PK11_DigestFinal() fails
+
+commit eea2a77ef0b242c295f2480349d8608273dfb184
+Merge: 815760d 38b9c0b
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Apr 6 10:42:44 2016 -0400
+
+    Merge pull request #7992 from SUSE/wip-13807-hammer
+    
+    hammer: segfault in Objecter::handle_watch_notify
+
+commit 815760dde45574f931e316f47c5990129375e9f2
+Merge: ba5c75a 01672b4
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Apr 6 10:41:41 2016 -0400
+
+    Merge pull request #8051 from xiaoxichen/wip-15024-hammer
+    
+    hammer: clock skew report is incorrect by ceph health detail command
+
+commit ba5c75a33ba10bbe7796715f2d8500ec1fe209fd
+Merge: dc09055 ac3569c
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Apr 6 10:40:51 2016 -0400
+
+    Merge pull request #8398 from asheplyakov/hammer-wip-15336
+    
+    hammer: monclient: avoid key renew storm on clock skew
+
+commit dc090558a3306feb2c50994950756e5599a92eaf
+Merge: 0f28a5b 744e9f8
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Apr 6 16:24:55 2016 +0200
+
+    Merge pull request #8272 from Yan-waller/yj-wip-fixbl
+    
+    hammer: tests: bufferlist: do not expect !is_page_aligned() after unaligned rebuild
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 0f28a5b033b47479ae6f1958d2f079bd6c57f1b4
+Merge: 374c3c1 f8d2abd
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Apr 6 16:20:15 2016 +0200
+
+    Merge pull request #7883 from tchaikov/wip-12848-hammer
+    
+    hammer: osd: use GMT time for the object name of hitsets
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 374c3c1d8cde50a4503cbd41ee2204aa6761557d
+Merge: add1674 0830275
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Apr 6 16:18:11 2016 +0200
+
+    Merge pull request #7917 from ceph/wip-10794-hammer
+    
+    hammer: ceph-objectstore-tool, osd: Fix import handling
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit add1674c759f946c7c456e96e52b7ca1a45829ec
+Merge: d08bae2 be4a9fe
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Apr 6 16:13:32 2016 +0200
+
+    Merge pull request #8026 from liewegas/wip-reweight-hammer
+    
+    hammer: mon: implement reweight-by-utilization feature
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit d08bae217e50dfa19dce3bb58349149c4210361e
+Merge: f68697c d4cf190
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Apr 6 16:12:55 2016 +0200
+
+    Merge pull request #8049 from xiaoxichen/wip-15022-hammer
+    
+    hammer: keyring permisions for mon deamon
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f68697c71ad4b6f49dae12a50fad73cacbcaf6e1
+Merge: e5beeef 57fd7f8
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Apr 6 16:09:42 2016 +0200
+
+    Merge pull request #8052 from xiaoxichen/wip-15051-hammer
+    
+    hammer: test/librados/tier.cc doesn't completely clean up EC pools
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit e5beeef1b142a86140791470785c69f865e5df5a
+Merge: 77fbf58 77a4ed0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Apr 6 16:03:07 2016 +0200
+
+    Merge pull request #8440 from dachary/wip-15382-hammer
+    
+    hammer: rpm package building fails if the build machine has lttng and babeltrace development packages installed locally
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 77fbf581cb2259146938a737c299d6cf762303d1
+Merge: 92a96e1 97f474f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Apr 6 15:05:11 2016 +0200
+
+    Merge pull request #8401 from Vicente-Cheng/wip-15129-hammer
+    
+    hammer: Protect against excessively large object map sizes
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 92a96e1b16d9cadf182d200dfef0de6c2b388202
+Merge: 7cfc739 281d63d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Apr 6 13:44:09 2016 +0200
+
+    Merge pull request #8011 from dillaman/wip-15033-hammer
+    
+    hammer: librbd: possible QEMU deadlock after creating image snapshots
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 77a4ed0a8986b2529edd8059dd3efa38f778c6f2
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Apr 5 11:53:54 2016 +0200
+
+    ceph.spec.in: disable lttng and babeltrace explicitly
+    
+    before this change, we do not pacakge tracepoint probe shared libraries
+    on rhel7. but "configure" script enables them if lttng is detected. and
+    rpm complains at seeing installed but not pacakged files. as EPEL-7 now
+    includes lttng-ust-devel and libbabeltrace-devel, we'd better
+    BuildRequire them, and build with them unless disabled otherwise. so in
+    this change
+    * make "lttng" an rpm build option enabled by default
+    * BuildRequire lttng-ust-devel and libbabeltrace-devel if the "lttng"
+     "lttng" option is enabled
+    * --without-lttng --without-babeltrace if the "lttng" option is disabled
+    
+    Fixes: #14844
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit f1a44907a2f83cb8080cd8ab5980a30306a33fbd)
+    
+    Conflicts:
+    	ceph.spec.in: trivial resolution, the context changed
+            (other packages, files etc.) but the semantic did not
+
+commit 97f474fb6b26ae7b7e2fdd21de4f4bd98cdd56ab
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Mar 14 13:57:28 2016 -0400
+
+    cls_rbd: protect against excessively large object maps
+    
+    Fixes: #15121
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4aff4ea3290dc7fb62c639bfc74fcfdde5fe9542)
+    
+      Conflicts:
+    	src/cls/rbd/cls_rbd.cc
+            `RBD_METADATA_KEY_PREFIX` is not introduced on hammer, so remove
+            it
+
+commit ac3569cd8bbb4b2b9a0edf298ce0b83bc18f1fea
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Mon Mar 21 18:20:18 2016 +0300
+
+    hammer: monclient: avoid key renew storm on clock skew
+    
+    Refreshing rotating keys too often is a symptom of a clock skew, try to
+    detect it and don't cause extra problems:
+    
+    * MonClient::_check_auth_rotating:
+      - detect and report premature keys expiration due to a time skew
+      - rate limit refreshing the keys to avoid excessive RAM and CPU usage
+        (both by OSD in question and monitors which have to process a lot
+        of auth messages)
+    * MonClient::wait_auth_rotating: wait for valid (not expired) keys
+    * OSD::init(): bail out after 10 attempts to obtain the rotating keys
+    
+    Fixes: #15336
+    
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+    (cherry picked from commit 918c12c2ab5d014d0623b1accf959b041aac5128)
+    
+    Conflicts:
+    	src/osd/OSD.cc
+    	  no loadavg checks in Hammer, hence the conflict. Move the
+    	  counter and max_attempts initialization to a suitable place.
+
+commit 7cfc739a513da84d76b959efc51dd21c1cc552d6
+Merge: 0418943 7eae05e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Mar 31 13:50:23 2016 +0200
+
+    Merge pull request #8187 from liewegas/wip-15171
+    
+    hammer: osd/ReplicatedPG: do not proxy read *and* process op locally
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+    Reviewed-by: David Disseldorp <ddiss@suse.de>
+
+commit f8d2abd2e41c5dd04977f85cc1d6e65853c9a1b2
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Mar 3 12:03:44 2016 +0800
+
+    osd/osd_types: encode pg_pool_t the old way
+    
+    as a workaround of #12410
+    
+    Fixes: #12968
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+
+commit 720a090eb67b3955b0cadb7633c5a28a934171a4
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 8 16:52:32 2015 +0800
+
+    mon: disable gmt_hitset if not supported
+    
+    the gmt_hitset is enabled by default in the ctor of pg_pool_t, this
+    is intentional. because we want to remove this setting and make
+    gmt_hitset=true as a default in future. but this forces us to
+    disable it explicitly when preparing a new pool if any OSD does
+    not support gmt hitset.
+    
+    Fixes: #12968
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit e48cec3dc93b3988dcd8924933deb1b3a43e1d0f)
+    
+    Conflicts:
+    	src/mon/OSDMonitor.cc
+    		remove unrelated hunk
+
+commit 7aec079f8a1bbe75625c438a17bb87e45398568e
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Aug 28 00:32:38 2015 +0800
+
+    osd: do not let OSD_HITSET_GMT reuse the feature bit
+    
+    * to ease the backport to hammer
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5a4f6a866bb675195ad83c38fd23238d078c78ed)
+    
+    Conflicts:
+    	src/include/ceph_features.h
+    		minor changes to resolve conflicts
+
+commit 370434136ef076c350db3db4fca6489f88f70453
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Aug 27 11:24:25 2015 -0700
+
+    osd: Decode use_gmt_hitset with a unique version
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 2bc5a48f4c5d3667213be3a7b5a0e0f5ef9daf4f)
+    
+    Conflicts:
+    	src/osd/osd_types.cc
+    		minor changes to resolve conflicts
+
+commit 64bca2a43b34b265621bad2ec1fb980217223847
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Aug 18 16:53:50 2015 +0800
+
+    mon: print use_gmt_hitset in "ceph osd pool get"
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit cc2bcf760f2d2f20fc4a2fa78ba52475a23e55bf)
+    
+    Conflicts:
+    	src/mon/OSDMonitor.cc
+    		use the old way to dump pool info
+
+commit 87df212cfca33efbbee6376f528cb7d4895d1dc0
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Aug 18 00:04:23 2015 +0800
+
+    mon: add "ceph osd pool set $pool use_gmt_hitset true" cmd
+    
+    allow "ceph osd pool set $pool use_gmt_hitset <true|1>" as long as
+    the cluster supports gmt hitset.
+    
+    Fixes: #9732
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 03a1a3cf023a9aeb2fa26820e49e5efe3f3b3789)
+
+commit 039240418060c9a49298dacc0478772334526dce
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Jun 5 21:06:48 2015 +0800
+
+    osd: use GMT time for the object name of hitsets
+    
+    * bump the encoding version of pg_hit_set_info_t to 2, so we can
+      tell if the corresponding hit_set is named using localtime or
+      GMT
+    * bump the encoding version of pg_pool_t to 20, so we can know
+      if a pool is using GMT to name the hit_set archive or not. and
+      we can tell if current cluster allows OSDs not support GMT
+      mode or not.
+    * add an option named `osd_pool_use_gmt_hitset`. if enabled,
+      the cluster will try to use GMT mode when creating a new pool
+      if all the the up OSDs support GMT mode. if any of the
+      pools in the cluster is using GMT mode, then only OSDs
+      supporting GMT mode are allowed to join the cluster.
+    
+    Fixes: #9732
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 42f8c5daad16aa849a0b99871d50161673c0c370)
+    
+     Conflicts:
+    	src/include/ceph_features.h
+    	src/osd/ReplicatedPG.cc
+    	src/osd/osd_types.cc
+    	src/osd/osd_types.h
+    		fill pg_pool_t with default settings in master branch.
+
+commit 744e9f837a94f95c5e877f074fd1479f8da64e3c
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Sep 4 01:23:31 2015 +0800
+
+    test/bufferlist: do not expect !is_page_aligned() after unaligned rebuild
+    
+    if the size of a bufferlist is page aligned we allocate page aligned
+    memory chunk for it when rebuild() is called. otherwise we just call
+    the plain new() to allocate new memory chunk for holding the continuous
+    buffer. but we should not expect that `new` allocator always returns
+    unaligned memory chunks. instead, it *could* return page aligned
+    memory chunk as long as the allocator feels appropriate. so, the
+    `EXPECT_FALSE(bl.is_page_aligned())` after the `rebuild()` call is
+    removed.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 8ed724222651812c2ee8cc3804dc1f54c973897d)
+
+commit 0830275b39afc408573c2c468fa3b0e7b1d5cc4e
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 6 14:35:35 2015 -0400
+
+    osd/PG: fix generate_past_intervals
+    
+    We may be only calculating older past intervals and have a valid
+    history.same_interval_since value, in which case the local
+    same_interval_since value will end at the newest old interval we had to
+    generate.
+    
+    This was introduced by 70316541bbb115d9a35954bfba373cf1dc084b7e.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 98cbf0380e9ee848725fc87d61c31a734305a84f)
+
+commit 0418943c6ef8c9649a58003444daeb4b6224fbab
+Merge: 35eae5f 2072a53
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Mar 18 10:13:40 2016 -0400
+
+    Merge pull request #8123 from liewegas/wip-15136-hammer
+    
+    qa/workunits: remove 'mds setmap' from workunits
+
+commit 7eae05e14231cb3476d14bf74b15bf9646849d09 (refs/remotes/me/wip-15171)
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 17 13:28:40 2016 -0400
+
+    osd/ReplicatedPG: do not proxy read *and* process op locally
+    
+    If we proxy the read, we cannot return false, even if we fail to also
+    trigger a promotion.
+    
+    Fixes: #15171
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit be4a9feae4c5f33685cceb386717a5733553df10 (refs/remotes/me/wip-reweight-hammer)
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Mar 14 08:07:19 2016 -0400
+
+    osd/OSDMap: fix typo in summarize_mapping_stats
+    
+    From ea9abe53d0e777b7dc3b22af71639f77c4de08c8
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit cc7e2329046ef7c528f321092a464f6db6e7edb9)
+
+commit 2072a53dc920b79ed76265d6097e50d3d2994b1d
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 15 09:44:13 2016 -0400
+
+    qa/workunits: remove 'mds setmap' from workunits
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6a80e2df8b31802a39ef6f89136ed357dce50c20)
+
+commit 01672b4afd6f886d6bb8527427663e062ab9d100
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Thu Jan 7 11:20:36 2016 +0000
+
+    mon: Monitor: get rid of weighted clock skew reports
+    
+    By weighting the reports we were making it really hard to get rid of a
+    clock skew warning once the cause had been fixed.
+    
+    Instead, as soon as we get a clean bill of health, let's run a new round
+    and soon as possible and ascertain whether that was a transient fix or
+    for realsies. That should be better than the alternative of waiting for
+    an hour or something (for a large enough skew) for the warning to go
+    away - and with it, the admin's sanity ("WHAT AM I DOING WRONG???").
+    
+    Fixes: #14175
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    
+    (cherry pick from commit 17d8ff429c7dca8fc1ada6e7cc8a7c4924a22e28)
+
+commit f90b8bc1a2b3ffa5ba52c0992c1c5ff52e5ea104
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Thu Jan 7 11:05:44 2016 +0000
+
+    mon: Monitor: adaptative clock skew detection interval
+    
+    When in the presence of a clock skew, adjust the checking interval
+    according to how many rounds have gone by since the last clean check.
+    
+    If a skew is detected, instead of waiting an additional 300 seconds we
+    will perform the check more frequently, gradually backing off the
+    frequency if the skew is still in place (up to a maximum of
+    'mon_timecheck_interval', default: 300s). This will help with transient
+    skews.
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    
+    (cherry pick from commit 45e16d0d7407cf4e886dde9321bb91c1a087776d)
+    
+    Conflicts:
+    	src/common/config_opts.h
+                Merge the change line.
+    	src/mon/Monitor.h
+                handle_timecheck_leader(MonOpRequestRef op) was replaced with handle_timecheck_leader(MTimeCheck *m)
+                also for handle_timecheck_peon and handle_timecheck.
+
+commit 57fd7f858c220af362b330d12db0e87de82b4346 (refs/remotes/gh/hammer-pr-8052)
+Author: Dan Mick <dan.mick@redhat.com>
+Date:   Wed Nov 25 19:20:51 2015 -0800
+
+    test/librados/test.cc: clean up EC pools' crush rules too
+    
+    SetUp was adding an erasure-coded pool, which automatically adds
+    a new crush rule named after the pool, but only removing the
+    pool.  Remove the crush rule as well.
+    
+    http://tracker.ceph.com/issues/13878 Fixes: #13878
+    
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 04b4795f81c15bfcb62ba5807745470ce0e5e949)
+
+commit d4cf19049349678065195659bb287b899f92a686
+Author: Owen Synge <osynge@suse.com>
+Date:   Wed Mar 2 12:15:48 2016 +0100
+
+    keyring permissions for mon daemon
+    
+    The command:
+    
+    sudo ceph-mon --cluster ceph --mkfs -i $NODE_NAME --keyring $PATH_MON_KEYRING
+    
+    Writes a new keyring with permissions set to "0644".
+    
+    Fixes: #14950
+    Signed-off-by: Owen Synge osynge@suse.com
+    (cherry picked from commit c2f91a8ce46974a72b960b7cb25af3d089fbb80d)
+
+commit 35eae5fbddd2b271a10db9a69d4c4e34029bca06
+Merge: c59b994 647723e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Mar 11 10:05:42 2016 +0700
+
+    Merge pull request #7911 from dzafman/wip-meta-list-hammer
+    
+    hammer: tools, test: Add ceph-objectstore-tool to operate on the meta collection
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit c59b9944a61174c7e29a26f4abfd7839020578af
+Merge: dd66eb9 d875620
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Mar 11 10:03:43 2016 +0700
+
+    Merge pull request #7903 from branch-predictor/bp-hammer-force-bench-smoke
+    
+    hammer: tools/rados: add bench smoke tests
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit dd66eb91d0129163ecfc8105f043fd0f6926dad5
+Merge: de43aba 5726463
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Mar 11 10:02:18 2016 +0700
+
+    Merge pull request #7876 from SUSE/wip-14946-hammer
+    
+    hammer: redhat-lsb-core dependency was dropped, but is still needed
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit de43aba0015927cec5ebd8f98ad26a801b1353ff
+Merge: 48dfac6 6a6754f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Mar 11 10:00:48 2016 +0700
+
+    Merge pull request #7817 from asheplyakov/hammer-bug-14873
+    
+    hammer: tools: fix race condition in seq/rand bench (part 2)
+    
+    Reviewed-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 48dfac64ba37e5d0ec262798fbec011c1016a4f0
+Merge: 7083829 3a5b102
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Mar 11 10:00:18 2016 +0700
+
+    Merge pull request #7896 from asheplyakov/hammer-bug-14873-p1
+    
+    hammer: tools: fix race condition in seq/rand bench (part 1)
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+
+commit 1b922e5431339d12e661de9fd4b16e8cf2c08ad5
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 10 08:50:57 2016 -0500
+
+    test/pybind/test_ceph_argparse: fix reweight-by-utilization tests
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d0f87131962b26cef9e31fa11651771856c41164)
+
+commit 06a2a7596b876a5c2addb184a48222bcb6e23b9a
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Mar 10 21:41:15 2016 +0800
+
+    man/8/ceph.rst: remove invalid option for reweight-by-*
+    
+    --yes-i-really-mean-it is not valid option for reweight-by-*
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 797f1d41d98738411079a7a16d8b1d5352359d33)
+
+commit 241f76262273bafd72e91fee3d46c412f7418767
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Mar 9 13:25:55 2016 -0500
+
+    mon: remove range=100 from reweight-by-* commands
+    
+    Specifying the range makes it skip instead of error when you give
+    it a bad value.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 661891aa28ca0600a62e25f6d10ce7fa94e32cce)
+
+commit 55ad2c7724e114728bb327a97726a6c9b660bb1d
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 3 13:46:12 2016 -0500
+
+    mon: make max_osds an optional arg
+    
+    We keep the config option with the default (4), but let you
+    specify the max # of osds to reweight as a third optional arg.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 651f05b6e6bec41b54c0a8f8ca6641ce3aa5dff3)
+
+commit f13cdeaf6657577c21a5bb2985d46971aa084c3f
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 3 13:38:57 2016 -0500
+
+    mon: make reweight max_change default configurable
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8a9429d85bc99874dc9871b43b2d0c7ad1086803)
+
+commit f4b4ef76295d40753c6acffa3d92128f7ebe8733
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 3 13:38:19 2016 -0500
+
+    mon/OSDMonitor: fix indentation
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 47b650c1497b8eaf37cfcb7e3b9fda12d8cf2d7a)
+
+commit 76eb3c8fbdb614a4cdde937a1dec680b4be82470
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Mar 2 17:34:48 2016 -0500
+
+    qa/workunits/cephtool/test.sh: test reweight-by-x commands
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 28b33a5b57e3f898808bd8ccc5e314667f84912a)
+
+commit 9a9d147aa14a637bbf8edddf85f6c36a29874694
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 10 08:28:59 2016 -0500
+
+    osd/MonCommand: add/fix up 'osd [test-]reweight-by-{pg,utilization}'
+    
+    - show before/after pg placement stats
+    - add test- variants that don't do anything
+    - only allow --no-increasing on the -utilization versions (where
+      it won't conflict with the optional pool list and confuse the
+      arg parsing)
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1de7e47833f2d14c6915d7d071fe0005408c8da0)
+    
+    - use non-C++11 sort
+    
+    # Conflicts:
+    #	src/mon/OSDMonitor.cc
+    #	src/mon/OSDMonitor.h
+
+commit 6ec676df083ec3078f5efd8058d6fb6a491c9b04
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Mar 2 14:00:15 2016 -0500
+
+    mon: add 'osd utilization' command
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5b757eff868dbce2393beadd9fe2522372288cc0)
+
+commit 94134d9148edfcc41d2d239db69d0b299fbf427f
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 10 08:30:45 2016 -0500
+
+    osd/OSDMap: add summarize_mapping_stats
+    
+    Calculate some basic stats about the PG distribution.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ea9abe53d0e777b7dc3b22af71639f77c4de08c8)
+    
+    - remove c++11 auto and range-based for loops
+
+commit 932f75d1f2105f296681b5fb02757087947e01f8
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 3 13:49:03 2016 -0500
+
+    mon: make reweight-by-* max_change an argument
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a70eaabcfc1cc1dfb9b6ba2d7f71b3d76496eb55)
+
+commit d8372cea0c51db89cc48e5592236d53f7b01b6b3
+Author: Dan van der Ster <daniel.vanderster@cern.ch>
+Date:   Fri Feb 26 22:29:46 2016 +0100
+
+    osd: add mon_reweight_max_osds to limit reweight-by-* commands
+    
+    Add configurable mon_reweight_max_osds which limits the number
+    of OSDs modified each time reweight-by-* is called (by default
+    to 4 OSDs).
+    
+    Also change the order in which we look at OSDs to go from most
+    to least utilized.
+    
+    Signed-off-by: Dan van der Ster <daniel.vanderster@cern.ch>
+    (cherry picked from commit 3aa0ccefddc1790df5b808bfccd5da4857951220)
+
+commit 6a422b2626d48098c5a2c13de677814429263cba
+Author: Dan van der Ster <daniel.vanderster@cern.ch>
+Date:   Fri Feb 26 22:28:46 2016 +0100
+
+    osd: add mon_reweight_max_change option which limits reweight-by-*
+    
+    Add a configurable mon_reweight_max_change which limits be default
+    the reweight-by-* commands to only change an OSDs weight by 0.05.
+    
+    Signed-off-by: Dan van der Ster <daniel.vanderster@cern.ch>
+    (cherry picked from commit 6f0a1fb69da84ee18bdb9b151772cef8f32413c5)
+
+commit d3635b796ae28e1bc6598daf4109bf371a1f4ed8
+Author: Dan van der Ster <daniel.vanderster@cern.ch>
+Date:   Fri Feb 26 21:58:00 2016 +0100
+
+    test: add simple test for new reweight-by-* options
+    
+    Test validity of --no-increasing and --yes-i-really-mean-it.
+    
+    Signed-off-by: Dan van der Ster <daniel.vanderster@cern.ch>
+    (cherry picked from commit 1a6ad5053e7fc4f8942855a17ce4540e340e9d4f)
+
+commit e993851b7abf80fd793aa5b7188f1af7dbe03041
+Author: Dan van der Ster <daniel.vanderster@cern.ch>
+Date:   Fri Feb 26 21:52:41 2016 +0100
+
+    osd: add sure and no-increasing options to reweight-by-*
+    
+    Add a --no-increasing option to reweight-by-* which can be used to only decrease
+    OSD weights without increasing any. This is useful for example if you need to
+    urgently lower the weight of nearly full OSDs.
+    
+    Also add a --yes-i-really-mean-it confirmation to reweight-by-*.
+    
+    Signed-off-by: Dan van der Ster <daniel.vanderster@cern.ch>
+    (cherry picked from commit ddf5c2b62316bb69839f220fe2527d1467863421)
+
+commit 281d63df792c4cdf25cca5428969c65d2ae379e8
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Mar 9 18:00:04 2016 -0500
+
+    librbd: complete cache reads on cache's dedicate thread
+    
+    If a snapshot is created out-of-band, the next IO will result in the
+    cache being flushed.  If pending writeback data performs a copy-on-write,
+    the read from the parent will be blocked.
+    
+    Fixes: #15032
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f9e5ae80e3b0799b5a7e1d3f93b2d85a4baff20f)
+
+commit 621e3aef85e4b720232ca9ee89a4620d58362047
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Mar 9 17:31:06 2016 -0500
+
+    test: reproducer for writeback CoW deadlock
+    
+    Refs: #14988
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 16b6efdd24b25ba1f6bc658681afa3d0878eb397)
+    (cherry picked from commit 7ffbe4099c9367e7405e2196803a0d8b12253a06)
+
+commit 38b9c0b2fbbb056c29cd0485a9831edce020eaa6
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Nov 16 11:32:34 2015 -0500
+
+    osdc/Objecter: call notify completion only once
+    
+    If we race with a reconnect we could get a second notify message
+    before the notify linger op is torn down.  Ensure we only ever
+    call the notify completion once to prevent a segfault.
+    
+    Fixes: #13805
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 987f68a8df292668ad241f4769d82792644454dd)
+
+commit 7083829c227403a77fcf35a2376dc02e4c9693c8 (refs/remotes/markhpc/hammer)
+Merge: 651e49e c4ba93a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Mar 8 10:00:15 2016 +0700
+
+    Merge pull request #6604 from ukernel/hammer-13268
+    
+    hammer: client: added permission check based on getgrouplist
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 651e49ea2037834fbbf48e642a78075140e06782
+Merge: 522e3e8 b2393dc
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Mar 8 09:58:30 2016 +0700
+
+    Merge pull request #6551 from ceph/hammer-client-inoderef
+    
+    hammer: client inoderef
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit f794adadd126bd77ae138794119725a046642712
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Fri Mar 4 13:06:47 2016 +1000
+
+    tests: Add TEST_no_segfault_for_bad_keyring to test/mon/misc.sh
+    
+    94da46b6e31cac206cb32fc5bd3159209ee25e8c adds
+    TEST_no_segfault_for_bad_keyring which requires changes to run
+    in hammer since test/mon/misc.sh is not written to run multiple tests in
+    succession in the hammer version.
+    
+    Refs: #14960
+    
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit 522e3e8d761a337796ed92868e00062b87e0f0a5
+Merge: d314849 e014ea8
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Mar 4 10:25:00 2016 +0700
+
+    Merge pull request #7797 from SUSE/wip-14861-hammer
+    
+    hammer: ceph init script unconditionally sources /lib/lsb/init-functions
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 94da46b6e31cac206cb32fc5bd3159209ee25e8c
+Author: Dunrong Huang <riegamaths@gmail.com>
+Date:   Tue Dec 8 00:22:02 2015 +0800
+
+    tests: make sure no segfault occurs when using some bad keyring
+    
+    Signed-off-by: Dunrong Huang <riegamaths@gmail.com>
+    (cherry picked from commit 2eb0bf32244fc174d56cc42e2298c67d8b5299ac)
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit a371c0f1d69fea131b56100492d3045c15ae072c
+Author: Dunrong Huang <riegamaths@gmail.com>
+Date:   Wed Nov 25 19:03:03 2015 +0800
+
+    auth: fix a crash issue due to CryptoHandler::create() failed
+    
+    In this case(e.g. user passes wrong key), attempts to call the CryptoKey.ckh will lead to a segfault.
+    
+    This patch fixes crash issue like following:
+    
+    Program received signal SIGSEGV, Segmentation fault.
+    [Switching to Thread 0x7fffed10e700 (LWP 25051)]
+    0x00007ffff59896c6 in CryptoKey::encrypt (this=0x7fffed10d4f0, cct=0x555555829c30, in=..., out=..., error=0x7fffed10d440) at auth/cephx/../Crypto.h:110
+    110	    return ckh->encrypt(in, out, error);
+    (gdb) bt
+        at auth/cephx/../Crypto.h:110
+        at auth/cephx/CephxProtocol.h:464
+    
+    Signed-off-by: Dunrong Huang <riegamaths@gmail.com>
+    (cherry picked from commit a7f520caabf997b3753f1200f19f21830531b3f5)
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit af5da4f24d7c2ea5a1322450180099278dd0da7c
+Author: Dunrong Huang <riegamaths@gmail.com>
+Date:   Wed Nov 25 18:24:30 2015 +0800
+
+    auth: fix double PK11_DestroyContext() if PK11_DigestFinal() failed
+    
+    Signed-off-by: Dunrong Huang <riegamaths@gmail.com>
+    (cherry picked from commit e9e05333ac7c64758bf14d80f6179e001c0fdbfd)
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit c3f031a32df5d434721a2b00f2c8961759b1a916 (refs/remotes/markhpc/wip-10794-hammer)
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed May 20 14:09:02 2015 -0700
+
+    ceph-objectstore-tool, osd: Fix import handling
+    
+    Use same_interval_since set to zero to communicate that an import has happened
+    and the correct value needs to be computed.
+    
+    Fixes: #10794
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 70316541bbb115d9a35954bfba373cf1dc084b7e)
+    
+    Conflicts:
+    	src/osd/PG.cc (trivial)
+
+commit 647723ecd283273657bdb8b35dbb904e55c37add
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Jun 4 10:37:42 2015 -0700
+
+    tools, test: Add ceph-objectstore-tool to operate on the meta collection
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 31fa89f24997c3e9b4f8795ea28938ab65493d10)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc
+    is_pg() still needs snap argument
+
+commit d8756208a3213e099574e57e9b654feab84b0497
+Author: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+Date:   Thu Mar 3 11:30:53 2016 +0100
+
+    common/obj_bencher.cc: make verify error fatal
+    
+    When run without "--no-verify", all verification errors are noted,
+    but they are not forwarded/reported anywhere else but to cerr, which
+    will cause automated testing to ignore them. Make seq_read_bench and
+    rand_read_bench return -EIO on any verification error which will,
+    in turn, return it back to caller.
+    
+    Fixes: #14974
+    Signed-off-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+    (cherry picked from commit 658bba85c488a38e25bc598637258c263f711631)
+
+commit 04fe9517cd049e97b845cef848f19a8b8dc3976c
+Author: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+Date:   Thu Mar 3 14:40:44 2016 +0100
+
+    test/test_rados_tool.sh: force rados bench rand and seq
+    
+    This forces rados bench rand and seq to execute, so if there's
+    a regression in any of above, it won't go unnoticed.
+    
+    Fixes: #14974
+    Signed-off-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+
+commit d314849ab9b59ae55c6a86b878d123eb1c13d88a
+Merge: 20b2a74 31a2fc4
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 3 08:35:37 2016 -0500
+
+    Merge pull request #7672 from dachary/wip-14783-hammer
+    
+    hammer: test_bit_vector.cc uses magic numbers against #defines that vary
+
+commit 6a6754f8dc834dbb13144cebdeeea2ff29e09127
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Fri Feb 26 15:01:11 2016 +0300
+
+    hammer: tools: fix race condition in seq/rand bench (part 2)
+    
+    Commit c2c6d02591519dfd15ddcb397ac440322a964deb which is intended to
+    cherry-pick 9bcf5f065c4ed4b10d8f98961d1f99493bcdc9b8 incorrectly resolved
+    a conflict by adding code where it should have been removed. The incorrect
+    conflict resolution can been displayed with
+    
+       commit=c2c6d02591519dfd15ddcb397ac440322a964deb
+       picked_from=9bcf5f065c4ed4b10d8f98961d1f99493bcdc9b8
+       diff -u --ignore-matching-lines '^[^+-]' <(git show $picked_from) <(git show $commit)
+    
+    --- /dev/fd/63    2016-03-03 14:09:51.354329129 +0700
+    +++ /dev/fd/62    2016-03-03 14:09:51.358329122 +0700
+    @@ -76,20 +79,18 @@
+          ++data.in_flight;
+     -    if (!no_verify) {
+     -      snprintf(data.object_contents, data.object_size, "I'm the %16dth object!", current_index);
+    --      lock.Unlock();
+    +       lock.Unlock();
+     -      if (memcmp(data.object_contents, cur_contents->c_str(), data.object_size) != 0) {
+     -        cerr << name[slot] << " is not correct!" << std::endl;
+     -        ++errors;
+     -      }
+    --    } else {
+    --        lock.Unlock();
+    --    }
+    --
+    -+    lock.Unlock();
+    -     name[slot] = newName;
+    -   }
+    -
+    -@@ -789,11 +791,14 @@ int ObjBencher::rand_read_bench(int seconds_to_run, int num_objects, int concurr
+    ++    if (memcmp(data.object_contents, cur_contents->c_str(), data.object_size) != 0) {
+    ++      cerr << name[slot] << " is not correct!" << std::endl;
+    ++      ++errors;
+    +     } else {
+    +       lock.Unlock();
+    +     }
+    +@@ -776,11 +785,14 @@ int ObjBencher::rand_read_bench(int seconds_to_run, int num_objects, int concurr
+            }
+            lc.cond.Wait(lock);
+          }
+    
+    http://tracker.ceph.com/issues/14873 Fixes: #14873
+    
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+
+commit 3a5b1029f8f30bca569b12195329224208855096
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Thu Mar 3 15:30:23 2016 +0300
+
+    hammer: tools: fix race condition in seq/rand bench (part 1)
+    
+    src/common/obj_bencher.cc:601: the lock should be taken before calling completion_ret,
+    not after. Also note that if r < 0 the lock will be unlocked twice in a row.
+    As a result rados bench seq fails with assertion in Mutex::Unlock().
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+    (cherry picked from commit 0c8faf7c9982c564002771c3a41362a833ace9bb)
+    
+    Conflicts:
+    	src/common/obj_bencher.cc
+    	src/common/obj_bencher.h
+      Pick only the lock related part to unbreak seq bench. The failure due
+      to the missing (or wrong sized) objects can be easily worked around, and
+      the changes required to fix this problem are way too intrusive for hammer.
+    
+    http://tracker.ceph.com/issues/14873 Related: #14873
+
+commit c4ba93a6529ab549d42b601432ec88d8243909bf
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Sep 29 15:24:41 2015 +0800
+
+    client: use fuse_req_getgroups() to get group list
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 0eb6d0ba1b65ac0aaffc72a82f07857cc65a6b88)
+
+commit a84ed87082076e1990fb79703aa5ac8eea5d34ea
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Sep 29 15:11:24 2015 +0800
+
+    client: use thread local data to track fuse request
+    
+    When handling an operation, libcephfs code may want to access fuse
+    request for extra information. by tracking fuse request in thread
+    local data, we can avoid adding extra parameter to Client::ll_foo
+    functions.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit a9f91d7d70893cdc5f832a26a5be298941d00315)
+
+commit e7f299ace762c7d02cad1f7253dba87404d944f5
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Aug 18 12:34:01 2015 +0200
+
+    client/Client.cc: remove only once used variable
+    
+    Fix for:
+    [src/client/Client.cc:4555]: (style) The scope of the variable
+     'initial_group_count' can be reduced.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit a29dd45dd89f59ff15018f541601ac5ede162174)
+
+commit 16e3e2f0a0dc465ed9079df051215f4758f0ee4d
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Wed Aug 12 18:38:38 2015 +0200
+
+    client/Client.cc: fix realloc memory leak
+    
+    Fix handling of realloc. If realloc() fails it returns NULL, assigning
+    the return value of realloc() directly to the pointer without checking
+    for the result will lead to a memory leak.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 4f98dab99c35663de89a06e2dfdbd874f56aed41)
+
+commit b13ddc0690742b11802e59f34b539987d632ac58
+Author: blinke <Burkhard.Linke@computational.bio.uni-giessen.de>
+Date:   Mon Aug 10 17:43:10 2015 +0200
+
+    client: added permission check based on getgrouplist
+    
+    Fixes: #13268
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit f04c8da5432174874ca97d11a5b2fef56f95d73d)
+
+commit 562c0a962b291b90112ccea7d8c6f7c92a14851f
+Author: blinke <Burkhard.Linke@computational.bio.uni-giessen.de>
+Date:   Mon Aug 10 15:31:44 2015 +0200
+
+    configure.ac: added autoconf check for getgrouplist
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 16b59c6cd68d532a47e8c0e7b1f088b41433f048)
+
+commit e014ea88861f4f27b4a5db14c32133a3b864411e
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Jul 23 15:07:45 2015 +0800
+
+    init-ceph: check if /lib/lsb/init-functions exists
+    
+    On OSX/FreeBSD, /lib/lsb/init-functions does not exist
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit bb1fa7f3993dfcd1243b2a9a0bb432c269df215f)
+
+commit 20b2a7413e8f7538ff0683313e7b19c41aaf69ad
+Merge: c1bc048 c63baeb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Mar 3 12:05:58 2016 +0700
+
+    Merge pull request #7671 from tchaikov/wip-14583-hammer
+    
+    hammer: global/pidfile: do not start two daemons with a single pid-file
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit c1bc048dd708a729137e1d28e0ccfedfe5d90933
+Merge: 546a7b9 28d42dc
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Mar 3 11:51:33 2016 +0700
+
+    Merge pull request #7702 from ceph/wip-14077-hammer
+    
+    hammer: scrub bogus results when missing a clone
+    
+    Reviewed-by: Martin Palma <martin@palma.bz>
+    Reviewed-by: Wei-Chung Cheng <freeze.vicente.cheng@gmail.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 5726463368ddcc688e6308d22992568ba4c0da32
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Feb 26 18:30:49 2016 +0100
+
+    packaging: lsb_release build and runtime dependency
+    
+    The lsb_release executable is being run in multiple places, not least in
+    src/common/util.cc, which calls it via shell in the collect_sys_info() code
+    path.
+    
+    This patch addresses this issue on SUSE- and Debian-derivatives, as well
+    as reinstating the dependency for RHEL/Fedora after it was dropped in
+    15600572265bed397fbd80bdd2b7d83a0e9bd918.
+    
+    http://tracker.ceph.com/issues/14906 Fixes: #14906
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit b87f4efb5689a8db77e06baf4f469afcd9df3677)
+    
+    Conflicts:
+    	ceph.spec.in
+                 The jewel specfile has diverged considerably from hammer:
+                 systemd, package split, etc. This is more of a hand backport
+                 than a cherry-pick.
+
+commit c63baebbb9bb5e6564955e920d0f658e5a3092af
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Feb 1 19:32:13 2016 +0700
+
+    global: do not start two daemons with a single pid-file (part 2)
+    
+    Fixes the following bugs:
+    
+    * the fd is open(O_WRONLY) and cannot be read from, safe_read
+      always fails and never removes the pid file.
+    
+    * pidfile_open(g_conf) is close(STDOUT_FILENO) and there is a risk that
+      pidfile_open gets STDOUT_FILENO only to have it closed and redirected
+      to /dev/null.
+    
+    * Before writing the file, ftruncate it so that overriding a file
+      containing the pid 1234 with the pid 89 does not end up being
+      a file with 8934.
+    
+    * Before reading the file, lseek back to offset 0 otherwise it
+      will read nothing.
+    
+    * tests_pidfile was missing an argument when failing
+      TEST_without_pidfile and killed all process with ceph in their name,
+      leading to chaos and no useful error message.
+    
+    * lstat(fd) cannot possibly return a result different from the one
+      obtained right after the file was open, stat(path) must be used
+      instead.
+    
+    In addition to fixing the bugs above, refactor the pidfile.cc
+    implementation to:
+    
+    * be systematic about error reporting (using cerr for when removing
+      the pidfile because derr is not available at this point and derr
+      when creating the pidfile).
+    
+    * replace pidfile_open / pidfile_write with just pidfile_write since
+      there never is a case when they are not used together.
+    
+    More test cases are added to test_pidfile to verify the bugs above are
+    fixed.
+    
+    http://tracker.ceph.com/issues/13422 Fixes: #13422
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 9828d49d6f3ccfc78d496153d263ea39b1722d4b)
+    
+    Conflicts:
+    	src/global/global_init.cc
+          - the `flag` argument of `global_init_prefork()` is not used, so
+            it was removed in master. but the cleanup commit was not
+            cherry-picked to hammer, thus the conflict. we can just keep it
+            around in hammer to minimize the code churn, although it may
+            stand in the way of future backports.)
+          - s/nullptr/NULL/ as hammer does not support c++11.
+
+commit 28d42dc8e0a0dd71d03c1b69fb7464b753bd4cc0
+Merge: 9144c38 9282c1d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:25:48 2016 -0800
+
+    Merge backport pull request #6545 into wip-14077-hammer
+    
+    A commit skipped because remove_coll() still needed in Hammer
+
+commit 9144c3837f284ac2c922d2438b0e0f0cc7304d7a
+Merge: fd74fbf 3226615
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:25:35 2016 -0800
+
+    Merge backport pull request #5783 into wip-14077-hammer
+
+commit fd74fbf04a965ca93dc9602f99e3cb7a3577340a
+Merge: 4309fa8 0fe3dfe
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:25:17 2016 -0800
+
+    Merge backport pull request #5031 into wip-14077-hammer
+
+commit 4309fa87e13a20368ca34f7699436b628c9d7876
+Merge: de5e11e 1bc8882
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:24:58 2016 -0800
+
+    Merge backport pull request #5842 into wip-14077-hammer
+
+commit de5e11ec69e7388a1a9c28614759b2e2d8b70ad8
+Merge: 7698e6c 6ff4217
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:24:31 2016 -0800
+
+    Merge backport pull request #5127 into wip-14077-hammer
+
+commit 7698e6c54aae1d06664d8e8324595400e6087f24
+Merge: c8057c6 de80bbf
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:23:14 2016 -0800
+
+    Merge backport 6 commits from pull request #5197 into wip-14077-hammer
+    
+    Excludes the ceph-objectstore-tool meta-list op
+
+commit c8057c693bf0ae7bb01ef238f4ef9f23454a2e33
+Merge: ecbf6d2 df0e11e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:22:26 2016 -0800
+
+    Merge backport branch 'wip-temp' into wip-14077-hammer
+
+commit ecbf6d2882073363ca1ddc5209e1e9537f51fa77
+Merge: 3415312 6c8884b
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:21:20 2016 -0800
+
+    Merge backport pull request #4932 into wip-14077-hammer
+
+commit 34153127963b5baec0c2b84f7aa19b29fc81d4fa
+Merge: e022b5b 87a7f99
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:20:41 2016 -0800
+
+    Merge backport pull request #4915 into wip-14077-hammer
+
+commit e022b5b55facb359f6fd27884e2408fda70db30a
+Merge: 4ed44e1 9ca2f35
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:20:00 2016 -0800
+
+    Merge backport 1 commit from pull request #4863 into wip-14077-hammer
+
+commit 4ed44e1e33df986468585d00a6dc401025263217
+Merge: 3f0fc84 78a59f8
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:18:38 2016 -0800
+
+    Merge backport 8 commits from pull request #4784 into wip-14077-hammer
+    
+    Only excludes actual fix for #10794
+
+commit 3f0fc84c4180473a87db01cb40d0488bcaa2c202
+Merge: db68c6c eab0f24
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:15:05 2016 -0800
+
+    Merge backport 41 commits from pull request #4473 into wip-14077-hammer
+
+commit db68c6cb683870e68bf6143c6902b8ac89ecdc3a
+Merge: c3d938d 18c49b6
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:13:37 2016 -0800
+
+    Merge backport branch 'wip-journal-header' of git://github.com/XinzeChi/ceph into wip-14077-hammer
+    
+    Includes a change to the code in the original merge commit itself
+
+commit c3d938d81020072c2d2eecf58f4639e773181b1a
+Merge: 7abb6ae a1fc101
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 25 14:13:02 2016 -0800
+
+    Merge backport 1 commit of pull request #3686 into wip-14077-hammer
+
+commit 9282c1d01c4f2d99fb3832a3cf495f99fdc6487e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Nov 5 19:06:17 2015 -0800
+
+    ceph-objectstore-tool: Add dry-run checking to ops missing it
+    
+    Update test to check most operations
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit e89f0e1c5e8a378809bec7c705ea2c23235d4b28)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc (trivial)
+
+commit efc2183bafacfcd29fbb1b42231d4c147199e528
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Nov 11 15:26:52 2015 -0800
+
+    test: Remove redundant test output
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit e6f1bdcd310bd6417f0f695fba3430fd0edba43d)
+
+commit 322661547ec437a9cdb893fcd280a48cd60ec990
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 28 21:34:44 2015 -0700
+
+    test: osd-scrub-snaps.sh uses ceph-helpers.sh and added to make check
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit d712737be1aadb38ecc3882200999d79d748cdb8)
+    
+    ceph-helpers.sh in different location in Hammer
+
+commit 995a0042a1b8ed04b77e6bc4757a10552450dc7f
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Nov 10 18:20:46 2015 -0800
+
+    test: Verify replicated PG beyond just data after vstart
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 9a29c5983cd392e4b479b6b685fb0c3797c10cd2)
+
+commit 6afb5d3be26931c670f22c892c925e2556c8e562
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Oct 26 11:36:43 2015 -0700
+
+    osd: Use boost::optional instead of snap 0 for "all_clones"
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 310bf784b7cd8d75f2ab7ee431bc764f5b312aa2)
+
+commit 750f81732bc803128a8a25c07e10e9f709dc2a1a
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Nov 10 17:57:59 2015 -0800
+
+    test: Fix verify() used after import to also check xattr and omap
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 92d00f22805599c519806c6ff6164847b5c33931)
+
+commit b8c9507f632510d7da8545d7afc49b618435ef3c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Sun Oct 25 18:57:18 2015 -0700
+
+    osd, test: When head missing a snapset, clones not an error
+    
+    Fix log message for this case and make into "info"
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit f508ddc6e9b40fe5c1c54e5faa569f2bc7b78c4b)
+
+commit 59fee8a65c1a39d7e3806852aabbf68d7ca37feb
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Nov 9 20:02:21 2015 -0800
+
+    test: Add test cases for xattr and omap ceph-objectstore-tool operations
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit cbaed6a8c737f77e68a0eb7f1fca32ec4155f045)
+
+commit 0988b1238d43630ff0b05d321952021ede73c2ae
+Author: David Zafman <dzafman@redhat.com>
+Date:   Sat Oct 24 16:23:13 2015 -0700
+
+    osd, test: Keep missing count and log number of missing clones
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 2e0bb0ae9498fd9a0c49f1b81bf0c86f7a7cac42)
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.h (trivial)
+
+commit 37be95996494f72d4fa3ecc62b54ee9478903c6e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Nov 11 08:28:01 2015 -0800
+
+    rados: Minor output changes for consistency across operations
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 016ed34ce85fa111f54857a8ca1002ae32b52b8e)
+
+commit 6c51e4831f34a67308e3f8190d9aad2c3d029bf7
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Oct 20 11:14:42 2015 -0700
+
+    test: Eliminate check for bogus "obj13/head snaps empty" error
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 8227b4b75c24b632e637decbe4b818606e8589ee)
+
+commit e92505bec093cb3070e4962bc86b82a257c64171
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 14 20:34:28 2015 -0700
+
+    ceph-objectstore-tool: Add new remove-clone-metadata object op
+    
+    Fixes: #12740
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit c6d283f0f16347a13418d8a0ed136913e1161893)
+    
+    No Sequencer in transactions in Hammer
+
+commit 8f88b442a26603cc321fccc7047cb8639b2a0f65
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 14 20:42:48 2015 -0700
+
+    osd: Fix trim_object() to not crash on corrupt snapset
+    
+    Add test case to check for trim_object() osd crash
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit eb0ca424815e94c78a2d09dbf787d102172f4ddf)
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.cc (trivial)
+
+commit 78b13f5d591d9d9676939abc4febeb7d2eebb1a3
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 14 13:12:17 2015 -0700
+
+    ceph-objectstore-tool: Improve object spec error handling
+    
+    Add test cases
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit caf2d5970b0a3e3ce4ef941d1955b7821a0fc916)
+    
+    Conflicts:
+    	src/test/ceph_objectstore_tool.py (trivial)
+
+commit 7b800b7c3efb5403d03fc0c4386821b433b85268
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 7 17:25:44 2015 -0700
+
+    ceph-objectstore-tool: Add undocumented clear-snapset command for testing
+    
+    Add test cases using new feature to corrupt SnapSet
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b6302acdd344997aabdb426104de9f24e9990437)
+    
+    No Sequencer in transactions in Hammer
+
+commit 7f398bd2a3264d1ac5bf40a5470c43454dbad571
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 14 21:59:50 2015 -0700
+
+    ceph-objectstore-tool: Add set-size command for objects
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 138a33b8946e6fea1e09527b8bc45b7aa4bee1d6)
+    
+    No Sequencer in transactions in Hammer
+
+commit 53dc87f82642de769a7ea11554552e6db06a3e65
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 14 22:04:05 2015 -0700
+
+    ceph-objectstore-tool: Enhanced dump command replaces dump-info
+    
+    Show object stat information from objectstore
+    Show SnapSet if present
+    Add --head for search by object name
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 1688debf7892d36c773c12d7d0b1b9b4cddc5f98)
+
+commit a46fc6613e1f1f6709184d68f23bccab0fd03bd6
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Oct 2 13:39:29 2015 -0700
+
+    test: Add some clones to ceph-objectstore-tool test
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b4ba3e66548a5dee58553df071b916c85d701caf)
+
+commit fd518e704e0e5c01f3b7efa8905ddde1e64cc842
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Oct 1 19:28:54 2015 -0700
+
+    ceph-objectstore-tool: For corrupt objectstores, don't abort listing on errors
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit d276d3286504ea702ae561b91dff4121c99feb9a)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc (trivial)
+
+commit ad7825a4beba879f6a38ef3eda0648f11f8783bd
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Oct 13 13:02:40 2015 -0700
+
+    ceph-objectstore-tool: Improve some error messages
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b0c884ba8b51eedf9def093c61f0f06398101ecc)
+
+commit 26cbf14ef1e44936f48fdb2be40b59ae2117666f
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Sep 3 14:22:20 2015 -0700
+
+    ceph-objectstore-tool: White space fixes
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0564f398269d920d90f6d6ecb008ebcd6609d611)
+
+commit 0f785646d2f52522706835a2f1c8e0101db9a622
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Oct 13 15:33:26 2015 -0700
+
+    tools/rados: Improve xattr import handling so future internal xattrs ignored
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 9222f56403cf52ec161a38fc437a3f04c24d1625)
+    
+    Conflicts:
+    	src/tools/rados/RadosImport.cc (Code in ceph_objectstore_tool.cc in Hammer)
+
+commit c8e27725249cf587e65af323e3ae0ba43c740c10
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Sep 29 19:57:43 2015 -0700
+
+    test: Test scrubbing of snapshot problems
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit e5ad33ec07ee9fffe862d84955a566261d101713)
+
+commit 113d5c75a5bd33b1b5fe399d83d94a671eb40658
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Sep 28 10:59:42 2015 -0700
+
+    osd: Don't crash if OI_ATTR attribute is missing or corrupt
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit e0b39650028205e9d9e314b75c444cc8cf055a31)
+
+commit 3af8f9e8044fb1e126f208903c41cde065db304e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Oct 13 18:02:32 2015 -0700
+
+    osd: Additional _scrub() check for snapset inconsistency
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 9e48e1801ea1a10ce495b3947da06676c2f5f72f)
+
+commit 7103e74e335652f2a28b1dad0a2f2028398a600d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Sep 3 19:43:35 2015 -0700
+
+    osd: Better SnapSet scrub checking (find issues instead of asserting)
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 3b381caaad20c683a330e8b7a4e1c017abcb60df)
+
+commit 18af852a2c71b733ec5c6bd397b2e712b676c693
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Aug 26 13:58:09 2015 -0700
+
+    osd: Make the _scrub routine produce good output and detect errors properly
+    
+    Catch decode errors so osd doesn't crash on corrupt OI_ATTR or SS_ATTR
+    Use boost::optional<> to make current state clearer
+    Create next_clone as needed using head/curclone
+    Add equivalent logic after getting to end of scrubmap.objects
+    
+    Fixes: #12738
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit a23036c6fd7de5d1dbc2bd30c967c0be51d94ca5)
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.cc (no num_objects_pinned in hammer)
+    	src/osd/ReplicatedPG.h (no get_temp_recovery_object() in hammer)
+
+commit 3a1b58838dba6bd482aa5786dc44962c4a977c17
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Sep 30 17:17:15 2015 -0700
+
+    osd: Fix log message name of ceph-objectstore-tool
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 545e4b2c582625f8b93502266ca59ee40e0d0e5f)
+
+commit 0fe3dfe8768b27743189add73cde9a44095f2a62
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Jun 16 11:57:08 2015 +0300
+
+    ceph-objectstore-tool: add mark-complete operation
+    
+    It is supposed to be used as a last resort to fix a cluster that has
+    PGs in 'incomplete' state, using the following procedure:
+    
+    1) stop the osd that is primary for the incomplete PG;
+    2) run:
+      ceph-objectstore-tool --data-path ... --journal-path ... --pgid $PGID --op mark-complete
+    3) start the osd.
+    
+    Fixes: #10098
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 6907778d767ba08bb80c495785056ed122b023fe)
+    
+    Conflicts:
+    	src/test/ceph_objectstore_tool.py (trivial)
+    	src/tools/ceph_objectstore_tool.cc (trivial)
+
+commit 1bc88827fc40f124f570dd8b7865c687f1b1f5a8
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Sep 8 12:33:44 2015 -0700
+
+    test: Fix failure test to find message anywhere in stderr
+    
+    Consolidate test_failure() and test_failure_tty()
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 95bd3c2c54312417b132ddb91c89fdbe63b01fd0)
+
+commit 6ff421738c93ff37521ccf67caf3d09d284e339c
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Jul 17 14:46:16 2015 +0800
+
+    test: add test for {get,set}-inc-osdmap commands.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 818de1a2c89272d98610328a958fea06d1cc41cc)
+    
+    Conflicts:
+    	src/test/ceph_objectstore_tool.py (trivial)
+
+commit de80bbf4d7c1b78017cf447a7f511ceefac39e50
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Jun 23 16:51:27 2015 -0700
+
+    test: Add debug argument to the ceph-objectstore-tool test
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit a3b14ede9e829ec85c49980ecbff11f584ca70ae)
+
+commit 0643797cb7a37b5601a7dd8da9a81f8194b8f25d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Sep 8 13:41:31 2015 -0700
+
+    rados: Fix usage for "notify" command
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b968fb3bcefb0cdafb1e84d7c679b61a201a940f)
+
+commit 5ba8649423fd5a2d5c212273c901f9ddd5f44632
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Jul 8 15:51:55 2015 +0800
+
+    test: add test for {get,set}-osdmap commands
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 0257c150fe552300c0d9f5a22a27aaa49fb6cc64)
+    
+    Conflicts:
+    	src/test/ceph_objectstore_tool.py (trivial)
+
+commit 3276258b1a9123caec8d635aa1a2de3ef02201e6
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Jun 25 17:15:39 2015 -0700
+
+    tools, test: Some ceph-objectstore-tool error handling fixes
+    
+    Improve various error messages generated with invalid syntax
+    Add test cases for most of these error messages
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 127a5f191d3d2442edd3a3fa820006bca3a08c93)
+    
+    Conflicts:
+    	src/test/ceph_objectstore_tool.py (trivial)
+
+commit cfabcc1066a06644b619fa027e27d338411120c9
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jul 14 10:33:47 2015 +0800
+
+    tools/ceph-objectstore-tool: add get-inc-osdmap command
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 7801bb20d23ce83fc4093c23331bcb75e43c878f)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc (trivial)
+
+commit c7d0fda01efc29d81723d4edfba2bfbbe6765760
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Jun 22 20:11:10 2015 -0700
+
+    tools: Check for valid --op earlier so we can get a better error message
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit d8460714fab61d0a3092b3e7b290c0c19ad58beb)
+    
+    Conflicts:
+    	src/test/ceph_objectstore_tool.py (trivial)
+    	src/tools/ceph_objectstore_tool.cc (meta-list op not in Hammer)
+
+commit be24c50bb74aa6cd7bb08519cdaf267d4107eef3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Jul 11 16:27:47 2015 +0800
+
+    tools/ceph-objectstore-tool: add set-inc-osdmap command
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 95344d07c3a1a9a6b1246e5eca3f7daeb062dd06)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc (trivial)
+
+commit 06dcf74374f57224536522ef46dba8a53a0d755d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Jun 10 17:04:57 2015 -0700
+
+    tools: Fix newlines in output of --op list
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 6aec38b51f0c14684912b5ee28fb5b1375cb8f8a)
+
+commit e44c042c3dffc4ca396dbdfac24fa65914c0b73b
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Jul 3 23:09:54 2015 +0800
+
+    tools/ceph-objectstore-tool: add "get-osdmap" command
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit f0c130d6b1bc2a8a0cce6d78a6e69995fe00a427)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc (trivial)
+
+commit 3f9e46779addaa07d04d501113eb95cea95b802b
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Jun 22 18:38:01 2015 -0700
+
+    tools: Fix dump-super which doesn't require pgid
+    
+    A valid pgid had to be specified in order for superblock to be displayed,
+    but it was otherwise ignored.
+    Fix message that didn't include dump-super
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit fd1772e0ac4cb801d6f3b056645eeb4d3d6ef001)
+
+commit c60eee1dcd64d4452e96ed937f94a3ec41faf9fe
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Jul 2 19:14:55 2015 +0800
+
+    tools/ceph-objectstore-tool: add "set-osdmap" command
+    
+    Fixes: #12193
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 3e30c1746fb8d90b04e4776849069db0b7737c87)
+    
+    Conflicts:
+    	src/tools/ceph_objectstore_tool.cc (trivial)
+
+commit cfe7d47d38fd103601c09eceb98b9c7365869fba
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Jun 22 19:16:22 2015 -0700
+
+    tools: Check and specify commands that require the pgid specification
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 5534bc864813d96a0d20791d79151c4f6695a013)
+
+commit df0e11e7783e239ae7be992b342df980d399ab95
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 11 16:25:49 2014 -0800
+
+    hobject_t: modify operator<<
+    
+    Put the most significant fields to the left so that it matches the sort
+    order.  Also use unambiguous separator when the nspace is present
+    (like we do with the key).
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ecdc8f697acf1627b35a83663c219a3bb0ada453)
+
+commit 6c8884b11b8211642662bfd7d612872621ffd8ff
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Jun 11 12:57:10 2015 -0700
+
+    test, tools: Improve ceph-objectstore-tool import error handling and add tests
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit ddc4d52782a04d99a4293c9fc278894dfe328515)
+
+commit 87a7f991d1f2f1a05fff18accc7e31ff393083f1
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Jun 9 18:14:09 2015 -0700
+
+    tools: For ec pools list objects in all shards if the pgid doesn't specify
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 1aa9655f10b0fcc42cb5114681b8b24a2ccb68f4)
+
+commit 9ca2f35d02169e6f82e3d45e00fe880c02cbd8ce
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Apr 20 17:28:15 2015 +0100
+
+    tools: clean up errors in ceph-objectstore-tool
+    
+    We use negative error codes everywhere else, should
+    use them here too.  Also add a couple of strategically
+    placed prints.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 8e5594da29e782b6efd046ed03f313599dafa1c5)
+    
+    Conflicts:
+    	src/tools/RadosDump.cc (Excluded, not in Hammer)
+
+commit 78a59f8b0c43adbd9a7a1db68dafcbf7bd2ad74d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Jun 1 17:22:18 2015 -0700
+
+    test/ceph-objectstore-tool: Don't need stderr noise
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit d7858c66454093dd6f2fab60e33e5b2472bcd80a)
+
+commit eab0f2403ea025f4e659dc055323b227023448ab
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Apr 9 16:26:07 2015 -0700
+
+    osd: Show number of divergent_priors in log message
+    
+    Report dirty_divergent_priors as true or false
+    Use dout(5) so tests can verify divergent_priors created
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 67da8feb0ff8659f1ff812be5a866ef5e97facf2)
+
+commit d58793d10d682fba2ceafa4d6a1668c4917be77f
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Jul 6 18:14:06 2015 -0700
+
+    osd, tools: Always filter temp objects since not being exported
+    
+    Can't use object_locator_to_pg() directly with a temp object
+    because the pool is negative.
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 557c6530a515cffacbaec439cabff8a079edbf5b)
+
+commit efc402ea9846d3995bf3d4c98188842cb69870cf
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Jun 1 17:15:21 2015 -0700
+
+    test/ceph-objectstore-tool: Show command that should have failed
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 23551deea66673c35d7183d18b63b5a8438ebf00)
+
+commit 88ac5198f542cdea87d0f600a2201132c47f158d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue May 12 15:28:07 2015 -0700
+
+    test: Add config changes to all tests to avoid order dependency
+    
+    ReplayCorrupt was crashing because journal_ignore_corruption wasn't set true
+    Improve ReplayCorrupt by checking setting of bool corrupt and use ASSERT_FALSE()
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 9016269a1c248da69a0e85df4ca3ba95011710c5)
+
+commit 3d99ecd6131f41ebe60c352266dca9193d61bbc9
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Jun 22 11:15:58 2015 -0700
+
+    tools: Don't export temporary objects until we have persistent-temp objects
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 4fcf5dd370e8cfac75fadbf709a785053eb26ad9)
+    
+    Remove "assert(!i->hobj.is_meta());" since is_meta() is not in Hammer
+
+commit 13360d3a55818fc5b63a2bb9889a67f3fc98d286
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri May 22 17:41:40 2015 -0700
+
+    test/ceph_objectstore_tool: Improve dump-journal testing
+    
+    Fix false failure when journal has no entries
+    Test dump-journal on all osds in 2 places
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit ce76f88777ec026521cce465095d6376efd460a8)
+
+commit 444ce0ac718e903b39c25cf58d0a503ead491790
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri May 22 11:52:00 2015 -0700
+
+    ceph-objectstore-tool: Allow --pgid specified on import (for post split)
+    
+    Revamp split import testing to use --pgid option post split
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 90202a70f8c32120625c99ef5131778b3c9f6303)
+
+commit aed1c49e7d3631d797776be1020f6df0f77ea1f3
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri May 22 10:53:48 2015 -0700
+
+    ceph-objectstore-tool: Invalidate pg stats when objects were skipped during pg import
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 51e27984ec131fd21bbc26411ae1540b9047eb37)
+
+commit af3f8ae1d9e210499fd0165e473f9b5d87666f75
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon May 18 20:22:03 2015 -0700
+
+    ceph-objectstore-tool: Add dump-super to show OSDSuperblock in format specified
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 99d7663f870abfa82711dbf0451334f7d6c883d8)
+
+commit 4dcf15bc71c815cfdb0f6bc6f90c9b603d18b37f
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon May 18 20:25:23 2015 -0700
+
+    mds, include: Fix dump() numeric char array to include additional alpha chars
+    
+    Caused by: c87f55b5a8b3e003726e1be22b9cb6894ef2f37d
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit f1520061bba1e052a9be3ae11161122c1f6c3425)
+
+commit feecacfb34dfd831291fc736ea39dbcf69e7d765
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon May 18 20:08:14 2015 -0700
+
+    ceph-objectstore-tool: Add dump-journal as not requiring --pgid in usage
+    
+    Additional comment describes --pgid requirement checking
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit bd2f62445be7153a9600bcf4399e8fc56f1efd3f)
+
+commit 5e8fbb1858f7a8dd65d1b9f873aaca05f3ddb1eb
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Mar 5 16:37:46 2015 -0800
+
+    test: ceph_test_filejournal: Conform to test infrastructure requirements
+    
+    Only call RUN_ALL_TESTS() once
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit e30efe6c15780a6239b9970297df1f2c1918c9b1)
+
+commit c161cbf81a29e523bb4ffb897f6d02112eb49b09
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Mar 5 16:13:50 2015 -0800
+
+    test: ceph_test_filejournal need to force aio because testing with a file
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit d540dde1faee47d01ee358ae384fa301fa59f00f)
+
+commit 06d3f512149dc9aa38d959ac31891b81634d0ea4
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Mar 5 16:04:36 2015 -0800
+
+    test: ceph_test_filejournal fix missing argument to FileJournal constructor
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit c94fe8db36e7d54227edde94a99bc1aa236b8ec8)
+
+commit 2078f639d91fbaa9b89832811315c7c41da25a4b
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 19 19:48:11 2015 -0800
+
+    test: ceph_test_filejournal  Add check of journalq in WriteTrim test
+    
+    Commit all journal entries and check for empty journalq
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 8da4d9f6330c57d6b437ea44f54f147075e24195)
+
+commit ab893d769d2728368f94b035edde750a3fa5e617
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Jan 29 17:14:39 2015 -0800
+
+    test: Fix ceph-objectstore-tool test missing fd.close()
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit ece630d5a741ba8c82f55f964f1887cb7fcaa493)
+
+commit b5f2ccd5bd77c7a473f38fdab183fd52577b7331
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Jan 29 17:13:41 2015 -0800
+
+    test: Fix ceph-objectstore-tool test error message
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 11c13eb7c182870727fbcb25a83f3348ac0bad13)
+
+commit 848822db38e000a7e7bfe49ae0fe03996bb7e8c1
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Apr 1 10:32:32 2015 -0700
+
+    test: ceph-objectstore-tool: Remove duplicate debug messages, keep cmd/log/call together
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 94b3d7632f2548c19beca5a2e024c95f601b61f3)
+
+commit 771dcd945281a091b165c0b1a90c1f6058dbf923
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Mar 27 19:47:01 2015 -0700
+
+    test: ceph-objectstore-tool import after split testing
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit d80ce6668c1aa3c9898ab49e847c4dadafe256f0)
+
+commit 4f387b1e1c8dbfe9ed2a70fc02ff5ccf93ffb905
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Mar 31 08:16:03 2015 -0700
+
+    test: Use CEPH_DIR where appropriate
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 1e18e8a88f88809081684b09f36691cd724720b5)
+
+commit b337d677daf5b6098a2dbcf2d8966703cf8db449
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Mar 30 17:35:07 2015 -0700
+
+    test: Limit how long ceph-objectstore-tool test will wait for health
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 868cf457a3990ba9989aef55823630e3189acf4a)
+
+commit 09cb8a41ed6a9642978e399b426b21704c704f46
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Mar 30 17:33:46 2015 -0700
+
+    test: Add optional arg to vstart() to provide additional args to vstart
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit dff17a800c9cc1cf0cc23d70479c031aa8e9cbfc)
+
+commit b4ac42be79bb11cb27282d97aec9e6ca8f658e5c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Mar 5 19:35:27 2015 -0800
+
+    test: Test ceph-objectstore-tool --op dump-journal output
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 9b08bcf95caf77fb7f9bd293559534ed8ff3eded)
+    
+    Conflicts:
+    	src/test/ceph_objectstore_tool.py (trivial)
+
+commit 729abf5a83a656ac2a95ad147909bf7c8fa18999
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Mar 17 14:56:06 2015 -0700
+
+    test: Pep8 fixes for ceph-objectstore-tool test
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b0141108c0596b535d25792a9102787b1cd3eca3)
+
+commit 33813b66fa4402fe0d66d95d6f9b790b11c9f1f1
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Mar 5 18:37:44 2015 -0800
+
+    test: Fix ceph-objectstore-tool test, overwrite OTHERFILE so second check is meaningful
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 1b9f85befa1efff3b05b2af0e62d568eea37d790)
+
+commit f7ab316be9ce3592e0bc1c3a1012b0a83dfea2b4
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Mar 5 19:16:45 2015 -0800
+
+    osd: FileJournal: Add _fdump() that takes Formatter instead of ostream
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 130eba48495d0e6acd8a472cbf0c47ebbc7a038c)
+
+commit 99d3e17fe8c03eb30b18dee185623a9c16553d9e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 18 12:34:35 2015 -0800
+
+    osd: Add simple_dump() to FileJournal for unit testing
+    
+    Allow a dump when entries aren't ObjectStore::Transaction's
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 8d5464c0f81aa9bb3be233a03bce64b505a265f4)
+
+commit 80fc57f79a9bd23e119be73fe01063e85c60ca83
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 18 12:29:38 2015 -0800
+
+    osd: FileJournal clean-up
+    
+    Move aio / directio adjustments to constructor
+    Indicate functions which only operate before journal is active
+    Make functions const when possible
+    Move push_back() of journalq to read_entry()
+    Change dump() to use do_read_entry() to minimize side effects
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 12fdf4ced0b2c17770c21204c8eccca8e4d0d2c9)
+    
+    Conflicts:
+    	src/os/FileJournal.cc (trivial)
+
+commit b8f4ea17274f9094efa169a28440701319c14516
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 5 18:48:22 2015 -0800
+
+    osd: Dump header in FileJournal::dump()
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 3c97b32131e83c934b3d340eeb75284e898380c4)
+
+commit 21c3c18c7b25ddea664179797527ea9854c3f450
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 5 18:40:25 2015 -0800
+
+    osd: FileJournal::read_entry() can't use a zero seq to check for corruption
+    
+    Seen when using dump() on a journal which has all entries committed
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit a7cda53d20eb6d35203fce6ec9f7ea1df432bf53)
+
+commit 288902fecd78b1926aa4a1d53587828fda63845d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 5 18:36:01 2015 -0800
+
+    osd: Fix flushing in FileJournal::dump()
+    
+    Use ostream arg since caller is specifying
+    Move flush after last section close
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit f2d83520f67fdc1b5a89b4ea742493a3df392989)
+
+commit a935ce501692f51be004f8cb20da29f18e649d44
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Apr 30 10:20:24 2015 -0700
+
+    osd: Add admin socket feature set_recovery_delay
+    
+    Set the value of osd_recovery_delay_start to specified seconds
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 73b17319379d904ffdf9479ca476da8e8e34e935)
+    
+    Conflicts:
+    	src/osd/OSD.cc (trivial)
+
+commit 4ae3f88cc4493413c4aab54e21759507b0d572b0
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Apr 27 15:37:14 2015 -0700
+
+    ceph-objectstore-tool: For import/export --debug dump the log
+    
+    Move log dumping code to common function dump_log()
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 545c78c65f33fcac60b5aa7ee2b574ceb0211fd5)
+
+commit cc5fa68c184f66d3ca6ff736eb308cba0a0d10b0
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Jan 30 16:56:10 2015 -0800
+
+    ceph-objectstore-tool: If object re-appears after removal, just skip it
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit c6e41663e7434b4dcbb1abbe62ac008388da268a)
+
+commit d8ae1a9b842b6871225f0cc2d8a8dbdcf01238e4
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Jan 30 16:11:13 2015 -0800
+
+    ceph-objectstore-tool: Add --no-overwrite flag for import-rados
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 35cdcabbdad7dc7249029630c64022c822c67ddc)
+
+commit 2dbf843560de2b9b74d9b0d0baf2a303e6c4e86b
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Feb 2 16:23:18 2015 -0800
+
+    ceph-objectstore-tool: Remove list-lost because now we have --dry-run flag
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 3e7fe7e85716f80e055ffbc5305f406b3e020608)
+
+commit 35991747c4cf5c4f9c150aae128f63c7b8c8a5cd
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Jan 23 07:34:11 2015 -0800
+
+    ceph-objectstore-tool: Add --dry-run option
+    
+    Fixes: #10290
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 2795161f470ab87cf4a82f455773c9b1b82946ef)
+
+commit 05d3b73ac6ef841af5037336996d1ed6bfdac4b5
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Jan 22 10:07:09 2015 -0800
+
+    ceph-objectstore-tool: Add dump-info command to show object info
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 244092d034781d2e980f9de7c6f9eea227967c5a)
+
+commit 2d764c54d4b028455be8b592712c6424f5f9f19e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Jan 27 14:32:42 2015 -0800
+
+    ceph-objectstore-tool: Use empty string for <object> to specify pgmeta object
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit c00290b836adba12212e8a57f13b0c9193ac0ef8)
+
+commit 3a533d767bafc466e653bdd89d064b5f2b9aa257
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Apr 20 17:28:15 2015 +0100
+
+    ceph-objectstore-tool: Add a couple of strategically placed prints
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit fdb1a4b676765c5d3ca5d436f34230dd6996d7a7)
+
+commit 7947f4f72f4bb6441ea5c353757d9ea3f464cff4
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Apr 20 16:10:18 2015 -0700
+
+    ceph-objectstore-tool: Clean up error handling
+    
+    Use negative errors throughout and positive error to specify exit status
+    cpp_strerror() handles negative errors
+    Clean up main() code to use local "ret"
+    Let end of main() decide how to treat the final value of "ret"
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit a50679a1a3313e4090bf24f6e4dda28e7d67d731)
+    
+    Adjust back cherry-pick f237ed97228839a1b412ad213945f0343df05bf5
+        Switched to use ret as in the original change from master
+
+commit 83de86e4e9dedc05fd65201715dadbd46074b7e0
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Apr 15 15:40:41 2015 -0700
+
+    ceph-objectstore-tool: Create section around log/missing/divergent_priors of --op log
+    
+    Remove extra new lines in the output
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 548147b412eebe57861768e5fd66360c2582d2c6)
+
+commit ddfaa70ade3b6d7ff061083fcd610b588fbd4dd7
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Apr 14 13:49:26 2015 -0700
+
+    ceph-objectstore-tool: Add divergent_priors handling
+    
+    Add to --op log output
+    Add to metadata_section for export
+    Add filtering of divergent_priors in case of splits
+    
+    Fixes: #11237
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 6ad7e9b362f437fe028fe59639489efe87b3525f)
+
+commit add937c437a4bd86c932ef49525aff039d9bee90
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Mar 18 10:05:24 2015 -0700
+
+    ceph-objectstore-tool: Add --force option which is used for import only
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 2f4090daaea715b896e49f678933abb8a13b5c98)
+
+commit f33274894a83072a483b0646a91b5182a72c7c90
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Mar 13 00:41:39 2015 -0700
+
+    ceph-objectstore-tool: Fix pgid scan to skip snapdirs
+    
+    Long standing bug where it wasn't skipping snapdirs if !debug
+    For debug output use stderr like all other cases
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 24f06598966319a62fa58a5831d9741fac1d471b)
+
+commit 3e68825e60a2e69a20198439347aadf8742d4ea6
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Feb 9 12:24:19 2015 -0800
+
+    ceph-objectstore-tool: Add dump-journal op
+    
+    Have dump-journal op directly dump a filestore journal without
+    mounting or try to dump after mounting.
+    
+    Fixes: #11135
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 381605e34a44edb9acfcfb5a56bd598b020e022f)
+
+commit aaff4d7bcbc3e48aa6c0342d66a9c30c93744064
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Feb 9 12:46:20 2015 -0800
+
+    ceph-objectstore-tool: On any exit release CephContext so logging can flush
+    
+    Add myexit() which will put() the g_ceph_context if present before exit
+    Must umount() once the objectstore has been mounted
+    
+    Fixes: #10809
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 409328bace0ffb3a457c0d384377bbbf37a2962d)
+
+commit 7445cf5a04d396dbc36e21aecd44bc4aadaff397
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Feb 9 17:28:04 2015 -0800
+
+    ceph-objectstore-tool: Check for keyvaluestore experimental feature
+    
+    Don't list keyvaluestore as a valid type if feature not enabled
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit fb2f9d500e52aa0834666550a684fe991701c9dd)
+
+commit 9da6c01b5e59e59a80d00e3d17cf5da05dd84f72
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Feb 9 17:14:50 2015 -0800
+
+    ceph-objectstore-tool: Eliminate obscure "Invalid params" error
+    
+    Will fail later with "No such file or directory" for an empty data-path
+    because we've already checked for vm.count("data-path")
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit cfca6835bd53d535bde2793551211845b0facebd)
+
+commit c5ac7ce09bc5b96722738a1884c6046aba6518f2
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Feb 9 15:37:57 2015 -0800
+
+    ceph-objectstore-tool: Check pgid validity earlier like we did before
+    
+    This code was not doing the umount() even though it was moved after mount()
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 565b413df9c562a4071b9372d6362a9105c3538e)
+
+commit 18c49b6474a563b5f7f7cef671050d46ac4c3c4f
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri May 1 09:22:01 2015 -0700
+
+    Backport the merge commit of branch 'wip-journal-header' of git://github.com/XinzeChi/ceph
+    
+    (cherry picked from commit 8d596bfa24c9c3e2aadbb87494284cb5fd824e55)
+    
+    Conflicts:
+    	src/os/FileJournal.cc (trivial)
+    	src/os/FileJournal.h (trivial)
+    
+    Renamed write_header() to write_header_sync() in a merge
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+
+commit 546a7b988dac56770730ddc9395ee0d52a666fa3
+Merge: 6b7660c 3352b14
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 25 22:39:06 2016 +0700
+
+    Merge pull request #7656 from dzafman/wip-14769-hammer
+    
+    ceph.in: Notify user that 'tell' can't be used in interactive mode
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 6b7660c91d1837b585b365068da835127752ebfc
+Merge: e34c209 14b5fea
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 25 22:37:58 2016 +0700
+
+    Merge pull request #7648 from tchaikov/wip-13783-hammer
+    
+    mon/LogMonitor: use the configured facility if log to syslog
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit cf433bac5b7bc6bdbb6fac56c420288544c79ef5
+Author: shun-s <songshun134@126.com>
+Date:   Wed Dec 2 10:36:35 2015 +0800
+
+    global/pidfile: do not start two daemons with a single pid-file
+    
+    add functions named pidfile_open and pidfile_verify to avoid starting two daemons by a single pid-file
+    
+    Fixes: #13422
+    Signed-off-by: shun song <song.shun3@zte.com.cn>
+    (cherry picked from commit f2c0ef40fd674fecc6e3e97cd6155b976e6759b4)
+
+commit e34c2094ef44e6e52896820f4dd90a1c5d5bbfe8
+Merge: ec33aec b43c5b2
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 25 19:39:49 2016 +0700
+
+    Merge pull request #7801 from dachary/wip-14863-hammer
+    
+    hammer: unittest_crypto: benchmark 100,000 CryptoKey::encrypt() calls
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.cz>
+
+commit b43c5b205f26198c79e21af3a6935a76f560a7fc
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jan 26 15:57:33 2015 -0800
+
+    unittest_crypto: benchmark 100,000 CryptoKey::encrypt() calls
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit fb4b6c52d8e31e8dedfa8aecc16c389f0e7d86cf)
+    
+    Conflicts:
+    	src/test/crypto.cc : complements the incorrect cherry-pick
+               df3f971eafda9c54881c13dcf47f996f18e17028 see
+               http://tracker.ceph.com/issues/14863 for more information
+
+commit ec33aec31fcac3b09bf06256b95c841a8c20d56f
+Merge: 4b46182 17d1b0d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 25 11:09:54 2016 +0700
+
+    Merge pull request #7185 from dachary/wip-14325-hammer
+    
+    hammer: rgw: radosgw-admin bucket check --fix not work
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 4b461829c24cd35656e86f507a78b0fff860c0b5
+Merge: 722ca62 5c92d1d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 25 11:09:28 2016 +0700
+
+    Merge pull request #7188 from dachary/wip-13799-hammer
+    
+    hammer: rgw: default quota params
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 722ca62e12ac977961d895dceaeb55f31d7b00ca
+Merge: 3c1e2d4 ad4df3b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 25 11:04:02 2016 +0700
+
+    Merge pull request #7414 from dachary/wip-14493-hammer
+    
+    hammer: make rgw_fronends more forgiving of whitespace
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 3c1e2d4216353c848f76b52d9c187e83574267a7
+Merge: b73a93d 3d84420
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Feb 24 10:15:27 2016 -0800
+
+    Merge pull request #7415 from dachary/wip-14554-hammer
+    
+    hammer: Unable to bring up OSD's after dealing with FULL cluster (OSD assert with /include/interval_set.h: 386: FAILED assert(_size >= 0))
+
+commit b73a93d8cd8bc8757a7a8960b2a9d448c7041515
+Merge: b3eac8e 5a450e6
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Feb 24 10:15:03 2016 -0800
+
+    Merge pull request #7576 from dachary/wip-14579-hammer
+    
+    hammer: mon sync does not copy config-key
+
+commit b3eac8e3dbb74ebcfdd22d10d6a6f2916d2bf127
+Merge: bf6dcc7 9e0a165
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Feb 24 10:14:37 2016 -0800
+
+    Merge pull request #7577 from dachary/wip-14689-hammer
+    
+    hammer: consume_maps clearing of waiting_for_pg needs to check the spg_t shard for acting set membership
+
+commit bf6dcc7c6da5d06b21e4a4fad3f7a65e8ca84f50
+Merge: 293e8c7 d5ba063
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Feb 24 10:03:55 2016 -0800
+
+    Merge pull request #7590 from SUSE/wip-14720-hammer
+    
+    hammer: OSDMap: reset osd_primary_affinity shared_ptr when deepish_copy_from
+
+commit 293e8c7b038a22e2135e7bf288a8ff4b8ce32a14
+Merge: 5b724fc 10d29da
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Feb 24 10:03:18 2016 -0800
+
+    Merge pull request #7645 from dachary/wip-14760-hammer
+    
+    hammer: OSD coredumps with leveldb compact on mount = true
+
+commit 5b724fcce0e260a04bd6889c3d4c6d9bf44971cf
+Merge: bb84692 1c8c708
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 24 12:56:13 2016 +0700
+
+    Merge pull request #7485 from dillaman/wip-14611-hammer
+    
+    librbd: flattening an rbd image with active IO can lead to hang
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit bb84692e96569823e23fdd02356446577959f438
+Merge: ee014d8 df3f971
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 24 12:55:18 2016 +0700
+
+    Merge pull request #7488 from ceph/wip-auth-hammer
+    
+    hammer: auth: use libnss more safely
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit ee014d8084c05ec3c876a9e46dcb6e558918a0e4
+Merge: 576225d 88ffcc2
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 24 12:48:05 2016 +0700
+
+    Merge pull request #7475 from dachary/wip-14602-hammer
+    
+    hammer: ceph-disk: use blkid instead of sgdisk -i
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 576225d6d5a64f0f43971ccca4d74b5dc44c5c63
+Merge: a6f7a2f eb048a3
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 24 12:47:07 2016 +0700
+
+    Merge pull request #7456 from dachary/wip-14587-hammer
+    
+    hammer: qa/workunits/post-file.sh: sudo
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit a6f7a2f1347393c1cf7564a2ba8d1ba6aff2122c
+Merge: 74cfe5c f04e007
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 24 12:38:47 2016 +0700
+
+    Merge pull request #6946 from dachary/wip-14087-shard-size
+    
+    osd: log inconsistent shard sizes
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 74cfe5c5f49cb566224a3ff0ef8ab515eadc78ac
+Merge: e832001 f209819
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 24 12:37:55 2016 +0700
+
+    Merge pull request #7110 from david-z/wip-hammer-fix-logrotate-ceph-fuse
+    
+    [ceph-fuse] fix ceph-fuse writing to stale log file after log rotation
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit a1fc101084c5aefeb64da344158783a186a28a93
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Feb 8 07:47:23 2015 -0800
+
+    crushtool: send --tree to stdout
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit dca345280e4f6124a8b5aea73e6219488111c356)
+    
+    Conflicts:
+    	src/tools/crushtool.cc (trivial)
+
+commit 4fb688d15c40a9b9d5a54700b3c7695a97ffd03b
+Author: Xinze Chi <xmdxcxz@gmail.com>
+Date:   Tue Mar 3 11:24:50 2015 +0000
+
+    osd: write journal header by force when journal write close
+    
+    osd update the journal header when ceph call FileStore::sync_entry(
+    ApplyManager::commit_finish -> FileJournal::committed_thru).
+    But, it doesnot write the journal head until the next transaction
+    (calling do_write or do_aio_write).
+    
+    So write jouranl header when journal write close, so that
+    committed_up_to in journal header is new enough. ceph may
+    not replay some transaction which has been writen to filestore.
+    
+    Signed-off-by: Xinze Chi <xmdxcxz@gmail.com>
+    (cherry picked from commit 8f80ae3fbaba7b0e64b6e4df6c2dddd10ac50e5a)
+
+commit 31a2fc475a2de8a656048c85595049ec4548ec6a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Feb 11 10:33:40 2016 -0500
+
+    common/bit_vector: use hard-coded value for block size
+    
+    The CEPH_PAGE_SIZE is not actually a constant. On aarch64 platforms
+    the page size is 64K instead of the assumed 4K.  The CRC block
+    computations require a true constant.
+    
+    Fixes: #14747
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ba2e6ce46fb5f781ffaf6483db8ebd99238419c6)
+
+commit 3352b144e0b1500651d8a194c3ebe2aae2f078d0
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Oct 20 18:10:39 2015 -0700
+
+    ceph.in: Notify user that 'tell' can't be used in interactive mode
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit a341d977d42a3010267b64e15cebae3085507297)
+    
+    Conflicts:
+    	src/ceph.in (trivial resolution)
+
+commit 14b5feab80f07b4ed2b8f6d77ea1fa920e98be05
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Jul 6 12:17:21 2015 +0800
+
+    mon/LogMonitor: use the configured facility if log to syslog
+    
+    Fixes: #13748
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 4d2ca37d3b8f067d0cf0f7cfea24964ac181115d)
+
+commit 10d29dab3176ec945dc7272323df0028ef7f0274
+Author: Xiaoxi Chen <xiaoxi.chen@intel.com>
+Date:   Thu Apr 16 13:42:42 2015 +0800
+
+    os/LevelDBStore:fix bug when compact_on_mount
+    
+    Initialize the performance counter(l_rocksdb_compact)
+    before use it in compact().
+    
+    Signed-off-by: Xiaoxi Chen <xiaoxi.chen@intel.com>
+    (cherry picked from commit 932d59d084090297c89cfa3ff7df0a952f807bab)
+
+commit d5ba063cdf3a8b92862ef6657001725d6192fe86
+Author: Xinze Chi <xinze@xsky.com>
+Date:   Sun Feb 7 18:50:29 2016 +0800
+
+    OSDMap: reset osd_primary_affinity shared_ptr when deepish_copy_from
+    
+    Base on current logic, OSDMonitor may call create_pending and
+    encode_pending twice for the some epoch.
+    
+    In encode_pending:
+    
+    tmp.deepish_copy_from(osdmap);
+    tmp.apply_incremental(pending_inc);
+    
+    This Op would change the tmp osd_primary_affinity, but the osd_primary_affinity
+    is declared as ceph::shared_ptr, so this would change the osdmap too. When this
+    round encode_pending is proposed fail. We may call encode_pending again, but the
+    osdmap is changed last round, so the pending_inc would be wrong.
+    
+    Fixes: #14686
+    Signed-off-by: Xinze Chi <xinze@xsky.com>
+    (cherry picked from commit 990b437f4e616a87f4f7438e51945d531170ca83)
+
+commit 9e0a1656200c2f91386215924304b495dd705dae
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 7 08:09:46 2016 -0800
+
+    OSD::consume_map: correctly remove pg shards which are no longer acting
+    
+    A particular shard may be non-active even if the osd is in the acting
+    set.
+    
+    Fixes: 14278
+    Backport: hammer
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit bd2fdf8c56c1aeed6cea3eb59046dbb6af470111)
+
+commit 5a450e6734b4ff9e5e483c74b67df385b9a3d640
+Author: shawn <cxwshawn@gmail.com>
+Date:   Tue Jan 26 05:56:12 2016 -0500
+
+    mon: add mon_config_key prefix when sync full
+    
+    Signed-off-by: Xiaowei Chen <chen.xiaowei@h3c.com>
+    (cherry picked from commit b77cd31b161e78467bbb6476776c0f1d43e615cf)
+
+commit eb048a393e7b54dcf5f2245e34a1aa0de281424d
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Sep 5 09:28:17 2015 -0400
+
+    qa/workunits/post-file.sh: sudo
+    
+    Only root can read the key in /usr/share/...
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 82b0243ec318457e376288a6b32487fbea059705)
+
+commit e9039f4fd3dfe9a3083fcfe64d8012568e8db96f
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Apr 28 17:40:20 2015 -0700
+
+    qa/workunits/post-file: pick a dir that's readable by world
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 04b00029e16971abf1937facbbff8da00d6a3452)
+
+commit 902abe7373c150000d3fc4d66d9c989ed208d0bf
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Apr 9 09:58:39 2015 -0700
+
+    qa/workunits/post-file.sh: use /etc/default
+    
+    base-files doesn't existing on rhel.  /etc/default is everywhere and it's
+    small.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 52235e32561ef3e91f07c59c8fd89c4df244f461)
+
+commit 1c8c708e43e4c97af17932027ae6b5d26c124c43
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Dec 23 12:06:50 2015 -0500
+
+    librbd: ensure librados callbacks are flushed prior to destroying image
+    
+    Fixes: #14092
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 98157ab3274bd960e4487e34f5a83e9c921a6ac8)
+
+commit f892566fc9b78ab38702b75422321a6d0a5bbb7b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jul 30 22:31:55 2015 -0400
+
+    librbd: simplify IO flush handling
+    
+    Add a new convenience method to ImageCtx for handling flush
+    requests and cleanup flush handling with dealing with the cache.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e8d8096babaf15fe8af717297d512f580aff6c18)
+
+commit e5dfd3dd7a53bf79f1bfc17b8a4a720aba08d7c3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Aug 14 13:28:13 2015 -0400
+
+    WorkQueue: PointerWQ drain no longer waits for other queues
+    
+    If another (independent) queue was processing, drain could
+    block waiting.  Instead, allow drain to exit quickly if
+    no items are being processed and the queue is empty for
+    the current WQ.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b118d7df1e34387b6e5649a5b205cf061598d0d4)
+
+commit edf60b452de3c6eb22c75be2547f51812e6eaf1e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Feb 2 10:54:53 2016 -0500
+
+    test: new librbd flatten test case
+    
+    AIO operations after a flatten operation were previously
+    hanging during the close of the parent image.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b59158d9f0c3dbe4b47be1dfd27aaa5ad92e6063)
+
+commit 88ffcc2cbd73602819ad653695de7e8718f6707d
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Dec 18 17:03:21 2015 +0100
+
+    ceph-disk: use blkid instead of sgdisk -i
+    
+    sgdisk -i 1 /dev/vdb opens /dev/vdb in write mode which indirectly
+    triggers a BLKRRPART ioctl from udev (starting version 214 and up) when
+    the device is closed (see below for the udev release note). The
+    implementation of this ioctl by the kernel (even old kernels) removes
+    all partitions and adds them again (similar to what partprobe does
+    explicitly).
+    
+    The side effects of partitions disappearing while ceph-disk is running
+    are devastating.
+    
+    sgdisk is replaced by blkid which only opens the device in read mode and
+    will not trigger this unexpected behavior.
+    
+    The problem does not show on Ubuntu 14.04 because it is running udev <
+    214 but shows on CentOS 7 which is running udev > 214.
+    
+    git clone git://anonscm.debian.org/pkg-systemd/systemd.git
+    systemd/NEWS:
+    CHANGES WITH 214:
+    
+            * As an experimental feature, udev now tries to lock the
+              disk device node (flock(LOCK_SH|LOCK_NB)) while it
+              executes events for the disk or any of its partitions.
+              Applications like partitioning programs can lock the
+              disk device node (flock(LOCK_EX)) and claim temporary
+              device ownership that way; udev will entirely skip all event
+              handling for this disk and its partitions. If the disk
+              was opened for writing, the close will trigger a partition
+              table rescan in udev's "watch" facility, and if needed
+              synthesize "change" events for the disk and all its partitions.
+              This is now unconditionally enabled, and if it turns out to
+              cause major problems, we might turn it on only for specific
+              devices, or might need to disable it entirely. Device Mapper
+              devices are excluded from this logic.
+    
+    http://tracker.ceph.com/issues/14094 Fixes: #14094
+    
+    Signed-off-by: Ilya Dryomov <idryomov@redhat.com>
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 9dce05a8cdfc564c5162885bbb67a04ad7b95c5a)
+    
+    Conflicts:
+    	src/ceph-disk: keep get_partition_type as it is because
+            some hammer users may rely on the fact that it is able
+            to fallback to sgdisk if blkid is old. Chances are an
+            old blkid also means an old udev that does not have the
+            problem this fix is adressing. The get_partition_uuid
+            is modified to try blkid first, with the same rationale.
+
+commit f209819fb676eadf099cc764e3ed3b20fd17b688
+Author: Zhi Zhang <willzzhang@tencent.com>
+Date:   Mon Feb 1 11:03:30 2016 +0800
+
+    [ceph-fuse] fix ceph-fuse writing to stale log file after log rotation
+    
+    This fix should be applied to hammer branch. It can't be directly applied to master branch, because logrotate.conf is changed on matser since ceph-osd, ceph-mon, etc, is controlled by systemd with user/group as 'ceph' by default, while ceph-fuse might be started as root privilege by external users.
+    
+    Signed-off-by: Zhi Zhang <zhangz.david@outlook.com>
+
+commit 3d84420847642e1cb4212f943ecf3ed479112d96
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Thu Jan 21 09:42:09 2016 +0300
+
+    PG::activate(): handle unexpected cached_removed_snaps more gracefully
+    
+    PGPool::update(): ditto
+    
+    Fixes: #14428
+    Backport: infernalis, hammer, firefly
+    
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+    (cherry picked from commit aba6746b850e9397ff25570f08d0af8847a7162c)
+
+commit ad4df3be0f4adba5e8e14829be724fa3a92263c8
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Dec 15 11:28:44 2015 -0500
+
+    rgw: warn on suspicious civetweb frontend parameters
+    
+    Warns about incorrect spacing near 'port' (e.g., 'port = 80').
+    
+    Fixes issue #12038.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 93497e576a946ed7e59f4a4299e447b65aa1774b)
+
+commit 5c92d1d2f11d59651eaa9c4ba6476b9f71990f1c
+Author: Daniel Gryniewicz <dang@redhat.com>
+Date:   Fri Sep 25 11:02:20 2015 -0400
+
+    rgw: Add default quota config
+    
+    Add a default quota config that is applied when no specific quota is
+    enabled.  There are both per-bucket and per-user defaults.  Defaults are
+    enabled if either max count or max size is enabled.
+    
+    This fixes issues 12912 and 12997.
+    
+    Signed-off-by: Daniel Gryniewicz <dang@redhat.com>
+    (cherry picked from commit 3047b5690eebb7fcd2d7a26acd20105e016a44c4)
+
+commit 17d1b0d859112dc9fb5b7a6e2140124790c5f260
+Author: Weijun Duan <duanweijun@h3c.com>
+Date:   Sun Jan 3 20:12:04 2016 -0500
+
+    rgw: radosgw-admin bucket check --fix not work
+    
+    Fixed:#14215
+    
+    Signed-off-by: Weijun Duan <duanweijun@h3c.com>
+    (cherry picked from commit a17f4e27d608ef29cf499fe76246929ec7962783)
+
+commit df3f971eafda9c54881c13dcf47f996f18e17028 (refs/remotes/markhpc/wip-auth-hammer, refs/remotes/gh/wip-auth-hammer)
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Feb 23 14:58:19 2015 -0800
+
+    auth: return error code from encrypt/decrypt; make error string optional
+    
+    This is simpler for a lot of callers.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 40203f711dd59c7f61f5f8eb56a97842c3206494)
+    
+    Conflicts:
+    	src/test/crypto.cc (trivial)
+
+commit 224bb3928de048b13a96ee0377b6d279ce84ac0a
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jan 27 09:17:28 2015 -0800
+
+    auth: optimize crypto++ key context
+    
+    The cbc appears to be stateful, but the key is not.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 7762f1886cab7f7b941851b0be4ec904723cb0e3)
+
+commit f11718de7873fe1fad1649358f208a4910837a7e
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Feb 1 15:07:35 2015 -0800
+
+    auth/Crypto: optimize libnss key
+    
+    Allocate the key and slot once per key instead of once per encrypt/
+    decrypt operation.  This improves things by a factor of almost 4x
+    for my microbenchmark.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 973cd1c00a7811e95ff0406a90386f6ead5491c4)
+
+commit d1b6096c79164f8a5830dee6b5f186c405cf73ab
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Feb 1 15:05:54 2015 -0800
+
+    auth: refactor crypto key context
+    
+    Cache all of the crypto key context in a new CryptoKeyHandler struct that
+    is attached to CryptoKey.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 16b3515af3df439fba7d545587285b50e9814165)
+
+commit 3249f48a07391c98634aebd87f18fc2edbe95ca2
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 22 15:49:25 2015 -0800
+
+    auth/cephx: optimize signature check
+    
+    The encode_encrypt() helper will generate a bufferlist with a simple
+    structure.  Profiles seem to indicate this is taking a large amount of time
+    in the message receive path.
+    
+    Avoid the encode overhead since we have a tiny buffer with a fixed and
+    known structure.
+    
+    Reported-by: Andreas Bluemle <andreas.bluemle@itxperts.de>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8d16d4ce14a82356007c14fb7535170933eb7812)
+
+commit 51eaf98e25287705c9c25d2d3a5df248ba066f2d
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 22 15:40:23 2015 -0800
+
+    auth/cephx: move signature calc into helper
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4e14a5fedbfacd164639f6a59a4d152404f3f9d1)
+    
+    Conflicts:
+    	src/auth/cephx/CephxSessionHandler.cc (trivial, const changes)
+
+commit c240da9e0693dc627162a0c70569dda519ec25e8
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jan 26 15:56:39 2015 -0800
+
+    auth/Crypto: avoid memcpy on libnss crypto operation
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e972a6951142cbea3fe7c2e08933c808693d60c4)
+
+commit 86cc0f0c5e10262bb494ecee415789fd6815b1ff
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jan 26 15:49:10 2015 -0800
+
+    auth: make CryptoHandler implementations totally private
+    
+    There is no need to expose these implementations outside Crypto.cc.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ad5a1549f401af3eae38f5c4de24e7b952e40f7f)
+
+commit f04e0075eabfb1b9d19c75761c321098546c4044
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Dec 15 18:28:29 2015 +0100
+
+    osd: log inconsistent shard sizes
+    
+    When an OSD asserts because it cannot recover from an unexpected erasure
+    coded shard size, the object needs to be manually fixed (i.e. the shard
+    must be removed so it can be reconstructed). Unfortunately the assert
+    does not display the name of the object and it is not displayed unless
+    the OSD logs are >= 10, which is uncommon on an actual cluster because
+    it creates log files that are too big.
+    
+    The problem has been fixed in infernalis as part of
+    https://github.com/ceph/ceph/pull/5173 and backporting it is non
+    trivial.
+    
+    The error condition is modified to make it easy for the system
+    administrator to identify which object is causing the crash and what
+    steps must be taken to fix the issue.
+    
+    http://tracker.ceph.com/issues/14009 Refs: #14009
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+
+commit b2393dcea4ca02fe093481d6a2815b0c857fddc9 (refs/remotes/markhpc/hammer-client-inoderef)
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Nov 20 15:42:05 2015 +0800
+
+    client: add InodeRef.h to make dist
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+
+commit fa79bd978f646001365b14009efe9fa5c562f681
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 24 15:54:31 2015 +0800
+
+    client: use smart pointer to track 'cwd' and 'root_parents'
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 8c45ae1239c934b6727071398a1cfccb6277069e)
+
+commit fcafc65e042c0ed913b5b564812180cd29eefb43
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 24 15:52:10 2015 +0800
+
+    client: convert Inode::snapdir_parent to smart pointer
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 89648e7ed56a1299a096b16080e4a52724893bdd)
+
+commit 78cca4e3b21308cc0a3f4838828a465482f619c7
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 24 15:39:31 2015 +0800
+
+    client: use smart pointer to track temporary inode reference
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 217837b89f6adabcf686377a9b3d33d5008b1097)
+
+commit bad6f335ccd8d277f31a27f04d10683c73d0037a
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 24 15:24:41 2015 +0800
+
+    client: convert CapSnap::in to smart pointer
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+
+commit 4bb48ee4ae49355e3a4bcd7424545c04ce284f89
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 24 15:50:10 2015 +0800
+
+    client: convert Fh::inode to smart pointer
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit dac11e6b4b1e8766ee5165366cf021d9ee91d3ae)
+
+commit 5bebb3acf1fa85ab301aa8ca4e99542c9d34d173
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 24 15:00:50 2015 +0800
+
+    client: use smart pointers in MetaRequest
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit fd02f0f245ccdc1d33c6d24f57267c8fb102618b)
+
+commit e18f1aec94ec021a687317bab00c59d3d223c2bc
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 24 11:21:23 2015 +0800
+
+    client: convert Dentry::inode to smart pointer
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 07f580977319110cbdd60c78fff892c85711ef3e)
+
+commit a7f6d2ffb84dee5bec4e373c1db723805e39a1bb
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat May 23 20:01:46 2015 +0800
+
+    client: hold reference for returned inode
+    
+    CLient::make_request() returns pointer of target inode, but it does not
+    increase reference count of the returned inode. The inode may get freed
+    when CLient::make_request() releases MetaRequest.
+    
+    he fix is hold a reference for the returned inode. Since there are many
+    of places use CLient::make_request() directly or indirectly, it's easy
+    to leak reference. This patch uses intrusive_ptr to track the reference.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit ad9c22a4ca56d7ad3c1666795bd8c9785eaccf70)
diff --git a/doc/changelog/v0.94.8.txt b/doc/changelog/v0.94.8.txt
new file mode 100644
index 000000000..6191ef9f7
--- /dev/null
+++ b/doc/changelog/v0.94.8.txt
@@ -0,0 +1,2059 @@
+commit 838cd35201e4fe1339e16d987cc33e873524af90 (tag: refs/tags/v0.94.8, refs/remotes/gh/hammer)
+Author: Jenkins Build Slave User <jenkins-build@trusty-huge--349f4788-0128-42d9-aca5-78c0aad008e6.localdomain>
+Date:   Fri Aug 26 00:02:49 2016 +0000
+
+    0.94.8
+
+commit eb27865f268eaa4abc4e4b46354f1ea1556d1a5c
+Merge: 08277b7 5248929
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 17 15:26:49 2016 +0200
+
+    Merge pull request #10750 from ceph/wip-14799-hammer
+    
+    hammer: rocksdb do not link against tcmalloc if it's disabled
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 5248929c296d6130ee99fe94fbe69151d241efb8
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Aug 16 23:45:18 2016 -0700
+
+    rocksdb: disable tcmalloc if disabled
+    
+    the rocksdb/configure.ac does not support --with-tcmalloc before,
+    and the updated rocksdb commit has the change for '--with-tcmalloc'
+    option, so let's pick it up.
+    
+    Fixes: http://tracker.ceph.com/issues/14799
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+
+commit fdfcd9b66cda1c637fbaad1c15d2ea04e426210d
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Sep 12 14:12:20 2015 -0400
+
+    ceph.spec: respect CEPH_EXTRA_CONFIGURE_ARGS
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 51abff11688f0201b8f4076ac515e4515929d4cb)
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit 08277b7bc7c0e533c3fd56a0040dc0ddc74637d6
+Merge: bcab369 d5274a3
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sat Aug 13 19:19:37 2016 +0200
+
+    Merge pull request #10718 from oritwas/wip-14328-hammer
+    
+    hammer: rgw: the swift key remains after removing a subuser
+
+commit bcab36992580e5666afbf8e7bada4e850dfc84c6
+Merge: da2960f 699b7c8
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sat Aug 13 18:22:19 2016 +0200
+
+    Merge pull request #10509 from yehudasa/wip-15557-hammer
+    
+    rgw: Have a flavor of bucket deletion to bypass GC and to trigger
+
+commit d5274a3aeb7cc40f5ac04ac1ffcd6e3fc3019be3
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed Jan 20 17:24:38 2016 +0100
+
+    rgw: fix subuser rm command failure
+    
+    Fixes: 14375
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit bbc06e70b49b2917ee63985727cb92724fb99209)
+
+commit f9637743efc7ba3164e3bb9ee7a768801e1599da
+Author: Sangdi Xu <xu.sangdi@h3c.com>
+Date:   Wed Dec 16 09:09:16 2015 +0800
+
+    rgw: add a method to purge all associate keys when removing a subuser
+    
+    Fixes: #12890
+    
+    When removing a subuser, make sure all of its keys, including the swift key and possible s3 keys, are also deleted.
+    
+    Signed-off-by: Sangdi Xu <xu.sangdi@h3c.com>
+    (cherry picked from commit e7b7e1afc7a81c3f97976f7442fbdc5118b532b5)
+
+commit da2960f974b9a64a683ac8b77faf08fda3a5d0ba
+Merge: 0f67539 3a30ffc
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Aug 12 12:23:41 2016 +0200
+
+    Merge pull request #10291 from badone/wip-13829-hammer
+    
+    hammer: config set with negative value results in "error setting 'filestore_merge_threshold' to '-40': (22) Invalid argument"
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 0f67539cf5efc46a92d7ea3ac031e0ce1c573a7a
+Merge: 1e01986 0d4b601
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 12 09:46:30 2016 +0200
+
+    Merge pull request #10699 from ktdreyer/wip-15867-nroff
+    
+    jewel: doc: fix by-parttypeuuid in ceph-disk(8) nroff
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 0d4b6013f0f130c93e2e488d1cd83527ad91274f
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Thu Aug 11 17:11:41 2016 -0600
+
+    doc: fix by-parttypeuuid in ceph-disk(8) nroff
+    
+    Commit 221efb0b893adbfd7a19df171cf967fee87afcc7 altered the rST source
+    for the ceph-disk man page. In Hammer, we also have to modify the nroff
+    sources, because static copies of the generated man pages are stored in
+    Git.
+    
+    Fixes: http://tracker.ceph.com/issues/15867
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 1e01986e9550700a11adfc1b175eca042c9cb188
+Merge: de28029 187d308
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Aug 11 22:58:33 2016 +0200
+
+    Merge pull request #9741 from SUSE/wip-16343-hammer
+    
+    hammer: boost uuid makes valgrind complain
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit de28029c25e4ff527c6021488381022538331473
+Merge: cb0488c 64211fa
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Aug 11 15:58:51 2016 +0200
+
+    Merge pull request #10554 from Vicente-Cheng/wip-16815-hammer
+    
+    hammer: radosgw-admin region-map set is not reporting the bucket quota correctly
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit cb0488c181a122de13d63686b2a64e1aa00791ca
+Merge: 0bc2d66 a3003f6
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Aug 11 15:56:39 2016 +0200
+
+    Merge pull request #10555 from Vicente-Cheng/wip-16317-hammer
+    
+    hammer: rgw: bucket listing following object delete is partial
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 0bc2d66db5feba6cd887d7c297839b231012b880
+Merge: 5866fef 81aef60
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Aug 11 15:55:01 2016 +0200
+
+    Merge pull request #10530 from yehudasa/wip-16412-hammer
+    
+    rgw: remove bucket index objects when deleting the bucket
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 5866fefb32cc443549891754a97eb8a6b8dd2af0
+Merge: da4f735 300c111
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Aug 11 11:14:53 2016 +0200
+
+    Merge pull request #8755 from kmroz/wip-hammer-convert-plain-object-to-versioned
+    
+    hammer: rgw: convert plain object to versioned (with null version) when removing
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit da4f735d7937aa623dfef41f921ad9381b1a07d7
+Merge: 4a12a88 d3eae0a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Aug 4 21:41:15 2016 +0200
+
+    Merge pull request #9271 from XinzeChi/wip-fix-digest-hammer
+    
+    osd: fix omap digest compare when scrub
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 4a12a88d7098c1f67365a82f3cf9a174dd5ada32
+Merge: 6f107f1 ec02d8b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Aug 4 21:38:57 2016 +0200
+
+    Merge pull request #8805 from SUSE/wip-15482-hammer
+    
+    hammer: is_split crash in handle_pg_create
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 6f107f1a7a42c070ef2da7f2b0b07d8bd2826429
+Merge: 3b9e73f 3cb72dd
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Aug 4 21:37:24 2016 +0200
+
+    Merge pull request #8593 from renhwsky/wip-15478-hammer
+    
+    hammer: Pool quota alarm is not in effect
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 3b9e73f319b28c0403871c7d842304aca74f4c6a
+Merge: d34ac2a b9a4ad9
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Aug 3 13:31:41 2016 +0100
+
+    Merge pull request #7542 from SUSE/wip-14668-hammer
+    
+    hammer: Wrong ceph get mdsmap assertion
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit d34ac2a4d76eda4a371cc16dbc6bfdc0223c81d8
+Merge: 8f51f14 3ecdedd
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Aug 3 13:31:30 2016 +0100
+
+    Merge pull request #8042 from ukernel/hammer-15050
+    
+    hammer: mds: fix stray purging in 'stripe_count > 1' case
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 8f51f146c98e281f065debdfdc4399624e0cce97
+Merge: 4773e9c 049bc8a
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Aug 3 13:31:22 2016 +0100
+
+    Merge pull request #8804 from SUSE/wip-15512-hammer
+    
+    hammer: Double decreased the count to trim caps which will cause failing to respond to cache pressure
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 4773e9c40849c2796aa16b7cacddbfe99536919c
+Merge: 85a25e4 f6076dd
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Aug 3 13:31:09 2016 +0100
+
+    Merge pull request #9404 from SUSE/wip-16082-hammer
+    
+    hammer: mds: wrongly treat symlink inode as normal file/dir when symlink inode is stale on kcephfs
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 85a25e43d4451b999ad517f089593d45895a7c59
+Merge: 3c4d14f 2633ec3
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Aug 3 13:30:58 2016 +0100
+
+    Merge pull request #10198 from SUSE/wip-16626-hammer
+    
+    hammer: Failing file operations on kernel based cephfs mount point leaves unaccessible file behind on hammer 0.94.7
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit a3003f6f4f02e37cf96b6e243203d86ad2e39549
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Feb 25 16:02:15 2016 +0100
+
+    rgw: reset return code in when iterating over the bucket the objects
+    
+    Fixes: #14826
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit e23fdcbbce47aaf041b9f8a0812c8a729aa5014b)
+
+commit 64211fa78f872648aad2b2cec2b4b3cb8f2b9953
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Oct 13 17:04:57 2015 -0700
+
+    rgw: fix compilation
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 9dfbae6d65bf6f8ab7cdf8597d556505b81cedd9)
+
+commit 3e45c6bb431f7763a12a48daac30fbe7aa3047cd
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Oct 12 11:14:19 2015 +0200
+
+    rgw: add bucket_quota to RGWRegionMap::encode_json
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit e18b72f7e97a65af681e368d3706a465603df89d)
+
+commit 699b7c8d5f51d895f5ff0578de5bf610436ee33b
+Author: root <root@ceph-node1.homeoffice.wal-mart.com>
+Date:   Wed Apr 20 14:15:55 2016 +0530
+
+    rgw: Have a flavor of bucket deletion to bypass GC and to trigger
+    object deletions async.
+    
+    Fixes: http://tracker.ceph.com/issues/15557
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+    (cherry picked from commit b7a69fca248afeef1de1278890076693b16cf6d3)
+    
+    Conflicts:
+    	src/rgw/rgw_admin.cc
+    	src/rgw/rgw_bucket.cc
+    	src/rgw/rgw_bucket.h
+    	src/test/cli/radosgw-admin/help.t
+
+commit 81aef600671f0fe523206cea16000e79bc6bb75c
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Jul 4 15:01:51 2016 +0200
+
+    rgw: remove bucket index objects when deleting the bucket
+    
+    Fixes: http://tracker.ceph.com/issues/16412
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 3ae276390641ad5fc4fef0c03971db95948880b4)
+
+commit 3c4d14fe73a97c52345369e732e445c78fb656c8
+Merge: 79753ed 8a39e5e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 23:51:47 2016 +0200
+
+    Merge pull request #10182 from gaurav36/wip-configure-xfs-issue-hammer
+    
+    hammer: configure: Add -D_LARGEFILE64_SOURCE to Linux build.
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 79753edf9c4763ae15d82502b53c6f3c9cd24349
+Merge: 75271a2 7dbace5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:40:46 2016 +0200
+
+    Merge pull request #7961 from dzafman/wip-15002
+    
+    hammer: objects unfound after repair (fixed by repeering the pg)
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 75271a2529b1504d249b528fe3fafd4c5e90a9bc
+Merge: 76fa3b1 43d1b92
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:38:15 2016 +0200
+
+    Merge pull request #9125 from dzafman/wip-15635
+    
+    hammer: [MON] Fixed calculation of %USED
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 76fa3b1097b3711ef75bd36aa2a4074e7a8fa9e6
+Merge: cf5f111 ca0beef
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:36:05 2016 +0200
+
+    Merge pull request #8464 from dzafman/wip-13039-15315
+    
+    hammer: "FAILED assert(pg->info.history.same_interval_since == p.same_interval_since)" in upgrade:hammer-x-jewel-distro-basic-openstack
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit cf5f1117afe9e03c683fd40e5af964eaf5975e2b
+Merge: 76ce775 488a787
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:32:47 2016 +0200
+
+    Merge pull request #8573 from Vicente-Cheng/wip-15293-hammer
+    
+    hammer: ECBackend: send write and read sub ops on behalf of client ops at normal priority
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 76ce775761f755bd3085ba22cf088c3a46b78d3a
+Merge: 8629d37 e20df80
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:31:16 2016 +0200
+
+    Merge pull request #8665 from Vicente-Cheng/wip-15525-hammer
+    
+    hammer: ReplicatedPG.cc: 430: FAILED assert(!actingbackfill.empty()): old watch timeout tries to queue repop on replica
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 8629d378b05d4f0374436373a0c972360bae7273
+Merge: 79fc401 7b3f1da
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:29:58 2016 +0200
+
+    Merge pull request #8806 from SUSE/wip-15320-hammer
+    
+    hammer: LibRadosMiscConnectFailure.ConnectFailure (not so intermittent) failure in upgrade/hammer-x
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 79fc401844c935283f43fbea83e113e298a830ba
+Merge: f4d56b0 cbd5aaf
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:28:32 2016 +0200
+
+    Merge pull request #9001 from Vicente-Cheng/wip-15730-hammer
+    
+    hammer: osd: acting_primary not updated on split
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit f4d56b03b1decf83bebbf6996909ee89783068f0
+Merge: f71c9e6 410ff15
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:26:55 2016 +0200
+
+    Merge pull request #9400 from Vicente-Cheng/wip-15796-hammer
+    
+    hammer: LibRadosWatchNotifyPPTests/LibRadosWatchNotifyPP.WatchNotify2Timeout/1 segv
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit f71c9e658d6f5af3cfa48a0dfb9a063f4965e7ce
+Merge: fdb730e 23498a9
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:24:42 2016 +0200
+
+    Merge pull request #9416 from chardan/wip-15770-hammer
+    
+    hammer: mon/OSDMonistor: improve reweight_by_utilization() logic
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit fdb730e169d8885ccd72e140c31797179c0a2c75
+Merge: f9e8dc8 2c18015
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:21:25 2016 +0200
+
+    Merge pull request #10189 from dreamhost/wip-16618-hammer
+    
+    rgw: fix multi-delete query param parsing.
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit f9e8dc824a5aa89e24ef12fc8fcfdbdd4f926aef
+Merge: f3cbf5f 24cc4f9
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:17:57 2016 +0200
+
+    Merge pull request #8379 from Vicente-Cheng/wip-15283-hammer
+    
+    hammer: RGW shouldn't send Content-Type nor Content-Length for 304 responses
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit f3cbf5f891029975f4f5a915366fce28fcabd196
+Merge: ba8de3a f28477c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:14:39 2016 +0200
+
+    Merge pull request #8313 from SUSE/wip-15257-hammer
+    
+    hammer: rgw: radosgw server abort when user passed bad parameters to set quota
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit ba8de3ab31567c8179b72fb1dc8dbf7ccd1f5f38
+Merge: 9562198 ed4ca7c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:13:09 2016 +0200
+
+    Merge pull request #8528 from wido/hammer-issue-15348
+    
+    rgw: Set Access-Control-Allow-Origin to a Asterisk if allowed in a rule
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 95621986d893029bc0a4a686c7da5cd6e0fa6f2b
+Merge: 28107d3 1ac920b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:10:57 2016 +0200
+
+    Merge pull request #9671 from ceph/hammer-rgw-copy-if-newer
+    
+    hammer: rgw: backport rgwx-copy-if-newer for radosgw-agent
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 28107d3ebfded3cd9046e5a23ab83d4e0252cce2
+Merge: 5e1a57a 850881c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:09:14 2016 +0200
+
+    Merge pull request #9961 from SUSE/wip-16190-hammer
+    
+    hammer: subuser rm fails with status 125
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 5e1a57a6f9c2afbdfc6d3e73d438551202c68229
+Merge: 91889db ce313cd
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 29 09:07:10 2016 +0200
+
+    Merge pull request #9402 from SUSE/wip-15966-hammer
+    
+    hammer: No Last-Modified, Content-Size and X-Object-Manifest headers if no segments in DLO manifest
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 23498a9620f792cd099dba028c5bdf96b1a625be
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Tue May 31 15:40:05 2016 -0700
+
+    mon/OSDMonitor: avoid potential expensive grace calculation
+    
+    The grace calculation during check_failure() is now very complicated
+    and time-consuming. Therefore we shall skip this when it is possible.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 3557903d5d57642179b2ae137bedc389974b1956)
+    
+     Conflicts:
+    	src/mon/OSDMonitor.cc
+    
+    Resolved by choosing the move-to-top implementation. Removed unused vars.
+
+commit 1b6f6f27b77803727a523b4337cbad411e8321ed
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Tue Apr 26 11:13:32 2016 +0800
+
+    mon/OSDMonitor: improve reweight_by_utilization() logic
+    
+    By calling reweight_by_utilization() method, we are aiming at an evener result
+    of utilization among all osds. To achieve this, we shall decrease weights of
+    osds which are currently overloaded, and try to increase weights of osds which
+    are currently underloaded when it is possible.
+    However, we can't do this all at a time in order to avoid a massive pg migrations
+    between osds. Thus we introduce a max_osds limit to smooth the progress.
+    
+    The problem here is that we have sorted the utilization of all osds in a descending
+    manner and we always try to decrease the weights of the most overloaded osds
+    since they are most likely to encounter a nearfull/full transition soon, but
+    we won't increase the weights from the most underloaded(least utilized by contrast)
+    at the same time, which I think is not quite reasonable.
+    
+    Actually, the best thing would probably be to iterate over teh low and high osds
+    in parallel, and do the ones that are furthest from the average first.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit e7a32534ebc9e27f955ff2d7a8d1db511383301e)
+    
+     Conflicts:
+    	src/mon/OSDMonitor.cc
+    
+    Resolved by picking the lambda implemenation.
+    NOTE: Because hammer does not support C++11, the lambda functionality from the
+    current master has been moved into the "Sorter" function object.
+
+commit 91889db73ab097502615b208c6e9f3b40411e682
+Merge: f3c4f7b b78a1be
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 27 08:28:38 2016 +0200
+
+    Merge pull request #10038 from tchaikov/wip-16297-hammer
+    
+    mon: Monitor: validate prefix on handle_command()
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit f3c4f7b61ab519c532a2218a11c5472daaaf5fa9
+Merge: 2fe8edb b31ac2d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jul 26 19:08:23 2016 +0200
+
+    Merge pull request #9090 from tchaikov/wip-13990-hammer
+    
+    hammer: osd: remove all stale osdmaps in handle_osd_map()
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 2fe8edb87e7384f206414db8a49e909e51f48de3
+Merge: 67b7f11 0498969
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 25 21:44:14 2016 +0200
+
+    Merge pull request #10227 from SUSE/wip-15081-hammer
+    
+    hammer: When logging to a file fails, ceph logs excessively to stderr
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 67b7f11c63313a5dcdf34b904137ef7a4673f4ec
+Merge: bd59c96 474abb8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 25 21:43:49 2016 +0200
+
+    Merge pull request #9893 from Vicente-Cheng/wip-16430-hammer
+    
+    hammer: OSDMonitor: drop pg temps from not the current primary
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit bd59c9695fadedef89b77166f60f5a8cf85825ff
+Merge: 3794e3a 055427c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 25 21:42:54 2016 +0200
+
+    Merge pull request #9238 from SUSE/wip-15962-hammer
+    
+    hammer: ceph-osd valgrind invalid reads/writes
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 3794e3ad44d1e429f110abe71fb2d2319567bcd1
+Merge: 89233e3 d96086a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 25 21:42:06 2016 +0200
+
+    Merge pull request #8849 from SUSE/wip-14326-hammer
+    
+    hammer: osd/PGLog.h: 88: FAILED assert(rollback_info_trimmed_to == head)
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 89233e32ffcdd7f795873771859caba1ce3ad23f
+Merge: 253143b fb1b40f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 25 21:40:57 2016 +0200
+
+    Merge pull request #8845 from SUSE/wip-15647-hammer
+    
+    hammer: osd: rados cppool omap to ec pool crashes osd
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 253143b559001eadb73ed9945abe2d0b8d946b65
+Merge: 81133dd 365f21b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Jul 25 15:05:54 2016 +0200
+
+    Merge pull request #8470 from dachary/wip-15403-api-test-timeout-hammer
+    
+    hammer: tests: be more generous with test timeout
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 81133dd6e2724ee78cc69bda8e89ca35b20b6b23
+Merge: d71845c 3bb248b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 25 15:05:31 2016 +0200
+
+    Merge pull request #9977 from shun-s/hammer
+    
+    replcatedBackend: delete one of the repeated op->mark_started in ReplicatedBackend::sub_op_modify_impl
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit d71845cec169c660c180a4c0313e0cdbc25329dc
+Merge: 387d5c1 5057c34
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 25 12:51:45 2016 +0200
+
+    Merge pull request #8960 from badone/wip-hammer-rados-bench-units
+    
+    hammer: Add units to rados bench output and clean up formatting
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 474abb8275cd7507d7c7eae0d0e5647612fc177e
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 2 10:43:17 2016 -0700
+
+    OSDMonitor: drop pg temp from sources other than the current primary
+    
+    Fixes: http://tracker.ceph.com/issues/16127
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 1a07123c38e3fecb3fb2e43bbbae962d8411d287)
+    
+    	change:
+                use NULL to replace the nullptr because we don't have C++11
+
+commit b31ac2d37ecb68ed9661923fbaeaedeac241ba4a
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu May 12 20:28:11 2016 +0800
+
+    osd: reset session->osdmap if session is not waiting for a map anymore
+    
+    we should release the osdmap reference once we are done with it,
+    otherwise we might need to wait very long to update that reference with
+    a newer osdmap ref. this appears to be an OSDMap leak: it is held by an
+    quiet OSD::Session forever.
+    
+    the osdmap is not reset in OSD::session_notify_pg_create(), because its
+    only caller is wake_pg_waiters(), which will call
+    dispatch_session_waiting() later. and dispatch_session_waiting() will
+    check the session->osdmap, and will also reset the osdmap if
+    session->waiting_for_pg.empty().
+    
+    Fixes: http://tracker.ceph.com/issues/13990
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 82b0af7cedc3071cd83ee53479f834c23c62b7d0)
+
+commit 3a30ffc21a5edd52726867302da6b358f93aa8dc
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Wed Jan 27 11:34:47 2016 +1000
+
+    qa: Add test for #13829
+    
+    qa/workunits/cephtool/test.sh: add test for setting negative int options
+    src/test/daemon_config.cc: remove tests for failed negative values
+    
+    Fixes: #13829
+    
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 994ac2942d04584d1617e6d4bbd5b880b1ea0448)
+
+commit f70e4adf8682ed82b267e21e5e63039048893243
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Wed Jan 27 11:18:16 2016 +1000
+
+    common: Allow config set with negative value
+    
+    A recent commit disabled negative values but they are required for variables
+    such as filestore_merge_threshold.
+    
+    Modified patch to remove C++11 specific elements so it will build for hammer
+    
+    Fixes: #13829
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 8b777a0c346bc70fd10d07e89368b3785b58f10e)
+
+commit 04989695915fd7ee5ef85ae34ede1fd680514f9d
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 10 09:50:07 2016 -0500
+
+    log: do not repeat errors to stderr
+    
+    If we get an error writing to the log, log it only once to stderr.
+    This avoids generating, say, 72 GB of ENOSPC errors in
+    teuthology.log when /var/log fills up.
+    
+    Fixes: #14616
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d9ac0474b864afda58a44b9012cca4bbc6aaf509)
+    
+    Conflicts:
+    	src/log/Log.cc (drop m_uid and m_gid which are not used in hammer;
+    	order of do_stderr, do_syslog, do_fd conditional blocks is reversed in
+    	hammer; drop irrelevant speed optimization code from
+    	5bfe05aebfefdff9022f0eb990805758e0edb1dc)
+
+commit 2633ec3934ad8f7629c55b4345f426cacfb7b140
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Apr 15 20:15:14 2016 +0800
+
+    mds: disallow 'open truncate' non-regular inode
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 0e4b6f2332bb4822cf324587a94144f1c98e4b97)
+    
+    Conflicts:
+    	src/mds/Server.cc (hammer has cur->inode.inline_version - in master
+    	this has been changed to cur->inode.inline_data.version)
+
+commit 3f0fb20ed08dfde614cff6c056d41b59af200c2c
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Apr 15 19:45:23 2016 +0800
+
+    mds: only open non-regular inode with mode FILE_MODE_PIN
+    
+    ceph_atomic_open() in kernel client does lookup and open at the same
+    time. So it can open a symlink inode with mode CEPH_FILE_MODE_WR.
+    Open a symlink inode with mode CEPH_FILE_MODE_WR triggers assertion
+    in Locker::check_inode_max_size();
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 4d15eb12298e007744486e28924a6f0ae071bd06)
+
+commit 2c180159c00aa184e1fe8b051b1aa1e9f0f332fc
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Thu Jul 7 14:14:36 2016 -0700
+
+    rgw: fix multi-delete query param parsing.
+    
+    Multi-delete is triggered by a query parameter on POST, but there are
+    multiple valid ways of representing it, and Ceph should accept ANY way
+    that has the query parameter set, regardless of what value or absence of
+    value.
+    
+    This caused the RubyGem aws-sdk-v1 to break, and has been present since
+    multi-delete was first added in commit 0a1f4a97da, for the bobtail
+    release.
+    
+    Fixes: http://tracker.ceph.com/issues/16618
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit a7016e1b67e82641f0702fda4eae799e953063e6)
+
+commit 387d5c1ba836833a0cf11ddf9a4fb8690a532878
+Merge: 2fd75cc 4eea92b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jul 7 10:16:31 2016 -0700
+
+    Merge pull request #9407 from SUSE/wip-15833-hammer
+    
+    hammer: RGW :: Subset of uploaded objects via radosgw are unretrievable when using erasure coded pool
+
+commit 8a39e5e67bc9a93cadb6c13f25c7ccd57421af6b
+Author: Ira Cooper <ira@redhat.com>
+Date:   Thu Mar 31 12:30:45 2016 -0400
+
+    configure: Add -D_LARGEFILE64_SOURCE to Linux build.
+    
+    Fixes: http://tracker.ceph.com/issues/16612
+    
+    Without -D_LARGEFILE64_SOURCE we can't build against libxfs,
+    because off64_t must be defined.
+    
+    Signed-off-by: Ira Cooper <ira@redhat.com>
+    (cherry picked from commit 602425a)
+
+commit 2fd75cced9dbcfdfdcba979eae90a6f4fb384758
+Merge: dac65d0 2c97cb3
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed Jul 6 20:25:20 2016 +0200
+
+    Merge pull request #8475 from Vicente-Cheng/wip-15343-hammer
+    
+    hammer: Multipart ListPartsResult has missing quotes on ETag
+    Reviewed-bu: Orit Wasserman <owasserm@redhat.com>
+
+commit 3bb248b7d530403b347e3ff68a8996b624fc1c98
+Author: shun-s <song.shun3@zte.com.cn>
+Date:   Tue Jun 28 15:30:16 2016 +0800
+
+    replcatedBackend: delete one useless op->mark_started as there are two in ReplicatedBackend::sub_op_modify_impl
+    delete one mark_start event as there are two same op->mark_started  in ReplicatedBackend::sub_op_modify_impl
+    Fixes: http://tracker.ceph.com/issues/16572
+    
+    Signed-off-by: shun-s <song.shun3@zte.com.cn>
+
+commit ed4ca7c92cdf7404754dab6f9d58c64b647136c5
+Author: Wido den Hollander <wido@42on.com>
+Date:   Tue Apr 5 11:14:16 2016 +0200
+
+    rgw: Set Access-Control-Allow-Origin to a Asterisk if allowed in a rule
+    
+    Before this patch the RGW would respond with the Origin send by the client in the request
+    if a wildcard/asterisk was specified as a valid Origin.
+    
+    This patch makes sure we respond with a header like this:
+    
+      Access-Control-Allow-Origin: *
+    
+    This way a resource can be used on different Origins by the same browser and that browser
+    will use the content as the asterisk.
+    
+    We also keep in mind that when Authorization is send by the client different rules apply.
+    In the case of Authorization we may not respond with an Asterisk, but we do have to
+    add the Vary header with 'Origin' as a value to let the browser know that for different
+    Origins it has to perform a new request.
+    
+    More information: https://developer.mozilla.org/en-US/docs/Web/HTTP/Access_control_CORS
+    
+    Fixes: #15348
+    
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit 0021e224480c7164330eaa7cc1078bb8795169bf)
+    
+    Conflicts:
+    	src/rgw/rgw_rest.cc
+    		hammer still uses s->cio->print() where master uses STREAM_IO(s)->print()
+
+commit b78a1be835706e7dabc505be343945d0ac05697d
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Jun 30 13:24:22 2016 +0800
+
+    mon: Monitor: validate prefix on handle_command()
+    
+    Fixes: http://tracker.ceph.com/issues/16297
+    
+    Signed-off-by: You Ji <youji@ebay.com>
+    (cherry picked from commit 7cb3434fed03a5497abfd00bcec7276b70df0654)
+    
+    Conflicts:
+        src/mon/Monitor.cc (the signature of Monitor::reply_command()
+                            changed a little bit in master, so adapt the
+                            commit to work with the old method)
+
+commit dac65d048919f701877de96d3271131853e532ed (refs/remotes/gh/wip-optracker-4354)
+Merge: 2e156d7 a2e7ca1
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jun 27 17:51:19 2016 +0200
+
+    Merge pull request #8538 from Vicente-Cheng/wip-15360-hammer
+    
+    hammer: Linking to -lrbd causes process startup times to balloon
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 850881cf787ccb23eca989861a6688f3a4de1a7c
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed Jan 20 17:24:38 2016 +0100
+
+    rgw: fix subuser rm command failure
+    
+    Fixes: 14375
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit bbc06e70b49b2917ee63985727cb92724fb99209)
+    
+    Conflicts:
+    	src/rgw/rgw_user.cc The "if (op_state.will_purge_keys())" block was
+    	later changed to "always purge all associated keys" by
+    	e7b7e1afc7a81c3f97976f7442fbdc5118b532b5 - keep the hammer version
+
+commit 055427cf58857eeadda11ed1b3381e84e384bc85
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu May 12 16:57:49 2016 -0700
+
+    Pipe: take a ref to existing while we are waiting
+    
+    Otherwise, if it is reaped while we are waiting, it'll be a
+    use-after-free.
+    
+    Fixes: http://tracker.ceph.com/issues/15870
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit b224912d249453d754fc0478d3680f8cfa1a5c22)
+    
+    Conflicts:
+        src/msg/simple/Pipe.cc: nullptr changed to 0 because hammer is not compiled
+        with -std=c++11
+
+commit 24cc4f9e67d9c61f68ccc8f7d80ac227d67aeda1
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Jun 15 16:53:16 2016 +0200
+
+    rgw: check for -ERR_NOT_MODIFIED in rgw_rest_s3.cc
+    
+    Fixes: http://tracker.ceph.com/issues/16327
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit fc38346c596d8b0bc156183970d6a327943cb577)
+    
+    Conflicts:
+    	src/rgw/rgw_rest_s3.cc
+    	  use ret instead of op_ret to check op result
+
+commit 7dbace59b83a059fcb364b78fc78d4e38164df6c
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu May 28 22:35:08 2015 +0200
+
+    erasure-code: s/unreocvery/unfound/
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 28820d1cd630ec466ed1069a437d740a025fb9aa)
+
+commit e726f211107cd84cc722744c38213b96dc59cdc7
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Wed Apr 22 09:45:04 2015 +0800
+
+    test: add test-case for repair unrecovery-ec pg.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 8f30db85219a6c57cb2fc03f4049a3f57446ca55)
+
+commit 40b1c2b231b62471d4f3cfba3ba2c3d596c9e170
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Tue Apr 14 13:29:54 2015 +0800
+
+    osd: Remove the duplicated func MissingLoc::get_all_missing.
+    
+    It is the same as MissingLoc::get_needs_recovery.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit bdd6205408cf20e039496728052a36c1a7d758c6)
+
+commit 47d5dfc0323e8d7614ad5793d85c1476b63c18e6
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Tue Apr 14 09:11:58 2015 +0800
+
+    osd: Fix ec pg repair endless when met unrecover object.
+    
+    In repair_object, if bad_peer is replica, it don't add soid in
+    MissingLoc for ec pool. If there are more bad replica for ec pool
+    which cause object can't recover, the later recoverying will endless.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d51806f5b330d5f112281fbb95ea6addf994324e)
+
+commit 187d308b5b0cb0512b442bbad1bddfda8ef9203f
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Aug 24 14:51:47 2015 -0400
+
+    uuid: use boost::random:random_device
+    
+    The boost mt code uses uninitialized memory for extra randomness,
+    which is a bad idea in general but more importantly makes valgrind
+    unhappy.  Use /dev/urandom instead.
+    
+    Unfortunately this introduces a link time dependency.. meh!
+    
+    Fixes: #12736
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit dbcaa544856fcc99ab912a101c4a28e1eb3fb94e)
+    
+    Conflicts:
+    	ceph.spec.in (trivial resolution)
+
+commit 174de7fce8080df6e02b363e7821e8cca6f8157f
+Author: Rohan Mars <code@rohanmars.com>
+Date:   Wed Aug 12 21:19:31 2015 -0700
+
+    moved to use boost uuid implementation, based on commit 4fe89a7b14c97b2ed7f357132901beb2bdcec551
+    
+    Signed-off-by: Rohan Mars <code@rohanmars.com>
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 62bfc7a1ab1587e81ed3bff0ddfbb1aa69d1c299)
+    
+    Conflicts:
+    	debian/control (trivial resolution)
+    	src/common/Makefile.am (trivial resolution)
+    	src/common/blkdev.cc (no get_device_by_uuid() function in hammer)
+
+commit 2e156d7ad4b9f4ffd6028df3a460b50b30c8b0d3
+Merge: 054a90e 5cd922c
+Author: Mykola Golub <to.my.trociny@gmail.com>
+Date:   Thu Jun 16 08:26:00 2016 +0300
+
+    Merge pull request #9722 from trociny/wip-16289-hammer
+    
+    hammer: qa/workunits/rbd: respect RBD_CREATE_ARGS environment variable
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 5cd922ca31d2eb987992b2afcbd45d5d92aeac69
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Jun 14 15:28:06 2016 +0300
+
+    qa/workunits/rbd: respect RBD_CREATE_ARGS environment variable
+    
+    Fixes: http://tracker.ceph.com/issues/16289
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 3048d3c7ce800a6174c54946b838e4e4079ec609)
+
+commit 1ac920bf291a58cf680108f8dc738197e9bae44d
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Sep 18 17:34:51 2015 -0700
+
+    rgw: fix identification of canceled operation
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 9f63b15b3a7c4cc633ff90f529934f071aea7c62)
+
+commit a38f157a09097d1eaef64813fc947aac19e3844a
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Mar 25 17:35:40 2015 -0700
+
+    rgw: identify racing writes when using copy-if-newer
+    
+    When copying an object from a different zone, and copy-if-newer is
+    specified, if the final meta write is canceled check whether the
+    destinatioin that was created is actually newer than our mtime,
+    otherwise retry.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit fe9c64b75caf3f53ce2dfdea5e39d794528dfc33)
+
+commit 02f6d8aaa86f4fdb9912763883d079d02589e5be
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Mar 24 22:48:03 2015 -0700
+
+    rgw: translate http error 304 to appropriate status
+    
+    So that we can return the appropriate response on a conditioinal multi
+    zone copy operation.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 20d24e965884af440f94c7e3093587c7b8643cb9)
+
+commit 7319d76b269d541249124f46890f1b04279ba2ea
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Mar 24 20:24:55 2015 -0700
+
+    rgw: fix if-modified-since boundary
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 27946e8ba4bc5dbfec2da319fa23266d5885d655)
+
+commit 5e4de5ad66dc24b059ef19664d1eef22be6890f7
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Mar 24 20:16:02 2015 -0700
+
+    rgw: add rgwx-copy-if-newer http header
+    
+    Similar to IF_MOD_SINCE, but does not take a time argument, and should
+    protected against races (not there yet).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 8813a4401198636e0f4d3a220a0f726625a5c5d4)
+
+commit 054a90edb2812f78426b8cb1dac2e768b2e7fc51
+Merge: 785ffd1 c722d00
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Tue Jun 7 15:48:00 2016 +0200
+
+    Merge pull request #8113 from Vicente-Cheng/wip-15023-hammer
+    
+    hammer: rgw: user quota may not adjust on bucket removal
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 785ffd1f616864909b24314c803d12566c885d14
+Merge: e0a3589 139691c
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Jun 6 12:42:14 2016 -0700
+
+    Merge pull request #9277 from ceph/wip-split-lfn-hammer
+    
+    Hammer LFN collection split fix: backport of http://tracker.ceph.com/issues/14766
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit e0a358972644d3718c80970fd7236a4df8e61afe
+Merge: 735c524 006ea56
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Jun 6 08:19:23 2016 -0700
+
+    Merge pull request #9521 from ceph/revert-9496-wip-16149-hammer
+    
+    Revert "hammer: Scrub error: 0/1 pinned"
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 006ea56f603e160c83c8b1fe35ee8022321bf3b2
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jun 6 11:15:54 2016 -0400
+
+    Revert "hammer: Scrub error: 0/1 pinned"
+
+commit 735c52423305d4ad548464e986ff88e57333b8ca
+Merge: a1a5f57 c294bd3
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jun 6 08:46:45 2016 -0400
+
+    Merge pull request #9496 from Vicente-Cheng/wip-16149-hammer
+    
+    hammer: Scrub error: 0/1 pinned
+
+commit a1a5f57558e30fc0fd581756a75da5abee3e897b
+Merge: a336291 d1ab71f
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Jun 6 11:39:34 2016 +0200
+
+    Merge pull request #8987 from dreamhost/hammer-wip-pr7712
+    
+    hammer: rgw: S3: set EncodingType in ListBucketResult
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit c294bd34e7e29f2c96e3277491c02ea26bf6da5c
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri May 20 13:59:10 2016 -0700
+
+    ReplicatedPG: adjust num_pinned in _delete_oid
+    
+    Fixes: http://tracker.ceph.com/issues/15952
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 907d4e25c676fd3e1c2be90ce8ab5b64b362b0bc)
+
+commit 43d1b92b97e0bb1c419341ba3c740fa632cafef0
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue May 17 17:14:07 2016 -0700
+
+    test: Fix grouping of mon_cephdf_commands by moving to MON_TESTS
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 2fb0530a5650bcdb9c89d8bdd2504a7dd1642678)
+    
+    Conflicts:
+    	qa/workunits/cephtool/test.sh (trivial)
+
+commit 300c11179290d9b119ff6f310efd1cf5b559b3e1
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Mar 22 18:14:57 2016 -0700
+
+    rgw: convert plain object to versioned (with null version) when removing
+    
+    Fixes #15243
+    
+    When removing a plain null versioned object (was created prior to bucket versioning
+    enabled), need to convert the bucket index representation to a versioned one. This
+    is needed so that all the versioning mechanics play together.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit c6334d430b17739fed2df2b4481ae395ef6568d9)
+    
+    Conflicts:
+            src/rgw/rgw_rados.cc
+              - hammer is missing get_zone() API from which log_data can be
+                obtained. Needed to fall back to zone_public_config
+                structure in bucket_index_unlink_instance() definition.
+              - olh_tag string parameter added to
+                bucket_index_unlink_instance() definition.
+            src/rgw/rgw_rados.h
+              - olh_tag string parameter added to
+                bucket_index_unlink_instance() declaration.
+
+commit 4eea92b4a50e06d850c39009d122f8bbbd48c857
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu May 5 14:02:25 2016 -0700
+
+    rgw: handle stripe transition when flushing final pending_data_bl
+    
+    Fixes: http://tracker.ceph.com/issues/15745
+    
+    When complete_writing_data() is called, if pending_data_bl is not empty
+    we still need to handle stripe transition correctly. If pending_data_bl
+    has more data that we can allow in current stripe, move to the next one.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit b7a7d41839558b1a8786a66e27d9d80e606f61c7)
+    
+    Conflicts:
+    	src/rgw/rgw_rados.cc (hammer write_data() takes rgw_obj reference as
+    	4th argument, hammer throttle_data() takes rgw_obj as 2nd argument)
+
+commit f6076dd0416dd4464955f208d8964ef136b6142b
+Author: Zhi Zhang <willzzhang@tencent.com>
+Date:   Tue May 3 17:12:26 2016 +0800
+
+    mds: wrongly treat symlink inode as normal file/dir when symlink inode is stale on kcephfs
+    
+    Signed-off-by: Zhi Zhang <zhangz.david@outlook.com>
+    (cherry picked from commit 88b6d669e37924536152a552db46ef43a7353562)
+    
+    Conflicts:
+    	src/mds/Server.cc (hammer has req->head.args.open.flags instead of just flags)
+
+commit ce313cd89e29c0cfc6b7c600a82bdea7b351cdbd
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue May 10 17:45:20 2016 +0200
+
+    rgw: handle errors properly during GET on Swift's DLO.
+    
+    Backport: Jewel, Hammer
+    Fixes: http://tracker.ceph.com/issues/15812
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 5ef0202a86a1e4cb56bd29d926555f114a1acdd8)
+    
+    Conflicts:
+    	src/rgw/rgw_op.cc (hammer uses ret instead of op_ret; enclosing if
+    			   block in hammer does not check skip_manifest)
+
+commit 410ff1528caafaec0cfd9ea6ee4b86ed6788b09d
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri May 6 09:09:43 2016 -0400
+
+    osdc/Objecter: upper bound watch_check result
+    
+    This way we always return a safe upper bound on the amount of time
+    since we did a check.  Among other things, this prevents us from
+    returning a value of 0, which is confusing.
+    
+    Fixes: http://tracker.ceph.com/issues/15760
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2e2ce365866a7569b55aff338a82c132aa927089)
+    
+    Conflicts:
+    	src/osdc/Objecter.cc
+    	    By https://github.com/ceph/ceph/pull/8967, simply add 1 to return vaule.
+
+commit d3eae0a01ae8d33988ffccee1f0af145aa10fa33
+Author: Xinze Chi <xinze@xsky.com>
+Date:   Mon May 23 21:38:39 2016 +0800
+
+    osd: fix omap digest compare when scrub
+    
+    Introduce by fe1c28dea4e5607a0c72eab1f046074616cd55a7.
+    
+    Fixes: http://tracker.ceph.com/issues/16000
+    Signed-off-by: Xinze Chi <xinze@xsky.com>
+
+commit a33629106ec64acdaba45bf9db18d2402a926b8e
+Merge: b9b5f96 dd29310
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat May 21 14:18:25 2016 -0400
+
+    Merge pull request #9240 from yehudasa/wip-15886-hammer
+    
+    rgw: keep track of written_objs correctly
+
+commit dd29310bd6273ec3643bf90e51f7cae4801629c7
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon May 16 14:35:12 2016 -0700
+
+    rgw: keep track of written_objs correctly
+    
+    Fixes: http://tracker.ceph.com/issues/15886
+    
+    Only add a rados object to the written_objs list if the write
+    was successful. Otherwise if the write will be canceled for some
+    reason, we'd remove an object that we didn't write to. This was
+    a problem in a case where there's multiple writes that went to
+    the same part. The second writer should fail the write, since
+    we do an exclusive write. However, we added the object's name
+    to the written_objs list anyway, which was a real problem when
+    the old processor was disposed (as it was clearing the objects).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 8fd74d11682f9d0c9085d2dc445fc3eb5631f6e0)
+
+commit c2ea6dbd652782bb227b7787c486660871b2d950
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon May 9 15:01:46 2016 +0800
+
+    osd: remove all stale osdmaps in handle_osd_map()
+    
+    in a large cluster, there are better chances that the OSD fails to trim
+    the cached osdmap in a timely manner. and sometimes, it is just unable
+    to keep up with the incoming osdmap if skip_maps, so the osdmap cache
+    can keep building up to over 250GB in size. in this change
+    
+    * publish_superblock() before trimming the osdmaps, so other osdmap
+      consumers of OSDService.superblock won't access the osdmaps being
+      removed.
+    * trim all stale osdmaps in batch of conf->osd_target_transaction_size
+      if skip_maps is true. in my test, it happens when the osd only
+      receives the osdmap from monitor occasionally because the osd happens
+      to be chosen when monitor wants to share a new osdmap with a random
+      osd.
+    * always use dedicated transaction(s) for trimming osdmaps. so even in
+      the normal case where we are able to trim all stale osdmaps in a
+      single batch, a separated transaction is used. we can piggy back
+      the commits for removing maps, but we keep it this way for simplicity.
+    * use std::min() instead MIN() for type safety
+    
+    Fixes: http://tracker.ceph.com/issues/13990
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 369db9930887d75b498927da9c97733bff4472b6)
+    
+    Conflicts:
+        remove the C++11ism
+
+commit ac0340ad30329df7919ce82938ad49d0a2e2d160
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Mar 16 21:15:35 2016 +0800
+
+    osd: populate the trim_thru epoch using MOSDMap.oldest_map
+    
+    instead of filling MOSDMap with the local oldest_map, we share
+    the maximum MOSDMap.oldest_map received so far with peers. That
+    way one OSD's failure to trim won't prevent it from sharing with
+    others that they are allowed to trim.
+    
+    Fixes: #13990
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 9789c29fe0f82b236703f7ae4d356251fde3a46f)
+    
+    Conflict: use atomic_t instead of std::atomic<>
+
+commit b9b5f966ddfe7e706c038db204271ee662e0d825
+Merge: b90c097 bb5e015
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed May 18 17:02:26 2016 -0400
+
+    Merge pull request #9164 from liewegas/wip-osdmapcrc-hammer
+    
+    osd: dump full map bl at 20 when crc doesn't match
+
+commit bb5e015d50cb50361dd6ce258c720fe798b3835a
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Sep 6 13:56:38 2015 -0400
+
+    osd: dump full map bl at 20 when crc doesn't match
+    
+    This will help us debug cases where the encoding doesn't match due
+    to a bug.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit bfe359af0b80f44ca04847f74d5a2d81097ce4e6)
+
+commit 5057c34ebe4f93792f058ea5f2e3c7e0a29b3d3d
+Author: Dmitry Yatsushkevich <dyatsushkevich@mirantis.com>
+Date:   Mon Mar 23 11:05:44 2015 -0700
+
+    obj_bencher: cosmetic display fixes
+    
+    fix indents in output
+    change `snprintf' to `setprecision'
+    
+    Signed-off-by: Dmitry Yatsushkevich <dyatsushkevich@mirantis.com>
+    (cherry picked from commit ddb422f40394ec2f4fc4a6a6cff49e43538121a9)
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit 6d8ad0edf6bae57a6e8ca5cc1a7add1659a92691
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Fri May 6 15:05:42 2016 +1000
+
+    common: Add space between timestamp and "min lat:" in bench output
+    
+    This change is taken from 069d95eaf49cadaa9a8fa1fa186455944a50ec7d
+    but I did not want to cherry-pick that patch since the rest of it
+    is purely cosmetic and would be unlikely to apply cleanly.
+    
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit b90c097ddd3e172df4534509309a554cb3bc3917
+Merge: aefe2cb aab3a40
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon May 16 13:02:44 2016 +0200
+
+    Merge pull request #8535 from asheplyakov/bug-14512-hammer
+    
+    hammer: rbd snap rollback: restore the link to parent
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 31849986896ba6b5fd2b0b13023b50f67842fd28
+Author: Adam Kupczyk <akupczyk@mirantis.com>
+Date:   Wed Mar 2 12:31:01 2016 +0100
+
+    [MON] Fixed calculation of %USED. Now it is shows (space used by all replicas)/(raw space available on OSDs). Before it was (size of pool)/(raw space available on OSDs).
+    
+    Signed-off-by: Adam Kupczyk akupczyk@mirantis.com
+    (cherry picked from commit 71c4e525f27b8efd2aa4f3b5e95f4a13f123d41a)
+
+commit fed256e40f5465959384735be957dec88733aeb7
+Author: Ruifeng Yang <yangruifeng.09209@h3c.com>
+Date:   Tue Jan 12 22:08:41 2016 +0800
+
+    mon: add a column 'RAW USED' for ceph df detail
+    
+    Signed-off-by: Ruifeng Yang <yangruifeng.09209@h3c.com>
+    (cherry picked from commit 2b597ae6d4d53974f28a2da090232577daf09979)
+    
+    Conflicts:
+    	qa/workunits/cephtool/test.sh (trivial)
+    	src/mon/PGMonitor.cc (trivial)
+    	src/mon/PGMonitor.h (trivial)
+
+commit aefe2cb4f262112f0a028797733f868b83da7bd2
+Merge: d56bdf9 221efb0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu May 12 23:17:51 2016 +0200
+
+    Merge pull request #9107 from SUSE/wip-15867-hammer
+    
+    hammer: ceph-disk s/by-parttype-uuid/by-parttypeuuid/
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 139691c646809f6c25d72c8de064f26b3c036197
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu May 12 13:50:58 2016 -0700
+
+    src/test/objectstore/store_test.cc: fix shards for new tests
+    
+    This didn't matter on master/jewel, but in hammer, keyvaluestore
+    asserts that gen != NO_GEN => shard != NO_SHARD.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+
+commit 221efb0b893adbfd7a19df171cf967fee87afcc7
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu May 12 20:06:06 2016 +0200
+
+    doc: s/by-parttype-uuid/by-parttypeuuid/ in ceph-disk
+    
+    Fixes: http://tracker.ceph.com/issues/15867
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 62f4fbe66f8af79fb874125743df0433aa92b731
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Mar 11 17:02:15 2016 -0800
+
+    store_test: improve synthetic coverage of the ec stash case
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit f3a78e046d54260de01a4d1db8e104130e35cf42)
+    
+    Conflicts:
+            src/test/objectstore/store_test.cc
+
+commit b6bc9cbaef99670bc0fc55936ab02a3656ec8bfd
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Mar 11 14:05:37 2016 -0800
+
+    store_test: improve synthetic test coverage of long object names
+    
+    1/2 of objects will now have long names, and we'll shift the region
+    of the hash space where new objects are created every 1024 objects
+    created.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 4b44720f752ebac63454ecd338e2b0c77c8fe501)
+    
+    Conflicts:
+    	src/test/objectstore/store_test.cc
+
+commit ec74c1281e641a45de039f2c168b50902e514729
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Mar 10 15:38:26 2016 -0800
+
+    TestRados: make half the objects have long names
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 8b64432d07a14df9dfd2ed1314783de16530b1d1)
+
+commit 9d1ee7cf390154c5939a01ff439e33d303a952c2
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Mar 10 15:19:15 2016 -0800
+
+    LFNIndex::lfn_translate: consider alt attr as well
+    
+    If the file has an alt attr, there are two possible matching
+    ghobjects.  We want to make sure we choose the right one for
+    the short name we have.  If we don't, a split while there are
+    two objects linking to the same inode will result in one of
+    the links being orphaned in the source directory, resulting
+    in #14766.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 850ce11ea3c4dacc6ff1a3ce7ff37000818267c2)
+
+commit 6b821cc8f394e59760e3d58d27f382976bb69e13
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Mar 10 11:06:06 2016 -0800
+
+    LFNIndex::created: fix return value with failure injection
+    
+    See comment for explanation.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit aec5560887a1c1dbbb5aba822280e2c1e05d5d8f)
+
+commit f500435da3ddf750cf0770c0f63e9cff0cebd7a7
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Mar 10 11:03:13 2016 -0800
+
+    store_test: add reproducer for #14766
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 0fb67c772561aaca9261ac9094b74d7524498f6f)
+    
+    Conflicts:
+            src/test/objectstore/store_test.cc
+
+commit cbd5aafe93a5c85392a460a183862f62af2836b8
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Apr 29 15:24:09 2016 -0400
+
+    osd/PG: update info.stats.* mappings on split
+    
+    These are updated in the init and start_peering_interval paths, but not
+    on split.
+    
+    Fixes: http://tracker.ceph.com/issues/15523
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8b42294d98dd70715184991da5ba495fbe93d009)
+
+commit d1ab71ff6084719e86e07a447f3054af73e40868
+Author: Vitja Makarov <vitja.makarov@gmail.com>
+Date:   Wed Feb 17 13:46:18 2016 +0300
+
+    hammer: rgw: S3: set EncodingType in ListBucketResult
+    
+    Signed-off-by: Victor Makarov <vitja.makarov@gmail.com>
+    (cherry picked from commit d2e281d2beb0a49aae0fd939f9387cb2af2692c8)
+    X-Github-PR: 7712
+    Backport: hammer
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit df4eadc2376bb4a53210e2fe176520887b840f2f
+Author: Gu Zhongyan <guzhongyan@360.cn>
+Date:   Wed Dec 30 16:42:07 2015 +0800
+
+    rados: Add units to rados bench output
+    
+    Fixes: #12248
+    Signed-off-by: Gu Zhongyan <guzhongyan@360.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 669b932934d60bbfa751cebd9340f11bd95e2853)
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit d96086af5d54a7450f3823210323b9fe08fc075a
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Dec 2 17:20:19 2015 -0800
+
+    PGLog::rewind_divergent_log: fix rollback_info_trimmed_to before index()
+    
+    Fixes: #13965
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 053ee919059b2671d40110e8dff8615dc13d34e0)
+
+commit e79162de4cb418576ca633d5c7ea6fb766706962
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Dec 2 17:17:57 2015 -0800
+
+    TestPGLog: add test for 13965
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 66c7246491f4b6e5b95d80cc93c9157725a1a778)
+
+commit fb1b40f5e2da1b70d9ff026a75196c79d09cf0c6
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Sun Mar 22 22:07:24 2015 +0800
+
+    osd/Replicated: For CEPH_OSD_OP_WRITE, set data digest.
+    
+    Add two cases which can add data digest for OP_WRITE:
+    a: offset = 0, and length > original size
+    b: offset = original size, and original has data_digest.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit c7702bf85d3617b3e1c6619b8ebeff34932fc3e4)
+
+commit f0242598ea7c990aef2d00f3dc6a0cd65656332c
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Mon Mar 23 09:54:52 2015 +0800
+
+    osd/ReplicatedPG: For obj has omap, it mean have omap data or omap header or have both.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit fc51ce2a837e8e878d46d8ca54531aa7bd5d01b2)
+
+commit 7b3f1da0ab7f60705b6bc802ddb1bfbce10c8738
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Mar 28 10:33:53 2016 -0400
+
+    mon/MonClient: fix shutdown race
+    
+    While we are shutting down, we should drop all subsequent mon
+    messages.  Currently it is possible to finish the authentication
+    sequence *while* we are shutting down, such that a subsequent
+    attempt to authenticate succeeds.
+    
+    Fix this by resetting cur_con early in the sequence, as this
+    makes us drop all incoming messages.
+    
+    Fixes: http://tracker.ceph.com/issues/13992
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5b44a348bfe4978446bbc7acf58eb8f60a47c965)
+
+commit ec02d8b7b1610bd7fd00c201fb01ef80d3b9f05b
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Apr 12 17:31:27 2016 +0800
+
+    PG: set epoch_created and parent_split_bits for child pg
+    
+    if we have a wrong "created", OSD receiving an osd_pg_create message
+    could fail to project its history and crash. a bad split_bits could
+    mislead the monitor and osd, and hence cause problems.
+    * set child.info.history.epoch_created to osdmap.get_epoch() instead of
+      using the parent's epoch_created
+    * set child.info.stats.parent_split_bits to the calculated split_bits
+      instead using the parent's split_bits
+    
+    Fixes: http://tracker.ceph.com/issues/15426
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 8b985561ba08d4ed2d7751a8e09f9d9067c5fbd3)
+
+commit 049bc8a498d4ab516c2daded8ac44699aa72c982
+Author: Zhi Zhang <willzzhang@tencent.com>
+Date:   Thu Jan 14 11:04:42 2016 +0800
+
+    ceph-fuse: double decreased the count to trim caps
+    
+    Signed-off-by: Zhi Zhang zhangz.david@outlook.com
+    (cherry picked from commit d8a3f6d839649243b12cd6f73e6cb8aec1f955c1)
+
+commit e20df8020458a7b7679d0105183f870220ed0ce7
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Apr 8 10:35:29 2016 -0400
+
+    osd/ReplicatedPG: make handle_watch_timeout no-op if !active
+    
+    During on_change, we clean up old events on the obcs.  This can
+    include a queued watch timeout:
+    
+     3: (ReplicatedPG::handle_watch_timeout(std::shared_ptr<Watch>)+0x125) [0x7f1fc21fe375]
+     4: (HandleDelayedWatchTimeout::finish(int)+0xd3) [0x7f1fc213e2e3]
+     5: (Context::complete(int)+0x9) [0x7f1fc20ead29]
+     6: (ReplicatedPG::finish_degraded_object(hobject_t const&)+0x354) [0x7f1fc22429e4]
+     7: (ReplicatedPG::on_change(ObjectStore::Transaction*)+0x2ba) [0x7f1fc224353a]
+     8: (PG::start_peering_interval(std::shared_ptr<OSDMap const>, std::vector<int, std::allocator<int> > const&, int, std::vector<int, std::allocator<int> > const&, int, ObjectStore::Transaction*)+0x7bd) [0x7f1fc219a0bd]
+    
+    In this case, handle_watch_timeout should not assume that we are
+    active and primary.
+    
+    Fixes: http://tracker.ceph.com/issues/15391
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4b0e39ee11e7bd5079ff6704bc74627d3ba8ba44)
+
+commit 3cb72dd57f901f400d19022269fb098c06318d6b
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Jun 9 14:57:54 2015 +0200
+
+    mon/OSDMonitor.cc: fix UNINTENDED_INTEGER_DIVISION
+    
+    Fix for:
+    
+    CID 1297885 (#1 of 2): Result is not floating-point (UNINTENDED_INTEGER_DIVISION)
+     integer_division: Dividing integer expressions g_conf->mon_pool_quota_warn_threshold
+     and 100, and then converting the integer quotient to type float. Any remainder,
+     or fractional part of the quotient, is ignored.
+    
+    CID 1297885 (#2 of 2): Result is not floating-point (UNINTENDED_INTEGER_DIVISION)
+     integer_division: Dividing integer expressions g_conf->mon_pool_quota_crit_threshold
+     and 100, and then converting the integer quotient to type float. Any remainder,
+      or fractional part of the quotient, is ignored.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit be7e07a52871e16e8a46accde6171077cca8db5a)
+
+commit aab3a40f95dafab34a7eadc2159d142a5f0f88f1
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Mon Apr 11 14:16:48 2016 +0300
+
+    hammer: rbd snap rollback: restore the link to parent
+    
+    So snapshot, flatten, rollback of a cloned image does not loose any data
+    
+    Fixes: #14512
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+
+commit 488a78794d800312f79f7a8cd75dfda3bf3d6ed0
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Jan 8 13:26:39 2016 -0800
+
+    ECBackend: send subop writes and reads at normal priority
+    
+    We want these to be processed ahead of new client ops since
+    there are resources being held.
+    
+    Fixes: 14313
+    Backport: hammer
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit b1151c4b8b7f22c7ac067da059d087007b0bc8ea)
+
+commit a2e7ca15d6fd0326526b77f443c15067eb1840ac
+Author: Richard W.M. Jones <rjones@redhat.com>
+Date:   Mon Mar 21 14:24:35 2016 +0000
+
+    common/Cycles: Do not initialize Cycles globally.
+    
+    Move initialization to the few tests that actually use it.
+    
+    Fixes: http://tracker.ceph.com/issues/15225
+    Signed-off-by: Richard W.M. Jones <rjones@redhat.com>
+    (cherry picked from commit 1c2831a2c1277c69f9649200d74a75c04a4b0296)
+    
+      Conflicts:
+    	src/test/msgr/perf_msgr_client.cc
+    	src/test/msgr/perf_msgr_server.cc
+    	src/test/perf_local.cc
+                These three files were not introduced on hammer, just remove
+
+commit ca0beef84812b09f1ab224e4796fd44c8fe37989
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Mar 18 10:42:40 2016 -0700
+
+    osd/OSD: fix build_past_intervals_parallel
+    
+    We may be only calculating older past intervals and have a valid
+    history.same_interval_since value, in which case the local
+    same_interval_since value will end at the newest old interval we had to
+    generate.
+    
+    Same as 0830275b39afc408573c2c468fa3b0e7b1d5cc4e change in generate_past_intervals()
+    This was introduced by 70316541bbb115d9a35954bfba373cf1dc084b7e.
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0fd674bbf0c17a673be40123645adee3d64375a0)
+
+commit fce79027034bef0cfedbecd9d2601589495f86e6
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Jul 20 17:48:15 2015 -0700
+
+    osd: When generating past intervals due to an import end at pg epoch
+    
+    Add assert() to make sure same_interval_since isn't too far forward
+    
+    Fixes: #12387
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 65dcc2da76750d0b6dd2cf0031c44f32749f33e5)
+
+commit 2c97cb33c8aa3f53af8a331075a3f79e70113cea
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Thu Mar 31 19:48:38 2016 +0800
+
+    rgw: fix compiling error
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 535eb0e608f3bf6b7c561f277e9977b91ab97b48)
+
+commit 2aa1ea6b505c768323abb99dcc8c8f24e594a898
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Thu Mar 31 06:24:40 2016 +0000
+
+    rgw: Multipart ListPartsResult ETag quotes
+    
+    ListPartsResult output has always missed quotes on the ETag since it was
+    first committed.
+    
+    Fixes: #15334
+    Backports: hammer, infernalis
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit a58b774e72cc1613d62e10b25322d6d15e9d2899)
+
+commit 365f21b1181b08e0bb89b916833301228e7c9efc
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Apr 6 15:39:23 2016 +0200
+
+    tests: be more generous with test timeout
+    
+    When the thrasher is in action together with a validater (lockdep or
+    valgrind), a single test may hang for more than 360 seconds. Increase to
+    1200: it does not matter if the value is large, only that it prevents
+    the test from hanging forever.
+    
+    Fixes: http://tracker.ceph.com/issues/15403
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit af89474b3fb2c4aa63680aa6b30d71fad2fdd373)
+
+commit c722d00af5286785ad23c574359220e47fd15076
+Author: Vicente Cheng <freeze.bilsted@gmail.com>
+Date:   Tue Feb 9 12:03:24 2016 -0800
+
+    rgw: user quota may not adjust on bucket removal
+    
+    Description:
+    If the user/admin removes a bucket using --force/--purge-objects options with s3cmd/radosgw-admin respectively, the user stats will continue to reflect the deleted objects for quota purposes, and there seems to be no way to reset them. User stats need to be sync'ed prior to bucket removal.
+    
+    Solution:
+    Sync user stats before removing a bucket.
+    
+    Fixes: #14507
+    Signed-off-by: Edward Yang <eyang@us.fujitsu.com>
+    (cherry picked from commit 06507992e3fad837c4667f467d59fc026a2ca0e8)
+    
+    Conflicts:
+    
+    	src/rgw/rgw_op.cc
+                reordering the check seqence and replace some op_ret to ret
+    
+    Backport Change:
+    We remove the `tenant` parameter because this feature is not introduced on hammer version.
+    The rgw multi-tenant feature is introduced on pr#6784 (https://github.com/ceph/ceph/pull/6784)
+    This feature is supported from v10.0.2 and later version.
+
+commit 20f300e2900ff123974312143c10dca59ee1a64a
+Author: Wido den Hollander <wido@42on.com>
+Date:   Tue Mar 22 11:35:05 2016 +0100
+
+    rgw: Do not send a Content-Length header on a 304 response
+    
+    We tell the client that the content has not changed. If we
+    send a Content-Length header RFC2616 describes that the client
+    MUST use that new value:
+    
+    "If a cache uses a received 304 response to update a cache entry,
+    the cache MUST update the entry to reflect any new field values
+    given in the response."
+    
+    Therefor we should not send a Content-Length header
+    
+    Fixes: #15119
+    
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit fb4e5cc5d6adc82423d294496855f56e43e8c11d)
+
+commit e53751d0e52f6fb611d9bd81fe2115cd7ce533b3
+Author: Wido den Hollander <wido@42on.com>
+Date:   Mon Mar 21 16:50:49 2016 +0100
+
+    rgw: Do not send a Content-Type on a '304 Not Modified' response
+    
+    When we say the Content has not changed we should not respond
+    with a content type which defaults to binary/octet stream.
+    
+    Fixes: #15119
+    
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit 471fa29e643fcfd5e1383fb987f99b5b21038ed7)
+
+commit 19dbc2598d15d0676017abe4fb75f7ddc3248119
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 18 10:18:22 2015 -0800
+
+    rgw: dump_status() uses integer
+    
+    so that we can avoid atoi() later
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 744a29a34b72d2a709a58a5ea5fb6223a39c676a)
+    
+      Conflicts:
+    	src/rgw/rgw_rest.cc
+            rename the input parameter from `err` to `http_ret`
+
+commit c79b481311b3cd992794abc6e2f194f350d43531
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 18 10:07:27 2015 -0800
+
+    rgw: move status_num initialization into constructor
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 2a12ffc3d7f98504ece8906a6fcb5dac8fd81aba)
+
+commit ceb8e19096f7d4ef35ed3ad8fad508aecbfa78ef
+Author: Wido den Hollander <wido@42on.com>
+Date:   Wed Nov 11 19:01:15 2015 +0100
+
+    rgw: Do not send a Content-Length header on status 204
+    
+    RFC7230 says:
+      A server MUST NOT send a Content-Length header field in any response with a
+      status code of 1xx (Informational) or 204 (No Content).
+    
+    Fixes: #13582
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit 4e5921dbc7d1c51feb4cc5c03aa59a432742765e)
+
+commit 3ecdedd5cfe67d85530123ed6684688b43910aa6
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Mar 11 11:13:43 2016 +0800
+
+    mds: fix stray purging in 'stripe_count > 1' case
+    
+    Fixes: #15050
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 99ec183973e0f4686673d93df02e5381543be51f)
+
+commit f28477c9cf904e9b27e6ad6bb38fbeeb80db5017
+Author: Dunrong Huang <riegamaths@gmail.com>
+Date:   Mon Dec 28 18:55:37 2015 +0800
+
+    rgw: do not abort when user passed bad parameters to set quota
+    
+    when user passes bad parameter(e.g. pass "null" to bool parameter), the ceph_json.cc::decode_json_obj()
+    will raise a exception with type of JSONDecoder::err(). If we do not catch this exception, the radosgw
+    server will abort.
+    
+    Fixes: #14191
+    
+    $ curl -X PUT http://ceph1:9090/admin/user?quota&uid=test5&quota-type=user -d '{"enabled": "null"}'
+    
+     0> 2015-12-28 11:25:20.864091 7f0d46fb5700 -1 *** Caught signal (Aborted) **
+     in thread 7f0d46fb5700
+    
+     ceph version 10.0.1-787-gc485d29 (c485d29a53f6a2c1a3b561c54632dbac1c4e9473)
+     1: (ceph::BackTrace::BackTrace(int)+0x2d) [0x7f0e13ef88c5]
+     2: (()+0x8fbfb7) [0x7f0e13ef7fb7]
+     3: (()+0xf100) [0x7f0e082c3100]
+     4: (gsignal()+0x37) [0x7f0e072fd5f7]
+     5: (abort()+0x148) [0x7f0e072fece8]
+     6: (__gnu_cxx::__verbose_terminate_handler()+0x165) [0x7f0e07c019b5]
+     7: (()+0x5e926) [0x7f0e07bff926]
+     8: (()+0x5e953) [0x7f0e07bff953]
+     9: (()+0x5eb73) [0x7f0e07bffb73]
+     10: (bool JSONDecoder::decode_json<bool>(char const*, bool&, JSONObj*, bool)+0x288) [0x7f0e13d74c2c]
+     11: (RGWQuotaInfo::decode_json(JSONObj*)+0x6c) [0x7f0e13d71128]
+     12: (void decode_json_obj<RGWQuotaInfo>(RGWQuotaInfo&, JSONObj*)+0x23) [0x7f0e13d21f16]
+     13: (int rgw_rest_get_json_input<RGWQuotaInfo>(CephContext*, req_state*, RGWQuotaInfo&, int, bool*)+0x11e) [0x7f0e13d21e48]
+     14: (RGWOp_Quota_Set::execute()+0x8c9) [0x7f0e13d1ed6d]
+     15: (()+0x7532d2) [0x7f0e13d4f2d2
+    
+    Reported-by: Coffee Chou <coffee.zyr@gmail.com>
+    Signed-off-by: Dunrong Huang <riegamaths@gmail.com>
+    (cherry picked from commit 59f9058467480d21bd98e0e90aa97313a92478a1)
+    
+    Conflicts:
+    	src/rgw/rgw_rest.h
+                hammer version lacks rgw_rest_get_json_input_keep_data()
+                template
+
+commit 9786394b2f75d883505125bc45c95fb685b96531
+Author: Dunrong Huang <riegamaths@gmail.com>
+Date:   Mon Dec 28 18:14:22 2015 +0800
+
+    rgw: do not abort when user passed bad parameters to set metadata
+    
+    Fixes: #14190
+    
+    when user passes bad parameter(e.g. pass "null" to bool parameter), the ceph_json.cc::decode_json_obj()
+    will raise a exception with type of JSONDecoder::err(). If we do not catch this exception, the radosgw
+    server will abort.
+    
+    $ curl -X PUT http://ceph1:9090/admin/metadata?user&format=json -d 'data = { "mtime": "null"}'
+    
+    terminate called after throwing an instance of 'JSONDecoder::err'
+    *** Caught signal (Aborted) **
+     in thread 7ff982f55700
+     ceph version 10.0.1-787-gc485d29 (c485d29a53f6a2c1a3b561c54632dbac1c4e9473)
+     1: (ceph::BackTrace::BackTrace(int)+0x2d) [0x7ffa814db8c5]
+     2: (()+0x8fbfb7) [0x7ffa814dafb7]
+     3: (()+0xf100) [0x7ffa758a6100]
+     4: (gsignal()+0x37) [0x7ffa748e05f7]
+     5: (abort()+0x148) [0x7ffa748e1ce8]
+     6: (__gnu_cxx::__verbose_terminate_handler()+0x165) [0x7ffa751e49b5]
+     7: (()+0x5e926) [0x7ffa751e2926]
+     8: (()+0x5e953) [0x7ffa751e2953]
+     9: (()+0x5eb73) [0x7ffa751e2b73]
+     10: (bool JSONDecoder::decode_json<long>(char const*, long&, JSONObj*, bool)+0x28c) [0x7ffa8135920a]
+     11: (RGWMetadataManager::put(std::string&, ceph::buffer::list&, RGWMetadataHandler::sync_type_t, obj_version*)+0x173) [0x7ffa81468029]
+     12: (RGWOp_Metadata_Put::execute()+0x172) [0x7ffa81308dcc]
+     13: (()+0x7532d2) [0x7ffa813322d2]
+     14: (RGWFCGXProcess::handle_request(RGWRequest*)+0x90) [0x7ffa813328a4]
+    
+    Reported-by: Coffee Chou <coffee.zyr@gmail.com>
+    Signed-off-by: Dunrong Huang <riegamaths@gmail.com>
+    (cherry picked from commit 83503239c81573232429a5fe98381f1a8f1357e9)
+    
+    Conflicts:
+    	src/rgw/rgw_user.cc
+                hammer version uses RGWUserInfo instead of RGWUserCompleteInfo
+
+commit b9a4ad9da99d82bd247e5d39108840f19535fa5e
+Author: Vicente Cheng <freeze.bilsted@gmail.com>
+Date:   Fri Mar 27 18:49:28 2015 +0800
+
+    Fixed the ceph get mdsmap assertion.
+    
+        When we want to get mdsmap, we try to get_version()
+        and the return value err = 0 means success.
+    
+        The assert verified r == 0. r would not change in this flow.
+        It always meet assert and lead mon failure.
+    
+        I think this verify should be:
+            assert(err == 0)
+        It will help to check return value of get_version().
+    
+    If you have any questions, feel free to let me know.
+    Thanks!
+    
+    Signed-off-by: Vicente Cheng <freeze.bilsted@gmail.com>
+    (cherry picked from commit f4398d2e6c245e3f81a6038425e1b8372b265b8c)
diff --git a/doc/changelog/v0.94.9.txt b/doc/changelog/v0.94.9.txt
new file mode 100644
index 000000000..52d883d44
--- /dev/null
+++ b/doc/changelog/v0.94.9.txt
@@ -0,0 +1,72 @@
+commit fe6d859066244b97b24f09d46552afc2071e6f90 (tag: refs/tags/v0.94.9, refs/remotes/gh/hammer)
+Author: Jenkins Build Slave User <jenkins-build@trusty-huge--d6d4765c-3c95-4fe1-a8cb-776094e0d416.localdomain>
+Date:   Tue Aug 30 11:33:34 2016 +0000
+
+    0.94.9
+
+commit d51d10407459bd337f674c6fb0bab1bf55e30c1e
+Merge: b97a974 27d8055
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Tue Aug 30 07:12:09 2016 -0400
+
+    Merge pull request #10913 from ceph/wip-revert-random-uuid-hammer
+    
+    hammer: revert use of boost for uuids
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit b97a974878881dc5963c0c8a78a9f83fdd31d5f5
+Merge: 838cd35 a219cf5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 30 03:03:57 2016 +0200
+
+    Merge pull request #10898 from SUSE/wip-release-notes-hammer
+    
+    doc: release-notes: add missing hammer releases
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 27d805589552b63a2a152335e4932f996e979d86
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Aug 29 15:29:29 2016 -0400
+
+    Revert "moved to use boost uuid implementation, based on commit 4fe89a7b14c97b2ed7f357132901beb2bdcec551"
+    
+    This reverts commit 174de7fce8080df6e02b363e7821e8cca6f8157f.
+
+commit 21f6f1da0764e1a003a43ef73d6788bdfc3ef848 (refs/remotes/gh/wip-revert-random-uuid-hammer)
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Aug 29 11:55:24 2016 -0400
+
+    Revert "uuid: use boost::random:random_device"
+    
+    This reverts commit 187d308b5b0cb0512b442bbad1bddfda8ef9203f.
+    
+    This breaks precise builds.
+
+commit a219cf5348b5a5d7b013f4dc1cb503d33d609f12
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Aug 28 09:27:00 2016 +0200
+
+    doc: release-notes.rst: add missing line to v0.94.8
+    
+    see 46264a3b6cb7f490680c79bf0957c36aeaea5f6c and
+    b83d25d2918cbd880fbaeec976bffcea8868d5d9 in master
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit a6ba10174559d100341e3684e08d74c6de2e79c7
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Aug 27 12:50:09 2016 +0200
+
+    doc: add missing changelogs up to 0.94.8
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit f3dad334fa3147490c8aa94323bee16dd2449681
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Aug 27 09:28:48 2016 +0200
+
+    doc: release-notes: add missing hammer releases
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
diff --git a/doc/changelog/v10.2.1.txt b/doc/changelog/v10.2.1.txt
new file mode 100644
index 000000000..5646e8a1a
--- /dev/null
+++ b/doc/changelog/v10.2.1.txt
@@ -0,0 +1,2249 @@
+commit 3a66dd4f30852819c1bdaa8ec23c795d4ad77269 (tag: refs/tags/v10.2.1, refs/remotes/gh/jewel)
+Author: Jenkins Build Slave User <jenkins-build@trusty-huge--11a52675-9585-4db4-a514-798db40d6da2.localdomain>
+Date:   Thu May 12 21:48:00 2016 +0000
+
+    10.2.1
+
+commit d93a90ec2743c9aec229891c3b0da72b038cffad
+Merge: cb35d7c 466b7fe
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu May 12 17:30:14 2016 +0200
+
+    Merge pull request #8853 from ceph/wip-jewel-backports
+    
+    jewel: several backports
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit cb35d7c25bb1df2a8eff01432d27c5d3bbb196af
+Merge: 0e1b633 b6ebb25
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Thu May 12 16:50:33 2016 +0200
+
+    Merge pull request #9041 from vshankar/jewel-15721-backport
+    
+    jewel: rbd: helpful error message on map failure
+    
+    Reviewed-by: Ilya Dryomov <idryomov@gmail.com>
+
+commit 0e1b6336b60af6bceaf9dbc8ce544fcb7af71961
+Merge: bfdf709 867f798
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed May 11 12:33:50 2016 -0700
+
+    Merge pull request #9081 from yehudasa/wip-rgw-period-commit-jewel
+    
+    rgw: period commit fix
+
+commit bfdf709dff9bd3d60d11950d4b2d8741ad0cb113
+Merge: bf1474f 9d5162f
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Wed May 11 12:24:23 2016 -0700
+
+    Merge pull request #8969 from Abhishekvrshny/wip-15731-jewel
+    
+    jewel: MDSAuthCap parse no longer fails on paths with hyphens
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit bf1474f86ce1a102d15c54ee0e1b11005fe48191
+Merge: 7a37155 52ca195
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Wed May 11 12:23:28 2016 -0700
+
+    Merge pull request #8970 from Abhishekvrshny/wip-15732-jewel
+    
+    jewel: MDS incarnation no longer gets lost after remove filesystem
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 7a37155d798aebc688fb4a06056dc4761062b098
+Merge: e83764c e6aa453
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Wed May 11 12:20:03 2016 -0700
+
+    Merge pull request #8971 from Abhishekvrshny/wip-15738-jewel
+    
+    jewel: handle standby-replay nodes properly in upgrades
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit e83764c03b9e88dd2a8ef848269768ffb8b6a703
+Merge: 4916950 dccb5ef
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed May 11 10:38:30 2016 -0700
+
+    Merge pull request #9054 from yehudasa/wip-rgw-admin-output-jewel
+    
+    jewel: rgw admin output
+
+commit 49169501a922034c2686dd88365d438e12605116
+Merge: aa24358 f071d8c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed May 11 10:38:17 2016 -0700
+
+    Merge pull request #9053 from yehudasa/wip-15745-jewel
+    
+    jewel: rgw: handle stripe transition when flushing final pending_data_bl
+
+commit aa243585508eedf87b2e9e558cc63c014f84b3b0
+Merge: f88e2df 9bb17db
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed May 11 10:38:02 2016 -0700
+
+    Merge pull request #9047 from ceph/wip-rgw-period-delete-jewel
+    
+    jewel: rgw: period delete fixes
+
+commit f88e2df881c045f52f78516aa08def8569be6ecf
+Merge: a614858 a496b70
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed May 11 12:30:18 2016 -0400
+
+    Merge pull request #9076 from dillaman/wip-cmake-test-rbd-mirror-jewel
+    
+    jewel: cmake: fix rbd compile errors
+
+commit a496b705ed48da187b341392530f7e8579637310
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 22:37:01 2016 -0400
+
+    cmake: fix rbd compile errors
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8d4c00d1bec431f8c721195c81684c4095079c1d)
+
+commit 9a46e13f3ca9915d71f01bb3506818db59c31a7b
+Author: runsisi <runsisi@zte.com.cn>
+Date:   Thu Apr 28 10:22:07 2016 +0800
+
+    cmake: add library cls_journal for target unittest_librbd
+    
+    otherwise if we want to generate target unittest_librbd we need to
+    (re)generate cls_journal manually
+    
+    Signed-off-by: runsisi <runsisi@zte.com.cn>
+    (cherry picked from commit 7efb29416b789fcb7e68aab554d6033e6b5ebb43)
+
+commit a6148582432ff36110145ba642d2e0f38e5872b3
+Merge: 7a4810d 48b732f
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Wed May 11 16:50:41 2016 +0800
+
+    Merge pull request #9063 from dachary/wip-15838-jewel
+    
+    jewel: tests: make check fails on ext4
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 48b732fbfac1fa6af10bec0ef900029b71204c26
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon May 2 10:42:24 2016 +0200
+
+    tests: enable make check on ext4 (part 2)
+    
+    Followup of 475cc08c330e639b226f2ab57dedbe4495f63022
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit b0b3a068d4fcc7b739c5dd3d68297e2d517c378e)
+
+commit e1c67d4684c485711b2b9af02edabe161f3d7779
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Apr 30 17:12:05 2016 -0400
+
+    qa/workunits/ceph-helpers.sh: make ceph-osd behave on ext4
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 475cc08c330e639b226f2ab57dedbe4495f63022)
+
+commit 867f798d9c364ab5400a61d83cbaef0da9a57b27
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue May 10 15:51:27 2016 -0400
+
+    radosgw-admin: 'period commit' sends to new master zone by default
+    
+    if no --remote or --url is given, set --remote = master_zone_id to
+    override the rest_master_conn (which refers to the old master zone)
+    
+    Fixes: http://tracker.ceph.com/issues/15828
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit fbe1a05355549dfe0c05750683eeb61e275950c2)
+
+commit a6a6c13c751bbb74ba4f12dc593cd054aabed87c
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue May 10 15:48:33 2016 -0400
+
+    radosgw-admin: allow --remote to specify zone or zonegroup id
+    
+    --remote was only looking for zonegroup connections. extended to check
+    for zone connections as well
+    
+    updated the usage to clarify that it expects zone/zonegroup ids
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 9fcff89ee1e60009fa367e33c915a90354c9a9cc)
+
+commit dccb5efb128233affaa544b43f93aa4b3d557259
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu May 5 09:59:49 2016 -0400
+
+    radosgw-admin: 'zonegroup add' prints updated zonegroup
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit d355208023fe08a7e16f1078c872286048c4bcc6)
+
+commit bd6b5fc5d49c08f574b84a2705a8de4a1d5f8ed1
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu May 5 09:59:32 2016 -0400
+
+    radosgw-admin: 'zonegroup modify' prints updated zonegroup
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit a033f40caf60083111886aa804fcdbf3ee60aef8)
+
+commit c1e92df545d0a8db64c84879bc09023c98783fe4
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu May 5 09:56:47 2016 -0400
+
+    radosgw-admin: 'zone modify' prints updated zone
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 8a639014a6f2dd950062aa405bbee1762ca23e09)
+
+commit f071d8c6885786ac5fd5400381369945704abaa8
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu May 5 14:02:25 2016 -0700
+
+    rgw: handle stripe transition when flushing final pending_data_bl
+    
+    Fixes: http://tracker.ceph.com/issues/15745
+    
+    When complete_writing_data() is called, if pending_data_bl is not empty
+    we still need to handle stripe transition correctly. If pending_data_bl
+    has more data that we can allow in current stripe, move to the next one.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit b7a7d41839558b1a8786a66e27d9d80e606f61c7)
+
+commit 7a4810d1f51c072f6406a7306743ad7577476c76
+Merge: 67809b8 10b393b
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue May 10 11:14:40 2016 -0700
+
+    Merge pull request #9052 from dillaman/wip-cmake-rbd-mirror-replay-status-jewel
+    
+    jewel: cmake: add missing source file to rbd_mirror/image_replayer
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 10b393bd90ed76ad41d2123aae5700e171a8e9bc
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Apr 29 17:33:03 2016 -0400
+
+    cmake: add missing source file to rbd_mirror/image_replayer
+    
+    fixes an undefined reference when linking librbd_mirror_internal.a
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 2f3f56170ab47fc10d4232101ae2e85398a4c299)
+
+commit 67809b8e3d71b3cba5ce91e542e840c099c927cd
+Merge: e15854c 8bb4c7f
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue May 10 14:10:05 2016 -0400
+
+    Merge pull request #9051 from athanatos/wip-15815-jewel
+    
+    Wip 15815 jewel
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 8bb4c7f92ca784a116d0f7a9a22677b514e7e6ca
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Apr 22 16:38:14 2016 -0400
+
+    ceph_test_keyvaluedb: clean up test dir
+    
+    Just like ceph_test_objectstore
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2b075aa52d4475338c0f3cd3360e1ee4de8befa7)
+
+commit 108cdfdff65bcce98cb3b2da799fe46bed4421f4
+Author: Allen Samuels <allen.samuels@sandisk.com>
+Date:   Fri Apr 22 14:06:56 2016 -0400
+
+    ceph_test_keyvaluedb: fixup unit test for keyvaluedb
+    
+    Fixes: http://tracker.ceph.com/issues/15435
+    Signed-off-by: Allen Samuels <allen.samuels@sandisk.com>
+    (cherry picked from commit 85c14de20936ff9ee224f9b6569a73f9bae75f19)
+
+commit e15854ce98291b3186e9cfadb6cb47a401a4c8ae
+Merge: 7686c62 815967a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 13:52:54 2016 -0400
+
+    Merge pull request #9050 from dillaman/wip-15821
+    
+    jewel: rbd-mirror: image sync needs to handle snapshot size and protection status
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 815967ab97631f65c44cd200971cc8056e94d89e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 9 18:25:39 2016 -0400
+
+    test: add missing header dependency for MockJournaler
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit fa91919c856ac8186545bc8f8da31a8f438a9b5d)
+
+commit 32189573c2da9e1922b6bfae232a62a909797e1a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 4 10:23:49 2016 -0400
+
+    rbd-mirror: image sync object copy was not properly mapping snapshots
+    
+    When the snapshot sequence is out-of-sync between remote and local
+    clusters (expected), the objects would not be written with the
+    correct snapshot context.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit dd8f08039bf5354a0413c9da4263d1075243e9c9)
+
+commit d3dc919991a013eb45d55a8b6a04628d089d000a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 4 10:23:25 2016 -0400
+
+    rbd-mirror: clean up image sync debug messages
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit acca475fe5c9ff024c7f76397e7f49e4c4558b09)
+
+commit 58e0612a7bb66f6c50dcfc27ed0f1e49d6ec08fa
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 4 00:12:05 2016 -0400
+
+    librbd: readahead should not read past end of image
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit adb14303ccf8a26ed5536bb692ae63d92ac40a9f)
+
+commit 7dadecb96a4f856d18fcccffb323941b1149ceaa
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 3 23:17:12 2016 -0400
+
+    test: randomize rbd-mirror test case IO
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0d75f2ce61cee164da4e5b7ffb4e69c286e6036a)
+
+commit 7aef2976084b8be858e354a05dae12df30743737
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 3 23:15:58 2016 -0400
+
+    rbd-mirror: don't set object map to nonexistent during image sync
+    
+    The object maps are initialized to nonexistent at snapshot creation,
+    so this operation is unnecessary.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit fe5d4a811f9d9d1160ff7104fe9243cc17ef1bef)
+
+commit 58e90f1bcb53a38be2491ac7797f170de324af9a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 3 23:01:30 2016 -0400
+
+    librbd: improve object map debug messages
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2c657752a5067e11405dca59461ce9cbdf062602)
+
+commit 6c1af50b67e1a1769bb3d30d28e45fee8b13bb71
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 3 22:53:52 2016 -0400
+
+    test: snapshot stress test rbd-mirror image sync
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 468287bd182fa0cd02f7551ad6d729ff67ab1a22)
+
+commit 0f84aaf011f1a62250948647069cc7fbd6c45699
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 3 21:42:05 2016 -0400
+
+    rbd-mirror: allocate empty object map during snapshot create
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 57a7fe0f44c5542a72120d0beed8b38e9ccd66db)
+
+commit fa8a61df1570893e222c0d9431ba61e192a23c40
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Apr 29 16:43:07 2016 -0400
+
+    librbd: optionally skip object map creation upon snapshot
+    
+    This feature is used by rbd-mirror during image bootstrap since
+    it will directly create an empty object map.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit db16240c79be7e28d65edac2088352b9b146da5f)
+
+commit a140753b6b01215f832cc9524e3532e825e4c2f2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Apr 25 15:11:28 2016 -0400
+
+    rbd-mirror: refactor snapshot create into its own state machine
+    
+    Initial implementation will resize the image (if needed) and create
+    the snapshot.  It is stubbed out for future state machines for handling
+    object map and clones.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8adc47fbf8efa3991bd247be43b786676fe5fff9)
+
+commit ef1e4c874adba51bc5a3acbbf3177174f25da088
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 14 21:47:19 2016 -0400
+
+    rbd-mirror: sync snapshot protection status during bootstrap
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e6fba2b2306c8f04ce088fb15ef70e1dac282ff3)
+
+commit c790da5aef0cb6a5573d86d0a8298d850b63ef25
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sat Apr 9 13:30:56 2016 -0400
+
+    test: stub test cases for rbd-mirror image bootstrap
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d0f732877213ba9eec9d3158b839c5a4c796e7ab)
+
+commit 7a7889572789e1bec9cb72e08a0d66d8302d7639
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Apr 5 00:03:12 2016 -0400
+
+    test: share journal::MockJournaler between librbd and rbd-mirror
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 04db7ddec661ef195077c3fab06f248705b17220)
+    
+    Conflicts:
+    	src/test/librbd/test_mock_Journal.cc: MockJournaler moved to new file
+
+commit 7686c628b199dd2f3f58b70e7ab3bbb9c1f199c1
+Merge: 63d2d23 f221fed
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 13:40:32 2016 -0400
+
+    Merge pull request #9040 from dillaman/wip-15818
+    
+    jewel: librbd: potential concurrent event processing during journal replay
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 63d2d2396d80fd7d9fd439488d08e8747846d181
+Merge: 1bb1a3a f18b14b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 13:39:46 2016 -0400
+
+    Merge pull request #9038 from dillaman/wip-15817
+    
+    jewel: journal: incorrectly computed object offset within set
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 1bb1a3a4862e5bb5a99c7549d499564deecbdafe
+Merge: abbe69e 8605040
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 13:39:06 2016 -0400
+
+    Merge pull request #9039 from dillaman/wip-15820
+    
+    jewel: librbd: client-side handling for incompatible object map sizes
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 86050402066d1f5d680ec777826386a5760e3456
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 4 17:00:59 2016 -0400
+
+    librbd: guard object map against incompatible image sizes
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 42d3d3ac2d33360aae071330cfa34b87c87eadc5)
+
+commit 7448d29027dafe1abd1e6ebff6f0fe5bb15e4b07
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 4 15:16:17 2016 -0400
+
+    librbd: properly handle object map open returning error code
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 570de56bfc7fd2edd5cf1cf5b7b92084cd3352a3)
+
+commit 764b3bd413b81d2c98f8c325100661ee41a571bc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 4 15:01:58 2016 -0400
+
+    librbd: add client-side memory constraint when loading object map
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2dd34b2c7b8c376817aafe5cfdda7aa9cfb60e1a)
+
+commit abbe69e19af6d453095e03adbaaaef680ed80958
+Merge: 05d5749 5ec1f79
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 13:30:49 2016 -0400
+
+    Merge pull request #9048 from dillaman/wip-15819
+    
+    jewel: librbd: constrain size of AioWriteEvent journal entries
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 5ec1f79086e4841aa27c12531a1220f69ed3cccb
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 6 00:08:38 2016 -0400
+
+    librbd: split large AioWrite journal events
+    
+    Ensure that AioWrite journal events fit within a single
+    journal object.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 70b9fbf2745c795fe345f0298a2f30af80a12985)
+
+commit f1a2e6aeaa22233f13878d004cbe905a42bdc744
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 5 21:30:18 2016 -0400
+
+    journal: prohibit journal entries larger than the soft object size
+    
+    This avoids complications like a librbd write event of 128MB trying
+    to fit within a 24MB journal object.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d5f307dec10429a1ffc042919b71ac25466a1b72)
+
+commit 05d574926df94b4fa2e0186f31d5050bd0b39ed3
+Merge: 495313f 2d906e6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 13:28:11 2016 -0400
+
+    Merge pull request #9036 from dillaman/wip-15814
+    
+    jewel: librbd: fix get/list mirror image status API
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 2d906e600a8d81aae15bcf3f8b2fc694537890d4
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue May 10 12:29:12 2016 +0300
+
+    rbd-mirror: fixup to get/list mirror image status API
+    
+    - embed name and rbd_mirror_image_info_t into rbd_mirror_image_status_t;
+    - index image status list by image id to make 'start' work correctly;
+    - provide rbd_mirror_image_status_list_cleanup function.
+    
+    Fixes: #15771
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit e6c709b3a8be489a900178b8b0d962705d5687e9)
+
+commit 495313f355955463f747be9e6650fe8796c0d124
+Merge: 9750610 617004e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 13:15:23 2016 -0400
+
+    Merge pull request #9003 from dillaman/wip-15743
+    
+    jewel: librbd: reduce log level for image format 1 warning
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 9750610ebf7f4d853dee23adaf6109c2c6b63ee0
+Merge: 0218f0b fb7eb4e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 13:13:27 2016 -0400
+
+    Merge pull request #9010 from dillaman/wip-15736
+    
+    jewel: rbd-mirror: admin socket commands to start/stop/restart mirroring
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit fb7eb4e6548b6536fccb6fd38b18a5ced0f9ee4b
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Apr 28 09:32:33 2016 +0300
+
+    rbd-mirror: admin socket commands to start/stop/restart mirroring
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit a22cf518f9cc0872f72a8a850ea01558094ebd00)
+
+commit 8e46e4383a15f022e9213c6710671e6241e7fb6c
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Sun May 1 22:04:46 2016 +0300
+
+    qa/workunits/rbd: fix rbd-mirror log file name
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 4c28fc2aa378c10ce6e78b1a2f98d5e14ac211e9)
+
+commit d73073a400807f2aa62d0eadc8ff6a8537ca04fa
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Apr 28 09:23:35 2016 +0300
+
+    rbd-mirror: make image replayer asok commands available when not started
+    
+    Initially the asok commands were registered only after the image
+    replayer start (and unregistered on stop) because their names were
+    built using remote pool and image names, which became known only after
+    start.
+    
+    Now, the asok commands are registered on the image replayer construction
+    using the temporary name "remote_pool_name/global_image_id". They are
+    re-registered using "remote_pool_name/remote_image_name" when the
+    image replayer is started.  Also the commands are not unregistered on
+    the image replayer stop.
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 21790484dc66e97e7625b5b2afabb7efe3a92b08)
+
+commit 24ea574a94c009d64d480bc3f887d4b0325d26d8
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri Apr 1 18:15:33 2016 -0700
+
+    qa/workunits/rbd: add env vars to adapt rbd_mirror.sh to teuthology
+    
+    With these set, this script can work in teuthology, where we want to
+    control the how daemons run and their log locations, valgrind
+    settings, etc. The workunit task always sets CEPH_ID, and allows
+    other env vars to be configured optionally.
+    
+    The RBD_MIRROR_USE_EXISTING_CLUSTER option can also speed up local testing.
+    
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit dcd11a1f262e0c56bb1b37d54283cda06abbf4c7)
+
+commit 21220a1fa668d4080eb7c4844cafeefffa77ecdc
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Apr 7 16:57:14 2016 -0700
+
+    qa: rbd_mirror.sh: change parameters to cluster rather than daemon name
+    
+    Daemon name is only useful to differentiate asok and pid filenames, so
+    just inline its use there. Everywhere else replace the 'daemon'
+    parameter with cluster or local_cluster as appropriate.
+    
+    Remove the CLUSTER{1,2}_DAEMON vars as well, since they are no longer
+    used.
+    
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 5fe64fa806f3bbdbdb950a5643238cc2508814b0)
+
+commit 0218f0b0750b88f308f3ee0837b4e4efc7aeea63
+Merge: a152b32 c1960de
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 13:04:09 2016 -0400
+
+    Merge pull request #9005 from dillaman/wip-15716
+    
+    jewel: rbd-nbd: fix rbd-nbd aio callback error handling
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit a152b32016529633a810fc901a2c980ab1169b27
+Merge: 54177d0 0e95ee4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 13:03:38 2016 -0400
+
+    jewel: librbd: unlock image if journal error encountered during lock
+    
+    jewel: librbd: unlock image if journal error encountered during lock
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 54177d00f06ab3d712c8837832de080e452205de
+Merge: b6b3cdd a73b9dd
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 13:02:24 2016 -0400
+
+    Merge pull request #9009 from dillaman/wip-15713
+    
+    jewel: librbd: possible race condition leads to use-after-free
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit a73b9dd670bcd3a77611c7240d86963b29527983
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 3 07:41:30 2016 -0400
+
+    librbd: possible race condition leads to use-after-free
+    
+    The invoke async operation state machine can complete before the
+    owner lock is released.  Use a stack reference to prevent
+    use-after-free.
+    
+    Fixes: http://tracker.ceph.com/issues/15690
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d9421a25b582e41550526714b71fb55f05abb1de)
+
+commit b6b3cdd40c60775c27b631cd8670e95bc84a5219
+Merge: a3e2e41 28ac027
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 12:53:30 2016 -0400
+
+    Merge pull request #9004 from dillaman/wip-15708
+    
+    jewel: rbd: help message distinction between commands and aliases
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit a3e2e410f5ef6308fcfd59baf607c49112f69138
+Merge: ade4359 82838e3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 12:53:00 2016 -0400
+
+    Merge pull request #8972 from Abhishekvrshny/wip-15697-jewel
+    
+    jewel: python: clone operation will fail if config overridden with "rbd default format = 1"
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit ade4359b15d18ab51ab00168339d1aadcdffacbb
+Merge: 7f95348 451246d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 12:50:43 2016 -0400
+
+    Merge pull request #9008 from dillaman/wip-15698
+    
+    jewel: rbd-mirror: lockdep error during bootstrap
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 9bb17dbb1ce2ae910927e120cf35da9ee4c98225 (refs/remotes/gh/wip-rgw-period-delete-jewel)
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri May 6 15:57:22 2016 -0400
+
+    rgw: fix for duplicates in list_periods()
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit abed30befd86c68922759a66db589285fe01e54b)
+
+commit 7ea6e78aa0aa3b86d663dab3f69998e5cd3a5177
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri May 6 15:18:46 2016 -0400
+
+    rgw: 'period delete' cleans up all period objects
+    
+    Fixes: http://tracker.ceph.com/issues/15469
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit a286b32224d4a9757058415f98a2feff683ac520)
+
+commit 451246df64890e39205d20d40eb5676893a7e1db
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 2 10:33:50 2016 -0400
+
+    librbd: disable automatic refresh of image upon lock message
+    
+    There is no need to refresh the image upon request/release/acquire
+    of the exclusive lock.  The next IO or maintenance op will kick off
+    the refresh.  This is interfering with the refresh state machine
+    unit test case (since two concurrent refreshes shouldn't be possible).
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ee617a0ca56d752daf62bfffa59219b2c540a428)
+
+commit 9e058fc852090ecb1b3a8daa3e0ee9df9d5eba3c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 2 10:26:29 2016 -0400
+
+    librbd: update_features should release lock if acquired
+    
+    There is a small window where requests are unblocked and before
+    the image is shut down where new requests can be accepted.  These
+    are correctly canceled during the shut down, but it can lead to
+    warning messages.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 608e09851396a9eeb7cf6dc1c7f687ef7a033601)
+
+commit e653a1541ca7df29d01e8524a3f3b597fa0fb67b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 2 09:27:29 2016 -0400
+
+    librbd: avoid applying refreshed image config within librados callback
+    
+    There is a potential that a synchronous API call could deadlock a
+    image refresh.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ce5c701bc47b0959f8453b6b92dee4804d3b1d75)
+
+commit a5996e5ad64e01fcde50fc46bf9861b2d163f0c4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 2 08:42:35 2016 -0400
+
+    rbd-mirror: don't use temporary string for snap operation
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 06faf83bbfab000c8200a9cbe781adb0082b3f7b)
+
+commit a891919b85f44a7d81fb2fb84d1b8d7258b3fbdc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 2 08:31:54 2016 -0400
+
+    librbd: avoid recursive locking within operation state machine
+    
+    Fixes: http://tracker.ceph.com/issues/15664
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 91a4890ee78c25391c1548fdacb2b51c46a47415)
+
+commit 7f953488b46642ed3825349d41a34b0faeb6a76b
+Merge: 8963af7 895c975
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 12:47:01 2016 -0400
+
+    Merge pull request #8869 from Abhishekvrshny/wip-15669-jewel
+    
+    jewel: Errors encountered disabling object-map while flatten is in-progress
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 8963af73823285e9261acf6305778903a8f4205d
+Merge: 978f30e 63738d4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 12:46:12 2016 -0400
+
+    Merge pull request #8870 from Abhishekvrshny/wip-15668-jewel
+    
+    jewel: rbd disk-usage CLI command should support calculating full image usage
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 978f30e100fa91c148fd2aeaadad2f6e7bb94f9a
+Merge: 1dceb38 32c0901
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 12:35:41 2016 -0400
+
+    Merge pull request #8868 from Abhishekvrshny/wip-15666-jewel
+    
+    jewel: rbd CLI to retrieve rbd mirror state for a pool / specific image
+    
+    Conflicts:
+    	src/tools/rbd_mirror/ImageReplayer.cc: trivial resolution
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 1dceb38538d9c8a13551ef8b9217cb5757089576
+Merge: 2495f08 357ad37
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 12:33:33 2016 -0400
+
+    Merge pull request #9046 from dillaman/wip-fix-snapc-validation-jewel
+    
+    jewel: librbd: put the validation of image snap context earlier
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 357ad37a4facc9917866c780892c7cd0e21653fd
+Author: runsisi <runsisi@zte.com.cn>
+Date:   Wed Apr 20 09:55:39 2016 +0800
+
+    librbd: put the validation of image snap context earlier
+    
+    we'd better validate the snap context right after we got the mutable
+    metadata before we could go any further
+    
+    Signed-off-by: runsisi <runsisi@zte.com.cn>
+    (cherry picked from commit ca8ae66ba42e7a598ad23ed911938589dc288456)
+
+commit 2495f08a64b352cbd21432b708fdd8e4090b67bb
+Merge: 5e9128d 125aab8
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 12:31:10 2016 -0400
+
+    Merge pull request #8867 from Abhishekvrshny/wip-15661-jewel
+    
+    jewel: Parent image is closed twice if error encountered while opening
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 5e9128dc8b4af16512f689d556baa7d6e08ae1f8
+Merge: c167514 af1c0bc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 12:30:12 2016 -0400
+
+    Merge pull request #8866 from Abhishekvrshny/wip-15660-jewel
+    
+    jewel: rbd-mirror can crash if start up is interrupted
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit c16751431baed0322765e502bf750b7f9be49e78
+Merge: 5e89e82 a2eb187
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 12:29:02 2016 -0400
+
+    Merge pull request #9044 from dillaman/wip-librbd-bad-header-jewel
+    
+    jewel: librbd: does not crash if image header is too short
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit a2eb187658d045160fb85f6c1d9437eaa49f0830
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Apr 28 00:59:34 2016 +0800
+
+    librbd: does not crash if image header is too short
+    
+    if something goes wrong with the object, and returns a chunk shorter
+    than expected, don't panic
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 7b52183addda83f98c8b1f37d3de20ca0fc4687b)
+
+commit 5e89e821f3e9b368b6d4963824e334acbd49cceb
+Merge: f99aba4 8cae07c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 12:26:48 2016 -0400
+
+    Merge pull request #8803 from SUSE/wip-15605-jewel
+    
+    jewel: Potential double free of SetSnapRequest instance
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit f99aba4a869bbb6dda5353a6f5c9c65da32dcd02
+Merge: 9d40896 9b2c173
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 12:16:09 2016 -0400
+
+    Merge pull request #8865 from Abhishekvrshny/wip-15649-jewel
+    
+    jewel: possible double-free of object map invalidation request upon error
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit b6ebb25afc3d75d834311a9931b08c84e858a5e4
+Author: Venky Shankar <vshankar@redhat.com>
+Date:   Thu May 5 15:28:16 2016 +0530
+
+    rbd: helpful error message on map failure
+    
+    Failure to map an rbd image provides pretty much nothing to
+    debug (or rectify) the nature of failure. This change tries
+    to improve the same by providing helpful error messages (by
+    pointing to dmesg, etc..).
+    
+    Fixes: http://http://tracker.ceph.com/issues/15816
+    Signed-off-by: Venky Shankar <vshankar@redhat.com>
+
+commit f221fedb3b34bccd80383096d6ccfa19a4a72db9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 6 11:16:37 2016 -0400
+
+    librbd: assertion to ensure no concurrent processing of replay events
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 44827a3e8e02529be84c829a76ef01f70e0d20ee)
+
+commit 37f08e6e15b21eec2dedfd5e497619165cd3d117
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 6 11:05:50 2016 -0400
+
+    journal: suppress notifications if client still in try_pop_front loop
+    
+    One such example is popping the last entry from an object.  The next
+    object will be automatically prefetched.  When that object is received,
+    we do not want to alert the user that entries are available since
+    try_pop_front already indicated more records were available.
+    
+    Fixes: http://tracker.ceph.com/issues/15755
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3b8d21ac13865791d8fc4c519f47c1f7d6096e76)
+
+commit 82c04c5ad3476805d8c4f7569e1eba41244ad2c7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 6 09:47:01 2016 -0400
+
+    librbd: delay processing of next journal entry until flush in-progress
+    
+    When replaying a journal flush event, do not start processing the next
+    journal entry until after the flush is in progress to ensure the barrier
+    is correctly guarding against future writes.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5d205ef33cb8b5517f141d46a62947dd583500e0)
+
+commit 9d40896d280f44080623b1d9a0d9aba35ae410c5
+Merge: bc764b6 075ee03
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 10 08:31:02 2016 -0700
+
+    Merge pull request #8930 from theanalyst/wip-15707-jewel
+    
+    jewel: multisite: Issues with Deleting Buckets
+
+commit bc764b6af7fb7f5aea6bc5fd8c8f5b7174949a90
+Merge: 8bbe2a9 7029b9c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 10 08:30:52 2016 -0700
+
+    Merge pull request #9029 from theanalyst/wip-15680
+    
+    jewel: rgw: radosgw-admin zone set cuts pool names short if name starts with a period
+
+commit 8bbe2a924d9b27582c130f045e1fede29171f159
+Merge: 80beeff 5fdca84
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 10 08:30:03 2016 -0700
+
+    Merge pull request #9022 from yehudasa/wip-rgw-leak-3-jewel
+    
+    jewel: rgw: leak fixes
+
+commit 80beeff4b1bf89fba6c36b59ffcec61aa8fc975e
+Merge: b9cb6bd d93fa13
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 10 08:29:52 2016 -0700
+
+    Merge pull request #9021 from yehudasa/wip-rgw-xstorageurl-fix-jewel
+    
+    jewel: rgw: fix printing wrong X-Storage-Url in Swift's TempAuth.
+
+commit b9cb6bd73593da4312889c928fba8f02a50b1533
+Merge: ce2f028 f37318a
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 10 08:29:27 2016 -0700
+
+    Merge pull request #9020 from yehudasa/wip-15597-jewel
+    
+    jewel: rgw: fix issue #15597
+
+commit ce2f028268509361700980d47dabe5429127910f
+Merge: 1cba195 a28fe02
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 10 08:29:18 2016 -0700
+
+    Merge pull request #9019 from yehudasa/wip-rgw-admin-zonegroup-jewel
+    
+    jewel: radosgw-admin: add missing --zonegroup-id to usage
+
+commit 1cba1951898e8d21dbfd6b8e354ebaa29f48b720
+Merge: 4ead0f9 1d28aac
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 10 08:29:03 2016 -0700
+
+    Merge pull request #9018 from yehudasa/wip-15626-jewel
+    
+    rgw: add AWS4 completion support for RGW_OP_SET_BUCKET_WEBSITE
+
+commit 4ead0f98dccf79f65878384ac0f45b9f3e2ad97c
+Merge: 6f23218 c08e90e
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 10 08:28:50 2016 -0700
+
+    Merge pull request #9017 from yehudasa/wip-15625-jewel
+    
+    jewel: rgw: segfault at RGWAsyncGetSystemObj
+
+commit 6f23218bbcee8190acfdf0ae7b76e6e742015d91
+Merge: 2650806 e69bfa1
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 10 08:28:37 2016 -0700
+
+    Merge pull request #9016 from yehudasa/wip-rgw-admin-modify-jewel
+    
+    jewel: radosgw-admin: update usage for zone[group] modify
+
+commit 26508062cbb13ba28252735449784dd4b63175af
+Merge: fdf8dcb 11d599c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 10 08:27:56 2016 -0700
+
+    Merge pull request #9015 from yehudasa/wip-rgw-zonegroup-remove-jewel
+    
+    jewel: radosgw-admin: zonegroup remove command
+
+commit f18b14b00fd3b8fbf3007f9eeb9424bbaf5ee14a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 9 11:22:48 2016 -0400
+
+    journal: incorrectly computed object offset within set
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f3372a1bcf81fcfd75e7ef5901dbcdc1daa062a3)
+
+commit 466b7fe3d5cd6da8de993169da4c0e4c0990e703
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Fri Apr 22 09:45:30 2016 +0800
+
+    tools/crushtool: add straw2 support for help message
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 6448ea283b4e1519682ecb29f18fbb767752d496)
+
+commit d6692ed70aebc5387997660e9e2b025e96eff96f
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Apr 20 16:23:55 2016 -0400
+
+    install-deps: remove distribute requirement
+    
+    Because it was meant for Ubuntu 12.04 and it is installing directly
+    from PyPI on newer releases that aren't being built for 12.04
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 64f14d43a829f998b9a2ad5f3d87095560914e2a)
+
+commit 3011edab65e78156346ff7de655707619589bd7a
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Apr 20 15:48:19 2016 -0400
+
+    tools: remove installation for distribute. It is no longer maintained and breaks builds
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d02bd9cb70479b7373c5b65e42e9c57f8eae55f1)
+
+commit 45a1f01aa4baa1913c335efb2f8b3941f3c0f791
+Author: Alexandre Derumier <aderumier@odiso.com>
+Date:   Fri Apr 29 03:30:05 2016 +0200
+
+    set 128MB tcmalloc cache size by bytes
+    
+    (cherry picked from commit d1d144c03bc9050931d86d635a16363bd2b73fd0)
+
+commit bb1aeb7c7bb296d9264f051519d100ecf1f50f1d
+Author: Alexandre Derumier <aderumier@odiso.com>
+Date:   Thu Apr 28 22:55:37 2016 +0200
+
+    etc/default/ceph: set 128MB tcmalloc cache size
+    
+    Signed-off-by: Alexandre Derumier <aderumier@odiso.com>
+    (cherry picked from commit 9decbd1026c6ab0474c73a26a9c84c24557beb8a)
+
+commit 617004ef26b6c97621a85a44b78cc7b0708fe161
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 10 08:33:03 2016 -0400
+
+    test: update rbd integration cram test to remove format 1 warning
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 098038fad47b181a9ddd11767397082b05405be5)
+
+commit 7029b9c3b0c02c9f20f6079243521458d28b2ab4
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Apr 25 15:09:47 2016 -0700
+
+    rgw: fix suffix in RGWZoneParams::fix_pool_names()
+    
+    Fixes: http://tracker.ceph.com/issues/15598
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 6f6b0c925994b0369eda390a6b3fd20005e70c9b)
+
+commit f37318af823f76ba7bf05e09da83141b569bf579
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon May 9 14:23:03 2016 -0700
+
+    rgw: code cleanup
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d53ffbfe65611425b08af71e0d24bfe2a0076f1e)
+
+commit a8b800b903f0f1144efe0cf3568b02b3e200c3b2
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Apr 26 11:31:43 2016 -0700
+
+    rgw: upgrade default zonegroup and set correct zone[group] id
+    
+    Fixes: http://tracker.ceph.com/issues/15597
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 27806c71922eef2f4cde360d7ba40dee9d5bd4bd)
+
+commit 5fdca846aaea9afd16828d33ccc279c4dcdbcd4c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu May 5 10:57:05 2016 -0700
+
+    rgw: RGWHTTPClient requests can unregister themselves early
+    
+    No need to wait for req_state to complete anymore.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 91f61d68bf5fc39152d75fbc633f088e17d53d9e)
+    
+    Conflicts:
+    	src/rgw/rgw_http_client.cc
+
+commit 5609eb1bae0fc55b24a3025621597590f30aaad4
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed May 4 18:08:38 2016 -0700
+
+    rgw: move around sync_env.init()
+    
+    call sync_env.init() even if we're initalized in RGWRemoteDataLog::init()
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 3d8442effc1c175e0b7c5064ed41bc642f6cdcb0)
+
+commit faf6b2d25e89b7a84ed262c08b0a858dbf4199da
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 3 12:01:55 2016 -0700
+
+    rgw: rados crs, explicit cleanup
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 1c876eb3873692042c0af3644f8ee3ad15ffdbf5)
+
+commit b55514c390b22605fabdf3a8e5011415f57e6da8
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 3 12:00:46 2016 -0700
+
+    rgw: RGWHTTPManager, avoid referring to req_data->client when completing
+    
+    this is past calling req_data->finish(), client can be destroyed by now
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 7ada4df124a71eae413e7dec0e25264a8deffcad)
+
+commit 2d0cd8a08bb26f519d2b9b59bfe54f014cc36a1f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 3 11:59:11 2016 -0700
+
+    rgw: civetweb_callback() refer to store inside lock
+    
+    pe->store might change when reconfiguring
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 2ca38dab071c7d20a224a4e1dacfd68548fe52dd)
+
+commit 2f65b313d08a5c9dfde220353eaa54204531b1ee
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 3 10:03:14 2016 -0700
+
+    rgw: RGWPeriodPusher, stop http manager explicitly
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 98433e589f2a8b519726ed23b1565c5eb331c68e)
+
+commit e5312b10aaa000d942468bf51d2164c689add279
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 3 10:02:02 2016 -0700
+
+    rgw: RGWHTTPManager, can call stop() more than once
+    
+    destructor calls stop(), but it can be also called explicitly, if caller
+    wants to ensure requests are not being handled anymore.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit bdce81a2ae00f154420386e107a104cbf100d96d)
+
+commit cbea993562b03f75ceb0bdb7c02d72e5b51ae0fa
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 3 10:01:33 2016 -0700
+
+    rgw: RGWReadRESTResourceCR, fix refcounting
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d1ec31def71505dc95139fbdd3b49c92fec1d296)
+
+commit 93a65f3306e21c1bc34f5a95d85deeba746adb62
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 3 09:59:47 2016 -0700
+
+    rgw: RGWReadRemoteDataLogShardCR fix destructor
+    
+    was missing destructor, constructor was broken
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 95162a4c1bf61f7329e363b7efd5686f7e1025f0)
+
+commit 2de16690b6a404a365af5d9f526de044873f9729
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon May 2 18:13:17 2016 -0700
+
+    rgw: RGWDataSyncStatusManager, cleanup if failing init
+    
+    init can be called more than once, ended up with leaking some resources
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 14cd44e0847fd914068f48dd0208ce7c1fe4eecb)
+
+commit 57266e6b78851f8e6167f15ad8dbf7195d561371
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Apr 29 13:04:13 2016 -0700
+
+    rgw: rest crs, explicitly call cleanup
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit a3a3f4f5883f3fc2bf47a7ebfa538206b7ab51d9)
+
+commit 6b86332ca3ee35eb522cbf88408b7fd5f3070772
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Apr 28 17:03:21 2016 -0700
+
+    rgw: more leaks fixes
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit cb797d4fd754b4a33eb34c249da3aabc65a7aba3)
+
+commit 0fb48546a9fe6eb497833a6566353229e0923d3f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Apr 27 16:39:36 2016 -0700
+
+    rgw: drop a reference to http op
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 4442d1c19acfc28c2dd4a011365ebebe08750963)
+
+commit d93fa132bba597e71c4dd483663d2069e7117bf1
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Sun Apr 24 17:41:55 2016 +0200
+
+    rgw: fix printing wrong X-Storage-Url in Swift's TempAuth.
+    
+    If the option "rgw_swift_account_in_url" is being set to true,
+    both user and subuser IDs will be present in the X-Storage-Url
+    header generated by the implementation of Swift API's TempAuth.
+    This patch rectifies this behavior and makes that only the user
+    identifier will be placed there.
+    
+    Fixes: http://tracker.ceph.com/issues/15667
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 88f4c67d822388d2038454a25ead7e9a11f404b5)
+
+commit a28fe02c933e2ec7ea969a974f961769052e9d38
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Apr 26 15:20:45 2016 -0400
+
+    radosgw-admin: add missing --zonegroup-id to usage
+    
+    also clarified that --rgw-zone takes the zone by name
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    Reported-by: John Wilkins <jowilkin@redhat.com>
+    (cherry picked from commit 97fad754565b262529073e1b9f8d1799bfdfe0de)
+
+commit c08e90ea20b8ed96c7fe8638f725cf5abefc6783
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Apr 27 15:09:55 2016 -0700
+
+    rgw: don't allow any concurrent sync requests on the same key
+    
+    We used to allow concurrent requests, as long as these had different op,
+    which wasn't correct.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit edea6d58dd25995bcc1ed4fc5be6f72ce4a6835a)
+
+commit 6c61341697b99db34466c9e7e90aa7867048d651
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Apr 26 15:59:33 2016 -0700
+
+    rgw: collect children after waiting for them
+    
+    Otherwise we're going to busy loop there.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 806fb85d18653910863ba6b8cb56b8db1a0fa3d2)
+
+commit 4a5f33dd975eb44f908b8a90a7ad2e4beb17b354
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Apr 26 15:29:41 2016 -0700
+
+    rgw: don't pass object context into async coroutines
+    
+    Fixes: http://tracker.ceph.com/issues/15625
+    
+    The async read data/attrs coroutines may outlive the callers. Instead
+    of introducing a complicated refcounting scheme, just don't pass it in.
+    Anyway, it was not thread safe, and the benefit of using it is not
+    clear.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit b225d1758057efc30721aad25447d54fa36e4cb2)
+
+commit 1d12f82a24f78769bcebb32cb379445e04f35e46
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Apr 21 17:24:16 2016 -0700
+
+    rgw: check for status >= 0
+    
+    Fixes: #15565
+    
+    check only verified that status was > 0, so we ended not updating mtime for
+    the target object. This resulted in a continuous sync of objects.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 255f19d9ef964536f2db17f70749f489330262a6)
+
+commit a288a054c18fedda9ac3230d392f5a41a140ca9a
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Apr 22 14:23:22 2016 -0700
+
+    rgw: don't try to delete object if does not exist
+    
+    If we know that the object does not exist, no need to remove it.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 8d59a30d56b5f197a293f7fba6ebb1a630a33a05)
+
+commit 308b9f54e9fa07d071335bb2d953a50a5095c464
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Apr 22 11:55:28 2016 -0700
+
+    rgw, cls/rgw: store removed object mtime in bi log
+    
+    Store the mtime of the object that was removed in the bucket index log.
+    This will be used in data sync.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 80d64049937e1930ea8273fdad76a51493a5e14a)
+
+commit 1d28aac642d940bb1c63733406999a7b027e05f2
+Author: Javier M. Mellid <jmunhoz@igalia.com>
+Date:   Wed Apr 27 20:28:08 2016 +0200
+
+    rgw: add AWS4 completion support for RGW_OP_SET_BUCKET_WEBSITE
+    
+    Fixes: http://tracker.ceph.com/issues/15626
+    
+    Signed-off-by: Javier M. Mellid <jmunhoz@igalia.com>
+    (cherry picked from commit 05373354e4725c3fc675df652fbf97da26160bcb)
+
+commit e69bfa14e184716c4ab6bf01746edf5c9663a8e1
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Apr 27 18:06:45 2016 -0400
+
+    radosgw-admin: update usage for zone[group] modify
+    
+    the help strings indicated that only --master could be changed
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 410d432d11541a29c3fd4608a070cecf8dc2766e)
+
+commit 11d599cda659a42ddde43f34307c5e3553596c54
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri May 6 14:26:46 2016 -0400
+
+    test/rgw: add test_zonegroup_remove
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit e2b27c7266f0ace372dbf94e062598fd7a575795)
+
+commit 928ccf4952052ab19044de1a8f20246075d54832
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri May 6 14:26:23 2016 -0400
+
+    test/rgw: index zones by name instead of insertion order
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 6327ade12bf0255c3b35bd6407f1dd2bcc6936fa)
+
+commit 5edacdccc61d565a0e91af452f9297d559a474e4
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon May 2 11:38:24 2016 -0400
+
+    radosgw-admin: add missing 'zonegroup remove'
+    
+    Fixes: http://tracker.ceph.com/issues/15684
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit db0fa48ed5958d37928f893acd44cd66b6b9b990)
+
+commit 6b1d9ebb80478c1dfe0e974377bd4fb070a5d417
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon May 2 11:36:58 2016 -0400
+
+    rgw: RGWZoneGroup::remove_zone() takes zone id
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 3a451511fb38a5f1cd83ca3610c91141e7279f63)
+
+commit 0e95ee4bd16f4d54d1da476b51fd0fd0829d69ed
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 3 10:15:08 2016 -0400
+
+    librbd: unlock image if journal error encountered during lock
+    
+    Explicitly unlock to prevent a client from accidentally blacklisting
+    itself when retrying the lock.
+    
+    Fixes: http://tracker.ceph.com/issues/15709
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a11f5e8e55fc448ed60616cbf66a3ea7db2247b9)
+
+commit fdf8dcb772d8313a3a801bf265f022505dba934d
+Merge: 030883f 0de6345
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon May 9 13:54:58 2016 -0400
+
+    Merge pull request #9006 from athanatos/wip-15655-jewel
+    
+    OSDMonitor: avoid underflow in reweight-by-utilization if max_change=1
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit c1960debf5c8696f365cd670dedeea01d2a3b9a8
+Author: cy.l@inwinstack.com <cy.l@inwinstack.com>
+Date:   Sat Apr 30 15:22:29 2016 +0800
+
+    rbd-nbd: Fix aio_callback error handling
+    
+    Change aio_callback return code handling for NBD_CMD_READ command.
+    
+    Tracker:
+    http://tracker.ceph.com/issues/15604
+    
+    Signed-off-by: Chang-Yi Lee <cy.l@inwinstack.com>
+    (cherry picked from commit ee211295441b126a995c07820176cfc966ab61c5)
+
+commit 4e87c1cd7e923f4254ce43e92e1cce4329fd86dd
+Author: cy.l@inwinstack.com <cy.l@inwinstack.com>
+Date:   Wed Apr 27 11:00:25 2016 +0800
+
+    Fix RBD-NBD aio_callback error handling
+    
+    Tracker-Id: #15604
+    
+    Signed-off-by: Chang-Yi Lee <cy.l@inwinstack.com>
+    (cherry picked from commit f8f6c1dc865b7ed642fec3f416e5eca5e91523d0)
+
+commit 28ac027f6f86635d320f0f5e25582452011d9bed
+Author: YongQiang <he.yongqiang@h3c.com>
+Date:   Fri Apr 22 11:20:31 2016 -0400
+
+    rbd:make a distinction of help message between ''rbd snap rollback" and "rbd snap revert"
+    
+    Fixes: https://tracker.ceph.com/issues/15521
+    
+    Signed-off-by: Yongqiang He <he.yongqiang@h3c.com>
+    (cherry picked from commit 1bf26509c6efd06a8facc0e45ab42255592ca74d)
+
+commit 039554d7ad8cfd82bcb6d4c6564f6dd46320a24f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 4 13:36:06 2016 -0400
+
+    librbd: reduce log level for image format 1 warning
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c87f383b65ccc2e91645394f1c75de1c2d52e1e1)
+
+commit 030883f41d3d2d8f297b3d3f6e7c7fa5a06ea4c0
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Tue Apr 26 15:12:17 2016 +0800
+
+    osdc/Objecter: fix race condition for sortbitwise flag detection
+    
+    During list_objects/list_nobjects(), we are checking osdmap's
+    sortbitwise flag setting without holding rwlock in hand, which
+    is a potential race condition.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 7fb6be706be2454cc2a831df29808997f0aa5ab3)
+
+commit 82838e340e0ab7c593618f5807ea5868ecd6bc48
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 2 15:06:51 2016 -0400
+
+    librbd: default clone operation to image format 2
+    
+    Image format 1 doesn't support clones, so unless the user
+    overrode the setting, default to format 2.
+    
+    Fixes: http://tracker.ceph.com/issues/15685
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f24c70f54343aa6698af8ac1f0774282b0659968)
+
+commit ca13a9554d8a9d97428c2096b5a2ae8e25d08a9e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 2 14:51:31 2016 -0400
+
+    librbd: add rbd_image_options_is_set helper method to API
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2633b045e0b57827cc10c2e7707bd5a5e344e59a)
+
+commit e6aa4533feb9c8e6d15c820f54dd719c378dd26e
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Apr 29 20:12:20 2016 +0100
+
+    mds: fix upgrades with replay daemons in map
+    
+    In the new world, a standby replay daemon which
+    is actively replaying has its rank set.  In the old
+    world it did not: map this during the upgrade/downgrade
+    paths in decode/encode.
+    
+    Fixes: http://tracker.ceph.com/issues/15591
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit d94115aa2a4bd92c56552e9fd11205d391a1c673)
+
+commit 6c1b792b7b2fba729f21b0f10ec70d14b830ffa9
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 30 12:06:27 2016 +0100
+
+    doc: add cephfs daemon management docs
+    
+    Principally to tell people how to configure
+    standby, but more generally as a place to define
+    our terms for GIDs etc.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 97dd6897efaa14d3f8859a791949f9d246769620)
+
+commit bf44c6d434a41b90c0739e493a022cb3d3df5554
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 30 12:05:43 2016 +0100
+
+    mds: omit fscid in single-filesystem status output
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit ec8f3db3e5c26b60b9333e86453e1b17324a8cfa)
+
+commit be5274c3f47ecb0dca0facca37d4a90c68b37027
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Mar 24 13:23:26 2016 +0000
+
+    mds: simplify standby/standby-replay logic
+    
+    This used to use an arcane set of constants
+    in standby_for_rank, combined with daemons sometimes
+    sending requests to enter state STANDBY_REPLAY.
+    
+    Simplify this so that there is only one place we
+    put daemons into standby replay, and that is
+    in tick->maybe_promote_staandby.
+    
+    There is a behavioural change in here, which is
+    that we used to sometimes promote standbys to
+    be standby-replay, even if they didn't have
+    "standby replay = true", when they did have
+    a standby_for_rank or standby_for_name set.
+    I'm calling that a bug, and making it so that
+    daemons will only go into standby-replay if
+    "standby replay = true" is set.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 2a2a56b2d0e9e4224cae27f80d990b787e209bed)
+
+commit ae3ce5b8741eaf65d03aeb7b74925a75e5ef2598
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Mar 24 13:13:55 2016 +0000
+
+    messages: add MMDSBeacon::standby_replay
+    
+    So that daemons can explicitly say that they
+    want to be standby replay, instead of using
+    magic standby_for_rank values.
+    
+    Because in some cases daemons would indicate this
+    with a beacon requesting STANDBY_REPLAY state, we
+    also munge these during decode to be normal STANDBY
+    requests with standby_replay=true.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit cdf8a29f81475a000824ce8942f02d9d570ec3bb)
+
+commit 02e3edd93c0f4ef6e0d11df1f35187f74c7ea2ff
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Mar 22 17:44:22 2016 +0000
+
+    mds: remove ONESHOT_REPLAY mode
+    
+    This predated cephfs-journal-tool.  Don't bother
+    with any backwards-compatibility mechanisms here, as
+    it was a seriously niche undocumented thing.
+    
+    On the off chance that someone ran an old ceph-mds against
+    a newer ceph-mon with oneshot enabled, it shouldn't break
+    anything, it just won't do what they want.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit edfda9c6b0a0c8b75ebff5f06017aa475339b1fd)
+
+commit 52ca195221ffa86b778346e847aeec817329c651
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Apr 7 12:53:00 2016 +0100
+
+    mds: remove stale comments
+    
+    A little bit of detritus from the MDS->MDSRank refactor.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit e9b58e7ff97646e557c5b3e2c86a067d2d136149)
+
+commit c1279d8ff8716a54622bf98bd29ce40faa2bf41d
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Apr 7 12:37:08 2016 +0100
+
+    mds: remove inc array from mdsmap
+    
+    Instead, use the MDSMap epochs to provide the `inc` that
+    we populate into mds_info_t for use by the daemons.
+    
+    It is not necessary to respect the current value of inc
+    in old maps, because the per-rank counters were necessarily
+    lower numbers than the overall map epoch (incrementing
+    the counters always corresponded to a map change).
+    
+    Fixes: http://tracker.ceph.com/issues/15399
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 30a375ebf814245fbbb5155fd39d8ff673f0a32a)
+
+commit 9d5162fe102301bc94065f99a20b3067395dde2a
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Sep 24 14:03:58 2015 +0100
+
+    test/mds: add test for symbols in paths
+    
+    Especially, hyphens.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 4e9327f64e5965c57c0eeea397060dc2ab43cfbf)
+
+commit af3a4e4af36b5159721b634e216edcab9a171155
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Apr 12 02:49:32 2016 +0100
+
+    mds: fix auth caps with hyphen in path
+    
+    Quirk of the boost spirit char() fn, hyphens
+    are treated as range delimiters unless they're
+    the last character in the set.
+    
+    Fixes: #15465
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit f82fa5f59a80851581573553be07e8f1620ee346)
+
+commit 075ee0310888d012c56c494291b873be47393e47
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Tue Apr 19 15:20:53 2016 +0200
+
+    rgw_op: pass delete bucket op to master zone first
+    
+    Basically first ask the master to process a delete bucket request before
+    we process it in the zone locally, otherwise we reach a situation where
+    we drop the bucket locally and master will still maintain an index
+    
+    Fixes: http://tracker.ceph.com/issues/15540
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit da6b44a6e47399493adae53ad3c36a029f1e2a23)
+
+commit 1527b56a888a65fa28223cd9c9cdb06d1575973c
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Tue Apr 19 15:20:07 2016 +0200
+
+    rgw: add errno entry for -ENOTEMPTY
+    
+    currently if master returns an -ENOTEMTPY, we return a -EIO as we dont
+    have an entry for this
+    
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit 40c3bfef8cc3bce7d15fbb0e135c1a909ee59368)
+
+commit 791eba81a5467dd5de4f1680ed0deb647eb3fb8b
+Author: Alexandre Derumier <aderumier@odiso.com>
+Date:   Thu Apr 28 22:48:52 2016 +0200
+
+    fix deb package /etc/default/ceph location
+    
+    commit 7384a14f243519547a61534d22e21e6069aae016 have introduce
+    the /etc/default/ceph file at a wrong location : /etc/default/ceph/ceph
+    
+    (cherry picked from commit 13804fafd4c33068d8ed6d0480b594daec0f9341)
+
+commit d9851351aeb6d45a2df1c107b23e77c992926d0a
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Tue Apr 26 11:13:32 2016 +0800
+
+    mon/OSDMonitor: improve reweight_by_utilization() logic
+    
+    By calling reweight_by_utilization() method, we are aiming at an evener result
+    of utilization among all osds. To achieve this, we shall decrease weights of
+    osds which are currently overloaded, and try to increase weights of osds which
+    are currently underloaded when it is possible.
+    However, we can't do this all at a time in order to avoid a massive pg migrations
+    between osds. Thus we introduce a max_osds limit to smooth the progress.
+    
+    The problem here is that we have sorted the utilization of all osds in a descending
+    manner and we always try to decrease the weights of the most overloaded osds
+    since they are most likely to encounter a nearfull/full transition soon, but
+    we won't increase the weights from the most underloaded(least utilized by contrast)
+    at the same time, which I think is not quite reasonable.
+    
+    Actually, the best thing would probably be to iterate over teh low and high osds
+    in parallel, and do the ones that are furthest from the average first.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit e7a32534ebc9e27f955ff2d7a8d1db511383301e)
+
+commit b0543fdfdebc6766fe927b4cc34a4369e7f86544
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Mon Apr 25 15:43:03 2016 +0800
+
+    mon/OSDMonitor: avoid potential expensive grace calculation
+    
+    The grace calculation during check_failure() is now very complicated
+    and time-consuming. Therefore we shall skip this when it is possible.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 3557903d5d57642179b2ae137bedc389974b1956)
+
+commit 53686dfac0622a981374d448ce8229eb3984e5c2
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Apr 29 22:01:56 2016 -0400
+
+    mon/OSDMonitor: max_osds must be > 0 for reweight-by-*
+    
+    Include a few test cases.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 84bd0dce18f6dd0c35d7e6cfee91ce8c5a0f4db7)
+
+commit ec416556e516b60ab490c424a4e45fa9ff8f8fd0
+Merge: 442c31b f4d63af
+Author: John Spray <jspray@redhat.com>
+Date:   Tue May 3 08:05:23 2016 -0500
+
+    Merge pull request #8787 from xiaoxichen/wip-15631
+    
+    jewel: CephFSVolumeClient should isolate volumes by RADOS namespace
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 0de63455cbf662435ff57a35b9d98187091dcec7 (refs/remotes/gh/wip-sam-testing-jewel)
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Apr 28 13:47:48 2016 -0700
+
+    OSDMonitor: avoid underflow in reweight-by-utilization if max_change=1
+    
+    Fixes: http://tracker.ceph.com/issues/15655
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit c121bc51a5a877a49094cfbd3bd16af1f3ae5090)
+
+commit 63738d4d154d983d98021a09daa24607df64b979
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 28 15:53:59 2016 -0400
+
+    rbd: disk usage now includes all snapshots by default
+    
+    Added a new "--from-snap" optional to restrict the calculation to
+    specific subsets of snapshots and the HEAD revision.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1ccdcb5b6c1cfd176a86df4f115a88accc81b4d0)
+
+commit 895c975651b722832befeb829a43acc55823740b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 28 12:48:59 2016 -0400
+
+    librbd: block RPC requests while updating features
+    
+    Disabling the journal and object map require acquiring the exclusive
+    lock locally.  We don't want to start executing long-running ops
+    for this quick process.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit db7aaff0f6d7a57e2b36d22b609f915f9b2b3893)
+
+commit 1f0056a427cd46793934c59c49aa225da2237628
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 28 12:43:24 2016 -0400
+
+    librbd: return -ESHUTDOWN when a refresh is requested on a closed image
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ed132a1eb10e6b2eb4bc465658eb0379daca3d67)
+
+commit 6da1a84f16f4478ec17bdd4f50d9b0a02178c444
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 28 12:42:49 2016 -0400
+
+    librbd: synchronous block_writes should return possible error result
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 88e76cb3c9a74b4718ad39c827f2194666f5be2a)
+
+commit 04ef40e7652ff05146289a6e9e15ce2290a1ee32
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 28 11:46:20 2016 -0400
+
+    librbd: refresh image before executing maint op
+    
+    If a maint op is interrupted and restarted, it's possible that the
+    image will need to be refreshed.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9cf6821bd9f58c181720a5c8156cec5ab33d35d4)
+
+commit 64cb6455f384f5ecc20abcf4d4ec6596569ac4c0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 28 11:10:25 2016 -0400
+
+    librbd: reduce log level when lock owner not detected
+    
+    This can happen under normal operating conditions and the
+    lock request is automatically retried.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5a4893b7a7e50cc399696e1d0cb4d4f8785e723d)
+
+commit 49cfb0e48014610d04233a6c7549bbc3756dd3f6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 28 10:52:15 2016 -0400
+
+    librbd: reduce log level for interrupted maint ops
+    
+    Fixes: http://tracker.ceph.com/issues/15572
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1617328402345c68bf5d54828da6d687e51ef42b)
+
+commit 32c09015a16ef8dee535d2c81825a2590f7f15e7
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Sun Apr 24 19:39:40 2016 +0300
+
+    qa/workunits/rbd: test mirror status in pool directory
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit f15cd51a1ddbd4079a8218bc2446a4bf68f95107)
+
+commit 9a30a89abc5749c37d26d39c175e51259f63caec
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Apr 21 09:52:47 2016 +0300
+
+    rbd-mirror: in replay status store number of entries behind master
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit e4229d1d4b163f1003e68fef1db63f796393ba23)
+
+commit 15a9131fd2d55191c3082c6692f5e8f714fcff71
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Fri Apr 15 08:42:17 2016 +0300
+
+    librbd: API to retrieve image mirroring status
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit a2045d61874f9e418b5b3c3136af4482603c8e7f)
+
+commit 98ca7f3904a9c8ff9a661f153069a8747a51431b
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Apr 6 14:57:29 2016 +0300
+
+    test: MockJournaler: add get_cached_client method
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 7633407ab584ce8c7777a525554773c2a358ce5f)
+
+commit e57c4d8f0832abcf431029b04da43e043a479d19
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Apr 5 14:42:09 2016 +0300
+
+    rbd: CLI to retrieve rbd mirror state for a pool / specific image
+    
+    Fixes: #15144
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit fe5207278d884464c05d6808245a0c4df03c441c)
+
+commit 6111a25448fda1824ca6b3007fcdece1970674d6
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Apr 5 14:40:50 2016 +0300
+
+    rbd-mirror: store replay status in mirroring object
+    
+    Fixes: #14420
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 52b2fe14cfc932b2aa53ce76a3150ce87ce4d377)
+
+commit f22514201c3b49d93593fad27360deb7b715493d
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Apr 5 14:25:11 2016 +0300
+
+    test: cls_rbd: methods to store/retrieve mirroring status
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 8aefad59813489cee112f46b3fe10331bb039b57)
+
+commit 4a3f0d23ba4b96a28de661e219dcb52165bbbeac
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Apr 5 14:24:07 2016 +0300
+
+    cls::rbd:: methods to store/retrieve mirroring status
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 17dd40df4f4201cf0c0c6d58b1728177c5ee726b)
+
+commit 62c60144311518e7e7947791a17608ed1765f971
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Fri Apr 1 08:27:03 2016 +0300
+
+    objclass: add method to list watchers
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 44a31d2b770ef016271f9285447c601921b9c838)
+
+commit 442c31bcf73899767c7fa198427712bbdf767067
+Merge: 6e5091e d302617
+Author: Boris Ranto <branto@redhat.com>
+Date:   Mon May 2 08:35:09 2016 +0200
+
+    Merge pull request #8843 from SUSE/wip-msd-typo-fix-jewel
+    
+    systemd: fix typo in preset file
+    
+    Reviewed-by: Boris Ranto <branto@redhat.com>
+
+commit 125aab8dc4300dc9c67b2d4ca219cfbd60ab0924
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 27 15:56:11 2016 -0400
+
+    librbd: propagate any image refresh errors back to caller
+    
+    Previously, the saved error code was only propagated when an image
+    flush was required.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 105ef8a24aa6ac8719f68c5a9366129ddb41ee88)
+
+commit 5fe4beac0ed68ebf33fa69ba7a3082d2072bb9ce
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 27 15:45:25 2016 -0400
+
+    librbd: avoid second close attempt of invalid parent image
+    
+    Fixes: http://tracker.ceph.com/issues/15574
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e91e7f8d9ce10f0d19c48facfb6318d6b49798e8)
+
+commit af1c0bce9184c55aa88b4082db9302239009af36
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 27 15:22:59 2016 -0400
+
+    rbd-mirror: interrupting image replay startup can cause crash
+    
+    Properly shut down the journaler if a shut down has been requested
+    while start up is in-progress.
+    
+    Fixes: http://tracker.ceph.com/issues/15630
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3a71052a5b14af34ed81f6ee77f862f576a7a5cc)
+
+commit 9b2c17392a2d9543f05d0cb8486183edf84eb2c3
+Author: runsisi <runsisi@zte.com.cn>
+Date:   Thu Apr 28 09:05:33 2016 +0800
+
+    librbd: fix potential assertion for object map invalidation
+    
+    if update_flags fails we should not go any further, or the async request
+    will be removed from m_image_ctx.async_requests twice
+    
+    Signed-off-by: runsisi <runsisi@zte.com.cn>
+    (cherry picked from commit dd0eb719a50e9fb24eca5a840c2e8adb53be41b2)
+
+commit 6e5091e454018e42ebe79bc4ec70cd24bfaeb36d
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Apr 30 16:21:13 2016 +0200
+
+    systemd: fix typo in preset file
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 80be4a8cbf1d42232ff61494d035e7abd90c2c14)
+
+commit d3026174f87ede746a0ab0aa96b0efbf5e77b161
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Apr 30 16:21:13 2016 +0200
+
+    systemd: fix typo in preset file
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 80be4a8cbf1d42232ff61494d035e7abd90c2c14)
+
+commit 9cdad1f348a67e3f4daec617210340b14a2dd49a
+Merge: 96b3726 37ccacf
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri Apr 29 08:22:01 2016 +0200
+
+    Merge pull request #8801 from SUSE/wip-15633-jewel
+    
+    jewel: ceph-{mds,mon,osd} packages need scriptlets with systemd code
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+    Reviewed-by: Boris Ranto <branto@redhat.com>
+
+commit 8cae07c10b9cb62af87d494724586c945f181127
+Author: runsisi <runsisi@zte.com.cn>
+Date:   Thu Apr 21 14:48:55 2016 +0800
+
+    librbd: fix potential double free of SetSnapRequest instance
+    
+    if image feature EXCLUSIVE_LOCK is not enabled we should not try to
+    initialize the exclusive lock, or we may end with two async Contexts
+    to finish the same SetSnapRequest instance
+    
+    Fixes: http://tracker.ceph.com/issues/15571
+    Signed-off-by: runsisi <runsisi@zte.com.cn>
+    (cherry picked from commit 0a6eaac12fe29762c3dd29068f9427840cfea100)
+
+commit 37ccacf7d1ce57291d6cbadd416fb96aed7db36a
+Author: Boris Ranto <branto@redhat.com>
+Date:   Wed Apr 27 18:10:51 2016 +0200
+
+    rpm: Add rpm scripts for ceph-rbd-mirror
+    
+    We are currently missing the systemd scripts for the new ceph-rbd-mirror
+    daemons. This patch introduces them.
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 14652f32165f72efde569a939179fc26584ab155)
+
+commit c729bdd4bcb974081899c65934d3fdb9e0e96316
+Author: Boris Ranto <branto@redhat.com>
+Date:   Wed Apr 27 18:00:08 2016 +0200
+
+    rpm: Start all the targets in %post
+    
+    We need to have the sub-targets active for the ceph.target to be able
+    to propagate its calls to the services. If the sub-target is inactive,
+    the main target won't propagate the stop/restart calls.
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 73f457986753b504fd088c1fa064608aa9d340c9)
+
+commit c72f0bc3d4fb9a3f5abfefcf8d2775721d95b33a
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Apr 19 14:48:41 2016 +0200
+
+    rpm: implement scriptlets for the post-split daemon packages
+    
+    This patch gives each of the ceph-{mds,mon,osd,radosgw} packages its own
+    %post, %preun, and %postun scriptlets dealing with the package's unit files.
+    
+    The scriptlets of ceph-base are adapted to handle the ceph.target unit file
+    only.
+    
+    The scriptlets of ceph-mon handle ceph-create-keys services in addition to ceph-mon.
+    
+    The scriptlets of ceph-osd handle ceph-disk services in addition to ceph-osd.
+    
+    Fixes: http://tracker.ceph.com/issues/14941
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 644aba9270714e5e231ac7d7e3437477837531eb)
+
+commit 81f407337da18b5b555c6737e394a0e054d7c7da
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Apr 19 14:51:01 2016 +0200
+
+    systemd: enable all the ceph .target services by default
+    
+    Some distros, like Fedora and openSUSE, have a policy that all services are
+    disabled by default.
+    
+    This patch changes that default for the ceph.target and
+    ceph-{mds,mon,osd,radosgw}.target services.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 53b1a6799c2ce08acc8382e4aa996d1bde700316)
+
+commit f4d63af533b22f740d809272e2e47ebc9d8d6845
+Author: Xiaoxi Chen <xiaoxchen@ebay.com>
+Date:   Wed Apr 6 22:50:05 2016 +0800
+
+    python_cephfs: rule out empty/None volume_id
+    
+    Signed-off-by: Xiaoxi Chen <xiaoxchen@ebay.com>
+    (cherry picked from commit 73368069dd466cdbde1ebeae97771d603ce431f5)
+
+commit b609017a52e3140e9ff6314b8d312a409af52a57
+Author: Xiaoxi Chen <xiaoxchen@ebay.com>
+Date:   Wed Apr 6 22:45:02 2016 +0800
+
+    python-cephfs: use rados namespace for data isolation.
+    
+    As cephfs dir layout supports rados namespace in jewel, it would be
+    good to use rados namespace instead of pool, since it is free,
+    we always enforce namespace isolation for volumes.
+    
+    The meaning of "data_isolated" flag changed a bit, it's no longer used
+    for security isolation, referring to physical isolation
+    (i.e. potentially using different OSDs).
+    
+    Also, (de)authorize based on namespace.
+    
+    Signed-off-by: Xiaoxi Chen <xiaoxchen@ebay.com>
+    (cherry picked from commit 0e1d013ea690c18f7fa6e2a19500f854a9949091)
+
+commit 96b3726d74cb7bbcacdd22e0ab0fe233e4c4d829
+Author: Tamil Muthamizhan <tmuthamizhan@MacBook-Air.local>
+Date:   Tue Apr 26 17:34:52 2016 -0700
+
+    Signed-off-by: Tamil Muthamizhan <tmuthami@redhat.com>
+    
+    s/ceph-deploy/ceph-tests, hence removing ceph-deploy folder
+
+commit 77fdbf1747d73fd809607efd3325cc38b2fbafc0
+Author: Tamil Muthamizhan <tamil@magna002.ceph.redhat.com>
+Date:   Fri Apr 22 18:25:46 2016 -0400
+
+    Signed-off-by: Tamil Muthamizhan <tmuthami@redhat.com>
+    
+      renamed the workunit and test to reflect reality and for better clarity
+    
+    (cherry picked from commit 0fd358b877c611bf67c94b4a2dacbe65e17deae9)
+
+commit 1fa533e83d8e541315aa4925a6192ab0401b688b
+Author: Alexandre Derumier <aderumier@odiso.com>
+Date:   Sun Apr 24 09:55:35 2016 +0200
+
+    fix ceph init script
+    
+    commit 65963739cd6815b8008282c8f64cd64365662e60 have introduce a bug
+    
+    test variables need to be quoted, or -n always return true, even if variables don't exist
+    
+    (cherry picked from commit b581dd56bea09520c154611ecea91ebe399058d3)
+
+commit 7acbefadace28412f395bb98aec5651d5ec70666
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Apr 18 18:27:52 2016 -0700
+
+    test: Fix ceph-objectstore-tool test to run manually from src non-cmake
+    
+    Fix init-ceph to handle it when CEPH_* values set
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 5e8e03b4d030db57c93047551d5b92db0b604063)
+
+commit af4b31cb9770d1710d038c557d547373f58a89a8
+Author: James Page <james.page@ubuntu.com>
+Date:   Tue Apr 5 16:58:58 2016 +0100
+
+    Drop --setuser/--setgroup from osd prestart
+    
+    These are not supported by /usr/lib/ceph/ceph-osd-prestart.sh,
+    resulting in warnings:
+    
+     ceph-osd-prestart.sh[23367]: getopt: unrecognized option '--setuser'
+     ceph-osd-prestart.sh[23367]: getopt: unrecognized option '--setgroup'
+    
+    --setuser and --setgroup are only needed for the ceph-osd process.
+    
+    Signed-off-by: James Page <james.page@ubuntu.com>
+    (cherry picked from commit 74977f78840fe95b67788f3aa135ac69a2353c43)
+
+commit 8a86d08e38a8b68ba5e04b2cc4f59a2983481ee7
+Author: Dan van der Ster <daniel.vanderster@cern.ch>
+Date:   Fri Apr 22 11:19:30 2016 +0200
+
+    osd: fix backwards min/max osd utilization
+    
+    Signed-off-by: Dan van der Ster <daniel.vanderster@cern.ch>
+    (cherry picked from commit 7529851f2a186501664461916f785da8456ad612)
diff --git a/doc/changelog/v10.2.10.txt b/doc/changelog/v10.2.10.txt
new file mode 100644
index 000000000..8493e7f4c
--- /dev/null
+++ b/doc/changelog/v10.2.10.txt
@@ -0,0 +1,4794 @@
+commit 5dc1e4c05cb68dbf62ae6fce3f0700e4654fdbbe
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Wed Oct 4 14:17:25 2017 +0000
+
+    10.2.10
+
+commit 951609bd3033f20560230d1d3514b535cffc0fde
+Merge: c4aa7c3df2 c6846df53c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 3 15:58:17 2017 -0400
+
+    Merge pull request #18100 from ceph/wip-yuriw-fix-clients-jewel
+    
+    tests: Added openstack.yaml bits to enable runs on ovh nodes
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit c6846df53c311ae5640a0bc28fbd752e38a745f3
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Tue Oct 3 12:50:24 2017 -0700
+
+    tests - Added openstack.yaml bits to enable runs on ovh nodes
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit c4aa7c3df2ead7d76aee9ed29d090c4433274af6
+Merge: 16110b985c add06614ab
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Oct 2 08:39:01 2017 -0700
+
+    Merge pull request #18044 from ceph/wip-yuriw-fix-clients-jewel
+    
+    qa: Changed distros symlink to point to supported OSs
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit add06614ab1b8f4a99feece1225a9c2e1cde1279
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Fri Sep 29 12:29:32 2017 -0700
+
+    Changed distros symlink to point to supported OSs
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 16110b985c4278b12f6884537a176cd85cfd0060
+Merge: 750e67cab8 06e8f38d21
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Sep 29 10:19:25 2017 -0700
+
+    Merge pull request #18027 from ceph/wip-yuriw-fix-clients-jewel
+    
+    tests - Added suit to test upgraded clients against jewel ceph clusters
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 06e8f38d216c049db4339bc291f7a495f2187d78
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Thu Sep 28 14:37:03 2017 -0700
+
+    tests - Added suit to test upgraded clients against jewel ceph clusters
+    
+    Replaces https://github.com/ceph/ceph/pull/17981
+    We need to run this suite using suite-branch option in
+    order to use jewel workloads agains ceph cluster luminous+ branches
+    
+    Added 'libcephfs1' to exclude_packages in upgrade_workload
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 750e67cab8fd0498ca6d843f25007904041d49cd
+Merge: 189f0c6f27 8398a8a033
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Sep 22 22:23:51 2017 +0200
+
+    Merge pull request #17892 from smithfarm/wip-p2p-s3-test
+    
+    jewel: tests: fix upgrade/jewel-x/point-to-point-x
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+    Reviewed-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 8398a8a0334718c68b9d9a04c820a875f7ec0056
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Sep 21 22:50:06 2017 +0200
+
+    qa: point-to-point-x: upgrade client.1 to -x along with cluster nodes
+    
+    The client.1 rgw in workload_x had not been upgraded to -x.
+    
+    Fixes: http://tracker.ceph.com/issues/21499
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit afc5a7d3208071b65c0d76d124bfc47a099a446c)
+
+commit d377266df600586d8889ca9ab4a6b7b110fcf0dc
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Sep 21 17:50:28 2017 +0200
+
+    tests: use special branch of ceph/s3-tests with pre-10.2.10
+    
+    Jewel v10.2.10 introduces a fix for S3 ACL code, for which a new test was added
+    to ceph/s3-tests.git (ceph-jewel branch). Since the jewel point-to-point-x
+    upgrade test runs s3-tests on 10.2.7, modify the test to use a special
+    ceph/s3-tests branch (ceph-jewel-10-2-7) that omits the new test.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 189f0c6f2703758a6be917a3c4086f6a26e42366
+Merge: 0ffa38ee4d de00e2bb5b
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Sep 19 18:03:29 2017 -0700
+
+    Merge pull request #17780 from smithfarm/wip-rh-74-jewel
+    
+    tests: CentOS 7.4 is now the latest
+    
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit 0ffa38ee4d074922737485cdd28d101449560107
+Merge: 94009032a8 0cd7df3649
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 19 14:22:52 2017 +0200
+
+    Merge pull request #16297 from smithfarm/wip-20518-jewel
+    
+    jewel: rbd: cli: map with cephx disabled results in error message
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 94009032a87026099fa2cf781e421508a061eb76
+Merge: d8aa3c502e d7d988d1f3
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 19 13:33:42 2017 +0200
+
+    Merge pull request #17781 from smithfarm/wip-drop-11429
+    
+    tests: Removed 11429 config
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit d7d988d1f3befc4443415931ee3f5c6fcb03b11a
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Wed Sep 21 09:14:09 2016 -0700
+
+    tests: Removed 11429 config
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+    (cherry picked from fde4d3bb96abc0c0ca5bca213a0a2423f95e91a2)
+    
+    Conflicts:
+        Cherry-picked manually because, nowadays, we have the entire ceph-qa-suite
+        under qa/ in ceph/ceph.git
+
+commit de00e2bb5b3c5f4026fd2d63c3799572655cd73a
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Sep 18 18:01:17 2017 +0200
+
+    tests: CentOS 7.4 is now the latest
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 2311b64025cdb6131035aaf01e7c97486da12e15)
+    
+    Conflicts
+        qa/distros/supported/centos_latest.yaml (renamed from centos_7.3.yaml)
+
+commit d8aa3c502e63462ab6a18eaed103eddfdf753855
+Merge: 2a525df77b f86dc69844
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Sep 18 11:05:28 2017 -0700
+
+    Merge pull request #16473 from smithfarm/wip-20723-jewel
+    
+    jewel: osd: rados ls on pool with no access returns no error
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit 2a525df77b91f5b9fdc62fa84677651e1b694bf4
+Merge: 955213635f 833c28e94f
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Sep 18 11:04:46 2017 -0700
+
+    Merge pull request #17626 from badone/wip-jewel-ceph-osd-flush-segfault
+    
+    jewel: core: kv: let ceph_logger destructed after db reset
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 955213635f1dd84b966a67e930e99a93d7880b51
+Merge: a536aa9878 c9b42c84e1
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Sep 12 15:50:42 2017 -0500
+
+    Merge pull request #17677 from mslovy/wip-disable-kstore-test
+    
+    jewel: qa: disable kstore to fix false positive case
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit a536aa98789f1c22e7d5aad241d1090ceb62f9dc
+Merge: 5c5085c7fc 0215989814
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 22:12:13 2017 +0200
+
+    Merge pull request #17597 from smithfarm/wip-21186-jewel
+    
+    jewel: rgw: rgw_file:  incorrect lane lock behavior in evict_block()
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 5c5085c7fc1309dc1678b520d88b6b0a9d246116
+Merge: 7429f50bb5 81810c1c7a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 22:11:33 2017 +0200
+
+    Merge pull request #17287 from smithfarm/wip-21109-jewel
+    
+    jewel: rgw: send data-log list infinitely
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 7429f50bb54c57356b98d103ae58aa85229c38c3
+Merge: adf2f8a5c9 9ab2e5fde0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 22:10:42 2017 +0200
+
+    Merge pull request #17285 from smithfarm/wip-20820-jewel
+    
+    jewel: rgw: Segmentation fault when exporting rgw bucket in nfs-ganesha
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit adf2f8a5c9bd9eb7ee30234e83893eea647753ff
+Merge: 694363a928 3c227e3f08
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 22:09:25 2017 +0200
+
+    Merge pull request #17281 from smithfarm/wip-20818-jewel
+    
+    jewel: rgw hangs in RGWRealmReloader::reload on SIGHUP
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 694363a928da9e0da3a284217a65c435fb44975d
+Merge: 9ac6c52bca cf65d63827
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 22:08:36 2017 +0200
+
+    Merge pull request #17280 from smithfarm/wip-20815-jewel
+    
+    jewel: rgw: uninitialized memory is accessed during creation of bucket's metadata
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 9ac6c52bca7624ac9ec39b6f233ee86efe76fc11
+Merge: 6180494692 442911ebad
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 22:07:48 2017 +0200
+
+    Merge pull request #17279 from smithfarm/wip-20813-jewel
+    
+    jewel: rgw: usage logging on tenated buckets causes invalid memory reads
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 618049469271eba8c819743688281e02e5009f08
+Merge: 4b37232d69 881bed7862
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 22:06:55 2017 +0200
+
+    Merge pull request #17277 from smithfarm/wip-20726-jewel
+    
+    jewel: rgw: user quota did not work well on multipart upload
+    
+    Reviewed-by: Daniel Gryniewicz <dang@redhat.com>
+
+commit 4b37232d69083d506ba5b7d3fc00b6e91a5cde31
+Merge: c1449b9a34 443e167b91
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 12 16:05:16 2017 -0400
+
+    Merge pull request #17649 from dillaman/wip-21346
+    
+    jewel: qa/workunits/rbd: relax greps to support upgrade formatting change
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit c1449b9a34687f873a6000f7787d047cfb3be447
+Merge: 9084ce8805 1516e24c85
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 22:04:20 2017 +0200
+
+    Merge pull request #17166 from smithfarm/wip-20719-jewel
+    
+    jewel: rgw: Truncated objects
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 9084ce88053d5b256c175e0712f4a619002de742
+Merge: 89b00a8e16 c47e5abd0f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 22:03:30 2017 +0200
+
+    Merge pull request #17165 from smithfarm/wip-20715-jewel
+    
+    jewel: rgw: radosgw-admin data sync run crash
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 89b00a8e1699cebc016286a528173f04faa01573
+Merge: 568cc0c25b 9b5b21def2
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 22:02:29 2017 +0200
+
+    Merge pull request #17164 from smithfarm/wip-20712-jewel
+    
+    jewel: rgw: not initialized pointer cause rgw crash with ec data pool
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 568cc0c25b1d5e047338e49aa7085c45ab0fbf3e
+Merge: 5b1e1cfae8 73c9d33ac0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 22:01:38 2017 +0200
+
+    Merge pull request #17159 from smithfarm/wip-20709-jewel
+    
+    jewel: rgw: radosgw-admin: bucket rm with --bypass-gc and without --purge-data doesn't throw error message
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 5b1e1cfae89d31bf33b47ba79fb9b75d451e34cd
+Merge: fbae1127a6 e76198751c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 22:00:54 2017 +0200
+
+    Merge pull request #17156 from smithfarm/wip-20673-jewel
+    
+    jewel: rgw: multisite: RGWRadosRemoveOmapKeysCR::request_complete return val is wrong
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit fbae1127a6e7a9f4dd0822a91c8a06e5b95e3c52
+Merge: f64a82ad4d 26541c07b4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 21:58:54 2017 +0200
+
+    Merge pull request #17148 from smithfarm/wip-20292-jewel
+    
+    jewel: rgw: multisite: log_meta on secondary zone causes continuous loop of metadata sync
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit f64a82ad4d931978a8cf53ca81190434da8017f6
+Merge: bd918be29c 38718ace48
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 21:56:44 2017 +0200
+
+    Merge pull request #17147 from smithfarm/wip-20290-jewel
+    
+    jewel: rgw: rgw_file: prevent conflict of mkdir between restarts
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit bd918be29cd31f0ef6dac64edbfaf40f38ae2e07
+Merge: 37dec0227a ff67388e24
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 21:55:14 2017 +0200
+
+    Merge pull request #16856 from prallabh/wip-20895
+    
+    jewel: rgw: bucket index check in radosgw-admin removes valid index
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 37dec0227a9522e730baabb9c7428f007ba9c202
+Merge: f4734fada4 5a7ea2790c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 21:53:38 2017 +0200
+
+    Merge pull request #16767 from linuxbox2/jewel-rgw-mpart-race
+    
+    jewel: rgw : fix race in RGWCompleteMultipart
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit c9b42c84e1f6850adea1cb9ba52f4a2a39fe8619
+Author: Ning Yao <yaoning@unitedstack.com>
+Date:   Tue Sep 12 18:16:29 2017 +0000
+
+    qa: disable kstore to fix false positive case
+    
+    Signed-off-by: Ning Yao <yaoning@unitedstack.com>
+
+commit f4734fada49ee6c2d4db9cd3e1e07d6bffe54071
+Merge: d3388d4601 3e70f3d662
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 18:18:40 2017 +0200
+
+    Merge pull request #17574 from smithfarm/wip-21297-jewel
+    
+    jewel: libradosstriper processes arbitrary printf placeholders in user input
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit c47e5abd0f6583bb6cf2a676e6b1194d29a96780
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Tue Jun 27 10:53:30 2017 +0800
+
+    rgw: fix radosgw-admin data sync run crash
+    
+    If sync thread have run before, then run data sync init. sync_status is still remain in rados pool. so no matter sync_status exists or not, if state is StateInit, sync_status.sync_info.num_shards should be updated.
+    
+    Fixes: http://tracker.ceph.com/issues/20423
+    
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit c307910d7131fc290f00bb8e33876e667afb72ec)
+    
+    Conflicts:
+        src/rgw/rgw_data_sync.cc (no data_sync_module or instance_id in jewel)
+
+commit d3388d4601d12c7c1e1187251c7221bbd8b58fbb
+Merge: 0277feb7d7 0a3227876c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 09:36:50 2017 +0200
+
+    Merge pull request #16299 from smithfarm/wip-20262-jewel
+    
+    jewel: rgw: datalog trim can't work as expected
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 0277feb7d74a91ad7e32a7b42e3bf98d5072e72e
+Merge: a683b04589 c90969031f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 09:34:36 2017 +0200
+
+    Merge pull request #17167 from smithfarm/wip-20720-jewel
+    
+    jewel: multisite: RGWPeriodPuller does not call RGWPeriod::reflect() on new period
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit f86dc698443bbaa6e2ad263875a17d0bccdff336
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Sep 12 09:30:37 2017 +0200
+
+    tests: use XFS explicitly in singleton-nomsgr/pool-access.yaml
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit a7e1ee22b03abc40188f3745ad86534df78d757e
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed May 24 16:46:00 2017 -0400
+
+    qa/suites/rados/singleton-nomsgr: fix syntax
+    
+    This parsed out as
+    
+      tasks:
+      - install: null
+      - ceph:
+          conf:
+            osd: osd max object name len = 400 osd max object namespace len = 64
+      - workunit:
+          clients:
+            all:
+            - rados/test_health_warnings.sh
+    
+    which is clearly not correct.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from 85e2f3f31d25dbfd9770cc3b1f232025df53b628)
+    
+    Conflicts:
+        applied changes to pool-access.yaml instead of health-warnings.yaml
+
+commit a683b04589763681553078bc31e7a838614bb910
+Merge: 12fa3ec34e 7966b846a8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 09:25:22 2017 +0200
+
+    Merge pull request #15556 from cbodley/wip-19847
+    
+    jewel: rgw: multisite: fixes for meta sync across periods
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 12fa3ec34e61e43525a013920ff89757a48bb0f2
+Merge: 151c9da33d 71262f1fae
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 09:24:34 2017 +0200
+
+    Merge pull request #17155 from smithfarm/wip-20641-jewel
+    
+    jewel: rgw: multisite: lock is not released when RGWMetaSyncShardCR::full_sync() fails to write marker
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 151c9da33db8c59e5a9b5d7a0a97dd517322ceb1
+Merge: c049777c74 8b52105701
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Sep 12 09:23:43 2017 +0200
+
+    Merge pull request #17278 from smithfarm/wip-20728-jewel
+    
+    jewel: rgw: multipart parts on versioned bucket create versioned bucket index entries
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 26541c07b4d107fa7b89e2ea7da821d1aacf22ca
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Jun 22 13:06:13 2017 +0300
+
+    rgw: we no longer use log_meta
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit ac8b0077c1f53ff0037c4163489b838eebf8c247)
+
+commit 63df57807486a3b2fd6c4ec154f2d70fa9e88948
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Jun 22 10:17:37 2017 +0300
+
+    rgw: is_single_zonegroup doesn't use store or cct
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 0b0090cee249c5380efa4bb416d724a7e2dbfc08)
+
+commit 3eb913e488c914d86eca2a4d31b050ea18914fa2
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed Jun 21 12:37:21 2017 +0300
+
+    rgw: log_meta only for more than one zone
+    
+    Fixes: http://tracker.ceph.com/issues/20357
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 088dba03ccc65609ef1c51306389ebcf1126ec8c)
+
+commit 381d85d14dc48bcea83fd10010ea1080af0e5c55
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Sat Jun 10 18:12:52 2017 -0400
+
+    rgw: only log metadata on metadata master zone
+    
+    Fixes: http://tracker.ceph.com/issues/20244
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit b8272f3607074a2f7cbfd08f7bbc82f22cf120ba)
+
+commit c049777c74710bf342379e8995b31ee02282b61d
+Merge: 9afc62a7cc 787ba33e5d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Sep 11 23:05:04 2017 +0200
+
+    Merge pull request #16296 from smithfarm/wip-20267-jewel
+    
+    jewel: rbd: api: is_exclusive_lock_owner shouldn't return -EBUSY
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 9afc62a7cc124dc92872779b18356aec85736f61
+Merge: dd0ead4815 475dda114a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Sep 11 23:04:09 2017 +0200
+
+    Merge pull request #17385 from dillaman/wip-19957
+    
+    jewel: librbd: reacquire lock should update lock owner client id
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit dd0ead48154bbe8c0775ab2d35dec00f22c15990
+Merge: 18085bd997 c755a0fccd
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Sep 11 23:03:29 2017 +0200
+
+    Merge pull request #17402 from dillaman/wip-20515
+    
+    jewel: librbd: fail IO request when exclusive lock cannot be obtained
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 18085bd9973fb2f0d0eb2b2e2209e9e193270536
+Merge: ac27f23fa6 f9acf56dce
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Sep 11 23:02:49 2017 +0200
+
+    Merge pull request #17412 from dillaman/wip-18704
+    
+    jewel: librbd: prevent self-blacklisting during break lock
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 443e167b91cec0d5c7ed51ce686d6bb3ccd8d036
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Sep 7 08:55:27 2017 -0400
+
+    qa/workunits/rbd: relax greps to support upgrade formatting change
+    
+    Fixes: http://tracker.ceph.com/issues/21181
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 273c84578b06905229a7c6eae7b1a12bbe60de95)
+    
+    Conflicts:
+            qa/workunits/rbd/import_export.sh: trivial resolution
+
+commit a0755a760a2d5017604564389240a9c975408b30
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon May 22 12:53:51 2017 -0400
+
+    qa/suites/rados/singleton-nomsgr/pool-access: behave on ext4
+    
+    We may land on an ext4 root partition.
+    
+    Fixes: http://tracker.ceph.com/issues/20043
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from 657453d34914832f6e8012fbd69200e9680bd9ff)
+    
+    Conflicts:
+        applied the changes to pool-access.yaml instead of health-warnings.yaml
+        to address a specific test failure in the jewel branch
+
+commit a2fd5279992770f111710edd55e323afd5803445
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Sep 11 12:13:35 2017 +0800
+
+    tasks/ceph: construct CephManager earlier
+    
+    Previously, if errors occurred during healthy(), then
+    the finally block would invoke osd_scrub_pgs, which relies
+    on CephManager being constructed, and it would die, hiding
+    the original exception.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit c444db12d455a1901da8041e92f5eff1a9875170)
+    
+    Conflicts:
+            qa/tasks/ceph.py: the `tasks` directory was moved into
+                    `qa` after the cherry-picked change was merged.
+                    so apply the change manually to the ceph.py
+                    under `qa` directory.
+    (cherry picked from commit bc71dabb4e311bd2461489e98e9b95d5b635effc)
+
+commit 833c28e94f9a71ddfdc8afbdfacc9627a1e476d6
+Author: wumingqiao <wumingqiao@inspur.com>
+Date:   Tue Jul 25 14:45:09 2017 +0800
+
+    kv: let ceph_logger destructed after db reset
+    
+    if ceph_logger is deleted earlier than db, it may still be used by db, which cause a segment fault.
+    
+    Signed-off-by: wumingqiao <wumingqiao@inspur.com>
+    (cherry picked from commit a5cd03c643d6cb9074dfd2952cde83435de1b9dd)
+
+commit ac27f23fa697c7ad9be897efc04af95e3908b5b3
+Merge: bcc85d178e 72379a4930
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sun Sep 10 21:38:15 2017 +0200
+
+    Merge pull request #15189 from mslovy/wip-19996-jewel
+    
+    jewel: osd: fix infinite loops in fiemap
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit bcc85d178e5900acbe1eaaf28de43ca53dadbb7e
+Merge: 2203d42ec6 314952c589
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sun Sep 10 21:15:35 2017 +0200
+
+    Merge pull request #16711 from cbodley/wip-20346
+    
+    jewel: rgw: meta sync thread crash at RGWMetaSyncShardCR
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 2203d42ec656b12c23de50874b36289354338aaa
+Merge: f212b361df 2d64d090c0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Sep 8 14:05:18 2017 -0400
+
+    Merge pull request #17552 from dillaman/wip-21279
+    
+    jewel: test/cls_rbd: mask newer feature bits to support upgrade tests
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 02159898147053fa492ec597bceb9096d160be08
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Aug 25 17:55:11 2017 -0400
+
+    rgw_file:  fix LRU lane lock in evict_block()
+    
+    Found by "Supriti Singh" <Supriti.Singh@suse.com>.
+    
+    Fixes http://tracker.ceph.com/issues/21141
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 3301596734adcbda1e8e76a742935efdeb7518fd)
+
+commit f212b361dfaed7f67e4b994b1f22b3c9897eeab2
+Merge: 26db2e7ff9 5adfc653dc
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 7 18:54:17 2017 +0200
+
+    Merge pull request #17514 from smithfarm/wip-21232-jewel
+    
+    jewel: core: interval_set: optimize intersect_of insert operations
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 26db2e7ff93eb351e0963e8ae25548367e2f63ff
+Merge: f4a0d00032 1bd4df1d7f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 7 18:39:58 2017 +0200
+
+    Merge pull request #16293 from smithfarm/wip-20460-jewel
+    
+    jewel: core: test_envlibrados_for_rocksdb.yaml fails on crypto restart
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit c755a0fccd007f95d709f124e931ab9758a6c93d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 5 16:20:26 2017 -0400
+
+    qa/suites/rbd: fixed cache override
+    
+    Fixes: http://tracker.ceph.com/issues/21251
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 98061bb3d7ce6309ddb04ea4d7e9d44a7ecd09c6)
+
+commit 3e70f3d662e3c051352e342fe89cd3b2186f4cb5
+Author: Jesse Williamson <jwilliamson@suse.de>
+Date:   Tue Jun 13 12:31:16 2017 -0700
+
+    libradosstriper: remove format injection vulnerability
+    
+    Fixes: http://tracker.ceph.com/issues/20240
+    
+    Signed-off-by: Stan K <redrampage@selectel.ru>
+    (cherry picked from commit e94d3b9661d287e500cdf4e6c102d2e3bb04475e)
+
+commit f4a0d000322992b9c67f51830a8396269a5caa1e
+Merge: 00d09fce6a 1415ecbd7d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 7 17:41:58 2017 +0200
+
+    Merge pull request #17084 from badone/wip-ceph-disk-dmcrypt-key-jewel
+    
+    jewel: ceph-disk: Use stdin for 'config-key put' command
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 00d09fce6a81bf7c8cd62caa5e0a3de44bd0ad84
+Merge: b014f3995d cd848c1e98
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 7 17:40:34 2017 +0200
+
+    Merge pull request #17008 from asheplyakov/19182-bp-jewel
+    
+    jewel: mon: fix force_pg_create pg stuck in creating bug
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit a4c3ef5276eb7d232f5706e6fe07b0808ee581d4
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Mon May 29 19:25:43 2017 +1000
+
+    osd: Reverse order of op_has_sufficient_caps and do_pg_op
+    
+    Fixes: http://tracker.ceph.com/issues/19790
+    
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit a921882e7c676d68ed67b54301b49fd9bed42945)
+    
+    Conflicts:
+            qa/suites/rados/singleton-nomsgr/all/pool-access.yaml - drop mgr.x
+                role, which is not needed in jewel and might even cause the test to
+                fail
+            src/osd/PrimaryLogPG.cc - this file doesn't exist in jewel, so apply
+                the change manually to ReplicatedPG.cc (i.e., by moving the
+                op->includes_pg_op() conditional below the
+                !op_has_sufficient_caps(op) conditional)
+
+commit b014f3995d7913fe5266c2e7d06f93f296e98991
+Merge: c3623149ef 53e97da166
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Thu Sep 7 22:51:10 2017 +0800
+
+    Merge pull request #17133 from smithfarm/wip-21035-jewel
+    
+    jewel: ceph-disk: systemd unit timesout too quickly
+    
+    Reviewed-by: David Disseldorp <ddiss@suse.de>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit c3623149efc2b25a3509ec26b671d835a81b8c28
+Merge: 3aede767bd 713a88a5c9
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 7 16:33:08 2017 +0200
+
+    Merge pull request #16316 from smithfarm/wip-20362-jewel
+    
+    jewel: rgw: VersionIdMarker and NextVersionIdMarker are not returned when listing object versions
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 72379a4930edfc917a15ea1ce5ba67ec5093fd96
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 15 17:29:45 2016 -0400
+
+    ceph_test_objectstore: disable filestore_fiemap
+    
+    This very reliably triggers a test failure for
+    ObjectStore/StoreTest.Synthetic/1.
+    
+    FIEMAP is bad!  Do not use it!
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 2d64d090c05be7792c934d8a3e060e48e68da129
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 5 21:37:16 2017 -0400
+
+    test/cls_rbd: mask newer feature bits to support upgrade tests
+    
+    Fixes: http://tracker.ceph.com/issues/21217
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit aeb8f29d21625f3570e2f3e6dd26594d1a6df810)
+
+commit 3aede767bdc391fe6521066aade80369081d23cc
+Merge: b2accb83ac e35c938208
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 7 10:21:10 2017 +0200
+
+    Merge pull request #16703 from ddiss/wip-20837-jewel
+    
+    jewel: ceph-disk: don't activate suppressed journal devices
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit b2accb83ac46a54669c4d0a603464ff117a66f06
+Merge: d7cc49d995 96de9d9078
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 7 09:47:41 2017 +0200
+
+    Merge pull request #17396 from smithfarm/wip-21113-jewel
+    
+    jewel: cephfs: get_quota_root sends lookupname op for every buffered write
+    
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+
+commit d7cc49d9950917e7afb6c14b4b063b51e7e136b5
+Merge: ae480caf68 3adbc35205
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 7 09:46:00 2017 +0200
+
+    Merge pull request #17009 from dreamhost/wip-16463-jewel
+    
+    jewel: rgw: aws4: add rgw_s3_auth_aws4_force_boto2_compat conf option
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit ae480caf68db8e6793c8f3db663e94215c9f75ea
+Merge: d4797bf193 3242b2b562
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 7 09:42:54 2017 +0200
+
+    Merge pull request #16963 from smithfarm/wip-20972-jewel
+    
+    jewel: cephfs: ceph-fuse segfaults at mount time, assert in ceph::log::Log::stop
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit d4797bf193354199a3182cc2c07ac46c76d506ba
+Merge: 02fef5c436 788cfa89dd
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 7 09:41:19 2017 +0200
+
+    Merge pull request #16952 from dreamhost/wip-20966-jewel
+    
+    jewel: rgw: radosgw-admin: fix bucket limit check argparse, div(0)
+    
+    Reviewed-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 02fef5c43687437a4b085b9f31d2e3eb982e637c
+Merge: d5e77009ac 6bd2dae193
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 7 09:34:52 2017 +0200
+
+    Merge pull request #16951 from linuxbox2/jewel-aws4-plus
+    
+    jewel: rgw: replace '+' with "%20" in canonical query string for s3 v4 auth
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit d5e77009acf009f1280d352ef6d0f8b6731fd3e8
+Merge: 7bdbd075a1 37b37e78f8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 7 09:33:25 2017 +0200
+
+    Merge pull request #16880 from linuxbox2/jewel-rgw-dir-suggest
+    
+    jewel: rgw: Fix up to 1000 entries at a time in check_bad_index_multipart
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 7bdbd075a11b9a8cd05eb0fd55d4787c554ac9a6
+Merge: 18e5db07f1 187e9eaa30
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Sep 7 09:28:03 2017 +0200
+
+    Merge pull request #16720 from prallabh/wip-jewel-20821
+    
+    jewel: rgw: Do not decrement stats cache when the cache values are zero
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 5adfc653dc201ec4faa60a5f78431ab68f0f4e19
+Author: Zac Medico <zmedico@gmail.com>
+Date:   Sun Aug 27 05:25:01 2017 -0700
+
+    interval_set: optimize intersect_of for identical spans
+    
+    Optimize comparisons for identical spans of intervals.
+    When this patch is combined with the previous map insert
+    optimization, a benchmark using 400000 identical
+    intervals shows a 7 times performance improvement in
+    comparison to without the patches.
+    
+    Signed-off-by: Zac Medico <zmedico@gmail.com>
+    (cherry picked from commit b6a035666c2765f8895ee9991348dbc025613ed7)
+
+commit 54dfe8733b32401a5a561bf544d91f44a41e378f
+Author: Zac Medico <zmedico@gmail.com>
+Date:   Fri Aug 25 09:41:07 2017 -0700
+
+    interval_set: optimize intersect_of insert operations
+    
+    Use the std::map insert method with hint iterator to optimize
+    inserts. This increases performance more than 3.5 times for
+    large numbers of intervals. This will help performance
+    especially in the PGPool::update method, where profiling data
+    has shown that intersection operations are a hot spot. The
+    following benchmark data is for 400000 intervals:
+    
+        4 +-+--+----+----+----+----+----+----+----+----+--+-+
+    P     +    +    +    +    +    +    +    +  *************
+    E     |                             ********            |
+    R 3.5 +-+                       ****                  +-+
+    F     |                   ******                        |
+    O     |                 **                              |
+    R   3 +-+           ****                              +-+
+    M     |          ***                                    |
+    A     |        **                                       |
+    N 2.5 +-+     *                                       +-+
+    C     |     **                                          |
+    E     |     *                                           |
+        2 +-+ **                                          +-+
+    R     |  **                                             |
+    A     | **                                              |
+    T 1.5 +**                                             +-+
+    I     |**                                               |
+    O     +*   +    +    +    +    +    +    +    +    +    +
+        1 +*+--+----+----+----+----+----+----+----+----+--+-+
+          0   0.1  0.2  0.3  0.4  0.5  0.6  0.7  0.8  0.9
+    
+                            SET SIZE RATIO
+    
+    The above chart was generated using benchmark results
+    from the following program:
+    
+    #include <iostream>
+    #include <sys/timeb.h>
+    #include "include/interval_set.h"
+    
+    int main(int argc, char *argv[])
+    {
+      const int interval_count = std::stoi(argv[1]);
+      const int interval_distance = 4;
+      const int interval_size = 2;
+      const int sample_count = 8;
+      const int max_offset = interval_count * interval_distance;
+      interval_set<int> a, b, intersection;
+    
+      for (int i = 0; i < max_offset; i+=interval_distance) {
+        a.insert(i, interval_size);
+      }
+    
+      for (int m = 1; m < 100; m++) {
+        float ratio = 1 / float(m);
+    
+        for (int i = 0; i < max_offset; i+=interval_distance*m) {
+          b.insert(i, interval_size);
+        }
+    
+        struct timeb start, end;
+        int ms = 0;
+        for (int i = 0; i < sample_count; i++) {
+          ftime(&start);
+          intersection.intersection_of(a, b);
+          ftime(&end);
+          ms += (int) (1000.0 * (end.time - start.time)
+            + (end.millitm - start.millitm));
+          intersection.clear();
+        }
+        b.clear();
+    
+        std::cout << ratio << "\t" << ms << std::endl << std::flush;
+      }
+    }
+    
+    Signed-off-by: Zac Medico <zmedico@gmail.com>
+    (cherry picked from commit 32bc0430f70b057d1bba623252e92ab9f279028d)
+
+commit 18e5db07f1e81eec8a2c8483c7bcc876be0d19c9
+Merge: 44cd2b19b5 178245d543
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Sep 6 09:36:13 2017 +0200
+
+    Merge pull request #15966 from prallabh/jewel
+    
+    jewel: rgw: Custom data header support
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 44cd2b19b5ee603a96b53e8286cfc0cbd6e1fc76
+Merge: c93d2c6cd6 d90b1b3395
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Sep 6 09:03:50 2017 +0200
+
+    Merge pull request #16144 from smithfarm/wip-20364-jewel
+    
+    jewel: mon: osd crush set crushmap need sanity check
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 1bd4df1d7f16b6cca02abb791acc298a1e6ec592
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Jun 24 00:50:40 2017 +0800
+
+    osdc/Objecter: release message if it is not handled
+    
+    Fixes: http://tracker.ceph.com/issues/19741
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5c2774234c20d809d56d005b4db34070f3029684)
+    
+    Conflicts:
+       src/osdc/Objecter.h (in master, ms_fast_dispatch function declaration has an
+                           "override" specifier; in jewel, this specifier is missing)
+
+commit 4e4b62b4c9ec8dc5eea025af31ae5761fdd127dd
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Jun 21 14:25:01 2017 +0800
+
+    crypto: allow PK11 module to load even if it's already initialized
+    
+    there is chance that other pieces of application loads PK11 module
+    already and does not finalize it before calling common_init_finish().
+    
+    also, upon fork, PK11 module resets its entire status including `nsc_init`,
+    by which PK11 module tell if it is initialized or not. so the behavior
+    of NSS_InitContext() could be different before and after fork. that's
+    another reason to ignore CKR_CRYPTOKI_ALREADY_INITIALIZED error (see
+    NSS_GetError()).
+    
+    Fixes: http://tracker.ceph.com/issues/19741
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit fcc3effd8b447ef0c54b4c806b8f6e996d7467dd)
+
+commit c93d2c6cd65273eef4e9187269e5d44622af9a1c
+Merge: e9eda5aa28 92eb6836bf
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Tue Sep 5 11:13:53 2017 -0700
+
+    Merge pull request #17210 from dzafman/wip-20730-jewel
+    
+    jewel: core: disable skewed utilization warning by default
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit e9eda5aa285ebf989c598b248c2c8d4fc660c6fa
+Merge: e4f02b2bef 153f775441
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Tue Sep 5 11:12:23 2017 -0700
+
+    Merge pull request #16405 from dzafman/wip-20041-jewel
+    
+    jewel: osd: scrub_to specifies clone ver, but transaction include head write ver
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit e4f02b2befa9203a51c7f146c85539180953e451
+Merge: baa5183c52 97fdaa753e
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Tue Sep 5 11:11:21 2017 -0700
+
+    Merge pull request #16169 from Vicente-Cheng/wip-20511-jewel
+    
+    jewel: osd: ReplicatedPG: solve cache tier osd high memory consumption
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit baa5183c524fc7af83dd46645a48e24464ab7c6e
+Merge: 81097b6fa1 f49c9c777f
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Tue Sep 5 10:55:32 2017 -0700
+
+    Merge pull request #16167 from Vicente-Cheng/wip-20492-jewel
+    
+    jewel: osd: omap threadpool heartbeat is only reset every 100 values
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 81097b6fa1543e332f0da7cd9857cb4877c8eee1
+Merge: d639aa9136 932342c1e3
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Sep 4 21:10:57 2017 +0200
+
+    Merge pull request #16151 from Vicente-Cheng/wip-20349-jewel
+    
+    jewel: cephfs: df reports negative disk "used" value when quota exceed
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit d639aa91369c6af4956a5186dd11bfb903f01421
+Merge: b6b08b72f0 6b6620ffef
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Sep 4 21:09:55 2017 +0200
+
+    Merge pull request #16150 from Vicente-Cheng/wip-20403-jewel
+    
+    jewel: mds: save  projected path into inode_t::stray_prior_path
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit b6b08b72f0516b74f502bcc98ad98d01b2012f7d
+Merge: bab324815d dac0ca797c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Sep 4 12:54:34 2017 +0200
+
+    Merge pull request #16141 from smithfarm/wip-20270-jewel
+    
+    jewel: tests: LibRadosMiscConnectFailure.ConnectFailure hang
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit bab324815da63ec395a4c036d451c7d3bb45fa99
+Merge: ecc5e960a0 3105f118bc
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Sep 2 17:16:57 2017 +0200
+
+    Merge pull request #16285 from smithfarm/wip-18137-jewel
+    
+    jewel: rbd-mirror: image sync should send NOCACHE advise flag
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit ecc5e960a0fdbcf9c0de7453626369938154d29c
+Merge: 0d53a36ea1 f9e87fef6a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Sep 2 17:16:08 2017 +0200
+
+    Merge pull request #16295 from smithfarm/wip-20265-jewel
+    
+    jewel: rbd: cli: ensure positional arguments exist before casting
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 0d53a36ea1b547f85172b79b84a61229af69d658
+Merge: 445833be4e 24bd60136e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Sep 2 13:12:17 2017 +0200
+
+    Merge pull request #16276 from smithfarm/wip-19563-jewel
+    
+    jewel: rgw: folders starting with _ underscore are not in bucket index
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 445833be4e941e5eea106d558f02afa6dc201db2
+Merge: 182a578157 39b2b0b39a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Sep 2 13:09:46 2017 +0200
+
+    Merge pull request #16268 from cbodley/wip-16072-jewel
+    
+    jewel: rgw: rgw_common.cc: modify the end check in RGWHTTPArgs::sys_get
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 182a5781575183fd74f6262b3d0f5137c356ec0c
+Merge: f61c562894 f16fa4b252
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Sep 2 13:08:21 2017 +0200
+
+    Merge pull request #16266 from cbodley/wip-20408
+    
+    jewel: rgw: multipart copy-part remove '/' for s3 java sdk request header
+    
+    Reviewed-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit f61c5628944f9b2b612f080d73296678a4a7da14
+Merge: 5d3c80eef4 00a7999552
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Sep 2 12:50:50 2017 +0200
+
+    Merge pull request #15988 from cbodley/wip-jewel-15983
+    
+    jewel: rgw: fix infinite loop in rest api for log list
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 5d3c80eef428b93edc7559709d4a2de7bd2750c1
+Merge: 6484a3bdab d3b34d3f9f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Sep 2 12:48:00 2017 +0200
+
+    Merge pull request #15449 from smithfarm/wip-19720-jewel
+    
+    jewel: rgw: rgw_file: fix size and (c|m)time unix attrs in write_finish
+    
+    Reviewed-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 6484a3bdab941b077ae942ae207b11ac679aeedc
+Merge: bd2a0c80ab d0492ea07a
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 1 14:27:31 2017 -0500
+
+    Merge pull request #17351 from liewegas/wip-prune-past-intervals-jewel
+    
+    osd: add osd_hack_prune_past_intervals
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit f9acf56dce26bb031cddb6676b2834b6bd426ae2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 31 21:30:47 2017 -0400
+
+    librbd: prevent self-blacklisting during break lock
+    
+    (derived from commit 5c590acaec4dd66a9a8c3aa0ec8ab904dd350216)
+    
+    Fixes: http://tracker.ceph.com/issues/18704
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit d90b1b339542408fb053571b64e705be208cb3be
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sat Mar 18 10:04:20 2017 +0100
+
+    mon: osd crush set crushmap need sanity check
+    
+    The sanity check verifying the new crushmap does not remove crush rules
+    that are in use is not exclusive to ceph setcrushmap.
+    
+    Fixes: http://tracker.ceph.com/issues/19302
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit ed760457bf154c10adf75c6df046eecab7eb8e4b)
+
+commit 2e7ddedf8acdd7a19753a0c376e9f29535329843
+Author: songbaisen <song.baisen@zte.com.cn>
+Date:   Thu Mar 31 11:39:44 2016 +0800
+
+     crush: when take place the crush map should consider the rule is in used
+    
+     Signed-off-by: song baisen song.baisen@zte.com.cn
+    
+    (cherry picked from commit 09bf6f2858d09c8c314a8c242a9c70d2834718dc)
+
+commit 26984b0b0506958da340b94e3a05eac348f20f2a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 31 08:03:03 2017 -0400
+
+    librbd: fix missing write block validation in IO work queue
+    
+    (derived from commit 8e76ebd93fd7525f6909a7da178e9b92c98e1ebe)
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 79f220d08063077b08cda3acc447f30790ee9dfd
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 27 14:05:58 2017 -0400
+
+    qa/suites/rbd: test dynamic features with cache disabled
+    
+    The journaling logic follows different paths depending on the
+    cache configuration.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 57a507d845339e9052122755bd460eb6d61d6060)
+
+commit 412f75a8f8203f236c1eb9f52da8876e476a052f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 27 14:03:34 2017 -0400
+
+    qa/tasks/qemu: rbd cache is enabled by default
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit fdc4c9221f2c54d5fb55e74491ebc5165f73e776)
+
+commit edf8b584f0856f6181579aa1b531963ad06ebbea
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 31 09:48:47 2017 -0400
+
+    test: unit tests for librbd IO work queue failure path
+    
+    (derived from commit 6e23ef358f29ca65a2a6b08b133416415e117b06)
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 1c8bd5f44a0a006094235254504772ae8eb90547
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 31 09:31:02 2017 -0400
+
+    librbd: cleanup interface between IO work queue and IO requests
+    
+    This will help mock the testing of the IO work queue
+    
+    (derived from commit e0834d12e91bf085a5c9503f1c69a2bee2d21edf)
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 75f0bb9e7bbdea0fa6d15d253139f0583be6abea
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 22 13:31:10 2017 -0400
+
+    common: improve the ability to mock PointerWQ classes
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2d90bd8fdb30705d05569d242fc0cc4e36a2c7db)
+    
+    Conflicts:
+            src/common/WorkQueue.h: trivial resolution
+            src/librbd/AioImageRequestWQ.cc: file renamed
+
+commit fbfafb75c70a1455e102bf09623e1f1d8cf2074f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 31 07:51:37 2017 -0400
+
+    librbd: exclusive lock failures should bubble up to IO
+    
+    (derived from commit 048d475127b600b6a40bd5e0c3a0daf8133294f4)
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 8fb8eb61dd8b2e44cb10da9b28fa334267784e2f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Aug 30 22:08:15 2017 -0400
+
+    librbd: directly inform IO work queue when locks are required
+    
+    Due to lock dependency issues between librbd locks and the the thread
+    pool lock, it's impossible to directly determine if the lock is
+    required within the _void_dequeue method. Therefore, this state needs
+    to be internally tracked by the work queue.
+    
+    (derived from commit 4a525671b3541a0a208dd039ac96f42bc8fca2cc)
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 4509765fee20aab082dfbb2ba02cf76f796d6292
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Aug 30 21:16:56 2017 -0400
+
+    librbd: clean up variable naming in IO work queue
+    
+    (derived from commit 9d8ae624858beb6fdc8ee2674b8e8d9b9cf14cbe)
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 08c63b3d0c8a3c68f68ded0b9122c5e094d362ef
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 31 08:25:29 2017 -0400
+
+    librbd: convert ImageRequestWQ to template
+    
+    (derived from commit b9569585fab5654b6c29552c5eb009dcddfaaa3b)
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 42780a8228da1440d7f3f44a09cddf3b88b90223
+Author: Ning Yao <yaoning@unitedstack.com>
+Date:   Mon Aug 28 15:48:50 2017 +0000
+
+    os:kstore fix unittest for FiemapHole
+    
+    kstore always return [0, object_size] regardless of offset and length
+    
+    Signed-off-by: Ning Yao <yaoning@unitedstack.com>
+    (cherry picked from commit dddae89fa7956fdfca1c09910b4e5b42ecbd22cf)
+
+commit 96de9d9078e5f638c74fdc68322ac4a199011597
+Author: Dan van der Ster <daniel.vanderster@cern.ch>
+Date:   Thu Aug 10 11:33:46 2017 +0200
+
+    client: skip lookupname if writing to unlinked file
+    
+    When a client writes to an unlinked file, get_quota_root tries to
+    resolve the filename via an MDS lookupname op. The op always results in
+    -13 permission denied for path-restricted caps or -2 no such file or
+    directory otherwise. More importantly, the repeated lookupname ops slow
+    down buffered writes significantly.
+    
+    Don't do the lookupname for unlinked files; use the root_ancentor's
+    quota instead.
+    
+    Fixes: http://tracker.ceph.com/issues/20945
+    Backport: jewel, luminous
+    Signed-off-by: Dan van der Ster <daniel.vanderster@cern.ch>
+    (cherry picked from commit 1d617eee2e8c32f7300cb2cb8dd37b3714a4699f)
+
+commit bd2a0c80ab1827f1ecdb349b6ebc65c4ad519322
+Merge: 05df53404b e9e7850b51
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Aug 31 10:24:21 2017 +0200
+
+    Merge pull request #16059 from vumrao/wip-vumrao-20353
+    
+    jewel: osd: Add heartbeat message for Jumbo Frames (MTU 9000)
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 05df53404b0e05cefe46d30174df71b53714f7f2
+Merge: 1b3c3c7f65 0068d9f333
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Aug 31 10:23:00 2017 +0200
+
+    Merge pull request #15726 from asheplyakov/20314-bp-jewel
+    
+    jewel: mon: fail to form large quorum; msg/async busy loop
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Haomai Wang <haomai@xsky.com>
+
+commit 475dda114a7e25b43dc9066b9808a64fc0c6dc89
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Aug 30 20:59:20 2017 -0400
+
+    librbd: reacquire lock should update lock owner client id
+    
+    (derived from commit 21ce5e16703776cbae20ac981dff4ec7cc2bc9c8)
+    
+    Fixes: http://tracker.ceph.com/issues/19957
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit d0492ea07abcc8652f9c713deaae792ef68dc491
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Nov 24 18:08:49 2016 -0500
+
+    osd: add osd_hack_prune_past_intervals
+    
+    Last ditch (but dangerous) method of reducing memory usage for
+    past_intervals, which can help very very unhappy clusters recovery.
+    A properly implemented version of this is in luminous.  This hacky
+    version was used successfully to recover multiple jewel-based clusters,
+    but is still only recommended for use when the OSD is otherwise unable
+    to recover.
+    
+    This change is not cherry-picked from master because luminous implements
+    a more sophisticated version of this that changes the past intervals
+    representation entirely, but it is too invasive to backport.  This
+    workaround to prune just on startup should be sufficient for emergencies.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 1b3c3c7f655b53b205a81d48d3fa5e6072d4cec5
+Merge: 4fcd7eea45 4be3db5973
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 29 18:51:27 2017 +0200
+
+    Merge pull request #15719 from Vicente-Cheng/wip-20146-jewel
+    
+    jewel: rgw: 'gc list --include-all' command infinite loop the first 1000 items
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 4fcd7eea45bc5158fd04b3b2bd8da717a59932b5
+Merge: ed19fdfc1c b786f7815b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 29 18:50:21 2017 +0200
+
+    Merge pull request #15477 from smithfarm/wip-19767-jewel
+    
+    jewel: rgw: Delete non-empty bucket in slave zonegroup
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit ed19fdfc1ce39f9cf127748e607ed0439a209f7c
+Merge: da648d66eb 7bf8b854ab
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 29 13:32:21 2017 +0200
+
+    Merge pull request #16015 from asheplyakov/19208-bp-jewel
+    
+    jewel: osd: osd_internal_types: wake snaptrimmer on put_read lock, too
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit da648d66eb67b608d7812d6b75c77df403c21fae
+Merge: 879a650f20 52a00b3bff
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 29 13:31:05 2017 +0200
+
+    Merge pull request #15762 from smithfarm/wip-17385-jewel
+    
+    jewel: ceph cli: Rados object in state configuring race
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 879a650f20ae74df389dfc58720c119bd6ee7330
+Merge: 2a8e0f6695 21b00c3ec0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 29 13:28:29 2017 +0200
+
+    Merge pull request #15474 from smithfarm/wip-19559-jewel
+    
+    jewel: osd: objecter full_try behavior not consistent with osd
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 2a8e0f6695c3f32c6e2a4c1230dce1db037a355a
+Merge: c40316e9d3 3fa277b479
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 29 09:15:37 2017 +0200
+
+    Merge pull request #15947 from asheplyakov/20428-bp-jewel
+    
+    jewel: osd: unlock sdata_op_ordering_lock with sdata_lock hold to avoid missing wakeup signal
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Liu-Chunmei <chunmei.liu@intel.com>
+    Reviewed-by: Xie Xingguo <xie.xingguo@zte.com.cn>
+
+commit c40316e9d3c4c230da234ffdeac6e4a3307a9243
+Merge: 3ec5cd9c61 648dfa1e93
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 29 09:04:24 2017 +0200
+
+    Merge pull request #15475 from tchaikov/wip-16239-jewel
+    
+    qa: add a sleep after restarting osd before "tell"ing it
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 3ec5cd9c61294a4b5ed4d17357f60b1544beb532
+Merge: 0916352610 5adc66bc2a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 29 08:56:01 2017 +0200
+
+    Merge pull request #15473 from smithfarm/wip-20080-jewel
+    
+    jewel: build/ops: rpm: set subman cron attributes in spec file
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+    Reviewed-by: Thomas Serlin <tserlin@redhat.com>
+
+commit 0916352610200ddd2a9b6bbe2dbba5d8ca4d470a
+Merge: 988559b4e2 36b1a35a44
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Aug 28 22:34:52 2017 +0200
+
+    Merge pull request #15460 from smithfarm/wip-17843-jewel
+    
+    jewel: rbd: object-map: batch updates during trim operation
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 988559b4e286360b3cc10ac0c6aa9238b98884dd
+Merge: 5d8ea26ebb b9ce1aa618
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Aug 28 22:33:13 2017 +0200
+
+    Merge pull request #15488 from dillaman/wip-20023-jewel
+    
+    jewel: rbd: rbd-mirror: ensure missing images are re-synced when detected
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 5d8ea26ebb51656ef435b3ae932bb172af9407f7
+Merge: 9d1b08e619 5c53ae3f9a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Aug 28 22:32:04 2017 +0200
+
+    Merge pull request #15602 from tchaikov/wip-20175-jewel
+    
+    jewel: tests: decouple ceph_test_librbd_api from libceph-common
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 9d1b08e6191d55027e9573ab721a320a67453d24
+Merge: cccacd7e0f 61f132078a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Aug 28 22:31:09 2017 +0200
+
+    Merge pull request #16124 from smithfarm/wip-19174-jewel
+    
+    jewel: rbd: rbd_clone_copy_on_read ineffective with exclusive-lock
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 178245d543d680bbdc7f2da07c431fc745da1143
+Author: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+Date:   Fri Apr 14 21:42:45 2017 +0530
+
+    rgw: add a field to store generic user data in the bucket index,
+    that can be populated/fetched via a configurable custom http header
+    
+    Signed-off-by: Pavan Rallabhandi <prallabhandi@walmartlabs.com>
+    (cherry picked from commit abca7a86c3cfbb58fafb5d057d9d6f5017a53704)
+    
+    Conflicts:
+            src/rgw/rgw_op.cc
+                    Number of arguments to RGWPutObjProcessor_Multipart::do_complete has been changed in master with an extra field accounted_size added, which is not required in Jewel.
+                    Number of arguments for the processor->complete() has been changed in master, which is not required in Jewel.
+    
+            src/rgw/rgw_op.h
+                    A new member field `crypt_http_responses` is added in the class `RGWPutObj` in master, which is not required in Jewel.
+    
+            src/rgw/rgw_rados.cc
+                    In the function RGWPutObjProcessor::complete() there is an extra argument to do_complete() in master, which is not required in Jewel.
+                    In the function RGWPutObjProcessor_Atomic::do_complete, a call to obj_op.write_meta has accounted_size, which is not required in Jewel.
+                    There is an extra argument accounted_size in RGWRados::Bucket::UpdateIndex::complete in master, which is not required in Jewel.
+                    In RGWRados::Bucket::UpdateIndex::complete, RGWObjEnt has been removed in master, which has to be retained in Jewel.
+                    In RGWRados::cls_obj_complete_op, user_data is added to the rgw_bucket_dir_entry_meta structure.
+                    In RGWRados::cls_bucket_list, the user_data field of RGWObjEnt is populated.
+    
+            src/rgw/rgw_rados.h
+                    In UpdateIndex::complete(), remove_objs is of type rgw_obj_key in Jewel instead of rgw_obj_index_key
+                    Corrected various function declarations as appropriate.
+                    RGWPutObjProcessor_Multipart is not part of this file in Jewel.
+    
+            src/rgw/rgw_rest_swift.cc
+                    In RGWListBucket_ObjStore_SWIFT::send_response(), accounted_size is not required in Jewel.
+    
+            src/rgw/rgw_common.h
+                    Add user_data field in RGWObjEnt structure.
+    
+            src/rgw/rgw_json_enc.cc
+                    Add user_data field while dumping RGWObjEnt.
+
+commit cccacd7e0fb7422e18020b919ad10bac8152b9e8
+Merge: 006804ab68 c2137f2f72
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Aug 28 11:03:32 2017 +0200
+
+    Merge pull request #15760 from smithfarm/wip-20325-jewel
+    
+    libradosstriper silently fails to delete empty objects in jewel
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 006804ab68818e73bdf822aa5994346b40cc6add
+Merge: 14a52865da 2efd7cd6f3
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Aug 28 10:50:27 2017 +0200
+
+    Merge pull request #16061 from weiqiaomiao/wqm-wip-ceph-disk-jewel
+    
+    jewel: ceph-disk: remove the special check to bcache devices
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Zhu Shangzhong <zhu.shangzhong@zte.com.cn>
+
+commit 14a52865da4de99b445560356533d7128df382d6
+Merge: d9bbf8c6e6 a3c487448b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Aug 28 10:47:59 2017 +0200
+
+    Merge pull request #15503 from Vicente-Cheng/wip-20011-jewel
+    
+    jewel: tools: ceph-disk: separate ceph-osd --check-needs-* logs
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 187e9eaa30514c93cd8e982ff67ddb6502810af9
+Author: Aleksei Gutikov <aleksey.gutikov@synesis.ru>
+Date:   Mon Aug 21 17:04:45 2017 +0300
+
+    rgw: Prevent overflow of stats cached values
+    
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+    Fixes: http://tracker.ceph.com/issues/20934
+    Signed-off-by: Aleksei Gutikov <aleksey.gutikov@synesis.ru>
+    (cherry picked from commit 634215eea1ddd4e4f5dc0066c4a2e745cfc20475)
+    
+    Conflicts:
+            src/rgw/rgw_quota.cc
+                           Have the quota stats structures to reflect rounded values in KB and honor the signed integer values
+
+commit 59854a158e8df803013ba1266d414af67feebac1
+Author: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+Date:   Tue Jul 18 14:40:04 2017 +0530
+
+    rgw: Do not decrement stats cache when the cache values are zero
+    
+    With RGWs configured in a load balancer, there is a possibility of
+    having the cached values going unbound, when PUT/DELETE operations
+    do not land up on the same RGW. To avoid such cases, make sure the
+    decrement of stats happen only when the cached values are sane.
+    
+    Fixes: http://tracker.ceph.com/issues/20661
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+    (cherry picked from commit 3903e213c7ac7624e3452f5f3b1ca1c339bf2ca2)
+    
+    Conflicts:
+            src/rgw/rgw_quota.cc
+
+commit 81810c1c7aa03a353975db500abeb1163e38fde7
+Author: fang.yuxiang <fang.yuxiang@eisoo.com>
+Date:   Wed Aug 9 11:48:53 2017 +0800
+
+    rgw: break sending data-log list infinitely
+    
+    radosgw send data-log list infinitely when opposite end trimmed the data-log and in quiescence.
+    Fixes: http://tracker.ceph.com/issues/20951
+    
+    Signed-off-by: fang yuxiang fang.yuxiang@eisoo.com
+    (cherry picked from commit 40db1fdbc7b24d56b2c7b463238e6b96f2d4bbd9)
+
+commit 9ab2e5fde02b705658d4e763519282cad3f0b173
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Jul 20 08:18:22 2017 -0400
+
+    rgw_file: properly & |'d flags
+    
+    Found by "Supriti Singh" <Supriti.Singh@suse.com>.
+    
+    Fixes http://tracker.ceph.com/issues/20663
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 5f838c05c72ad46ce01884a916f81275ef43dacd)
+
+commit d3b34d3f9fece0d12c19132852dcb2c9b38e29d5
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Apr 18 09:19:13 2017 -0400
+
+    rgw_file:  pre-compute unix attrs in write_finish()
+    
+    New serialized Unix attrs need to reflect the change being made,
+    and should be reverted if the change fails.
+    
+    Fixes: http://tracker.ceph.com/issues/19653
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit ed91d23ccaaac4e72a4c28a58e77485395949f04)
+    
+    Conflicts:
+        src/rgw/rgw_file.cc (processor->complete() takes different arguments in
+            jewel than in master)
+
+commit d9bbf8c6e6d6c2a8b978a5f31cdb0c5c52184d0f
+Merge: de27247717 ab2d3582f4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sun Aug 27 18:10:39 2017 +0200
+
+    Merge pull request #14874 from ceph/wip-19805-jewel
+    
+    jewel: rbd: default features should be negotiated with the OSD
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit de2724771785b190cff88996bb09097413a08e84
+Merge: d3677e9f50 618a82e11e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sun Aug 27 18:09:03 2017 +0200
+
+    Merge pull request #14977 from dillaman/wip-19873-jewel
+    
+    jewel: rbd: rbd-mirror: failover and failback of unmodified image results in split-brain
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit d3677e9f5054362b70f9569a47967930b954a67c
+Merge: d47a2e05cf b63b7886dc
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sun Aug 27 17:53:15 2017 +0200
+
+    Merge pull request #15461 from smithfarm/wip-19795-jewel
+    
+    jewel: tests: test_notify.py: assert(not image.is_exclusive_lock_owner()) on line 147
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit d47a2e05cf4004bb4d7ea2a1c20642d17e2f3b14
+Merge: 27fd5ec473 5c0986153e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sun Aug 27 17:51:35 2017 +0200
+
+    Merge pull request #15463 from smithfarm/wip-20017-jewel
+    
+    jewel: rbd: rbd-nbd: kernel reported invalid device size (0, expected 1073741824)
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 27fd5ec4739b87e260bd337ddfe9593943b5f802
+Merge: 2b759d2f74 bc8c5d6c77
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sun Aug 27 17:49:36 2017 +0200
+
+    Merge pull request #15464 from smithfarm/wip-20153-jewel
+    
+    jewel: rbd: Potential IO hang if image is flattened while read request is in-flight
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 3c227e3f086cbfbfff23d11515d9261906c7219d
+Author: fang.yuxiang <fang.yuxiang@eisoo.com>
+Date:   Wed Jul 19 18:49:11 2017 +0800
+
+    rgw: fix rgw hang when do RGWRealmReloader::reload after go SIGHUP
+    
+    Quota async processer reference count err when bucket has no explicit shard
+    
+    Fixes: http://tracker.ceph.com/issues/20686
+    
+    Signed-off-by: fang yuxiang fang.yuxiang@eisoo.com
+    (cherry picked from commit 957b9bdee2024723da240e488b17d777942ac77d)
+    
+    Conflicts:
+        src/rgw/rgw_rados.cc - master has diverged from jewel; applied change
+            manually to the RGWGetBucketStatsContext ctor call
+
+commit cf65d638277b8cb36e70a0eae96dfbd47ca0bf37
+Author: Radoslaw Zarzynski <rzarzyns@redhat.com>
+Date:   Wed Jul 26 08:16:11 2017 -0400
+
+    rgw: rgw_website.h doesn't assume inclusion of the std namespace anymore.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
+    (cherry picked from commit 06f18c163ec843f1fc852f078f539e1821644506)
+
+commit c348ec29947c034779937d6611f4d15545fd3e44
+Author: Radoslaw Zarzynski <rzarzyns@redhat.com>
+Date:   Wed Jul 26 08:12:30 2017 -0400
+
+    rgw: never let http_redirect_code of RGWRedirectInfo to stay uninitialized.
+    
+    Fixes: http://tracker.ceph.com/issues/20774
+    Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
+    (cherry picked from commit 6318b4990fe7fe8f25fff4755eee64fef069dee9)
+
+commit 442911ebad8a4b6f69fb9caa2e1f7e44486de166
+Author: Radoslaw Zarzynski <rzarzyns@redhat.com>
+Date:   Wed Jul 26 09:25:36 2017 -0400
+
+    rgw: fix the UTF8 check on bucket entry name in rgw_log_op().
+    
+    Fixes: http://tracker.ceph.com/issues/20779
+    Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
+    (cherry picked from commit e9a787163080474a1d1a3b8db4db824161326982)
+
+commit 8b52105701e8bbc5e3eb2cc1b251e739376540df
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Thu Apr 13 17:14:48 2017 +0800
+
+    rgw: fix the bug that part's index can't be removed after completing
+    multipart upload when the bucket versioning is enabled.
+    
+    Fixes: http://tracker.ceph.com/issues/19604
+    
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit decc01e2b1be9e4f809ff052149e1e6377c35047)
+
+commit 881bed78620a2c55c9530abefab433c7341895b9
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Wed Nov 16 10:52:54 2016 +0800
+
+    rgw: Fix a bug that multipart upload may exceed quota ...
+    
+    ... because of improperly operation to the cache.
+    
+    Fixes: http://tracker.ceph.com/issues/19602
+    
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit ffe95d3c11f05937109b72e8e54c0cfb7dc09a36)
+    
+    Conflicts:
+        src/rgw/rgw_rados.cc - jewel does not have 885606c76cf8009570098cde48c0a23a21e6020d
+            so use state->size instead of state->accounted_size
+
+commit 2b759d2f74f3870c96c33de62d304b703141beb2
+Merge: f024855cae 89a2f6476d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Aug 26 19:43:55 2017 +0200
+
+    Merge pull request #15455 from smithfarm/wip-19838-jewel
+    
+    jewel: rgw: reduce log level of 'storing entry at' in cls_log
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit f024855cae7c41c3ef61365333672932369ff409
+Merge: ccf9d7109a 938e5b0728
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Aug 26 19:42:23 2017 +0200
+
+    Merge pull request #15456 from smithfarm/wip-20003-jewel
+    
+    jewel: rgw: rgw_file:  v3 write timer does not close open handles
+    
+    Reviewed-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit ccf9d7109a74875107736c017999437d233814d4
+Merge: 9302a44a69 8cf3f18c7a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Aug 26 19:38:29 2017 +0200
+
+    Merge pull request #15457 from smithfarm/wip-20018-jewel
+    
+    jewel: rgw: multisite: memory leak on failed lease in RGWDataSyncShardCR
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 9302a44a69eb29d27b184eba76b9c9df613e6e3e
+Merge: dea17eaa1a 0897afe74d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Aug 26 19:33:42 2017 +0200
+
+    Merge pull request #15459 from smithfarm/wip-20144-jewel
+    
+    jewel: rgw: rgw_file: cannot delete bucket w/uxattrs
+    
+    Reviewed-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit dea17eaa1ab5cdc712b18567c1fed702ee655c2f
+Merge: e335808d26 2e278d821d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Aug 26 19:32:40 2017 +0200
+
+    Merge pull request #15465 from smithfarm/wip-20155-jewel
+    
+    jewel: rgw: fix crash caused by shard id out of range when listing data log
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 36b1a35a449d7d1e0b3e506e021fc6bdb274750c
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Aug 26 10:06:25 2017 +0200
+
+    librbd: clean up object map update interface, revisited
+    
+    In master, the "batch update" change [1] was merged before the "order
+    concurrent updates" [2], while in jewel the latter is already
+    backported [3]. A backport of [1] to jewel was attempted, and was
+    necessarily applied on top of [3] - i.e. in the reverse order compared
+    to how the commits went into master. This reverse ordering caused the
+    automated cherry-pick to miss some parts of [1] which this commit is
+    adding manually.
+    
+    [1] https://github.com/ceph/ceph/pull/11510
+    [2] https://github.com/ceph/ceph/pull/12420
+    [3] https://github.com/ceph/ceph/pull/12909
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit ca5577853cacb76ee46d50d644884102353caa21
+Author: Venky Shankar <vshankar@redhat.com>
+Date:   Sat Oct 15 17:18:30 2016 +0530
+
+    librbd: batch ObjectMap updations upon trim
+    
+    Shrinking a clone which has snapshots and does not share
+    majority of objects with its parent (i.e., there are less
+    objects to be copied up) involves huge number of object
+    map updates -- two (pre, post) per object. This results
+    in lots of requests to be send to OSDs especially when
+    trimming a gigantus image. This situation can be optimized
+    by sending batch ObjectMap updates for an object range
+    thereby significantly cutting down OSD traffic resulting
+    in faster trim times.
+    
+    Fixes: http://tracker.ceph.com/issues/17356
+    Signed-off-by: Venky Shankar <vshankar@redhat.com>
+    (cherry picked from commit 05653b7c512334533b801013f7e426363237301b)
+
+commit e335808d267435414dc1351e0f2e99e773fce805
+Merge: 47e3ff8408 244a2ae6a9
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Aug 25 14:26:21 2017 -0700
+
+    Merge PR #16248 into jewel
+    
+    * refs/remotes/upstream/pull/16248/head:
+            client: don't re-send interrupted flock request
+            mds/flock: properly remove item from global_waiting_locks
+            mds/flock: properly update ceph_lock_state_t::client_waiting_lock_counts
+            mds/flock: don't import ceph_lock_state_t::waiting_locks
+    
+    Reviewed-by: Zheng Yan <zyan@redhat.com>
+
+commit 47e3ff8408af027fe60f1260c3c46c7ac7f8135b
+Merge: aec54d8ecd 01890abd95
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Aug 25 20:09:58 2017 +0200
+
+    Merge pull request #15454 from smithfarm/wip-19775-jewel
+    
+    jewel: rgw: multisite: realm rename does not propagate to other clusters
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit aec54d8ecdf3bfdb39b3f03a9e3971f78c13d4d4
+Merge: 494c634c23 a36211c84d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Aug 25 20:09:15 2017 +0200
+
+    Merge pull request #15453 from smithfarm/wip-19768-jewel
+    
+    jewel: rgw: multisite: operating bucket's acl&cors is not restricted on slave zone
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 494c634c236dff709b1d6856b5f25c419a0ec067
+Merge: 0da1195f5f afd036cb67
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Aug 25 20:08:26 2017 +0200
+
+    Merge pull request #15452 from smithfarm/wip-19765-jewel
+    
+    jewel: rgw: when uploading objects continuously into a versioned bucket, some objects will not sync
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 0da1195f5f2f6d272c6da5ed4583f11246da07c4
+Merge: beb6e229b4 50f140768d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Aug 25 20:06:37 2017 +0200
+
+    Merge pull request #15451 from smithfarm/wip-19764-jewel
+    
+    jewel: rgw: set latest object's acl failed
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit beb6e229b48d4d9f8c4feb18fa0868aa03109f0c
+Merge: 88126f86bf 61df8eaa02
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Aug 25 20:05:54 2017 +0200
+
+    Merge pull request #15450 from smithfarm/wip-19758-jewel
+    
+    jewel: rgw: multisite: after CreateBucket is forwarded to master, local bucket may use different value for bucket index shards
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 92eb6836bfb3350f11fcc2d55a4710be45a34e4f
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Aug 23 15:31:50 2017 -0700
+
+    config: disable skewed utilization warning by default
+    
+    This has a few problems:
+    
+    1- It does not do it's analysis over CRUSH rule roots/classes, which
+    means that an innocent user of classes will see skewed usage (bc hdds are
+    more full than ssds, say)
+    
+    2- It does not take degraded clusters into account, which means the warning
+    will appear when a fresh OSD is added.
+    
+    See http://tracker.ceph.com/issues/20730
+    
+    Based on master commit 7832c53 but we've decided not to remove the code
+    in the older releases so it can be used if appropriate for a
+    particular cluster configuration.  Also, 7832c53 won't cleanly
+    cherry-pick so this is easier.
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+
+commit 88126f86bf240db8bd4573832a75709a8ae8ddba
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 24 10:27:22 2017 -0400
+
+    qa/workunits/rados/test-upgrade-v11.0.0: skip ListObjectsError
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 8199f3998f01ae31c4ae88fda7cd0b4704ae7c8e
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 24 10:20:00 2017 -0400
+
+    qa/workunits/rados/test-upgrade-v11.0.0: skip more tests
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 55cd3d90f8f3cd1d54df922badaf818acdcf140b
+Merge: b0814346f3 f965bdd013
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Aug 24 09:16:19 2017 -0500
+
+    Merge pull request #16870 from liewegas/wip-20929-jewel
+    
+    jewel: ceph-disk: dmcrypt cluster must default to ceph
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit b0814346f3e7553654a999b27ba12ea7985a2ff3
+Merge: d79618b004 cd8c51569b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Aug 24 10:15:57 2017 +0200
+
+    Merge pull request #14659 from asheplyakov/19473-jewel
+    
+    jewel: rgw: add the remove-x-delete feature to cancel swift object expiration
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit d79618b004bdb9270cbd50b98e37d69721004bce
+Merge: 14804ad5c6 01abfed8f2
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Aug 24 09:13:23 2017 +0200
+
+    Merge pull request #15322 from gregsfortytwo/jewel-19931-snaptrim-shutdown
+    
+    jewel: osd: Reset() snaptrimmer on shutdown and do not default-abort on leaked pg refs
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 14804ad5c6c4e4a8ea7951e2e7be562f2ac46d53
+Merge: 03fb91cfef 328e2cc71a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Aug 24 09:11:52 2017 +0200
+
+    Merge pull request #15236 from asheplyakov/20036-bp-jewel
+    
+    jewel: mon: factor mon_osd_full_ratio into MAX AVAIL calc
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 03fb91cfef5b75baecc56d795f64a47793974c56
+Merge: a07f4d3a90 644bb1e81c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Aug 24 09:07:08 2017 +0200
+
+    Merge pull request #14346 from ceph/wip-ceph-disk-fix-jewel
+    
+    jewel: build/ops: Add fix subcommand to ceph-disk, fix SELinux denials, and speed up upgrade from non-SELinux enabled ceph to an SELinux enabled one
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit a07f4d3a90f0dd9a30801f813c91183f77e5eb6c
+Merge: 3e75beb162 e3cba06bd0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Aug 24 09:04:49 2017 +0200
+
+    Merge pull request #15083 from asheplyakov/19926-bp-jewel
+    
+    jewel: mon: crash on shutdown, lease_ack_timeout event
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 3e75beb1624046759694823b3406ce4e451bd5c5
+Merge: 6c51a81a82 e6582c43be
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Aug 23 20:56:38 2017 +0200
+
+    Merge pull request #14663 from smithfarm/wip-19228-jewel
+    
+    jewel: rbd: Enabling mirroring for a pool with clones may fail
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 6c51a81a822408ac5b45706b888e22a02e3e99ca
+Merge: 9502460f7f 03ca042248
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Aug 23 20:52:04 2017 +0200
+
+    Merge pull request #15050 from asheplyakov/19265-bp-jewel
+    
+    jewel: core: an OSD was seen getting ENOSPC even with osd_failsafe_full_ratio passed
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 9502460f7f15e6ef9a79a09ccdb0463966aabb83
+Merge: a70259a441 9ec5d8b1a5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Aug 23 20:50:01 2017 +0200
+
+    Merge pull request #15065 from asheplyakov/19915-bp-jewel
+    
+    jewel: osd: PrimaryLogPG: do not call on_shutdown() if (pg.deleting)
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit a70259a44173adeb1046a01e9c3d40b71052b3bd
+Merge: fd197dcbeb 09f076ed96
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Aug 23 20:48:38 2017 +0200
+
+    Merge pull request #14943 from shinobu-x/wip-18293-jewel
+    
+    jewel: osd: leaked MOSDMap
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit fd197dcbeb4c416ad903a7795ce91ac3ffd0f296
+Merge: 5dc3b45b92 3776d960e0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Aug 23 20:12:01 2017 +0200
+
+    Merge pull request #14699 from smithfarm/wip-19679-jewel
+    
+    jewel: mds: damage reporting by ino number is useless
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 5dc3b45b92e9fb4b7025d3373dfd85c5867b6b60
+Merge: 8581349c8b a4c181fa7a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Aug 23 20:10:07 2017 +0200
+
+    Merge pull request #14673 from smithfarm/wip-19466-jewel
+    
+    jewel: mds: log rotation doesn't work if mds has respawned
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 8581349c8b1b1e06522707d0b6a3bd2d0c3e35b1
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 23 13:53:48 2017 -0400
+
+    qa/workunits/rados/test-upgrade-v11.0.0.sh: more tests skipped
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit c90969031f8f0c49a974c3e001fc44b1544547d7
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue May 2 12:16:45 2017 -0400
+
+    rgw: call update_latest_epoch() on all period updates
+    
+    when updating the period, callers use the atomic result of
+    update_latest_epoch() to determine whether they need to call
+    RGWPeriod::reflect() and RGWRealm::notify_new_period()
+    
+    this adds a missing call to RGWPeriod::reflect() to RGWPeriodPuller,
+    which was previously not safe to do without atomic updates to
+    latest_epoch
+    
+    Fixes: http://tracker.ceph.com/issues/19817
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit d825b9d854864b815890cf9204d7e72e1e1a9ada)
+
+commit 6c8a7ca22bffc334032ae93953cc2b7ad47a363d
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue May 2 11:45:39 2017 -0400
+
+    rgw: remove set_latest_epoch from RGWPeriod::store_info
+    
+    split the latest_epoch update out of RGWPeriod::store_info(), so callers
+    that need to call the atomic update_latest_epoch() can do so and
+    interpret its result separately
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit c1731dd2d3de5a337903004f835045d609a731cf)
+    
+    Conflicts:
+            src/rgw/rgw_rados.cc (trivial resolution)
+
+commit 03cf10aefb9dcc253170472b3c95c829fb68cc40
+Merge: 1f9693be5e c49e249fb5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Aug 23 08:41:55 2017 +0200
+
+    Merge pull request #15442 from Vicente-Cheng/wip-20025-jewel
+    
+    jewel: cephfs: osdc/Filer: truncate large file party by party
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+
+commit 1f9693be5ef8c64c7d58ccf08ef27cf2d7cb7cdb
+Merge: 198f7fac8f 0bb08832c0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Aug 23 08:40:20 2017 +0200
+
+    Merge pull request #14691 from smithfarm/wip-19571-jewel
+    
+    jewel: tests: upgrade:client-upgrade/firefly-client-x: drop CentOS
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit ab2d3582f42a45b5d3cf63e00007e0ca65df1c2f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Apr 28 12:52:18 2017 -0400
+
+    librbd: default features should be negotiated with the OSD
+    
+    Derived from f066ce8f80bce929edc209590efd47cce2196ae2
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 5b5468189ea74da5db63bcba0ae6450d83cc99f7
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Sat Nov 5 22:44:33 2016 +0200
+
+    cls/rbd: add get_all_features on client side
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 84f6d5c109911923c25414de639308921983e438)
+    
+    Conflicts:
+            src/cls/rbd/cls_rbd_client.h: trivial resolution
+
+commit ff312a4e6fb08dc93311e6969ef16d621388c395
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue May 2 10:44:20 2017 -0400
+
+    rgw: add atomic RGWPeriod::update_latest_epoch
+    
+    update_latest_epoch() uses RGWObjVersionTracker to implement atomic
+    updates to the period's latest_epoch, returning -EEXIST if we already
+    have an epoch >= the one given
+    
+    Fixes: http://tracker.ceph.com/issues/19816
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 55000d7fb3d4ab1ba4c19f3b3730823b5b056b67)
+
+commit e91a3945afa4ca27678c6adf8d1c792333b6b554
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue May 2 09:47:57 2017 -0400
+
+    rgw: period latest_epoch ops take optional objv tracker
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit b855c39656d7556c7895850d12177ca6c1db01af)
+
+commit c864d5cac25d77307e9fa429c4d9a4f3f5f9df56
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue May 2 09:46:44 2017 -0400
+
+    rgw: remove unused RGWPeriod::use_next_epoch
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 5770a01ebe8b5913cb5446bd4fd09d8029ac1381)
+
+commit 1516e24c8583237bc75c4185b416cd3fd4ec3df1
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Jun 13 11:23:37 2017 -0700
+
+    cls/refcount: store and use list of retired tags
+    
+    Fixes: http://tracker.ceph.com/issues/20107
+    
+    Keep around the list of retired tags, make sure we don't
+    drop a refcount using the same tag.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 2c4fb020a13333bba6d3c29318c8a87099d915db)
+
+commit 9b5b21def271b1b85834fb32b762a0c0dc688d12
+Author: Aleksei Gutikov <aleksey.gutikov@synesis.ru>
+Date:   Thu Jul 6 11:27:42 2017 +0300
+
+    rgw: fix not initialized pointer which cause rgw crash with ec data pool
+    
+    In RGWPutObjProcessor_Atomic::complete_writing_data()
+    with pending_data_bl.length() > 0 and next_part_ofs==data_ofs
+    not initialized void *handle leads to invalid pointer librados::AioCompletion::pc
+    which leads to rgw crash.
+    
+    Fixes: http://tracker.ceph.com/issues/20542
+    Signed-off-by: Aleksei Gutikov <aleksey.gutikov@synesis.ru>
+    (cherry picked from commit 3e938dd9fe681fa8652dc4b0ec1dc56781d884c0)
+    
+    Conflicts:
+            src/rgw/rgw_rados.cc (trivial resolution)
+
+commit 7d199110fb0a46796bfe8422ee103b837deb964f
+Author: fang yuxiang <fang.yuxiang@eisoo.com>
+Date:   Wed May 3 13:46:50 2017 +0800
+
+    rgw: don't do unneccesary write if buffer with zero length
+    
+    Don't do unneccesary write if buffer with zero length,
+    or there will be one more shadow stripe rados object with size 0
+    
+    Signed-off-by: fang yuxiang <fang.yuxiang@eisoo.com>
+    (cherry picked from commit 4e21f7ea2167be7793273f31faeb0aa95f0fa30a)
+
+commit 73c9d33ac0d761ef0b616a4beae56dc15c7c3f5e
+Author: Abhishek Varshney <abhishek.varshney@flipkart.com>
+Date:   Wed Jul 19 11:57:08 2017 +0000
+
+    rgw: fix error message in removing bucket with --bypass-gc flag
+    
+    Fixes: http://tracker.ceph.com/issues/20688
+    
+    Signed-off-by: Abhishek Varshney <abhishek.varshney@flipkart.com>
+    (cherry picked from commit 596b4bc05f167baca3ae6e3ed9bed5b04d27409e)
+
+commit e76198751cf876ee1137194e6cdb1aa761b31d21
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Fri Jul 7 11:09:02 2017 +0800
+
+    rgw:multisite: fix RGWRadosRemoveOmapKeysCR
+    
+    RGWRadosRemoveOmapKeysCR::request_complete return val is wrong. The return val should get from completion.  Some member variables is not used, clear up those variables.
+    
+    Fixes:http://tracker.ceph.com/issues/20539
+    
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit 82a0b6ac2130a7f326473e5f376c214f59f50829)
+    
+    Conflicts:
+            src/rgw/rgw_cr_rados.cc (trivial resolution)
+            src/rgw/rgw_cr_rados.h (master does not have IoCtx, jewel does not have override)
+
+commit 71262f1faeaa26481cec15e2fe3894a1ab2a3f3f
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Tue Nov 29 08:57:30 2016 +0800
+
+    rgw: lock is not released when set sync marker is failed.
+    
+    Fixes: http://tracker.ceph.com/issues/18077
+    
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit 65ef596ce1d83515338367f69b1bf0ce0ab96ae5)
+
+commit 198f7fac8f67b5e2996f2b53420c6c58854e54b3
+Merge: ed53df2bb9 7e3db1519e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 22 16:25:14 2017 +0200
+
+    Merge pull request #15448 from smithfarm/wip-19615-jewel
+    
+    jewel: rgw: multisite: bucket zonegroup redirect not working
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 38718ace488d2f1ca93c50b2d6474d6095b1f544
+Author: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+Date:   Tue Jun 13 14:39:14 2017 +0800
+
+    rgw_file: prevent conflict of mkdir between restarts
+    
+    Fixes: http://tracker.ceph.com/issues/20275
+    
+    Signed-off-by: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+    (cherry picked from commit 87cec5ff9a2f516321543076f536fcfff433c03b)
+
+commit 0bb08832c0ba20cc85a50d9dfbc4efc0f083b7ae
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Apr 20 19:15:29 2017 +0200
+
+    tests: upgrade:client-upgrade/firefly-client-x: drop CentOS
+    
+    The RBD suite needs ceph-cm-ansible to install qemu-kvm on CentOS, but doing
+    that breaks the firefly install on CentOS because:
+    
+    1. the qemu-kvm that gets installed is from hammer (0.94.5)
+    2. qemu-kvm brings in librados2, librbd1 as dependencies
+    
+    As a result, the hammer librados2 and librbd1 are installed on the test nodes
+    even before the teuthology install task starts. When it does start and tries
+    to install firefly, it fails because firefly librados2 and librbd1 cannot be
+    installed over their hammer versions.
+    
+    Fixes: http://tracker.ceph.com/issues/19571
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit ed53df2bb9d9fed42f47f73168f77c7346942601
+Merge: 7a599849d3 66c17ff10b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 22 13:47:21 2017 +0200
+
+    Merge pull request #15197 from smithfarm/wip-20007-jewel
+    
+    jewel: rgw: remove unnecessary 'error in read_id for object name: default'
+    
+    Reviewed-by: Jos Collin <jcollin@redhat.com>
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 53e97da166c6e36c6808063f90c4a2cef2d52a77
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Jun 8 22:29:48 2017 +0200
+
+    ceph-disk: set the default systemd unit timeout to 3h
+    
+    There needs to be a timeout to prevent ceph-disk from hanging
+    forever. But there is no good reason to set it to a value that is less
+    than a few hours.
+    
+    Each OSD activation needs to happen in sequence and not in parallel,
+    reason why there is a global activation lock.
+    
+    It would be possible, when an OSD is using a device that is not
+    otherwise used by another OSD (i.e. they do not share an SSD journal
+    device etc.), to run all activations in parallel. It would however
+    require a more extensive modification of ceph-disk to avoid any chances
+    of races.
+    
+    Fixes: http://tracker.ceph.com/issues/20229
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit a9eb52e0a4c06a80e5dbfaac394aac940edf4c68)
+
+commit 7a599849d3f62c4e6001e2c7e3973b5c368d85d1
+Merge: c79524153c 592293cdfb
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 22 10:00:07 2017 +0200
+
+    Merge pull request #15051 from asheplyakov/19910-bp-jewel
+    
+    jewel: build/ops: extended, customizable systemd ceph-disk timeout
+    
+    Reviewed-by: David Disseldorp <ddiss@suse.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit c79524153cb52b1022dba0102aaef942f81d6fd9
+Merge: 161cba9cea 6b99bc9c60
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 22 09:54:12 2017 +0200
+
+    Merge pull request #15428 from smithfarm/wip-20162-jewel
+    
+    jewel: build/ops: deb: Fix logrotate packaging
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 161cba9cea5d83911e1ee7d42d58004ec327b9bb
+Merge: 1f0549ca49 9a5425d5de
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 22 09:52:49 2017 +0200
+
+    Merge pull request #15447 from smithfarm/wip-19613-jewel
+    
+    jewel: rgw: multisite: rest api fails to decode large period on "period commit"
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit e6582c43bebac315ef2d76fded92373c7b1300be
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 27 16:21:33 2017 -0400
+
+    test: remove hard-coded image name from RBD metadata test
+    
+    Fixes: http://tracker.ceph.com/issues/19798
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8f72e745e157cc12d76ca6babe956c5698ee297f)
+
+commit 8f23ea4fc0938c35998193ea76d7a029422b89b0
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Mar 2 17:18:18 2017 +0100
+
+    librbd: relax "is parent mirrored" check when enabling mirroring for pool
+    
+    If the parent is in the same pool and has the journaling feature enabled
+    we can assume the mirroring will eventually be enabled for it.
+    
+    Fixes: http://tracker.ceph.com/issues/19130
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit fe31bca22f90ce02f461d6421a4f66539db888d3)
+
+commit 1f0549ca49cc60ed62f52ae5ee22d1044f70aab0
+Merge: 648fabaa9f 00ce80b52f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 22 09:31:04 2017 +0200
+
+    Merge pull request #15196 from smithfarm/wip-20006-jewel
+    
+    jewel: build/ops: rpm: fix python-Sphinx package name for SUSE
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+    Reviewed-by: Abhishek Lekshmanan <abhishek@suse.com>
+
+commit 1415ecbd7de21ed2ab0c8f6e20f707cbe4f1390c
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Mon Aug 21 14:06:13 2017 +1000
+
+    ceph-disk: Use stdin for 'config-key put' command
+    
+    The 'osd new' command is not available in jewel so backport support for
+    stdin '-i -' and use it for the 'config-key put' command in order to
+    not log the dmcrypt key.
+    
+    Fixes: http://tracker.ceph.com/issues/21059
+    
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit 28cfb01dddab94b849a8d8bce8d8bd27c3818e25
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jul 17 09:38:52 2017 -0400
+
+    ceph: allow '-' with -i and -o for stdin/stdout
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3a4931b0e467e3b64a2637b96d7a3cab9add1c80)
+    
+    Conflicts:
+            qa/workunits/cephtool/test.sh (post-jewel functions removed)
+            src/ceph.in
+
+commit a6da22fcf646ff9db45a01a43fa719982bf9bca4
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Jun 8 19:00:05 2017 +0200
+
+    ceph-disk: implement command_with_stdin
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 597393d5d09c52170b7db6f609ce142edfd4d8fe)
+
+commit 3adbc35205f663c8c312476f3694d2c14294c5d0
+Author: Javier M. Mellid <jmunhoz@igalia.com>
+Date:   Mon Aug 1 21:00:28 2016 +0200
+
+    rgw: aws4: add rgw_s3_auth_aws4_force_boto2_compat conf option
+    
+    Runtime bugfix to handle presigned urls computed with canonical requests using
+    the port number once.
+    
+    Boto2 computes canonical requests using the port number twice although it
+    should be used once only. This behaviour is a bug supported by AWS S3. Boto2 is
+    used in RGW S3 as reference implementation.
+    
+    The client-side tools not supporting this boto2 bug will fail although they
+    should work too.
+    
+    In order to support both presigned url implementations this patch adds a config
+    option to compute a second signature. With this option disabled, the code will
+    compute two signatures when the first signature is not valid. The aws4 auth
+    succeed if some of the two signatures is valid.
+    
+    The config option rgw_s3_auth_aws4_force_boto2_compat, is enabled by default so
+    one signature, working with boto2, is computed only.
+    
+    Fixes: http://tracker.ceph.com/issues/16463
+    
+    Signed-off-by: Javier M. Mellid <jmunhoz@igalia.com>
+    (cherry picked from commit 078c513b6bc6b1d1da50db1d51fbbb65bddd44b9)
+
+commit cd848c1e9852c89a06e3baa669314cba2fdef965
+Author: Alexey Sheplyakov <asheplyakov@yandex.ru>
+Date:   Sun Aug 13 16:33:00 2017 +0400
+
+    jewel: mon: fix force_pg_create pg stuck in creating bug
+    
+    Register the creating PG through the common path to get the PG mapped to an OSD.
+    Adapted from 9a41a0b7289fa59f4b747a63e152e88af6e8abd5
+    
+    Fixes: http://tracker.ceph.com/issues/19182
+    
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@yandex.ru>
+
+commit 648fabaa9fc1e78aa808a3a2c7c91dee464dbc76
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Aug 12 09:00:22 2017 -0400
+
+    qa/workunits/rados/test/test-upgrade-v11.0.0: skip Quota test
+    
+    This makes a pool go full which confounds the wait for healthy check
+    during restarts.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 37b37e78f858a2d981698de2e9e1aafef7953642
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Jul 31 14:45:25 2017 +0300
+
+    rgw: Fix up to 1000 entries at a time in check_bad_index_multipart
+    
+    Fixes: http://tracker.ceph.com/issues/20772
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 25cef9a09721e1a67d217535d4d624ab60adca02)
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 6bd2dae1934f5f2a752b2c0edaa937b5a4163c36
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Wed Aug 9 13:51:40 2017 -0400
+
+    rgw: replace '+' with "%20" in canonical query string for s3 v4 auth.
+    
+    Fixes: http://tracker.ceph.com/issues/20501
+    
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit 416bc51101b4ae5da569c9bc3d8d738eeadc25a6)
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 3242b2b5629b951acfe90011a3c07533db1ab0a3
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Tue Dec 6 15:07:19 2016 -0800
+
+    ceph-fuse: start up log on parent process before shutdown
+    
+    Otherwise, we hit an assert in the Ceph context and logging teardown.
+    
+    Fixes: http://tracker.ceph.com/issues/18157
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit cbf18b1d80d214e4203e88637acf4b0a0a201ee7)
+
+commit ff67388e24c93ca16553839c16f51030fa322917
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Tue Jan 10 16:36:13 2017 +0800
+
+    rgw: bucket index check in radosgw-admin removes valid index.
+    
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit a786252ed9c0e1ba08a88a43902f7aa1e91c10cd)
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+    
+    Conflicts:
+            src/rgw/rgw_bucket.cc
+                    Jewel has RGWObjEnt, honor the same while populating obj oid
+
+commit 788cfa89dd2abc3db8dc35803e275d4c2fbb6837
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri May 26 12:41:40 2017 -0400
+
+    rgw-admin: fix bucket limit check argparse, div(0)
+    
+    1. normalize arg parsing for "bucket limit check"
+    1.1 s/buckets/bucket/
+    
+    2. avoid dividing by num_shards when it is 0
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit fb8e2fa61f0b4621b5fdc83048467982a7ca2961)
+    Fixes: http://tracker.ceph.com/issues/20966
+
+commit e6f86ba31d3092dbb45bdd35f344aba7b2c5a61b
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 9 09:56:21 2017 -0400
+
+    qa/workunits/rados/test-upgrade-v11.0.0: skip api_io *CmpExt*
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 5a7ea2790c391544a3a8ea4369ac8e46a0b3c4ec
+Author: Abhishek Varshney <abhishek.varshney@flipkart.com>
+Date:   Tue Aug 1 14:46:52 2017 +0000
+
+    rgw : fix race in RGWCompleteMultipart
+    
+    Fixes : http://tracker.ceph.com/issues/20861
+    
+    Signed-off-by: Abhishek Varshney <abhishek.varshney@flipkart.com>
+    (cherry picked from commit dd7dc62eaddb47447c01c03c87ad56cd7690693e)
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 6d5ef4b27d0fad62646ddf82b59d9c277179d293
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Aug 7 14:01:58 2017 -0400
+
+    qa/workunits/rados/test-upgrade-11.0.0: skip RoundTrip3
+    
+    This does the crc op, which isn't in jewel.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit d4fa37323596394feadaed6168b49ccbaf6fc089
+Merge: 5776904626 d3e81588ee
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 7 10:07:13 2017 -0400
+
+    Merge pull request #16343 from dillaman/wip-20630-jewel
+    
+    jewel: tests: qa/tasks: rbd-mirror daemon not properly run in foreground mode
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit f965bdd0134cbc3d947ac83bf96e5ce08541a198
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Aug 3 10:56:24 2017 +0200
+
+    ceph-disk: dmcrypt cluster must default to ceph
+    
+    If ceph_fsid is not found, which is the case for legacy dmcrypted OSD,
+    the cluster must default to ceph, as it was before support for non
+    standard cluster names was introduced.
+    
+    Fixes: http://tracker.ceph.com/issues/20893
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit ed22159fdd7071a110e81cd7b63d7e0f2296761b)
+
+commit 5776904626e17fd1ed73bcebeee742655523e833
+Merge: 8db71d796e 10636ef77f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 1 13:06:54 2017 +0200
+
+    Merge pull request #13507 from shinobu-x/wip-18468-jewel
+    
+    jewel: mon: Disallow enabling 'hashpspool' option to a pool without some kind of --i-understand-this-will-remap-all-pgs flag
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit c49e249fb5a5232dc25b8250f4b7d2b4e3d47bef
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Apr 25 16:21:24 2017 +0800
+
+    osdc/Filer: truncate large file party by party
+    
+    Fixes: http://tracker.ceph.com/issues/19755
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 5fab215e461e5ecc36c0f9d9ea867f6c45e80263)
+    
+    Conflicts:
+            src/osdc/Filer.h
+            src/osdc/Filer.cc
+                - add parameter to fit _modify (need onack parameter)
+            src/mds/MDCache.cc
+                - make truncate() consistency
+
+commit 314952c58993e6ea6a3fa1bb3b6f9a7118c52ea4
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jul 6 20:17:49 2017 +0200
+
+    rgw: lease_stack: use reset method instead of assignment
+    
+    It seems that the intent of 45877d38fd9a385b2f8b13e90be94d784898b0b3 was to
+    change all instances of "lease_stack = ..." to "lease_stack.reset(...)", but
+    this one was missed.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit fc425afeb2e2a6ba3c98c612b3977aea619c9f73)
+
+commit 0930eb07d1c983412e27f7c42836a319a3161c7a
+Author: fang.yuxiang <fang.yuxiang@eisoo.com>
+Date:   Tue Jun 13 16:40:16 2017 +0800
+
+    rgw: meta sync thread crash at RGWMetaSyncShardCR
+    
+    Fixes: http://tracker.ceph.com/issues/20251
+    
+    Signed-off-by: fang yuxiang fang.yuxiang@eisoo.com
+    (cherry picked from commit 45877d38fd9a385b2f8b13e90be94d784898b0b3)
+
+commit e35c938208a7e265387ba01cde9d84d08ec55ec0
+Author: David Disseldorp <ddiss@suse.de>
+Date:   Wed Jul 5 16:23:37 2017 +0200
+
+    ceph-disk: don't activate suppressed journal devices
+    
+    Multipath device activation currently requires that devices representing
+    individual paths making up a multipath device are suppressed, to avoid
+    errors such as:
+      ceph_disk.main.Error: Error: /dev/sda1 is not a multipath block device
+    
+    Activation of suppressed OSD devices is currently handled by checks in
+    main_activate() and main_activate_space(). However, suppressed journal
+    and block devices are not detected.
+    
+    (Along with udev change) Fixes: http://tracker.ceph.com/issues/19489
+    
+    Signed-off-by: David Disseldorp <ddiss@suse.de>
+    (cherry picked from commit c729627450d4032ede7932f4c1b57a6d0cff73b2)
+
+commit 153f77544118613e19d5e88c030c3901234cf950
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Jul 18 15:08:14 2017 -0700
+
+    osd: scrub_to specifies clone ver, but transaction include head write ver
+    
+    Fixes: http://tracker.ceph.com/issues/20041
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit fd598a0d23d61c645633ae774c3404a43d035e3c)
+    
+    Conflicts:
+            src/osd/ReplicatedPG.cc (trivial)
+
+commit 2efd7cd6f32aa780fea5d26c8314c48cae734793
+Author: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+Date:   Mon Jul 3 09:34:08 2017 +0800
+
+    jewel:ceph-disk:remove the special check to bcache devices
+    
+    if we skip partition when coming cross the bcache device in ceph-disk,
+    the udev can't detect the osd in such devices when we reboot the osd's host,
+    so the osd who deploy in bcache device can't auto start when the host restart.
+    
+    And partition is now supported by bcache devices (https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git/commit/drivers/md/bcache?id=b8c0d911ac5285e6be8967713271a51bdc5a936a),
+    we can remove this special check now.
+    
+    This cannot be cherry-picked from master because 'accept bcache devices as data disks' [21f0216](https://github.com/ceph/ceph/commit/21f0216287485e6cce8811f324fee804ef733524)
+    is a jewel-only feature not present in master.
+    
+    Signed-off-by: Wei Qiaomiao <wei.qiaomiao@zte.com.cn>
+
+commit 03ca042248586d1e7612f4d026551891701048c3
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Jul 7 10:53:41 2017 -0700
+
+    jewel: mon: Fix status output warning for mon_warn_osd_usage_min_max_delta
+    
+    Fixes: http://tracker.ceph.com/issues/20544
+    
+    Caued by: 489e810c37ed6fb9d32d1015634041a577501ee4
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 56f9808016ab289bdc0ce7cfbb0503c78b509593)
+    
+    Conflicts:
+            apply changes to src/mon/PGMonitor.cc instead of src/mon/PGMap.cc
+
+commit 8db71d796e50bcdbb6bd711feba6e4fdbecfcd29
+Merge: 2ee413f771 8885f3ab6a
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Jul 20 15:00:32 2017 -0400
+
+    Merge pull request #16289 from cbodley/wip-20513
+    
+    jewel: rgw: cls: ceph::timespan tag_timeout wrong units
+
+commit e9e7850b51c824a50dde96332e9f47fd3a62e60b
+Author: Piotr Dałek <piotr.dalek@corp.ovh.com>
+Date:   Fri Jun 16 13:34:19 2017 +0200
+
+    messages/MOSDPing: optimize encode and decode of dummy payload
+    
+    The dummy payload doesn't need to be processed, we can just skip over
+    it when decoding and we can use a single bufferptr instead of entire
+    bufferlist to encode it.
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@corp.ovh.com>
+    (cherry picked from commit 15ce0772841f913d1eef1daebad0834e5f862383)
+
+commit 8acfd7189a4a2fe937e06d12132df55af5d90fcb
+Author: Piotr Dałek <piotr.dalek@corp.ovh.com>
+Date:   Fri Jun 16 13:10:36 2017 +0200
+
+    messages/MOSDPing: fix the inflation amount calculation
+    
+    If user specifies a min_message_size small enough (or zero to disable
+    it altogether), OSDs will crash and burn while trying to allocate
+    almost 4GB of payload (both min_message_size and payload.length() are
+    unsigned, so it'll roll over back to 4GB and MAX(4GB, 0) will use 4GB).
+    If the size of dummy payload is 0, don't bother constructing bufferptr
+    and bufferlist, then encoding that.
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@corp.ovh.com>
+    (cherry picked from commit d959735a7d684018df564573a2ff88092d870fc1)
+
+commit 52f221b2103333facbc4ead30adc0eba8063983d
+Author: Piotr Dałek <piotr.dalek@corp.ovh.com>
+Date:   Fri Jun 16 13:05:10 2017 +0200
+
+    OSD: mark two heartbeat config opts as observed
+    
+    "osd heartbeat min size" and "osd heartbeat interval" can be changed
+    at runtime, because their values, when used, are always taken from
+    global Ceph configuration. Mark them as observed, so the message
+    the user sees once they're changed doesn't confuse them.
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@corp.ovh.com>
+    (cherry picked from commit 44b7839c788b20e0b9da76bc3838ae10d3ad2f89)
+
+commit f840864a5100dc65bb09a49e91fe4db776f69413
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jun 15 22:18:08 2017 -0400
+
+    messages/MOSDPing: initialize MOSDPing padding
+    
+    This memory must be initialized or else valgrind will be very unhappy.
+    
+    Avoid the cost of zeroing (or even allocating) the buffer for normal
+    padding values by (re)using a static zero buffer.
+    
+    Fixes: http://tracker.ceph.com/issues/20323
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9beaf5efd74daa8c15e50b42583264d1252a85f5)
+
+commit 71645604b6e07369f45034112710661167db052f
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Mon Jun 5 13:33:14 2017 -0700
+
+    osd: heartbeat with packets large enough to require working jumbo frames.
+    
+    We get periodic reports that users somehow misconfigure one of their switches
+    so that it drops jumbo frames, yet the servers are still passing them along. In
+    that case, MOSDOp messages generally don't get through because they are much
+    larger than the 1500-byte non-jumbo limit, but the MOSDPing messages have kept
+    going (as they are very small and dispatched independently, even when the
+    server is willing to make jumbo frames). This means peer OSDs won't mark down
+    the ones behind the broken switch, despite all IO hanging.
+    Push the MOSDPing message size over the 1500-byte limit so that anybody in
+    this scenario will see the OSDs stuck behind a bad switch get marked down.
+    
+    Fixes: http://tracker.ceph.com/issues/20087
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 2096113d9e1589c571d96e34dd9cd841308a2567)
+    
+     Conflicts:
+            src/messages/MOSDPing.h
+             - Changed HEAD_VERSION to 3 and kept COMPAT_VERSION to 1.
+             - In class MOSDPing removed following line:
+                   if (header.version >= 2)
+             - To keep ::decode(stamp, p) without condition because HEAD_Version
+               is already 3 now and this condition is removed in the backport commit.
+
+commit d3e81588eef9a3745d2cc27382b3a26b12bc8005
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jul 14 10:32:28 2017 -0400
+
+    qa/tasks: rbd-mirror daemon not properly run in foreground mode
+    
+    Fixes: http://tracker.ceph.com/issues/20630
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4fa1918717b25a5ffafbf649eedcfe7d5ab829c2)
+
+commit 713a88a5c9ec8e1140227ddcf0df8badfd96706d
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Tue May 9 16:39:39 2017 +0800
+
+    rgw: VersionIdMarker and NextVersionIdMarker should be returned when listing
+    object versions if necessary.
+    
+    Fixes: http://tracker.ceph.com/issues/19886
+    
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit f805c3e08948e379b7d2c4f2faf9e7f550e4cb23)
+
+commit 0a3227876c3142d60d137ef351cf91795227883a
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Tue Jun 6 15:43:20 2017 +0800
+
+    rgw: datalog trim and mdlog trim handles the result returned by osd
+    incorrectly.
+    
+    Fixes: http://tracker.ceph.com/issues/20190
+    
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit 7fd6e031e5b0b1f3eca70c5b459d50f6f214171f)
+
+commit 0cd7df3649d7486d444a61cab89c48a89ddd3e8d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 29 14:54:40 2017 -0400
+
+    rbd: do not attempt to load key if auth is disabled
+    
+    Fixes: http://tracker.ceph.com/issues/19035
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8b9c8df6d7f0b75c5451953bb322bc1f9afb6299)
+
+commit 787ba33e5dba285dff874955a8f0d7aabd3f87fe
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 5 08:17:05 2017 -0400
+
+    librbd: filter expected error codes from is_exclusive_lock_owner
+    
+    Fixes: http://tracker.ceph.com/issues/20182
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d4daaf54e6bc42cd4fb2111ea20b2042941b0c31)
+
+commit f9e87fef6a4771545cc23fe94c9cb9656ca5b773
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 5 13:17:19 2017 -0400
+
+    rbd: properly decode features when using image name optional
+    
+    Fixes: http://tracker.ceph.com/issues/20185
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f1b05a2b062a59ec1b6682f7683bfd816433a931)
+
+commit 9a5425d5de16045b57c1307cb3b8263c739909b1
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Apr 5 14:20:20 2017 -0400
+
+    rgw: allow larger payload for period commit
+    
+    testing with 3 zonegroups and 3 zones each, the period json grew larger
+    than 4k and caused decode failures on period commit
+    
+    updated to use the new config variable rgw_max_put_param_size
+    
+    Fixes: http://tracker.ceph.com/issues/19505
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 7f2871fe59d933b03f37fde40f1781b2320d0d50)
+    
+    Conflicts:
+        src/rgw/rgw_rest_realm.cc - in RGWOp_Period_Post::execute(), set max_size
+            explicitly instead of backporting rgw_max_put_param_size
+
+commit 61df8eaa02d3fcdc814568be20631933c34aa8a3
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Tue May 9 15:05:03 2017 +0800
+
+    rgw: when create_bucket use the same num_shards with info.num_shards
+    
+    pr #14388 only fix the num_shards in BucketInfo, "init_bucket_index" function still use local num_shards
+    
+    Fixes: http://tracker.ceph.com/issues/19745
+    
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit 4ce64a190b4ff36985e785e574c077d39796feea)
+    
+    Conflicts:
+            src/rgw/rgw_rados.cc: init_bucket_index() called earlier
+
+commit 93799374a97e5acec00737bb7fd46dce314ec19a
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Fri Apr 7 15:34:27 2017 +0800
+
+    rgw: using the same bucket num_shards as master zg when create bucket in secondary zg
+    
+    create bucket in secondary zonegroup will forward to master. The master may have different num_shards option.
+    So when create bucket in local, should use master's num_shards instead of local num_shards option.
+    
+    Fixes: http://tracker.ceph.com/issues/19745
+    
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit a34c4b8fb13dd5590eb3c6ecb5e55207ed8e3ee8)
+    
+    Conflicts:
+        src/rgw/rgw_op.cc
+        - different argument list of store->create_bucket() in jewel, compared to
+          master
+        - no RGWBulkUploadOp:: in jewel; modifications to
+          RGWBulkUploadOp::handle_dir() omitted
+
+commit 8885f3ab6a529c451581046fb3c8f72fceec6e80
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Jun 29 15:19:36 2017 -0400
+
+    rgw: cls: ceph::timespan tag_timeout wrong units
+    
+    In rgw_dir_suggest(), the ceph::timespan tag_timeout is intended
+    to be a value in seconds, but has been taken as something much
+    smaller (millis?).  The incorrect time scale likely induces a race
+    condition with object deletes.
+    
+    Fixes: http://tracker.ceph.com/issues/20380
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit e6763ebd6124c1b4736c5c7850077df7644bc8f5)
+
+commit 3105f118bccb5ae9209cb289cec5ccdba6d9d7b6
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Fri Dec 2 13:38:29 2016 +0200
+
+    rbd-mirror: set SEQUENTIAL and NOCACHE advise flags on image sync
+    
+    Fixes: http://tracker.ceph.com/issues/17127
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 1167865e3b4c5284938cb1797ebeff15974f7260)
+    
+    Conflicts:
+        src/tools/rbd_mirror/image_sync/ObjectCopyRequest.cc (sparse reads are not
+            being backported to jewel)
+
+commit 24bd60136e033f14a556f8e6220cf4ea49520598
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Jun 26 14:28:21 2017 +0300
+
+    rgw: fix next marker to pass test_bucket_list_prefix in s3test
+    
+    Fixes: http://tracker.ceph.com/issues/19432
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 8fff351d7607c6183b77acc29057c9d55b11f9de)
+
+commit 28f0a8dba36a20b482a70e42241e4e5ee4771e5f
+Author: Giovani Rinaldi <gr.gigio@terra.com.br>
+Date:   Wed Apr 5 16:46:14 2017 -0300
+
+    rgw: fix listing of objects that start with underscore
+    
+    Current marker and prefix search utilized in rgw's function list_objects should respect index key name in order to correctly list objects which have names starting with underscore.
+    
+    Fixes: http://tracker.ceph.com/issues/19432
+    
+    Signed-off-by: Giovani Rinaldi <giovani.rinaldi@azion.com>
+    (cherry picked from commit c1a1539af596e97dc412a505d5411ee6cfdd3980)
+    
+    Conflicts:
+        src/rgw/rgw_rados.cc - in jewel, the call to cur_marker.name.find() is in a
+            conditional block; not so in master
+
+commit 39b2b0b39aedea1c7ee5eab66ca7fb6c59a41e45
+Author: zhaokun <jswps2011@163.com>
+Date:   Mon May 16 09:30:48 2016 +0800
+
+    rgw/rgw_common.cc: modify the end check in RGWHTTPArgs::sys_get
+    
+    Fixes: http://tracker.ceph.com/issues/16072
+    
+    Signed-off-by: zhao kun <jswps2011@163.com>
+    (cherry picked from commit 7e91c81702284f1eb68567fc5d9385ce22405e69)
+
+commit f16fa4b252b1601b00149f17ed1516bc90cbedf3
+Author: root <root@dlp-dev01.novalocal>
+Date:   Tue May 23 13:56:59 2017 +0800
+
+    rgw: multipart copy-part remove '/' for s3 java sdk request header.
+    
+    Fixes: http://tracker.ceph.com/issues/20075
+    Signed-off-by: donglinpeng@cmss.chinamobile.com
+    (cherry picked from commit d60ddddfaf6964364faed40f45b18eb88a841023)
+
+commit 244a2ae6a91e2f288cf1ef8c930dec712b588e37
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat Jun 3 12:33:36 2017 +0800
+
+    client: don't re-send interrupted flock request
+    
+    don't re-send interrupted flock request in cases of mds failover
+    and receiving request forward.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 044fabd669fdc8287f5cab0f38a82dfbfe41f0cd)
+    
+     Conflicts:
+            src/client/Client.cc: UserPerms not in jewel
+
+commit 93c8ca6450146cc2a3e50597d91f29455b42c742
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat Jun 3 12:06:10 2017 +0800
+
+    mds/flock: properly remove item from global_waiting_locks
+    
+    ceph_lock_state_t::remove_waiting() uses wrong key to search
+    global_waiting_locks. It should use item in waiting_locks as
+    key.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 0d71c6120e61f31b803c3fb6488fc7e97134e348)
+
+commit e795086776e9dfc8d9068161b6e96e79e8c8f193
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat Jun 3 12:01:27 2017 +0800
+
+    mds/flock: properly update ceph_lock_state_t::client_waiting_lock_counts
+    
+    no item is added to waiting_locks when deadlock check fails
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit a6b01939a199833568961ae4d30d8540a25a7740)
+
+commit 917240a291c6366b55ddab1a381e23e77b3264c1
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat Jun 3 11:42:12 2017 +0800
+
+    mds/flock: don't import ceph_lock_state_t::waiting_locks
+    
+    Item in waiting_locks is associated with flock mds request in
+    exporter mds. If client re-sends the flock mds request to the
+    importer, the importer starts a new MDRequest. The new one's
+    'flock_was_waiting' is false.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 8d777c4855b0c8e58a8d6966371698b8b7721a42)
+
+commit 644bb1e81ca29e244cf2962bc1c06bdecaac427a
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri Jul 7 12:37:55 2017 +0200
+
+    rpm: Fix undefined FIRST_ARG
+    
+    If FIRST_ARG is undefined, the rpms will show an error on upgrade
+    because the condition in the postun script gets broken.
+    
+    This was a regression introduced by commit for issue 20077 that moved
+    ceph-disk unit files to ceph-base.
+    
+    Fixes: http://tracker.ceph.com/issues/20077
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 562816914ccca8e4e7d9c31f333db2f0da6f7c99)
+
+commit e9608e0c4097caa3c2e344fc3456ac3d6c816d5e
+Author: Boris Ranto <branto@redhat.com>
+Date:   Mon Jun 5 18:44:18 2017 +0200
+
+    selinux: Install ceph-base before ceph-selinux
+    
+    We need to have ceph-base installed before ceph-selinux to use ceph-disk
+    in %post script. The default ordering is random and so the installation
+    randomly failed to relabel the files.
+    
+    Fixes: http://tracker.ceph.com/issues/20184
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit e69086645d3411a2ed781609b670eb5f16ac4810)
+    Conflicts:
+            ceph.spec.in: No _epoch_prefix in jewel
+
+commit 5c53ae3f9ac03445a99e84796f1b50ae32a365f2
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Jun 12 11:55:36 2017 +0800
+
+    test/librbd: decouple ceph_test_librbd_api from libceph-common
+    
+    Fixes: http://tracker.ceph.com/issues/20175
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b7287fdc4d70c5ecedda78ae367b98e5d8f61c5b)
+    
+    Conflicts: master has switched from autotools to cmake, so this is
+      somewhat an equivalent in automake to the cherry-picked change on
+      CMakeLists.txt
+
+commit 14fcdb7b699185e6e4ca1f158d34b47c63f6563b
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Jun 12 11:55:04 2017 +0800
+
+    test/librados: extract functions using libcommon in test.cc into test_common.cc
+    
+    Fixes: http://tracker.ceph.com/issues/20175
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    
+    (cherry picked from commit 82a848c2053ea69ebc6d3ec1003e18921e2d08d2)
+    
+    Conflicts: since master has switched to cmake. the changes in this
+      commit on .am files are ported from counterpart of CMakeLists.txt.
+
+commit 7d959e02f79ed3c268b03bf6fdf689cbe73063dc
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Jun 12 11:01:06 2017 +0800
+
+    test/librbd: replace libcommon classes using standard library
+    
+    so ceph_test_librbd_api has less dependencies on libcommon
+    
+    Fixes: http://tracker.ceph.com/issues/20175
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    
+    (cherry picked from commit bc8fa0193c8b2fb4b749ce9976bf6efbea833e96)
+
+commit 61f132078a535d10d10d2feb0688853855319ece
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jul 6 18:01:33 2017 +0200
+
+    tests: librbd: adapt test_mock_RefreshRequest for jewel
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 97fdaa753e812dfeb5256ced3af1dbe5adfff95d
+Author: Peng Xie <peng.hse@xtaotech.com>
+Date:   Thu Jun 29 18:48:27 2017 +0800
+
+    osd/PrimaryLogPG solve cache tier osd high memory consumption
+    
+    during cache tier dirty data flushing, cache tier osd memroy consumption
+    will get increasing due to the accumulative pg log which will not be trimmed
+    
+    Fixes: http://tracker.ceph.com/issues/20464
+    
+    Signed-off-by: Peng Xie <peng.hse@xtaotech.com>
+    (cherry picked from commit da605aa35e2d0897597a0ab6e19be70d94e6da24)
+    
+    Conflicts:
+            src/osd/PrimaryLogPG.cc
+                - should modify on ReplicatedPG.cc
+
+commit f49c9c777ffca31fa4c338dd22154ae6ad4e2ecf
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Jun 21 15:05:49 2017 -0700
+
+    osd/ReplicatedBackend: reset thread heartbeat after every omap entry in deep-scrub
+    
+    Doing this every 100 entries could be after 100MB of reads. There's
+    little cost to reset this, so remove the option for configuring it.
+    
+    This reduces the likelihood of crashing the osd due to too many omap
+    values on an object.
+    
+    Fixes: http://tracker.ceph.com/issues/20375
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 15ce60830aed7c4c2a0b10d2cb69d8daef418d20)
+    
+    Conflicts:
+            src/osd/ReplicatedBackend.cc
+                - remain the iterator checker (do not check status)
+
+commit 932342c1e3d704201dc4b4b42513c32a73ef0adf
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Jun 5 11:30:45 2017 +0100
+
+    client: avoid returning negative space available
+    
+    ...when a quota is set and the used bytes exceed
+    the quota.
+    
+    Fixes: http://tracker.ceph.com/issues/20178
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 0144fde1b1639a6917e487f78490f2b84dceeae6)
+
+commit 6b6620ffefc6481d629e19cd9a6c50b39dba222c
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 21 15:48:54 2017 +0800
+
+    mds: save  projected path into inode_t::stray_prior_path
+    
+    Otherwise, path string like #10000000000/xxx may get saved into
+    inode_t::stray_prior_path.
+    
+    Fixes: http://tracker.ceph.com/issues/20340
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit ba7472cbf2a3147136a31b611f7b88f883be5f6d)
+    
+    Conflicts:
+            src/mds/Server.cc
+              - remove counter `change_attr` because it does not involved
+
+commit dac0ca797cada404b4d5eeb29039b8eac84ce7d6
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jun 6 15:13:50 2017 -0400
+
+    ceph_test_rados_api_misc: fix LibRadosMiscConnectFailure.ConnectFailure retry
+    
+    Fixes: http://tracker.ceph.com/issues/19901
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 087dff80cac707ee9bcb5bcfc98cb3ec047bd49f)
+
+commit ab70ffc815e23e4bffa1e563ade21a0e1e84b8bf
+Author: Venky Shankar <vshankar@redhat.com>
+Date:   Mon Feb 20 12:04:10 2017 +0530
+
+    librbd: acquire exclusive-lock during copy on read
+    
+    Fixes: http://tracker.ceph.com/issues/18888
+    Signed-off-by: Venky Shankar <vshankar@redhat.com>
+    (cherry picked from commit 7dba5311b12011a4a6e8564e68150e54c5af5ddd)
+    
+    Conflicts:
+        src/librbd/AioImageRequestWQ.h:
+          - in master this file has morphed into src/librbd/io/ImageRequestWQ.h
+          - jewel has AioImageRequest<ImageCtx> instead of ImageRequest<ImageCtx>
+        src/librbd/image/RefreshRequest.cc:
+          - rename image context element to "aio_work_queue" (from "io_work_queue")
+            because jewel doesn't have de95d862f57b56738e04d77f2351622f83f17f4a
+        src/test/librbd/image/test_mock_RefreshRequest.cc:
+          - rename image context element to "aio_work_queue" (from "io_work_queue")
+            because jewel doesn't have de95d862f57b56738e04d77f2351622f83f17f4a
+
+commit 7bf8b854ab573016bba10fe2904b4d4f4e9b1ee1
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 2 15:03:57 2017 -0600
+
+    osd/osd_internal_types: wake snaptrimmer on put_read lock, too
+    
+    The snaptrimmer can block taking a write lock, which might happen due to
+    a conficting EC read.  When the EC read completes, we need to wake up the
+    snaptrimmer.
+    
+    Fixes: http://tracker.ceph.com/issues/19131
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 993179870ef7eece47cb9ccfb7275c29bf8536eb)
+    
+    Conflicts:
+            src/osd/osd_internal_types.h: ObjectContext::put_lock_type() is
+              defined in src/osd/osd_types.h in Jewel, patch that file
+
+commit 00a79995523ae3b9ed6753b645188cb6759c1d07
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jun 28 13:32:35 2017 -0400
+
+    rgw: rest handlers for mdlog and datalog list dont loop
+    
+    the do-while loops are unnecessary, because cls_log_list() will already
+    give us the requested number of entries, unless truncated. and these
+    rest operations return the truncated flag and last_marker, which the
+    client side is already using to run the same loop
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit d75e6b967d2ddf84d6e82c78a53061578129294a)
+
+commit e89036abb20d33df60c9a7dc7db4c8cd9825c068
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jun 28 12:19:52 2017 -0400
+
+    rgw: fix RGWMetadataLog::list_entries() for null last_marker
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit dd2893ba05c1095405b3a5edc16b3b1c7916cd7d)
+
+commit e5902f442bf87d7ed319f8e74b2f1e85791a39b9
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jun 28 12:18:33 2017 -0400
+
+    rgw: RGWMetadataLog::list_entries() no longer splices
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 112464483ecb407ef1388ba8a3f8d29cb0725594)
+
+commit 6f597f073fa2b4ad0941351606405d79d0899b67
+Author: Xie Rui <jerry.xr86@gmail.com>
+Date:   Thu Jun 22 17:12:30 2017 +0800
+
+    fix infinite loop in rest api for log list
+    
+    as max_entries_str is empty, it will be infinite loop when log is more than MAX_ENTRIES(1000) from marker.
+    and max_entries maybe overflow if use it to count the left entries.
+    
+    Fixes: http://tracker.ceph.com/issues/20386
+    Signed-off-by: xierui <jerry.xr86@gmail.com>
+    (cherry picked from commit 9fbc5378d2737d38e2ac915f20b44d308e18bc68)
+
+commit 3fa277b479d69699bf5a6875cd4a5efcf9ae0788
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Tue Jun 27 16:07:01 2017 +0400
+
+    jewel: osd: unlock sdata_op_ordering_lock with sdata_lock hold to avoid missing wakeup signal
+    
+    Based on commit bc683385819146f3f6f096ceec97e1226a3cd237. The OSD code has
+    been refactored a lot since Jewel, hence cherry-picking that patch introduces
+    a lot of unrelated changes, and is much more difficult than reusing the idea.
+    
+    Fixes: http://tracker.ceph.com/issues/20428
+    
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+
+commit a36211c84d8c475c88820f33a467936e0b270420
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Wed May 24 16:06:07 2017 +0800
+
+    rgw: update bucket cors in secondary zonegroup should forward to master
+    
+    pr#14082 is incomplete, cors should be redirect to master
+    
+    Fixes: http://tracker.ceph.com/issues/16888
+    
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit 7dd099588a47024b410d513cb8ea919731718fa7)
+    
+    Conflicts:
+            src/rgw/rgw_op.cc (jewel does not have 5fff6371d8e)
+            src/test/rgw/rgw_multi/tests.py (jewel does not have this file)
+
+commit fa419a87b64f7c88a302ac594af81e07eb0ea430
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Apr 6 13:13:54 2017 -0400
+
+    rgw: fix for EINVAL errors on forwarded bucket put_acl requests
+    
+    PutACL rejects request that include both a canned_acl and a request
+    body. when forwarding requests with canned_acls, we were also including
+    the generated policy in the body
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 1c863a78ed73cf1d89f05276627c7f0bde53ec9a)
+
+commit 1d7bd61d3f5f07a3f6d30faa9325f93c89800ade
+Author: Guo Zhandong <guozhandong@cmss.chinamobile.com>
+Date:   Wed Mar 22 18:00:37 2017 +0800
+
+    rgw: enable to update acl of bucket created in slave zonegroup
+    
+    Fixes: http://tracker.ceph.com/issues/16888
+    
+    Signed-off-by: Guo Zhandong <guozhandong@cmss.chinamobile.com>
+    (cherry picked from commit 5f89d37506adab0aa59e7a8a46eb962030565c72)
+
+commit a3c487448bbded31c98e752ff1c9518aa76c2e67
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue May 9 12:32:51 2017 +0200
+
+    ceph-disk: separate ceph-osd --check-needs-* logs
+    
+    It is using the OSD id zero but have nothing to do with OSD zero and
+    this is confusing to the user. The log themselves do not need to be kept
+    around and are stored in the run directory so that they can be disposed
+    of after reboot.
+    
+    Fixes: http://tracker.ceph.com/issues/19888
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit c7b3c46bd63b78475868e405bf20d9c142f0336a)
+
+commit a4c181fa7ab6ced676b9ff4f1381555215d77152
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu Mar 16 21:45:46 2017 -0400
+
+    mds: set ceph-mds name uncond for external tools
+    
+    External tools like logrotate which use killall rely on the "ceph-mds" name to
+    find it in /proc/*/comm. This is normally the case but when ceph-mds respawns
+    using /proc/self/exe (on Linux), its name will change to "exe". This makes
+    logrotate fail to signal ceph-mds the log needs to be reopened and will lead to
+    the log growing until the disk is full.
+    
+    This patch unconditionally sets the name so on respawn the name is correct (and
+    in any other possible scenario, such as a misnamed executable).  Note, there is
+    still a very small race between execve and ceph_pthread_setname where the
+    process name is wrong.
+    
+    Problem was introduced by 66a122025f6cf023cf7b2f3d8fbe4964fb7568a7.
+    
+    Fixes: http://tracker.ceph.com/issues/19291
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 4f177bb6b72cf9c8eb363051b27496c026b345f0)
+    
+    Conflicts:
+        src/ceph_mds.cc - use pthread_setname_np() instead of
+            ceph_pthread_setname(), drop compat.h include
+
+commit 3776d960e07570727cea9c6ef8516ffa9953b54e
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Sep 29 17:14:54 2016 +0100
+
+    suites: update log whitelist for scrub msg
+    
+    Fixes: http://tracker.ceph.com/issues/16016
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 795094628b65eb9c3f5d51c6c895fe1443c5f4cf)
+    
+    Conflicts:
+            suites/fs/recovery/tasks/forward-scrub.yaml (file does not exist in jewel)
+
+commit d9234d47a9abc290475a317bb3fd070192df1840
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Mar 23 09:07:32 2017 -0400
+
+    mds: include advisory `path` field in damage
+    
+    This will just be whatever path we were looking
+    at at the point that damage was notified -- no
+    intention whatsoever of providing any up to date
+    path or resolution when there are multiple paths
+    to an inode.
+    
+    Fixes: http://tracker.ceph.com/issues/18509
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit c0bff51ef409eb6e4b2fc248e06e5a7e43faf51e)
+    
+    Conflicts:
+        src/mds/CDir.cc - omit dout(10) because jewel does not have cb86740a
+        src/mds/ScrubStack.cc - jewel does not have 7b456109 which changed
+           in->make_path_string_projected() call to in->make_path_string() but
+           it's moot because that line is dropped
+
+commit 7675e41c88a38e107e9478bde32e0ece564f8878
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 19 20:25:58 2016 +0100
+
+    mds: populate DamageTable from scrub and log more quietly
+    
+    Fixes: http://tracker.ceph.com/issues/16016
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 9c82040b1a762a3498c6794a29a43b3866f30dab)
+
+commit 88605fc76bc943697ca5b4e235d82f439766d14c
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 19 18:26:42 2016 +0100
+
+    mds: tidy up ScrubHeader
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 111d2cf2d8504cd4486180a95e52f253018364b3)
+    
+    Conflicts:
+            src/mds/CInode.cc (jewel does not have 5259683e7819c22c14b21b1dd678a33e14574f21)
+
+commit c15fce6e50eb5775687feb121da5ed59df34e28b
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 19 17:35:54 2016 +0100
+
+    mds: remove redundant checks for null ScrubHeader
+    
+    This was originally optional but now all the paths
+    that kick off a scrub should be going through
+    enqueue_scrub and thereby getting a header set.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 0c890282699f877f42870408d674ec1e9f9322a3)
+    
+    Conflicts:
+            src/mds/CInode.cc (jewel does not have 5259683e7819c22c14b21b1dd678a33e14574f21)
+
+commit c17e3c1b7b6961f3fb866af76aa2d798cdb97c9f
+Author: Michal Jarzabek <stiopa@gmail.com>
+Date:   Tue Oct 11 21:23:57 2016 +0100
+
+    mds/DamageTable: move classes to .cc file
+    
+    Signed-off-by: Michal Jarzabek <stiopa@gmail.com>
+    (cherry picked from commit 96018b0a85d0bc7eec285398dd596ee01d16fae7)
+
+commit 52a00b3bffc2db22c15d0e5b1e9580f9bc593555
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Sep 16 10:56:51 2016 +0200
+
+    test: timeout verification that mon is unreachable
+    
+    Without a timeout on the command, it may hang for a very long time,
+    hunting for new mons. If it hangs for more than 60 seconds, it is
+    safe to assume the mon is indeed down.
+    
+    Fixes: http://tracker.ceph.com/issues/16477
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit a5e5119bd138a97f35737c2b50d1e621fa8286d6)
+
+commit 3b5f9924b3db66fae952cf67331b69789045916f
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Sep 15 10:42:24 2016 +0200
+
+    cli: retry when the mon is not configured
+    
+    The mon may return on error if a command is sent to it while it is
+    still in configuring state.
+    
+    Fixes: http://tracker.ceph.com/issues/16477
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit d5a3c8d6844ba632767baf3c2c790fa4d947a95e)
+    
+    Conflicts:
+            src/pybind/ceph_argparse.py (trivial resolution; jewel does not have
+                8fc67075c6d7d4443747f53687e498439f80b57a)
+
+commit 00ce80b52f6f231125cc165ab477544238ed3a02
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue May 9 11:42:58 2017 +0200
+
+    build/ops: rpm: fix python-Sphinx package name for SUSE
+    
+    This commit moves "BuildRequires: python-sphinx" down to the RH/CentOS/Fedora
+    distro conditional and adds a "BuildRequires: python-Sphinx" to the SUSE
+    conditional.
+    
+    Signed-off-by: Jan Matejek <jmatejek@suse.com>
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 0688f57d1c35692cdddf219d9ac3571d03dbb8e0)
+
+commit 66c17ff10b1703d0d9098d0ddcaf34a644ff5fbe
+Author: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+Date:   Tue Jun 14 15:34:08 2016 +0800
+
+    rgw: remove unnecessary output
+    
+    a lot of radosgw-admin command will output
+    "error in read_id for object name: default : (2) No such file or directory"
+    when the zone named 'default' is not exist:
+    
+    Signed-off-by: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+    (cherry picked from commit 4e9c7be08b9828db2795536ce170a675d25bc1ed)
+
+commit b786f7815b94e7659a07778a6318ffe5e5079fc0
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Mon Mar 20 15:19:22 2017 +0800
+
+    rgw: delete non-empty buckets in slave zonegroup returns error but the
+    buckets have actually been deleted.
+    
+    Fixes: http://tracker.ceph.com/issues/19313
+    
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit 4714b17f53a9f30f7f155a69cfbfc682f809b4e4)
+    
+    Conflicts:
+            src/rgw/rgw_op.cc - first argument to delete_bucket() and
+                check_bucket_empty() is "s->bucket" in jewel (master:
+                "s->bucket_info")
+            src/rgw/rgw_rados.cc - RGWRados::delete_bucket() takes bucket instead
+                of bucket_info; adapt RGWRados::check_bucket_empty() to take bucket
+                instead of bucket_info as well
+            src/rgw/rgw_rados.h - ditto
+
+commit 7e3db1519e40ecdde5d7e83798a35f3fca7e5cd8
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Apr 4 10:42:44 2017 -0400
+
+    rgw: fix for zonegroup redirect url
+    
+    local dest_url variable was shadowing the one in the enclosing scope, so
+    the changes were not applied and no Location header was written on redirect
+    
+    Fixes: http://tracker.ceph.com/issues/19488
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 542e188a40f0495720b48308372366951ae41e62)
+
+commit 45f0396cffe528a752fb8330d2323e1de8d8b8ac
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Apr 4 10:41:51 2017 -0400
+
+    rgw: use zonegroup's master zone endpoints for bucket redirect
+    
+    if no zonegroup endpoints are set, fall back to master zone endpoints
+    
+    Fixes: http://tracker.ceph.com/issues/19488
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 92f63c6392bdc4633a2e57cb3867051bb1a3fd55)
+
+commit 50f140768d56d7813efac7ab2def3619e9639b4c
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Tue Jan 24 15:20:32 2017 +0800
+
+    rgw: put object's acl can't work well on the latest object when versioning
+    is enabled.
+    
+    Reported-by: Liu Hong <liuhong@cmss.chinamobile.com>
+    
+    Fixes: http://tracker.ceph.com/issues/18649
+    
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit fe5f95fff30cc872484de6efb6cba1dd48b09316)
+    
+    Conflicts:
+         src/rgw/rgw_op.cc: read_op.prepare() takes no arguments in master; replace
+             this with read_op.prepare(NULL, NULL) to make jewel happy
+
+commit 01890abd953881ab1529652d148f7a19d171e18c
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Apr 21 15:04:48 2017 -0400
+
+    radosgw-admin: warn that 'realm rename' does not update other clusters
+    
+    Fixes: http://tracker.ceph.com/issues/19746
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 8a459c768ffecd689a53a79dfe33eb8f1bbc318f)
+
+commit 89a2f6476d81bd233d76771915e080fce66ff228
+Author: Willem Jan Withagen <wjw@digiware.nl>
+Date:   Sat Apr 29 13:36:07 2017 +0200
+
+    cls/log/cls_log.cc: reduce logging noise
+    
+     - The other reference in the source as already at 20.
+          ./src/cls/timeindex/cls_timeindex.cc:85:
+            CLS_LOG(20, "storing entry at %s", index.c_str());
+    
+       And we need not always know where in the log items are stored.
+       So it looks like a leftover debug feature.
+    
+    Fixes: http://tracker.ceph.com/issues/19835
+    Signed-off-by: Willem Jan Withagen <wjw@digiware.nl>
+    (cherry picked from commit d76010900bf9012f2e66335787710531772766b7)
+
+commit 938e5b0728c07997abf09d30c670ffe1737a65ab
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon May 15 17:30:29 2017 -0400
+
+    rgw_file: v3: fix write-timer action
+    
+    For now, unify with v4 write-on-close path, by calling
+    RGWFileHandle::close() on write-timer expire, since it will
+    call write_finish() as a side-effect.
+    
+    Fixes: http://tracker.ceph.com/issues/19932
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit ce6ecb553b85ea158af28c22827b93135a75d159)
+
+commit 0897afe74d381c19a9b1bc1c2b398f93568a86fe
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Mar 7 09:48:57 2017 -0500
+
+    rgw_file:  fix fs_inst progression
+    
+    Reported by Gui Hecheng<guimark@126.com>.  This change is a
+    variation on proposed fix by Dan Gryniewicz<dang@redhat.com>
+    to take root_fh.state.dev as fs_inst for new handles.
+    
+    Fixes: http://tracker.ceph.com/issues/19214
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 0e988edfb6ab9085d6d37cfc444d46e8a2841943)
+
+commit a54e6ccba31109d59abd8e7b9ba1f8fada30ecda
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon May 22 17:37:11 2017 -0400
+
+    rgw_file:  remove post-unlink lookup check
+    
+    This could induce asserts in multi-nfs-gateway scenarios.
+    
+    Fixes: http://tracker.ceph.com/issues/20047
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit b3db617ddd6fa2477726b54828b6410b36989ac7)
+
+commit 7042733db61e8a773969d3ce5fc24a1df6ae417b
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue May 23 15:05:45 2017 -0400
+
+    rgw_file: release rgw_fh lock and ref on ENOTEMPTY
+    
+    An early return in new unlink bucket num_entries check was
+    missing a conditional unlock and unref.
+    
+    Fixes: http://tracker.ceph.com/issues/20061
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 51c25593dd741b4df2e839fcef82b143f8c8cfca)
+
+commit 4a32ef856554921dab503d70257bd4922159831d
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon May 22 14:51:19 2017 -0400
+
+    rgw_file: remove hidden uxattr objects from buckets on delete
+    
+    If a setattr (e.g., chown) has been performed on a bucket, then like
+    any directory it has a hidden object storing its attributes. This must
+    be deleted before attempting bucket delete, otherwise, actually empty
+    buckets will not be removable via NFS.
+    
+    Fixes: http://tracker.ceph.com/issues/20045
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 01e15c3ec6cf43a7a7c4e2d13aad5f385c9f9748)
+    
+    Conflicts:
+            src/rgw/rgw_file.cc (jewel does not have
+                    "inline int valid_fs_bucket_name()" and
+                    "inline int valid_fs_object_name()")
+
+commit b63b7886dc1fb704f8baa9749a20e09faa93c1d1
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Apr 24 16:23:21 2017 +0200
+
+    test/librbd/test_notify.py: don't disable feature in slave
+    
+    On jewel it will have stolen the exclusive lock. Instead, ensure that
+    object map and fast diff are already disabled on the clone before the
+    start of the test.
+    
+    Fixes: http://tracker.ceph.com/issues/19716
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit e009e1bdd4b3997462feb9a050bd2eb201e028ba)
+
+commit 5c0986153e79d7ec4bc165e497689319bd95ada9
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Fri May 5 15:59:44 2017 +0200
+
+    rbd-nbd: relax size check for newer kernel versions
+    
+    Fixes: http://tracker.ceph.com/issues/19871
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 8e912b9a0564a57f1b761e9e567771197bd0fb98)
+
+commit bc8c5d6c77880d21248c683609c3814145008f36
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue May 23 12:07:45 2017 +0200
+
+    librbd: potential read IO hang when image is flattened
+    
+    Fixes: http://tracker.ceph.com/issues/19832
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from 10d58618e7c632ef01b9537492239e0a042dc17e)
+    
+    Conflicts:
+        src/librbd/AioObjectRequest.cc: manual backport because the master changes
+            are in a different file (src/librbd/io/ObjectRequest.cc)
+
+commit 2e278d821d1e5992f77e6ac1f0b3e1c7fcba7e12
+Author: redickwang <redickwang@tencent.com>
+Date:   Fri May 19 15:08:12 2017 +0800
+
+    rgw: segment fault when shard id out of range
+    
+    Fixes: http://tracker.ceph.com/issues/19732
+    
+    Signed-off-by: redickwang <redickwang@tencent.com>
+    (cherry picked from commit ff4c40fc2e3c092d17a218ae6132de4e6034c8ee)
+
+commit 5adc66bc2abe5b8d58111d52b23619edfbebbc7f
+Author: Thomas Serlin <tserlin@redhat.com>
+Date:   Wed May 24 12:07:41 2017 -0400
+
+    Set subman cron attributes in spec file
+    
+    Fixes: http://tracker.ceph.com/issues/20074
+    
+    Signed-off-by: Thomas Serlin <tserlin@redhat.com>
+    (cherry picked from commit a9eb6ecea7b9512722e2283e5acb4fc6a1b7b734)
+
+commit 21b00c3ec0ae60848f6677b5fe23e6321b0f8de8
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 30 13:50:41 2017 -0400
+
+    osd/PrimaryLogPG: do not expect FULL_TRY ops to get resent
+    
+    The objecter will not resend FULL_TRY requests that were sent prior to
+    becoming full, so we should not discard them.
+    
+    Fixes: http://tracker.ceph.com/issues/19430
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3f7acdbc9a942fd18937dbcf07fbc7b752c50ba3)
+
+commit afd036cb6725f9e91cfd38995d6536d92deb934d
+Author: lvshuhua <lvshuhua@cmss.chinamobile.com>
+Date:   Wed Dec 7 15:47:47 2016 +0800
+
+    rgw: fix versioned bucket data sync fail when upload is busy
+    
+    Fixes: http://tracker.ceph.com/issues/18208
+    
+    Signed-off-by: lvshuhua <lvshuhua@cmss.chinamobile.com>
+    (cherry picked from commit ce7d00ac1642d84c1d9111156a544c37801c5adf)
+    
+    Conflicts:
+            src/rgw/rgw_data_sync.cc (in jewel, inc_marker is declared to be of
+                type rgw_bucket_shard_inc_sync_marker, while in master it is of
+                type rgw_bucket_shard_inc_sync_marker& - i.e. reference to
+                rgw_bucket_shard_inc_sync_marker)
+
+commit c2137f2f725d8536066b6b495f8565b126a720ed
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Jun 19 13:51:50 2017 +0200
+
+    libradosstriper: delete striped objects of zero length
+    
+    This patch fixes the bug: "libradosstriper fails to delete striped objects of
+    zero length without returning any kind of error"
+    
+    N.B.: This is not cherry picked from master because
+    7a50ea479e7e5c2909d899d89d33d3fb082257f8, which fixes the issue in master, is
+    too complicated to backport.
+    
+    Fixes: http://tracker.ceph.com/issues/20325
+    Signed-off-by: Stan K <redrampage@selectel.ru>
+
+commit fb79932be69545b286497d0ee24c96d5e466344b
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Apr 18 13:54:56 2017 -0400
+
+    mon/PGMonitor: clean up min/max span warning
+    
+    Clean up option naming.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 489e810c37ed6fb9d32d1015634041a577501ee4)
+    
+    Conflicts:
+            src/common/config_opts.h: Jewel's options are quite different from ones of master,
+              s/mon_warn_osd_usage_percent/mon_warn_osd_usage_min_max_delta/
+
+commit 8180f1612a3dfac6ba5e8914f9e6a3faa8cf0297
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 16 22:23:06 2017 -0800
+
+    osd: Round fullness in message to correspond to df -h
+    
+    This really only works after journal drains because
+    we adjust for the journal.
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 26dcb591f9af01ed444aa758c3d601e7f67261b2)
+
+commit 728097af06e1e49b5f02111887ce6716d3729213
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 16 17:25:12 2017 -0800
+
+    filestore: Account for dirty journal data in statfs
+    
+    Fixes: http://tracker.ceph.com/issues/16878
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 78adb70c21c6b8e6a9191b76917919b125a9490f)
+    
+    Conflicts:
+            src/os/filestore/FileStore.cc: adapt the code to use struct
+              statfs to keep changes minimal
+
+commit c6e4fff5e3f5c5a45b3c4fb647548bd2a2dfea0c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Feb 14 16:37:07 2017 -0800
+
+    mon: Add warning if diff in OSD usage > config mon_warn_osd_usage_percent (10%)
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit c8004e6558359fb542e45bb4b483a6c91afdc0b4)
+    
+    Conflicts:
+            src/common/config_opts.h: some options in the original patch hunk
+              are not supported in Jewel, manually add the new `mon_warn_osd_usage_percent'
+              option.
+
+commit 0412bf2adaf07e527ac6659c9bce0d406edeca93
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Feb 14 14:40:05 2017 -0800
+
+    mon: Bump min in ratio to 75%
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 830cc7aa7be1ccd8f54f056b6a58e923cadd1c2b)
+
+commit 21e2a0337092d57f3e23d282b02b99146ed50a63
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Feb 14 14:38:53 2017 -0800
+
+    osd: Fix ENOSPC crash message text
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 50cfe03fcba253c8380b21043ed03879134d6836)
+    
+    Conflicts:
+            src/os/bluestore/BlueStore.cc: leave bluestore code as is,
+              bluestore users should use Kraken or Luminous
+
+commit 0068d9f3335fc8d6b7bfb5c29aa8cba87c0d01f2
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Tue Jun 13 10:19:55 2017 +0800
+
+    msg/async: go to open new session when existing already closed
+    
+    Fixes: http://tracker.ceph.com/issues/20230
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 99f580a3959240f99061a9ad48ec591b39a9fd46)
+    
+    Changes: src/msg/async/AsyncConnection.cc: Mutex::Unlock() in Jewel
+               versus mutex::unlock() in master
+
+commit 28aa104c0c8f9d6c556aeab881231034191b55d1
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Tue Jun 13 10:16:47 2017 +0800
+
+    msg/async: fix accept_conn not remove entry in conns when lazy delete
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit bf98babb3289a7714543ff3cbd3872d80f0dc196)
+
+commit 9212d62007394d9219a1c629d875610aa9aeab4b
+Author: Michal Jarzabek <stiopa@gmail.com>
+Date:   Sun Jun 19 16:01:31 2016 +0100
+
+    msg/AsyncMessenger.h:remove unneeded use of count
+    
+    Signed-off-by: Michal Jarzabek <stiopa@gmail.com>
+    (cherry picked from commit b9ccc756614870142ba878ffdf82c7c7c89e49a2)
+
+commit 8cf3f18c7af3a096da4f2e40f6c273c3e39c7e8f
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed May 3 11:32:34 2017 -0400
+
+    rgw: RGWPeriodPusher spawns http thread before cr thread
+    
+    Fixes: http://tracker.ceph.com/issues/19834
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 21a943da45032a276997a0a885e869e2c2bc321d)
+
+commit 730c88e800ceedea51c92d9f3a59aeba709b692d
+Author: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+Date:   Mon Jun 27 19:40:13 2016 +0800
+
+    rgw: should delete in_stream_req if conn->get_obj(...) return not zero value
+    
+    Signed-off-by: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+    (cherry picked from commit 494fbe9ce595b0b59597bd426cc3bc4d14ba6644)
+
+commit 9b12fe08098a686afe96f7965b24ee331c7902ab
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Apr 20 16:33:32 2017 -0400
+
+    rgw: dont spawn error_repo until lease is acquired
+    
+    if RGWDataSyncShardCR fails to acquire its lease, it doesn't call
+    error_repo->finish() to stop the coroutine. wait until the lease
+    succeeds before spawning the error_repo
+    
+    Fixes: http://tracker.ceph.com/issues/19446
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 1524a5eb249113fed8da1d47a4f92dec98db4566)
+    
+    Conflicts:
+            src/rgw/rgw_data_sync.cc (different argument list of RGWOmapAppend()
+                function in jewel, compared to master)
+
+commit 6b99bc9c6083190e65825f2fd4a2d18e7f03a4ec
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jun 2 09:45:06 2017 +0200
+
+    build/ops: deb: fix logrotate packaging
+    
+    This minimal jewel-only fix is not cherry-picked from master because
+    the Debian packaging was refactored between jewel and master and the
+    master fix is totally different.
+    
+    Fixes: http://tracker.ceph.com/issues/20316
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 4be3db597321fd61caa202af49f8c2d941db552d
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Thu Jan 5 11:50:42 2017 +0800
+
+    rgw: fix 'gc list --include-all' command infinite loop the first 1000 items
+    
+    When the items to gc over 1000, 'gc list --include-all' command will infinite loop the first 1000 items.
+    Add next_marker to move to the next 1000 items.
+    
+    Fixes: http://tracker.ceph.com/issues/19978
+    
+    Signed-off-by: fang yuxiang <fang.yuxiang@eisoo.com>
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit fc29f52ebca63104a05515484088ff136dfb0b15)
+
+commit 7966b846a875c60b5272f32baec648a719a10b9e
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jun 7 12:58:05 2017 -0400
+
+    test/rgw: wait for realm reload after set_master_zone
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 5cc99db52423f9ee98a549b215559c677f178f48)
+
+commit 97ffe49defb213c26288744ed572fbb5b158583a
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jun 7 12:56:20 2017 -0400
+
+    test/rgw: fixes for test_multi_period_incremental_sync()
+    
+    test was only creating objects in subsequent periods, which wasn't
+    adding any entries to the mdlog. this wasn't correctly testing
+    incremental metadata sync across periods
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 2976fd36cded95f68766f8ba485d43f932e97db2)
+
+commit 7c19e375be379cd8366097b57adea9f6010d783c
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jun 7 12:00:15 2017 -0400
+
+    test/rgw: meta checkpoint compares realm epoch
+    
+    avoid marker comparisons between different periods
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 20df35ad94aea0c03be0c2c9739bd239354a46d8)
+
+commit cb957226083f7644db5b5826a3d7ab21c47def8e
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Mar 21 12:19:01 2017 -0400
+
+    rgw: remove rgw_realm_reconfigure_delay
+    
+    when the master zone is changed, this config variable was increasing the
+    window of time where the old master zone would continue to handle
+    requests to modify metadata. those changes would not be reflected by the
+    new metadata master zone, and would be lost to the cluster
+    
+    it was an attempt to optimize for the unlikely case of multiple period
+    changes in a short period of time, but the logic in reload() handles this
+    case correctly as is
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit f422d4f1841e15a4ecf2d9304aa77021e8bd8626)
+
+commit 48d596df141f5014ae5ca9ab4e8955a6c6ba755e
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Mar 21 16:10:27 2017 -0400
+
+    rgw: require --yes-i-really-mean-it to promote zone with stale metadata
+    
+    if a zone is promoted to master before it has a chance to sync from the
+    previous master zone, any metadata entries after its sync position will
+    be lost
+    
+    print an error if 'period commit' is trying to promote a zone that is
+    more than one period behind the current master, and only allow the
+    commit to proceed if the --yes-i-really-mean-it flag is provided
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 721e3d6ee5917b19cfc15e3e9582d23623b8cca7)
+
+commit 81ced03d203ec00376fef5a143e4cac24aa65093
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Mar 20 16:13:03 2017 -0400
+
+    rgw: period commit uses sync status markers
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit bb49e2fbed3a87de837329cfa0c11f8d97633a94)
+    
+    Conflicts:
+            src/rgw/rgw_rados.cc: RGWPeriod::update removed
+
+commit a9830dbdd43572995b3e8e157ecbc706d6481fac
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Aug 1 15:21:52 2016 -0400
+
+    rgw: use RGWShardCollectCR for RGWReadSyncStatusCoroutine
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit a4bf014b8642073f3eac226a93f6360cdd9cee25)
+    
+    Conflicts:
+            src/rgw/rgw_sync.cc: rgw_pool, rgw_raw_obj
+
+commit 0ab8e0ab5db988724bb0e26c88a279d099d3b8cd
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Aug 1 14:35:53 2016 -0400
+
+    rgw: change metadata read_sync_status interface
+    
+    makes the same change to read_sync_status() in RGWMetaSyncStatusManager,
+    needed to support multiple concurrent readers for the rest interface
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 0372df20be86cb616600626ee4c755b31032f134)
+
+commit ac9045abba05818108655c75ecb57cdf065f56e8
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Mar 17 09:55:47 2017 -0400
+
+    rgw: store realm epoch with sync status markers
+    
+    sync status markers can't be compared between periods, so we need to
+    record the current period's realm epoch with its markers. when the
+    rgw_meta_sync_info.realm_epoch is more recent than the marker's
+    realm_epoch, we must treat the marker as empty
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 574ff5f5df62a0a1e9a8ff7e4a87cd65cc952d7e)
+    
+    Conflicts:
+            src/rgw/rgw_sync.cc: rgw_pool
+
+commit 013d349c791318a03cf7bd0bbbe6d4bec708359d
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Feb 27 15:19:54 2017 -0500
+
+    rgw: RGWBackoffControlCR only retries until success
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit a898fb76a4179add68ccb526f2917768736ac52b)
+
+commit 237368752b013ad639581b506fdb46e2119f155a
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Mar 7 12:28:33 2017 -0500
+
+    rgw: clean up RGWInitDataSyncStatusCoroutine
+    
+    RGWInitDataSyncStatusCoroutine operates on a given rgw_data_sync_status
+    pointer, which saves us from having to read it back from rados
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 69be410d0e29cd8b4cf5171800e4a7fa938eb8c6)
+    
+    Conflicts:
+            src/rgw/rgw_data_sync.cc: rgw_pool, rgw_raw_obj
+
+commit 67011e51a7ba1538ad2c5f426311ae71bb5f8c34
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Mar 7 11:16:43 2017 -0500
+
+    rgw: fix marker comparison to detect end of mdlog period
+    
+    Fixes: http://tracker.ceph.com/issues/18639
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 7c23713b1a3c14b135f8eb6fdb36d113ea860e4e)
+
+commit c19c72058e9a09ccfc316ebe0c53348be1e2ba32
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Aug 22 22:06:15 2016 -0400
+
+    rgw: add == and != operators for period history cursor
+    
+    RGWMetaSyncCR was using operator== but it always returned true!
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 09c847ff790f55004871fb7304361ae6c9845b1a)
+
+commit 1c344e254efcd802e417cc45b6678989c871ad0a
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jun 8 09:37:26 2016 -0400
+
+    rgw: add empty_on_enoent flag to RGWSimpleRadosReadCR
+    
+    RGWSimpleRadosReadCR won't currently fail with ENOENT, but instead
+    passes an empty object to handle_data(). add an empty_on_enoent flag to
+    the constructor, defaulting to true, to make this behavior optional for
+    callers that do want to fail on ENOENT
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit c5c95e7f225d59a8bdd8eda3742053b77492c40c)
+
+commit bf973c8400db08dd541152812d42379cf113755d
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri May 26 09:52:25 2017 +0200
+
+    rpm: Move ceph-disk to ceph-base
+    
+    The SELinux package now requires the ceph-disk binary but that one was
+    part of the ceph-osd package. The ceph-disk python library is already
+    packaged in ceph-base so moving ceph-disk to ceph-base seems like a
+    reasonable next step.
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 6991764f3bff7b8f6be699603927aff882377878)
+    
+    Conflicts:
+            ceph.spec.in: we have ceph-disk-udev in jewel
+            ceph.spec.in: few if conditions were different in jewel
+
+commit 6ca61b1b478f5dbbd33ddf87e0fc1af999ed75cb
+Author: Boris Ranto <branto@redhat.com>
+Date:   Thu May 25 14:36:13 2017 +0200
+
+    ceph-disk: Fix the file ownership, skip missing
+    
+    This commit fixes the file ownership for the /usr/bin/ and /etc/ceph
+    files and skips missing files as some of the files that we do specify
+    now can be missing from the system (not installed, e.f. radosgw).
+    
+    Fixes: http://tracker.ceph.com/issues/20077
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 077038b4393a28ccbd38ca4a90105dbd4c1ffcd5)
+
+commit 7d6eea0440f9856894570af80a14ba6c19a3c910
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri Apr 28 12:29:46 2017 +0200
+
+    selinux: Do parallel relabel on package install
+    
+    We can take advantage of ceph-disk fix subcommand when doing a package
+    install. We will keep using the differential fixfiles command otherwise.
+    
+    We also need to add relabel for /usr/bin/ daemons so that we could use
+    this.
+    
+    Fixes: http://tracker.ceph.com/issues/20077
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 1cecddf031991f1c64ea203f173189624f11940e)
+
+commit b9ce1aa618d1f488f91f349bed8c87a421fcd5ba
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 3 21:36:21 2017 -0400
+
+    rbd-mirror: ensure missing images are re-synced when detected
+    
+    Fixes: http://tracker.ceph.com/issues/19811
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 74bd4f230a0cb7b709f2cb5c6db3dc79f0d8dede)
+    
+    Conflicts:
+            src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc: trivial resolution
+            src/tools/rbd_mirror/image_replayer/BootstrapRequest.h: trivial resolution
+
+commit 648dfa1e939fa65ef629b7e4a31f2eda2b0b55c0
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Jun 5 15:40:10 2017 +0800
+
+    qa: add a sleep after restarting osd before "tell"ing it
+    
+    without the fast-fail feature, the monitor does not mark osd down after
+    a grace time. so we cannot truest the "healthy()" in ceph.restart task.
+    
+    also, "restart" task wait-for-healthy by default, so no need to do it
+    explicitly.
+    
+    Fixes: http://tracker.ceph.com/issues/16239
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    Conflicts: this change is not cherry-picked from master. because, in
+               master we have the fast-fail feature, but we have'not
+               backport it to jewel. so adding a timeout would help to
+               make sure that the OSD is back online even "restart"
+               waits for "healthy" already.
+
+commit 01abfed8f26eab70dc3242aeaa4c44599f542af2
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Thu May 25 22:32:01 2017 -0700
+
+    osd: do not default-abort on leaked pg refs
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 4caf2df0c380a1281db9509b3feb342705512b58)
+    
+    Conflicts:
+            qa/clusters/fixed-4.yaml
+            src/osd/OSD.cc
+    
+    Fixes: http://tracker.ceph.com/issues/20084
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 967d6a54055b438d6730ab7b7c1fc1a1b980c5e2
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Thu May 25 21:52:49 2017 -0700
+
+    osd: Reset() the snaptrimmer on shutdown
+    
+    We were failing to exit various wait states which held PGRefs. Error!
+    
+    Fixes: http://tracker.ceph.com/issues/19931
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit b0e9deeea8a8e90f6d7e9d56b6b4aed890e01d7b)
+    
+    Conflicts:
+            src/osd/ReplicatedPG.cc
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 328e2cc71aaefa07a04381a7f752dac46b566da0
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Feb 3 10:08:33 2017 -0500
+
+    mon/PGMap: factor mon_osd_full_ratio into MAX AVAIL calc
+    
+    If we only fill OSDs to 95%, we should factor that into
+    the MAX AVAIL calculation for the pool.
+    
+    Fixes: http://tracker.ceph.com/issues/18522
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f223ac92917f4bc18e5b9b3ad61afa155e4d088a)
+    
+    Conflicts:
+            src/mon/PGMap.cc: get_rule_avail() is a method of the PGMonitor
+              class in Jewel (and osd_stat is pg_map.osd_stat)
+
+commit b52bfe6b443f0ff88c8614441752102058063699
+Author: Ning Yao <yaoning@unitedstack.com>
+Date:   Thu Apr 6 11:12:04 2017 +0000
+
+    os/filestore: fix infinit loops in fiemap()
+    
+    since fiemap can get extents based on offset --> len
+    but we should consider last extents is retrieved when len == 0
+    even though it is not last fiemap extents
+    
+    Signed-off-by: Ning Yao <yaoning@unitedstack.com>
+    (cherry picked from commit 36f6b668a8910d76847674086cbc86910c78faee)
+
+commit e3cba06bd02b1fca07414c987cf147f1ded99b68
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri May 5 12:02:05 2017 +0800
+
+    mon: check is_shutdown() in timer callbacks
+    
+    introduce a helper class: C_MonContext, and initialize all timer events
+    using it, to ensure that they do check is_shutdown() before doing their
+    work.
+    
+    Fixes: http://tracker.ceph.com/issues/19825
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 561cbded0c7e28231b1c7ce18663b8d7d40aad6d)
+    
+    Conflicts:
+            src/mon/MgrMonitor.cc: no such service in Jewel
+
+commit ef5265f107858d37bbdeea165cec571db614b4a3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu May 4 22:49:04 2017 +0800
+
+    mon/Elector: call cancel_timer() in shutdown()
+    
+    instead of doing it manually.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 12139ae529a49b6caedea89f910d034ddca094b6)
+
+commit 9693b6bd1685f3ff1a3a44972b7e7f0e03d5da12
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Wed May 17 17:50:10 2017 +0400
+
+    jewel: mon: add override annotation to callback classes
+    
+    The only purpose of this patch is to avoid merge conflicts while
+    cherry-picking commit 561cbded0c7e28231b1c7ce18663b8d7d40aad6d.
+    Alternatively one could cherry-pick 1effdfe19bf9fd6d546620b96eaf452e889b15dc,
+    but that one brings a lot of unrelated changes.
+    
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+
+commit ea1b21d9604f19a48edeacc68ebabf8716143c2e
+Author: Michal Jarzabek <stiopa@gmail.com>
+Date:   Mon Aug 1 16:52:46 2016 +0100
+
+    mon/PaxosService: move classes to cc file
+    
+    Signed-off-by: Michal Jarzabek <stiopa@gmail.com>
+    (cherry picked from commit a4e979a41c7d3649cf70f9d0768015d7ff4aca8a)
+
+commit 1869ed9b2d4225e2bf0f366baf75e0e14f202aea
+Author: Michal Jarzabek <stiopa@gmail.com>
+Date:   Fri Sep 23 19:43:56 2016 +0100
+
+    mon/Paxos: move classes to .cc file
+    
+    Signed-off-by: Michal Jarzabek <stiopa@gmail.com>
+    (cherry picked from commit d21357a7e042424754d9ab41c134cae9e89945f9)
+
+commit 18cf302377aa7a09ad4922c07c64fefc047f2bdb
+Author: Michal Jarzabek <stiopa@gmail.com>
+Date:   Sun Jul 24 10:51:54 2016 +0100
+
+    mon/Elector:move C_ElectionExpire class to cc file
+    
+    Signed-off-by: Michal Jarzabek <stiopa@gmail.com>
+    (cherry picked from commit c819596e59fd5f66b3258fedd1d10b20cf70f0a9)
+
+commit fa33e7600b0205ca9113a1c5be535516886a0701
+Author: Michal Jarzabek <stiopa@gmail.com>
+Date:   Sat Jul 30 09:58:30 2016 +0100
+
+    mon/Monitor: move C_Scrub, C_ScrubTimeout to .cc
+    
+    Signed-off-by: Michal Jarzabek <stiopa@gmail.com>
+    (cherry picked from commit e58750127cbe8728f4047411904af3890966490f)
+
+commit 9ec5d8b1a5ea7fa437b161a49d246a7f0ff653c8
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu May 11 13:13:39 2017 +0800
+
+    osd/PrimaryLogPG: do not call on_shutdown() if (pg.deleting)
+    
+    when a callback is called, it could be facing a PG already shut down by
+    OSD. but if that callback wants to shut that PG down. it should check
+    the PG's status first.
+    
+    Fixes: http://tracker.ceph.com/issues/19902
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit f3c44a0dfc859f6f625a92e727e0e521ed4a9207)
+
+commit 592293cdfb397337c2808117ae9b06d5dc77f7f2
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Tue Jan 31 13:54:38 2017 +0400
+
+    systemd/ceph-disk: make it possible to customize timeout
+    
+    When booting a server with 20+ HDDs udev has to process a *lot* of
+    events (especially if dm-crypt is used), and 2 minutes might be not
+    enough for that. Make it possible to override the timeout (via systemd
+    drop-in files), and use a longer timeout (5 minutes) by default.
+    
+    Fixes: http://tracker.ceph.com/issues/18740
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+    (cherry picked from commit 22332f6bae57a6975a99523a115eb70608f26331)
+
+commit 3828c1443b0a580fa23680da6d0b624d76a6de2a
+Author: Boris Ranto <branto@redhat.com>
+Date:   Wed Mar 8 09:38:39 2017 +0100
+
+    ceph-disk: Add --system option for fix command
+    
+    This adds the ability to restore the labels of the underlying system
+    data in addition to ceph data.
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 8d81af42fd507c7b92c8279eb114b0a733ac1da6)
+
+commit 5b4132266cc8a4d53f9e045bc4b21706a9eeabc1
+Author: Boris Ranto <branto@redhat.com>
+Date:   Thu Feb 16 11:34:27 2017 +0100
+
+    ceph-disk: Add more fix targets
+    
+    It turns out I forgot several more directories that needs to be fixed by
+    this script. We need to fix /var/log/ceph, /var/run/ceph and /etc/ceph
+    as well.
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit ae139307d6b2bfba47e21d29d6dbd3c8dc01b5b5)
+
+commit 7341ee445829114e3dfb57a61ca3a0e02b666a6e
+Author: Boris Ranto <branto@redhat.com>
+Date:   Thu Feb 9 19:17:12 2017 +0100
+
+    ceph-disk: Add unit test for fix command
+    
+    This will simulate the command* functions to not actually run anything
+    thus excercising the python code directly. It also checks that the
+    proper (sub-strings) are in the output.
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 1ec53dee9a690134936bdc3a09c9a02fecf13a9d)
+
+commit 3122fa8ca37475fb1cbab296b89c5904ea7b6f76
+Author: Boris Ranto <branto@redhat.com>
+Date:   Tue Jan 31 13:19:33 2017 +0100
+
+    ceph-disk: Add fix subcommand
+    
+    This subcommand will fix the SELinux labels and/or file permissions on
+    ceph data (/var/lib/ceph).
+    
+    The command is also optimized to run the commands in parallel (per
+    sub-dir in /var/lib/ceph) and do restorecon and chown at the same time
+    to take advantage of the caching mechanisms.
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 6d5d30f6ed7538271579cc2ef4e2e364f01a4a6f)
+    
+    Conflicts:
+            doc/man/8/ceph-disk.rst: whitespacing changed
+            src/ceph-disk/ceph_disk/main.py: command_check_call changed type
+
+commit 618a82e11e53cf2f1d5d306a2eb330cdf5e459ca
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 4 17:56:22 2017 -0400
+
+    librbd: add no-op event when promoting an image
+    
+    The rbd-mirror process needs an event in the journal
+    to properly detect the transition between primary and
+    non-primary state between peers.
+    
+    Fixes: http://tracker.ceph.com/issues/19858
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4031555dda7597d24e9eb04b9ff29173909586f7)
+    
+    Conflicts:
+            src/librbd/journal/DemoteRequest.cc: logic exists in Journal.cc
+    
+    (cherry picked from commit 7970ec586bebd26b1ca4955136ad8f48bb833af6)
+    
+    Conflicts:
+            src/librbd/journal/PromoteRequest.[h|cc]: logic exists in Journal.cc
+            src/librbd/journal/Types.[h|cc]: trivial resolution
+            src/test/librbd/journal/test_mock_PromoteRequest.cc: does not exist
+
+commit 7ec6e8bc50b62b43f27cf572c9d267c1a5f9520e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 4 17:57:34 2017 -0400
+
+    rbd-mirror: prevent infinite loop when computing replay status
+    
+    If the image had a non-primary predecessor epoch whose tag tid
+    duplicates an epoch within its own journal, an infinite loop
+    would result.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3f179bf86216540d8e25aad469c604f96f6aecd8)
+
+commit 09f076ed968209507b1f8a27388c3959cb91c82b
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Apr 14 13:21:38 2017 -0400
+
+    osd: fix occasional MOSDMap leak
+    
+    _committed_osd_maps() may return early (without putting
+    the ref) on shutdown.
+    
+    Fixes: http://tracker.ceph.com/issues/18293
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a74632f964e10a57fb8065aec90dc7b69bddba5b)
+
+commit cd8c51569baaccf3ea57507f427db373c415934e
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Thu Apr 20 12:13:13 2017 +0400
+
+    rgw: add the remove-x-delete feature to cancel swift object expiration
+    
+    In openstack swift, it also support the feature to cancel the object expiration,
+    which could be found at last point in
+    https://docs.openstack.org/user-guide/cli-swift-set-object-expiration.html. we
+    can remove the object expiration by set 'X-Remove-Delete-At:'.
+    
+    This patch also could fix the bug that when we set the object expiration and
+    then upload the same object to the container again. The previous object expiration
+    also works, which is not compatible with the openstack swift.
+    
+    Fixes: http://tracker.ceph.com/issues/19074
+    Signed-off-by: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+    (cherry picked from commit 230429ebc4ac9b5840bb93c7e0eeb5edbb949106)
+    
+    Conflicts:
+            src/rgw/rgw_file.cc:
+              - processor->complete() has no object_size argument
+                (compression is not supported in Jewel)
+            src/rgw/rgw_op.cc:
+              - processor->complete() has no object_size argument
+                (compression is not supported in Jewel)
+              - RGWPostObj::execute(): the code changed a lot in master,
+                adjust delete_at manually (patch(1) tries to apply
+                the corresponding hunk to a different function)
+            src/rgw/rgw_rest_swift.cc:
+              - trivial: STREAM_IO in Jewel versus dump_header in master
+
+commit 10636ef77f7327b85d9128c3ba0c0f05829ee821
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Sat Jan 21 01:44:31 2017 +0530
+
+    osd/Pool: Disallow enabling 'hashpspool' option to a pool without
+              '--yes-i-really-mean-it'
+    
+    Fixes: http://tracker.ceph.com/issues/18468
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit 3715362883acf9b8a477eb058a9ad6ee6d81ae01)
diff --git a/doc/changelog/v10.2.2.txt b/doc/changelog/v10.2.2.txt
new file mode 100644
index 000000000..cda69548d
--- /dev/null
+++ b/doc/changelog/v10.2.2.txt
@@ -0,0 +1,3002 @@
+commit 45107e21c568dd033c2f0a3107dec8f0b0e58374 (tag: refs/tags/v10.2.2)
+Author: Jenkins Build Slave User <jenkins-build@trusty-huge--abdffd45-81df-4aa8-a769-e169993c7a0f.localdomain>
+Date:   Tue Jun 14 11:43:24 2016 +0000
+
+    10.2.2
+
+commit 008c5af59b045f9b6b8046a4c6d5e915cbe18bb3
+Merge: b733244 dd1ea65
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 14 07:32:44 2016 -0400
+
+    Merge pull request #9692 from dillaman/wip-16292
+    
+    jewel: librbd: remove should ignore mirror errors from older OSDs
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit dd1ea65c2c8ae966c1f07ce78f0d1f16c9a315d9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 13 15:29:43 2016 -0400
+
+    librbd: remove should ignore mirror errors from older OSDs
+    
+    Fixes: http://tracker.ceph.com/issues/16268
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5a97a9b7ef238c4ceb16225a0b937f9a0cdbb2b6)
+
+commit b733244b3dc46e11689ddfb46187877c01347d3a
+Merge: 1feddef 2656881
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 14 07:24:28 2016 -0400
+
+    Merge pull request #9691 from dillaman/wip-16260-jewel
+    
+    jewel: librbd: do not shut down exclusive lock while acquiring'
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 1feddef07c182d5a50b3aa3c4a7ff87f21dbd3e1
+Merge: ffb64cd a38caf9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 14 06:57:38 2016 -0400
+
+    Merge pull request #9690 from dillaman/wip-16290
+    
+    jewel: rbd-mirror: FAILED assert(!m_status_watcher)
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 26568819e331ef3adec6667a801f07058d94845b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 13 20:20:33 2016 -0400
+
+    librbd: track in-flight flush events for journaling
+    
+    The tracking for flush events was accidentally dropped. This
+    can result in the journal being closed too early since it
+    might miss in-flight flush ops.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a1ea1edce9332fa1730625629e3b4e16f02caaf4)
+
+commit a85fbb4e9a9a409738e9f031c20a8e2beb1b514f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 13 12:00:28 2016 -0400
+
+    librbd: do not shut down exclusive lock while acquiring
+    
+    Fixes: http://tracker.ceph.com/issues/16260
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c5694fc6766fb8e213c4b65d2cd7b9d066b07ff7)
+
+commit a38caf90c98a04d57b81519803271f9d85fb5d56
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Sun Jun 12 20:32:45 2016 +0300
+
+    rbd-mirror: FAILED assert(!m_status_watcher)
+    
+    Fixes: http://tracker.ceph.com/issues/16245
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit b8bfc0f2efb44d3eff7a5fd8bca24516c8815d62)
+
+commit ffb64cd0538cee05b4f360fa8334829da268d1b9
+Merge: 271406c edef63d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 14 06:54:51 2016 -0400
+
+    Merge pull request #9654 from dillaman/wip-16246
+    
+    jewel: librbd: recursive lock possible when disabling journaling
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 271406c77a6176057ba73b55a110a193d6872a48
+Merge: b5e344e 0579a48
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 14 06:54:31 2016 -0400
+
+    Merge pull request #9673 from dillaman/wip-16265
+    
+    jewel: rbd-mirror: do not re-use image id from mirror directory if creating image
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit b5e344e1e3670214da803ff6f7eb6e50bf52e46f
+Merge: 1720664 739f343
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 13 23:37:06 2016 -0400
+
+    Merge pull request #9674 from dillaman/wip-16267
+    
+    jewel: qa/workunits: ensure replay has started before checking position
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 739f343ec880eacd13b935d493b5dbfcc5efa387
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 13 00:19:37 2016 -0400
+
+    qa/workunits: ensure replay has started before checking position
+    
+    Fixes: http://tracker.ceph.com/issues/16248
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6a68b68e0d2741e6694fd28d31766808ba881815)
+
+commit 0579a483bf7b0f1cc590fb1dbc6c740c17679dac
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 13 02:09:00 2016 -0400
+
+    rbd-mirror: reset local image id when creating new image
+    
+    Fixes: http://tracker.ceph.com/issues/16253
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1948047acbc129de2982d53e130da1f577af224b)
+
+commit ec809870cc2902d2bdf6543d911e23e71567303c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 13 02:07:54 2016 -0400
+
+    rbd-mirror: handle an attempt to delete a non-mirrored image
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 45dda67b3ef3d5dc5abb327fc1f242140f004d90)
+
+commit 6c303007af20a317f02b5caf7d3cb9f2214d2ee5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 13 02:03:29 2016 -0400
+
+    cls_rbd: disallow setting an empty mirror image entry to disabling
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7b2f99a97ff64451f3ae72081186dff41c507271)
+
+commit edef63d56bc329638f79ce8017e72810dcd768f1
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun Jun 12 21:42:59 2016 -0400
+
+    librbd: avoid holding owner_lock waiting for exclusive lock
+    
+    Fixes: http://tracker.ceph.com/issues/16235
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b96ab805eed49d4bdb494c8919d2e1ee7ad0a7e6)
+
+commit 1720664ddcb32dd300b2bd046ff15ed4916676a4
+Merge: f902309 2eaaf2e
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Mon Jun 13 01:26:53 2016 -0700
+
+    Merge pull request #9656 from gregsfortytwo/wip-jewel-16024
+    
+    Jewel client: fix bugs accidentally disabling readahead
+
+commit f902309bc9c159c0d6a798c0398a95f64bb682ed
+Merge: 89d6545 d61e3dd
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Mon Jun 13 01:22:22 2016 -0700
+
+    Merge pull request #9655 from gregsfortytwo/wip-jewel-15508
+    
+    Jewel mds: order directories by hash and fix simultaneous readdir races
+
+commit d61e3ddf6d99a4b9b72fd769d635bf9b2bffaf0e
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon May 9 22:12:48 2016 +0800
+
+    client: fix simultaneous readdirs race
+    
+    Current readdir code uses list to track the order of the dentries
+    in readdir replies.  When handling a readdir reply, it pushes the
+    resulting dentries to the back of directory's dentry_list. After
+    readdir finishes, the dentry_list reflects how MDS sorts dentries.
+    
+    This method is racy when there are simultaneous readdirs. The fix
+    is use vector instead of list to trace how dentries are sorted in
+    its parent directory. As long as shared_gen doesn't change, each
+    dentry is at fixed position of the vector. So cocurrent readdirs
+    do not affect each other.
+    
+    Fixes: http://tracker.ceph.com/issues/15508
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 9d297c5e98f814b282dadc379ab70dfa678db73e)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit ba9fa11a235dc6c55b3b03e3a98b17316d06cf27
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Apr 25 23:24:46 2016 +0800
+
+    client: move dir_{release,ordered}_count into class Inode
+    
+    We close Inode::dir when it's empty. Once closing the dir, we lose
+    track of {release,ordered}_count. This causes direcotry to be wrongly
+    marked as complete. (dir is trimmed to empty in the middle of readdir)
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 235fcf68f5d96a4d9d6cc260d12da912fa7ea4a8)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit f5db278c8cba873e580255dea6dba5227d508df7
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Apr 25 17:59:35 2016 +0800
+
+    ceph_test_libcephfs: check order of entries in readdir result
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 9b17d145c15a469caa29853312b8dcb0b19f55f0)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit 51a75063f59fcb47d3a4d5f06aa51cab83b76616
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Apr 25 15:52:32 2016 +0800
+
+    mds: don't reset readdir offset if client supports hash order dentry
+    
+    Now the ordering of dentries is stable across directory fragmentation.
+    There is no need to reset readdir offset if directory get fragmented
+    in the middle of readdir.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 98a01af2cd8e01f14ab7be32c31a43571ef8bb87)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit 3fe5a09f3e7aa9fc89cd47a261785110c2bf38a2
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Apr 25 15:31:27 2016 +0800
+
+    client: using hash value to compose dentry offset
+    
+    If MDS sorts dentries in dirfrag in hash order, we use hash value to
+    compose dentry offset. dentry offset is:
+    
+      (0xff << 52) | ((24 bits hash) << 28) |
+      (the nth entry hash hash collision)
+    
+    This offset is stable across directory fragmentation.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 680766ec131b95271e320f54dfe6d69ea8d4fbb3)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit a65b3ef30123e5d9d3199cf2f5dafd7d9ab8a755
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Apr 25 10:50:17 2016 +0800
+
+    client: record 'offset' for each entry of dir_result_t::buffer
+    
+    This is preparation for using hash value as dentry 'offset'
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit bd6546e5e8bfb4ef8927807492d63d5285534618)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit 8361b988441d073b82809dbafd82fba1c23b13be
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon May 9 11:51:36 2016 +0800
+
+    client: fix cached readdir after seekdir
+    
+    Client::seekdir doesn't reset dirp->at_cache_name for a forward seek
+    within same frag. So the dentry with name == at_cache_name may not be
+    the one prior to the readdir postion.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 0e32115bae2f1ac2c59b57852976b0de5587abac)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit cf261257f6ad9b0d51f39af5d26beb15e5759672
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Apr 21 17:31:10 2016 +0800
+
+    mds: define end/complete in readdir reply as single u16 flags
+    
+    so that we can introduce new flags for readdir reply.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 92cfbdf526d1030891da366299b635e589dd5f8e)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit 9ce73cdeb6056060b299cf92e4c858d1ab2aef2c
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat Apr 23 16:03:48 2016 +0800
+
+    mds: sort dentries in CDir in hash order
+    
+    This gives us stable ordering of dentries. (Previously ordering of
+    dentries changes after directory gets fragmented)
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit f483224a0bc70b9eb5f62cd6cbb8b97df5b65953)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit 288160b7e4bec94238d3b24622b4d16808a273e1
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Apr 22 09:43:54 2016 +0800
+
+    client: save readdir result into dir_result_t directly
+    
+    Current code saves the readdir result into MedaRequest, then updates
+    dir_result_t according to MetaRequest. I can't see any reason why
+    we need to do this.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit db5d60d1b8d01afd43da09a063f00aa6ab312903)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit 75edb5b1c8bcdbc24b4cd6cb811e9c07608f09f5
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Apr 22 09:31:00 2016 +0800
+
+    client: don't allocate dir_result_t::buffer dynamically
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit c41ceb9f1425432295a52965de8d4a23b15fe5e5)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit 98e36d17d42d7b43720685745348317cab4a847f
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Apr 21 20:53:55 2016 +0800
+
+    client: simplify 'offset in frag'
+    
+    don't distinguish leftmost frag from other frags. always use 2 as
+    first entry's offset.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 6572c2a24ea2abec4c7eecef92a292c2cc72c762)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit 2eaaf2e524c8b95b1b22b6bd7ba153b52746fd01
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Jun 10 17:33:04 2016 -0400
+
+    client: move readahead ref get/put to obj constr
+    
+    The readahead context was incrementing the file handle reference count but only
+    decreasing in finish which is not called if readahead is a no-op.
+    
+    This fixes a bug caught in testing where readahead was causing inode to become
+    disconnected:
+    
+        2016-06-10 19:46:48.953018 7f2a4351be80  1 client.4125 dump_inode: DISCONNECTED inode 10000000502 #10000000502 ref 110000000502.head(faked_ino=2307 ref=1 ll_ref=0 cap_refs={1024=0,2048=0,4096=0,8192=0} open={1=0,2=0} mode=100666 size=4194304/0 mtime=2016-06-10 19:29:45.107417 caps=-(0=pAsLsXsFscr) objectset[10000000502 ts 2/4012653 objects 0 dirty_or_tx 0] 0x7f2a24300d00)
+        2016-06-10 19:46:48.953032 7f2a4351be80  2 client.4125 cache still has 0+155 items, waiting (for caps to release?)
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit b5a0773259ce9f2ef44e8e6925ab11dc1a5e55e6)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit 1ec2ef876d0e5cfe595ebb4a98fabf4958e9e937
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu May 26 16:32:58 2016 -0400
+
+    client: use fh min readahead to enable readahead
+    
+    The client was using the configured maximum readahead of 0 which was changed in
+    95ee69989129750fddce6a3b5644238c4b88ed74. This prevented readahead from ever
+    running (without setting a different default from 0).
+    
+    Fixes: http://tracker.ceph.com/issues/16024
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 01c179bcce2dc8a269d78eb1be8198b12bd161f7)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit b0bccc3dc11731466f9809013d6deb09f020693b
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri May 27 11:43:31 2016 -0400
+
+    client: update comment
+    
+    Default was changed to unlimited in 11e4680e7394e6e02612441c964e05207b97737b.
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit bee02a073f4c551798bc490618a0e8d862893c25)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit e4104689b2698feccd9ea8432f1d57847effbd07
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu May 26 15:08:52 2016 -0400
+
+    client: use layout.get_period() helper
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit b331d12a43a4177369a003c3ab9d525da6a89950)
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com
+
+commit e5759fa8d016d3f8e2a1ba79edb0bf90ab24565b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 15:44:26 2016 -0400
+
+    librbd: recursive lock possible when disabling journaling
+    
+    If pool-level mirroring is enabled, deleting the journal
+    could cause a deadlock attempting to delete remote peer
+    sync-point snapshots.
+    
+    Fixes: http://tracker.ceph.com/issues/16235
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit fb255e6c3cd44c8d24c53e3cd70395a11a712574)
+
+commit 89d65456ed1303b09e9bc054bbb62c0808db4565
+Merge: c526386 e15bfff
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun Jun 12 08:16:35 2016 -0400
+
+    Merge pull request #9553 from Abhishekvrshny/wip-16038-jewel
+    
+    jewel: pybind: rbd API should default features parameter to None
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit c526386b4b087029e6fc00315b7726a2cf9caecb
+Merge: 6f9f830 6af7b22
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun Jun 12 08:13:16 2016 -0400
+
+    Merge pull request #9552 from Abhishekvrshny/wip-15981-jewel
+    
+    jewel: qa: dynamic_features.sh races with image deletion
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 6f9f8307d7cff313bd2f476236591141d3c45c71
+Merge: 7edf393 dc38a26
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun Jun 12 08:08:39 2016 -0400
+
+    Merge pull request #9629 from dillaman/wip-16231
+    
+    jewel: rbd-mirror: fix deletion propagation edge cases
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 7edf393743b2f7811d784e02a42fce13af19e8bc
+Merge: 39f530a bc658ed
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun Jun 12 08:08:23 2016 -0400
+
+    Merge pull request #9630 from dillaman/wip-16233
+    
+    jewel: rbd-mirror: do not propagate deletions when pool unavailable
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 39f530a224d85cd12294ce125d8548cbe1e3e1c0
+Merge: bd454ff cb50679
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun Jun 12 08:07:42 2016 -0400
+
+    Merge pull request #9628 from dillaman/wip-16224
+    
+    jewel: rbd-mirror: fix journal shut down ordering
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit bd454ff3a95244d2be2177180814439667db73f1
+Merge: 3fe2f1b 3a2cd16
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun Jun 12 08:07:10 2016 -0400
+
+    Merge pull request #9627 from dillaman/wip-16217
+    
+    jewel: rbd-mirror: refresh image after creating sync point
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 3fe2f1b652d7d6d80f9375174885b002db503f40
+Merge: 2a9fee9 99ff1fc
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jun 10 21:48:26 2016 -0400
+
+    Merge pull request #9579 from Abhishekvrshny/wip-16006-jewel
+    
+    jewel: BackoffThrottle spins unnecessarily with very small backoff while the throttle is full
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 2a9fee94f10cba7675f564edbeac35620e8a6602
+Merge: ac1b354 4a967eb
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jun 10 17:34:19 2016 -0400
+
+    Merge pull request #9638 from ceph/wip-16211-jewel
+    
+    Jewel: remove reliance on FLAG_OMAP for reads
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit bc658ede5b2e68210852b66585bfa5560964f405
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 08:15:19 2016 -0400
+
+    rbd-mirror: do not propagate deletions when pool unavailable
+    
+    Fixes: http://tracker.ceph.com/issues/16229
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d321634414fa4b90c66474557ec990dfc935ae29)
+
+commit dc38a26e2e48f0a1fabdb51f1819e402e5b214ce
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 00:12:06 2016 -0400
+
+    rbd-mirror: image deleter properly handles multiple snapshots
+    
+    Fixes: http://tracker.ceph.com/issues/16226
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d72a8cb67fbdc7361f8f9e265c659ddb96dc0f74)
+
+commit 0a74552b5a4c297855f5e04336a9d5cf00441d9e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 9 22:32:33 2016 -0400
+
+    rbd-mirror: propagate deletions even if image replayer is stopped
+    
+    If an image deletion which causes the replayer to stop (due to an error)
+    before the deletion is detected, the deletion should still occur.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a0c526a23783fe543737f6445a2d9438cc73b137)
+
+commit da6d409ab670e45e34c1dcf7de3f775632423d3f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 9 21:14:27 2016 -0400
+
+    qa/workunits/rbd: improve deletion propagation test case
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit de15b198cd7b5e94016605c8016a5cf38c07c6f6)
+
+commit cb50679bad91fa1a0e360dca0331996b543fdadd
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jun 8 10:12:04 2016 -0400
+
+    librbd: quickly check for duplicate snap name upon create
+    
+    During journal replay, quickly check for a duplicate snap name
+    before allocating a snapshot sequence from the OSDs and
+    reverting due to the duplicate name.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8374d45c7644666103f88c410a2b1b781f3a0cb8)
+
+commit 43f3bbf130c9914f895bf4834305f951d0228954
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 7 14:45:16 2016 -0400
+
+    rbd-mirror: stop external replayer before closing remote journaler
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 88bd3002082184c045bb82547929b30da1526cfb)
+
+commit f0bf15e9d38e3de63c959bac0ae90b609dafc514
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 7 12:42:52 2016 -0400
+
+    librbd: stop journal recorder before starting external replay
+    
+    Fixes: http://tracker.ceph.com/issues/16165
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5aa6eb2782bdb5cde6a050a5f7a4e683800414c0)
+
+commit 3a2cd16e6c35b7b8ba0dd113b9a8cc586c1086b0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jun 8 12:49:04 2016 -0400
+
+    rbd-mirror: refresh image after creating sync point snapshot
+    
+    Fixes: http://tracker.ceph.com/issues/16196
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c9a3e3be7ddc31992167d55de12bbe6849447b43)
+
+commit 6b77d2268e7b17911f0a6809045d356d97d1621a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jun 8 12:03:33 2016 -0400
+
+    rbd-mirror: consistently use dout/derr for debug logging
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 03f5e4a917b3c18f8ccdaa5ce676ed9731dd2a8f)
+
+commit ac1b35478eea6bad574fac8641b47c0fcaee20f4
+Merge: 966186f 03466f0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 12:32:21 2016 -0400
+
+    Merge pull request #9612 from dillaman/wip-16208
+    
+    jewel: rbd-mirror: support bootstrap canceling
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 966186f3c20080475d3e692e918cc363cac1ae47
+Merge: 49ed4ff bddf944
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 12:32:02 2016 -0400
+
+    Merge pull request #9610 from dillaman/wip-16209
+    
+    jewel: librbd: prevent error messages when journal externally disabled
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 49ed4ffeb2c17f266559137030ab32182750b321
+Merge: 8e533c8 0014a9e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 12:19:47 2016 -0400
+
+    Merge pull request #9611 from dillaman/wip-16210
+    
+    jewel: librbd: journal IO error results in failed assertion in AioCompletion
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 8e533c877e5d7bac84867d418aada1998425ef72
+Merge: a768a1c e424482
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 12:16:49 2016 -0400
+
+    Merge pull request #9580 from Abhishekvrshny/wip-15996-jewel
+    
+    jewel: librbd: reuse ImageCtx::finisher and SafeTimer for lots of images case
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit a768a1cb7b0062963714361ce94f8a834db44bd4
+Merge: ba5d490 bb279f1
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 12:15:51 2016 -0400
+
+    Merge pull request #9609 from dillaman/wip-16216
+    
+    jewel: librbd: refresh image if needed in mirror functions
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit ba5d490824efa63c63713fee418631cd684293c4
+Merge: 41c5c27 3b8cb63
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 12:15:32 2016 -0400
+
+    Merge pull request #9608 from dillaman/wip-16189
+    
+    jewel: cls_rbd: mirror image status summary should read full directory
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 41c5c273b1f52d0284c6b08f15db3edb17cd674c
+Merge: 7d217b9 cec6870
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 12:15:08 2016 -0400
+
+    Merge pull request #9556 from Abhishekvrshny/wip-16162-jewel
+    
+    jewel: ceph_test_librbd_fsx crashes during journal replay shut down
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 7d217b9cc55778fa73f0ce4a0c2c5994f5f31c87
+Merge: 32f90a7 6c0e202
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 12:14:49 2016 -0400
+
+    Merge pull request #9554 from Abhishekvrshny/wip-16084-jewel
+    
+    jewel: librbd: validate image metadata configuration overrides
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 32f90a7c98ebd5a272ece71b1af7009be2742cb5
+Merge: 3715d20c 09200d4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 12:13:06 2016 -0400
+
+    Merge pull request #9555 from Abhishekvrshny/wip-16147-jewel
+    
+    jewel: Object Map is showing as invalid, even when Object Map is disabled for that Image.
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 3715d20c25c19ec1d1c58d275cef4e7ddb5b7778
+Merge: 7ea13df 73464af
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 12:12:47 2016 -0400
+
+    Merge pull request #9550 from Abhishekvrshny/wip-15969-jewel
+    
+    jewel: Initial python APIs to support mirroring
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 7ea13dfb9d85613d5c4daf92811a786293ffc2be
+Merge: 2c22131 3084cf3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 10 12:12:37 2016 -0400
+
+    Merge pull request #9548 from Abhishekvrshny/wip-15963-jewel
+    
+    jewel: Disabling journaling feature results in "Transport endpoint is not connected" error
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 2c221318e27fb4c4b68780c0037b9acae5862d6e
+Merge: 4d68ae0 69a9500
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jun 10 00:55:47 2016 +0200
+
+    Merge pull request #9424 from SUSE/wip-16107-jewel
+    
+    jewel: ceph gid mismatch on upgrade from hammer with pre-existing ceph user (SUSE)
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 4d68ae062b90599c184282b807f7d16cfaf2f6ad
+Merge: 57b4542 b443fdf
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jun 10 00:54:52 2016 +0200
+
+    Merge pull request #8802 from SUSE/wip-15606-jewel
+    
+    jewel: ceph-{mds,mon,osd,radosgw} systemd unit files need wants=time-sync.target
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 57b45426b5efa7ec4e293f9276f1f4299b5bf3ac
+Merge: 3cf6533 555cec9
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 9 15:44:13 2016 -0700
+
+    Merge pull request #9320 from SUSE/wip-16017-jewel
+    
+    jewel: ceph: cli: exception when pool name has non-ascii characters
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 3cf653328c988da5cf6be678cf3f0cb4c80c094d
+Merge: a422d03 663d7c2
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 9 15:43:48 2016 -0700
+
+    Merge pull request #9270 from Abhishekvrshny/wip-15852-jewel
+    
+    jewel: mon/Monitor: memory leak on Monitor::handle_ping()
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit a422d0372719841ea4cc63f87e77ddc1dee7d815
+Merge: 34b193b 8356021
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 9 15:43:04 2016 -0700
+
+    Merge pull request #9237 from SUSE/wip-15961-jewel
+    
+    jewel: ceph-osd valgrind invalid reads/writes
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 34b193be043f0d88c42c5ae177a660b747e81486
+Merge: e2a035e 64f15b3
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 9 15:42:32 2016 -0700
+
+    Merge pull request #9103 from Abhishekvrshny/wip-15797-jewel
+    
+    jewel: deadlock in OSD::_committed_osd_maps
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit e2a035eeb19e9043c41e5132d2ea0b8a59ac1bed
+Merge: 2c35fe7 adfbe95
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 9 15:41:52 2016 -0700
+
+    Merge pull request #9101 from Abhishekvrshny/wip-15799-jewel
+    
+    jewel: osd boot race with noup being set
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 2c35fe7e739e99452cf4d039281185e8c4a40b1c
+Merge: 6528e43 f4306de
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 9 15:40:43 2016 -0700
+
+    Merge pull request #9104 from Abhishekvrshny/wip-15795-jewel
+    
+    jewel: LibRadosWatchNotifyPPTests/LibRadosWatchNotifyPP.WatchNotify2Timeout/1 segv
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 6528e4371e0ab1c385e539646d08f4ea1dded893
+Merge: 50143ed b989084
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 9 15:37:43 2016 -0700
+
+    Merge pull request #9427 from SUSE/wip-16104-jewel
+    
+    jewel: ceph-disk: workaround gperftool hang
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 50143ed004ef198f2161b9f360bfeccd16503954
+Merge: b328e99 a32820d
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jun 9 14:55:31 2016 -0400
+
+    Merge pull request #9614 from ceph/wip-16113-jewel
+    
+    osd: hobject_t::get_max() vs is_max() discrepancy
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 03466f096adcfe3322cad49c959fec976d887860
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed May 4 15:30:05 2016 +0300
+
+    rbd-mirror: don't return split-brain error if we still in MIRROR_PEER_STATE_SYNCING
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 3ac620c5d66d8516e68264fcdb5e97b0e803d0ec)
+
+commit dfefd343e35b1c69fc2c938c7ebc30352e1320d4
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue May 3 12:48:06 2016 +0300
+
+    rbd-mirror: refactor split-brain detection code
+    
+    to make it more readable and extendable.
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 039cd2d2e3bb6a9c5042c125d2c67ec9adc02c47)
+
+commit c94e5b740628b98008e5342b8a3b2b888ff7b794
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue May 31 15:00:30 2016 +0300
+
+    rbd-mirror: support bootstrap canceling
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 21f895f48498e518c3675a9e851559f4d071f6dd)
+
+commit c8a3e759298c3a2adfa67d932178d284619d460e
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon May 30 16:24:24 2016 +0300
+
+    rbd-mirror: return ECANCELED when ImageReplayer start is canceled
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit f581dbcb2eca108ce9440629b8f00f3229c07b95)
+
+commit 0014a9e284c6cfc7678e63b56d707a8ad9b93127
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 7 01:01:09 2016 -0400
+
+    librbd: flush journal commit positions before starting op
+    
+    Ensure all IO has been properly flushed and committed to the
+    journal before starting an op that could affect the IO path.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 636129653c7cac9665d0c423f6aafad5fd0c480a)
+
+commit 3cec45a17a3d8bdcd19c63c31dad6b9150a9fbfd
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 7 00:11:34 2016 -0400
+
+    librbd: leaked AioCompletion if IO issued after image shut down
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d3d4e068396e01c58832e0ea1224ca34bafb808e)
+
+commit ef12536830082194da52df3a0dc8e22a3f69cb54
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 6 19:00:47 2016 -0400
+
+    librbd: resize and snap create can hang on journal replay failure
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1032f19b3c9d9c7916669d04af909131574b3cab)
+
+commit c8b4cabea0e1a74a62fd9b25ccb5e0b15df38733
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 6 14:54:23 2016 -0400
+
+    librbd: missing journal state for flushing/restarting replay
+    
+    Fixes: http://tracker.ceph.com/issues/16077
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a7f5bc181c9cd677d39f95ad78134e0f85c22acb)
+
+commit bc7649be5867fca46c1e86dacb99c1f8b18ad2e8
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 12 21:22:56 2016 -0400
+
+    librbd: track IO in the order issued
+    
+    Fixes: http://tracker.ceph.com/issues/15034
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit eb8dda5296eb5c19aa27e96d6e92eaa0c352cb62)
+
+commit 602ffd78f3de00d6655236dd914aa83006a8f862
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 12 15:15:04 2016 -0400
+
+    librbd: AioCompletion should retrieve CephContext from ImageCtx
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 133b1ffb21c982003c7aa9dafd585062f9b2eb51)
+
+commit 6e20662a9f5b9eabbd83eb91760532d8171d96b8
+Author: Hector Martin <marcan@marcan.st>
+Date:   Wed May 25 20:41:40 2016 +0900
+
+    librbd: only mark async completion as done after callback
+    
+    This change in behavior was introduced in fde9f78.
+    
+    Signed-off-by: Hector Martin <marcan@marcan.st>
+    (cherry picked from commit 77f7c1cf829e5417eb356ebf3bf1fbc8ff7a25a5)
+
+commit de29be73c043c0cd49b4d35a160ddd3910d423e8
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 9 18:17:49 2016 -0400
+
+    librbd: Journal no longer requires AioCompletion for IO events
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 82587a31f1d8ba1765f164391796753cf97a1878)
+
+commit 65556aa507e3e70352bb61c1549f340af0c4a9fb
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 9 17:59:09 2016 -0400
+
+    librbd: bubble journal error up via existing AioObjectRequests
+    
+    The journal cannot directly fail the AioCompletion because it
+    might have one-or-more outstanding requests associated with
+    it.
+    
+    Fixes: http://tracker.ceph.com/issues/15791
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 79c934742fb15c803780519b36070408599a74c5)
+
+commit bddf944797df5ffb93b587f8ac245846a48bd2e3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 6 22:34:30 2016 -0400
+
+    librbd: potential duplicate snap removal can result in crash
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 276fed6b7078158c2cd04fba14a11531c27898e0)
+
+commit 4ce4130d0e6deef315dcbd5dc899faf5de9ab0e7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 2 00:43:53 2016 -0400
+
+    ObjectCacher: do not merge uncommitted journal bhs
+    
+    Avoid the possibility of an overwrite callback for regions
+    that were never associated with a journal write.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c6060ade16205337de3941a10d05e234b738fa07)
+
+commit 6e645985e77bd0597eab6755b6e4071b79967584
+Author: Zhiqiang Wang <zhiqiang@xsky.com>
+Date:   Wed Feb 24 21:54:34 2016 +0800
+
+    ObjectCacher: don't loop the extents of OSDRead in map_read
+    
+    Looping the extents of OSDRead is done in _readx. The looping in
+    map_read is not needed.
+    
+    Signed-off-by: Zhiqiang Wang <zhiqiang@xsky.com>
+    (cherry picked from commit fae912c3856cab6d03183411eecab90c6e6732d2)
+
+commit c5f281d64221c62e3736f9630204f85d67328670
+Author: Zhiqiang Wang <zhiqiang@xsky.com>
+Date:   Wed Feb 24 21:49:56 2016 +0800
+
+    ObjectCacher: fix the bytes_written in writex
+    
+    The BufferHead returned after map_write may include other dirty data to
+    the left and/or right. The actual length of data been written is in the
+    ObjectExtent.
+    
+    Signed-off-by: Zhiqiang Wang <zhiqiang@xsky.com>
+    (cherry picked from commit ca6f1681e4baf61682e258c5761536d67bf6bdb8)
+
+commit 1c00d813791cdd8c6e685c1ebf6f93ef4aa0adbe
+Author: Zhiqiang Wang <zhiqiang@xsky.com>
+Date:   Wed Feb 24 21:45:45 2016 +0800
+
+    osdc: don't loop the extents of OSDWrite in map_write
+    
+    We are looping all the extents of the OSDWrite in writex. The looping in
+    map_write is not needed. And if there are same object in these
+    extents,they will be mapped multiple times.
+    
+    Signed-off-by: Zhiqiang Wang <zhiqiang@xsky.com>
+    (cherry picked from commit 322f1059670898762a890a01af6d3b0dd3ccf826)
+
+commit b48cdb21da2ba02ae4ee5398c38b48e857dba8cf
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 2 00:42:30 2016 -0400
+
+    librbd: notify image feature update before releasing lock
+    
+    If a client is waiting for the lock, ensure it sees the latest
+    version of the image features.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f601ea835bb78554ac767ccb11789df960765bf9)
+
+commit 680efbdcc3d2e28b6de991e7ad5c18e0d71cc551
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jun 1 23:19:20 2016 -0400
+
+    librbd: refresh image after acquiring exclusive lock
+    
+    It's possible that the object map or journaling features have
+    been disabled while the lock was not owned.
+    
+    Fixes: http://tracker.ceph.com/issues/16114
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 03c54f52d15d6283c630bac6f75427e6829f7d0a)
+
+commit bb279f1b288f1069b83d7661588f883716c80d9d
+Author: Jon Bernard <jbernard@tuxion.com>
+Date:   Wed Jun 1 21:01:30 2016 -0400
+
+    librbd: refresh image if needed in mirror functions
+    
+    Fixes: http://tracker.ceph.com/issues/16096
+    
+    Signed-off-by: Jon Bernard <jbernard@tuxion.com>
+    (cherry picked from commit a355c8f6d9e792517a8d0e096a99a64474369cfc)
+
+commit 3b8cb634b34dd7742bbdea2ca354a358cb0fe089
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 7 09:36:02 2016 -0400
+
+    cls_rbd: mirror image status summary should read full directory
+    
+    Previously only retrieved the status for the first 64 images in
+    the rbd_mirroring directory.
+    
+    Fixes: http://tracker.ceph.com/issues/16178
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 61f86e7ec14233ea0dee5b3370595ce050ee28c3)
+
+commit 4a967eb42a7409b9d78d569585af2155414234d2 (refs/remotes/origin/wip-16211-jewel)
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 9 10:36:44 2016 -0700
+
+    Revert "osd/ReplicatedPG: for copy_get get omap, firstly check ob whether has omap."
+    
+    This reverts commit 771c869651132013054599f3decfc1017092a25f.
+
+commit fd8f8af2456bc197877793a60d4401751f828032
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 9 10:36:39 2016 -0700
+
+    Revert "osd/ReplicatedPG: For omap read ops, it should check object wether has omap"
+    
+    This reverts commit 98caba1d0b4dd334748e3bcaf1bfb3f79f75eb65.
+
+commit d59ca31a4fcc472809b1f1bd3379fc3521c28fb0
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 9 10:36:31 2016 -0700
+
+    Revert "osd/ReplicatedPG: When do omapclear, it should check object whether is omap."
+    
+    This reverts commit aeb8141c29cdd337961430bbf3e93a79e3e1bcd3.
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.cc
+
+commit b328e998b2aa066f5f42577abd0c74a5f3d42694
+Merge: b1c4c4c 188318b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jun 8 12:00:31 2016 +0200
+
+    Merge pull request #8938 from SUSE/wip-15739-jewel
+    
+    jewel: ceph.spec.in: drop support for RHEL<7 and SUSE<1210 in jewel and above
+    
+    Reviewed-by: Boris Ranto <branto@redhat.com>
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit f4306de66d6c071cb240a0d77e8f877841f87033
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri May 6 09:09:43 2016 -0400
+
+    osdc/Objecter: upper bound watch_check result
+    
+    This way we always return a safe upper bound on the amount of time
+    since we did a check.  Among other things, this prevents us from
+    returning a value of 0, which is confusing.
+    
+    Fixes: http://tracker.ceph.com/issues/15760
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2e2ce365866a7569b55aff338a82c132aa927089)
+
+commit 64f15b3cfd15faa65e7a36c9779713ea2436cf16
+Author: Xinze Chi <xinze@xsky.com>
+Date:   Wed May 4 09:49:40 2016 +0800
+
+    OSD: fix deadlock in OSD::_committed_osd_maps
+    
+    Fixes: http://tracker.ceph.com/issues/15701
+    Signed-off-by: Xinze Chi <xinze@xsky.com>
+    (cherry picked from commit 4fbf3891aab16347d02782f7c1c0400f9e0896af)
+
+commit adfbe9554be2cb79a6cbccc7c4a556139b4aed50
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon May 2 23:28:18 2016 -0400
+
+    osd: handle boot racing with NOUP set
+    
+    This is a follow-on to 7139a232d26beef441ffbc13bc087baab3505ea8,
+    which handled the NOUP set + clear case when the OSD found out
+    about the flag being cleared.  However, it's possible that the
+    flag will get cleared but the OSD won't get a map update (because
+    it hasn't subscribed and is not doing any work).
+    
+    This means that it is *more* likely than before that we will
+    restart the boot process even though the OSD did successfully
+    mark us up.  However, as before, it is unavoidable because there
+    is no notification of whether our boot request succeeds or not.
+    
+    And it is still mostly harmless (an extra mark down + up cycle).
+    
+    Fixes: http://tracker.ceph.com/issues/15678
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 11e4242fbdb2f2f6f654d4cb3a7c95d5b38a88c2)
+
+commit e424482d0e79ad0a0c44b14f0977cfa69d1b7f72
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Sun Feb 28 23:35:17 2016 +0800
+
+    TaskFinisher: use singleton SafeTimer and Finisher
+    
+    Fix #13938
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    
+    (cherry picked from commit dc6d6ce8b29de51b69242e569517f69acfe226f4)
+
+commit 99ff1fc34215da3339ccd227c4f17caf8294a32f
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 18 12:09:10 2016 -0700
+
+    BackoffThrottle: wait() if past max
+    
+    Otherwise, we risk spinning on wait_for() with a small
+    delay.
+    
+    Fixes: http://tracker.ceph.com/issues/15953
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 6f835726a7df5b29a96849fc1362186fd2116d9b)
+
+commit 5ce43eb7682535deeee149600f1226b87e41defd
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed May 18 11:49:19 2016 -0700
+
+    BackoffThrottle: use wait_for instead of wait_until
+    
+    On some platforms, wait_until won't surrender the lock
+    with a negative argument.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 846808ff3f3837b1772f293ae71dba5eda609cc4)
+
+commit cec6870572621ed9abbe3791ba6e1a460333dfc0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 2 10:41:18 2016 -0400
+
+    test/rbd: fsx needs to shut down the journal
+    
+    Fixes: http://tracker.ceph.com/issues/16123
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d10ffaafbd2a5831154757b848ed8504dad99069)
+
+commit 09200d4b5002f8ef9e75a9218fe85aaa5da33a8b
+Author: xinxin shu <shuxinxin@chinac.com>
+Date:   Thu Jun 2 14:13:09 2016 +0800
+
+    remove invalid objectmap flag when objectmap is disabled
+    Fixes: http://tracker.ceph.com/issues/16076
+    Signed-off-by: xinxin shu <shuxinxin@chinac.com>
+    
+    (cherry picked from commit b2d475686ee7617bb2023d753941e3d6952f0878)
+
+commit 6c0e202573110de522d531e0aa8978e843b5167e
+Author: zhuangzeqiang <zhuang.zeqiang@h3c.com>
+Date:   Sat Oct 7 16:52:07 2017 +0800
+
+    rbd: check value of config override when setting image-meta
+    
+    Fixes: http://tracker.ceph.com/issues/15522
+    
+    Signed-off-by: zhuangzeqiang <zhuang.zeqiang@h3c.com>
+    (cherry picked from commit 4538f8152d0e73bdefc09874113c87467bdda622)
+
+commit e15bfff2e822e15e0a34d09825e04cee381d8737
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon May 23 09:18:34 2016 +0300
+
+    pybind/rbd: create/clone/copy: default to None for features param
+    
+    If None is specified don't set the features so that the defaults is
+    used.
+    
+    Fixes: http://tracker.ceph.com/issues/15982
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit b8b3a9757f7d517210a91f2893b0961b6b35afab)
+
+commit f9e32ac68a83932945165612dc64c307f0411b19
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon May 23 09:16:38 2016 +0300
+
+    librbd: clone: default to parent features
+    
+    Fixes: http://tracker.ceph.com/issues/15982
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit d305eaad8d57a6f64e6d80eebcd0839a65c9a668)
+
+commit 6af7b223f3d0a8d51cd472cb7024a5ec70d30f61
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu May 12 09:15:53 2016 +0300
+
+    qa: dynamic_features.sh: return error only if it failed on alive QEMU
+    
+    Fixes: #15500
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 8a71a79d24f2e1f999a5fc0a4154a10401d5a261)
+
+commit 73464af844b45137049b0fff294449ed3a6f13bb
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed May 11 14:33:00 2016 +0300
+
+    test: initial python APIs to support mirroring
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit bfad0ca33b9d65dcec65bc1a18596d35fb3098f3)
+
+commit 5c312669ba2d208acaa8e819812e50ce23418208
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed May 11 14:29:17 2016 +0300
+
+    rbd: initial python APIs to support mirroring
+    
+    Fixes: http://tracker.ceph.com/issues/15656
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit ef0ea8ee3c926a31b54e410c18e887415f6ea3cf)
+
+commit 3084cf32463a07cecec66befa1562a187bc1734f
+Author: Yuan Zhou <yuan.zhou@intel.com>
+Date:   Mon May 16 16:18:59 2016 +0800
+
+    rbd: close journal before removing
+    
+    Otherwise there is some unwanted error message.
+    
+    Fixes: http://tracker.ceph.com/issues/15863
+    
+    Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>
+    (cherry picked from commit ecdad4acca635d4b9f0b407889dde9985ab3a506)
+
+commit a32820d15fd5d2f817bc5fc94b731f4e7adafbf3 (refs/remotes/origin/wip-16113-jewel)
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 2 17:13:09 2016 -0700
+
+    src/: remove all direct comparisons to get_max()
+    
+    get_max() now returns a special singleton type from which hobject_t's
+    can be assigned and constructed, but which cannot be directly compared.
+    
+    This patch also cleans up all such uses to use is_max() instead.
+    
+    This should prevent some issues like 16113 by preventing us from
+    checking for max-ness by comparing against a sentinel value.  The more
+    complete fix will be to make all fields of hobject_t private and enforce
+    a canonical max() representation that way.  That patch will be hard to
+    backport, however, so we'll settle for this for now.
+    
+    Fixes: http://tracker.ceph.com/issues/16113
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 7c5f2acaa57bd6caaf4b13b48154df3ad6fbe84d)
+
+commit f869594332fb8ffa4d29aae22573afd094c39f54
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 2 17:39:09 2016 -0700
+
+    PG::replica_scrub: don't adjust pool on max object
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 8b7b7c7a2379ce77173bbdfd90c98c561d786155)
+
+commit 1737ff3f49080904ef07a7519d200ee608fab446
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 2 17:36:21 2016 -0700
+
+    hobject: compensate for non-canonical hobject_t::get_max() encodings
+    
+    This closes a loop-hole that could allow a non-canonical in memory
+    hobject_t::get_max() object which would return true for is_max(), but
+    false for *this == hobject_t::get_max().
+    
+    Fixes: http://tracker.ceph.com/issues/16113
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 42fad4b76def50a0984bf3de06f78ed434d56954)
+
+commit b1c4c4c456d1d20ccc91de4f762dbe46d8e339a3
+Merge: 24404c9 5d9ee88
+Author: John Spray <jspray@redhat.com>
+Date:   Fri Jun 3 12:37:00 2016 +0100
+
+    Merge pull request #9430 from ceph/wip-manila-backports
+    
+    jewel: backports needed for Manila
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 24404c928b878d28a34439b8adaf43854547d317
+Merge: 1aa81ce 208659f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 2 12:03:12 2016 -0400
+
+    Merge pull request #9423 from dillaman/wip-16101
+    
+    jewel: rbd-mirror: replicate cloned images
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 5d9ee88e2a0ee57455581350bafe613fef8e3da2
+Author: Xiaoxi Chen <xiaoxchen@ebay.com>
+Date:   Tue Apr 26 17:51:48 2016 -0500
+
+    pybind: configurable cephfs_vol_client prefix and ns_prefix.
+    
+    Which is useful if we would like to have multiple backends
+    in manila and potentially set different layout based on prefix.
+    
+    And also make ns_prefix configurable also in order to support
+    multiple backend.
+    
+    Fixes #15417
+    
+    Signed-off-by: Xiaoxi Chen <xiaoxchen@ebay.com>
+    (cherry picked from commit d1a21b07ede77ab3f2be7a753426d76de0343086)
+
+commit 470605c38f772e5bcc466e71ac232294b41be276
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Wed Apr 13 14:03:51 2016 +0530
+
+    ceph_volume_client: evict client also based on mount path
+    
+    Evict clients based on not just their auth ID, but also based on the
+    volume path mounted. This is needed for the Manila use-case, where
+    the clients using an auth ID are denied further access to a share.
+    
+    Fixes: http://tracker.ceph.com/issues/15855
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 68714b9dda8ed6010cad6730e0ddf54b3219150d)
+    (cherry picked from commit aaa0e9aea9fcf2f63e73e544bd40c10a0d694408)
+
+commit 726292eb305d58661993803dea613e9d98a64f4f
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed May 11 13:18:23 2016 +0100
+
+    client: report root's quota in statfs
+    
+    When user is mounted a quota-restricted inode
+    as the root, report that inode's quota status
+    as the filesystem statistics in statfs.
+    
+    This allows us to have a fairly convincing illusion
+    that someone has a filesystem to themselves, when
+    they're really mounting a restricted part of
+    the larger global filesystem.
+    
+    Fixes: http://tracker.ceph.com/issues/15599
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit b6d2b6d1a51969c210ae75fef93c71ac21f511a6)
+
+commit 46c2bd0864f8f7ace80a8cad07b873d9aa57ead9
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Apr 1 14:27:31 2016 +0100
+
+    pybind: fix unicode handling in CephFSVolumeClient::purge
+    
+    os.path.join is sensitive to string encoding, but
+    just doing a straight substitution should not be.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit e590f4d05fdb46747e83e35e66a26d9f4aa0314d)
+
+commit b989084dfc28659445fe08847537f8e5ad275cdd
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu May 26 12:55:51 2016 +0200
+
+    ceph-disk: workaround gperftool hang
+    
+    Temporary workaround: if ceph-osd --mkfs does not
+    complete within 5 minutes, assume it is blocked
+    because of https://github.com/gperftools/gperftools/issues/786
+    
+    References http://tracker.ceph.com/issues/13522
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit c092321c24b9ca493d90a9ccc8f0b4b9a38677f1)
+
+commit 69a9500d900b2d5b9bdb0e989f83fe60ddddeda0
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu May 12 19:23:31 2016 +0200
+
+    rpm: unconditionally set ceph user's primary group to ceph (SUSE)
+    
+    This commit brings the user/group creation into greater semantic alignment
+    with the Debian packaging.
+    
+    Fixes: http://tracker.ceph.com/issues/15869
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit b3dfa8d90962e60b6841555db830c8ea635f4cce)
+
+commit 208659fdebc888cab35239ec848f27320477420f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 26 12:49:45 2016 -0400
+
+    qa/workunits/rbd: basic cloned image test
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 943ff829874f96f720d6c6299e1212e4da24c6c5)
+
+commit af6be1bc15dfc2757cf74772078dd8516ed889be
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 26 08:22:16 2016 -0400
+
+    rbd-mirror: copy snapshot parent image settings
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f5fd09373714ebe8886bcf51bb97732038fffe55)
+
+commit 8405c2c377c7f9e755a349bfc90494f7ee7cc477
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 26 12:01:15 2016 -0400
+
+    cls_rbd: asynchronous helper method to set parent
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit fcfdebb471650a1730306f6a309f1775ba7a5eee)
+
+commit 26f4edcbb3b24eed98f111a2c77a5380cbf16927
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 25 17:38:06 2016 -0400
+
+    rbd-mirror: add support for cloning images from mirrored parent
+    
+    Fixes: http://tracker.ceph.com/issues/14937
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ae6e03dfb1fa3be61e6deac4fe39982e7e7c18aa)
+
+commit b8b01c57949955b1ff6a6a35dcae05108a2dda58
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 25 17:36:06 2016 -0400
+
+    librbd: extend internal API to clone from open parent image
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit bfaa112c30da7091fd2d539b80a4a211176a6031)
+
+commit 9081c58a31aa8fa6f966f3b682b9870e2b6f8297
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 25 17:13:59 2016 -0400
+
+    cls_rbd: asynchronous helper methods to retrieve mirror image metadata
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5a26080447cfa4b7bfdacdc13242569258a64076)
+
+commit 2c2f4b251d0c8b870728293d06e9c853856c8092
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 25 15:16:28 2016 -0400
+
+    rbd-mirror: helper state machine to open remote/read-only images
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3731c3c34c15f8dcfbab444bdbf3c05976b03431)
+
+commit 1aa81ce5ba492e8eeea2b20f08b77a18a94cd738
+Merge: 54bfefb 0bcc295
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 31 13:11:22 2016 -0400
+
+    Merge pull request #9409 from dillaman/wip-16068
+    
+    jewel: rbd-mirror: support multiple replicated pools
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 0bcc295363d9ca20d43fd313084370158513bc62
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 27 11:40:58 2016 -0400
+
+    rbd-mirror: cluster-level asok commands need to support multiple pools
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f4339ee8cb64d5f1a164e017fea942600ffb0ea9)
+
+commit e4547c036ef641d48ab03808f59e035b56136745
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 26 23:26:20 2016 -0400
+
+    rbd-mirror: replayer should only handle a single pool
+    
+    The pool watcher now only needs to scan a single pool for
+    its associated replayer since a peer is inherently tied to
+    a single pool.
+    
+    Fixes: http://tracker.ceph.com/issues/16045
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c5a8b780edeb9b62a601f0718459837f3d66e63d)
+
+commit 6fc5328e93b7d005479f925cca1268e7e7e10234
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 26 16:03:59 2016 -0400
+
+    rbd-mirror: group peers by local pools
+    
+    The peer structure is automatically unique per pool due to its
+    UUID, so grouping local pools by a single peer doesn't work.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 020f0282a0771c0d321732a62f53cee10a193715)
+    
+     Conflicts:
+    	src/tools/rbd_mirror/ClusterWatcher.cc: trivial resolution
+
+commit 0745cc3db35a349799891c8397249bf8d088ef83
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 25 14:00:34 2016 -0400
+
+    rbd-mirror: stop stale replayers before starting new replayers
+    
+    If the connection details are tweaked for a remote peer, stop
+    the existing replayer before potentially starting a new replayer
+    against the same remote.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6a2adca2a47b384f64c20996b9fc77010a70bc22)
+
+commit 91f8373613a38d48d71f2762e75e21ec817305be
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 26 13:29:49 2016 -0400
+
+    rbd-mirror: normalize debug log message prefix
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d52843190d84d533ab50a1702cf4a829b71ad68b)
+
+commit 5f0d89a9258fdf2f7392c63ca5d6488c4e3f210d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 26 13:26:57 2016 -0400
+
+    qa/workunits/rbd: create secondary replicated pool
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 839544ee6446b959f01290667592e44fc5ee6991)
+
+commit 54bfefb7d108667058eda56fda24b0029a560746
+Merge: 522c7f9 61542c1
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 31 11:51:05 2016 -0400
+
+    Merge pull request #9373 from dillaman/wip-16055
+    
+    jewel: journal: support asynchronous shutdown
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 522c7f9432e029f4db2b7455fb83b4aeb3ff4a4e
+Merge: f04972c 6375c78
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 31 11:49:39 2016 -0400
+
+    Merge pull request #9372 from dillaman/wip-15995
+    
+    jewel: rbd-mirror: Delete local image mirror when remote image mirroring is disabled
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 61542c14d34c4192f1468e2d7865fe913cce21b8
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 25 02:31:11 2016 -0400
+
+    rbd-mirror: image replay now uses asynchronous journal shutdown
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a1b0a1b1b9a237ae363a52c7b4038b19e87052a5)
+
+commit bf4c45830330b025bf818286e13693b837f1f2c5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 25 00:21:14 2016 -0400
+
+    librbd: integrate with async journaler shutdown API
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ad297850b1be8ed38f77b86913c6821748f3368b)
+
+commit 36cf42a3f47c23db82a8aca7a5560857cc3e0027
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 24 16:12:16 2016 -0400
+
+    journal: extend asynchronous shutdown to facade
+    
+    Fixes: http://tracker.ceph.com/issues/14530
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f40c595255a6828e27abfe63b4209ebae03e237b)
+
+commit 1b0422d7272788561253479a43f5b3dcdbb6282b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 24 15:53:12 2016 -0400
+
+    journal: trimmer now has asynchronous shutdown
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 59ca9916ba2e2b144eff5ee471a314b9c4391936)
+
+commit f913946ff74924fc14c7336fef6c73a4c6edfdfa
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 24 15:36:17 2016 -0400
+
+    journal: metadata init and shutdown now asynchronous
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0543e1b29eb253bf1d9e9af7af6f22fa394dd854)
+
+commit 1a9d60f8244d6ba8697b2254d967907fdb618cdb
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 24 12:06:26 2016 -0400
+
+    journal: player shutdown is now handled asynchronously
+    
+    Fixes: http://tracker.ceph.com/issues/15949
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 79b41346678b3acdeb547fe07e44d8d0543d37a7)
+
+commit 4af5bb1d2d86de68be0cef3713e09dfe0f1a9030
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 23 22:26:05 2016 -0400
+
+    test: use randomized write sizes for rbd-mirror stress test
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b9edff2174ff04bb8b6353dd2d2218c64757aae1)
+
+commit df2b8d9f0f399dcf9bb0e3eba3a39e14de504c49
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 23 22:21:33 2016 -0400
+
+    journal: eliminate watch delay for object refetches
+    
+    The randomized write sizes of the modified rbd-mirror stress
+    test results in a lot of journal object with few entries.
+    Immediately fetch objects when performing a refetch check prior
+    to closing an empty object.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d7b511b854c8ab633dcdb1c935014218b83f4e09)
+
+commit c65614f155ceccebd1960d6042d55e13b31e62ba
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 23 14:57:03 2016 -0400
+
+    journal: keep active tag to assist with pruning watched objects
+    
+    It's possible that there might be additional entries to prune in
+    objects that haven't been prefetched yet. Keep the active tag
+    to allow these entries to be pruned after they have been loaded.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0fb730e6be632e25d76695e1f37eba542162d620)
+
+commit 30c7f4dbe0a7c515d14be98f3f59ba8a8cdb1dcf
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 23 12:15:49 2016 -0400
+
+    journal: update commit entry object number upon overflow
+    
+    Otherwise the recorded object positions might point to an older
+    object that doesn't contain the actual entry.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5fbf5f82de8214c5c676d38feae7682a006fcab1)
+
+commit fa08330dadb9ce973db866ad641a79d5d2dd290c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 23 11:01:05 2016 -0400
+
+    journal: cleanup watch refetch flag handling
+    
+    Clear the refetch required flag while scheduling the watch
+    and remove the stale object after the watch completes if still
+    empty. Previously, it was possible for the flag to become
+    out-of-sync with whether or not it was actually refreshed
+    and pruned.
+    
+    Fixes: http://tracker.ceph.com/issues/15993
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ff2cc27ae592646b495bf1b614d35bd50c091a3d)
+
+commit 6375c7831de551c52f1dbf74bf9f65be3b6d8adf
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Fri May 13 16:47:25 2016 +0100
+
+    rbd-mirror: test: added image-deleter test case when mirroring is disabled
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit d85e312ead1da7c35d5ee4fe4be3d6e69a68b572)
+
+commit 1ba650539c1e90b82727a6bd9f1c16796b92b821
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Fri May 13 16:44:53 2016 +0100
+
+    rbd-mirror: Unregister clients from non-primary images journal
+    
+    A non-primary image may have registered clients on its journal
+    (for instance a primary image that was later demoted). We must
+    unregister the clients when disabling image mirroring with the
+    force option.
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 9cb0b180bc8a12ea2ebb0d7e32279b696ddd127e)
+
+commit a53cf28ab95a704b78cfa46cf758c2eec8058619
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Tue Apr 19 12:35:49 2016 +0100
+
+    rbd-mirror: test: Added unit test for testing image-deleter thread
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit ac9514941b0accb6c738ce4aa0953c36cdcf69a6)
+
+commit 78b13a504c32685090bb5b6512be9c050df2d8b6
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Mon Apr 18 12:22:19 2016 +0100
+
+    rbd-mirror: test: Fixed test_PoolWatcher to support the new changes in PoolWatcher class
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit e3677f115e1f369a49227fba51a1d34505d487ac)
+
+commit fef7456f8bc85f32cbdb28ce81198a9eb7d56d3b
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Tue Apr 5 09:47:32 2016 +0100
+
+    rbd-mirror: Replayer: bootstrap existing mirrored images
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 3c2615aa6da837bfef2365f701f56a34c49b536f)
+
+commit 97eb85f971a74678e4075b16cb5a8e836a985b56
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Thu Apr 28 11:49:43 2016 +0100
+
+    rbd-mirror: replayer: Added image-deleter thread to replayer
+    
+    Fixes: http://tracker.ceph.com/issues/14421
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit a6d698829d925894e4afcdec0ee42b1fc1205db0)
+
+commit 24406c5f8ea12d2c19573116ea84c90ba7446f8b
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Thu Apr 28 11:48:05 2016 +0100
+
+    rbd-mirror: Added image-deleter thread to rbd-mirror
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 011984d40bce448fe0febfc6a9fcf57d2c7b4768)
+
+commit 0780230b4af7e4ca923dab5f77735d0a11409202
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Wed Apr 13 15:17:40 2016 +0100
+
+    rbd-mirror: Added implementation of image-deleter thread
+    
+    image-deleter thread will handle the deletion of local images that have
+    been disabled for mirroring by the primary site.
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 6cc6e663ed2ec336a9475d93ab2cb605644e6ee0)
+
+commit 38514adc847a227dbd28318650931a8c2a7df7db
+Author: runsisi <runsisi@zte.com.cn>
+Date:   Wed May 11 20:42:37 2016 +0800
+
+    rbd-mirror: librbd::mirror_peer_list never returns -ENOENT
+    
+    no need to check -ENOENT specially
+    
+    Signed-off-by: runsisi <runsisi@zte.com.cn>
+    (cherry picked from commit 81ff42253369a9f57118bd26b29115c86a1748c5)
+
+commit 4e7233ad43d368a5f6e812870cec77545e034186
+Author: runsisi <runsisi@zte.com.cn>
+Date:   Wed May 11 20:52:38 2016 +0800
+
+    rbd-mirror: fix typo
+    
+    Signed-off-by: runsisi <runsisi@zte.com.cn>
+    (cherry picked from commit 25bfd4c786d0701fe6ae06e3d120098bd82f8aa0)
+
+commit c3b1bf1e530a4cf55585886a4b4038b9d2862471
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed May 18 10:07:55 2016 +0300
+
+    rbd-mirror: calculate behind_master only if mirror tag is not newer than master
+    
+    Fixes: http://tracker.ceph.com/issues/15916
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit cbd8d526f94523c9de3c575d426063e63d7e1802)
+
+commit f04972c8be28fc6d3133363b5474f22ebac5fd15
+Merge: c194c5d cb950fc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 31 11:48:06 2016 -0400
+
+    Merge pull request #9377 from dillaman/wip-16063
+    
+    jewel: test: failure in journal.sh workunit test
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit cb950fca12395a1c62421d4b0bce64b4a52b2fd4
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed May 25 21:54:16 2016 +0300
+
+    test: workaround failure in journal.sh
+    
+    With the changes to ensure that the commit position of a new
+    client is initialized to the minimum position of other clients,
+    the 'journal inspect/export' commands return zero records because
+    the master client has committed all of its entries.
+    
+    Workaround this by restoring the initial commit position after
+    writing to the image.
+    
+    Fixes: http://tracker.ceph.com/issues/16011
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit e9ed8ac33b1884c6ed5cdfb23023094ec790b116)
+
+commit f92c2a5604d4a8e83504b7283114d3d59bc45f38
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed May 25 14:51:43 2016 +0300
+
+    cls::journal: treat empty commit position as minimal
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 16846e8085b0c594f065322a768e6dfcbd5d2444)
+
+commit c194c5d8d454d0b387e191d03fec9d916271ff7f
+Merge: 4d3b668 8fbb555
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 31 11:46:30 2016 -0400
+
+    Merge pull request #9376 from dillaman/wip-15994
+    
+    jewel: cls_journal: initialize empty commit position upon client register
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 4d3b66842d4477ec4c4a418a91723fc9b800101c
+Merge: dafe375 e5ebb51
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 31 11:45:43 2016 -0400
+
+    jewel: librbd: write-after-write might result in an inconsistent replicated image
+    
+    jewel: librbd: write-after-write might result in an inconsistent replicated image
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit dafe375640967e10d1a94b02970d399c51980076
+Merge: 409fc64 305ebbc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 31 11:30:47 2016 -0400
+
+    Merge pull request #9318 from Abhishekvrshny/wip-15957-jewel
+    
+    jewel: Metadata config overrides are applied synchronously
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 409fc64799aec36e183e6ba2bb25c3bf7e3d603b
+Merge: cd6ed4d 0c99028
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 31 11:29:45 2016 -0400
+
+    Merge pull request #9317 from Abhishekvrshny/wip-15956-jewel
+    
+    jewel: rbd-mirror should disable the rbd cache for local images
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit cd6ed4deb0f4b66a10ea7bd5e027daee3a93d1f4
+Merge: 9fa8596 be9e85d
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Tue May 31 18:48:58 2016 +0800
+
+    Merge pull request #9403 from dachary/wip-16088-jewel
+    
+    jewel: tests: rm -fr /tmp/*virtualenv*
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit be9e85d83f3d025cec16b7da85d8a2ce1f64e8fa
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri May 20 13:19:07 2016 +0200
+
+    tests: rm -fr /tmp/*virtualenv*
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 8f0eefba00d1c6ad656302a608217fc3395d0401)
+
+commit 9fa85963a8accd669b0f0fbb17f4b721290432c6
+Merge: a046d2a 3674341
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Tue May 31 13:46:31 2016 +0800
+
+    Merge pull request #8968 from Abhishekvrshny/wip-15728-jewel
+    
+    jewel: osd: acting_primary not updated on split
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit a046d2a8c37182652d5f300f58673f70c8485114
+Merge: 954af78 8bbb5ad
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Tue May 31 13:42:28 2016 +0800
+
+    Merge pull request #9100 from Abhishekvrshny/wip-15856-jewel
+    
+    jewel: Hammer (0.94.3) OSD does not delete old OSD Maps in a timely fashion (maybe at all?)
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 954af787526a77b923fe85ed1282ba98277738e4
+Merge: e8e572c fae360f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon May 30 13:53:45 2016 +0300
+
+    Merge pull request #9245 from dreamhost/wip-15974-jewel
+    
+    rgw: fix manager selection when APIs customized
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit e8e572c6454816119448240fa7813be7995fd250
+Merge: c11db6f ffd545b
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon May 30 11:15:18 2016 +0200
+
+    Merge pull request #9239 from yehudasa/wip-15886-jewel
+    
+    rgw: keep track of written_objs correctly
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 8fbb555befb1415ef5218c8aaa149ca44017d73d
+Author: Venky Shankar <vshankar@redhat.com>
+Date:   Thu May 19 22:45:34 2016 +0530
+
+    cls_journal: Select min commit position for new clients
+    
+    When a new client registers, its commit position is initialized
+    to the least commit position of all other clients.
+    
+    Fixes: http://tracker.ceph.com/issues/15757
+    Signed-off-by: Venky Shankar <vshankar@redhat.com>
+    (cherry picked from commit dd2e3dae8adb3778e7e381db5dbfca0948571c55)
+
+commit 576ff0ca804779d1eb0b1c44d6ce38e0658b6a05
+Author: runsisi <runsisi@zte.com.cn>
+Date:   Mon May 9 15:15:31 2016 +0800
+
+    cls_journal: remove duplicated key generation
+    
+    Signed-off-by: runsisi <runsisi@zte.com.cn>
+    (cherry picked from commit 715e99c83e1ebc01ceb9eff0f66da8e2b598766a)
+
+commit fae360f0e419266b80f001a25a6864668e772d08
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Fri May 20 16:00:33 2016 -0700
+
+    rgw: fix manager selection when APIs customized
+    
+    When modifying rgw_enable_apis per RGW instance, such as for staticsites, you
+    can end up with RESTManager instance being null in some cases, which returns a
+    HTTP 405 MethodNotAllowed to all requests.
+    
+    Example configuration to trigger the bug:
+    rgw_enable_apis = s3website
+    
+    Backport: jewel
+    X-Note: Patch from Yehuda in private IRC discussion, 2016/05/20.
+    Fixes: http://tracker.ceph.com/issues/15973
+    Fixes: http://tracker.ceph.com/issues/15974
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit 7c7a465b55f7100eab0f140bf54f9420abd1c776)
+
+commit 8bbb5add6208a552979270baf4623430cf45e1b3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri May 13 11:26:31 2016 +0800
+
+    osd/OpRequest: reset connection upon unregister
+    
+    this helps to free the resources referenced by the connection, among
+    other things, in the case of MOSDOp, the OSD::Session and OSDMap. this
+    helps to free the resource earlier and trim the osdmaps in time.
+    
+    Fixes: http://tracker.ceph.com/issues/13990
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 87850e96ea6fa45a8368bacabee50f9e95b40ae9)
+
+commit e97cc2d1e9bcee0a43f54ed592d3078182571874
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu May 12 20:28:11 2016 +0800
+
+    osd: reset session->osdmap if session is not waiting for a map anymore
+    
+    we should release the osdmap reference once we are done with it,
+    otherwise we might need to wait very long to update that reference with
+    a newer osdmap ref. this appears to be an OSDMap leak: it is held by an
+    quiet OSD::Session forever.
+    
+    the osdmap is not reset in OSD::session_notify_pg_create(), because its
+    only caller is wake_pg_waiters(), which will call
+    dispatch_session_waiting() later. and dispatch_session_waiting() will
+    check the session->osdmap, and will also reset the osdmap if
+    session->waiting_for_pg.empty().
+    
+    Fixes: http://tracker.ceph.com/issues/13990
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 82b0af7cedc3071cd83ee53479f834c23c62b7d0)
+
+commit 555cec9a92568666880d70b9ae8f6e9748eca204
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Tue May 17 18:04:28 2016 +0100
+
+    ceph.in: fix exception when pool name has non-ascii characters
+    
+    When deleting a pool without the --i-really-really-mean-it option, if
+    the pool name has non-ascii characters, the format of the command
+    message raises a UnicodeEncodeError exception.
+    
+    Fixes: http://tracker.ceph.com/issues/15913
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 805873fed0135616a4cf8b499d1ba0b10dac004c)
+
+commit 305ebbc36473cf8bfd22513ffe55d3a4a99d5ef1
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 18 19:19:24 2016 -0400
+
+    librbd: metadata retrieval added to open image state machine
+    
+    Fixes: http://tracker.ceph.com/issues/15928
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b64cb31f8e7796175b4709929c017b3236649462)
+
+commit 5c9ecea87d3eb90201a967d8da827018cd728f0d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 18 17:50:07 2016 -0400
+
+    cls_rbd: async version of metadata_list helper method
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 985cb38211c51c95d84479df231c4f53847cb2ec)
+
+commit 0c99028582326c4ac142a03f168ab1bcf0ddd3c2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 18 20:53:26 2016 -0400
+
+    rbd-mirror: disable librbd caching for replicated images
+    
+    Each image has its own cache and each cache uses its own thread. With
+    a large replicated cluster, this could result in thousands of extra
+    threads and gigabytes of extra memory.
+    
+    Fixes: http://tracker.ceph.com/issues/15930
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ea35f148257282fe3f3ae02fe7a26cf245cda952)
+
+commit 188318b7b567f206d60dfb754fedbb0bfc74ef91
+Author: Boris Ranto <branto@redhat.com>
+Date:   Mon May 23 17:06:19 2016 +0200
+
+    rpm: Restore /var/run/ceph context manually
+    
+    The fixfiles command won't fix label for /var/run/ceph (/run is probably
+    excluded from relabel), we need to restore the context manually.
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 6de39ce73737f79d4511dfc4e1d10d7f3733ab1b)
+
+commit ab1f65dd0429647c30fdae1dd97ff6f393dbc1a8
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri May 20 08:09:42 2016 +0200
+
+    rpm: Mute semodule in postun script
+    
+    Currently, we don't mute the semodule output in postun script. This
+    results in the following message when removing ceph-selinux package:
+    
+    libsemanage.semanage_direct_remove_key: Removing last ceph module (no
+    other ceph module exists at another priority).
+    
+    The fix is to simply mute the output of the command.
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 3c104b106540fabdba43e0d04a3828d77f83e89d)
+
+commit 587fb3df704973071114a30bc55288768dbc2720
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Apr 27 11:13:06 2016 +0200
+
+    install-deps.sh: systemd-rpm-macros must be installed on SUSE
+    
+    Otherwise the script breaks.
+    
+    Fixes: http://tracker.ceph.com/issues/15627
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit a7f5ad5f6f9c01ef81f90a7071b6e2243b6da4e7)
+
+commit 69470d406d2b539e8021afd4def7de931ac91857
+Author: Boris Ranto <branto@redhat.com>
+Date:   Tue May 10 18:20:13 2016 +0200
+
+    rpm: Drop SELinux priority setting
+    
+    Older versions of semodule binary that are in Centos/RHEL do not
+    support priority setting, dropping it.
+    
+    Fixes: #15822
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 7e606f29f499eac83ecb7053ea6874fcfd092924)
+
+commit 128dbc8af0095622d48b962ddd9476aa4140bc16
+Author: Boris Ranto <branto@redhat.com>
+Date:   Wed May 4 09:09:47 2016 +0200
+
+    rpm: Fix SELinux relabel on fedora
+    
+    The SELinux userspace utilities stopped providing versions when they
+    switched to CIL language. We need to use a different technique to
+    relabel the files.
+    
+    Fixes: #15725
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit bd26641d705a34a1148a781eea08d203e81b750d)
+
+commit 8182b9a4d2750bb8f0f035d84e9d8476abc62c3c
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Apr 12 15:33:45 2016 +0200
+
+    Test-enable XIO messenger and IB/RDMA support
+    
+    Fixes: http://tracker.ceph.com/issues/15472
+    
+    Signed-off-by: Lars Marowsky-Bree <lmb@suse.com>
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit d4b74c56d6cbb2c6852bf66f2f34b8a877d7a9ef)
+
+commit 50626cec7a55637764ba379d6cb71e0b485290a8
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Apr 28 19:29:00 2016 +0200
+
+    rpm: do a full make check when --with-tests is given
+    
+    The check-local target tests the CLI tools only.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit e1514809ae3d2df08a0c47882f9b3bd0ed68d200)
+
+commit fb6858c283d96de15c70fbe8ffc33b5dada82739
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Apr 26 17:03:28 2016 +0200
+
+    rpm: drop sysvinit bits from relabel_files function
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 20df58799a4853ed3798501176988ffaf888936b)
+
+commit db4f2d677df5c9fc00592fc9e003bccb44af6da6
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Oct 11 18:27:45 2015 +0200
+
+    rpm: replace literal paths with RPM macros
+    
+    The only place we should write out literal paths is in the RPM scriptlets,
+    and there only for things that are not installed by this package.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit e0346b36fc0e6e17c80c047b13997448ccf1c76e)
+
+commit 3abf89517b0b2142698610debbaa3c6b535424f3
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Oct 11 18:05:12 2015 +0200
+
+    rpm: drop udev/95-ceph-osd-alt.rules
+    
+    This udev rules file was needed on older RHEL platforms, which are
+    unsupported as of jewel.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit ee4acb31593dd33da99138998fd0f30ebc2f1a6e)
+
+commit a7e01dd4ff08607fc6d99e41189e3db1a3c1d4da
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Oct 11 17:20:46 2015 +0200
+
+    rpm: global replace $RPM_BUILD_ROOT with %{buildroot}
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit bed92b6a86cdef784e89c38952fa779ce5fc5a91)
+
+commit d81cec99421e544e2cf7d212680b04d51ef675f3
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Oct 11 10:37:47 2015 +0200
+
+    rpm: put dependencies in alphabetical order
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 66042c03df012754215ed848e475f2195ee988a0)
+
+commit e8338a5844aea6dad8e61f00dbd94ca4e04df7be
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Oct 11 10:32:46 2015 +0200
+
+    rpm: drop sysvinit-specific dependencies
+    
+    Obsolete in jewel and above.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit c655cb896ad10e001b95848f4a4b64dd1ef6984f)
+
+commit b18d8ac0d65f91929fc956980d3b5977bb2970e6
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Oct 11 10:32:05 2015 +0200
+
+    rpm: move boost-random dependency to appropriate section
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 46b16c19f73d82cc357b16e6201926a3479046f2)
+
+commit 94efaef9c28347baebc374e27d3e81f143e642e0
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Apr 23 19:57:49 2016 +0200
+
+    rpm: move unified python-sphinx build dependency
+    
+    Now that the python-sphinx build dependency is unified, move it
+    to the proper section of the spec file.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 1d897da861c6134778136fa2b8b33e36c08faff9)
+
+commit 54dc8bf5703e2fb8962a97f7737a6114132b9b20
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Apr 23 19:41:06 2016 +0200
+
+    rpm: drop init-ceph.in-fedora.patch
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit e8cf501d0f22c757a0860a0c91dcb1d7a472ed34)
+
+commit 641e5bb0808a14d2b59cba141bad4acf5bbfb63a
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Oct 11 10:22:37 2015 +0200
+
+    rpm: drop dead conditionals
+    
+    This commit drops conditionals that no longer serve any purpose, since
+    jewel and above do not support the distro versions they are checking for.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 1332fac70e537014b5e03810aa983e2c9210d8dc)
+
+commit 4317a17997f21019e1dec59606928a2a9f4d94b9
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Oct 11 10:18:33 2015 +0200
+
+    rpm: drop python-argparse dependency
+    
+    This was only necessary for older (now unsupported) distro versions.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 6c42066d488d03142a56f92cff99d9c29d106214)
+
+commit 70466ffd386795082f02d718e804ad9e9c494401
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Oct 11 10:02:40 2015 +0200
+
+    rpm: drop _with_systemd
+    
+    Ceph versions jewel and above only support systemd.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 2ca6dd250b6a049f27fa17c5e8ca11674acb63f6)
+
+commit 928a8158aa71012db9f451a4302e8661783b5d37
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Oct 11 09:43:06 2015 +0200
+
+    rpm: drop python_sitelib/sitearch conditional
+    
+    This conditional was required to support older versions of RHEL/CentOS that are
+    no longer supported in infernalis and above.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 98d52120f89af3e93896d0518dbea4c4f660af48)
+
+commit 9f76b9ff31525eac01f04450d72559ec99927496
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Apr 18 09:16:02 2016 -0400
+
+    udev: remove 60-ceph-partuuid-workaround-rules
+    
+    These were added to get /dev/disk/by-partuuid/ symlinks to work on
+    wheezy.  They are no longer needed for the supported distros (el7+,
+    jessie+, trusty+), and they apparently break dm by opening devices they
+    should not.
+    
+    Fixes: http://tracker.ceph.com/issues/15516
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9f77244b8e0782921663e52005b725cca58a8753)
+
+commit 1ad390197af21d8cd7e507e2959f803028d3d09d
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Apr 20 17:21:56 2016 +0200
+
+    rpm: refrain from installing systemd unit files twice
+    
+    These files are already installed by systemd/Makefile.am
+    
+    Fixes: http://tracker.ceph.com/issues/15549
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 6d3cbaa509525f1807ac34bf5fed0fe99255f40f)
+
+commit 9c81f25b76ef2211910e453ba2bc4577fc921d7f
+Author: Boris Ranto <branto@redhat.com>
+Date:   Tue Feb 23 05:57:14 2016 +0100
+
+    test/ec: build the libs only when 'make check'
+    
+    Currently, we are always building the erasure code libraries while we
+    need them only when 'make check' is run. Moving the test libraries to
+    check_LTLIBRARIES should fix this for us.
+    
+    We no longer need to remove the libec libs manually, remove the lines
+    that do that.
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 0bbfea5cf13c138bbb2f0993607fa0dbf756281b)
+
+commit 663d7c2901ddf0b6e4184ec344bf0af00582eaf2
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Wed May 4 16:53:18 2016 +0800
+
+    mon/Monitor: use CEPH_MON_PORT for CRC output
+    
+    The default monitor port will be changed from 6789 to 3300, which has
+    been assigned to us by IANA. So update this accordingly for better
+    tracing and understanding.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 2b3d5686d76ee65fceca31d0d3343c2eaa4f2364)
+
+commit 64b948c21a4608a7680012e247f23dd2a3393f3f
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Wed May 4 15:03:55 2016 +0800
+
+    mon/Monitor: fix memory leak
+    
+    Fixes: http://tracker.ceph.com/issues/15793
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit ae5184ac07523a050584a3d621954baeb0dca01a)
+
+commit 9c3c6d57552a0a2cd448f0e60ded2f96ef9802ef
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Wed May 4 14:54:32 2016 +0800
+
+    mon/Monitor: use VOID_TEMP_FAILURE_RETRY wrapper for fd close().
+    
+    This is safer and much more robust.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit bfa40363b2d8fc25b4aed0164579d55137ac9e8b)
+
+commit 544c538d3720d7f18fca29ba15fd47200d3e05fc
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Wed May 4 10:02:01 2016 +0800
+
+    mon/Monitor: add "host" to tracked_conf_key
+    
+    The get_tracked_conf_keys() method gets the specified keys
+    registered and works as filters for configuration changing,
+    so we can track these key configuration changing and react
+    properly.
+    
+    Right now graylog prints host in its log entry, so
+    Monitor::update_log_clients() also reads from conf.host
+    and pass the host read from conf to graylog. That is
+    why we shall get "host" to the tracked_conf_key too.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit c79fc3305be42bd6126e8a60ae1f220d86754830)
+
+commit e5ebb51f22c43d49d716129ffcb5f87390b7acab
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 19 15:52:16 2016 -0400
+
+    journal: replay position might change after pruning stale tags
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0f5f1c375d8ef7b301cfd20b6848bdc81a761746)
+
+commit 9ecc3dcb76cc8f6d2904118d6b246d77d10cd9cf
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 19 15:50:04 2016 -0400
+
+    librbd: delay commit of overwritten journal event
+    
+    With the cache enabled and write-after-write IOs to the same
+    object extents, it was possible for the overwritten journal event
+    to be committed before the overwriter journal event was written
+    to disk.  If a client crash occurs before the event is written,
+    the image will be inconsistent on replay.
+    
+    Fixes: http://tracker.ceph.com/issues/15938
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e8bf64cc85ffe3d2dda23eab1834f7a5f104f6fe)
+
+commit 070dc7c98daaacaa05f72f50561d47ce459984da
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 19 14:13:46 2016 -0400
+
+    ObjectCacher: pass new journal tid when overwriting extent
+    
+    librbd needs to ensure the new journal event has been safely
+    committed to disk before it can mark an overwritten journal
+    event as committed.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0051316effe0b25804378975b117691d7ad08a3b)
+
+commit 0dfc787ef58fe9aa8177742f7c6e294e3b5d4b89
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 19 13:48:22 2016 -0400
+
+    qa/workunits/rbd: record rbd CLI debug messages during mirror stress
+    
+    The debug messages from 'rbd bench-write' and 'rbd snap create',
+    in addition to the existing debug messages from rbd-mirror, make
+    it possible to determine the source of any image inconsistency.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 714ed5e64e65fed9ac886537becf3a85ae6ae7fd)
+
+commit ffd545bf173c5a076c47541cbe3889175188d73e
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon May 16 14:35:12 2016 -0700
+
+    rgw: keep track of written_objs correctly
+    
+    Fixes: http://tracker.ceph.com/issues/15886
+    
+    Only add a rados object to the written_objs list if the write
+    was successful. Otherwise if the write will be canceled for some
+    reason, we'd remove an object that we didn't write to. This was
+    a problem in a case where there's multiple writes that went to
+    the same part. The second writer should fail the write, since
+    we do an exclusive write. However, we added the object's name
+    to the written_objs list anyway, which was a real problem when
+    the old processor was disposed (as it was clearing the objects).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 8fd74d11682f9d0c9085d2dc445fc3eb5631f6e0)
+
+commit 8356021ef3d3d21fd632d3caa660ba9a03cf80d3
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu May 12 16:57:49 2016 -0700
+
+    Pipe: take a ref to existing while we are waiting
+    
+    Otherwise, if it is reaped while we are waiting, it'll be a
+    use-after-free.
+    
+    Fixes: http://tracker.ceph.com/issues/15870
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit b224912d249453d754fc0478d3680f8cfa1a5c22)
+
+commit c11db6f103b2dd0049894a606916a84d66933833
+Merge: 7834070 e0dfc55
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 20 14:57:00 2016 -0400
+
+    Merge pull request #9232 from jdurgin/wip-mirror-workunit-jewel
+    
+    jewel: qa/workunits/rbd: fixed rbd_mirror teuthology runtime errors
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit e0dfc55ac874f2531bd0b88aa1c81156d7f7aec9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 18 14:17:13 2016 -0400
+
+    qa/workunits/rbd: fixed rbd_mirror teuthology runtime errors
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8ef09c4d8d3c7c6fbc9b6f7ef17cca0179e73468)
+
+commit 7834070fd8ead19e49fb8c766ea3ce8bfe63f032
+Merge: 9873fc1 5f09b9a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 20 09:02:51 2016 -0400
+
+    Merge pull request #9226 from dillaman/wip-15950
+    
+    jewel: rbd-mirror: potential crash during image status update
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 9873fc13d5647e94ec6f0152f903dd67b2882b0d
+Merge: aee6f7e 6c2973c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 20 09:02:06 2016 -0400
+
+    Merge pull request #9069 from dillaman/fixup-mirror_image_status_list-jewel
+    
+    jewel: cls::rbd: mirror_image_status_list returned max 64 items
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 5f09b9aef8f30d9dc7e5ec934b228d885e6456e2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 19 10:11:12 2016 -0400
+
+    journal: reset watch step after pruning expired tag
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 48c2f4e079000021b6ebe807bb1dba0dc1a6a11d)
+
+commit 2ee1e0a583d0ca851db4671b561eff9d493ca7d3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 18 11:01:22 2016 -0400
+
+    rbd-mirror: additional debug messages during image replayer start/stop
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit afc891f7fcb000d106b1d722386f6d81ddeddab6)
+
+commit f9e6bd8f1b6750d500fba3dd134257b7baac08c3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 18 00:55:01 2016 -0400
+
+    rbd-mirror: ensure proper handling of status updates during shutdown
+    
+    Previously, several shutdown race conditions could occur due to the
+    use of the async work queue for scheduling updates.
+    
+    Fixes: http://tracker.ceph.com/issues/15909
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9ae4edc05660665c3a135536a7c51d4070d28a40)
+
+commit f789d83cbb9dbb1a6daa15585e2be2980ea19616
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 17 23:31:02 2016 -0400
+
+    rbd-mirror: track bootstrap state within image status
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e5dd97b4bf4a993911febf34dce90470a2a91d59)
+
+commit e0b3879692d38bcc01d928cef09f74bc850cef86
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 17 23:25:25 2016 -0400
+
+    rbd-mirror: combine ImageReplayer stopped and uninitialized states
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a8f6dde4dbae8039380f31e026030a22a89fb4bd)
+
+commit 9c7977da45aef44160efadb48cb2d606568d4a7b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue May 17 16:14:42 2016 -0400
+
+    rbd-mirror: lock ordering issue in status update callback
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5c04c1361074ca9e9df1bc1c2dd82dfafc2a1304)
+
+commit 66f7f2d7bda7f2db6871870f5622594d6391b921
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed May 4 15:23:11 2016 +0300
+
+    rbd-mirror: don't unregister asok commands if image replayer start failed
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 2fd6fdd3dc9ecb915362f5e3aa8a224210273540)
+
+commit 569f06f1241309a710a0570d4ede950c1dc6e5d1
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed May 4 15:25:04 2016 +0300
+
+    rbd-mirror: avoid potential deadlock
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 46bf727d43af4975beeb514630a99d59cfb0a406)
+
+commit aee6f7ed791abd63ff9d1158a258d51db6c5ad28
+Merge: 2d0669e 0d19390
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 20 09:01:50 2016 -0400
+
+    Merge pull request #9217 from dillaman/wip-15945
+    
+    jewel: journal: live replay might skip entries from previous object set
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 0d193905b1d552600b542a466759af054c80e498
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 16 21:17:09 2016 -0400
+
+    qa/workunits/rbd: rbd-mirror daemon stress test
+    
+    This test repeatedly runs rbd bench-write, kills the process
+    randomly to create an unclean journal shutdown, and verifies
+    that the image content replicates correctly.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e2ab3128a37f4f36c9a1de6ddf2c0fe2e48aa7e2)
+
+commit d020b6ab1d10daa92fc99d1506222987b4306b26
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 16 18:08:35 2016 -0400
+
+    journal: helper method to detect newer tags
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0a8a6126ea35344e85af7eb64ffc490938edba51)
+
+commit 56515070710a6723ae4adc3e3b166e9ae9a35fdf
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun May 15 09:52:41 2016 -0400
+
+    journal: skip partially complete tag entries during playback
+    
+    If a journal client does not fully write out its buffered entries
+    before quiting, replay should skip over all remaining out-of-
+    sequence entries for the tag.
+    
+    Fixes: http://tracker.ceph.com/issues/15864
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9454f7e4c62437b1c288f371009feba1fd374584)
+
+commit 18f663d834edd03b42e2c08b9428e72fdc6bae9d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sat May 14 18:58:41 2016 -0400
+
+    journal: close, advance, and open object set ordering
+    
+    Flush in-flight appends to open objects before advancing the
+    active object set.  Additionally, don't start recording to the
+    new objects until after advancing the active set.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit de830057d0f7286914e019540c6263423cb60428)
+
+commit 27fba6ad7e7cb566d32c6e63f35aff966fc60bd7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sat May 14 18:13:38 2016 -0400
+
+    journal: new ObjectRecorder closed callback
+    
+    The callback will be invoked if there were in-flight appends
+    when the close was requested.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit bba91437dbe3b7a9b6da8a61ccc4c597858c8efc)
+
+commit 870c21c738e42d2cf34478bfa0631d602175dadc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 13 16:34:44 2016 -0400
+
+    journal: do not flag append as full if already known to be full
+    
+    Once an object has overflowed or a close is in-progress, re-attempting
+    to close the object is not needed since the async process is already
+    underway.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 45620dc312ca2aeaf084638144aaa4783b622c4a)
+
+commit 1cdebc7b896dd36bfee30f12aa3abd602c15af3e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 13 16:28:50 2016 -0400
+
+    journal: delay object overflow event until in-flight appends settled
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ee8d311a1fcdf7a85cdc168e1740fbaa252e87d3)
+
+commit 821d92af2fcc43ff560f01c78ff699ad7df33daa
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 13 16:17:37 2016 -0400
+
+    journal: ignore flush on closed/overflowed object
+    
+    The journal would be in-progress on transitioning to a new
+    object recorder in a newer object set.  Once the records
+    re-attach to the new object player they will automatically
+    flush.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a13cb821f71b6b1b3378464bcc2d8318eb0da48a)
+
+commit 3d860d5db37f77c1e8c76d214529266062caaa75
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 13 16:10:11 2016 -0400
+
+    journal: implicitly detach future's flush handler on append
+    
+    If the future is already in-flight, there is no purpose served
+    by requesting the future be flushed to disk.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1cb9b1a1b25d0f4c2d9a9e22661636e195fc70a0)
+
+commit f98e1289ce10aeb059a0b7d7cc8d17d6f1d60219
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 13 15:22:30 2016 -0400
+
+    journal: async callback for advancing the active object set
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ca89abfd9e6f0e8edc3234afea387757ef694826)
+
+commit 86e51ebcb301d823fb5c8b30bdc07107704679b4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri May 13 14:49:07 2016 -0400
+
+    journal: re-fetch active object before advancing set during replay
+    
+    During a live replay, it's possible that an append and and overflow
+    into the next object could race with the live playback of the same
+    object.  Re-fetch an "empty" object at least once before advancing
+    to next set to ensure all records have been read.
+    
+    Fixes: http://tracker.ceph.com/issues/15665
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6056f8c45c99bd37cb18933a37cc238c7e9a7c7d)
+
+commit 2d0669e172c37d38f118332574a474a534d27020
+Merge: f3979cd e885f1e
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu May 19 13:08:00 2016 -0700
+
+    Merge pull request #9210 from yehudasa/wip-15926-jewel
+    
+    radosgw-admin: fix 'period push' handling of --url
+
+commit e885f1e3be0f9a0c6457c5387f1c38b6e8b6a492
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon May 16 09:37:05 2016 -0400
+
+    radosgw-admin: fix 'period push' handling of --url
+    
+    was calling send_to_remote_gateway(), but passing 'url' instead of
+    'remote'. now uses send_to_remote_or_url() to accept either
+    
+    Fixes: http://tracker.ceph.com/issues/15926
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+
+commit f3979cd1d2fe910185db6dc3894722dd385379bd
+Merge: 1356d29 37ecfa6
+Author: Boris Ranto <branto@redhat.com>
+Date:   Thu May 19 12:56:15 2016 +0200
+
+    Merge pull request #9194 from ceph/wip-jewel-no-lttng-global-link
+    
+    Do not link lttng into libglobal
+    
+    Reviewed-by: Boris Ranto <branto@redhat.com>
+
+commit 37ecfa6c754683cac1c6f4c66a1a491d2fd5db1d
+Author: Karol Mroz <kmroz@suse.com>
+Date:   Mon May 2 14:01:27 2016 +0200
+
+    global: don't link lttng into libglobal
+    
+    Rely on dynamic initialization instead. Linking lttng in this way had
+    the unfortunate side effect of causing radosgw to segfault (when
+    daemonized) during sigterm processing (ie. during lttng_ust_exit()).
+    
+    This was originally removed in 638738f, but accidentally re-added via
+    5f61d36.
+    
+    Signed-off-by: Karol Mroz <kmroz@suse.com>
+
+commit 1356d29c70afedcbd55599d29aab93d2f79a02ad
+Merge: 15c4ad4 16ba13c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 18 13:23:39 2016 -0400
+
+    Merge pull request #9180 from dillaman/wip-15746-jewel
+    
+    jewel: doc: update mirroring guide to include pool/image status commands
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 16ba13c83d678656723173c061318dd83805768a
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu May 12 11:15:34 2016 +0300
+
+    doc: update mirroring guide to include pool/image status commands
+    
+    Fixes: http://tracker.ceph.com/issues/15746
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 4381e341c435c80793b847115eed616b8359c4e6)
+
+commit 818166e25d2ccac5d9bb3541f6166fcb8c25fd64
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu May 12 11:11:55 2016 +0300
+
+    doc: fixup: "rbd-mirror daemon" instead of "rbd-daemon"
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit ddd6345f8f7edcff638ed435aef316ba3e0e40dd)
+
+commit 15c4ad44010c798af7804e287ba71dcc289f806f
+Merge: 3a66dd4 7be281d
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed May 18 05:11:55 2016 -0400
+
+    Merge pull request #8815 from ktdreyer/wip-15646-jewel-system-targets
+    
+    jewel: debian: install systemd target files
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 7be281dab96c15b53fcfc3c2ffe6d17aec84b315
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Apr 28 09:13:50 2016 -0400
+
+    debian/control: dh_systemd_start is in the dh-systemd package
+    
+    Fixes: http://tracker.ceph.com/issues/15573
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 89154d1e01dbc58dc92f37741031f7ddb0448d2e)
+
+commit e463aa8b92dabdc7c13e150585eb09790da638ba
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Apr 22 21:07:58 2016 +0800
+
+    debian: install systemd target files
+    
+    * enable it using dh_systemd_enable
+    * start the target using dh_systemd_start
+    * move the dh_installinit, dh_systemd_enable, dh_systemd_start calls
+      down, so they can identify the service files if they care about them.
+    
+    Fixes: http://tracker.ceph.com/issues/15573
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit f950a0a7f5b35e13668379bc69dcb5d98a2dfc1f)
+
+commit 72c9b6f70752faf779b9272724f428e68d6499bb
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon May 9 14:15:36 2016 +0800
+
+    osd: remove all stale osdmaps in handle_osd_map()
+    
+    in a large cluster, there are better chances that the OSD fails to trim
+    the cached osdmap in a timely manner. and sometimes, it is just unable
+    to keep up with the incoming osdmap if skip_maps, so the osdmap cache
+    can keep building up to over 250GB in size. in this change
+    
+    * publish_superblock() before trimming the osdmaps, so other osdmap
+      consumers of OSDService.superblock won't access the osdmaps being
+      removed.
+    * trim all stale osdmaps in batch of conf->osd_target_transaction_size
+      if skip_maps is true. in my test, it happens when the osd only
+      receives the osdmap from monitor occasionally because the osd happens
+      to be chosen when monitor wants to share a new osdmap with a random
+      osd.
+    * always use dedicated transaction(s) for trimming osdmaps. so even in
+      the normal case where we are able to trim all stale osdmaps in a
+      single batch, a separated transaction is used. we can piggy back
+      the commits for removing maps, but we keep it this way for simplicity.
+    * use std::min() instead MIN() for type safety
+    
+    Fixes: http://tracker.ceph.com/issues/13990
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 369db9930887d75b498927da9c97733bff4472b6)
+
+commit 6c2973cd08503a3d91eeb881732c79213e42599f
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed May 11 14:14:46 2016 +0300
+
+    cls::rbd: mirror_image_status_list returned max 64 items
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit e925ce86243c91e6d64ff8d7b45166e4e3f2650b)
+
+commit 36743419ba49df4f97cdbe9ac25d9b08b0685108
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Apr 29 15:24:09 2016 -0400
+
+    osd/PG: update info.stats.* mappings on split
+    
+    These are updated in the init and start_peering_interval paths, but not
+    on split.
+    
+    Fixes: http://tracker.ceph.com/issues/15523
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8b42294d98dd70715184991da5ba495fbe93d009)
+
+commit b443fdf9402df621400ab7ca66799f597e8b2c23
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Apr 23 11:42:25 2016 +0200
+
+    ceph.spec.in: recommend ntp-daemon on SUSE
+    
+    SUSE has settled on "ntp-daemon" as the generic package name. The "ntp" and
+    "chrony" etc. packages have "Provides: ntp-daemon" in their respective spec
+    files.
+    
+    References: http://tracker.ceph.com/issues/15419
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 23108547454f53b95e193086d82d433570312ad4)
+
+commit 299f84ccdbf4d91412e216042fb248fd802af89d
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Apr 7 20:54:21 2016 +0200
+
+    deb: make ceph-base and radosgw recommend time-daemon
+    
+    References: http://tracker.ceph.com/issues/15419
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit f27cda03b7a961e445cce4ad35db6417e9e55959)
+
+commit 2da7cb2c7daad3ce2f9d2614e019291f75d4737d
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Apr 7 20:17:44 2016 +0200
+
+    systemd: make Ceph daemon units "want" time-sync.target
+    
+    Fixes: http://tracker.ceph.com/issues/15419
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit df893f395e62c8b1db18a277b38b44cab8b2016f)
diff --git a/doc/changelog/v10.2.3.txt b/doc/changelog/v10.2.3.txt
new file mode 100644
index 000000000..a21d1b7a1
--- /dev/null
+++ b/doc/changelog/v10.2.3.txt
@@ -0,0 +1,5957 @@
+commit ecc23778eb545d8dd55e2e4735b53cc93f92e65b
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Tue Sep 20 22:04:23 2016 +0000
+
+    10.2.3
+
+commit 9bfc0cf178dc21b0fe33e0ce3b90a18858abaf1b
+Merge: aeee80e 1cbc839
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Aug 29 13:43:15 2016 +0200
+
+    Merge pull request #10804 from dachary/wip-17089-jewel
+    
+    jewel: OSD failed to subscribe skipped osdmaps after ceph osd pause
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit aeee80e64af2f685ac9785c3444f40b60d0877f2
+Merge: 3168a84 dc154cd
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Aug 29 13:43:00 2016 +0200
+
+    Merge pull request #10760 from dachary/wip-16865-jewel
+    
+    jewel: saw valgrind issues in ReplicatedPG::new_repop
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 3168a8482211a2aaf9b4d715c0f7a920e896d9bb
+Merge: 12b374f ce986aa
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 26 22:12:39 2016 +0200
+
+    Merge pull request #10662 from dachary/wip-16901-jewel
+    
+    jewel: segfault in RGWOp_MDLog_Notify
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 12b374fccfb9792998ab88a4d7777f8b8134cb5c
+Merge: 91ee75f f298643
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Fri Aug 26 10:31:06 2016 +0200
+
+    Merge pull request #10763 from dachary/wip-17034-jewel
+    
+    jewel: rgw: object expirer's hints might be trimmed without processing in some circumstances
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 91ee75f39c05fe0c25aab2664f1fa71f858e7c6f
+Merge: d8bdc22 a865f26
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Fri Aug 26 10:28:10 2016 +0200
+
+    Merge pull request #10658 from dachary/wip-16862-jewel
+    
+    jewel: default zone and zonegroup cannot be added to a realm
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit d8bdc2245ddec91c2f55d367045e4407112d2af1
+Merge: 6d270f6 fa2e42d
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Fri Aug 26 10:26:53 2016 +0200
+
+    Merge pull request #10659 from dachary/wip-16863-jewel
+    
+    jewel: use zone endpoints instead of zonegroup endpoints
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 6d270f6084a035040638638ff65c14f5d76eeef1
+Merge: fe23c2e 2485efc
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Fri Aug 26 10:24:20 2016 +0200
+
+    Merge pull request #10660 from dachary/wip-16864-jewel
+    
+    jewel: multisite segfault on ~RGWRealmWatcher if realm was deleted
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit fe23c2edc70d1ad15e640195570e624b31415aef
+Merge: a57c21a 577336e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 26 00:46:17 2016 +0200
+
+    Merge pull request #10847 from gregsfortytwo/jewel-fix-mds
+    
+    mds: fix double-unlock on shutdown
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit a57c21afa686967a3afc4ddabb97c665e5894b21
+Merge: 49022d4 ce5e250
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 25 23:04:37 2016 +0200
+
+    Merge pull request #10663 from dachary/wip-16934-jewel
+    
+    jewel: Add zone rename to radosgw_admin
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 49022d4b2e2c5506dd83f3d894ac0a4ba83f8c7b
+Merge: f7afbbf 22c9df1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 25 23:04:22 2016 +0200
+
+    Merge pull request #10664 from dachary/wip-16958-jewel
+    
+    jewel: Bug when using port 443s in rgw.
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit f7afbbf094e5eabc1a651379f522d0b1544b01a3
+Merge: 7b672cb 73cc608
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 25 23:04:07 2016 +0200
+
+    Merge pull request #10764 from dachary/wip-17032-jewel
+    
+    jewel: multisite: RGWPeriodPuller tries to pull from itself
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 7b672cbe343955228b2a701eba91a30f5cac7fe5
+Merge: 28bc826 55460e2
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 25 23:03:40 2016 +0200
+
+    Merge pull request #10765 from dachary/wip-17006-jewel
+    
+    jewel: Increase log level for messages occuring while running rgw admin command
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 28bc826e9140520e86e2eb34470103979d10ae49
+Merge: 8b37c68 21da103
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 25 22:25:21 2016 +0200
+
+    Merge pull request #10845 from cbodley/wip-16702
+    
+    jewel: rgw: add tenant support to multisite sync
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 8b37c68f761d59025678c184a4f9b1755c90a628
+Merge: 94eb163 2266287
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 25 09:17:52 2016 +0200
+
+    Merge pull request #9405 from SUSE/wip-16083-jewel
+    
+    jewel: mds: wrongly treat symlink inode as normal file/dir when symlink inode is stale on kcephfs
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 94eb163c19e02ffae06659ced714140bd55584f0
+Merge: 9de2e93 f5db5a4
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 25 09:02:17 2016 +0200
+
+    Merge pull request #10654 from dachary/wip-9577-jewel
+    
+    jewel: mon: "mon metadata" fails when only one monitor exists
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 577336e20f68aa289311103e54d4def793b0cc01
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Wed Aug 24 13:34:43 2016 -0700
+
+    mds: fix double-unlock on shutdown
+    
+    We did a bad backport or something and accidentally ended up with two Unlock()
+    calls on mds_lock. Don't.
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 21da1039fc57bcf4054c0e7a13bb2732781770f6
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Jul 12 20:36:35 2016 -0700
+
+    rgw: collect skips a specific coroutine stack
+    
+    Fixes: http://tracker.ceph.com/issues/16665
+    
+    Instead of drain_all_but() that specifies number of stacks to leave behind,
+    added drain_all_but_stack() that has a specific stack specified. This is needed
+    so that we don't call wakeup() through lease_cr->go_down() on a cr stack that
+    was already collected.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5a2e8f0526db92a290c711f82627fc5042c290ea)
+
+commit 9de2e93d1b55c5f5f54b3dbf334d1513abb3f174
+Merge: 1d5dedf d1e0512
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 24 22:08:58 2016 +0200
+
+    Merge pull request #10791 from dachary/wip-17004-jewel
+    
+    jewel: rbd-mirror: FAILED assert(m_state == STATE_STOPPING)
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 1d5dedf0307f0fddf55f6487486a281c2625ef92
+Merge: e30c76e ecea6dc
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 24 22:08:30 2016 +0200
+
+    Merge pull request #10827 from dillaman/wip-16950-jewel
+    
+    jewel: librbd: delay acquiring lock if image watch has failed
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 98779c32de98efc367e888b99fd154ad4e5274d9
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Jul 11 17:38:43 2016 -0700
+
+    rgw: fix compilation
+    
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 874de58024e12d893ae050cd421ef67c9521c753)
+
+commit f5d9537d3e69197c7eadb21a6d0d01373530e2de
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jul 7 14:42:00 2016 -0400
+
+    rgw: remove datalog keys from error repo on ENOENT
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 16976eedb70292e821193e39bb577a68df1bc95a)
+
+commit 32505b2c8d703d09224b6a2f7007b20a79e0fb3d
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Jul 1 12:36:04 2016 -0400
+
+    test/rgw: add optional --tenant flag to test_multi.py
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 6f65d192cd3917cdcf7d9deb5396750d2c641842)
+
+commit f68337aaa535ea182258eb4934af739fb703f556
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jun 30 15:50:00 2016 -0400
+
+    rgw: add tenant id to GetBucketInstanceInfo
+    
+    use the rgw_bucket overload of get_bucket_instance_info() so it can
+    supply the tenant id
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 3cf715c6828d7d9732bf6a6fd9b46bbd8c08de5f)
+
+commit a0ffffa5aad423f363bf10a6b8dbaf19a47262a2
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jun 29 11:58:55 2016 -0400
+
+    rgw: carry tenant id with data sync
+    
+    use rgw_bucket_shard to track buckets, which includes tenant id
+    
+    Fixes: http://tracker.ceph.com/issues/16469
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit d945e2099525e39588b572e357b115df98c8cdca)
+
+commit bff626f7b32b3fe918510c147f88c49ead5007fa
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jun 30 17:43:40 2016 -0400
+
+    rgw: add tenant to url for RGWRESTStreamWriteRequest
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 2a1e1a776c58f62fbcb57877d56b92eb1e8933e6)
+
+commit dc50687f9ee7291192b2291c3c5770c46c852677
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jun 30 17:42:54 2016 -0400
+
+    rgw: add tenant to rgw_bucket json format
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit fc0df4802b48fa095587b93557c82da5446d9ca3)
+
+commit 06223adc51d2ccc1494c5e6891ad367edde69a91
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jun 30 00:13:47 2016 -0400
+
+    rgw: add rgw_bucket_parse_bucket_key()
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit d5ac140040ea2f17e33d2846870cb033c7ae6ffa)
+
+commit b4f687ee7fce682a41c707af7e2feb6b3a1cbe15
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Jun 28 11:24:46 2016 -0400
+
+    rgw: add tenant name to datalog entries
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit c8aa668fc3e34a7651ce6e2dda62902ced808541)
+
+commit dda0ee035fe9bb0c2fc4366df8355184db5034e3
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jul 6 11:50:47 2016 -0400
+
+    rgw: convert bucket instance listings back to metadata key format
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 3eae201c8a1513bae23233700c754ec076360664)
+
+commit 6225a6d45b8e76f9afaa741fa61d4b9742ef60fb
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Jul 5 09:00:19 2016 -0400
+
+    rgw: use tenant/ for bucket instance metadata keys
+    
+    to work around the ambiguity of parsing tenant: and :shard in the same
+    bucket instance metadata key, use tenant/ instead
+    
+    to preserve backward compatibility with existing objects, new helper
+    function rgw_bucket_instance_key_to_oid() converts this / back to a :
+    before being used as an object name
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit a0befa1e7be3aeb37209bac2e2a7de0edf5d5a95)
+
+commit cac6612a8a4f344e910d0c880ee8d020fac1ec79
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jun 30 18:09:03 2016 -0400
+
+    rgw: add get_key() methods to format rgw_buckets
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 51ff8ef7c6bb7c9f55f3ca8ad059b029a4088cdf)
+
+commit ac557e04e2d104fe9fdeae1d1ca31e212cfc89f6
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Jun 6 16:16:33 2016 -0700
+
+    rgw: data sync debug logging
+    
+    a high level debug logging of the data sync process.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 97ef16c0cc0ad8345b5a897108717d83ab0aa9ab)
+
+commit 6bb8c15a38e1ebf4e2b03f43bb527d460cd82dc3
+Author: Pritha Srivastava <prsrivas@redhat.com>
+Date:   Thu Jun 16 14:45:57 2016 +0530
+
+    rgw: modifying multi-site log messages.
+    
+    Increasing the log level and removing 'ERROR' from messages that appear during normal multi-site operation.
+    Fixes: http://tracker.ceph.com/issues/16121
+    
+    Signed-off-by: Pritha Srivastava <prsrivas@redhat.com>
+    (cherry picked from commit e1737ddaa035f23757d021dd34b0c87d24ce372c)
+
+commit e30c76ee33ddcb1a58835637c5e506c6c4b61b12
+Merge: df71e57 308f514
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 24 18:57:23 2016 +0200
+
+    Merge pull request #10653 from dachary/wip-10495-jewel
+    
+    jewel : 60-ceph-partuuid-workaround-rules still needed by debian jessie (udev 215-17)
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit df71e5740255ffc0a809b2c437ce37826aed11d3
+Merge: 286b30f f80d10c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 24 18:51:54 2016 +0200
+
+    Merge pull request #10357 from SUSE/wip-16748-jewel
+    
+    jewel: mount.ceph: move from ceph-base to ceph-common and add symlink in /sbin for SUSE
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 286b30fc54c1ea6a41c38a80c5227f9deeb13741
+Merge: eb28401 48ee3aa
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 24 17:48:00 2016 +0200
+
+    Merge pull request #10790 from dachary/wip-17005-jewel
+    
+    jewel: ImageReplayer::is_replaying does not include flush state
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit eb28401a59e2ca1a30d0f29aab7a2ea5eb704aea
+Merge: 4f3605b d336735
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 24 17:47:45 2016 +0200
+
+    Merge pull request #10792 from dachary/wip-16978-jewel
+    
+    jewel: rbd-mirror: FAILED assert(m_on_update_status_finish == nullptr)
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 4f3605b0c3cba436b362318487bec4358ea8c4d0
+Merge: 0c60d47 8316b6a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 24 17:47:36 2016 +0200
+
+    Merge pull request #10646 from dachary/wip-16576-jewel
+    
+    jewel: rbd-mirror: FAILED assert(m_local_image_ctx->object_map != nullptr)
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 0c60d47579591f07bc4e8d24a52c0355fbd17917
+Merge: 6dd0327 5968664
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 24 17:47:18 2016 +0200
+
+    Merge pull request #10647 from dachary/wip-16593-jewel
+    
+    jewel: FAILED assert(object_no < m_object_map.size())
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 6dd0327b40b35e19cb14ce30dec0d75d5a7da086
+Merge: cb4cb4a e098fb1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 24 17:47:08 2016 +0200
+
+    Merge pull request #10786 from dachary/wip-17061-jewel
+    
+    jewel: bashism in src/rbdmap
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit cb4cb4a02b160d9dd65e624cd762c12284eed241
+Merge: 089bb43 46246e3
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 24 14:12:34 2016 +0200
+
+    Merge pull request #10421 from SUSE/wip-16659-jewel
+    
+    jewel: ReplicatedBackend doesn't increment stats on pull, only push
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 089bb43b146408e5e670c3788990b712d8d26543
+Merge: 59b6c20 28697fc
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 23 15:42:11 2016 +0200
+
+    Merge pull request #10501 from Abhishekvrshny/wip-16621-jewel
+    
+    jewel: mds:  tell command blocks forever with async messenger (TestVolumeClient.test_evict_client failure)
+    
+    Reviewed-by: Douglas Fuller <dfuller@redhat.com>
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 59b6c20ea5bfb2d7062f7119ffbecd312d342bc4
+Merge: e1ddce7 f4fb598
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 23 15:23:34 2016 +0200
+
+    Merge pull request #10816 from SUSE/wip-17092-jewel
+    
+    jewel: build/ops: need rocksdb commit 7ca731b12ce for ppc64le build
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit ecea6dcf1c36bc5d478cf030f7ba1e01ca35a2d0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 4 13:24:54 2016 -0400
+
+    librbd: delay acquiring exclusive lock if watch has failed
+    
+    Fixes: http://tracker.ceph.com/issues/16923
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit dfe9f3eac9cca3b83962e0e1b7eac38e6e76d7a5)
+
+commit 49a39ebf6f7c6b7b0b19e4486fc10e57637e143c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 4 13:47:33 2016 -0400
+
+    librbd: convert ImageWatcher class to template
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 814c305ce8c35b5ce01d7e29a912d5ef3978754b)
+    
+     Conflicts:
+            src/librbd/ImageWatcher.cc: no shrink guard
+            src/librbd/Operations.cc: no shrink guard
+
+commit e1ddce7654698f3e41d89406981b1eb26f8f72a3
+Merge: 2c387d5 a7987f0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 23 08:48:19 2016 +0200
+
+    Merge pull request #10103 from xiaoxichen/wip-16037-jewel
+    
+    jewel: MDSMonitor::check_subs() is very buggy
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 2c387d5cd2785d518982f1c0d209235c240cb7ae
+Merge: 128251c 64d99b1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 23 08:48:01 2016 +0200
+
+    Merge pull request #10105 from xiaoxichen/wip-16515-jewel
+    
+    jewel: Session::check_access() is buggy
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 128251ceddf6129a1836a6d991a727adc93869df
+Merge: d27c6ac bce5646
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 23 08:47:37 2016 +0200
+
+    Merge pull request #10106 from xiaoxichen/wip-16215-jewel
+    
+    jewel: client: crash in unmount when fuse_use_invalidate_cb is enabled
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit d27c6ac57b5cd2f5566c5ecf476b8fe37041ab27
+Merge: 63422d0 fd7ff96
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 23 08:47:23 2016 +0200
+
+    Merge pull request #10108 from xiaoxichen/wip-16320-jewel
+    
+    jewel: fuse mounted file systems fails SAMBA CTDB ping_pong rw test with v9.0.2
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 63422d08ea6d401cefd6b438a7de48c1528da114
+Merge: 9fd5291 aa39361
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 23 08:47:10 2016 +0200
+
+    Merge pull request #10199 from SUSE/wip-16625-jewel
+    
+    jewel: Failing file operations on kernel based cephfs mount point leaves unaccessible file behind on hammer 0.94.7
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 9fd5291345e4f7d08faf8110ba78df244d030db6
+Merge: 2823714 2beb56a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 23 08:46:53 2016 +0200
+
+    Merge pull request #10499 from Abhishekvrshny/wip-16299-jewel
+    
+    jewel: mds: fix SnapRealm::have_past_parents_open()
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 282371465476e37408f65fc33145f8a76cf35ac7
+Merge: e2f7de6 1d3a816
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 23 08:46:38 2016 +0200
+
+    Merge pull request #10500 from Abhishekvrshny/wip-16620-jewel
+    
+    jewel: Fix shutting down mds timed-out due to deadlock
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit e2f7de68393dcf008e366cfc40b199b5acfcd428
+Merge: 6f287bf 7c2eab1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 23 08:46:13 2016 +0200
+
+    Merge pull request #10502 from Abhishekvrshny/wip-16797-jewel
+    
+    jewel: MDS Deadlock on shutdown active rank while busy with metadata IO
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 6f287bf2b99b8f6b1ae21f017d905dfa4a569649
+Merge: d68d41e d244b7a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 23 08:45:41 2016 +0200
+
+    Merge pull request #10104 from xiaoxichen/wip-16560-jewel
+    
+    jewel: mds: enforce a dirfrag limit on entries
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit d68d41ebdae92666b4dcf6cc8455cf17ab1d903a
+Merge: bc6e329 a5f5513
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 23 01:19:33 2016 +0200
+
+    Merge pull request #10074 from ceph/jewel-16002
+    
+    jewel: ObjectCacher split BufferHead read fix
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit f4fb59896ccb0d8ac01434cd4cf6ad67776fb0a0
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Aug 22 20:56:38 2016 +0200
+
+    build/ops: bump rocksdb submodule
+    
+    Fixes a FTBFS on ppc64le
+    
+    Fixes: http://tracker.ceph.com/issues/17092
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit bc6e3291b4960779fb84b2976079490d5f8f4935
+Merge: 46106f0 e825dd2
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Aug 22 10:37:14 2016 +0200
+
+    Merge pull request #10537 from theanalyst/wip-16778-jewel
+    
+    jewel: rgw multisite: preserve zone's extra pool
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 46106f0c0cfeb8746a521559bb466a5021a2c221
+Merge: 1bf76cd 96ad2d1
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Aug 22 10:31:39 2016 +0200
+
+    Merge pull request #10655 from dachary/wip-16163-jewel
+    
+    jewel: rgw: can set negative max_buckets on RGWUserInfo
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 1bf76cd58ec3635291e5c0080df47f0845088b77
+Merge: 3cef399 8b9954d
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Aug 22 10:29:40 2016 +0200
+
+    Merge pull request #10580 from dreamhost/wip-16928-jewel
+    
+    jewel: rgw: Fix civetweb IPv6
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 3cef399857b5cb234e0f4cd54933d58744736f62
+Merge: e1bc847 9e5a3ae
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Aug 22 10:28:14 2016 +0200
+
+    Merge pull request #10216 from SUSE/wip-16637-jewel
+    
+    jewel: add socket backlog setting for via ceph.conf
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit e1bc84776e7a95062311fb6b00a85a5327926c92
+Merge: 089967f 429e9c0
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Aug 22 10:27:03 2016 +0200
+
+    Merge pull request #10167 from jmunhoz/aws4-streaming-backport-jewel
+    
+    jewel: rgw: aws4: add STREAMING-AWS4-HMAC-SHA256-PAYLOAD support
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 089967f4374569495d177c7dea1e345ca08c772a
+Merge: 8e2de35 44decb4
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Aug 22 10:13:51 2016 +0200
+
+    Merge pull request #9544 from Abhishekvrshny/wip-16085-jewel
+    
+    jewel: A query on a static large object fails with 404 error
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 8e2de35324258cdbe44dfe8e7ce757ca1b3ef99b
+Merge: 3f007a1 059ed62
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Aug 22 10:10:25 2016 +0200
+
+    Merge pull request #10525 from Abhishekvrshny/wip-16732-jewel
+    
+    jewel: Bucket index shards orphaned after bucket delete
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 3f007a192f8629b7a6000846db41550eb94cdcb2
+Merge: 1019325 8655803
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Aug 22 10:09:21 2016 +0200
+
+    Merge pull request #10188 from dreamhost/wip-16618-jewel
+    
+    jewel: rgw: fix multi-delete query param parsing.
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 1019325af6c7153893a74b126f2c5e6534841bc4
+Merge: 0cd574f 4a3c9f3
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Aug 22 10:06:51 2016 +0200
+
+    Merge pull request #9266 from Abhishekvrshny/wip-15964-jewel
+    
+    jewel: rgw: realm pull fails when using apache frontend
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 0cd574f5a9a1db4ddfe262e2c4eb4643c5af9075
+Merge: cdd4ab2 dbf8cf0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Aug 22 10:02:20 2016 +0200
+
+    Merge pull request #10710 from rzarzynski/wip-16393-jewel
+    
+    jewel: rgw: improve support for Swift's object versioning.
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 1cbc839ebd11acb2ed0ebf0b0117eab09b5141c3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Aug 15 15:42:29 2016 +0800
+
+    osd: set objecter.osdmap when starts
+    
+    if any OSD is full or the pause flag is set in cluster, objecter will
+    always try to subscribe the next osdmap using the mon client shared with
+    OSD. but if the OSD's osdmap is very far behind on osd maps due to some
+    reasons, `MonClient::sub_want_increment()` will reject the subscribe
+    request from OSD, because it's asking for older maps while the mon client
+    is about to send the request from objecter for the *next* osdmap.
+    
+    so we need to update objecter with the latest local osdmap, so it is
+    able to skip the new osdmaps if OSD needs to catch up with the cluster
+    first.
+    
+    Fixes: http://tracker.ceph.com/issues/17023
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 1515e08e7d425f01d2e9d02c34b2ae40cdf2611a)
+
+commit 4d37cfb6eed1737b21d9f3f18eee6169f48445b2
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Aug 12 19:19:51 2016 +0800
+
+    mon/MonClient: should update sub_sent with sub_new
+    
+    std::map::insert() does not overwrite existing items with the same key.
+    so we need to do this in a different way. and we will check sub_sent
+    aftewards, so we need to keep an updated sub_sent around.
+    
+    Fixes: http://tracker.ceph.com/issues/17023
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 550814900ebca9dd89e088b9abe24f66eb5e3e6d)
+
+commit cdd4ab2897a3d0004b1e50401754004f95559e18
+Merge: 38433ad 5498377
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Aug 21 21:48:20 2016 +0200
+
+    Merge pull request #10797 from trociny/wip-17080-jewel
+    
+    jewel: the option 'rbd_cache_writethrough_until_flush=true' dosn't work
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 5498377205523052476ed81aebb2c2e6973f67ef
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Aug 17 12:08:37 2016 -0400
+
+    librbd: cache was not switching to writeback after first flush
+    
+    Fixes: http://tracker.ceph.com/issues/16654
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2d9840af39555ce00246b50e4a5c186798bd88ff)
+
+commit 471871e752963bf8cc42dcc1ec7e44e34e7e0ca2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 26 23:28:43 2016 -0400
+
+    test: unit test cases for disabling librbd journal by policy
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4a256f5044ac54a8a0a9598ee7fd151b8ea08f84)
+    
+    Conflicts:
+            src/test/librbd/CMakeLists.txt (test_ConsistencyGroups.cc)
+
+commit 062162f88f1bdaaaa96713429007757b72ec92f9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jul 27 10:56:48 2016 -0400
+
+    librbd: utilize factory methods to create AioObjectRequest objects
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2e5076eb19172919deeb0f4d11f2b2b6cc03fab3)
+
+commit d8eddc6b58fa2b0589719fab54beb42bc8e440bf
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jul 27 09:07:00 2016 -0400
+
+    librbd: convert AioObjectRequest/AioObjectRead classes to templates
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 840a473e7fb94124ca7c571fc3dd2c787e0d265a)
+
+commit 1a7cb60ee4b898aaea43a9963e95ed575ec66f56
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jul 27 09:54:44 2016 -0400
+
+    librbd: move read callback helpers to narrowest scope
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 65b336f68596a430629692a8682c5cfe883c31f1)
+
+commit 026f6d2da25d41f21f5bb5f8ee367cc7601296f7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 26 23:26:08 2016 -0400
+
+    librbd: convert AioImageRequest derived classes to templates
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a945c2c235d9259b1d4a8a579d7e6efc2ed35916)
+
+commit 61f0acb82ba344139bc2aa486fb096e5576e2553
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jul 27 08:49:44 2016 -0400
+
+    librbd: removed namespace indentation from legacy classes
+    
+    Better follows the Ceph C++ style guide
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f1e391982b43ddfb363ff913260460368a6d5834)
+
+commit e83866bcf01f32c35af94a47ee614361b4a787e6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 26 22:58:24 2016 -0400
+
+    librbd: do not record journal events if append is disabled by policy
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 405142c615352613e8bacee46e92484eb0c08f26)
+
+commit a9a84bad431f2d9916bdf88980b1cd2d0e36a1e5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 26 20:57:40 2016 -0400
+
+    librbd: remove unused journal replay canceled callback
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f453554edcb16ec72714b2456c08bab9e339b1eb)
+
+commit 942950007f9c424949eacf1d1cf3a74ef3b95725
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 26 20:48:13 2016 -0400
+
+    librbd: optionally support disabling journal appends via policy
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f7eeacd02f8f80c5d1d5d9cfed35bbd23f64b018)
+
+commit 1f63307a339fcfe3045cc001a72d1ecc197a7010
+Author: shawn <cxwshawn@gmail.com>
+Date:   Fri Jun 17 01:45:58 2016 -0400
+
+    librbd: optimize header file dependency & modify related file.
+    
+    Signed-off-by: Xiaowei Chen <chen.xiaowei@h3c.com>
+    (cherry picked from commit 2185a6275c89d9462611e965da407ea0d504987d)
+
+commit c49398dcd92bab90ab852407427bcc2c23147779
+Author: shawn <cxwshawn@gmail.com>
+Date:   Tue Jun 7 03:12:55 2016 -0400
+
+    librbd: optimize operation header file dependency
+    
+    Signed-off-by: Xiaowei Chen <chen.xiaowei@h3c.com>
+    (cherry picked from commit 88afa408683238b803606f3d63de326ead8cefee)
+
+commit 5366973897c1103a1c291c2f886692da4170d901
+Author: shawn <cxwshawn@gmail.com>
+Date:   Tue Jun 7 01:59:05 2016 -0400
+
+    librbd: optimize journal header file dependency
+    
+    Signed-off-by: Xiaowei Chen <chen.xiaowei@h3c.com>
+    (cherry picked from commit 59397e27a04fb0f2b990a86dcaf3c418457d1b72)
+
+commit c95b83e7c0cae0b322af713045745fb1c8fa411a
+Author: shawn <cxwshawn@gmail.com>
+Date:   Mon Jun 6 23:36:06 2016 -0400
+
+    librbd: optimize image_watcher header file dependency
+    
+    Signed-off-by: Xiaowei Chen <chen.xiaowei@h3c.com>
+    (cherry picked from commit aee1559f312e9493d6580e5ffcb606fe21ee3c59)
+
+commit e955496fb638163427c71b0734f6506aa6636d28
+Author: shawn <cxwshawn@gmail.com>
+Date:   Mon Jun 6 23:22:39 2016 -0400
+
+    librbd: optimize image header file dependency
+    
+    Signed-off-by: Xiaowei Chen <chen.xiaowei@h3c.com>
+    (cherry picked from commit 461958ebc7e8ccef76b3f08a979482762267c10b)
+
+commit f403abafd5da00e5afc13abba7514fa2bdce9760
+Author: shawn <cxwshawn@gmail.com>
+Date:   Mon Jun 6 22:56:49 2016 -0400
+
+    librbd: optimize exclusive_lock  header file dependency
+    
+    Signed-off-by: Xiaowei Chen <chen.xiaowei@h3c.com>
+    (cherry picked from commit 01282eb5cdeda7bbbb77438c7d1953d023ea54b6)
+
+commit 0ca8071e13492a9124be8322843cd96ca2bba531
+Author: shawn <cxwshawn@gmail.com>
+Date:   Mon Jun 6 03:37:22 2016 -0400
+
+    librbd: optimize object-map header file dependency
+    
+    Signed-off-by: Xiaowei Chen <chen.xiaowei@h3c.com>
+    (cherry picked from commit 1a2276927d5956918f7a4830b4a44048ac090328)
+
+commit 03314145d1bb5f4330a92d945af9df8856284724
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Jun 30 14:31:23 2016 +0300
+
+    test: fix librbd tests for rbd_skip_partial_discard
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 9717417b7b8aa1b24a82a0bfddbfc23748188641)
+    
+    Conflicts:
+            src/rocksdb this was a mistake in the original commit
+
+commit d0c0c2fce39091915428e815bc6aee265ac9e351
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Jun 30 14:30:08 2016 +0300
+
+    librbd: discard hangs when 'rbd_skip_partial_discard' is enabled
+    
+    Fixes: http://tracker.ceph.com/issues/16386
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit dc41731fbfd73d9fbb63d3ff360d4c5dd62deaf1)
+
+commit 5bf4398e373ecc26958d8480171f4322341a9d82
+Author: xinxin shu <shuxinxin@chinac.com>
+Date:   Tue Jun 7 12:07:55 2016 +0800
+
+    librbd: object_may_exist always return true when you write an empty object
+    
+    if you write an empty object, object map is updated firstly
+    
+    Signed-off-by: xinxin shu <shuxinxin@chinac.com>
+    (cherry picked from commit a54073808146d205d54d6a932a6e0fd750f1dc38)
+
+commit 67861061fcd85f143e7d26077c71a582c6439745
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jul 25 12:43:13 2016 -0400
+
+    librbd: ensure that AIO ops are started prior to request
+    
+    Fixes: http://tracker.ceph.com/issues/16708
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6064f2346de0a8bf2878bf5bfac9a992cda7c4ca)
+
+commit 47279f8e0158d0483011bca01455ef9735453b34
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jul 25 12:42:26 2016 -0400
+
+    librbd: helper method for creating and starting AioCompletions
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3df7213c0a9f0186e3a37c9e4a10f1c8bc84446e)
+
+commit 38433ad90b81b4e398ae6f4b8e196af1a9ea20b8
+Merge: b98e27c 9fc5b41
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 19 20:35:19 2016 +0200
+
+    Merge pull request #10649 from dachary/wip-16867-jewel
+    
+    jewel: mkfs.xfs slow performance with discards and object map
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit d3367354b02be7ee231a50b9ad8bca098f840f46
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 8 14:41:00 2016 -0400
+
+    rbd-mirror: potential assertion failure during error-induced shutdown
+    
+    Fixes: http://tracker.ceph.com/issues/16956
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6a465d9dad417e8b52909c5478f7e3e433748948)
+
+commit d1e05127b73c53a02944edc267548656732231fd
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Aug 10 12:50:53 2016 -0400
+
+    rbd-mirror: potential race condition during failure shutdown
+    
+    Fixes: http://tracker.ceph.com/issues/16980
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 74ec7c91f17630c77647cfc9813090d688b3410d)
+
+commit 48ee3aacef8b1628f41936bd13cc0327c44e59ef
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Aug 9 08:13:30 2016 -0400
+
+    rbd-mirror: replaying state should include flush action
+    
+    Fixes: http://tracker.ceph.com/issues/16970
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7246f1b771b4d1f336ac11b9e881c9eb32bcd4e1)
+
+commit e098fb14f106bf294780abd5d580244e7ce3b20c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 15 14:10:27 2016 -0400
+
+    doc: added rbdmap man page to RBD restructured index
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4e05cbf7e6eda797a9b08e0081aead19523d5762)
+
+commit b7793d757fca99aa2eb371934e667b3cc555d666
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 15 14:07:43 2016 -0400
+
+    rbdmap: specify bash shell interpreter
+    
+    Fixes: http://tracker.ceph.com/issues/16608
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 51237c33d1ec4034c5c07f5d63d63838a76bce88)
+
+commit b98e27c42f29f4fa44cbe7f4b7d9ab3633404f0c
+Merge: eb706ab 92d7882
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 19 07:12:40 2016 +0200
+
+    Merge pull request #10684 from dillaman/wip-16904-jewel
+    
+    jewel: rbd-mirror: reduce memory footprint during journal replay
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 92d7882661d632af9e9f07817610297411c0248f
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Jun 30 16:18:56 2016 +0300
+
+    rbd-mirror: remove ceph_test_rbd_mirror_image_replay test case
+    
+    Fixes: http://tracker.ceph.com/issues/16539
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 06a333b116383de4d39c9e098e6e5fa195ceb370)
+
+commit 0d9d57385bcb544115ccdb00d6bbd67b987dbad0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jul 20 16:18:23 2016 -0400
+
+    qa/workunits/rbd: override rbd-mirror integration test poll frequency
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 574be7486ad737892422aed0322f80e5750a75a0)
+
+commit 96cdb11934e359024c1a2f7cbffd55a5c7f6715b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jul 21 07:28:54 2016 -0400
+
+    rbd-mirror: do not cancel maintenance ops with missing finish events
+    
+    librbd will replay these ops when opening an image, so rbd-mirror
+    should also ensure these ops are replayed.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 862e581553fff510286b58135a1fd69705c06096)
+
+commit d47a23272765633df2303e142d67d78af5bd2bf4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 19 15:46:49 2016 -0400
+
+    rbd-mirror: potential memory leak when attempting to cancel image sync
+    
+    The cancel request could race with the actual scheduling of the image
+    sync operation.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e6cdf955bad500561ddada2791641eba5fb27762)
+
+commit 1e4d98cdd4c5d671870735823a80e80751ba13ae
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 19 15:42:27 2016 -0400
+
+    rbd-mirror: fix issues detected when attempting clean shut down
+    
+    Fixed lockdep issue from status update callback and fixed the
+    potential for a stuck status state.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0275c7ca23b27dc5250cd33f317e2273470a9fe8)
+
+commit e4c43190b49e94ccfeda05ac7a7a8835d81b49a5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 19 13:50:20 2016 -0400
+
+    rbd-mirror: shut down image replayers in parallel
+    
+    When multiple pools are being replicated, start the shut down
+    process concurrently across all pool replayers.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 73cdd08007c27d2c3c41fe644601e7a144f21c82)
+
+commit 1e3821268fbd17581a1352401af626a3e56ee3d4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 19 00:50:14 2016 -0400
+
+    rbd-mirror: configuration options to control replay throttling
+    
+    Fixes: http://tracker.ceph.com/issues/16223
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 24883e0605907d1f9bcd1206c8a95c3bde30d5dc)
+
+commit 549aada7f90b9f7d4a4447d2794c18d5f61807a5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jul 20 08:11:53 2016 -0400
+
+    librbd: new configuration option to restrict journal payload size
+    
+    Ensure that, by default, IO journal events are broken up into manageable
+    sizes when factoring in that an rbd-mirror daemon might be replaying
+    events from thousands of images.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 11d7500b9bcda7b7c1d8756ade3373f404257f32)
+
+commit 99195e89e4d229922c6e88aed6a024619e939040
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 19 00:42:16 2016 -0400
+
+    librbd: wait for journal commit op event to be safely recorded
+    
+    Operation request op finish events should not be fire and forget.
+    Instead, ensure the event is committed to the journal before
+    completing the op. This will avoid several possible split-brain
+    events during mirroring.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 47e0fbf231e52d00069c97b72c57c3158445bcf0)
+    
+    Conflicts:
+            src/test/librbd/operation/test_mock_ResizeRequest.cc: no shrink restriction
+
+commit 3ae52ebadb5ef0de63ba30f937dcbcad507a7048
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jul 20 16:17:41 2016 -0400
+
+    journal: optimize speed of live replay journal pruning
+    
+    When streaming playback, avoid the unnecessary watch delay when
+    one or more entries have been pruned.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 08a8ee98c03b5dfb30341c8d209f0c231b2c5d27)
+
+commit 3850ded99728d1d6acfcaa72cf3923e791dd8fed
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jul 20 10:04:21 2016 -0400
+
+    journal: possible deadlock during flush of journal entries
+    
+    If a future flush is requested at the exact same moment that an
+    overflow is detected, the two threads will deadlock since locks
+    are not taken in a consistent order.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2c65471de4b0f54b8ed722f5deaf51ba62632e37)
+
+commit b4b984325f67a617c890ce6ccfbea5f42322cec5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jul 20 09:15:26 2016 -0400
+
+    journal: improve debug log messages
+    
+    rbd-mirror debugging involved potentially thousands of journals
+    concurrently running. The instance address will correlate log
+    messages between journals.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 11475f4fe740cccdfea459ebeabdca8cb94dc911)
+
+commit 196de276d5f1102a1cb4c7600252c9f1e783acc6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jul 18 15:34:53 2016 -0400
+
+    journal: support streaming entry playback
+    
+    Now that it's possible for the ObjectPlayer to only read a
+    partial subset of available entries, the JournalPlayer needs
+    to detect that more entries might be available.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 28d5ca16cbcb445f985469413b2a8a3048ab66b7)
+
+commit b08335d052ba04765690397f70d0d7f9b54898ac
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jul 18 15:15:58 2016 -0400
+
+    journal: replay should only read from a single object set
+    
+    Previously it was prefetching up to 2 object sets worth of journal
+    data objects which consumed too much memory.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2666d366645b22a5db2a2bcbfce466726bf0b3c0)
+
+commit c6f5303fb2d12830d0c9ac8605884b0c89d16539
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jul 20 08:06:13 2016 -0400
+
+    journal: optionally restrict maximum entry payload size
+    
+    Journal playback will need to read at least a full entry which was
+    currently limited to the maximum object size. In memory constrained
+    environment, this new optional limit will set a fix upper bound on
+    memory usage.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8c1877b82fee0db1dba76252b32ff348226d41a7)
+
+commit 6dc609f24e54c8363a5acecef982aa67beb6130e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jul 18 11:01:26 2016 -0400
+
+    journal: optionally fetch entries in small chunks during replay
+    
+    Support fetching the full object or incremental chunks (with a
+    minimum of at least a single decoded entry if available).
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f7362e9a57e484fffd840ca0eef01778dcacb65b)
+
+commit 91c70f34edabfb3983da643478d746568ed6b738
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jul 18 09:31:40 2016 -0400
+
+    journal: helper class for organizing optional settings
+    
+    Additional runtime configuration settings will be needed. The
+    new class will avoid the need to expand the constructor.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit dad8328f2d502d18923c35f7b86a0cc2ccec133a)
+
+commit d68fe79eedc475c11858c9da5cb3fd2b66ed2b2f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jul 11 15:32:45 2016 -0400
+
+    rbd-mirror: preprocess journal events prior to applying
+    
+    Fixes: http://tracker.ceph.com/issues/16622
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4df913d10b2dd0444db806fccb2812547efa1b56)
+
+commit 01f5f3f15c0da44588b4644905953e234a551def
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jul 8 16:19:52 2016 -0400
+
+    rbd-mirror: event preprocessor to handle snap rename operations
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit fdfca557370c9d86acb81d50edb6aafc42044747)
+
+commit f71dc87e509ada03b40b2bec5ea82579709e0d2d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jul 8 15:16:04 2016 -0400
+
+    librbd: improve journaling debug log messages
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 270cb74bc276bfb1f828a6a6933fa827f6cdce42)
+    
+    Conflicts:
+            src/librbd/journal/Replay.cc: no snap limit restriction
+
+commit 14db53587aa9a918174b616f006c38d1b4c99e58
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jul 8 14:37:14 2016 -0400
+
+    librbd: separate journal event decoding and processing
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 57cd75e8058b84b5dce38f3d8f4b7b4138ac6c9a)
+
+commit 0c4a73fb422ea78e4c5db0763866206cbb54f120
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jul 8 09:14:58 2016 -0400
+
+    librbd: record original snap name in snap rename journal event
+    
+    Remote peers need a key to map snapshot ids between clusters.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f70b90c48d9520bbb4bb29058375e8205cf63771)
+
+commit 1a25490367343d7d4083961163c62f1c32cac105
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jul 8 09:13:07 2016 -0400
+
+    librbd: simple duplicate op checks for all maintenance operations
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 77699bfe749bc7a898024638fb8347c53fe12123)
+    
+    Conflicts:
+            src/test/librbd/mock/MockOperations.h: no shrink restriction
+
+commit 45a0b74a3eb50395a8e5ebb0f2a87add5090ddc1
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jul 7 22:16:51 2016 -0400
+
+    qa/workunits/rbd: exercise snapshot renames within rbd-mirror test
+    
+    Snapshot rename operations utilize the (cluster) unique snapshot
+    sequence to prevent attempts at replays. When mirroring to a
+    different cluster, these sequences will not align.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2f4cb26d8bbd1457bc261547103b56ad40b3c464)
+
+commit c7ab24e7ed63dd1754753c7b0e5c8981a8454a0b
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Jun 15 11:31:14 2016 +0300
+
+    librbd: re-register watch on old format image rename
+    
+    The watching object name is changed when renaming an old format image,
+    so unregister the watcher before the rename, and register back after,
+    to avoid "Transport endpoint is not connected" error.
+    
+    Fixes: http://tracker.ceph.com/issues/16321
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 1a3973c8814200dc449a557a4fc8212911633985)
+
+commit 46fdba458750a13d217708ae3209b0f8bd058072
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 23 20:28:33 2016 -0400
+
+    rbd-mirror: gracefully restart pool replayer when blacklisted
+    
+    Fixes: http://tracker.ceph.com/issues/16349
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2f55aa5e33b2fe242ebb9702ba9ff6f8d5cef96c)
+
+commit a806cdbb057e6bf14322ec6c3da8670ea2e9cfdc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 23 20:28:02 2016 -0400
+
+    rbd-mirror: do not clear image replayer error code after shut down
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1fc27545c2092c178293ed636b6aadb11bc8cbd3)
+
+commit b88a851d96b224d8332aa76a61d17a659dd5d14e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 23 16:57:50 2016 -0400
+
+    rbd-mirror: image deleter should use provided librados connection
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 03c2aec4c613f11fcfed5bcd47855005f42abd79)
+
+commit 4b05677581ce9910e356e84203000251c92d8405
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 23 16:03:03 2016 -0400
+
+    rbd-mirror: each pool replayer should use its own librados connection
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 48f301decbf1f27937bb77a3b47e54933f272d7d)
+
+commit 569fbee99d1367bc0d51e96e42dcf64eeb930648
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 27 09:05:33 2016 -0400
+
+    rbd-mirror: clean up ImageReplayer before stopping state machine
+    
+    Fixes: http://tracker.ceph.com/issues/16489
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c97f7245a015dbdda25584134840d023fd65cdd1)
+
+commit eb706abd814fbd5cc704f21f3f9839ff09aaea6e
+Merge: 3167918 fcc00f7
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 17 19:17:56 2016 +0200
+
+    Merge pull request #10679 from dillaman/wip-16735-jewel
+    
+    jewel: rbd-nbd does not properly handle resize notifications
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 3167918679b519ae7d2625ea08b96419111c9992
+Merge: e6f9f28 2afc176
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 17 19:10:04 2016 +0200
+
+    Merge pull request #10148 from SUSE/wip-16599-jewel
+    
+    jewel: rgw: Swift API returns double space usage and objects of account metadata
+    
+    Reviewed-by: Pritha Srivastava <prsrivas@redhat.com>
+
+commit 55460e2e7b0f1298796b37b38a64820a8f23f6e2
+Author: Shilpa Jagannath <smanjara@redhat.com>
+Date:   Sat Aug 6 14:25:48 2016 +0530
+
+    rgw: raise log levels for common radosgw-admin errors
+    
+    Currently while running radosgw-admin command we see these messages thrown at
+    the endusers since debug level is set to 0:
+    
+    2016-08-02 14:48:28.687251 7f9e630639c0  0 Cannot find zone
+    id=7a2a89b9-4f5e-4f60-a29e-451ac9acc5a8 (name=us-west), switching to local
+    zonegroup configuration
+    2016-08-02 14:48:35.742185 7f9e630639c0  0 error read_lastest_epoch
+    .rgw.root:periods.a3db7884-2445-4bab-a165-6730f8573b8f:staging.latest_epoch
+    
+    Raising the log level to 1.
+    
+    Fixes: http://tracker.ceph.com/issues/16935
+    Signed-off-by: Shilpa Jagannath <smanjara@redhat.com>
+    (cherry picked from commit 8ff2b8e71abd34648c8eefb6b158ae5de46bd709)
+
+commit 73cc60862891cba8581370e3d5726a9f75ba1651
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Aug 5 11:28:13 2016 -0400
+
+    rgw: fixes for period puller
+    
+    * reset epoch=0 before RGWPeriod::init() so we get the latest_epoch
+    * if we're the metadata master, don't try to pull periods from ourself
+    
+    Fixes: http://tracker.ceph.com/issues/16939
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 06c384f1c5aa5b4b6404a3caf8a5f88ee40b0b03)
+
+commit f29864302648f38050626086bb87d47448a72aaf
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Jul 15 17:18:23 2016 +0200
+
+    rgw: ONLY improve code formatting in rgw_object_expirer_core.{cc,h}.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit bc1ecdfb42bf9c9c5825af035409359536ed5b1e)
+
+commit e7be5a544a712142742323d573586dc9e98cbd77
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Jul 15 17:11:04 2016 +0200
+
+    rgw: fix skipping some objects to delete by RadosGW's object expirer.
+    
+    Wei Qiaomiao has found that expired objects, which should be removed by
+    the object expirer of RadosGW, might be left unprocessed till next
+    restart of a RadosGW's instance. This happens when process_single_shard
+    method of RGWObjectExpirer class exhaust a time slot for a single round
+    and finishes without informing caller about the situation.
+    
+    Fixes: http://tracker.ceph.com/issues/16705
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 99f7d6eb565cee116c662305411df2adef002cae)
+
+commit af7e1e0dde953a56f33197c80da499e89f3dfc93
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Jul 15 17:05:37 2016 +0200
+
+    rgw: fix trimming object expirer's hints.
+    
+    Fixes: http://tracker.ceph.com/issues/16684
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 83cd7852ed1fd8350d7c95a1b8811ffd34d7789f)
+
+commit dc154cd1e497b02386b91fde28a658d79ec6659e
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Jul 25 11:12:14 2016 +0800
+
+    osd: fix the mem leak of RepGather
+    
+    ReplicatedPG::new_repop() returns a pointer to RepGather with two refcounts,
+    one is held by ReplicatedPG::repop_queue, the other is supposed to be
+    held by the caller of this function. but it's caller
+    ReplicatedPG::submit_log_entries() assigns it to a
+    boost::intrusive_ptr<RepGather>() directly, why by default add_ref() in
+    its constructor. this makes the refcount 3. that's why we have a leak of
+    RepGather in `ReplicatedPG::new_repop(ObcLockManager&&,
+    boost::optional<std::function<void ()>>&&)`.
+    
+    Fixes: http://tracker.ceph.com/issues/16801
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d3a28465fc7b7002f38cff364fdf601f7486add3)
+
+commit e6f9f287e226e44035920827be2a45753f7452d1
+Merge: 84e3810 90d29fc
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 17 13:41:12 2016 +0200
+
+    Merge pull request #10678 from dillaman/wip-16701-jewel
+    
+    jewel: rbd-mirror: include local pool id in resync throttle unique key
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 84e38109472b4a2a169eb8286c6632da4b1edb6e
+Merge: c2defb0 0ff0960
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 17 13:39:02 2016 +0200
+
+    Merge pull request #10614 from SUSE/wip-16959-jewel
+    
+    jewel: rpm: OBS needs ExclusiveArch
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit c2defb0cea4ba9eed88af3f234c248c57ba48f4f
+Merge: 6d7ebb1 461782e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 17 13:38:27 2016 +0200
+
+    Merge pull request #10519 from Abhishekvrshny/wip-16312-jewel
+    
+    jewel: selinux denials in RGW
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 6d7ebb10e6b20f7c3656fd763d96e64ac247e59f
+Merge: 338a58c b26acc0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 17 13:37:51 2016 +0200
+
+    Merge pull request #10364 from SUSE/wip-16750-jewel
+    
+    jewel: ceph-osd-prestart.sh contains Upstart-specific code
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 338a58ca2ed9ba27f9d66c22e725322ce15466af
+Merge: c7a1c81 c1a47c7
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 17 13:36:47 2016 +0200
+
+    Merge pull request #10008 from dachary/wip-16099-jewel
+    
+    jewel: ceph-base requires parted
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit c7a1c812c4aab478ee38db60947e6f686e8a7407
+Merge: 2d5ada4 ad3b788
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 17 13:35:54 2016 +0200
+
+    Merge pull request #9917 from SUSE/wip-16461-jewel
+    
+    jewel: ceph Resource Agent does not work with systemd
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 96ad2d16df6ec43cd730a4c21b866524a188c532
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jun 2 04:28:30 2016 -0700
+
+    rgw: can set negative max_buckets on RGWUserInfo
+    
+    This can be used to disable user's bucket creation
+    
+    Fixes: http://tracker.ceph.com/issues/14534
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 626d795139a6c5104be098780b70500c2de9b8f4)
+
+commit 2d5ada4ef8ee7d6803a903b7395e0c2b19b781ea
+Merge: aa03e1d 00f3fd4
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 16 16:39:22 2016 +0200
+
+    Merge pull request #10303 from ukernel/jewel-16655
+    
+    jewel: ceph-fuse: link to libtcmalloc or jemalloc
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit aa03e1dbe6b243c56ed40eee8734051f434c2ed2
+Merge: 0280773 3da251f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 16 16:21:01 2016 +0200
+
+    Merge pull request #10420 from SUSE/wip-16798-jewel
+    
+    jewel: ceph command line tool chokes on ceph –w (the dash is unicode 'en dash' &ndash, copy-paste to reproduce)
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit dbf8cf0b5d46cc49eaf0a14bcc05a4fb215d98fb
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri May 13 18:23:12 2016 +0200
+
+    rgw: improve support for Swift's object versioning.
+    
+    This patch allows RadosGW to pass the RefStack with an accuracy
+    to the RFC7230 violation issue which is clearly a Tempest bug.
+    
+    Fixes: http://tracker.ceph.com/issues/15925
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 237ad1210f5a6e4f191293ce532ca15869612a93)
+    
+    Conflicts:
+            src/rgw/rgw_op.cc
+              In contrast to master, Jewel doesn't support
+              container quota of Swift API. All tracks of this
+              feature have been eradicated from the patch.
+            src/rgw/rgw_rest_swift.cc
+              Jewel doesn't include boost/optional.hpp but has
+              boost/utility/in_place_factory.hpp. The conflict
+              has been resolved to pull in both headers.
+
+commit 02807732b0f66826c2d110fe06ece15751924760
+Merge: 1e04a35 67eb961
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 16 10:40:05 2016 +0200
+
+    Merge pull request #10520 from Abhishekvrshny/wip-16319-jewel
+    
+    jewel: radosgw-admin: inconsistency in uid/email handling
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 1e04a357a6e9e67c81b5104f57ab61e5ce07a5e3
+Merge: f6e927c 3f76e4a
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Mon Aug 15 17:18:36 2016 -0400
+
+    Merge pull request #10073 from SUSE/wip-16565-jewel
+    
+    jewel: rgw: data sync stops after getting error in all data log sync shards
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit f6e927cfe3e296ebd5e3b22c3f05d717f87cd4ed
+Merge: d2e8692 93d7875
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Mon Aug 15 17:18:06 2016 -0400
+
+    Merge pull request #10524 from Abhishekvrshny/wip-16731-jewel
+    
+    jewel: failed to create bucket after upgrade from hammer to jewel
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit d2e869255869ae68ca419d5f0a3bbfbcf3a91710
+Merge: 87e71aa dc96383
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Mon Aug 15 17:16:41 2016 -0400
+
+    Merge pull request #10523 from Abhishekvrshny/wip-16700-jewel
+    
+    jewel: rgw: segmentation fault on error_repo in data sync
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 87e71aae7a2d73b7b5444e1a5ab283ece570e3fb
+Merge: cc60230 fe57ace
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Mon Aug 15 17:16:06 2016 -0400
+
+    Merge pull request #10518 from Abhishekvrshny/wip-16272-jewel
+    
+    jewel: rgw ldap: fix ldap bindpw parsing
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit cc60230eef82173eafa52df3d2015c8c0473d236
+Merge: 31264f4 489f8ce
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Aug 15 17:59:51 2016 +0200
+
+    Merge pull request #10144 from dillaman/wip-fix-missing-return-jewel
+    
+    jewel: librbd: fix missing return statement if failed to get mirror image state
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 31264f4126f79c26cff73b2a4d2bea77426c5cd9
+Merge: 4ee9117 c6546bb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Aug 15 17:55:42 2016 +0200
+
+    Merge pull request #10552 from ceph/wip-16507-jewel
+    
+    jewel: expose buffer const_iterator symbols
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 4ee9117d52d41a72444722afe81ce2074dbf58d3
+Merge: 5b7899e 3d6d36a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Aug 15 17:53:40 2016 +0200
+
+    Merge pull request #10497 from Abhishekvrshny/wip-16586-jewel
+    
+    jewel: partprobe intermittent issues during ceph-disk prepare
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 5b7899e89154eceb53625cd6f7ae33984298857f
+Merge: c65536c bd70d6d
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Aug 15 08:35:57 2016 -0700
+
+    Merge pull request #10716 from ceph/wip-16975
+    
+    qa: add rados test script for upgrades
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit c65536cf4f3433fea2cc4683da930676f2def223
+Merge: 40e3e2e 5ffdc34
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 15 08:28:32 2016 -0400
+
+    Merge pull request #10732 from dillaman/wip-rbd-default-format-jewel
+    
+    doc: format 2 now is the default image format
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 5ffdc344401e91c4db89f9ed017adb15e535ebb4
+Author: Chengwei Yang <yangchengwei@qiyi.com>
+Date:   Fri Aug 12 17:44:16 2016 +0800
+
+    doc: format 2 now is the default image format
+    
+    Signed-off-by: Chengwei Yang <yangchengwei@qiyi.com>
+    (cherry picked from commit c8c92f8a2c2106d11b18c70c4c183c3724c2f3c2)
+
+commit 40e3e2e075f21c2e4d36219747d6fdba530d5ed8
+Merge: f58ca19 2c39d36
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Aug 15 11:14:58 2016 +0200
+
+    Merge pull request #10217 from SUSE/wip-16636-jewel
+    
+    jewel: rgw: document multi tenancy
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f58ca19f36179a64e50e2d14555b1efbb8fbfa79
+Merge: 40fc75e 546141c
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Aug 15 10:02:12 2016 +0200
+
+    Merge pull request #9453 from wido/jewel-issue-15348
+    
+    jewel: rgw: Set Access-Control-Allow-Origin to a Asterisk if allowed in a rule
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 40fc75ef4fafada6aa3fac47963da060f50646b0
+Merge: ec14cf5 3ff6e8f
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Fri Aug 12 17:41:07 2016 +0200
+
+    Merge pull request #10693 from dreamhost/wip-15975-jewel
+    
+    rgw: Fallback to Host header for bucket name.
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit bd70d6ddba1bc237a6ffafa0dadea16b2c65a39c
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Aug 11 20:39:20 2016 -0700
+
+    qa: remove tmap_migrate tests from upgrade testing
+    
+    tmap_upgrade only exists in jewel
+    
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 49db733bad375645214ddb8f9b95b8817f9b41e1
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Aug 11 16:05:04 2016 -0700
+
+    qa: add rados test script for upgrades
+    
+    Disable master tests that are not meant to work on jewel
+    
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 3ff6e8fa852346f4b69cd8c10b2f14ae4983d527
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Thu May 26 15:41:20 2016 -0700
+
+    rgw: Fallback to Host header for bucket name.
+    
+    RGW should fallback to using the Host header as the bucket name if valid &
+    possible even when it is NOT a suffix match against the DNS names, or a match
+    against the CNAME rule.
+    
+    This mirrors AWS S3 behavior for these cases (The AWS S3 does not do any DNS
+    lookups regardless).
+    
+    Backport: jewel
+    Fixes: http://tracker.ceph.com/issues/15975
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit 46aae19eeb91bf3ac78a94c9d4812a788d9042a8)
+
+commit 90d29fc94a826b1f56ff99e5b5d81a735abbc4cd
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jul 11 21:58:45 2016 -0400
+
+    rbd-mirror: include local pool id in resync throttle unique key
+    
+    Fixes: http://tracker.ceph.com/issues/16536
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8ad36cab7cbf3492bfa972b43e4a5f75a110bfe6)
+
+commit 03c76148ef9e10fad649a407374f4fc855529377
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 28 15:18:51 2016 -0400
+
+    test: ensure unique template specializations per test case
+    
+    With optimizations disabled, there can be cross translation unit
+    symbol leaking that is not present when inlining is enabled.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 995a16ebcbe79fd14e6de3b25087373744238294)
+
+commit ea9e031e2144ee32b3776cfe2237681f7065861b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 28 14:59:17 2016 -0400
+
+    test: missing template specialization for unittest_rbd_mirror
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 187e2c87b50d64e93736a42fcf449c55e8550125)
+
+commit c2e6d08469ea733dac984770c30d8ab6356f3eca
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jun 15 17:49:54 2016 -0400
+
+    qa/workunits/rbd: increase writes in large image count test
+    
+    This will help to test edge cases where the remote image does
+    or does not own the exclusive lock when the sync starts.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d16698f96c65b9627cc1f64ef80b7f1e39d69d45)
+
+commit 88a7c5c1b28c6a077d35577787757b0a98b7b2a5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jun 15 17:42:59 2016 -0400
+
+    rbd-mirror: prune sync points referencing missing snapshots
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 378914f6b8a8a8c5517fee8fa186268078fbcbfb)
+
+commit 9a6bad3bd7eaa733042c0a23aaf48be4fa6f84ce
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jun 15 16:34:44 2016 -0400
+
+    rbd-mirror: ignore empty snapshot sequence mapping
+    
+    This invalid condition will be gracefully detected and handled
+    when the snapshot mappings are computed.
+    
+    Fixes: http://tracker.ceph.com/issues/16329
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 58ed8a18e91401333bc3f3f957ce5d715b6687b6)
+
+commit c8089dea98fd40651664ae80b0e16834bf727540
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 27 13:01:13 2016 -0400
+
+    qa/workunits/rbd: remove temporary image exports
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 45498d04990861a034e2aad69a93658e018c991a)
+
+commit bdb2189c94badb6e389a3a05794ad8af3eceda1c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 27 11:13:29 2016 -0400
+
+    rbd-mirror: fix potential image replayer state transition race
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3e224c6c03a6a5de0e179bd788387cfa3ff49e9d)
+
+commit e7d7990958bdd8add7cfdcf037bcda5dd05c59d4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 27 11:06:57 2016 -0400
+
+    rbd-mirror: cancel image deletion callback on shut down
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c3f1cb3a34e1b4d258877d519e683e25bf65c65a)
+
+commit bba7811489170e01471ef7c086066a006f57fb6b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 27 10:58:09 2016 -0400
+
+    rbd-mirror: fixed potential leaking image deletion context callback
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9d1cd90c220f95d23d2694b4f1fb6b2fed073ced)
+
+commit b71efb090708e30225502d0cbf974b880def1eb8
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 27 09:45:25 2016 -0400
+
+    rbd-mirror: ImageReplayer doesn't need image deleted callback
+    
+    The Replayer will detect that the ImageReplayer is stopped and will
+    schedule the "wait for deleted" callback before attempting to
+    restart the ImageReplayer.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7e5afc71dfd3ec3e7b0ddaca96f92ac6e0414006)
+
+commit b657d1849a37bc4461edb18337078041a5460818
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 27 09:21:05 2016 -0400
+
+    rbd-mirror: use async callback when deletion not in-progress
+    
+    Fixes: http://tracker.ceph.com/issues/16491
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c136f443366fe804057a9c1f8c01c3f0a33a4d63)
+
+commit c9c1216b3c06e49d3f0addeb7a4f6f5796c00d30
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Thu Jun 9 10:40:23 2016 +0100
+
+    rbd-mirror: tests: ImageSyncThrottler unit tests
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 8ca9a84f36efa73ad17fd27f545dff3716cd798c)
+
+commit c56d6ec4c1898e710aad307b5d3696b9b159ba0c
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Wed Jun 8 16:38:01 2016 +0100
+
+    rbd-mirror: Usage of image-sync throttler in BootstrapRequest
+    
+    Fixes: http://tracker.ceph.com/issues/15239
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 6a91146255d84229688bd8b378732be5975a778b)
+
+commit 5323ebd40a96484f5341574ed8a783c2e696f5dc
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Wed Jun 8 16:37:20 2016 +0100
+
+    rbd-mirror: Implementation of image-sync throttler
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit c4f926d6980d1efd95771885a11d8cc4ebd2e4c3)
+
+commit 783bd686c451ae816ea9c7001b3c6a0ef1b5f554
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Tue Jun 14 16:43:19 2016 +0100
+
+    rbd-mirror: tests: Support for inflight image sync point update
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit cee543bd96bc99f219024aedbfafc5dcd99abb5a)
+
+commit b878f5ffc75e70b1f09eee461f93b1cb5031586e
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Thu Jun 2 10:04:41 2016 +0100
+
+    rbd-mirror: image-sync: Periodically update sync point object number
+    
+    Fixes: http://tracker.ceph.com/issues/15108
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit c2eedf4d27b73d7eaf3fda54f9b0ee74e455bc1d)
+
+commit 2ba61a73b1c616fab5acb1b92fe0a0b7ce7b64cb
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Mon Jun 27 11:07:41 2016 +0100
+
+    rbd-mirror: image-replayer: Fix bug in resync listener remotion
+    
+    Fixes: http://tracker.ceph.com/issues/16488
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 11a5851e90bca2c1813156bf12dda0192965db37)
+
+commit cbd1ab461d5d75baf6e8387788740a6bcb3520de
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Thu May 19 12:06:08 2016 +0100
+
+    rbd-mirror: resync: Added unit tests
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 8953825956f2a560d17c7b651d1724a95f2f7d1d)
+
+commit 5fe93a4383521b7b67d7f168e4f20735440bb8ca
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Thu May 12 18:12:33 2016 +0100
+
+    rbd-mirror: image-replayer: Implementation of resync operation
+    
+    Fixes: http://tracker.ceph.com/issues/15670
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit f2b114238103315197a9eefc8490786c484c8520)
+
+commit c2b786ca92728ef6ceeb836133598a837cd18a2c
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Thu May 12 18:10:38 2016 +0100
+
+    rbd: journal: Support for listening updates on client metadata
+    
+    Currently we only support listening for image resync requests.
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 0dd85739e9034912b86250ced2834dd7fb3d92fd)
+
+commit 21d2ba5eefd1ef904360ce09b4335df77c952d95
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Thu May 12 18:09:09 2016 +0100
+
+    journal: Support for registering metadata listeners in the Journaler
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit c535eb0399cd649371db233ee1ed01cf5441a4b3)
+
+commit fcc00f7ac2c9ed517d12ef9ebab735bdd0d1a162
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Jun 27 10:53:09 2016 +0300
+
+    test: fsx: fix rbd-nbd daemon logging
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 09710c46d3811e602250f858fe7292925c1e3c06)
+
+commit 63fbed0f39da22374f6818030674c6b5e73f41dd
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Fri Jun 3 21:11:21 2016 +0300
+
+    qa/workunits/rbd: image resize test for rbd-nbd
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit fa58acbb6fcc79e5a9bfd4b86250a0e2b9c8efa1)
+
+commit 3db1a3c8ea5a6829e1e44ecd0c96316c759bf048
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue May 24 15:52:00 2016 +0300
+
+    rbd-nbd: use librbd API method to watch image size update
+    
+    Fixes: http://tracker.ceph.com/issues/15715
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 1e1d3b1609432ea80d6fd1c5b3df49f38ad579f1)
+
+commit 8f94f5e3f7ba0cb3578952ffbca85c0dc6a772a0
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue May 24 15:50:22 2016 +0300
+
+    librbd: API: methods to watch image stat update
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 9952b75a0c8c9bed9792cb77671fd5588f32630f)
+    
+    Conflicts:
+            src/include/rbd/librbd.h: removed CG references
+            src/tracing/librbd.tp: removed CG references
+
+commit ec14cf5ce3e8d04eee3f13aadcfc2d7f22c5a633
+Merge: c587e9e 393bf7e
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Aug 11 11:31:40 2016 +0200
+
+    Merge pull request #9790 from SUSE/wip-16381-jewel
+    
+    jewel: comparing return code to ERR_NOT_MODIFIED in rgw_rest_s3.cc (needs minus sign)
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 22c9df1d9bef3f48ab2a12357035fc54fbde141d
+Author: Pritha Srivastava <prsrivas@redhat.com>
+Date:   Fri Jul 1 14:15:42 2016 +0530
+
+    rgw: Fix for using port 443 with pre-signed urls.
+    
+    Fixes http://tracker.ceph.com/issues/16548
+    
+    Signed-off-by: Pritha Srivastava <prsrivas@redhat.com>
+    (cherry picked from commit 44b9ed37ac659d83fbd4bb99c69da6b9dec06b8d)
+
+commit ce5e250405449c6035b805bbcdcca73ad4399d68
+Author: Shilpa Jagannath <smanjara@redhat.com>
+Date:   Fri Jun 17 18:29:42 2016 +0530
+
+    rgw: added zone rename to radosgw_admin
+    
+    Added zone rename option to radosgw-admin help
+    
+    Signed-off-by: Shilpa Jagannath <smanjara@redhat.com>
+    (cherry picked from commit da3235234eaf41f9dbd378fb0d9994d2c4096947)
+
+commit ce986aa6e8b3ee54697802a361246c256e8097b4
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jul 14 13:38:44 2016 -0400
+
+    rgw: RGWMetaSyncCR holds refs to stacks for wakeup
+    
+    because RGWCoroutine::wakeup() calls RGWCoroutinesStack::wakeup(), the
+    stack must also stay alive
+    
+    Fixes: http://tracker.ceph.com/issues/16666
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit e4bc16044e1b80636855dbc39da1d121a3508308)
+
+commit 2485efc26d68b6848cda02d66b54dd58769fcbd8
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Jul 26 12:50:13 2016 -0400
+
+    rgw: clear realm watch on failed watch_restart
+    
+    if the realm is deleted while the gateway has a watch, it's disconnected
+    and watch_restart() will fail. this results in a watch handle of 0,
+    which leads to a segfault on ~RGWRealmWatcher when we pass it to
+    unwatch()
+    
+    this commit cleans up the watch when watch_restart() fails, so we don't
+    try to unwatch() on destruction
+    
+    Fixes: http://tracker.ceph.com/issues/16817
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit e5b8b5b27502fdacdac38effd7a55fd719911e85)
+
+commit fa2e42d4614619f761a5cf4c86e3db7d7e101e11
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jul 27 12:16:57 2016 -0400
+
+    rgw: use endpoints from master zone instead of zonegroup
+    
+    Fixes: http://tracker.ceph.com/issues/16834
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit d4872ec9f7554ff49f4be336fe59cdd2051ee2a5)
+
+commit a865f26e68a991eaf55dc5f105b7fd7cb4634bc3
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jul 28 09:55:05 2016 -0400
+
+    radosgw-admin: zone[group] modify can change realm id
+    
+    allows the default zone and zonegroup (created with empty realm_id) to
+    be later added to a realm. the 'modify' command now accepts either
+    --realm_id=id or --rgw-realm=name
+    
+    Fixes: http://tracker.ceph.com/issues/16839
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 064b7e953dd6ace2c32b94150e70959e95a01761)
+
+commit c587e9e5c73cd8a990106708a43222fcb5a2a709
+Merge: aacb793 3250c4d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Aug 10 13:33:12 2016 +0200
+
+    Merge pull request #10026 from SUSE/wip-16392-jewel
+    
+    jewel: master: build failures with boost > 1.58
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f5db5a4b0bb52fed544f277c28ab5088d1c3fc79
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue May 17 17:53:56 2016 +0100
+
+    mon: tolerate missing osd metadata
+    
+    Just because one OSD's metadata is missing,
+    don't give up on outputting all the other
+    OSDs' metadata.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 2c7dd5f22288c4aee814573baa6af131421f36a4)
+
+commit 36e5c86469eda59f7d5d15b16e15d0aab81a34fa
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue May 17 14:03:09 2016 +0100
+
+    mon: fix metadata dumps for empty lists
+    
+    Because 'r' was set inside loop, these would
+    fail to set it (leaving ENOSYS) if no
+    metadata is present.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 5fe4fe4d92f5b2d67896beac9b5b3b48b309a6b4)
+    
+    Conflicts:
+            src/mon/Monitor.cc and src/mon/MDSMonitor.cc
+            changes relate to features that are not backported to jewel
+
+commit de99bd53f5efe84647e1a1d79a8db330887b7704
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sun May 15 10:32:28 2016 +0800
+
+    mon: 'std::move` Metadata when updating it
+    
+    to avoid copying of the std::map
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit dd67eaab00698d747bf8669a3065c72ea68f7784)
+
+commit 828df3995430ce89fe9be40c6c19c44fb6f14752
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu May 12 15:49:23 2016 +0100
+
+    mon: fix 'mon metadata' for lone monitors
+    
+    Previously, writing to the store was only
+    triggered when MMonMetadata was received
+    from peers, so if you had a single mon then
+    you would always get ENOENT from "mon metadata"
+    
+    Fixes: http://tracker.ceph.com/issues/15866
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 93ab00c77059e93b52ac4caa703e259944be1d1e)
+
+commit 308f5143a9e9b121fb5c4df0a227ba30bda4636e
+Author: runsisi <runsisi@zte.com.cn>
+Date:   Mon Jul 4 13:52:03 2016 +0800
+
+    ceph.spec.in: fix rpm package building error as follows:
+    
+          error: Installed (but unpackaged) file(s) found:
+          /usr/lib/udev/rules.d/60-ceph-by-parttypeuuid.rules
+    
+    Signed-off-by: runsisi <runsisi@zte.com.cn>
+    (cherry picked from commit 49dfad1f53d7abe0e9efc7817aaaf1735df5a9c9)
+
+commit 727a704e79995c8a74f2b5768c63e90eecf9a883
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Jun 23 09:23:09 2016 +0200
+
+    udev: always populate /dev/disk/by-parttypeuuid
+    
+    ceph-disk activate-all walks /dev/disk/by-parttypeuuid at boot time. It
+    is not necessary when udev fires ADD event for each partition and
+    95-ceph-osd.rules gets a chance to activate a ceph disk or journal.
+    
+    There are various reasons why udev ADD events may not be fired at
+    boot (for instance Debian Jessi 8.5 never does it and CentOS 7.2 seems
+    to be racy in that regard when a LVM root is being used).
+    
+    Populating /dev/disk/by-parttypeuuid fixes ceph-disk activate-all that
+    would not work without it. And it guarantees disks are activated at boot
+    time regardless of wether udev fires ADD events at the right time (or at
+    all).
+    
+    The new udev file is a partial resurection of the
+    60-ceph-partuuid-workaround-rules that was removed by
+    9f77244b8e0782921663e52005b725cca58a8753. It is given a name that
+    reflects its new purpose.
+    
+    Fixes http://tracker.ceph.com/issues/16351
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 35004a628b2969d8b2f1c02155bb235165a1d809)
+
+commit 9fc5b41598f09414572e89572e8980fd3d3c76e0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jul 18 14:03:01 2016 -0400
+
+    librbd: optimize away unnecessary object map updates
+    
+    Fixes: http://tracker.ceph.com/issues/16689
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e5b4188635c4ee1ee0c4353cfc5ecd6e887d536b)
+
+commit 81a2534742e5051fa92549029303cef3f3d187a6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jul 18 13:37:37 2016 -0400
+
+    rbd-replay: decode and replay discard IO operations
+    
+    Fixes: http://tracker.ceph.com/issues/16707
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b7a4db213d0d4812aa6cd9c54e9646ff57d10411)
+
+commit 5968664fd4ddf8e26989360ebbc048faa8431251
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 30 15:00:32 2016 -0400
+
+    librbd: failed assertion after shrinking a clone image twice
+    
+    Fixes: http://tracker.ceph.com/issues/16561
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 34d2297eed040f694b4d1c5e2606af04752c9e1a)
+
+commit 8316b6a6bad28343f8bade63f05b5fb93f4fe415
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 30 10:24:01 2016 -0400
+
+    rbd-mirror: gracefully fail if object map is unavailable
+    
+    If the exclusive lock was lost due to a watch failure from an
+    overloaded cluster, gracefully abort the image sync.
+    
+    Fixes: http://tracker.ceph.com/issues/16558
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6f573ca2d63784e4a2b1e7a410044afdf58f6801)
+
+commit aacb793948a69294e518f8458be6e670df23fe53
+Merge: 954e978 5ae0e43
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 17:09:42 2016 +0200
+
+    Merge pull request #10625 from dachary/wip-16969-jewel
+    
+    jewel: src/script/subman fails with KeyError: 'nband'
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 5ae0e43e7f75a3a6d96d9fc5f03c4b0a86c7481b
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Aug 9 09:13:17 2016 +0200
+
+    subman: use replace instead of format
+    
+    Otherwise all {} are assumed to be substituted. Add a test.
+    
+    Fixes: http://tracker.ceph.com/issues/16961
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit b006c136761746e12704e349a672325b8b6e51f5)
+
+commit 0ca27727173501856e9f17785b9f555684661709
+Author: Ali Maredia <amaredia@redhat.com>
+Date:   Thu Apr 21 20:32:47 2016 -0400
+
+    cmake: script that sets env vars for unit tests
+    
+    Signed-off-by: Ali Maredia <amaredia@redhat.com>
+    (cherry picked from commit 15a669435aaa78562746f6728bb185b1e99a4274)
+
+commit 954e978bc8cd233ac149a2e490d8730d9782583a
+Merge: d387bf4 cbc9636
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:41:30 2016 +0200
+
+    Merge pull request #10054 from dachary/wip-16484-jewel
+    
+    jewel: ExclusiveLock object leaked when switching to snapshot
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit d387bf44d405a3a8658697134e6bfc9ae077dfef
+Merge: 14a4484 cf65ed9
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:28:44 2016 +0200
+
+    Merge pull request #10041 from dachary/wip-16315-jewel
+    
+    jewel: When journaling is enabled, a flush request shouldn't flush the cache
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 14a448466e9bda6b8d565f34b248a0ab6f9f90b3
+Merge: fc39bd8 757babb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:27:50 2016 +0200
+
+    Merge pull request #10055 from dachary/wip-16485-jewel
+    
+    jewel: Whitelist EBUSY error from snap unprotect for journal replay
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit fc39bd804e479fbaefa6f81befd899b8a1a82581
+Merge: fc084a3 d9c3f28
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:27:22 2016 +0200
+
+    Merge pull request #10053 from dachary/wip-16483-jewel
+    
+    jewel: Close journal and object map before flagging exclusive lock as released
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit fc084a32ecf7c1868cbbbd808b273f2d095746f0
+Merge: 543dd04 b1d9698
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:27:08 2016 +0200
+
+    Merge pull request #10052 from dachary/wip-16482-jewel
+    
+    jewel: Timeout sending mirroring notification shouldn't result in failure
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 543dd0460228f778ca10e8ff238c8db3f1b239fe
+Merge: 8fb4121 51defea
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:26:50 2016 +0200
+
+    Merge pull request #10051 from dachary/wip-16460-jewel
+    
+    jewel: Crash when utilizing advisory locking API functions
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 8fb4121c53850f0f60eadb6e2a3bfc8c750e5935
+Merge: 64d5ff9 db28ddc
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:26:40 2016 +0200
+
+    Merge pull request #10050 from dachary/wip-16459-jewel
+    
+    jewel: rbd-mirror should disable proxied maintenance ops for non-primary image
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 64d5ff97ced5cc0cfa4f15b66b54cc2fb11b2c22
+Merge: 4fe02a6 70bf746
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:26:21 2016 +0200
+
+    Merge pull request #10047 from dachary/wip-16426-jewel
+    
+    jewel: Possible race condition during journal transition from replay to ready
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 4fe02a62b7ad2c6b4a32962741a73d8b1db9fb11
+Merge: b86aec9 f3f4a4a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:26:11 2016 +0200
+
+    Merge pull request #10046 from dachary/wip-16425-jewel
+    
+    jewel: rbd-mirror: potential race condition accessing local image journal
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit b86aec99e8375eb2de1d495338ab212e1880a0f7
+Merge: 4a6e48e 0399958
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:26:00 2016 +0200
+
+    Merge pull request #10045 from dachary/wip-16424-jewel
+    
+    jewel: Journal needs to handle duplicate maintenance op tids
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 4a6e48ed85914413b919e4da5a67ee4f95f34ae2
+Merge: adcf5a7 6a28d63
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:25:48 2016 +0200
+
+    Merge pull request #10044 from dachary/wip-16423-jewel
+    
+    jewel: Journal duplicate op detection can cause lockdep error
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit adcf5a7fbf181505c6319c4399572cd3e9259a47
+Merge: eab8994 1e85da9
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:25:35 2016 +0200
+
+    Merge pull request #10043 from dachary/wip-16371-jewel
+    
+    jewel: rbd-mirror: ensure replay status formatter has completed before stopping replay
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit eab89942c53ef5bff15cd837dea8ba42a228dc8f
+Merge: 289c2d2 bf58eab
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:25:11 2016 +0200
+
+    Merge pull request #10042 from dachary/wip-16372-jewel
+    
+    jewel: Unable to disable journaling feature if in unexpected mirror state
+
+commit 289c2d2a512723e039bc1465353dbb39084edb80
+Merge: e01ebb9 dd635e4
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:24:55 2016 +0200
+
+    Merge pull request #10010 from dachary/wip-16486-jewel
+    
+    jewel: Object map/fast-diff invalidated if journal replays the same snap remove event
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit e01ebb9319d17fd71b83bd0ccf9cb0f231a1783e
+Merge: 28575db db7ce96
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Aug 9 08:24:38 2016 +0200
+
+    Merge pull request #10009 from dachary/wip-16514-jewel
+    
+    jewel: Image removal doesn't necessarily clean up all rbd_mirroring entries
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 0ff096073f4c26bbccd9125a345372f090754d63
+Author: Michel Normand <normand@linux.vnet.ibm.com>
+Date:   Fri Aug 5 16:34:48 2016 +0200
+
+    ExclusiveArch for suse_version
+    
+    for SLES supports only x86_64 and aarch64 targets
+    for openSUSE (Tumbleweed and Leap) add ppc64/ppc64le targets.
+    
+    fixes: http://tracker.ceph.com/issues/16936
+    Signed-off-by: Michel Normand <normand@linux.vnet.ibm.com>
+    (cherry picked from commit a8db0f4d96d71223083bf6c2275acd892666a13e)
+
+commit 28575db3fb1579cdfa85b14b0484363cc0634a2e
+Merge: 3e36f8c 1f19dbd
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Aug 8 08:47:34 2016 +0200
+
+    Merge pull request #9952 from dillaman/wip-fix-use-after-free-jewel
+    
+    jewel: librbd: potential use after free on refresh error
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 3e36f8c66326d3de97a79859eab363e509dfc8c2
+Merge: 5c98730 6b41d76
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Aug 8 08:45:58 2016 +0200
+
+    Merge pull request #9752 from dillaman/wip-fix-task-finisher-jewel
+    
+    jewel: librbd: cancel all tasks should wait until finisher is done
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 5c98730854f11b0efb3b3e03be426ce2b7a999af
+Merge: f014619 7e1ba28
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Aug 5 12:46:10 2016 -0700
+
+    Merge pull request #10561 from dzafman/wip-16672-jewel
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit f0146196ccfbcfd923191f63d93e4e81219523b1
+Merge: e400999 c025010
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 5 15:30:52 2016 +0200
+
+    Merge pull request #9562 from Abhishekvrshny/wip-16152-jewel
+    
+    jewel: client: fstat cap release
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit e400999a2cb0972919e35dd8510f8d85f48ceace
+Merge: 0cb5ca6 c3f6d82
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 5 15:30:41 2016 +0200
+
+    Merge pull request #9561 from Abhishekvrshny/wip-16136-jewel
+    
+    jewel: MDSMonitor fixes
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 0cb5ca6db018461c6275636f0d9cc1b1c8f56e53
+Merge: 67f2eb0 19c1366
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 5 15:29:33 2016 +0200
+
+    Merge pull request #9557 from Abhishekvrshny/wip-15898-jewel
+    
+    jewel: Confusing MDS log message when shut down with stalled journaler reads
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 67f2eb01deff699016c16a312448d665cbef3922
+Merge: 7ec15f6 eea0e91
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 5 15:29:23 2016 +0200
+
+    Merge pull request #9560 from Abhishekvrshny/wip-16135-jewel
+    
+    jewel: MDS: fix getattr starve setattr
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 7ec15f69f83e78ac862515c1068a64769caaff91
+Merge: e51060a 2c7fc95
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 5 15:29:08 2016 +0200
+
+    Merge pull request #9559 from Abhishekvrshny/wip-16041-jewel
+    
+    jewel: mds/StrayManager.cc: 520: FAILED assert(dnl->is_primary())
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit e51060ad8190bddf2b9b367c4d0a67e8cce7daef
+Merge: fda98f7 edd3f79
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 5 15:17:22 2016 +0200
+
+    Merge pull request #9996 from dachary/wip-16437-jewel
+    
+    jewel: async messenger mon crash
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+    Reviewed-by: Haomai Wang <haomai@xsky.com>
+
+commit fda98f780134c26e491ce15fca0890dc240d67cf
+Merge: 56bbcb1 18fdc1c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 5 15:10:27 2016 +0200
+
+    Merge pull request #9997 from dachary/wip-16431-jewel
+    
+    jewel: librados,osd: bad flags can crash the osd
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 56bbcb1aa11a2beb951de396b0de9e3373d91c57
+Merge: 7586d69 a826bb8
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 5 15:10:08 2016 +0200
+
+    Merge pull request #9998 from dachary/wip-16429-jewel
+    
+    jewel: OSDMonitor: drop pg temps from not the current primary
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 7586d693e1c4f08a17e60f89fda5abcb31d2f80d
+Merge: 3b28428 6554d46
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Aug 5 15:09:52 2016 +0200
+
+    Merge pull request #10001 from dachary/wip-16427-jewel
+    
+    jewel: prepare_pgtemp needs to only update up_thru if newer than the existing one
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 3b28428f901c1a6ed08b6c23b4f9991b87fe0960
+Merge: 6996b76 762db30
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Fri Aug 5 09:09:23 2016 -0400
+
+    Merge pull request #9743 from vumrao/wip-vumrao-16339
+    
+    jewel : rgw: support size suffixes for --max-size in radosgw-admin command
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 8b9954d0ccab9694c7519002209be8cb9dc03362
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Thu Aug 4 17:15:18 2016 -0700
+
+    rgw: Fix civetweb IPv6
+    
+    Commit c38e3cbb6f7c6221209f2b512cba18c564c895a4 introduced a second
+    compiled version of src/civetweb/src/civetweb.c, but did not pass the
+    configuration header (civetweb/include/civetweb_conf.h).
+    
+    As a result, USE_IPV6 was not defined when it was compiled, and that
+    copy was included into the radosgw binary. This caused breakage for the
+    civetweb frontend when used with IPv6:
+      rgw frontends = civetweb port=[::]:7480
+    
+    Reintroduce the header so that civetweb is compiled correctly again.
+    
+    Fixes: http://tracker.ceph.com/issues/16928
+    Backport: jewel
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit 921c556b65eafe2136f2afcc8234681711aa7348)
+
+commit 6996b76d35b20a558d13d35fafaaad47fe331434
+Merge: 8586e9b 74dd035
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 4 14:45:05 2016 +0200
+
+    Merge pull request #10007 from dachary/wip-15806-jewel
+    
+    jewel: New pools have bogus stuck inactive/unclean HEALTH_ERR messages until they are first active and clean
+    
+    Reviewed-by: xie xingguo <xie.xingguo@zte.com.cn>
+
+commit 8586e9bd49d1fc083a7c37a2b1865744bbcce26a
+Merge: e85b58b a2e8ae6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 4 14:43:36 2016 +0200
+
+    Merge pull request #10006 from dachary/wip-16249-jewel
+    
+    jewel: sparse_read on ec pool should return extends with correct offset
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit e85b58bbe6aa217e4f9c5cef052aabedf728b9fe
+Merge: f05eaae f8e4911
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 4 14:43:22 2016 +0200
+
+    Merge pull request #9740 from vumrao/wip-vumrao-16338
+    
+    jewel : rados: Add cleanup message with time to rados bench output
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit f05eaaea3f39159cc5e1a127a7d9913f479a7bf4
+Merge: 093f30d b8f7aa2
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 4 14:42:53 2016 +0200
+
+    Merge pull request #10004 from dachary/wip-16374-jewel
+    
+    jewel: AsyncConnection::lockmsg/async lockdep cycle: AsyncMessenger::lock, MDSDaemon::mds_lock, AsyncConnection::lock
+    
+    Reviewed-by: Haomai Wang <haomai@xsky.com>
+
+commit 093f30d1788b4a7225e3ab1c3345db14e40a5237
+Merge: 648442b 3dbb08a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 4 14:42:32 2016 +0200
+
+    Merge pull request #10003 from dachary/wip-16380-jewel
+    
+    jewel: msg/async: connection race hang
+    
+    Reviewed-by: Haomai Wang <haomai@xsky.com>
+
+commit 648442bfea4ea72c04fad441fc933d03d980f80c
+Merge: a35e2fa caad884
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 4 12:35:14 2016 +0200
+
+    Merge pull request #10036 from liewegas/wip-16297-jewel
+    
+    mon: Monitor: validate prefix on handle_command()
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit a35e2fa5f1a22ea5798c559196d50d1d38d7679e
+Merge: e499d1c 7b0318a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Aug 4 09:53:10 2016 +0200
+
+    Merge pull request #9547 from jcsp/wip-jewel-15705
+    
+    jewel backport: mds: fix mdsmap print_summary with standby replays
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 7e1ba28b1e218deb7d54ad6bef8853c9d19411a0
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Aug 2 22:32:02 2016 -0700
+
+    os: Fix HashIndex::recursive_remove() to remove everything but original path
+    
+    Fixes: http://tracker.ceph.com/issues/16672
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit ba88a3aac0d3b620986e32ed718237513d6592f6)
+
+commit cb1f17f05422ac802378df0de698ce20a1f2976e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Aug 2 23:26:41 2016 -0700
+
+    filestore: Clear objects before calling collection_list() again
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 892303cd02b64b622669183c2acffcdd7a8f3547)
+
+commit bdcfcaf821add71c518b01d6fb3dccb7c2e074a7
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Aug 2 23:24:12 2016 -0700
+
+    filestore: Improve logging
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b78c457785553e3a6e4f901bfebb41fc9c4d9bd5)
+
+commit e499d1c8fb216b490a739f2712b3c733623d45d1
+Merge: 269d742 8981f3b
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Tue Aug 2 10:28:00 2016 -0400
+
+    Merge pull request #10293 from theanalyst/wip-16589-jewel
+    
+    jewel: multisite sync races with deletes
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 269d7427dfa9de7b1a288da0532505810bdf283b
+Merge: 894a5f8 e1eb8af
+Author: John Spray <jspray@redhat.com>
+Date:   Tue Aug 2 12:31:18 2016 +0100
+
+    Merge pull request #10453 from ajarr/wip-manila-backports-jewel
+    
+    jewel: essential backports for OpenStack Manila
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit e1eb8afea9f202947eef33e8361a0aac0e955eea
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Thu Jul 7 17:15:13 2016 +0530
+
+    ceph_volume_client: version on-disk metadata
+    
+    Version on-disk metadata with two attributes,
+    'compat version', the minimum CephFSVolume Client
+    version that can decode the metadata, and
+    'version', the version that encoded the metadata.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 1c1d65a45f4574ca5f33cc9d949089a5c956e363)
+
+commit e8dd1eeec7ee2fd11a2985843825a3ec0f522b0c
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Wed Jul 6 15:33:06 2016 +0530
+
+    ceph_volume_client: add versioning
+    
+    Add class attributes to CephFSVolumeClient to version
+    its capabilities.
+    
+    'version' attribute stores the current version number
+    of CephFSVolumeClient.
+    
+    'compat_version' attribute stores the earliest version
+    number of CephFSVolumeClient that the current version is
+    compatible with.
+    
+    Fixes: http://tracker.ceph.com/issues/15406
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 46876fb2ceb22082c0a1703fe77ad1694b508ad8)
+
+commit a0ffc859f5763850342a165ec4c49b3b1e83bb95
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Thu Jun 23 23:22:12 2016 +0530
+
+    ceph_volume_client: disallow tenants to share auth IDs
+    
+    Restrict an auth ID to a single OpenStack tenant to enforce
+    strong tenant isolation of shares.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 82445a20a258a4c8800f273dc5f2484aace0e413)
+
+commit d3e22db9841404d6c305012032427d6bb5311253
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Thu Jun 23 17:11:33 2016 +0530
+
+    ceph_volume_client: cleanup auth meta files
+    
+    Remove auth meta files on last rule for an auth ID deletion
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit ec2e6e37d01c961d269d83661d0b95ada6a8449e)
+
+commit 2e9c37adfb7b9f7e67801ffc2e1fa6c85674c9c1
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Thu Jun 23 16:55:30 2016 +0530
+
+    ceph_volume_client: fix log messages
+    
+    Log the path of the volume during creation and deletion of volumes.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 7731287761f91e1fdee0e6306d9ecf9b04ad363c)
+
+commit 7e93d3d0e631f375a496673174667934d5ecc08e
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Thu Jun 23 16:31:23 2016 +0530
+
+    ceph_volume_client: create/delete VMeta for create/delete volume
+    
+    Create and delete volume meta files during creation and deletion of
+    volumes.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 37fbfc7aa8f10d37f5202603a906425507271ff5)
+
+commit 7cea0eee455c83fbdd64fff192c6aa10235a2c40
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Thu Jun 23 16:06:53 2016 +0530
+
+    ceph_volume_client: modify locking of meta files
+    
+    File locks are applied on meta files before updating the meta
+    file contents. These meta files would need to be cleaned up
+    sometime, which could lead to locks being held on unlinked meta
+    files. Prevent this by checking whether the file had been deleted
+    after lock was acquired on it.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit f7c037229bcf3f5a3d06897ec7fe0c5419dd7143)
+
+commit e77684fc68cfd3dfb33a80e7c5604356921a40b0
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Thu Jun 23 17:39:32 2016 +0530
+
+    cephfs.pyx: implement python bindings for fstat
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit f58403f3d19e22edeb8f91b6f87a0b7947b0ff21)
+
+commit d2792ef1babd452d76528ce5ca511106d8c836b4
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Wed Jun 8 16:57:01 2016 +0530
+
+    ceph_volume_client: restrict volume group names
+    
+    Prevent craftily-named volume groups from colliding with meta files.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 7f7d2a76ae9b556c1de418f0eab8461c538f91d9)
+
+commit 4f874dd9a9c4ce58f623c635c31bb67012a72199
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Wed Jun 8 16:34:56 2016 +0530
+
+    ceph_volume_client: use fsync instead of syncfs
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 27eb51baab4cda6b385aef53fc7c3962a9debae5)
+
+commit bde2c8f111d8276b026a3b1e37a5c9081a498ec9
+Author: Xiaoxi Chen <xiaoxchen@ebay.com>
+Date:   Fri Apr 29 14:47:42 2016 -0500
+
+    pybind: expose fsync in cephfs binding.
+    
+    So we don't necessary to syncfs when want to persistent
+    some file.
+    
+    Signed-off-by: Xiaoxi Chen <xiaoxchen@ebay.com>
+    (cherry picked from commit 1c952fbaf0fd393ef2dcb83a3db721a077b4274e)
+
+commit 0c13bf2d028eab4cd062209be7ed985cdf28960c
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Wed Jun 8 00:42:18 2016 +0530
+
+    ceph_volume_client: recover from dirty auth and auth meta updates
+    
+    Check dirty flag after locking something and call recover() if we are
+    opening something dirty (racing with another instance of the driver
+    restarting after failure) -- only required if someone running multiple
+    manila-share instances with Ceph loaded.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 647a2447f0c4354dc21d1083043591d2b6f6f94f)
+
+commit 8f7defb5d084f8e781fa44e9c21edf6c472cfb4a
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Tue Jun 21 12:14:56 2016 +0530
+
+    ceph_volume_client: modify data layout in meta files
+    
+    Notable changes to data layout in auth meta and volume meta files:
+    
+    In the auth meta files, add a 'dirty' flag to track the status of auth
+    updates to a single volume.
+    
+    In the volume meta file, make the 'dirty' flag track the status of
+    auth updates for a single ID.
+    
+    Optimize the recovery of partial auth update changes to auth meta,
+    volume meta, and the Ceph backend, facilitated by changes in the
+    data layout in the meta files.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 7c8a28a7e817d030a4d419b0cc627c30c1064270)
+
+commit 748a5a08ecc1e212e4bd10e0c8eff0a805fc931f
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 2 12:30:45 2016 +0000
+
+    pybind: ceph_volume_client authentication metadata
+    
+    Store a two-way mapping between auth IDs and volumes.
+    
+    Enables us to record some metadata on auth ids (which
+    openstack tenant created it) so that we can avoid exposing
+    keys to other tenants who try to use the same ceph
+    auth id.
+    
+    Enables us to expose the list of which auth ids have access
+    to a volume, so that Manila's update_access() can be
+    implemented efficiently.
+    
+    DNM: see TODOs inline.
+    
+    Fixes: http://tracker.ceph.com/issues/15615
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit d2e9eb55ca6ed5daa094cf323faf143615b9380b)
+
+commit e14dc25e1fca7fd74971cbda74d1fcadff5be3b4
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Mar 7 13:06:41 2016 +0000
+
+    pybind: enable integer flags to libcephfs open
+    
+    The 'rw+' style flags are handy and convenient, but
+    they don't capture all possibilities.  Change to
+    optionally accept an integer here for advance users
+    who want to specify arbitrary combinations of
+    flags.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 5678584f4176d07301acd7f57acc4efd7fb20e43)
+
+commit c6546bb29b25fb96f270257b959426f949eb9418
+Author: Noah Watkins <noahwatkins@gmail.com>
+Date:   Wed Mar 16 14:12:05 2016 -0700
+
+    buffer: fix iterator_impl visibility through typedef
+    
+    The following program doesn't compile because of symbol visibility issues.
+    While bufferlist::iterator is a class implementation with visibility specified,
+    it is unclear after google-fu how to do the same through typedef.
+    
+    int main()
+    {
+     ceph::bufferlist bl;
+     ceph::bufferlist::const_iterator it = bl.begin();
+     (void)it;
+     return 0;
+    }
+    
+    [nwatkins@bender ~]$ g++ -Wall -std=c++11 -Iinstall/include -Linstall/lib -o test test.cc -lrados
+    /tmp/cciR9MUj.o: In function `main':
+    test.cc:(.text+0x43): undefined reference to `ceph::buffer::list::iterator_impl<true>::iterator_impl(ceph::buffer::list::iterator const&)'
+    /usr/bin/ld: test: hidden symbol `_ZN4ceph6buffer4list13iterator_implILb1EEC1ERKNS1_8iteratorE' isn't defined
+    /usr/bin/ld: final link failed: Bad value
+    collect2: error: ld returned 1 exit status
+    
+    Signed-off-by: Noah Watkins <noahwatkins@gmail.com>
+    (cherry picked from commit 16bc3e59325d4057d36cd63a57111ab36fbc50fd)
+
+commit e825dd2f3fc9048995062bfe96e32e6df7ef37d9
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Fri Jul 22 10:57:45 2016 +0200
+
+    rgw multisite: preserve zone's extra pool
+    
+    In current multisite scenarios,if a bucket is created in master, we end
+    up storing multipart metadata in `$source-zone.rgw.buckets.non-ec` pool
+    instead of the zone's own non-ec pool, so we end up additionally
+    creating this pool and storing multipart metadata entries in it. Also if
+    a bucket is created in a secondary zone, and we initiate a multipart
+    upload, before mdlog sync with master, we end up getting errors during
+    complete multipart requests as omap entries are partly stored in the
+    `$zone.rgw.buckets.non-ec` as well as `$source-zone.rgw.buckets.non-ec`
+    pools which leads to total number of parts mismatch.
+    
+    Fixes: http://tracker.ceph.com/issues/16712
+    
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit d4ecc956f603a567bd4043c2be39b3e7e3402c75)
+
+commit 059ed62ff937ce63b7255d72bf9b3cf687c4b532
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Jul 4 15:01:51 2016 +0200
+
+    rgw: remove bucket index objects when deleting the bucket
+    
+    Fixes: http://tracker.ceph.com/issues/16412
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 3ae276390641ad5fc4fef0c03971db95948880b4)
+
+commit 93d787557939d77c6a15854520a42f3d0f358370
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Fri Jul 8 10:41:59 2016 +0200
+
+    rgw: add missing master_zone when running with old default region config
+    
+    Fixes: http://tracker.ceph.com/issues/16627
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit a0420741d446341a4979d78db7e2e58f396fdc4b)
+
+commit dc963833af2d81984c7fd023f718f5b7e58a37dc
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jul 6 09:15:29 2016 -0400
+
+    rgw: fix error_repo segfault in data sync
+    
+    RGWDataSyncShardCR will only allocate an error_repo if it's doing
+    incremental sync, so RGWDataSyncSingleEntryCR needs to guard against a
+    null error_repo
+    
+    also, RGWDataSyncShardCR::stop_spawned_services() was dropping the last
+    reference to the error_repo before calling drain_all(), which meant that
+    RGWDataSyncSingleEntryCR could still be holding a pointer. now uses a
+    boost::intrusive_ptr in RGWDataSyncSingleEntryCR to account for its
+    reference
+    
+    Fixes: http://tracker.ceph.com/issues/16603
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 28609029cf1be2fc9f8c8e3f47320636db29014a)
+
+commit 67eb961927664cbded6b01c5f76bfd9db8481f8c
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Jan 18 16:06:15 2016 -0500
+
+    rgw: add line space between inl. member function defns (rgw_user.h)
+    
+    This is a cosmetic change only.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 5a7f7f5560e57f8dc64a3c84c1e668be834d81e0)
+
+commit 3f8298a049e255f542a012688b4148456d465d8d
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Jan 18 15:58:07 2016 -0500
+
+    rgw-admin: return error on email address conflict (add user)
+    
+    Fixes the error return inconsistency documented in upstream
+    Ceph issue #13598
+    
+    Revised after upstream review, permits changing email address as
+    originally (but use case-insensitive comp).
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 90c143efa2156e55dc017ebe81005315cacaf2c5)
+
+commit 962e7dc47dcaa64f4774a197ae21d0ee59f7854f
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Jan 18 15:06:19 2016 -0500
+
+    rgw-admin: convert user email addresses to lower case
+    
+    Fixes the email address inconsistency documented in upstream
+    Ceph BUG #13598
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 4c438dbbc0e6eda6b9a3018d60019a1a780d6f65)
+
+commit 461782ee6b755b14f6ae90ea56ebcb656e305b94
+Author: Boris Ranto <branto@redhat.com>
+Date:   Mon Jun 13 12:34:39 2016 +0200
+
+    selinux: allow chown for self and setattr for /var/run/ceph
+    
+    Fixes: http://tracker.ceph.com/issues/16126
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 2a6c738abda35f540af6f9398406d4e49337c34d)
+
+commit fe57aceeb02ad9163feb2d196589b5927cedfa0f
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Jun 6 16:19:17 2016 -0400
+
+    rgw ldap: fix ldap bindpw parsing
+    
+    Also add additional LDAP debugging output at 0, 10, and 15 to make
+    troubleshooting easier.
+    
+    Fixes DN search issue using QE configuration of MS AD.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 72b2f3e8dcb879be3dac7ac03142fd4a13ff28ac)
+
+commit 429e9c03f3f47e3804f54cac380db0add18584f2
+Author: Javier M. Mellid <jmunhoz@igalia.com>
+Date:   Tue Jun 14 11:04:23 2016 +0200
+
+    rgw: aws4: fix buffer sharing issue with chunked uploads
+    
+    Signed-off-by: Javier M. Mellid <jmunhoz@igalia.com>
+    (cherry picked from commit dd9f53115a452e1e4268a1036f8fb69f9c1b86f7)
+
+commit 7c2eab19e2c088e97de924f7f1a633a9f291439d
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Jul 1 21:06:17 2016 -0400
+
+    mds: move Finisher to unlocked shutdown
+    
+    This commit resolves a deadlock reported in i16042 where the thread calling
+    MDSRankDispatcher::shutdown would hold the mds_lock while asynchronous
+    callbacks in the Finisher would attempt to lock mds_lock.
+    
+    For simplicity, I merged the finisher stop with the messenger shutdown as both
+    need the mds_lock dropped.
+    
+    Fixes: http://tracker.ceph.com/issues/16042
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit b1d6689b6cae579d5f1b2047fd3c653cda422454)
+    
+    Conflicts:
+            src/mds/MDSRank.cc
+                    trivial conflict in MDSRankDispatcher::shutdown()
+
+commit 28697fc5101395be2a1bd778445983e61a2dc089
+Author: Douglas Fuller <dfuller@redhat.com>
+Date:   Mon Jun 27 12:30:11 2016 -0700
+
+    mds: Kill C_SaferCond in evict_sessions()
+    
+    MDSRankDispatcher::evict_sessions waits on a C_SaferCond for
+    kill_session to complete on each of its victims. Change the
+    command handling flow to pass command messages all the way down
+    to MDSRankDispatcher. Extract the MDSDaemon's reply path into a
+    static function callable from a new context in the MDSRankDispatcher.
+    
+    See: http://tracker.ceph.com/issues/16288
+    Signed-off-by: Douglas Fuller <dfuller@redhat.com>
+    (cherry picked from commit 3a4d63ee67765010a8e53af5a89aef4f49fafd56)
+
+commit 1d3a816814c6cb87f645f205362de800cc84d484
+Author: Zhi Zhang <willzzhang@tencent.com>
+Date:   Mon Jun 27 13:00:26 2016 +0800
+
+    mds: fix shutting down mds timed-out due to deadlock
+    
+    Signed-off-by: Zhi Zhang <zhangz.david@outlook.com>
+    (cherry picked from commit ca069149654ecd10778cd5327bf59b9643e23967)
+
+commit 49a1ce936d76b4bbf7a3b46be300ad5ebb6d301e
+Author: Zhi Zhang <willzzhang@tencent.com>
+Date:   Mon Jun 27 12:59:08 2016 +0800
+
+    msg/async: remove the unnecessary checking to wakup event_wait
+    
+    Signed-off-by: Zhi Zhang <zhangz.david@outlook.com>
+    (cherry picked from commit 560dc32771be00664c4b22a0ebca5190cd246e0a)
+
+commit 2beb56a10a0319b6d061caf1da769bb3c9f5f90b
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Jun 2 17:03:05 2016 +0800
+
+    mds: fix SnapRealm::have_past_parents_open()
+    
+    the '!' got delete accidentally in commit f7fb2cb52c (mds: fix open
+    snap parents tracking)
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit f28f72635c7d11f0db4f156a6d108a480295056d)
+
+commit 3d6d36a12bd4823352dc58e2135d03f261d18dbe
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu May 26 09:38:47 2016 +0200
+
+    ceph-disk: partprobe should block udev induced BLKRRPART
+    
+    Wrap partprobe with flock to stop udev from issuing BLKRRPART because
+    this is racy and frequently fails with a message like:
+    
+        Error: Error informing the kernel about modifications to partition
+        /dev/vdc1 -- Device or resource busy.  This means Linux won't know about
+        any changes you made to /dev/vdc1 until you reboot -- so you shouldn't
+        mount it or use it in any way before rebooting.
+    
+    Opening a device (/dev/vdc for instance) in write mode indirectly
+    triggers a BLKRRPART ioctl from udev (starting version 214 and up)
+    when the device is closed (see below for the udev release note).
+    
+    However, if udev fails to acquire an exclusive lock (with
+    flock(fd, LOCK_EX|LOCK_NB); ) the BLKRRPART ioctl is not issued.
+    
+    https://github.com/systemd/systemd/blob/045e00cf16c47bc516c0823d059b7548f3ce9c7c/src/udev/udevd.c#L1042
+    
+    Acquiring an exclusive lock before running the process that opens the
+    device in write mode is therefore an effective way to control this
+    behavior.
+    
+    git clone git://anonscm.debian.org/pkg-systemd/systemd.git
+    systemd/NEWS:
+    CHANGES WITH 214:
+    
+      * As an experimental feature, udev now tries to lock the
+           disk device node (flock(LOCK_SH|LOCK_NB)) while it
+           executes events for the disk or any of its partitions.
+           Applications like partitioning programs can lock the
+           disk device node (flock(LOCK_EX)) and claim temporary
+           device ownership that way; udev will entirely skip all event
+           handling for this disk and its partitions. If the disk
+           was opened for writing, the close will trigger a partition
+           table rescan in udev's "watch" facility, and if needed
+           synthesize "change" events for the disk and all its partitions.
+           This is now unconditionally enabled, and if it turns out to
+           cause major problems, we might turn it on only for specific
+           devices, or might need to disable it entirely. Device Mapper
+           devices are excluded from this logic.
+    
+    Fixes: http://tracker.ceph.com/issues/15176
+    
+    Signed-off-by: Marius Vollmer <marius.vollmer@redhat com>
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 8519481b72365701d01ee58a0ef57ad1bea2c66c)
+
+commit 8cd7f44d8066a796cccd872dd2593582ac061331
+Author: Javier M. Mellid <jmunhoz@igalia.com>
+Date:   Fri Jun 3 17:34:10 2016 +0200
+
+    rgw: aws4: add STREAMING-AWS4-HMAC-SHA256-PAYLOAD support
+    
+    When authenticating requests using the Authorization header in AWS4, you have
+    the option of uploading the payload in chunks. You can send data in fixed size
+    or variable size chunks.
+    
+    This patch enables streaming mode and signed headers support with chunked
+    uploads.
+    
+    Fixes: http://tracker.ceph.com/issues/16146
+    
+    Signed-off-by: Javier M. Mellid <jmunhoz@igalia.com>
+    (cherry picked from commit 5de5876a535537f7878615898bb9cf7887204cb1)
+    
+    Conflicts:
+            src/rgw/rgw_rest_s3.cc
+    
+    No change required to resolve the conflict. Manual merge was enough.
+
+commit 276ec72e85d6650782aef2c96f4b1fbc47918f42
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Mar 11 17:34:33 2016 +0100
+
+    rgw: use std::unique_ptr for rgw_aws4_auth management.
+    
+    This is purely for code maintainability.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 6f273eb52c07b27c7aa2d69be3e5ff5d6578558c)
+
+commit 2c422e306989f65d9112f95a131ff29479b2d0f0
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Mar 11 17:33:47 2016 +0100
+
+    rgw: add handling of memory allocation failure in AWS4 auth.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit ddbb0ab9d5046672795ec876221de9ebd466364f)
+
+commit 2cd3ed8a59786be28b55a0983a438af1aab226ea
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Wed Apr 27 23:26:44 2016 +0530
+
+    ceph_volume_client: allow read-only authorization for volumes
+    
+    Allow clients to be restricted to read-only mount of the volume
+    by restricting their ceph auth ID's MDS and OSD caps to read-only.
+    
+    Fixes: http://tracker.ceph.com/issues/15614
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 011ea5e7fb35ee07848e0c3abac24702a778ad63)
+
+commit 46246e39b8cadc14aede930559eddd09059301fc
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Jul 6 19:41:54 2016 +0800
+
+    osd: increment stas on recovery pull also
+    
+    PGMap::recovery_rate_summary() summaries the recovery progress from
+    a pool's
+    pool_stat_t.stats.sum.num_{objects_recovered,bytes_recovered,keys_recovered},
+    now we only increment stats on completion of recovery push, but there
+    are chances that a PG recovers by pulling data from replicas to primary.
+    in that case, teuthology will erroneously consider recovery hung: a zero
+    recovering_keys_per_sec or recovering_bytes_per_sec or
+    recovering_objects_per_sec. so we should increment stats on pull as
+    well.
+    
+    Fixes: http://tracker.ceph.com/issues/16277
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 9c6c4d8fef67380e2a6d93f6c101744c926d8b9b)
+
+commit 3da251fe7f3bafa9a936df209b1906824764e5f6
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Jul 22 15:55:16 2016 +0800
+
+    pybind/ceph_argparse: handle non ascii unicode args
+    
+    we raise UnicodeDecodeError at seeing non-ascii args if we fail to match
+    it with any command signatures. instead, we should use a unicode string
+    for representing the error in that case. please note, the exception is
+    not printed at all in real-world. =)
+    
+    Fixes: http://tracker.ceph.com/issues/12287
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5864626d275c423cdb8d3e52a91fe4cc6b8e6f90)
+
+commit b01af21ceeb83e3b2fb7cee608f2a9229e6eed7e
+Author: Oleh Prypin <oleh@pryp.in>
+Date:   Wed Jun 15 00:50:20 2016 +0300
+
+    Fix tabs->whitespace in ceph_argparse
+    
+    Signed-off-by: Oleh Prypin <oleh@pryp.in>
+    (cherry picked from commit bef2c83c1da7dcc32fc2ff98cf214fc93bef75b8)
+
+commit bb2dc9537612b56a79f1237618126faecbeccc9e
+Author: Oleh Prypin <oleh@pryp.in>
+Date:   Wed Jun 15 00:54:08 2016 +0300
+
+    Make usage of builtins in ceph_argparse compatible with Python 3
+    
+    Signed-off-by: Oleh Prypin <oleh@pryp.in>
+    (cherry picked from commit ddf06041ef80ac9606da8d57f048d2e23233d122)
+
+commit f80d10c82034fcd9d53b3890afe62a4dbfeb1799
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Jul 20 10:23:48 2016 +0200
+
+    rpm: move libatomic_ops-devel to non-distro-specific section
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 94e4f2190b6ab3ff7b57b0b6095fea6d9228d917)
+
+commit a28810c69b356d1865d1ce9e989bcc9e8a66017a
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Jul 20 10:22:37 2016 +0200
+
+    rpm: move gperftools-devel to non-distro-specific section
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 06940e29b702c7437dc9f8df365fcc383cb882e1)
+
+commit e6b7a4ba8a1a8ea13d6f7178bd69432a3705754b
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Jul 9 22:55:54 2016 +0200
+
+    rpm: use new name of libatomic_ops-devel
+    
+    The previous form, libatomic-ops-devel, has been deprecated since May 23, 2012.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit c835f6ddf5839c418d29db1c2f46da32fbf05d2c)
+
+commit 9bbf2e8d874201460c3cf1dfd14cfd17752d397d
+Author: Dan Horák <dan@danny.cz>
+Date:   Fri May 6 13:29:03 2016 +0200
+
+    fix tcmalloc handling in spec file
+    
+    - there is no gperftools/tcmalloc on s390(x) yet
+    - explicitly disable tcmalloc when built without
+    
+    Signed-off-by: Dan Horák <dan@danny.cz>
+    (cherry picked from commit efa7f7b365d27797573bf4e5a9878f94f41aede2)
+
+commit 894a5f8d878d4b267f80b90a4bffce157f2b4ba7
+Merge: b15cf42 2538b77
+Author: John Spray <jspray@redhat.com>
+Date:   Thu Jul 21 13:58:16 2016 +0100
+
+    Merge pull request #10373 from ceph/jewel-mds-snap-failover
+    
+    Jewel mds snap failover
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit b26acc03ab42ba7eacb804ef8dc08c03a6e64e2d
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Jun 13 10:27:18 2016 +0200
+
+    ceph-osd-prestart.sh: drop Upstart-specific code
+    
+    Before this patch, the ceph-osd-prestart.sh script would do Upstart-specific
+    things if the journal device failed to appear after a 5-second timeout.
+    
+    Fixes: http://tracker.ceph.com/issues/15984
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 47a06e608b7f686252d42c5a879373d8ebb65d9d)
+
+commit 1e622a5d3b79a89a6affb56f282e1305c48ec835
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Tue Jul 19 15:40:36 2016 +0100
+
+    rpm: Fix creation of mount.ceph symbolic link for SUSE distros
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 976c9d4f31793712b2af334e4a7a58745af8c099)
+
+commit 89cb116ca14abf5764edf07373d045bb127ca7eb
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Jul 18 19:09:54 2016 +0200
+
+    build/ops: build mount.ceph and mount.fuse.ceph as client binaries
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit b2675ff82445cda9873292ac2be8186e054098ed)
+
+commit 84b45b75c47e6234aac9d3ee1cf1c6844b898ae9
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Jul 18 16:16:11 2016 +0200
+
+    rpm: move mount.ceph from ceph-base to ceph-common
+    
+    Ceph clients use mount.ceph to mount CephFS filesystems, and
+    ceph-base is not expected to be installed on client systems.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit b8c24bf2f8c6a6b125778ca92d68d8e64ccf9ae0)
+
+commit e028cd140840b15a8f99962dcf8ec646da3f1f60
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Jul 6 10:13:03 2016 +0200
+
+    rpm: create mount.ceph symlink in /sbin (SUSE only)
+    
+    Fixes: http://tracker.ceph.com/issues/16598
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit b090e9da326a72068260682599c886c2d11f06b3)
+
+commit ad67d1c15f49233fe1108258d9b968c8b2301330
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Jun 2 02:08:45 2016 +0800
+
+    makefile: install mount.fuse.ceph,mount.ceph into /usr/sbin
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 52021ff9f0b23c36a012f19c7388dcfb2281b523)
+    
+    Conflicts:
+            src/CMakeLists.txt (Install mount.ceph to ${CMAKE_INSTALL_SBINDIR}
+            instead of sbin; install mount.fuse.ceph to ${CMAKE_INSTALL_SBINDIR} as
+            well)
+
+commit 2538b77a2f219a76c19289ab3e675927f6914149
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Jul 11 15:32:41 2016 +0800
+
+    client: fix MetaRequest::set_other_inode()
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 265f96bda7e106883063f074133450ce39ee262c)
+
+commit ffcfe69ef7abaaa3c663de86fb04ea986dc90b15
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Jul 11 11:25:37 2016 +0800
+
+    client: close directory's snapdir when deleting directory
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 3099cabd11a16d22c6707631861bef0aa176ed02)
+
+commit b9007025f3dd49532a45267d7478dcdc5308ce3a
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Jul 11 11:07:01 2016 +0800
+
+    client: invalidate snap inodes after removing snapshot
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit f180ad149ab510626fcd6cbd8221f550f9858126)
+
+commit d9f957a5cda55adcdba5db28689aeb344d10e413
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Jul 11 10:51:13 2016 +0800
+
+    mds: fix incorrect "unconnected snaprealm xxx" warning
+    
+    If a snaprealm has no child/parent snaprelam, and the snaprealm inode
+    is not in the cache while client reconnects. The snaprealm does not
+    get properly removed from MDCache::reconnected_snaplrealm. This causes
+    incorrect "unconnected snaprealm xxx" warning
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 85e687de876ca7e0174734ba81130949c4ab6a40)
+
+commit 3e745ef3172aeef07038c9156d4668ca08078f18
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Jun 30 16:05:57 2016 +0800
+
+    qa/workunits/fs: fix expect_failure function in test scripts
+    
+    The origin expect_failure function return 0 regardness of command's
+    return value.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 16f7d7c71e65cd68cecde3e5b89d189ab5950548)
+
+commit 37157d146f1983b94588135e4b0468a9c59c3ead
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 29 20:49:40 2016 +0800
+
+    client: make sure snapflush is sent before normal cap message
+    
+    MDS does null snapflush when it receives normal cap message. So client
+    must send snapflush first.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit d3916717e2edc8000400f678fa7134ca1406a074)
+
+commit 326d46b54280ff5612ee571671cc4956fcb7e8eb
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Jun 28 20:39:08 2016 +0800
+
+    client: unify cap flush and snapcap flush
+    
+    This patch includes following changes
+    - assign flush tid to snapcap flush
+    - remove session's flushing_capsnaps list. add inode with snapcap
+      flushes to session's flushing_caps list instead.
+    - when reconnecting to MDS, re-send one inode's snapcap flushes and
+      cap flushes at the same time.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit a05e996b2a2c36496abd8538829ac4897392f6eb)
+
+commit 5c2ff04061cc686c8ece37cee3393365769d2bf1
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 29 17:15:01 2016 +0800
+
+    mds: handle partly purged directory
+    
+    For a snapshoted direcotry whose snaprealm parents are being opened,
+    MDS does not know if the directory is purgeable. So MDS can't skip
+    committing dirfrags of the directory. But if the direcotry is purgeale,
+    some dirfrags could have already been deleted during MDS failover.
+    Committing them could return -ENOENT.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit bc50e0309280c08c3ca79dfa5514ac3a15f81a23)
+
+commit 57b39f04087f3ba855248950723c949e3b7dcd7c
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 29 11:42:42 2016 +0800
+
+    mds: do files recovery after processing cap flushes
+    
+    File recovery may update inode and trigger inode COW. MDS relies on
+    client caps to setup CInode::client_need_snapflush. But for a given
+    client, the reconnected caps may not include the flushing caps.
+    (Before MDS failover, client released and flushed some caps at the
+    same time. When MDS recovers, client re-send the cap flush and send
+    cap reconnect to the MDS.) This may cause later snapflush to get
+    dropped.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit dd98448d3d0bbb7d338f03f7ad1e40f217eebe0d)
+
+commit bace1c8c7ce7d29676b9ed6925cdee41af8a8425
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 29 11:25:12 2016 +0800
+
+    mds: combine MDCache::{reconnected_caps,cap_imports_dirty}
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 57067e032e84bbdb69c499aa7ea93ca68082569b)
+
+commit 365bda18e3fe960be69d81882e11dcb9932a002c
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Jun 24 17:09:34 2016 +0800
+
+    mds: remove CEPH_LOCK_IFLOCKL from cinode_lock_info
+    
+    Currently we don't support dirty CEPH_CAP_FLOCK_EXCL
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit cfc3ec17b6f245e6d8a0be4fdf6cfa64d2fb725f)
+
+commit 55367c5cda09167778839eb8474f86903857f53a
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Jun 23 16:59:46 2016 +0800
+
+    mds: rebuild the internal states that tracking pending snapflush
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 1b7d198f633a8608b704f82f01a3f4a4a3a4892b)
+
+commit 0897fc4f15804a25a667bf7e495dc4ff1209127b
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 22 20:34:41 2016 +0800
+
+    mds: using cap_reconnect_t to track cap recconect
+    
+    Previous commit extended cap_reconnect_t to include snapflush
+    related information. This information are needed in various
+    places
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 25b6011dcf07c1fc663cbe29ffd119b66545a0ac)
+
+commit ac508dc9202ebdb8f39bf1304bb459637cae1cb9
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 22 15:56:29 2016 +0800
+
+    mds: add 'follows' of first pending snapflush to cap reconnect message
+    
+    This helps the recovering MDS to reconstruct the internal states that
+    tracking pending snapflush.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit dd3963a878d4bb2fb5992278ccbc9f7633fb8786)
+
+commit c258f52afa68bf991b4d6c76d36271333c3e49bf
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Jun 21 17:17:56 2016 +0800
+
+    mds: journal snap inodes that need flush when expiring log segment
+    
+    Treat snap inodes that need flush in the same way as open files.
+    When MDS recovers, this make sure that journal replay bring snap
+    inodes that need flush into the cache
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 306153b3d012832bdfa20402077fa60a9a5d626c)
+
+commit 42dd72df1a514175be05d5d613d9b00296cb1b1e
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Jun 21 16:20:58 2016 +0800
+
+    mds: properly update client_snap_caps when splitting snap inode
+    
+    update the new snap inode's client_snap_caps according to the old
+    snap inode.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 4883779935219817c7e391940a025be1679daeb5)
+
+commit b15cf42a4be7bb290e095cd5027d7f9ac604a97d
+Author: Oleh Prypin <oleh@pryp.in>
+Date:   Thu Jul 14 01:06:51 2016 +0300
+
+    install-deps: Get the latest virtualenv in a temporary folder
+    to work around a bug in old virtualenv
+    
+    Signed-off-by: Oleh Prypin <oleh@pryp.in>
+    (cherry picked from commit 2699b6d89b8c1c193fd86b5233d1ea86458753a0)
+
+commit 00f3fd4a39f35780e602acfa4023bf59031b3617
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Jul 12 16:07:53 2016 +0800
+
+    ceph-fuse: link to libtcmalloc or jemalloc
+    
+    Fixes: http://tracker.ceph.com/issues/16655
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit e644f6eb64037b1e21ce55d4dcffa43571ac1327)
+
+commit d794e295786599102d5ea1a4c10002f4f8d85649
+Merge: bb94997 7178f23
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu Jul 14 12:10:11 2016 -0400
+
+    Merge remote-tracking branch 'upstream/pull/10298/head' into jewel
+    
+    * upstream/pull/10298/head:
+      doc: fix standby replay config
+
+commit 7178f23c1bcb800bec2c7bec138ac02b22dbad2b
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Tue Jul 12 15:43:23 2016 -0400
+
+    doc: fix standby replay config
+    
+    I tried using these settings in tests without success. The correct config names
+    are prefixed with "mds".
+    
+    Fixes: http://tracker.ceph.com/issues/16664
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 8981f3beb5732f51197a5be84df18edde64217ca
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Jun 21 15:09:53 2016 -0400
+
+    rgw: add pg_ver to tombstone_cache
+    
+    a tombstone cache was added to remember the mtime of deleted objects for
+    use with the HTTP_IF_MODIFIED_SINCE header, but the comparison was still
+    failing because of a missing pg_ver. added pg_ver to the tombstone cache
+    so it can be passed with HTTP_DEST_PG_VER
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit adb529f2fb983df671a1db58a0b17862a29762f0)
+
+commit 6c32fe8a75f535f21d3bec089bf06a82db5ec876
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jun 10 14:35:01 2016 -0700
+
+    rgw: add obj tombstone cache
+    
+    The obj tombstone cache is used in multi-zone environmet to keep
+    track of removed objects' mtime. This is then used to fetch remote
+    object only if its newer than the object that was removed, otherwise
+    we're just fetching ghost of the past.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit eb10214920c23b24edd94ca53d0f36c85404644d)
+
+commit f651a8f0cc2a11eb1a037338e35de3cf165f5ac2
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jun 10 14:34:41 2016 -0700
+
+    rgw: rgw_obj comparison also accounts for bucket id
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit a7f32c4b1bd951b1a18b7d13013dc8e2822d6ffc)
+
+commit 354e81c1800066e4e8f525706214c41d90816b9d
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jun 9 14:30:42 2016 -0700
+
+    cls/rgw: reduce tag_timeout significantly
+    
+    The tag timeout is used for applying dir_suggest changes. Shorten it
+    so that changes will be reported early and can be used in data sync.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 99efdc673b7aed439eebdaa92ff117ba9646dd7c)
+
+commit 24d483119035a78973a1ee3827f5f7c8cb20ce5a
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jun 9 14:22:07 2016 -0700
+
+    cls_rgw: dir_suggest entries bucket index logging
+    
+    Fixes: http://tracker.ceph.com/issues/16222
+    
+    Log entries that were set through the dir_suggest mechanism in
+    the bucket index log.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit ae00c5529219d81987425160dbb2a4e937661a6c)
+
+commit bdef8b23714eaff933992ed8c02fb35cd8d11818
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jun 9 12:59:35 2016 -0700
+
+    cls/rgw: fix timespan comparison
+    
+    Fixes: http://tracker.ceph.com/issues/16220
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit e0a07f70f7740a66ffa2646e0a57e8bdc0285373)
+
+commit c34dbd4b19dcc35483306f95932b3ed6ed8332fd
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat Jun 4 05:47:50 2016 -0700
+
+    rgw: data sync squash, prioritize object removal
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 2fcd8b1d49aae2fd03b986dd10bb3f98d3b8f32e)
+
+commit 09eee3be8fcd79ef46ecfbd277e8cc2bf4f28d93
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat Jun 4 03:29:12 2016 -0700
+
+    rgw: squash bi complete bi log entries
+    
+    only process the newest complete operation in a bilog listing.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f69db8e455f07c5594363c5beac329cb964be1ff)
+
+commit 571a13255aff4ca072ff88bb4ce54da086fbad86
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jun 3 02:03:13 2016 -0700
+
+    rgw: stop bucket lease only after draining object sync operations
+    
+    Fixes: http://tracker.ceph.com/issues/16143
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit be6ad9a76c3008ea25a737c0de718faab8fca611)
+
+commit 7af0306de5778340fc7c9395b237bf4e73716d0a
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue May 24 12:57:19 2016 -0400
+
+    rgw: don't leak RGWRados::cr_registry on error
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 53a7f1a9951fa21cb46b5fb3914db3547b568aa5)
+
+commit 9591e505c1d2c7b66b4c17421f94b6fc7e68913f
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue May 24 10:40:25 2016 -0400
+
+    rgw: dont need atomic RGWRados::next_rados_handle
+    
+    next_rados_handle is only accessed under an exclusive handle_lock
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 1adff94b720d01240040fdffebdbf53efdd528a5)
+
+commit 047379c2521434f2dbbe67ebbf53e59ed92654f0
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue May 24 10:23:26 2016 -0400
+
+    rgw: remove unneeded RGWRados::num_rados_handles
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 72d5a485e41ac5824c30556b6cfe659094cd303c)
+
+commit 7848482988711406c9cde48d828a0d118f764ad1
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue May 3 10:35:41 2016 -0400
+
+    rgw: use vector for librados handles
+    
+    using a vector instead of an array of pointers cleans up our
+    initialization/shutdown logic
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 02fb39d7f5835ada4d6304f318203444dc7eedc9)
+
+commit bb94997bf4548a430b686a7ceb98052fdb051223
+Merge: 6ae4d13 a40cfe4
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Tue Jul 12 10:09:34 2016 -0400
+
+    Merge pull request #9268 from theanalyst/wip-15992-jewel
+    
+    jewel: radosgw-admin: EEXIST messages for create operations
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 6ae4d13e81d4b98e34d7094e939a8eab5820f608
+Merge: bf73c9a 21f0216
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Jul 10 23:03:16 2016 +0200
+
+    Merge pull request #8497 from sabaini/jewel
+    
+    ceph-disk: Accept bcache devices as data disks
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 2c39d3652acbab77e8da50fa1b7ef537374ac109
+Author: Pete Zaitcev <zaitcev@kotori.zaitcev.us>
+Date:   Tue Jun 7 17:44:20 2016 -0600
+
+    rgw: Add documentation for the Multi-tenancy feature
+    
+    Signed-off-by: Pete Zaitcev <zaitcev@redhat.com>
+    (cherry picked from commit 0e622020fa2a185eaf0546d93a20b06d44e7f691)
+
+commit 9e5a3ae3cd35ccbd7a9f48c555fb93d5beee57cb
+Author: Feng Guo <diluga@gmail.com>
+Date:   Fri Jun 24 09:26:16 2016 +0800
+
+    RGW:add socket backlog setting for via ceph.conf
+    http://tracker.ceph.com/issues/16406
+    
+    Signed-off-by: Feng Guo <diluga@gmail.com>
+    (cherry picked from commit 3e4df832d34c782795a32b5a30f5c7414a1c3fa9)
+
+commit aa3936131d6525e656cd56315947bbf9ddc508ce
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Apr 15 20:15:14 2016 +0800
+
+    mds: disallow 'open truncate' non-regular inode
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 0e4b6f2332bb4822cf324587a94144f1c98e4b97)
+
+commit 3af7b422a3a97c0e89dbff757c5ed0f4557a6602
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Apr 15 19:45:23 2016 +0800
+
+    mds: only open non-regular inode with mode FILE_MODE_PIN
+    
+    ceph_atomic_open() in kernel client does lookup and open at the same
+    time. So it can open a symlink inode with mode CEPH_FILE_MODE_WR.
+    Open a symlink inode with mode CEPH_FILE_MODE_WR triggers assertion
+    in Locker::check_inode_max_size();
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 4d15eb12298e007744486e28924a6f0ae071bd06)
+
+commit 8655803d8fb9ac21f75b636fb6d666f387462934
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Thu Jul 7 14:14:36 2016 -0700
+
+    rgw: fix multi-delete query param parsing.
+    
+    Multi-delete is triggered by a query parameter on POST, but there are
+    multiple valid ways of representing it, and Ceph should accept ANY way
+    that has the query parameter set, regardless of what value or absence of
+    value.
+    
+    This caused the RubyGem aws-sdk-v1 to break, and has been present since
+    multi-delete was first added in commit 0a1f4a97da, for the bobtail
+    release.
+    
+    Fixes: http://tracker.ceph.com/issues/16618
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit a7016e1b67e82641f0702fda4eae799e953063e6)
+
+commit bf73c9ad08ed2cc8db821c08694b11461549fb26
+Merge: f672ddf f858f94
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Jul 6 14:18:44 2016 -0400
+
+    Merge pull request #9545 from Abhishekvrshny/wip-16117-jewel
+    
+    jewel: rgw: aws4 parsing issue
+
+commit 546141c94a1c5e45dcb70e2d5fd06fe1ac0b1599
+Author: Wido den Hollander <wido@42on.com>
+Date:   Tue Apr 5 11:14:16 2016 +0200
+
+    rgw: Set Access-Control-Allow-Origin to a Asterisk if allowed in a rule
+    
+    Before this patch the RGW would respond with the Origin send by the client in the request
+    if a wildcard/asterisk was specified as a valid Origin.
+    
+    This patch makes sure we respond with a header like this:
+    
+      Access-Control-Allow-Origin: *
+    
+    This way a resource can be used on different Origins by the same browser and that browser
+    will use the content as the asterisk.
+    
+    We also keep in mind that when Authorization is send by the client different rules apply.
+    In the case of Authorization we may not respond with an Asterisk, but we do have to
+    add the Vary header with 'Origin' as a value to let the browser know that for different
+    Origins it has to perform a new request.
+    
+    More information: https://developer.mozilla.org/en-US/docs/Web/HTTP/Access_control_CORS
+    
+    Fixes: #15348
+    
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit 0021e224480c7164330eaa7cc1078bb8795169bf)
+
+commit f672ddfc5ad6ac86051987985067318436f0667b
+Merge: aa211cb ec884a3
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jul 6 11:55:13 2016 +0200
+
+    Merge pull request #9568 from dreamhost/wip-16182
+    
+    jewel: rgw: backport merge of static sites fixes
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 2afc176a2792a9ef389f34c7de1aba697fca6a31
+Author: Albert Tu <albert.t@inwinstack.com>
+Date:   Tue Jun 28 10:38:07 2016 +0800
+
+    rgw: fix double counting in RGWRados::update_containers_stats()
+    
+    Fixes: http://tracker.ceph.com/issues/16188
+    Signed-off-by: Albert Tu <albert.t@inwinstack.com>
+    (cherry picked from commit 5dd825aed25588843dc4834be3f5fdf10d93bc68)
+
+commit 489f8ce48b30c708879a002a55aecc080421b5a1
+Author: runsisi <runsisi@zte.com.cn>
+Date:   Tue Jul 5 16:08:40 2016 +0800
+
+    librbd: fix missing return statement if failed to get mirror image state
+    
+    Signed-off-by: runsisi <runsisi@zte.com.cn>
+    (cherry picked from commit ea775178b61ba38237343b07a90d19802f0b7dac)
+
+commit a7987f060479f81d6181f8949e8ed2c8b3029dfd
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed May 25 18:33:49 2016 +0800
+
+    MDSMonitor.cc: fix mdsmap.<namespace> subscriptions
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 8f09dd15cb07597c57d0a6ae981f15a47de11bb9)
+
+commit d244b7a0c6eb4a57a424297d4293184dff28b94c
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Jun 17 11:53:32 2016 -0400
+
+    mds: add maximum fragment size constraint
+    
+    This commit adds a new config option
+    
+        mds_bal_fragment_size_max = 10000*10
+    
+    which is an order of magnitude larger than mds_bal_split_size.
+    
+    This limit prevents a fragment from getting too large which results in large
+    omap directories.
+    
+    Right now the limit is enforced only in the RPC paths and in stray directory
+    entry creation.
+    
+    Fixes http://tracker.ceph.com/issues/16164
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 60af83c80910070d8fb10ac7a4f6f24d49521c1b)
+
+commit 64d99b160d9d6a8758b7a9b3783bd1d153e15c9a
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Jun 17 11:58:13 2016 +0800
+
+    mds: fix Session::check_access()
+    
+    It calls CInode::make_path_string(...) with the second argument is
+    false. The second argument makes the third argument useless. For
+    newly created inode, the path string is something like #1xxxxxxxxx.
+    This can cause the access check to fail.
+    
+    Fixes: http://tracker.ceph.com/issues/16358
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit a94ef56523a383c44c7a52e473e37a43fa0cb6a2)
+
+commit bce5646cd7c9bc9c753bfcefeff37991763b9066
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Jun 6 16:41:49 2016 +0800
+
+    client: skip executing async invalidates while umounting
+    
+    Fixes: http://tracker.ceph.com/issues/16137
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit b95e603a3f9568debeb76fc49aae9a6ee4b75c6b)
+
+commit fd7ff96cbc4a2e9b38d805f36cd4e72a32f04925
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Aug 10 15:15:48 2015 +0800
+
+    ceph-fuse: add option to disable kernel pagecache
+    
+    When multiple clients read/write to a file at the same time, we need
+    to disable cache and let reads/writes go to OSDs. In ceph-fuse case,
+    there are two caches, one is in ceph-fuse, another one is in kernel.
+    We can't disable/enable the kernel cache dynamically. So adding an
+    config option to completely disable kernel cache. Disabling kernel
+    cache may reduce performance, so option is disabled by default.
+    
+    Fixes: 12653
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 0f11ec237d4692d313a038ed61aa07a3f40555ba)
+    
+    Conflicts:
+            src/common/config_opts.h
+               follow the content in patchset, set fuse_use_invalidate_cb
+               to True. Which is also the backport
+               for http://tracker.ceph.com/issues/15634
+
+commit 3f76e4acf243dda26c44ae95919b3d4444bd7d88
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jun 29 14:50:12 2016 -0700
+
+    rgw: finish error_repo cr in stop_spawned_services()
+    
+    Fixes: http://tracker.ceph.com/issues/16530
+    
+    Need to call finish, otherwise drain_all() wouldn't be able to
+    drain it.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 492bb608733c83a5d85319cf47c4d86402344376)
+
+commit a5f5513ede87d7d77d83a9327e8d178767f734ea
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Tue Jun 28 14:39:46 2016 -0700
+
+    test: fix CMake build of ceph_test_objectcacher_stress
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 2ee3d02c8f5de8d2a0ae3e22f3029b269fe4a212)
+
+commit 17f1bffdb891a155532d966b7d3ab4983c9016cf
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Mon May 23 15:14:21 2016 -0700
+
+    ObjectCacher: fix bh_read_finish offset logic
+    
+    If we have an incoming read split across multiple BufferHeads, we want to
+    line up the BufferHead's bl with the incoming OSDOp's bl at the right offset. We
+    were erroneously using this nonsense calculation (always equal to zero!) when
+    a much simpler comparison of the BufferHead's logical object offset to the
+    incoming OSDOp's logical offset will do the trick nicely.
+    
+    Fixes: http://tracker.ceph.com/issues/16002
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 9ec6e7f608608088d51e449c9d375844631dcdde)
+
+commit 73bc6d11745246df45ea6c4b5fbf9be3b9e91c09
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Thu Jun 23 14:23:51 2016 -0700
+
+    osd: provide some contents on ObjectExtent usage in testing
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit a555d9a0c7d3b6c2206520cf78445234b0834639)
+
+commit 3446fa498266ec2902a1d4d9215de4e4b1d9c455
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Fri Jun 10 17:01:09 2016 -0700
+
+    test: build a correctness test for the ObjectCacher
+    
+    For now it's very specifically designed to reproduce
+    http://tracker.ceph.com/issues/16002, but it can
+    be extended to other patterns in the future.
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 0fd55a9886dd8da344c23a5e9898ee5c5061e8f9)
+
+commit b6684914e29fdc375d91be7c80cdf9615d637c8c
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Tue May 31 16:18:19 2016 -0700
+
+    test: split objectcacher test into 'stress' and 'correctness'
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit cc9aab1b0a22c3f7320046b97f75dccf2b86cc6d)
+
+commit 74f59203a13da4ec5ca673a921ed3d07e8d5fc9b
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Thu Jun 23 13:41:46 2016 -0700
+
+    test: add a data-storing MemWriteback for testing ObjectCacher
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit ed5801492bf2850e80a328ce5a61cb1ff2709215)
+
+commit aa211cb4ff0a7e9cac4779187848b3ba00c39bf9
+Merge: e3ccf0f d48a1ed
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Jun 30 10:36:20 2016 +0200
+
+    Merge pull request #9099 from Abhishekvrshny/wip-15841-jewel
+    
+    jewel: s3website: x-amz-website-redirect-location header returns malformed HTTP response
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit e3ccf0f9d31c9e35edb551dc24da7af7265799c1
+Merge: 448deed a08caa6
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Jun 30 10:35:23 2016 +0200
+
+    Merge pull request #9265 from Abhishekvrshny/wip-15965-jewel
+    
+    jewel: No Last-Modified, Content-Size and X-Object-Manifest headers if no segments in DLO manifest
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 448deed8484b06bdbbdad62cbe34cfab7958bbc1
+Merge: ad182de ff9c29a
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Jun 30 10:34:42 2016 +0200
+
+    Merge pull request #9267 from Abhishekvrshny/wip-15960-jewel
+    
+    jewel: rgw: custom metadata aren't camelcased in Swift's responses
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit ad182de1ca849d90fbbbf2ec057b476f32e86d8b
+Merge: 3ccfac6 108638f
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Jun 30 10:28:45 2016 +0200
+
+    Merge pull request #9316 from Abhishekvrshny/wip-15954-jewel
+    
+    jewel: rgw: initial slashes are not properly handled in Swift's BulkDelete
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 3ccfac6b8e98afef19e6c113bd38ae7ab0c51c24
+Merge: 4e1ebac 0bdc8fd
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Jun 30 10:26:09 2016 +0200
+
+    Merge pull request #9390 from vumrao/wip-vumrao-16071
+    
+    jewel: rgw : cleanup radosgw-admin temp command as it was deprecated
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 4e1ebac3048ee1750978e11115144df074923e22
+Merge: 4957014 4eded9a
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Jun 30 10:25:51 2016 +0200
+
+    Merge pull request #9542 from Abhishekvrshny/wip-15967-jewel
+    
+    jewel: rgw: account/container metadata not actually present in a request are deleted during POST through Swift API
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 4957014a0e7a744af6c782e6d4e9e5a89c934ace
+Merge: bb02d3e e8b7dd4
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Jun 30 10:25:30 2016 +0200
+
+    Merge pull request #9543 from Abhishekvrshny/wip-16040-jewel
+    
+    jewel: rgw: updating CORS/ACLs might not work in some circumstances
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 757babb03a9024d6aa42fb327244fd983c066545
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 23 09:03:24 2016 -0400
+
+    librbd: memory leak possible if journal op event failed
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f5069750849c2c43caa4bff766301169a4bfc6ca)
+
+commit e7ec20e6c54f913dd4bf52a949488ac9d258c150
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 23 08:45:59 2016 -0400
+
+    librbd: ignore snap unprotect -EBUSY errors during journal replay
+    
+    Fixes: http://tracker.ceph.com/issues/16445
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit beef0b4b024e9692305f4b413e6c6b520fdaa7f9)
+
+commit cbc963684def355b9c27932fdb4c605bf373ed6e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 23 09:23:32 2016 -0400
+
+    librbd: delete ExclusiveLock instance when switching to snapshot
+    
+    Fixes: http://tracker.ceph.com/issues/16446
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 677832ceb24096ddbeaf2d60e0ac72d28c399f02)
+
+commit d9c3f28f1200e99c8ecf92f50ec4e8840d74a0f1
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 23 13:34:56 2016 -0400
+
+    librbd: mark exclusive lock as released after journal is closed
+    
+    Fixes: http://tracker.ceph.com/issues/16450
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1811e62aa0ba3bab85c536eaab8f3adec6d1fffa)
+
+commit b1d969868bdd898958236212ee847f7a401c6406
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 24 10:03:53 2016 -0400
+
+    librbd: do not propagate mirror status notification failures
+    
+    These should be treated as a broadcast since no ACK is required. The
+    peer will eventually see the notification or its watch will have timed
+    out and it will re-sync.
+    
+    Fixes: http://tracker.ceph.com/issues/16470
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 97bade9f76af62765d7aa8c2154e51a7b231e40f)
+
+commit 51defeada069c0e33b03b96be962f155ce3df295
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jun 22 18:19:52 2016 -0400
+
+    librbd: fix crash while using advisory locks with R/O image
+    
+    Fixes: http://tracker.ceph.com/issues/16364
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1007aea3d78ca5aead8a11f932da0c1d9d67259e)
+
+commit db28ddcf88c13aef80e5a7131db463b305102abe
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jun 22 10:14:21 2016 -0400
+
+    rbd-mirror: block proxied ops with -EROFS return code
+    
+    When replicating to a local image, the daemon will own the
+    exclusive lock and will receive any proxied maintenance ops
+    from other clients. Since the image is non-primary, respond
+    with -EROFS.
+    
+    Fixes: http://tracker.ceph.com/issues/16411
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 07b49df24e5f30460ce3ab584a89370ea3ff7cc8)
+
+commit ebce8ceb9353052d1d43d18e2bb76c68e581272e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jun 22 10:13:45 2016 -0400
+
+    librbd: optionally block proxied requests with an error code
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 93e2faf38e866fb3e32a7b3f3527d97215c60d31)
+
+commit bb02d3e7a4db89d693ed1555d2ac82c5452978e1
+Merge: 8542898 b4a80cb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Jun 30 09:41:20 2016 +0200
+
+    Merge pull request #9631 from dillaman/wip-16232
+    
+    jewel: test: improve rbd-mirror test case coverage
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 70bf7465ad7c75c9c7623a446218ab501f329bd3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 20 09:39:24 2016 -0400
+
+    librbd: potential race when replaying journal ops
+    
+    Fixes: http://tracker.ceph.com/issues/16198
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 75b0f58e37b0ede5e8cf8dcaea980cf71a5ca908)
+
+commit f3f4a4a20ab3039f8dfeda23c773141bf6d95792
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 16 09:27:50 2016 -0400
+
+    librbd: journal callback to interrupt replay
+    
+    If the exclusive lock is lost while the journal is in the
+    replay state, the journal close will block until the replay
+    completes. The new callback will facilitate stopping replay
+    in a timely fashion so that the journal can be closed.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9687e5e34aca98934fcf04089ead2794629455a1)
+
+commit b203168e21c0cfba9df55e9ff43e73a5905119d6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jun 15 23:04:48 2016 -0400
+
+    rbd-mirror: keep local pointer to image journal
+    
+    Fixes: http://tracker.ceph.com/issues/16230
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 47e25e59a8ed4c4e64ae58b41c03125c6d3c4104)
+
+commit 0399958f6cc2a16487f0962b5d5a317dc1d9ff21
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 20 11:41:31 2016 -0400
+
+    rbd-mirror: keep events from different epochs independent
+    
+    Fixes: http://tracker.ceph.com/issues/16362
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3a5b491ec61134dc2e18cbe4e27a54e64b17f7d2)
+
+commit 6a28d63ac609c4e3eb43a31cabe36f2c79c875d2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 20 10:32:04 2016 -0400
+
+    librbd: fix lockdep issue when duplicate event detected
+    
+    Fixes: http://tracker.ceph.com/issues/16363
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 86ef725c34ae950c0e41e89c1aa0c6a15e40f369)
+
+commit 1e85da9cb97f6262eb25d8de234d45e8daccd461
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 16 17:32:20 2016 -0400
+
+    rbd-mirror: ensure replay status formatter has completed before stopping
+    
+    Fixes: http://tracker.ceph.com/issues/16352
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit cccdca460b4be310f99877cf43c48300713150df)
+
+commit bf58eabf561fdb041ed170e54e5d35c46cbe3258
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 16 10:37:56 2016 -0400
+
+    journal: do not log watch errors against deleted journal
+    
+    The peer rbd-mirror process will eventually notice that the journal has
+    been removed and clean itself up. This avoids flooding the log with
+    expected error messages when journaling is disabled.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8317ce1611c39ad6a58bf2d760a010587d91ec60)
+
+commit deb6ca84e4083b1cf569d22b84f3194b1fb27a4b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 16 09:38:28 2016 -0400
+
+    librbd: force-remove journal when disabling feature and removing image
+    
+    If mirroring is enabled, it's nearly certain that the peer rbd-mirror
+    daemon will be a registered client within the journal. Without
+    forcing the removal, this would prevent the removal from occurring.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d83aa1e0e44784e1f38698a839f8a353c87027eb)
+
+commit 415ea77fdad016faf7da150630e933930c5ac4b6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 16 09:31:03 2016 -0400
+
+    librbd: ignore ENOENT error when removing image from mirror directory
+    
+    Fixes: http://tracker.ceph.com/issues/16348
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4255afd34927ad6c303074aa6814538a8b5ae96e)
+
+commit cf65ed998b175e5d766364ff18741554b775b632
+Author: Yuan Zhou <yuan.zhou@intel.com>
+Date:   Thu May 12 18:22:12 2016 +0800
+
+    rbd: Skip rbd cache flush if journaling is enabled under aio_flush
+    
+    With journaling rbd writes will be persisteted on rbd journal objects.
+    The journal will be replayed if crash happen. So it's not necessary to
+    flush rbd_cache in this case. This will improve the flush latency.
+    
+    This patch adds checking on handling aio_flush: if journaling is
+    enabled, rbd cache flushing is skipped.
+    In a system flush(ImageCtx::flush) the cache is flushed even w/ journaling
+    where we truly do need to flush all IO out to disk.
+    
+    Fixes: http://tracker.ceph.com/issues/15761
+    
+    Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>
+    (cherry picked from commit f2def83b7a4a98bc60db5ba8936d78a49abace88)
+
+commit caad884704ce22ae0a860a12693d7529a5837212
+Author: youji <youji@ebay.com>
+Date:   Tue Jun 14 11:12:16 2016 -0700
+
+    mon: Monitor: validate prefix on handle_command()
+    
+    Fixes: http://tracker.ceph.com/issues/16297
+    
+    Signed-off-by: You Ji <youji@ebay.com>
+    (cherry picked from commit 7cb3434fed03a5497abfd00bcec7276b70df0654)
+
+commit 3250c4d5a0bd9fc32eb6b1831a38363581f7c1bd
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Mon Jun 20 15:56:50 2016 +0200
+
+    rgw_swift: newer versions of boost/utility no longer include in_place
+    
+    boost > 1.58 no longer includes in_place in boost/utility, we need to
+    include in_place_factory explicitly. This causes build failures in
+    distros that ship with a higher version of boost. Since the only call is for
+    swift_ver_location, another possibility is to use emplace()
+    instead (though this requires boost ~ 1.56)
+    
+    Fixes: http://tracker.ceph.com/issues/16391
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit a14f0f3869bb7defa7587ad9ccb18c3f086b2c3d)
+    
+    Conflicts:
+            src/rgw/rgw_rest_swift.cc
+                jewel does not have #include <boost/optional.hpp>
+
+commit 85428983070c8df663056404a7c63959bb8ef693
+Merge: 2817f99 e9f9916
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jun 29 16:47:15 2016 +0200
+
+    Merge pull request #9294 from theanalyst/wip-16009-jewel
+    
+    jewel: radosgw-admin: failure for user create after upgrade from hammer to jewel
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit dd635e46ee2425335264ae493cb6b8437cd712fc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 23 10:25:54 2016 -0400
+
+    librbd: ignore missing object map during snap remove
+    
+    Fixes: http://tracker.ceph.com/issues/16350
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 68b296b0f107db39b16fa90c186fdc920d97170d)
+
+commit db7ce96a3093e661f535efdd5e7e37177ba0732b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jun 24 10:52:16 2016 -0400
+
+    librbd: removal of partially deleted image needs id lookup
+    
+    Several operations depend on the image id but if the image cannot be
+    opened to retrieve the id, these cleanup operations cannot be executed.
+    
+    Fixes: http://tracker.ceph.com/issues/16471
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6a9e7da6c3da40c7b25ff3d5ac84027e94beb287)
+
+commit c1a47c7728a17dbfc71280270b0c3079e5961ffb
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue May 31 17:50:05 2016 -0600
+
+    packaging: move parted requirement to -osd subpkg
+    
+    Prior to this change, ceph-base required the "parted" package, which
+    meant that any installation of ceph-osd, ceph-mon, or ceph-mds would
+    pull in the parted package.
+    
+    Move the parted dependency to ceph-osd, since ceph-disk is the only
+    thing that uses parted.
+    
+    The advantage of this change is that Mon and MDS systems will not need
+    to install the parted package.
+    
+    Fixes: http://tracker.ceph.com/issues/16095
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 115349680bad520a0aa56ce3a346d93f541a905c)
+
+commit 74dd0359d4d6db3c6dac4fd41703270e5020aad7
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon May 2 14:28:55 2016 -0400
+
+    osd/PG: set last_* PG fields on a creating PG
+    
+    Use the value from pg_history_t that ultimately came from the
+    mon if last_epoch_started == 0.  This establishes a sane lower
+    bound on these timestamps, and prevents a mon health error from
+    coming up about how a PG is stuck in whatever state while it is
+    being created.
+    
+    (We addressed half of this problem in commit 6ca6aeac, but the
+    warning comes back as soon as the OSD reports with a creating
+    state.)
+    
+    Fixes: http://tracker.ceph.com/issues/14952
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3be3bc60c12448a36f607c8d4fbf3300c7bbdbee)
+
+commit 2c03e02a04c217b8461b858fd3b46b73c4a370d1
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon May 2 14:27:16 2016 -0400
+
+    osd: set pg history last_clean_scrub_stamp on create
+    
+    We were setting the other two; set this one as well.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 47f540d5b7ecc4ae193057df429db24ca6b3de8d)
+
+commit a2e8ae68ea452d03519359cb0cd344e71e603fa9
+Author: kofiliu <liu.xuan@h3c.com>
+Date:   Fri May 27 03:45:06 2016 -0400
+
+    osd: sparse_read offset may not be zero for ecpool
+    
+    Signed-off-by: kofiliu <liu.xuan@h3c.com>
+    (cherry picked from commit 65e8738611cde0090619b3566a2e25f83b4c8468)
+
+commit 2817f9978f73014b1b1363cecacabdd98e499c72
+Merge: 38a2542 8b82bb8
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Jun 29 11:42:43 2016 +0300
+
+    Merge pull request #9721 from trociny/wip-16289-jewel
+    
+    jewel: qa/workunits/rbd: respect RBD_CREATE_ARGS environment variable
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit b8f7aa28c394830bac9aa73429131e92f7499aa0
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Sat Jun 11 13:39:23 2016 +0800
+
+    msg/async/AsyncConnection: make verify auth callback without connection lock
+    
+    Fixes: http://tracker.ceph.com/issues/16237
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 2b7776545c3f87d7f54a53190e65ec48378eaa05)
+
+commit 3dbb08a069c8e8238d2884a78b04f0a0d01a36e3
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Thu May 12 12:01:35 2016 +0800
+
+    AsyncConnection: create writable event for in progress connection
+    
+    Previously we use a tricky with ceph msgr protocol, if initiator side got
+    in progress connection state, it will wait until read event. Because if
+    tcp session built successfully server side will send the banner firstly
+    and initiator side will get read event. Otherwise, if connection failed,
+    read event also be called.
+    
+    But actually man(2)[http://man7.org/linux/man-pages/man2/connect.2.html]
+    specify if we want to get notification whether connection built, we need
+    to listen writable event. It means when connection built, send buffer
+    is ready to be written.
+    
+    This patch follow the strict nonblock connect process. Not fully sure fix
+    http://tracker.ceph.com/issues/15849
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit a74ce419133881ff8618733a0501c4a47e1368e3)
+
+commit 6554d462059b68ab983c0c8355c465e98ca45440
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Jun 7 17:15:05 2016 -0700
+
+    OSDMonitor::prepare_pgtemp: only update up_thru if newer
+    
+    Fixes: http://tracker.ceph.com/issues/16185
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 5f2bd7b6b28aad96d68444b22c04b8b24564616b)
+
+commit a826bb818b02edac71b69fdd97e9318bb42ebc36
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jun 2 10:43:17 2016 -0700
+
+    OSDMonitor: drop pg temp from sources other than the current primary
+    
+    Fixes: http://tracker.ceph.com/issues/16127
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 1a07123c38e3fecb3fb2e43bbbae962d8411d287)
+
+commit 18fdc1c9f061b396df8095907c5b0ffb0591e76d
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue May 24 14:02:32 2016 -0400
+
+    osd: reject PARALLELEXEC ops with EINVAL
+    
+    Errors are better than crashing.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 407495197dd878b62f1356f6d939e33ab50d78c6)
+
+commit f91da931bcca9fd9f055a8d42d86ba3c3d3ea25f
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue May 24 14:02:14 2016 -0400
+
+    ceph_test_rados_api_misc: test rados op with bad flas
+    
+    Pass the bad PARALLELEXEC flag to remove(), which takes a flags arg.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1aa807f4f29b4b016cc737ffa443e8f30c3b7693)
+
+commit edd3f799fb1e5d70244412c5a1be17f74397aae2
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Sun Jun 19 23:42:36 2016 +0800
+
+    msg/async: close STATE_WAIT connection in short period
+    
+    1. in practice, STATE_WAIT connection caused by racing connect should be
+    resolved in milliseconds level. we don't need to keep this connection
+    forever.
+    2. it will avoid unexpected osd peering hang because of outside network
+    problem.
+    
+    Fixes: http://tracker.ceph.com/issues/16378
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 4de5407ac96686748497253e4daf51177f809a95)
+
+commit af8e86c20e9b1124abe33f0dff58817fc8aad7c9
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Sun Jun 19 23:12:10 2016 +0800
+
+    test_msgr: add assert if wait for shutdown hang
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit ca22e0bc09c1a13c8dac14b25c00501e912b8006)
+
+commit 42ef435554a21dfd5daec53002440a38f4d9f705
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Sun Jun 19 23:04:46 2016 +0800
+
+    test/test_msgr: add shutdown hang debug info
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 37b53d69d3d8c848d4627475fd63c80e15e118e7)
+
+commit d4c531dca281560b904ec14b2749bc2924f470fb
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Sun May 22 23:43:47 2016 +0800
+
+    test_msgr: add verbose info for pending connections
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 39515b0f7c49d0cedbec0cf0dc2196b9d6d6339c)
+
+commit 90ce35c4f449bfd48398b8a164e423f3d72609b2
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Sun Jun 19 23:03:07 2016 +0800
+
+    msg/async/AsyncConnection: lower racing case debug log level
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit d8cc985fe0ebc18d4ff725f50304a5ff36a77cd0)
+
+commit 38a25420a0120288029d65cf924f1cbe5f3eb3fd
+Merge: 980abcf 75d3261
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jun 29 09:56:58 2016 +0200
+
+    Merge pull request #9425 from cbodley/wip-rgw-sync-retry-jewel
+    
+    jewel: rgw: retry on bucket sync errors
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 980abcfd3966c29b4871d5a0913051a8312d866c
+Merge: e230023 933fdef
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jun 29 09:55:26 2016 +0200
+
+    Merge pull request #9327 from cbodley/wip-rgw-zone-modify-master-jewel
+    
+    jewel: rgw: add_zone only clears master_zone if --master=false
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit e2300232b07175a12b92f23352deb34210d8ae41
+Merge: e63dcb0 8163c4d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jun 29 09:50:04 2016 +0200
+
+    Merge pull request #9242 from yehudasa/wip-15911-jewel
+    
+    jewel: rgw: don't unregister request if request is not connected to manager
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit e63dcb08083dff3a8382900f90df9f3311c9ec99
+Merge: c9f2fda b751d48
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 28 15:53:03 2016 -0400
+
+    Merge pull request #9883 from dillaman/wip-16422
+    
+    jewel: librbd: flag image as updated after proxying maintenance op
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit c9f2fda32c55f85f717ec26dfcf9140b16fa8cc6
+Merge: 9aafefe 70018bf
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Jun 27 12:43:11 2016 -0700
+
+    Merge pull request #9105 from Abhishekvrshny/wip-15768-jewel
+    
+    jewel: FileStore: umount hang because sync thread doesn't exit
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 9aafefeab6b0f01d7467f70cb2f1b16ae88340e8
+Merge: 25c807b cb9e9e1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Jun 27 17:50:39 2016 +0200
+
+    Merge pull request #9633 from liewegas/wip-async-jewel
+    
+    msg/async: backports of all asyncmsgr fixes to jewel
+    
+    Reviewed-by: Haomai Wang <haomai@xsky.com>
+
+commit 25c807b1919fe11b8d8183b06cdfc5465357d509
+Merge: ae34e37 6619bd9
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jun 27 09:12:14 2016 -0400
+
+    Merge pull request #9576 from Abhishekvrshny/wip-16148-jewel
+    
+    jewel: Scrub error: 0/1 pinned
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit ae34e3742651f7b679e4b524676d26d4fb906562
+Merge: e16830c b7f0e73
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jun 27 09:11:43 2016 -0400
+
+    Merge pull request #9575 from Abhishekvrshny/wip-16150-jewel
+    
+    jewel: crash adding snap to purged_snaps in ReplicatedPG::WaitingOnReplicas
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit e16830c6287323aae4fdddd9f4192793a6c48e74
+Merge: 2cf8d58 6c1163c
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jun 27 09:10:36 2016 -0400
+
+    Merge pull request #8754 from kmroz/wip-backport-fix-run-dir-chown
+    
+    jewel: global-init: fixup chown of the run directory along with log and asok files
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 1f19dbd0f092390b44acfb5fe9dc3936c6732fdd
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Jun 27 14:45:02 2016 +0300
+
+    librbd: potential use after free on refresh error
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit e92b3950cd261d36de422ccaa4a2441d41c80d0c)
+
+commit 2cf8d58beb164017d065e91fdbf8c7a380a4dddb
+Merge: b861c30 3f8642d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Jun 27 10:12:35 2016 +0200
+
+    Merge pull request #9578 from Abhishekvrshny/wip-16080-jewel
+    
+    jewel: osd:sched_time not actually randomized
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit b861c30bdf86378845379bd3e66db32f264b23ee
+Merge: eea4851 1c7abcb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Jun 27 10:08:25 2016 +0200
+
+    Merge pull request #9574 from Abhishekvrshny/wip-16153-jewel
+    
+    jewel: Missing export for rados_aio_get_version in src/include/rados/librados.h
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit eea48519d76e53ebe94947165c9404259397a37e
+Merge: 057ff74 9415d38
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Jun 27 10:06:54 2016 +0200
+
+    Merge pull request #9565 from dzafman/wip-15767-jewel
+    
+    jewel: Fixes for list-inconsistent-*
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 057ff74ede5510e1b28b19e3f99d823781d5da39
+Merge: 50e242c b2d1df1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Jun 27 09:42:28 2016 +0200
+
+    Merge pull request #8904 from SUSE/wip-15700-jewel
+    
+    jewel: rados/test.sh workunit timesout on OpenStack
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit ad3b788b0c83ffb1339d940cd86555dbc3b1b55f
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Apr 23 20:33:17 2016 +0200
+
+    Drop ceph Resource Agent
+    
+    This RA wraps the ceph sysvinit script. As of Jewel, none of the supported
+    distros are using sysvinit anymore. So, drop it.
+    
+    Incidentally, Pacemaker can control systemd units without any wrappers.
+    
+    References: http://tracker.ceph.com/issues/14828
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit bb624c7334ee4241ea3bf892f88e25d165dc3477)
+
+commit b751d4854aa8629f46f0e3dd835aff27b8be709d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jun 21 14:56:43 2016 -0400
+
+    librbd: flag image as updated after proxying maintenance op
+    
+    Fixes: http://tracker.ceph.com/issues/16404
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 478dd5f173591914b41d87b8c6d035c32cac8d3e)
+
+commit 50e242c28b766bd35c924f0f01c0298ed166d20a
+Merge: a34b227 9b75275
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Tue Jun 21 16:43:20 2016 -0700
+
+    Merge branch 'wip-dmick-install-deps-jewel' into jewel
+    
+    install-deps.sh: use mk-build-deps instead of processing control
+    
+    Reviewed-by: Erwan Velu <erwan@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 9b7527537547b388d1cd75cb70a522712420a455
+Author: Dan Mick <dan.mick@redhat.com>
+Date:   Wed Mar 23 17:43:53 2016 -0700
+
+    install-deps.sh: use mk-build-deps instead of processing control
+    
+    mk-build-deps is designed to handle installing build dependencies;
+    use that instead, so '|' indications are handled properly.
+    
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    (cherry picked from commit 99bca09b8d09c41077a9d2141ff556b74328e57a)
+
+commit a34b227c1ed11f51d830ea48a6fbfc0c74a981a6
+Author: Roi Dayan <roid@mellanox.com>
+Date:   Wed Apr 20 17:56:16 2016 +0300
+
+    xio: add MNop.h to dist tarball
+    
+    This file is needed when we want to build a package with xio messenger
+    but was not added to dist tarball.
+    
+    Signed-off-by: Roi Dayan <roid@mellanox.com>
+    (cherry picked from commit ac50842d81d271569d4149737834ac53b2f51db2)
+
+commit 393bf7ef409253123e0721cdfe3b78aa4fd55148
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Jun 15 16:53:16 2016 +0200
+
+    rgw: check for -ERR_NOT_MODIFIED in rgw_rest_s3.cc
+    
+    Fixes: http://tracker.ceph.com/issues/16327
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit fc38346c596d8b0bc156183970d6a327943cb577)
+
+commit 6b41d76f53337a696523cddb47b53b703cdfa571
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Tue Jun 14 11:03:19 2016 +0800
+
+    TaskFinisher: cancel all tasks wait until finisher done
+    
+    Otherwise, caller may think pending task won't be executed but actually
+    finisher may execute callback which may cause refer to freed object.
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 40c5679ef038375e2bde982f401d78c1f3e05c6c)
+
+commit 762db30a3c2c43e56b227017b993802369c07219
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Tue May 24 09:29:32 2016 +0530
+
+    rgw: support size suffixes for --max-size in
+    radosgw-admin command
+    
+    Fixes: http://tracker.ceph.com/issues/16339
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit 62eec902c70463173e758e55fa38594c594f868f)
+
+commit e3a99c082e3ebd56d5b40d7d94d98e35629df81e
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Thu May 26 23:30:25 2016 +0530
+
+    common: add int64_t template for strict_si_cast()
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit 8e429d05370fbe7935212d0ae9608e7547f39860)
+
+commit f8e491168d4802ab071e30e4fdd654dca018bfe6
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Mon May 9 18:00:28 2016 +0530
+
+    rados: Add cleanup message with time to rados bench output
+    
+    Fixes: http://tracker.ceph.com/issues/16338
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit 885ebb1650bb4a355600d85ca436c1ecd4916dce)
+
+commit 8b82bb85e4f2d6fb81c3c4ccc6789856c78f4984
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Jun 14 15:28:06 2016 +0300
+
+    qa/workunits/rbd: respect RBD_CREATE_ARGS environment variable
+    
+    Fixes: http://tracker.ceph.com/issues/16289
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 3048d3c7ce800a6174c54946b838e4e4079ec609)
+
+commit 9415d38139a66b86ff40e0521b769665f31fd36b
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu May 5 23:20:58 2016 -0700
+
+    rados: Improve list-inconsistent json format
+    
+    Use array "errors" of strings for the error bits
+    Change snapshot number to integer instead of hex
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 07dc844d6804db93055573ec83e6753773a4c049)
+
+commit 2fd6e7cf78f2eabbec669f3f9c4e60f1aa3ee8bd
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri May 6 19:54:11 2016 -0700
+
+    test: Fix test to not use jq -S which isn't avail in all distributions
+    
+    Fixes: http://tracker.ceph.com/issues/15766
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit a2147ad3cb256c34541ff7dc285594a22a24dfc6)
+
+commit c789aa39f27dc83f52465ba2d5b513a06b99c9d2
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue May 3 12:10:38 2016 -0700
+
+    test: Add testing of new scrub commands in rados
+    
+    Test list-inconsistent-pg
+    Test list-inconsistent-obj
+    Test inconsistent-snapset
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit db517ba9176f1ee829453b016b1bd43c6054a555)
+
+commit a6f3f762505849994c91efea98c345da16561a22
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Apr 29 11:25:18 2016 -0700
+
+    rados: Don't bother showing list-inconsistent-* errors that aren't set
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit a551b6bd5c4eba11c170afe63994258e7598c3d2)
+
+commit 4c72195a6c5cb9eeca952f12edf62a10ec666f10
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Apr 29 17:09:13 2016 -0700
+
+    osd, rados: Fixes for list-inconsistent-snapset
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit fdca113fc24adbb4f60bfdc55fbbd5044b711b5d)
+
+commit 4e4e562926d38371f70ba2cf2f7003daa251b9f0
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Apr 29 11:26:39 2016 -0700
+
+    include, rados: Fixes for list-inconsistent-obj and librados
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 4fc4326b36aa00092043514746aa39aedad06b0f)
+
+commit 80f0dd348f32ff6e1f98ca29ea2ed38b5b7f5854
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Apr 26 18:16:22 2016 -0700
+
+    rados: Balance format sections in same do_get_inconsistent_cmd()
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit de4681b80a5d24e72bbc7ac6dfee4775987fd834)
+
+commit d248aba10ed232df116729591fea5b195a245735
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Mar 21 20:41:03 2016 -0700
+
+    rados: Include epoch in the list-inconsistent-* command output
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit df4bc16c4d49489f7398eb1ecc7b9aef93258414)
+
+commit 43141e383ebac822221b3f22392f02b95b015ef5
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Mar 21 20:39:53 2016 -0700
+
+    rados: Improve error messages for list-inconsistent commands
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit cea7cf56ee3b9593c935d0c74ce6e4b28b14c2da)
+
+commit 502540faf67308fa595e03f9f446b4ba67df731d
+Author: Jenkins Build Slave User <jenkins-build@trusty-huge--dfeed223-b008-47e6-b689-a0d96992b62c.localdomain>
+Date:   Fri May 13 17:50:34 2016 +0000
+
+    10.2.1
+
+commit b4a80cb1e30f8da8e2b0fc2b18fc846609282879
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Sat Jun 11 20:01:31 2016 +0300
+
+    qa/workunits/rbd: specify source path
+    
+    otherwise it looks in $PATH for the script
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 698242e7435c9c4d1db7cb6d5aab3faf57eda840)
+
+commit 70c97bd07e6764e1c6ff83225f6a2a9dcdfb989e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 9 16:23:52 2016 -0400
+
+    qa/workunits/rbd: additional rbd-mirror stress tests
+    
+    Fixes: http://tracker.ceph.com/issues/16197
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit db3e583a5fe4a7985b1e7f1740114da414835af5)
+
+commit c7cfb4825fbcc82e74b5b1461fc86591098a8599
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 9 17:14:36 2016 -0400
+
+    vstart: add --nolockdep option
+    
+    rbd-mirror stress test cases need a way to disable lockdep when
+    initiated in a vstart environment.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d5eec7b2f4222dde0cfdaa9e0e4b43be015c3692)
+
+commit d48a1ed7845843cfc7598d97dce6b10992079ec3
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Sun Apr 17 08:23:23 2016 -0700
+
+    rgw/s3website: Fix x-amz-website-redirect-location support.
+    
+    Support for the x-amz-website-redirect-location header was broken in the
+    feature merging of Jewel, as it previously depended on the error handler to
+    redirect, which was overkill. Simplify it to work.
+    
+    Backport: jewel
+    Fixes: http://tracker.ceph.com/issues/15531
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit 7cbb63bb748c4c78c02926acb4ad9bcea5593695)
+
+commit 3f8642d134c473be331a53f420d122b4f390dee6
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Sat May 14 11:28:04 2016 +0800
+
+    osd: fix sched_time not actually randomized
+    
+    The test program:
+    
+    main()
+    {
+      for (int i = 0; i < 1000; i++) {
+        double before = rand() / RAND_MAX;
+        double after = rand() / (double)RAND_MAX;
+        cout << "before: " << before << " after: " << after << endl;
+      }
+    }
+    
+    And the output(partial and with "-std=c++11" option):
+    
+    before: 0 after: 0.394383
+    before: 0 after: 0.79844
+    before: 0 after: 0.197551
+    before: 0 after: 0.76823
+    before: 0 after: 0.55397
+    before: 0 after: 0.628871
+    before: 0 after: 0.513401
+    before: 0 after: 0.916195
+    before: 0 after: 0.717297
+    before: 0 after: 0.606969
+    before: 0 after: 0.242887
+    before: 0 after: 0.804177
+    before: 0 after: 0.400944
+    before: 0 after: 0.108809
+    before: 0 after: 0.218257
+    before: 0 after: 0.839112
+    before: 0 after: 0.296032
+    before: 0 after: 0.524287
+    
+    Fixes: http://tracker.ceph.com/issues/15890
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit d3b7767a64af0cd57d17b888e4ec3bdae6fdab87)
+
+commit 6619bd9dbe257bfeb19931cf7f94c56520ae1ebf
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri May 20 13:59:10 2016 -0700
+
+    ReplicatedPG: adjust num_pinned in _delete_oid
+    
+    Fixes: http://tracker.ceph.com/issues/15952
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 907d4e25c676fd3e1c2be90ce8ab5b64b362b0bc)
+
+commit b7f0e73223687035e470fcd7ffc7b851c04aba00
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu May 19 16:00:35 2016 -0700
+
+    PG: update PGPool to detect map gaps and reset cached_removed_snaps
+    
+    Fixes: http://tracker.ceph.com/issues/15943
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 5798fb3bf6d726d14a9c5cb99dc5902eba5b878a)
+
+commit 1c7abcbfe58f77d9d915dd2fc2fc4c75883d46c1
+Author: Jim Wright <jim@quadraturecapital.com>
+Date:   Thu Jun 2 15:12:30 2016 +0100
+
+    librados: Added declaration for rados_aio_get_version
+    
+    Was missing from librados.h and hence had an incorrect
+    visability.
+    
+    Fixes: #15535
+    (cherry picked from commit a5a1c1732e50be7d27f8742e794eba88d35c1c7f)
+
+commit ec884a3406568ccb4c997b4da8924d25d3e2473b
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Tue May 17 17:38:38 2016 -0700
+
+    rgw/s3website: whitespace style fixes
+    
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit fd35ad001c9430753cc2b1f91a7d48f591c6754e)
+
+commit bf26b6eeb7fb806bff0ee306467277a273dcc887
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Tue May 17 17:35:41 2016 -0700
+
+    rgw/s3website: Fix ErrocDoc memory leak.
+    
+    Yehuda pointed [1] out a memory leak with the RGWGetObj not being handed back, and
+    suggested that shared_ptr was used to handle it, rather than the prior approach
+    of explicitly calling handler->put_op.
+    
+    This commit implements just that change, in isolation from other changes, for
+    ease of comparision testing.
+    
+    X-Reference-1: https://github.com/ceph/ceph/pull/8669#discussion_r63445516
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit 2fd7bc8cbe4a2e23553e716ce0be75937853e07b)
+
+commit 36672c61be8289ba00dd694c92cd9057dbf5cf29
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Sun Apr 17 08:23:23 2016 -0700
+
+    rgw/s3website: Fix x-amz-website-redirect-location support.
+    
+    Support for the x-amz-website-redirect-location header was broken in the
+    feature merging of Jewel, as it previously depended on the error handler to
+    redirect, which was overkill. Simplify it to work.
+    
+    Backport: jewel
+    Fixes: http://tracker.ceph.com/issues/15531
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit 7cbb63bb748c4c78c02926acb4ad9bcea5593695)
+
+commit 3c0ac8eb9c9d19a6418e3d84df36e4a87f24fb23
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Wed Apr 20 15:52:51 2016 -0700
+
+    rgw/s3website: Implement ErrorDoc & fix Double-Fault handler
+    
+    Fix more last minute breakage from merges, now has has a working ErrorDoc as
+    well as working double-fault. Also moves some s3website-specific code out of
+    the main S3 codepath.
+    
+    Fixes: #15532
+    Fixes: #15555
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit fcb3cf169f1fa7cf878eb154dc3f1ff78e278056)
+
+commit cb9e9e1c322cb7fb2150b15b17bde4371fd1e703
+Author: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+Date:   Fri Feb 26 13:54:20 2016 +0100
+
+    msg/async: Implement smarter worker thread selection
+    
+    This changeset makes AsyncMessenger a bit smarter when it comes
+    to assigning worker threads to AsyncConnections. Each time a worker
+    is assigned, its reference count is increased. Next time when Async
+    Messenger needs to assign another worker to new AsyncConnection, it
+    picks the one with the lowest reference count. If it cannot find an
+    idle one, and number of currently instantiated workers is less than
+    specified with "ms async op max threads", the new worker is created
+    and returned.
+    Once AsyncConnection goes away, the reference count on assigned
+    worker is decreased.
+    This does not prevent, but greatly reduces chances of having a single
+    async worker thread doing most (or even all) of the ops, and also
+    removes the need to manually tune the "ms async op threads" option.
+    
+    Signed-off-by: Piotr Dałek <git@predictor.org.pl>
+    (cherry picked from commit 3e80f8d74a535e14d4092b27ea5417bacff8394e)
+
+commit 578ac8aacfdd0ecfecb3ae3ebeb8a0b3ff53b67d
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Wed Apr 20 14:23:20 2016 -0400
+
+    Event: fix delete_time_event while in processing list
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 1ddeede83200566666fce80867eb7cb5a61a4f62)
+
+commit 8c7a13fdfa46095621b96a7da8d3b9ce09439509
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Fri Apr 15 13:33:35 2016 +0800
+
+    test_msgr: add delay inject test
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 0a2392919f9ad7286ae4b5924566197c1069474f)
+
+commit 846992f025586fa83c69eaec3ed09b6ab6677fcc
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Fri Apr 15 11:43:42 2016 +0800
+
+    AsyncConnection: make delay message happen within original thread
+    
+    Fixes: http://tracker.ceph.com/issues/15503
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 83f7db58aad2509e1a8742e862d4e8bbfd85c37c)
+
+commit 9b199d0b19220bf4adb9b0754f493e7c1ad78a4e
+Author: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+Date:   Tue Apr 5 09:37:23 2016 +0200
+
+    msg/async: add missing DelayedDelivery and delay injection
+    
+    Delay injection was missing from a few spots, also, DelayedDelivery
+    was added.
+    
+    Fixes: http://tracker.ceph.com/issues/15372
+    Signed-off-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>
+    (cherry picked from commit 49a0c9981bd4bf61b520ece8fb8adfdf7439185b)
+
+commit 0e6324a726e85e498946a49393e0f7c228c2f913
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Sun May 1 08:32:24 2016 +0800
+
+    Event: replace ceph_clock_now with coarse_real_clock
+    
+    reduce cpu utilization on real clock latency
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 79343a8a093630f1c0696f135c6e3bef0c23da28)
+
+commit 6597fab4cc27de6d6f1dcfa070ed401612bfed76
+Author: Yan Jun <yan.jun8@zte.com.cn>
+Date:   Mon Apr 25 09:45:20 2016 +0800
+
+    msg/async: fix some return values and misspellings.
+    
+    Signed-off-by: Yan Jun <yan.jun8@zte.com.cn>
+    (cherry picked from commit 0519f938e13d0adf53c1328fd636b9fa5e6c5b93)
+
+commit 18f18c7beee3c7a072009838c02d5ba1f97fef2c
+Author: Yan Jun <yan.jun8@zte.com.cn>
+Date:   Mon Mar 28 15:33:30 2016 +0800
+
+    msg/async: delete the confused comments.
+    
+    Signed-off-by: Yan Jun <yan.jun8@zte.com.cn>
+    (cherry picked from commit 69a587b139815812433f2b651db6bd723353605d)
+
+commit 79354f4223191c9d8ce6f92aaadf3481f9abf72d
+Author: Yan Jun <yan.jun8@zte.com.cn>
+Date:   Fri Mar 25 17:34:12 2016 +0800
+
+    msg/async: add numevents statistics for external_events
+    
+    Maybe we need to add the statistics for external_events.
+    
+    Signed-off-by: Yan Jun <yan.jun8@zte.com.cn>
+    (cherry picked from commit f08ca0a0892767a8c40e06e336297109aa6142a1)
+
+commit 4005a5193a1cb3e3a0ac1e6f019bead0837ea552
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Sun Mar 13 12:24:35 2016 +0800
+
+    AsyncConnection: remove unnecessary "send" flag
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 5c0a689d720f1016846ff945ca75b9f91756170d)
+
+commit 441847d9951e230b48776fa0fbe639d7b23a595b
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Sat Mar 12 21:02:51 2016 +0800
+
+    async: skip unnecessary steps when parsing simple messages
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit a9ac1c4f88ffb5f66c43527ea0ae1deef1be3a96)
+
+commit c02501005b0316ba9f0ae1cd316a92c567b37c3b
+Author: Noah Watkins <nwatkins@redhat.com>
+Date:   Wed May 4 17:22:14 2016 +0800
+
+    client: fstat should take CEPH_STAT_CAP_INODE_ALL
+    
+    Fixes: http://tracker.ceph.com/issues/15723
+    Signed-off-by: Noah Watkins <nwatkins@redhat.com>
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 4b1a5d4ef9c3a116bb08100edd576819930047e3)
+
+commit c3f6d82d86f9720a373ec9792b927292f01897c0
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Wed Jun 1 14:28:17 2016 +0800
+
+    mon/MDSMonitor: fix wrongly set expiration time of blacklist
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 0c3a92bb999e90212a9f38f9f4dc3bf89bd20acb)
+
+commit d4017ae915e8355f9146844a443942c0dce32476
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Wed Jun 1 11:17:32 2016 +0800
+
+    mon/MDSMonitor: fix wrong positive of jewel flag check
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 24b82bafffced97384135e55ab2a97091e9a7b4b)
+
+commit eea0e916640c3ac2d69ffb9c335dde6332b03938
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri May 6 19:07:07 2016 +0800
+
+    mds: finish lock waiters in the same order that they were added.
+    
+    Current code first processes lock waiters who have smaller wait mask.
+    Lock waiters who have large wait mask can starve if client keeps
+    sending requests that add waiter with small mask.
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit d463107473382170c07d9250bb7ace0e5a2a7de2)
+
+commit 2c7fc95e06f6b1b52bef89904399de4088efaff1
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon May 23 17:40:05 2016 +0800
+
+    mds: fix race between StrayManager::{eval_stray,reintegrate_stray}
+    
+    StrayManager::eval_stray() is called after Server::respond_to_request()
+    drops locks. So it can race with StrayManager::reintegrate_stray()
+    
+    Fixes: http://tracker.ceph.com/issues/15920
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 65827a0fd4d4ecb6d5e2eccf3a1818026ce1f10c)
+
+commit 19c13669ecb1bc9788b6741753d3eedbde713620
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon May 2 22:23:04 2016 +0100
+
+    osdc: send error to recovery waiters on shutdown
+    
+    ...instead of sending them '0', which gets things
+    confused.
+    
+    Fixes: http://tracker.ceph.com/issues/15689
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 62c7a3c59ce479d5240eb5fbef01edf11388c7bb)
+
+commit 7b0318aad79a08f1549bbf9054519a18c9e8c379
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon May 2 22:14:07 2016 +0100
+
+    mds: fix mdsmap print_summary with standby replays
+    
+    Fixes: http://tracker.ceph.com/issues/15705
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 899a16e50a6c22359e7a5c5ac720a605f6a4b67e)
+
+commit f858f94c9dcd2b2845fe59ff9d213b59574144c5
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jun 1 04:24:34 2016 -0700
+
+    rgw: reduce string copy
+    
+    As suggested by Casey Bodley.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f8f1f217314c32cf65ac1fa4e8e0132b501ee184)
+
+commit 67e3be38fdfcec8fa4b00dfc703a45ffa101679b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu May 19 15:02:21 2016 -0700
+
+    rgw: rework aws4 header parsing
+    
+    Fixes: #15940
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit e3618c87026b5ced8ef81adbcafc7f9b34f2d48d)
+
+commit 5bdd13e6f2bc8c52f9a4829a5cc80691947233bb
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu May 19 12:52:54 2016 -0700
+
+    rgw: don't add port to aws4 canonical string if using default port
+    
+    Fixes: #15939
+    
+    When either port 80 is used, or if it's a secure connection and port 443 is used,
+    and when going through the presigned url auth, don't add the port to the signed
+    string.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 033888bbd0e4d8d81358bf61a099276dddb5692b)
+
+commit 474739e87231dc1fc3e3a805584c3f15e1dd1f94
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu May 19 11:30:44 2016 -0700
+
+    rgw: use correct method to get current epoch
+    
+    Fixes: #15939
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 310f5bdf56a9deb09347aadc158da25750fb6735)
+
+commit fc34fbd469dd1c35804305a96d4e650828049d51
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed May 18 17:21:28 2016 -0700
+
+    rgw: check for aws4 headers size where needed
+    
+    Fixes: #15940
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 493cc5d1241693f3ea52f4d7f3a194d9e0ec1905)
+
+commit 44decb4ea1d8e60ba929500e0ccbdac3417c3647
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed May 25 14:23:29 2016 +0200
+
+    rgw: properly handle initial slashes in SLO's segment path.
+    
+    Fixes: http://tracker.ceph.com/issues/16015
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit d384b2b8e0ed670f229eb889a14f521fa8d194fc)
+
+commit e8b7dd47018ab115ffa27b7e72470956de738fd3
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Mon May 23 13:27:24 2016 +0200
+
+    rgw: remove unnecessary data copying in RGWPutMetadataBucket.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 19c12bbc233a118496f8ad5d640d19bb0e2c5d05)
+
+commit 63e0993e33b10adc4d9e1f80c4fe4c5ee9c5f4ff
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Sun May 22 15:32:19 2016 +0200
+
+    rgw: Fix updating CORS/ACLs during POST on Swift's container.
+    
+    Introduced in: 7a7de9b75265b978ba4e53966f614fac033972cb
+    Fixes: http://tracker.ceph.com/issues/15976
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    
+    (cherry picked from commit d9e4727b7e6ffa1cb2918b610381d41439a056e8)
+
+commit 4eded9aa94384e60e765accb4c9f093bd2534970
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Sat May 21 02:47:12 2016 +0200
+
+    rgw: fix update of already existing account/bucket's custom attributes.
+    
+    Introduced in: 3f3b18bff16f6a5b36987f888ba3f2a0d1ea3155.
+    Fixes: http://tracker.ceph.com/issues/15977
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    
+    (cherry picked from commit d6129e664fc8d25e70bfaf83e340703005f8f73f)
+
+commit 30ee18018ae890a058ae40a6006e1045258d36d5
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Mon May 9 17:22:45 2016 +0200
+
+    rgw: fix updating account/container metadata of Swift API.
+    
+    This patch rectifies an issue with handling of user metadata that are
+    actually stored by an updated resource (account or container).
+    The expected behaviour is they will be merged with new medadata coming
+    from an HTTP client.
+    
+    Backport: Jewel
+    Fixes: http://tracker.ceph.com/issues/15779
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 3f3b18bff16f6a5b36987f888ba3f2a0d1ea3155)
+
+commit 75d326117dd57acdae355cec5ac25112e70ff2ba
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 13 14:13:27 2016 -0700
+
+    rgw: back off if error repo is empty
+    
+    Don't check it every time
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit dbf6dcb29faa853c2439457c767d550d5fcdc0f2)
+
+commit 8dcd2a1d93f08db6ede05b3028c3fc601defe932
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 13 11:13:48 2016 -0700
+
+    rgw: data sync retries sync on prevously failed bucket shards
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f1ccc4cd973d16e7676b2374eeefe4ee6f6a4630)
+
+commit 3e5210ddedd1d98473e0bcff04a03958b0b850fc
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed May 11 15:59:27 2016 -0700
+
+    rgw: store failed data sync entries in separate omap
+    
+    so that we can reiterate over them
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit b7deb7cb09304f1b0963139296bdb3abb22895ff)
+
+commit d08ca528ea869cf5a899fd684caecf030b96f5fe
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri May 13 11:20:49 2016 -0700
+
+    rgw: configurable window size to RGWOmapAppend
+    
+    We want to be able to disable buffering for certain operations.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 94ff675e2e7763650d14f62aaf1ff9ddb05cc380)
+
+commit 368e88409e57d6827c7b10f72761d0320cdb4dc2
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu May 12 09:14:29 2016 -0700
+
+    rgw: add a cr for omap keys removal
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 20c1b4b8a3695b818c5c64d61d5ea4f84c48c782)
+
+commit 933fdefb56ce1ab883e6d3ee09c34fb2aa14808e
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon May 16 11:03:45 2016 -0400
+
+    rgw: add_zone only clears master_zone if --master=false
+    
+    Fixes: http://tracker.ceph.com/issues/15901
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 40c4d150757cba4b05938c6c760ab9f0296c2ff6)
+
+commit e9f99166f7fc1b4b468e146a47ec9ac71541bec9
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Fri May 20 09:52:01 2016 +0200
+
+    rgw: add missing metadata_heap pool to old zones
+    
+    Fixes: http://tracker.ceph.com/issues/15937
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit b79856bd3f7db8498231d7f35b53b4bcb44a0422)
+
+commit 22662875320717d1ee939732f9cf7939db697106
+Author: Zhi Zhang <willzzhang@tencent.com>
+Date:   Tue May 3 17:12:26 2016 +0800
+
+    mds: wrongly treat symlink inode as normal file/dir when symlink inode is stale on kcephfs
+    
+    Signed-off-by: Zhi Zhang <zhangz.david@outlook.com>
+    (cherry picked from commit 88b6d669e37924536152a552db46ef43a7353562)
+
+commit 0bdc8fd58ce790638c62bbe5264294ed5e1f710a
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Thu May 26 15:43:41 2016 +0530
+
+    rgw : cleanup radosgw-admin temp command as it was deprecated
+    and also implementation code for this command was removed in commit
+    8d7c8828b02c46e119adc4b9e8f655551512fc2d
+    
+    Fixes: http://tracker.ceph.com/issues/16071
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit 949f917af80ae0bd9a5448129d3ce8979acf7e0b)
+
+commit 108638f366a70ca634a21b7fe4c9206368791514
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue Mar 15 16:48:26 2016 +0100
+
+    rgw: handle initial slashes properly in BulkDelete of Swift API.
+    
+    Fixes: http://tracker.ceph.com/issues/15948
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 1dde062c21c3d6fa3cc01e8c922d7f89c6973bfa)
+
+commit a40cfe4df81c03b197dc31888f5d77927eeae7c5
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Tue May 3 14:56:38 2016 +0200
+
+    rgw: remove -EEXIST error msg for ZoneCreate
+    
+    currently for any admin operations like user create etc. you would
+    always see:
+    
+    `RGWZoneParams::create(): error creating default zone params: (17) File
+    Exists`
+    
+    in stdout as the debug level is set to 0, which doesn't make much sense
+    for an end user, so skip the error message, callers of the function handle
+    the error message anyway, so we skip it here
+    
+    Fixes: http://tracker.ceph.com/issues/15720
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit 2ace9d7f349ef09d3ed87fb216cda3e305ef706f)
+
+commit ff9c29a0e7a3719858ad94f632b6d1af3f3ec73e
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Mon May 16 18:24:55 2016 +0200
+
+    rgw: camelcase names of custom attributes in Swift's responses.
+    
+    Fixes: http://tracker.ceph.com/issues/15902
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit cfde4c42c0248b52ef1b855f7de79ee1e229e73f)
+
+commit 4a3c9f338fb0f2cee71f31652cf31a8ba61d5e4e
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Fri May 13 15:39:32 2016 +0200
+
+    rgw: fix realm pull and period pull for apache frontend
+    
+    add missing content-type and content-length headers
+    
+    Fixes: http://tracker.ceph.com/issues/15846
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 8454ff1a91322697224332f0e6f0c7a9c856ec9a)
+
+commit a08caa6d3cbca0395a9ed5f487f9de33b615f59f
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue May 10 17:45:20 2016 +0200
+
+    rgw: handle errors properly during GET on Swift's DLO.
+    
+    Backport: Jewel, Hammer
+    Fixes: http://tracker.ceph.com/issues/15812
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 5ef0202a86a1e4cb56bd29d926555f114a1acdd8)
+
+commit 8163c4dd34d24249bf662c1110c49136ff335bf9
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue May 10 15:34:19 2016 -0700
+
+    rgw: don't unregister request if request is not connected to manager
+    
+    That means that request is already complete and done with the manager.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 16c0605bf276b245fbf41cb3e000ecdf7b049e15)
+
+commit 70018bfe6f5e298791c34ae4ad6eac2140d6f761
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue May 3 17:20:04 2016 +0800
+
+    os/FileStore::sync_entry check for stop in after wait
+    
+    there is chance that the sync_entry() gets signaled in the
+    WaitInterval(max_interval) call because of sync_cond.Signal() call after
+    "stop = true" in umount(), so without this change, sync_entry() will
+    continue wait until min_interval is reached. this forces umount() to
+    wait even it has called d_force_sync(), and hence slows down the
+    progress of umount(). so we need to check for `stop` if we are not
+    signalled because of `force_sync`.
+    
+    Fixes: http://tracker.ceph.com/issues/15695
+    Reported-by: Haomai Wang <haomaiwang@gmail.com>
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 65426a522d9d052fd7c38964f143087f277816c5)
+
+commit b2d1df129bc4403d98b66873ee6bca64ba1ea264
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Apr 6 15:39:23 2016 +0200
+
+    tests: be more generous with test timeout
+    
+    When the thrasher is in action together with a validater (lockdep or
+    valgrind), a single test may hang for more than 360 seconds. Increase to
+    1200: it does not matter if the value is large, only that it prevents
+    the test from hanging forever.
+    
+    Fixes: http://tracker.ceph.com/issues/15403
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit af89474b3fb2c4aa63680aa6b30d71fad2fdd373)
+
+commit 6c1163cd6e39ba293be1be4a3f82e60fd054a348
+Author: Karol Mroz <kmroz@suse.com>
+Date:   Mon Apr 25 17:24:07 2016 +0200
+
+    admin-socket: use chown instead of fchown
+    
+    fchown() returns success, but does not change asok file
+    ownership. chown() does.
+    
+    Signed-off-by: Karol Mroz <kmroz@suse.com>
+    (cherry picked from commit 3d051a58cc117ff79d4cdd768603d1dec8ca7f60)
+
+commit 3963de78635d760924a36eeb1e8b7782dd5a276e
+Author: Karol Mroz <kmroz@suse.com>
+Date:   Sat Apr 23 13:55:18 2016 +0200
+
+    global-init: fixup inconsistent use of ceph ctx and conf
+    
+    Signed-off-by: Karol Mroz <kmroz@suse.com>
+    (cherry picked from commit 088cc66a94886130df647976050c9d98fcd7572e)
+
+commit 2bc41945ca6a8e8e8e852ae0b7e888bc223dd7a3
+Author: Karol Mroz <kmroz@suse.com>
+Date:   Sat Apr 23 13:54:45 2016 +0200
+
+    global-init: chown pid files
+    
+    Signed-off-by: Karol Mroz <kmroz@suse.com>
+    (cherry picked from commit 9c82d8466a264c0f07fd3ec4cc58c1ddb88cc84f)
+
+commit 5681b78ac74728db70a89bc86845a74ff7d81217
+Author: Karol Mroz <kmroz@suse.com>
+Date:   Sat Apr 23 13:47:58 2016 +0200
+
+    global-init: chown run dir
+    
+    Also performs log file chown() operations if permission drop has been
+    deferred. Documents that admin socket chown() happen in the common init
+    path.
+    
+    Signed-off-by: Karol Mroz <kmroz@suse.com>
+    (cherry picked from commit 9d513793b3fe4b7e1402e36a4a675553edd6f317)
+
+commit c2075448f052ef7e49bc95c1f8240c2ff559b5ae
+Author: Karol Mroz <kmroz@suse.com>
+Date:   Sat Apr 23 13:40:08 2016 +0200
+
+    common-init: chown admin socket after service thread started
+    
+    Signed-off-by: Karol Mroz <kmroz@suse.com>
+    (cherry picked from commit 206585c1d0fc6826c91f7830cde8ded86288abef)
+
+commit f7e6b3cf9c363a3a65da4e176c7324d027352420
+Author: Karol Mroz <kmroz@suse.com>
+Date:   Sat Apr 23 13:36:01 2016 +0200
+
+    global-init: check init flags and set accordingly
+    
+    If init flags have changed between the time ceph context was
+    instantiated and now, update them. This can happen, for example, if
+    between manual global_pre_init() and global_init() calls, a daemon adds
+    modifies it's init flags.
+    
+    Signed-off-by: Karol Mroz <kmroz@suse.com>
+    (cherry picked from commit a0ad13db7aba95537808a570ad4c90f6ac4ca0b8)
+
+commit d4afe947578a7ae4956a99fbc9350920e2239e0b
+Author: Karol Mroz <kmroz@suse.com>
+Date:   Sat Apr 23 13:35:02 2016 +0200
+
+    global-init: add a path chown wrapper function
+    
+    Signed-off-by: Karol Mroz <kmroz@suse.com>
+    (cherry picked from commit 860977b105595dbb50320a30dc0edb922279cb00)
+
+commit 770ae9eecf5c8e064529efec29020f38a3675a63
+Author: Karol Mroz <kmroz@suse.com>
+Date:   Sat Apr 23 13:32:45 2016 +0200
+
+    ceph-context: add function to set init flags
+    
+    Signed-off-by: Karol Mroz <kmroz@suse.com>
+    (cherry picked from commit aecab14089bcfd33d6a18adabede6a54040460d4)
+
+commit 21f0216287485e6cce8811f324fee804ef733524
+Author: Peter Sabaini <peter@sabaini.at>
+Date:   Fri Apr 8 00:30:55 2016 +0200
+
+    ceph-disk: Accept bcache devices as data disks
+    
+    Fixes: http://tracker.ceph.com/issues/13278
+    Signed-off-by: Peter Sabaini <peter@sabaini.at>
diff --git a/doc/changelog/v10.2.4.txt b/doc/changelog/v10.2.4.txt
new file mode 100644
index 000000000..f9b388f6c
--- /dev/null
+++ b/doc/changelog/v10.2.4.txt
@@ -0,0 +1,4586 @@
+commit 9411351cc8ce9ee03fbd46225102fe3d28ddf611
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Mon Dec 5 22:15:20 2016 +0000
+
+    10.2.4
+
+commit a9444915a10038c35d5726485e110e56349fe3df
+Merge: 4d9a28d 4d6f848
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Dec 5 14:50:23 2016 +0100
+
+    Merge pull request #12167 from liewegas/wip-osdmap-encoding-jewel
+    
+    jewel: osd: condition OSDMap encoding on features
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 4d9a28d26d84a1dab90d22a09b1a044c193a41ad
+Merge: 6d17188 d194db8
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sat Dec 3 10:57:18 2016 +0100
+
+    Merge pull request #12067 from SUSE/wip-17953-jewel
+    
+    jewel: mon: OSDMonitor: only reject MOSDBoot based on up_from if inst matches
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit d194db826bb26483f847235bf062ba7b225be147
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Nov 14 11:50:23 2016 -0800
+
+    OSDMonitor: only reject MOSDBoot based on up_from if inst matches
+    
+    If the osd actually restarts, there is no guarrantee that the epoch will
+    advance past up_from.  If the inst is different, it can't really be a
+    dup.  At worst, it might be a queued MOSDBoot from a previous inst, but
+    in that case, the real inst would see itself marked up, and then back
+    down causing it to try booting again.
+    
+    Fixes: http://tracker.ceph.com/issues/17899
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 033ad5b46c0492134e72a8372e44e3ef1358d2df)
+
+commit 6d17188786ef7bd574dd8336d1e97e069db9de74
+Merge: b168c26 79be070
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri Dec 2 08:16:27 2016 -0800
+
+    Merge pull request #12207 from jdurgin/wip-librados-setxattr-overload-jewel
+    
+    librados: remove new setxattr overload to avoid breaking the C++ ABI
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit b168c262f4c04d748b8a71aa6289e13385398764
+Merge: be5c828 00de014
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Dec 2 10:01:39 2016 +0100
+
+    Merge pull request #12267 from dachary/wip-17904-jewel
+    
+     jewel: Error EINVAL: removing mon.a at 172.21.15.16:6789/0, there will be 1 monitors
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 00de0140410ec8457ca25920866f8409f0d54a10
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Wed Nov 2 15:38:36 2016 +0000
+
+    mon: MonmapMonitor: drop unnecessary 'goto' statements
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit 20dcb597e35e6961db81831facefbe22cecddec3)
+
+commit 25f1b390525d975f90d0c7232b785415431d0f3e
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Wed Nov 2 15:33:52 2016 +0000
+
+    mon: MonmapMonitor: return success when monitor will be removed
+    
+    Fixes: http://tracker.ceph.com/issues/17725
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit c9d46cfbf2512bc3495c6901de2b8f711bef9bae)
+
+commit be5c82829916c581765f7c0130b738395a27720c
+Merge: 427f357 3cc29c6
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Dec 1 11:08:04 2016 -0800
+
+    Merge pull request #12001 from dachary/wip-17915-jewel
+    
+    jewel: filestore: can get stuck in an unbounded loop during scrub
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 79be070a4d442229d62b168ab87c95b662df1a9c
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Nov 28 22:06:56 2016 -0800
+
+    librados: remove new setxattr overload to avoid breaking the C++ ABI
+    
+    Fixes: http://tracker.ceph.com/issues/18058
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit b8ff781ddcf737882163cf56d7c9b11e815fb699)
+    
+    Conflicts:
+    	src/include/rados/librados.hpp (trivial namespace change in removed line)
+
+commit 4d6f8481b22ecfa5e55b2a6f8f8660f2d0445030
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 23 14:15:50 2016 -0500
+
+    crush: condition latest tunable encoding on features
+    
+    This avoids throwing hammer OSDMap encodings off.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9e5ff86487bd1f5979866b5e16300dd4a3979f97)
+
+commit bf96b30eaf552148249953ed4fb654cbb101c3d0
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Nov 28 14:35:53 2016 -0500
+
+    crush/CrushWrapper: encode with features
+    
+    No behavior change yet; just fixing callers.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b7c9e055848c8aa951bc48c957cff3ef323ea441)
+    
+    [Updated write_file to use all feaetures]
+    [Updated OSDMonitor.cc to use mon->quorum_features instead of the
+     mon->get_quorum_con_featuers() helper]
+    [trivial conflict from removed write_file and read_file]
+    
+    Conflicts:
+    	src/crush/CrushWrapper.h
+    	src/mgr/PyModules.cc
+    	src/mon/OSDMonitor.cc
+    	src/tools/ceph_monstore_tool.cc
+
+commit c5f5b94fc2e18b38d05f00fec04d2d7e3c35c54c
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Nov 28 14:35:24 2016 -0500
+
+    crush/CrushWrapper: drop unused 'lean' encode() argument
+    
+    No callers, no users.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 638a38bbb55c07ad0358a35a56418e66874d1c26)
+    
+    Conflicts:
+    	src/crush/CrushWrapper.h
+    
+    [trivial conflict due to removal of write_file and read_file]
+
+commit c66c556852b08e18d409e769eb7bd945c35e43cf
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 23 13:51:59 2016 -0500
+
+    osd/osd_types: encode pg_pool_t like hammer if features indicate hammer
+    
+    If the target features are missing the new OSDOp encoding, the
+    first feature we added post-hammer, encode like hammer.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2f8cfb632823ba4e63eaff394392d6af7979d7c8)
+
+commit 85caf34924bbfb0e872abd096ec571fff73035ae
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 23 13:48:35 2016 -0500
+
+    osd/osd_types: conditional pg_pool_t encoding
+    
+    Align this with decode.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 01d9e8a20bbc3c039f67b040da95018e2c7b00b6)
+
+commit 3cc29c6736007c97f58ba3a77ae149225e96d42a
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Nov 10 13:56:24 2016 -0500
+
+    os/filestore/HashIndex: fix list_by_hash_* termination on reaching end
+    
+    If we set *next to max, then the caller (a few lines up) doesn't terminate
+    the loop and will keep trying to list objects in every following hash
+    dir until it reaches the end of the collection.  In fact, if we have an
+    end bound we will never to an efficient listing unless we hit the max
+    first.
+    
+    For one user, this was causing OSD suicides when scrub ran because it
+    wasn't able to list all objects before the timeout.  In general, this would
+    cause scrub to stall a PG for a long time and slow down requests.
+    
+    Broken by refactor in 921c4586f165ce39c17ef8b579c548dc8f6f4500.
+    
+    Fixes: http://tracker.ceph.com/issues/17859
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c5180262a086c2d3895aff4bf0fb0ff9a6666149)
+
+commit 427f357f0eed32c9ce17590ae9303a94e8b710e7
+Merge: 0c38c46 8b595f5
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Nov 9 20:53:18 2016 +0100
+
+    Merge pull request #11822 from SUSE/wip-17816-jewel
+    
+    jewel: Missing comma in ceph-create-keys causes concatenation of arguments
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 8b595f567407be2a759e4987f33ce79e3763ea49
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sun Sep 18 16:26:29 2016 -0400
+
+    ceph-create-keys: add missing argument comma
+    
+    The arguments "get" and "client.admin" were being concatenated into
+    "getclient.admin".
+    
+    Found using ceph-ansible + strace:
+    
+        13031 execve("/usr/bin/ceph", ["ceph", "--cluster=ceph", "--name=mon.", "--keyring=/var/lib/ceph/mon/ceph-ceph-mon0/keyring", "auth", "getclient.admin"], ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin", "LANG=en_US.UTF-8", "CLUSTER=ceph", "TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=134217728", "CEPH_AUTO_RESTART_ON_UPGRADE=no"] <unfinished ...>
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 482022233d845b75876b04ca23fb137281a9f6ab)
+
+commit 0c38c464fff2280a6345b470f1c83aa6229776cc
+Merge: eb67259 eea546f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Nov 7 14:39:48 2016 +0100
+
+    Merge pull request #11679 from dachary/wip-17734-jewel
+    
+    jewel: Upgrading 0.94.6 -> 0.94.9 saturating mon node networking
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit eea546fafcf22573e33332ce91e3d112202ac207
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Sat May 21 14:11:55 2016 +0800
+
+    mon: expose require_jewel_osds flag to user
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 83ffc2b761742d563777e50959faa6a6010edae0)
+
+commit f8ee076ac4559dc9dbf828121618e78ad11687fd
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 21 12:25:08 2016 -0400
+
+    mon/OSDMonitor: encode OSDMap::Incremental with same features as OSDMap
+    
+    The Incremental encode stashes encode_features, which is
+    what we use later to reencode the updated OSDMap.  Use
+    the same features so that the encoding will match!
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 916ca6a0aaa32bd9c2b449e0d7fbd312c29f06e5)
+
+commit 1f629b2ba91c793db34614f319d12594646f5ce2
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 13 12:16:40 2016 -0400
+
+    mon/OSDMonitor: health warn if require_{jewel,kraken} flags aren't set
+    
+    We want to prompt users to set these flags as soon as their
+    upgrades complete.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 12e508313dbd5d1d38c76859cb7de2ce22404e12)
+    
+    Conflicts:
+       src/mon/OSDMonitor.cc: remove references to kraken
+    
+        if ((osdmap.get_up_osd_features() & CEPH_FEATURE_SERVER_KRAKEN) &&
+    	!osdmap.test_flag(CEPH_OSDMAP_REQUIRE_KRAKEN)) {
+          string msg = "all OSDs are running kraken or later but the"
+    	" 'require_kraken_osds' osdmap flag is not set";
+          summary.push_back(make_pair(HEALTH_WARN, msg));
+          if (detail) {
+    	detail->push_back(make_pair(HEALTH_WARN, msg));
+          }
+        } else
+
+commit 34555f11b068eb335866d4b536c9e10fe1de62e5
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 30 18:02:39 2016 -0400
+
+    mon/OSDMonitor: encode canonical full osdmap based on osdmap flags
+    
+    If the JEWEL or KRAKEN flags aren't set, encode the full map without
+    those features.  This ensure that older OSDs in the cluster will be able
+    to correctly encode the full map with a matching CRC.  At least, that is
+    true as long as the encoding changes are guarded by those feature bits.
+    That appears to be true currently, and we plan to ensure that it is true
+    in the future as well.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5e0daf6642011bf1222c4dc20aa284966fa5df9f)
+    
+    Conflicts:
+       src/mon/OSDMonitor.cc: removed reference to kraken
+    
+        if (!tmp.test_flag(CEPH_OSDMAP_REQUIRE_KRAKEN)) {
+          dout(10) << __func__ << " encoding without feature SERVER_KRAKEN" << dendl;
+          features &= ~CEPH_FEATURE_SERVER_KRAKEN;
+        }
+
+commit eb672598ad8209dcb7b7454fea31f921d255af1f
+Merge: 5e079cf d48e603
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Nov 4 15:31:05 2016 +0100
+
+    Merge pull request #11742 from tchaikov/wip-17728-jewel
+    
+    jewel: test/ceph_test_msgr: do not use Message::middle for holding transient…
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 5e079cfd46e4649e0eb24b15cb2c45887f5762fe
+Merge: 7b3ec11 eb30cc5
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Nov 3 15:54:28 2016 +0100
+
+    Merge pull request #11746 from liewegas/wip-post-file-key-jewel
+    
+    jewel: ceph-post-file: use new ssh key
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit eb30cc5ad6b439417298b7c615ff8ae15bf00fa2
+Author: David Galloway <dgallowa@redhat.com>
+Date:   Fri Aug 19 16:11:32 2016 -0400
+
+    ceph-post-file: Ignore keys offered by ssh-agent
+    
+    In my case, I had multiple private keys in ssh-agent which resulted in
+    the sftp connection failing despite explicitly specifying the private
+    key to use
+    
+    Signed-off-by: David Galloway <dgallowa@redhat.com>
+    (cherry picked from commit a61fcb2eac35a149b49efdc9b2ffa675afb968e8)
+
+commit 43282b0657ff19060dad25df79981ce17a76900f
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 2 09:37:41 2016 -0400
+
+    ceph-post-file: migrate to RSA SSH keys
+    
+    DSA keys are being deprecated: http://www.openssh.com/legacy.html
+    
+    drop.ceph.com will continue to allow the old DSA key but eventually,
+    users submitting logs using ceph-post-file will run into issues when
+    OpenSSH completely drops support for the algorithm.
+    
+    Fixes: http://tracker.ceph.com/issues/14267
+    
+    Signed-off-by: David Galloway <dgallowa@redhat.com>
+    (cherry picked from commit ecd02bf3f1c7a07a3271b2736a9e12dd6e897821)
+    
+    # Conflicts:
+    #	src/CMakeLists.txt
+
+commit d48e603d6c6715fbc127003226d327b79f30713a
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Oct 23 18:40:57 2016 -0500
+
+    msg: adjust byte_throttler from Message::encode
+    
+    Normally we never call encode on a message that has a byte_throttler set
+    because we only use it for messages we received.  However, for forwarded
+    messages that we clear_payload() before resending, we *do* reencode, and in
+    that case we need to retake the appropriate number of bytes from the
+    throttler--just like we release them in clear_payload().
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a9651282f7c16df872757b82d3d2995d92458d5c)
+
+commit 1bc616138fea897f36c1c25851f87df91404011c
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Oct 23 18:10:00 2016 -0500
+
+    msg/Message: fix set_middle vs throttler
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e7bf50b27a495ed75def67025d1ceca83861ba35)
+
+commit 5d5b5952322bb6a571c9cd58fd3e683eb32a2509
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Oct 22 14:01:34 2016 -0400
+
+    messages/MForward: reencode forwarded message if target has differing features
+    
+    This ensures we reencode the payload with the
+    appropriate set of features if the client, us, or the
+    target do not have identical features.  Otherwise we
+    may forward an encoding with more features than the
+    target can handle.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a433455e59067a844c3df4a0d6080db2ceb4ec59)
+
+commit e068c9206a3d618b3b04975d03f61ca64a92c4d4
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 28 11:44:28 2016 -0400
+
+    messages/MForward: fix encoding features
+    
+    We were encoding the message with the sending client's
+    features, which makes no sense: we need to encode with
+    the recipient's features so that it can decode the
+    message.
+    
+    The simplest way to fix this is to rip out the bizarre
+    msg_bl handling code and simply keep a decoded Message
+    reference, and encode it when we send.
+    
+    We encode the encapsulated message with the intersection
+    of the target mon's features and the sending client's
+    features.  This probably doesn't matter, but it's
+    conceivable that there is some feature-dependent
+    behavior in the message encode/decode that is important.
+    
+    Fixes: http://tracker.ceph.com/issues/17365
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d4f5e88f36e5388ae9e062c4bc49ac1c684a3f3c)
+
+commit 158b003751d8bedafdca60d859aef67e69d9a732
+Author: Michal Jarzabek <stiopa@gmail.com>
+Date:   Sat Jun 4 23:24:06 2016 +0100
+
+    all: add const to operator<< param
+    
+    Signed-off-by: Michal Jarzabek <stiopa@gmail.com>
+    (cherry picked from commit 0a157e088b2e5eb66177421f19f559ca427240eb)
+
+commit 3e1edde98610b11b94c59d23de979d6cd79dd8fe
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Oct 29 01:54:58 2016 +0800
+
+    test/ceph_test_msgr: do not use Message::middle for holding transient data
+    
+    Message::middle is used for holding encoded data, so we we can not stuff
+    it with payload and leave the "payload" field empty. this change
+    refactors the ceph_test_msgr by introducing a Payload class which
+    encodes all test data in it.
+    
+    Fixes: http://tracker.ceph.com/issues/17728
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 56896a7ed20869ce91ade4c77c1d6cbab8d50de1)
+    Conflicts:
+    	src/test/msgr/test_msgr.cc: do not use the new-style DENC()
+    framework for implementing the encoder of Payload class. DENC() was
+    introduced after jewel was released.
+
+commit 8f75bd64768b6984c537dd15b5d9159070b86d91
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Aug 8 23:20:58 2016 +0800
+
+    test/ceph_test_msgr: fix circular locking dependency
+    
+    * do not acquire lock when sending message
+    * remove lock in session
+    * reduce the scope guarded by locks for better performance.
+    
+    Fixes: http://tracker.ceph.com/issues/16955
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit cf1801c260c42aa93850538eea7a194440ebe350)
+
+commit f960db4646a9535bcee6d53740811b84e0678c93
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Thu Jul 7 14:59:51 2016 +0800
+
+    ceph_test_msgr: use ceph log infrastructure to output
+    
+    because we want to get the right log sequence which mixes ceph logginer and
+    cerr. Otherwise, cerr output make the logs a little disordered.
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit d1268a6aa895ee93cd9fee6fc6c759317e681a85)
+
+commit 7b3ec119cb968a26526ad95355c5bf7525fb5346
+Merge: 5efb6b1 779af22
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Tue Nov 1 17:05:27 2016 -0700
+
+    Merge pull request #11728 from ceph/wip-librados-upgrade-jewel
+    
+    qa: remove EnumerateObjects from librados upgrade tests
+
+commit 5efb6b1c2c9eb68f479446e7b42cd8945a18dd53
+Merge: 3c9fe54 39b8e78
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Oct 28 14:29:20 2016 +0200
+
+    Merge pull request #11669 from ceph/wip-jewel-11566
+    
+    rgw: fix put_acls for objects starting and ending with underscore
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 3c9fe545dccf13413bb491098c089d3e4a76bcd2
+Merge: 5ca8791 6356664
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 27 21:40:26 2016 +0200
+
+    Merge pull request #11472 from dachary/wip-17510-jewel
+    
+    jewel: ERROR: got unexpected error when trying to read object: -2
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 39b8e783defb930b1dd8eeecdfee68d0d886d03b
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed Oct 19 21:51:01 2016 +0200
+
+    rgw: fix put_acls for objects starting and ending with underscore
+    
+    Fixes: http://tracker.ceph.com/issues/17625
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 14d4d912c7b47c56b16ae0bdc6bc08d208de3461)
+
+commit 5ca879114fcc98b906cac64a5ef5cb2a8568cb60
+Merge: ed9a824 dc2ffda
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Oct 27 13:52:22 2016 -0400
+
+    Merge pull request #11662 from linuxbox2/jewel-17635
+    
+    jewel: rgw: handle empty POST condition
+
+commit ed9a824f2075e15ac245b9aac683cb28183ecd3a
+Merge: e08b0f3 f5e37ab
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 27 16:48:43 2016 +0200
+
+    Merge pull request #11634 from dillaman/wip-17590-jewel
+    
+    jewel: journal: do not prematurely flag object recorder as closed
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 779af22fc920f1fdfdd3fa2b01d8587088372bd4
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Oct 26 16:33:53 2016 -0700
+
+    qa: remove EnumerateObjects from librados upgrade tests
+    
+    These rely on new rados functionality not present in jewel
+    
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+
+commit e08b0f308f5af45c1b1867ab5b757486bba51333
+Merge: 0aee633 4cb83c1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 26 23:00:35 2016 +0200
+
+    Merge pull request #11644 from ceph/wip-17695
+    
+    jewel: librbd: discard after write can result in assertion failure
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 4cb83c14dbe09d4b371f7b728d9b5c0549e59f1a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 25 09:43:06 2016 -0400
+
+    librbd: discard after write can result in assertion failure
+    
+    With journaling enabled, the proper lock is not held when handling
+    a discard after write to overlapping extents. This issue is only present
+    on the jewel branch due to design changes on the master branch.
+    
+    Fixes: http://tracker.ceph.com/issues/17695
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit dc2ffda7819d2ebeed3526d9e6da8f53221818de
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Oct 20 10:17:36 2016 -0700
+
+    rgw: handle empty POST condition
+    
+    Fixes: http://tracker.ceph.com/issues/17635
+    
+    Before accessing json entity, need to check that iterator is valid.
+    If there is no entry return appropriate error code.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 23cb642243e09ca4a8e104f62a3bb7b2cbb6ea12)
+
+commit 0aee6331ad93a3caf212b84412bb648171758fff
+Merge: 06f7d7a cd99a64
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 26 16:47:03 2016 +0200
+
+    Merge pull request #11657 from dachary/wip-17707-jewel
+    
+    jewel: ceph-disk: using a regular file as a journal fails
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 06f7d7a9b2403cf8b7af5301dae575d6f21c71ce
+Merge: 7ff2c10 7964187
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 26 14:04:20 2016 +0200
+
+    Merge pull request #11321 from linuxbox2/jewel-11051
+    
+    jewel: rgw: Do not archive metadata by default
+    
+    Reviewed-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 7ff2c108ea12451b6dcb25dab4574f2de68162d1
+Merge: 0fb486d 2f9a5be
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 26 11:55:49 2016 +0200
+
+    Merge pull request #11626 from ceph/wip-jewel-11567
+    
+    jewel: don't loop forever when reading data from 0 sized segment.
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 0fb486d59e1574e936564e4b048a089dee8b267b
+Merge: bf5c47a 86eef27
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 26 11:39:40 2016 +0200
+
+    Merge pull request #11478 from dachary/wip-17312-jewel
+    
+    jewel: build/ops: allow building RGW with LDAP disabled
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit cd99a642a088c7cd010e45a81d6680cdfca16ca6
+Author: Jayashree Candadai <jayaajay@indiana.edu>
+Date:   Fri Oct 21 13:52:05 2016 -0400
+
+    ceph-disk: allow using a regular file as a journal
+    
+    Because of a missing return, ceph-disk prepare would fail if given a
+    regular file as a journal. If the journal file does not exist, ceph-disk
+    will create it but fail to ensure that the ceph user owns it. The
+    symlink to the journal file is not set when the journal file is
+    specified on the command line and the journal file does not exist at
+    all. The ceph-osd daemon will silently create it as a file but it will
+    not be the file given in argument.
+    
+    Add a test case to verify using a regular file as a journal works as
+    expected.
+    
+    Fixes: http://tracker.ceph.com/issues/17662
+    
+    Signed-off-by: Jayashree Candadai <jayaajay@indiana.edu>
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit db917d50eb5f86a07a5487e130f46a6b1d27672a)
+
+commit a80040230cddc395809d2323392c87a4a1fef923
+Author: Anirudha Bose <ani07nov@gmail.com>
+Date:   Wed Aug 17 12:19:33 2016 +0530
+
+    ceph-disk: PEP8ify
+    
+    Signed-off-by: Anirudha Bose <ani07nov@gmail.com>
+    (cherry picked from commit f7a15ccfa2cc214c3688a2b41ac45be1abfb7700)
+    
+    Conflicts:
+    	src/ceph-disk/setup.py: python 2.7 restriction not backported
+
+commit e200b17016a1a7cd103affde2d2b265916755498
+Author: Anirudha Bose <ani07nov@gmail.com>
+Date:   Wed Aug 17 12:15:25 2016 +0530
+
+    ceph-disk: Set space_symlink to the path, not file object
+    
+    Signed-off-by: Anirudha Bose <ani07nov@gmail.com>
+    (cherry picked from commit d290454cf5f660c1681eefd70a38e79da683525f)
+
+commit 9191825c06b3d92e2caa26c9efdb69a79cc3964b
+Author: Anirudha Bose <ani07nov@gmail.com>
+Date:   Wed Aug 17 12:13:53 2016 +0530
+
+    ceph-disk: Use os.path.isabs instead of .startswith('/')
+    
+    Signed-off-by: Anirudha Bose <ani07nov@gmail.com>
+    (cherry picked from commit 6f795356de00dd7c33282183b11a03da800fde7b)
+
+commit f1c2de768535ce18259d4fc187cd370766c1e6cf
+Author: Anirudha Bose <ani07nov@gmail.com>
+Date:   Sun Jun 19 05:10:27 2016 +0530
+
+    ceph-disk: Use context manager with FileLock
+    
+    acquire and release methods of FileLock are dropped
+    
+    Signed-off-by: Anirudha Bose <ani07nov@gmail.com>
+    (cherry picked from commit c1011d514ce0c7d340a5acd6f9c640165e169156)
+
+commit 26cb5b6bfbd6eee46bfa3cf005b1cd9f06b0e262
+Author: Anirudha Bose <ani07nov@gmail.com>
+Date:   Sun Jun 19 04:05:42 2016 +0530
+
+    ceph-disk: Fix bug in FileLock
+    
+    Python fcntl.lockf() accepts a file descriptor, not a file object
+    
+    Signed-off-by: Anirudha Bose <ani07nov@gmail.com>
+    (cherry picked from commit df9cc2ce938a969f4044b63fd80030d00f64f060)
+
+commit 91a9ca374f646966e31b015928a2cc60bc5225aa
+Author: Anirudha Bose <ani07nov@gmail.com>
+Date:   Thu Jun 16 16:37:00 2016 +0530
+
+    ceph-disk: Use true integer division in get_dev_size
+    
+    Signed-off-by: Anirudha Bose <ani07nov@gmail.com>
+    (cherry picked from commit f1bb72c82806cc03e85e0b19c83d61409c6b2d51)
+
+commit 2373ccb7a991069406e344b91fa70c0b2d1aa5e0
+Author: Anirudha Bose <ani07nov@gmail.com>
+Date:   Tue Jun 14 21:42:02 2016 +0530
+
+    ceph-disk: Compatibility fixes for Python 3
+    
+    ceph-disk: Misc cleanups
+    
+    Signed-off-by: Anirudha Bose <ani07nov@gmail.com>
+    (cherry picked from commit d0e29c74f84a2ed3014a516c0106172619314bdc)
+    
+    Conflicts:
+         src/ceph-disk/tox.ini: python3 is not supported in jewel
+
+commit 79cf6330d67ad52af5690f4d4efd29aa2722acb2
+Author: Shylesh Kumar <shmohan@redhat.com>
+Date:   Thu Jul 7 20:45:57 2016 +0530
+
+    ceph-disk: change ownership of init file to ceph:ceph
+    
+    Fixes: http://tracker.ceph.com/issues/16280
+    Signed-off-by: Shylesh Kumar <shmohan@redhat.com>
+    (cherry picked from commit aab9d03e1b50ba10a383663088400b9fabe306cb)
+
+commit 796418746ecd2a4971e365499c6952e131e74913
+Author: root <root@ceph-node1.homeoffice.wal-mart.com>
+Date:   Mon Sep 12 14:30:43 2016 +0530
+
+    rgw: Do not archive metadata by default
+    
+    Fixes: http://tracker.ceph.com/issues/17256
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    
+    (cherry picked from commit c617ea83e0cca0061af18e0811c7ef8b4e836519)
+
+commit bf5c47a28abe6d91dfdace8d6803a7016bbf02fb
+Merge: 7714689 1eedf18
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 25 15:49:55 2016 +0200
+
+    Merge pull request #11642 from tchaikov/wip-17685-jewel
+    
+    jewel: mon: fix missing osd metadata (again)
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 77146891ab254bc796e38e2a45b8500c0119745f
+Merge: 3ccc17b e8e1acb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 25 15:40:01 2016 +0200
+
+    Merge pull request #11467 from dachary/wip-17262-jewel
+    
+    jewel: rbd-nbd IO hang
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 1eedf181176ce75212a8cfbb87ac927350a826ed
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed May 25 17:56:51 2016 +0100
+
+    mon: fix missing osd metadata (again)
+    
+    The JSON output was getting broken by continuing
+    in the wrong place.
+    
+    Fixes: http://tracker.ceph.com/issues/17685
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit c5700ce4b45b3a385fe4c2111da852bea7d86da2)
+
+commit f5e37abefcb015b2cb58295cfdf109fd6f5833b0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Oct 17 09:48:20 2016 -0400
+
+    journal: do not prematurely flag object recorder as closed
+    
+    Fixes: http://tracker.ceph.com/issues/17590
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2be6367cb0119d86dfecfa94eb4a3a102c40162a)
+
+commit 2f9a5beb9a903220b70a780b30564d9741e9f0e5
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Thu Oct 13 21:12:36 2016 -0400
+
+    Don't loop forever when reading data from 0 sized segment.
+    
+    The 0 sized segment can arise depending on how
+    the client uploads the object in the first place.
+    The cpu loop then happens during a swift `GET'.
+    
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    (cherry picked from commit 46c5f9773246522e66bb2cca49345d0b62a16c42)
+
+commit 3ccc17b81d2794406d803ff4210e930a1fa67455
+Merge: 112b89d 0b30a1d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 24 12:03:13 2016 +0200
+
+    Merge pull request #10757 from dachary/wip-17056-jewel
+    
+    jewel: mon/osdmonitor: decouple adjust_heartbeat_grace and min_down_reporters
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 112b89dc315dd1240d68eb50fba1438057002327
+Merge: 7ab1e02 0dcefd2
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 24 12:03:01 2016 +0200
+
+    Merge pull request #10759 from dachary/wip-16866-jewel
+    
+    jewel: OSD: ceph osd df does not show summarized info correctly if one or more OSDs are out
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 7ab1e0275101a30c2d318bcd55787f61c687328f
+Merge: e053b20 4f9e02c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 24 11:55:41 2016 +0200
+
+    Merge pull request #11590 from dachary/wip-17642-jewel
+    
+    jewel: TestJournalReplay: sporadic assert(m_state == STATE_READY || m_state == STATE_STOPPING) failure
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 4f9e02c40ac5359f344d29f20332ec6ed8575b8e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 13 10:04:24 2016 -0400
+
+    test: TestJournalReplay test cases need to wait for committed journal event
+    
+    Fixes: http://tracker.ceph.com/issues/17566
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1bdf7a425eb7436838c9445ea2c8ab42dfd3a1b6)
+
+commit e8e1acb1d5154b749d251efa88b45e8ad3edb2bb
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Sep 8 11:51:34 2016 -0400
+
+    librbd: ignore cache busy errors when shrinking an image
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4ce663845679dc35f2f15b893c6f988c4a60b25b)
+    
+    Conflicts:
+         src/test/librbd/operation/test_mock_ResizeRequest.cc:
+         when_resize does not have the allow_shrink argument because
+         d1f2c557b2c039730baca9efa3f5244bc19dcb1a has not been
+         backported
+
+commit ba2e87e0443069b0552b698cc7a508898a3f585d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Sep 8 09:59:37 2016 -0400
+
+    librbd: invalidate cache before trimming image
+    
+    Any potential writeback outside the extents of a shrunk image
+    would result in orphaned objects.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3f93a1917437ba7b69b306e4ff971b79e8b79c89)
+
+commit d7c0873ea77234b7b736080c3de4012d2f6adaee
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Sep 7 11:49:26 2016 -0400
+
+    rbd-nbd: mask out-of-bounds IO errors caused by image shrink
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c6cfb616a846959a2cd1c0e540b14668c61a2afd)
+
+commit 0ce342d39f359d23c92ab94efb910b84a634094a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Sep 5 10:01:45 2016 -0400
+
+    rbd-nbd: fix kernel deadlock during teuthology testing
+    
+    Fixes: http://tracker.ceph.com/issues/16921
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ce7c1520a8019f011fa34dd898af317f78974577)
+
+commit e053b2085b0d0be0e8c9912c82c7142bb0871783
+Merge: f6d7290 caf08d7
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Thu Oct 20 11:17:49 2016 -0400
+
+    Merge pull request #11519 from dachary/wip-17576-jewel
+    
+    jewel: RGW loses realm/period/zonegroup/zone data: period overwritten if somewhere in the cluster is still running Hammer
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit f6d729082359b450950513080c9c08461d6f557c
+Merge: 9e9a38d 40689d1
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Thu Oct 20 11:17:22 2016 -0400
+
+    Merge pull request #11471 from dachary/wip-17511-jewel
+    
+    jewel: s3tests-test-readwrite failing with 500
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 9e9a38df303749147a638f0df42ad875c829d294
+Merge: 9b0fa2b 5a53ffa
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Thu Oct 20 11:16:38 2016 -0400
+
+    Merge pull request #11469 from dachary/wip-17538-jewel
+    
+    jewel: rgw:user email can modify to empty when it has values
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 9b0fa2b27386dd0c09aead832c9f049ef10b26fb
+Merge: 0eac635 4babd3f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 16:31:10 2016 +0200
+
+    Merge pull request #11466 from dachary/wip-17290-jewel
+    
+    jewel: ImageWatcher: use after free within C_UnwatchAndFlush
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 0eac63545da063a8bbab4b1ea6c9a765023617fc
+Merge: 51d5ed0 b73356b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 16:30:53 2016 +0200
+
+    Merge pull request #11464 from dachary/wip-17373-jewel
+    
+    jewel: image.stat() call in librbdpy fails sometimes
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 51d5ed0ddbe7e1018be1fa46c912d2e0ca07a984
+Merge: d2150fb b410c8e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 16:30:44 2016 +0200
+
+    Merge pull request #11463 from dachary/wip-17384-jewel
+    
+    jewel: helgrind: TestLibRBD.TestIOPP potential deadlock closing an image with read-ahead enabled
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit d2150fb54e894039e03855352708fad6b08d75d0
+Merge: f8989ef 32d84e0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 16:30:32 2016 +0200
+
+    Merge pull request #11462 from dachary/wip-17404-jewel
+    
+    jewel: update_features API needs to support backwards/forward compatibility
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit f8989ef7a79dd37f58fc9bddbd1e3fcd83e14f42
+Merge: ac2ce6d 7d6801b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 16:28:18 2016 +0200
+
+    Merge pull request #11459 from dachary/wip-17483-jewel
+    
+    jewel: RBD should restrict mirror enable/disable actions on parents/clones
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit ac2ce6dcda27a70a45e17270cab4d5e9a856abc7
+Merge: cd48521 391936a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 16:27:36 2016 +0200
+
+    Merge pull request #11460 from dachary/wip-17482-jewel
+    
+    jewel: Enable/Disable of features is allowed even the features are already enabled/disabled
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit cd48521cde8bb69b6879977f51404a3014571a84
+Merge: 410a368 dd93e00
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 16:26:53 2016 +0200
+
+    Merge pull request #11461 from dachary/wip-17481-jewel
+    
+    jewel: Proxied operations shouldn't result in error messages if replayed
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 410a368e9bdf31f6512be5e9a783c7990ba9003f
+Merge: 07a8d8a d66bb7a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 12:42:01 2016 +0200
+
+    Merge pull request #11431 from ceph/wip-jewel-tarball
+    
+    jewel: build/ops: include more files in "make dist" tarball
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.cz>
+    Reviewed-by: Boris Ranto <branto@redhat.com>
+
+commit 07a8d8aedfa101a52e4764cc211fd2a1d8f36582
+Merge: 62f7f64 50fd48f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 11:48:49 2016 +0200
+
+    Merge pull request #11474 from dachary/wip-17350-jewel
+    
+    jewel: rgw:response information is error when geting token of swift account
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 62f7f646b85f81c0c812f3d5adc1926d4cee3f7d
+Merge: 638590c 91bd342
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 11:47:43 2016 +0200
+
+    Merge pull request #11492 from SUSE/wip-17575-jewel
+    
+    jewel: aarch64: Compiler-based detection of crc32 extended CPU type is broken
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 638590c97a5365df118c72aab2e510ffcd2a9470
+Merge: c58d626 6575545
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 11:28:29 2016 +0200
+
+    Merge pull request #11475 from dachary/wip-17349-jewel
+    
+    jewel: Modification for TEST S3 ACCESS section in INSTALL CEPH OBJECT GATEWAY page
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit c58d62677449b08023b761762a52cc4ceaf69def
+Merge: 3cb4bf4 7423661
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 11:27:54 2016 +0200
+
+    Merge pull request #11473 from dachary/wip-17509-jewel
+    
+    jewel: Config parameter rgw keystone make new tenants in radosgw multitenancy does not work
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 3cb4bf43b923a71872525b0222b4b0fc73664fb0
+Merge: fb74b16 de672a0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 11:27:18 2016 +0200
+
+    Merge pull request #11201 from ceph/wip-backport-logrotate-jewel
+    
+    jewel: build/ops: backport 'logrotate: Run as root/ceph'
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit fb74b1688603c0351e80ed504be4faad320a67e6
+Merge: 9d3373d 25a35d4
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 11:22:45 2016 +0200
+
+    Merge pull request #11126 from tchaikov/wip-17179-jewel
+    
+    jewel: add a tool to rebuild mon store from OSD
+    
+    Reviewed-by: huanwen ren <ren.huanwen@zte.com.cn>
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 9d3373d71f8af16a8f0220aa0d0682e2f3bb5a8c
+Merge: 0c83eb3 eb6c3cb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 20 09:39:15 2016 +0200
+
+    Merge pull request #11563 from ceph/wip-jewel-acl-underscore
+    
+    rgw: fix regression with handling double underscore
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit eb6c3cbcc6cadd4eff9de0d6332f42c785486f7b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 18:35:59 2016 -0700
+
+    rgw: set correct instance on the object
+    
+    Fixes: http://tracker.ceph.com/issues/17443
+    
+    This was broken by commit bc840afafdfe5e528e5c1b711e71420ac3cb5a67
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d2ada35c8c0d938a704ecf9974988ea30a9c0105)
+
+commit 0c83eb355e989fb6ed38a3b82f9705fd5d700e89
+Merge: 7496388 f400ff2
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 19 23:14:23 2016 +0200
+
+    Merge pull request #11548 from dachary/wip-17609-jewel
+    
+    jewel: tests: ceph-disk must ignore debug monc
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 7496388468688763cab7fab9cc2f61cfaeeb4e7c
+Merge: f8055a9 d205b74
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 19 23:11:24 2016 +0200
+
+    Merge pull request #11411 from dachary/wip-17245-jewel
+    
+    jewel: tests: scsi_debug fails /dev/disk/by-partuuid
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f8055a9453e228079a8c61ba398e93e9fbdcf55a
+Merge: 3b2d360 dea93dc
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 19 23:11:04 2016 +0200
+
+    Merge pull request #10884 from dachary/wip-17149-jewel
+    
+    jewel: ceph-disk: expected systemd unit failures are confusing
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 084108e440b7b9b8f0d36282dd4eba64a4b96f4b
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed Aug 31 22:25:57 2016 +0200
+
+    rgw: fix regression with handling double underscore
+    
+    Fixes: http://tracker.ceph.com/issues/16856
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit bc840afafdfe5e528e5c1b711e71420ac3cb5a67)
+
+commit 3b2d36039305bc070ac1eaf5142d5b5bfa7accb1
+Merge: d15d6dc 483d8c4
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 19 16:50:44 2016 +0200
+
+    Merge pull request #11408 from dachary/wip-17345-jewel
+    
+    jewel: Ceph Status - Segmentation Fault
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit d15d6dc896d0ac30ddff4354600d7c2ecb61b54a
+Merge: d4197dc 5ae4f31
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 19 16:50:22 2016 +0200
+
+    Merge pull request #11407 from dachary/wip-17360-jewel
+    
+    jewel: ceph-objectstore-tool crashes if --journal-path <a-directory>
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit d4197dcae43b073d121c4b12ff7cca2f619745e1
+Merge: 54bb909 c94244d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 19 09:47:00 2016 +0200
+
+    Merge pull request #11158 from dillaman/wip-rbdmap-jewel
+    
+    jewel: systemd: add install section to rbdmap.service file
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 54bb9092a8d658ab2f352ed8c850d719b2468d83
+Merge: dc59575 c3c2910
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 19 09:46:19 2016 +0200
+
+    Merge pull request #10862 from dachary/wip-17095-jewel
+    
+    jewel: rpm: ceph installs stuff in %_udevrulesdir but does not own that directory
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f400ff222e531feb00de3ebf1860a0d4bcca736e
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Oct 18 17:33:23 2016 +0200
+
+    tests: ceph-disk: force debug monc = 0
+    
+    The sh function will collect both stderr and stdout and debug
+    will mess the json parsing.
+    
+    Fixes: http://tracker.ceph.com/issues/17607
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 469a53a4adc335ea4ff8e34a958b12f0f222a6c1)
+
+commit dc59575b2fec477bdcf152f91a02532fcdfd1a10
+Merge: 778d884 c2cf21d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 19 08:56:46 2016 +0200
+
+    Merge pull request #10860 from dachary/wip-17121-jewel
+    
+    jewel: the %USED of ceph df is wrong
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 778d884512d2cf298c60634cda9a6f66dc2b948b
+Merge: 82edc5b a25a483
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Oct 19 01:35:13 2016 +0200
+
+    Merge pull request #10886 from dachary/wip-17144-jewel
+    
+    jewel: mark_all_unfound_lost() leaves unapplied changes
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 82edc5bc526e02d7c535d01642d8ef7f00d19939
+Merge: c4f4f8a e6ac214
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 18 13:05:43 2016 +0200
+
+    Merge pull request #10784 from dachary/wip-17067-jewel
+    
+    jewel: Request exclusive lock if owner sends -ENOTSUPP for proxied maintenance op
+    
+    Reviewed-by: Ilya Dryomov <idryomov@redhat.com>
+
+commit 25a35d43a8bf7e1a7d78d5d1d2a5556dff98f5e8
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Oct 10 18:43:39 2016 +0800
+
+    doc: fill keyring with caps before passing it to ceph-monstore-tool
+    
+    to make sure the recovered monitor store is ready for use.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit af8e21163735377071b6832d8a81b035bb835257)
+
+commit 73ea9264803bdf53a5da64992c1b91e94633f5e3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Oct 10 16:32:27 2016 +0800
+
+    tools/ceph_monstore_tool: bail out if no caps found for a key
+    
+    we take it as an error if no caps is granted to an entity in the
+    specified keyring file when rebuilding the monitor db.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b4bd4004a836121c11b0bb97d8123df54c271f04)
+
+commit 4ebf87bccdb17bd04b2c615e6278a7816ae1ff43
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Sep 30 17:58:14 2016 +0800
+
+    tools/ceph_monstore_tool: update pgmap_meta also when rebuilding store.db
+    
+    we should rebuild pgmap_meta table from the collected osdmaps
+    
+    Fixes: http://tracker.ceph.com/issues/17400
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit cdfa7a69f63d047205dcfccd63b5d58ab0d4695b)
+
+commit d2deed6bfbc5693e3bd8a10a2f9be8df8ef1b0e5
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Sun Sep 18 11:40:56 2016 +0800
+
+    tools/rebuild_mondb: kill compiling warning
+    
+    As follow:
+    
+    [ 72%] Building CXX object src/tools/CMakeFiles/ceph-objectstore-tool.dir/RadosDump.cc.o
+    /home/jenkins-build/build/workspace/ceph-pull-requests/src/tools/rebuild_mondb.cc: In function ‘int update_mon_db(ObjectStore&, OSDSuperblock&, const string&, const string&)’:
+    /home/jenkins-build/build/workspace/ceph-pull-requests/src/tools/rebuild_mondb.cc:289:22: warning: ‘crc’ may be used uninitialized in this function [-Wmaybe-uninitialized]
+             if (have_crc && osdmap.get_crc() != crc) {
+                          ^
+    /home/jenkins-build/build/workspace/ceph-pull-requests/src/tools/rebuild_mondb.cc:238:14: note: ‘crc’ was declared here
+         uint32_t crc;
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit f16a31476a3f9b44a7c3dabf0dfd2a0d015b11b9)
+
+commit 09701269de225e556099b9e5c511faa44acae024
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Sun Sep 18 10:33:56 2016 +0800
+
+    tools/rebuild_mondb: avoid unnecessary result code cast
+    
+    In general we return negative codes for error cases, so there is
+    no need perform the cast here.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 6a1c01d334fe65124043aa68a6e0cfaea43836b5)
+
+commit 5191b06294cf505716d0c64ac36528e9ea57b0a8
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Oct 1 14:18:55 2016 +0800
+
+    doc: add rados/operations/disaster-recovery.rst
+    
+    document the process to recover from leveldb corruption.
+    
+    Fixes: http://tracker.ceph.com/issues/17179
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 79a9f299253e24d20547131b3c9c9e0667e3b869)
+    Conflicts:
+            src/tools/rebuild_mondb.cc:
+    		remove the code change in this file from this commit.
+    		and the code gets removed is added in anther commit.
+
+commit 8c8d5ce529fa826bc0d453edf5fb5e98e29294d3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Aug 29 19:53:11 2016 +0800
+
+    tools/ceph_monstore_tool: add "rebuild" command
+    
+    Fixes: http://tracker.ceph.com/issues/17179
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d909fa035c8fbbded786b2ca072acc10ea6b6052)
+
+commit 1fcb0bb81a8989b31a79bc565354f6817ceb12d7
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Aug 29 19:52:19 2016 +0800
+
+    tools/ceph-objectstore-tool: add "update-mon-db" command
+    
+    Fixes: http://tracker.ceph.com/issues/17179
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    Conflicts:
+    	src/tools/CMakeLists.txt: this file was added in master, so
+    		update src/CMakeLists.txt instead
+    	src/tools/Makefile-server.am: jewel is still using autotools,
+    		so update this file also.
+            src/tools/rebuild_mondb.cc: move the code spilled into
+                    doc/rados/troubleshooting/troubleshooting-mon.rst
+                    by accident back to this commit.
+    (cherry picked from commit 24faea7ce446bbf09cbd4a9d3434dd5444a6c295)
+
+commit 416750258fb63064a36eaf53f586a51fc3ea63e0
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Aug 31 13:11:24 2016 +0800
+
+    mon/AuthMonitor: make AuthMonitor::IncType public
+    
+    so ceph-objectstore-tool is able to use it when rebuilding monitor
+    db.
+    
+    Fixes: http://tracker.ceph.com/issues/17179
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 19ef4f16b3aba04119ac647cd6261c74a57ce829)
+
+commit c4f4f8a72421e9c43c09eb9dfb5a3c6fe2123886
+Merge: ded7c27 5ef9085
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Oct 17 11:56:30 2016 -0700
+
+    Merge pull request #10885 from dachary/wip-17145-jewel
+    
+    jewel: PG::choose_acting valgrind error or ./common/hobject.h: 182: FAILED assert(!max || (*this == hobject_t(hobject_t::get_max())))
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit ded7c27a2d0fc0ae307321de2bf419e065c36ba1
+Merge: bdcdedd 58b7c52
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Oct 17 11:54:29 2016 -0700
+
+    Merge pull request #10883 from dachary/wip-17141-jewel
+    
+    jewel: PG::_update_calc_stats wrong for CRUSH_ITEM_NONE up set items
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit bdcdedd32f9e8a71735ae1d4d3ca18b41bde3378
+Merge: 757f8e8 3bb2a9e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 18:08:46 2016 +0200
+
+    Merge pull request #11420 from dachary/wip-17556-jewel
+    
+    jewel: librbd::Operations: update notification failed: (2) No such file or directory
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 757f8e87e0aa39af05f0bc7b8146c826191e8032
+Merge: 4d2bc33 bd63666
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 18:08:32 2016 +0200
+
+    Merge pull request #10857 from dachary/wip-16984-jewel
+    
+    jewel: Disabling pool mirror mode with registered peers results orphaned mirrored images
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 4d2bc33101aaadbab73cdae3327fc0888a15e25a
+Merge: 23d91cf 18a66cb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 17:55:10 2016 +0200
+
+    Merge pull request #10645 from dachary/wip-16458-jewel
+    
+    jewel: Potential crash during journal::Replay shut down
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 23d91cfcd1c77766972577639ef72db353076e37
+Merge: 51c2794 775c78d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 17:54:55 2016 +0200
+
+    Merge pull request #10652 from dachary/wip-16951-jewel
+    
+    jewel: ceph 10.2.2 rbd status on image format 2 returns (2) No such file or directory
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 51c279425e26626884ec54add6043c7ed3873920
+Merge: ee32be2 bb81f9d
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 17:37:41 2016 +0200
+
+    Merge pull request #11337 from SUSE/wip-17060-jewel
+    
+    jewel: librbd: cannot disable journaling or remove non-mirrored, non-primary image
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit ee32be2ae3c328238fd2bbb953c18d3b9f0a51b1
+Merge: 7d0dd1c 2b27212
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 17:37:06 2016 +0200
+
+    Merge pull request #10650 from dachary/wip-16868-jewel
+    
+    jewel: Prevent the creation of a clone from a non-primary mirrored image
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 7d0dd1cf6f6568f86fa593de0a7845462a510d31
+Merge: f1c21c6 1c76ef4
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 17:06:51 2016 +0200
+
+    Merge pull request #11433 from dillaman/wip-17416-jewel
+    
+    jewel: rbd-mirror: improve resiliency of stress test case
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f1c21c6f01e20949753198e5f3ad160e0213cc5b
+Merge: 986a8b5 80e25b0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 17:04:11 2016 +0200
+
+    Merge pull request #10796 from dillaman/wip-17059-jewel
+    
+    jewel: rbd: bench io-size should not be larger than image size
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 986a8b5eacbf0d6084121eef8407088a24d27ffd
+Merge: c7b7f2e 10e603b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 14:28:06 2016 +0200
+
+    Merge pull request #11414 from dachary/wip-17477-jewel
+    
+    jewel: Crash in Client::_invalidate_kernel_dcache when reconnecting during unmount
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit c7b7f2e8c0de596255e651ae5e499c586f1e4493
+Merge: ada7edd 3320da0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 14:27:53 2016 +0200
+
+    Merge pull request #11412 from dachary/wip-17479-jewel
+    
+    jewel: Duplicate damage table entries
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit ada7edd1d21073b71c1d4538ab66b15b978cf418
+Merge: 1412b17 3a79db1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 14:27:43 2016 +0200
+
+    Merge pull request #11415 from dachary/wip-17476-jewel
+    
+    jewel: Failure in snaptest-git-ceph.sh
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 1412b171c057fabd05abaf9adcf31aa29a4f5b00
+Merge: 368c96c 0a17741
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 14:27:30 2016 +0200
+
+    Merge pull request #11416 from dachary/wip-17474-jewel
+    
+    jewel: Failure in dirfrag.sh
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 368c96c513cd5424d7935fc02edd6297fb680bb7
+Merge: 4486e3b 2c4e1c1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 14:27:17 2016 +0200
+
+    Merge pull request #11418 from dachary/wip-17246-jewel
+    
+    jewel: Log path as well as ino when detecting metadata damage
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 4486e3b5e00e809bc3391613cc5160810ce5bf1e
+Merge: b174220 5173563
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 14:27:04 2016 +0200
+
+    Merge pull request #11419 from dachary/wip-17244-jewel
+    
+    jewel: Failure in snaptest-git-ceph.sh
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit b1742204e10d563573bbda82917b843c01b20420
+Merge: 54240c4 c0db9fb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 14:26:30 2016 +0200
+
+    Merge pull request #10877 from dachary/wip-16946-jewel
+    
+    jewel: client: nlink count is not maintained correctly
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 54240c4d3dfeef97c6fba343fe501f0303c73f1b
+Merge: 8b5aa5d 430ab1b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 17 13:40:13 2016 +0200
+
+    Merge pull request #10758 from dachary/wip-17007-jewel
+    
+    jewel: ceph-disk should timeout when a lock cannot be acquired
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit caf08d731c7cec40fe6635189de877d1f047e8b4
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Tue Oct 11 14:18:29 2016 +0200
+
+    rgw: avoid corruption when running old radosgw-admin on a newer rgw
+    
+    Fixes:http://tracker.ceph.com/issues/17371
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 8eab9454b4d45ed8092a1292bd1904ef928c2120)
+
+commit e5f7854ddfc2a6c123d77b1d33e659b4f870fbe6
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Tue Oct 11 11:19:01 2016 +0200
+
+    rgw: Fix missing master zone for default region conversion
+    
+    Fixes:http://tracker.ceph.com/issues/17371
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit cd3bccdef2ecd1d294ab1192ab3b9e731b5592e0)
+
+commit ec2fb022307eea93a802f80307b957bfb2581249
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Tue Oct 11 11:00:00 2016 +0200
+
+    rgw: mark configuration as converted to avoid corruption when running older admins
+    
+    Fixes: http://tracker.ceph.com/issues/17371
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit ceafa2863dd9c02da8a30c5a5324b472ed5d3404)
+
+commit 8b5aa5d94fd24793c160372f02b02735632b5fd0
+Merge: 6baac41 2311ab2
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Oct 14 18:13:09 2016 +0200
+
+    Merge pull request #10864 from dachary/wip-17131-jewel
+    
+    jewel: Jewel: segfault in ObjectCacher::FlusherThread
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 6baac417eb95df880b97cdc720e6b00f6f8f27d8
+Merge: 19922c5 ca8fc6f
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 14 09:26:32 2016 -0500
+
+    Merge pull request #11417 from dachary/wip-17347-jewel
+    
+    jewel: ceph-create-keys: sometimes blocks forever if mds allow is set
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 19922c5b818e82c297dfc0d2eb914276945e9a2e
+Merge: 952a81a 35660d1
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 14 09:25:42 2016 -0500
+
+    Merge pull request #11272 from aiicore/wip-17402-jewel
+    
+    jewel: OSDMonitor: Missing nearfull flag set
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 952a81aed3bae88239e448fcae1ce73e725d9966
+Merge: e34b7da 165e5ab
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 14 09:24:42 2016 -0500
+
+    Merge pull request #11193 from SUSE/wip-17377-jewel
+    
+    jewel: LIBRADOS modify Pipe::connect() to return the error code
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit e34b7dad7e11f1db45f2bd636b1f3ed73742540a
+Merge: f9c969e 7d92e2e
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 14 09:23:30 2016 -0500
+
+    Merge pull request #10861 from dachary/wip-17135-jewel
+    
+    jewel: ceph mon Segmentation fault after set crush_ruleset ceph 10.2.2
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit f9c969eb54fef6f474bc8f784735123039ffeccc
+Merge: 4bcc21b 117aa35
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 14 09:22:05 2016 -0500
+
+    Merge pull request #10855 from dachary/wip-16657-jewel
+    
+    jewel: i386 tarball gitbuilder failure on master
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 4bcc21b9288ed046a2781e95e781bfc71636b9c1
+Merge: 92a3538 5d0e2f8
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Oct 14 15:37:25 2016 +0200
+
+    Merge pull request #11409 from dachary/wip-17341-jewel
+    
+    jewel: librados memory leaks from ceph::crypto (WITH_NSS)
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 92a353847852dbec003e4795e3322d2a92752b9d
+Merge: 67d0ded 8833c64
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Oct 14 13:45:46 2016 +0200
+
+    Merge pull request #11311 from dzafman/wip-scrub-boundary-jewel
+    
+    jewel: osd: adjust scrub boundary to object without SnapSet
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 67d0ded8d3ceda56ab7c0aaca25f3c1c51731c72
+Merge: 52e596f 9cb45e1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Oct 14 13:43:27 2016 +0200
+
+    Merge pull request #11231 from badone/wip-17376
+    
+    jewel: common: Log.cc: Assign LOG_INFO priority to syslog calls
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 52e596ffe0b8a83bfeae0ad836227a73d7d6facb
+Merge: 65e8bbc cca589f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Oct 14 13:30:05 2016 +0200
+
+    Merge pull request #9872 from odivlad/fix-init-el7-jewel
+    
+    jewel: remove SYSTEMD_RUN from initscript
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 91bd342df6de15263004f3a41c285f6658a16d45
+Author: Alexander Graf <agraf@suse.de>
+Date:   Mon Sep 26 10:26:30 2016 +0200
+
+    AArch64: Detect crc32 extension support from assembler
+    
+    The used compiler may or may not be recent enough to recognize the
+    crc32 extended cpu type. However, it does not really have to know about
+    them either, since all we do is pass inline assembly instructions to
+    the assembler.
+    
+    This patch moves the crc cpu extension detection from compiler based
+    to assembler based, so that we can build optimized code even when the
+    compiler does not know about the cpu type yet.
+    
+    Signed-off-by: Alexander Graf <agraf@suse.de>
+    
+    (manual backport of e70ab48b7f6d39a281b3ec65098535a55018b681 - manual backport
+    was undertaken because jewel uses autotools)
+
+commit 65e8bbccdb73cc6696736367a3df59ea3dcfda1f
+Merge: 7c9f1c7 086f6e0
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Thu Oct 13 14:52:30 2016 -0400
+
+    Merge pull request #10891 from dachary/wip-16793-jewel
+    
+    jewel: rgw: upgrade from old multisite to new multisite fails
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 7c9f1c7940c77451b2d44ef22a278fdfb2938cea
+Merge: e3839ab 27626ba
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Thu Oct 13 14:51:59 2016 -0400
+
+    Merge pull request #10889 from dachary/wip-17143-jewel
+    
+    jewel: rgw: rgw file uses too much CPU in gc/idle thread
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit e3839ab5ce63abd248d886dcf362f3a6c49f382e
+Merge: 0561550 416ec6f
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Thu Oct 13 14:51:38 2016 -0400
+
+    Merge pull request #10868 from dachary/wip-17064-jewel
+    
+    jewel: rgw: radosgw daemon core when reopen logs
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 05615509d946c340d03af7e407a31e3e53447ac8
+Merge: abc8e3d f034fd0
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Thu Oct 13 14:51:17 2016 -0400
+
+    Merge pull request #10867 from dachary/wip-17118-jewel
+    
+    jewel: rgw： period commit return error when the current period has a zonegroup which doesn't have a master zone
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit abc8e3dd0388778d01a19f3de15a1ffcb662549f
+Merge: 8adc55d 83a91ba
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 13 17:43:52 2016 +0200
+
+    Merge pull request #11367 from linuxbox2/jewel-s3-versioning
+    
+    jewel: rgw: S3 object versioning fails when applied on a non-master zone
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 8adc55d0d3f98acde4d91ba6f8ab9fb379ec10d9
+Merge: 50404ae 77ea511
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 13 16:43:04 2016 +0200
+
+    Merge pull request #11342 from SUSE/wip-17505-jewel
+    
+    jewel: rgw: doc: description of multipart part entity is wrong
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 50404aef1f36cd2035c4384ecdf608910e6cb215
+Merge: f1c55ad de0c4e1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 13 16:42:45 2016 +0200
+
+    Merge pull request #11330 from cbodley/wip-17073
+    
+    jewel: rgw: RGWDataSyncCR fails on errors from RGWListBucketIndexesCR
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f1c55ad7eeeb633e5ad3414e37ebb5dacccb930b
+Merge: a336110 e2ce857
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 13 16:39:06 2016 +0200
+
+    Merge pull request #11139 from cbodley/wip-radosgw-admin-man-jewel
+    
+    jewel: add orphan options to radosgw-admin --help and man page
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit a336110785ea06f4404f136dddcce61660d85b2f
+Merge: 79885ae cf47628
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 13 16:35:57 2016 +0200
+
+    Merge pull request #10832 from ceph/jewel-default-quota
+    
+    jewel: rgw - default quota fixes
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 79885ae86be1e444985446ab8315a21ad6d24634
+Merge: 9d1f756 54b6b5e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Oct 13 14:58:33 2016 +0200
+
+    Merge pull request #10107 from xiaoxichen/wip-16313-jewel
+    
+    jewel: client: FAILED assert(root_ancestor->qtree == __null)
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+
+commit d205b74e07ab364378afb4fc63cb08814a8a1f4e
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Aug 23 12:17:00 2016 +0200
+
+    tests: populate /dev/disk/by-partuuid for scsi_debug
+    
+    The scsi_debug SCSI devices do not have a symlink in /dev/disk/by-partuuid
+    because they are filtered out by 60-persistent-storage.rules. That was
+    worked around by 60-ceph-partuuid-workaround-rules which has been
+    removed by 9f76b9ff31525eac01f04450d72559ec99927496.
+    
+    Add create rules targetting this specific case, only for tests since the
+    problem does not show in real use cases.
+    
+    Fixes: http://tracker.ceph.com/issues/17100
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 7cbf1f0a5eced402e6c7242015550668e3b568f3)
+
+commit 86eef272304f3896a9cda5cde7e882d09d0269c0
+Author: Daniel Gryniewicz <dang@redhat.com>
+Date:   Fri Jun 10 11:33:56 2016 -0400
+
+    Allow building RGW with LDAP disabled
+    
+    Signed-off-by: Daniel Gryniewicz <dang@redhat.com>
+    (cherry picked from commit 38abfcb5d036c050a363533a2c7b658d528d3612)
+
+commit 657554574414dcece6126bfa1b4008b038c3f729
+Author: la-sguduru <SGuduru@walmartlabs.com>
+Date:   Thu May 12 11:53:25 2016 +0530
+
+    doc: Modification for "TEST S3 ACCESS" section
+    
+    Signed-off-by: SirishaGuduru sirishaguduru99@gmail.com
+    (cherry picked from commit 27d6cf0c6958de9f0533bedb90a74c4020b53675)
+
+commit 50fd48fbdc8f4f0e9151eb405eb112b710304bb9
+Author: qiankunzheng <zheng.qiankun@h3c.com>
+Date:   Fri Mar 18 13:58:22 2016 -0400
+
+    rgw:response information is error when geting token of swift account
+    The header is X-Trans-Id instead of x-amz-request-id in the response header
+    
+    Fixes:#15195
+    Signed-off-by: Qiankun Zheng <zheng.qiankun@h3c.com>
+    (cherry picked from commit 08e909d0b844e8be3cb82d6add3e87d38e19e80e)
+
+commit 742366116d368d3adff4b7941a4751d362073b35
+Author: SirishaGuduru <SGuduru@walmartlabs.com>
+Date:   Mon Sep 19 10:21:06 2016 +0530
+
+    doc: Radosgw multitenancy config paramater change
+    
+    Radosgw multitenancy configuration parameter
+    "rgw keystone make new tenants" never works even
+    applied. When gone through the code, itseems this
+    parameter is not used. But "rgw keystone implicit
+    tenants" works as the code looks for this.
+    
+    Modified the configuration parameter in two files
+    mentioned below from "rgw keystone make new tenants"
+    to "rgw keystone implicit tenants"
+    
+    Fixes: http://tracker.ceph.com/issues/17293
+    
+    Signed-off-by: SirishaGuduru <SGuduru@walmartlabs.com>
+    (cherry picked from commit bd9695d50cfd472508a8a704f1ea5fc62dc08faf)
+
+commit 635666412c464228edb014b525abb0af3a55ac50
+Author: Yang Honggang <joseph.yang@xtaotech.com>
+Date:   Tue Oct 4 09:18:09 2016 +0800
+
+    rgw: fix versioned object IO error
+    
+        When accessing a copied destination object, its source object's instance ID
+        information is needed, however it's missing now in the destination object's
+        manifest.
+    
+        In order to fix this problem, we can record source object's version_id/instance
+        into dest object's manifest(a new filed 'tail_instance' is added). When creating
+        a new object(not copy), 'tail_instance' should be equal to its instance value.
+        When copy/get a object, 'tail_instance' should always be used to get the right
+        tail objects.
+    
+        Fixes: http://tracker.ceph.com/issues/17111
+        Signed-off-by: Yang Honggang <joseph.yang@xtaotech.com>
+    
+    (cherry picked from commit d43b69e529328f73da6c29cd746557788a989ae0)
+
+commit 40689d1acab4da963d05540eb837fd12a4411269
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Sep 30 16:13:00 2016 -0700
+
+    rgw: don't fail if lost race when setting acls
+    
+    Fixes: http://tracker.ceph.com/issues/16930
+    
+    When concurrently setting acls on object/bucket, we could lose in a race.
+    Instead of retry, just return success (same effect as if we won and then
+    other writer overwrote us).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 6e9a915b565923081f609048072b8d75716a74ea)
+
+commit 5a53ffa8a9e8590ea0d1992fad9a30cde135cbac
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Oct 5 11:41:42 2016 -0700
+
+    rgw: remove user email index using helper function
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 10dbfc540aef5c3d45b7e3aabe61090c302f47ab)
+
+commit ad54bf9a0dc541cb6d664edec9ca7f3ec6ffbc93
+Author: Weijun Duan <duanweijun@h3c.com>
+Date:   Sat Feb 27 04:28:14 2016 -0500
+
+    rgw:user email can modify to empty
+    
+    Fixes: http://tracker.ceph.com/issues/13286
+    
+    Signed-off-by: Weijun Duan <duanweijun@h3c.com>
+    (cherry picked from commit ebfd713d9a40e1cb33dfdf2198c20d621e387e36)
+
+commit 4babd3fa335351bd4327cded47a1ffe5a8cfd897
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sat Sep 17 08:29:15 2016 -0400
+
+    librbd: corrected use-after-free in ImageWatcher
+    
+    Fixes: http://tracker.ceph.com/issues/17289
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2f4d4868e3b721c932c35ae7e8f0dd96b36a37fc)
+
+commit 1ca4dc6d1bd6aebece500c6e91f6a9871af0e1f1
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Sep 9 10:42:50 2016 -0400
+
+    librbd: possible deadlock if cluster connection closed after image
+    
+    Fixes: http://tracker.ceph.com/issues/17254
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 818c2f2abc972f689acb7d783c2a684bcc9e6e51)
+
+commit b73356b500f9ff364c09e6d78e62fc0e79fbfbac
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 20 07:25:36 2016 -0400
+
+    librbd: block name prefix might overflow fixed size C-string
+    
+    The issue which resulted in too large v2 image ids was fixed
+    under #16887.
+    
+    Fixes: http://tracker.ceph.com/issues/17310
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 61734d266c6ee476c2f5fcfbbaefc7d0c7939617)
+
+commit b410c8ea2164eb53ee7882859f0a977b67cf8b80
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 20 10:19:45 2016 -0400
+
+    librbd: potential deadlock closing image with in-flight readahead
+    
+    Fixes: http://tracker.ceph.com/issues/17198
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c971d58f8a5550b19374b74bb89d69143423479b)
+
+commit 32d84e0eed671f0cfe37057d59f0afe1b63f3c4d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 20 14:47:36 2016 -0400
+
+    librbd: update_features should handle Infernalis OSDs
+    
+    Fixes: http://tracker.ceph.com/issues/17330
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8cb2ccda1b847e0f33c0d34bf57e9ec29bbbb43b)
+
+commit b4e13808b004fd10d342e3daa17a3d4e830d4d00
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 20 14:34:16 2016 -0400
+
+    cls/rbd: set_features should adjust the mask to known features
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a26c7a5cacebf8814e20a2f3d5b69c20d8798871)
+
+commit dd93e009a222e8e86041661742b4df9c92b097f7
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Mon Jun 27 23:36:09 2016 +0530
+
+    rbd: cleanup - Proxied operations shouldn't result
+    in error messages if replayed
+    
+    Fixes: http://tracker.ceph.com/issues/16130
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit d09c9c471f40f15c14f392a93a04353ca30b1c5e)
+
+commit 391936a30cc6022f067cb11c2e39bd47b9e58d61
+Author: Lu Shi <shi.lu@h3c.com>
+Date:   Wed Jun 15 09:24:43 2016 +0800
+
+    librbd: enable/disable of features is not allowed when already enabled/disabled
+    
+    Fixes: http://tracker.ceph.com/issues/16079
+    
+    Signed-off-by: Lu Shi <shi.lu@h3c.com>
+    (cherry picked from commit a8a633396a4105991c9643c2b39391621934c26d)
+
+commit 7d6801bdd376f8eb5e1fbfa8f3a9f4de5dc710a1
+Author: zhuangzeqiang <zhuang.zeqiang@h3c.com>
+Date:   Sat Jun 25 10:21:25 2016 +0800
+
+    rbd: restrict mirror enable/disable actions on parents/clones
+    
+    Fixes: http://tracker.ceph.com/issues/16056
+    
+    Signed-off-by: zhuangzeqiang zhuang.zeqiang@h3c.com
+    (cherry picked from commit 11dee0bbf0a85f2c197192d0560bd486bc2ad6fc)
+
+commit cf476284ae9089e73e5fdf5ddfefdd5842246190
+Author: root <root@ceph-node1.homeoffice.wal-mart.com>
+Date:   Sun Jun 19 13:06:06 2016 +0530
+
+    rgw: Add documentation for RGW default quota
+    
+    Fixes http://tracker.ceph.com/issues/16447
+    
+    This was modified to use size in kB, rather than size in bytes, since
+    Jewel has not yet been converted to use size in bytes.
+    
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+    Signed-off-by: Daniel Gryniewicz <dang@redhat.com>
+    (cherry picked from commit e3ab3476146cf0fe604783e9e5a0d63237271735)
+
+commit 22348154ad51c20d73aa7715695244b8d85fac37
+Author: root <root@ceph-node1.homeoffice.wal-mart.com>
+Date:   Sun Jun 19 12:48:42 2016 +0530
+
+    rgw: Do not bother to check against conf values during quota handling
+    
+    Fixes http://tracker.ceph.com/issues/16447
+    
+    This was modified to use size in kB, rather than size in bytes, since
+    Jewel has not yet been converted to use size in bytes.
+    
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+    Signed-off-by: Daniel Gryniewicz <dang@redhat.com>
+    (cherry picked from commit 17d2c1712a5b72315a47ab0f8380331bfd478c0b)
+
+commit 0b8ecce6ac5abafe0175c941dbf55862b1515b1d
+Author: root <root@ceph-node1.homeoffice.wal-mart.com>
+Date:   Mon May 23 15:34:58 2016 +0530
+
+    rgw: Let the default quota settings take effect during user creation
+    
+    Fixes http://tracker.ceph.com/issues/16447
+    
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+    Signed-off-by: Daniel Gryniewicz <dang@redhat.com>
+    (cherry picked from commit 400d7c982f18efd7cf96acfc3a63078791d1ec0a)
+
+commit 1c76ef4e3ea7cb401345af0938d6b76652061d05
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Sat Oct 1 11:21:21 2016 +0300
+
+    journal: ensure in-flight ops are complete destroying journaler
+    
+    Fixes: http://tracker.ceph.com/issues/17446
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a25b1d7cbebb7b19cebee4cfb362cd744bbb4607)
+
+commit 9bc5e444543fade1f6d78788cadeb7bfdd29710b
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Sep 29 16:55:22 2016 +0300
+
+    journal: complete action only after notification completed
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 55762cca2bf586d037cb9f32775ec158dc3287c1)
+
+commit 74873322da011d0e6221c90ff39d137de80720ce
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 4 21:29:31 2016 -0400
+
+    qa/workunits: reduce amount of client IO for rbd-mirror stress
+    
+    Journal IO is faster now, resulting in OSDs quickly filling up
+    under the stress test case.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a6dd6b50d632aa6995a0f4b6fc5e1a1bda23e2a0)
+
+commit d51a2f6689391274bba41f9359acb9395574e629
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 4 12:22:06 2016 -0400
+
+    rbd-mirror: potential race allocating tag during shut down
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8c2ff9bce61a8af024150b8be9dee484f339f7df)
+
+commit cea000c6225c809b35b6c643919a01d940522c55
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Sep 30 12:32:37 2016 -0400
+
+    librbd: ignore attempts to request lock if already lock owner
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 355c79cb7c955e62e3153cf28046a9b8a6d0d25c)
+
+commit 912ce3728c09b2bf613da287f5013b97920cc27c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Sep 30 12:59:16 2016 -0400
+
+    journal: clean up object recorder closed/overflow callback
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 24faead086a50ea1b9614268d4dd5f3ea7bbe445)
+
+commit 310f3f79cf54daeefa8f00aba0d7b692261e4a33
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Sep 29 08:36:53 2016 -0400
+
+    journal: delay moving overflowed buffers until quiesced
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4483531aa3aa3d0f298778062e2b9a339ad05075)
+
+commit 9f3614377043efb56c606905a4b4f5c86b3e074e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Sep 29 08:19:52 2016 -0400
+
+    journal: use reverse iterator search to find flush record
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a96065815558e50361af4c701c23e5248962dfe0)
+
+commit 21502d90ecc72e70faa19b1202f9c7095341aaec
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Sep 28 08:35:36 2016 -0400
+
+    journal: avoid holding lock while sending journal append
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit dc77a629ed353d586b63f0bd8e20f54a7595afba)
+
+commit e0de824f7e728b06025a572a0c8213ef9fb1f112
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Sep 21 15:41:55 2016 -0400
+
+    rbd-mirror: wait for in-flight event commit notifications
+    
+    Fixes: http://tracker.ceph.com/issues/17355
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f810c106ad4e9ae94494801fea6c580d81b9156b)
+
+commit efa12f7187b6e037c22ba62563030eb883dab5f9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 11 13:55:21 2016 -0400
+
+    journal: fix AsyncOpTracker compiler error
+    
+    The master branch has an async JournalTrimmer which shouldn't be
+    backported to the Jewel branch yet. This change addresses the
+    missing sync AsyncOpTracker::wait_for_ops method from the backported
+    class.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 2460a3dbe7d04f1a4daa44d7367d24d47fa070fb
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Sep 21 13:02:34 2016 -0400
+
+    common: move AsyncOpTracker to common library
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 72d8992f054a7e36f92fdd2e01278ce3b9ede2eb)
+    
+    Conflicts:
+    	src/journal/CMakeLists.txt: doesn't exist in Jewel
+
+commit 1748b38da48beb8e25dc5ca831a540d389a963ee
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jul 28 16:35:48 2016 -0400
+
+    librbd: support deleting image by id instead of name
+    
+    The rbd-mirror daemon will use this API to delete images instead
+    of attempting to use the local image name.
+    
+    Fixes: http://tracker.ceph.com/issues/16227
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 87b32d15914207f61595c1c943817d983faceacd)
+
+commit bb5f6b6872e81291176543e954bf3654eb88120f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jul 13 14:49:06 2016 -0400
+
+    rbd-mirror: use pool id + global image id as deletion primary key
+    
+    Fixes: http://tracker.ceph.com/issues/16538
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 25203a8a9d59ff025d223ec1afaeb14946d54993)
+
+commit 96d551d7e8f10f98cc0a6ae5d90d33e6bca968a5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Sep 15 18:14:15 2016 -0400
+
+    rbd-mirror: concurrent access of event might result in heap corruption
+    
+    Fixes: http://tracker.ceph.com/issues/17283
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ac9ad37ab80db8913f9f1149707bd0009f8235c4)
+
+commit 8070d6a560044e5b45af7bd6995345ab30489d84
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Wed Sep 21 23:08:18 2016 +0100
+
+    rbd-mirror: test: Fixed timeout problem in rbd_mirror_stress.sh
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 5c737038dd6d44bd76605587486ddf9457bc2d96)
+
+commit 6ea4cde78ac7db6081e83d2530552946368021ef
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Tue Sep 6 17:28:22 2016 +0100
+
+    rbd: Fix race between journal flush and append events
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit aa959e71fe5a8cec43de75007fc9cef8de5ee3a5)
+
+commit 180a86b61498e6a27a59c7673f74e32ce1ae02cf
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Wed Sep 7 15:26:34 2016 +0100
+
+    journal: make librados call async in ObjectRecorder
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 7b740f5b4ac1c66ac3c80782d2d34e846d00fddd)
+
+commit 5edbfe278532225aaaa6bb3fe6bef84a4c693d83
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Mon Jul 25 17:00:50 2016 +0100
+
+    journal: increase concurrency of journal recorder
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 5c88edd68a1ee7c77f11e4113251fbe5768b8d99)
+
+commit 9ad132e670564c738e448bf719503cc73525fd8a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Sep 19 11:24:17 2016 -0400
+
+    journal: send update notification when tag allocated
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f1cd613e3d8ccb6d05b9adfe1956927991a4f4fe)
+
+commit f9aae06152e281c271f50201a8dd1852a132447f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 13 21:54:46 2016 -0400
+
+    librbd: new journal listener event for force promotion
+    
+    Fixes: http://tracker.ceph.com/issues/16974
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit fd005490e95d7fca85be4cad34344a58986f64d6)
+
+commit 4f5ce86349e246e12abe8e0c55380f400bf05ebb
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 13 16:38:51 2016 -0400
+
+    librbd: helper class for quiescing in-flight async ops
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 39d9e5cc9b38de2ee9ad2faf8e04253314160811)
+
+commit 829ff8c5fa9d9470f1e5370cf601509809b39674
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 13 12:37:53 2016 -0400
+
+    librbd: unify journal event callbacks into single interface
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit dbbcecf4a289ca36b734b7bda9530cc0a59f84ac)
+
+commit 41cf3d93356bde801c32dec4b7a21ae065295aa7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 13 16:28:50 2016 -0400
+
+    journal: expose ability to retrieve partial tag list
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 277b6dd9f13a8390cbf7c9ac7a313813ecad4d27)
+
+commit 2f9e6412982826fe0712a0b98e004c405b60fac3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Sep 14 08:49:39 2016 -0400
+
+    qa/workunits/rbd: fix remove mirrored image race conditions
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a0f65b968b350629dfad978c191fc878ca26d093)
+
+commit 2ec5e93806f8487bdc5f8df23740aab5738cde1f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 13 12:32:45 2016 -0400
+
+    qa/workunits/rbd: new mirroring forced promotion test
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a43268a4a30c06233152d531cbf2550224fb8a15)
+
+commit 5d1d898e1132325cae7045dc764a533878d56e00
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 20 13:31:36 2016 -0400
+
+    test/rbd: fix possible mock journal race conditions
+    
+    Fixes: http://tracker.ceph.com/issues/17317
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 471898392372ba4c404376410fb56f3af5287c80)
+
+commit 775a999e8bac9d0dc02cb40f0206b96c5864b8d1
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Jul 6 12:59:25 2016 +0300
+
+    qa/workunits/rbd: before removing image make sure it is not bootstrapped
+    
+    If an image is being bootstrapped, it implies that the rbd-mirror
+    daemon currently has the image open. The removal API will prevent the
+    removal of any image that is opened by another client.
+    
+    Works-around: http://tracker.ceph.com/issues/16555
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 9737a8d6cbaf1b79bbc6008249d39acbae883941)
+
+commit 3bbd8ba4282ea7f3fa286d0a1944e9e93d321365
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 27 14:47:02 2016 -0400
+
+    librbd: fix journal create compilation error
+    
+    The synchronous journal create method no longer exists on the master
+    branch and the associated change to introduce an asynchronous journal
+    creation state machine should be delayed to provide more testing time
+    on the master branch before being backported to jewel.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 89c6618df89dc0d5d3eb1f855f6f93c72be75939
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 11 21:11:07 2016 -0400
+
+    test: fixed unused function warnings in unittest_rbd_mirror
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3545d9ed19be8d3956f0db901ea9d3bb8b10d13d)
+
+commit ca94f25aa960bb352043a9d53eee361071d537da
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 11 20:48:27 2016 -0400
+
+    rbd-mirror: prevent syncing to non-primary image after failover
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit beaef377d69f555277b706afff944a15086da28e)
+
+commit 7a434842ac2a2799b611aa87422009c244418922
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 8 23:45:46 2016 -0400
+
+    rbd-mirror: demote/promote in same cluster results in split-brain
+    
+    Fixes: http://tracker.ceph.com/issues/16855
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a6901ca1a065419426b3ad704e27e43ba8d591b8)
+
+commit f8f3bbd7246d9a1d5a82f6b0b112f185323a5fb3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 11 19:39:21 2016 -0400
+
+    librbd: normalize journal promotion/demotion events
+    
+    A non-primary image's commit possition won't accurately reflect
+    the current demotion/promotion chain. Therefore, directly specify
+    the predecessor for promotion events.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ac590e85a472533005cad73c82b2d61db161ab7a)
+
+commit 4baea6ace14bd834a3cb2c1d9d5202e94546e264
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 11 19:09:09 2016 -0400
+
+    librbd: block RPC requests when demoting the image
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8b195e1fc8fe70a0e5417934302d5831b1f8dfb3)
+
+commit 8ccdad6d9d09e0f09431df6d3ebb2fbed47040c6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Aug 9 12:24:19 2016 -0400
+
+    librbd: separate journal::TagData predecessor data into new struct
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 718befdae711141ef4a1e2e9f5e9aca97f1b5513)
+
+commit 13daaffb451e7187d5492a82bb85488c1bfe55d2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 8 20:57:45 2016 -0400
+
+    rbd-mirror: include tag tid in bootstrap debug log messages
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 47d1e625006d554164f020e496a847735240ec95)
+
+commit d775680dc4eb53c08a692e746dd65c57a560496d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 8 10:39:01 2016 -0400
+
+    qa/workunits/rbd: demote/promote image on same cluster
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e6aa18ea0df6dc4d1add597bc2d972c79699bf4c)
+
+commit f7ffbfa71d6f8c417b140ce434b7714cd0053b09
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Aug 10 13:46:46 2016 +0300
+
+    rbd-mirror: option to automatically resync after journal client disconnect
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 77fd6a1c2016262d734b0bb5387e6b6a41232e8b)
+    
+    Conflicts:
+    	src/common/config_opts.h: trivial resolution
+
+commit 0b402390381dfbfcf4d3810f3179f90c4aa995ef
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Jul 27 13:45:32 2016 +0300
+
+    rbd-mirror: stop replay when client is disconnected
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 330dba00ba3153ba2862eef52714e0dceae05192)
+
+commit 80aa7e0b4fae6f71a34e28ad1225e3540f9606ed
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Aug 10 11:22:16 2016 +0300
+
+    rbd-mirror: resync was possible only when image replayer start had succeeded
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 4bf6912f3c75560b89324fc29286028750f122c1)
+
+commit 170476e408308fad622063eddbd282a38a28ed89
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Jul 26 16:10:30 2016 +0300
+
+    rbd-mirror: decode_client_meta should return false on error
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit cd5eb36e98f46a1d84bddeafa4e7dcad415aa4a2)
+
+commit 7382e1a5a2b801cdfcbf7cda109343c365d005f3
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Aug 3 14:19:51 2016 +0300
+
+    rbd: new command to disconnect journal client
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit fc3ba54b3c101498a08a3f34ac8f7eab0152ad7c)
+
+commit d3ad2ff9dff192d93d2fbef82ec0ccd37809d2ca
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Jul 27 14:06:42 2016 +0300
+
+    librbd: optionally flag "laggy" journal clients disconnected
+    
+    Fixes: http://tracker.ceph.com/issues/14738
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit b8eafefba9f2221a0ce927568795cb1c3ac0fa9c)
+
+commit 4056e360117b3aacfba2ae98cd4ecb60e356730c
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Jul 13 15:49:40 2016 +0300
+
+    journal: allow to trim journal for "laggy" clients
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 0b8b1aaedc10f7f46e91bf6ad809414feb770c8d)
+
+commit 3aec576572066a4329488c0b4420fe863cbbeeb2
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Jul 27 13:42:19 2016 +0300
+
+    cls/journal: add async client_update_state method
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 58b8c66d5bfa60e6dd3ad2ec79360c2eca165c58)
+
+commit d66bb7a991affb8b77eb9440d7c8874cc0b3bbf4
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Mon Oct 3 14:43:25 2016 -0600
+
+    build: include more files in "make dist" tarball
+    
+    Include all the documentation, qa, and CMakeLists.txt files in the "make
+    dist" tarball.
+    
+    Kraken and newer releases will already do this. This change is
+    restricted to releases that still use autotools.
+    
+    The purpose of this change is to make it easier to apply downstream
+    patches on an unpacked upstream tarball, because often those patches are
+    cherry-picks of commits that touch files under doc/, qa/, or
+    CMakeLists.txt.
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 9d1f756dfa1775338c2fa1cf6bfafd45b842b75f
+Merge: 59f0f4c 57f08fe
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 11 12:30:44 2016 +0200
+
+    Merge pull request #11389 from jcsp/wip-17466-jewel
+    
+    jewel: mon: don't crash on invalid standby_for_fscid
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 59f0f4c7cc81b20fc584b8e6fff84887ec9c26dd
+Merge: b747903 c2d4239
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 11 12:30:25 2016 +0200
+
+    Merge pull request #10997 from batrick/i17105-backport
+    
+    jewel: multimds: allow_multimds not required when max_mds is set in ceph.conf at startup
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit b7479037472c7ce633e2130fb6dbc302dbb3affb
+Merge: db2e822 cf211d7
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 11 12:30:03 2016 +0200
+
+    Merge pull request #10958 from ukernel/jewel-16764
+    
+    jewel: client: fix shutdown with open inodes
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit db2e8224853dc344b71871317204f13a0a53d808
+Merge: 4769027 1bc047b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 11 12:29:36 2016 +0200
+
+    Merge pull request #10921 from jcsp/wip-client-lock-backport
+    
+    jewel: client: add missing client_lock for get_root
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 4769027f0c83cb22f25186356b56e86b49579b8b
+Merge: f49bac2 494687b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 11 12:29:05 2016 +0200
+
+    Merge pull request #11400 from ceph/wip-rbd-cli-jewel
+    
+    jewel: krbd-related CLI patches
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f49bac2cbbda6aefef676b03f6656a98ec390b87
+Merge: 2a348d2 5ab5e82
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Oct 11 11:03:40 2016 +0200
+
+    Merge pull request #11171 from ceph/wip-rasize-doc-jewel
+    
+    jewel: doc: fix description for rsize and rasize
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 3bb2a9ed9602b0d280a357f53bc90c6cf83ffe32
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Oct 10 11:57:03 2016 -0400
+
+    librbd: ignore notify errors on missing image header
+    
+    The rename op on v1 images fails since the header no longer exists. In
+    the general case, the removal of the header object will also fail the
+    watcher which has its own recovery path.
+    
+    Fixes: http://tracker.ceph.com/issues/17549
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit fe3e2eafa087f81c9ab59f3fbc39600d6adaa9c2)
+
+commit 5173563fbfc2799cc2328468bb197d65a94b4d7a
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Sep 2 16:19:29 2016 +0800
+
+    client: properly set inode number of created inode in replay request
+    
+    Fixes: http://tracker.ceph.com/issues/17172
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit e59385f16afc607ec700397b0bea5229ce69df30)
+
+commit 2c4e1c1cd1d01e51d5e4ad500d6b31832591d201
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Sep 6 13:16:04 2016 +0100
+
+    mds: log path with CDir damage messages
+    
+    Previously you just got the inode number, which
+    wasn't terribly useful for e.g. a missing fragment
+    object, as you couldn't readily resolve the parent
+    path.
+    
+    Fixes: http://tracker.ceph.com/issues/16973
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 439cd5e1b4725e594786785189a37184243828d9)
+    
+    Conflicts:
+    	src/mds/CDir.cc: the go_bad() prototype which is part of the
+            context of the patch has changed.
+
+commit d52f190d449a2e68baed0659367795ebfb0dcb3d
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Sep 6 13:07:38 2016 +0100
+
+    mds: s/used_ions/used_inos/
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit bd3645fb07714d43fb7bd3d66b7e34b33768a8e8)
+
+commit 494687b7f11d0c9ca56d96b6b4480de96847bec2
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Fri Oct 7 12:32:43 2016 +0200
+
+    rbd: expose rbd unmap options
+    
+    Reuse rbd map -o infrastructure to expose rbd unmap options in
+    a similar fashion.  Currently it's just one bool option, but we may
+    need more in the future.
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit 620f5e1455fdcb05cd3873c1e260141849829e35)
+    
+    Conflicts:
+    	doc/man/8/rbd.rst [ PR #9151 ("rbd: add methods to set and get
+    	  snapshot limits") not in jewel ]
+
+commit 6b0226774e2f09f98751e6fa3b0125a960417cc0
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Fri Oct 7 11:14:16 2016 +0200
+
+    rbd: fix up terminology in help texts
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit ba8f9ee791d55c4cf25ad751213ea6ba770b7434)
+
+commit 071bd4778547930d9aab9340dcabbb6d9ffdd890
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Mon Oct 3 14:09:11 2016 +0200
+
+    rbd: recognize lock_on_read option
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit e857b7896527b676155d6e01c78567337dc33b1c)
+
+commit 7d0714e17557ad281bd92907e10462894a482ce9
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Tue Oct 4 10:21:36 2016 +0200
+
+    doc: clarify rbd size units
+    
+    It wasn't clear from the man page that --size defaults to M.
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit d38dc29ed588b8933da3b66e46e1f0dd4cbb2cf4)
+    
+    Conflicts:
+    	doc/man/8/rbd.rst [ PR #9066 ("rbd: introduce a simple bench
+    	  for read") not in jewel ]
+
+commit ca8fc6fc8baae2e4da842e16e25b91d304702cb5
+Author: John Spray <john.spray@redhat.com>
+Date:   Sat Jul 23 22:56:45 2016 +0100
+
+    ceph-create-keys: fix existing-but-different case
+    
+    We just have to refrain from calling get-or-create
+    if the named key already exists, to avoid potentially
+    having an error when the default creation args don't
+    match the key as it has already been created, such
+    as on certain upgrades.
+    
+    Fixes: http://tracker.ceph.com/issues/16255
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 278196d86c52b0be3cb9c17ed7f8f3e3502a217a)
+
+commit 0a17741861faa1643bc338459397072268e469bd
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Sep 20 18:07:56 2016 +0800
+
+    client: fix readdir vs fragmentation race
+    
+    following sequence of events tigger the race
+    
+    client readdir frag 0* -> got item 'A'
+    MDS merges frag 0* and frag 1*
+    client send readdir request (frag 1*, offset 2, readdir_start 'A')
+    MDS reply items (that are after item 'A') in frag *
+    
+    Fixes: http://tracker.ceph.com/issues/17286
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit feb63776d4f2b03ece189715fdd75da5cae2afc2)
+
+commit 3a79db10f259d66ca17c2152cf639a63f245505d
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Sep 14 18:22:01 2016 +0800
+
+    mds: remove fail-safe queueing replay request
+    
+    MDSRank::queue_one_replay() does not create active request
+    immediately, it just queues corresponding C_MDS_RetryMessage
+    for execution. So the fail-safe code can queue an extra replay
+    request. This can cause replay requests be processed out-of-order
+    
+    Fixes: http://tracker.ceph.com/issues/17271
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit d755a4a78242293dd3103bdd0748292df3034563)
+
+commit 10e603b1e3024c760cd5567dbbc73fbb3c1281f3
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Sep 21 21:57:25 2016 +0800
+
+    client: fix segment fault in Client::_invalidate_kernel_dcache().
+    
+    when umounting, root can be NULL
+    
+    Fixes: http://tracker.ceph.com/issues/17253
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit ddea9a5b78151c0abe834e4a9872f7cf5f918ebc)
+
+commit 3320da0adb346af15a5d50a45e0aea5ca234eb79
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 19 15:18:24 2016 +0100
+
+    mds: catch duplicates in DamageTable
+    
+    There was an implicit assumption in the code that callers
+    wouldn't hit the notify_*damaged paths twice because they would
+    have checked is_*_damaged paths first.
+    
+    However, that's not really true in all cases, e.g. scrub
+    code isn't required to respect existing damage entries
+    when trying to load a CDir.
+    
+    Simply fix this by having the DamageTable notify* functions
+    check the key they're inserting doesn't already exist.
+    
+    Fixes: http://tracker.ceph.com/issues/17173
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit c9cfaef104e9aaefad55583d7e54f8b4665904b3)
+
+commit 5d0e2f8690b08418fc404dc43df7a3a16ac83309
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 2 14:13:19 2016 -0400
+
+    common: only call crypto::init once per CephContext
+    
+    Fixes: http://tracker.ceph.com/issues/17205
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 9dfc1537564c4ce65e4d0230a83f0d8d47503b16)
+
+commit 483d8c41a241aa917878284765aef2ba2ced0bb3
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Tue Jun 14 17:34:44 2016 +1000
+
+    cephx: Fix multiple segfaults due to attempts to encrypt or decrypt
+    an empty secret and a null CryptoKeyHandler
+    
+    Fixes: http://tracker.ceph.com/issues/16266
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 009e777fbd18602e5fd66f97bdad95e977e6fecc)
+
+commit 5ae4f316b2cf37ce5d13b1ced4b65962c5583882
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 20 17:39:24 2016 +0800
+
+    os/filestore/FileJournal: fail out if FileJournal is not block device or regular file
+    
+    otherwise JournalingFileStore will assert when deleting FileJournal
+    which still has the non block/regular file opened.
+    
+    Fixes: http://tracker.ceph.com/issues/17307
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 7431eec6fd24cd08ca6c76a9893e3f6e8c63a916)
+
+commit 2a348d23e5a047bf6a9c80140cf31ba2d6d4113f
+Merge: da04ac0 29a8701
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 10 20:30:54 2016 +0200
+
+    Merge pull request #10812 from SUSE/wip-17094-jewel
+    
+    jewel: build/ops: ceph-osd-prestart.sh fails confusingly when data directory does not exist
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit da04ac055188e2e84d21b29feee3069bb95e5ebe
+Merge: a80aacf 1a7faefd
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Oct 10 10:46:33 2016 -0400
+
+    Merge pull request #11335 from linuxbox2/jewel-nfs
+    
+    jewel: rgw: nfs backports
+
+commit a80aacf048c2809bc2b7fd7e2f41e12b5470c5ab
+Merge: e795450 a10d700
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Oct 10 10:45:07 2016 -0400
+
+    Merge pull request #11332 from linuxbox2/jewel-ldap
+    
+    jewel: rgw: combined LDAP backports
+
+commit c2d4239331e397e6a48b1ee7a4c69dad0c951f94
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Mon Aug 29 16:25:10 2016 -0400
+
+    mds: remove max_mds config option
+    
+    It is now required that all changes to max_mds use the run-time `ceph fs
+    set max_mds` command. The rationale for this change is that it is
+    confusing to have a configuration for max_mds which is only observed at
+    file system creation.
+    
+    Fixes: http://tracker.ceph.com/issues/17105
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 52f4e545e0ce4782d5f2239d31080d392c12eba2)
+
+commit e7954500dab4f3e3bead824ba0da66105940e36a
+Merge: eac384a 0b93a9a
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 10 08:54:02 2016 -0500
+
+    Merge pull request #10761 from dachary/wip-16377-jewel
+    
+    jewel: msgr/async: Messenger thread long time lock hold risk
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit eac384a5a158191e3276940d340ee2fe0b4b4b73
+Merge: c21e1de d94e5bc
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 10 08:52:59 2016 -0500
+
+    Merge pull request #10278 from SUSE/wip-16667-jewel
+    
+    jewel: incorrect value of CINIT_FLAG_DEFER_DROP_PRIVILEGES
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 57f08fe72269b573d95791b44f25de044d40fba6
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Sep 30 21:57:35 2016 +0100
+
+    mds: make mds_role_t member order match constructor
+    
+    Previously this was mildy confusing.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 63d711d4e9decd2940c55aa697e6a588593da23a)
+
+commit c19326cef28ca3764340434cacc1f77e5899b887
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Oct 3 07:33:27 2016 +0100
+
+    mon: don't crash on invalid standby_for_fscid
+    
+    Fixes: http://tracker.ceph.com/issues/17466
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 92fdcf36bd7d57b97f5d887d0370a181860e388c)
+
+commit cbd9a39e83258a7fa98d3d33aea652def17d3906
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Oct 3 07:40:12 2016 +0100
+
+    messages: fix MMDSBeacon constructor
+    
+    This was leaving garbage in some fields during decode
+    when decoding messages with an older version.
+    
+    Fixes: http://tracker.ceph.com/issues/17466
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit f1fb667dfd21cd241f4f45ce9ef3f8393d114c60)
+
+commit c21e1de83294fea247bd2a458143639544f91f6d
+Merge: 8a84877 f7d23d5
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 10 12:26:02 2016 +0200
+
+    Merge pull request #10794 from SUSE/wip-17082-jewel
+    
+    jewel: disable LTTng-UST in openSUSE builds
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 8a8487711339596f2f22da572e59a194c6166c69
+Merge: 4c526a4 47605a2
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 10 12:21:59 2016 +0200
+
+    Merge pull request #9739 from vumrao/wip-vumrao-16337
+    
+    jewel: osd: add peer_addr in heartbeat_check log message
+    
+    Reviewed-by: Samuel Just <sjust@redhat.com>
+
+commit 4c526a4211f29a5151c5d79d3f0aa221f3d24221
+Merge: c362c71 5ffee35
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 10 12:20:30 2016 +0200
+
+    Merge pull request #9388 from vumrao/wip-vumrao-16069
+    
+    jewel: mon: Display full flag in ceph status if full flag is set
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit c362c71ef34b136112852cce7a769a8bac30626c
+Merge: f707d1d 79e2acb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 10 12:03:51 2016 +0200
+
+    Merge pull request #10496 from Abhishekvrshny/wip-16583-jewel
+    
+    jewel: mon crash: crush/CrushWrapper.h: 940: FAILED assert(successful_detach)
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit f707d1d4f1cae66927e630da9a4524d2af6c9a11
+Merge: 84adafe f337a07
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Oct 10 09:31:29 2016 +0200
+
+    Merge pull request #11018 from ceph/wip-17223-jewel
+    
+    jewel: krbd qa scripts and concurrent.sh test fix
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit cca589fdb18147ea61e79b8d98b886ef30cd65d5
+Author: Vladislav Odintsov <odivlad@gmail.com>
+Date:   Mon Jul 4 15:29:22 2016 +0300
+
+    init-radosgw: do not use systemd-run in sysvinit
+    
+    `systemd-run` logic in initscripts was introduced because of ticket
+    http://tracker.ceph.com/issues/7627.
+    
+    If we have systemd-based operating system, we should use systemd unit files
+    from systemd directory to start/stop ceph daemons.
+    Otherwise, `daemon()` from `/etc/init.d/functions` on systemd distro starts service
+    in `system.slice` and everything works well.
+    
+    `systemd-run` can not be run on non-systemd distros, so it's not needed
+    on SysV systems.
+    
+    also, ceph-disk is now able to run the "systemctl"
+    to enable and start the ceph-osd, and ceph-deploy is also now
+    playing well with systemd when it comes to ceph-mon and ceph-mds
+    
+    Fixes: http://tracker.ceph.com/issues/16440
+    
+    Signed-off-by: Vladislav Odintsov <odivlad@gmail.com>
+    (cherry picked from commit 1fd4f92a025e80092b8d08d9b7da2e0b73a52f0f)
+
+commit 83a91baff05f03f907fb80702ae1c68194b5308b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jun 30 17:36:16 2016 -0700
+
+    rgw: forward input data when forwarding set_bucket_version to master
+    
+    Fixes: http://tracker.ceph.com/issues/16494
+    
+    Needed to keep input data around to be forwarded correctly. Also, master
+    does not send any data back, so don't try to parse anything.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 1eec00bef1e5540bf3e31d1f8fb1645eb64b1e62)
+
+commit 0986aff2aafdf64847a7e9608abb84edaea59eb1
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jun 30 17:34:38 2016 -0700
+
+    rgw: set args when rebuilding req_info
+    
+    Was missing args, so it didn't sign requests with subresources
+    correctly when forwarding.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit dd1068c54e50671d8904b99189823418a5a5bc07)
+
+commit bb81f9dd83da76704a5b6ba3e718fef3ac918290
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Aug 12 12:39:45 2016 -0400
+
+    librbd: permit disabling journaling if in corrupt state
+    
+    Fixes: http://tracker.ceph.com/issues/16740
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6afb884d755697528684fad54dbb69b15d4386f1)
+
+commit b14d7f148c8f64d2e2da533fe2b35661536810c4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Aug 12 12:21:02 2016 -0400
+
+    librbd: new journal policy to disable initializing the journal
+    
+    This will be used in the case where the journal is being disabled.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 73e4c65c809a1e4161229f49285b21b2cfc623ca)
+    
+    Conflicts:
+    	src/librbd/Makefile.am (no CreateRequest.h or RemoveRequest.h in jewel)
+
+commit 77ea51138f05b4fa1c603ade0e62f961ec708441
+Author: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+Date:   Wed Aug 10 14:11:43 2016 +0800
+
+    doc/radosgw: fix description of response elements 'Part'
+    
+    Signed-off-by: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+    (cherry picked from commit 9c3fe46a7eb89aedc52582737c200e58d63738a5)
+
+commit 1a7faefd035aa8f31ab11ddfe1046989d0e69aac
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Sep 21 17:18:20 2016 -0400
+
+    rgw_file:  pre-assign times
+    
+    Set unix timestamps based on RGW values for creation, modification
+    time for buckets, objects in stat requests.  Regard any saved
+    value of these in unix attributes as an overlay.
+    
+    Fixes: http://tracker.ceph.com/issues/17367
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 969750e96380859a350e25015bc8cbf9bdf88512)
+    
+    Fixes: http://tracker.ceph.com/issues/17394
+
+commit 0c932b9078a249bcb9dbb8c425788787f1cd48f2
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Sep 20 17:32:03 2016 -0400
+
+    rgw_file: fix set_attrs operation
+    
+    The effective part of this change is to always set a value
+    for RGW_ATTR_UNIX_KEY1 (because it is expected later).
+    
+    Secondarily, do not pass the address of the to-set attributes
+    buffer::list as remove attrs--this is confusing.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 08afb40b2812270a6af3cd1965e8c03c10392ace)
+    
+    Fixes: http://tracker.ceph.com/issues/17393
+
+commit 0de33909012f2a8305ff2038906669c78070cbcd
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Apr 14 19:18:37 2016 -0400
+
+    rgw_file: implement rgw_setattr
+    
+    Introduce a new RGWSetattrs RGWOp descendant, to create or replace
+    sets of attrs on buckets or objects.
+    
+    This version of the change uses the standard RGWRADOS::set_attrs op
+    (we want attribute changes to (e.g.) sync with other changes).
+    
+    Previous versions of this changed incorrectly masked the values
+    of st->st_ino in RGWFileHandle::stat(), now fixed.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 4de1c3c260265f821ebee842d49cb35bf49d8e4e)
+    
+    Fixes: http://tracker.ceph.com/issues/17311
+    Fixes: http://tracker.ceph.com/issues/17332
+
+commit 00f3fbe7dc940d8c9176a8da35fccd42e6ddc10f
+Author: zhangweibing <zhangweibing@unitedstack.com>
+Date:   Fri Aug 26 10:49:48 2016 +0800
+
+    rgw: remove duplicated calls to getattr
+    
+    Signed-off-by: Weibing Zhang <zhangweibing@unitedstack.com>
+    (cherry picked from commit 87b550ed2fba472d4ef2e1f771bb5f9fd248804f)
+    
+    Fixes: http://tracker.ceph.com/issues/17327
+
+commit c53992856d6058b71bed55337c3830dbaefb6823
+Author: Min Chen <chenmin@xsky.com>
+Date:   Fri Aug 19 01:02:50 2016 -0400
+
+    rgw: rgw file fix bug of rgw_lookup can not exact match file name
+    
+    bug reproduce steps:
+    in nfs-client (connect to nfs-ganesha server):
+    1. mv file file.rename
+    2. stat file // get file attrs, not return with -ENOENT
+    
+    the reason is that:
+    RGWStatLeafRequest does not exact match the file name,
+    just take file name as a prefix filter
+    
+    Signed-off-by: Min Chen <chenmin@xsky.com>
+    (cherry picked from commit 9d813bafc8e197507457c58ab4f365ccdb7f3589)
+    
+    Fixes: http://tracker.ceph.com/issues/17326
+
+commit 9f5c7377143c56b169a62a96dba5c6ff800e4d5c
+Author: Yan Jun <yan.jun8@zte.com.cn>
+Date:   Tue Aug 23 12:51:10 2016 +0800
+
+    rgw: fix the wrong return value
+    
+    Here `parent->readdir` will return negative numbers When there is an error.
+    so we should just return it.
+    
+    Signed-off-by: Yan Jun <yan.jun8@zte.com.cn>
+    (cherry picked from commit c979be7810793534715c24c6a9f7c8b0ed31de8d)
+    
+    Fixes: http://tracker.ceph.com/issues/17325
+
+commit cf1353c24917ecb9056272da7f6b3e73bba11fa8
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Aug 17 10:28:48 2016 -0400
+
+    rgw_file: restore local definition of RGWLibFS gc interval
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 31936caa09b46e86700faad969adfa9d43176206)
+    
+    Fixes: http://tracker.ceph.com/issues/17323
+
+commit 7a648ed991630541b5875a329820b67b269af570
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Aug 23 16:50:42 2016 -0400
+
+    rgw_file:  explain semantics of RGWFileHandle::commit
+    
+    Add a comment explaining why the method currently returns 0
+    unconditionally.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit b1da1354320cc5f69277b225a293a03cc2a5054f)
+
+commit 0135da74e2a08fccc2e2da9f1b7e55478bf79157
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Aug 23 15:57:06 2016 -0400
+
+    ceph_timer: prefer using directive to typedef
+    
+    This change replaces the existing member-hook typedef as well as
+    the new set-type typedefs, so committed separately.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit f7d4d418658d2091652033045abdb4bfbeb637aa)
+
+commit 2a9233f1fbff38c12d378d5b6528a7e576a62e3e
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Aug 23 14:18:43 2016 -0400
+
+    librgw: add API version defines for librgw and rgw_file
+    
+    This change borrows the major, minor+extra format used by libcephfs.
+    The version numbering is starting at 1,1,0 on the theory that the
+    implicit version at Jewel is 1,0,0.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 355ccc5ded7f0b459bca24bc8b504b41807c583d)
+
+commit 2d6966f7e1f4c6dab12f7500746f1021b52d91af
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Aug 17 11:24:01 2016 -0400
+
+    rgw file: allow RGWLibFS::write_completion_interval_s to be set in conf
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 70dad0fba561edaf0bb4e746e29b92a4e9b014b2)
+
+commit 7071845f2e477648c322f0072e54490c47ed4154
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Aug 17 11:00:22 2016 -0400
+
+    rgw_file:  log on write completion events at level 10
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit d1e512d8c0e20df5a96ccb5f775a7a2ca123efbe)
+
+commit 95310c7f960eead2e22bc02c0e9d94ec863cafd6
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Aug 17 10:52:58 2016 -0400
+
+    rgw_file: add RGW_OPEN_FLAG_STATELESS as a synonym for V3
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit da596eb3af69c2cdf191893eb44a8947662634cf)
+
+commit 697d4ef7837061bb6b5ebaad07ae86836cdbe126
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Aug 16 18:17:53 2016 -0400
+
+    rgw_file: implement stateless write completion timer
+    
+    Implements a temporal mechanism to enforce write completion for setups
+    which lack open state tracking (e.g., NFS3).
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 2c83ed4bd43fcb5d5497151a157b1dc08022fed9)
+
+commit 4a31509c5ae7e40dd1538b2075287d8cc202f159
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Aug 12 16:52:51 2016 -0400
+
+    rgw_file: fix/update rgw_create flag arguments
+    
+    Allow passing POSIX open flags as well as api call flags.  Needed
+    for NFS3 support.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 2d0ea2b4649e76cbbb87f44c57754d7ea9c46dce)
+
+commit 73f9a0b147914f3ef4f7b69a3b57899034406bca
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Aug 12 10:39:21 2016 -0400
+
+    rgw_file: fix/update rgw_open flag arguments
+    
+    Allow passing POSIX open flags as well as api call flags.  Needed
+    for NFS3 support.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit ada29f7f8323be2301588941bcd5c60190b32a4b)
+
+commit 4ceb70eba4d777ecc34cce9d1d10002807a6e770
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Aug 9 16:49:41 2016 -0400
+
+    rgw file: remove busy-wait in RGWLibFS::gc()
+    
+    This is a background thread.  However, CPU is wasted.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit ca33241286f52d849dbde8092507131b8b1108cc)
+    
+    Fixes: http://tracker.ceph.com/issues/17321
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 8e1515b068991567136042fb11384fb0fd0620fe
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Aug 9 14:23:00 2016 -0400
+
+    rgw_file: unlock() must precede out label
+    
+    In lookup_handle(...).
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 9f64f34e481278a683e962b17c7aa811005783a8)
+
+commit 88eeaa967494ba6b536d493d06bc4bdbe92e452f
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Aug 7 18:39:33 2016 -0400
+
+    rgw nfs: fix write continuations
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit b4d079f2b5a4d49b2ba7576b272a2a3f3b0d66c8)
+
+commit f0b1799910564706766fccb8ec2b1c2c27a10467
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Aug 5 10:03:33 2016 -0400
+
+    rgw nfs: don't leak fh->mtx in lookup_handle()
+    
+    This change fixes a serious latent locking problem, noticed after
+    updating the ganesha/rgw driver invalidation after renames.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit d74d46170d7104a6553674f111bbdbe3a116cf54)
+
+commit f16ac90f9c097352de2763215f0c83812503e815
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Aug 8 10:18:35 2016 -0400
+
+    rgw file: refuse to rename directories
+    
+    The representation of paths as an aggregate involving any
+    number of objects argues against ever permitting such
+    operations.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 297220fd2a96c0cf34701127b8cf724e24e6865f)
+
+commit 23b749f93ed9a7de93aabc871df0adeeebac4e0f
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Aug 3 13:53:15 2016 -0400
+
+    rgw_file: refuse ops on deleted targets
+    
+    Detect these illegal cases, as they are indicate various incorrect
+    behaviors/bugs.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 34224ecd8360e03603a1a5dd1c1c9f0a31f1c1d5)
+
+commit 345de45cbf5932174222b733bf71820e818fbfd5
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Apr 14 19:18:37 2016 -0400
+
+    rgw_file: implement rgw_setattr
+    
+    Introduce a new RGWSetattrs RGWOp descendant, to create or replace
+    sets of attrs on buckets or objects.
+    
+    This version of the change uses the standard RGWRADOS::set_attrs op
+    (we want attribute changes to (e.g.) sync with other changes).
+    
+    Previous versions of this changed incorrectly masked the values
+    of st->st_ino in RGWFileHandle::stat(), now fixed.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 4de1c3c260265f821ebee842d49cb35bf49d8e4e)
+
+commit 88b5027850618f14c562d07ba25a664fd4c22771
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Jul 13 10:16:59 2016 -0400
+
+    rgw_file: refuse partial, out-of-order writes
+    
+    A single file object may be opened only once per gateway
+    instance, and writes to that object must be complete, and in-order.
+    Enforce this.
+    
+    If an invalid write is seen, deletes the current write transaction.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 12aded803e24539266ce9698c678088e2158a82a)
+
+commit be243695c1309acfc293f7080cf213edbc558d14
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Jul 12 17:43:43 2016 -0400
+
+    rgw_file: fix rename cases and unify unlink
+    
+    Rather ordinary rename cases failed in the unlink step.  Fix this,
+    unifying the unlink path while at it.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 05af1fbb870a905ea8903594dfc607d346c9dd31)
+
+commit a10d700697d8f432aca2b1bff6766e2478ac060d
+Author: Harald Klein <hari@vt100.at>
+Date:   Wed Aug 31 17:41:26 2016 +0200
+
+    add ldap auth custom search filter feature patch - http://tracker.ceph.com/issues/17185
+    
+    Signed-off-by: Harald Klein <hari@vt100.at>
+    (cherry picked from commit c935885ae9d5fd413495448a0b0e5fce899c9b73)
+    
+    Fixes: http://tracker.ceph.com/issues/17185
+
+commit 7486638563c1eeda7781dcf58b0c536d11f17a0b
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Aug 18 10:54:16 2016 -0400
+
+    rgw ldap:  protect rgw::from_base64 from non-base64 input
+    
+    Also adds unit tests for:
+    1. empty output from from_base64 (turns out to be harmless)
+    2. random and specific non-base64 and sort strings
+    (modified from upstream to avoid alteration of src/test/test_rgw_token.cc)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 0a4c91ec7652d02673a9b156cd16144d778a3844)
+    
+    Fixes: http://tracker.ceph.com/issues/17324
+
+commit 68788fb898118c826b136b7cd7f60265a6dfef79
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Aug 5 10:02:03 2016 -0400
+
+    rgw ldap: enforce simple_bind w/LDAPv3
+    
+    Found by Harald Klein <hklein@redhat.com>.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 08d54291435e4d1cb5e02cda3951bc6e8510b0e2)
+
+commit 871caeb5011b06d54bd41370ee20ba38e84c9ac0
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Aug 1 17:36:17 2016 -0400
+
+    rgw: add reinit/rebind logic (ldap)
+    
+    Gracefully handle stale LDAP connections by rebinding--verified
+    with MS AD.  Rebind is attempted once per request until rebound--
+    not worse than misconfiguring the environment.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 9b8fccf68789ba5c5986766c544b23aeada0e87e)
+
+commit de0c4e1281b5b67af1459aa11fbb292a0cae2ab2
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Aug 16 16:58:51 2016 -0400
+
+    rgw: RGWDataSyncCR fails on errors from RGWListBucketIndexesCR
+    
+    Fixes: http://tracker.ceph.com/issues/17073
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit e77a523f1d74768f4fef58c05cc167705d219036)
+
+commit 8833c64459edb77fa0d394b2eda2f79cd0f6dba9
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Oct 3 10:34:51 2016 -0700
+
+    PG: block writes and scan log for scrub using inclusive upper bound
+    
+    See comment in commit.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+
+commit 0e2338e5ec5441e5fc173a7af69343c775b02a13
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Oct 3 14:34:19 2016 -0700
+
+    osd_types,PG: force the map used in _scrub to be sorted correctly
+    
+    Previously, ScrubMap::objects was always sorted bitwise (nibblewise
+    before the comparator change was made.  It didn't matter because we
+    always scrubbed whole hash values.  Now, we need the objects in the
+    objectstore ordering because we may be missing objects at the tail of
+    the scanned range and need them to show up at the tail of the
+    ScrubMap::objects mapping.  We don't need to do anything else to handle
+    the upgrade process since the actual objects *in* the map were
+    determined by the objectstore ordering.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 6d410e97232afdad4f226151333b209d8b50f4ed)
+
+commit 16f9d95204f3eca6908ae241e5bacc48b006cb23
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Sep 28 13:24:56 2016 -0700
+
+    src/osd: relax the requirement that we scrub a whole hash value
+    
+    Previously, we needed to scrub all objects in clones in a single
+    hash value mainly to ensure that _scrub had access to all clones
+    of a single object at the same time.  Instead, just avoid letting
+    head or snapdir be a boundary (see the comment in the commit
+    for details).
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 27bdc8ce6d0a7d8ae47f29540f281ba417e16b4c)
+
+commit 2176c847d588fbec654653db48c4ad156a87aac2
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Sep 28 13:25:42 2016 -0700
+
+    hobject: clarify is_snap and has_snapset for max
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit a2c27c9423b43b8c5633fb0af95f28b0de94b365)
+
+commit 84adafe3fe6f57d49b4bdf1e036e1b686e6b0f55
+Merge: 5246f81 fb36dd0
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Sep 30 14:02:43 2016 -0400
+
+    Merge pull request #10656 from dachary/wip-16564-jewel
+    
+    jewel: cors auto memleak
+
+commit 35660d1aabd8b27bf3ba71e50d00ba1cf1cecf00
+Author: Igor Podoski <igor.podoski@ts.fujitsu.com>
+Date:   Wed Sep 14 07:36:43 2016 -0400
+
+    mon: OSDMonitor: Missing nearfull flag set
+    
+    Output from 'ceph -s -f json-pretty' wans't showing nearfull flag set.
+    
+    Signed-off-by: Igor Podoski <igor.podoski@ts.fujitsu.com>
+    (cherry picked from commit 754887b575795ef96cda095f7f0d8c898e20f00f)
+
+commit 5246f8135124dd49466268ead3c8c10d241c94f9
+Merge: 9c464a5 3cb0a5e
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Sep 29 15:54:05 2016 -0700
+
+    Merge pull request #11252 from ceph/wip-offline-split-jewel
+    
+    ceph-objectstore-tool: add a way to split filestore directories offline
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 9c464a55c8ffe7278450c88cc208f2f2e7bce1a0
+Merge: 085e9e3 457d78f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 11:34:44 2016 -0700
+
+    Merge pull request #10866 from dachary/wip-17122-jewel
+    
+    jewel: COPY broke multipart files uploaded under dumpling
+
+commit 085e9e369c5c6fbe48f888009e3997d86dcaf968
+Merge: cb46489 23d73dc
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 11:32:09 2016 -0700
+
+    Merge pull request #10890 from dachary/wip-17140-jewel
+    
+    jewel: rgw: period commit loses zonegroup changes: region_map converted repeatedly
+
+commit cb46489b92b14c5722368e88a4c66712ecc1623d
+Merge: a0fce63 cefd6f5
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 10:51:09 2016 -0700
+
+    Merge pull request #11021 from cbodley/wip-16794
+    
+    jewel: rgw: back off bucket sync on failures, don't store marker
+
+commit a0fce6302013b2e9ddad8466d7ab34c370d3786e
+Merge: 4e61538 92581a3
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 10:49:28 2016 -0700
+
+    Merge pull request #10657 from dachary/wip-16792-jewel
+    
+    jewel: metadata sync can skip markers for failed/incomplete entries
+
+commit 3cb0a5e0b0517701ad3b44bdf6d4b60d6a51e8ee
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri Aug 5 11:45:00 2016 -0700
+
+    ceph-objectstore-tool: add a way to split filestore directories offline
+    
+    Use the usual split code, and split each dir that meets the
+    usual split criteria.
+    
+    This can be run with lower than usual split settings, to avoid costly
+    online splits. To make sure the directories are not merged again, use
+    a load merge threshold (e.g. filestore merge threshold = 1), and
+    adjust the split multiplier accordingly.
+    
+    Fixes: http://tracker.ceph.com/issues/17220
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit e7b0428e0e8d8f5459311dc698d94a3ac4f04684)
+    
+    Conflicts:
+    	src/os/filestore/FileStore.cc
+    	src/tools/ceph_objectstore_tool.cc
+    (trivial resolution)
+
+commit de672a09f1e2b29de1e0e6e15d26be99315c41b9
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri Aug 5 11:14:53 2016 +0200
+
+    logrotate: Run as root/ceph
+    
+    Currently, we run the logrotate scripts as ceph/ceph but that way we
+    cannot rotate the scripts created by qemu (they are root/ceph and 644).
+    The original su line was introduced in commit 73d7bed9 because logrotate
+    complained that the directory was writable by a non-root group and it
+    needed a su line to supress that error. This way, we keep the group
+    settings and we can access and rotate the qemu logs as well.
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+
+commit 4e61538d9f1b372f28e61ad94c368c782be09d92
+Merge: ecc2377 e72e7a2
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 27 15:37:17 2016 -0500
+
+    Merge pull request #10888 from dachary/wip-17062-jewel
+    
+    jewel: tests: fix TestClsRbd.mirror_image failure in upgrade:jewel-x-master-distro-basic-vps
+
+commit 9cb45e11ec392da7c22578539b99619bdbf37e35
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Tue May 10 16:44:44 2016 +1000
+
+    log: Log.cc: Assign LOG_INFO priority to syslog calls
+    
+    LOG_DEBUG prio messages are not logged by a default syslog
+    configuration so log at LOG_INFO instead.
+    
+    Fixes: http://tracker.ceph.com/issues/15808
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 3ab5a660a45506d6c4c41bfeb5babdf188e62b3d)
+
+commit 165e5abdbf6311974d4001e43982b83d06f9e0cc
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Fri Apr 1 16:55:36 2016 +0530
+
+    librados: modify Pipe::connect() to return the error code
+    
+    Fixes: http://tracker.ceph.com/issues/15308
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit cad38dca0c90fb2ee4b589f336f8272122f50ad6)
+
+commit 5ab5e8268174bd34a095a1e96a1a969ad9e3531b
+Author: Andreas Gerstmayr <andreas.gerstmayr@gmail.com>
+Date:   Thu Sep 15 22:02:23 2016 +0200
+
+    doc: fix description for rsize and rasize
+    
+    Signed-off-by: Andreas Gerstmayr <andreas.gerstmayr@gmail.com>
+    (cherry picked from commit a004254a02cafdee848a382f893f89c3c5a9845a)
+
+commit c94244d3b132b6cf1f8ad309644cf48739282f04
+Author: Jelle vd Kooij <vdkooij.jelle@gmail.com>
+Date:   Thu Sep 1 00:42:34 2016 +0200
+
+    Add Install section to systemd rbdmap.service file
+    
+    Signed-off-by: Jelle vd Kooij <vdkooij.jelle@gmail.com>
+    (cherry picked from commit 57b6f656e17124a5ab4cd1400840d9c7c87a3cc3)
+
+commit e2ce8576f547b1e073c169ef331c7063319d9783
+Author: tserlin <tserlin@redhat.com>
+Date:   Mon Sep 19 14:40:12 2016 -0400
+
+    Add two options to radosgw-admin.rst manpage
+    
+    Add '--job-id' and '--max-concurrent-ios' to Orphan Search Options
+    
+    Fixes: http://tracker.ceph.com/issues/17281
+    Signed-off-by: Thomas Serlin <tserlin@redhat.com>
+    (cherry picked from commit 697f30d86f49b73c981c06375ab2937570b1db01)
+
+commit 4e66f9e9d1179204a6fdf08e43608a852d2b7fd3
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Thu Sep 15 12:56:39 2016 -0600
+
+    radosgw-admin: add "--orphan-stale-secs" to --help
+    
+    The radosgw-admin --help did not include the description of the
+    `--orphan-stale-secs` option of the `orphans find` command. The option
+    sets the number of seconds to wait before declaring an object to be an
+    orphan.
+    
+    Fixes: http://tracker.ceph.com/issues/17280
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 354059ae43b4f4cc797da1669715399cd96a4738)
+
+commit bfa90a152867e0277ae05c64b6bdc429b659b9a5
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Thu Sep 15 11:40:14 2016 -0600
+
+    doc: add "--orphan-stale-secs" to radosgw-admin(8)
+    
+    The radosgw-admin(8) manual page did not include the description of the
+    `--orphan-stale-secs` option of the `orphans find` command. The option sets
+    the number of seconds to wait before declaring an object to be an
+    orphan.
+    
+    Fixes: http://tracker.ceph.com/issues/17280
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit a676c516069cc448591018ecf4f7d1f7f7bc3bfd)
+
+commit cefd6f51068df3ed04c755b1167dd146b85a0741
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jul 20 12:59:50 2016 -0700
+
+    rgw: fix collection of object sync errors
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit aacc31f72cef6f151459e4a0543850edeeaf1938)
+
+commit aa369817c9bf10cffa420bfb88a27957407fb625
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jul 20 12:43:48 2016 -0700
+
+    rgw: fix marker tracker completion handling
+    
+    Was not tracking high markers correctly. Could only work if there was a single
+    hole in the completion range. Just keep a map of all the complete entries.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f5801c726efdc2f3067a071e6bb5ac83fd0cd147)
+
+commit bce19a39b6f6ff23b6373f82ed2e6e12d65a77ef
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jul 20 12:42:52 2016 -0700
+
+    rgw: collect() stops if error encoutered
+    
+    and returns true if needed to be called again
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 5528932e4c327a0fb0ce34a676eb59707a575325)
+
+commit f82b59363d12e38446b4f06f554b932a3cbdb1fb
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Jul 19 15:32:03 2016 -0700
+
+    rgw: back off bucket sync on failures, don't store marker
+    
+    Fixes: http://tracker.ceph.com/issues/16742
+    
+    If we fail on any single entry in bucket, skip updating the marker tracker
+    so that next time we'll go over that entry, and back off. This will trigger
+    a report to the data sync error repo and eventually a retry on the failing
+    object.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 1f3fec807043fd313ef7c66ff48f18b82c8bfa66)
+
+commit f337a0720713329b5305ddd9e5c18e8d2b11b475
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Fri Jun 3 17:24:48 2016 +0200
+
+    krbd: don't segfault if images are unmapped concurrently
+    
+    "rbd map c" can die from a NULL dereference on any of this_pool,
+    this_image or this_snap in wait_for_udev_add():
+    
+        <image a is mapped>
+        rbd map c
+                                        rbd map b
+                                        rbd unmap a
+                                        rbd unmap b
+    
+    However unlikely, this segfault is triggered by the rbd/concurrent.sh
+    workunit on a regular basis.
+    
+    Similarly, "rbd showmapped" can die if an image to be listed is
+    unmapped.
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit 2bfecb1c0a0b2a314a5d137e6ca2cccd9ddc9b54)
+
+commit 2399d4ae54780cb18c313a8da79b697a70a87146
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Fri May 13 17:57:12 2016 +0200
+
+    qa: rbd/simple_big.sh: drop extraneous sudo
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit 6643f4d9723e5d9c5195680ac9abb9915ef42a56)
+
+commit 472cb29c5712c4fbbeaef26cdfddc1afd5f2945d
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Fri May 13 15:05:53 2016 +0200
+
+    qa: rbd/concurrent.sh: suppress rbd map output
+    
+    rbd_map_image() echoes the id of the image and is used in command
+    substitution:
+    
+        id=$(rbd_map_image "${image}")
+    
+    Output from rbd map isn't consumed and clobbers the return.
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit a7d4189587e343a581660f1a118f6667c26e5bd4)
+
+commit 667d42aad7e19e36200ee7cd979efdd19154876a
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Thu May 12 15:07:12 2016 +0200
+
+    qa: rbd: don't modprobe, chown sysfs files or udevadm settle
+    
+    This is a followup to 38a572011dbb ("qa: rbd/map-snapshot-io.sh: don't
+    chown sysfs files") for the rest of the workunits:
+    
+    - /sys/bus/rbd/add_single_major is used if present, so chown'ing
+      /sys/bus/rbd/add doesn't help.  Use sudo on rbd map instead.
+    
+    - Don't modprobe or udevadm settle - rbd CLI tool takes care of that.
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit 6bddef8ecff97f732bf707d9ba6d2386bc147f5a)
+
+commit 866c3e594b754d10bf960375f8498ba573c531dd
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Wed May 11 17:26:39 2016 +0200
+
+    qa: rbd/map-snapshot-io.sh: don't chown sysfs files
+    
+    /sys/bus/rbd/add_single_major is used if present, so chown'ing
+    /sys/bus/rbd/add doesn't help.  Use sudo on rbd map instead.
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit 38a572011dbbf8298cb7ba8ebf43909967eec689)
+
+commit cf211d7dc2eef46cb409b425caa72aac1b24bd7c
+Author: John Spray <john.spray@redhat.com>
+Date:   Sun Jul 24 17:19:55 2016 +0100
+
+    client: fix shutdown with open inodes
+    
+    This piece of code was dereferencing an invalid
+    iterator (invalidated by call to erase())
+    
+    Fixes:  http://tracker.ceph.com/issues/16764
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit d642b4faec3266f609e4871ccdccdcd73707dc23)
+
+commit 1bc047b1c6ccc1a1757c97e52da5e4db5bc8f801
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Wed Jun 29 15:45:30 2016 -0400
+
+    client: add missing client_lock for get_root
+    
+    A segmentation fault was observed in a recent test failure
+    (test_client_cache_size) [1]. An analysis of the core dump revealed that the
+    client->root inode was NULL. The client log [2] showed that the root inode was
+    deleted prior to the segfault:
+    
+        2016-06-27 14:24:13.358500 7fe75cff9700 20 client.4161 _ll_put 0x7fe76000ba40 100000003e9 1 -> 0
+        2016-06-27 14:24:13.358502 7fe75cff9700 10 client.4161 put_inode on 100000003e9.head(faked_ino=0 ref=1 ll_ref=0 cap_refs={} open={} mode=41777 size=0/0 mtime=2016-06-27 14:24:10.700562 caps=pAsLsXsFs(0=pAsLsXsFs) 0x7fe76000ba40)
+        2016-06-27 14:24:13.358513 7fe75cff9700 15 inode.put on 0x7fe76000ba40 100000003e9.head now 0
+        2016-06-27 14:24:13.358515 7fe75cff9700 10 client.4161 remove_cap mds.0 on 100000003e9.head(faked_ino=0 ref=0 ll_ref=0 cap_refs={} open={} mode=41777 size=0/0 mtime=2016-06-27 14:24:10.700562 caps=pAsLsXsFs(0=pAsLsXsFs) 0x7fe76000ba40)
+        2016-06-27 14:24:13.358538 7fe75cff9700 15 client.4161 remove_cap last one, closing snaprealm 0x7fe76000c1b0
+        2016-06-27 14:24:13.358544 7fe75cff9700 20 client.4161 put_snap_realm 1 0x7fe76000c1b0 1002 -> 1001
+        2016-06-27 14:24:13.358546 7fe75cff9700 10 client.4161 put_inode deleting 100000003e9.head(faked_ino=0 ref=0 ll_ref=0 cap_refs={} open={} mode=41777 size=0/0 mtime=2016-06-27 14:24:10.700562 caps=- 0x7fe76000ba40)
+    
+    After looking through the reference count inc/dec, I noticed this mismatched inc/dec:
+    
+        2016-06-27 14:24:13.352622 7fe75dffb700  3 client.4161 ll_forget 100000003e9 1
+        2016-06-27 14:24:13.352601 7fe7767fc700  1 -- 172.21.15.58:0/3762258427 >> 172.21.15.57:6804/11121 conn(0x7fe789bf3b60 sd=20 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=7 cs=1 l=0). == rx == mds.0 seq 1473 0x7fe764037a80 client_reply(???:470 = 0 (0) Success safe) v1
+        2016-06-27 14:24:13.352665 7fe75dffb700 20 client.4161 _ll_put 0x7fe76000ba40 100000003e9 1 -> 1
+        2016-06-27 14:24:13.352687 7fe75d7fa700  3 client.4161 ll_lookup 0x7fe76000ba40 testdir
+        2016-06-27 14:24:13.352699 7fe75d7fa700 20 client.4161 _lookup have dn testdir mds.-1 ttl 0.000000 seq 0
+        2016-06-27 14:24:13.352705 7fe75d7fa700 15 inode.get on 0x7fe760011da0 100000003ea.head now 537
+        2016-06-27 14:24:13.352693 7fe76effd700  1 -- 172.21.15.58:0/3762258427 <== mds.0 172.21.15.57:6804/11121 1473 ==== client_reply(???:470 = 0 (0) Success safe) v1 ==== 27+0+0 (3458149698 0 0) 0x7fe764037a80 con 0x7fe789bf3b60
+        2016-06-27 14:24:13.352708 7fe75d7fa700 10 client.4161 _lookup 100000003e9.head(faked_ino=0 ref=3 ll_ref=1 cap_refs={} open={} mode=41777 size=0/0 mtime=2016-06-27 14:24:10.700562 caps=pAsLsXsFs(0=pAsLsXsFs) 0x7fe76000ba40) testdir = 100000003ea.head(faked_ino=0 ref=537 ll_ref=999 cap_refs={} open={} mode=40755 size=0/0 mtime=2016-06-27 14:24:13.325297 caps=pAsLsXsFsx(0=pAsLsXsFsx) COMPLETE parents=0x7fe750001f50 0x7fe760011da0)
+    
+    This sequence of ll_forget/ll_lookup is repeated in this test several hundred times. This is the prior sequence (in one thread):
+    
+        2016-06-27 14:24:13.324896 7fe75dffb700  3 client.4161 ll_forget 100000003e9 1
+        2016-06-27 14:24:13.324904 7fe75dffb700 20 client.4161 _ll_put 0x7fe76000ba40 100000003e9 1 -> 1
+        2016-06-27 14:24:13.324915 7fe75dffb700  3 client.4161 ll_lookup 0x7fe76000ba40 testdir
+        2016-06-27 14:24:13.324921 7fe75dffb700 20 client.4161 _lookup have dn testdir mds.-1 ttl 0.000000 seq 0
+        2016-06-27 14:24:13.324931 7fe75dffb700 15 inode.get on 0x7fe760011da0 100000003ea.head now 621
+        2016-06-27 14:24:13.324904 7fe7767fc700  1 -- 172.21.15.58:0/3762258427 >> 172.21.15.57:6804/11121 conn(0x7fe789bf3b60 sd=20 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=7 cs=1 l=0). == rx == mds.0 seq 1387 0x7fe764026720 client_reply(???:385 = 0 (0) Success safe) v1
+        2016-06-27 14:24:13.324934 7fe75dffb700 10 client.4161 _lookup 100000003e9.head(faked_ino=0 ref=3 ll_ref=2 cap_refs={} open={} mode=41777 size=0/0 mtime=2016-06-27 14:24:10.700562 caps=pAsLsXsFs(0=pAsLsXsFs) 0x7fe76000ba40) testdir = 100000003ea.head(faked_ino=0 ref=621 ll_ref=998 cap_refs={} open={} mode=40755 size=0/0 mtime=2016-06-27 14:24:13.321021 caps=pAsLsXsFsx(0=pAsLsXsFsx) COMPLETE parents=0x7fe750001f50 0x7fe760011da0)
+    
+    The key line is `_lookup 100000003e9.head(...)`. ll_ref=2 is the expected count
+    after this sequence but, as we see at 14:24:13.352708, ll_ref=1.
+    
+    This racing behavior is not serialized by client_lock because
+    Client::get_root() does not lock the client_lock. Additionally, this race is
+    not easy to identify in the logs because Client::get_root() does not use
+    Client::_ll_get to increment the reference, which hides the increments in the
+    logs. Instead it directly increments the reference using Inode:ll_get().
+    
+    This commit adds the client_lock for Client::get_root() and
+    Client::get_root_ino() (which may not strictly be necessary but can't hurt).
+    
+    [1] http://pulpito.ceph.com/pdonnell-2016-06-27_06:54:23-fs-wip-pdonnell-i16164---basic-smithi/280361/
+    [2] http://qa-proxy.ceph.com/teuthology/pdonnell-2016-06-27_06:54:23-fs-wip-pdonnell-i16164---basic-smithi/280361/remote/smithi058/log/ceph-client.0.23448.log.gz
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 208e3b6fd58237b309aae15b18dcd82672b133af)
+
+commit 086f6e0c07c807dc5001e40d18a72d53c5a10f66
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed Jul 20 13:02:03 2016 +0200
+
+    rgw: fix upgrade from old multisite to new multisite configuration
+    
+    We need to store the updated current period after adding the old converted regions
+    
+    Fixes: http://tracker.ceph.com/issues/16751
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 6d7841c6f050f6bd813d54df65d38e7f6934d22d)
+
+commit 23d73dca4e8fd9472b8e39d9219fda90323478fb
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Aug 23 15:10:44 2016 -0400
+
+    rgw: delete region map after upgrade to zonegroup map
+    
+    convert_regionmap() reads the region map and uses it to initialize the
+    zonegroup map.  but it doesn't remove the region_map afterwards, so
+    radosgw (and some radosgw-admin commands) will keep doing this on
+    startup, overwriting any changes made to the period/zonegroup map
+    
+    Fixes: http://tracker.ceph.com/issues/17051
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 215cd6f6ff7b8cb74df6dc3b94a5928f56ab9540)
+
+commit 27626ba9cc7495e312bf5caba6be8d6ee2f1ea64
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Aug 17 10:28:48 2016 -0400
+
+    rgw_file: restore local definition of RGWLibFS gc interval
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 31936caa09b46e86700faad969adfa9d43176206)
+
+commit 91b65e7b4976103ea45c923bac9d21321d962773
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Aug 9 16:49:41 2016 -0400
+
+    rgw file: remove busy-wait in RGWLibFS::gc()
+    
+    This is a background thread.  However, CPU is wasted.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit ca33241286f52d849dbde8092507131b8b1108cc)
+
+commit e72e7a26f22022e6520232481f66f509367073a1
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Aug 12 14:44:09 2016 -0400
+
+    qa/workunits: support filtering cls_rbd unit test cases
+    
+    Fixes: http://tracker.ceph.com/issues/16529
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0891ede57cd254aa75c66301fde4a2eb6957a3d6)
+
+commit a25a4835ef91f2c249107a3f402c299d8c5c40da
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Jun 3 14:41:13 2016 -0700
+
+    ReplicatedPG: call op_applied for submit_log_entries based repops
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 8bde0ae122f384817f4e522604379d1e9b21d8d9)
+
+commit 5ef90856ce26761b31cfd4eb1bea4dfbe88cfb13
+Author: Tao Chang <changtao@hihuron.com>
+Date:   Mon May 9 09:50:17 2016 -0400
+
+    Cleanup: delete find_best_info again
+    
+    After called find_best_info find a auth_pg_shard, it must be not incomplete,
+    so it will not enter find_best_info again.
+    
+    Signed-off-by: Tao Chang <changtao@hihuron.com>
+    (cherry picked from commit a25cfc4d46c03d8d78e0254c728ea5c29e2246e1)
+
+commit dea93dc35859ebf31366dfe5f2b8c2ebebc4f6a6
+Author: Boris Ranto <branto@redhat.com>
+Date:   Mon Jun 27 11:48:17 2016 +0200
+
+    ceph-disk: do not activate device that is not ready
+    
+    If the journal (or data) device is not ready when we are activating the
+    data (or journal) device, just print an info message and exit with 0 so
+    that the ceph-disk systemd service won't fail in this case.
+    
+    Fixes: http://tracker.ceph.com/issues/15990
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 73a7a65f65b1f6e90c4cae101bd43fabaeac3045)
+
+commit 58b7c52db2df0105aea3fbc36a0b2bc0d299415f
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Aug 11 08:57:51 2016 -0700
+
+    PG: use upset rather than up for _update_calc_stats
+    
+    Fixes: http://tracker.ceph.com/issues/16998
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 200cae8c9db99b3dede377f74413cc0b15033e1f)
+
+commit 4e2b4917c30b431f87b0da0d090664f2210f7ad2
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Aug 11 08:57:35 2016 -0700
+
+    PG: introduce and maintain upset
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 80a5c21d97116e7a66351a0e1f5d9b561f8663ac)
+
+commit c0db9fb831dc67ae138056c2f03c31dcc7f6adbf
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Mon Aug 1 09:01:15 2016 -0400
+
+    ceph: don't fudge the ctime in stat() unless it's really older than the mtime
+    
+    The current code just looks at the sec field, but the difference may
+    only be evident in the nsec field.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 14ee7bcbf0bdd6553dc1b78898ae429aae644100)
+
+commit 66cd43bfedddbbc82e3c303554f089f20eabf0f6
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Mon Aug 1 09:01:15 2016 -0400
+
+    client: only skip querying the MDS in _lookup when we have the necessary caps
+    
+    If we don't have the caps we'll need later, then we must reissue the
+    lookup to get them regardless of whether we have a lease on the
+    dentry.
+    
+    Fixes: http://tracker.ceph.com/issues/16668
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 9e8476743eef195a0a9bc6ef82392a8781e82549)
+
+commit fb4a9398a8d60f40ed4c8f57965de54dac1df983
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Mon Aug 1 09:01:14 2016 -0400
+
+    client: plumb a mask argument into _lookup
+    
+    ...and attempt to pass in a sane value there, based on what we'll do
+    with the resulting inode.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit f3605d39e53b3ff777eb64538abfa62a5f98a4f2)
+
+commit b5cbd5766fbf2a9ff5ee0e58b4ea42c706f4505a
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Mon Aug 1 09:01:14 2016 -0400
+
+    client: add mask parameter to _do_lookup
+    
+    We need to allow callers to specify caps to acquire during a lookup, as
+    they may need to scrape certain info out of the inode later. Allow them
+    to pass in a mask. For now, _lookup just passes in 0 for the mask,
+    but verify_reply_trace passes in the regetattr_mask to match what we'd
+    request in the _getattr request if there were a traceless reply.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit a2ce16f8bfdb16ac485b8c4ad9a51ade5c256a5b)
+
+commit 416ec6ff91c602f385385acd8ac458ac58f13222
+Author: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+Date:   Tue Aug 16 14:19:20 2016 +0800
+
+    rgw: fix radosgw daemon core when reopen logs
+    
+    the rgw_user_st_syn thread use the pointer meta_mgr in RGWUserStatsCache::sync_all_users() fuction, so we should close the thread before delete the pointer in RGWRados::finalize() function when close storage before reopen_logs.
+    
+    Fixes: http://tracker.ceph.com/issues/17036
+    
+    Signed-off-by: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+    (cherry picked from commit ea5d7fdd7fa978deadeee42bc0ea33fb11662e59)
+
+commit f034fd0a7b959dc3553fb0708da989ba6ac77db3
+Author: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+Date:   Wed Aug 24 09:38:03 2016 +0800
+
+    rgw: fix period update --commit return error
+    
+    period commit return error when the current period has a zonegroup which doesn't have a master zone
+    
+    Fixes: http://tracker.ceph.com/issues/17110
+    Signed-off-by: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+    (cherry picked from commit fcedda6e5dfbaff3975b4474d8546cf77f78cc83)
+
+commit 457d78f4f497fdaa0f0f116228ebf135a1ce7fb6
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jul 7 18:01:54 2016 -0700
+
+    rgw: adjust manifest head object
+    
+    adjust the manifest head object:
+     - when reading manifest, set the head object to the object
+       we read the manifest from (and its size). Some manifests are
+       broken and point at a different object
+     - when copying multipart object, set the manifest head object to
+       point at the new head object
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 419264586fc46337322f0d60b0ffcdcce3bb5c5a)
+
+commit 77e497da5c618a30fb908cb3915aa1ca7279ce7f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jul 7 15:49:07 2016 -0700
+
+    rgw: adjust objs when copying obj with explicit_objs set
+    
+    If the head object contains data, need to set it in the list of
+    objs (in addition to being pointed at by head_obj).
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 1e012112b99004745952accc6fb11ada5e8e0045)
+
+commit c1250ee6cbc4a7710399e603ff3ee0e46aa92d49
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jul 7 15:36:33 2016 -0700
+
+    rgw: patch manifest to handle explicit objs copy issue
+    
+    Fixes: http://tracker.ceph.com/issues/16435
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit b51476466e5bb03bbaa6e428bb6bb189a259b9fd)
+
+commit 2311ab24049805ea274451607949ba9b54028470
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Jul 15 11:32:18 2016 +0800
+
+    ObjectCacher: fix last_write check in bh_write_adjacencies()
+    
+    Make the last_write check in bh_write_adjacencies() match corresponding
+    check in flush().
+    
+    Fixes: http://tracker.ceph.com/issues/16610
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit ce166c2cd706b8573deabd331d29544e75e85972)
+
+commit c3c2910fbe8c6cc206e1cd70d267503f8ab36e74
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Aug 7 23:08:48 2016 +0200
+
+    rpm: proper packaging of udev directories
+    
+    The issue here is that ceph.spec.in does not package the directories
+    /usr/lib/udev and /usr/lib/udev/rules.d. The problem was not showing because
+    hdparm, which is brought in as a build dependency, packages these directories.
+    However, in SUSE a recent update to hdparm changes that and the problem
+    manifests.
+    
+    This PR addresses the issue by adding udev as a build dependency, which should
+    cover /usr/lib/udev, and by adding explicit "%dir %{_udevrulesdir}" to the
+    packages that put files in this directory.
+    
+    Fixes: http://tracker.ceph.com/issues/16949
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 1a164bc94d661b9d83b4a80e957712ed6f5cd122)
+
+commit 7d92e2ea1855a21f9a444c716c590e90dcb9f978
+Author: songbaisen <song.baisen@zte.com.cn>
+Date:   Mon Apr 25 10:34:42 2016 +0800
+
+    crush: When consider rule size use the correct method.
+    
+    Signed-off-by: song baisen <song.baisen@zte.com.cn>
+    (cherry picked from commit 4cf4791ea1200528ccc0d65b3d4a0fbc234c5df0)
+
+commit c2cf21dcce9f481a53c8309800707c80c77754e1
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Aug 5 14:27:43 2016 +0800
+
+    mon/PGMonitor: calc the %USED of pool using used/(used+avail)
+    
+    we were using
+    "the raw space used by this pool" / "the raw space of the whole cluster"
+    as the %USED. but it's wrong:
+    
+    - there is chance that not all OSDs are assigned to the pool in question
+    - we can not utilize all the raw space for the pool: there is overhead.
+      and the size of available space for a pool is capped by the assigned
+      OSD with minimal free space.
+    
+    so we should use
+     USED / (USED + AVAIL)
+    as the %USED. so once we have no AVAIL space left, %USED will be 100%.
+    
+    Fixes: http://tracker.ceph.com/issues/16933
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 08532ea166dfd97c548d9f1fb478e163021cdda3)
+
+commit 109da6fd2fa8a6f8e56a1fe2daa6a738bb2a1fde
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sun Aug 7 00:58:04 2016 +0800
+
+    mon/PGMonitor: mark dump_object_stat_sum() as static
+    
+    so we can test this method without create an instance of PGMonitor.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit af3d3e25823ca9b40ddf09835edb82795ac68f33)
+
+commit bd636662a2ff26d9af13955b08f9ab20f29de771
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Aug 12 09:52:21 2016 -0400
+
+    librbd: fix possible inconsistent state when disabling mirroring
+    
+    Fixes: http://tracker.ceph.com/issues/16984
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7cfedb54ea0cf496cc4b55d08a787abc2d6a4bbe)
+
+commit 117aa35094c059dbf5770b01ac13a583471e54aa
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sun Jun 26 01:02:03 2016 +0800
+
+    common: instantiate strict_si_cast<long> not strict_si_cast<int64_t>
+    
+    this fixes the build on armf.
+    
+    on 32bit platforms, cstdint is very likely to
+    
+     typedef long long int int64_t;
+    
+    this results in compilation error like
+    
+     `common/strtol.cc:190:75: error: duplicate explicit instantiation of 'T
+     strict_si_cast(const char, std::string) [with T = long long int;
+     std::string = std::basic_string]'
+    
+     [-fpermissive]
+     template int64_t strict_si_cast(const char *str, std::string *err);
+     ^`
+    
+    we can address this by instantiate the primitive type of `long long`
+    instead of `in64_t`.
+    
+    Fixes: http://tracker.ceph.com/issues/16398
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 31db4c5f9f725e13e38f3c90744e299e023d02a4)
+
+commit d93eda88048d2bcefe4be3ea0aaa6ca0289eabbf
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Thu May 26 23:30:25 2016 +0530
+
+    common: add int64_t template for strict_si_cast()
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit 8e429d05370fbe7935212d0ae9608e7547f39860)
+
+commit f7cd28460147530cfd265a593b32d02adb93abe6
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Sat Apr 30 18:31:37 2016 +0800
+
+    common/config: cast OPT_U32 options using uint32_t
+    
+    the OPT_U32 options was translated using strict_si_cast<int>(), and then
+    cast the converted result to uint32_t. this could cause integer
+    underflow. we could have lifted the burden of checking invalid input
+    from the user of this option to the strict_si_cast<>() function. so in
+    this change, we use strict_si_cast<uint32_t>() instead, before casting
+    the converted value into `uint32_t`.
+    
+    Signed-off-by: Kefu Chai <tchaikov@gmail.com>
+    (cherry picked from commit b7babd6aa671d688eef0af61ca17fd11eec22773)
+
+commit 518883d939f34ec0afa03aea1bac35960fb579f2
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Aug 25 09:09:40 2016 +0200
+
+    Revert "common: add int64_t template for strict_si_cast()"
+    
+    This reverts commit e3a99c082e3ebd56d5b40d7d94d98e35629df81e.
+
+commit 29a87012f560c213e34b893cd8dd9dedfd0b11da
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Aug 22 11:52:45 2016 +0200
+
+    ceph-osd-prestart.sh: check existence of OSD data directory
+    
+    Fixes: http://tracker.ceph.com/issues/17091
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit d6ffa770784f0efb88827ee7c65ff2e5f9029434)
+
+commit 80e25b00554e3955f7bee2e03eba6c55a955999b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Aug 12 08:26:49 2016 -0400
+
+    rbd: bench io-size should not be larger than image size
+    
+    Fixes: http://tracker.ceph.com/issues/16967
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 113775eb583fa28a669c6dc1f8dbf47a8ecd789c)
+    
+    Conflicts:
+    	src/tools/rbd/action/BenchWrite.cc: trivial resolution
+
+commit f7d23d59535dee004686c5baa79c7b0815ab610a
+Author: Michel Normand <normand@linux.vnet.ibm.com>
+Date:   Fri Aug 5 15:19:50 2016 +0200
+
+    LTTng-UST disabled for openSUSE
+    
+    LTTng-UST not yet supported in openSUSE so do not enable lltng for it.
+    The (1) is where is defined "is_opensuse"
+    
+    Remove value for test of suse_version in spec file
+    and change related comment from SLES12 to SLE
+    as per comment in https://github.com/ceph/ceph/pull/10592
+    
+    (1) https://en.opensuse.org/openSUSE:Build_Service_cross_distribution_howto#Detect_a_distribution_flavor_for_special_code
+    
+    Fixes: http://tracker.ceph.com/issues/16937
+    Signed-off-by: Michel Normand <normand@linux.vnet.ibm.com>
+    (cherry picked from commit 7da19b6db7eaefb5fe159375cc5a465a722d3897)
+
+commit e6ac214f8a6d8773de032c83f5fdd7c65b68d53f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jul 28 15:09:53 2016 -0400
+
+    librbd: request exclusive lock if current owner cannot execute op
+    
+    The initial krbd implementation will not support executing maintenance
+    ops and instead will return -EOPNOTSUPP. In this case, librbd can take
+    the lock and execute the operation.
+    
+    Fixes: http://tracker.ceph.com/issues/16171
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d2d2d90d64663905c2b81f7809f1d636db6b7fb1)
+
+commit 0b93a9a1e7929bc0709ec139fbc6686783f99044
+Author: Wei Jin <wjin.cn@gmail.com>
+Date:   Thu Apr 28 19:00:48 2016 +0800
+
+    fast dispatch heartbeat message
+    
+    Signed-off-by: Wei Jin <wjin.cn@gmail.com>
+    (cherry picked from commit 6f1f717446314618db7a165c75dfd6a76a01f55e)
+
+commit 0dcefd2352c06b719960a2df970ac3cbdcce8f52
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Mon Jul 18 13:57:08 2016 +0800
+
+    mon/osdmonitor: fix incorrect output of "osd df" due to osd out
+    
+    If an osd is automatically marked as out, the output of "osd df"
+    is not right, as follow:
+    
+    -5 10.00999        -  5586G  2989G  2596G     0    0     host ceph192-9-9-8
+    11  0.90999  1.00000   931G   542G   388G 58.25 0.99         osd.11
+    14  0.90999  1.00000   931G   530G   400G 56.97 0.97         osd.14
+    20  0.90999  1.00000   931G   716G   214G 76.99 1.31         osd.20
+    22  0.90999  1.00000   931G   477G   453G 51.29 0.87         osd.22
+    26  0.90999        0      0      0      0     0    0         osd.26
+    28  0.90999  1.00000   931G   587G   343G 63.09 1.07         osd.28
+    30  0.90999  1.00000   931G   602G   328G 64.75 1.10         osd.30
+    16  0.90999  1.00000   931G   589G   341G 63.34 1.08         osd.16
+    18  0.90999  1.00000   931G   530G   400G 56.93 0.97         osd.18
+    24  0.90999  1.00000   931G   202G   728G 21.77 0.37         osd.24
+    32  0.90999  1.00000   931G   477G   454G 51.23 0.87         osd.32
+    
+    Two problems are identified from the above output:
+    
+    1. the total capacity(total, total used, total avial)
+    only includes osd.32, osd.24, osd.18, osd.16, osd.30, osd.28, and other
+    healthy osds such as osd.11, osd.14 etc. are excluded.
+    
+    2. the average utilization/deviation are forced resetted.
+    
+    Fixes: http://tracker.ceph.com/issues/16706
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 1e4735440ca329f4b95d9bd6f58d9efab7d28d20)
+
+commit ac27352743a4ac1e4a4bec422d41a0ad8f17e41a
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Fri Jul 15 16:08:33 2016 +0800
+
+    mon/osdmonitor: initialize local variable "kb_avail_i"
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 98f50cebe1d1708adeeb6fc6d0aa2cdc85ac942c)
+
+commit 430ab1b83e67dfb697b034e669b06b7a600bcc6b
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Jul 12 16:56:52 2016 +0200
+
+    ceph-disk: timeout ceph-disk to avoid blocking forever
+    
+    When ceph-disk runs from udev or init script, it is in the background
+    and should it block for any reason, it may keep a lock forever. All
+    calls to ceph-disk in these context are changed to timeout.
+    
+    The TimeoutStartSec= and TimeoutStopSec= which are both set via
+    TimeoutSec= do not apply to Type=oneshot services.
+    
+    https://www.freedesktop.org/software/systemd/man/systemd.service.html
+    
+    Fixes: http://tracker.ceph.com/issues/16580
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit bed1a5cc05a9880b91fc9ac8d8a959efe3b3d512)
+
+commit 0b30a1d210710593678462b287ff33f22de52323
+Author: Zengran Zhang <zhangzengran@h3c.com>
+Date:   Mon Apr 25 22:45:56 2016 -0400
+
+    mon/osdmonitor: decouple adjust_heartbeat_grace and min_down_reporters
+    
+    if cancel mon_osd_adjust_heartbeat_grace, we must set mon_osd_min_down_reporters to zero.
+    otherwise the next checking for
+    	reporters_by_subtree.size() >= g_conf->mon_osd_min_down_reporters
+    will allways be fail.the two options should take effect respectively...
+    
+    Signed-off-by: Zengran Zhang <zhangzengran@h3c.com>
+    (cherry picked from commit 7ac5ca5698cfdaab019bf282b537bcc897f9444c)
+
+commit 92581a388462039a7f4dc748e8318c4226f068d3
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jul 20 12:43:48 2016 -0700
+
+    rgw: fix marker tracker completion handling
+    
+    Was not tracking high markers correctly. Could only work if there was a single
+    hole in the completion range. Just keep a map of all the complete entries.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f5801c726efdc2f3067a071e6bb5ac83fd0cd147)
+
+commit fb36dd0c7a72ce5f3affd69d1f77e6222e26de94
+Author: Yan Jun <yan.jun8@zte.com.cn>
+Date:   Tue Jun 28 16:17:58 2016 +0800
+
+    rgw: fix potential memory leaks in RGWPutCORS_ObjStore_S3::get_params
+    
+    We should free `data` before return or jump to `done_err` label.
+    
+    Signed-off-by: Yan Jun <yan.jun8@zte.com.cn>
+    (cherry picked from commit 69c6cf551754e51debb4e645716a89e29517f4ee)
+
+commit 775c78d4bb31640a006d255ec0cb3818b6a5d23a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 4 20:40:10 2016 -0400
+
+    librbd: prevent creation of v2 image ids that are too large
+    
+    The librbd API is capped at 24 characters for expressing the
+    object prefix for data blocks (including trailing null byte).
+    
+    Fixes: http://tracker.ceph.com/issues/16887
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4de7c8d0a7222c83268d03b99015c6b9d25f124d)
+
+commit 2b272126de6ea6e75b9dcf4fedac5c216093e36d
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Jul 4 13:54:32 2016 +0300
+
+    librbd: prevent creation of clone from non-primary mirrored image
+    
+    Fixes: http://tracker.ceph.com/issues/16449
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit ba849e3b04a5c513849d40a7fe4151375265302a)
+
+commit 18a66cb375bf6540a3908f843b004c652eacc2dc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jun 22 18:09:29 2016 -0400
+
+    librbd: journal::Replay no longer holds lock while completing callback
+    
+    Fixes: http://tracker.ceph.com/issues/16433
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3112a93b49c24f9ae101a7f252c8e708e0d3e260)
+
+commit 79e2acb9a585ede5b97d804550b5394f277e7cc4
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Jul 1 20:44:35 2016 +0800
+
+    crush: reset bucket->h.items[i] when removing tree item
+    
+    * crush: so we don't see the reference after the removing, this keeps
+      check_item_loc() happy, and move_bucket() use check_item_loc() to see if
+      the removed bucket disappears after the removal.
+    * test: also add unittest_crush_wrapper::CrushWrapper.insert_item
+    
+    Fixes: http://tracker.ceph.com/issues/16525
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit a7069c7aab6b3f605f3d8f909af96f87977e8698)
+
+commit d94e5bc0903a40d8ea353327ddaa7b8567b95f5f
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Apr 27 16:53:26 2016 -0400
+
+    common: fix value of CINIT_FLAG_DEFER_DROP_PRIVILEGES
+    
+    0x16 (binary 10110) was overlapping two other flags
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit dfb897ad576e4861a3cb75d85dffbbeaad5e980e)
+
+commit 54b6b5e42bca2b7f05b9ff77315de61d3ba14daf
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jun 8 17:47:58 2016 +0800
+
+    client: kill QuotaTree
+    
+    Multiple clients can modify cephfs at the same time. It is
+    very tricky to keep QuotaTree consistant with the global FS
+    hiberarchy. This patch kills the quota tree.
+    
+    After removing the quota tree, we traverse inode's path to
+    find quota root.
+    
+    Fixes: http://tracker.ceph.com/issues/16066
+    Fixes: http://tracker.ceph.com/issues/16067
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 161954bbfeb13d5588668637d5258221948128ea)
+
+commit 47605a2fdde65265e3c4dc60aac206c6ae712be5
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri May 6 09:46:06 2016 -0400
+
+    msg/msg_types: update sockaddr, sockaddr_storage accessors
+    
+    Return sockaddr_storage by value.  New sockaddr-based accessors.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit 0038d362d38181bd348881bc0fbe717211eb8d15)
+
+commit d5c77c8ff6ad0e4b4242669e4f218f607f433310
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Fri May 20 15:47:23 2016 +0530
+
+    osd: add peer_addr in heartbeat_check log message
+    
+    Fixes: http://tracker.ceph.com/issues/16337
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit f235b9cc1541d33c210e2d56e8061e8908b91ac9)
+
+commit 5ffee353cb947bfb79925ca4d62daec248c3ba4a
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Fri May 20 23:09:46 2016 +0530
+
+    mon : Display full flag in ceph status if
+    full flag is set
+    
+    Fixes: http://tracker.ceph.com/issues/16069
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit 6b1c894b2b083bf6cead21e9f96d304b2eb7887d)
diff --git a/doc/changelog/v10.2.5.txt b/doc/changelog/v10.2.5.txt
new file mode 100644
index 000000000..53d3f1006
--- /dev/null
+++ b/doc/changelog/v10.2.5.txt
@@ -0,0 +1,37 @@
+commit c461ee19ecbc0c5c330aca20f7392c9a00730367
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Fri Dec 9 20:08:24 2016 +0000
+
+    10.2.5
+
+commit 2c7d2b99d3be6f23188af4bf95f2d6f4a4fb516a
+Merge: 9411351 f7abffe
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Dec 8 07:55:27 2016 -0800
+
+    Merge pull request #12376 from liewegas/wip-msgr-eagain-loop-jewel
+    
+    msg/simple/Pipe: avoid returning 0 on poll timeout
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit f7abffec751e454d119df273dc6e49e5f7106078
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 7 18:25:55 2016 -0600
+
+    msg/simple/Pipe: avoid returning 0 on poll timeout
+    
+    If poll times out it will return 0 (no data to read on socket).  In
+    165e5abdbf6311974d4001e43982b83d06f9e0cc we changed tcp_read_wait from
+    returning -1 to returning -errno, which means we return 0 instead of -1
+    in this case.
+    
+    This makes tcp_read() get into an infinite loop by repeatedly trying to
+    read from the socket and getting EAGAIN.
+    
+    Fix by explicitly checking for a 0 return from poll(2) and returning
+    EAGAIN in that case.
+    
+    Fixes: http://tracker.ceph.com/issues/18184
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6c3d015c6854a12cda40673848813d968ff6afae)
diff --git a/doc/changelog/v10.2.6.txt b/doc/changelog/v10.2.6.txt
new file mode 100644
index 000000000..4628c7cad
--- /dev/null
+++ b/doc/changelog/v10.2.6.txt
@@ -0,0 +1,5165 @@
+commit 656b5b63ed7c43bd014bcafd81b001959d5f089f
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Tue Mar 7 13:29:40 2017 +0000
+
+    10.2.6
+
+commit 420a9a0796e327676cb704bc18b2020f9941d7b6
+Merge: 9267dec 88f4895
+Author: Zack Cerza <zack@cerza.org>
+Date:   Thu Mar 2 14:00:13 2017 -0700
+
+    Merge pull request #13749 from zmc/wip-openstack-volumes-jewel
+    
+    qa/suites/upgrade/hammer-x: Add some volumes
+
+commit 88f48955900e9401680f5434f9a162907b146e74
+Author: Zack Cerza <zack@redhat.com>
+Date:   Thu Mar 2 10:57:54 2017 -0700
+
+    qa/suites/upgrade/hammer-x: Add some volumes
+    
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+
+commit 9267dec7e794afb67d0904bf9795e9ac7a0b2a78
+Merge: c66683d 0c242d1
+Author: Zack Cerza <zack@cerza.org>
+Date:   Wed Mar 1 16:35:37 2017 -0700
+
+    Merge pull request #13708 from zmc/wip-openstack-volumes-jewel
+    
+    qa/suites/ceph-deploy: Drop OpenStack volume count
+
+commit 0c242d18c4f4b2009467d6cbc0ef26a08a9d9865
+Author: Zack Cerza <zack@redhat.com>
+Date:   Tue Feb 28 13:07:15 2017 -0700
+
+    qa/suites/ceph-deploy: Drop OpenStack volume count
+    
+    Looks like we only need two per node, since there is only one OSD per
+    node, and ceph-deploy wants two disks per OSD to account for the
+    journal.
+    
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit 87072e277c9ef259c9ee2ae1f761e252aa216713)
+
+commit c66683d908de4a747de9ddffee353206f50bf50a
+Merge: d5a778c ccd0265
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Mar 1 07:26:30 2017 -0800
+
+    Merge pull request #13710 from tmuthamizhan/wip-mv-centos-version
+    
+    qa: replace centos 7.2 with centos 7.3
+    
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit d5a778ced47a7601cfcea79b660ba6367e740e01
+Merge: 7187b61 2cbec5b
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Feb 28 16:05:32 2017 -0800
+
+    Merge pull request #13705 from ceph/wip_remove_dumpling_jewel
+    
+    qa: Removed dumplin test 13234.yaml as not needed anymore
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit ccd0265a0540e4c4e9423b177b04fae60716bc1c
+Author: Tamil Muthamizhan <tmuthami@redhat.com>
+Date:   Tue Feb 28 16:02:57 2017 -0800
+
+    qa: replace centos 7.2 with centos 7.3
+    
+    s/centos_7.2/centos_7.3
+    
+    Signed-off-by: Tamil Muthamizhan <tmuthami@redhat.com>
+
+commit 2cbec5b909920a05afd34e83c6fa00c5ece4a991
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Tue Feb 28 15:17:54 2017 -0800
+
+    Removed dumplin test 13234.yaml as not needed anymore
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 7187b617ccdc7ba70134ba468eb4c3848a555a41
+Merge: ac3ba2a 771e1d9
+Author: Zack Cerza <zack@cerza.org>
+Date:   Mon Feb 27 12:23:18 2017 -0700
+
+    Merge pull request #13674 from zmc/wip-openstack-volumes-jewel
+    
+    qa/suites/{ceph-ansible,rest}: OpenStack volumes
+
+commit 771e1d98a2b5cd437dcf68e03d504f3dec4f2e06
+Author: Zack Cerza <zack@redhat.com>
+Date:   Mon Feb 27 09:14:41 2017 -0700
+
+    qa/suites/rest: Openstack volumes
+    
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit 99d942145f4206c00aca30c0bb74f0edc4bac798)
+
+commit a18640fcd7df45eea0baf2b0cfb1d717f982490b
+Author: Zack Cerza <zack@redhat.com>
+Date:   Mon Feb 27 09:06:26 2017 -0700
+
+    qa/suites/ceph-ansible: Openstack volumes
+    
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit 964b983bdbd412311bce56184d12d1b6d43c7f28)
+
+commit ac3ba2adcd21ac011ad556ac4506623e61fbe696
+Merge: 016238b 841688b
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 24 17:22:17 2017 -0700
+
+    Merge pull request #13642 from zmc/wip-fs-openstack-jewel
+    
+    qa/suites/fs: Add openstack volume configuration
+
+commit 841688bb2e46bade3ede4a902c0b42a99e6a62f6
+Author: Zack Cerza <zack@redhat.com>
+Date:   Fri Feb 24 15:25:19 2017 -0700
+
+    qa/suites/fs: Add openstack volume configuration
+    
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit b076d89a3f1fbad7d477913812b2e17529abeacf)
+
+commit 016238b06938064afb6debb2c6dd753037c05279
+Merge: 48bc625 9778743
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 24 14:44:10 2017 -0700
+
+    Merge pull request #13639 from zmc/wip-openstack-volumes-jewel
+    
+    qa/suites/{knfs,hadoop,samba}: OpenStack volume configuration
+
+commit 48bc625c721f426b29eed4f19f030f1243048649
+Merge: f5643f8 ba35859
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 24 14:43:40 2017 -0700
+
+    Merge pull request #13636 from zmc/wip-kcephfs-openstack-jewel
+    
+    qa/suites/kcephfs: Openstack volume configuration
+
+commit 9778743547fb9337e98e636fb7ad801fe1ff39ca
+Author: Zack Cerza <zack@redhat.com>
+Date:   Fri Feb 24 13:46:54 2017 -0700
+
+    qa/suites/samba: Openstack volume configuration
+    
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit e0296d706422ea4dc01d84f8786f6f7104c3d996)
+
+commit cd1e8ef4fc2ccfe5abc11b6282a496185af49455
+Author: Zack Cerza <zack@redhat.com>
+Date:   Fri Feb 24 13:45:18 2017 -0700
+
+    qa/suites/hadoop: Openstack volume configuration
+    
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit 3fef0a49da2ccfdceba7b98e9096be8305da1111)
+
+commit ac7add155f0b787d6a7a55b597da84278b785ae8
+Author: Zack Cerza <zack@redhat.com>
+Date:   Fri Feb 24 13:44:33 2017 -0700
+
+    qa/suites/knfs: Add openstack volume configuration
+    
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit 62c6fd3371adf0f420c12d9c7e2b3a2a0c69256b)
+
+commit ba35859ef26b49ecab750cd36c87a98315e8c023
+Author: Zack Cerza <zack@redhat.com>
+Date:   Fri Feb 24 13:37:23 2017 -0700
+
+    qa/suites/kcephfs: Openstack volume configuration
+    
+    (cherry picked from commit ec6fb28eaf8e2db327e4afc115879a40c7664e07)
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+
+commit f5643f8818772a4778e2c553d483c7c6ab67ac18
+Merge: bfac1be aced718
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 24 12:15:41 2017 -0700
+
+    Merge pull request #13632 from zmc/wip-krbd-openstack-jewel
+    
+    qa/suites/krbd: Add openstack volume configuration
+
+commit bfac1bee09c90ec81b6d969ecaccbbe047a82604
+Merge: d9eaab4 94d5888
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 24 12:15:22 2017 -0700
+
+    Merge pull request #13612 from ceph/wip-rgw-openstack-jewel
+    
+    qa/suites/rgw: Add openstack volume configuration
+
+commit aced718032fdfedfdfad441a5761f26f04028af3
+Author: Zack Cerza <zack@redhat.com>
+Date:   Fri Feb 24 11:17:45 2017 -0700
+
+    qa/suites/krbd: Add openstack volume configuration
+    
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit 201b4d0d1e92bf95ac6a8f2951b664763030f12d)
+
+commit 94d5888cea0ddac8fa03b32225c937955d83dd86
+Author: Zack Cerza <zack@redhat.com>
+Date:   Thu Feb 23 10:14:05 2017 -0700
+
+    qa/suites/rgw: Add openstack volume configuration
+    
+    Without this, OSDs will fail to create on instances whose root fs isn't
+    xfs.
+    
+    (cherry picked from commit 8af4c35f9577ef5a88307ea5cbbe2561a473926c)
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+
+commit d9eaab456ff45ae88e83bd633f0c4efb5902bf07
+Merge: ecdfb7c bf3400f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Feb 23 16:32:20 2017 +0100
+
+    Merge pull request #13184 from smithfarm/wip-18720-jewel
+    
+    jewel: build/ops: systemd restarts Ceph Mon to quickly after failing to start
+
+commit ecdfb7ce0c29aa5eb954b3644e5be1544e018515
+Merge: 96b4cb8 5c328f0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 23 07:32:01 2017 +0100
+
+    Merge pull request #13240 from smithfarm/wip-18804-jewel
+    
+    jewel: tests: ignore bogus ceph-objectstore-tool error in ceph_manager
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 96b4cb8eab5e7531eeaf412cf4b10db5947bef70
+Merge: 27095f3 20e7502
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 23 07:30:30 2017 +0100
+
+    Merge pull request #13058 from wido/issue-18635-jewel
+    
+    jewel: systemd: Restart Mon after 10s in case of failure
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 27095f36b51a91ff5575bd42d12a16d2af11b731
+Merge: 0766a8b 67e7a90
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 23 07:28:16 2017 +0100
+
+    Merge pull request #13048 from SUSE/wip-18457-jewel
+    
+    jewel: selinux: Allow ceph to manage tmp files
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 0766a8bc604d0c4ee7a7d93427022105f5eb6425
+Merge: 9f36610 463e88e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 22 22:59:41 2017 +0100
+
+    Merge pull request #13131 from ovh/bp-osdmap-hammer-compat
+    
+    jewel: mon: OSDMonitor: clear jewel+ feature bits when talking to Hammer OSD
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 9f366100bb7c9882146def8b5ed9ce9eddd4db21
+Merge: 083e0f4 78d296b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 22 22:36:07 2017 +0100
+
+    Merge pull request #13255 from liewegas/wip-enxio-jewel
+    
+    jewel: osd: do not send ENXIO on misdirected op by default
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 083e0f4ac06f922c32b806d6d027f738c5057cae
+Merge: 04a8492 8a75f98
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Feb 21 21:40:14 2017 +0100
+
+    Merge pull request #13050 from SUSE/wip-18406-jewel
+    
+    jewel: tests: Cannot reserve CentOS 7.2 smithi machines
+
+commit 04a8492dfed7ec6f235cb201ab6de14201e27bbd
+Merge: d10a8e6 ebb2f73
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Feb 21 10:27:51 2017 +0100
+
+    Merge pull request #13459 from ukernel/jewel-18708
+    
+    jewel: mds: fix incorrect assertion in Server::_dir_is_nonempty()
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit d10a8e6d927ea8171e29b7a7c10afcc2c10c5392
+Merge: a009942 cb7bb0c
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon Feb 20 10:20:58 2017 -0600
+
+    Merge pull request #13533 from smithfarm/wip-stress-split-ec
+    
+    jewel: qa/suites/upgrade/hammer-x: break stress split ec symlinks
+
+commit a00994299fb360c1c086baad3b65e6889579a4ba
+Merge: 40eedcc 1e75e23
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon Feb 20 10:20:51 2017 -0600
+
+    Merge pull request #13222 from liewegas/wip-hammer-x-jewel
+    
+    jewel: tests: qa/suites/upgrade/hammer-x/stress-split: finish thrashing before final upgrade
+
+commit 40eedcc9428abc8bc988594b6e624bbc3c732405
+Merge: 786b318 e9a6dec
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 20 16:42:59 2017 +0100
+
+    Merge pull request #13273 from asheplyakov/jewel-bp-18773
+    
+    jewel: rgw: fix period update crash
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 786b31877fcf1c13bfb5034ca92e5044d5d6a78f
+Merge: 7566139 b06d6f5
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 20 16:41:56 2017 +0100
+
+    Merge pull request #13341 from dreamhost/wip-17550-jewel
+    
+    jewel: rgw: bucket resharding
+    
+    Reviewed-by: Yehuda Sadeh <ysadehwe@redhat.com>
+
+commit 7566139391f43616e7847e806c657c527e7a098d
+Merge: b3bb9cc b1d6c2e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 20 16:41:29 2017 +0100
+
+    Merge pull request #13004 from asheplyakov/jewel-bp-18563
+    
+    jewel: rgw: leak from RGWMetaSyncShardCR::incremental_sync
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit cb7bb0c49315cc66130336625ede92c70beaec74
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Feb 20 11:44:49 2017 +0100
+
+    tests: reduce stress-split-erasure-code-x86_64 dependency on stress-split
+    
+    This patch breaks the symlinks for two files from stress-split that
+    are being changed by https://github.com/ceph/ceph/pull/13222 in a way
+    that would break this test otherwise.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit b3bb9cc382e11f535a874675d65015829d1a3254
+Merge: 7d6e915 7cf37f6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 20 11:48:26 2017 +0100
+
+    Merge pull request #12754 from Abhishekvrshny/wip-18285-jewel
+    
+    jewel: rbd: partition func should be enabled When load nbd.ko for rbd-nbd
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 7d6e915a2f7d78741d8fcdfdc7d44e71236ac48d
+Merge: 7e844b6 401271e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 20 11:45:29 2017 +0100
+
+    Merge pull request #13183 from smithfarm/wip-17313-jewel
+    
+    jewel: build/ops: add ldap lib to rgw lib deps based on build config
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 8ae18863d013655e05a46e53de53f9f97fc4d42d
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Feb 20 11:44:49 2017 +0100
+
+    tests: reduce stress-split-erasure-code dependency on stress-split
+    
+    This patch breaks the symlinks for two files from stress-split that
+    are being changed by https://github.com/ceph/ceph/pull/13222 in a way
+    that would break this test otherwise.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 7e844b66848ba6e911aa357ebfca0ebf20aa46d3
+Merge: 5cda572 4d4b38e
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 17 14:43:00 2017 -0800
+
+    Merge pull request #13502 from tmuthamizhan/wip-ansible-rm-trusty
+    
+    qa: drop ubuntu trusty support
+    
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit 4d4b38eca81f7b57e3d3b31e1c13e7ab0ba5b30f
+Author: Tamil Muthamizhan <tmuthami@redhat.com>
+Date:   Fri Feb 17 21:06:43 2017 +0000
+
+    qa: drop ubuntu trusty support
+    
+    ceph-ansible dropped support for OS that doesnt support systemd
+    
+    Signed-off-by: Tamil Muthamizhan <tmuthami@redhat.com>
+
+commit 5cda5722e05784163190b6110dcc2dd164142e90
+Merge: 87f8341 260801f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 16 22:27:53 2017 +0100
+
+    Merge pull request #13276 from rzarzynski/wip-rgw-18364-2-jewel
+    
+    jewel: rgw: be aware abount tenants on cls_user_bucket -> rgw_bucket conversion
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 87f8341909e791e420a4b9344810e3840c1e83cc
+Merge: 3d9a584 a0ee8b9
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 16 16:30:56 2017 +0100
+
+    Merge pull request #12729 from liupan1111/wip-fix-disable-rgw-compile-error
+    
+    jewel: build/ops: fixed compilation error when --with-radowsgw=no
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 3d9a584bf0f335caf65265345faf2a940f901519
+Merge: 9f1e1ce 89248e1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 16 15:43:32 2017 +0100
+
+    Merge pull request #12239 from liupan1111/wip-fix-makefile-bug
+    
+    jewel: build/ops: fixed the issue when --disable-server, compilation fails.
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit ebb2f73e48092c8d2ecf18c75ce70f1a88c9bb22
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jan 18 16:23:49 2017 +0800
+
+    mds: fix incorrect assertion in Server::_dir_is_nonempty()
+    
+    when filelock is in XLOCKDONE state. client of xlocker can rdlock
+    the filelock. In that case, only client of xlocker can read the lock.
+    
+    Fixes: http://tracker.ceph.com/issues/18708
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit fe4ab52b30079a785be053a9fd0197d6990737fe)
+
+commit 9f1e1ce394c7779420ef27bda793bc719cdb9096
+Merge: 792a0c8 36ff758
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 15 23:39:11 2017 +0100
+
+    Merge pull request #12380 from dachary/wip-18183-jewel
+    
+    jewel: cephfs metadata pool: deep-scrub error omap_digest != best guess omap_digest
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 792a0c8160a1e239f7388cb1ca5be412d850aa82
+Merge: f95cbe0 3eff1ac
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 15 17:08:26 2017 +0100
+
+    Merge pull request #13130 from rjfd/wip-18608-jewel
+    
+    jewel: librbd: allow to open an image without opening parent image
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit f95cbe064639d31c218079d93c5831821e7cd13c
+Merge: 49b16ee 214ce1b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 15 17:06:53 2017 +0100
+
+    Merge pull request #13233 from smithfarm/wip-18556-jewel
+    
+    jewel: rbd: Potential race when removing two-way mirroring image
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 49b16eed6bf5303667df6657238735b47f573ebb
+Merge: cd9e5df 8742203
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 15 10:48:21 2017 +0100
+
+    Merge pull request #13113 from jcsp/wip-18679
+    
+    jewel: mds: finish clientreplay requests before requesting active state
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit cd9e5df1593f999c943376b82de49a3eba291f8c
+Merge: 3192ef6 21622c1
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 15 10:47:59 2017 +0100
+
+    Merge pull request #13139 from jcsp/wip-18100
+    
+    jewel: mds: fix MDSMap upgrade decoding
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 3192ef6a034bf39becead5f87a0e48651fcab705
+Merge: e84b3f8 384e5c0
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 15 10:18:04 2017 +0100
+
+    Merge pull request #12490 from linuxbox2/jewel-rgw-header
+    
+    jewel: rgw: add option to log custom HTTP headers (rgw_log_http_headers)
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit e84b3f8e073ec02f7a45387768490b90d6d777b4
+Merge: 9b4175e d7a479c4
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 15 10:14:23 2017 +0100
+
+    Merge pull request #12079 from rzarzynski/wip-rgw-17961
+    
+    jewel: rgw: TempURL properly handles accounts created with the implicit tenant
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 9b4175e491286f0578bcba5152a72e0e3b934051
+Merge: 2420d47 732405e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 15 10:13:32 2017 +0100
+
+    Merge pull request #12044 from Abhishekvrshny/wip-17886-jewel
+    
+    jewel: rgw: multisite: ECANCELED & 500 error on bucket delete
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 2420d474a8d22f3237b9fb28a78f0ff3f168ba34
+Merge: 81efc40 256b850
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 15 00:39:48 2017 +0100
+
+    Merge pull request #13422 from smithfarm/wip-fix-hammer-x-fix
+    
+    tests: remove extra indentation in exec block
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 81efc40b11fc0f146f5d2388033333aeb9ad9d56
+Merge: 994df97 4824ad2
+Author: Tamilarasi Muthamizhan <tmuthami@redhat.com>
+Date:   Tue Feb 14 13:59:58 2017 -0800
+
+    Merge pull request #13364 from tmuthamizhan/wip-ansible-jewel
+    
+    qa: ceph-ansible smoke suite modified for jewel
+
+commit 256b850fc1a80f71b0b8365c6d66abea84482f9e
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Feb 14 22:13:37 2017 +0100
+
+    tests: remove extra indentation in exec block
+    
+    The exec block was indented by an extra space, causing
+    
+    line 439, in parse_block_mapping_key
+        "expected <block end>, but found %r" % token.id, token.start_mark)
+    yaml.parser.ParserError: while parsing a block mapping
+      in "<string>", line 111, column 3:
+          sequential:
+          ^
+    expected <block end>, but found '<block sequence start>'
+      in "<string>", line 126, column 4:
+           - exec:
+           ^
+    
+    when trying to run upgrade/hammer-x
+    
+    Reported-by: Yuri Weinstein <yweins@redhat.com>
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 994df972dd64fc39ef7acce1acfff0e9e8440e1f
+Merge: 9bf8dcf 898702d
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon Feb 13 18:44:37 2017 -0600
+
+    Merge pull request #13404 from smithfarm/wip-fix-hammer-x
+    
+    tests: add require_jewel_osds to upgrade/hammer-x/tiering
+    
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit 898702d58f3e26c41713e44625fcf357841f52af
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Feb 13 22:04:17 2017 +0100
+
+    tests: add require_jewel_osds to upgrade/hammer-x/tiering
+    
+    Without this, the test tends to fail, but sometimes passes (apparently because
+    the "wait-for-healthy: true" in the last ceph.restart is racy - HEALTH_OK is
+    received before the MONs notice that all OSDs are running jewel without
+    require_jewel_osds.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 214ce1b96c112c193614aeeabaa278bd1004f6b2
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Aug 31 22:49:36 2016 +0300
+
+    librbd: async method to check journal tag owner
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 953561f84140efcc870ec4a8f1ecb732b8e99409)
+    
+    Conflicts:
+    	src/librbd/Journal.cc - trivial context difference (master does not
+                                    have "class SafeTimerSingleton"),
+                                    added "typedef ::journal::Journaler Journaler;" in
+                                    C_IsTagOwner template to fix build failure
+
+commit 5723b932cfb12ed67cb54347a127c5f3539d0bcf
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Jan 9 10:40:46 2017 +0100
+
+    rbd-mirror: check image mirroring state when bootstrapping
+    
+    Fixes: http://tracker.ceph.com/issues/18447
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 5fc5a8ac895524f05eed6e7db20b0dda3a8cb60f)
+
+commit 8361a60c3cb272d2d4895840dadbbc6c50294f51
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Jan 9 09:23:19 2017 +0100
+
+    rbd-mirror: async request to test if image is primary
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 0a1cb35caacdf85029f31a0364dc07a5d7462f5f)
+    
+    Conflicts:
+    	src/tools/rbd_mirror/CMakeLists.txt - file doesn't exist in jewel
+                (ported the patch to src/tools/Makefile-client.am)
+    	src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc - Journal is
+                 implemented by a typedef in jewel
+    	src/tools/rbd_mirror/image_replayer/OpenLocalImageRequest.cc -
+                 no compat.h include in jewel, Journal is implemented by a typedef
+                 in jewel
+
+commit 9bf8dcf914e76a340fed29d2b46346f8ada6d0c7
+Merge: 748f75c 7515a77
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 13 08:20:53 2017 +0100
+
+    Merge pull request #13331 from smithfarm/wip-18869-jewel
+    
+    jewel: tests: SUSE yaml facets in qa/distros/all are out of date
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek@suse.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 748f75cd845e3b33687d9d235508f9bc7ebdcefb
+Merge: c27aba8 21ded74
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Mon Feb 13 14:52:59 2017 +0800
+
+    Merge pull request #11410 from dachary/wip-17334-jewel
+    
+    jewel: crushtool --compile is create output despite of missing item
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 21ded748e7bbf2c02b3bf381cf796afe99a62ce0
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Sep 21 22:58:49 2016 +0800
+
+    test/cli/crushtool: fix the test of compile-decompile-recompile.t
+    
+    should read the map from $TESTDIR,
+    it's a regression introduced by b2c0a07
+    
+    Fixes: http://tracker.ceph.com/issues/17306
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 404043980d7882f320f48fbaed4b18f31bb15eb7)
+
+commit d92738c0c9491c97fd189ff49ca57427316b9eb7
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 20 14:10:16 2016 +0800
+
+    crush/CrushCompiler: error out as long as parse fails
+    
+    do not output compiled crush map if anything goes wrong when parsing
+    crush map.
+    
+    Fixes: http://tracker.ceph.com/issues/17306
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b2c0a079dc074c907e1bc429861230096534f597)
+
+commit 4824ad27345f32ed0043c2cb20fe5fc7e019555a
+Author: Tamil Muthamizhan <tmuthami@redhat.com>
+Date:   Fri Feb 10 12:37:51 2017 -0800
+
+    qa: ceph-ansible smoke suite modified for jewel
+    
+    Signed-off-by: Tamil Muthamizhan <tmuthami@redhat.com>
+
+commit c27aba8f723b013baeb0704fe84c6956d1cb005d
+Merge: 8c87d09 5ed454d
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Feb 10 15:04:02 2017 -0500
+
+    Merge pull request #13358 from linuxbox2/wip-jewel-marker
+    
+    jewel: rgw: fix use of marker in List::list_objects()
+
+commit 5ed454dd8f470749ef661782a8f4e866ae8193e2
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jan 26 16:26:42 2017 -0800
+
+    rgw: fix use of marker in List::list_objects()
+    
+    Fixes: http://tracker.ceph.com/issues/18331
+    
+    List marker is an index key, so treat it as such. This
+    fixes infinite loop in orphans find command.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit a5d1fa0587184f43c69d8e03114b58d43f320781)
+
+commit b06d6f5f8fb6693dcf4f480d1f80b1081cd63305
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Thu Feb 9 14:38:02 2017 -0800
+
+    Backport bucket reshard to jewel.
+    
+    Fixes: http://tracker.ceph.com/issues/17756
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit 25af6e5c2448d2478ac3d8def6141fcbfb086f3c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 3 17:00:41 2016 -0700
+
+    rgw_admin: add a few admin commands to the usage
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 91ed1216d7465a42d11546cb967c70997ea5b1d8)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit b429331f4fb22e845edcd526b618b10c82db6286
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Oct 3 16:43:44 2016 -0700
+
+    rgw_admin: add bi purge command
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit aaf0d213eb39192ceb252c9c7db68c1a48ba1272)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit eb65394363676629726aed2aac8182a745ba96e6
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Sep 28 12:54:47 2016 -0700
+
+    rgw: bucket resharding, adjust logging
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit e2b8dc6113e2625bdf65ea6f2c42510229d04c87)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit e197ec0312cb07d576c0bf045b803454702ccc81
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Sep 28 10:41:45 2016 -0700
+
+    cls/rgw: bi_list() fix is_truncated returned param
+    
+    is_truncated was never set. Also, make sure that we don't return
+    more entries than requested.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 47f422a4e0382d53023af6f651433011606b8625)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit 81daefa473a4e6826be5afa4dae5ea63d66bd1be
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 27 15:13:37 2016 -0700
+
+    rgw_admin: require --yes-i-really-mean-it for bucket reshard
+    
+    in the case where num shards are less or equal to current bucket
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 75da4375ee0e36c58f96cbf2920c3b9aadf12733)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit 4adb247e56e8e77cbdfecca8f90240edbd04742f
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 27 14:35:31 2016 -0700
+
+    rgw_admin: better bucket reshard logging
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit ac88b5d9e6bc3c8b59c500bf79a0e3daa923a47c)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit 139842d7451ef44cb0548676a818d3056e2e4ac6
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 27 14:11:41 2016 -0700
+
+    rgw: limit bucket reshard num shards to max possible
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d9c88afec4b52848f9ad8957bab5a86fdafecded)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit 7e4493a4cca99bfa817803c0255755e1cd330cbf
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 27 11:55:32 2016 -0700
+
+    rgw_admin: fix bi list command
+    
+    Changes scoping of json section, and push the --object param in so that
+    an object can be specified as a filter.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 716d096bbb8e836aefa6a451b799389d3bd85620)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit 1604b521730577fc6aa5345127381b1bfc5625b7
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 15:49:37 2016 -0700
+
+    rgw_admin: use aio operations for bucket resharding
+    
+    also created shards manager to make things slightly cleaner
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 97e7ee9ca213ccf4b8f537e02125bd0c4ef24103)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit cd2e35e84599d25e94f28fad8e4a743883fec94d
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 10:45:17 2016 -0700
+
+    rgw: bucket reshard updates stats
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit dd712384ffe72ee23cbe0a20d7400aff7fb779a8)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit 678dac9289ce52f1e5f13e603b5858a3b867c216
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 09:52:29 2016 -0700
+
+    cls/rgw: add bucket_update_stats method
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 4cc7d3a33a28602b45ec04ff5384e6bc62b376cb)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit d39eac038266a61a6081c820360c2165279bd8c1
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat Sep 24 10:46:36 2016 -0700
+
+    rgw_admin: reshard also links to new bucket instance
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 094fe3f0cfeb27b32abfc93b07054b60de363a20)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit 29c9e9872c2cef6b6e874bf95c797a72a580e84c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat Sep 24 10:33:57 2016 -0700
+
+    rgw: rgw_link_bucket, use correct bucket structure for entry point
+    
+    The bucket structure might be different than the one that we were using
+    before.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f2d9fc015556d6d70e596f75c382cc7157add411)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit b40ce2738964c6fc1c9c390af18758b16820cdb9
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat Sep 24 10:01:00 2016 -0700
+
+    radosgw-admin: bucket reshard needs --num-shards to be specified
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d0569f913340cb251c1a49f1e470b176d8b34346)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit d9df7451a70a4fe64443e7f4669e995c54d74361
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Sat Sep 24 05:49:37 2016 -0700
+
+    cls/rgw: fix bi_list objclass command
+    
+    was filtering entries, even if filter was not specified, and need to
+    set boundary for plain entries. Also, list_instance_entries() was not
+    working correctly, and added list_olh_entries().
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f87c4b2c2a6ecadaf9f0e0cfef4b6061878a023b)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit 4892def3643555720619b987bc1ccb508cdd8ac7
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 16:09:34 2016 -0700
+
+    rgw_admin: bucket rehsrading, initial work
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit d8c5931bceaa42ad47ae5ad9b2c32bb6c321484a)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    
+    Conflicts:
+    	src/rgw/rgw_admin.cc
+
+commit c1cf61f3612844fc644a832f7c61f716f28ecb46
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 16:09:15 2016 -0700
+
+    rgw: utilities to support raw bucket index operations
+    
+    and other related changes.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit b01e732f3d597670f4f781be3db81786e63d5053)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    
+    Conflicts:
+    	src/rgw/rgw_rados.cc
+
+commit 0ce2a125f4d400abaf2a65ff90d1d40ee000aea9
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 16:06:17 2016 -0700
+
+    rgw: use bucket_info.bucket_id instead of marker where needed
+    
+    We used to use these interchangeably, but they actually have diffent meaning.
+    The marker is the prefix we assign to the objects in this bucket, whereas
+    the bucket_id is the bucket instance's id. These used to hold the same
+    value, but with bucket resharding it's not going to be true anymore.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 94afaf722fc8289af6e75025ba7d783f11c4b7d0)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit 5fe58c5bf22ad593d650d68b059d462be49214ed
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Sep 26 16:01:39 2016 -0700
+
+    cls/rgw: utilities to support raw bucket index operations
+    
+    New flavour of bi_put() call, and a function to extract key off
+    a raw bi entry.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 9b3a2a40a5732689be98a940f7e6c3c4a6e73c53)
+    See: http://tracker.ceph.com/issues/17556
+    See: https://github.com/ceph/ceph/pull/11368
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit 7515a77bf1d06ec7cea448adecc56e50c417540f
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Feb 8 21:23:54 2017 +0100
+
+    tests: drop buildpackages.py
+    
+    The buildpackages suite has been moved to teuthology. This cleans up a file
+    that was left behind by https://github.com/ceph/ceph/pull/13297
+    
+    Fixes: http://tracker.ceph.com/issues/18846
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 6b7443fb50c117ee7f20d53bbc7530bb0eb7ebd5)
+
+commit 36f96f40e7ce9abf1fc21878c91365d300dee281
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Feb 8 15:27:00 2017 +0100
+
+    tests: update SUSE yaml facets in qa/distros/all
+    
+    Fixes: http://tracker.ceph.com/issues/18856
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 0bd56e871a8549d4b0b1211f09dad2d1120fb606)
+
+commit 8c87d0927447233697d623c995163fde25524fa5
+Merge: a00efd8 bb3c594
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 8 13:16:25 2017 +0100
+
+    Merge pull request #13299 from dachary/wip-18848-jewel
+    
+    jewel: tests: remove qa/suites/buildpackages
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.cz>
+
+commit 7cf37f68e14935d6d6c3d540ab5fd6b7a2ebb605
+Author: Pan Liu <pan.liu@istuary.com>
+Date:   Tue Dec 6 21:04:03 2016 +0800
+
+    rbd-nbd: support partition for rbd-nbd mapped raw block device.
+    
+    Fixes: http://tracker.ceph.com/issues/18115
+    Signed-off-by: Pan Liu pan.liu@istuary.com
+    (cherry picked from commit 42645a301869b08b4be860fcac491ae4189b313a)
+    
+    Conflicts:
+    	src/tools/rbd_nbd/rbd-nbd.cc
+    		Removed exclusive option
+
+commit bb3c5941d15f9e6ae35c51334f0774dc3ef17f64
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 7 18:33:29 2017 +0100
+
+    buildpackages: remove because it does not belong
+    
+    It should live in teuthology, not in Ceph. And it is currently broken:
+    there is no need to keep it around.
+    
+    Fixes: http://tracker.ceph.com/issues/18846
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 5a43f8d57925da227c95480501ceec10a29395d8)
+    
+    Conflicts:
+    	qa/tasks/buildpackages/common.sh (remove)
+    	qa/tasks/buildpackages/make-rpm.sh (remove)
+
+commit 5c328f0ec51f435e51357f015bb088a450277ece
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Feb 6 18:43:49 2017 +0100
+
+    tests: fix regression in qa/tasks/ceph_master.py
+    
+    https://github.com/ceph/ceph/pull/13194 introduced a regression:
+    
+    2017-02-06T16:14:23.162 INFO:tasks.thrashosds.thrasher:Traceback (most recent call last):
+      File "/home/teuthworker/src/github.com_ceph_ceph_master/qa/tasks/ceph_manager.py", line 722, in wrapper
+        return func(self)
+      File "/home/teuthworker/src/github.com_ceph_ceph_master/qa/tasks/ceph_manager.py", line 839, in do_thrash
+        self.choose_action()()
+      File "/home/teuthworker/src/github.com_ceph_ceph_master/qa/tasks/ceph_manager.py", line 305, in kill_osd
+        output = proc.stderr.getvalue()
+    AttributeError: 'NoneType' object has no attribute 'getvalue'
+    
+    This is because the original patch failed to pass "stderr=StringIO()" to run().
+    
+    Fixes: http://tracker.ceph.com/issues/16263
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit db2582e25e390fcaf75952eb59a73dcff643f49c)
+
+commit a00efd8d2d22aaca89747f4db818866b9d5fc982
+Merge: 84a0fdb 0cf7a61
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Feb 6 16:31:06 2017 -0800
+
+    Merge pull request #13280 from athanatos/wip-revert-jewel-18581
+    
+    Revert "Merge pull request #12978 from asheplyakov/jewel-18581"
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 0cf7a6133ee0d4609242d94088dd77e83665aa93
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Feb 6 10:20:55 2017 -0800
+
+    Revert "Merge pull request #12978 from asheplyakov/jewel-18581"
+    
+    See: http://tracker.ceph.com/issues/18809
+    
+    This reverts commit 8e69580c97622abfcbda73f92d9b6b6780be031f, reversing
+    changes made to c05730ceb3387fb43c35937f0506297a34a44452.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+
+commit 260801f9bb78169648c3f2052ebc3e1328113367
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed Feb 1 19:05:50 2017 +0100
+
+    rgw: be aware abount tenants on cls_user_bucket -> rgw_bucket conversion.
+    
+    Fixes: http://tracker.ceph.com/issues/18364
+    Fixes: http://tracker.ceph.com/issues/16355
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 871e1f51afe9d6c8b88debc07460b4316121f999)
+
+commit e9a6dec55b894b34e3278dbf6b226063fd461cd5
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 22 15:05:30 2017 +0200
+
+    rgw: add check for update return value
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 125955e0625461065dc4755b900e51c3598cadb4)
+
+commit a27f6a72ec836af1672e8aa344fb80d067ed20a8
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 22 14:42:14 2017 +0200
+
+    rgw: we need to reinit the zonegroup after assignment to avoid invalid cct and store
+    
+    Fixes: http://tracker.ceph.com/issues/18631
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit ac9a7565ddf801121f22476cf3f66668f311833e)
+
+commit 006140adf7413ec3d482971bb44bfc3ffada2fbf
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 22 14:40:16 2017 +0200
+
+    rgw: fix init_zg_from_period when default zone is not set as default
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 5393077e07bed45b9fc007591d365f1229d3e815)
+
+commit 78d296b076a284e35ef4bae931756e7d17a5c09d
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Feb 3 17:40:29 2017 -0500
+
+    osd: do not send ENXIO on misdirected op by default
+    
+    In practice this tends to get bubbled up the stack as an error on
+    the caller, and they usually do not handle it properly.  For example,
+    with librbd, this turns into EIO and break the VM.
+    
+    Instead, this will manifest as a hung op on the client.  That is
+    also not ideal, but given that the root cause here is generally a
+    bug, it's not clear what else would be better.
+    
+    We already log an error in the cluster log, so teuthology runs will
+    continue to fail.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 923e7f5ce5ed437af15e178299a61029ff48e4a2)
+    
+    # Conflicts:
+    #	PendingReleaseNotes
+    #	src/common/config_opts.h
+
+commit 30fb615c3686154275cf10a567a0c3ad961cbb50
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Jan 31 00:46:22 2017 +0100
+
+    tests: ignore bogus ceph-objectstore-tool error in ceph_manager
+    
+    Fixes: http://tracker.ceph.com/issues/16263
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 046e873026c59e733f1844b28ffdc030cbe57b36)
+
+commit 3eff1ac2d680d2f6ae1f7ff0d8fe6b6329a17522
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Tue Jan 10 15:11:19 2017 +0000
+
+    librbd: allow to open an image without opening parent image
+    
+    Fixes: http://tracker.ceph.com/issues/18325
+    
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 61af1c25015de087a2423811548d975dd7d430b4)
+    
+    Conflicts:
+      src/librbd/ImageState.cc - added missing arg to RefreshRequest::create
+      src/librbd/exclusive_lock/PostAcquireRequest.cc -
+                                            deleted, does not exist in jewel
+      src/librbd/image/OpenRequest.cc - added missing arg to
+                                        RefreshRequest::create
+      src/librbd/internal.cc - added missing arg to ImageState::open
+      src/librbd/librbd.cc - added missing arg to ImageState::open
+      src/test/librbd/exclusive_lock/test_mock_PostAcquireRequest.cc -
+                                            deleted, does not exist in jewel
+      src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc -
+                                            added missing arg to ImageState::open
+      src/test/rbd_mirror/test_PoolWatcher.cc - added missing arg to
+                                                ImageState::open
+
+commit bee1d2c24ab463a0f5c1e28cb70f6b89e0278d22
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Jan 9 15:02:02 2017 +0100
+
+    rbd-mirror: hold owner lock when testing if lock owner
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 82aa89668d666f434cd19ff444223017b5512c6a)
+
+commit 463e88e24378470ef405bdfb2eb823b04e017e5e
+Author: Piotr Dałek <piotr.dalek@corp.ovh.com>
+Date:   Fri Jan 20 16:07:10 2017 +0100
+
+    OSDMonitor: clear jewel+ feature bits when talking to Hammer OSD
+    
+    During upgrade from Hammer to Jewel, when upgrading MONs first and OSDs
+    last, Jewel MONs send OSDMaps with components in encoding version not
+    encodable by Hammer OSDs, generating extra load on MONs due to requests
+    for full OSDMap after failing the CRC check.
+    Fix this by not including CEPH_FEATURE_NEW_OSDOP_ENCODING (which
+    is responsible for encoding pg_pool_t in version 24 instead of 21) and
+    CEPH_FEATURE_CRUSH_TUNABLES5 (responsible for adding chooseleaf_stable
+    field into encoded CRUSH map) when CEPH_OSDMAP_REQUIRE_JEWEL flag
+    is not present.
+    Note that this issue applies only to upgrade from Hammer to Jewel,
+    because direct upgrade from Hammer to any other later release is not
+    supported. For that reason, there is no need to have this patch in any
+    release other than Jewel.
+    
+    Fixes: http://tracker.ceph.com/issues/18582
+    Signed-off-by: Piotr Dałek <piotr.dalek@corp.ovh.com>
+
+commit b1d6c2ebee2e6216cb6aa2afaa9e32ce5e571aa9
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jan 5 13:10:50 2017 -0500
+
+    rgw: RGWCloneMetaLogCoroutine uses RGWMetadataLogInfoCompletion
+    
+    Fixes: http://tracker.ceph.com/issues/18412
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 34a2edbf53f26a21e3206027dc61efc70d9c6aad)
+    
+    Minor changes:
+    	RGWCloneMetaLogCoroutine::state_read_shard_status(): use
+    	boost::intrusive_ptr ctor instead of reset() to be compatible
+    	with older boost versions (there's no bundled boost in jewel)
+
+commit 7ca400b9bd7b6fd382a9143b450de5a2dbcf057a
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jan 5 13:10:18 2017 -0500
+
+    rgw: expose completion for RGWMetadataLog::get_info_async()
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 7b3532e2ad0f478a1670ceb61fd68209df87aa8b)
+
+commit 335a7326a9e8416b114467096d938fafe6b09605
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Dec 21 14:32:04 2016 -0500
+
+    rgw: RGWMetaSyncShardCR drops stack refs on destruction
+    
+    if the coroutine is canceled before collect_children() can clean up
+    all of its child stacks, those stack refs will leak. store these
+    stacks as boost::intrusive_ptr so the ref is dropped automatically on
+    destruction
+    
+    Fixes: http://tracker.ceph.com/issues/18300
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 060fe72faf6a483a36d481207c6624c46a414231)
+
+commit e5646a0192c1968a542b42dbfe907565b604971e
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Sep 26 11:19:50 2016 -0400
+
+    rgw: librados aio wait_for_safe, not wait_for_complete
+    
+    We want to wait for the op to be durable, not ordered.
+    
+    Note that wait_for_safe works for read ops as well, despite a
+    name that implies an update.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2e447eafb34a7019ca15189d73a3ad3341640dc5)
+
+commit 84a0fdb8b464734a58bfc98edb010f1278fa25f2
+Merge: 96626c2 73cef9f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Feb 2 00:08:29 2017 +0100
+
+    Merge pull request #11872 from dachary/wip-17838-jewel
+    
+    jewel: leak in RGWFetchAllMetaCR
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 96626c2555b4bab7c293ee8a6f3382614c6306d1
+Merge: 3322556 73d2114
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 23:58:29 2017 +0100
+
+    Merge pull request #13182 from smithfarm/wip-18498-jewel
+    
+    jewel: rgw: Realm set does not create a new period
+
+commit 3322556f3ebabcd82be703bcffbd539454b191e0
+Merge: bc704b1 33c6ef5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 23:57:40 2017 +0100
+
+    Merge pull request #13180 from smithfarm/wip-18547-jewel
+    
+    jewel: rgw: multisite: segfault after changing value of rgw_data_log_num_shards
+
+commit bc704b1beb1897286e6eaa62f03ba8ddabb17ee9
+Merge: f46c125 8b124c8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 23:56:21 2017 +0100
+
+    Merge pull request #13177 from smithfarm/wip-18676-jewel
+    
+    jewel: rgw: librgw: objects created from s3 apis are not visible from nfs mount point
+
+commit f46c1259dcd63ce335715817e3aeb18def59c503
+Merge: a3cb5e6 2cb0307
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 23:55:25 2017 +0100
+
+    Merge pull request #13175 from smithfarm/wip-18684-jewel
+    
+    jewel: rgw: multisite: sync status reports master is on a different period
+
+commit a3cb5e61d18cc48d176b0630fb21ebc603472b4c
+Merge: 73ccbdc 0a47342
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 23:54:33 2017 +0100
+
+    Merge pull request #13173 from smithfarm/wip-18710-jewel
+    
+    jewel: rgw: slave zonegroup cannot enable the bucket versioning
+
+commit 73ccbdce630350a4601255483469959f00829c45
+Merge: 5d0f2f3 7e51bec
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 23:53:39 2017 +0100
+
+    Merge pull request #13171 from smithfarm/wip-18712-jewel
+    
+    jewel: rgw: radosgw-admin period update reverts deleted zonegroup
+
+commit 5d0f2f3ec893f5990860f901f40b870a6fd2e737
+Merge: 2f4990d e8f55f6
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 23:13:11 2017 +0100
+
+    Merge pull request #13001 from asheplyakov/jewel-bp/18559
+    
+    jewel: rgw multisite: fix ref counting of completions
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 2f4990d7a6cb355b059e5b8cfb97b6e371155dde
+Merge: a379c01 c21622d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 23:10:37 2017 +0100
+
+    Merge pull request #12997 from asheplyakov/jewel-bp-18569
+    
+    jewel: radosgw valgrind "invalid read size 4" RGWGetObj
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit a379c01ee916e73c12da4ba8ef1c7620be81e3b8
+Merge: 5834732 9a59ce9
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 23:02:14 2017 +0100
+
+    Merge pull request #12764 from SUSE/wip-17342-jewel
+    
+    jewel: tests: assertion failure in a radosgw-admin related task
+
+commit 5834732ad44d67b9414d782799f41e320d66bcf2
+Merge: 9486ccd 9a1258d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 23:01:02 2017 +0100
+
+    Merge pull request #12738 from SUSE/wip-18286-jewel
+    
+    jewel: rgw: multisite: coroutine deadlock in RGWMetaSyncCR after ECANCELED errors
+
+commit 9486ccd9bd13e3020947a0d669c2e70248f24bfb
+Merge: edf2e6e 5aa9387
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 23:00:12 2017 +0100
+
+    Merge pull request #12678 from Abhishekvrshny/wip-18348-jewel
+    
+    jewel: rgw ldap: enforce simple_bind w/LDAPv3 redux
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit edf2e6eaccf5d0fa848d56c0920d44c980a86298
+Merge: 2f087a3 d584f9e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 22:45:54 2017 +0100
+
+    Merge pull request #12622 from wido/jewel-15776
+    
+    jewel: rgw: log name instead of id for SystemMetaObj on failure
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek@suse.com>
+
+commit 2f087a37bc3d9ed86c4a6268bd21fe42c843cbb7
+Merge: 2d5f8fd 87a2a95
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 22:39:52 2017 +0100
+
+    Merge pull request #12542 from dreamhost/wip-17076-jewel
+    
+    jewel: rgw: Replacing '+' with "%20" in canonical uri for s3 v4 auth.
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 2d5f8fda98cba07ce48a3e85896c83b3459a5973
+Merge: 9897021 ddb5403
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 22:38:17 2017 +0100
+
+    Merge pull request #12428 from SUSE/wip-18216-jewel
+    
+    jewel: rgw-admin: missing command to modify placement targets
+
+commit 9897021fd29716974ba547814c35f8a606dce88f
+Merge: bf1e63b 26c87fd
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 22:36:19 2017 +0100
+
+    Merge pull request #12426 from SUSE/wip-18214-jewel
+    
+    jewel: add max_part and nbds_max options in rbd nbd map, in order to keep consistent with
+    
+    http://tracker.ceph.com/issues/17851#note-59
+
+commit bf1e63b6441af02404ad680462de5380e8714c43
+Merge: 9c83af8 8b10d3b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 22:34:15 2017 +0100
+
+    Merge pull request #12419 from SUSE/wip-18217-jewel
+    
+    jewel: rgw sends omap_getvals with (u64)-1 limit
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 9c83af87ee8716dd7785c2e21fcc8007a92f9c92
+Merge: c0e845c fe753db
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 22:32:24 2017 +0100
+
+    Merge pull request #12405 from jan--f/wip-18199-jewel
+    
+    jewel: build/ops: install-deps.sh based on /etc/os-release
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit c0e845c4f6b03f3c28c31ba2278be5b20e5be13c
+Merge: 45281fb 4eb7c73
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 22:29:24 2017 +0100
+
+    Merge pull request #12397 from linuxbox2/jewel-next
+    
+    jewel: rgw: do not abort when accept a CORS request with short origin
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 45281fb4d3ddca81d051cd776269cb111cc26bc1
+Merge: 1f85f84 7db6d1d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 22:15:35 2017 +0100
+
+    Merge pull request #12316 from dachary/wip-18101-jewel
+    
+    jewel: Add workaround for upgrade issues for older jewel versions
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 1f85f847b6a8413047891d6d46e838ad83d5a8b7
+Merge: 735be97 3839727
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 21:37:45 2017 +0100
+
+    Merge pull request #12890 from dillaman/wip-18453-jewel
+    
+    jewel: librbd: new API method to force break a peer's exclusive lock
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 1e75e23b9973c9c5203b3c3ed2cec35333b40d77
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Feb 1 15:24:50 2017 -0500
+
+    qa/suites/upgrade/hammer-x: wrap thrash and workloads
+    
+    We need the thrashing to stop before we do the final upgrade step.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 735be9772f0029eb6c7f9aef7d0469107897521a
+Merge: ce309b8 5400673
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 1 20:23:36 2017 +0100
+
+    Merge pull request #12302 from SUSE/wip-18135-jewel
+    
+    jewel: build/ops: add hostname sanity check to run-{c}make-check.sh
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit ce309b8493a3e69498d3495bbbdfa7b2c2e31cf5
+Merge: 4cbe0e3 478e40a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 16:18:04 2017 +0100
+
+    Merge pull request #12315 from dachary/wip-18102-jewel
+    
+    jewel: rgw: Unable to commit period zonegroup change
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 4cbe0e398ad9b46eefafbfd2d0202d0cbb65402a
+Merge: e371af6 b502b96
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 16:14:15 2017 +0100
+
+    Merge pull request #12314 from dachary/wip-18107-jewel
+    
+    jewel: multisite: failed assertion in 'radosgw-admin bucket sync status'
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit e371af6b7d17020c1bda09f0de28036a920303fd
+Merge: 056f8ac ece622d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 16:12:40 2017 +0100
+
+    Merge pull request #12313 from dachary/wip-18112-jewel
+    
+    jewel: rgw: multisite requests failing with '400 Bad Request' with civetweb 1.8
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 056f8ac054c82358ec4ea20782a7da327e6ad19e
+Merge: 41fcf74 d9c1d86
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 16:02:05 2017 +0100
+
+    Merge pull request #12258 from rzarzynski/wip-rgw-17931-jewel
+    
+    jewel: rgw: add support for the prefix parameter in account listing of Swift API
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 41fcf740ba9205c08a2995fd69dc29051ed81b6a
+Merge: 8a86bf5 a0b4e60
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 15:56:48 2017 +0100
+
+    Merge pull request #12156 from dachary/wip-17969-jewel
+    
+    jewel: rgw: multisite upgrade from hammer -> jewel ignores rgw_region_root_pool
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 8a86bf508503f7b22b47c6663b56f6fd7012df4c
+Merge: d71c75a d1df8f7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 15:34:45 2017 +0100
+
+    Merge pull request #11990 from asheplyakov/jewel-17908
+    
+    jewel: rgw: for the create_bucket api, if the input creation_time is zero, we …
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit d71c75ac6bc2ed576344e0f75fe99ce860b8914d
+Merge: 7d0c6de 6a3c10f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 15:31:21 2017 +0100
+
+    Merge pull request #11876 from dachary/wip-17839-jewel
+    
+    jewel: rgw: the value of total_time is wrong in the result of 'radosgw-admin log show' opt
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 7d0c6dee032c1254a2c78b589bb48bea335ca0f2
+Merge: a695e8d 3239ce8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 15:26:21 2017 +0100
+
+    Merge pull request #11868 from dachary/wip-17512-jewel
+    
+    jewel: multisite: metadata master can get the wrong value for 'oldest_log_period'
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit a695e8d167ada37260b4ef8ed4595f758fc4421b
+Merge: 4395560 0b7577e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 15:24:11 2017 +0100
+
+    Merge pull request #11866 from dachary/wip-17709-jewel
+    
+    jewel: rgw: multisite: coroutine deadlock assertion on error in FetchAllMetaCR
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 4395560d686cc7ccdc28eaba1f6cbadf57e1bba5
+Merge: 1bc9432 8d3b0e7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 15:21:33 2017 +0100
+
+    Merge pull request #11497 from rzarzynski/wip-rgw-slashinfo-jewel
+    
+    jewel: rgw: add suport for Swift-at-root dependent features of Swift API
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 1bc9432e744c42fae98c570f09ddc3f4606d2297
+Merge: 92d385e 6c1edcd
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 15:18:42 2017 +0100
+
+    Merge pull request #11477 from dachary/wip-17119-jewel
+    
+    jewel: rgw: multisite: assert(next) failed in RGWMetaSyncCR
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 92d385e8cb4cff91051464a8552ab5aed4cbfae4
+Merge: a61be36 d54b354
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 15:17:03 2017 +0100
+
+    Merge pull request #11476 from dachary/wip-17162-jewel
+    
+    jewel: rgw: multisite doesn't retry RGWFetchAllMetaCR on failed lease
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit a61be367615f7ef5f8101d0fbbe40615782d8c5c
+Merge: 5821e8d aa24a8f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 15:15:01 2017 +0100
+
+    Merge pull request #11470 from dachary/wip-17514-jewel
+    
+    jewel: rgw：bucket check remove _multipart_ prefix
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 5821e8d9c448f95f9a4b73e6353db8f7d2cfb534
+Merge: bb702bc 19a836b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 15:12:51 2017 +0100
+
+    Merge pull request #10661 from dachary/wip-16871-jewel
+    
+    jewel: rgw: Have a flavor of bucket deletion in radosgw-admin to bypass garbage collection
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit bb702bc4b9b720af86f759d0e040d57a9a91b3da
+Merge: c5461bc 18545a2
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 1 14:02:56 2017 +0100
+
+    Merge pull request #13187 from asheplyakov/jewel-bp-18729
+    
+    jewel: cli: ceph-disk: convert none str to str before printing it
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit c5461bc9703a9062c3c2ea8022b9f4d4842e0b65
+Merge: 1c6eacb 1481c8f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 13:29:50 2017 +0100
+
+    Merge pull request #13153 from smithfarm/wip-no-firefly-on-centos
+    
+    jewel: tests: upgrade:hammer-x: install firefly only on Ubuntu 14.04
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 1c6eacb6f905cd39248663383da8f797aaef9ebe
+Merge: 5784855 5c4fffa
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 13:17:48 2017 +0100
+
+    Merge pull request #13118 from smithfarm/wip-18274-jewel
+    
+    jewel: Memory leaks in object_list_begin and object_list_end
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 5784855782385186a40696c8bcec151403bfae0c
+Merge: cc867e1 173ea7f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 1 13:15:09 2017 +0100
+
+    Merge pull request #13025 from SUSE/wip-18605-jewel
+    
+    jewel: ceph-disk prepare writes osd log 0 with root owner
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit cc867e1ed2b631c6dd380a575c4475973b5a2a4d
+Merge: 864f915 3a02868
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 13:12:57 2017 +0100
+
+    Merge pull request #13106 from ceph/wip-cd-dev-option
+    
+    qa/tasks: jewel backport -  ceph-deploy use of dev option
+    
+    Reviewed-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 864f9159a97b671f4332b917b72a29600c79aa4d
+Merge: 8e69580 ff91688
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 13:02:54 2017 +0100
+
+    Merge pull request #13062 from asheplyakov/jewel-bp-18379
+    
+    jewel: msg/simple: clear_pipe when wait() is mopping up pipes
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 8e69580c97622abfcbda73f92d9b6b6780be031f
+Merge: c05730c 509de4d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 12:53:06 2017 +0100
+
+    Merge pull request #12978 from asheplyakov/jewel-18581
+    
+    jewel: ReplicatedBackend: take read locks for clone sources during recovery
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit c05730ceb3387fb43c35937f0506297a34a44452
+Merge: 36005e9 fcdd5e7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 12:33:19 2017 +0100
+
+    Merge pull request #12755 from Abhishekvrshny/wip-18284-jewel
+    
+    jewel: Need CLI ability to add, edit and remove omap values with binary keys
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 36005e9efa4a5b75efecfd0eec6effc17639d7db
+Merge: 615549a dcc9483
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 12:31:13 2017 +0100
+
+    Merge pull request #12677 from SUSE/wip-18104-jewel
+    
+    jewel: mon: ceph osd down detection behaviour
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 615549abbf7eed157289dde4603e3c0b25a40d2e
+Merge: 90c9ad7 8a774cc
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 12:28:50 2017 +0100
+
+    Merge pull request #12291 from asheplyakov/jewel-18108
+    
+    jewel: msg/simple/Pipe: error decoding addr
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 90c9ad7fe60b89a87f53b2b224100befc334dbe1
+Merge: ef7d826 a9da605
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 12:27:46 2017 +0100
+
+    Merge pull request #11998 from Abhishekvrshny/wip-17877-jewel
+    
+    jewel: FileStore: fiemap cannot be totally retrieved in xfs when the number of extents > 1364
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit ef7d82679e91d1e2364e3fb433542940511598ee
+Merge: d1a019a fff2127
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 12:26:28 2017 +0100
+
+    Merge pull request #11991 from asheplyakov/jewel-17909
+    
+    jewel: osd: limit omap data in push op
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit d1a019ad3b0a35bd8f339f0ceefd5f8617ee18fa
+Merge: 0b33f2c b044361
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Jan 31 11:40:41 2017 -0500
+
+    Merge pull request #12045 from Abhishekvrshny/wip-17875-jewel
+    
+    jewel: rgw: file: remove spurious mount entries for RGW buckets
+
+commit 0b33f2caf7bf2762ac66779fba0032868e9488e5
+Merge: 92827ae 75d0580
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Jan 31 16:42:14 2017 +0100
+
+    Merge pull request #13161 from smithfarm/wip-lfn-upgrade-hammer
+    
+    jewel: tests: add require_jewel_osds before upgrading last hammer node
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 92827aeb4ea557f204f850a86d69eeed74812c93
+Merge: 790f2a1 077290b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 16:36:17 2017 +0100
+
+    Merge pull request #12425 from SUSE/wip-18190-jewel
+    
+    jewel: rbd-mirror: gmock warnings in bootstrap request unit tests
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 790f2a11860378785bab7fb0195a455ec1d70857
+Merge: 32890c1 649b1d4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 15:16:45 2017 +0100
+
+    Merge pull request #13129 from smithfarm/wip-18558-jewel
+    
+    jewel: rbd: bench-write will crash if --io-size is 4G
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 32890c153dcd8abb3e1ea942f16718582bd0c836
+Merge: c0e3ae3 dd1f425
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 15:15:30 2017 +0100
+
+    Merge pull request #13155 from smithfarm/wip-18494-jewel
+    
+    jewel: rbd: [rbd-mirror] sporadic image replayer shut down failure
+
+commit c0e3ae3332a51dab3a17f1546b6e8258eaea09eb
+Merge: 7849ea5 c2f86a4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 15:14:31 2017 +0100
+
+    Merge pull request #13128 from smithfarm/wip-18633-jewel
+    
+    jewel: rbd: [qa] crash in journal-enabled fsx run
+    
+    http://tracker.ceph.com/issues/13512
+
+commit 7849ea5166030e7f0b331ef21d0867b235fedb2d
+Merge: 833c58e 61e1b0c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 15:11:43 2017 +0100
+
+    Merge pull request #13104 from idryomov/wip-fio-unmap-devices-jewel
+    
+    qa/tasks: backport rbd_fio fixes to jewel
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 833c58edc0445fc54364cc5ae7a58e1c17a96422
+Merge: 26798a1 16a2fec
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 15:10:03 2017 +0100
+
+    Merge pull request #13156 from smithfarm/wip-18455-jewel
+    
+    jewel: rbd: Attempting to remove an image w/ incompatible features results in partial removal
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 26798a1850c2ee6de2e6ec2fd9b9597e447f63dd
+Merge: dff349c 99bafc1
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 14:59:56 2017 +0100
+
+    Merge pull request #13157 from smithfarm/wip-18434-jewel
+    
+    jewel: rbd: Improve error reporting from rbd feature enable/disable
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit dff349c8249388e5b31314dc6297446bf9f0f9e3
+Merge: f9b56db 1555638
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 14:59:04 2017 +0100
+
+    Merge pull request #13168 from trociny/wip-18550-jewel
+    
+    jewel: librbd: metadata_set API operation should not change global config setting
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit f9b56dbdb28cd02ea665f0dfde630f8419a1067c
+Merge: 586f4ea 9c84a65
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 14:55:58 2017 +0100
+
+    Merge pull request #12739 from SUSE/wip-18323-jewel
+    
+    jewel: JournalMetadata flooding with errors when being blacklisted
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 586f4ead083518e95115d31dbecfbf4d5995e7ed
+Merge: 9af8b21 ad869de
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 14:54:19 2017 +0100
+
+    Merge pull request #12416 from SUSE/wip-18219-jewel
+    
+    jewel: msg: don't truncate message sequence to 32-bits
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 9af8b2191268d1b2b4e4d4ba234ba12d265e3731
+Merge: 1a9eb84 336c351
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 14:52:00 2017 +0100
+
+    Merge pull request #13115 from smithfarm/wip-18404-jewel
+    
+    jewel: mon: cache tiering: base pool last_force_resend not respected (racing read got wrong version)
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 1a9eb8454368160be35e966b5264c296db87ac0a
+Merge: b03280d 35e10a0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 14:50:52 2017 +0100
+
+    Merge pull request #13045 from SUSE/wip-18553-jewel
+    
+    jewel: mon: peon wrongly delete routed pg stats op before receive pg stats ack
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit b03280d67c2464599289da659152a6877da65827
+Merge: e524035 7bbb5a8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 14:49:56 2017 +0100
+
+    Merge pull request #13117 from smithfarm/wip-18280-jewel
+    
+    jewel: mon: osd flag health message is misleading
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit e5240357a6901a7e37440f01883ecac56fbc771f
+Merge: c4f8684 820ab7d
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Jan 31 08:34:22 2017 -0500
+
+    Merge pull request #12320 from dachary/wip-18061-jewel
+    
+    jewel: rgw:fix for deleting objects name beginning and ending with underscores of one bucket using POST method of js sdk.
+
+commit c4f868406274881a8b5bbe97bc19e32146efe861
+Merge: e47969e 13fa5db
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Jan 31 08:31:34 2017 -0500
+
+    Merge pull request #11759 from Abhishekvrshny/wip-17783-jewel
+    
+    jewel: rgw: json encode/decode of RGWBucketInfo missing index_type field
+
+commit e47969e5d5cdf51149c23da4eec04dd0a2faec6f
+Merge: b278ece edac06f
+Author: John Spray <jspray@redhat.com>
+Date:   Tue Jan 31 14:10:46 2017 +0100
+
+    Merge pull request #12153 from dachary/wip-18010-jewel
+    
+    jewel: Cleanly reject session evict command when in replay
+
+commit b278ece85be7ed4f57dec6f524f187d1d10f4616
+Merge: 2d2e4af e1af490
+Author: John Spray <jspray@redhat.com>
+Date:   Tue Jan 31 14:10:23 2017 +0100
+
+    Merge pull request #12324 from dachary/wip-18103-jewel
+    
+    jewel: truncate can cause unflushed snapshot data lose
+
+commit 2d2e4afa3eef29ee82ec8057d6172f5706f4b872
+Merge: 916e95d 7819adb
+Author: John Spray <jspray@redhat.com>
+Date:   Tue Jan 31 14:09:59 2017 +0100
+
+    Merge pull request #12783 from SUSE/wip-18413-jewel
+    
+    jewel: cephfs: lookup of /.. in jewel returns -ENOENT
+
+commit 916e95d13692e9a025e47b11724aa65d51860411
+Merge: 384a425 8404426
+Author: John Spray <jspray@redhat.com>
+Date:   Tue Jan 31 14:08:56 2017 +0100
+
+    Merge pull request #12921 from xiaoxichen/wip-18520-jewel
+    
+    Jewel: speed up readdir by skipping unwanted dn
+
+commit 384a425cf69b7554e460d528e7f788c3b0c2eff1
+Merge: 4fd62b6 b147022
+Author: John Spray <jspray@redhat.com>
+Date:   Tue Jan 31 14:08:30 2017 +0100
+
+    Merge pull request #13119 from smithfarm/wip-18565-jewel
+    
+    jewel: MDS crashes on missing metadata object
+
+commit 4fd62b65c2097052378043ee82c937bb702263c1
+Merge: 01a6b1f bc9b779
+Author: John Spray <jspray@redhat.com>
+Date:   Tue Jan 31 14:08:20 2017 +0100
+
+    Merge pull request #13120 from smithfarm/wip-18551-jewel
+    
+    jewel: ceph-fuse crash during snapshot tests
+
+commit 01a6b1f31eafc66cea06702984a77391562e0ad1
+Merge: 831c1d8 fef3de8
+Author: John Spray <jspray@redhat.com>
+Date:   Tue Jan 31 14:08:01 2017 +0100
+
+    Merge pull request #13123 from smithfarm/wip-18282-jewel
+    
+    jewel: monitor cannot start because of FAILED assert(info.state == MDSMap::STATE_STANDBY)
+
+commit 831c1d8c8fc3e8777559da8a154aff104f1c831d
+Merge: 7fbe164 df4558c
+Author: John Spray <jspray@redhat.com>
+Date:   Tue Jan 31 14:07:52 2017 +0100
+
+    Merge pull request #13125 from smithfarm/wip-18195-jewel
+    
+    jewel: cephfs: fix missing ll_get for ll_walk
+
+commit 7fbe164962d488a6b2c2b921619794fea254b848
+Merge: 3793798 5eda4aa
+Author: John Spray <jspray@redhat.com>
+Date:   Tue Jan 31 14:07:41 2017 +0100
+
+    Merge pull request #13126 from smithfarm/wip-18192-jewel
+    
+    jewel: standby-replay daemons can sometimes miss events
+
+commit 75d05809a66bee219031a7ccb64d414a2d6c8775
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jan 27 22:27:18 2017 +0100
+
+    tests: add require_jewel_osds before upgrading last hammer node
+    
+    Note: this commit was inspired by
+    http://github.com/ceph/ceph-qa-suite/commit/50758a4810794d265c5d36a71d1e16799251a00d
+    
+    As of 10.2.4, when upgrading a cluster from hammer to jewel, after the last
+    node is upgraded the MON will put the cluster into HEALTH_WARN and say: "all
+    OSDs are running jewel or later but the 'require_jewel_osds' osdmap flag is not
+    set". The release notes say:
+    
+        This is a signal for the admin to do "ceph osd set require_jewel_osds" – by
+        doing this, the upgrade path is complete and no more pre-Jewel OSDs may be
+        added to the cluster.
+    
+    Fixes: http://tracker.ceph.com/issues/18719
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 3793798049ba74f8f3a5742c50398c8e89bbac0b
+Merge: a1a209e 23680e0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 11:19:39 2017 +0100
+
+    Merge pull request #13049 from SUSE/wip-18433-jewel
+    
+    jewel: rados bench seq must verify the hostname
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit a1a209ed1b33fbb78a4335f69f1f2d5d9085bda4
+Merge: 25bd0be c2bbf7f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 08:40:52 2017 +0100
+
+    Merge pull request #13040 from SUSE/wip-fs-thrash-jewel
+    
+    jewel: tests: run fs/thrash on xfs instead of btrfs
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 25bd0befb0749bb2759059559405f5bafe933d20
+Merge: bdb5e36 7a341a8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 08:39:35 2017 +0100
+
+    Merge pull request #13029 from SUSE/wip-18611-jewel
+    
+    jewel: cephfs: client segfault on ceph_rmdir path /
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit bdb5e362dfec1debf1de5db6c14b23a63437a0d6
+Merge: 1bc16a4 f24c3ff
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 08:34:46 2017 +0100
+
+    Merge pull request #12875 from asheplyakov/jewel-bp-18485
+    
+    jewel: osd/PG: publish PG stats when backfill-related states change
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 1bc16a4333e27c9c3b603af6c52722183ce10a51
+Merge: 1722b46 2296c87
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 08:32:58 2017 +0100
+
+    Merge pull request #12789 from SUSE/wip-18417-jewel
+    
+    jewel: osd: leveldb corruption leads to Operation not permitted not handled and assert
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 1722b4684808eb968b0997b8584477a51cd8940a
+Merge: 345bcdc bbf4c27
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 08:28:22 2017 +0100
+
+    Merge pull request #12761 from SUSE/wip-18402-jewel
+    
+    jewel: tests: objecter_requests workunit fails on wip branches
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 345bcdc0a41ac362ae813a64756fcd3f1d2ff2f4
+Merge: 85ae998 ef2709a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 08:10:20 2017 +0100
+
+    Merge pull request #11947 from SUSE/wip-17884-jewel
+    
+    jewel: mon: OSDs marked OUT wrongly after monitor failover
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 85ae998886c5f8b429b20a8a7d80b2ed4ce57e37
+Merge: 0ddbaf3 4f8287f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jan 31 08:07:40 2017 +0100
+
+    Merge pull request #11508 from SUSE/wip-17583-jewel
+    
+    jewel: utime.h: fix timezone issue in round_to_* funcs.
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 3a02868be40f3431d2bfedf16737b37ebf1a7e89
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Thu Jan 26 13:21:30 2017 -0800
+
+    Revert "use the create option during instantiation"
+    
+    jewel cephfs still uses old Filesystem initializtion method
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+
+commit ebdc0d2ce82ba67aecfaa25e66270c9529238700
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Thu Dec 15 14:11:00 2016 -0800
+
+    use dev option instead of dev-commit
+    
+    Fixes: http://tracker.ceph.com/issues/18736
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+
+commit 0ddbaf384a76e87f218af1babb99f4a08da5abce
+Merge: 4ef0a63 016b059
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Jan 30 13:25:23 2017 -0500
+
+    Merge pull request #13169 from linuxbox2/jewel-mg-errortrans
+    
+    jewel:rgw: fix interface compliance of RGWCivetWeb::write_data()
+
+commit 3839727f2969337a6d70e948db3de034a1346e90
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jan 30 11:38:21 2017 -0500
+
+    librbd: ensure owner lock is held before purging cache
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit d0c12edc52d45a0e429e0d4cca78f8724e39e926
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jan 6 11:17:10 2017 -0500
+
+    librados: blacklist_add should wait for latest OSD map
+    
+    This ensures that future operations against the OSDs force
+    a OSD map update to notice the blacklisted client.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9242a2e4e1a5a9fcea48d8135b1589493fc28242)
+
+commit bf8b78c3a6fb8a6b516793493c4169ceee15d9f8
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 5 13:31:57 2017 -0500
+
+    librbd: prevent assertion failure when journal IO is blacklisted
+    
+    Fixes: http://tracker.ceph.com/issues/18429
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c720f6e3704ed7e8cf41dffdb931dbb05d59a003)
+
+commit 2ca703073fd7563c06a310b66191fb4a99d4fa63
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jan 4 12:12:27 2017 -0500
+
+    librbd: ignore blacklist error when releasing exclusive lock
+    
+    This ensures the journal and object map are properly closed so that the
+    image can be properly closed w/o failing any assertions.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 418dcf29cb8c450049047e09a4dad2941af87018)
+
+commit 7aa424ab450afb6ff308272cedc854a33420d081
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jan 3 15:29:17 2017 -0500
+
+    librbd: fail immediately if the exclusive lock cannot be acquired
+    
+    Fixes: http://tracker.ceph.com/issues/16988
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 47b89f4d48a0b99876462167389df28893a8d418)
+
+commit 5d96332700cbb05f541c211c375ca97506b41aad
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jan 3 14:51:14 2017 -0500
+
+    librbd: add new lock_get_owners / lock_break_lock API methods
+    
+    If the client application supports failover, let the application
+    force break the current lock and blacklist the owner. This is
+    required in case the current lock owner is alive from the point-of-view
+    of librbd but failover was required due to a higher level reason.
+    
+    Fixes: http://tracker.ceph.com/issues/18327
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9a5a8c75a025143cee6f92f3dbc3a12f2b6a9ad7)
+    
+    Conflicts:
+    	src/pybind/rbd/rbd.pyx: trivial resolution
+    	src/test/pybind/test_rbd.py: trivial resolution
+
+commit 245898aa9ae3d6fe03668f3a88b2fac84d11cb29
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Dec 22 15:00:23 2016 -0500
+
+    librbd: separate break lock logic into standalone state machine
+    
+    The current lockers are now queried before the lock is attempted to
+    prevent any possible race conditions when one or more clients attempt
+    to break the lock of a dead client.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 23f60fee86d1ff9b261fbb6411746a2a9479cf19)
+
+commit 968a10b2adc56a45f292ac9faf9b671f2f66b996
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Dec 22 17:24:47 2016 -0500
+
+    librbd: separate locker query into standalone state machine
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 03533b912c59d5e433d0f006e1a063e014468ca5)
+    
+    Conflicts:
+    	src/test/librbd/exclusive_lock/test_mock_AcquireRequest.cc: trivial resolution
+
+commit 652e65a2143f9fa2a69822fe9924e59a83ba5c21
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Mon Feb 22 15:17:20 2016 +0100
+
+    librbd/exclusive_lock/AcquireRequest.cc: init lock_type
+    
+    Fixup for:
+    
+    CID 1351687 (#1 of 1): Uninitialized scalar variable (UNINIT)
+     var_decl: Declaring variable lock_type without initializer
+     uninit_use: Using uninitialized value lock_type.
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit da9ede90cd257ff605ab3ebfcc1aa995f655f1cc)
+
+commit d4085d34f31fa9eed73d69f241184b99e5198b22
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Jun 8 15:11:02 2016 +0300
+
+    librbd: API methods to directly acquire and release the exclusive lock
+    
+    Fixes: http://tracker.ceph.com/issues/15632
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 8f1b47fd5da021ec320fd0b5fc0fd68ffff8a706)
+    
+    Conflicts:
+    	src/common/config_opts.h: trivial resolution
+    	src/include/rbd/librbd.h: trivial resolution
+    	src/librbd/CMakeLists.txt: trivial resolution
+    	src/librbd/Makefile.am: trivial resolution
+    	src/test/librbd/test_librbd.cc: trivial resolution
+
+commit 476e2b14364daa4363425b392199e9cc23c2f02b
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Fri Jun 10 14:04:04 2016 +0300
+
+    rbd-mirror: fix error messages formatting
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 2af72480d94b2b90ed6eac7b3e131437864eada7)
+    
+    Conflicts:
+    	src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc: trivial resolution
+
+commit 374d89f3706aea019f053350c9d790cd8c2c1423
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun Sep 11 09:08:41 2016 -0400
+
+    librbd: ignore partial refresh error when acquiring exclusive lock
+    
+    Fixes: http://tracker.ceph.com/issues/17227
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 24396dcba77a97342d19916fdd285bae0c38fd19)
+
+commit 0bd843aa7bcf212bf370800a4c0c3176af5d6e9e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Sep 9 08:31:52 2016 -0400
+
+    librbd: potential seg fault when blacklisting an image client
+    
+    Fixes: http://tracker.ceph.com/issues/17251
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7025fe8976f8672a2fce05ea693c06a8e9faed19)
+
+commit 273fd99085f13fb5f76e9e60a81012c22c49418d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun Sep 4 10:48:48 2016 -0400
+
+    librbd: potential double-unwatch of watch handle upon error
+    
+    Fixes: http://tracker.ceph.com/issues/17210
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1068ded0cba59831a0712f347946731689a68553)
+
+commit 33e037a089337b5c96a03befac44549e680db13d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Aug 31 21:33:54 2016 -0400
+
+    librbd: deadlock when replaying journal during image open
+    
+    Fixes: http://tracker.ceph.com/issues/17188
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3dc13067f5f0d140ee76b0166eb4cec568610211)
+
+commit bca65c46fdac4829132308f7fcf2ac59e2aaea6c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Aug 31 20:56:54 2016 -0400
+
+    librbd: improve image state machine debug log messages
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c71182a73146c65dfe7bf955ad67ebeebcf7b1fd)
+
+commit a12f435041860f3329e2465586805a168dcc87c9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Aug 17 15:16:37 2016 -0400
+
+    librbd: remove unused refresh request logic
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5156b438d5ad69803b4d4529083039db825d6c8c)
+
+commit a475bfb4d2c8906cfad52a7d81196a87c4c2f0bc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Aug 17 14:58:22 2016 -0400
+
+    librbd: interlock image refresh and lock operations
+    
+    Fixes: http://tracker.ceph.com/issues/16773
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 583ac91872859e81d68c9d346516522c6aa1614c)
+
+commit 3d61b69b11ef67719e78d77fec000403d43d70d3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Aug 17 14:02:32 2016 -0400
+
+    librbd: image state machine now has hooks for lock requests
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a5b8c9c1d81ed609e71a5a8abe4d0e90194408cc)
+
+commit 4ea8d55c95304e07a90a6c4b8890787ccaeecedf
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Aug 16 16:23:57 2016 -0400
+
+    librbd: integrate asynchronous image rewatch state machine
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d0d97fcca2cb427adbdf5c32f0e07830c38aaf4b)
+
+commit 62f265b30531141dfda8a7490d18b1d0d787fe13
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 15 15:46:23 2016 -0400
+
+    librbd: helper state machine for asynchronous watch recovery
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 32180aaf42050a01981c33f84edd95eff931ee6c)
+    
+    Conflicts:
+    	src/librbd/CMakeLists.txt: trivial resolution
+    	src/librbd/Makefile.am: trivial resolution
+
+commit ead7201874ed8617d86dc548798aa4bde7168fdb
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Aug 16 14:20:20 2016 -0400
+
+    librbd: exclusive lock now supports reacquiring a lost lock
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 66c605573f840c0db8b3630315ea50e9fc987509)
+
+commit 38ca4ff1d8d21db17f382a6821d098e472be4657
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Aug 16 13:11:19 2016 -0400
+
+    librbd: store exclusive lock cookie instead of recalculating
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit aa53f74ad261f453e971bf3cef0b96bba4932b7b)
+
+commit 73a445572e8fab135c7d87b80e374355e77d2e28
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Aug 16 12:28:09 2016 -0400
+
+    librbd: helper state machine to update lock cookie
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d523df8dafac472f95233805d3a82edb3b3b02ea)
+
+commit 98a5e116c28ec9096ce68308be06e794883ce29e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Aug 15 16:50:43 2016 -0400
+
+    cls_lock: support updating the lock cookie without releasing the lock
+    
+    Fixes: http://tracker.ceph.com/issues/17015
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 377f57652f8ddae53f44c59f21e89c51b2bf0f7b)
+
+commit 8a75f980d3cb00cc31c479f5e4253a6385563663
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Jan 30 14:19:05 2017 +0100
+
+    tests: make distros/all/centos.yaml be a symlink to centos_7.3
+    
+    Before this commit, tests that used distros/all/centos.yaml would
+    fail on VPS because VPS defaults to centos 7.0 when os_version is not
+    specified.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 4ef0a63e7a29eafad6dda52ea3c92846f03f268d
+Merge: efad365 900f2ac
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jan 30 13:26:37 2017 +0100
+
+    Merge pull request #12323 from dachary/wip-18024-jewel
+    
+    jewel: rbd: FAILED assert(m_processing == 0) while running test_lock_fence.sh
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit efad365753b7847ecce1280fe267d97d2ccf2888
+Merge: a1e7c06 64c0cae
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jan 30 13:25:48 2017 +0100
+
+    Merge pull request #12649 from xiaoxichen/wip-18278-jewel
+    
+    jewel: rbd: librbd: use proper snapshot when computing diff parent overlap
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit a1e7c06dcb1b47eceea273e9b6ae5044c5958a07
+Merge: ed50d14 8877ee4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jan 30 13:24:22 2017 +0100
+
+    Merge pull request #12741 from SUSE/wip-18320-jewel
+    
+    jewel: rbd status: json format has duplicated/overwritten key
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit ed50d14b4fa4acd13b4197e881666b2120525601
+Merge: b7481a8 d313e42
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jan 30 13:23:44 2017 +0100
+
+    Merge pull request #12753 from Abhishekvrshny/wip-18288-jewel
+    
+    jewel: rbd-mirror: image sync object map reload logs message
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit b7481a85453ed059b7ab40932e57ac5354b7581f
+Merge: 2ebb894 944ec03
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jan 30 13:23:09 2017 +0100
+
+    Merge pull request #12756 from Abhishekvrshny/wip-18276-jewel
+    
+    jewel: rbd-nbd: invalid error code for "failed to read nbd request" messages
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 2ebb8948858a33c077f6a1481a1ce3e3f23d5d74
+Merge: 60a2037 b359935
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jan 30 13:22:37 2017 +0100
+
+    Merge pull request #12822 from SUSE/wip-18450-jewel
+    
+    jewel: tests: update rbd/singleton/all/formatted-output.yaml to support ceph-ci
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 60a2037ef52a6fc75171260067cee9c9bcc66bd9
+Merge: 60bc353 cdd6cbf
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jan 30 13:21:25 2017 +0100
+
+    Merge pull request #12909 from dillaman/wip-18290-jewel
+    
+    jewel: librbd: properly order concurrent updates to the object map
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 60bc35357cdbe39e8c85ebd217526c1b9ad4fc76
+Merge: 18cb72c aa8e57d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jan 30 13:20:07 2017 +0100
+
+    Merge pull request #12529 from SUSE/wip-18270-jewel
+    
+    jewel: rbd: add image id block name prefix APIs
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 18cb72c129d6c90afa489fd1cda6713fe8a9e1ed
+Merge: 20a480d 4a157ea
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jan 30 13:18:32 2017 +0100
+
+    Merge pull request #12322 from dachary/wip-18110-jewel
+    
+    jewel: diff calculate can hide parent extents when examining first snapshot in clone
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 18545a27eccfa0b22b1782bb52e3f47afef8ec39
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jan 3 20:40:00 2017 +0800
+
+    ceph-disk: convert none str to str before printing it
+    
+    Error('somethings goes wrong', e) is thrown if exception `e` is caught
+    in ceph-disk, where e is not a string. so we can not just concat it in
+    Error's __str__(). so cast it to str before doing so.
+    
+    introduced by d0e29c7
+    
+    Fixes: http://tracker.ceph.com/issues/18371
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5e0dd1e7df43a3be589d17878714756a22052d8e)
+
+commit 077290b873e2cd991a995de14a116d85d83ba66e
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Dec 6 23:19:54 2016 +0200
+
+    rbd-mirror: fix gmock warnings in bootstrap request unit tests
+    
+    Fixes: http://tracker.ceph.com/issues/18156
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 376026d7b24c77a59ef95d0f66686494caf0e9d0)
+
+commit ce32297c880779532ba3482edb93fc4e4340d94b
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Fri Dec 2 10:10:52 2016 +0200
+
+    qa/workunits/rbd: test_status_in_pool_dir: explicitly check grep return value
+    
+    Otherwise, it does not work as supposed to work in statements like below:
+    
+     set -e
+     test_status_in_pool_dir ... && ...
+    
+    (e.g. in wait_for_status_in_pool_dir)
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 7c078eda0fc260f6a885fa43f377b47844867d12)
+
+commit 6d729d231401f9253aa1cbde06e57cd8bd066a90
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Nov 29 11:44:05 2016 +0200
+
+    rbd-mirror: make 'rbd mirror image resync' work after split-brain
+    
+    Fixes: http://tracker.ceph.com/issues/18051
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 6cb1ed485f89f30fe6dda31e77e16e23f9b5b2ab)
+
+commit e34a403df5e247b0937abee617758e0838fc0c8e
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Nov 29 11:40:14 2016 +0200
+
+    rbd-mirror: split-brain issues should be clearly visible in mirror status
+    
+    Fixed: http://tracker.ceph.com/issues/16991
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit cccca67d5f898c43207a19a6e029a1abb86efbcb)
+
+commit 8f9a93c68947c05636a633c92dbbd69d2ed70c3d
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Sun Nov 27 16:27:51 2016 +0200
+
+    qa/workunits/rbd: use image id when probing for image presence
+    
+    This fixes a race in resync tests leading to false negative results.
+    
+    Fixes: http://tracker.ceph.com/issues/18048
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit c23f56b9835f4e59d72f20a2218de8236574e65f)
+
+commit 1f2d30cbb735d2c7fd3eea6431f1a1e9555d5942
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Oct 3 10:10:33 2016 +0300
+
+    qa/workunits/rbd: check status also in pool dir after asok commands
+    
+    wait_for_image_replay_stopped returns not when the state is stopped,
+    but when the state is not replaying. So a race was possible when an
+    asok command was running when the previos stop command was still in
+    progress, leading to unexpected results.
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 2882f8c01a42ee3cfe22a838b64f21c93cf84d16)
+    
+    Conflicts:
+    	qa/workunits/rbd/rbd_mirror.sh (tests for not backported features)
+
+commit 5d0fba44501130165416a57a10490ff8ba292d48
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Aug 2 20:06:17 2016 +0300
+
+    qa/workunits/rbd: wait for image deleted before checking health
+    
+    This is a fixup to the previous commit.
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit a5f63f726146c2c230d172f6909d8ca1ad46895a)
+
+commit 38e06fbfcd8ba3a529af0a902c1af6168cf7b0f6
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Jul 28 14:45:56 2016 +0300
+
+    qa/workunits/rbd: wait for image deleted before checking health
+    
+    When a primiry image is being deleted, the mirrored image might
+    temporary be reported in error state, before deletion is propagated.
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 8dbe42d34520d0b44b189ae4d8b96559752a57da)
+
+commit 735e32bb53fd370058ed8dfb0e29e4a9cfc6e86f
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Jul 28 14:49:37 2016 +0300
+
+    qa/workunits/rbd: small fixup and improvements for rbd-mirror tests
+    
+    - log to stderr;
+    - log status if a `wait_for` function failed;
+    - don't needlessly sleep in `wait_for` functions after the last
+      unsuccessful iteration;
+    - make `wait_for_pool_images` work for image removal case;
+    - fix `wait_for_pool_images` reset timeout (last_count set).
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit fdb971a2660111d35f0a3077f17a9ca85ca2ef54)
+
+commit bf3400f7f1dbc14917f3392c8f9221b6b02df185
+Author: Wido den Hollander <wido@42on.com>
+Date:   Mon Jan 23 08:18:27 2017 +0100
+
+    systemd: Restart Mon after 10s in case of failure
+    
+    In some situations the IP address the Monitor wants to bind to
+    might not be available yet.
+    
+    This might for example be a IPv6 Address which is still performing
+    DAD or waiting for a Router Advertisement to be send by the Router(s).
+    
+    Have systemd wait for 10s before starting the Mon and increase the amount
+    of times it does so to 5.
+    
+    This allows the system to bring up IP Addresses in the mean time while
+    systemd waits with restarting the Mon.
+    
+    Fixes: #18635
+    
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit e73eb8cc1e0d45af1f0b7852c551f2ddfb82a520)
+
+commit 20a480defb607e6e5a72f2bcc3868fb14f9e56de
+Merge: 6b4bcd3 bcd4698
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sun Jan 29 13:22:49 2017 +0100
+
+    Merge pull request #13043 from SUSE/wip-18570-jewel
+    
+    jewel: Python Swift client commands in Quick Developer Guide don't match configuration in vstart.sh
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 401271e353fcfd59bf0285c53643b667281954d0
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Jan 29 13:12:43 2017 +0100
+
+    build/ops: add libldap dependency for RGW
+    
+    This is a minimal manual backport of the relevant parts of
+    a4c7e13d17ceff3ee15fc311c2a344cd4573821d and
+    b3b3185008a0a2149dcba59813e0f0400d2e47de
+    
+    Fixes: http://tracker.ceph.com/issues/17313
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 73d2114ed60e8297d26b7f31f0500034b3d784e7
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 1 12:56:44 2017 +0200
+
+    radosgw-admin: check for name mistmatch in realm set
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 4facc5f4f3e842d371115a9a04d86257280014f0)
+
+commit e4b6cf633a88dc7fcd48f9a4a54d10fea7bef7e2
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 1 12:40:37 2017 +0200
+
+    radosgw-admin: relam set can use input redirection
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit b8b3ae3be3f8e4c05cb23062d25c701b15900475)
+    
+    Conflicts:
+    	src/rgw/rgw_admin.cc ("return EINVAL" in master, "return -EINVAL" in
+                  jewel; but it's irrelevant because the whole block is deleted)
+
+commit 3cd42f4b75c9ea9d253c276e61789d5a2b2395b4
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 1 12:36:04 2017 +0200
+
+    radosgw-admin: realm set should create a new realm
+    
+    Fixes: http://tracker.ceph.com/issues/18333
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit e23339c9ef34f6b9df90b1ab64b550af9b541d9e)
+
+commit 33c6ef53413ef83f84be3bc3825125030aa2281b
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jan 11 09:32:59 2017 -0500
+
+    rgw: fix off-by-one in RGWDataChangesLog::get_info
+    
+    Fixes: http://tracker.ceph.com/issues/18488
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit a0974fdcf62e60cf31bc15588e7b718da6f6ade3)
+
+commit 8b124c84749e0fdd6663aa8302e0bd93a52626e7
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Jan 19 18:14:30 2017 -0500
+
+    rgw_file: add timed namespace invalidation
+    
+    With change, librgw/rgw_file consumers can provide an invalidation
+    callback, which is used by the library to invalidate directories
+    whose contents should be forgotten.
+    
+    The existing RGWLib GC mechanism is being used to drive this.  New
+    configuration params have been added.  The main configurable is
+    rgw_nfs_namespace_expire_secs, the expire timeout.
+    
+    Updated post Yehuda review.
+    
+    Fixes: http://tracker.ceph.com/issues/18651
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit deb2c1ea985fcb906e47b93fd3d0117794e2d0a1)
+    
+    Conflicts:
+    	src/rgw/rgw_lib_frontend.h - in class RGWLibProcess : public RGWProcess
+                           there was no public method stop() in jewel (now there is)
+
+commit 2cb0307e7522a8dac9e6a400d987e22b9c09b56a
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Fri Jan 13 16:32:55 2017 +0100
+
+    rgw_admin: read master log shards from master's current period
+    
+    Also make the sync output look similar to the output of data sync
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    
+    (cherry picked from commit cc306c506ca6607223cb89cd388f8e18673c4fe2)
+
+commit b2fedddd3daf544d935ba4f80a5feef35400f2e3
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Fri Jan 13 16:29:47 2017 +0100
+
+    rgw: allow getting master log shards info on specified period
+    
+    This is needed for rgw admin's sync status or else we end up always
+    publishing that we're behind since we are always checking against
+    master's first period to sync from
+    
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit 063c949d4409a18a22b64791d497e20f7473bc01)
+
+commit a0e08935b342ff3713ab9172a541e284fd1bb006
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Thu Jan 12 22:09:01 2017 +0100
+
+    rgw_admin: get master's period from store's current period info
+    
+    This ensures that we get the current period in contrast to the admin log
+    which gets the master's earliest period.
+    
+    Fixes: http://tracker.ceph.com/issues/18064
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit 4ca18df7198a9f0ded8b0100a70b5db7187c3de4)
+
+commit 0a4734261bd2f02f8b1acfae8ae65daf6b54d7d7
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Dec 12 14:00:05 2016 +0100
+
+    rgw: complete versioning enablement after sending it to meta master
+    
+    Fixes: http://tracker.ceph.com/issues/18003
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 2d8aafb9dbe64bd9dd2b7d5ed50c6e9550cbe1ab)
+
+commit 7e51bec0dd77bedc248a595e1ab63f87d7a30458
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Dec 25 12:36:34 2016 +0200
+
+    rgw: clear master_zonegroup when reseting RGWPeriodMap
+    
+    Fixes: http://tracker.ceph.com/issues/17239
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit d8f42fe6be659c1d48bf04b30aa54ad616936145)
+
+commit 509de4d9a2d3ba95e9f9e0375bd13239280b0e66
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Jan 18 10:24:13 2017 -0800
+
+    PrimaryLogPG::try_lock_for_read: give up if missing
+    
+    The only users calc_*_subsets might try to read_lock an object which is
+    missing on the primary.  Returning false in those cases is perfectly
+    reasonable and avoids the problem.
+    
+    Fixes: http://tracker.ceph.com/issues/18583
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 3833440adea6f8bcb0093603c3a9d16360ed57ec)
+
+commit cedaecf88efd3e4807e764d023bb956f806051da
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Nov 23 15:41:13 2016 -0800
+
+    ReplicatedBackend: take read locks for clone sources during recovery
+    
+    Otherwise, we run the risk of a clone source which hasn't actually
+    come into existence yet being used if we grab a clone which *just*
+    got added the the ssc, but has not yet actually had time to be
+    created (can't rely on message ordering here since recovery messages
+    don't necessarily order with client IO!).
+    
+    Fixes: http://tracker.ceph.com/issues/17831
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 68defc2b0561414711d4dd0a76bc5d0f46f8a3f8)
+    
+    Conflicts:
+    	src/osd/ReplicatedBackend.cc:
+    	  PGBackend::failed_push() takes a single pg_shard_t in jewel
+    	src/osd/ReplicatedPG.h:
+    	  trivial: get_obc() is not declared as override in jewel,
+    	           no pgb_{set,clear}_object_snap_mapping() in jewel
+
+commit 6b4bcd388829d6c5b78a7acb6d75d6f905a60f53
+Merge: c9ece04 7b74238
+Author: vasukulkarni <vasu.kulkarni@gmail.com>
+Date:   Sat Jan 28 17:58:45 2017 -0800
+
+    Merge pull request #13166 from smithfarm/wip-drop-ext4-test
+    
+    tests: Remove ext4 option from rados:thrash tests
+
+commit 016b059fcf2993f721e97212d50b5e6da8180a03
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Sat Jan 28 14:17:10 2017 -0500
+
+    rgw: fix interface compliance of RGWCivetWeb::write_data()
+    
+    Backport of (portions of) civet web error handling bugfixes.
+    
+    Adapted from 3a9f50c55e0be6733893a7ae1a5b4f504a3b0f61.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 15556389c3c3bf9d6855aaa4699545a787d9fe7a
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Jan 9 18:31:21 2017 +0100
+
+    librbd: metadata_set API operation should not change global config setting
+    
+    Fixes: http://tracker.ceph.com/issues/18465
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 27465b5916b55ac3c2846c74b89f4362ad17ff1e)
+    
+    Conflicts:
+        src/librbd/Operations.cc (after jewel set_metadata migrated to Operations)
+
+commit 7b7423837ea4f400507cf775f609c676a4cf3c8e
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Jan 28 12:11:08 2017 +0100
+
+    tests: Remove ext4 option from rados:thrash tests
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (manual cherry-pick from ceph-qa-suite 1fcc4457144278f77dd8462ecf34948a11dcb2a9)
+
+commit c9ece0444611994ddc1292a83e6c091f7773212f
+Merge: efcbcac 4e60be5
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jan 27 12:57:32 2017 -0800
+
+    Merge pull request #13160 from ceph/revert-13068-jewel-rgw-multipart-upload-copy
+    
+    Revert "jewel: rgw: multipart upload copy"
+    
+    Reverting, will require further testing before merging. Some tests fail.
+
+commit 4e60be5867975b4ae5fe6f53aacd6abe128a18c3
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Jan 27 15:53:47 2017 -0500
+
+    Revert "jewel: rgw: multipart upload copy"
+
+commit efcbcac34e034cd3f3bc542fb71d527d64e66688
+Merge: a632cc4 514e2ba
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Jan 27 13:04:46 2017 -0500
+
+    Merge pull request #13068 from linuxbox2/jewel-rgw-multipart-upload-copy
+    
+    DNM: jewel:rgw: multipart upload copy
+
+commit 99bafc1bab8eb8349e44b601f5971af97c131bd9
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jan 27 18:41:25 2017 +0100
+
+    rbd: enabling/disabling rbd feature should report missing dependency
+    
+    Currently while enabling or disabling rbd feature command does not
+    give missing dependency for eg: attempting to enable the journaling
+    feature on an image that doesn't have the exclusive-lock feature
+    enabled should give missing dependency error message.
+    
+    Fixes: http://tracker.ceph.com/issues/16985
+    
+    Reported-by:  Jason Dillaman <dillaman@redhat.com>
+    Signed-off-by: Gaurav Kumar Garg <garg.gaurav52@gmail.com>
+    (manual cherry pick of bd023cfec8e9aaa8fb0095a8d9534c21b3209020)
+
+commit 16a2feca3523a4b0495769c2750dd1f5d70de25c
+Author: Dongsheng Yang <dongsheng.yang@easystack.cn>
+Date:   Thu Dec 22 21:00:41 2016 -0500
+
+    librbd: don't remove an image w/ incompatible features
+    
+    Fixes: http://tracker.ceph.com/issues/18315
+    Signed-off-by: Dongsheng Yang <dongsheng.yang@easystack.cn>
+    (cherry picked from commit f76127b5e617923d14adb62bfb836a635c14f209)
+
+commit dd1f4252ed3a89407063cd283eebdbdc7cf5653c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jan 6 15:59:22 2017 -0500
+
+    rbd-mirror: avoid processing new events after stop requested
+    
+    Fixes: http://tracker.ceph.com/issues/18441
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d30873b269441815b5fc7de14c7d9a1077a17d8d)
+
+commit de1ebc36943208057e1aa427f9fda631d0bd717d
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jan 27 16:59:44 2017 +0100
+
+    tests: explicitly use centos 7.3 in distros/supported
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 0133d6316fc83373977c132cdbaa6f99d4f1140e
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Tue Jan 3 15:59:18 2017 -0800
+
+    qa: fixed distros links
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+    (cherry picked from commit fbb560c90101937317380a6621a70564f10e0ae3)
+
+commit 1481c8fac899e395757ef81ae5b7baffe8e492ab
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jan 27 15:29:27 2017 +0100
+
+    tests: upgrade: install firefly only on Ubuntu 14.04
+    
+    The upgrade:hammer-x/f-h-x-offline test installs firefly, but firefly does not
+    build on CentOS anymore, just Ubuntu 14.04.
+    
+    References: http://tracker.ceph.com/issues/18089
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 514e2ba64f2babdf6f58c57a7e8e1263abbf187c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 17:44:08 2016 -0700
+
+    rgw: minor optimization
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 4919dc9987c6376d3d4e143702c26417449524c5)
+
+commit 7f76bb1229f40c4b535d97f163f26ee642798e2b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 17:43:00 2016 -0700
+
+    rgw: rgw_obj_key use adjustment in multipart copy part
+    
+    This fixes a case where objects start with double underscore.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 29fece3545cc1df404a25eec46706b16f893a5df)
+
+commit f99ead1570a23a9050e396383006deb2e7e1d9f8
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 17:43:35 2016 -0700
+
+    rgw: multipart copy-part handle versionId
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 53521efffb1cb92e5f5ce992d4127bf9498d7c33)
+
+commit 679348947910a92bf9bb7920c071142536272b19
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 14:24:13 2016 -0700
+
+    rgw: multipart copy part minor parsing cleanup
+    
+    no need for range.size()
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 6e9b824d5d4017239d58b4752ebc43bfad8f698d)
+
+commit 2ca1bcdacc37e74cb61c2ed9855150b4cc9537e9
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 14:11:43 2016 -0700
+
+    rgw: multipart copy, check for empty bucket, improve logging
+    
+    also reduce log level for non critical user errors.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 2bcb3d286b230ef917d5ba96c8276a942f544689)
+
+commit e5ac1204c26a26b29b146d350ff5034bfd9fc798
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 14:07:14 2016 -0700
+
+    rgw: multipart copy part, chunked read
+    
+    Don't read the entire range from source object, read it in parts.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 4049e47a0cfc1eef6efd502590b68ba7234589d3)
+
+commit a54a7ada65fafc88251897b2739b364bf59be128
+Author: Javier M. Mellid <jmunhoz@igalia.com>
+Date:   Tue Jul 26 14:56:50 2016 +0200
+
+    rgw: doc: add multipart uploads copy part feature as supported
+    
+    Signed-off-by: Javier M. Mellid <jmunhoz@igalia.com>
+    (cherry picked from commit 8a7ebeee2ff3f10ceb23b7fa43e43c3c450efe22)
+
+commit 987b42561e4c98c54963055f357955d243881238
+Author: Javier M. Mellid <jmunhoz@igalia.com>
+Date:   Tue Jul 26 14:53:44 2016 +0200
+
+    rgw: multipart uploads copy part support
+    
+    Add multipart uploads copy part feature.
+    
+    Fixes: http://tracker.ceph.com/issues/12790
+    
+    Signed-off-by: Javier M. Mellid <jmunhoz@igalia.com>
+    (cherry picked from commit 949480c2e9760855ed6a0501d364d5e766c8207d)
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 21622c13468d85f017d139a376097267943ea328
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Nov 21 12:10:05 2016 +0000
+
+    src/mds: fix MDSMap upgrade decoding
+    
+    Hammer MDSMonitors did not validate the state in beacons
+    and would apply anything to the mds_info_t for a standby,
+    such as setting it to down:dne.  We must handle this
+    case during upgrade.
+    
+    Fixes: http://tracker.ceph.com/issues/17837
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit a977c9029ff80b4eb4f3b128965693694b729425)
+
+commit be8bc1195270b87bdc0840b36684863d990404df
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Jul 15 23:40:17 2016 -0400
+
+    mds: use FSMap::insert to add to standby_daemons
+    
+    This reduces the number of code sites which modify standby_daemons.
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 9566ae27b33dd4d266ee9223dc73738922b6eceb)
+
+commit 649b1d4253a1753c3f4d783a8e2d7b48f861db08
+Author: Gaurav Kumar Garg <garg.gaurav52@gmail.com>
+Date:   Tue Jan 10 15:25:13 2017 +0100
+
+    rbd: bench-write should return error if io-size >= 4G
+    
+    Currently if user perform bench-write with io-size > 4G
+    then its crashing because currently during memory allocation
+    bufferptr taking size of buffer as a unsigned and io-size > 4G
+    will overflow with unsigned. so during memset operation it will
+    try to set io_size size of memory area pointed by bufferptr,
+    (bufferptr area is:  (4G - io_size)), so it will cause
+    segmentation fault.
+    
+    Fix is to return error if io-size >= 4G
+    
+    Fixes: http://tracker.ceph.com/issues/18422
+    
+    Reported-by:  Jason Dillaman <dillaman@redhat.com>
+    Signed-off-by: Gaurav Kumar Garg <garg.gaurav52@gmail.com>
+    (cherry picked from commit 6ab73e5f420e89e19b52e39dab28fa4c94e00197)
+    
+    Conflicts:
+    	src/tools/rbd/action/BenchWrite.cc - jewel has rbd_bencher b(&image)
+                while in master this has become rbd_bencher b(&image, io_type,
+                io_size)
+
+commit c2f86a461ea556cc5f9d93a7143cd80fb3e7dc96
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jan 20 14:26:43 2017 -0500
+
+    journal: don't hold future lock during assignment
+    
+    It's possible that the future raced with its owner and reaches
+    an empty reference count. This was resulting in the future being
+    destructed while its lock was still held.
+    
+    Fixes: http://tracker.ceph.com/issues/18618
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0f21ceef8336e35ca16148a9d58f511037911418)
+
+commit 5eda4aac77200b858371ebab50d8779f70fe0dca
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Nov 18 21:11:54 2016 +0000
+
+    mds: fix dropping events in standby replay
+    
+    Ensure that we never drop the last segment during
+    standby replay -- this avoids the case where we
+    start ignoring events because we think we're
+    still waiting to see a subtreemap.
+    
+    Fixes: http://tracker.ceph.com/issues/17954
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit e3f2fa356f419cbac8d72bd068313f64769ef3af)
+
+commit df4558c9d5779ac8916811ea63d1106c2390638f
+Author: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+Date:   Thu Nov 17 18:01:22 2016 +0800
+
+    cephfs: fix missing ll_get for ll_walk
+    
+    Fixs: http://tracker.ceph.com/issues/18086
+    
+    Signed-off-by: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+    (cherry picked from commit ad846d10d5fbab2ded5fddc47475b95a788c223c)
+    
+    Conflicts:
+    	src/client/Client.cc - jewel has fill_stat(in, attr) instead of
+                                   fill_statx(in, mask, stx) in master
+
+commit fef3de88d0381322d49ce7842f777454b0b40b63
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Dec 8 16:49:04 2016 +0000
+
+    mon/MDSMonitor: fix iterating over mutated map
+    
+    If a standby is promoted, this was modifying the
+    standby_daemons map that we were iterating over
+    in maybe_promote_standby.
+    
+    Fixes: http://tracker.ceph.com/issues/18166
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit db3deb80d6e4a8e8ca7a2cf6278d5d5cb24eb616)
+
+commit 05916271c17050fd4c1ad2351b6da17d3b89fc15
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu Jul 7 19:09:56 2016 -0400
+
+    mon: use clearer code structure
+    
+    The failed map is not changed in the first if, so the second if should be an
+    else.
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit adba77e34c3fbf983d221729dd340afcc942778b)
+
+commit bc9b7792e4d44c199b526c2d5b17916d948b0354
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Jan 10 17:16:40 2017 +0800
+
+    client: fix Client::handle_cap_flushsnap_ack() crash
+    
+    Struct CapSnap holds a reference to its parent inode. So erasing
+    struct CapSnap from Inode::cap_snaps may drop inode's last reference.
+    The inode gets freed in the middle of erasing struct CapSnap
+    
+    Fixes: http://tracker.ceph.com/issues/18460
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 525c52fd491ed1ced385c8047872e3f557f8423f)
+    
+    Conflicts:
+    	src/client/Client.cc (jewel does in->cap_snaps.erase(follows), master
+                                 does not; put it after the tmp_ref assignment)
+
+commit b14702209f5bad7d3aa65d3acba89b7757f725dd
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jan 11 15:50:52 2017 +0800
+
+    qa/tasks: add test_open_ino_errors
+    
+    Validate that errors encountered during opening inos are properly
+    propagated
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 6526ecc084733b34129aa1f21085fa41fb53b785)
+
+commit 3385419e8f41d0af986dc088e1c9ce83b569eb19
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Jan 3 11:11:12 2017 +0800
+
+    mds: propagate error encountered during opening inode by number
+    
+    Fixes: http://tracker.ceph.com/issues/18179
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 2213cc2dcc0e8fb01bcae3863d0d8a4a1fd8873f)
+
+commit 5c4fffa878c6ea881cc29b0ee4f99cde5e0e4ea8
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Wed Dec 14 16:29:08 2016 +1000
+
+    librados: Memory leaks in object_list_begin and object_list_end
+    
+    We allocate a cursor in the constructor but simply reassign it in these
+    functions without cleaning up the original. We have a utility setter that
+    handles this exact case so we should use it.
+    
+    Fixes: http://tracker.ceph.com/issues/18252
+    
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 8d9d84bce923a009054ad2b223a97d7eb00e6774)
+
+commit 7bbb5a8918c7828d925f4796d2c25a583c8323e3
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 7 09:50:26 2016 -0500
+
+    mon/OSDMonitor: only show interesting flags in health warning
+    
+    Also add PAUSEREC flag into the list.
+    
+    Fixes: http://tracker.ceph.com/issues/18175
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 26220f0608f5ed4a7d97fb8d10d0d12a0fcf5583)
+
+commit 336c3515043f89494e452266069f778d7d8fe821
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 29 12:08:28 2016 -0500
+
+    mon/OSDMonitor: set last_force_op_resend on overlay pool too
+    
+    We currently set the last_force_op_resend field on the
+    base pool when we set or clear the overlay.  Set it on
+    the cache/overlay pool too.  The Objecter should resend
+    even with a change only to the base pool, but the OSD
+    needs to see the change on the overlay pool to correctly
+    discard the op.
+    
+    Fixes: http://tracker.ceph.com/issues/18366
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 08c3618771b852840aa88cff1ca98d980d802941)
+
+commit 874220371890a59b8459bb6306a5db047e32563e
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Jan 10 11:16:47 2017 +0800
+
+    mds: finish clientreplay requests before requesting active state
+    
+    All clientreplay requests' finish contexts should be executed
+    before MDCache::export_remaining_imported_caps(). Otherwise
+    MDCache::try_reconnect_cap() may fail to reconnect client caps.
+    
+    Fixes: http://tracker.ceph.com/issues/18461
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 80dae314ee90e79d60e2cfee301e43a435c10801)
+
+commit b3599351225bd8f6299121e84b62a582f698199b
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Jan 7 13:48:51 2017 +0100
+
+    tests: subst repo and branch in qemu test urls
+    
+    References: http://tracker.ceph.com/issues/18440
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 2484a5546160560a4050d35400a6edce37ae48ad)
+    
+    Conflicts:
+    	qa/suites/rbd/qemu/workloads/#qemu_xfstests.yaml#
+                (file does not exist in jewel)
+
+commit f66bd81d7082af3c1e28cfee19e46906a35835ee
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Jan 7 13:45:10 2017 +0100
+
+    tests: subst branch and repo in qa/tasks/qemu.py
+    
+    References: http://tracker.ceph.com/issues/18440
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 74689df754561e11a3377998840efcea9e780755)
+
+commit 69a0efa7a689b5ec80d44119858cdcf86ca85588
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jan 6 21:09:23 2017 +0100
+
+    tests: subst repo name in krbd/unmap/tasks/unmap.yaml
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit ed0e8be0b2c4d7a3e6e0716a0211d19e8b93f125)
+
+commit 2931aefc952ca0592eca26606864cbfa724b4c07
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jan 6 21:06:11 2017 +0100
+
+    tests: subst repo name in qa/tasks/cram.py
+    
+    Inspired by bcbe45d948f1c4da02e27b3be5f29a8b52745e28
+    
+    Fixes: http://tracker.ceph.com/issues/18440
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 56e37e41f4dddd289dd3c1886b192cd328ed311b)
+
+commit 205403b02156e4ed6b2b53a2b21189b5122162f8
+Author: Venky Shankar <vshankar@redhat.com>
+Date:   Fri Dec 9 15:11:49 2016 +0530
+
+    cram: support fetching from sha1 branch, tag, commit hash
+    
+    Signed-off-by: Venky Shankar <vshankar@redhat.com>
+    (cherry picked from commit d2f0d745987a2e2eee4e1822146aad8da5d42708)
+    
+    Conflicts:
+    	suites/rbd/singleton/all/formatted-output.yaml (has been moved
+               under qa/)
+
+commit a632cc41755be4393a12dd768726ade86a49bbc9
+Merge: e55b84e ea65450
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jan 25 10:40:25 2017 -0500
+
+    Merge pull request #13103 from dillaman/wip-18672
+    
+    jewel: qa/workunits/rbd: use more recent qemu-iotests that support Xenial
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit ea65450c49083f186781a3cbffe6cd24ed3c7a00
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Dec 5 13:46:02 2016 -0500
+
+    qa/workunits/rbd: use more recent qemu-iotests that support Xenial
+    
+    Fixes: http://tracker.ceph.com/issues/18149
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4314cb945a9c2296e2f7cd357b09015777f233c0)
+
+commit f449e3d1745ffd09e32058ea66342b486090d394
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Dec 7 09:59:39 2016 -0500
+
+    qa/workunits/rbd: removed qemu-iotest case 077
+    
+    The test case is not stable due to racing console output. This
+    results in spurious failures.
+    
+    Fixes: http://tracker.ceph.com/issues/10773
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2c70df978d605a45ff81971b86f5afbefbdaabb6)
+
+commit 61e1b0caddfbc3721cd9fa2de5b90b2f65740f79
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Wed Aug 31 19:30:19 2016 +0200
+
+    tasks/rbd_fio: unmap rbd devices on cleanup
+    
+    Not doing so leads to issues and can interfere with subsequent jobs.
+    One example is the invocation of vgs(8) during the inital test setup:
+    it will issue a read to the left-behind rbd device(s) whose backing
+    cluster is long gone, locking up the job.
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit 15be2d29be3e42644b0c541f5c70461f1874f24f)
+
+commit 5fcfa32f21709054f054bcda719df090777f52d5
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Wed Aug 31 19:05:25 2016 +0200
+
+    tasks/rbd_fio: don't use sudo unnecessarily
+    
+    Creating and cloning images doesn't require sudo.
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit ec97445740bbf7848488fbb6213e74d50a355547)
+
+commit e55b84e4cdb13839d8466cf162adb001d4134687
+Merge: ce3e387 ea9665b
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 25 14:57:17 2017 +0100
+
+    Merge pull request #12137 from jcsp/wip-17974
+    
+    jewel: client: fix stale entries in command table
+
+commit ce3e387766f2d6839798dfac96eacbf6334e4674
+Merge: 9dd1251 d0b0d41
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 25 14:56:24 2017 +0100
+
+    Merge pull request #12686 from SUSE/wip-18272-jewel
+    
+    jewel: tests: Workunits needlessly wget from git.ceph.com
+
+commit 9dd1251efdcd20ec9ce3467118b16b4d10a5360e
+Merge: a9c1fe0 85fbddd
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 25 14:56:03 2017 +0100
+
+    Merge pull request #12836 from SUSE/wip-18462-jewel
+    
+    jewel: Decode errors on backtrace will crash MDS
+
+commit a9c1fe0f693020d5e01aad5f7c2264939591cbf4
+Merge: 0bcd904 bf873a7
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 25 14:55:46 2017 +0100
+
+    Merge pull request #13023 from SUSE/wip-18603-jewel
+    
+    jewel: cephfs test failures (ceph.com/qa is broken, should be download.ceph.com/qa)
+
+commit 0bcd9049b469d4afcc547169ee1f9585557d3e0d
+Merge: 966bdbd 05e5a5a
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 25 14:55:28 2017 +0100
+
+    Merge pull request #12155 from dachary/wip-17956-jewel
+    
+    jewel: Clients without pool-changing caps shouldn't be allowed to change pool_namespace
+
+commit 966bdbd7a1ef7fbe1023b6e7bde0f78c2eb4ab7a
+Merge: d276861 e725605
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 25 14:55:11 2017 +0100
+
+    Merge pull request #12325 from dachary/wip-18026-jewel
+    
+    jewel: ceph_volume_client.py : Error: Can't handle arrays of non-strings
+
+commit d276861915979e6a82662e7b4eba0b399c6ddca8
+Merge: dd703bc 50a3fa1
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 25 14:54:51 2017 +0100
+
+    Merge pull request #13060 from asheplyakov/jewel-bp-18615
+    
+    jewel: mds: fix null pointer dereference in Locker::handle_client_caps
+
+commit dd703bc028eeb9c58dc1ef39ae506c73642ea948
+Merge: 347f71d 36186d0
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 25 14:54:35 2017 +0100
+
+    Merge pull request #11656 from ajarr/wip-17705-jewel
+    
+    jewel: ceph_volume_client: fix recovery from partial auth update
+
+commit 347f71dc21c51d61ea42269236568756c7d0742a
+Merge: e90396f 6efad69
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 25 14:54:06 2017 +0100
+
+    Merge pull request #12154 from dachary/wip-18008-jewel
+    
+    jewel: Cannot create deep directories when caps contain path=/somepath
+
+commit e90396f16c495df0ac74acff841703b919e596b9
+Merge: 72b24f0 89dcd8b
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 25 14:53:45 2017 +0100
+
+    Merge pull request #13085 from jcsp/wip-18361-jewel
+    
+    jewel: client: populate metadata during mount
+
+commit 85fbddd4f68ab159425c9198740c0d5787adc739
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Jan 24 15:49:24 2017 +0100
+
+    qa/tasks/cephfs/filesystem.py: backport _write_data_xattr() function
+    
+    This is a partial manual backport of 5f77f09b019b607b84e6a8f89ce19065383ca108
+    
+    It is needed by test_corrupt_backtrace() in qa/tasks/cephfs/test_damage.py
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 89dcd8b6287da7d97af0e7cad45ee023a5a39811
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Jan 13 00:30:28 2017 +0000
+
+    client: populate metadata during mount
+    
+    This way we avoid having to over-write the "root"
+    metadata during mount, and any user-set overrides (such
+    as bad values injected by tests) will survive.
+    
+    Because Client instances may also open sessions without
+    mounting to send commands, add a call into populate_metadata
+    from mds_command as well.
+    
+    Fixes: http://tracker.ceph.com/issues/18361
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 1dbff09ad553f9ff07f4f4217ba7ece6c2cdc5d2)
+
+commit ff9168803ebb2e3e0c4b42268765cd4b53e50d68
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 22 17:18:08 2016 -0500
+
+    msg/simple: clear_pipe when wait() is mopping up pipes
+    
+    When wait is mopping up connections it may hit one that
+    is in the process of accepting.  It will unregister it
+    whilst the accept() thread is trying to set it up,
+    aborting the accept and getting it reaped.  However,
+    the pipe mop-up does not clear_pipe() the way that
+    mark_down(), mark_down_all(), and fault() do, which
+    leads to this assert.
+    
+    Pipe is accepting...
+    
+      -161> 2016-12-22 17:31:45.460613 37353700 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=0 pgs=0 cs=0 l=1 c=0x3e2a6f40).accept:  setting up session_security.
+      -160> 2016-12-22 17:31:45.460733 37353700 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=0 pgs=0 cs=0 l=1 c=0x3e2a6f40).accept new session
+      -159> 2016-12-22 17:31:45.460846 37353700 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=2 pgs=7 cs=1 l=1 c=0x3e2a6f40).accept success, connect_seq = 1, sending READY
+      -158> 2016-12-22 17:31:45.460959 37353700 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=2 pgs=7 cs=1 l=1 c=0x3e2a6f40).accept features 1152921504336314367
+    
+    wait() is shutting down...
+    
+      -156> 2016-12-22 17:31:45.461882 9506ac0 20 -- 172.21.15.14:6804/20738 wait: stopping accepter thread
+      -155> 2016-12-22 17:31:45.462994 9506ac0 10 accepter.stop accept listening on: 15
+    ...
+      -116> 2016-12-22 17:31:45.482137 9506ac0 10 -- 172.21.15.14:6804/20738 wait: closing pipes
+      -115> 2016-12-22 17:31:45.482850 9506ac0 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=2 pgs=7 cs=1 l=1 c=0x3e2a6f40).unregister_pipe
+      -114> 2016-12-22 17:31:45.483421 9506ac0 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=2 pgs=7 cs=1 l=1 c=0x3e2a6f40).stop
+    
+    ...which interrupts the accept()...
+    
+      -113> 2016-12-22 17:31:45.484164 37353700 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=4 pgs=7 cs=1 l=1 c=0x3e2a6f40).accept fault after register
+    
+    and makes accept() return failure, and reader() to exit
+    and reap...
+    
+      -110> 2016-12-22 17:31:45.486103 9506ac0 10 -- 172.21.15.14:6804/20738 wait: waiting for pipes 0x3e2a5c20 to close
+      -109> 2016-12-22 17:31:45.487146 37353700 10 -- 172.21.15.14:6804/20738 queue_reap 0x3e2a5c20
+      -108> 2016-12-22 17:31:45.487658 9506ac0 10 -- 172.21.15.14:6804/20738 reaper
+      -107> 2016-12-22 17:31:45.487722 9506ac0 10 -- 172.21.15.14:6804/20738 reaper reaping pipe 0x3e2a5c20 172.21.15.35:0/146098963
+      -106> 2016-12-22 17:31:45.487816 9506ac0 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=4 pgs=7 cs=1 l=1 c=0x3e2a6f40).discard_queue
+      -105> 2016-12-22 17:31:45.494742 37353700 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=4 pgs=7 cs=1 l=1 c=0x3e2a6f40).reader done
+    ...
+       -92> 2016-12-22 17:31:45.527589 9506ac0 -1 /mnt/jenkins/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/11.1.0-6151-ge1781dd/rpm/el7/BUILD/ceph-11.1.0-6151-ge1781dd/src/msg/simple/SimpleMessenger.cc: In function 'void SimpleMessenger::reaper()' thread 9506ac0 time 2016-12-22 17:31:45.488264
+    /mnt/jenkins/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/11.1.0-6151-ge1781dd/rpm/el7/BUILD/ceph-11.1.0-6151-ge1781dd/src/msg/simple/SimpleMessenger.cc: 235: FAILED assert(!cleared)
+    
+    Fixes: http://tracker.ceph.com/issues/15784
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 948f97b3bdd39269a38277238a61f24e5fec6196)
+
+commit 50a3fa1ba1ab94ab736abf73830762afc0d05352
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Jan 6 15:42:52 2017 +0800
+
+    mds: fix null pointer dereference in Locker::handle_client_caps
+    
+    Locker::handle_client_caps delays processing cap message if the
+    corresponding inode is freezing or frozen. When the message gets
+    processed, client can have already closed the session.
+    
+    Fixes: http://tracker.ceph.com/issues/18306
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit e281a0b9c1fdeaf09f1b01f34cecd62e4f49d02e)
+
+commit 20e75023d6c9498ac19d3ee55e556063e94ea6e6
+Author: Wido den Hollander <wido@42on.com>
+Date:   Mon Jan 23 08:18:27 2017 +0100
+
+    systemd: Restart Mon after 10s in case of failure
+    
+    In some situations the IP address the Monitor wants to bind to
+    might not be available yet.
+    
+    This might for example be a IPv6 Address which is still performing
+    DAD or waiting for a Router Advertisement to be send by the Router(s).
+    
+    Have systemd wait for 10s before starting the Mon and increase the amount
+    of times it does so to 5.
+    
+    This allows the system to bring up IP Addresses in the mean time while
+    systemd waits with restarting the Mon.
+    
+    Fixes: #18635
+    
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit e73eb8cc1e0d45af1f0b7852c551f2ddfb82a520)
+
+commit aa0e45040f8b02c38a97670c41b8993742171cfb
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 22 17:19:42 2016 -0500
+
+    qa/distros: centos_7.yaml -> centos.yaml
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 004ef2c648c0c70a2efcd9b5f211369a2eb14f1d)
+
+commit b0c1e8863d84cbd04c74c54e6141a6a8355a3dee
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 22 15:41:25 2016 -0500
+
+    qa/suites: centos_7.2.yaml -> centos_7.yaml
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e9f119bda70554b79ca7f344571902c936c92f49)
+    
+    Conflicts:
+    	qa/suites/upgrade/jewel-x/point-to-point-x/distros (this directory used
+                to be a symlink to distros/supported)
+
+commit 8a98f06f25951a0113f2044ce2c062500eaea2e5
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 22 15:39:41 2016 -0500
+
+    qa/distros: add centos 7.3
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c168ce92aa1c1d834adecfe8fe32fe442ea7d21c)
+
+commit a45ad83c8f057da7054892d8ebc9767fc9238611
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 22 15:39:35 2016 -0500
+
+    qa/distros: add centos 7 yaml; use that instead
+    
+    No need to be picky about the point release here.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6e7db2329b917cf179390bd87937098f153aa513)
+
+commit 23680e080758be00a950e15594235b88b1e55b01
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Oct 6 19:39:20 2016 +0200
+
+    doc: document hostname constraints for rados bench
+    
+    Fixes: http://tracker.ceph.com/issues/17526
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 1d7e8188b9067f4f51958a13a23da1e105d89c63)
+
+commit 67e7a904373ee0863ca21eaa7da6edd559bb463d
+Author: Boris Ranto <branto@redhat.com>
+Date:   Thu Sep 29 12:08:39 2016 +0200
+
+    selinux: Allow ceph to manage tmp files
+    
+    Two new denials showed up in testing that relate to ceph trying to
+    manage (rename and unlink) tmp files. This commit allows ceph to manage
+    the files.
+    
+    Fixes: http://tracker.ceph.com/issues/17436
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit f8a0e201ee54759695ef44f7ed98b3b9705fafe3)
+
+commit 35e10a010f58fe50d9f236ae57db46e96b0ffed3
+Author: Mingxin Liu <mingxin@xsky.com>
+Date:   Mon Jan 2 13:20:10 2017 +0800
+
+    mon: do not send duplicated osdmap msg to not sync'ed osd
+    
+    prior to this change:
+    a peon may forward the pgstats to leader, and record it locally, but leader will
+    check if osd has the latest map before process, if not, will use a route op to
+    indicate peon to send it, then poen will delete routed op when fininaly send
+    out which make peon cannot send pgstatack when leader has processed the
+    pgstat update. so osd will always track it util reach a threshold block pgstats
+    sending, at worst, reopen mon session.
+    also, both leader and peon will send out the osdmap message to the osd.
+    
+    after this change:
+    only the peon will send out the osdmap message. and the pgstatack message
+    will be routed to the osd as expected. so the osd will not keep track of the
+    "acked" pg stats in its queue forever before times out.
+    
+    Fixes: http://tracker.ceph.com/issues/18458
+    Signed-off-by: Mingxin Liu <mingxin@xsky.com>
+    (cherry picked from commit 57274488c072ec6912b700288ce5b1ea8372d162)
+
+commit bcd4698115011810e7a8370ef04a5c3f0b9a3fb0
+Author: Ronak Jain <ronakjain@outlook.in>
+Date:   Fri Jan 13 16:57:45 2017 +0530
+
+    Doc: Fixes Python Swift client commands
+    
+    Fixes: http://tracker.ceph.com/issues/17746
+    Signed-off-by: Ronak Jain <ronakjain@outlook.in>
+    (cherry picked from commit 8c79959557d60f619adf1a3ed1b5bd1112ceaabb)
+
+commit c2bbf7ff4f929a63e0c7d6f70bb66e906a6c6cab
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Jan 21 14:24:25 2017 +0100
+
+    tests: run fs/thrash on xfs instead of btrfs
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 7a341a8886c93a8c4f266f8035784dd2e9528e3f
+Author: Michal Jarzabek <stiopa@gmail.com>
+Date:   Thu Jan 12 21:22:20 2017 +0000
+
+    client/Client.cc: prevent segfaulting
+    
+    The segfaulting in the rmdir function is caused by calling
+    filepath::last_dentry() function.
+    last_dentry() function assumes that the bits vector has always at
+    least one element, which is not the case for the the filepath object
+    created with "/" input.
+    This commit also fixes other functions affected by this bug:
+    link, unlink, rename, mkdir, mknod and symlink.
+    
+    Fixes: http://tracker.ceph.com/issues/9935
+    Signed-off-by: Michal Jarzabek <stiopa@gmail.com>
+    (cherry picked from commit 6ed7f2364ae5507bab14c60b582929aa7b0ba400)
+    
+    Conflicts:
+    	src/client/Client.cc (Client.cc - path_walk(), may_create(), and
+                                  _link() take fewer parameters in jewel)
+    	src/test/libcephfs/test.cc (preceding tests are missing in jewel)
+
+commit 173ea7f9856e37ee4febc5d1c13709bb81a4f37b
+Author: Samuel Matzek <smatzek@us.ibm.com>
+Date:   Mon Jan 16 11:11:31 2017 -0600
+
+    Ceph-disk to use correct user in check_journal_req
+    
+    The ceph-disk tool calls ceph-osd to check the journal requirements
+    using OSD id 0. This creates a log file for osd-0 on the system
+    using the current user/group for file permissions.  When ceph-disk
+    is run as root this makes the file owned by root which makes
+    the osd daemon for osd.0 unable to write to its own log file.
+    
+    This commit changes the journal reqs calls of ceph-osd to pass
+    the ceph user and group so ceph-osd creates the log file with the
+    appropriate permissions.
+    
+    Fixes: http://tracker.ceph.com/issues/18538
+    
+    Signed-off-by: Samuel Matzek <smatzek@us.ibm.com>
+    (cherry picked from commit bcf7514bf53693ec61e482341787c80494589faf)
+
+commit bf873a73cc3ee7e0fa5d7a1d2280fc3e3dd26ab0
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Jan 17 17:12:46 2017 +0100
+
+    qa: update remaining ceph.com to download.ceph.com
+    
+    Fixes: http://tracker.ceph.com/issues/18574
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 549d993d3fd8ffffa280ed4a64aca41d1c6f2da1)
+    
+    Conflicts:
+    	qa/tasks/cram.py (trivial resolution)
+
+commit 72b24f0f47bebaed793d7b75ce0eacc3c1533ab5
+Merge: 174ed80 95edad2
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jan 20 15:50:16 2017 +0100
+
+    Merge pull request #12766 from jtlayton/wip-18408-jewel
+    
+    client: Fix lookup of "/.." in jewel
+    
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+    Reviewed-by: Gregory Farnum <gfarnum@redhat.com>
+
+commit 174ed80fb77b172b6a7d3edb7c97cb49afe98ae8
+Merge: 01ff675 bee7e3a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jan 20 12:31:26 2017 +0100
+
+    Merge pull request #12147 from dachary/wip-18007-jewel
+    
+    jewel: ceph-disk: ceph-disk@.service races with ceph-osd@.service
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.cz>
+
+commit e8f55f65cc908cb43fe44c271866ed7672773f1f
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jan 5 16:06:45 2017 -0500
+
+    rgw: RGWAsyncRadosRequest drops notifier ref on cancel
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 1d586f76a11fed937fc7bb0f7cf6a44ca0506881)
+
+commit 3f509aac28fc81c464d3fc2a0b4c011260533e93
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Nov 7 10:58:50 2016 -0500
+
+    rgw: remove circular reference in RGWAsyncRadosRequest
+    
+    RGWAsyncRadosRequest does not need a reference to its calling coroutine,
+    it only needs the completion notifier. this eliminates a circular
+    reference between RGWAsyncRadosRequest and the coroutines that create
+    them
+    
+    Fixes: http://tracker.ceph.com/issues/17792
+    Fixes: http://tracker.ceph.com/issues/17793
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 7f670c0ea5de7c6969e1d332824d80c55c8d6af0)
+
+commit 0ef1bdf973ae94e982c1e678862645f87ea85bdd
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jan 4 11:36:00 2017 -0500
+
+    rgw: release RGWAioCompletionNotifier refs on destruction
+    
+    Fixes: http://tracker.ceph.com/issues/18407
+    Fixes: http://tracker.ceph.com/issues/18414
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 7a4975adba8b890e211d47662896e445a9822970)
+
+commit c21622d31d830d59278808805238e11888d39350
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Jan 6 12:30:42 2017 -0500
+
+    rgw_rados: add guard assert in add_io()
+    
+    Use the iterator-returning insert operation in std::map, check
+    assert the insert case.  As a side effect, this makes use of the
+    inserted object record more clear.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit d10c37adf925d8b34daa81b419463ea376ec27aa)
+
+commit 585eb484956c1666bfb870e67a9c506a8154294e
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Jan 6 15:16:32 2017 -0500
+
+    rgw_rados: sanitize dout print in GWRados::get_obj_iterate_cb(...)
+    
+    We cannot assume pbl may be deferenced.  Per review, move the dout
+    print into the r < 0 condition--since it's now an error, make it's
+    trace level 0.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit c0233c4abb9f99304e3d82272bbb4385086bbfc0)
+
+commit 01ff6751e1705094fe1c8057685fa7216bd56118
+Merge: 25a9e5f e2eaae4
+Author: vasukulkarni <vasu.kulkarni@gmail.com>
+Date:   Wed Jan 18 12:43:34 2017 -0800
+
+    Merge pull request #12983 from ceph/wip-cherry-pick-4vasu
+    
+    qa: Wip cherry pick https://github.com/ceph/ceph/pull/12969
+
+commit e2eaae4b4260df473cb5b7d9a467d5e4ca82d417
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Mon Jan 9 16:45:01 2017 -0800
+
+    Add ceph-create-keys to explicitly create admin/bootstrap keys
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 68f9b7eb3c0548c88650f67fb72c6ff9bc0f3ead)
+
+commit 2adc0ee858062a646380c8f594fa6a6207da29a3
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Mon Jan 9 17:59:20 2017 -0800
+
+    Remove debug overrides
+    
+    the high level of debug for mon/osd is causing remoto to hang during get key
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit f7dcc74cd3f119a2f65584fdb544c08d115f8c39)
+
+commit 8f36e23b49191f5a37184f9ee9b670025eecc4eb
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Tue Jan 10 15:43:12 2017 -0800
+
+    use the create option during instantiation
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit be836bb30960000468c79e08fb416ceefd79d7db)
+
+commit 25a9e5f526d2758543357d7072dbdc2ec9031e5f
+Merge: 601fb96 0ab5b7a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Jan 18 17:12:54 2017 +0100
+
+    Merge pull request #12210 from ddiss/tracker18049_ceph_disk_trigger_flock_timeout_jewel
+    
+    jewel: systemd/ceph-disk: reduce ceph-disk flock contention
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.cz>
+
+commit 601fb96599ebcb07017951542ddc58b2ab2abfd4
+Merge: de70003 1ea9de2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jan 17 08:41:25 2017 -0500
+
+    Merge pull request #12959 from SUSE/wip-18545-jewel
+    
+    jewel: [teuthology] update Ubuntu image url after ceph.com refactor
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 1ea9de2c14ebf6a4683cb0b796e92f86a58f3d59
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jan 16 22:12:51 2017 -0500
+
+    qa/tasks/qemu: update default image url after ceph.com redesign
+    
+    Fixes: http://tracker.ceph.com/issues/18542
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6d17befb3bbc3d83c9d23d763ad95e1e7b2e4be0)
+
+commit 36186d03ebd5c6eb14a720a0fca3a5bbeb49c0a9
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Tue Oct 11 14:18:29 2016 +0530
+
+    test_volume_client: remove superfluous arguments
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit bb60e01904187db417e8c7d6e57401823a0072fd)
+
+commit 75496812dc3d35a1737bc091b588ea7ad533ca54
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Tue Oct 11 13:40:43 2016 +0530
+
+    test_volume_client: check volume size
+    
+    Check that the total size shown by the df output of a mounted volume
+    is same as the volume size and the quota set on the volume.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 91c74f4778ce5433968226345ffe26e876eb56a7)
+
+commit 3e3ffcf7362a9a6c8fc8eebcc527e92330cf5a5e
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Tue Sep 6 17:31:04 2016 +0530
+
+    tasks/cephfs: test recovery of partial auth update
+    
+    ... in ceph_volume_client.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit f0134a3db576282ed05d4b94b969b9593297669d)
+
+commit 3320ef1944a10cc5835510ee752750237cbefc7a
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Tue Oct 4 13:55:46 2016 +0530
+
+    ceph_volume_client: fix partial auth recovery
+    
+    ... for volumes whose group_id is None.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 0ab8badcf3ffe685135af17dc28b238f6e686922)
+
+commit 5115c217bb9070aa9d0f0ea15a84cfb7862345b3
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Wed Sep 28 14:06:54 2016 +0530
+
+    ceph_volume_client: check if volume metadata is empty
+    
+    ... when recovering from partial auth updates.
+    
+    Auth update happens in the following order:
+    auth metadata update, volume metadata update, and then Ceph auth
+    update.
+    
+    A partial auth update can happen such that auth metadata is updated,
+    but the volume metadata isn't updated and is empty, and the auth
+    update did not propogate to Ceph. When recovering from such a
+    scenario, check if volume metadata is empty and if so remove the
+    partial auth update info in auth metadata.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit a95de7882cdf70e04e3c918ff41fc690d0d9bda3)
+
+commit bf33cd52a395b99453f72af016ea8f559ad3a13a
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Tue Oct 4 16:50:13 2016 +0530
+
+    ceph_volume_client: fix _recover_auth_meta() method
+    
+    It needs to be an instance method.
+    
+    Fixes: http://tracker.ceph.com/issues/17216
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 675cb91b68c1b54698708d604253ab9d1b2abdec)
+
+commit 84044262635cd8674dbda79b9b6a7ab29cb554c8
+Author: Xiaoxi Chen <xiaoxchen@ebay.com>
+Date:   Tue Jan 10 19:11:08 2017 -0700
+
+    mds/server: skip unwanted dn in handle_client_readdir
+    
+    We can skip unwanted dn which  < (offset_key, snap) via map.lower_bound, rather than
+    iterate across them.
+    
+    Previously we iterate and skip dn which < (offset_key, dn->last), as dn->last >= snap
+     means (offset_key, dn->last) >= (offset_key, snap), and such iterate_and_skip logic
+    still keep, so this commit doesnt change code logic but an optimization.
+    
+    Signed-off-by: Xiaoxi Chen <xiaoxchen@ebay.com>
+    (cherry picked from commit 52fe52baf920c672ac7f63a3087dcd31137891b6)
+
+commit de70003fda092d3fdf86b3323ab5a42038139c11
+Merge: 15c081c 06e40eb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Fri Jan 13 11:10:39 2017 +0100
+
+    Merge pull request #12745 from SUSE/wip-18386-jewel
+    
+    jewel: tests: use ceph-jewel branch for s3tests
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit a0ee8b930a41b56cd964b232e7230859d8d3b4aa
+Author: Pan Liu <pan.liu@istuary.com>
+Date:   Sun Jan 1 17:32:39 2017 +0800
+
+    jewel: fix compile error for dencode test case when --with-radosgw=no
+    
+    If cannot disable radosgw, the user has to always compile radosgw part,
+    even only want to use block device or file storage. Cherry-pick cannot
+    be done because ceph master doesn't have Makefile.am any more.
+    
+    Fixes: http://tracker.ceph.com/issues/18512
+    Signed-off-by: Pan Liu <pan.liu@istuary.com>
+
+commit 5e6c72946374895dca65f25d590615428ab3d8ef
+Author: Pan Liu <pan.liu@istuary.com>
+Date:   Sun Jan 1 17:19:35 2017 +0800
+
+    jewel: fixed compile error when --with-radosgw=no
+    
+    If cannot disable radosgw, the user has to always compile radosgw part,
+    even only want to use block device or file storage. Cherry-pick cannot
+    be done because ceph master doesn't have Makefile.am any more.
+    
+    Fixes: http://tracker.ceph.com/issues/18512
+    Signed-off-by: Pan Liu <pan.liu@istuary.com>
+
+commit cdd6cbfdfe5a77008ba298667bb7add8c236027a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Dec 9 09:39:39 2016 -0500
+
+    librbd: block concurrent in-flight object map updates for the same object
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7d743bfce61c6ede0a34fc0982e52be1d367d772)
+
+commit 15c081c6433e9ee4b6dd7c145e8e6aaddf334e69
+Merge: 988cd95 04cee05
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Jan 12 13:58:42 2017 -0800
+
+    Merge pull request #12912 from liewegas/wip-workunits-jewel
+    
+    qa/tasks/workunits: backport misc fixes to jewel
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 04cee0521681a49eca733d196011be3213c34682
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 22 13:05:22 2016 -0500
+
+    qa/tasks/workunit: clear clone dir before retrying checkout
+    
+    If we checkout ceph-ci.git, and don't find a branch,
+    we'll try again from ceph.git. But the checkout will
+    already exist and the clone will fail, so we'll still
+    fail to find the branch.
+    
+    The same can happen if a previous workunit task already
+    checked out the repo.
+    
+    Fix by removing the repo before checkout (the first and
+    second times).  Note that this may break if there are
+    multiple workunit tasks running in parallel on the same
+    role.  That is already racy, so if it's happening, we'll
+    want to switch to using a truly unique clonedir for each
+    instantiation.
+    
+    Fixes: http://tracker.ceph.com/issues/18336
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2a7013cd5a033c5be43350505d75f088e831e201)
+
+commit 1a98850578577dfe8a20df84be5e1638dc682e8b
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 16 15:06:16 2016 -0500
+
+    qa/tasks/workunit: retry on ceph.git if checkout fails
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 72d73b8c8836ae35c518fa09f44805a74038f02a)
+
+commit c101fba58b97015e77e5362d2db4112f8a8586fc
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 15 13:26:14 2016 -0500
+
+    qa/tasks/workunit.py: add CEPH_BASE env var
+    
+    Root of git checkout
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 27b8eac24922f8b4bd065e6e7f0bc8e2ba37b5d5)
+
+commit c7b74cda893e651750dc744185f009e46d115c0e
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 15 13:25:23 2016 -0500
+
+    qa/tasks/workunit: leave workunits inside git checkout
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4602884ab8f5a256d13091f7239d938990482d95)
+
+commit 384e5c05b4986e19a0ca45173d23da7d573d894a
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Dec 8 10:11:42 2016 -0500
+
+    rgw: add 'rgw log http headers' gloss to config-ref.rst
+    
+    Explain the option for upstream doc.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 04bf0f8ca1dfb949bc912b93a4ff84bf8bac57aa)
+
+commit 9fd29b440990c3458e4e65f22733d9da0c2606bc
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Dec 14 16:31:19 2016 -0500
+
+    use std::map
+    
+    (adaptation commit for #7639)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 8838bcbabcd2c9b3d7aec2403cad0bad4ad9a057
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Feb 12 17:02:53 2016 -0500
+
+    rgw: add rgw_log_http_headers option
+    
+    Tracks headers that should be handled conditionally (currently,
+    can only log, so using minimal structure to represent the
+    mapping).
+    
+    Adds map of custom headers to rgw_log_entry, and populate it with
+    headers pre-selected for custom logging in RGWREST.  Added to encoder
+    and Formatter output.
+    
+    Some additional optimization possible.
+    
+    (cherry picked from commit b82919a152217b3cd49afdc28bb890f329c2742a)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 87557754f591607a1667fecfc8dabc7e52bbbca0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Dec 7 22:38:47 2016 -0500
+
+    librbd: new block guard helper to prevent concurrent IO to blocks
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b1d624b43ec159d4a07616e86557ea48f089b7a1)
+    
+    Conflicts:
+    	src/librbd/BlockGuard.h: fixed compile issue
+    	src/librbd/Makefile.am: added BlockGuard
+    	src/test/Makefile-client.am: added BlockGuard test
+    	src/test/librbd/CMakeLists.txt: trivial resolution
+
+commit 5d306fd015d3e6d0aa35f368d3ca6cde6e7ca77b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Dec 8 18:27:33 2016 -0500
+
+    librbd: convert ObjectMap to template for unit testing
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ea7b30a4fb105e052603c55ac2dc2aca11e66545)
+    
+    Conflicts:
+    	src/librbd/image/CreateRequest.cc: not in jewel
+    	src/librbd/internal.cc: trivial resolution
+    	src/librbd/object_map/CreateRequest.cc: not in jewel
+    	src/librbd/object_map/RemoveRequest.cc: not in jewel
+    	src/test/librbd/test_ObjectMap.cc: trivial resolution
+
+commit c53df3780dd9221cfe602c09651eeee06046ebeb
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Dec 7 22:41:56 2016 -0500
+
+    librbd: clean up object map update interface
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 477ae54a568bd1081fd2c77b570b04dd1b983cd9)
+    
+    Conflicts:
+    	src/librbd/AioObjectRequest.cc: trivial resolution
+    	src/librbd/ObjectMap.cc: trivial resolution
+    	src/librbd/operation/TrimRequest.cc: removed optimizations
+    	src/tools/rbd_mirror/image_sync/ObjectCopyRequest.cc: trivial resolution
+
+commit 6fe9be8de391aa239fc82f5483508d762a44cb99
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Dec 7 16:28:22 2016 -0500
+
+    librbd: update in-memory object map after on-disk update committed
+    
+    Concurrent IO to the same object would previously result in the first
+    IO pausing to update the object map while the other IO would proceed
+    to directly update the object before the object map state was properly
+    updated.
+    
+    Fixes: http://tracker.ceph.com/issues/16176
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 378b810cbaadb1a12a7f0d21ed9a33e2a9640f55)
+    
+    Conflicts:
+    	test/librbd/object_map/test_mock_UpdateRequest.cc: resize op signature
+
+commit 988cd95d84be7973f774a4dae2d09faf4dbc30c3
+Merge: d4b1341 895ab24
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Jan 12 07:29:34 2017 +0100
+
+    Merge pull request #12791 from athanatos/wip-15943-jewel
+    
+    jewel: crash adding snap to purged_snaps in ReplicatedPG::WaitingOnReplicas (part 2)
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f24c3fffc0ee26fddbc416620bbb9c0b353a0cf2
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 30 17:28:59 2016 -0500
+
+    osd/PG: publish PG stats when backfill-related states change
+    
+    These frequently get flushed because other updates
+    happen, but we should explicitly ensure that the mon
+    sees these state changes.
+    
+    Fixes: http://tracker.ceph.com/issues/18369
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d4adeb7500a113097fdd717ada0231f68badafbb)
+    
+    Conflicts:
+    	src/osd/PG.cc: trivial - PG::RecoveryState::Recovering::Recovering()
+    	hunk fails to apply since there's no PG::queue_recovery(),
+    	it's pg->osd->queue_for_recovery(pg) in jewel
+    
+    Fixes: http://tracker.ceph.com/issues/18485
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+
+commit d4b1341cf38f430f047f7263d713c0161b44730f
+Merge: 5b402f8 182babf
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Jan 10 16:25:18 2017 -0800
+
+    Merge pull request #12868 from athanatos/wip-17899-jewel
+    
+    OSDMonitor: only reject MOSDBoot based on up_from if inst matches
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit fe753dbc9b4bd03d7a6063724edc59c2f860fccf
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Tue Jan 10 09:42:18 2017 +0100
+
+    install-deps.sh: unify indentation in case statement
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+
+commit 94ab8d85b59ff8bd8919930b9d912d1bf55eb9ed
+Author: Nitin A Kamble <Nitin.Kamble@Teradata.com>
+Date:   Mon Oct 31 11:46:13 2016 -0700
+
+    install-deps.sh: allow building on SLES systems
+    
+    Avoids this error on SLES systems:
+    
+    > ./install-deps.sh
+    sles is unknown, dependencies will have to be installed manually.
+    
+    Signed-off-by: Nitin A Kamble <Nitin.Kamble@Teradata.com>
+    (cherry picked from commit 266f4d4f20a1756e825ee54d79fe5f8d931d4b77)
+    
+        Conflict in install-deps.sh due to indentation. Trivial resolution.
+
+commit 7159265ac154ab798f64272b97aa5d7154d0b303
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jan 5 13:40:41 2017 +0000
+
+    qa/tasks: add test_corrupt_backtrace
+    
+    Validate that we get EIO and a damage table entry
+    when seeing a decode error on a backtrace.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 5f6cdab80f6e2f09af5783c8f616d8ddd6d9f428)
+
+commit 893d4ab72fec1529550b28ee930be8de5a74ee2b
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Dec 20 18:04:47 2016 +0000
+
+    mds: check for errors decoding backtraces
+    
+    Fixes: http://tracker.ceph.com/issues/18311
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 6f489c74ac0040631fde0ceb0926cbab24d3ad55)
+
+commit 895ab243fe5d3bd07bff0bd422d1dc9cb561f0a0
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Dec 12 10:35:38 2016 -0800
+
+    PG: fix cached_removed_snaps bug in PGPool::update after map gap
+    
+    5798fb3bf6d726d14a9c5cb99dc5902eba5b878a actually made 15943 worse
+    by always creating an out-of-date cached_removed_snaps value after
+    a map gap rather than only in the case where the the first map after
+    the gap did not remove any snapshots.
+    
+    Introduced: 5798fb3bf6d726d14a9c5cb99dc5902eba5b878a
+    Fixes: http://tracker.ceph.com/issues/15943
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 5642e7e1b3bb6ffceddacd2f4030eb13a17fcccc)
+
+commit 299478a1e69c9da8dc902e2f102d5d272412fdc8
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Dec 14 15:48:59 2016 -0800
+
+    qa/config/rados.yaml: enable osd_debug_verify_cached_snaps
+    
+    Also, make map gaps more likely.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit d4b6615a49e4635113f9ba900e9c57147b224357)
+
+commit b492c0089de858cfe2d7030eddd046acac6872ab
+Author: Samuel Just <sjust@redhat.com>
+Date:   Mon Dec 12 10:33:13 2016 -0800
+
+    PG::handle_advance_map: add debugging option to verify cached_removed_snaps
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit aeb8fef92469831d94f06db457a4ba15b5b0e3c5)
+
+commit 2296c87e2e8c885c5c6bee1ea1807aa7862f88fc
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Nov 24 18:23:43 2016 +0100
+
+    osd: improve error message when FileStore op fails due to EPERM
+    
+    References: http://tracker.ceph.com/issues/18037
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit f7723e59b6e10e1aef35543639e0ea1fcac65574)
+
+commit 7819adb3b5f9af813d4df05d3483175ee54e10df
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Fri Aug 12 18:48:12 2016 -0400
+
+    client: don't use special faked-up inode for /..
+    
+    The CEPH_INO_DOTDOT thing is quite strange. Under most OS (Linux
+    included), the parent of the root is itself. IOW, at the root, '.' and
+    '..' refer to the same inode.
+    
+    Change the ceph client to do the same, as this allows users to get
+    valid stat info for '..', as well as elimnating some special-casing.
+    
+    Also in several places, we're checking dn_set.empty as an indicator
+    of being the root. While that is true for the root, it's also true
+    for unlinked directories.
+    
+    This patch has treats them the same. An unlinked directory will
+    be reparented to itself, effectively acting as a root of its own.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 30d4ca01db0de9a1e12658793ba9bf9faf0331dd)
+
+commit 95edad2d058575231bb9b5468559f7b42667f40e
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Tue Jan 3 12:56:51 2017 -0500
+
+    client: don't use special faked-up inode for /..
+    
+    The CEPH_INO_DOTDOT thing is quite strange. Under most OS (Linux
+    included), the parent of the root is itself. IOW, at the root, '.' and
+    '..' refer to the same inode.
+    
+    Change the ceph client to do the same, as this allows users to get
+    valid stat info for '..', as well as elimnating some special-casing.
+    
+    Also in several places, we're checking dn_set.empty as an indicator
+    of being the root. While that is true for the root, it's also true
+    for unlinked directories.
+    
+    This patch has treats them the same. An unlinked directory will
+    be reparented to itself, effectively acting as a root of its own.
+    
+    Fixes: http://tracker.ceph.com/issues/18408
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 30d4ca01db0de9a1e12658793ba9bf9faf0331dd)
+
+commit 9a59ce946c4f73d200969d9352b7049c6690eaf4
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Sep 19 15:05:22 2016 +0200
+
+    rgw: fix decoding of creation_time and last_update.
+    
+    Fixes: http://tracker.ceph.com/issues/17167
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 3afe715381ca63539bb20609b614f5e60561711a)
+
+commit bbf4c27cab6476e284feed1fa151cca8763cd442
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 15 13:35:02 2016 -0500
+
+    qa/tasks/admin_socket: subst in repo name
+    
+    It is either ceph.git or ceph-ci.git.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit bcbe45d948f1c4da02e27b3be5f29a8b52745e28)
+
+commit 944ec0348e5b4cd69c8aedda833cf5e468a98dc9
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Dec 13 18:54:17 2016 +0200
+
+    rbd-nbd: invalid error code for "failed to read nbd request" messages
+    
+    Fixes: http://tracker.ceph.com/issues/18242
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 42c2514fe22d34e1bd5a5d2686e06f7e01b7c0c3)
+
+commit fcdd5e75c3a3107e4b8c8562036fd54e8ea40ddf
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Dec 2 13:36:35 2016 -0500
+
+    rados: optionally support reading omap key from file
+    
+    Fixes: http://tracker.ceph.com/issues/18123
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 286ceb1e035f060cc564c1ef7400382331893101)
+
+commit d313e42db210e3ab04b5e7e513f7f2660ba1aa1d
+Author: runsisi <runsisi@zte.com.cn>
+Date:   Wed Dec 14 15:18:18 2016 +0800
+
+    librbd: ignore error when object map is already locked by current client
+    
+    otherwise when using rbd cli to rollback image with object-map feature
+    enabled, the following error message will be printed out on the screen,
+    which is confusing to users:
+      librbd::object_map::LockRequest: failed to lock object map: (17) File exists
+    
+    Fixes: http://tracker.ceph.com/issues/16179
+    
+    Signed-off-by: runsisi <runsisi@zte.com.cn>
+    (cherry picked from commit 711da4e2396a8967cd457133a847d40cb46b0a5a)
+
+commit d0b0d4114bb5933bbc6b707e781f1a9262b56fd5
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Jan 2 22:49:13 2017 +0100
+
+    tests: rbd/test_lock_fence.sh: fix rbdrw.py relative path
+    
+    This commit fixes a regression introduced by
+    cf294777ea92f0911813a7132068584d4f73a65a
+    
+    Fixes: http://tracker.ceph.com/issues/18388
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 91231de16dbe4d0e493ec617165a2b38078d122b)
+
+commit 06e40eb27f65df861b8f9992f0706fbc464d3f39
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Jan 4 10:03:08 2016 +0100
+
+    tests: use ceph-jewel branch for s3tests
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 8877ee4d48b025c6dc171f3ba6064c72b9171ddc
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Fri Dec 16 12:50:50 2016 +0200
+
+    rbd: fix json formatting for image and journal status output
+    
+    Fixes: http://tracker.ceph.com/issues/18261
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 4c2659a0f49ff5f709a8275d6adaa817daf8f76a)
+
+commit 9c84a657e25ccb32be7f775fef3d4c563d4e95b9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Dec 14 20:49:46 2016 -0500
+
+    journal: prevent repetitive error messages after being blacklisted
+    
+    Fixes: http://tracker.ceph.com/issues/18243
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 60f1b1a2f2a96cfe72acfc6855b9cc26b480732d)
+
+commit d069464be70c3a6286cb7918eb83d555e1d52759
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Dec 13 14:40:23 2016 -0500
+
+    journal: avoid logging an error when a watch is blacklisted
+    
+    Fixes: http://tracker.ceph.com/issues/18243
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b82a1c1d1fc43e13aa6e76f5468f43a1f779f7fa)
+
+commit 9a1258de21d4ee2b965b3cdeb65bd71ceac573c1
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Dec 12 16:42:15 2016 -0500
+
+    rgw: use explicit flag to cancel RGWCoroutinesManager::run()
+    
+    RGWCoroutinesManager::run() was setting ret = -ECANCELED to break out of
+    the loop when it sees going_down. coroutines that failed with -ECANCELED
+    were confusing this logic and leading to coroutine deadlock assertions
+    below. when we hit the going_down case, set a 'canceled' flag, and check
+    that flag when deciding whether to break out of the loop
+    
+    Fixes: http://tracker.ceph.com/issues/17465
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 73cd8df887fb5e45f2d49275cedfeab31809ddc8)
+
+commit a67dca4a45cf68625b6f673b251e101fe713e5b5
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 22 13:05:22 2016 -0500
+
+    qa/tasks/workunit: clear clone dir before retrying checkout
+    
+    If we checkout ceph-ci.git, and don't find a branch,
+    we'll try again from ceph.git. But the checkout will
+    already exist and the clone will fail, so we'll still
+    fail to find the branch.
+    
+    The same can happen if a previous workunit task already
+    checked out the repo.
+    
+    Fix by removing the repo before checkout (the first and
+    second times).  Note that this may break if there are
+    multiple workunit tasks running in parallel on the same
+    role.  That is already racy, so if it's happening, we'll
+    want to switch to using a truly unique clonedir for each
+    instantiation.
+    
+    Fixes: http://tracker.ceph.com/issues/18336
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2a7013cd5a033c5be43350505d75f088e831e201)
+
+commit e5c81c34bb766d94d2ef21b407d70d2d089c425b
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 16 15:06:16 2016 -0500
+
+    qa/tasks/workunit: retry on ceph.git if checkout fails
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 72d73b8c8836ae35c518fa09f44805a74038f02a)
+
+commit efaedb355dbc4697a3f9c3147053a5c92642e785
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 19 14:08:11 2016 -0500
+
+    qa/workunits: include extension for nose tests
+    
+    When you have a relative path you have to include the extension.
+    Weird.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5666fd61d6dbd40be1d79354227cabd562e829ea)
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    
+    Conflicts:
+    	qa/workunits/rados/test_python.sh (nosetests instead of nose)
+
+commit de15912c3bb714f9ff48ad8e10b8b4dd6961069e
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 15 15:10:28 2016 -0500
+
+    qa/workunits: use relative path instead of wget from git
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit cf294777ea92f0911813a7132068584d4f73a65a)
+    
+    Conflicts:
+    	qa/workunits/rados/test_python.sh (nosetests instead of nose)
+
+commit 74aac99ce20462de0aca042b6128ea479d17bb4b
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 15 13:26:14 2016 -0500
+
+    qa/tasks/workunit.py: add CEPH_BASE env var
+    
+    Root of git checkout
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 27b8eac24922f8b4bd065e6e7f0bc8e2ba37b5d5)
+
+commit 117d38e2382bc0016729387aba1dfaae9bb7a684
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 15 13:25:23 2016 -0500
+
+    qa/tasks/workunit: leave workunits inside git checkout
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4602884ab8f5a256d13091f7239d938990482d95)
+
+commit 5aa9387555c98666a83f909646b62c8688ac24d7
+Author: Weibing Zhang <atheism.zhang@gmail.com>
+Date:   Thu Dec 22 14:38:27 2016 +0800
+
+    rgw: ldap: simple_bind() should set ldap option on tldap
+    
+        ldap_set_option() should set option for "tldap" other than "ldap".
+        The current code will lead to "Protocol error" as the ldap version
+        of tldap is not set.
+    
+    Signed-off-by: Weibing Zhang <zhangweibing@unitedstack.com>
+    (cherry picked from commit 96cf7fa06a1192293b24c823ad9d08456f81ac8a)
+
+commit dcc9483cbd605997876983ec65f6fb80d5da8a52
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Thu Apr 7 16:43:17 2016 +0800
+
+    mon: OSDMonitor: trigger an immediate propose if any newly down osd is detected during tick()
+    
+        Currently we rely on OSDs to watch over each other and perform failure detection
+        and report to OSDMonitor. Before we can safely and undoubtedly mark an OSD as down,
+        enough reports from a certain number of different reporters must have been collected.
+        Also, the victimed OSD has to be declared failed long enough before we make any final
+        decision in order to avoid temperary problems such as network failure, network traffic jam etc.,
+        which if handled carelessly, may cause even serious problem such as flapping.
+    
+        Form the above analysis, even if we have gathered enough witnesses, we have to wait long
+        enough to sentence the guilty OSD to death. Therefore we rely on the tick() thread to
+        do such an hourglass job. However, the problem here is currently the tick() thread is
+        unable to trigger a propose even if it has witnessed such a memont, and this is our goal
+        to solve such an embrassing situation.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit a80d6c500ab247013a1c068c457c1b9bfbc750b2)
+
+commit 64c0caeab21f1f764bb48c0355b24f24dda2cfe2
+Author: Xiaoxi Chen <xiaoxchen@ebay.com>
+Date:   Fri Dec 9 02:40:03 2016 +0800
+
+    librbd/diff_iterator: use proper snap to query parent overlap
+    
+    If no snapshot has been made, we will use CEPH_NOSNAP instead of 0,
+    to query the parent overlap.
+    
+    Fixes bug http://tracker.ceph.com/issues/18200
+    
+    Signed-off-by: Xiaoxi Chen <xiaoxchen@ebay.com>
+    (cherry picked from commit a88ceff2767761483aee73590767c412116a7489)
+
+commit d584f9e124cfd1da1b8d8dbfa1a0d370185c6835
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Mon May 9 11:41:19 2016 +0200
+
+    rgw: log name instead of id for SystemMetaObj on failure
+    
+    Currently if we fail to read a SystemMetaObj we try to log the
+    MetaObject id, however this will not be set mostly as read_id has
+    failed, so we end up logging an empty id, changing this to log
+    the object name instead
+    
+    Fixes: http://tracker.ceph.com/issues/15776
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit e9f896a9efea74c42d56dd5a7feb8b8710d6becc)
+
+commit 1a0becf4e45e753503d27719540b966d74fdafff
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Mon May 9 14:05:06 2016 +0200
+
+    rgw: drop unnecessary spacing in rgw zg init log
+    
+    Dropping unneeded space when we're printing the failed reading zg info
+    message
+    
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit a43ac6e56857ac9b2bf7d6aa2956b7bcc045d5ce)
+
+commit 5b402f8a7b5a763852e93cd0a5decd34572f4518
+Merge: e565d89 3dbf0c9
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Dec 22 00:18:11 2016 +0100
+
+    Merge remote-tracking branch 'ceph/jewel-next' into jewel
+
+commit e565d8924241ba0520a836fc96040de41582906f
+Merge: f97b399 5799d5f
+Author: jtlayton <jlayton@redhat.com>
+Date:   Wed Dec 21 09:18:18 2016 -0500
+
+    Merge pull request #12591 from jtlayton/wip-18308-jewel
+    
+    Clear setuid bits on ownership changes
+
+commit 5799d5fdba9b890634a9b3d6fa038dfde3818779
+Merge: a0ae9a8 f97b399
+Author: jtlayton <jlayton@redhat.com>
+Date:   Tue Dec 20 15:36:39 2016 -0500
+
+    Merge branch 'jewel' into wip-18308-jewel
+
+commit f97b399896261612aa0b8bab2c720446d908a644
+Merge: c067b58 f15c8da
+Author: jtlayton <jlayton@redhat.com>
+Date:   Tue Dec 20 15:35:54 2016 -0500
+
+    Merge pull request #12592 from jtlayton/wip-18307-jewel
+    
+    Fix mount root for ceph_mount users and change tarball format
+
+commit f15c8dafbcc15982265f072aeddb9d5dfc7166d6
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Tue Dec 20 14:44:04 2016 -0500
+
+    ceph_disk: fix a jewel checkin test break
+    
+    Silly python:
+    
+        ceph_disk/main.py:173:1: E305 expected 2 blank lines after class or function definition, found 1
+        ceph_disk/main.py:5011:1: E305 expected 2 blank lines after class or function definition, found 1
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+
+commit 8e0cffdd435eeb24f4ea3658cf1db9973759dd29
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Tue Dec 20 11:54:25 2016 -0500
+
+    automake: convert to tar-pax
+    
+    We hit some recent build issues with the merge of ceph-qa-suite into
+    the main repo. The ustar format barfs on >100 character symlink
+    paths.
+    
+    Convert to using "tar-pax" which should make it use the posix format.
+    Any build machine that we're reasonably targeting should support it.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+
+commit a0ae9a83df222fce75f41d3b5a3e3d0396dd748a
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Tue Dec 20 08:17:21 2016 -0500
+
+    client: drop setuid/setgid bits on ownership change
+    
+    When we hold exclusive auth caps, then the client is responsible for
+    handling changes to the mode. Make sure we remove any setuid/setgid
+    bits on an ownership change.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 18d2499d6c85a10b4b54f3b8c335cddf86c4588f)
+
+commit d49e628361a17cca65f31f85350468228a0ec2d7
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Tue Dec 20 08:16:43 2016 -0500
+
+    mds: clear setuid/setgid bits on ownership changes
+    
+    If we get a ownership change, POSIX mandates that you clear the
+    setuid and setgid bits unless you are "appropriately privileged", in
+    which case the OS is allowed to leave them intact.
+    
+    Linux however always clears those bits, regardless of the process
+    privileges, as that makes it simpler to close some potential races.
+    Have ceph do the same.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 6da72500882d9749cb2be6eaa2568e6fe6e5ff4d)
+
+commit f2dfc200dbf93927ca1059ae203c8242b23eeb40
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Tue Dec 20 08:07:23 2016 -0500
+
+    client: set metadata["root"] from mount method when it's called with a pathname
+    
+    Currently, we only set the root properly config file or the
+    --client_metadata command line option. If a userland client program
+    tries to call ceph_mount with a pathname, it's not being properly
+    set.
+    
+    Since we already hold the mutex, we can just update it directly.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 9f8810008c82eebe6e354e7e321e33a3dcba8407)
+
+commit 87a2a95ad1103cecf4b710776d35182920e72823
+Author: Pritha Srivastava <prsrivas@redhat.com>
+Date:   Mon Aug 29 14:32:42 2016 +0530
+
+    rgw: Replacing '+' with "%20" in canonical uri for s3 v4 auth.
+    
+    s3cmd encodes space as "%20" while signature computation and
+    encodes space as '+' while sending the canonical uri. This
+    results in a SignatureMismatch Error in rgw, since rgw
+    computes the signature based on the request received from
+    the client (s3cmd in this case).
+    
+    Fixes http://tracker.ceph.com/issues/17076.
+    
+    Signed-off-by: Pritha Srivastava <prsrivas@redhat.com>
+    (cherry picked from commit 20e5ff023ebad89c386a520d07613547d4836399)
+
+commit aa8e57d0d4fa855ec6b2201274dd78504ab9e5c9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 6 12:56:31 2016 -0400
+
+    rbd: utilize new API methods for image id and block name prefix
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0a0a88c71552aa5858384fa802a3161da90e7c86)
+    
+     Conflicts:
+    	src/tools/rbd/action/Info.cc (jewel does not have
+                    653bc453e3c8f1062cdbc4d0d8f77f623f48915b)
+
+commit 05295eff568aec5ff9f5d01bb73b7c7bd1dee581
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 6 12:48:22 2016 -0400
+
+    librbd: new API methods to retrieve image id and block name prefix
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 366e6075cab2748efab395cd23882eaee4ba402f)
+
+commit 240431bea219412edf9588490f6d72d16b1f01b6
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Sep 5 15:40:33 2016 +0200
+
+    build/ops: fix /etc/os-release parsing in install-deps.sh
+    
+    85a370e35fc42031a7f7e24dea9e50a649c0f309 introduced the DISTRO variable whose
+    value was obtained by parsing /etc/os-release like this:
+    
+    DISTRO=$(grep  "^ID=" /etc/os-release | sed "s/ID=//")
+    
+    This unfortunately picks up the double-quotes, so on a CentOS system DISTRO
+    will be equal to '"centos"'.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit fd37e87f6de0b8e272a2859ec52d6ceab277db7a)
+
+commit f6519509049362ec3f378e2178e577f761a72c42
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Wed Jun 29 09:40:07 2016 +0200
+
+    install-deps.sh: initial distro detection based on /etc/os-release
+    
+    This avoids initial distribution detection (before lsb_release is available)
+    based on the presence of a specific package manager. This caused an
+    issue with openSUSE Tumbleweed before since both zypper and apt-get are
+    available.
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 85a370e35fc42031a7f7e24dea9e50a649c0f309)
+
+commit c067b589128fac3fb382f64ac91ade3be3c69a51
+Merge: c461ee1 d2380d7
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 14 11:39:56 2016 -0600
+
+    Merge pull request #12454 from liewegas/qa-suite-jewel
+    
+    jewel: merge ceph-qa-suite
+
+commit d2380d7d1d4fd568b286e0fb90d8d192f361f7c9
+Merge: c461ee1 1c28e7f
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 14 11:29:59 2016 -0600
+
+    merge ceph-qa-suite
+
+commit 1c28e7f08d0e42a14e28b78e12b57a219ce8796c
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 14 11:29:55 2016 -0600
+
+    move ceph-qa-suite dirs into qa/
+
+commit 282451d8cf8d98ca90502e29081aaf7c4a4c9508
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 14 12:27:58 2016 -0500
+
+    Revert "tasks/workunit.py: depth 1 clone"
+    
+    This reverts commit e6f61ea9f19d0f1fad4a6547775fa80616eeeb89.
+
+commit e6f61ea9f19d0f1fad4a6547775fa80616eeeb89
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 14 12:19:44 2016 -0500
+
+    tasks/workunit.py: depth 1 clone
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4faf77a649cb3f8ddf497ca81937b3dbf63a18dc)
+
+commit 426f7cf2d151a1122cb603a2c9f224a25c9702a2
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 14 12:18:29 2016 -0500
+
+    tasks/workunit: remove kludge to use git.ceph.com
+    
+    This was hard-coded to ceph.git (almost) and breaks when
+    you specify --ceph-repo.  Remove it entirely.  We'll see if
+    github.com is better at handling our load than it used to
+    be!
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 159c455a0326eef2c017b3e3cf510f918b5ec76c)
+
+commit 1ba5995b7d9816bab936bc1ba557504821e94c89
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Dec 10 02:36:52 2016 +0800
+
+    tasks/ceph: restore context of osd mount path before mkfs
+    
+    all newly created files and directories under the mount dir inherit the
+    SELinux type of their parent directory. so we need to set it before
+    mkfs.
+    
+    Fixes: http://tracker.ceph.com/issues/16800
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 53225d5272a1d35d4183fcfa55a139f55f77e122)
+
+commit 26c87fd1f1d105e5aa1c4c32c73f75414ea3d04f
+Author: Pan Liu <pan.liu@istuary.com>
+Date:   Thu Dec 8 20:03:30 2016 +0800
+
+    rbd: --max_part and --nbds_max options for nbd map
+    
+    Fixes: http://tracker.ceph.com/issues/18186
+    Signed-off-by: Pan Liu <pan.liu@istuary.com>
+    (cherry picked from commit 45ceb389b685a5ec7bc0b8ef9e180ce851646082)
+    
+    Conflicts:
+    	src/test/cli/rbd/help.t (no --exclusive in jewel-next)
+    	src/tools/rbd/action/Nbd.cc (no --exclusive in jewel-next)
+
+commit ddb5403605f843b4bc5abd1093b034e087c07f8b
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Nov 30 14:36:38 2016 -0500
+
+    radosgw-admin: 'zone placement modify' doesnt require pool names
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit b59afea4dbb06454b1e14f03dd65ba2449674a15)
+    
+    Conflicts:
+    	src/rgw/rgw_admin.cc (jewel-next lacks --tier* options)
+
+commit 7cfc346cf1427e37be38ebb92ad9518afa982b83
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Nov 30 13:43:28 2016 -0500
+
+    radosgw-admin: add 'zonegroup placement default' command
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 711cc132528f2dfeb6c1f8defb19493ffe7cb9be)
+
+commit dbc1b614add263fe80ef6fc0c09c0597fb9cddff
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Nov 30 13:08:38 2016 -0500
+
+    radosgw-admin: fix 'placment' typos
+    
+    Fixes: http://tracker.ceph.com/issues/18078
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 0cf08985683820cd11492a9ad684d06565a04f34)
+
+commit 4f7147cb79aa0637e156a3a29739fcbadc37e62d
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Nov 29 14:55:52 2016 -0800
+
+    rgw_admin: commands to manage placement targets
+    
+    Fixes: http://tracker.ceph.com/issues/18078
+    
+    added the following commands:
+    
+     radosgw-admin zone placement add
+     radosgw-admin zone placement modify
+     radosgw-admin zone placement rm
+     radosgw-admin zone placement list
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit ead132ae84bf3b2738fa0443f4f9666d59edbb43)
+    
+    Conflicts:
+    	src/rgw/rgw_admin.cc (jewel-next lacks the --tier* and --sync-from* options)
+    	src/test/cli/radosgw-admin/help.t (jewel-next lacks the --tier* and --sync-from* options)
+
+commit 155641f9ce6a16ee1fdb3231d76bea0eafc89ff8
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Nov 29 14:15:09 2016 -0800
+
+    rgw-admin: add commands to manage zonegroup placement fields
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit a91c7b5cc699b2c37d449974d2ebf2eece9506a0)
+    
+    Conflicts:
+    	src/rgw/rgw_admin.cc (chose raw_storage_ops_list from master)
+
+commit bb9678bf44b8ed0dd8bb1f895da3c78edc792eeb
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Nov 29 14:05:27 2016 -0800
+
+    rgw: use set for zonegroup placement target tags
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 12162d654ce24db0617e4bc3de24aebc28b50996)
diff --git a/doc/changelog/v10.2.7.txt b/doc/changelog/v10.2.7.txt
new file mode 100644
index 000000000..473f45e1d
--- /dev/null
+++ b/doc/changelog/v10.2.7.txt
@@ -0,0 +1,1530 @@
+commit 50e863e0f4bc8f4b9e31156de690d765af245185
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Mon Apr 10 11:43:44 2017 +0000
+
+    10.2.7
+
+commit a64d3e4b33e904aa6585464df8ffff6aafdec10c
+Merge: c92640a 6c2a40a
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Fri Mar 31 10:27:07 2017 -0600
+
+    Merge pull request #14230 from linuxbox2/wip-jewel-expand-argv
+    
+    jewel: rgw_file: expand argv
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit c92640a113843a491678c11319d2352f14486f17
+Merge: 7c35658 fcc3ada
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Fri Mar 31 10:17:10 2017 -0600
+
+    Merge pull request #14233 from ktdreyer/wip-19421-jewel
+    
+    jewel: librbd: possible race in ExclusiveLock handle_peer_notification
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 7c3565899af5f1f474d82d8802d15c6c216ea894
+Merge: 12467f6 eedb9f7
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Fri Mar 31 10:16:48 2017 -0600
+
+    Merge pull request #14231 from linuxbox2/wip-jewel-fhcache
+    
+    jewel: rgw_fh: RGWFileHandle dtor must also cond-unlink from FHCache
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 12467f6c3f346072d81c444ed68eb206402d8c33
+Merge: 2d2521c 78c8be7
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Fri Mar 31 10:09:14 2017 -0600
+
+    Merge pull request #14215 from mdw-at-linuxbox/wip-jewel-rgw-openssl-fix1
+    
+    rgw: fix openssl
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 2d2521c0300000421740c611a5133c82d2e4d0f6
+Merge: 0840cf8 7e4e290
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Fri Mar 31 10:07:48 2017 -0600
+
+    Merge pull request #14206 from linuxbox2/wip-gui-jewel
+    
+    jewel: rgw_file: various fixes
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 0840cf86214c793794c6f913b63cc67e0ab3fc94
+Merge: 0bf6360 b24a8c2
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Fri Mar 31 10:07:03 2017 -0600
+
+    Merge pull request #14205 from linuxbox2/wip-dir-orphan-jewel
+    
+    jewel:  rgw_file: wip dir orphan
+    
+     Conflicts:
+    	src/rgw/rgw_file.cc (whitespace, trivial resolution)
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 0bf636057ba853d1d1a777147014a740d5149b17
+Merge: 83b5a7f fb85c68
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Fri Mar 31 10:05:40 2017 -0600
+
+    Merge pull request #14169 from linuxbox2/wip-18650-jewel
+    
+    jewel: rgw: rgw_file: FHCache residence check should be exhaustive
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 83b5a7f1c589546645e0f377d5d5f7b1f7363a59
+Merge: 6a175f2 a969c44
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Fri Mar 31 10:02:58 2017 -0600
+
+    Merge pull request #13869 from smithfarm/wip-19161-jewel
+    
+    jewel: rgw: rgw_file: fix marker computation
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit fb85c68dfbfcfb299cac3a7e6723067f216483ef
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Feb 28 15:49:06 2017 -0500
+
+    rgw_file:  use fh_hook::is_linked() to check residence
+    
+    Previously we assumed that !deleted handles were resident--there
+    is an observed case where a !deleted handle is !linked.  Since
+    we currently use safe_link mode, an is_linked() check is
+    available, and exhaustive.
+    
+    Fixes: http://tracker.ceph.com/issues/19111
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit c0aa515f8d8c57ec5ee09e3b60df3cac60453c40)
+
+commit fcc3ada733bac74a642ccd0f1ed6301ad7318a30
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Mar 28 15:55:11 2017 +0200
+
+    jewel: librbd: possible race in ExclusiveLock handle_peer_notification
+    
+    This is a cherry-pick from kraken -- the master diverged after
+    ManagedLock refactoring and is not affected.
+    
+    Fix: http://tracker.ceph.com/issues/19368
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit df59d6d5f7deb586cf14a6ef6984e6847db08852)
+
+commit eedb9f7d2d2a4047b473f23c5a6956c40a3da126
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Feb 28 20:24:12 2017 -0500
+
+    rgw_file: RGWFileHandle dtor must also cond-unlink from FHCache
+    
+    Formerly masked in part by the reclaim() action, direct-delete now
+    substitutes for reclaim() iff its LRU lane is over its high-water
+    mark, and in particular, like reclaim() the destructor is certain
+    to see handles still interned on the FHcache when nfs-ganesha is
+    recycling objects from its own LRU.
+    
+    Fixes: http://tracker.ceph.com/issues/19112
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit d51a3b1224ba62bb53c6c2c7751fcf7853c35a4b)
+
+commit 6c2a40a800277ded302f3183ac6c68b01ca3ed41
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Jan 17 11:23:45 2017 -0500
+
+    rgw_file:  split last argv on ws, if provided
+    
+    This is intended to allow an "extra" unparsed argument string
+    containing various cmdline options to be passed as the last argument
+    in the argv array of librgw_create(), which nfs-ganesha is
+    expecting to happen.
+    
+    While at it, hook env_args() too.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit fbc19e4effc736c98cc1cc283e5c7b131a0fa766)
+
+commit d2ca03b391340c6944ba453b106f3e1c45b75f1d
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Feb 12 20:18:26 2017 -0500
+
+    rgw_file: fix hiwat behavior
+    
+    Removed logic to skip reclaim processing conditionally on hiwat,
+    this probably meant to be related to a lowat value, which does
+    not exist.
+    
+    Having exercised the hiwat reclaim behavior, noticed that the
+    path which moves unreachable objects to LRU, could and probably
+    should remove them altogether when q.size exceeds hiwat.  Now
+    the max unreachable float is lane hiwat, for all lanes.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit b8791b2217e9ca87b2d17b51f283fa14bd68b581)
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 166cb7f85c240eeaffc70968abf5352d9cd45bd9
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Feb 12 18:20:43 2017 -0500
+
+    rgw_file: refcnt bugfixes
+    
+    This change includes 3 related changes:
+    
+    1. add required lock flags for FHCache updates--this is a crash
+       bug under concurrent update/lookup
+    
+    2. omit to inc/dec refcnt on root filehandles in 2 places--the
+       root handle current is not on the lru list, so it's not
+       valid to do so
+    
+    3. based on observation of LRU behavior during creates/deletes,
+       update (cohort) LRU unref to move objects to LRU when their
+       refcount falls to SENTINEL_REFCNT--this cheaply primes the
+       current reclaim() mechanism, so very significanty improves
+       space use (e.g., after deletes) in the absence of scans
+       (which is common due to nfs-ganesha caching)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit beaeff059375b44188160dbde8a81dd4f4f8c6eb)
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 31a71be9c9f24e22cf7e6eb390d3b39811ee3577
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sat Feb 11 16:38:05 2017 -0500
+
+    rgw_file:  add refcount dout traces at debuglevel 17
+    
+    These are helpful for checking RGWFileHandle refcnt invariants.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 462034e17f919fb783ee33e2c9fa8089f93fd97d)
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 1d6c72fe9a59561fb8f33889895b6708342b2856
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Feb 10 17:14:16 2017 -0500
+
+    rgw_file: add pretty-print for RGWFileHandle
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit ef330f385d3407af5f470b5093145f59cc4dcc79)
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit a969c449272ce88ddcbbd509432134f19520c8c3
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Feb 20 15:05:18 2017 -0500
+
+    rgw_file: fix marker computation
+    
+    Fixes: http://tracker.ceph.com/issues/19018
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 4454765e7dd08535c50d24205858e18dba4b454c)
+
+commit a70e83c90eb42439ba194ca197f507f972a18c3c
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Feb 19 20:34:31 2017 -0500
+
+    rgw_file: rgw_readdir can't list multi-segment dirs
+    
+    This issue has one root cause in librgw, namely that the marker
+    argument to these requests was incorrectly formatted (though the
+    marker cache was working as intended).
+    
+    Secondarily, for nfs-ganesha users, there is a compounding issue
+    that the RGW fsal was required by "temporary" convention to
+    populate the entire dirent cache for a directory on a single
+    readdir() invocation--the cache_inode/mdcache implementations
+    invariantly pass (before future 2.5 changesets, currently in
+    progress) a null pointer for the start cookie offset, intended
+    to convey this.
+    
+    Fixes: http://tracker.ceph.com/issues/18991
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 2cd60ee9712291b906123aca1704288b18a9742b)
+
+commit 209987e1c764cac7574bf0bb969e93fcf55b4361
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Feb 19 18:21:06 2017 -0500
+
+    rgw_file: allow setattr on placeholder directories
+    
+    When a POSIX path <bucket>/foo/ is known only as an implicit path
+    segment from other objects (e.g., <bucket>/foo/bar.txt), a case
+    that would usually arise from S3 upload of such an object, an
+    RGWFileHandle object representing "<bucket>/foo/" will be constructed
+    as needed, with no backing in RGW.
+    
+    This is by design, but subsequently, if a setattr is performed on
+    such a handle, we must be ready to create the object inline with
+    storing the attributes.
+    
+    Fixes: http://tracker.ceph.com/issues/18989
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 55eec1c0a0e136736961423b7b6244d0f3693c1a)
+
+commit 6a175f27961bd4f0fda8f94c200458f17865c9f9
+Merge: d32ae21 f3face6
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 16:36:35 2017 -0600
+
+    Merge pull request #14140 from smithfarm/wip-19341-jewel
+    
+    jewel: 'period update' does not remove short_zone_ids of deleted zones
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit d32ae21128d664370f7d07ed14427b75e10da7f8
+Merge: 4dfeed9 0be4e89
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 16:35:16 2017 -0600
+
+    Merge pull request #13886 from cbodley/wip-17786
+    
+    jewel: rgw: clear data_sync_cr if RGWDataSyncControlCR fails
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 4dfeed949655a5b5041f612b868225c234e86bba
+Merge: d69c54a 1985662
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 16:34:57 2017 -0600
+
+    Merge pull request #13867 from smithfarm/wip-19159-jewel
+    
+    jewel: rgw: multisite: RGWMetaSyncShardControlCR gives up on EIO
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit d69c54aca9dde3d64b56e5ee72f1cd29057a80c6
+Merge: 0830135 dfaaec0
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 16:34:24 2017 -0600
+
+    Merge pull request #13858 from smithfarm/wip-19152-jewel
+    
+    jewel: rgw_file:  restore (corrected) fix for dir partial match (return of FLAG_EXACT_MATCH)
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 0830135f40ac37ac9027bece65b8aac64c2c7ec9
+Merge: 4dd3a9f 97fed01
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 16:33:18 2017 -0600
+
+    Merge pull request #13848 from smithfarm/wip-19150-jewel
+    
+    jewel: rgw_file:  avoid interning .. in FHCache table and don't ref for them
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 4dd3a9f628053784cff2aa5849ab3c4cdb2acf4e
+Merge: 5eeba9c d333add
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 16:29:30 2017 -0600
+
+    Merge pull request #13844 from smithfarm/wip-19148-jewel
+    
+    jewel: rgw: DUMPABLE flag is cleared by setuid preventing coredumps
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 5eeba9cb85e9111ef124acd36dccf77c6f00b3e4
+Merge: ae632b0 f7ce5df
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 16:26:48 2017 -0600
+
+    Merge pull request #13823 from asheplyakov/19176-bp-jewel
+    
+    jewel: rgw: fix swift cannot disable object versioning with empty X-Versions-Location
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit ae632b0da6cc90a993fcae31b56f97c834d15aa1
+Merge: 874120f e8041f6
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 16:22:23 2017 -0600
+
+    Merge pull request #13778 from smithfarm/wip-18811-jewel
+    
+    jewel: librgw: RGWLibFS::setattr fails on directories
+    
+    Reviewed-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 874120f627f509c75b28d1be699812fce881f4bb
+Merge: b0108ea e475bfa
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 16:21:44 2017 -0600
+
+    Merge pull request #13717 from asheplyakov/19115-bp-jewel
+    
+    jewel: rgw_file: ensure valid_s3_object_name for directories, too
+    
+    Reviewed-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit b0108ea3bae1662e90c7fb9a911257f45193798c
+Merge: 6518d70 db928d6
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 16:19:03 2017 -0600
+
+    Merge pull request #13596 from dillaman/wip-19038-jewel
+    
+    jewel: rbd-mirror: deleting a snapshot during sync can result in read errors
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 6518d70b89686b9a4e8854a4c6290381fbcdd8c3
+Merge: db50938 8941881
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 16:10:43 2017 -0600
+
+    Merge pull request #13583 from asheplyakov/jewel-bp-18901
+    
+    jewel: rgw_file:  interned RGWFileHandle objects need parent refs
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit db50938013f0b1c1fe8a23247c91bbfc38d36a8c
+Merge: 7536871 d44263f
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 16:02:43 2017 -0600
+
+    Merge pull request #13503 from linuxbox2/wip-jewel-10156
+    
+    jewel: rgw: make sending Content-Length in 204 and 304 controllable
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit 75368712aafc51b062b674b263354cca2dd4b49a
+Merge: 250071e 2f20328
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 15:53:49 2017 -0600
+
+    Merge pull request #13232 from ovh/wip-rework-recovery-priorities-jewel
+    
+    jewel: osd: Increase priority for inactive PGs backfill
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 250071eb8dc744237acfed01df2cf04f2d88bc85
+Merge: 6f57a77 9910eac
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 15:51:46 2017 -0600
+
+    Merge pull request #13219 from linuxbox2/jewel-rgw-multipart-upload-copy-4
+    
+    jewel: rgw: multipart uploads copy part support
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 6f57a775301edce7a1039a4aa4479bbd8883cbf6
+Merge: 8c02e54 3fce77a
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Mar 28 15:51:24 2017 -0600
+
+    Merge pull request #13108 from vumrao/wip-vumrao-jewel-18657
+    
+    jewel: osd: fix OSD network address in OSD heartbeat_check log message
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit b24a8c2f3890b6fcc60f3b52cba93d573b9d45a2
+Author: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+Date:   Thu Mar 2 17:21:57 2017 +0800
+
+    rgw_file: posix style atime,ctime,mtime
+    
+    As an ganesha FSAL backend, rgw_file should properly maintain
+    the atime,ctime,mtime properly against operations such as:
+    	(read,write) for file
+    	(create,unlink,mkdir,rmdir,rename) for dir
+    	(setattr) for file and dir
+    
+    Signed-off-by: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+    (cherry picked from commit ac25da2479b9be876cbdb820560ac46a6e2b17d7)
+
+commit b6181833be925b7eb11afffff7f03486bdde2d25
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Feb 20 15:05:18 2017 -0500
+
+    rgw_file: fix marker computation
+    
+    Fixes: http://tracker.ceph.com/issues/19018
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 4454765e7dd08535c50d24205858e18dba4b454c)
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit ed9308cba2af227991a50a9535df30cf9bb18a82
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Feb 19 20:34:31 2017 -0500
+
+    rgw_file: rgw_readdir can't list multi-segment dirs
+    
+    This issue has one root cause in librgw, namely that the marker
+    argument to these requests was incorrectly formatted (though the
+    marker cache was working as intended).
+    
+    Secondarily, for nfs-ganesha users, there is a compounding issue
+    that the RGW fsal was required by "temporary" convention to
+    populate the entire dirent cache for a directory on a single
+    readdir() invocation--the cache_inode/mdcache implementations
+    invariantly pass (before future 2.5 changesets, currently in
+    progress) a null pointer for the start cookie offset, intended
+    to convey this.
+    
+    Fixes: http://tracker.ceph.com/issues/18991
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 2cd60ee9712291b906123aca1704288b18a9742b)
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 26a38e2e1525714978feda018c03698d7af65129
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Feb 19 18:21:06 2017 -0500
+
+    rgw_file: allow setattr on placeholder directories
+    
+    When a POSIX path <bucket>/foo/ is known only as an implicit path
+    segment from other objects (e.g., <bucket>/foo/bar.txt), a case
+    that would usually arise from S3 upload of such an object, an
+    RGWFileHandle object representing "<bucket>/foo/" will be constructed
+    as needed, with no backing in RGW.
+    
+    This is by design, but subsequently, if a setattr is performed on
+    such a handle, we must be ready to create the object inline with
+    storing the attributes.
+    
+    Fixes: http://tracker.ceph.com/issues/18989
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 55eec1c0a0e136736961423b7b6244d0f3693c1a)
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 7e4e29038bd820e171d007360cf383c85f67879b
+Author: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+Date:   Wed Mar 8 16:23:11 2017 +0800
+
+    rgw_file: fix reversed return value of getattr
+    
+    When ::getattr returns -ESTALE, rgw_getattr returns ESTALE,
+    which is a not expected postive.
+    
+    Signed-off-by: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+    (cherry picked from commit 39203cf872b8f4af86eb0e4a0f96dffd9cc92b41)
+
+commit 3c02ee4fe9ae70f30328a7b015f5cac4a1804c67
+Author: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+Date:   Mon Mar 20 10:53:46 2017 +0800
+
+    rgw_file: fix non-negative return code for open operation
+    
+    The nfs-ganesha expects a negative retcode for errors.
+    
+    Signed-off-by: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+    (cherry picked from commit b5f70ef7b066f9ff44770cc8a50ccadaa02f4691)
+
+commit e9a4903c6e77caae6f161056e8aaa5ba10ae5ef2
+Author: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+Date:   Wed Mar 15 15:01:05 2017 +0800
+
+    rgw_file: fix double unref on rgw_fh for rename
+    
+    Skip unref after unlink to fix the problem.
+    
+    Signed-off-by: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+    (cherry picked from commit bff228734c73b536d2482e2e2fa4ad38b206ebff)
+
+commit 9910eac98474930369d694b236c06ffd627fee04
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 22:20:10 2016 -0700
+
+    rgw: multipart part copy, fix regression
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 66fbe9384703c004c01783eb664f55895d310439)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit a3fdf0e246637ef4145b6b5ba1f4b114ec7ebc62
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 17:44:08 2016 -0700
+
+    rgw: minor optimization
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 4919dc9987c6376d3d4e143702c26417449524c5)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 2161376baf413acfbf02df07e404d2918729bfcc
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 17:43:00 2016 -0700
+
+    rgw: rgw_obj_key use adjustment in multipart copy part
+    
+    This fixes a case where objects start with double underscore.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 29fece3545cc1df404a25eec46706b16f893a5df)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 135f84d860d10a2961f430708983113a87ddf899
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 17:43:35 2016 -0700
+
+    rgw: multipart copy-part handle versionId
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 53521efffb1cb92e5f5ce992d4127bf9498d7c33)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 9ab65f5501fb59aac17cfdde57371b00af03d84b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 14:24:13 2016 -0700
+
+    rgw: multipart copy part minor parsing cleanup
+    
+    no need for range.size()
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 6e9b824d5d4017239d58b4752ebc43bfad8f698d)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 3eb9dc96ff8e655415500a3595a78cab80739826
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 14:11:43 2016 -0700
+
+    rgw: multipart copy, check for empty bucket, improve logging
+    
+    also reduce log level for non critical user errors.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 2bcb3d286b230ef917d5ba96c8276a942f544689)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 2588e95d9d422d3b33fd710de1f5884873465483
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Sep 29 14:07:14 2016 -0700
+
+    rgw: multipart copy part, chunked read
+    
+    Don't read the entire range from source object, read it in parts.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 4049e47a0cfc1eef6efd502590b68ba7234589d3)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit bd118b556562176ce2390a935b3bb8c25a62d0c4
+Author: Javier M. Mellid <jmunhoz@igalia.com>
+Date:   Tue Jul 26 14:56:50 2016 +0200
+
+    rgw: doc: add multipart uploads copy part feature as supported
+    
+    Signed-off-by: Javier M. Mellid <jmunhoz@igalia.com>
+    (cherry picked from commit 8a7ebeee2ff3f10ceb23b7fa43e43c3c450efe22)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit b56b719299becc38ec36d427a36b8c29f2416c08
+Author: Javier M. Mellid <jmunhoz@igalia.com>
+Date:   Tue Jul 26 14:53:44 2016 +0200
+
+    rgw: multipart uploads copy part support
+    
+    Add multipart uploads copy part feature.
+    
+    Fixes: http://tracker.ceph.com/issues/12790
+    
+    Signed-off-by: Javier M. Mellid <jmunhoz@igalia.com>
+    (cherry picked from commit 949480c2e9760855ed6a0501d364d5e766c8207d)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit d44263fc91da12ea0ad4fec0cd2877b36ddb9e9f
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Feb 17 00:56:34 2017 +0100
+
+    rgw: make sending Content-Length in 204 and 304 controllable
+    
+    This commit introduces a new configurable "rgw print prohibited
+    content length" to let operator decide whether RadosGW complies
+    to RFC 7230 (a part of the HTTP specification) or violates it
+    but follows the Swift's behavior.
+    
+    Fixes: http://tracker.ceph.com/issues/16602
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit d8e3e64ec97a3c222a56bb6f510e5e23d7858615)
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit f3face61351a61f3b71dfb8268a4b645c4d92da2
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Mar 9 15:24:08 2017 -0500
+
+    rgw: clear old zone short ids on period update
+    
+    the short ids of old, removed zones were being kept in the period to
+    guard against hash collisions with new zones
+    
+    but for a hash collision to cause a wrong object to sync, that object
+    would have to be uploaded simultaneously to two different zones that had
+    the same short id
+    
+    to avoid this, we just have to prevent the period from containing two
+    colliding zones at the same time - we don't have to remember old zone
+    short ids forever
+    
+    Fixes: http://tracker.ceph.com/issues/15618
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 9c45633c836c966ab1f75ea2b1ad3fa0a4886600)
+
+commit 8c02e54a747644d24feb98dbc15a15cdd55d1afd
+Merge: 9d222b9 952f34f
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Mar 24 08:13:02 2017 -0700
+
+    Merge pull request #13146 from dzafman/wip-18502-jewel-zafman
+    
+    jewel: osd: Scrub improvements and other fixes
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 9d222b943217a2ec022678e50b34766d231004ce
+Merge: ce2e989 cb9fae5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Mar 23 16:49:13 2017 +0100
+
+    Merge pull request #14100 from smithfarm/wip-18781
+    
+    Manually merge PR#13143 which was mistakenly merged to jewel-next
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit cb9fae56ebdfda28b0606ee89631deb60e5f7de7
+Merge: ce2e989 ee3c67c
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Mar 23 09:46:53 2017 +0100
+
+    Merge branch 'jewel-next' into jewel
+    
+    Branch "jewel-next" was merged to jewel on Dec 22 (5b402f8a7b), and then PR
+    https://github.com/ceph/ceph/pull/13143 merged to jewel-next much later (Feb
+    1st), and then jewel-next was never merged to jewel again before we tagged
+    v10.2.6.
+    
+    This merge brings in a single commit, i.e. the one from
+    https://github.com/ceph/ceph/pull/13143
+    
+    Reported-by: Ken Dreyer <kdreyer@redhat.com>
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit ce2e9897c08775cfbe318f637d61e07eb5433df5
+Merge: 656b5b6 d0a0d2f
+Author: Zack Cerza <zack@cerza.org>
+Date:   Tue Mar 14 09:19:28 2017 -0600
+
+    Merge pull request #13952 from smithfarm/wip-fix-merge-error
+    
+    tests: fix merge error in rgw/singleton/all/radosgw-admin.yaml
+
+commit 0be4e89419f2083d081ac784891e4653290cd530
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 11 15:21:42 2016 -0400
+
+    rgw: hold a reference on data_sync_cr over run()
+    
+    run() will drop its reference to data_sync_cr, so we need to hold a
+    reference until we can reacquire the lock
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 4cf0d2a768e7402e71280ca16b47353ca2a68505)
+
+commit 18ffdb7baf4aff1fd7f894af1054081f3ee61d28
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 11 15:19:37 2016 -0400
+
+    rgw: clear data_sync_cr if RGWDataSyncControlCR fails
+    
+    async notifications will still try to call wakeup() on RGWDataSyncControlCR
+    if it fails, leading to segfault
+    
+    Fixes: http://tracker.ceph.com/issues/17569
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 5cc599b9bf2dde31de16a5b2831baf06851d69d5)
+    
+     Conflicts:
+    	src/rgw/rgw_data_sync.cc: declaration of 'int r'
+
+commit d0a0d2f4bf1dacf7c1f38b968be84a5e905554a8
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Mar 14 01:26:23 2017 +0100
+
+    tests: fix merge error in rgw/singleton/all/radosgw-admin.yaml
+    
+    Introduced by 94d5888c
+    
+    Reported-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 19856624cd5ea8cb82bc5b46e062beb39674bd34
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Feb 20 16:00:01 2017 -0500
+
+    rgw: RGWMetaSyncShardControlCR retries with backoff on all error codes
+    
+    RGWBackoffControlCR only treats EBUSY and EAGAIN as 'temporary' error
+    codes, with all other errors being fatal when exit_on_error is set
+    
+    to RGWMetaSyncShardControlCR, a 'fatal' error means that no further sync
+    is possible on that shard until the gateway restarts
+    
+    this changes RGWMetaSyncShardControlCR to set exit_on_error to false, so
+    that it will continue to retry with backoff no matter what error code it
+    gets
+    
+    Fixes: http://tracker.ceph.com/issues/19019
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 3e4059557fd6cad5d31014327f60832b36d04a6c)
+
+commit dfaaec0446c3db458bffe1d725518ab4f7cc2fa8
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Feb 23 10:21:38 2017 -0500
+
+    rgw_file:  return of RGWFileHandle::FLAG_EXACT_MATCH
+    
+    Allow callers of rgw_lookup() on objects attested in an
+    rgw_readdir() callback the ability to bypass exact match in
+    RGWLibFS::stat_leaf() case 2, but restore exact match enforcement
+    for general lookups.
+    
+    This preserves required common_prefix namespace behavior, but
+    prevents clients from eerily permitting things like "cd sara0" via
+    partial name match on "sara01."
+    
+    Fixes: http://tracker.ceph.com/issues/19059
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 70ef7d45e0abf2661bd4e23161d4e70cf5178079)
+
+commit 9b3784d924112d9ba42b2088d5fb2656ef74fadc
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Feb 19 17:43:17 2017 -0500
+
+    rgw_file: invalid use of RGWFileHandle::FLAG_EXACT_MATCH
+    
+    The change which introduced this flag also caused it to be
+    given as the flags argument to RGWLibFS::stat_leaf() when called
+    from rgw_lookup().
+    
+    This was incorrect:  in particular, when a directory is known only
+    as a common prefix of other objects, the AWS namespace mapping
+    convention requires lookup("foo") to match a non-materialized
+    instance of "foo/" (case 2 in RGWLibFS::stat_leaf's stat loop).
+    
+    Fixes: http://tracker.ceph.com/issues/18992
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit e31e9eb980f958640150e8d7f17de1b9e5478b1e)
+
+commit 7761376c92eb38a2dd3d19f0b3d81895a4b1167c
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Feb 22 14:57:59 2017 -0500
+
+    rgw_file: rgw_lookup: don't ref for "/" or ".."
+    
+    These refs won't be returned by nfs-ganesha, and are sufficiently
+    magical that other consumers should be persuaded to understand
+    their specialness.
+    
+    Fixes: http://tracker.ceph.com/issues/19060
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit dea8d1ee373399a21851690a9753388b659b8ede)
+
+commit 97fed013f66a89c404bac5145080e5d556ff5c42
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Feb 22 10:24:29 2017 -0500
+
+    rgw_file: avoid stranding invalid-name bucket handles in fhcache
+    
+    To avoid a string copy in the common mkdir path, handles for
+    proposed buckets currently are staged in the handle table, before
+    being rejected.  They need to be destaged, not just marked deleted
+    (because deleted objects are now assumed not to be linked, as of
+    beaeff059375b44188160dbde8a81dd4f4f8c6eb).
+    
+    This triggered an unhandled Boost assert when deleting staged
+    handles, as current safe_link mode requires first removing from
+    the FHCache.
+    
+    Fixes: http://tracker.ceph.com/issues/19036
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 6cde812c92e5bba9f85fbf8486ebe69b55952370)
+
+commit d333addb71a0a92fbac5a7b922dbd69fc9e0604e
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Mon Feb 27 13:06:59 2017 +1000
+
+    rgw: set dumpable flag after setuid post ff0e521
+    
+    ff0e521 resolved the issue for the other daemons but not for rgw since
+    it calls setuid (via civetweb) after the new code sets PR_SET_DUMPABLE.
+    Add another prctl call before wait_shutdown.
+    
+    Fixes: http://tracker.ceph.com/issues/19089
+    
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit bc458d39630b599e0e1ca9fe25ad7455fcffdd10)
+
+commit f7ce5df4064e5538156cb44f9525d7552a0dd098
+Author: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+Date:   Wed Feb 8 15:07:43 2017 +0800
+
+    rgw: fix swift cannot disable object versioning with empty X-Versions-Location
+    
+    we should be able to disable object verioning by removing its X-Versions-Location
+    metadata header by sending an empty key value. this description can be found at
+    No.8 in http://docs.openstack.org/user-guide/cli-swift-set-object-versions.html.
+    
+    Fixes: http://tracker.ceph.com/issues/18852
+    Signed-off-by: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+    (cherry picked from commit 17c5a0edd2227703cec867f0f588d4eae36dfe1c)
+
+commit 2f2032814189a4ecbf8dc01b59bebfae8ab3f524
+Author: Bartłomiej Święcki <bartlomiej.swiecki@corp.ovh.com>
+Date:   Fri Dec 2 16:54:46 2016 +0100
+
+    osd: Increase priority for inactive PGs backfill
+    
+    This change does prioritize backfill of PGs which don't
+    have min_size active copies. Such PGs would cause IO stalls
+    for clients and would increase throttlers usage.
+    
+    This change also fixes few subtlle out-of-bounds bugs.
+    
+    Signed-off-by: Bartłomiej Święcki <bartlomiej.swiecki@corp.ovh.com>
+    (cherry picked from commit 6a76adcdb1f92c136841d960aa7cd4e5b94addec)
+    
+    Conflicts:
+    	PendingReleaseNotes (removed version number, merged conflicts)
+
+commit e8041f627c6a05a59dba29819ed610ea0896f4dd
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Feb 3 13:44:45 2017 -0500
+
+    rgw_file:  fix RGWLibFS::setattr for directory objects
+    
+    Fixes:  http://tracker.ceph.com/issues/18808
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 4ad5a9226852d6d564baf2e63278ed6c4c185ecb)
+
+commit e475bfaf7d3a1b0e54172083a92546560219665a
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Feb 23 16:02:07 2017 -0500
+
+    rgw_file: ensure valid_s3_object_name for directories, too
+    
+    The logic in RGWLibFS::mkdir() validated bucket names, but not
+    object names (though RGWLibFS::create() did so).
+    
+    The negative side effect of this was not creating illegal objects
+    (we won't), but in a) failing with -EIO and b) more importantly,
+    not removing up the proposed object from FHCache, which produced a
+    boost assert when recycled.
+    
+    Fixes: http://tracker.ceph.com/issues/19066
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit eb1cd3b30c0504385f05bf2d2dd5e2251b7efed7)
+
+commit 78c8be7a0df3d1c669f8a2a8fd7a5676d0823209
+Author: Marcus Watts <mdw@linuxbox.com>
+Date:   Thu Feb 23 02:30:52 2017 -0500
+
+    rgw/openssl fix: xenial autoconf logic problem: gcc/ld got too smart...
+    
+    On xenial, cc -o foo -lssl -lcrypto doesn't always
+    record libssl.so libcrypto.so as runtime library dependencies.
+    It is necessary to actually *use* a function from the library before
+    it gets recorded.  The ld(1) options "--as-needed" and "no-as-needed"
+    control this.  Evidently the default has changed in xenial.
+    That caused my smart "soname" detecting logic for openssl to
+    stop working.  To make it work, the test program has to
+    actually reference routines or variables inside the library.
+    
+    This is a quick fix for xenial / autoconf.  There needs to be
+    a better fix for cmake and master.
+    
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+
+commit db928d6b3f983b3a1ccc07440fcd3680700a1188
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Feb 21 15:33:01 2017 -0500
+
+    rbd-mirror: retry object copy after -ENOENT error
+    
+    Fixes: http://tracker.ceph.com/issues/18990
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b4f36d5dc3f4f3cbb23f61cbb945b222248a50df)
+    
+    Conflicts:
+    	src/test/librados_test_stub/MockTestMemIoCtxImpl.h: sparse reads not supported
+    	src/test/rbd_mirror/image_sync/test_mock_ObjectCopyRequest.cc: sparse reads not supported
+
+commit a643fa80e03b41edcc720ff77b69ebaf24a23f3e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Feb 21 13:09:39 2017 -0500
+
+    rbd-mirror: object copy should always reference valid snapshots
+    
+    If a remote snapshot is deleted while an image sync is in-progress,
+    associate the read request against the most recent, valid remote
+    snapshot for a given snapshot object clone.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9a91efc3047963364944f8be91cee8e8f6afc49a)
+
+commit 37bbc95d56f73e4ec9c6e13ddbae199b14b104b4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Feb 21 11:52:00 2017 -0500
+
+    rbd-mirror: replace complex object op tuple with struct
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0c181527c0e151784a0f7c466aaa70b0772f91b1)
+    
+    Conflicts:
+    	src/tools/rbd_mirror/image_sync/ObjectCopyRequest.cc: sparse reads not supported
+    	src/tools/rbd_mirror/image_sync/ObjectCopyRequest.h: sparse reads not supported
+
+commit 8941881e8986af0b2871c15d188d94d49e95dc02
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Dec 30 23:30:16 2016 -0500
+
+    rgw_file:  interned RGWFileHandle objects need parent refs
+    
+    RGW NFS fhcache/RGWFileHandle operators assume existence of the
+    full chain of parents from any object to the its fs_root--this is
+    a consequence of the weakly-connected namespace design goal, and
+    not a defect.
+    
+    This change ensures the invariant by taking a parent ref when
+    objects are interned (when a parent ref is guaranteed).  Parent
+    refs are returned when objects are destroyed--essentially by the
+    invariant, such a ref must exist.
+    
+    The extra ref is omitted when parent->is_root(), as that node is
+    not in the LRU cache.
+    
+    Fixes: http://tracker.ceph.com/issues/18650
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 0e5299f3f43e633a5d8a9360893b4b11f6217d81)
+
+commit 952f34f39cdd0438e4a4fb369ea8ca20b26488a8
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Dec 12 15:53:25 2016 -0800
+
+    test: Update for new error message when doing scrub with deep-scrub errors
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 85e0774a7dded4aa6c67f237416041e25a7680bc)
+
+commit b655b98e48989ae954dc150fcb3d8976978cd90d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Dec 9 12:14:06 2016 -0800
+
+    osd: Add "trigger_scrub" admin socket command
+    
+    This allows testing to fake a schedule scrub (must_scrub not set)
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit c8dc945260ee2ee841aca00fbc802d41036033d6)
+
+commit 94c958d2c9570e55305384ac86185e328746d2ff
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Dec 8 23:00:13 2016 -0800
+
+    test: Add test for keeping deep-scrub information
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 64a7012e986ec88994c073b738fd08e8958395c3)
+
+commit 42eb24f18d5114410bd2a3e84e9219584d9e165e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Dec 1 16:39:42 2016 -0800
+
+    osd: When deep-scrub errors present upgrade regular scrubs
+    
+    Previously, if a weekly deep-scrub found errors the next daily scrub
+    would cause the deep-scrub information to be removed.
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 532a759dca466181f37a329f86045c34f1a2506f)
+
+commit 76a457aa7c411dc5ed22f171954b8ed3c4661845
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 19 17:10:29 2016 -0700
+
+    tasks/scrub_test.py: Make test deterministic by updating digests
+    
+    ceph_test_rados: Get coverage of digest updates in deep-scrub/repair
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit e000ab2f46b7b98a223176a2a25bb7195e601af6)
+
+commit bd1f1983129d6e1787b8b5eb4884c5e908f7b274
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 19 17:10:07 2016 -0700
+
+    repair_test, scrub_test: Fix whitelists for scrub changes
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 5fe8522df47f56842d227e08b2912623504afe24)
+
+commit 28106f06404407df7d05b35f9f570a80e785e635
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Aug 31 12:02:31 2016 -0700
+
+    scrub_test: Fix for list-inconsistent-obj output changes
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 5ae0f5c75a8236d4a026f9ddcb5ff59964d90579)
+
+commit abcefc1395f76e5b05a988b970457a7f6dac6e8e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Aug 30 12:11:44 2016 -0700
+
+    doc, test: Add schemas for list-inconsistent-* rados command output
+    
+    If jsonschema cmd is available use it to test output against schema
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit eb73dd473600fbbb45fad00194f7c46b565d6b81)
+
+commit 2fa2b64676555a91883a5aabb4e4237124308629
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Aug 30 12:22:55 2016 -0700
+
+    test: Update testing for new list-inconsistent-obj output
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b7bacd219b000869b9c93e21edba4f8a3ace60d9)
+    
+    Adapted for difference in Jewel object_info
+
+commit 304f697ff1106695188b572e0da2415437040a6c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Aug 30 12:22:29 2016 -0700
+
+    rados, osd: Improve attrs output of list-inconsistent-obj
+    
+    Persist the user_version and shard id of scrubbed obj
+    Rados command dump inconsistent obj's version and shard-id
+        so they can be passed to repair command
+    Rados list-inconsistent-obj output of attrs
+        Make attrs an array since there are more than one
+        Use base64 encode for values with non-printable chars
+        Add indication if base64 encoding used
+    Add checking for ss_attr_missing and ss_attr_corrupted
+        Rename attr errors to attr_key_mismatch and attr_value_mismatch
+    Add missing size_mismatch_oi scrub checking
+    For erasure coded pools add ec_size_error and ec_hash_error not just read_error
+    Use oi_attr_missing and oi_attr_corrupted just like list-inconsistent-snap does
+    Pick an object info based on version and use that to find specific shards in error
+        Check for object info inconsistency which should be rare
+    Make all errors based on comparing shards to each other object errors
+        We don't want give the impression that we've picked the correct one
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit df3ff6dafeadb3822b35c424a890db9a14d7f60f)
+
+commit ea52f527e530b111b1bb26d10824c0230d662c4a
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Sep 1 14:45:01 2016 -0700
+
+    osd: Fix logging to help with diagnostics
+    
+    These messages were not outputing the intended information
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b39001ec6ff0996860bcb5a3578bc2c66355c781)
+
+commit 5e8f8a2288aa1a9f1de86d3f7d0f9c66c795113e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Jun 21 18:05:25 2016 -0700
+
+    test: Fix use of wait_for_clean()
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit d0503a52d0fe5505bdb38dfd613c03a20500c05d)
+
+commit 2230e6adc66ca3e5dbe81aecccdb4a435e93e256
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Aug 30 10:56:06 2016 -0700
+
+    common: Change cleanbin() to use base64 encoding, update ceph-objectstore-tool
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 92e982c71995b863466d83671468f84761cb1793)
+
+commit 83ea077ee560d31a5c302a62b55451a2571fda8d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Aug 30 11:05:16 2016 -0700
+
+    common: Move cleanbin() function to common/util.cc
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 5c79074ffaee34b2956d9dfc67b1eff9f39b47f3)
+    
+    Conflicts:
+    	src/tools/CMakeLists.txt (changes goes in src/CMakeLists.txt)
+
+commit ba84ca9fa391d963d1d4e46fbf27ce3dbff397be
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Jun 8 09:48:00 2016 -0700
+
+    test: Add test support for deep-scrub
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 907e79e2b77835d1aca9e8ba2fae5fa2fd437e5a)
+
+commit 6a421d19d0be4d59beed5c69bb0aa4477d65a14e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Sat Jun 18 17:58:36 2016 -0700
+
+    common: Fix indentation
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit a74418a29e4e469117fc0c4edd80f78b62944c98)
+
+commit c7c3e070336dc898460d4338b4d78b8963467c8a
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Aug 12 12:06:31 2016 -0700
+
+    osd: Handle corrupt attributes in get_object_context()
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 30f8b0d6593718dd10bcf6ff459c95b4bb68c05c)
+
+commit 8006ba7e86ebb8201a040c427cc95197901064be
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Nov 11 21:37:50 2016 +0800
+
+    ReplicatedPG::failed_push: release read lock on failure
+    
+    and requeue the blocked ops.
+    
+    Fixes: http://tracker.ceph.com/issues/17857
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b3224a18f6acc7ed54c2162b140a33b6146a16be)
+
+commit 5ca69d57dbe2ee7acc64d28ca35bb390bf463199
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Oct 14 16:17:55 2016 -0700
+
+    test.sh: Make check for flags more robust
+    
+    Low space broke test, saw "flags nearfull,pauserd,pausewr...."
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b4c080b1b4378d659c1ca8a17811cd6f84595166)
+
+commit 1be38e5ebd2d01d5d527b05e64b026df955ffe69
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Jul 21 17:36:34 2016 -0700
+
+    test: Remove extra objectstore_tool call which causes a recovery
+    
+    Caused by: 70e000a9a42c50eda48f5d3b9e432ffc3a70f75b
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 6904529d1b53993304de2927500937ba0d493e9e)
+
+commit 026f181e72f2e09e769a8821dfe2c99f6213e6a3
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Jun 14 20:09:15 2016 -0700
+
+    test: Handle object removals in a non-racey way
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit bfe3ebd94fdc1ef36ebe6e8f9d38acea322eca22)
+
+commit c5ef385c25086dc1582ebe5343481d05283b0cc6
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu May 26 22:09:42 2016 -0700
+
+    osd: Fix hang on unfound object after mark_unfound_lost is done
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 73a27533beba3587f8447b4d41d200427c45042b)
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.cc (trivial)
+
+commit 6284f745157458439699c76e8616455c50d1eb71
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri May 20 15:20:18 2016 -0700
+
+    osd: Handle recovery read errors
+    
+    Fixes: http://tracker.ceph.com/issues/13937
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit c51d70e1e837c972e42ddd5fa66f7ca4477b95cc)
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.h (trivial)
+
+commit 27393a8c05d4656f342ecd32817307e558a2e400
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri May 20 18:19:42 2016 -0700
+
+    osd: Fix log messages
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b40ec3fe890679b4c73778815e1bedd492cb264b)
+
+commit cbf66f3b16f194dd0c67b46e0fec247e02e84134
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri May 20 13:58:32 2016 -0700
+
+    osd: CLEANUP: Remove unused pending_read member
+    
+    Remove unused struct
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 36fd68c96653e83f87767feb08530a9fc6e841b0)
+
+commit 228b91d72a15b62adc10591604c4e8a849df53d5
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Oct 11 18:08:12 2016 -0700
+
+    test/osd-scrub-repair.sh: Use test case specific object names to help with diagnostics
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0bf4da589155ee50969812492cfbc66368efb54c)
+
+commit ee3c67c9cbfdeeb3e628bac34b708cf150b3862e
+Merge: 3dbf0c9 39848e4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Feb 1 23:28:22 2017 +0100
+
+    Merge pull request #13143 from linuxbox2/jewel-mdw-rgw-lf
+    
+    jewel: rgw: radosgw/swift: clean up flush / newline behavior.
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 39848e41b7c517cc5faab1ccf77c2804fd7d2628
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Wed Jan 11 00:06:15 2017 -0500
+
+    radosgw/swift: clean up flush / newline behavior.
+    
+    The current code emits a newline after swift errors, but fails
+    to account for it when it calculates 'content-length'.  This results in
+    some clients (go github.com/ncw/swift) producing complaints about the
+    unsolicited newline such as this,
+    	Unsolicited response received on idle HTTP channel starting with "\n"; err=<nil>
+    
+    This logic eliminates the newline on flush.  This makes the content length
+    calculation correct and eliminates the stray newline.
+    
+    There was already existing separator logic in the rgw plain formatter
+    that can emit a newline at the correct point.  It had been checking
+    "len" to decide if previous data had been emitted, but that's reset to 0
+    by flush().  So, this logic adds a new per-instance variable to separately
+    track state that it emitted a previous item (and should emit a newline).
+    
+    Fixes: http://tracker.ceph.com/issues/18473
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 5f229d6a33eae4906f22cdb90941835e47ee9f02)
+
+commit 3fce77ab3662496368e25cbbf6d1b37d3c140db9
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Wed Jan 25 21:54:27 2017 +0530
+
+    osd: jewel fix OSD network address in OSD heartbeat_check log message
+    
+    Fixes: http://tracker.ceph.com/issues/18657
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+
+commit 14a6aabe22f68436ea3297ce0851700f86ee5b12
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Wed Aug 3 17:36:55 2016 -0400
+
+    rgw: Handle multiple listening addreses w/ optional ssl "correctly" with civetweb.
+    
+    For civetweb: accept a range of port numbers joined with '+'.
+    Port numbers may include an ipaddress: prefix and 's' suffix.
+    Additionally, use "mg_get_local_addr" to correctly deduce host port per
+    incoming connection.
+    
+    civetweb can accept connections on multiple ports, some of which
+    might have SSL turned on and some not.  Both s3 and swift have various
+    authorization protocols in which the port number matters.  In the generic
+    radosgw frontend process, each frontend only has one port number, but
+    we should want to have both ssl and non-ssl connections managed within
+    one rgw frontend, because the thread pool is also per front-end, and
+    that *is* a scarce resource.
+    
+    So, this patch enables the use of multiple ports with a single civetweb
+    frontend.  To indicate https: append an 's' to portno.  To use multiple
+    ports, use +.  So 80+443s indicates use of the usual default http ports.
+    The parsed port is not stored in the frontend structure,
+    
+    So instead, this patch adds logic to use the results of
+    mg_get_local_addr() on a per-connection basis insetad of the generic
+    front-end port number.  This will affect "v4" s3 authorization, and also
+    affect swift pre-signed URLs.
+    
+    mg_get_local_addr() is a new customization to civetweb; that submodule
+    was updated (in a temporary repository) by the previous commit to this.
+    
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    (cherry picked from commit 8bc6decc0774fae9ac881f91e73da55deebe3360)
+
+commit 698250563ccc4c69e5ca5aebf65dc352d80a8bdd
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Tue Dec 20 00:22:02 2016 -0500
+
+    rgw: s3: secure_port should override port, also apply ssl default right.
+    
+    Without https, only port is set.  With https, secure_port and port are
+    both set to the same value.  The previous logic looked at port first and
+    had overly simplified conditional logic which was liable to try to apply
+    both non-default cases.  The correct behavior is: look secure_port first,
+    and if secure_port is set, then only check to see if it's a non-default
+    port.
+    
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    (cherry picked from commit a113cf5ff5a642d2ee4cc83f5c7001b4bfe0a5df)
+
+commit 28f2841c8b9a832e486b9f89c574a4591bf3d448
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Wed Nov 2 00:18:48 2016 -0400
+
+    rgw: Get civetweb ssl enhancement: wip-listen3 = mg_get_local_addr
+    
+    The logic inside of radosgw that computes aws v4 signatures wants to know
+    what server port the client connected.  The patch to civetweb patch adds a
+    call mg_get_local_addr() which will permit that code to actually find out
+    on what address a connection was received, rather than merely guessing
+    based on configuration as it previously did.
+    
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    (cherry picked from commit 46ced9ddd2795f00f014e22e5637070b49e7a6d5)
+
+commit 8d83dfb4176ede7490d0cab589e9566bc7d4e387
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Wed Mar 23 17:32:14 2016 -0400
+
+    rgw: Document that radosgw now supports SSL.
+    
+    This includes information on file format and configuration file syntax.
+    
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    (cherry picked from commit d4e72dfed30274b3cfbad4ac58c0746a98c0148b)
+
+commit e3f80c9d0ed6762a39fc242561f5ea26f0f26546
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Fri Jan 13 03:39:57 2017 -0500
+
+    rgw: civetweb/openssl: automagic: load libssl.so and libcrypto.so by soname.
+    
+    If building with radosgw, always look for openssl library (even when
+    building with nss).  Then, use objdump to fetch SONAME from the copies
+    of libssl and libcrypto that were found.  When building civetweb; pass
+    the library soname values in as the libraries to load with "dlopen".
+    
+    The issue reported here against master
+    http://tracker.ceph.com/issues/16535
+    reflects stuff that doesn't seem to have made it into jewel
+    	(which had: -lssl -lcrypto hard-wired into it.)
+    Still, since people were pretty riled about making that change,
+    this puts things in congruence to the final result in master.
+    	(which is: runtime shared object load of ssl crypto by soname.)
+    
+    Fixes: http://tracker.ceph.com/issues/11239
+    
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    (inspired by commit 7caa0bd002110b62514da83a37a2a3deb841267a)
+
+commit 045551485415309ba9bad77e1aee28a0008881ca
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Mon Jul 11 17:54:07 2016 -0400
+
+    rgw: civetweb/openssl: Load libssl.so and libcrypto.so by soname.
+    
+    If building with radosgw, always look for openssl library (even when
+    building with nss).  Then, use objdump to fetch SONAME from the copies
+    of libssl and libcrypto that were found.  When building civetweb; pass
+    the library soname values in as the libraries to load with "dlopen".
+    
+    This is a problem that went away for a bit, but came back with some
+    changes for
+    http://tracker.ceph.com/issues/16535
+    
+    Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1341775
+    Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1258961
+    
+    Fixes: http://tracker.ceph.com/issues/11239
+    
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    (cherry picked from commit 7caa0bd002110b62514da83a37a2a3deb841267a)
+
+commit 386640865dee30d38f17e55fc87535e419bc3cb5
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Thu Nov 3 23:33:44 2016 -0400
+
+    rgw: cmake: remove useless civetweb include path side effect.
+    
+    For 'target_include_directories" for the cmake object library
+    'civetweb_common_objs', change from PUBLIC to PRIVATE.  This doesn't
+    break anything, so it wasn't doing anything useful.  If it has it
+    any effect, it would be to cause everything that linked against this
+    "library" to also use the indictated include path.  Which would be great
+    except everything in ceph wants to include "civetweb/civetweb.h" and
+    not "civetweb.h".  We already make separate arrangements elsewhere for
+    that to work.  Additionally, static object libraries in cmake aren't
+    really libraries, so I'm not entirely sure this even does anything.
+    So: making this public is not useful, and could be harmful.  Making it
+    private makes this only take effect for building civetweb.c itself,
+    exactly the effect we we require, and no more.
+    
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    (cherry picked from commit 8308a13b0257c9460fd2a721c20b0c37cb9e7c57)
diff --git a/doc/changelog/v10.2.8.txt b/doc/changelog/v10.2.8.txt
new file mode 100644
index 000000000..575a7b062
--- /dev/null
+++ b/doc/changelog/v10.2.8.txt
@@ -0,0 +1,5435 @@
+commit f5b1f1fd7c0be0506ba73502a675de9d048b744e
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Thu Jul 6 14:56:18 2017 +0000
+
+    10.2.8
+
+commit 66dbf9beef04988dbd3653591e51afa6d84e3990
+Merge: 2f491b2e5e f46ccf2cb4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jul 4 17:43:57 2017 +0200
+
+    Merge pull request #14710 from smithfarm/wip-start-race
+    
+    tests: rados: sleep before ceph tell osd.0 flush_pg_stats after restart
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 2f491b2e5e8b1f340b28415c3bd3d9628603c377
+Merge: 552a573f84 a372b4eca1
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jul 4 07:31:50 2017 -0700
+
+    Merge pull request #16089 from ceph/wip_fix_point_jewel
+    
+    qa/Fixed upgrade sequence to 10.2.0 -> 10.2.7 -> latest -x (10.2.8)
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 552a573f8426ecfec1a0df21a6c3941afd4e460c
+Merge: 53a3be7261 55eeaadfc4
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jul 3 17:23:43 2017 -0700
+
+    Merge pull request #16088 from smithfarm/wip-fix-client-upgrade-centos
+    
+    tests: run upgrade/client-upgrade on latest CentOS 7.3
+    
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit a372b4eca1f25647541943918ae737f20783db11
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Mon Jul 3 14:18:14 2017 -0700
+
+    Fixed upgrade sequence to 10.2.0 -> 10.2.7 -> latest -x (10.2.8)
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 55eeaadfc4025c83cb63c951265710868df0325f
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Jul 3 22:55:21 2017 +0200
+
+    tests: run upgrade/client-upgrade on latest CentOS 7.3
+    
+    Before this patch, all centos jobs were failing because there are no longer any
+    CentOS 7.2 machines in Sepia.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 53a3be7261cfeb12445fbdba8238eefa40ed09f5
+Merge: 84bd162978 d33b30cdb0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jun 30 16:43:38 2017 +0200
+
+    Merge pull request #15504 from Vicente-Cheng/wip-20151-jewel
+    
+    jewel: ceph-disk: do not setup_statedir on trigger
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 84bd162978e48eead40335bcbd92e4ab18e9c590
+Merge: c710689109 8e0e4a0ce7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jun 28 10:20:23 2017 +0200
+
+    Merge pull request #15904 from smithfarm/wip-20413-jewel
+    
+    jewel: tests: upgrade:hammer-x/stress-split-erasure-code-x86_64 fails in 10.2.8 integration testing
+    
+    Reviewed-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit c7106891096c895f0cc5c2cef438078ea48de95d
+Merge: 5c6cb14806 9d3110c276
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jun 28 08:16:33 2017 +0200
+
+    Merge pull request #14930 from smithfarm/wip-19829-jewel
+    
+    jewel: tests: New upgrade test for #14930
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 5c6cb1480699f5ce464e25b9cacdda770ce3660d
+Merge: bdc085d02a d43e19d886
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jun 28 08:16:04 2017 +0200
+
+    Merge pull request #14392 from asheplyakov/19508-jewel
+    
+    jewel: osd: pg_pool_t::encode(): be compatible with Hammer <= 0.94.6
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 9d3110c276917055b078cd14c181b2bda2625821
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Jun 25 10:32:16 2017 +0200
+
+    tests: upgrade/hammer-x/v0-94-6-mon-overload: tweak packages list
+    
+    Include some hammer dependencies that aren't in the jewel default packages
+    list, and exclude some java packages that may not be in the hammer repo and are
+    not needed for the upgrade test in any case.
+    
+    N.B.: This cannot be cherry-picked from master because upgrade/hammer-x was
+    dropped in master.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 6a64f8901bb3b218a8dc58b11d6c13033d45f067
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed May 3 11:39:27 2017 +0200
+
+    tests: upgrade/hammer-x: new v0-94-6-mon-overload subsuite
+    
+    This is not a cherry-pick from master because direct upgrades
+    from hammer to kraken+ are not supported.
+    
+    Fixes: http://tracker.ceph.com/issues/19829
+    References: http://tracker.ceph.com/issues/19508
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit bdc085d02ab9723f6b90b6a3047bc51cf224b930
+Merge: e41ae4a7b1 d2d4b7202d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jun 27 12:41:59 2017 +0200
+
+    Merge pull request #15936 from batrick/i20412
+    
+    qa: enable quotas for pre-luminous quota tests
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit d2d4b7202d77e5696eb18c4da4f7d614116ced36
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Mon Jun 26 19:04:48 2017 -0700
+
+    qa: enable quotas for pre-luminous quota tests
+    
+    This cannot be cherry-picked from master because the config option is removed
+    since 0f250a889dba2100d3afcea0a18e4f6a8d086b86.
+    
+    Fixes: http://tracker.ceph.com/issues/20412
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit e41ae4a7b1b9cc4394473e21f6e6d6ef9cab1d59
+Merge: a21af3b7a1 682b4d717c
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon Jun 26 21:24:11 2017 -0500
+
+    Merge pull request #15933 from smithfarm/wip-hammer-jewel-x
+    
+    jewel: tests: drop upgrade/hammer-jewel-x
+
+commit 682b4d717c96b516c315a01b1174af3503dedba6
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Jun 27 02:27:22 2017 +0200
+
+    tests: drop upgrade/hammer-jewel-x
+    
+    This suite doesn't have any test logic in it. Its existence in the jewel branch
+    appears to be an oversight.
+    
+    This cannot be cherry-picked from master because the upgrade/hammer-jewel-x
+    suite is present (and justified) in master and is not currently being dropped
+    there.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit a21af3b7a1ea5abfa4f344800e413d6249824204
+Merge: 615a6ab9b7 a744340790
+Author: John Spray <jspray@redhat.com>
+Date:   Mon Jun 26 16:25:12 2017 -0400
+
+    Merge pull request #15438 from Vicente-Cheng/wip-20027-jewel
+    
+    jewel: mds: issue new caps when sending reply to client
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 615a6ab9b723d204c6de567750fe9450742fcedb
+Merge: 9b13b48b3e 8ac0e5c363
+Author: John Spray <jspray@redhat.com>
+Date:   Mon Jun 26 16:23:48 2017 -0400
+
+    Merge pull request #15000 from jan--f/wip-19846-jewel
+    
+    jewel: cephfs: normalize file open flags internally used by cephfs
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 9b13b48b3ed919340789a41d065eb4a9a27110de
+Merge: d217da1742 b429fa1807
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jun 26 18:25:03 2017 +0200
+
+    Merge pull request #15383 from asheplyakov/20014-bp-jewel
+    
+    jewel: cls/rgw: list_plain_entries() stops before bi_log entries
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit d217da174252f9126d530868aa472230ecba31ca
+Merge: e520040ece 4028774122
+Author: Zack Cerza <zack@cerza.org>
+Date:   Mon Jun 26 10:17:03 2017 -0600
+
+    Merge pull request #15870 from smithfarm/wip-swift-task-move-jewel
+    
+    tests: move swift.py task from teuthology to ceph, phase one (jewel)
+
+commit e520040ecec756ce181f716dad1c0bad41c77a7d
+Merge: dde8656e6b 1c0c9093ab
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jun 26 09:24:04 2017 +0200
+
+    Merge pull request #15842 from smithfarm/wip-sortbitwise-jewel
+    
+    qa/suites/upgrade/hammer-x: set "sortbitwise" for jewel clusters
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit dde8656e6b5e7ffe66a6fd695cbc17dfb18fb43e
+Merge: 498c96e66a 06cf9f3650
+Author: John Spray <jspray@redhat.com>
+Date:   Sun Jun 25 19:59:40 2017 -0400
+
+    Merge pull request #15468 from smithfarm/wip-20140-jewel
+    
+    jewel: cephfs: Journaler may execute on_safe contexts prematurely
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 4028774122954023265d7825fbf9e91dc526fdee
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Jun 25 12:42:36 2017 +0200
+
+    tests: swift.py: tweak imports
+    
+    The ".." form only works within the teuthology repo. With swift.py now in the
+    Ceph repo, we have to be explicit.
+    
+    Error message was: "ValueError: Attempted relative import beyond toplevel
+    package
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 8e0e4a0ce7489542f47522e0a5161a5bf123c744
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Jun 25 10:27:58 2017 +0200
+
+    tests: upgrade/hammer-x/stress-split: tweak packages list
+    
+    Include some hammer dependencies that aren't in the jewel default packages
+    list, and exclude some java packages that may not be in the hammer repo and are
+    not needed for the upgrade test in any case.
+    
+    N.B.: This cannot be cherry-picked from master because upgrade/hammer-x was
+    dropped in master.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit a86ce728954a765797ce634025d43650d990e480
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Jun 25 09:27:47 2017 +0200
+
+    tests: swift.py: clone the ceph-jewel branch
+    
+    The master branch of ceph/swift.git contains tests that are incompatible with
+    Jewel and Hammer. The ceph-jewel branch omits these tests.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 498c96e66a91edc8bd614cfc8fc5a14b3d210a76
+Merge: 38af498f9f cda721bbbf
+Author: John Spray <jspray@redhat.com>
+Date:   Fri Jun 23 08:02:48 2017 -0400
+
+    Merge pull request #15472 from smithfarm/wip-20148-jewel
+    
+    jewel: mds: Too many stat ops when trying to probe a large file
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 3d5b489369bb2cecccb1f36347654c0a37069d1c
+Merge: 38af498f9f 7b58ac97e9
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jun 23 08:35:27 2017 +0200
+
+    Merge branch 'master' of /home/smithfarm/src/ceph/upstream/teuthology into wip-swift-task-move-jewel
+
+commit 7b58ac97e9dd195f4170e9e0ea00bae76d1f3ccd
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jun 23 08:27:42 2017 +0200
+
+    tests: move swift.py task to qa/tasks
+    
+    In preparation for moving this task from ceph/teuthology.git into ceph/ceph.git
+    
+    The move is necessary because jewel-specific changes are needed, yet teuthology
+    does not maintain a separate branch for jewel. Also, swift.py is a
+    Ceph-specific task so it makes more sense to have it in Ceph.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 38af498f9f7b62f9f851364ae7f2691832423198
+Merge: d0ae1de51f aa0cd461df
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jun 22 22:07:23 2017 +0200
+
+    Merge pull request #15529 from badone/wip-async-sleep-timer-fix-jewel
+    
+    jewel:  osd: Implement asynchronous scrub sleep
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 1c0c9093ab913a82c1dc5656a54b4009bdc35c9c
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jun 22 11:32:42 2017 +0200
+
+    qa/suites/upgrade/hammer-x: set "sortbitwise" for jewel clusters
+    
+    Inspired by 3734280522a913ca8340ebc98b80978f63bade6f
+    
+    This cannot be cherry-picked from master because master does not have
+    qa/suites/upgrade/hammer-x
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit d0ae1de51f5faf26a2f4b0d5b7f494a4923f870d
+Merge: 64c011a8c4 de76fdbb9f
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Thu Jun 22 11:59:51 2017 +0800
+
+    Merge pull request #15824 from tchaikov/jewel
+    
+    qa/workunits/rados/test-upgrade-*: whitelist tests the right way
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit de76fdbb9f435652e2c15326d00b01d26ab007a7
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Jun 22 08:06:43 2017 +0800
+
+    qa/workunits/rados/test-upgrade-*: whitelist tests the right way
+    
+    --gtest_filter=POSTIVE_PATTERNS[-NEGATIVE_PATTERNS], so we cannot add
+    multiple exclusive patterns using -pattern:-pattern, instead, we should
+    use: -pattern:pattern
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    Conflicts:
+            qa/workunits/rados/test-upgrade-v11.0.0.sh: this change is not
+        cherry-picked from master, because the clone-range op was removed
+        from master. and only supported in pre-luminous releases.
+
+commit 64c011a8c4af27dc095b1a9190ccf1ca76d2cc8f
+Merge: e8da5e376f ab78cd040f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jun 20 22:54:00 2017 +0200
+
+    Merge pull request #14661 from smithfarm/wip-19575-jewel
+    
+    jewel: rgw: unsafe access in RGWListBucket_ObjStore_SWIFT::send_response()
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit e8da5e376fc426f85fcab84a5ae71b3c17ed0068
+Merge: 5a1e849ecf 1af6781d3c
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Tue Jun 20 22:42:47 2017 +0800
+
+    Merge pull request #15778 from tchaikov/wip-upgrade-without-clone-range-jewel
+    
+    qa/workunits/rados/test-upgrade-*: whitelist tests for master
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit d43e19d88692bd318f0569559867df919c26d8db
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Fri Apr 7 12:34:20 2017 +0400
+
+    jewel: osd: pg_pool_t::encode(): be compatible with Hammer <= 0.94.6
+    
+    This patch is necessary for Jewel only since direct upgrades from Hammer
+    to Kraken and newer are not supported.
+    
+    Fixes: http://tracker.ceph.com/issues/19508
+    
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+
+commit 1af6781d3c60421930087d31124e62cae530ca24
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jun 20 19:49:14 2017 +0800
+
+    qa/workunits/rados/test-upgrade-*: whitelist tests for master
+    
+    The jewel-x upgrade test now runs this script against a mixed cluster on
+    a machine with code from master installed.  That means we have to
+    skip any new tests that will fail on a mixed cluster. CloneRange was
+    removed in 0d7b0b7.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    Conflicts:
+        qa/workunits/rados/test-upgrade-v11.0.0.sh: this change is not
+    cherry-picked from master, because the clone-range op was removed from
+    master. and only supported in pre-luminous releases.
+
+commit a74434079088129244b7aae6ccc6df7094282eba
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat Apr 22 12:27:12 2017 +0800
+
+    mds: issue new caps when sending reply to client
+    
+    After Locker::issue_new_caps() adds new Capability data struct,
+    do not issue caps immediately. Let CInode::encode_inodestate()
+    do the job instead. This can avoid various races that early reply
+    is not allowed, caps that haven't been sent to client gets revoked.
+    
+    Fixes: http://tracker.ceph.com/issues/19635
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 799703a4acb49db0b6cc99a23e4326767e694c3a)
+
+commit d33b30cdb019937ff88f9724599f52f4e00d37cf
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Jun 1 11:37:20 2017 +0200
+
+    ceph-disk: do not setup_statedir on trigger
+    
+    trigger may run when statedir is unavailable and does not use it.
+    
+    Fixes: http://tracker.ceph.com/issues/19941
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 16bfbdd3d9988523bba31aace516c303057daa58)
+
+commit f46ccf2cb4701cd93cd9b15a4e57b5b97798b947
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Apr 21 11:05:05 2017 +0200
+
+    tests: rados: sleep before ceph tell osd.0 flush_pg_stats after restart
+    
+    Even though we wait for HEALTH_OK after restarting the daemons, they are not
+    ready to respond to flush_pg_stats.
+    
+    The reason why the osd is not ready for "tell" command after "ceph health"
+    shows that the cluster is "HEALTH_OK" is that the monitor fails to be notified
+    that the osd in question is not up in "heatbeat_interval". Because infernalis
+    does not have the osd_fast_fail_on_connection_refused support, the monitor
+    needs longer to detect that an osd is down, and osd_heartbeat_grace is used to
+    determine if an osd is down.
+    
+    References: http://tracker.ceph.com/issues/16239
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+
+commit ab78cd040f6d3946ed40b6638ebcf52969a7cbb6
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Mar 8 14:52:34 2017 -0800
+
+    rgw: fix crash when listing objects via swift
+    
+    Fixes: http://tracker.ceph.com/issues/19249
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit a9ec5e8ce184e19c009863db4d3519f9d8af91bd)
+    
+    Conflicts:
+            src/rgw/rgw_rest_swift.cc ("key" element of RGWObjEnt struct
+               is not a reference; fix)
+    
+    (cherry picked from commit 92b35155ff7b7492f3c50bf4f2ff0ffef2ca1c55)
+
+commit 5a1e849ecf215d82e31b9bdd0970cb04200de2c9
+Merge: 2469085d57 66c3db7aee
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jun 19 22:46:47 2017 +0200
+
+    Merge pull request #14752 from cbodley/wip-19474
+    
+    jewel: rgw: allow system users to read SLO parts
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit cda721bbbfae00ec4244718ae20cbd9ae914c630
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri May 19 09:37:15 2017 +0800
+
+    client: update the 'approaching max_size' code
+    
+    The old 'approaching max_size' code expects MDS set max_size to
+    '2 x reported_size'. This is no longer true. The new code reports
+    file size when half of previous max_size increment has been used.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 9316b0442c6f828dcf8da952e4c7a63c4db1398d)
+    
+    Conflicts:
+            src/client/Client.cc - in jewel, second argument to check_caps() is
+                a bool (see 0df562a8e13 which is not in jewel)
+
+commit 439f39128ec278ce78139d4b96ed098c68efa3f5
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed May 17 19:08:37 2017 +0800
+
+    mds: limit client writable range increment
+    
+    For very large file, setting the writable range to '2 * file_size'
+    causes file recovery to run a long time. To recover a 1T file, Filer
+    needs to probe 2T~1T range.
+    
+    Fixes: http://tracker.ceph.com/issues/19955
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 538f35bef944b18e9bca2b15ed7f4e8807ef0554)
+    
+    Conflicts:
+            src/mds/Locker.h - in jewel, file_update_finish() has different
+                arguments than it does in master
+
+commit 06cf9f365033f7913051bdf4060f0bc6fc0444d7
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue May 23 21:46:54 2017 +0800
+
+    osdc/Journaler: avoid executing on_safe contexts prematurely
+    
+    Journaler::_do_flush() can skip flushing some data when prezered
+    journal space isn't enough. Before updating Journaler::next_safe_pos,
+    we need to check if Journaler::_do_flush() has flushed enough data.
+    
+    Fixes: http://tracker.ceph.com/issues/20055
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 6511e7a9e35a14216c03cd6921ca4d232274f953)
+
+commit 2e299b50de4a297fee2aec21290632336d239857
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Apr 12 16:00:18 2017 +0800
+
+    osdc/Journaler: make header write_pos align to boundary of flushed entry
+    
+    This can speed up the process that detects and drops partial written
+    entry in the log tail.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 8ae2962b79903e217fda83cea4140af64b5d6883)
+    
+    Conflicts:
+            src/osdc/Journaler.cc - 8d4f6b92cba is not being backported to jewel
+            src/osdc/Journaler.h - Journaler::Journaler initializer list is different in jewel, compared to master
+
+commit 2469085d57a05933589165f6f99a67b2e28c7022
+Merge: 42c3fbc129 d57437e338
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 10:01:27 2017 -0400
+
+    Merge pull request #14672 from smithfarm/wip-19334-jewel
+    
+    jewel: MDS heartbeat timeout during rejoin, when working with large amount of caps/inodes
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 8ac0e5c363bd6439071d26874b6714cf2376736f
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri May 12 10:38:51 2017 +0800
+
+    pybind: fix cephfs.OSError initialization
+    
+    Traceback (most recent call last):
+      File "<stdin>", line 1, in <module>
+      File "cephfs.pyx", line 672, in cephfs.LibCephFS.open (/home/zhyan/Ceph/ceph-2/build/src/pybind/cephfs/pyrex/cephfs.c:10160)
+      File "cephfs.pyx", line 155, in cephfs.OSError.__init__ (/home/zhyan/Ceph/ceph-2/build/src/pybind/cephfs/pyrex/cephfs.c:1889)
+    TypeError: __init__() takes exactly 3 positional arguments (2 given)
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit e6493f64ba4592b8dca54ece4464efa6c7f331a7)
+
+commit 09b9410c2e69a466b001d92fc14eb44d768009f1
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed May 10 08:13:52 2017 +0800
+
+    pybind: fix open flags calculation
+    
+    (O_WRONLY | O_RDWR) is invaild open flags
+    
+    Fixes: http://tracker.ceph.com/issues/19890
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 2c25c99cb4572ffae97555a56b24a4c4097dcdec)
+
+commit 42c3fbc129cbb60d447c1a6b0402a9def1656446
+Merge: 71d45e1905 7347f11939
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:43:05 2017 -0400
+
+    Merge pull request #14677 from smithfarm/wip-19665-jewel
+    
+    jewel: mds: C_MDSInternalNoop::complete doesn't free itself
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 71d45e190528124a8ff7e4674f7bfb7c340f80ee
+Merge: 27c915f5d2 e6daee8a9f
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:35:08 2017 -0400
+
+    Merge pull request #15466 from smithfarm/wip-19762-jewel
+    
+    jewel: cephfs: non-local quota changes not visible until some IO is done
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 27c915f5d21bd84502e35eb269d955fafc47de0b
+Merge: a76357622a db053da618
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:32:47 2017 -0400
+
+    Merge pull request #14700 from smithfarm/wip-19709-jewel
+    
+    jewel: mds: enable start when session ino info is corrupt
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit a76357622a2773850153aa1f6ea02b1737942c4b
+Merge: 85aab833f7 db86a24e79
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:31:22 2017 -0400
+
+    Merge pull request #14685 from smithfarm/wip-19675-jewel
+    
+    jewel: cephfs: Test failure: test_data_isolated (tasks.cephfs.test_volume_client.TestVolumeClient)
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 85aab833f7f66ac81d8b12f4203fa215787f8d2a
+Merge: f6b395115e 7b9283beec
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:30:53 2017 -0400
+
+    Merge pull request #14684 from smithfarm/wip-19673-jewel
+    
+    jewel: cephfs: mds is crushed, after I set about 400 64KB xattr kv pairs to a file
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit f6b395115e4de15d73269ff6b96f2ee0fd0ea9c3
+Merge: ff60fceb5f b52c508861
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:30:13 2017 -0400
+
+    Merge pull request #14683 from smithfarm/wip-19671-jewel
+    
+    jewel: cephfs: MDS assert failed when shutting down
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit ff60fceb5f5f7f9df4a48da4ad8bd863b9a04e2d
+Merge: 6a6d57d2de 96e801fb53
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:29:50 2017 -0400
+
+    Merge pull request #14682 from smithfarm/wip-19668-jewel
+    
+    jewel: cephfs: MDS goes readonly writing backtrace for a file whose data pool has been removed
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 6a6d57d2de674c5a971e999cb2731b2d8ae1b523
+Merge: 8260669efa f34489dd52
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:27:13 2017 -0400
+
+    Merge pull request #14679 from smithfarm/wip-19666-jewel
+    
+    jewel: cephfs: The mount point break off when mds switch hanppened.
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 8260669efa74f41d3b1b1039fb7b34e070951c34
+Merge: 013529b61f 824b19a9a6
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:26:40 2017 -0400
+
+    Merge pull request #14676 from smithfarm/wip-19619-jewel
+    
+    jewel: cephfs: MDS server crashes due to inconsistent metadata.
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 013529b61fc4fbf02656f7c6cb0baa1bc6004758
+Merge: b518522f64 eab56dae67
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:24:38 2017 -0400
+
+    Merge pull request #14674 from smithfarm/wip-19482-jewel
+    
+    jewel: cephfs: No output for ceph mds rmfailed 0 --yes-i-really-mean-it command
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit b518522f64b66f4a14618a9345b6314ca0f2c54c
+Merge: 388e0d1bc3 63f41d543f
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:23:27 2017 -0400
+
+    Merge pull request #14671 from smithfarm/wip-19044-jewel
+    
+    jewel: tests: buffer overflow in test LibCephFS.DirLs
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 388e0d1bc35985a6916d3eb1ca5184a0907b6e8b
+Merge: d7c7ce7ebb 7146816065
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:23:13 2017 -0400
+
+    Merge pull request #14670 from smithfarm/wip-18949-jewel
+    
+    jewel: mds: avoid reusing deleted inode in StrayManager::_purge_stray_logged
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit d7c7ce7ebbb663dab0dfa8058c845d494d7615cc
+Merge: d717ef73bc d8b139b584
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:22:28 2017 -0400
+
+    Merge pull request #14669 from smithfarm/wip-18900-jewel
+    
+    jewel: cephfs: Test failure: test_open_inode
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit d717ef73bc0d5ef24551ec9157385c8b0521380b
+Merge: c2a3b7567f 36c86f71ef
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jun 14 09:21:43 2017 -0400
+
+    Merge pull request #14668 from smithfarm/wip-18705-jewel
+    
+    jewel: mds: fragment space check can cause replayed request fail
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit c2a3b7567fbe0b2f62bcd38cd9bb9a5a2a238743
+Merge: fd9256b770 5b56214519
+Author: John Spray <jspray@redhat.com>
+Date:   Tue Jun 13 19:05:55 2017 +0100
+
+    Merge pull request #14698 from smithfarm/wip-19677-jewel
+    
+    jewel: cephfs: ceph-fuse does not recover after lost connection to MDS
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit fd9256b77010066c934fd0016eb6d3f9c1fb54e1
+Merge: 26ada59c81 c49b114e8d
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Jun 9 19:39:17 2017 +0200
+
+    Merge pull request #14766 from smithfarm/wip-19757-jewel
+    
+    jewel: rgw: fix failed to create bucket if a non-master zonegroup has a single zone
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek@suse.com>
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 26ada59c810bfda5c16f2d935a2b98711e7b2c76
+Merge: fdd25c2bdb e552d91f73
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Jun 9 19:38:46 2017 +0200
+
+    Merge pull request #14787 from linuxbox2/jewel-rgw-shard-limit-ck
+    
+    jewel: rgw: add bucket size limit check to radosgw-admin
+    
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek@suse.com>
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit fdd25c2bdb9ec6bb8e3061088b8782bfb2331bc4
+Merge: cfd6750416 fb3ee2efcc
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Jun 9 19:38:23 2017 +0200
+
+    Merge pull request #14789 from mdw-at-linuxbox/wip-jewel-rgw-rvk
+    
+    jewel: rgw: swift: disable revocation thread if sleep == 0 || cache_size == 0
+    
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek@suse.com>
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit cfd6750416baeb2bbd3836bc7dc55d83fc214c20
+Merge: ce1fc3492e 86980a045b
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Jun 9 19:36:02 2017 +0200
+
+    Merge pull request #14815 from smithfarm/wip-19786-jewel
+    
+    jewel: rgw: failure to create s3 type subuser from admin rest api
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek@suse.com>
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit ce1fc3492e87c669f7059c2047a3bed077418a89
+Merge: 7ca0252560 aa99558934
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jun 7 20:13:15 2017 +0200
+
+    Merge pull request #15312 from theanalyst/wip-20078
+    
+    jewel: rgw: only append zonegroups to rest params if not empty
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 7ca0252560d1ff23384afb50a9c2ae2aad2ce85c
+Merge: 62c500f522 59bd6711a4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jun 7 13:48:06 2017 +0200
+
+    Merge pull request #15382 from theanalyst/wip-mem-leak2
+    
+    jewel: rgw:fix memory leaks in data/md sync
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit aa0cd461df5fee6d143bc07440ec6de829761cef
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Mon May 22 13:21:25 2017 +1000
+
+    osd: Move scrub sleep timer to osdservice
+    
+    PR 14886 erroneously creates a scrub sleep timer for every pg resulting
+    in a proliferation of threads. Move the timer to the osd service so
+    there can be only one.
+    
+    Fixes: http://tracker.ceph.com/issues/19986
+    
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit f110a82437df79dc20207d296e8229fc0e9ce18b)
+    
+    Conflicts:
+            src/osd/PG.cc - ceph_clock_now requires a CephContext argmunent
+            in Jewel
+
+commit c47bd0562b1187ffb0b1b2c1ef5f105aa7951d10
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Mon Apr 24 14:10:47 2017 +1000
+
+    osd: Implement asynchronous scrub sleep
+    
+    Rather than blocking the main op queue just do an async sleep.
+    
+    Fixes: http://tracker.ceph.com/issues/19497
+    
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 7af3e86c2e4992db35637864b83832535c94d0e6)
+
+commit e6daee8a9fbc576da2a03550a81056d093a516c9
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Jun 4 20:39:58 2017 +0200
+
+    Client.cc: adjust Client::_getattr calls
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit a2c7a2262ac8ecbea78f09e6e8e6a37498568d57
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 15 19:36:08 2017 +0000
+
+    qa/cephfs: use getfattr/setfattr helpers
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit dd43d3bc646aeab88486b0963fc83de0b18800c4)
+    
+    Conflicts:
+            qa/tasks/cephfs/test_data_scan.py: difference in the
+                self._mount.run_shell() call in NonDefaultLayout::write (which is
+                being dropped by this commit) - in jewel it has "sudo", and in
+                master it doesn't
+
+commit 12aa35a6d50f612df77199ac4f35c7baeed0583e
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Jun 22 13:00:44 2016 +0100
+
+    tasks/cephfs: fix race while mounting
+    
+    This could fail if the mount hadn't finished
+    coming up.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit adfb757c898a80f18c15dafd02e29840c5931c87)
+
+commit a7b699269b65c76361fcb1d10593812be40c7612
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 15 19:26:30 2017 +0000
+
+    qa: add test for reading quotas from different clients
+    
+    Fixes: http://tracker.ceph.com/issues/17939
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 61617f8f10a6322603a9add77980865cd972ef97)
+
+commit 8b8ee392b8093b9b140dbbe895691f69ae40440f
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 15 17:51:44 2017 +0000
+
+    client: _getattr on quota_root before using in statfs
+    
+    ...so that after someone adjusts the quota settings
+    on an inode that another client is using as its mount root,
+    the change is visible immediately on the other client.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 3d25941aadd223669448d0f5d3c0bd1fefa72308)
+
+commit dd7d59a08141d6a24b172c22f5e27c8962e25fb9
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 15 15:32:47 2017 +0000
+
+    client: getattr before read on ceph.* xattrs
+    
+    Previously we were returning values for quota, layout
+    xattrs without any kind of update -- the user just got
+    whatever happened to be in our cache.
+    
+    Clearly this extra round trip has a cost, but reads of
+    these xattrs are fairly rare, happening on admin
+    intervention rather than in normal operation.
+    
+    Fixes: http://tracker.ceph.com/issues/17939
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 532dc4b68e538c189ef828f67cecd0d647a62250)
+
+commit 62c500f52240eaa5faadd3795bd9ec84bdcbc6c7
+Merge: 2badc2416c 8dd93cabd5
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Jun 2 09:54:22 2017 -0700
+
+    Merge pull request #15416 from dzafman/wip-20126
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 8dd93cabd52cbafc29a47862f343431eb6f1cfe3
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed May 31 15:39:19 2017 -0700
+
+    osd: Object level shard errors are tracked and used if no auth available
+    
+    Shards with object mismatch are tracked to mark them inconsistent
+    Fix test because storing omap_digest in object_info not behaving as before
+    
+    Fixes: http://tracker.ceph.com/issues/20089
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    
+    (cherry picked from commit 1cacbea763c7aabfeaaf4bd5e878301044184117)
+    
+    Conflicts:
+            src/test/osd/osd-scrub-repair.sh (no alloc_hint in object_info)
+
+commit 59bd6711a47c354117a612dd4ef033d70d449383
+Author: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+Date:   Wed Jun 1 17:20:49 2016 +0800
+
+    rgw:fix memory leaks
+    
+    Signed-off-by: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+    (cherry picked from commit 73e5be2b6133cf4caa0e5e5c8c9eae748b785dbf)
+
+commit b429fa1807062716c9705ddcf316ed9b2741cc43
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri May 5 14:56:40 2017 -0400
+
+    cls/rgw: list_plain_entries() stops before bi_log entries
+    
+    list_plain_entries() was using encode_obj_versioned_data_key() to set
+    its end_key, which gives a prefix of BI_BUCKET_OBJ_INSTANCE_INDEX[=2]
+    
+    that range between start_key and end_key would not only span the
+    BI_BUCKET_OBJS_INDEX[=0] prefixes, but BI_BUCKET_LOG_INDEX[=1] prefixes
+    as well. this can result in list_plain_entries() trying and failing to
+    decode a rgw_bi_log_entry as a rgw_bucket_dir_entry
+    
+    Fixes: http://tracker.ceph.com/issues/19876
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit b29a1633a57abf443d5790c13d680d2917f86037)
+
+commit 2badc2416c9e16babbc91364502dcb03877e88bf
+Merge: 5d2a68eb90 1f895c2403
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue May 30 09:39:47 2017 -0500
+
+    Merge pull request #15360 from liewegas/wip-jewel-master-mixed
+    
+    qa/workunits/rados/test-upgrade-*: whitelist tests for master
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 1f895c24030b79ae9f2eae16b09582fc72928b6a
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue May 30 09:58:09 2017 -0400
+
+    qa/workunits/rados/test-upgrade-*: whitelist tests for master
+    
+    The jewel-x upgrade test now runs this script against a mixed cluster on
+    a machine with code from master installed.  That means we have to skip
+    any new tests that will fail on a mixed cluster.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 5d2a68eb903f533689d1a1cbc410ca940baeaff3
+Merge: 998d1ee4f5 81e35b9416
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat May 27 09:21:44 2017 +0200
+
+    Merge pull request #15208 from liewegas/wip-sortbitwise-jewel
+    
+    mon: fix 'sortbitwise' warning on jewel
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 998d1ee4f503cb4875283fe8d2140f3b10a26643
+Merge: 54bc1e13ea 99c65bbc18
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat May 27 02:46:55 2017 +0200
+
+    Merge pull request #14851 from yehudasa/wip-rgw-support-ragweed-jewel
+    
+    jewel: rgw: add apis to support ragweed suite
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit aa99558934c6143cb35e722148b87f35ccecc397
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Aug 23 10:22:42 2016 -0700
+
+    rgw: rest conn functions cleanup, only append zonegroup if not empty
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 4e41af19846db75081cb0ddb7b33dc2bb9321ace)
+
+commit 5d90798432a6446636699fc03b2f548010b1212f
+Author: Karol Mroz <kmroz@suse.com>
+Date:   Thu Mar 17 10:32:14 2016 +0100
+
+    rgw: rest and http client code to use param vectors
+    
+    Replaces param/header lists with vectors. In these cases, we're only ever
+    adding to the back of the list, so a vector should be more efficient.
+    Also moves param_pair_t/param_vec_t higher up the include chain for
+    cleaner function signatures.
+    
+    Signed-off-by: Karol Mroz <kmroz@suse.com>
+    (cherry picked from commit d4a2527872e0f5c3ae2874bb7d0ff459ae83cfd4)
+    
+    Conflicts:
+            src/rgw/rgw_http_client.cc
+    trivial ws conflict on rebase
+
+commit 54bc1e13ea19642c8d5893f29e96ee5c053f9c59
+Merge: 966f222917 43327f83ef
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Thu May 25 12:45:30 2017 -0400
+
+    Merge pull request #14765 from smithfarm/wip-18972-jewel
+    
+    jewel: ceph-disk does not support cluster names different than 'ceph'
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit 81e35b941659a0f6dff8a935c27c7dd6d5cc4213
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed May 24 09:48:11 2017 -0400
+
+    qa/suites/rados/singleton-nomsgr/*: set sortbitwise after upgrade
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit f2814e4dbb2821a415310559fce405c2eab23947
+Author: huanwen ren <ren.huanwen@zte.com.cn>
+Date:   Tue Dec 27 10:54:45 2016 +0000
+
+    mon/OSDMonitor: fixup sortbitwise flag warning
+    
+    "ceph -s" does not report warning when using
+    command "ceph osd unset sortbitwise" to drop
+    sortbitwise flag.
+    we should use "osdmap.get_up_osd_features() &
+    CEPH_FEATURE_OSD_BITWISE_HOBJ_SORT"
+    instead of "(osdmap.get_features(CEPH_ENTITY_TYPE_OSD, NULL) &
+    CEPH_FEATURE_OSD_BITWISE_HOBJ_SORT)",
+    because osdmap.get_features only get local "features"
+    
+    Signed-off-by: huanwen ren <ren.huanwen@zte.com.cn>
+    (cherry picked from commit c25ee187e28724846d0011cd8145e16956d3636e)
+
+commit 02617188688eebde759c375a2257e076e4538491
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Nov 1 12:09:57 2016 -0400
+
+    mon: remove config option to disable no sortbitwise warning
+    
+    We'll require this soon.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 04e3319e8aecde9ca58ccb7c89016f9079c7d657)
+
+commit 966f2229178c5d72722982d0f6a40f4d01210d9a
+Merge: 6537fc741c 4ceaa7cce9
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed May 17 09:29:21 2017 +0200
+
+    Merge pull request #13450 from dreamhost/wip-18887-jewel
+    
+    jewel: msg: IPv6 Heartbeat packets are not marked with DSCP QoS - simple messenger
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit bb79663490468b4ac2832aa9e3cbac019c1f712d
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Thu Apr 20 18:38:43 2017 +0200
+
+    fs: normalize file open flags internally used by cephfs
+    
+    The file open flags (O_foo) are platform specific. Normalize these flags
+    before they are send to the MDS. For processing of client messages the
+    MDS should only compare to these normalized flags.
+    Otherwise this can lead to bogus flags getting transmitted on ppc64.
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 88d2da5e93198e69435e288ce00d216d5fe27f80)
+    
+    Conflicts:
+            src/client/Client.cc
+              Conflicts can be resolved by choosing changes from HEAD and
+              adding a call to ceph_flags_sys2wire where flags are logged.
+            src/mds/Server.cc
+              Conflicts can be resolved by choosing changes from HEAD and
+              while making sure that the MDS compares request O_ flags the
+              the CEPH_O_ flags, since all wire O_ flags are normalized.
+
+commit 4ceaa7cce9c9132d47564d79204b48b1d02e531c
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Wed May 3 22:31:40 2017 -0700
+
+    msg/simple/Pipe: manual backport of fix in PR#14795
+    
+    Manual backport of errno fixup from PR#14795
+    (6f1037e22c2a304795895498cdc955e0ef80f8e3), as noted by
+    https://github.com/ceph/ceph/pull/13450#discussion_r114696885.
+    
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit 6537fc741c3267472e607e581c002854d8eafd46
+Merge: af31b453f4 82ea0971b3
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed May 3 13:04:27 2017 +0200
+
+    Merge pull request #14667 from smithfarm/wip-18699-jewel
+    
+    jewel: client: fix the cross-quota rename boundary check conditions
+    
+    Reviewed-by: Gregory Farnum <gfarnum@redhat.com>
+
+commit af31b453f404cb73ee7bdb8b6b02d71ad8aab7e7
+Merge: 472034cbe0 335258f975
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed May 3 10:33:24 2017 +0200
+
+    Merge pull request #14332 from shinobu-x/wip-19396-jewel
+    
+     jewel: Objecter::epoch_barrier isn't respected in _op_submit()
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 472034cbe0735131f3419205e9f7e6530377bfba
+Merge: afe98ba074 acf608a903
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed May 3 10:32:37 2017 +0200
+
+    Merge pull request #14204 from dzafman/wip-18533-jewel
+    
+    jewel: core: two instances of omap_digest mismatch
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit afe98ba07436ea105f4c9c42a52795ce3757419e
+Merge: 0353a91bfd 043d70461c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed May 3 10:31:38 2017 +0200
+
+    Merge pull request #13884 from shinobu-x/wip-19119-jewel
+    
+    jewel: pre-jewel "osd rm" incrementals are misinterpreted
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 0353a91bfd799077ddd87de901e8a07f5da6a13d
+Merge: 460b12c259 608785a007
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed May 3 10:30:32 2017 +0200
+
+    Merge pull request #13647 from mslovy/wip-19083-jewel
+    
+    jewel: osd: preserve allocation hint attribute during recovery
+    
+    Reviewed-by: Gregory Farnum <gfarnum@redhat.com>
+
+commit 460b12c259f5563d9d1b2477149fe79486ba5bcd
+Merge: 630cfca36c 905c4acb99
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Apr 28 10:24:35 2017 +0200
+
+    Merge pull request #14791 from smithfarm/wip-19774-jewel
+    
+    jewel: osd: promote throttle parameters are reversed
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 630cfca36c5a42a58882966c5598752aac9ff54e
+Merge: 013e781b45 3ec1a9bf16
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Apr 28 10:23:45 2017 +0200
+
+    Merge pull request #14763 from smithfarm/wip-19562-jewel
+    
+    jewel: api_misc: [  FAILED  ] LibRadosMiscConnectFailure.ConnectFailure
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 99c65bbc1899663d0b23e2cec3d6e516d0e95ad7
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Jan 9 13:04:43 2017 -0800
+
+    rgw: new rest api to retrieve object layout
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 2768583dc486109e49d209243675b99fdd39e92c)
+
+commit 33745a342433716ad8a07ef846dbea9b57de5017
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jan 5 13:47:24 2017 -0800
+
+    rgw: rest api to read zone config params
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit a220a2efbfd675e6abf14ae33c21005bcbf6dadf)
+
+commit 013e781b45e28af15da4ddc32025aad713f128e8
+Merge: d144d99f00 b698d1fa4c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 27 21:46:41 2017 +0200
+
+    Merge pull request #14481 from dillaman/wip-19468-jewel
+    
+    jewel: librbd: is_exclusive_lock_owner API should ping OSD
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit d144d99f008776dc02c5838feb5a9eb9b84a400b
+Merge: 0f0cd82eda dbe90c79b8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 27 21:05:11 2017 +0200
+
+    Merge pull request #14666 from smithfarm/wip-19612-jewel
+    
+    jewel: librbd: Issues with C API image metadata retrieval functions
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 0f0cd82edabcb2feb0b29793e5b45eb424f1c42d
+Merge: eba821ce9c 216156b5d4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 27 21:04:27 2017 +0200
+
+    Merge pull request #14664 from smithfarm/wip-19325-jewel
+    
+    jewel: rbd: [api] temporarily restrict (rbd_)mirror_peer_add from adding multiple peers
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit eba821ce9c61bcb197895e6949ce30a5c07097fa
+Merge: 25e29c9215 b8fd297eb6
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 27 20:46:08 2017 +0200
+
+    Merge pull request #14694 from ceph/wip-bp-systemd
+    
+    [backport] qa/tasks: systemd test backport to jewel
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 25e29c921557883da4bf48ff2496bcec6448e6ec
+Merge: d7b63e21ca 7c6c3c753c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 27 08:44:26 2017 +0200
+
+    Merge pull request #13544 from shinobu-x/wip-18932-jewel
+    
+    jewel: tests: 'ceph auth import -i' overwrites caps, should alert user before overwrite
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit acf608a9034e915e38ccea6002ee808c46620433
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Mar 20 17:28:45 2017 -0700
+
+    filestore, tools: Fix logging of DBObjectMap check() repairs
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 1704f62c0831e6b07138f7dd14a89fef3c9ed2c1)
+
+commit fecc52338b2a58bf1730f7b7a3e4a293e45160d2
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Mar 3 15:04:02 2017 -0800
+
+    osd: Simplify DBObjectMap by no longer creating complete tables
+    
+    Bump the version for new maps to 3
+    Make clone less efficient but simpler
+    Add rename operation (use instead of clone/unlink)
+    For now keep code that understands version 2 maps
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 738156a99ed1caf61e5a8230eb8048360056c08e)
+    
+    No ghobject_t::operator>() so use Kraken cmp_bitwise() instead
+    Need to use MIN_GHOBJ/MAX_GHOBJ instead of std::min/std::max
+
+commit 6902c3141eeaefaacd92f33877cf319872f626c7
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 15 16:17:32 2017 -0800
+
+    ceph-osdomap-tool: Fix seg fault with large amount of check error output
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 1dda0411f4fbb14ce1e0062da9f14ec3af505d39)
+
+commit 4a3e4bcf40cd004fc53e7be467a29084dedc3e1c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 15 15:02:33 2017 -0800
+
+    osd: Add automatic repair for DBObjectMap bug
+    
+    Add repair command to ceph-osdomap-tool too
+    
+    Under some situations the previous rm_keys() code would
+    generated a corrupt complete table.  There is no way to
+    figure out what the table should look like now.  By removing
+    the entries we fix the corruption and aren't much worse off
+    because the corruption caused some deleted keys to re-appear.
+    
+    This doesn't breaking the parent/child relationship during
+    repair because some of the keys may still be contained
+    in the parent.
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 4cd3c74c928a32e065ed9543d6c91d8718a6ae3d)
+    
+    Conflicts:
+            src/os/filestore/DBObjectMap.h (trivial)
+
+commit d4f0ac0a405266f638b25ec475c0110741e3c431
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 15 14:59:40 2017 -0800
+
+    ceph-osdomap-tool: Fix tool exit status
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 666f14ed90655a2d1bedde8561949625db7a9e6c)
+
+commit 5f36c319cd7fa4be15efd27b8aabbebc99d8999c
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Feb 10 15:51:42 2017 -0800
+
+    DBObjectMap: rewrite rm_keys and merge_new_complete
+    
+    Leverage the updated in_complete_region and needs_parent to simplify
+    these methods.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit c4dffb68eaafe724f7fdae93a4285a7f8003ea29)
+
+commit 1fe4b856a37b29c85a9317e514c7f15f8e4905d5
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Feb 10 15:50:57 2017 -0800
+
+    DBObjectMap: strengthen in_complete_region post condition
+    
+    Previously, in_complete_region didn't guarantee anything about
+    where it left complete_iter pointing.  It will be handy for
+    complete_iter to be pointing at the lowest interval which ends
+    after to_test.  Make it so.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 97b35f4d7d4862da4b6f50ecaef0d292a671fd04)
+
+commit 85f2151fec991e5db13d8e6f44b27e092605fb35
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Feb 10 15:48:57 2017 -0800
+
+    DBObjectMap: fix next_parent()
+    
+    The previous implementation assumed that
+    lower_bound(parent_iter->key()) always leaves the iterator
+    on_parent().  There isn't any guarantee, however, that that
+    key isn't present on the child as well.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 74a7631d0938d7b44894f022224eab10a90d5cec)
+
+commit 484ccda5c53a8ee151b9f97687edd160ad7ebbd7
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Feb 9 10:47:59 2017 -0800
+
+    test_object_map: add tests to trigger some bugs related to 18533
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit f131dbcf5bb17107c029f942a57e9bf4432a26ee)
+
+commit cdeb690869bf6dd52226476c21514cf03ff37d6e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Feb 14 12:40:33 2017 -0800
+
+    test: Add ceph_test_object_map to make check tests
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0e97a01bd7291458881ee53cece2d887f6333669)
+
+commit cf5d588d39498c8d65bc64bd2935aecac8546e40
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 8 18:56:27 2017 -0800
+
+    ceph-osdomap-tool: Add --debug and only show internal logging if enabled
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 5fb2b2d13953979e5da9f571ab8c4b0b510b8368)
+
+commit 4c4a06ff525d9fa2271099db73701c7994054d36
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 8 18:55:48 2017 -0800
+
+    osd: DBOjectMap::check: Dump complete mapping when inconsistency found
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit fcf1e17c645e8fad5216c3e59627c817e5c858c7)
+
+commit 6c128ff8c6944e57059008959f49bd03635c5417
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 8 15:38:51 2017 -0800
+
+    test_object_map: Use ASSERT_EQ() for check() so failure doesn't stop testing
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 053a273cbc02d6902a4bb1f11db1ea946498df3a)
+
+commit aa769a9d08ac7490f5063bb44c3a44c77a5d7232
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 8 10:02:40 2017 -0800
+
+    tools: Check for overlaps in internal "complete" table for DBObjectMap
+    
+    Changed check to return an error count and fix tool error message
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit e5e8eb962db6187ea19b96ba29ac83469c90b4ea)
+    
+    Conflicts:
+            src/os/filestore/DBObjectMap.h (trivial)
+
+commit 761ee7c6af8802ab6b668a0b7ccaa819b2764456
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 8 09:40:49 2017 -0800
+
+    tools: Add dump-headers command to ceph-osdomap-tool
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit f4101591ad701a62fe027c4744ca8ea505f44bdc)
+    
+    Conflicts:
+            src/os/filestore/DBObjectMap.h (trivial)
+
+commit 117db1c6d6952d181614f4fe22f0b1866eed6f10
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Feb 6 21:09:42 2017 -0800
+
+    tools: Add --oid option to ceph-osdomap-tool
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 2d94889e9ee3359017b1efd560f3557ce03ccee6)
+
+commit 4d8120d3b3975ce101f6272240e5e43bcfe0e742
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Feb 6 21:31:18 2017 -0800
+
+    osd: Remove unnecessary assert and assignment in DBObjectMap
+    
+    Fix and add comment(s)
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 937e6a03ea4692cc44d53faa0615f8e808c9eb03)
+
+commit 86980a045b7176428062a19758d145e38e515b40
+Author: snakeAngel2015 <li.jieA@h3c.com>
+Date:   Mon Jul 18 14:51:37 2016 +0800
+
+    rgw: add suport for creating S3 type subuser of admin rest api
+    
+    Fixes: http://tracker.ceph.com/issues/16682
+    
+    The original code cannot support create s3 type subuser of admin rest api as when i execute the following command:
+    
+      ./s3curl.pl --id=personal --put -- http://radosgw.h3c.com:8000/admin/user?subuser\&uid=yrf2\&subuser=yrf2:yrf1\&key-type=s3 -v
+    
+    it would return msg as follows :
+    
+             < HTTP/1.1 403 Forbidden
+            < Date: Thu, 14 Jul 2016 07:04:40 GMT
+            * Server Apache/2.4.7 (Ubuntu) is not blacklisted
+            < Server: Apache/2.4.7 (Ubuntu)
+            < x-amz-request-id: tx00000000000000006608f-0057873988-8551-slave
+            < Accept-Ranges: bytes
+            < Content-Length: 114
+            < Content-Type: application/json
+            <
+            * Connection #0 to host slave.com left intact
+            {"Code":"InvalidAccessKeyId","RequestId":"tx00000000000000006608f-0057873988-8551-slave","HostId":"8551-slave-us"}
+    
+     But i have modified the codes for support it ,and it will return actual msg as follows :
+    
+      "subusers": [
+            {
+                "id": "yrf2:yrf1",
+                "permissions": "<none>"
+            }
+        ],
+        "keys": [
+            {
+                "user": "yrf2",
+                "access_key": "B46PXYFEWUX0IMHGKP8C",
+                "secret_key": "2JYxywXizqwiiMd74UXrJdSJMPNlBtYwF5z8rNvh"
+            },
+            {
+                "user": "yrf2:yrf1",
+                "access_key": "INO55WXJ7JQ1ZZGSAB6B",
+                "secret_key": "GgCKEfF9hArV2hglunbO7KtvKZnbhmsDpqjSj5DL"
+            }
+        ],
+    
+    Please check it ,thanks .
+    
+    Signed-off-by: snakeAngel2015 <li.jieA@h3c.com>
+    (cherry picked from commit 6535f6ad2137ee55bf5531e865c05aa10bd39bd0)
+
+commit d7b63e21ca9818e21afeb2945b882e24b7a9b10b
+Merge: a3fae531cd f32b5c613a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 26 21:32:31 2017 +0200
+
+    Merge pull request #14809 from tchaikov/wip-18193-jewel
+    
+    jewel: tests: test/librados/tmap_migrate: g_ceph_context->put() upon return
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit a3fae531cdb8c2c64c66d38c947ca139f4ede83d
+Merge: 89d6ddb1f5 6c4826606d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 26 21:32:09 2017 +0200
+
+    Merge pull request #14701 from smithfarm/wip-18193-jewel
+    
+    jewel: core: transient jerasure unit test failures
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 89d6ddb1f5ee08b0f019dfaadc5857a788c92a09
+Merge: c5f1fce21d be9e83281b
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Apr 26 14:28:29 2017 -0400
+
+    Merge pull request #14776 from linuxbox2/jewel-pullup-civet-chunked
+    
+    [DNM] jewel: pullup civet chunked
+
+commit c5f1fce21d0996cce751b6bcca5e57da2cafc135
+Merge: 4d97e0ba8e 97cd21afc5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 26 19:32:07 2017 +0200
+
+    Merge pull request #14416 from smithfarm/wip-19557-jewel
+    
+    jewel: tests: upgrade/hammer-x failing with OSD has the store locked when Thrasher runs ceph-objectstore-tool on down PG
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit be9e83281b8c765cd111d5687a516fcd3ca521a5
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Apr 25 09:44:39 2017 -0400
+
+    civetweb:  pullup chunked encoding by Marcus
+    
+    Fixes: http://tracker.ceph.com/issues/19736
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 608785a0079f807ff860c56d96b1b67bf6a2ed74
+Author: yaoning <yaoning@unitedstack.com>
+Date:   Fri Jun 24 09:51:07 2016 +0800
+
+    os: make zero values noops for set_alloc_hint() in FileStore
+    
+    Signed-off-by: yaoning <yaoning@unitedstack.com>
+    (cherry picked from commit e2ec24f61b55457caccefecd56f9f08b98264802)
+
+commit f32b5c613a8acd50e32747c3581131d28d209efa
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Apr 26 22:58:30 2017 +0800
+
+    test/librados/tmap_migrate: g_ceph_context->put() upon return
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    Conflict: test/librados/tmap_migrate.cc
+      this change is not cherry-picked from master, because tmap_migrate was
+    removed in master. so we are applying the same change in
+    cb1cda96713b2ec0f6418c4cbe3d964c2020729c to this test.
+
+commit 905c4acb99f9ea78ff615034dae969ab089bda06
+Author: Mark Nelson <mnelson@redhat.com>
+Date:   Tue May 3 09:56:47 2016 -0500
+
+    Fix reveresed promote throttle default parameters.
+    
+    Signed-off-by: Mark Nelson <mnelson@redhat.com>
+    (cherry picked from commit 793ceac2f3d5a2c404ac50569c44a21de6001b62)
+
+commit fb3ee2efcc13c37db90faa1bc8bad584bab22efa
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Thu Apr 13 05:33:55 2017 -0400
+
+    rgw: swift: disable revocation thread if sleep == 0 || cache_size == 0
+    
+    Keystone tokens can be revoked.  This causes them to fail
+    validation.  However, in ceph, we cache them.  As long as
+    they're in the cache we trust them.  To find revoked tokens
+    there's a call OSI-PKI/revoked but that's only useful for
+    pki tokens.  Installations using fernet/uuid may not even
+    have the proper credentials to support the call, in which
+    case the call blows up in various ways filling up logs
+    with complaints.
+    
+    This code makes the revocation thread optional; by disabling it,
+    the complaints go away.  A further fix is in the works
+    to use other more modern calls available in modern keystone
+    installations to properly deal with non-PKI/PKIZ tokens.
+    
+    (NB: jewel has this logic in src/rgw/rgw_swift.cc not in src/rgw/rgw_keystone.h)
+    
+    To disable the revocation thread, use at least one of these:
+            rgw_keystone_token_cache_size = 0
+                    using this will cause tokens to be validated on every call.
+    You may instead want to set
+            rgw_keystone_revocation_interval = 0
+                    using just this will disable the revocation thread,
+                    but leaves the cache in use.  That avoids the extra
+                    validation overhead, but means token revocation won't
+                    work very well.
+    
+    Fixes: http://tracker.ceph.com/issues/9493
+    Fixes: http://tracker.ceph.com/issues/19499
+    
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    (cherry picked from commit 003291a8cbca455c0e8731f66759395a0bb1f555)
+
+commit 4d97e0ba8ebc89b7797b0936a9e046ef59cc3899
+Merge: cc820a0d4c c2efeb4b62
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Apr 25 21:06:26 2017 +0200
+
+    Merge pull request #14686 from smithfarm/wip-19686-jewel
+    
+    jewel: osd: Give requested scrubs a higher priority
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit e552d91f73d996c44821f5fbfb28cfc3e5cddc9a
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sat Nov 5 13:13:47 2016 -0400
+
+    rgw: add bucket size limit check to radosgw-admin
+    
+    The change adds a new list of all buckets x all users, with
+    fields for bucket name, tenant name, current num_objects,
+    current num_shards, current objects per shard, and the
+    corresponding fill_status--the latter consisting of 'OK',
+    'WARN <n>%', or 'OVER <n>%.'
+    
+    The warning check is relative to two new tunables.  The threshold
+    max objects per shard is set as rgw_bucket_safe_max_objects_per_shard,
+    which defaults to 100K.  The value rgw_bucket_warning_threshold is
+    a percent of the current safe max at which to warn (defaults to
+    90% of full).
+    
+    From review:
+    
+    * fix indentation (rgw_admin)
+    * if user a user_id is provided, check only buckets for that user
+    * update shard warn pct to be pct-of-fill (not 100 - pct-of-fill)
+    * print only buckets near or over per-shard limit, if --warnings-only
+    * s/bucket limitcheck/bucket limit check */
+    * sanity shard limit should be 90, not 10 (because that changed)
+    * fixes for memleaks and other points found by cbodley
+    
+    Fixes: http://tracker.ceph.com/issues/17925
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 7bc144ce36fedc16a3dedc54598b0d75fb8c68bc)
+
+commit cc820a0d4c2676799383aeaf49a9269e104853ce
+Merge: 5ee54cfe8f d079b91479
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Apr 25 17:38:21 2017 +0200
+
+    Merge pull request #14605 from asheplyakov/19476-jewel
+    
+    jewel: rgw: don't return skew time in pre-signed url
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 5ee54cfe8f0a955b11edb4819a652ee420f43e5d
+Merge: 37254aee6c c05bd1cb1f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Apr 25 17:37:16 2017 +0200
+
+    Merge pull request #14660 from smithfarm/wip-19478-jewel
+    
+    jewel: rgw: zonegroupmap set does not work
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 37254aee6c3f725cf60deb5c7a3fc28405c51216
+Merge: 09919f938f faeb8088ac
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Apr 25 17:36:23 2017 +0200
+
+    Merge pull request #14607 from asheplyakov/19607-jewel
+    
+    jewel: rgw: multisite: fetch_remote_obj() gets wrong version when copying from remote
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 09919f938f3c1e9e81963f676a9dd13114d54e73
+Merge: d5e1345a2a 527911fab7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Apr 25 15:50:26 2017 +0200
+
+    Merge pull request #14587 from asheplyakov/19617-jewel
+    
+    jewel: mon/MonClient: make get_mon_log_message() atomic
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 6c4826606dc81fdd847959a49454c69a958bb1d8
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Dec 5 20:23:21 2016 +0800
+
+    test/ceph_crypto: do not read ceph.conf in global_init()
+    
+    ForkDeathTest.MD5 expect an empty output while global_init() complains
+    if ceph.conf is missing if 0 is passed in as the `flags`. this test
+    passes if ceph.conf is in current working directory, but jenkins does
+    not prepare this file for this test.
+    
+    Fixes: http://tracker.ceph.com/issues/18128
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit c72a2271a8012a66d7bbccf5766a73da5bb878d6)
+    
+    Conflicts:
+            src/test/ceph_crypto.cc - jewel does not have
+                 5af29540675b674c1985ff98b28a783ed124acf6
+
+commit aca2659952528aa1b40b02828293342128657194
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Dec 8 12:40:42 2016 +0100
+
+    tests: fix erasure-code premature deallocation of cct
+    
+    The setup function returns before the run function, the cct variable
+    must be a data member, not a local variable that gets de-allocated
+    before run() starts.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit efa1e54362423d4cfd1541fb8c68237b7b9ebbe3)
+
+commit 8bddd427d9a4bca75a352ca333847dbd6d2c369c
+Author: Pan Liu <pan.liu@istuary.com>
+Date:   Thu Feb 16 22:17:52 2017 +0800
+
+    rbd-nbd: no need create asok file for unmap and list-mapped commands.
+    
+    Fixes: http://tracker.ceph.com/issues/17951
+    Signed-off-by: Pan Liu <liupan1111@gmail.com>
+    (cherry picked from commit 72352653d585ef89043a4ece371b5c0cb3f6f32a)
+
+commit 328bfbd25c63b1b0e253865abedada7a9e5858e4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Nov 29 12:36:00 2016 -0500
+
+    rbd-nbd: restart parent process logger after forking
+    
+    Fixes: http://tracker.ceph.com/issues/18070
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 29baf254d72cc593572b5a6215360ba51e3be198)
+
+commit 192e7bcdcd1bc02b40b29eff86a335cc8919f663
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Nov 23 19:45:57 2016 +0800
+
+    crushtool: do not release g_ceph_context at exit
+    
+    it is but a work around of occasionally timeout.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d305cc51b18cbf4b2757bbacb5d43324461306a9)
+
+commit 8a2f27cc632c26d7c2b8e8528b4d459b1d78705b
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Nov 15 14:21:03 2016 +0800
+
+    common,test: g_ceph_context->put() upon return
+    
+    prior to this change, global_init() could create a new CephContext
+    and assign it to g_ceph_context. it's our responsibilty to release
+    the CephContext explicitly using cct->put() before the application
+    quits. but sometimes, we fail to do so.
+    
+    in this change, global_init() will return an intrusive_ptr<CephContext>,
+    which calls `g_ceph_context->put()` in its dtor. this ensures that
+    the CephContext is always destroyed before main() returns. so the
+    log is flushed before _log_exp_length is destroyed.
+    
+    there are two cases where global_pre_init() is called directly.
+    - ceph_conf.cc: g_ceph_context->put() will be called by an intrusive_ptr<>
+      deleter.
+    - rgw_main.cc: global_init() is called later on on the success code
+      path, so it will be taken care of.
+    
+    Fixes: http://tracker.ceph.com/issues/17762
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit cb1cda96713b2ec0f6418c4cbe3d964c2020729c)
+    
+    Conflicts:
+            src/ceph_fuse.cc
+            src/ceph_mgr.cc
+            src/global/global_init.cc
+            src/rgw/rgw_main.cc
+            src/test/compressor/test_compression.cc
+            src/test/compressor/test_compression_plugin.cc
+            src/test/compressor/test_compression_plugin_snappy.cc
+            src/test/compressor/test_compression_plugin_zlib.cc
+            src/test/compressor/test_compression_snappy.cc
+            src/test/compressor/test_compression_zlib.cc
+            src/test/erasure-code/TestErasureCode.cc
+            src/test/erasure-code/TestErasureCodeExample.cc
+            src/test/erasure-code/TestErasureCodeIsa.cc
+            src/test/erasure-code/TestErasureCodeJerasure.cc
+            src/test/erasure-code/TestErasureCodeLrc.cc
+            src/test/erasure-code/TestErasureCodePlugin.cc
+            src/test/erasure-code/TestErasureCodePluginIsa.cc
+            src/test/erasure-code/TestErasureCodePluginJerasure.cc
+            src/test/erasure-code/TestErasureCodePluginLrc.cc
+            src/test/erasure-code/TestErasureCodePluginShec.cc
+            src/test/erasure-code/TestErasureCodeShec.cc
+            src/test/erasure-code/TestErasureCodeShec_thread.cc
+            src/test/fio/fio_ceph_objectstore.cc
+            src/test/librados/misc.cc
+            src/test/mon/PGMap.cc
+            src/test/msgr/test_async_networkstack.cc
+            src/test/msgr/test_userspace_event.cc
+            src/test/objectstore/Allocator_test.cc
+            src/test/objectstore/BitAllocator_test.cc
+            src/test/objectstore/test_bluefs.cc
+            src/test/objectstore/test_bluestore_types.cc
+            src/test/objectstore/test_memstore_clone.cc
+            src/test/osd/TestPGLog.cc
+            src/test/rgw/test_http_manager.cc
+            src/test/rgw/test_rgw_compression.cc
+            src/test/test_mempool.cc
+            src/tools/rados/rados.cc
+
+commit d5e1345a2a6bd8456417db6bb60d61871165e6bd
+Merge: 256f48f463 5096fc9c7c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Apr 25 15:42:43 2017 +0200
+
+    Merge pull request #14665 from smithfarm/wip-19610-jewel
+    
+    jewel: [librados_test_stub] cls_cxx_map_get_XYZ methods don't return correct value
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 256f48f46354f11a2f238f24fa9d890b55f4f4fc
+Merge: 4f67da1b9d 21a83e1276
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Apr 25 09:20:19 2017 +0200
+
+    Merge pull request #14653 from smithfarm/wip-19662-jewel
+    
+    jewel: rgw_file: fix event expire check, don't expire directories being read
+    
+    Reviewed-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 4f67da1b9d554a1a5371665573b1f56d3f903de3
+Merge: c2452c5364 33af18e592
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Apr 25 09:17:33 2017 +0200
+
+    Merge pull request #14635 from smithfarm/wip-19690-jewel
+    
+    jewel: doc: Improvements to crushtool manpage
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 040ff013d5c8aded8beedc59bbc4f5afccc64e46
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Nov 16 11:56:09 2016 +0800
+
+    crushtool: s/exit(EXIT_FAILURE)/return EXIT_FAILURE/
+    
+    so the destructor(s) can be called.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit e01b89ed6be6b99fec5c725f4bc5769b42468cac)
+    
+    Conflicts:
+            src/tools/crushtool.cc - jewel does not have 17feefbcb3105553b763cb7ce123b20b77b95857
+
+commit 8e993e6e43ecd6e2b444961d1e6c945081b64207
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Nov 16 11:19:04 2016 +0800
+
+    global/signal_handler: reset injected segv after test
+    
+    ~CephContext() => ~TypedSingletonWrapper() => ~MempoolObs =>
+    unregister_command() => ldout() << "unregister_command" =>
+    Log::submit_entry() => *(volatile int *)(0) = 0xdead;
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d932c8f2f23263924103a900714db82ee87f6eef)
+    
+    Conflicts:
+            src/log/Log.cc - jewel has "namespace log" instead of "namespace
+                             logging" (trivial resolution)
+
+commit f1c0042b831826e5bd72ada79a4918a9c26bda24
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Nov 15 14:42:35 2016 +0800
+
+    test_cors.cc: fix the mem leak
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit c8a3777203482cabf6739a8ba69b127df8697628)
+
+commit c49b114e8dde6c3f3c5a5b663bd8299b4f2feafb
+Author: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+Date:   Tue Sep 6 16:34:52 2016 +0800
+
+    rgw: fix failed to create bucket if a non-master zonegroup has a single zone
+    
+    If a non-master zonegroup has a single zone, the metadata sync thread not running and
+    the non-master zonegroup can't sync user from master zonegroup,
+    so we can't create bucket(or other metadata update) in it
+    because the authenticated user not found in the zone of non-master zonegroup.
+    
+    Signed-off-by: weiqiaomiao <wei.qiaomiao@zte.com.cn>
+    (cherry picked from commit 949af79b21098e6410bc29274cf36eae2d89faea)
+    
+    Conflicts:
+            src/rgw/rgw_rados.cc - retain d32654b7cd60ccc4e23d3f05b9e4385a697bacd6
+                which was merged after this commit
+
+commit 43327f83efcbc5ce54be866a30da8dd1e58d6707
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Feb 22 01:49:12 2017 +0100
+
+    ceph-disk: dmcrypt activate must use the same cluster as prepare
+    
+    When dmcrypt is used, the fsid cannot be retrieved from the data
+    partition because it is encrypted. Store the fsid in the lockbox to
+    enable dmcrypt activation using the same logic as regular activation.
+    
+    The fsid is used to retrive the cluster name that was used during
+    prepare, reason why activation does not and must not have a --cluster
+    argument.
+    
+    Fixes: http://tracker.ceph.com/issues/17821
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 7f66672b675abbc0262769d32a38112c781fefac)
+    
+    Conflicts:
+            src/ceph-disk/ceph_disk/main.py - in master, self.create_key() takes an
+                argument (self.args.cluster) but in jewel it takes no argument
+
+commit c2452c53641fca416268c2e31c774b8b7e609c88
+Merge: 82b8c89e47 01d04e28db
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Apr 25 07:20:41 2017 +0200
+
+    Merge pull request #13608 from smithfarm/wip-19063-jewel
+    
+    jewel: tests: eliminate race condition in Thrasher constructor
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 3ec1a9bf16e2c305096e11223aaa8db94dc4084d
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Mar 31 10:06:42 2017 -0400
+
+    ceph_test_librados_api_misc: fix stupid LibRadosMiscConnectFailure.ConnectFailure test
+    
+    Sometimes the cond doesn't time out and it wakes up instead.  Just repeat
+    the test many times to ensure that at least once it times out (usually
+    it doesn't; it's pretty infrequent that it doesn't).
+    
+    Fixes: http://tracker.ceph.com/issues/15368
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8bc197400d94ee2716d3f2fa454247379a676cf9)
+
+commit 82b8c89e477610641a6e21123b7c929e9c560729
+Merge: 28c7ce595a 30c9527353
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Apr 24 22:27:33 2017 +0200
+
+    Merge pull request #14195 from cbodley/wip-19353
+    
+    jewel: rgw: use separate http_manager for read_sync_status
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 28c7ce595ab8fef7c0076f091f249f6b3548ad54
+Merge: c05ecff3f0 2e50fe1684
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Apr 24 22:26:50 2017 +0200
+
+    Merge pull request #14066 from asheplyakov/19321-bp-jewel
+    
+    jewel: rgw: fix break inside of yield in RGWFetchAllMetaCR
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit c05ecff3f0cb5af2f3aa52b68188742796335ddb
+Merge: 3240cbf4ef dc4e7a1a86
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Apr 24 22:26:14 2017 +0200
+
+    Merge pull request #14064 from asheplyakov/19211-bp-jewel
+    
+    jewel: rgw: "cluster [WRN] bad locator @X on object @X...." in cluster log
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 3240cbf4ef60733f55800039051e85f2a7ace61d
+Merge: d4672acbb8 85fbb00f6e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Apr 24 22:25:21 2017 +0200
+
+    Merge pull request #13842 from smithfarm/wip-19145-jewel
+    
+    jewel: rgw: a few cases where rgw_obj is incorrectly initialized
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit d4672acbb82daf81eaf259e659dd627ec9f3bc79
+Merge: 61ed719d75 ec0668c201
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Apr 24 22:24:29 2017 +0200
+
+    Merge pull request #13837 from smithfarm/wip-19048-jewel
+    
+    jewel: rgw: multisite: some yields in RGWMetaSyncShardCR::full_sync() resume in incremental_sync()
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 61ed719d75bf3606fa9e3ae8c8ed776aa2ee4313
+Merge: f5e51db564 ced799f9c6
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Apr 24 22:23:24 2017 +0200
+
+    Merge pull request #13724 from asheplyakov/18626-bp-jewel
+    
+    jewel: rgw: Use decoded URI when verifying TempURL
+    
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 66c3db7aee5b53f83e87ee8c8f081ab9b0336177
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Mar 8 16:31:34 2017 -0500
+
+    rgw: data sync skips slo data when syncing the manifest object
+    
+    Fixes: http://tracker.ceph.com/issues/19027
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 8b69847d7b3e92c70090d1dddf7cea5c44fb6b20)
+    
+    Conflicts: bucket cleanup, overrides
+
+commit 303a62f7f15c69413165ed604869909587714a94
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Mar 13 11:33:02 2017 -0400
+
+    rgw: RGWGetObj applies skip_manifest flag to SLO
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 987377ae34382e107e1d54f0bfc1121fcedb4513)
+
+commit f3d99ae8aafaa65e91cd233b5ce8054678d8fa11
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Feb 21 10:27:13 2017 -0500
+
+    rgw: allow system users to read SLO parts
+    
+    multisite data sync relies on fetching the object as the system user
+    
+    Fixes: http://tracker.ceph.com/issues/19027
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit d50d18c500fd5dd89e7cada1162cf453b36df370)
+    
+    Conflicts: auth rework
+
+commit f5e51db5644d4eafdbe3b2d541582a17febed139
+Merge: c90cfb7a32 eac0e27193
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Apr 21 22:51:59 2017 +0200
+
+    Merge pull request #14643 from smithfarm/wip-revert-14427
+    
+    Wip revert 14427
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit db053da61807e26876d97786550a769295c7955a
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Mar 27 12:56:31 2017 +0100
+
+    mds: validate prealloc_inos on sessions after load
+    
+    Mitigates http://tracker.ceph.com/issues/16842
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit c39aaf90ed1b23343eba2b341bb8ee6a50a4ea74)
+    
+    Conflicts:
+            src/mds/InoTable.cc - no 5259683e7819c22c14b21b1dd678a33e14574f21 in jewel
+            src/mds/InoTable.h - no 5259683e7819c22c14b21b1dd678a33e14574f21 in jewel
+
+commit 2b5eb8fa141fa8bd9173dee206ec3530d702fc3a
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Mar 27 12:33:59 2017 +0100
+
+    mds: operator<< for Session
+    
+    Use this to get a nice human readable name
+    when available (also including the session id in
+    parentheses)
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 0f89787d8312f132ebb621f16c44e950b17a395a)
+
+commit 5b562145197833c6a6f48f006dfb13b6e38f9084
+Author: Henrik Korkuc <henrik@uber.com>
+Date:   Sun Feb 19 11:44:20 2017 +0200
+
+    client/Client.cc: add feature to reconnect client after MDS reset
+    
+    Client.cc marks session as stale instead of reconecting after received
+    reset from MDS. On MDS side session is closed so MDS is ignoring cap
+    renew. This adds option to reconnect stale client sessions instead of
+    just marking sessions stale.
+    
+    Fixes: http://tracker.ceph.com/issues/18757
+    
+    Signed-off-by: Henrik Korkuc <henrik@kirneh.eu>
+    (cherry picked from commit e0bbc704676ef4aed510daff075ef63c9e73b7b3)
+
+commit 8f21038d30097622e319ab986631b03f87f5d907
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Mar 21 12:49:45 2017 +0800
+
+    doc: cephfs: fix the unexpected indent warning
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit e423f0b59711422b40c4b3de0bdc73b0947c04d3)
+
+commit f9a1954e113e807b5f9ee6fd56351adc2841730a
+Author: Barbora An�incová <bancinco@redhat.com>
+Date:   Thu Feb 16 10:45:36 2017 +0100
+
+    doc: additional edits in FUSE client config
+    
+    Signed-off-by: Bara Ancincova (bara@redhat.com)
+    (cherry picked from commit b6cad3364c020abd3acf906643fa0b6cbb862a0a)
+
+commit 018649f8a4628881c706a6df9b3eef48403fe91e
+Author: Barbora An�incová <bancinco@redhat.com>
+Date:   Thu Jan 26 12:23:34 2017 +0100
+
+    doc: Dirty data are not the same as corrupted data
+    
+    Signed-off-by: Bara Ancincova (bara@redhat.com)
+    (cherry picked from commit 80db40f8559128baadad42b925ae813e51a31409)
+
+commit 1d8a5b6d64a26a36fb3a227c52908631d95bef79
+Author: Barbora An�incová <bancinco@redhat.com>
+Date:   Mon Jan 23 16:34:55 2017 +0100
+
+    doc: minor changes in fuse client config reference
+    
+    Signed-off-by: Bara Ancincova (bara@redhat.com)
+    (cherry picked from commit e57605681f10436f4b2c85e95179a2904b8c80da)
+
+commit 1ae46b2b94280b8b4a1db89b2cbed8f12bcbcc53
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sun Jul 24 23:21:29 2016 -0400
+
+    doc: add client config ref
+    
+    Fixes: http://tracker.ceph.com/issues/16743
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 9ad2ccf29830d5309336fc7de877b6926e5dbacd)
+
+commit b8fd297eb64cd29c58596d0a5b0be8fbe9d94a72
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Tue Apr 11 13:51:47 2017 -0700
+
+    use sudo to check check health
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 7af157ad4ce7f7e2b8de97ee10eeaf64b9099bc0)
+
+commit 1b91ffc0fbe76c5475f17d4e15ea295ee3680688
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Wed Mar 29 09:27:20 2017 -0700
+
+    Add reboot case for systemd test
+    
+    test systemd units restart after reboot
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 7b587304a54d9b21041ffdfbc85fad8d87859c49)
+
+commit 3d8d1dad8a698a9fb45be7b31c92f8b71ee67720
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Wed Mar 29 09:56:11 2017 -0700
+
+    Fix distro's, point to latest version
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 1947648669971c1bd1ca189870ed9b25bbd48d3a)
+
+commit c90cfb7a327fcbb9508e617c2353becb7e5cb45f
+Merge: 327276cf3d 9b77b16b88
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 22:05:57 2017 +0200
+
+    Merge pull request #14602 from asheplyakov/19646-jewel
+    
+    jewel: ceph-disk: enable directory backed OSD at boot time
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 327276cf3d4fb7a2558620a8a7d9cc90e5d6e5c5
+Merge: 7008c64c51 25e43ac256
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 22:04:41 2017 +0200
+
+    Merge pull request #14449 from smithfarm/wip-test-doc-oversight
+    
+    tests: fix oversight in yaml comment
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f34489dd52f07aeb88e1dbf361ead63f09bb9c65
+Author: YunfeiGuan <yunfei.guan@xtaotech.com>
+Date:   Mon Apr 10 05:48:47 2017 +0000
+
+    cephfs: fix mount point break off problem after mds switch occured
+    
+    The hot-standby become active as we expected but the mount piont broken strangely
+    when the active mds is down. The root reason is the new mds use last_cap_renews
+    decoded from ESesson::replay in find_idle_sessions and wrongly killed the session.
+    Maybe we should reset session->last_cap_renew to the current time when server send
+    OPEN to client in reconnect stage.
+    
+    Fixes: http://tracker.ceph.com/issues/19437
+    Signed-off-by: Guan yunfei <yunfei.guan@xtaotech.com>
+    (cherry picked from commit 4ef830c5d6f22bf0d4f82a8624c772ecbbda44a6)
+    
+    Conflicts:
+            src/mds/Server.cc (leave '\n' in because jewel does not have
+            693132eb00b1803d5e97a79908521d5a6903e9f8; jewel ceph_clock_now takes a
+            CephContext object)
+
+commit 7008c64c5143a3ea19d6679a3e521897f74ff69a
+Merge: d62644abcd f8aa6be06c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 19:26:52 2017 +0200
+
+    Merge pull request #13606 from smithfarm/wip-19062-jewel
+    
+    jewel: build/ops: enable build of ceph-resource-agents package on rpm-based os
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit d62644abcde0fd99da24922ce7638d173952ceb4
+Merge: 8df8960553 482bd1adab
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 18:18:05 2017 +0200
+
+    Merge pull request #14654 from smithfarm/wip-19461-jewel
+    
+    jewel: admin ops: fix the quota section
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit c2efeb4b62e6dc4e6975561eb8b20cfca66f7237
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Apr 17 14:58:02 2017 -0700
+
+    osd: Give requested scrub work a higher priority
+    
+    Once started we now queue scrub work at higher priority than
+    scheduled scrubs.
+    
+    Fixes: http://tracker.ceph.com/issues/15789
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit ebab8b1f4f67fbdec1e147c580329c1e2b5cf7cd)
+    
+    Conflicts:
+            src/osd/OSD.h - in jewel, the PGScrub() call is enclosed within
+                op_wq.queue(make_pair(...)) instead of enqueue_back()
+
+commit db86a24e7906ca7c70c60b1752f1230d56361bcd
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Feb 22 17:33:05 2017 +0800
+
+    client: wait for lastest osdmap when handling set file/dir layout
+    
+    Fixes: http://tracker.ceph.com/issues/18914
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 76f5eb86cdd61dde4e6c7cfeb5cf34f0c0334f21)
+    
+    Conflicts:
+            src/client/Client.cc (jewel does not have 201c56039)
+            src/client/Client.h (jewel does not have 201c56039)
+
+commit 7b9283beec0c4bf8e2067558fc4e03e336849b77
+Author: Yang Honggang <joseph.yang@xtaotech.com>
+Date:   Thu Apr 13 20:09:07 2017 +0800
+
+    cephfs: fix write_buf's _len overflow problem
+    
+    After I have set about 400 64KB xattr kv pair to a file,
+    mds is crashed. Every time I try to start mds, it will crash again.
+    The root reason is write_buf._len overflowed when doing
+    Journaler::append_entry().
+    
+    This patch try to fix this problem through the following changes:
+    
+     1. limit file/dir's xattr size
+     2. throttle journal entry append operations
+    
+    Fixes: http://tracker.ceph.com/issues/19033
+    Signed-off-by: Yang Honggang joseph.yang@xtaotech.com
+    (cherry picked from commit eb915d0eeccbe523f8f70f6571880003ff459459)
+
+commit b52c5088618011a569f157616c5c667c2fc1e9fe
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 8 12:13:46 2017 +0000
+
+    mds: shut down finisher before objecter
+    
+    Some of the finisher contexts would try to call into Objecter.
+    We mostly are protected from this by mds_lock+the stopping
+    flag, but at the Filer level there's no mds_lock, so in the
+    case of file size probing we have a problem.
+    
+    Fixes: http://tracker.ceph.com/issues/19204
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 177a97d5c55ee6a2d5dcd3cf0893546190b10f7a)
+    
+    Conflicts:
+            src/mds/MDSRank.cc (no purge_queue.shutdown() in jewel because jewel
+            does not have 8ebf7d95a9071de24bb1e56a6423c505169cb4de)
+
+commit 96e801fb53941214c669ac01c1804a4d1f27086c
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Mar 28 14:13:33 2017 -0400
+
+    mds: ignore ENOENT on writing backtrace
+    
+    We get ENOENT when a pool doesn't exist.  This can
+    happen because we don't prevent people deleting
+    former cephfs data pools whose files may not have
+    had their metadata flushed yet.
+    
+    http://tracker.ceph.com/issues/19401
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 3fccc2372f2715d075b05e459140360cf6e7ca96)
+
+commit 8df896055366d777ca309dd4a7840f2a2a5a72cc
+Merge: da888fa871 7468689314
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 14:53:30 2017 +0200
+
+    Merge pull request #14680 from smithfarm/wip-19711-jewel
+    
+    jewel: [test] test_notify.py: rbd.InvalidArgument: error updating features for image test_notify_clone2
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 74686893147f2d7e1fd1287de37587554fea96c6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 19 09:26:31 2017 -0400
+
+    test: rbd master/slave notify test should test active features
+    
+    Fixes: http://tracker.ceph.com/issues/19692
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0dcba41cba96566d0b8da54cf0316d523b88ded2)
+
+commit 7347f1193963666ce07ff9163064aeb0cf343d38
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Apr 5 21:29:10 2017 +0800
+
+    mds: make C_MDSInternalNoop::complete() delete 'this'
+    
+    Fixes: http://tracker.ceph.com/issues/19501
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 424e0c6744e7f63459ca0ff7deab751726aa30cd)
+    
+    Conflicts:
+            src/mds/MDSContext.h (omit "override" because jewel does not have
+            1a91aeab987870b3ccbcf2f1e476fac8b534d449)
+
+commit 824b19a9a66261a90c4e4de64e877a42438febd2
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 29 19:38:37 2017 +0100
+
+    tools/cephfs: set dir_layout when injecting inodes
+    
+    When we left this as zero, the MDS would interpret it was HASH_LINUX
+    rather than the default HASH_RJENKINS.  Potentially that
+    could cause problems if there perhaps were already dirfrags in
+    the metadata pool that were set up using rjenkins.  Mainly
+    it just seems more appropriate to explicitly set this field
+    rather than hit the fallback behaviour.
+    
+    Related: http://tracker.ceph.com/issues/19406
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 7d6d542885bd29b71214f9ca52bd26e9183c5d01)
+
+commit eab56dae6714706cf9650dff172b20a764f5481c
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Mar 9 13:15:46 2017 +0000
+
+    mon: fix hiding mdsmonitor informative strings
+    
+    Local `stringstream ss` declarations were hiding
+    the real variable used to feed back to the user.
+    
+    Fixes: http://tracker.ceph.com/issues/16709
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 00404ae9bd4cce0518a44d36d2d6a5612f4f9d04)
+
+commit da888fa8713178cc23c1069ce2651e02c98aad05
+Merge: 8068b546af 2271cd8128
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Thu Apr 20 19:10:33 2017 +0800
+
+    Merge pull request #14402 from shinobu-x/wip-17331-jewel
+    
+     jewel: ceph-disk: ceph-disk list reports mount error for OSD having mount options with SELinux context
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Brad Hubbard <bhubbard@redhat.com>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 8068b546afbe8e774c9d699205a32215e38c5d4f
+Merge: 118ccad0ba 754b4a482c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 13:06:33 2017 +0200
+
+    Merge pull request #13865 from smithfarm/wip-19158-jewel
+    
+    jewel: rgw: health check errors out incorrectly
+    
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit d57437e338984c5db84f9d16387e082ada5a0a33
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Mar 6 11:51:31 2017 +0000
+
+    mds: reset heartbeat in export_remaining_imported_caps
+    
+    This loop can be very long.
+    
+    Fixes: http://tracker.ceph.com/issues/19118
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 85071f1509beba4a390730e6a3a4332484646d63)
+
+commit 6adf1904ed7209b70328b20f248a701fbdd3127c
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Mar 6 11:24:50 2017 +0000
+
+    mds: heartbeat_reset in dispatch
+    
+    Previously we only heartbeated in tick.  However, our locking is
+    not guaranteed to be fair, so on a super-busy dispatch queue it may be
+    possible for the heartbeat to time out while the tick() function
+    is waiting for mds_lock.
+    
+    Fixes: http://tracker.ceph.com/issues/19118
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 819394549af10532419d88742fae3a69d2ea487d)
+
+commit 63f41d543f8a5f1f55a12612d39c6a2a1cf9c114
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Feb 15 11:45:26 2017 +0800
+
+    test/libcephfs: avoid buffer overflow when testing ceph_getdents()
+    
+    The buffer size should be at least "2 * sizeof(struct dirent)".
+    Otherwise, the code that checks dentry '..' overflow.
+    
+    Fixes: http://tracker.ceph.com/issues/18941
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit fa6671345b8f3a82dcd232f99e55a982b0a641f1)
+
+commit 7146816065bea55b4e3fec59048a459bcff50f55
+Author: Zhi Zhang <willzzhang@tencent.com>
+Date:   Fri Feb 10 10:56:46 2017 +0800
+
+    mds/StrayManager: aviod reusing deleted inode in StrayManager::_purge_stray_logged
+    
+    Signed-off-by: Zhi Zhang <zhangz.david@outlook.com>
+    (cherry picked from commit 4978e57419482384279d7e784a625f5e5c10961a)
+
+commit d8b139b5847cb46aa13486ee85e26f26421f36d6
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Feb 1 00:38:08 2017 +0000
+
+    tasks/cephfs: switch open vs. write in test_open_inode
+    
+    Do the write after opening the file, so that we get good
+    behaviour wrt the change in Mount.open_background that uses
+    file existence to confirm that the open happened.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit a027dba78fc8bc84ae39d7998b386ce21c01e1bf)
+
+commit e8ae80fc8721e500782c7e87a7cb89128c2a9687
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jan 26 16:48:58 2017 +0000
+
+    qa: fix race in Mount.open_background
+    
+    Previously a later remote call could end up executing
+    before the remote python program in open_background
+    had actually got as far as opening the file.
+    
+    Fixes: http://tracker.ceph.com/issues/18661
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit c6d91dd91252e703d08b8ac62ac6a47ee82c0bed)
+
+commit 36c86f71efc10f9a651aedf01dc6fb9d1bfed703
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jan 25 15:28:23 2017 +0800
+
+    mds: don't purge strays when mds is in clientreplay state
+    
+    MDS does not trim log when it's in clientreplay state. If mds hang
+    at clientreplay state (due to bug), purging strays can submit lots
+    of log events and create very large mds log.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 86bbc7fff02668077f27d0924ba3efe6544b77f6)
+
+commit 8b01cf33575783661a1e1151c8214d327a08b6f9
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jan 25 11:03:45 2017 +0800
+
+    mds: skip fragment space check for replayed request
+    
+    when handling replayed request, stray directory can be different
+    from the stray directory used by the original request. The fragment
+    space check for stray directory can fail.
+    
+    Fixes: http://tracker.ceph.com/issues/18660
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit afe889cbc5baab196567c2aad01f49fe90901fda)
+
+commit 82ea0971b3cb07c32ec837cb85de63d4068a70d7
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Wed Dec 14 12:09:44 2016 -0800
+
+    client: fix the cross-quota rename boundary check conditions
+    
+    We were previously rejecting a rename if either of the involved directories
+    was a quota root, even if the other directory was part of the same quota
+    "tree". What we really want to do is identify the correct quota root
+    (whether local or ancestral) for each directory and compare them. So
+    now we do.
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 8e8892aa46accb519faa4bb9fecf66618f1b11b2)
+    
+    Conflicts:
+            src/client/Client.cc (do not pass perm to get_quota_root() because
+            jewel does not have 3caa4d233633fb7a67747f2c79c4a0ab89112294)
+
+commit dbe90c79b86743c7d143d33eb5389fcee0ac76ef
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Apr 11 22:31:43 2017 +0200
+
+    librbd: fix rbd_metadata_list and rbd_metadata_get
+    
+    - properly check for val_len in rbd_metadata_list
+    - don't expect output buffers are zero pre-filled
+    
+    Fixes: http://tracker.ceph.com/issues/19588
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 75afc74ea681402e22b6dec8b83276d145fc786b)
+
+commit 5096fc9c7c62e3043a9a0638eb2516792526fdd7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 12 10:47:28 2017 -0400
+
+    test/librados_test_stub: fixed cls_cxx_map_get_keys/vals return value
+    
+    Fixes: http://tracker.ceph.com/issues/19597
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9ffd464dac102f684d6dfa78e58d2cb45e165ed6)
+
+commit 216156b5d4e2666ca592eaab2211940028422bc0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Mar 10 10:56:38 2017 -0500
+
+    rbd: prevent adding multiple mirror peers to a single pool
+    
+    The rbd-mirror daemon does not currently support replication
+    from multiple peers. Until that is supported, add a temporary
+    restriction to prevent confusion.
+    
+    Fixes: http://tracker.ceph.com/issues/19256
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c0c9d1014d57b3d5b95e7513fcc38d04b9ea5165)
+
+commit c05bd1cb1f570c42b1999fbc70c86b0a72a7fcb3
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed Apr 5 13:31:08 2017 +0300
+
+    radosgw-admin: use zone id when creating a zone
+    
+    Fixes: http://tracker.ceph.com/issues/19498
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 3fea36d635fcba8ca584a1c0ec9f07840009402c)
+
+commit ba81cbbdee1ccf95ceff56eef0a1b2b06be4024a
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Mar 14 15:43:13 2017 -0400
+
+    qa: rgw task uses period instead of region-map
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit e3e3a71d1f1fb43bb4172ce2dfac9a28ca89df0f)
+
+commit a755c95262e10fac774716e460e2ba2eaee9df70
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Mar 14 14:18:15 2017 -0400
+
+    rgw-admin: remove deprecated regionmap commands
+    
+    Fixes: http://tracker.ceph.com/issues/18725
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 5830c1849a0c0110d17c37784808e456e6dcb7b3)
+    
+    Conflicts:
+            src/rgw/rgw_admin.cc (trivial resolution)
+
+commit 482bd1adabf6f54c8f2bd73090d2e4c02d07f8a3
+Author: hrchu <petertc@gmail.com>
+Date:   Wed Mar 29 02:17:04 2017 +0000
+
+    doc: rgw: correct the quota section
+    
+    Add the missing option and fix typo.
+    
+    Fixes: http://tracker.ceph.com/issues/19397
+    
+    Signed-off-by: Chu, Hua-Rong <hrchu@cht.com.tw>
+    (cherry picked from commit 51a88267f0d7f51aeb62092949b66b9f6c062e15)
+
+commit 21a83e1276e415e98a3780d0374bb9d4feb191d9
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Mar 15 16:35:16 2017 -0400
+
+    rgw_file:  remove unused rgw_key variable
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 1100a1c26e76485569cfebcf863b18cf908f6161)
+
+commit ebad040b3ab62e702afc52edd9d99d984b24cdc1
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Apr 14 15:56:37 2017 -0400
+
+    rgw_file: fix readdir after dirent-change
+    
+    Also, fixes link count computation off-by-one, update of state.nlink
+    after computation, link computation reset at start, and a time print
+    in debug log.
+    
+    Fixes: http://tracker.ceph.com/issues/19634
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    
+    link count
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit e0f80266ecd424bf9466579b3edc03911a7c5719)
+
+commit dd9833cacaec20e5bd1a70ec46a427a7352eb5d6
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Apr 11 06:42:07 2017 -0400
+
+    rgw_file: don't expire directories being read
+    
+    If a readdir expire event turns out to be older than last_readdir,
+    just reschedule it (but actually, we should just discard it, as
+    another expire event must be in queue.
+    
+    Fixes: http://tracker.ceph.com/issues/19625
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 007b7451c26716c51207c161dc347e9a00da53f1)
+
+commit 30a5e857aade2474c5bd621c57938dfe333c3ae5
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Mar 15 16:40:35 2017 -0400
+
+    rgw_file:  rgw_readdir:  return dot-dirs only when *offset is 0
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 61482c2b85a07519f2256b1a3f2b6d8aa99d5f06)
+
+commit fe836bfb7a286c87c57576d29d6862be2514ada5
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Apr 11 05:56:13 2017 -0400
+
+    rgw_file:  chunked readdir
+    
+    Adjust readdir callback path for new nfs-ganesha chunked readdir,
+    including changes to respect the result of callback to not
+    continue.
+    
+    Pending introduction of offset name hint, our caller will just be
+    completely enumerating, so it is possible to remove the offset map
+    and just keep a last offset.
+    
+    Fixes: http://tracker.ceph.com/issues/19624
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit e0191d74e3aef06bf300df045a53a3952a71f651)
+
+commit 16eeb8c2acc8a1f843fa87967c2ee4c0ed2cd8bd
+Author: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+Date:   Fri Mar 31 10:42:40 2017 +0800
+
+    rgw_file: fix missing unlock in unlink
+    
+    Fixes: http://tracker.ceph.com/issues/19435
+    
+    Signed-off-by: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+    (cherry picked from commit cb6808a6366a70f54d0cc16437d16aa1b7819c84)
+
+commit 8c7cb8227c39b09060cdca4e11dec75cf7f2336c
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Mar 13 21:52:08 2017 -0400
+
+    rgw_file: implement reliable has-children check (unlink dir)
+    
+    Bug report and discussion provided by
+    Gui Hecheng <guihecheng@cmss.chinamobile.com> in nfs-ganesha upstream
+    github.  Briefly, while a reliable check is potentially costly,
+    it is necessary.
+    
+    Fixes: http://tracker.ceph.com/issues/19270
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit b05f1c6d61aa4501a971e87de6dcaf3e58c3d9b4)
+
+commit 18f14dd86400b50f46930a9ef56666d82035507e
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Apr 4 20:16:13 2017 -0400
+
+    rgw_file: introduce rgw_lookup type hints
+    
+    The new type hints optimize object type deduction, when the
+    rgw_lookup is called from an rgw_readdir callback.
+    
+    Fixes: http://tracker.ceph.com/issues/19623
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 2e66c7a7cc763c5c0d6f5db04855f60f2b2ceed3)
+
+commit 118ccad0ba763b98c9ee3de225a947be5372dc0d
+Merge: bf30ecd7cb 65465356b5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 11:08:45 2017 +0200
+
+    Merge pull request #14383 from smithfarm/wip-19547-jewel
+    
+    jewel: build/ops: rbdmap.service not included in debian packaging (jewel-only)
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit bf30ecd7cb8dea43a9d55cb1b02c72bfda6bad09
+Merge: 376c5e4753 9e123e6d6c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 11:02:39 2017 +0200
+
+    Merge pull request #14143 from smithfarm/wip-19355-jewel
+    
+    jewel: rgw: when converting region_map we need to use rgw_zone_root_pool
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 376c5e47531a2803ac9b7ea236b989c528268a9f
+Merge: da306df4de e2ee70a8ad
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 11:01:49 2017 +0200
+
+    Merge pull request #14136 from smithfarm/wip-19330-jewel
+    
+    jewel: rgw: upgrade to multisite v2 fails if there is a zone without zone info
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit da306df4deaaa782c0bad72539affd3ee1ec7efc
+Merge: 57b210da41 5ee8feaba4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 11:00:35 2017 +0200
+
+    Merge pull request #13872 from smithfarm/wip-19163-jewel
+    
+    jewel: doc: radosgw-admin: add the 'object stat' command to usage
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 57b210da4185d624524d05bcd9ad01c2df16ca76
+Merge: db92019e9a 9cd7dd8490
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 10:59:33 2017 +0200
+
+    Merge pull request #13863 from smithfarm/wip-19155-jewel
+    
+    jewel: rgw: typo in rgw_admin.cc
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit db92019e9af43c2367fdf7865c0fab8cb38453e6
+Merge: 6dcd5fa847 6add2a457e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 10:56:35 2017 +0200
+
+    Merge pull request #13779 from smithfarm/wip-18866-jewel
+    
+    jewel: rgw: 'radosgw-admin sync status' on master zone of non-master zonegroup
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit eac0e27193c67ffca60b5e7c61e7769ad8ace6aa
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Apr 19 16:53:57 2017 +0200
+
+    Revert "osdc/Objecter: If osd full, it should pause read op which w/ rwordered flag."
+    
+    This reverts commit 2d68822c784eb4d62d3b0198ed4ec04404dbffb3.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 0efe16d2566f0d6040f61fafd38c6661f08da1cd
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Apr 19 16:53:49 2017 +0200
+
+    Revert "osdc/Objecter: resend RWORDERED ops on full"
+    
+    This reverts commit f2474042ecd6560323673170c13f2cb964406e70.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 33af18e59298926d9636023faf871a191d582c1a
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Apr 18 08:06:01 2017 +0200
+
+    doc: mention --show-mappings in crushtool manpage
+    
+    Fixes: http://tracker.ceph.com/issues/19649
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit b48b6f4ed8b5f5b5852cbbfd5b3d5b650efb7f1b)
+
+commit 6dcd5fa8474c3a43ffca2394557b11f4906ee1ff
+Merge: 7c006fc164 68fcb01211
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Wed Apr 19 02:47:27 2017 -0400
+
+    Merge pull request #14596 from gregsfortytwo/wip-17916-divergent
+    
+    Wip 17916 divergent
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 97cd21afc54efe3afb482b041f9c34ab6cdc682e
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Apr 9 20:11:27 2017 +0200
+
+    tests: Thrasher: handle "OSD has the store locked" gracefully
+    
+    On slower machines (VPS, OVH) it takes time for the OSD to go down.
+    
+    Fixes: http://tracker.ceph.com/issues/19556
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit a5b19d2d73540b730392f8001c8601f2cecc1b51)
+
+commit faeb8088ac3bd16bc04a1e5a55fec50285a7253f
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Apr 5 16:19:57 2017 -0400
+
+    rgw: fix for null version_id in fetch_remote_obj()
+    
+    commit 8b43c9781206c22d9aedb4beb8d669bf1e23169f fixed the wrong use of
+    the dest_obj's version, but removed the check for "null" version
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 915370776df5b964c2ee8d9f9329562919eef8d5)
+
+commit 6180fcb4ec33bdade37f5693193712f8016b2560
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Fri Mar 17 16:26:56 2017 +0800
+
+    rgw: version id doesn't work in fetch_remote_obj
+    
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit 8b43c9781206c22d9aedb4beb8d669bf1e23169f)
+    
+    Conflicts:
+            src/rgw/rgw_rados.cc: trivial: dest_obj.key.instance in master
+              versus dest_obj.get_instance() in Jewel
+
+commit d079b91479abfb474fad4fafe54c119f009ea572
+Author: liuchang0812 <liuchang0812@gmail.com>
+Date:   Fri Feb 10 18:02:03 2017 +0800
+
+    rgw: don't return skew time in pre-signed url
+    
+    Fixes: http://tracker.ceph.com/issues/18828
+    
+    Signed-off-by: liuchang0812 <liuchang0812@gmail.com>
+    (cherry picked from commit dd8b348f4aad0124e8a4457117bf3f5f76af7bdb)
+
+commit 9b77b16b888b8efbf7d50d333e1880a6ec70d87a
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Apr 13 23:49:50 2017 +0200
+
+    ceph-disk: enable directory backed OSD at boot time
+    
+    https://github.com/ceph/ceph/commit/539385b143feee3905dceaf7a8faaced42f2d3c6
+    introduced a regression preventing directory backed OSD from starting at
+    boot time.
+    
+    For device backed OSD the boot sequence starts with ceph-disk@.service
+    and proceeds to
+    
+        systemctl enable --runtime ceph-osd@.service
+    
+    where the --runtime ensure ceph-osd@12 is removed when the machine
+    reboots so that it does not compete with the ceph-disk@/dev/sdb1 unit at
+    boot time.
+    
+    However directory backed OSD solely rely on the ceph-osd@.service unit
+    to start at boot time and will therefore fail to boot.
+    
+    The --runtime flag is selectively set for device backed OSD only.
+    
+    Fixes: http://tracker.ceph.com/issues/19628
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit f425a127b7487d2093c8c943f0bcdec3d673d601)
+    
+    Conflicts:
+            src/ceph-disk/ceph_disk/main.py: trivial: Jewel does not support
+               OpenRC and other inits, hence no corresponding 'elif'
+
+commit 68fcb01211e064f0d200cc9c9576254e9a6b949c
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Mon Apr 17 18:09:55 2017 -0700
+
+    pglog: require users set a config option before ignoring divergent_priors
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit b9477303b010b3653934f77fa533df01aeff1c3c
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Fri Apr 7 14:33:20 2017 -0700
+
+    osd: pglog: clean up divergent_priors off disk when running; don't assert on startup
+    
+    Fixes: http://tracker.ceph.com/issues/17916
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 7c006fc1640d2f6b9dc9002bfd994ecdf25510ee
+Merge: e31a540dce 721b2083cd
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Mon Apr 17 17:41:19 2017 -0400
+
+    Merge pull request #14492 from gregsfortytwo/wip-jewel-snaptrim
+    
+    Backport snap trimming improvements to Jewel
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 721b2083cd2ed8567d41aba6b17caf30b678e6b3
+Merge: a84dc8fe74 e31a540dce
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Mon Apr 17 17:39:59 2017 -0400
+
+    Merge branch 'jewel' into wip-jewel-snaptrim
+
+commit a84dc8fe747d1e17c7910c9857f7f5d3d56e2dff
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Mon Apr 17 14:32:38 2017 -0700
+
+    PendingReleaseNotes: discuss snap trim improvements
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 360a9d9af003c650cdf00534909d6488c702c413
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Wed Apr 12 16:30:55 2017 -0700
+
+    PrimaryLogPG: reimplement osd_snap_trim_sleep within the state machine
+    
+    Rather than blocking the main op queue, just pause for that amount of
+    time between state machine cycles.
+    
+    Also, add osd_snap_trim_sleep to a few of the thrasher yamls.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 2ed7759cfeb03e71f0fbd98fe7c2db2bb741861c)
+    
+    Conflicts:
+            src/osd/PrimaryLogPG.cc
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 67336454a4cee66522bc0ca01b2c58b8960f75ec)
+    
+    Conflicts:
+            qa/suites/rados/thrash/thrashers/default.yaml
+            qa/suites/rados/thrash/thrashers/pggrow.yaml
+            src/osd/OSD.h
+            src/osd/ReplicatedPG.cc
+            src/osd/ReplicatedPG.h
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 18dbf6a0245e35dbbdb5ddb760182795b37983c0
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 26 15:41:21 2017 -0800
+
+    rados: check that pool is done trimming before removing it
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 4aebf59d906fa3e03d21bdac182f89fe3cd4c802)
+    (cherry picked from commit 34398c29b3c57f00d932cf96570f882dce64a82b)
+
+commit 7f78450bd184335ba0098f3ded0fefa2c79a5dd5
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Tue Apr 11 14:04:19 2017 -0700
+
+    osd/ReplicatedPG: limit the number of concurrently trimming pgs
+    
+    This patch introduces an AsyncReserver for snap trimming to limit the
+    number of pgs on any single OSD which can be trimming, as with backfill.
+    Unlike backfill, we don't take remote reservations on the assumption
+    that the set of pgs with trimming work to do is already well
+    distributed, so it doesn't seem worth the implementation overhead to get
+    reservations from the peers as well.
+    
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 21cc515adfb225ba70f1d80b1b76f0345c214c22)
+    
+    Conflicts:
+            src/osd/PrimaryLogPG.cc
+            src/osd/PrimaryLogPG.h
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    
+    (cherry picked from commit 68ea24396ca6450d4d8042a7c5f51306b7d199fa)
+    (cherry picked from commit c7176b869898c870b56b1762958652d801af4c4c)
+    
+    Conflicts: Many. As evidenced by involving two distinct patches
+    in this one commit, it wasn't a clean backport.
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 527911fab78b4752313a4a2a5d3ab0ae736bc50f
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Apr 10 14:53:46 2017 +0800
+
+    mon/MonClient: make get_mon_log_message() atomic
+    
+    * LogClient: move reset_session() into get_mon_log_message() and add a
+      "flush" param to the latter. so it can get_mon_log_message()
+      atomically. otherwise another call changing the log queue could sneak
+      in between reset_session() and get_mon_log_message().
+    * MonClient: add a "flush" param to do_send() so we can reset the
+      LogClient session once we are connected to a monitor.
+    
+    Fixes: http://tracker.ceph.com/issues/19427
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5215e291da2b527d85e129eda86043490843178e)
+    
+    Conflicts:
+            src/mon/MonClient.cc: handle_auth: replaced 'log_client->reset_session();
+            send_log();' sequence with newly introduced 'send_log(true);' like
+            the original patch does
+
+commit e31a540dcea96b3d5b4f7ecd20e2d54e81a68e2b
+Merge: 7c36d1650f 06916a8798
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Apr 14 22:28:51 2017 +0200
+
+    Merge pull request #13834 from smithfarm/wip-18969-jewel
+    
+    jewel: rgw: Change loglevel to 20 for 'System already converted' message
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 7c36d1650fdb7a357ff724b37cf59b4da413f76b
+Merge: 0e3aa2cb01 4c1f302f7d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Apr 14 22:26:51 2017 +0200
+
+    Merge pull request #13833 from smithfarm/wip-18908-jewel
+    
+    jewel: rgw: the swift container acl does not support field .ref
+    
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit b698d1fa4ce4aca5e392eeec600e3357c3cf71a9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Mar 16 12:28:41 2017 -0400
+
+    librbd: is_exclusive_lock_owner API should ping OSD
+    
+    This is required to detect if a peer has been silently blacklisted
+    and is therefore no longer the lock owner.
+    
+    Fixes: http://tracker.ceph.com/issues/19287
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e15db05960a284bdf3701256722299d553cfd5aa)
+    
+    Conflicts:
+            src/librbd/ManagedLock.[h|cc]: logic moved to ExclusiveLock
+    
+    (cherry picked from commit 7e30b630e2806c73ea503871599f958b58df7934)
+
+commit 0e3aa2cb011be8a2af69040b94ee240ea6c1e663
+Merge: 8d5a5ddfec 419c9926d9
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 13 11:09:27 2017 +0200
+
+    Merge pull request #13214 from ovh/bp-osd-updateable-throttles-jewel
+    
+    jewel: osd: allow client throttler to be adjusted on-fly, without restart
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 8d5a5ddfecc7c020eeb8aeb927e007bc04885534
+Merge: 091aaa2ab7 bcd3c906e5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 13 10:44:23 2017 +0200
+
+    Merge pull request #14326 from shinobu-x/wip-15025-jewel
+    
+    jewel: osd: new added OSD always down when full flag is set
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit d311eea6bc5b2c88984cfc16340970ea1459b74a
+Author: Robin H. Johnson <robin.johnson@dreamhost.com>
+Date:   Sat Feb 11 10:32:53 2017 -0800
+
+    msg/simple/Pipe: support IPv6 QoS.
+    
+    Extend DSCP marking for heartbeat packets to IPv6, as commit
+    9b9a682fe035c985e416ee1c112fa58f9045a27c only implemented
+    support for IPv4.
+    
+    Conflicts: Cherry-picked 91a29bc490fdfbbef0875fa620c7ba1a1a6492ae from master to avoid conflict.
+    Backport: jewel, luminious
+    Fixes: http://tracker.ceph.com/issues/18887
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+    (cherry picked from commit 2d6021fbf7a728f73c2998be17e9224f14b83a30)
+
+commit 332b5174c769f395074255e075de8d2cc1ee4021
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Mar 16 12:27:08 2017 -0400
+
+    pybind: fix incorrect exception format strings
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 68617455f534a612ade1331f43b032ab524704ae)
+
+commit 091aaa2ab768858e840e2d05e0896c229ce69984
+Merge: 3f2e4cd2d5 d30c4d55ad
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 11:38:48 2017 +0200
+
+    Merge pull request #13874 from smithfarm/wip-19171-jewel
+    
+    jewel: doc: rgw S3 create bucket should not do response in json
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek@suse.com>
+
+commit 3f2e4cd2d53f1cbdbdb1fc8687c5ffe0d46b346b
+Merge: ea0bc6c553 0e11a938c5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 11:06:40 2017 +0200
+
+    Merge pull request #13492 from shinobu-x/wip-18516-jewel
+    
+    jewel: build/ops: "osd marked itself down" will not recognised if host runs mon + osd on shutdown/reboot
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit ea0bc6c553454f7641a2594013412aee142fbd11
+Merge: 845972f4de d012c381e8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 11:01:23 2017 +0200
+
+    Merge pull request #13254 from shinobu-x/wip-14609-jewel
+    
+    jewel: common: radosstriper: protect aio_write API from calls with 0 bytes
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 845972f4de1d803aa4dab0e1afaed693bbba088f
+Merge: a3deef997f cfa37d6a16
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:52:10 2017 +0200
+
+    Merge pull request #13489 from shinobu-x/wip-18955-jewel
+    
+    jewel: ceph-disk: bluestore --setgroup incorrectly set with user
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit a3deef997ff0800b9e3e2d141cfc6fdc73cac837
+Merge: 702edb5519 39aab763a4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:51:42 2017 +0200
+
+    Merge pull request #14070 from smithfarm/wip-19339-jewel
+    
+    jewel: tests: dummy suite fails in OpenStack
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 702edb5519e67bc5f8c5b65c6f63c9635cd758cf
+Merge: f509ccc4b7 a20d2b89ee
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:51:12 2017 +0200
+
+    Merge pull request #14329 from smithfarm/wip-19493-jewel
+    
+    jewel: ceph-disk: Racing between partition creation & device node creation
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Sébastien Han <seb@redhat.com>
+
+commit f509ccc4b734697267cec35ff57dd7f1c5aaaddb
+Merge: c8c4bff2af f2474042ec
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:48:29 2017 +0200
+
+    Merge pull request #14427 from smithfarm/wip-19140-jewel
+    
+    jewel: osdc/Objecter: If osd full, it should pause read op which w/ rwordered flag
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit c8c4bff2afa7fe0339a97f32e69bb9d0546f1318
+Merge: 349baea1a4 b5b441abaa
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:48:08 2017 +0200
+
+    Merge pull request #14324 from shinobu-x/wip-19371-jewel
+    
+    jewel: common: monitor creation with IPv6 public network segfaults
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 349baea1a4486e475e0381a6b316d64a6ce0139c
+Merge: dd466b7d9a 72e2476a13
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:45:35 2017 +0200
+
+    Merge pull request #14112 from shinobu-x/wip-19192-jewel
+    
+    jewel: tools: brag fails to count "in" mds
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit dd466b7d9acb03c8830bdd83b3b73602c68083c2
+Merge: b8f8bd0a94 ee06517547
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:45:11 2017 +0200
+
+    Merge pull request #14150 from smithfarm/wip-18823-jewel
+    
+    jewel: tests: run-rbd-unit-tests.sh assert in lockdep_will_lock, TestLibRBD.ObjectMapConsistentSnap
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit b8f8bd0a949c917b119eca91aec95c6a971a1fb4
+Merge: 222916a375 1cc8d0d085
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:44:47 2017 +0200
+
+    Merge pull request #14152 from smithfarm/wip-18893-jewel
+    
+    jewel: librbd: Incomplete declaration for ContextWQ in librbd/Journal.h
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 222916a3758b0253bcb851b5e955f6970f171306
+Merge: 49f84b1a14 b85677397e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:44:25 2017 +0200
+
+    Merge pull request #14154 from smithfarm/wip-18948-jewel
+    
+    jewel: tests: additional rbd-mirror test stability improvements
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 49f84b1a14295e80ef35204ed691b3131c5f744c
+Merge: 2a232d43a1 f6489d01ca
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:44:04 2017 +0200
+
+    Merge pull request #14148 from smithfarm/wip-18778-jewel
+    
+    jewel: rbd: rbd --pool=x rename y z does not work
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 2a232d43a18b47399f96abeac3ac800f204a9eaf
+Merge: 413ac584d6 8bed107b84
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:43:41 2017 +0200
+
+    Merge pull request #14083 from smithfarm/wip-19357-jewel
+    
+    jewel: rbd: systemctl stop rbdmap unmaps all rbds and not just the ones in /etc/ceph/rbdmap
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 413ac584d63fd469ea28defa68c39538444d01b1
+Merge: 23d595b1b0 547e867628
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:43:21 2017 +0200
+
+    Merge pull request #13154 from smithfarm/wip-18496-jewel
+    
+    jewel: librbd: Possible deadlock performing a synchronous API action while refresh in-progress
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 23d595b1b0fb874adfd1507d86db261cf6a193a9
+Merge: 4add6f5580 915dbace5d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:42:54 2017 +0200
+
+    Merge pull request #13244 from smithfarm/wip-18775-jewel
+    
+    jewel: rbd: qemu crash triggered by network issues
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 4add6f5580b7d65571bc426e95fac35be7150ff4
+Merge: 37ab19cc29 1a4e1e09b1
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:42:23 2017 +0200
+
+    Merge pull request #13809 from asheplyakov/18321-bp-jewel
+    
+    jewel: librbd: remove image header lock assertions
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 37ab19cc29d61970e08af1b4627137cfcaa99474
+Merge: f7c04e3ca6 8d0140a9ed
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:41:56 2017 +0200
+
+    Merge pull request #13107 from smithfarm/wip-18669-jewel
+    
+    jewel: tests: [  FAILED  ] TestLibRBD.ImagePollIO in upgrade:client-upgrade-kraken-distro-basic-smithi
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit f7c04e3ca69c7fc134b267e825277eccea228c9b
+Merge: d2909bdb4c a18a2dd108
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:39:26 2017 +0200
+
+    Merge pull request #13585 from asheplyakov/jewel-bp-16585
+    
+    jewel: msg: set close on exec flag
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit d2909bdb4cf135de850cc865f735cc61eb3d4ea6
+Merge: cd748603da 6d47615c11
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:56:22 2017 +0200
+
+    Merge pull request #14371 from tchaikov/wip-19429-jewel
+    
+    jewel: tests: clone workunit using the branch specified by task
+    
+    Nathan Cutler <ncutler@suse.com>
+
+commit cd748603da43c0b7377b5ef07aff618711b4879c
+Merge: 1a20c12355 dd25a8f36b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:54:11 2017 +0200
+
+    Merge pull request #14325 from shinobu-x/wip-18619-jewel
+    
+    jewel: osd: degraded and misplaced status output inaccurate
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 1a20c123556e2f566af7e87ab3da6fc2ca328bb9
+Merge: 4838c4db4c 7fdf4d41c5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:53:25 2017 +0200
+
+    Merge pull request #14236 from smithfarm/wip-19392-jewel
+    
+    jewel: mon: remove bad rocksdb option
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 4838c4db4c610926edc599677cb7e74f3bdc3077
+Merge: e26b7033eb 3860ccf16d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:52:41 2017 +0200
+
+    Merge pull request #14181 from mslovy/wip-19394-jewel
+    
+    jewel: osd: bypass readonly ops when osd full
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit e26b7033ebd4a09210d482a54677806fe2c03505
+Merge: 389150bcc3 819af9e413
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:49:49 2017 +0200
+
+    Merge pull request #14113 from shinobu-x/wip-19319-jewel
+    
+     jewel: cli: RadosImport::import should return an error if Rados::connect fails
+    
+    Reviewed-by: Brad Hubbard <bhubbard@redhat.com>
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 389150bcc376702022cf15c0c79dbe856a97dffa
+Merge: a8b10082f5 90de64bd81
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:45:33 2017 +0200
+
+    Merge pull request #14047 from asheplyakov/reindex-on-pg-split
+    
+    jewel: osd: reindex properly on pg log split
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit a8b10082f5761ea1132b85bd916a84f87e0276e2
+Merge: 32ed9b7897 ae498e84ff
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:45:15 2017 +0200
+
+    Merge pull request #14044 from mslovy/wip-19311-jewel
+    
+    jewel: core: os/filestore: fix clang static check warn use-after-free
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 32ed9b789733e21a2d597c7016eabf95763e50ae
+Merge: 6705e911a2 335b5fa4a9
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:44:25 2017 +0200
+
+    Merge pull request #13932 from asheplyakov/18911-bp-jewel
+    
+    jewel: rbd-nbd: check /sys/block/nbdX/size to ensure kernel mapped correctly
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 6705e911a229cee50d5ac992c7dd9af834f2400d
+Merge: 3d21a0080d 714eb863c3
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:42:13 2017 +0200
+
+    Merge pull request #13831 from jan--f/wip-19206-jewel
+    
+    jewel: fs: Invalid error code returned by MDS is causing a kernel client WARNING
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 3d21a0080dfd00c0b89ed23fd8049e46c3041af4
+Merge: 8a6d64331b 6b5322c5f6
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:41:36 2017 +0200
+
+    Merge pull request #13827 from tchaikov/wip-19185-jewel
+    
+    jewel: osd: ReplicatedPG: try with pool's use-gmt setting if hitset archive not found
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 8a6d64331b73a4a24a59f4dcdb6771f1dc638879
+Merge: f96392a589 cebba011e5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:40:09 2017 +0200
+
+    Merge pull request #13788 from shinobu-x/wip-18235-jewel
+    
+    jewel: core: os/filestore/HashIndex: be loud about splits
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit f96392a58970ab1e81653d5657d3d680120a02b6
+Merge: 8fe6ffcfef 1d054c3856
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:39:27 2017 +0200
+
+    Merge pull request #13786 from shinobu-x/wip-19129-jewel
+    
+    jewel: build/ops: ceph-base package missing dependency for psmisc
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 8fe6ffcfef8918fd8634e87255ad3fe7a544aa16
+Merge: 6f589a159e b249fd5bd8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:38:34 2017 +0200
+
+    Merge pull request #13732 from liewegas/wip-19119-jewel
+    
+    jewel: doc: PendingReleaseNotes: note about 19119
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 6f589a159e4eb5f6c96634a742acaf6cd6e174c2
+Merge: b8f2d35884 8c7a1df251
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:36:21 2017 +0200
+
+    Merge pull request #13541 from shinobu-x/wip-18929-jewel
+    
+     jewel: osd: restrict want_acting to up+acting on recovery completion
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit b8f2d35884a52586a28f1ff4eaf99c8c3ba1c43f
+Merge: 40d1443d68 2d17092fab
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:34:54 2017 +0200
+
+    Merge pull request #13477 from asheplyakov/jewel-bp-18951
+    
+    jewel: osd: --flush-journal: sporadic segfaults on exit
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 40d1443d68b788737f83c10db3811bc37dbe8dda
+Merge: 50e863e0f4 07501dec6f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 03:33:25 2017 +0200
+
+    Merge pull request #13261 from shinobu-x/wip-18587-jewel
+    
+    jewel: mon: OSDMonitor: make 'osd crush move ...' work on osds
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 25e43ac25619d883d5a04e2df1cf6f57fea73fd0
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Apr 11 08:23:23 2017 +0200
+
+    tests: fix oversight in yaml comment
+    
+    When the file was copied from the hammer version, the word "hammer"
+    was not changed to "infernalis".
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    
+    This cannot be cherry-picked from master because the test has been dropped.
+
+commit f2474042ecd6560323673170c13f2cb964406e70
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 2 21:20:08 2017 -0600
+
+    osdc/Objecter: resend RWORDERED ops on full
+    
+    Our condition for respecting the FULL flag is complex, and involves
+    the WRITE | RWORDERED flags vs the FULL_FORCE | FULL_TRY flags.  Previously,
+    we could block a read bc of RWORDRED but not resend it later.
+    
+    Fix by capturing the complex condition in a respects_full() bool and using
+    it both for the blocking-on-send and resending-on-possibly-notfull-later
+    checks.
+    
+    Fixes: http://tracker.ceph.com/issues/19133
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c4b73f19a7be13ff412eef804efcd8c18ed4dae6)
+
+commit 2d68822c784eb4d62d3b0198ed4ec04404dbffb3
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Thu May 5 23:44:57 2016 +0800
+
+    osdc/Objecter: If osd full, it should pause read op which w/ rwordered flag.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit 07b2a22210e26eac1b2825c30629788da05e5e12)
+
+commit 2271cd81282f3f026316134cbab630f3e4f47782
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Mon Apr 3 13:37:17 2017 +1000
+
+    ceph-disk: Populate mount options when running "list"
+    
+    Also tidy up by moving duplicated code into a function
+    
+    Fixes: http://tracker.ceph.com/issues/17331
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 7943ab2e01e24f2dfc5b6f1d3ffdc8a49e01af45)
+
+commit 65465356b5e7cb6cba67f8cbb81259d21e888dfb
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Tue Feb 16 12:56:34 2016 -0700
+
+    debian: replace SysV rbdmap with systemd service
+    
+    Stop shipping /etc/init.d/rbdmap in the Debian packages. Ship the
+    rbdmap.service systemd unit file instead.
+    
+    The corresponding change has already been made for RPMs, in
+    9224ac2ad25f7d017916f58b642c0ea25305c3e5.
+    
+    For Upstart-based systems (eg Ubuntu Trusty), the Debian packages
+    already contain rbdmap.conf.
+    
+    (This gets us a tiny bit closer to being able to remove the rbdmap SysV
+    script from our tree entirely.)
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 839807118dda2fb4d57ed9d50ec46e3ee0e2820a)
+    
+    Conflicts:
+        debian/ceph-common.install (retain /etc/init.d/rbdmap so jewel users can choose sysvinit or systemd)
+        debian/rules (retain /etc/init.d/rbdmap so jewel users can choose sysvinit or systemd)
+
+commit 6d47615c11a216733ba368f0dbd9a0a1b9c8fd35
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Apr 1 23:04:22 2017 +0800
+
+    qa/tasks/workunit.py: use "overrides" as the default settings of workunit
+    
+    otherwise the settings in "workunit" tasks are always overridden by the
+    settings in template config. so we'd better follow the way of how
+    "install" task updates itself with the "overrides" settings: it uses the
+    "overrides" as the *defaults*.
+    
+    Fixes: http://tracker.ceph.com/issues/19429
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 47080150a17d238f38d9da824d227393ad767aad)
+
+commit fdc71e75cd6361be49c4c20e77d3bdff017b38bf
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Mar 30 12:37:01 2017 +0800
+
+    tasks/workunit.py: specify the branch name when cloning a branch
+    
+    c1309fb failed to specify a branch when cloning using --depth=1, which
+    by default clones the HEAD. and we can not "git checkout" a specific
+    sha1 if it is not HEAD, after cloning using '--depth=1', so in this
+    change, we dispatch "tag", "branch", "HEAD" using three Refspec classes.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    (cherry picked from commit 9ca7ccf5f1739f731da8bf31260594aea3a2932d)
+    
+    Conflicts:
+            qa/tasks/workunit.py (trivial resolution)
+
+commit 6c14a803894d71bae858705855551a44bdf9bf67
+Author: Dan Mick <dan.mick@redhat.com>
+Date:   Tue Mar 28 20:08:13 2017 -0700
+
+    tasks/workunit.py: when cloning, use --depth=1
+    
+    Help avoid killing git.ceph.com.  A depth 1 clone takes about
+    7 seconds, whereas a full one takes about 3:40 (much of it
+    waiting for the server to create a huge compressed pack)
+    
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    (cherry picked from commit c1309fbef300a062138ac40eb5d3e5081b833072)
+
+commit f8aa6be06cc628b1cf64c9196c30045d020d657e
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Apr 7 06:59:13 2017 +0200
+
+    build/ops: rpm: move $CEPH_EXTRA_CONFIGURE_ARGS to right place
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit bb3eb4284fe74cbf78e4a406c7b5f67a8e3c84b3
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Apr 7 06:48:51 2017 +0200
+
+    build/ops: rpm: explicitly provide --with-ocf to configure
+    
+    Fixes: http://tracker.ceph.com/issues/19546
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    
+    (Note: This cannot be cherry-picked because master uses cmake, but
+    the fix does bring the jewel spec file into better alignment its master
+    counterpart, at least as far as this one little bit is concerned.)
+
+commit 30c952735327d05b6049e9364c1a053ebf651751
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Mon Feb 27 15:52:43 2017 +0800
+
+    rgw: use separate http_manager for read_sync_status
+    
+    concurrent users of read_sync_status() use different cr managers, when get_resource must
+    use http_manager related to the cr manager.
+    
+    Fixes: http://tracker.ceph.com/issues/19236
+    
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit c412024889f8995d98096ac863bafee71624bd70)
+
+commit 87cb8474f523be1b281882c46a8a3597977a51c9
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jul 28 20:20:29 2016 -0400
+
+    rgw: pass cr registry to managers
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit ef4d7eab11fb5d2a41c9c28b9eb8b075aaff0d05)
+    
+     Conflicts:
+            src/rgw/rgw_rados.cc: removed ref to RGWSyncLogTrimThread (not backported)
+
+commit 1a6d7c0506d4bcda775dda05bb357d4d5695dabb
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Jul 22 11:00:16 2016 -0400
+
+    rgw: use separate cr manager for read_sync_status
+    
+    RGWCoroutinesManager::run() is not reentrant, so concurrent users of
+    read_sync_status() must use different managers
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 6b1e40d7a21c549b55e6576ec56973c8d3c092d1)
+
+commit c466adee9c8f7a97ff7e99bee56da8ce51bf0f00
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jul 21 23:46:20 2016 -0400
+
+    rgw: change read_sync_status interface
+    
+    RGWDataSyncStatusManager::read_sync_status() now operates on the given
+    parameter, rather than its internal member variable. this allows
+    multiple concurrent readers, which is needed for the rest interface
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit b7cd4e0e8b879b5e528de75bea3307585b96cbf2)
+
+commit 36921a3f3d01547508dc6270f6b19f2576e067d9
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jun 8 11:24:11 2016 -0400
+
+    rgw: don't ignore ENOENT in RGWRemoteDataLog::read_sync_status()
+    
+    rest handlers for sync status need to return ENOENT errors. the only
+    other callers are in radosgw-admin, so the ENOENT errors are ignored at
+    those call sites instead
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 2cc533b30c0f23c0750ea8d02c51b3b3d3b4821a)
+
+commit b249fd5bd816a63b445db12c8f846cfda199c8b8
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Mar 1 13:18:44 2017 -0600
+
+    PendingReleaseNotes: warning about 'osd rm ...' and #19119
+    
+    See http://tracker.ceph.com/issues/19119
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit be96003c464481d8e84825178d600234a0d64d22)
+
+commit 335258f975a8e8539774e8cb22690d746ec90d9f
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Tue Mar 28 11:49:08 2017 +0200
+
+    osdc/Objecter: respect epoch barrier in _op_submit()
+    
+    Epoch barrier instructs us to avoid sending (i.e. pause) any OSD ops
+    until we see a barrier epoch.  The only thing epoch_barrier check in
+    target_should_be_paused() does is keep already paused ops paused.  We
+    need to actually pause incoming OSD ops in _op_submit().
+    
+    Fixes: http://tracker.ceph.com/issues/19396
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit f8e8efc0a53d7bd807cc0c2178aef7c4bed62ab7)
+
+commit a20d2b89ee13e311cf1038c54ecadae79b68abd5
+Author: Erwan Velu <erwan@redhat.com>
+Date:   Fri Mar 31 14:54:33 2017 +0200
+
+    ceph-disk: Adding retry loop in get_partition_dev()
+    
+    There is very rare cases where get_partition_dev() is called before the actual partition is available in /sys/block/<device>.
+    
+    It appear that waiting a very short is usually enough to get the partition beein populated.
+    
+    Analysis:
+    update_partition() is supposed to be enough to avoid any racing between events sent by parted/sgdisk/partprobe and
+    the actual creation on the /sys/block/<device>/* entrypoint.
+    On our CI that race occurs pretty often but trying to reproduce it locally never been possible.
+    
+    This patch is almost a workaround rather than a fix to the real problem.
+    It offer retrying after a very short to be make a chance the device to appear.
+    This approach have been succesful on the CI.
+    
+    Note his patch is not changing the timing when the device is perfectly created on time and just differ by a 1/5th up to 2 seconds when the bug occurs.
+    
+    A typical output from the build running on a CI with that code.
+            command_check_call: Running command: /usr/bin/udevadm settle --timeout=600
+            get_dm_uuid: get_dm_uuid /dev/sda uuid path is /sys/dev/block/8:0/dm/uuid
+            get_partition_dev: Try 1/10 : partition 2 for /dev/sda does not in /sys/block/sda
+            get_partition_dev: Found partition 2 for /dev/sda after 1 tries
+            get_dm_uuid: get_dm_uuid /dev/sda uuid path is /sys/dev/block/8:0/dm/uuid
+            get_dm_uuid: get_dm_uuid /dev/sda2 uuid path is /sys/dev/block/8:2/dm/uuid
+    
+    fixes: #19428
+    
+    Signed-off-by: Erwan Velu <erwan@redhat.com>
+    (cherry picked from commit 93e7b95ed8b4c78daebf7866bb1f0826d7199075)
+
+commit 2d5d0aec60ec9689d44a53233268e9b9dd25df95
+Author: Erwan Velu <erwan@redhat.com>
+Date:   Wed Mar 22 10:11:44 2017 +0100
+
+    ceph-disk: Reporting /sys directory in get_partition_dev()
+    
+    When get_partition_dev() fails, it reports the following message :
+        ceph_disk.main.Error: Error: partition 2 for /dev/sdb does not appear to exist
+    The code search for a directory inside the /sys/block/get_dev_name(os.path.realpath(dev)).
+    
+    The issue here is the error message doesn't report that path when failing while it might be involved in.
+    
+    This patch is about reporting where the code was looking at when trying to estimate if the partition was available.
+    
+    Signed-off-by: Erwan Velu <erwan@redhat.com>
+    (cherry picked from commit 413c9fcfbe8e6ab33d73b8428090ccacc33c5d15)
+
+commit bcd3c906e5b57e7f44df1963c6e11b78ff89482c
+Author: Mingxin Liu <mingxin@xsky.com>
+Date:   Mon Mar 13 23:41:58 2017 +0800
+
+    osd: don't share osdmap with objecter when preboot
+    
+    Signed-off-by: Mingxin Liu <mingxin@xsky.com>
+    (cherry picked from commit a5a3644eecc49b4eea890c6999fe87536495dcbe)
+
+commit dd25a8f36bef1901f3ce6193cfcbdaf7ab2424a1
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Jan 18 08:33:40 2017 -0800
+
+    osd: Calculate degraded and misplaced more accurately
+    
+    Calculate num_object_copies based on the larger of pool size,
+    up set size and acting set size.
+    
+    Calculate num_objects_degraded as the difference between num_object_copies
+    and all copies found on acting set and backfilling up set OSDs.
+    
+    Calculate num_objects_misplaced as all copies on acting set OSDs not in up set
+    less copies that have been backfilled to up set OSDs.
+    
+    Fixes: http://tracker.ceph.com/issues/18619
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 8423bc40759cca137f61e7b755411719a84369d4)
+
+commit b5b441abaa852e85ddefd8b22835c9b85898cc06
+Author: Fabian Grünbichler <f.gruenbichler@proxmox.com>
+Date:   Wed Mar 22 16:13:50 2017 +0100
+
+    common: fix segfault in public IPv6 addr picking
+    
+    sockaddr is only 16 bytes big, so declaring net as sockaddr
+    and then casting to sockaddr_in6 in case of IPv6 cannot
+    work.
+    
+    using sockaddr_storage works for both IPv4 and IPv6, and is
+    used in other code parts as well.
+    
+    note that the tests did not find this issue as they declared
+    the bigger structs and casted the references to (sockaddr *)
+    
+    Fixes: http://tracker.ceph.com/issues/19371
+    Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
+    (cherry picked from commit ae2ee3d3835fe25b35eeb1a841ee5234cd69eb65)
+
+commit 7fdf4d41c5bef14269cb302301a08d5a3a57a768
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon May 2 15:29:12 2016 -0400
+
+    mon: remove bad rocksdb option
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 0ac671ece258e509f71a05253e62a9878e279840)
+
+commit 3860ccf16d7dfb137face9886c3d7f29cd527835
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Thu May 5 23:07:06 2016 +0800
+
+    osd: bypass readonly ops when osd full.
+    
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit e2a0ae8e88e6b7354b14adb503fd8ba8525bee39)
+    See: http://tracker.ceph.com/issues/19394
+    
+    Signed-off-by: yaoning <yaoning@unitedstack.com>
+
+commit b85677397ef9a3fe16c087e67d3f752851bbe070
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 21 15:31:43 2016 -0500
+
+    qa/workunits/rbd: resolve potential rbd-mirror race conditions
+    
+    Fixes: http://tracker.ceph.com/issues/18935
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 63eae97afc1a92412525468263fb8696a243ebac)
+
+commit 1cc8d0d08560af4e8785d7c4c6a925b1fc9f988e
+Author: Boris Ranto <branto@redhat.com>
+Date:   Wed Feb 8 23:47:57 2017 +0100
+
+    librbd: Include WorkQueue.h since we use it
+    
+    We use m_work_queue of type ContextWQ in handle_update function but we
+    do not include common/WorkQueue.h that defines ContextWQ. This results
+    in dereference of an incomplete type and causes build error in latest
+    Fedora rawhide (future 26).
+    
+    Fixes: http://tracker.ceph.com/issues/18862
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 480f82847ad1fc7959f1fe5a90761a5a24550993)
+
+commit ee06517547ae174472d739f966c0a27d3a97d742
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jan 18 20:54:22 2017 -0500
+
+    librbd: avoid possible recursive lock when racing acquire lock
+    
+    Fixes: http://tracker.ceph.com/issues/17447
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5e46e8eb664f573bd70ae7c96a6d9a98b0deb09e)
+
+commit f6489d01ca41d6979b5de28e3cde6b43fcaa8edb
+Author: Gaurav Kumar Garg <garg.gaurav52@gmail.com>
+Date:   Mon Jan 30 13:03:20 2017 +0100
+
+    rbd: destination pool should be source pool if it is not specified
+    
+    Currently if user perform image rename operation and user give pool
+    name as a optional parameter (--pool=<pool_name>) then currently
+    its taking this optional pool name for source pool and making
+    destination pool name default pool name.
+    With this fix if user provide pool name as a optional pool name
+    parameter then it  will consider both soruce and destination pool
+    name as optional parameter pool name.
+    
+    Fixes: http://tracker.ceph.com/issues/18326
+    
+    Reported-by: М�РК КОРЕ�БЕРГ <socketpair@gmail.com>
+    Signed-off-by: Gaurav Kumar Garg <garg.gaurav52@gmail.com>
+    (cherry picked from commit 01f23aa99fb694da326ab408e75b33c640ce660b)
+
+commit 9e123e6d6c8bbd54514b498df5a22d961f0cefbb
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Mar 12 12:11:28 2017 +0200
+
+    rgw: use rgw_zone_root_pool for region_map like is done in hammer
+    
+    Fixes: http://tracker.ceph.com/issues/19195
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit c91dd6d9efd148e0fe0f027dde537e977de9aa26)
+
+commit e2ee70a8ad51992bbd763d2465f6d8a01dad6a31
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Mar 9 13:03:24 2017 +0200
+
+    rgw: skip conversion of zones without any zoneparams
+    
+    Fixes: http://tracker.ceph.com/issues/19231
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 36cf5a5c8179c6313346b2e29286c537c6fefce8)
+
+commit c7d292bf6714d7aaf10412e5109badb90f5dc208
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Mar 9 11:16:26 2017 +0200
+
+    rgw: better debug information for upgrade
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit e9f3bf8eab1dd46a92f54b0f7afe1f4c0e4204db)
+
+commit 11f5c841c7698e6239017478fa05f742b7c0ab1c
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Jan 31 18:01:32 2017 +0100
+
+    rgw/rgw_rados.cc: prefer ++operator for non-primitive iterators
+    
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 7086cf9a73f2ec1eb96c0e752beb1b74fca18570)
+
+commit 819af9e4139997cd845dc24a137d43218d8a40a8
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Tue Mar 21 12:22:20 2017 +1000
+
+    tools/rados: Check return value of connect
+    
+    Fail gracefully if Rados::connect returns an error.
+    
+    Fixes: http://tracker.ceph.com/issues/19319
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit c119091ef0844e4a1ddd790a8bfef8f06bb57d58)
+
+commit 72e2476a130e14abcd541ff61328454cb69ad9c3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Mar 6 11:33:27 2017 +0800
+
+    brag: count the number of mds in fsmap not in mdsmap
+    
+    this change was introduced in 4e9b953
+    
+    Fixes: http://tracker.ceph.com/issues/19192
+    Signed-off-by: Peng Zhang <zphj1987@gmail.com>
+    (cherry picked from commit 2d25a9c0c760664d3de33ecca0e0272c1031cd46)
+
+commit 3cb192779a1c3662d27bba7715eb31c5f7b6a5b7
+Author: Oleh Prypin <oleh@pryp.in>
+Date:   Thu Jun 30 00:51:50 2016 +0300
+
+    brag: Assume there are 0 MDS instead of crashing when data is missing
+    
+    Signed-off-by: Oleh Prypin <oleh@pryp.in>
+
+commit 8bed107b84efdc8c735245cdfb51bfd8d07da13b
+Author: David Disseldorp <ddiss@suse.de>
+Date:   Fri Feb 10 19:19:46 2017 +0100
+
+    doc: update description of rbdmap unmap[-all] behaviour
+    
+    Fixes: http://tracker.ceph.com/issues/18884
+    
+    Signed-off-by: David Disseldorp <ddiss@suse.de>
+    (cherry picked from commit f987396e126d5e61240a6645ffed439f79b072b4)
+
+commit da4e0b56c60f4bc2c67daa5dfe4d5255ab8bfc03
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Dec 15 18:23:41 2016 +0100
+
+    doc: add verbiage to rbdmap manpage
+    
+    Fixes: http://tracker.ceph.com/issues/18262
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit fbac4a081547d83bb2436cd60b0b7ee7250f8a6c)
+
+commit 167d4fd7ccf0cdac536f95250bbfa3e9879ab769
+Author: David Disseldorp <ddiss@suse.de>
+Date:   Fri Feb 10 17:50:12 2017 +0100
+
+    rbdmap: unmap RBDMAPFILE images unless called with unmap-all
+    
+    When called with a "map" parameter, the rbdmap script iterates the list
+    of images present in RBDMAPFILE (/etc/ceph/rbdmap), and maps each entry.
+    When called with "unmap", rbdmap currently iterates *all* mapped RBD
+    images and unmaps each one, regardless of whether it's listed in the
+    RBDMAPFILE or not.
+    
+    This commit adds functionality such that only RBD images listed in the
+    configuration file are unmapped. This behaviour is the new default for
+    "rbdmap unmap". A new "unmap-all" parameter is added to offer the old
+    unmap-all-rbd-images behaviour, which is used by the systemd service.
+    
+    Fixes: http://tracker.ceph.com/issues/18884
+    
+    Signed-off-by: David Disseldorp <ddiss@suse.de>
+    (cherry picked from commit e58413abf408cbe254232e563f3e30d2dc0d707c)
+
+commit 39aab763a44e45e025c311cdfff95116df11a4c4
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Dec 15 13:01:02 2016 +0100
+
+    Revert "dummy: reduce run time, run user.yaml playbook"
+    
+    This reverts commit d4e3cec1851ae35889127b90912e133178085bc6.
+    
+    Fixes: http://tracker.ceph.com/issues/18259
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit a8a2a8c3e68b910dbaeb3186576898bf9f89f9fd)
+
+commit 2e50fe1684f73ebe96969c341242b6f20c8470a0
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Oct 20 15:01:01 2016 -0400
+
+    rgw: fix break inside of yield in RGWFetchAllMetaCR
+    
+    the yield macro is implemented with for/switch, so the breaks in
+    RGWFetchAllMetaCR weren't being applied to the for loop as expected -
+    so any of these breaks send RGWFetchAllMetaCR into an infinite loop
+    
+    removed the yield {} block, so that breaks will apply to the for loop as
+    intended, then added a single yield; statement to allow the
+    entries_index consumer to run one per iteration
+    
+    Fixes: http://tracker.ceph.com/issues/17655
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 190bd385a7be52867d65740c410884f5c8cbc21f)
+
+commit dc4e7a1a865ea0ae7362c1b6a7a542aa5f72107d
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Mar 3 12:10:40 2017 -0500
+
+    rgw: delete_system_obj() fails on empty object name
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 67401193f871db95a6045915fa59dce8c5dd1012)
+
+commit e9a577c8c535702d1eb285429978bdbb395e2d5c
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Mar 3 11:42:45 2017 -0500
+
+    rgw: if user.email is empty, dont try to delete
+    
+    Fixes: http://tracker.ceph.com/issues/18980
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 022ecf0fcc8e44912c8758ee1d9a452dc23cbbce)
+
+commit 90de64bd81fedcb9540e40d50420e169a4a81248
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Mon Mar 20 14:05:17 2017 +0400
+
+    jewel: osd/PGLog: reindex properly on pg log split
+    
+    When PGLog::IndexedLog::split_into runs it builds the list, which means
+    the old indices are wrong (point to bad memory), but index() will not
+    rebuild them because ever since b858e86 we won't rebuild them if they
+    are already built. Fix that by calling unindex() before the split.
+    
+    Based on 643ae42cf27f16dd6ed4e1402acc0483bb9fca74. Notice that both
+    the child and the parent log are re-indexed in Jewel, so the only
+    problem is missing unindex().
+    
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+
+commit ae498e84ffcff7424721f0d2704ec739d1cc092d
+Author: liuchang0812 <liuchang0812@gmail.com>
+Date:   Tue Dec 20 13:21:40 2016 +0800
+
+    os/filestore: fix clang static check warn use-after-free
+    
+    Signed-off-by: liuchang0812 <liuchang0812@gmail.com>
+    (cherry picked from commit 1d359455b3dd6abb383542ba596a03f14ac54dbd)
+    See: http://tracker.ceph.com/issues/19311
+    
+    Signed-off-by: yaoning <yaoning@unitedstack.com>
+
+commit 335b5fa4a9694620546422f9a02bdcc16549d7cc
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Feb 2 11:11:35 2017 +0100
+
+    rbd-nbd: check /sys/block/nbdX/size to ensure kernel mapped correctly
+    
+    Fixes: http://tracker.ceph.com/issues/18335
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 596e5ea8a5df72002672eef0a6d20572ca6f60f0)
+    
+    Conflicts:
+            qa/workunits/rbd/rbd-nbd.sh: the original commit removes
+              TOO_LARGE_IMAGE test, do the same thing
+            src/tools/rbd_nbd/rbd-nbd.cc: help git to add
+              "include/stringify.h"
+    
+    Other changes:
+            src/tools/rbd_nbd/rbd-nbd.cc: #include <fstream> so
+              the code compiles
+    
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+
+commit ced799f9c6558482d538f8dec854c62162685ad0
+Author: Michal Koutný <mkoutny@suse.com>
+Date:   Thu Jan 26 16:08:09 2017 -0500
+
+    rgw: Use decoded URI when verifying TempURL
+    
+    Instead of calliing url_decode directly, we reuse s->decoded_uri that is
+    initialized in RGWREST::preprocess().
+    
+    Fixes: http://tracker.ceph.com/issues/18590
+    
+    Adapted from 4e1318f4dcbfd64c3ec94f4addf6e38ddd6c013a. Cherry-picking
+    that patch requires a quite a number of unrelated changes, hence this
+    patch does s/s->info.request_uri/s->decoded_uri/ to keep the fix as
+    minimal as possible.
+    
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+
+commit 043d70461c1eb874d9185f9bd671930fad05ff65
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Wed Mar 1 17:19:04 2017 +0100
+
+    osd/OSDMap: don't set weight to IN when OSD is destroyed
+    
+    Since commit 4e28f9e63644 ("osd/OSDMap: clear osd_info, osd_xinfo on
+    osd deletion"), weight is set to IN when OSD is deleted.  This changes
+    the result of applying an incremental for clients, not just OSDs.
+    Because CRUSH computations are obviously affected, pre-4e28f9e63644
+    servers disagree with post-4e28f9e63644 clients on object placement,
+    resulting in misdirected requests.
+    
+    Fixes: http://tracker.ceph.com/issues/19119
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit a6009d1039a55e2c77f431662b3d6cc5a8e8e63f)
+
+commit 6b5322c5f62f4b90c4206c6ddcc70d090fa7eeb9
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Mar 7 18:49:46 2017 +0800
+
+    osd/ReplicatedPG: try with pool's use-gmt setting if hitset archive not found
+    
+    due to http://tracker.ceph.com/issues/19185, a hammer OSD could store
+    pg_hit_set_info_t with "use-gmt = true" even the pool setting is false.
+    so we use the pool setting as a fallback if the hitset archive is not
+    found locally and the pool.use_gmt does not match with hitset.use_gmt.
+    
+    Fixes: http://tracker.ceph.com/issues/19185
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    
+    Conflicts:
+      osd/ReplicatedPG.cc:  This cannot be cherry-picked from master
+        because: hammer should upgrade to jewel first before moving to a
+        higher version. so there is no necessary to include this workaround
+        in releases later than jewel.
+
+commit d30c4d55ad52e2b63fdbd06ce256d92fc3fd36c9
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Thu Feb 16 17:40:50 2017 +0100
+
+    doc: rgw: make a note abt system users vs normal users
+    
+    Mention that system users don't behave like normal users in context of
+    normal rest operations
+    
+    Fixes: http://tracker.ceph.com/issues/18889
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit a47bcf70c9f51a6601b809cba219f5615b204d34)
+    
+    Conflicts:
+            doc/radosgw/multisite.rst (trivial whitespace difference)
+
+commit 5ee8feaba469886f9e3bd3909475ffef62ba261d
+Author: root <root@ceph-node1.homeoffice.wal-mart.com>
+Date:   Tue Feb 7 14:37:36 2017 +0530
+
+    rgw: Let the object stat command be shown in the usage
+    
+    Fixes: http://tracker.ceph.com/issues/19013
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+    (cherry picked from commit 0fe76f83d19be098ef54fb0492a376fef3aa9e23)
+
+commit 754b4a482cb0369215beed58103a1e241231cf77
+Author: root <root@ceph-node1.homeoffice.wal-mart.com>
+Date:   Tue Feb 21 16:33:29 2017 +0530
+
+    rgw: Correct the return codes for the health check feature
+    Fixes: http://tracker.ceph.com/issues/19025
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+    
+    (cherry picked from commit 4da2bf310f6d43423554c32e43ebf90ad2c3f3a9)
+
+commit 9cd7dd84909abdb9e603ff3aeb9958cdab8c70ad
+Author: Ronak Jain <ronakjain@outlook.in>
+Date:   Wed Feb 22 12:03:46 2017 +0530
+
+    rgw: Fixes typo in rgw_admin.cc
+    
+    Issue: http://tracker.ceph.com/issues/19026
+    Signed-off-by: Ronak Jain <ronakjain@outlook.in>
+    (cherry picked from commit 58837ef6ce8cbcfc2cac29d5f833b2cf62d8737a)
+
+commit 85fbb00f6ef5f11bc5d615ccd8e2202ce3896fd1
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Feb 27 10:35:01 2017 -0800
+
+    rgw: don't init rgw_obj from rgw_obj_key when it's incorrect to do so
+    
+    Fixes: http://tracker.ceph.com/issues/19096
+    
+    rgw_obj_key currently deals with the bucket index key, and not
+    representing a (name, instance, ns) tupple. Need to initialize
+    it in two steps.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 392c5d9dae6ba699014ffe6e1e67818fa62d7e41)
+
+commit ec0668c201a71b4a17ef0ab3c5908f57229aa6ef
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Nov 29 11:29:41 2016 -0500
+
+    rgw: fix for broken yields in RGWMetaSyncShardCR
+    
+    Fixes: http://tracker.ceph.com/issues/18076
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit e62d48a9bf2e309eab1a863f167af5267ebcc371)
+
+commit 6afe3efa4b636ede1cd77086cb2e70ed09fa2e95
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Mon Jul 25 11:21:11 2016 +0200
+
+    rgw: kill a compile warning for rgw_sync
+    
+    killing the compile warning for
+    
+    ```
+    /ceph/src/rgw/rgw_sync.cc:1462:12:
+    warning: suggest explicit braces to avoid ambiguous ‘else’ [-Wparentheses]
+             if (can_adjust_marker) yield {
+                ^
+    ```
+    
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 831640bb46621a6f003ad562cef7928ffa9a7ad3)
+
+commit 06916a8798439ec033294d791749ce7381d92f51
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Mon Feb 13 23:07:25 2017 +0530
+
+    rgw: change log level to 20 for 'System already converted' message
+    
+    Fixes: http://tracker.ceph.com/issues/18919
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit 55b567c767830170d04de4cdc8f10aba30a3f379)
+
+commit 4c1f302f7d71bedb0dead220f17eeb84e7e3f737
+Author: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+Date:   Wed Jan 11 05:28:43 2017 +0800
+
+    rgw: the swift container acl should support field .ref
+    
+    On the openstack-swift. The container acl supports .ref, which is ignored on ceph swift.
+    
+    Fixes: http://tracker.ceph.com/issues/18484
+    Signed-off-by: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+    (cherry picked from commit b06f9cd9f0900db7b0d0fbcaea69cdd0d4b10132)
+    
+    Conflicts:
+            src/rgw/rgw_acl_swift.cc - no extract_referer_urlspec() in jewel
+                see https://github.com/ceph/ceph/pull/8657
+
+commit 714eb863c30df4e653068e6ea16630504e58b704
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Tue Jun 14 19:32:01 2016 +0800
+
+    server: negative error code when responding to client
+    
+    As the comment suggests. Also a zero or positive return code
+    shall indicates a success, which does not match our intention here.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 26931f888ce4661765cca106b3a3dc66702266df)
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+
+commit 1a4e1e09b1e562bf97cfe96f5cb9f937b6987165
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Dec 13 14:10:58 2016 -0500
+
+    librbd: remove image header lock assertions
+    
+    This assertions can sporadically fail if the watch is lost and
+    recovered in the background. Upon a true loss of the lock, the
+    client would either be blacklisted or it would have completed
+    all in-flight ops before releasing.
+    
+    Fixes: http://tracker.ceph.com/issues/18244
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ce4f2a52ec0a794d89e7576b59c9b9aefe3db288)
+    
+    Conflicts:
+            src/librbd/operation/SnapshotCreateRequest.cc: rbd class
+               does not support the snapshot namespaces in Jewel, skip
+               the corresponding argument
+
+commit cebba011e502f7009208bbddc295eb17f88f1bb9
+Author: Dan van der Ster <daniel.vanderster@cern.ch>
+Date:   Fri Dec 9 22:06:26 2016 +0100
+
+    os/filestore/HashIndex: be loud about splits
+    
+    Filestore splits are a rare yet important enough event that an
+    OSD should visibly report when they happen.
+    
+    Without this reporting an operator could spend hours trying to
+    understand the cause of any split-induced slow requests.
+    
+    Fixes: http://tracker.ceph.com/issues/18235
+    Signed-off-by: Dan van der Ster <daniel.vanderster@cern.ch>
+    (cherry picked from commit 61c47acd3a1f3e01f0106d4a541bb7f28a1301d8)
+
+commit 1d054c3856a63ceebe44f66ff83fda691c374f71
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Mar 2 12:41:07 2017 +0100
+
+    build/ops: add psmisc dependency to ceph-base (deb and rpm)
+    
+    Fixes: http://tracker.ceph.com/issues/19129
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 769b695465162bc8424abf8e2f259e6765b5bbff)
+    
+    Conflicts:
+            debian/control (jewel does not have f11acf2b 7e71cd2c)
+
+commit 6add2a457e2826b71c0e9e82c6f6686cecbc4584
+Author: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+Date:   Fri Nov 25 21:31:22 2016 +0800
+
+    rgw: metadata sync info should be shown at master zone of slave zonegroup
+    
+    When executing 'radosgw-admin sync status', the metadata sync info should be shown on the srceen at master zone of slave zonegroup.
+    
+    Using the function store->is_meta_master() instead of 'zonegroup.is_master && zone.id == zonegroup.master_zone'
+    
+    Fixes: http://tracker.ceph.com/issues/18091
+    Signed-off-by: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+    (cherry picked from commit c12d0af2f98b246a73cc3ee027449a22192795b3)
+
+commit 0e11a938c5c9acd8a50efa9a154ea3bf21bcafc5
+Author: Boris Ranto <branto@redhat.com>
+Date:   Wed Jan 25 12:39:40 2017 +0100
+
+    systemd: Start OSDs after MONs
+    
+    Currently, we start/stop OSDs and MONs simultaneously. This may cause
+    problems especially when we are shutting down the system. Once the mon
+    goes down it causes a re-election and the MONs can miss the message
+    from the OSD that is going down.
+    
+    Resolves: http://tracker.ceph.com/issues/18516
+    
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 7f4acf45dd0d86e7d9992a8c30e5876fb57b1914)
+    
+    Conflicts:
+            systemd/ceph-osd@.service (jewel does not have 4179aa8d)
+
+commit 3bdd4398f1dcad0b7e22f1750ca524b97feca15a
+Author: yaoning <yaoning@unitedstack.com>
+Date:   Mon Jun 6 13:31:52 2016 +0800
+
+    osd: preserve allocation hint attribute during recovery
+    
+    Signed-off-by: yaoning <yaoning@unitedstack.com>
+    (cherry picked from commit e15be792960da6bac2bd469acf7d30007be61781)
+    
+    Conflicts:
+        src/osd/ReplicatedBackend.cc (in master, it contains alloc_hint_flags for set_alloc_hint)
+        src/osd/ReplicatedPG.cc (in master, it contains alloc_hint_flags in object_info_t struct)
+        src/osd/osd_types.cc (in master, it contains alloc_hint_flags in message serialization)
+        alloc_hint_flags is used in master bluestore, filestore does not use alloc_hint_flags.
+        therefore, remove alloc_hint_flags here in jewel
+    
+    Signed-off-by: yaoning <yaoning@unitedstack.com>
+
+commit 8d0140a9eda814beadf1f59c9b4205f30a1d2e35
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jan 24 09:24:52 2017 -0500
+
+    librbd: improve debug logging for lock / watch state machines
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit cc046597983bd491cc66081cc33d9046264fe24b)
+    
+    Conflicts:
+        NOTE: cherry-picked from kraken commit to avoid conflicts
+
+commit 62ce3461c3b205eaa9062113526cf572184d0a27
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jan 23 21:24:41 2017 -0500
+
+    test: use librados API to retrieve config params
+    
+    The CephContext object is not ABI-stable, so it is necessary to
+    use the "conf_get" librados methods to safely retrieve a setting.
+    
+    Fixes: http://tracker.ceph.com/issues/18617
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2ed02f3cd56bf89984c3538ac3f21ec2321cd3b7)
+    
+    Conflicts:
+            src/test/librbd/test_librbd.cc (jewel does not have
+                006138e2d80b779d8c15b141002bb4b3852f6c4a or
+                cb3712e08cdc2c37a983b479f4692bbdfe83b220)
+
+commit 01d04e28db7c2969b86df5b38a20b9eb156cf393
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Feb 2 23:23:54 2017 +0100
+
+    tests: Thrasher: eliminate a race between kill_osd and __init__
+    
+    If Thrasher.__init__() spawns the do_thrash thread before initializing the
+    ceph_objectstore_tool property, do_thrash races with the rest
+    of Thrasher.__init__() and in some cases do_thrash can call kill_osd() before
+    Trasher.__init__() progresses much further. This can lead to an exception
+    ("AttributeError: Thrasher instance has no attribute 'ceph_objectstore_tool'")
+    being thrown in kill_osd().
+    
+    This commit eliminates the race by making sure the ceph_objectstore_tool
+    attribute is initialized before the do_thrash thread is spawned.
+    
+    Fixes: http://tracker.ceph.com/issues/18799
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit b519d38fb1967628ad8a1c46fcfb3f984de58790)
+    
+    Conflicts:
+        qa/tasks/ceph_manager.py (jewel has only one if statement after
+            "self.thread = gevent.spawn(self.do_thrash)" while master has four;
+            jewel lacks 66836c957ffd974dec136997e23261ec7de2f0aa which disables
+            ceph-objectstore-tool testing in master)
+
+commit 08a667883b68ccc72e3a4bc3013856deef1df93d
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Feb 18 14:33:25 2017 +0100
+
+    rpm: build ceph-resource-agents by default
+    
+    To align with debian build
+    
+    Fixes: http://tracker.ceph.com/issues/17613
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 3e157bf16c3020ac11cb26df5df3ed331faf3c25)
+
+commit d22becab0f2c541584ce891d392760a5c4f1d153
+Author: Yan Jun <yan.jun8@zte.com.cn>
+Date:   Thu Jul 14 19:10:29 2016 +0800
+
+    msg/simple: cleanups
+    
+    should save the `errno` which may be changed by `ldout` and/or `<<` operator
+    
+    Signed-off-by: Yan Jun <yan.jun8@zte.com.cn>
+    (cherry picked from commit 91a29bc490fdfbbef0875fa620c7ba1a1a6492ae)
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit a18a2dd108678d2e4b57e08b559c1f9a262d6923
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Jun 17 13:58:55 2016 +0800
+
+    msg/simple: set close on exec on server sockets
+    
+    mds execv() when handling the "respawn" command, to avoid fd leakage,
+    and enormous CLOSE_WAIT connections after respawning, we need to set
+    FD_CLOEXEC flag for the socket fds.
+    
+    Fixes: http://tracker.ceph.com/issues/16390
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit f019ad563ce90f5aea0d8dd8b7b98688441596e0)
+
+commit 91a968b8fc7b363cae351b8648259211a1e71d18
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Jun 17 01:17:05 2016 +0800
+
+    msg/async: set close on exec on server sockets
+    
+    mds execv() when handling the "respawn" command, to avoid fd leakage,
+    and enormous CLOSE_WAIT connections after respawning, we need to set
+    FD_CLOEXEC flag for the socket fds.
+    
+    Fixes: http://tracker.ceph.com/issues/16390
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit eaf68c724144d07f9506037a14d9192cb9f16d70)
+    
+    Conflicts:
+            src/msg/async/AsyncMessenger.cc: Processor::accept(): applied
+                    the hunk manually (invoke set_close_on_exec on a socket
+                    returned by accept)
+
+commit 547e867628975c7144590e9332aa62b0ef82a433
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 5 12:12:57 2017 -0500
+
+    librbd: possible deadlock with flush if refresh in-progress
+    
+    Fixes: http://tracker.ceph.com/issues/18419
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b95f92a5572d3035c20eba07e76d2c825a9853f7)
+    
+    Conflicts:
+            src/librbd/ImageState.h (master commit just adds a function
+                          declaration, so just add it to jewel as well)
+
+commit 07501dec6f1c70afd4e4c2a50d7f874c39f2220b
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jan 18 17:02:54 2017 -0600
+
+    mon/OSDMonitor: make 'osd crush move ...' work on osds
+    
+    Currently it only allows you to move buckets, which is annoying and much
+    less useful.  To move an OSD you need to use create-or-move, which is
+    harder to use.
+    
+    Fixes: http://tracker.ceph.com/issues/18587
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 47956475dea8bb8e07331dd76344a60b776b5158)
+    
+    Conflicts:
+        qa/workunits/mon/crush_ops.sh: adapted "ceph osd find" to jewel syntax
+
+commit 7c6c3c753ccdd3baea834338e1a761f05b4e0a12
+Author: Vikhyat Umrao <vumrao@redhat.com>
+Date:   Thu Feb 16 23:51:11 2017 +0530
+
+    auth: 'ceph auth import -i' overwrites caps, if caps are not specified
+    in given keyring file, should alert user and should not allow this import.
+    Because in 'ceph auth list' we keep all the keyrings with caps and importing
+    'client.admin' user keyring without caps locks the cluster with error[1]
+    because admin keyring caps are missing in 'ceph auth'.
+    
+    [1] Error connecting to cluster: PermissionDeniedError
+    
+    Fixes: http://tracker.ceph.com/issues/18932
+    
+    Signed-off-by: Vikhyat Umrao <vumrao@redhat.com>
+    (cherry picked from commit 90144aa64c11a685b6a7cb3aafea75d427f569be)
+
+commit 8c7a1df251e8289e7cf2df5b3096b91d8640695d
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Feb 14 15:00:09 2017 -0500
+
+    osd/PG: restrict want_acting to up+acting on recovery completion
+    
+    On recovery completion we recalculate want_acting to see if we
+    should add recently backfilled osds into acting.  However, at
+    this point we may have gotten infos from others OSDs outside
+    of up/acting that could be used for want_acting.  We currently
+    assert that only up/acting osds are used in
+    PG::RecoveryState::Active::react(const AdvMap&), so we must
+    restrict want_acting to up/acting here.
+    
+    We could remove this restriction, but it would mean
+    
+    1) checking on every map change that want_acting hasn't been
+    invalidated, and if so, recalculating want_acting and requesting
+    a new pg_temp.  Also, presumably
+    
+    2) on each new info, checking whether we can construct a better
+    want_acting, and if so, doing it.
+    
+    That would be a good thing, but is a more complicated change.  In
+    reality this case comes up very rarely, so simply make our
+    post-recovery want_acting calculation limit itself to up+acting.
+    
+    See 1db67c443d84dc5d1ff53cc820fdfd4a2128b680 for the assertion.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 0f2dee9aa48a00a7f2f809cd4d20e98df771da81)
+
+commit cfa37d6a1674e3f6f8eef4d8519823a7af70df01
+Author: craigchi <craig10624@gmail.com>
+Date:   Thu Feb 16 19:21:48 2017 +0800
+
+    ceph-disk: Fix getting wrong group name when --setgroup in bluestore
+    
+    ceph-disk prepare --setgroup <GROUP NAME> will be wrong when using with
+    bluestore
+    
+    Signed-off-by: craigchi <craig10624@gmail.com>
+    (cherry picked from commit a8c0870e7370a0948e8e7fd53d3376b85bf9c649)
+
+commit 2d17092fab8080f819369d74d4c76d8ae58d899b
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Tue Feb 7 16:47:45 2017 +0400
+
+    ceph-osd: --flush-journal: sporadic segfaults on exit
+    
+    FileStore holds a number of recources like op thread pool and work
+    queue, key/value DB threads, etc. These should be properly stopped
+    (released) before exiting to avoid segfaults on exit.
+    
+    Note: more code paths (mkfs, dump_journal, etc) need similar fixes,
+    these will be submitted as separate patches.
+    
+    Fixes: http://tracker.ceph.com/issues/18820
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+    (cherry picked from commit 00184814c156f6194a6ba4b696073ca1c18a3f8f)
+    
+    Adjustments:
+     - release g_ceph_context in the same way as the main code path does
+
+commit d012c381e8c59994ea9a40dc006d23f1bdd6a026
+Author: Sebastien Ponce <sebastien.ponce@cern.ch>
+Date:   Tue May 10 11:27:59 2016 +0200
+
+    radosstriper : protect aio_write API from calls with 0 bytes
+    
+    an assertion was failing so far, while we only have to return without doing anything
+    
+    Signed-off-by: Sebastien Ponce <sebastien.ponce@cern.ch>
+    (cherry picked from commit 7cce1e8c51640f466d8bb37a21c0d5f1b00db8ab)
+
+commit 915dbace5ddea69fff29f7965f213229b6fbc0ac
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jan 17 11:55:00 2017 -0500
+
+    osdc: cache should ignore error bhs during trim
+    
+    A read error (such as injecting a timeout into an OSD op) might result
+    in a bh in an error state. These should be trimable by the cache.
+    
+    Fixes: http://tracker.ceph.com/issues/18436
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5910ed9de9856b5821488a1836487bbbd3d6460e)
+
+commit 419c9926d9ed57cb60228bc95956a9a1471b92cb
+Author: Piotr Dałek <piotr.dalek@corp.ovh.com>
+Date:   Tue Jan 31 16:07:18 2017 +0100
+
+    OSD: allow client throttler to be adjusted on-fly, without restart
+    
+    This patch allows the osd_client_message_cap and
+    osd_client_message_size_cap to be adjusted on-fly, using admin socket
+    functionality.
+    
+    Fixes: http://tracker.ceph.com/issues/18791
+    Signed-off-by: Piotr Dałek <piotr.dalek@corp.ovh.com>
+    (cherry picked from commit 64c309d7e18a975931b526e6f5d6f610c3a0d632)
+    
+    Conflicts:
+            src/osd/OSD.cc (suppressed post-jewel option)
+
+commit 957c19b844fb44cde78ad59f872815f82bbf23b8
+Author: Ali Maredia <amaredia@redhat.com>
+Date:   Thu Nov 10 13:58:35 2016 -0500
+
+    swift: added "--cluster" to rgw-admin command for multisite support
+    
+    Signed-off-by: Ali Maredia <amaredia@redhat.com>
+
+commit 8423bc1eefa45366bdd215a17c61701c9b05dfdd
+Merge: 6dc30c4140 c078534376
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Thu Apr 23 14:49:30 2015 -0500
+
+    Merge pull request #470 from ceph/wip-remote
+    
+    Add timeouts to Remote connection functions
+
+commit 6dc30c4140a833fd6cd126f8b5c1eceebad90510
+Merge: d55484f9e5 c078534376
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Mon Apr 20 12:38:29 2015 -0500
+
+    Merge pull request #466 from ceph/wip-11426
+    
+    Log stderr in get_latest_image_version_deb()
+
+commit d55484f9e562779e7d47a8f63ff029337dc01eef
+Merge: c078534376 cd72cf2b31
+Author: Dan Mick <dan.mick@redhat.com>
+Date:   Wed Apr 15 10:30:46 2015 -0700
+
+    Merge pull request #462 from ceph/wip-ssh-keys
+    
+    When modifying authorized_keys, store a backup
+
+commit c078534376d594aa3bf70d1d2e5dfc09ca8ae248
+Merge: cd72cf2b31 fce2ed683f
+Author: Zack Cerza <zack@cerza.org>
+Date:   Tue Apr 14 11:38:15 2015 -0600
+
+    Merge pull request #460 from zhouyuan/mkdir_p
+    
+    Make parent directories as needed
+
+commit cd72cf2b3132e7c6371aa4dbfe7564ad3ad0509e
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Nov 24 09:59:30 2014 -0800
+
+    swift: set full access to subusers creation
+    
+    Default subuser permissions are 'none'.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit fce2ed683fcd3798db968a40200f8e8f215595fa
+Author: Zack Cerza <zack.cerza@inktank.com>
+Date:   Wed Aug 6 10:06:34 2014 -0600
+
+    Remove most ceph-specific tasks. They are in ceph-qa-suite now.
+    
+    Signed-off-by: Zack Cerza <zack.cerza@inktank.com>
+
+commit fac452ae55594aea482db5c13a0bd0207b6ecff6
+Author: Zack Cerza <zack@cerza.org>
+Date:   Thu Mar 27 11:35:28 2014 -0500
+
+    Revert "Lines formerly of the form '(remote,) = ctx.cluster.only(role).remotes.keys()'"
+    
+    This reverts commit d693b3f8950ffd1f2492a4db0f8234fee31f00f0.
+
+commit e98b107302e06fa5c3c628a7ab0e7455de9ab568
+Author: Warren Usui <warren.usui@inktank.com>
+Date:   Fri Feb 28 19:13:40 2014 -0800
+
+    Lines formerly of the form '(remote,) = ctx.cluster.only(role).remotes.keys()'
+    and '(remote,) = ctx.cluster.only(role).remotes.iterkeys()' would fail with
+    ValueError and no message if there were less than 0 or more than 1 key.
+    Now a new function, get_single_remote_value() is called which prints out
+    more understandable messages.
+    
+    Fixes: 7510
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+    Signed-off-by: Warren Usui <warren.usui@inktank.com>
+
+commit e5fe884edfec66cc7e520938bd520aa8f1344f85
+Merge: 8e2cdbf5ed f8bf53c4fe
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Wed Feb 19 16:40:17 2014 -0500
+
+    Merge pull request #186 from ceph/wip-7369
+    
+    Fix #7369: "sed expression must be raw string"
+
+commit 8e2cdbf5ede871ebde260e6bdaec13daae03cfc5
+Merge: 132f3e8ae6 f8bf53c4fe
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 14 14:59:04 2014 -0600
+
+    Merge pull request #188 from ceph/wip-calamari-onefile
+    
+    Add Calamari test tasks, test script
+
+commit 132f3e8ae664a35cad89896b7300be13b5c604ec
+Merge: 24363351c3 7b63876676
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 14 11:50:55 2014 -0600
+
+    Merge pull request #192 from ceph/wip-6537-wusui
+    
+    Readjust the indentation of mon_clock_skew_check.py and mon_thrash.py.
+
+commit 24363351c3b1d23a8d6ef4e3f952855415f97ba8
+Merge: 4eb147291f 7b63876676
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 14 11:45:51 2014 -0600
+
+    Merge pull request #194 from ceph/wip-6534-wusui
+    
+    Add docstrings to internal.py
+
+commit 4eb147291fccdb2164a46e977ee43dbf50894e5a
+Merge: ad9aaf8fa3 f4284b520a
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 14 11:45:19 2014 -0600
+
+    Merge pull request #193 from ceph/wip-6538-wusui
+    
+    Add doc strings to Swift tests
+
+commit ad9aaf8fa35d49c33373fa69df7b38d3aca6abc5
+Merge: 7b63876676 f8bf53c4fe
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 14 11:43:15 2014 -0600
+
+    Merge pull request #187 from ceph/wip-better-debug
+    
+    Debug output improvements
+
+commit f4284b520a554b1cbe130731741e53a7fcf4e35f
+Author: Warren Usui <warren.usui@inktank.com>
+Date:   Thu Feb 13 21:11:34 2014 -0800
+
+    Add doc strings to Swift tests
+    
+    Fixes: 6538
+    Signed-off-by: Warren Usui <warren.usui@inktank.com>
+
+commit 7b63876676f1a6845ba3b9147cf7bb2348ef2468
+Author: Warren Usui <warren.usui@inktank.com>
+Date:   Tue Feb 11 20:21:06 2014 -0800
+
+    Add docstrings to s3 related tasks.
+    
+    Fixes: 6536
+    Signed-off-by: Warren Usui <warren.usui@inktank.com>
+
+commit f8bf53c4fe52009abdf730b05e2cb2ddbb412dea
+Author: Zack Cerza <zack@cerza.org>
+Date:   Tue Sep 24 14:19:24 2013 -0500
+
+    Fix namespace collision
+
+commit db6efe3e0ba7446ca42baf2a50eef18a10cf4a10
+Merge: 611733c8b5 66555a4039
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Tue Sep 24 08:17:22 2013 -0700
+
+    Merge pull request #106 from ceph/wip-mirror
+    
+    Remove lots of ceph.com hardcoding; default to upstream sources
+
+commit 66555a4039c61db9b96d6eecf8d2f298c98b6bad
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Sep 20 15:53:58 2013 -0500
+
+    Don't hardcode the git://ceph.com/git/ mirror
+    
+    Default to https://github.com/ceph/ but add a ~/teuthology.yaml option
+
+commit 611733c8b5ea55206df50c122efac612cb146c2f
+Merge: 2346f1d735 6e8a3807c7
+Author: Sage Weil <sage@inktank.com>
+Date:   Fri Sep 6 13:24:34 2013 -0700
+
+    Merge pull request #78 from ceph/wip-6247
+    
+    Move helper scripts to /usr/local/bin to clean up logs.
+
+commit 6e8a3807c766f728027c3099eebfa24cdc645bd1
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Sep 6 15:08:01 2013 -0500
+
+    Helper scripts live in /usr/local/bin now!
+
+commit 2346f1d735ccb40d00b8ff61d4acb446f684b3b2
+Author: Joe Buck <jbbuck@gmail.com>
+Date:   Fri Aug 23 19:54:53 2013 -0700
+
+    s3tests: extend for multi-region tests
+    
+    Added code to the s3tests task to extract
+    multi-region info so that that data
+    can be added to the S3TEST_CONF file
+    used to run S3 tests.
+    
+    Signed-off-by: Joe Buck <jbbuck@gmail.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit d3b6d633e161bc422f4781c7be5011f8a7ed8a32
+Merge: 442a36c57c 09b01b27a3
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Mon Aug 19 14:10:18 2013 -0700
+
+    Merge pull request #41 from ceph/wip-3791
+    
+    Various usability and documentation fixes
+
+commit 442a36c57c3173b36ce17e5f85a49abf7e80ea93
+Merge: 09b01b27a3 9b2c4fa4ad
+Author: wusui <warren.usui@inktank.com>
+Date:   Fri Aug 16 14:47:59 2013 -0700
+
+    Merge pull request #40 from ceph/wip-teutharm-wusui
+    
+    Wip teutharm wusui
+
+commit 09b01b27a3a1310d4257133def60896ad37fb575
+Author: Zack Cerza <zack@cerza.org>
+Date:   Thu Aug 15 08:49:35 2013 -0500
+
+    Fix some instances where print is being used instead of log
+
+commit 9b2c4fa4ad4c4258b26526afb0c16c71ce47f593
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed Jul 31 13:32:58 2013 -0700
+
+    s3/swift tests: call radosgw-admin as the right client
+    
+    This allows the right region and zone info to be read from ceph.conf
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 3b3816df58a3ba1f2f850faf8969ad070aa0046e
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Thu Jul 25 16:47:34 2013 -0700
+
+    s3tests: clone correct branch
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit f5170fb460a2183aed33122b4e6d0117a4220fec
+Merge: 7207a31e58 bd56af707a
+Author: Sandon Van Ness <sandon@inktank.com>
+Date:   Thu Jul 25 19:50:39 2013 -0700
+
+    Merge branch 'master' of github.com:ceph/teuthology
+
+commit 7207a31e5812dd0b29f2d6378360015622ddf4aa
+Merge: bd56af707a 3da945512e
+Author: Sandon Van Ness <sandon@inktank.com>
+Date:   Thu Jul 25 19:50:02 2013 -0700
+
+    Merge remote-tracking branch 'origin/wip-sandon-vm'
+    
+    Conflicts:
+            teuthology/lock.py
+            teuthology/misc.py
+            teuthology/task/install.py
+
+commit bd56af707ae50c98ec46344cf57eb333061847b1
+Merge: 3da945512e 343a42c0d8
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Fri Jul 19 14:44:51 2013 -0700
+
+    Merge branch 'wip-centos-rgw'
+
+commit 343a42c0d86af5b8630a30716c03fc84ba22f944
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Jul 9 18:50:52 2013 -0700
+
+    s3tests: fix client configurations that aren't dictionaries
+    
+    They're always used as dictionaries later on.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 3da945512ed78081be5dbe9ba59c836a311e1973
+Merge: 2c34d1971f 253cc98d98
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Jun 24 16:18:36 2013 -0700
+
+    Merge pull request #15 from ceph/wip-ulimits
+    
+    Reviewed-by: Warren Usui <warren.usui@inktank.com>
+
+commit 253cc98d98855d65be7ebcdd46a39aa1004f8e67
+Author: Sage Weil <sage@inktank.com>
+Date:   Sun Jun 23 09:15:28 2013 -0700
+
+    enable-coredump -> adjust-ulimits
+    
+    and set max_files to be big, too!
+
+commit 2c34d1971f1e82311b364bf8efe60b223158d676
+Merge: 61dba20d1f b366ad334a
+Author: Warren Usui <warren.usui@inktank.com>
+Date:   Tue May 7 19:27:51 2013 -0700
+
+    Merge branch 'wip-teuth4768a-wusui'
+    
+    Conflicts:
+            teuthology/task/install.py
+
+commit 61dba20d1fce774eac5b56f0d61d4229460875c0
+Merge: a9f3eb6310 5a7267f85c
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon May 6 21:31:36 2013 -0700
+
+    Merge branch 'next'
+
+commit a9f3eb631064931cbdde7ef218c16e154bdb9991
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu May 2 13:47:46 2013 -0700
+
+    s3tests: add force-branch with higher precdence than 'branch'
+    
+    This way we can force a branch despite something in overrides.
+    
+    Signed-off-by: Sage Weil <sage@inktank.com>
+
+commit b366ad334af55867ba781e22c8f87b6ac7775bf2
+Merge: 2a51e32891 5a7267f85c
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Wed May 1 09:52:02 2013 -0700
+
+    Merge remote branch 'origin/next'
+
+commit 5a7267f85c80f88aca1b0081b07de1de3909f2e7
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Apr 30 17:07:53 2013 -0700
+
+    fix some errors found by pyflakes
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit f866037f045887ccc5da15404935ce3361a74a08
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Apr 30 13:23:22 2013 -0700
+
+    s3tests: revert useless portion of 1c50db6a4630d07e72144dafd985c397f8a42dc5
+    
+    Perhaps it was attempting to debug something, but it shouldn't have been committed.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 2dcce57c00264b85cfb906223dfb89db9cc61ba5
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Apr 30 16:49:04 2013 -0700
+
+    rgw tests: remove users after each test
+    
+    These should all be cleanup up at some point. They're
+    almost all the same code.
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 3c604251d9014dba45f3adbb2008bac16ff6346d
+Author: Josh Durgin <josh.durgin@inktank.com>
+Date:   Tue Apr 30 16:47:34 2013 -0700
+
+    rgw tests: clean up immediately after the test
+    
+    There's no need for an explicit cleanup function, so move it back
+    to where it came from (except in s3roundtrip, which did not have it).
+    
+    Instead, since these use a nested contextmanager, pass through
+    and yield to the top-level run_tasks after the nested
+    contextmanager has finished (and thus run all the cleanup steps
+    in the subtasks for this test).
+    
+    Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 022bd4aa42312b80917169282cbfba655bbad6f1
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Tue Apr 30 07:06:03 2013 -0700
+
+    swift, s3readwrite: add missing yield
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 820c72b8d0177b01f69887ec64d98702db37077c
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Apr 29 11:24:04 2013 -0700
+
+    s3tests, s3readwrite, swift: cleanup explicitly
+    
+    Cleaning up test dir explicitly after run, so that
+    consecutive runs don't fail.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 2a51e328913f16917a881129b475a4aeeab24ed0
+Merge: cccadb9b03 617534e769
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Wed Feb 20 14:10:50 2013 -0800
+
+    Merge remote-tracking branch 'origin/wip-3634'
+
+commit cccadb9b03ca4421a5fd841a61bf252c329e3649
+Merge: 3eb19c8107 fa1f89478a
+Author: Sage Weil <sage@inktank.com>
+Date:   Tue Feb 19 21:04:24 2013 -0800
+
+    Merge branch 'unstable'
+    
+    Conflicts:
+            teuthology/task/workunit.py
+
+commit fa1f89478a76373cb33cf2524e2ebf68b3cd622c
+Author: Sander Pool <sander.pool@inktank.com>
+Date:   Wed Feb 6 19:16:52 2013 +0000
+
+    Install ceph debs and use installed debs
+    
+    The ceph task installs ceph using the debian
+    packages now, and all invocations of binaries installed
+    in {tmpdir}/binary/usr/local/bin/ are replace with
+    the use of the binaries installed in standard locations
+    by the debs.
+    
+    Author:    Sander Pool <sander.pool@inktank.com>
+    Signed-off-by: Sam Lang <sam.lang@inktank.com>
+
+commit 3eb19c810725b011baacdb8a6f5b172f4720a39a
+Author: Sam Lang <sam.lang@inktank.com>
+Date:   Wed Jan 23 14:37:39 2013 -0600
+
+    Replace /tmp/cephtest/ with configurable path
+    
+    Teuthology uses /tmp/cephtest/ as the scratch test directory for
+    a run.  This patch replaces /tmp/cephtest/ everywhere with a
+    per-run directory: {basedir}/{rundir} where {basedir} is a directory
+    configured in .teuthology.yaml (/tmp/cephtest if not specified),
+    and {rundir} is the name of the run, as given in --name.  If no name
+    is specified, {user}-{timestamp} is used.
+    
+    To get the old behavior (/tmp/cephtest), set test_path: /tmp/cephtest
+    in .teuthology.yaml.
+    
+    This change was modivated by #3782, which requires a test dir that
+    survives across reboots, but also resolves #3767.
+    
+    Signed-off-by: Sam Lang <sam.lang@inktank.com>
+    Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
+
+commit 2f829870e140c87b30e5b7aa3ad237a90dcb2179
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Fri Dec 21 10:20:02 2012 -0800
+
+    task/swift: change upstream repository url
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 334d6386753510c312898552c6f92313942786ef
+Merge: b8e6ce4db9 26df886d82
+Author: Joao Eduardo Luis <jecluis@gmail.com>
+Date:   Thu Nov 29 00:53:59 2012 +0000
+
+    Merge branch 'wip-mon-thrasher'
+
+commit b8e6ce4db9a603ce3523b1759c65eeadee55daa7
+Author: Sage Weil <sage@inktank.com>
+Date:   Thu Nov 22 13:59:58 2012 -0800
+
+    s3tests: fix typo
+
+commit 26df886d825e28c25b630887b8dcc1c8c6d687d8
+Author: Yehuda Sadeh <yehuda@inktank.com>
+Date:   Mon Nov 19 16:19:06 2012 -0800
+
+    rgw-logsocket: a task to verify opslog socket works
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
+
+commit 617534e76978acb09a9f925f18bba475a65a7dd2
+Author: Sage Weil <sage@inktank.com>
+Date:   Mon Sep 10 11:08:57 2012 -0700
+
+    s3tests: run against arbitrary branch/sha1 of s3-tests.git
+
+commit 7d5c7ee8c6f2bfedd193a8d3b7102b4cfe0bf74f
+Author: Sage Weil <sage@newdream.net>
+Date:   Wed Jun 6 16:00:55 2012 -0700
+
+    pull s3-tests.git using git, not http
+
+commit ce951cf4caffd5d6883cc8dcd24372bcdf03690a
+Author: Sage Weil <sage@newdream.net>
+Date:   Sat May 5 09:30:41 2012 -0700
+
+    ceph.newdream.net -> ceph.com
+
+commit 2b879905fcfd660e242ed1a804d1c8301d17ab84
+Merge: 1ac4bb10fc 1970713a2f
+Author: Mark Nelson <nhm@clusterfaq.org>
+Date:   Wed Mar 14 15:32:23 2012 -0500
+
+    Merge branch 'master' of github.com:ceph/teuthology
+
+commit 1970713a2fc43e3afae376712356ca93a65d9e1f
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Mar 2 10:55:19 2012 -0800
+
+    github.com/NewDreamNetwork -> github.com/ceph
+
+commit 1ac4bb10fc4b7d8d07c44b0e92b1627c721ab925
+Author: Josh Durgin <josh.durgin@dreamhost.com>
+Date:   Tue Feb 21 14:54:33 2012 -0800
+
+    Add necessary imports for s3 tasks, and keep them alphabetical.
+
+commit 92110e5a4460281139233dcea3f629d01182d398
+Author: Yehuda Sadeh <yehuda.sadeh@dreamhost.com>
+Date:   Tue Feb 21 12:12:03 2012 -0800
+
+    rgw: access key uses url safe chars
+    
+    Signed-off-by: Yehuda Sadeh <yehuda.sadeh@dreamhost.com>
+
+commit 709d9441127fec93da74c7702cafa54a47e10e8f
+Author: Sage Weil <sage@newdream.net>
+Date:   Sun Jan 15 22:48:33 2012 -0800
+
+    use local mirrors for (most) github urls
+    
+    A cronjob on ceph.newdream.net updates these every 15 minutes.  Sigh.
+
+commit 9598e47949ba65030c722947dc433e38875b1bd6
+Author: Tommi Virtanen <tommi.virtanen@dreamhost.com>
+Date:   Mon Dec 5 10:07:25 2011 -0800
+
+    Rename "testrados" and "testswift" tasks to not begin with "test".
+    
+    Anything "test*" looks like a unit test, and shouldn't be used for
+    actual code.
+
+commit 6236e7db22edac7b51fc6329188b6afa74f3fc78
+Author: Yehuda Sadeh <yehuda.sadeh@dreamhost.com>
+Date:   Thu Nov 17 16:53:21 2011 -0800
+
+    testswift: fix config
+
+commit 1dd607cabb07126769b4beb1ba6677e21c448719
+Author: Yehuda Sadeh <yehuda.sadeh@dreamhost.com>
+Date:   Wed Nov 16 16:00:01 2011 -0800
+
+    rgw: add swift task
+    
+    still not completely working (for some reason it skips all the tests)
+
+commit cb425c158085568cd92c239a071e282c74eddf1a
+Author: Greg Farnum <gregory.farnum@dreamhost.com>
+Date:   Fri Sep 30 09:26:42 2011 -0700
+
+    s3-tests: use radosgw-admin instead of radosgw_admin
+    
+    Signed-off-by: Greg Farnum <gregory.farnum@dreamhost.com>
+
+commit 37d7d515345ab04c333d6fada722e432e5816eb3
+Author: Tommi Virtanen <tommi.virtanen@dreamhost.com>
+Date:   Fri Sep 16 11:09:45 2011 -0700
+
+    s3tests: Clone repository from github.
+    
+    Signed-off-by: Tommi Virtanen <tommi.virtanen@dreamhost.com>
+
+commit 29a242d97dd4a5a9110710027cdddb244b8b0e29
+Author: Tommi Virtanen <tv@eagain.net>
+Date:   Tue Sep 13 14:53:02 2011 -0700
+
+    Move orchestra to teuthology.orchestra so there's just one top-level package.
+
+commit ec49a5f263f71aa473257e3fd49d86e475fe9456
+Author: Tommi Virtanen <tommi.virtanen@dreamhost.com>
+Date:   Fri Sep 9 13:22:03 2011 -0700
+
+    Callers of task s3tests.create_users don't need to provide dummy "fixtures" dict.
+
+commit d7d995e82b45e6077040b467c8ef9a82a573faf7
+Author: Stephon Striplin <stephon.striplin@dreamhost.com>
+Date:   Tue Aug 9 13:43:46 2011 -0700
+
+    allow s3tests.create_users defaults be overridden
+
+commit 0086109767d5bfbbc370ca13d3fe91895b207821
+Author: Josh Durgin <josh.durgin@dreamhost.com>
+Date:   Thu Jul 14 16:47:29 2011 -0700
+
+    Make targets a dictionary mapping hosts to ssh host keys.
+
+commit 1b2c96416f554c0890b2690291b9a2dc8a6dc17a
+Author: Tommi Virtanen <tommi.virtanen@dreamhost.com>
+Date:   Wed Jul 6 14:17:24 2011 -0700
+
+    Skip s3-tests marked fails_on_rgw, they will fail anyway.
+
+commit 06fb9b95e39985630d89e1635dcd12510686d9cd
+Author: Tommi Virtanen <tommi.virtanen@dreamhost.com>
+Date:   Tue Jul 5 09:27:28 2011 -0700
+
+    The shell exits after the command, hence there is no need for pushd/popd.
+
+commit cd524a6904bf8254edc73a9148308f642638e33d
+Author: Josh Durgin <josh.durgin@dreamhost.com>
+Date:   Fri Jun 24 17:09:47 2011 -0700
+
+    Add s3tests task.
diff --git a/doc/changelog/v10.2.9.txt b/doc/changelog/v10.2.9.txt
new file mode 100644
index 000000000..551f268ee
--- /dev/null
+++ b/doc/changelog/v10.2.9.txt
@@ -0,0 +1,63 @@
+commit 2ee413f77150c0f375ff6f10edd6c8f9c7d060d0
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Thu Jul 13 13:04:57 2017 +0000
+
+    10.2.9
+
+commit 9295f588535c45431d19b9601b4063c8de88752d
+Merge: 7b10d629ae fef1c8718f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jul 13 10:31:31 2017 +0200
+
+    Merge pull request #16282 from smithfarm/wip-20599-jewel
+    
+    jewel: cephfs: Damaged MDS with 10.2.8
+    
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+
+commit fef1c8718f77c190a0908d353f38b16b7c3832ab
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Jul 12 08:40:20 2017 +0200
+
+    Revert "osdc/Journaler: make header write_pos align to boundary of flushed entry"
+    
+    This reverts commit 2e299b50de4a297fee2aec21290632336d239857.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 3f89971e9edb88e313e1c190f7d05a83b52e6d91
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Jul 12 08:40:13 2017 +0200
+
+    Revert "osdc/Journaler: avoid executing on_safe contexts prematurely"
+    
+    This reverts commit 06cf9f365033f7913051bdf4060f0bc6fc0444d7.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 7b10d629ae32122b267486fe9e176f5cd0e330cf
+Merge: f5b1f1fd7c 6b479c275a
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Jul 11 15:58:04 2017 -0500
+
+    Merge pull request #16273 from smithfarm/wip-jewel-pending-release-notes
+    
+    jewel: doc: clarify status of jewel PendingReleaseNotes
+
+commit 6b479c275a24451c278074d81758a385eca12869
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Jul 11 22:53:56 2017 +0200
+
+    doc: zero PendingReleaseNotes in preparation for v10.2.9
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 55de93f9d711e13980168cc884dcb04d8849708e
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Jul 11 22:27:33 2017 +0200
+
+    doc: clarify status of jewel PendingReleaseNotes
+    
+    Status as of 10.2.8 release
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
diff --git a/doc/changelog/v11.2.1.txt b/doc/changelog/v11.2.1.txt
new file mode 100644
index 000000000..0decf8f65
--- /dev/null
+++ b/doc/changelog/v11.2.1.txt
@@ -0,0 +1,8020 @@
+commit e0354f9d3b1eea1d75a7dd487ba8098311be38a7
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Tue Aug 8 19:07:05 2017 +0000
+
+    11.2.1
+
+commit 11d5c2b0aae372f9c993a7dea5a914ee74e7480c
+Merge: 397b553932 6b428b77f2
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Aug 2 15:08:48 2017 +0200
+
+    Merge pull request #14702 from ceph/wip-bp-kraken-systemd
+
+    kraken: qa/tasks: misc systemd updates
+
+    Reviewed-by: Nathan Cutler <ncutler@suse.com
+
+commit 397b5539320236bffc66caab579422b1bd4570f7
+Merge: 0bab3f3a10 d24edde072
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Aug 2 07:55:48 2017 +0200
+
+    Merge pull request #15509 from ceph/wip-ceph-disk-fix-kraken
+
+    kraken: selinux: Do parallel relabel on package install
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 0bab3f3a1026c41736e06f72f8ede68b3386d772
+Merge: 646f2b3c26 a91253ec50
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 1 20:19:15 2017 +0200
+
+    Merge pull request #14612 from smithfarm/wip-19651
+
+    tests: backport Sage's fixes to qa/suites/upgrade/jewel-x
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 646f2b3c260ea5af3657d5f289753b114ba0d71e
+Merge: e9c255689f cf06edd36b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 1 14:39:37 2017 +0200
+
+    Merge pull request #16298 from smithfarm/wip-20517-kraken
+
+    kraken: rbd: cli: map with cephx disabled results in error message
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit e9c255689fcf2a7afe1deda3b987591162767daa
+Merge: 143e431d76 1fc14857a9
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 1 14:17:40 2017 +0200
+
+    Merge pull request #16175 from smithfarm/wip-20263-kraken
+
+    kraken: rgw: datalog trim can't work as expected
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 143e431d769ad68101df436735154c1230f3732b
+Merge: c9a545dba6 b758348447
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 1 14:16:54 2017 +0200
+
+    Merge pull request #15985 from prallabh/kraken
+
+    kraken: rgw: Custom data header support
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit c9a545dba68fe09e7aa971aec00c7e5aa1f217d2
+Merge: 245bf9205d 45b4c86452
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 1 14:15:33 2017 +0200
+
+    Merge pull request #16186 from smithfarm/wip-20264-kraken
+
+    kraken: rbd: [cli] ensure positional arguments exist before casting
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 245bf9205dc811b4bbe99e53bf20b32daeb85f8d
+Merge: 3d19b8acf0 f2d61c199d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 1 14:12:58 2017 +0200
+
+    Merge pull request #16342 from dillaman/wip-20630-kraken
+
+    kraken: tests: qa/tasks: rbd-mirror daemon not properly run in foreground mode
+
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 3d19b8acf0f76794c2b9c48f6ab386b6e5d14139
+Merge: a57772db99 f72ea68186
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 1 13:14:09 2017 +0200
+
+    Merge pull request #16290 from smithfarm/wip-19759-kraken
+
+    kraken: rgw: multisite: after CreateBucket is forwarded to master, local bucket may use different value for bucket index shards
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit a57772db99068c5e05b7b8cd1e346bdfb1fba292
+Merge: a1ff4e61af af0a6df270
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 1 13:12:37 2017 +0200
+
+    Merge pull request #16180 from smithfarm/wip-20347-kraken
+
+    kraken: rgw: meta sync thread crash at RGWMetaSyncShardCR
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit a1ff4e61aff4497e84d07070de984e34c6539bf0
+Merge: d0d98362e7 17fd055a4f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 1 12:47:45 2017 +0200
+
+    Merge pull request #16190 from smithfarm/wip-20026-kraken
+
+    kraken: mds: unresponsive when truncating a very large file
+
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit d0d98362e7363fd84334941085203aa3bfd7a2d2
+Merge: 72e5d6116c 719ed0101b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Aug 1 11:08:47 2017 +0200
+
+    Merge pull request #15526 from badone/wip-async-sleep-timer-fix-kraken
+
+    kraken: osd: Implement asynchronous scrub sleep
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 72e5d6116cc1afd83f96e26e32c5949548c1dcbc
+Merge: 16ca3c7f40 533ff8a540
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 31 16:47:45 2017 +0200
+
+    Merge pull request #16137 from smithfarm/wip-20024-kraken
+
+    kraken: tests: HEALTH_WARN pool rbd pg_num 244 > pgp_num 224 during upgrade
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 16ca3c7f40d37f1fb6e38665dd0c33fe3ab5a060
+Merge: a1c2cbd074 4d1272e5c6
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 31 15:31:12 2017 +0200
+
+    Merge pull request #14734 from smithfarm/wip-19670-kraken
+
+    kraken: build/ops: logrotate is missing from debian package (kraken, master)
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit a1c2cbd074dbc2954fea0d2808a80958f17fe9c3
+Merge: fd6816bb72 ccb33bab37
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 31 15:21:02 2017 +0200
+
+    Merge pull request #16166 from smithfarm/wip-19840-kraken
+
+    kraken: rgw: civetweb frontend segfaults in Luminous
+
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit a91253ec50dd5c2ab85b00d52361fd22580e1ed3
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Mar 7 22:35:02 2017 -0500
+
+    qa/suites/upgrade/jewel-x: do not thrash cluster full during upgrade
+
+    The mon command varies.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 39fdc53fe5f33678fbbd00cf8810b6d523d0040c)
+
+commit afb5f4b0b6eb0bbbfbdd628ff466e1fad6e6e4ef
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Mar 6 15:16:13 2017 -0500
+
+    qa/suites/upgrade/jewel-x/parallel: expand workload matrix
+
+    These should run independently against a racing upgrade.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1a0ad2b48839fea75d4d3339f7d198404ff1ac37)
+
+commit a88bd845df01b9fa8d1b0482b1c4d6bfae301f40
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Mar 6 14:11:53 2017 -0500
+
+    qa/suites/upgrade/jewel-x/stress-split-erasure-code: box thrashosds
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b2d354d56359d08b35cbea58f811c7bafb700d31)
+
+commit d4521fc2b9b13cee58abb4b578e8ba00c6aca0ae
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Mar 6 14:07:53 2017 -0500
+
+    qa/suites/upgrade/jewel-x/stress-split: finish client.0 upgrade too
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 56f9387736eed136b38c087a4805821063e9f8ab)
+
+commit 907888f881cc7224f9f9c17f38cc98773ca33906
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Mar 5 14:14:40 2017 -0500
+
+    qa/suites/upgrade/jewel-x: remove kraken references
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 468285b65d6bef2d86c80ebdfecf0920294ca5cd)
+
+commit 70415451e90d5aba473dbb3b1bd00ce4de0d077b
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Mar 1 19:01:20 2017 -0600
+
+    qa/suite/upgrade/jewel-x: drop x86 ec test
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1e0e53c80d8fa88970cee1d61aaa8514004547ac)
+
+commit fa7934b3abe395d52f127c6e170a54296d936211
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Feb 20 14:38:49 2017 -0500
+
+    qa/suites/upgrade/jewel-x: fix upgrade order; fix split grouping
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 269eafb2027ea6dfbad730f6fb1481a1cabc5e60)
+
+    Conflicts:
+        stress-split/6-kraken.yaml (do not delete)
+        stress-split/6-luminous.yaml (do not add)
+        stress-split-erasure-code/6-kraken.yaml (do not delete)
+        stress-split-erasure-code/6-luminous.yaml (do not add)
+
+commit b574cb341c9ce0f9d986c79c03951c7c9eafb0b8
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Mar 6 19:01:12 2017 -0500
+
+    qa/suites/upgrade/jewel-x/parallel: upgrade mons before osds
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3c80e15c3b34ac2adc4e70f09929e3bc01785594)
+
+commit 4eb3c2dbf321cf7a110b2ac7022d5d9cc80b8f3a
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Mar 1 13:58:07 2017 -0600
+
+    qa/suites/upgrade/jewel-x/parallel: fix upgrade to luminous
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2e3ea53b41cacc4b4bbc4ffe7342fb00233009cf)
+
+    Conflicts:
+       4-kraken.yaml (do not delete)
+       4-luminous.yaml (do not add)
+
+commit fd6816bb725f7699537276c13f68d815a7406228
+Merge: a6fdfccd11 b7503d3a80
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 31 12:32:45 2017 +0200
+
+    Merge pull request #13871 from smithfarm/wip-19162-kraken
+
+    kraken: rgw: rgw_file: fix marker computation
+
+commit a6fdfccd117180cf3906e1349b7af19462eae0e1
+Merge: b50909c861 a7af766aa8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 31 12:26:47 2017 +0200
+
+    Merge pull request #16133 from smithfarm/wip-18378-kraken
+
+    kraken: msg/simple/SimpleMessenger.cc: 239: FAILED assert(!cleared)
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit b50909c861450f1aeb40d3fa1a0ca4d997d09a0c
+Merge: 938b723717 c6542ac7e6
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 31 12:23:08 2017 +0200
+
+    Merge pull request #16134 from smithfarm/wip-19340-kraken
+
+    kraken: An OSD was seen getting ENOSPC even with osd_failsafe_full_ratio passed
+
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 17fd055a4f68cb3c51baa1ac1fb089d15b818312
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Apr 25 16:21:24 2017 +0800
+
+    osdc/Filer: truncate large file party by party
+
+    Fixes: http://tracker.ceph.com/issues/19755
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 5fab215e461e5ecc36c0f9d9ea867f6c45e80263)
+
+    Conflicts:
+            src/osdc/Filer.h
+            src/osdc/Filer.cc
+            src/mds/MDCache.cc
+
+commit 938b723717745eff4e3ab18b9a0f11cfc5190876
+Merge: e56d4c481d b8dfa2f73a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 31 11:58:20 2017 +0200
+
+    Merge pull request #16131 from smithfarm/wip-swift-kraken
+
+    tests: swift.py: clone the ceph-jewel branch
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 719ed0101b1bfdd4b71ef84101515492597153f9
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Mon May 22 13:21:25 2017 +1000
+
+    osd: Move scrub sleep timer to osdservice
+
+    PR 14886 erroneously creates a scrub sleep timer for every pg resulting
+    in a proliferation of threads. Move the timer to the osd service so
+    there can be only one.
+
+    Fixes: http://tracker.ceph.com/issues/19986
+
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit f110a82437df79dc20207d296e8229fc0e9ce18b)
+
+commit 460a820a3b2fbd48c8a7966502b235aae8d5d298
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Mon Apr 24 14:10:47 2017 +1000
+
+    osd: Implement asynchronous scrub sleep
+
+    Rather than blocking the main op queue just do an async sleep.
+
+    Fixes: http://tracker.ceph.com/issues/19497
+
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 7af3e86c2e4992db35637864b83832535c94d0e6)
+
+commit e56d4c481dead2f8ccb7baaae80db2f4acfc7bf0
+Merge: 024272160d bdb16fd0f5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jul 25 07:14:37 2017 +0200
+
+    Merge pull request #16493 from smithfarm/wip-suppress-upgrade-fail
+
+    tests: run certain upgrade/jewel-x tests on Xenial only
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Gregory Farnum <gfarnum@redhat.com>
+
+commit 024272160de37c3ed2a2df701fc1e8f4c11c2da5
+Merge: d8e0ddc57b 838d7840f0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jul 25 07:13:48 2017 +0200
+
+    Merge pull request #14597 from gregsfortytwo/wip-kraken-snaptrim
+
+    kraken: core: improve control and throttling of the snap trimmer
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit bdb16fd0f533691b7b7a01b176764ad907143814
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Jul 22 10:13:22 2017 +0200
+
+    tests: upgrade/jewel-x/parallel: drop duplicate kraken.yaml
+
+    This yaml file has the effect of re-running "ceph osd set require_kraken_osds"
+    at the very end of the test. Drop it.
+
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit dc8c2231cefdbc8811cdea380d7c856b57c62251
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sat Jul 22 00:06:48 2017 +0200
+
+    tests: run certain upgrade/jewel-x tests on Xenial only
+
+    This PR drops two upgrade/jewel-x test cases that are not compatible with
+    https://github.com/ceph/ceph/pull/14597
+
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit b8dfa2f73aff0998a4aebf6f61865983f476b8d1
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Jun 25 09:27:47 2017 +0200
+
+    tests: swift.py: clone the ceph-jewel branch
+
+    The master branch of ceph/swift.git contains tests that are incompatible with
+    Jewel and Hammer. The ceph-jewel branch omits these tests.
+
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit a86ce728954a765797ce634025d43650d990e480)
+
+    Conflicts:
+        qa/tasks/swift.py: clone ceph-kraken branch instead of ceph-jewel
+
+commit d8e0ddc57b0e2632baf70bebf409d52808c202fa
+Merge: c539ea49a8 19c7524ace
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 21 08:22:25 2017 +0200
+
+    Merge pull request #16111 from smithfarm/wip-20497-kraken
+
+    kraken: tests: insufficient timeout in radosbench task
+
+    Reviewed-by: Gregory Farnum <gfarnum@redhat.com>
+
+commit af0a6df270f3c469280cae877c177086ddf10245
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Jul 6 20:17:49 2017 +0200
+
+    rgw: lease_stack: use reset method instead of assignment
+
+    It seems that the intent of 45877d38fd9a385b2f8b13e90be94d784898b0b3 was to
+    change all instances of "lease_stack = ..." to "lease_stack.reset(...)", but
+    this one was missed.
+
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit fc425afeb2e2a6ba3c98c612b3977aea619c9f73)
+
+    Conflicts:
+            src/rgw/rgw_sync.cc (trivial)
+
+commit c539ea49a8dab85bade53140d80f2a05f3f1860c
+Merge: 0a9badb3e2 640a7a2629
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jul 20 16:38:27 2017 +0200
+
+    Merge pull request #16178 from smithfarm/wip-20268-kraken
+
+    kraken: rgw: get wrong content when download object with specific range when compression was enabled
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit d24edde072ec63d35965e30a58aaaf02547f9c57
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri Jul 7 12:37:55 2017 +0200
+
+    rpm: Fix undefined FIRST_ARG
+
+    If FIRST_ARG is undefined, the rpms will show an error on upgrade
+    because the condition in the postun script gets broken.
+
+    This was a regression introduced by commit for issue 20077 that moved
+    ceph-disk unit files to ceph-base.
+
+    Fixes: http://tracker.ceph.com/issues/20077
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 562816914ccca8e4e7d9c31f333db2f0da6f7c99)
+
+commit eac6a0dd475e22512cc24bb3b8fe018c70cf7bc8
+Author: Boris Ranto <branto@redhat.com>
+Date:   Mon Jun 5 18:44:18 2017 +0200
+
+    selinux: Install ceph-base before ceph-selinux
+
+    We need to have ceph-base installed before ceph-selinux to use ceph-disk
+    in %post script. The default ordering is random and so the installation
+    randomly failed to relabel the files.
+
+    Fixes: http://tracker.ceph.com/issues/20184
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit e69086645d3411a2ed781609b670eb5f16ac4810)
+    Conflicts:
+            ceph.spec.in: No _epoch_prefix in kraken.
+
+commit 0a9badb3e2be41be5e7de6d865a8e323b3706fa8
+Merge: 13b04089ce baa772372c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 23:09:32 2017 +0200
+
+    Merge pull request #16114 from smithfarm/wip-20500-kraken
+
+    kraken: cephfs: src/test/pybind/test_cephfs.py fails
+
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 13b04089ceff7136785c37595f526841574cdcd9
+Merge: 34c32b9d07 b941ca7eac
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 23:03:19 2017 +0200
+
+    Merge pull request #16108 from smithfarm/wip-19763-kraken
+
+    kraken: cephfs: non-local quota changes not visible until some IO is done
+
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 34c32b9d0721407511a8420ad3888cbee2fd8f5f
+Merge: bb95e12790 4241a6eef0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 23:02:09 2017 +0200
+
+    Merge pull request #16107 from smithfarm/wip-19710-kraken
+
+    kraken: mds: enable daemon to start when session ino info is corrupt
+
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit bb95e127909052679a883375feb55935ccb127d5
+Merge: 8202ddcf11 0b1dee0edc
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 23:01:01 2017 +0200
+
+    Merge pull request #16106 from smithfarm/wip-19680-kraken
+
+    kraken: mds: damage reporting by ino number is useless
+
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 8202ddcf1156093c58a0c161d6cdc3455fc440b5
+Merge: e2efa6e9e7 f458d60838
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 23:00:13 2017 +0200
+
+    Merge pull request #16105 from smithfarm/wip-19678-kraken
+
+    kraken: cephfs: ceph-fuse does not recover after lost connection to MDS
+
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit e2efa6e9e742dba2b9779e8b7382c173abfea0af
+Merge: 055319bfe7 7674f84ddf
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 22:58:56 2017 +0200
+
+    Merge pull request #16104 from smithfarm/wip-19676-kraken
+
+    kraken: cephfs: Test failure: test_data_isolated (tasks.cephfs.test_volume_client.TestVolumeClient)
+
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 055319bfe7e6dd15417e30b708590a4040d2a884
+Merge: 5c6a770308 bee73d2429
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 22:56:04 2017 +0200
+
+    Merge pull request #16103 from smithfarm/wip-19674-kraken
+
+    kraken: cephfs: mds is crushed, after I set about 400 64KB xattr kv pairs to a file
+
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 5c6a770308200788231173c270833a15d04765ed
+Merge: 8d86192e34 32c7ebe3e6
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 22:55:17 2017 +0200
+
+    Merge pull request #16102 from smithfarm/wip-19672-kraken
+
+    kraken: mds: assert fail when shutting down
+
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 8d86192e34dfd3d91a99a354545fb3e5b7b94f79
+Merge: 99d0afda7f b267a1a8be
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 22:54:07 2017 +0200
+
+    Merge pull request #16101 from smithfarm/wip-19669-kraken
+
+    kraken: mds: daemon goes readonly writing backtrace for a file whose data pool has been removed
+
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 99d0afda7f8748b13940267c3d5286b8a98d5a8f
+Merge: 818ecc66c7 93e81a87ac
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 22:53:11 2017 +0200
+
+    Merge pull request #16100 from smithfarm/wip-19667-kraken
+
+    kraken: cephfs: mount point break off problem after mds switch
+
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 818ecc66c71e3ccb351353a36b32b3dd0c6c2123
+Merge: c8ccfc72b3 e72d6362c1
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 22:05:12 2017 +0200
+
+    Merge pull request #16099 from smithfarm/wip-19664-kraken
+
+    kraken: mds: C_MDSInternalNoop::complete doesn't free itself
+
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit c8ccfc72b3556969722c24eacdd78828a3194b7f
+Merge: fbcef7d42a 3c4a5ea385
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 22:03:52 2017 +0200
+
+    Merge pull request #14998 from jan--f/wip-19845-kraken
+
+    kraken: cephfs: normalize file open flags internally used by cephfs
+
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit fbcef7d42a3832c2e404aa0d141238252045d3d9
+Merge: dbaae75029 f32e26e909
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 16:51:05 2017 +0200
+
+    Merge pull request #16168 from smithfarm/wip-20031-kraken
+
+    kraken: rgw: Swift's at-root features (/crossdomain.xml, /info, /healthcheck) are broken
+
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit dbaae750299d9169e2e1dda9d0af98521c8abb5e
+Merge: e26575be67 e773b304ee
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 16:15:21 2017 +0200
+
+    Merge pull request #16164 from smithfarm/wip-19777-kraken
+
+    kraken: rgw: swift: disable revocation thread under certain circumstances
+
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit e26575be67e00dbf674ff77ef6e7729f22750d8b
+Merge: 415174d29e 53024570e7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 15:43:16 2017 +0200
+
+    Merge pull request #16191 from smithfarm/wip-20028-kraken
+
+    kraken: cephfs: Deadlock on two ceph-fuse clients accessing the same file
+
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+
+commit 415174d29e20f73884e81773fd0b4d1055d439b4
+Merge: 2879b80281 b27595ae9c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 15:32:52 2017 +0200
+
+    Merge pull request #16183 from smithfarm/wip-20405-kraken
+
+    kraken: rgw: Lifecycle thread will still handle the bucket even if it has been removed
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+    Reviewed-by: Daniel Gryniewicz <dang@redhat.com>
+
+commit 2879b80281c1d4bd8b8daeb5662879a296274fca
+Merge: eed9a165bd 3105e2327e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 15:31:50 2017 +0200
+
+    Merge pull request #16181 from smithfarm/wip-20363-kraken
+
+    kraken: rgw: VersionIdMarker and NextVersionIdMarker are not returned when listing object versions
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit eed9a165bd1dd95f5afe8ccd78a742fbf16a4503
+Merge: b759d4362b d0e742cafa
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 15:31:14 2017 +0200
+
+    Merge pull request #16179 from smithfarm/wip-20269-kraken
+
+    kraken: rgw: wrong object size after copy of uncompressed multipart objects
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit b759d4362b7a3ae2bc4174eb0ef328f919644e6c
+Merge: 82e9c73fe7 90288afc10
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 15:30:33 2017 +0200
+
+    Merge pull request #16174 from smithfarm/wip-20261-kraken
+
+    kraken: rgw: 'radosgw-admin usage show' listing 0 bytes_sent/received
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 82e9c73fe70b5e78bc170ad566fa7900191ece70
+Merge: 2c5b063f43 dda4d912cd
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 15:29:57 2017 +0200
+
+    Merge pull request #16173 from smithfarm/wip-20156-kraken
+
+    kraken: fix: rgw crashed caused by shard id out of range when listing data log
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 2c5b063f4386663f00ff897beee0523fa6e95acd
+Merge: 423d6c319f f8235c5c0a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 15:29:22 2017 +0200
+
+    Merge pull request #16165 from smithfarm/wip-19839-kraken
+
+    kraken: rgw: reduce log level of 'storing entry at' in cls_log
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 423d6c319f55fb3eddc0602954645b3b1f1d7bd8
+Merge: 0af35a358e 68a853a14d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 15:28:30 2017 +0200
+
+    Merge pull request #16163 from smithfarm/wip-19766-kraken
+
+    kraken: rgw: when uploading the objects continuesly in the versioned bucket, some objects will not sync
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 0af35a358ea8a8b3376faf4d77bd75edb1e808fe
+Merge: 872a8b53ea 9a7a73edf1
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 15:27:38 2017 +0200
+
+    Merge pull request #16162 from smithfarm/wip-19725-kraken
+
+    kraken: rgw: S3 v4 authentication issue with X-Amz-Expires
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 872a8b53ea022923e9c442011432df5e403520ae
+Merge: 129345c319 d52aeec031
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 15:25:48 2017 +0200
+
+    Merge pull request #16161 from smithfarm/wip-19614-kraken
+
+    kraken: multisite: rest api fails to decode large period on 'period commit'
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 129345c3196effdaea08f9678d1a9f1cd8265d81
+Merge: 72c3a04f23 91569f6385
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 15:24:59 2017 +0200
+
+    Merge pull request #16139 from smithfarm/wip-20147-kraken
+
+    kraken: rgw: 'gc list --include-all' command infinite loop the first 1000 items
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 72c3a04f23a227713f50527b8b8651580556d45a
+Merge: 762901080c 6d2f959735
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 14:29:31 2017 +0200
+
+    Merge pull request #16096 from smithfarm/wip-19336-kraken
+
+    kraken: rbd: refuse to use an ec pool that doesn't support overwrites
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 762901080c3e7982a0f38705cd09578be1c6d940
+Merge: ac8bd5ea97 5dab0825d7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 14:28:08 2017 +0200
+
+    Merge pull request #16097 from smithfarm/wip-19609-kraken
+
+    kraken: tests: [librados_test_stub] cls_cxx_map_get_XYZ methods don't return correct value
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit ac8bd5ea97b7f87533d0320048335f80dcc66224
+Merge: c5d53bd299 ac706f2b4a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 14:27:11 2017 +0200
+
+    Merge pull request #16184 from smithfarm/wip-20154-kraken
+
+    kraken: rbd: Potential IO hang if image is flattened while read request is in-flight
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit c5d53bd299807d90d3311625554e055696898e71
+Merge: f73412843d 18f1830e8a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 14:24:05 2017 +0200
+
+    Merge pull request #16187 from smithfarm/wip-20266-kraken
+
+    kraken: rbd: [api] is_exclusive_lock_owner shouldn't return -EBUSY
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit f73412843db6cb9e3416d66a2ddf92e1b8d7038f
+Merge: 0d01813a8c ce874ab676
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 14:22:57 2017 +0200
+
+    Merge pull request #16195 from dillaman/wip-20351-kraken
+
+    kraken: tests: test/librbd: decouple ceph_test_librbd_api from libceph-common
+
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 0d01813a8c72e0d797d100e8e45d99f1ff99ba5a
+Merge: 771ebef418 379309b552
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 12:56:47 2017 +0200
+
+    Merge pull request #15486 from dillaman/wip-20022-kraken
+
+    kraken: rbd-mirror: ensure missing images are re-synced when detected
+
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 771ebef418eb0d585ca9bd9dd565ea9f1caf3766
+Merge: 9d6e0f2ef8 3173da6e3b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 12:50:22 2017 +0200
+
+    Merge pull request #14540 from smithfarm/wip-18910-kraken
+
+    kraken: rbd-nbd: check /sys/block/nbdX/size to ensure kernel mapped correctly
+
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 9d6e0f2ef86b34b0bd2443f8c8a747bb15732e7c
+Merge: bb1b7c3b10 9a26882289
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 19 08:05:11 2017 +0200
+
+    Merge pull request #16140 from smithfarm/wip-20271-kraken
+
+    kraken: tests: LibRadosMiscConnectFailure.ConnectFailure hang
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit bb1b7c3b1060b983eeb8e3a3f9607306b47ead77
+Merge: 5ea47f79c9 76fedab576
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jul 18 23:14:59 2017 +0200
+
+    Merge pull request #16112 from smithfarm/wip-20499-kraken
+
+    kraken: tests: ObjectStore/StoreTest.OnodeSizeTracking/2 fails on bluestore
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 5ea47f79c91229b57981bac77472102c868cfa89
+Merge: 1913303660 47f751f48b
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Jul 18 13:58:14 2017 -0700
+
+    Merge pull request #14960 from yehudasa/wip-19704
+
+    kraken: civetweb: move to post 1.8 version
+
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 838d7840f023c5802507727231983311182a1c05
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Mon Jul 10 13:33:26 2017 -0700
+
+    osd: do not default-abort on leaked pg refs
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 4caf2df0c380a1281db9509b3feb342705512b58)
+
+commit 4bf34200896ce120bba2e2d974c4f3dadb342821
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Thu May 25 22:14:38 2017 -0700
+
+    osd: shutdown our reserver_finisher earlier in the process
+
+    This finisher thread has a lot of callbacks which can hold PGRefs. Make
+    sure we drain them out before checking that all the PGs have finished
+    and have no outstanding references.
+
+    Moving this should be safe; we've already stopped the op thread et al
+    and the only things still running are the OSDService's objecter_finisher,
+    recovery_request_timer, and snap_sleep_timer (which has definitely been emptied
+    by the time we get here as it's synchronously cleared out on PG shutdown).
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 66ea9c1f66ae31035e62bd4335b08948b1e8e5e2)
+
+commit 2d5cafdf6461c213aa2642210a3c4efabce61efd
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Thu May 25 21:52:49 2017 -0700
+
+    osd: Reset() the snaptrimmer on shutdown
+
+    We were failing to exit various wait states which held PGRefs. Error!
+
+    Fixes: http://tracker.ceph.com/issues/19931
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit b0e9deeea8a8e90f6d7e9d56b6b4aed890e01d7b)
+
+commit cc0046a999600b2e684c17fddaebcf451c259443
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Wed May 24 18:43:34 2017 -0700
+
+    osd: print out pg debug ref counts on acquire/drop
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit a3b028a6d1ba74ed975ebd665b1b50fb7e5039a4)
+
+commit c6542ac7e6408f26ce7376d3a58eed0661a17099
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Jul 7 10:53:41 2017 -0700
+
+    mon: Fix status output warning for mon_warn_osd_usage_min_max_delta
+
+    Fixes: http://tracker.ceph.com/issues/20544
+
+    Caued by: 489e810c37ed6fb9d32d1015634041a577501ee4
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 56f9808016ab289bdc0ce7cfbb0503c78b509593)
+
+    Conflicts:
+        src/mon/PGMap.cc:
+            kraken has no daa0793c393a21bd2dd2ec52a0efd181e1032400 so we manually
+            apply the changes from 56f9808016ab289bdc0ce7cfbb0503c78b509593 to
+            src/mon/PGMonitor.cc instead of to src/mon/PGMap.cc
+
+commit 2f26f8cbd3e81700c42202f8d933f1576772e212
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Apr 18 13:54:56 2017 -0400
+
+    mon/PGMonitor: clean up min/max span warning
+
+    Clean up option naming.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 489e810c37ed6fb9d32d1015634041a577501ee4)
+
+commit 90b7fd8a0accc2f8f458d35199361344652d8859
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Feb 17 12:27:36 2017 -0800
+
+    bluestore: Fix BlueStore::statfs available amount to not include bluefs min reserved
+
+    This fixes OSD crashes because checking osd_failsafe_full_ratio won't work
+    without accurate statfs information.
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 72d83f848a35e8831d66e8529c4e26f51e845da6)
+
+commit d69e4e5e6ad633e38fc62ca0859cea1f146744b1
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 16 22:23:06 2017 -0800
+
+    osd: Round fullness in message to correspond to df -h
+
+    This really only works after journal drains because
+    we adjust for the journal.
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 26dcb591f9af01ed444aa758c3d601e7f67261b2)
+
+commit eb887e3cb1f55f016ca70ac3d4441aa03478d934
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Feb 16 17:25:12 2017 -0800
+
+    filestore: Account for dirty journal data in statfs
+
+    Fixes: http://tracker.ceph.com/issues/16878
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 78adb70c21c6b8e6a9191b76917919b125a9490f)
+
+commit bc324e0446d5fd6b35651b024926c5a3099a0aff
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Feb 14 16:37:07 2017 -0800
+
+    mon: Add warning if diff in OSD usage > config mon_warn_osd_usage_percent (10%)
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit c8004e6558359fb542e45bb4b483a6c91afdc0b4)
+
+commit 2c2e0a353bd518f1e44435376ad1ad9a0ee6fe67
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Feb 14 14:40:05 2017 -0800
+
+    mon: Bump min in ratio to 75%
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 830cc7aa7be1ccd8f54f056b6a58e923cadd1c2b)
+
+commit 899f3fca7f89ab37d20cb1ec06b911ba039c52a3
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Feb 14 14:38:53 2017 -0800
+
+    osd: Fix ENOSPC crash message text
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 50cfe03fcba253c8380b21043ed03879134d6836)
+
+commit 19133036602d9dc9eb234b5e4dec6b845b4ade4b
+Merge: 7224ebc1f1 85cefa113b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jul 18 19:50:46 2017 +0200
+
+    Merge pull request #16143 from smithfarm/wip-20365-kraken
+
+    kraken: mon: osd crush set crushmap need sanity check
+
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 7224ebc1f18fec1f472688f6494ed10d2b574f88
+Merge: 47359c12e6 ea186807c3
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jul 18 19:48:51 2017 +0200
+
+    Merge pull request #16138 from smithfarm/wip-20034-kraken
+
+    kraken: ceph-disk: Racing between partition creation & device node creation
+
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 47359c12e69fbde4c36403ccd7992fe4ecf9223b
+Merge: 9860e65818 f04eb66d36
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jul 18 19:47:34 2017 +0200
+
+    Merge pull request #16135 from smithfarm/wip-20010-kraken
+
+    kraken: ceph-disk: separate ceph-osd --check-needs-* logs
+
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f2d61c199d7ba103290178573984db98e8c8ed21
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jul 14 10:32:28 2017 -0400
+
+    qa/tasks: rbd-mirror daemon not properly run in foreground mode
+
+    Fixes: http://tracker.ceph.com/issues/20630
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4fa1918717b25a5ffafbf649eedcfe7d5ab829c2)
+
+commit cf06edd36b176a10e6ce3d62eb957bfdb0146703
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jun 29 14:54:40 2017 -0400
+
+    rbd: do not attempt to load key if auth is disabled
+
+    Fixes: http://tracker.ceph.com/issues/19035
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8b9c8df6d7f0b75c5451953bb322bc1f9afb6299)
+
+commit f72ea6818691e1824d4c22630e856d3b280a008c
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Tue May 9 15:05:03 2017 +0800
+
+    rgw: when create_bucket use the same num_shards with info.num_shards
+
+    pr #14388 only fix the num_shards in BucketInfo, "init_bucket_index" function still use local num_shards
+
+    Fixes: http://tracker.ceph.com/issues/19745
+
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit 4ce64a190b4ff36985e785e574c077d39796feea)
+
+    Conflicts:
+            src/rgw/rgw_rados.cc - init_bucket_index() called earlier
+
+commit 9182c279cf2f2228471d506113c61566cbfea4bb
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Fri Apr 7 15:34:27 2017 +0800
+
+    rgw: using the same bucket num_shards as master zg when create bucket in secondary zg
+
+    create bucket in secondary zonegroup will forward to master. The master may have different num_shards option.
+    So when create bucket in local, should use master's num_shards instead of local num_shards option.
+
+    Fixes: http://tracker.ceph.com/issues/19745
+
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit a34c4b8fb13dd5590eb3c6ecb5e55207ed8e3ee8)
+
+    Conflicts:
+        src/rgw/rgw_op.cc - no RGWBulkUploadOp:: methods in kraken; modifications
+            to RGWBulkUploadOp::handle_dir() omitted
+
+commit b758348447e60af23b114aa47c28f151ffd97792
+Author: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+Date:   Fri Apr 14 21:42:45 2017 +0530
+
+    rgw: add a field to store generic user data in the bucket index,
+    that can be populated/fetched via a configurable custom http header
+
+    Signed-off-by: Pavan Rallabhandi <prallabhandi@walmartlabs.com>
+    (cherry picked from commit abca7a86c3cfbb58fafb5d057d9d6f5017a53704)
+
+    Conflicts:
+            src/rgw/rgw_op.cc
+                    Signature fixes for RGWPutObjProcessor_Multipart::do_complete().
+
+            src/rgw/rgw_op.h
+                    A new member field `crypt_http_responses` is added in the class `RGWPutObj` in master version, which is not required in Kraken.
+
+            src/rgw/rgw_rados.cc
+                    In RGWRados::Bucket::UpdateIndex::complete, RGWObjEnt has been removed in master, which has to be retained in Kraken.
+                    In RGWRados::cls_obj_complete_op, user_data is added to the rgw_bucket_dir_entry_meta
+                    In RGWRados::cls_bucket_list, the user_data field of RGWObjEnt is populated.
+
+            src/rgw/rgw_rados.h
+                    In UpdateIndex::complete(), remove_objs is of type rgw_obj_key in Kraken instead of rgw_obj_index_key
+                    RGWPutObjProcessor_Multipart is not part of this file in Kraken.
+
+            src/rgw/rgw_rest_swift.cc
+                    In RGWListBucket_ObjStore_SWIFT::send_response(), there is no meta struct in Kraken.
+
+            src/rgw/rgw_common.h
+                    Add user_data field in RGWObjEnt structure.
+
+            src/rgw/rgw_json_enc.cc
+                    Add user_data field while dumping RGWObjEnt.
+
+commit 4fa19ce669338bb2d4c3233c70da8a3829a468f6
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Mon Jul 10 13:31:21 2017 -0700
+
+    osd: repair the PG_DEBUG_REFS build
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit ec4185d1980b48b01687a177248d0894f326dc37)
+
+    Conflicts:
+            src/osd/OSD.cc
+            src/osd/Session.h
+
+commit 2a3e66c388892efcecebd8eb3d7bf082be485bab
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Wed May 24 00:19:51 2017 -0700
+
+    osd: use PGRef instead of boost::intrusive_ptr<PG> everywhere
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 306ad85f0fb4597ba3bb3c04a67abdaba556ba74)
+
+commit 9860e658189e3ac775102b3f66676b0a77a80ba7
+Merge: 6e1023ad24 c65eaaafb1
+Author: Sage Weil <sage@newdream.net>
+Date:   Sat Jul 8 21:39:59 2017 -0500
+
+    Merge pull request #16215 from liewegas/wip-17743
+
+    kraken: CMakeLists.txt: disable memstore make check test
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit c65eaaafb174ac1f62fe8fc08ae63deadeffd0ad
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jul 7 11:38:25 2017 -0400
+
+    CMakeLists.txt: disable memstore make check test
+
+    This fails frequently on kraken, but is probably a wontfix.
+
+    See http://tracker.ceph.com/issues/17743
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 379309b552db51dcab9b08ba5437c1dae5cd45a7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed May 3 21:36:21 2017 -0400
+
+    rbd-mirror: ensure missing images are re-synced when detected
+
+    Fixes: http://tracker.ceph.com/issues/19811
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 74bd4f230a0cb7b709f2cb5c6db3dc79f0d8dede)
+
+    Conflicts:
+            src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc: trivial resolution
+            src/tools/rbd_mirror/image_replayer/BootstrapRequest.h: trivial resolution
+
+commit 6e1023ad2482f5829c06a54adb4271643b1e53c4
+Merge: 6353508134 076e4b7991
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:49:51 2017 +0200
+
+    Merge pull request #14067 from asheplyakov/19322-bp-kraken
+
+    kraken: rgw: fix break inside of yield in RGWFetchAllMetaCR
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 6353508134bd4d2bcbca7d0f9a42788895fe5c27
+Merge: 425992f92d 87811ff161
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:48:57 2017 +0200
+
+    Merge pull request #14509 from smithfarm/wip-18499-kraken
+
+    kraken: rgw: Realm set does not create a new period
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 425992f92d30c8503053311471322a9849a37746
+Merge: 39bf493984 6860fce424
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:48:01 2017 +0200
+
+    Merge pull request #14511 from smithfarm/wip-18772-kraken
+
+    kraken: rgw: crash when updating period with placement group
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 39bf493984cb98c5a3f7691458188fea7a246c70
+Merge: eb397c1780 1cb9868456
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:45:15 2017 +0200
+
+    Merge pull request #14513 from smithfarm/wip-18843-kraken
+
+    kraken: rgw: usage stats and quota are not operational for multi-tenant users
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit eb397c178028530d061e9edc32f84ddd6d826a89
+Merge: 1ed342d75d 5f4c38fa6b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:44:21 2017 +0200
+
+    Merge pull request #14515 from smithfarm/wip-18904-kraken
+
+    kraken: rgw: first write also tries to read object
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 1ed342d75d0befb602597385b5f49d27a244d450
+Merge: d3a2417cf1 935cc11595
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:42:36 2017 +0200
+
+    Merge pull request #14516 from smithfarm/wip-18909-kraken
+
+    kraken: rgw: the swift container acl does not support field .ref
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit d3a2417cf11e6ccebf25de8fe12242c1e8eda626
+Merge: ce95f0f7d8 ffe5bbfc6e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:25:43 2017 +0200
+
+    Merge pull request #14517 from smithfarm/wip-19047-kraken
+
+    kraken: rgw: fix use of marker in List::list_objects()
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit ce95f0f7d81af4bfbad7afb4f429b9cfa2e37726
+Merge: aab812c517 03e5be4498
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:24:13 2017 +0200
+
+    Merge pull request #14524 from smithfarm/wip-19477-kraken
+
+    kraken: rgw: S3 v4 authentication issue with X-Amz-Expires
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit aab812c51706fc3395af3c34b0b3720e84c30b61
+Merge: a441042e00 a5fe812483
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:22:26 2017 +0200
+
+    Merge pull request #14525 from smithfarm/wip-19479-kraken
+
+    kraken: rgw: zonegroupmap set does not work
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit a441042e00cb752737e303c4c38322fadf13a2f6
+Merge: 3b128e458f cac2647189
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:21:22 2017 +0200
+
+    Merge pull request #14528 from smithfarm/wip-19534-kraken
+
+    kraken: rgw: Error parsing xml when get bucket lifecycle
+
+    Reviewed-by: Daniel Gryniewicz <dang@redhat.com>
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 3b128e458f86a23e8b6ff987b0f0774e7d5597bb
+Merge: 6366b2382b 8f81bb33c7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:19:08 2017 +0200
+
+    Merge pull request #14606 from asheplyakov/19608-kraken
+
+    kraken: rgw: multisite: fetch_remote_obj() gets wrong version when copying from remote
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 6366b2382b8d457ab9e544cc889aa77f024a626f
+Merge: 19280ae91e 727653623f
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:18:10 2017 +0200
+
+    Merge pull request #15384 from asheplyakov/20015-bp-kraken
+
+    kraken: cls/rgw: list_plain_entries() stops before bi_log entries
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 19280ae91e710cbb9569fc1dc3a6a24b6ea05ca4
+Merge: eb56af6c0d 595c31f390
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:03:16 2017 +0200
+
+    Merge pull request #14522 from smithfarm/wip-19472-kraken
+
+    kraken: rgw: add the remove-x-delete feature to cancel swift object expiration
+
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit eb56af6c0d8eea0e19a16da4e186e8edbfe13148
+Merge: 4ccd32619f f8db63f24d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 09:00:43 2017 +0200
+
+    Merge pull request #14523 from smithfarm/wip-19475-kraken
+
+    kraken: rgw: multisite: EPERM when trying to read SLO objects as system/admin user
+
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit 4ccd32619f39a4f6fa7a6700c8c90c622c83d67a
+Merge: e59eba0ff3 c3d8444890
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 08:58:44 2017 +0200
+
+    Merge pull request #14519 from smithfarm/wip-19175-kraken
+
+    kraken: swift API: cannot disable object versioning with empty X-Versions-Location
+
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit e59eba0ff375011c18b3a8dcdb228c71c9085fe9
+Merge: aaa7992ee5 faf80bc1dd
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 08:56:47 2017 +0200
+
+    Merge pull request #14526 from smithfarm/wip-19524-kraken
+
+    kraken: rgw: 'radosgw-admin zone create' command with specified zone-id creates a zone with different id
+
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit aaa7992ee5fd2767d8a144c47255489080f5fb47
+Merge: 6c3daeb297 e5719234b4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Jul 7 08:55:30 2017 +0200
+
+    Merge pull request #14529 from smithfarm/wip-19573-kraken
+
+    kraken: rgw: Response header of swift API returned by radosgw does not contain x-openstack-request-id. But Swift returns it.
+
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit ce874ab676e8b5a09e32db19f077c650cff9979b
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Jun 10 23:59:19 2017 +0800
+
+    test/librbd: decouple ceph_test_librbd_api from libceph-common
+
+    Fixes: http://tracker.ceph.com/issues/20175
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b7287fdc4d70c5ecedda78ae367b98e5d8f61c5b)
+
+    Conflicts:
+            src/test/librbd/CMakeLists.txt: trivial resolution
+
+commit 57f7213d9fd5a9a82e46c4ca3f3dda87c3dde717
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Jun 10 23:31:53 2017 +0800
+
+    test/librbd: replace libcommon classes using standard library
+
+    so ceph_test_librbd_api has less dependencies on libcommon
+
+    Fixes: http://tracker.ceph.com/issues/20175
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit bc8fa0193c8b2fb4b749ce9976bf6efbea833e96)
+
+    Conflicts:
+            src/test/librbd/test_librbd.cc: trivial resolution
+
+commit 7d044c6fbdcabc92d4dae7d9759dc63f87b2e4f4
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Jun 10 23:58:47 2017 +0800
+
+    test/librados: extract functions using libcommon in test.cc into test_common.cc
+
+    Fixes: http://tracker.ceph.com/issues/20175
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 82a848c2053ea69ebc6d3ec1003e18921e2d08d2)
+
+    Conflicts:
+            src/test/librados/CMakeLists.txt: trivial resolution
+            src/test/librados/test.cc: trivial resolution
+
+commit 53024570e75f2963da84f3a045f1ac1f9ee4081e
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat Apr 22 12:27:12 2017 +0800
+
+    mds: issue new caps when sending reply to client
+
+    After Locker::issue_new_caps() adds new Capability data struct,
+    do not issue caps immediately. Let CInode::encode_inodestate()
+    do the job instead. This can avoid various races that early reply
+    is not allowed, caps that haven't been sent to client gets revoked.
+
+    Fixes: http://tracker.ceph.com/issues/19635
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 799703a4acb49db0b6cc99a23e4326767e694c3a)
+
+commit 18f1830e8acaa386d0113171ec649b5b33698dd9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 5 08:17:05 2017 -0400
+
+    librbd: filter expected error codes from is_exclusive_lock_owner
+
+    Fixes: http://tracker.ceph.com/issues/20182
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d4daaf54e6bc42cd4fb2111ea20b2042941b0c31)
+
+commit 45b4c86452d19eaf3078704ba951e2fdb2e49a30
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jun 5 13:17:19 2017 -0400
+
+    rbd: properly decode features when using image name optional
+
+    Fixes: http://tracker.ceph.com/issues/20185
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f1b05a2b062a59ec1b6682f7683bfd816433a931)
+
+commit d463de9b8ae357a4edbfcf63e65740ecd4845b26
+Author: fang.yuxiang <fang.yuxiang@eisoo.com>
+Date:   Tue Jun 13 16:40:16 2017 +0800
+
+    rgw: meta sync thread crash at RGWMetaSyncShardCR
+
+    Fixes: http://tracker.ceph.com/issues/20251
+
+    Signed-off-by: fang yuxiang fang.yuxiang@eisoo.com
+    (cherry picked from commit 45877d38fd9a385b2f8b13e90be94d784898b0b3)
+
+    Conflicts:
+            src/rgw/rgw_data_sync.cc
+              - kraken RGWContinuousLeaseCR() has slightly different options
+            src/rgw/rgw_sync.cc
+              - kraken RGWContinuousLeaseCR() has slightly different options
+              - added "override" qualifier to ~RGWFetchAllMetaCR() definition
+
+commit d52aeec0310d3f1dab986a43bee2f4fc768efc06
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Sat Jun 10 18:12:52 2017 -0400
+
+    rgw: only log metadata on metadata master zone
+
+    Fixes: http://tracker.ceph.com/issues/20244
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit b8272f3607074a2f7cbfd08f7bbc82f22cf120ba)
+
+commit 9651ed95894707d6d5517979b18a8acdbcc1b004
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Apr 21 15:04:48 2017 -0400
+
+    radosgw-admin: warn that 'realm rename' does not update other clusters
+
+    Fixes: http://tracker.ceph.com/issues/19746
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 8a459c768ffecd689a53a79dfe33eb8f1bbc318f)
+
+commit 4e4cd5453670af8d87b217c60dc84dd39e192952
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Apr 4 10:42:44 2017 -0400
+
+    rgw: fix for zonegroup redirect url
+
+    local dest_url variable was shadowing the one in the enclosing scope, so
+    the changes were not applied and no Location header was written on redirect
+
+    Fixes: http://tracker.ceph.com/issues/19488
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 542e188a40f0495720b48308372366951ae41e62)
+
+commit 693d88081358f2059a65de5ddb4d35272ef1f3fe
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Apr 4 10:41:51 2017 -0400
+
+    rgw: use zonegroup's master zone endpoints for bucket redirect
+
+    if no zonegroup endpoints are set, fall back to master zone endpoints
+
+    Fixes: http://tracker.ceph.com/issues/19488
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 92f63c6392bdc4633a2e57cb3867051bb1a3fd55)
+
+commit f23df08430c6b62314898a4c95f81d8cb01f93c3
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Apr 5 14:20:20 2017 -0400
+
+    rgw: allow larger payload for period commit
+
+    testing with 3 zonegroups and 3 zones each, the period json grew larger
+    than 4k and caused decode failures on period commit
+
+    updated to use the new config variable rgw_max_put_param_size
+
+    Fixes: http://tracker.ceph.com/issues/19505
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 7f2871fe59d933b03f37fde40f1781b2320d0d50)
+
+commit dea9a69c3f85504070a56f8a2dbf1a4c129b9ef4
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Fri Mar 3 16:50:33 2017 +0100
+
+    rgw: make a configurable size for requests with xml params
+
+    We currently read all user input for a few apis accepting xml, avoid
+    doing this and error out early, most s3 apis limit to about 1000 xml
+    entries, make this configurable via a new config param,
+    `rgw_max_put_param_size` defaulting to 1MB. Also modified
+    `rgw_rest_read_all_input` with an additional param to not allow for
+    chunked uploads, though we error out in a better way by responding with
+    content length required rather than S3's 501 if chunked is set.
+
+    This also adds the same behavior in RGWPutCORS reusing
+    `rgw_rest_read_all_input`, and using a data_deleter to manage the char*
+
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+
+    rgw: rgw_rest_s3: make PUT CORS accept only a configured max input
+
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit 35375b75160dc93beeb04c7fe010dc0d21952534)
+
+commit ac706f2b4a4aee7b813a0b5ba0589431b84f3dcd
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue May 23 12:07:45 2017 +0200
+
+    librbd: potential read IO hang when image is flattened
+
+    Fixes: http://tracker.ceph.com/issues/19832
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 10d58618e7c632ef01b9537492239e0a042dc17e)
+
+commit b27595ae9c3490a4ff1d39b0d864a4a47a941981
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Wed Jun 14 10:29:53 2017 +0800
+
+    rgw: lifecycle thread shouldn't process the bucket which has been deleted.
+
+    Fixes: http://tracker.ceph.com/issues/20285
+
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit a2b042fe9f7e6503273fa4c2a4c56e399d7c338f)
+
+commit 3105e2327ea29558fdd97dfb6da2c7ee9ec08f8f
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Tue May 9 16:39:39 2017 +0800
+
+    rgw: VersionIdMarker and NextVersionIdMarker should be returned when listing
+    object versions if necessary.
+
+    Fixes: http://tracker.ceph.com/issues/19886
+
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit f805c3e08948e379b7d2c4f2faf9e7f550e4cb23)
+
+commit d0e742cafae325cf794a2160b6df19296ac398a4
+Author: fang.yuxiang <fang.yuxiang@eisoo.com>
+Date:   Thu May 4 15:58:37 2017 +0800
+
+    rgw: set object accounted size correctly
+
+    sometimes, object accounted size is set wrong,
+    because we don't konw the object size if don't resort to the compression info or manifest.
+    e.g, when i use s3cmd do copy object(bucket_A/obj_A -> bucket_B/obj_B, assume the size of obj_A is 4M).
+    then i use s3cmd do list bucket, I got obj_B size is 512K, it is the head size apparently.
+
+    Fixes: http://tracker.ceph.com/issues/20071
+
+    Signed-off-by: fang yuxiang <fang.yuxiang@eisoo.com>
+    (cherry picked from commit 539985a99eebdc72c8d2446acc1108664a162f68)
+
+commit 640a7a26296e652ffb8b8bfab87839d7bfd66d1b
+Author: fang.yuxiang <fang.yuxiang@eisoo.com>
+Date:   Sat May 27 15:20:30 2017 +0800
+
+    rgw: get wrong content when download object with specific range when
+    compression was enabled
+
+    look at the prototype:
+    RGWGetObj_Decompress::handle_data(bufferlist& bl, off_t bl_ofs, off_t bl_len)
+    we should trim the bl using bl_ofs and bl_len.
+
+    Fixes: http://tracker.ceph.com/issues/20100
+
+    Signed-off-by: fang yuxiang <fang.yuxiang@eisoo.com>
+    (cherry picked from commit afe9d99570059b88837690950a4f4525e0e60262)
+
+commit 6c3daeb2970f52970145eb31d4bef56b33df8b40
+Merge: 08b8a736e2 bd9aec00ec
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jul 6 17:46:06 2017 +0200
+
+    Merge pull request #16113 from smithfarm/wip-19807-kraken
+
+    kraken: tests: remove hard-coded image name from TestLibRBD.Mirror
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 1fc14857a9dd81b1b4726428f115ef5cfb495781
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Tue Jun 6 15:43:20 2017 +0800
+
+    rgw: datalog trim and mdlog trim handles the result returned by osd
+    incorrectly.
+
+    Fixes: http://tracker.ceph.com/issues/20190
+
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit 7fd6e031e5b0b1f3eca70c5b459d50f6f214171f)
+
+commit 90288afc10d878b59dacb99383ada309377e4bb4
+Author: Pritha Srivastava <prsrivas@redhat.com>
+Date:   Mon Mar 20 11:41:48 2017 +0530
+
+    rgw: Added code to correctly account for bytes sent/ received during a 'PUT' operation.
+
+    Currently, the bytes sent/ received are both set to zero after
+    an object is uploaded to a bucket. Added code to correct the logic.
+
+    Signed-off-by: Pritha Srivastava <prsrivas@redhat.com>
+    (cherry picked from commit 85735c972f5db1a110f55e01a4f6249202152553)
+
+commit dda4d912cd753e1443beac4e7f92d97f24b57992
+Author: redickwang <redickwang@tencent.com>
+Date:   Fri May 19 15:08:12 2017 +0800
+
+    rgw: segment fault when shard id out of range
+
+    Fixes: http://tracker.ceph.com/issues/19732
+
+    Signed-off-by: redickwang <redickwang@tencent.com>
+    (cherry picked from commit ff4c40fc2e3c092d17a218ae6132de4e6034c8ee)
+
+commit f32e26e909c3d56160e9a33d051d690fa4366bbd
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Thu Apr 6 17:08:23 2017 +0200
+
+    rgw: fix broken /crossdomain.xml, /info and /healthcheck of Swift API.
+
+    Fixes: http://tracker.ceph.com/issues/19520
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit fe2afd42aaf706d38d6ed611796a181ae9e02ae8)
+
+commit ccb33bab37e45bf47dad5577602e9be9d1d683c6
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Tue Apr 25 14:46:09 2017 +0200
+
+    rgw: use a vector for options passed to civetweb
+
+    Since the array we used needs additional check to ensure that the size
+    is correct, and causes undefined behaviour in a few platforms, using a
+    vector and passing the c array back to mg_start so that we don't go past
+    the end of array.
+
+    Fixes: http://tracker.ceph.com/issues/19749
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    Signed-off-by: Jesse Williamson <jwilliamson@suse.de>
+    (cherry picked from commit 3959a8b52c2910e4060f7d273c8939bbdc83e48a)
+
+commit f8235c5c0a64eb409957d80d4c4a22a87280d0b2
+Author: Willem Jan Withagen <wjw@digiware.nl>
+Date:   Sat Apr 29 13:36:07 2017 +0200
+
+    cls/log/cls_log.cc: reduce logging noise
+
+     - The other reference in the source as already at 20.
+          ./src/cls/timeindex/cls_timeindex.cc:85:
+            CLS_LOG(20, "storing entry at %s", index.c_str());
+
+       And we need not always know where in the log items are stored.
+       So it looks like a leftover debug feature.
+
+    Fixes: http://tracker.ceph.com/issues/19835
+    Signed-off-by: Willem Jan Withagen <wjw@digiware.nl>
+    (cherry picked from commit d76010900bf9012f2e66335787710531772766b7)
+
+commit e773b304eefa3d2ca7c1fe0817c89082bf574a38
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Thu Apr 13 05:33:55 2017 -0400
+
+    rgw: swift: disable revocation thread if sleep == 0 || cache_size == 0
+
+    Keystone tokens can be revoked.  This causes them to fail
+    validation.  However, in ceph, we cache them.  As long as
+    they're in the cache we trust them.  To find revoked tokens
+    there's a call OSI-PKI/revoked but that's only useful for
+    pki tokens.  Installations using fernet/uuid may not even
+    have the proper credentials to support the call, in which
+    case the call blows up in various ways filling up logs
+    with complaints.
+
+    This code makes the revocation thread optional; by disabling it,
+    the complaints go away.  A further fix is in the works
+    to use other more modern calls available in modern keystone
+    installations to properly deal with non-PKI/PKIZ tokens.
+
+    To disable the revocation thread, use at least one of these:
+            rgw_keystone_token_cache_size = 0
+                    using this will cause tokens to be validated on every call.
+    You may instead want to set
+            rgw_keystone_revocation_interval = 0
+                    using just this will disable the revocation thread,
+                    but leaves the cache in use.  That avoids the extra
+                    validation overhead, but means token revocation won't
+                    work very well.
+
+    Fixes: http://tracker.ceph.com/issues/9493
+    Fixes: http://tracker.ceph.com/issues/19499
+
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    (cherry picked from commit 003291a8cbca455c0e8731f66759395a0bb1f555)
+
+commit 68a853a14d89261fcd0a132a1a7a7151721b84c8
+Author: lvshuhua <lvshuhua@cmss.chinamobile.com>
+Date:   Wed Dec 7 15:47:47 2016 +0800
+
+    rgw: fix versioned bucket data sync fail when upload is busy
+
+    Fixes: http://tracker.ceph.com/issues/18208
+
+    Signed-off-by: lvshuhua <lvshuhua@cmss.chinamobile.com>
+    (cherry picked from commit ce7d00ac1642d84c1d9111156a544c37801c5adf)
+
+commit 9a7a73edf1f432df98984ab7ad60306aa6aca6e3
+Author: liuchang0812 <liuchang0812@gmail.com>
+Date:   Fri Feb 10 18:02:03 2017 +0800
+
+    rgw: don't return skew time in pre-signed url
+
+    Fixes: http://tracker.ceph.com/issues/18828
+
+    Signed-off-by: liuchang0812 <liuchang0812@gmail.com>
+    (cherry picked from commit dd8b348f4aad0124e8a4457117bf3f5f76af7bdb)
+
+commit 08b8a736e2b49bba39cf8fe51fe3435a7d381e72
+Merge: f45de0f39d 405503472e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jul 6 09:28:26 2017 +0200
+
+    Merge pull request #16132 from smithfarm/wip-20522-kraken
+
+    kraken: FAILED assert(object_contexts.empty()) (live on master only from Jan-Feb 2017, all other instances are different)
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit f45de0f39db316219207d674198ecc578ed76f7e
+Merge: 4ca8f1f4a9 ff19977fbc
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jul 6 09:27:37 2017 +0200
+
+    Merge pull request #15962 from asheplyakov/20443-bp-kraken
+
+    kraken: osd: unlock sdata_op_ordering_lock with sdata_lock hold to avoid missing wakeup signal
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 4ca8f1f4a9c5cdb15bd273f74df7ac6cda218f66
+Merge: d45c64d1b9 2aec591cfe
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jul 6 09:26:40 2017 +0200
+
+    Merge pull request #15066 from asheplyakov/19916-bp-kraken
+
+    kraken: osd: osd/PrimaryLogPG: do not call on_shutdown() if (pg.deleting)
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit d45c64d1b94fb6a91c78e8333f924a2be08fcddb
+Merge: 4890ba0df0 62bb2086af
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jul 6 09:24:26 2017 +0200
+
+    Merge pull request #14942 from shinobu-x/wip-18293-kraken
+
+    kraken: osd: leaked MOSDMap
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 4890ba0df02403b8bc5348235d1aadb76ab8f4c1
+Merge: c45570b4a6 d5aea7a68b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jul 6 09:20:56 2017 +0200
+
+    Merge pull request #14736 from smithfarm/wip-19326-kraken
+
+    kraken: bluestore bdev: flush no-op optimization is racy
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit c45570b4a61243839e7d4e64c5e1f1b017bb3948
+Merge: 5a1116c0d5 e76725f4e9
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jul 6 09:19:50 2017 +0200
+
+    Merge pull request #14732 from smithfarm/wip-19560-kraken
+
+    kraken: objecter: full_try behavior not consistent with osd
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 5a1116c0d5c1fa8810031104cd093ed1b1e8cb3a
+Merge: 7133d4db9b 1bc9cfff42
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jul 6 09:18:24 2017 +0200
+
+    Merge pull request #13542 from shinobu-x/wip-18929-kraken
+
+     kraken: osd: bogus assert when checking acting set on recovery completion in rados/upgrade
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 7133d4db9b31b0627c6278f2f72b690ea366551e
+Merge: 6db580b6cf 56ca3b0190
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jul 6 09:11:09 2017 +0200
+
+    Merge pull request #14939 from rzarzynski/wip-rgw-19754-kraken
+
+    kraken: rgw: fix RadosGW hang during multi-chunk upload of AWSv4
+
+    Reviewed-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 6db580b6cfdf3ef14403adbbcd27a2f16cbf15fc
+Merge: be8b7610d1 59391c40db
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jul 6 08:54:16 2017 +0200
+
+    Merge pull request #14530 from smithfarm/wip-19574-kraken
+
+    kraken: rgw: unsafe access in RGWListBucket_ObjStore_SWIFT::send_response()
+
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit be8b7610d164bf2798bb4b9676db330ef5fb6c27
+Merge: e12eae92b0 1f86be6e19
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Jul 6 08:40:16 2017 +0200
+
+    Merge pull request #13868 from smithfarm/wip-19160-kraken
+
+    kraken: rgw: multisite: RGWMetaSyncShardControlCR gives up on EIO
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 85cefa113b31c9dc6b8a7a88e3343de923278c7d
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sat Mar 18 10:04:20 2017 +0100
+
+    mon: osd crush set crushmap need sanity check
+
+    The sanity check verifying the new crushmap does not remove crush rules
+    that are in use is not exclusive to ceph setcrushmap.
+
+    Fixes: http://tracker.ceph.com/issues/19302
+
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit ed760457bf154c10adf75c6df046eecab7eb8e4b)
+
+commit 9a26882289e1746fe8ca1443666ed0a583e07021
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jun 6 15:13:50 2017 -0400
+
+    ceph_test_rados_api_misc: fix LibRadosMiscConnectFailure.ConnectFailure retry
+
+    Fixes: http://tracker.ceph.com/issues/19901
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 087dff80cac707ee9bcb5bcfc98cb3ec047bd49f)
+
+commit 91569f63852f12d958175bf970967fa9d25b2cfc
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Thu Jan 5 11:50:42 2017 +0800
+
+    rgw: fix 'gc list --include-all' command infinite loop the first 1000 items
+
+    When the items to gc over 1000, 'gc list --include-all' command will infinite loop the first 1000 items.
+    Add next_marker to move to the next 1000 items.
+
+    Fixes: http://tracker.ceph.com/issues/19978
+
+    Signed-off-by: fang yuxiang <fang.yuxiang@eisoo.com>
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit fc29f52ebca63104a05515484088ff136dfb0b15)
+
+commit ea186807c3fb7ac79f8ef1a8ae874ced69aec6e1
+Author: Erwan Velu <erwan@redhat.com>
+Date:   Fri Mar 31 14:54:33 2017 +0200
+
+    ceph-disk: Adding retry loop in get_partition_dev()
+
+    There is very rare cases where get_partition_dev() is called before the actual partition is available in /sys/block/<device>.
+
+    It appear that waiting a very short is usually enough to get the partition beein populated.
+
+    Analysis:
+    update_partition() is supposed to be enough to avoid any racing between events sent by parted/sgdisk/partprobe and
+    the actual creation on the /sys/block/<device>/* entrypoint.
+    On our CI that race occurs pretty often but trying to reproduce it locally never been possible.
+
+    This patch is almost a workaround rather than a fix to the real problem.
+    It offer retrying after a very short to be make a chance the device to appear.
+    This approach have been succesful on the CI.
+
+    Note his patch is not changing the timing when the device is perfectly created on time and just differ by a 1/5th up to 2 seconds when the bug occurs.
+
+    A typical output from the build running on a CI with that code.
+            command_check_call: Running command: /usr/bin/udevadm settle --timeout=600
+            get_dm_uuid: get_dm_uuid /dev/sda uuid path is /sys/dev/block/8:0/dm/uuid
+            get_partition_dev: Try 1/10 : partition 2 for /dev/sda does not in /sys/block/sda
+            get_partition_dev: Found partition 2 for /dev/sda after 1 tries
+            get_dm_uuid: get_dm_uuid /dev/sda uuid path is /sys/dev/block/8:0/dm/uuid
+            get_dm_uuid: get_dm_uuid /dev/sda2 uuid path is /sys/dev/block/8:2/dm/uuid
+
+    fixes: #19428
+
+    Signed-off-by: Erwan Velu <erwan@redhat.com>
+    (cherry picked from commit 93e7b95ed8b4c78daebf7866bb1f0826d7199075)
+
+commit f7f6375b54233f1085ba8d64df1053c038ff3991
+Author: Erwan Velu <erwan@redhat.com>
+Date:   Wed Mar 22 10:11:44 2017 +0100
+
+    ceph-disk: Reporting /sys directory in get_partition_dev()
+
+    When get_partition_dev() fails, it reports the following message :
+        ceph_disk.main.Error: Error: partition 2 for /dev/sdb does not appear to exist
+    The code search for a directory inside the /sys/block/get_dev_name(os.path.realpath(dev)).
+
+    The issue here is the error message doesn't report that path when failing while it might be involved in.
+
+    This patch is about reporting where the code was looking at when trying to estimate if the partition was available.
+
+    Signed-off-by: Erwan Velu <erwan@redhat.com>
+    (cherry picked from commit 413c9fcfbe8e6ab33d73b8428090ccacc33c5d15)
+
+commit 533ff8a540b0b30378a9562953f1da46291e9832
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed May 3 18:28:01 2017 +0800
+
+    qa/tasks/ceph_manager: always fix pgp_num when done with thrashosd task
+
+    Fixes: http://tracker.ceph.com/issues/19771
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit da1161cbd8b50b8a980e8a3b48154a97f988426c)
+
+commit f04eb66d365c088e0e6985864d54bdc4fc3d3bfd
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue May 9 12:32:51 2017 +0200
+
+    ceph-disk: separate ceph-osd --check-needs-* logs
+
+    It is using the OSD id zero but have nothing to do with OSD zero and
+    this is confusing to the user. The log themselves do not need to be kept
+    around and are stored in the run directory so that they can be disposed
+    of after reboot.
+
+    Fixes: http://tracker.ceph.com/issues/19888
+
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit c7b3c46bd63b78475868e405bf20d9c142f0336a)
+
+commit a7af766aa89b9435c72cdb93e5d67a9e9331c635
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 22 17:18:08 2016 -0500
+
+    msg/simple: clear_pipe when wait() is mopping up pipes
+
+    When wait is mopping up connections it may hit one that
+    is in the process of accepting.  It will unregister it
+    whilst the accept() thread is trying to set it up,
+    aborting the accept and getting it reaped.  However,
+    the pipe mop-up does not clear_pipe() the way that
+    mark_down(), mark_down_all(), and fault() do, which
+    leads to this assert.
+
+    Pipe is accepting...
+
+      -161> 2016-12-22 17:31:45.460613 37353700 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=0 pgs=0 cs=0 l=1 c=0x3e2a6f40).accept:  setting up session_security.
+      -160> 2016-12-22 17:31:45.460733 37353700 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=0 pgs=0 cs=0 l=1 c=0x3e2a6f40).accept new session
+      -159> 2016-12-22 17:31:45.460846 37353700 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=2 pgs=7 cs=1 l=1 c=0x3e2a6f40).accept success, connect_seq = 1, sending READY
+      -158> 2016-12-22 17:31:45.460959 37353700 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=2 pgs=7 cs=1 l=1 c=0x3e2a6f40).accept features 1152921504336314367
+
+    wait() is shutting down...
+
+      -156> 2016-12-22 17:31:45.461882 9506ac0 20 -- 172.21.15.14:6804/20738 wait: stopping accepter thread
+      -155> 2016-12-22 17:31:45.462994 9506ac0 10 accepter.stop accept listening on: 15
+    ...
+      -116> 2016-12-22 17:31:45.482137 9506ac0 10 -- 172.21.15.14:6804/20738 wait: closing pipes
+      -115> 2016-12-22 17:31:45.482850 9506ac0 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=2 pgs=7 cs=1 l=1 c=0x3e2a6f40).unregister_pipe
+      -114> 2016-12-22 17:31:45.483421 9506ac0 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=2 pgs=7 cs=1 l=1 c=0x3e2a6f40).stop
+
+    ...which interrupts the accept()...
+
+      -113> 2016-12-22 17:31:45.484164 37353700 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=4 pgs=7 cs=1 l=1 c=0x3e2a6f40).accept fault after register
+
+    and makes accept() return failure, and reader() to exit
+    and reap...
+
+      -110> 2016-12-22 17:31:45.486103 9506ac0 10 -- 172.21.15.14:6804/20738 wait: waiting for pipes 0x3e2a5c20 to close
+      -109> 2016-12-22 17:31:45.487146 37353700 10 -- 172.21.15.14:6804/20738 queue_reap 0x3e2a5c20
+      -108> 2016-12-22 17:31:45.487658 9506ac0 10 -- 172.21.15.14:6804/20738 reaper
+      -107> 2016-12-22 17:31:45.487722 9506ac0 10 -- 172.21.15.14:6804/20738 reaper reaping pipe 0x3e2a5c20 172.21.15.35:0/146098963
+      -106> 2016-12-22 17:31:45.487816 9506ac0 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=4 pgs=7 cs=1 l=1 c=0x3e2a6f40).discard_queue
+      -105> 2016-12-22 17:31:45.494742 37353700 10 -- 172.21.15.14:6804/20738 >> 172.21.15.35:0/146098963 pipe(0x3e2a5c20 sd=31 :6804 s=4 pgs=7 cs=1 l=1 c=0x3e2a6f40).reader done
+    ...
+       -92> 2016-12-22 17:31:45.527589 9506ac0 -1 /mnt/jenkins/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/11.1.0-6151-ge1781dd/rpm/el7/BUILD/ceph-11.1.0-6151-ge1781dd/src/msg/simple/SimpleMessenger.cc: In function 'void SimpleMessenger::reaper()' thread 9506ac0 time 2016-12-22 17:31:45.488264
+    /mnt/jenkins/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/11.1.0-6151-ge1781dd/rpm/el7/BUILD/ceph-11.1.0-6151-ge1781dd/src/msg/simple/SimpleMessenger.cc: 235: FAILED assert(!cleared)
+
+    Fixes: http://tracker.ceph.com/issues/15784
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 948f97b3bdd39269a38277238a61f24e5fec6196)
+
+commit 6d2f95973561c9f0911fce04dff2036f73a77442
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 28 13:36:01 2016 -0500
+
+    test: added missing IoCtx copy/assignment methods in librados_test_stub
+
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c35d307c62961072b4c00f94e37486e3371c1a21)
+
+commit 405503472ea6b3a2965455c8c064dc0e4d713ad4
+Author: Samuel Just <sjust@redhat.com>
+Date:   Tue Feb 14 12:47:37 2017 -0800
+
+    ReplicatedBackend: don't queue Context outside of ObjectStore with obc
+
+    We only flush the ObjectStore callbacks, not everything else.  Thus,
+    there isn't a guarrantee that the obc held by pull_complete_info will
+    be cleaned up before the Flush callback is triggered.  Instead, just
+    defer clearing the pull state until the callback (it'll be cleaned up
+    during the interval change) and remove the ObjectContext from
+    pull_complete_info.
+
+    Introduced: 68defc2b0561414711d4dd0a76bc5d0f46f8a3f8
+    Fixes: http://tracker.ceph.com/issues/18927
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 51eee55c475a3d931844831e040ed3d66ee59af4)
+
+commit 5257d1a643fb1af36b80650ea2b014b4887dc10f
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Feb 3 13:12:47 2017 -0800
+
+    osd/: don't leak context for Blessed*Context or RecoveryQueueAsync
+
+    This has always been a bug, but until
+    68defc2b0561414711d4dd0a76bc5d0f46f8a3f8, nothing deleted those contexts
+    without calling complete().
+
+    Fixes: http://tracker.ceph.com/issues/18809
+    Bug shadowed until: 68defc2b0561414711d4dd0a76bc5d0f46f8a3f8
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 91b74235027c8a4872dcab6b37767b12c3267061)
+
+commit e12eae92b04d30d322740a6c7a33d86e448bf9c5
+Merge: 7bdc3ef238 73fe367bc4
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 22:04:41 2017 +0200
+
+    Merge pull request #15792 from liewegas/wip-bluestore-leaks-kraken
+
+    kraken: os/bluestore: deep decode onode value
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 7bdc3ef238228bb156f8f97867a66138fcc93807
+Merge: 0a71e1899b 27e9644ceb
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 22:01:58 2017 +0200
+
+    Merge pull request #15729 from asheplyakov/20315-bp-kraken
+
+    kraken: mon: fail to form large quorum; msg/async busy loop
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 0a71e1899bd8ee84640059e18126461d3a1748a3
+Merge: d7598df528 d38a752999
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 22:00:17 2017 +0200
+
+    Merge pull request #15084 from asheplyakov/19928-bp-kraken
+
+    kraken: mon crash on shutdown, lease_ack_timeout event
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit d7598df5281e1648c5890a4b18b7b64d74b8dc95
+Merge: 0dbbd5001e 47a4544176
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 21:59:03 2017 +0200
+
+    Merge pull request #14735 from smithfarm/wip-19685-kraken
+
+    kraken: osd: Give requested scrubs a higher priority
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 0dbbd5001ea84906d15bb27c99b73493d20eab75
+Merge: 0d177d5c99 7a275769b6
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 21:57:53 2017 +0200
+
+    Merge pull request #14733 from smithfarm/wip-19561-kraken
+
+    kraken: api_misc: [  FAILED  ] LibRadosMiscConnectFailure.ConnectFailure
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 0d177d5c992ca8cd99afa8e9be3f429a933d490a
+Merge: 11ed483791 0227920438
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 21:51:26 2017 +0200
+
+    Merge pull request #13883 from shinobu-x/wip-19119-kraken
+
+    kraken: pre-jewel "osd rm" incrementals are misinterpreted
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 11ed483791849905910c879152d1eea41d95604e
+Merge: 74b0c9da46 fa2753245d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 19:39:54 2017 +0200
+
+    Merge pull request #14646 from gregsfortytwo/wip-17916-kraken
+
+    kraken: osd: pglog: with config, don't assert in the presence of stale diverg…
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 74b0c9da461825ef53ddf79ec00d08168e33025c
+Merge: 7a87735243 648c6adf52
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 18:41:59 2017 +0200
+
+    Merge pull request #15237 from asheplyakov/20035-bp-kraken
+
+    kraken: mon/PGMap: factor mon_osd_full_ratio into MAX AVAIL calc
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 7a877352430e49f94dff614dc0a697a4697f6a77
+Merge: 4a7c26083f 5c7a07677a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 18:19:05 2017 +0200
+
+    Merge pull request #14331 from shinobu-x/wip-19396-kraken
+
+    kraken: Objecter::epoch_barrier isn't respected in _op_submit()
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 4a7c26083feebca273044691c7f7ef0dbf8305eb
+Merge: f978b2514c 071f111abb
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 17:54:30 2017 +0200
+
+    Merge pull request #16083 from smithfarm/wip-18677-kraken
+
+    kraken: osd: metadata reports filestore when using bluestore
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit f978b2514cb17dabd341b5c801bd541a489071a7
+Merge: 56c65b120f 7054ff69d0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 17:51:36 2017 +0200
+
+    Merge pull request #16098 from smithfarm/wip-19621-kraken
+
+    kraken: rbd-nbd: add signal handler
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 56c65b120f1ef7323109889c6a82287e5e4f5371
+Merge: e13f5467fa 27cbff1114
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 17:50:21 2017 +0200
+
+    Merge pull request #15612 from smithfarm/wip-19611-kraken
+
+    kraken: rbd: Issues with C API image metadata retrieval functions
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit e13f5467fac1e296d2d56b8ed4dc89d5cdfa0f5d
+Merge: 19deb31d39 b19d6eb311
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 17:48:24 2017 +0200
+
+    Merge pull request #14833 from smithfarm/wip-19794-kraken
+
+    kraken: tests: test_notify.py: assert(not image.is_exclusive_lock_owner()) on line 147
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 19deb31d399715716b02c8fdf58ea2845b42d58c
+Merge: cc5241e4aa c79d45ff52
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 17:26:58 2017 +0200
+
+    Merge pull request #14543 from smithfarm/wip-19173-kraken
+
+    kraken: rbd: rbd_clone_copy_on_read ineffective with exclusive-lock
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit cc5241e4aab227b8b272774bb5d4ff1f37ed3446
+Merge: e705528331 9efeb6bc2d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 09:37:01 2017 +0200
+
+    Merge pull request #14974 from dillaman/wip-19872-kraken
+
+    kraken: rbd-mirror: failover and failback of unmodified image results in split-brain
+
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit e70552833137eed1f9ede6c3eb59d345a50ac9ce
+Merge: 2b73b570e0 d51b755391
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 09:36:14 2017 +0200
+
+    Merge pull request #14934 from dillaman/wip-19833-kraken
+
+    kraken: cls_rbd: default initialize snapshot namespace for legacy clients
+
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 2b73b570e060d2c77b6c86579cecc84221b166f9
+Merge: dbe8a8ba84 e0f90f03dd
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Jul 5 09:30:07 2017 +0200
+
+    Merge pull request #14539 from smithfarm/wip-18771-kraken
+
+    kraken: rbd: Improve compatibility between librbd + krbd for the data pool
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit baa772372c52ca2f5b27bad59798734def8656f1
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri May 12 10:38:51 2017 +0800
+
+    pybind: fix cephfs.OSError initialization
+
+    Traceback (most recent call last):
+      File "<stdin>", line 1, in <module>
+      File "cephfs.pyx", line 672, in cephfs.LibCephFS.open (/home/zhyan/Ceph/ceph-2/build/src/pybind/cephfs/pyrex/cephfs.c:10160)
+      File "cephfs.pyx", line 155, in cephfs.OSError.__init__ (/home/zhyan/Ceph/ceph-2/build/src/pybind/cephfs/pyrex/cephfs.c:1889)
+    TypeError: __init__() takes exactly 3 positional arguments (2 given)
+
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit e6493f64ba4592b8dca54ece4464efa6c7f331a7)
+
+commit ac9aed9c2c46bd64384ae047f461b990cb3f6ffe
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed May 10 08:13:52 2017 +0800
+
+    pybind: fix open flags calculation
+
+    (O_WRONLY | O_RDWR) is invaild open flags
+
+    Fixes: http://tracker.ceph.com/issues/19890
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 2c25c99cb4572ffae97555a56b24a4c4097dcdec)
+
+commit bd9aec00ec284e2e828625b5e92165465aee38b6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Apr 27 16:21:33 2017 -0400
+
+    test: remove hard-coded image name from RBD metadata test
+
+    Fixes: http://tracker.ceph.com/issues/19798
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8f72e745e157cc12d76ca6babe956c5698ee297f)
+
+commit 474d504dd67378588d2dd243b06b844737ab64d5
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Mar 2 17:18:18 2017 +0100
+
+    librbd: relax "is parent mirrored" check when enabling mirroring for pool
+
+    If the parent is in the same pool and has the journaling feature enabled
+    we can assume the mirroring will eventually be enabled for it.
+
+    Fixes: http://tracker.ceph.com/issues/19130
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit fe31bca22f90ce02f461d6421a4f66539db888d3)
+
+commit 44c116dfd6f519538b6c1401e3c3bc283d31b78a
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Jan 9 10:40:46 2017 +0100
+
+    rbd-mirror: check image mirroring state when bootstrapping
+
+    Fixes: http://tracker.ceph.com/issues/18447
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 5fc5a8ac895524f05eed6e7db20b0dda3a8cb60f)
+
+commit d5fa6a65b52c3f2cf5363ebb55af7e96d8be90ab
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Jan 9 09:23:19 2017 +0100
+
+    rbd-mirror: async request to test if image is primary
+
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 0a1cb35caacdf85029f31a0364dc07a5d7462f5f)
+
+commit 4035dc6fdaf9df06ffeb8b86813046a1b5e0cff2
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Jan 9 15:02:02 2017 +0100
+
+    rbd-mirror: hold owner lock when testing if lock owner
+
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 82aa89668d666f434cd19ff444223017b5512c6a)
+
+commit 76fedab57612ea0e15d18729e779ec43bf830704
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Tue Dec 27 19:39:51 2016 +0800
+
+    os/bluestore: fix OnodeSizeTracking testing
+
+    [  FAILED  ] 1 test, listed below:
+    [  FAILED  ] ObjectStore/StoreTest.OnodeSizeTracking/2, where GetParam() = "bluestore"
+
+     1 FAILED TEST
+
+    The above test failure happens as the bluestore mount() process
+    will try to load all collections and put them into the coll_map,
+    which will be also tracked as mempool::bluestore_meta_other.
+    So total_bytes from mempool won't be equal to zero.
+
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 1d97862f716ce24bfe227b192ecc5e1eb3f76757)
+
+commit 19c7524ace7aeb4f1bc7986162191f4a88bcb4d2
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jun 23 11:12:01 2017 -0400
+
+    qa/tasks/radosbench: increase timeout
+
+    The current timeout isn't enough in some cases (powercycle thrashing leaves
+    osds down for a long time because rebooting is so slow).
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit dfa95db57d2c73923918ecf7014c801a925ddfb7)
+
+commit fa88fc77b4df0c548cda8783334862e33867abac
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Mar 15 14:49:13 2017 -0400
+
+    librbd: image create validates that pool supports overwrites
+
+    Fixes: http://tracker.ceph.com/issues/19081
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a16beba7843539bef4e5035148253a7b594a5cfd)
+
+    Conflicts:
+        src/librbd/image/CreateRequest.cc - kraken uses create_rados_ack_callback;
+            in master, this has been renamed ceph_rados_callback
+
+commit d5aea7a68b8b2603cb9fa57df3a3ae5a44ec450a
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Mar 24 12:15:50 2017 -0400
+
+    os/bluestore/KernelDevice: fix uninit value
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 42db0c70bc7ef595f0925657c043ce081799b2b9)
+
+    Conflicts:
+        src/os/bluestore/KernelDevice.h
+          - flush_lock mutex is still used in kraken, so keep it
+
+commit c08720553f6ae787fe3b0edbdd1497859cdfe0d4
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 9 16:51:21 2017 -0500
+
+    os/bluestore/BlueFS: fix flush_bdev placement
+
+    We need to flush any new writes on any fsync().  Notably, this includes
+    the rocksdb log. However, previously _fsync was only doing a bdev flush if
+    we also had a dirty bluefs journal and called into _sync_and_flush_journal.
+    If we didn't, we weren't doing a flush() at all, which could lead to
+    corrupted data.
+
+    Fix this by moving the first flush_bdev *out* of _sync_and_flush_log.  (The
+    second one is there to flush the bluefs journal; the first one was to
+    ensure prior writes are stable.)  Instead, flush prior writes in all of the
+    callers prior to calling _sync_and_flush_log.  This includes _fsync (and
+    fixes the bug by covering the non-journal-flush path) as well as several
+    other callers.
+
+    Fixes: http://tracker.ceph.com/issues/19250
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2924a96493d8570317e55854a25fc64911ecf151)
+
+commit 594332631e442f3da1b7e6aa49248a6e3c46a30f
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 9 16:51:05 2017 -0500
+
+    os/bluestore/KernelDevice: make flush() thread safe
+
+    flush() may be called from multiple racing threads (notably, rocksdb can call fsync via
+    bluefs at any time), and we need to make sure that if one thread sees the io_since_flush
+    command and does an actual flush, that other racing threads also wait until that flush is
+    complete.  This is accomplished with a simple mutex!
+
+    Also, set the flag on IO *completion*, since flush is only a promise about
+    completed IOs, not submitted IOs.
+
+    Document.
+
+    Fixes: http://tracker.ceph.com/issues/19251
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6b3c52643c8e5fa820c53d96608862b7649c3fd0)
+
+commit dbe8a8ba848d5567db935d7cc945fdd85d8e9d3f
+Merge: ccde4da15c 5dcf60702c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jul 4 13:32:46 2017 +0200
+
+    Merge pull request #14065 from asheplyakov/19212-bp-kraken
+
+    kraken: rgw: "cluster [WRN] bad locator @X on object @X...." in cluster log
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit b941ca7eac3090bf219b6ced4a9239e315f663de
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 15 19:36:08 2017 +0000
+
+    qa/cephfs: use getfattr/setfattr helpers
+
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit dd43d3bc646aeab88486b0963fc83de0b18800c4)
+
+    Conflicts:
+        qa/tasks/cephfs/test_data_scan.py - the master commit drops p =
+            self._mount.run_shell(...) assignment which isn't present in kraken
+            (trivial resolution)
+
+commit f8843445aa1b027c7a2ab5729b715d0da616bfb7
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 15 19:26:30 2017 +0000
+
+    qa: add test for reading quotas from different clients
+
+    Fixes: http://tracker.ceph.com/issues/17939
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 61617f8f10a6322603a9add77980865cd972ef97)
+
+commit b7f46973b03bc9e113ccd55ede0f766a1b1eb0d3
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 15 17:51:44 2017 +0000
+
+    client: _getattr on quota_root before using in statfs
+
+    ...so that after someone adjusts the quota settings
+    on an inode that another client is using as its mount root,
+    the change is visible immediately on the other client.
+
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 3d25941aadd223669448d0f5d3c0bd1fefa72308)
+
+commit 5a69e33057982d78157b75bce3fc458e03260d9e
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 15 15:32:47 2017 +0000
+
+    client: getattr before read on ceph.* xattrs
+
+    Previously we were returning values for quota, layout
+    xattrs without any kind of update -- the user just got
+    whatever happened to be in our cache.
+
+    Clearly this extra round trip has a cost, but reads of
+    these xattrs are fairly rare, happening on admin
+    intervention rather than in normal operation.
+
+    Fixes: http://tracker.ceph.com/issues/17939
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 532dc4b68e538c189ef828f67cecd0d647a62250)
+
+commit 4241a6eef03e93918d18e4a4c2ac38d4a55b82b6
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Mar 27 12:56:31 2017 +0100
+
+    mds: validate prealloc_inos on sessions after load
+
+    Mitigates http://tracker.ceph.com/issues/16842
+
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit c39aaf90ed1b23343eba2b341bb8ee6a50a4ea74)
+
+commit 41be67e80909962a004c66491815195e9a6fbf39
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Mar 27 12:33:59 2017 +0100
+
+    mds: operator<< for Session
+
+    Use this to get a nice human readable name
+    when available (also including the session id in
+    parentheses)
+
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 0f89787d8312f132ebb621f16c44e950b17a395a)
+
+commit ccde4da15c9ea50a4663b0697d0cef827c26f9c7
+Merge: a1ed9bac52 52ee6a4c49
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jul 4 13:18:48 2017 +0200
+
+    Merge pull request #16092 from smithfarm/wip-20150-kraken
+
+    kraken: ceph-disk: fails if OSD udev rule triggers prior to mount of /var
+
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 0b1dee0edcf3fe38289c10262655875d61a0dbbf
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Mar 23 09:07:32 2017 -0400
+
+    mds: include advisory `path` field in damage
+
+    This will just be whatever path we were looking
+    at at the point that damage was notified -- no
+    intention whatsoever of providing any up to date
+    path or resolution when there are multiple paths
+    to an inode.
+
+    Fixes: http://tracker.ceph.com/issues/18509
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit c0bff51ef409eb6e4b2fc248e06e5a7e43faf51e)
+
+    Conflicts:
+        src/mds/CDir.cc - trivial resolution (kraken does not have
+            cb86740a5f4aa3eed43c7f09ac5e7e525a5c1d67)
+
+commit f458d60838628c4ed08448998956bc6ce7228f9b
+Author: Henrik Korkuc <henrik@uber.com>
+Date:   Sun Feb 19 11:44:20 2017 +0200
+
+    client/Client.cc: add feature to reconnect client after MDS reset
+
+    Client.cc marks session as stale instead of reconecting after received
+    reset from MDS. On MDS side session is closed so MDS is ignoring cap
+    renew. This adds option to reconnect stale client sessions instead of
+    just marking sessions stale.
+
+    Fixes: http://tracker.ceph.com/issues/18757
+
+    Signed-off-by: Henrik Korkuc <henrik@kirneh.eu>
+    (cherry picked from commit e0bbc704676ef4aed510daff075ef63c9e73b7b3)
+
+commit 7674f84ddf8f8f8f0a3fe8c3b336bacc0718eeed
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Feb 22 17:33:05 2017 +0800
+
+    client: wait for lastest osdmap when handling set file/dir layout
+
+    Fixes: http://tracker.ceph.com/issues/18914
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 76f5eb86cdd61dde4e6c7cfeb5cf34f0c0334f21)
+
+commit bee73d2429628e7d27cf9b1ca67eb5d5f049f285
+Author: Yang Honggang <joseph.yang@xtaotech.com>
+Date:   Thu Apr 13 20:09:07 2017 +0800
+
+    cephfs: fix write_buf's _len overflow problem
+
+    After I have set about 400 64KB xattr kv pair to a file,
+    mds is crashed. Every time I try to start mds, it will crash again.
+    The root reason is write_buf._len overflowed when doing
+    Journaler::append_entry().
+
+    This patch try to fix this problem through the following changes:
+
+     1. limit file/dir's xattr size
+     2. throttle journal entry append operations
+
+    Fixes: http://tracker.ceph.com/issues/19033
+    Signed-off-by: Yang Honggang joseph.yang@xtaotech.com
+    (cherry picked from commit eb915d0eeccbe523f8f70f6571880003ff459459)
+
+commit 32c7ebe3e6d19d37f198a8ae6bdc46d68fb3ed05
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 8 12:13:46 2017 +0000
+
+    mds: shut down finisher before objecter
+
+    Some of the finisher contexts would try to call into Objecter.
+    We mostly are protected from this by mds_lock+the stopping
+    flag, but at the Filer level there's no mds_lock, so in the
+    case of file size probing we have a problem.
+
+    Fixes: http://tracker.ceph.com/issues/19204
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 177a97d5c55ee6a2d5dcd3cf0893546190b10f7a)
+
+    Conflicts:
+        src/mds/MDSRank.cc - kraken has no 7189b53b410424d4a662486c1081181b2bd662ff
+            so we omit the purge_queue.shutdown() call
+
+commit b267a1a8be0ea4893abb77695f1d1b2a0dac2c90
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Mar 28 14:13:33 2017 -0400
+
+    mds: ignore ENOENT on writing backtrace
+
+    We get ENOENT when a pool doesn't exist.  This can
+    happen because we don't prevent people deleting
+    former cephfs data pools whose files may not have
+    had their metadata flushed yet.
+
+    http://tracker.ceph.com/issues/19401
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 3fccc2372f2715d075b05e459140360cf6e7ca96)
+
+commit 93e81a87ac99080a8670b7485dc9c8c801477518
+Author: YunfeiGuan <yunfei.guan@xtaotech.com>
+Date:   Mon Apr 10 05:48:47 2017 +0000
+
+    cephfs: fix mount point break off problem after mds switch occured
+
+    The hot-standby become active as we expected but the mount piont broken strangely
+    when the active mds is down. The root reason is the new mds use last_cap_renews
+    decoded from ESesson::replay in find_idle_sessions and wrongly killed the session.
+    Maybe we should reset session->last_cap_renew to the current time when server send
+    OPEN to client in reconnect stage.
+
+    Fixes: http://tracker.ceph.com/issues/19437
+    Signed-off-by: Guan yunfei <yunfei.guan@xtaotech.com>
+    (cherry picked from commit 4ef830c5d6f22bf0d4f82a8624c772ecbbda44a6)
+
+commit 78ccba49d3a1f53c3fd7ffe0d6b6bb45eb66c801
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Mar 5 21:59:45 2017 -0500
+
+    common: remove \n on clog messages
+
+    Confirmed these aren't needed, and are inconsistently included.
+    Remove them where they do appear.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 693132eb00b1803d5e97a79908521d5a6903e9f8)
+
+    Conflicts:
+            src/mon/OSDMonitor.cc - trivial resolution
+            src/osd/PrimaryLogPG.cc - trivial resolution
+
+commit e72d6362c15668e2340212dcdeaa07fc94d12ba7
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Apr 5 21:29:10 2017 +0800
+
+    mds: make C_MDSInternalNoop::complete() delete 'this'
+
+    Fixes: http://tracker.ceph.com/issues/19501
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 424e0c6744e7f63459ca0ff7deab751726aa30cd)
+
+    Conflicts:
+        src/mds/MDSContext.h - omit "override" because kraken does not have
+            1a91aeab987870b3ccbcf2f1e476fac8b534d449
+
+commit a1ed9bac523d3115fe5120b2f74f450a739d89b1
+Merge: 4c8d3ed4dc 8d64dd4f29
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jul 4 11:37:29 2017 +0200
+
+    Merge pull request #14604 from asheplyakov/19647-kraken
+
+    kraken: ceph-disk: enable directory backed OSD at boot time
+
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 7054ff69d0450ecd86aff5c9c78a0a93f536d179
+Author: Pan Liu <liupan1111@gmail.com>
+Date:   Fri Mar 31 00:23:12 2017 +0800
+
+    rbd-nbd: remove debug message from do_unmap
+
+    Global context is not initialized when do_unmap is called.
+
+    Signed-off-by: Pan Liu <liupan1111@gmail.com>
+    (cherry picked from commit cd748f9b260a3c388b45091ff0bfc041257acd0c)
+
+commit 36e2199f0bfd869054506476f64af7f603ee6541
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Mar 29 19:07:34 2017 +0800
+
+    rbd-nbd: s/cpp_error/cpp_strerror/ to fix FTBFS
+
+    the build failure was introduced by ff4dcf0
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit c12ecd984df0f9ac0d7de81b9d689f6182a3646a)
+
+commit 4d310c24af51bb0539426a5a7ca6f2b1ca31a1d0
+Author: Pan Liu <liupan1111@gmail.com>
+Date:   Tue Mar 28 16:48:21 2017 +0800
+
+    rbd-nbd: polish the output info before and after ioctl NBD_DISCONNECT.
+
+    Signed-off-by: Pan Liu <liupan1111@gmail.com>
+    (cherry picked from commit ff4dcf029028e8a3636ac71a6c5ac5380bf274e4)
+
+commit e6a01244e6ae2862b1c6e78be67b3c2be6899364
+Author: Pan Liu <liupan1111@gmail.com>
+Date:   Tue Mar 28 16:33:25 2017 +0800
+
+    rbd-nbd: support signal handle for SIGHUP, SIGINT, and SIGTERM.
+
+    Fixes: http://tracker.ceph.com/issues/19349
+    Signed-off-by: Pan Liu <liupan1111@gmail.com>
+    (cherry picked from commit 3ba01aa6ce052d1afa42132feffc2353d73caae6)
+
+commit 5dab0825d705d84926fd5f2f106fa945492bda34
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 12 10:47:28 2017 -0400
+
+    test/librados_test_stub: fixed cls_cxx_map_get_keys/vals return value
+
+    Fixes: http://tracker.ceph.com/issues/19597
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9ffd464dac102f684d6dfa78e58d2cb45e165ed6)
+
+commit 8ed81b033a7a9120a24611c9e193d85e109ad2e5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Mar 15 16:33:58 2017 -0400
+
+    librbd: clean up debug messages within image create state machine
+
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d4e27c48aa3f96f650d6cbfe103be5aa980e0f56)
+
+commit fbd296933bb64239f49909a123fcd654000f910f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Mar 15 14:12:35 2017 -0400
+
+    librbd: removed legacy state machine callbacks from CreateRequest
+
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8d79bf2bcba1d09d8e5589c5a8ee6dca1d27f800)
+
+    Conflicts:
+        src/librbd/image/CreateRequest.cc - kraken does not have 5f2689b62ae
+
+commit f1cf9465984801b17ce536a6a94b2789b482d1a7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Mar 2 10:29:36 2017 -0500
+
+    librbd: avoid duplicating librados IoCtx objects if not needed
+
+    This introduces the potential for shutdown race conditions within
+    the unit tests.
+
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit cc5ac6aa66f4c97cbe1c7d6334b3f710610f6742)
+
+commit 4c8d3ed4dc7fc51d7116bb65915ed5cf8ac468bd
+Merge: 8452c0c134 d186951730
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jul 4 10:26:43 2017 +0200
+
+    Merge pull request #14616 from smithfarm/wip-18723-kraken
+
+    kraken: osd: calc_clone_subsets misuses try_read_lock vs missing
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit b7503d3a80b14dd99f6e46fe63414815afcff814
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Apr 14 15:56:37 2017 -0400
+
+    rgw_file: fix readdir after dirent-change
+
+    Also, fixes link count computation off-by-one, update of state.nlink
+    after computation, link computation reset at start, and a time print
+    in debug log.
+
+    Fixes: http://tracker.ceph.com/issues/19634
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+    link count
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit e0f80266ecd424bf9466579b3edc03911a7c5719)
+
+commit ea3aec3334871c5952986fcf14aac61348f1cc00
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Apr 11 06:42:07 2017 -0400
+
+    rgw_file: don't expire directories being read
+
+    If a readdir expire event turns out to be older than last_readdir,
+    just reschedule it (but actually, we should just discard it, as
+    another expire event must be in queue.
+
+    Fixes: http://tracker.ceph.com/issues/19625
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 007b7451c26716c51207c161dc347e9a00da53f1)
+
+commit 247f897c31f15c48fa0521311b1afc557e5a10ae
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Apr 11 05:56:13 2017 -0400
+
+    rgw_file:  chunked readdir
+
+    Adjust readdir callback path for new nfs-ganesha chunked readdir,
+    including changes to respect the result of callback to not
+    continue.
+
+    Pending introduction of offset name hint, our caller will just be
+    completely enumerating, so it is possible to remove the offset map
+    and just keep a last offset.
+
+    Fixes: http://tracker.ceph.com/issues/19624
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit e0191d74e3aef06bf300df045a53a3952a71f651)
+
+commit ac1dd8d323c5d90e20fd88e56031441f704ee95a
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Feb 28 20:24:12 2017 -0500
+
+    rgw_file: RGWFileHandle dtor must also cond-unlink from FHCache
+
+    Formerly masked in part by the reclaim() action, direct-delete now
+    substitutes for reclaim() iff its LRU lane is over its high-water
+    mark, and in particular, like reclaim() the destructor is certain
+    to see handles still interned on the FHcache when nfs-ganesha is
+    recycling objects from its own LRU.
+
+    Fixes: http://tracker.ceph.com/issues/19112
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit d51a3b1224ba62bb53c6c2c7751fcf7853c35a4b)
+
+commit 78e251632dd3103c5476544ea7386203bd5ff7a2
+Author: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+Date:   Thu Mar 2 17:21:57 2017 +0800
+
+    rgw_file: posix style atime,ctime,mtime
+
+    As an ganesha FSAL backend, rgw_file should properly maintain
+    the atime,ctime,mtime properly against operations such as:
+            (read,write) for file
+            (create,unlink,mkdir,rmdir,rename) for dir
+            (setattr) for file and dir
+
+    Signed-off-by: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+    (cherry picked from commit ac25da2479b9be876cbdb820560ac46a6e2b17d7)
+
+commit 59a965a8e017da3dc6c79a25d55441b945cd50bc
+Author: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+Date:   Wed Mar 8 16:23:11 2017 +0800
+
+    rgw_file: fix reversed return value of getattr
+
+    When ::getattr returns -ESTALE, rgw_getattr returns ESTALE,
+    which is a not expected postive.
+
+    Signed-off-by: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+    (cherry picked from commit 39203cf872b8f4af86eb0e4a0f96dffd9cc92b41)
+
+commit e5c78f8bf2733b803f4e8e051884c8130809e506
+Author: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+Date:   Wed Mar 15 15:01:05 2017 +0800
+
+    rgw_file: fix double unref on rgw_fh for rename
+
+    Skip unref after unlink to fix the problem.
+
+    Signed-off-by: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+    (cherry picked from commit bff228734c73b536d2482e2e2fa4ad38b206ebff)
+
+commit f97ab40539385494f9d4a66806c2bfc08d3654bb
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Feb 22 14:57:59 2017 -0500
+
+    rgw_file: rgw_lookup: don't ref for "/" or ".."
+
+    These refs won't be returned by nfs-ganesha, and are sufficiently
+    magical that other consumers should be persuaded to understand
+    their specialness.
+
+    Fixes: http://tracker.ceph.com/issues/19060
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit dea8d1ee373399a21851690a9753388b659b8ede)
+
+commit 6e0b260154328b14c45ecbea00007e537ce220ac
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Feb 23 16:02:07 2017 -0500
+
+    rgw_file: ensure valid_s3_object_name for directories, too
+
+    The logic in RGWLibFS::mkdir() validated bucket names, but not
+    object names (though RGWLibFS::create() did so).
+
+    The negative side effect of this was not creating illegal objects
+    (we won't), but in a) failing with -EIO and b) more importantly,
+    not removing up the proposed object from FHCache, which produced a
+    boost assert when recycled.
+
+    Fixes: http://tracker.ceph.com/issues/19066
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit eb1cd3b30c0504385f05bf2d2dd5e2251b7efed7)
+
+commit 349de8a5b30691b0579708ce95f91ce8b929a393
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Feb 3 13:44:45 2017 -0500
+
+    rgw_file:  fix RGWLibFS::setattr for directory objects
+
+    Fixes:  http://tracker.ceph.com/issues/18808
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 4ad5a9226852d6d564baf2e63278ed6c4c185ecb)
+
+commit 2b9a77edf203ef39049677a94e13db0fdf0fed8d
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Apr 4 20:16:13 2017 -0400
+
+    rgw_file: introduce rgw_lookup type hints
+
+    The new type hints optimize object type deduction, when the
+    rgw_lookup is called from an rgw_readdir callback.
+
+    Fixes: http://tracker.ceph.com/issues/19623
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 2e66c7a7cc763c5c0d6f5db04855f60f2b2ceed3)
+
+commit 64cce346c0fcc1402d4dab548f6f7536430703e7
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Feb 23 10:21:38 2017 -0500
+
+    rgw_file:  return of RGWFileHandle::FLAG_EXACT_MATCH
+
+    Allow callers of rgw_lookup() on objects attested in an
+    rgw_readdir() callback the ability to bypass exact match in
+    RGWLibFS::stat_leaf() case 2, but restore exact match enforcement
+    for general lookups.
+
+    This preserves required common_prefix namespace behavior, but
+    prevents clients from eerily permitting things like "cd sara0" via
+    partial name match on "sara01."
+
+    Fixes: http://tracker.ceph.com/issues/19059
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 70ef7d45e0abf2661bd4e23161d4e70cf5178079)
+
+commit fa208008dad8cb165ee72a8cf96ead36d855ba30
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Feb 28 15:49:06 2017 -0500
+
+    rgw_file:  use fh_hook::is_linked() to check residence
+
+    Previously we assumed that !deleted handles were resident--there
+    is an observed case where a !deleted handle is !linked.  Since
+    we currently use safe_link mode, an is_linked() check is
+    available, and exhaustive.
+
+    Fixes: http://tracker.ceph.com/issues/19111
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit c0aa515f8d8c57ec5ee09e3b60df3cac60453c40)
+
+commit 29fe4cfeb71a1b0a74fb7a073c2348cf2d1d143d
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Feb 12 18:20:43 2017 -0500
+
+    rgw_file: refcnt bugfixes
+
+    This change includes 3 related changes:
+
+    1. add required lock flags for FHCache updates--this is a crash
+       bug under concurrent update/lookup
+
+    2. omit to inc/dec refcnt on root filehandles in 2 places--the
+       root handle current is not on the lru list, so it's not
+       valid to do so
+
+    3. based on observation of LRU behavior during creates/deletes,
+       update (cohort) LRU unref to move objects to LRU when their
+       refcount falls to SENTINEL_REFCNT--this cheaply primes the
+       current reclaim() mechanism, so very significanty improves
+       space use (e.g., after deletes) in the absence of scans
+       (which is common due to nfs-ganesha caching)
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit beaeff059375b44188160dbde8a81dd4f4f8c6eb)
+
+commit 99941105a982f77513c8df576176f4a707daf26a
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sat Feb 11 16:38:05 2017 -0500
+
+    rgw_file:  add refcount dout traces at debuglevel 17
+
+    These are helpful for checking RGWFileHandle refcnt invariants.
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 462034e17f919fb783ee33e2c9fa8089f93fd97d)
+
+commit a446c724aa2404f4f4ff8efc2da3e12ca2bf6892
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Feb 10 17:14:16 2017 -0500
+
+    rgw_file: add pretty-print for RGWFileHandle
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit ef330f385d3407af5f470b5093145f59cc4dcc79)
+
+commit 1553877b1673b21694a4a3b3ef9811800e67144e
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Dec 30 23:30:16 2016 -0500
+
+    rgw_file:  interned RGWFileHandle objects need parent refs
+
+    RGW NFS fhcache/RGWFileHandle operators assume existence of the
+    full chain of parents from any object to the its fs_root--this is
+    a consequence of the weakly-connected namespace design goal, and
+    not a defect.
+
+    This change ensures the invariant by taking a parent ref when
+    objects are interned (when a parent ref is guaranteed).  Parent
+    refs are returned when objects are destroyed--essentially by the
+    invariant, such a ref must exist.
+
+    The extra ref is omitted when parent->is_root(), as that node is
+    not in the LRU cache.
+
+    Fixes: http://tracker.ceph.com/issues/18650
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 0e5299f3f43e633a5d8a9360893b4b11f6217d81)
+
+commit 4af9f6fef77597a826cfdf5d0b3a814ce4629bc3
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Feb 22 10:24:29 2017 -0500
+
+    rgw_file: avoid stranding invalid-name bucket handles in fhcache
+
+    To avoid a string copy in the common mkdir path, handles for
+    proposed buckets currently are staged in the handle table, before
+    being rejected.  They need to be destaged, not just marked deleted
+    (because deleted objects are now assumed not to be linked, as of
+    beaeff059375b44188160dbde8a81dd4f4f8c6eb).
+
+    This triggered an unhandled Boost assert when deleting staged
+    handles, as current safe_link mode requires first removing from
+    the FHCache.
+
+    Fixes: http://tracker.ceph.com/issues/19036
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 6cde812c92e5bba9f85fbf8486ebe69b55952370)
+
+commit e03a5cba8ff4a2ceebf85a2697e90d525185b2d2
+Author: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+Date:   Fri Mar 31 10:42:40 2017 +0800
+
+    rgw_file: fix missing unlock in unlink
+
+    Fixes: http://tracker.ceph.com/issues/19435
+
+    Signed-off-by: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+    (cherry picked from commit cb6808a6366a70f54d0cc16437d16aa1b7819c84)
+
+commit 34553f919ccf0324138b990ff928b32dc0e4baf2
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Mar 15 16:35:16 2017 -0400
+
+    rgw_file:  remove unused rgw_key variable
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 1100a1c26e76485569cfebcf863b18cf908f6161)
+
+commit b04d051fd765a88a270041c5238a65481180528c
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Wed Mar 15 16:40:35 2017 -0400
+
+    rgw_file:  rgw_readdir:  return dot-dirs only when *offset is 0
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 61482c2b85a07519f2256b1a3f2b6d8aa99d5f06)
+
+commit b350b61bd2c75bc176f854bb3a39a453a2faec97
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Mar 13 21:52:08 2017 -0400
+
+    rgw_file: implement reliable has-children check (unlink dir)
+
+    Bug report and discussion provided by
+    Gui Hecheng <guihecheng@cmss.chinamobile.com> in nfs-ganesha upstream
+    github.  Briefly, while a reliable check is potentially costly,
+    it is necessary.
+
+    Fixes: http://tracker.ceph.com/issues/19270
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit b05f1c6d61aa4501a971e87de6dcaf3e58c3d9b4)
+
+commit 7faa917cdb1c484018dd47559c5f25b6b33b5529
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Mon Feb 20 15:05:18 2017 -0500
+
+    rgw_file: fix marker computation
+
+    Fixes: http://tracker.ceph.com/issues/19018
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 4454765e7dd08535c50d24205858e18dba4b454c)
+
+commit 50955a5aef10f18f050735af03734189f519f585
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Jan 19 18:14:30 2017 -0500
+
+    rgw_file: add timed namespace invalidation
+
+    With change, librgw/rgw_file consumers can provide an invalidation
+    callback, which is used by the library to invalidate directories
+    whose contents should be forgotten.
+
+    The existing RGWLib GC mechanism is being used to drive this.  New
+    configuration params have been added.  The main configurable is
+    rgw_nfs_namespace_expire_secs, the expire timeout.
+
+    Updated post Yehuda review.
+
+    Fixes: http://tracker.ceph.com/issues/18651
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit deb2c1ea985fcb906e47b93fd3d0117794e2d0a1)
+
+    Conflicts:
+        src/rgw/rgw_lib_frontend.h - in class RGWLibProcess : public RGWProcess
+                               there was no public method stop() in kraken (now there is)
+
+commit c613ee7e5853c40c89215efdc231c9e5274aec58
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Feb 19 20:34:31 2017 -0500
+
+    rgw_file: rgw_readdir can't list multi-segment dirs
+
+    This issue has one root cause in librgw, namely that the marker
+    argument to these requests was incorrectly formatted (though the
+    marker cache was working as intended).
+
+    Secondarily, for nfs-ganesha users, there is a compounding issue
+    that the RGW fsal was required by "temporary" convention to
+    populate the entire dirent cache for a directory on a single
+    readdir() invocation--the cache_inode/mdcache implementations
+    invariantly pass (before future 2.5 changesets, currently in
+    progress) a null pointer for the start cookie offset, intended
+    to convey this.
+
+    Fixes: http://tracker.ceph.com/issues/18991
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 2cd60ee9712291b906123aca1704288b18a9742b)
+
+commit 2e27aa9bb2484512620d885c0a19e35b1de7b652
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Feb 19 18:21:06 2017 -0500
+
+    rgw_file: allow setattr on placeholder directories
+
+    When a POSIX path <bucket>/foo/ is known only as an implicit path
+    segment from other objects (e.g., <bucket>/foo/bar.txt), a case
+    that would usually arise from S3 upload of such an object, an
+    RGWFileHandle object representing "<bucket>/foo/" will be constructed
+    as needed, with no backing in RGW.
+
+    This is by design, but subsequently, if a setattr is performed on
+    such a handle, we must be ready to create the object inline with
+    storing the attributes.
+
+    Fixes: http://tracker.ceph.com/issues/18989
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 55eec1c0a0e136736961423b7b6244d0f3693c1a)
+
+commit 70ebe961de881734b637c7d13e7dd63bc09066c6
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Feb 19 17:43:17 2017 -0500
+
+    rgw_file: invalid use of RGWFileHandle::FLAG_EXACT_MATCH
+
+    The change which introduced this flag also caused it to be
+    given as the flags argument to RGWLibFS::stat_leaf() when called
+    from rgw_lookup().
+
+    This was incorrect:  in particular, when a directory is known only
+    as a common prefix of other objects, the AWS namespace mapping
+    convention requires lookup("foo") to match a non-materialized
+    instance of "foo/" (case 2 in RGWLibFS::stat_leaf's stat loop).
+
+    Fixes: http://tracker.ceph.com/issues/18992
+
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit e31e9eb980f958640150e8d7f17de1b9e5478b1e)
+
+commit 52ee6a4c49e5749463811b57f4af4a0824875b70
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Jun 1 11:37:20 2017 +0200
+
+    ceph-disk: do not setup_statedir on trigger
+
+    trigger may run when statedir is unavailable and does not use it.
+
+    Fixes: http://tracker.ceph.com/issues/19941
+
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 16bfbdd3d9988523bba31aace516c303057daa58)
+
+commit 8452c0c134a1cae30ac9e80eb7e08c6e54b16239
+Merge: e8312132ff 75cdc3f3a1
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jul 4 00:02:05 2017 +0200
+
+    Merge pull request #14852 from yehudasa/wip-rgw-support-ragweed-kraken
+
+    kraken: rgw: add apis to support ragweed suite
+
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit e8312132ff8caee9b0f83b641bfc0a1875bacf3f
+Merge: 7540396a34 6e5ac14ab6
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 3 22:28:08 2017 +0200
+
+    Merge pull request #13514 from rzarzynski/wip-rgw-controllable-204-kraken
+
+    kraken: rgw: make sending Content-Length in 204 and 304 controllable
+
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit 7540396a34147fbb34558356fde18f0a11798d8f
+Merge: f31033cb5b 910b989970
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 3 17:05:34 2017 +0200
+
+    Merge pull request #14403 from shinobu-x/wip-17331-kraken
+
+    kraken: ceph-disk list reports mount error for OSD having mount options with SELinux context
+
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 071f111abbd7d6c7a90fc58f52c84907cf4d9860
+Author: Wido den Hollander <wido@42on.com>
+Date:   Mon Jan 23 21:06:05 2017 +0100
+
+    osd: Return correct osd_objectstore in OSD metadata
+
+    Do not simply read the configuration value as it might have changed
+    during OSD startup by reading the type from disk.
+
+    Fixes: http://tracker.ceph.com/issues/18638
+
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit 8fe6a0303b02ac1033f5bfced9f94350fe3e33de)
+
+    Conflicts:
+        src/osd/OSD.cc
+          - g_conf->osd_objectstore was changed to cct->_conf->osd_objectstore by
+            1d5e967a05ddbcceb10efe3b57e242b3b6b7eb8c which is not in kraken
+
+commit ffe5bbfc6e09a47009dd281a38b35a8a3528812e
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jan 26 16:26:42 2017 -0800
+
+    rgw: fix use of marker in List::list_objects()
+
+    Fixes: http://tracker.ceph.com/issues/18331
+
+    List marker is an index key, so treat it as such. This
+    fixes infinite loop in orphans find command.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit a5d1fa0587184f43c69d8e03114b58d43f320781)
+
+commit 56ca3b01906ad7ee25b4bc2db0b6df51d2b162cb
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue Apr 25 12:22:54 2017 +0200
+
+    rgw: fix RadosGW hang during multi-chunk upload of AWSv4.
+
+    Fixes: http://tracker.ceph.com/issues/19754
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 72c1e2e351d984d0425a20f2c772951cbc36f13e)
+
+commit cac26471897b6126a9b95e02f1a93a3a5386dae1
+Author: liuchang0812 <liuchang0812@gmail.com>
+Date:   Mon Mar 27 13:08:12 2017 +0800
+
+    rgw/lifecycle: do not send lifecycle rules when GetLifeCycle failed
+
+    Now, RGW will send two HTTP responses when GetLifeCycle failed. The first one is
+    Error Respnse like 404, and the second is lifecycle rules. It will breaks s3 sdk
+    and s3 utilities.
+
+    Fixes: http://tracker.ceph.com/issues/19363
+    Signed-off-by: liuchang0812 <liuchang0812@gmail.com>
+    (cherry picked from commit c3c0c828da5a64ca896475c1b0c369fde1bbd76a)
+
+commit e5719234b48a90bbe59a693c96da3634884a0578
+Author: tone-zhang <tone.zhang@linaro.org>
+Date:   Thu Apr 6 17:56:05 2017 +0800
+
+      rgw: fix response header of Swift API
+
+    Response header of Swift API returned by radosgw does not contain
+    "x-openstack-request-id", but Swift returns it. Enhance the
+    compatibility of radosgw.
+
+    Fixes: http://tracker.ceph.com/issues/19443
+
+    Signed-off-by: tone-zhang <tone.zhang@linaro.org>
+    (cherry picked from commit e96db213079ab5e026156ab4b38418d1d4c23d27)
+
+commit 59391c40db969f2a9e97505cfcd2997d02ea86a3
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Mar 8 14:52:34 2017 -0800
+
+    rgw: fix crash when listing objects via swift
+
+    Fixes: http://tracker.ceph.com/issues/19249
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit a9ec5e8ce184e19c009863db4d3519f9d8af91bd)
+
+    Conflicts:
+            src/rgw/rgw_rest_swift.cc ("key" element of RGWObjEnt struct
+               is not a reference; fix)
+
+commit e0f90f03ddf2824b448f6affc34ed40d4cead5d2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 26 14:02:11 2017 -0500
+
+    librbd: introduce new constants for tracking max block name prefix
+
+    Fixes: http://tracker.ceph.com/issues/18653
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2c08629c99d90aa7676b59263c055c9f1f577039)
+
+commit 3173da6e3bc7083ed2bdeea3a25bb3fa790bbc86
+Author: Pan Liu <pan.liu@istuary.com>
+Date:   Thu Feb 16 22:17:52 2017 +0800
+
+    rbd-nbd: no need create asok file for unmap and list-mapped commands.
+
+    Fixes: http://tracker.ceph.com/issues/17951
+    Signed-off-by: Pan Liu <liupan1111@gmail.com>
+    (cherry picked from commit 72352653d585ef89043a4ece371b5c0cb3f6f32a)
+
+commit 61c5957b30f807f5eab4c072deb00f19433f0a46
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Feb 2 11:11:35 2017 +0100
+
+    rbd-nbd: check /sys/block/nbdX/size to ensure kernel mapped correctly
+
+    Fixes: http://tracker.ceph.com/issues/18335
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 596e5ea8a5df72002672eef0a6d20572ca6f60f0)
+
+commit c79d45ff52a636c136e0c5f8aec7911c04601b6f
+Author: Venky Shankar <vshankar@redhat.com>
+Date:   Mon Feb 20 12:04:10 2017 +0530
+
+    librbd: acquire exclusive-lock during copy on read
+
+    Fixes: http://tracker.ceph.com/issues/18888
+    Signed-off-by: Venky Shankar <vshankar@redhat.com>
+    (cherry picked from commit 7dba5311b12011a4a6e8564e68150e54c5af5ddd)
+
+    Conflicts:
+        src/librbd/AioImageRequestWQ.h:
+          - in master this file has morphed into src/librbd/io/ImageRequestWQ.h
+          - kraken has AioImageRequest<ImageCtx> instead of ImageRequest<ImageCtx>
+        src/librbd/image/RefreshRequest.cc:
+          - rename image context element to "aio_work_queue" (from "io_work_queue")
+            because kraken doesn't have de95d862f57b56738e04d77f2351622f83f17f4a
+        src/test/librbd/image/test_mock_RefreshRequest.cc:
+          - rename image context element to "aio_work_queue" (from "io_work_queue")
+            because kraken doesn't have de95d862f57b56738e04d77f2351622f83f17f4a
+
+commit d186951730d947881fea0e638a6e112718820ae0
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Jan 18 10:24:13 2017 -0800
+
+    PrimaryLogPG::try_lock_for_read: give up if missing
+
+    The only users calc_*_subsets might try to read_lock an object which is
+    missing on the primary.  Returning false in those cases is perfectly
+    reasonable and avoids the problem.
+
+    Fixes: http://tracker.ceph.com/issues/18583
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 3833440adea6f8bcb0093603c3a9d16360ed57ec)
+
+commit 30ff11c84fd6d53c6439aaed8f7742b98a5b6218
+Author: Samuel Just <sjust@redhat.com>
+Date:   Wed Nov 23 15:41:13 2016 -0800
+
+    ReplicatedBackend: take read locks for clone sources during recovery
+
+    Otherwise, we run the risk of a clone source which hasn't actually
+    come into existence yet being used if we grab a clone which *just*
+    got added the the ssc, but has not yet actually had time to be
+    created (can't rely on message ordering here since recovery messages
+    don't necessarily order with client IO!).
+
+    Fixes: http://tracker.ceph.com/issues/17831
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 68defc2b0561414711d4dd0a76bc5d0f46f8a3f8)
+
+commit e76725f4e9fb4a6cd7c76480d0313a8e0264eb73
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Mar 30 13:50:41 2017 -0400
+
+    osd/PrimaryLogPG: do not expect FULL_TRY ops to get resent
+
+    The objecter will not resend FULL_TRY requests that were sent prior to
+    becoming full, so we should not discard them.
+
+    Fixes: http://tracker.ceph.com/issues/19430
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3f7acdbc9a942fd18937dbcf07fbc7b752c50ba3)
+
+commit 7a275769b6a281eaec1a659975ec50bfe847e679
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Mar 31 10:06:42 2017 -0400
+
+    ceph_test_librados_api_misc: fix stupid LibRadosMiscConnectFailure.ConnectFailure test
+
+    Sometimes the cond doesn't time out and it wakes up instead.  Just repeat
+    the test many times to ensure that at least once it times out (usually
+    it doesn't; it's pretty infrequent that it doesn't).
+
+    Fixes: http://tracker.ceph.com/issues/15368
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8bc197400d94ee2716d3f2fa454247379a676cf9)
+
+commit 4d1272e5c654c32a844a791b08a97ea8c28c5e5d
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Apr 18 14:07:04 2017 +0800
+
+    debian: package ceph.logroate properly
+
+    see also "man dh_installlogrotate"
+
+    Fixes: http://tracker.ceph.com/issues/19390
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 17ca501fe8927d541d50a502da53a4d7628d2b4a)
+
+commit 47a45441762e610eef548273c719cf70643bfeac
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Apr 17 14:58:02 2017 -0700
+
+    osd: Give requested scrub work a higher priority
+
+    Once started we now queue scrub work at higher priority than
+    scheduled scrubs.
+
+    Fixes: http://tracker.ceph.com/issues/15789
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit ebab8b1f4f67fbdec1e147c580329c1e2b5cf7cd)
+
+    Conflicts:
+            src/osd/OSD.h - in kraken, the PGScrub() call is enclosed within
+                       op_wq.queue(make_pair(...)) instead of enqueue_back()
+
+commit f31033cb5bc7764b44871c15717aed5a34921881
+Merge: b7a2e67460 795538210c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jul 3 11:39:42 2017 +0200
+
+    Merge pull request #16069 from smithfarm/wip-20345-kraken
+
+    kraken: make check fails with Error EIO: load dlopen(build/lib/libec_FAKE.so): build/lib/libec_FAKE.so: cannot open shared object file: No such file or directory
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 795538210ce76adc7521fac2caf5de27856aee39
+Author: Kyr Shatskyy <kyrylo.shatskyy@gmail.com>
+Date:   Wed Feb 8 16:02:52 2017 +0100
+
+    ceph.spec.in, debian/control: Add bc to build dependencies
+
+    The bc is missing for ceph-helpers.sh
+
+    Fixes: http://tracker.ceph.com/issues/18876
+    Signed-off-by: Kyr Shatskyy <kyrylo.shatskyy@gmail.com>
+    (cherry picked from commit 3ff4be6d6896346b7b9ec7f158fcde8866faeb38)
+
+commit 977770d828252c25004cb6344c43d85abd8ee6f5
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Apr 12 17:31:52 2017 +0800
+
+    qa/workunits/ceph-helpers: do not error out if is_clean
+
+    it would be a race otherwise, because we cannot be sure that the cluster
+    pgs are not all clean or not when run_osd() returns, but we can be sure
+    that they are expected to active+clean after a while. that's what
+    wait_for_clean() does.
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 6cb4503a40ae4ebee5690fda024cae8d1a506bce)
+
+commit b1d3c1a322dd611b3608732e7ef31cf15e671845
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Apr 12 12:33:53 2017 +0800
+
+    qa/workunits/ceph-helpers: display rejected string
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 0196e154ed2e164cf55b0d7ed9f9cdd1f4f50100)
+
+commit 75cdc3f3a16d3799ae417862b9b5ee41cfcaee3e
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Jan 9 13:04:43 2017 -0800
+
+    rgw: new rest api to retrieve object layout
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 2768583dc486109e49d209243675b99fdd39e92c)
+
+commit df30ac30112a0c15798c868dc1fb6bd58bcc05a2
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu Jan 5 13:47:24 2017 -0800
+
+    rgw: rest api to read zone config params
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit a220a2efbfd675e6abf14ae33c21005bcbf6dadf)
+
+commit 47f751f48ba972f2c93ecad919c66ee03b4aec70
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Thu May 4 10:59:07 2017 -0700
+
+    civetweb: move to post 1.8 version
+
+    Fixes: http://tracker.ceph.com/issues/19704
+
+    Version that went into kraken was pre 1.8, and had a few issues.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit ff19977fbc4cf4a787af80b6c0457df932004fd3
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Tue Jun 27 16:07:01 2017 +0400
+
+    kraken: osd: unlock sdata_op_ordering_lock with sdata_lock hold to avoid missing wakeup signal
+
+    Based on commit bc683385819146f3f6f096ceec97e1226a3cd237. The OSD code has
+    been refactored a lot since Kraken, hence cherry-picking that patch introduces
+    a lot of unrelated changes, and is much more difficult than reusing the idea.
+
+    Fixes: http://tracker.ceph.com/issues/20443
+
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+
+commit 9efeb6bc2df559f8d91c480c396b4dd97b981d34
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 4 17:56:22 2017 -0400
+
+    librbd: add no-op event when promoting an image
+
+    The rbd-mirror process needs an event in the journal
+    to properly detect the transition between primary and
+    non-primary state between peers.
+
+    Fixes: http://tracker.ceph.com/issues/19858
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4031555dda7597d24e9eb04b9ff29173909586f7)
+
+    Conflicts:
+            src/librbd/journal/DemoteRequest.cc: logic exists in Journal.cc
+
+commit 1d71475ff561c7f7d4936774d0ff821786b15fe6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu May 4 17:57:34 2017 -0400
+
+    rbd-mirror: prevent infinite loop when computing replay status
+
+    If the image had a non-primary predecessor epoch whose tag tid
+    duplicates an epoch within its own journal, an infinite loop
+    would result.
+
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3f179bf86216540d8e25aad469c604f96f6aecd8)
+
+commit b7a2e674601133f481449d3f8cdd3512ccbad51c
+Merge: 2f5c65bfc2 9fd233c2d4
+Author: Zack Cerza <zack@cerza.org>
+Date:   Mon Jun 26 14:11:56 2017 -0600
+
+    Merge pull request #15869 from smithfarm/wip-swift-task-move-kraken
+
+    tests: move swift.py task from teuthology to ceph, phase one (kraken)
+
+commit 9fd233c2d406a31bfc35993539d60e0b41a53cc0
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Sun Jun 25 12:42:36 2017 +0200
+
+    tests: swift.py: tweak imports
+
+    The ".." form only works within the teuthology repo. With swift.py now in the
+    Ceph repo, we have to be explicit.
+
+    Error message was: "ValueError: Attempted relative import beyond toplevel
+    package
+
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 5d8ffee4d07d21f12a748af2a86820b1307f2e7c
+Merge: 2f5c65bfc2 7b58ac97e9
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jun 23 08:34:53 2017 +0200
+
+    Merge branch 'master' of /home/smithfarm/src/ceph/upstream/teuthology into wip-swift-task-move-kraken
+
+commit 7b58ac97e9dd195f4170e9e0ea00bae76d1f3ccd
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Jun 23 08:27:42 2017 +0200
+
+    tests: move swift.py task to qa/tasks
+
+    In preparation for moving this task from ceph/teuthology.git into ceph/ceph.git
+
+    The move is necessary because jewel-specific changes are needed, yet teuthology
+    does not maintain a separate branch for jewel. Also, swift.py is a
+    Ceph-specific task so it makes more sense to have it in Ceph.
+
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+
+commit 73fe367bc4bf27730479bdc46931dd7c3c710c39
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon May 29 21:55:33 2017 -0400
+
+    os/bluestore: deep decode onode value
+
+    In particular, we want the attrs (map<string,bufferptr>) to be a deep
+    decode so that we do not pin this buffer, and so that any changed attr
+    will free the previous memory.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f53f56350b893bfdd47fe730a4339ca5793791a0)
+
+commit 2f5c65bfc229cb43ac5c193fed7c9f51bd20cf79
+Merge: 4b1e8bcf4c f96262fc9c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jun 20 22:59:27 2017 +0200
+
+    Merge pull request #13181 from smithfarm/wip-18548-kraken
+
+    kraken: rgw: multisite: segfault after changing value of rgw_data_log_num_shards
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 4b1e8bcf4c86168ec66808a9a88a6fa556e97445
+Merge: 7706a386e2 291ffe1801
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jun 20 22:56:59 2017 +0200
+
+    Merge pull request #13838 from smithfarm/wip-19049-kraken
+
+    kraken: rgw: multisite: some yields in RGWMetaSyncShardCR::full_sync() resume in incremental_sync()
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 7706a386e255eec46fafa58ea1435293581218cc
+Merge: 2f4a775d5f d1ec59544b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jun 20 22:52:13 2017 +0200
+
+    Merge pull request #13224 from smithfarm/wip-18780-kraken
+
+    kraken: rgw: Swift API: spurious newline after http body causes weird errors
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit d38a7529995cd13b1a3d13a8e00d22ce0172aa5c
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri May 5 12:02:05 2017 +0800
+
+    mon: check is_shutdown() in timer callbacks
+
+    introduce a helper class: C_MonContext, and initialize all timer events
+    using it, to ensure that they do check is_shutdown() before doing their
+    work.
+
+    Fixes: http://tracker.ceph.com/issues/19825
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 561cbded0c7e28231b1c7ce18663b8d7d40aad6d)
+
+commit e7361176f4ad1d4b43a393e6e34c50233a73e829
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu May 4 22:49:04 2017 +0800
+
+    mon/Elector: call cancel_timer() in shutdown()
+
+    instead of doing it manually.
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 12139ae529a49b6caedea89f910d034ddca094b6)
+
+commit bf347e92a2d67032f4365a125f3bb9975930e912
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Wed May 17 17:50:10 2017 +0400
+
+    kraken: mon: add override annotation to callback classes
+
+    The only purpose of this patch is to avoid merge conflicts while
+    cherry-picking commit 561cbded0c7e28231b1c7ce18663b8d7d40aad6d.
+    Alternatively one could cherry-pick 1effdfe19bf9fd6d546620b96eaf452e889b15dc,
+    but that one brings a lot of unrelated changes.
+
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+
+commit 3c4a5ea385c7d2f2dfe88328e2c5f7778928d92d
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Thu Apr 20 18:38:43 2017 +0200
+
+    fs: normalize file open flags internally used by cephfs
+
+    The file open flags (O_foo) are platform specific. Normalize these flags
+    before they are send to the MDS. For processing of client messages the
+    MDS should only compare to these normalized flags.
+    Otherwise this can lead to bogus flags getting transmitted on ppc64.
+
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 88d2da5e93198e69435e288ce00d216d5fe27f80)
+
+    Conflicts:
+            src/client/Client.cc - trivial resolution, add
+              ceph_flags_sys2wire when flags are logged
+
+commit b19d6eb3110cdf121c086a754176005a0e62797f
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Apr 24 16:23:21 2017 +0200
+
+    test/librbd/test_notify.py: don't disable feature in slave
+
+    On jewel it will have stolen the exclusive lock. Instead, ensure that
+    object map and fast diff are already disabled on the clone before the
+    start of the test.
+
+    Fixes: http://tracker.ceph.com/issues/19716
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit e009e1bdd4b3997462feb9a050bd2eb201e028ba)
+
+commit 2f4a775d5ffd170903c4412e4274ab2a188922ef
+Merge: 358081d27a 4776067797
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Jun 20 08:46:28 2017 +0200
+
+    Merge pull request #13174 from smithfarm/wip-18711-kraken
+
+    kraken: rgw: slave zonegroup cannot enable the bucket versioning
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit f8db63f24dcb0d3aa93eba84e80a3450fb4bd666
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Mar 8 16:31:34 2017 -0500
+
+    rgw: data sync skips slo data when syncing the manifest object
+
+    Fixes: http://tracker.ceph.com/issues/19027
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 8b69847d7b3e92c70090d1dddf7cea5c44fb6b20)
+
+    Conflicts:
+            src/rgw/rgw_rest_conn.cc (kraken is missing
+                50c522ea89a756123bf74ab615138cf8478b2cee)
+            src/rgw/rgw_rest_s3.h (omitted override because kraken is missing
+                3b247475a2c35526c129535021adfa621ecb5327)
+
+commit e3d4d5791867009c8c4b4bc867b65a23116653ae
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Mar 13 11:33:02 2017 -0400
+
+    rgw: RGWGetObj applies skip_manifest flag to SLO
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 987377ae34382e107e1d54f0bfc1121fcedb4513)
+
+commit 6c1ba33a7bdffb7fdaf288596acf628ea8ed471c
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Feb 21 10:27:13 2017 -0500
+
+    rgw: allow system users to read SLO parts
+
+    multisite data sync relies on fetching the object as the system user
+
+    Fixes: http://tracker.ceph.com/issues/19027
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit d50d18c500fd5dd89e7cada1162cf453b36df370)
+
+commit 6860fce4243f0a02bf424a477bf554da608cf90c
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 22 15:05:30 2017 +0200
+
+    rgw: add check for update return value
+
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 125955e0625461065dc4755b900e51c3598cadb4)
+
+commit b9c7c6503eff88bccea3bb19b4edeba89cb73718
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 22 14:42:14 2017 +0200
+
+    rgw: we need to reinit the zonegroup after assignment to avoid invalid cct and store
+
+    Fixes: http://tracker.ceph.com/issues/18631
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit ac9a7565ddf801121f22476cf3f66668f311833e)
+
+commit db6fb3c005e05f0c6f503a367919e31943044e6b
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 22 14:40:16 2017 +0200
+
+    rgw: fix init_zg_from_period when default zone is not set as default
+
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 5393077e07bed45b9fc007591d365f1229d3e815)
+
+commit 5f4c38fa6b2162cb97162089a8a22095a7cde99d
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jan 20 17:05:24 2017 -0800
+
+    rgw: don't update bucket index multiple times in overwrite
+
+    Instead of this for overwrites:
+     prepare (index),
+       write (head) [-EEXIST]
+     cancel (index)
+     read (head)
+     prepare (index)
+       write (head)
+     complete (index)
+
+    We now do:
+     prepare (index),
+       write (head) [-EEXIST]
+       read (head)
+       write (head)
+     complete (index)
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 7f4818f9611ea8f7b3fac7df8c5c314964f657a2)
+
+commit 96c8161a786783b84a164afef162dd24c46abdc4
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jan 20 16:17:49 2017 -0800
+
+    rgw: assume obj write is a first write
+
+    if fails and object already exists then retry. This improves first obj
+    write performance on the expense of overwrites.
+
+    Fixes: http://tracker.ceph.com/issues/18622
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 6f27f6089238d2a20e7a0f13066eddfc31192dc8)
+
+commit f2e39fec64df7592bc3b6191f122cdfb5d407d6a
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jan 20 14:40:58 2017 -0800
+
+    rgw: configurable write obj window size
+
+    Fixes: http://tracker.ceph.com/issues/18623
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 66a82b4266acedfdd71c64394d68d9e50ed11b20)
+
+commit 8c67dd10c63009e5bc67c20493de732f7bf4abf7
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Jan 20 12:03:54 2017 -0800
+
+    rgw: change default chunk size to 4MB
+
+    Fixes: http://tracker.ceph.com/issues/18621
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 54ef04bc957a3fb034dce903d62a42d77950dd72)
+
+commit f96262fc9c3805aa5fa04348f520dba8be1e4ae8
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jan 11 09:32:59 2017 -0500
+
+    rgw: fix off-by-one in RGWDataChangesLog::get_info
+
+    Fixes: http://tracker.ceph.com/issues/18488
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit a0974fdcf62e60cf31bc15588e7b718da6f6ade3)
+
+commit 935cc11595f4a0b56313c8c98aae0c7684bfff05
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Feb 3 14:41:50 2017 +0100
+
+    rgw: improve handling of illformed Swift's container ACLs.
+
+    Fixes: http://tracker.ceph.com/issues/18796
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit f780fc6ec40395ad0941d4e0309d464fe33836b1)
+
+commit 500eac722d3e59ca8408d1bf9a23cc7368164c58
+Author: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+Date:   Wed Jan 11 05:28:43 2017 +0800
+
+    rgw: the swift container acl should support field .ref
+
+    On the openstack-swift. The container acl supports .ref, which is ignored on ceph swift.
+
+    Fixes: http://tracker.ceph.com/issues/18484
+    Signed-off-by: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+    (cherry picked from commit b06f9cd9f0900db7b0d0fbcaea69cdd0d4b10132)
+
+commit d1ec59544b3894f8ce7a9d00b5e8fb937fb20b7a
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Wed Jan 11 00:06:15 2017 -0500
+
+    radosgw/swift: clean up flush / newline behavior.
+
+    The current code emits a newline after swift errors, but fails
+    to account for it when it calculates 'content-length'.  This results in
+    some clients (go github.com/ncw/swift) producing complaints about the
+    unsolicited newline such as this,
+            Unsolicited response received on idle HTTP channel starting with "\n"; err=<nil>
+
+    This logic eliminates the newline on flush.  This makes the content length
+    calculation correct and eliminates the stray newline.
+
+    There was already existing separator logic in the rgw plain formatter
+    that can emit a newline at the correct point.  It had been checking
+    "len" to decide if previous data had been emitted, but that's reset to 0
+    by flush().  So, this logic adds a new per-instance variable to separately
+    track state that it emitted a previous item (and should emit a newline).
+
+    Fixes: http://tracker.ceph.com/issues/18473
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 5f229d6a33eae4906f22cdb90941835e47ee9f02)
+
+commit 1cb98684565427a28b286dcaaa6cce48626ba79b
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed Feb 1 19:05:50 2017 +0100
+
+    rgw: be aware abount tenants on cls_user_bucket -> rgw_bucket conversion.
+
+    Fixes: http://tracker.ceph.com/issues/18364
+    Fixes: http://tracker.ceph.com/issues/16355
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 871e1f51afe9d6c8b88debc07460b4316121f999)
+
+commit 87811ff161d3c6273050a6be5d7eb7582ecca23c
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 1 12:56:44 2017 +0200
+
+    radosgw-admin: check for name mistmatch in realm set
+
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 4facc5f4f3e842d371115a9a04d86257280014f0)
+
+commit ff888f8f9a4b81347b78ae6681a868cc9a7910e9
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 1 12:40:37 2017 +0200
+
+    radosgw-admin: relam set can use input redirection
+
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit b8b3ae3be3f8e4c05cb23062d25c701b15900475)
+
+commit 68d2d4bb91d6c508c0e1c154aff3b5de3dc347eb
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 1 12:36:04 2017 +0200
+
+    radosgw-admin: realm set should create a new realm
+
+    Fixes: http://tracker.ceph.com/issues/18333
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit e23339c9ef34f6b9df90b1ab64b550af9b541d9e)
+
+commit 291ffe18018029f1418afb2dc038c605a1e34157
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Nov 29 11:29:41 2016 -0500
+
+    rgw: fix for broken yields in RGWMetaSyncShardCR
+
+    Fixes: http://tracker.ceph.com/issues/18076
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit e62d48a9bf2e309eab1a863f167af5267ebcc371)
+
+commit 4776067797b46b4d38d987caa5da4e6954d392aa
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Dec 12 14:00:05 2016 +0100
+
+    rgw: complete versioning enablement after sending it to meta master
+
+    Fixes: http://tracker.ceph.com/issues/18003
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 2d8aafb9dbe64bd9dd2b7d5ed50c6e9550cbe1ab)
+
+commit 27e9644ceb469fb2f54cea468d873ad13c1fe6c4
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Tue Jun 13 10:19:55 2017 +0800
+
+    msg/async: go to open new session when existing already closed
+
+    Fixes: http://tracker.ceph.com/issues/20230
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 99f580a3959240f99061a9ad48ec591b39a9fd46)
+
+commit 67ee45bbed3975f338b798caf81cd89f80ceb785
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Tue Jun 13 10:16:47 2017 +0800
+
+    msg/async: fix accept_conn not remove entry in conns when lazy delete
+
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit bf98babb3289a7714543ff3cbd3872d80f0dc196)
+
+commit 27cbff111458e77baae617c7a222610c8ff0abee
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Apr 11 22:31:43 2017 +0200
+
+    librbd: fix rbd_metadata_list and rbd_metadata_get
+
+    - properly check for val_len in rbd_metadata_list
+    - don't expect output buffers are zero pre-filled
+
+    Fixes: http://tracker.ceph.com/issues/19588
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 75afc74ea681402e22b6dec8b83276d145fc786b)
+
+commit 30e3ae27e0b99ae9b7f3234dbb93260ebda91466
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri May 26 09:52:25 2017 +0200
+
+    rpm: Move ceph-disk to ceph-base
+
+    The SELinux package now requires the ceph-disk binary but that one was
+    part of the ceph-osd package. The ceph-disk python library is already
+    packaged in ceph-base so moving ceph-disk to ceph-base seems like a
+    reasonable next step.
+
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 6991764f3bff7b8f6be699603927aff882377878)
+
+    Conflicts:
+            ceph.spec.in: ceph-disk-udev is present in kraken
+
+commit fa100ed57c3002b357d56398c3589f33cc651fcf
+Author: Boris Ranto <branto@redhat.com>
+Date:   Thu May 25 14:36:13 2017 +0200
+
+    ceph-disk: Fix the file ownership, skip missing
+
+    This commit fixes the file ownership for the /usr/bin/ and /etc/ceph
+    files and skips missing files as some of the files that we do specify
+    now can be missing from the system (not installed, e.f. radosgw).
+
+    Fixes: http://tracker.ceph.com/issues/20077
+
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 077038b4393a28ccbd38ca4a90105dbd4c1ffcd5)
+
+commit 4e0cfecd593b267811e6bf92e81ba98dbede5317
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri Apr 28 12:29:46 2017 +0200
+
+    selinux: Do parallel relabel on package install
+
+    We can take advantage of ceph-disk fix subcommand when doing a package
+    install. We will keep using the differential fixfiles command otherwise.
+
+    We also need to add relabel for /usr/bin/ daemons so that we could use
+    this.
+
+    Fixes: http://tracker.ceph.com/issues/20077
+
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 1cecddf031991f1c64ea203f173189624f11940e)
+
+commit 358081d27a9b2c39e9daed10244819df5291da48
+Merge: ae0eab53f0 bf26c56555
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Jun 2 09:54:45 2017 -0700
+
+    Merge pull request #15421 from dzafman/wip-20125
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit bf26c5655505369753142646ac357df2b8897ab7
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed May 31 15:39:19 2017 -0700
+
+    osd: Object level shard errors are tracked and used if no auth available
+
+    Shards with object mismatch are tracked to mark them inconsistent
+    Fix test because storing omap_digest in object_info not behaving as before
+
+    Fixes: http://tracker.ceph.com/issues/20089
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 1cacbea763c7aabfeaaf4bd5e878301044184117)
+
+commit 727653623f6cc616bdeeab2f0ce98f41a649ceee
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri May 5 14:56:40 2017 -0400
+
+    cls/rgw: list_plain_entries() stops before bi_log entries
+
+    list_plain_entries() was using encode_obj_versioned_data_key() to set
+    its end_key, which gives a prefix of BI_BUCKET_OBJ_INSTANCE_INDEX[=2]
+
+    that range between start_key and end_key would not only span the
+    BI_BUCKET_OBJS_INDEX[=0] prefixes, but BI_BUCKET_LOG_INDEX[=1] prefixes
+    as well. this can result in list_plain_entries() trying and failing to
+    decode a rgw_bi_log_entry as a rgw_bucket_dir_entry
+
+    Fixes: http://tracker.ceph.com/issues/19876
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit b29a1633a57abf443d5790c13d680d2917f86037)
+
+commit 648c6adf52a7847a5b46d1523da45a29504a68da
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Feb 3 10:08:33 2017 -0500
+
+    mon/PGMap: factor mon_osd_full_ratio into MAX AVAIL calc
+
+    If we only fill OSDs to 95%, we should factor that into
+    the MAX AVAIL calculation for the pool.
+
+    Fixes: http://tracker.ceph.com/issues/18522
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f223ac92917f4bc18e5b9b3ad61afa155e4d088a)
+
+commit 2aec591cfe1ba69631860b4886d34de80f752bd7
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu May 11 13:13:39 2017 +0800
+
+    osd/PrimaryLogPG: do not call on_shutdown() if (pg.deleting)
+
+    when a callback is called, it could be facing a PG already shut down by
+    OSD. but if that callback wants to shut that PG down. it should check
+    the PG's status first.
+
+    Fixes: http://tracker.ceph.com/issues/19902
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit f3c44a0dfc859f6f625a92e727e0e521ed4a9207)
+
+commit ae0eab53f0bb05c954fc98e019e2d5a054dbba45
+Merge: 9df2d772ff 28e1fd0f5d
+Author: Boris Ranto <branto@redhat.com>
+Date:   Thu May 11 08:06:47 2017 +0200
+
+    Merge pull request #14345 from ceph/wip-ceph-disk-fix-kraken
+
+    ceph-disk: Add fix subcommand kraken back-port
+
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 28e1fd0f5d1442d4b56194082014853db551def4
+Author: Boris Ranto <branto@redhat.com>
+Date:   Wed Mar 8 09:38:39 2017 +0100
+
+    ceph-disk: Add --system option for fix command
+
+    This adds the ability to restore the labels of the underlying system
+    data in addition to ceph data.
+
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 8d81af42fd507c7b92c8279eb114b0a733ac1da6)
+
+commit dfa721f48df5eac565d27388adc17c40991502c6
+Author: Boris Ranto <branto@redhat.com>
+Date:   Thu Feb 16 11:34:27 2017 +0100
+
+    ceph-disk: Add more fix targets
+
+    It turns out I forgot several more directories that needs to be fixed by
+    this script. We need to fix /var/log/ceph, /var/run/ceph and /etc/ceph
+    as well.
+
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit ae139307d6b2bfba47e21d29d6dbd3c8dc01b5b5)
+
+commit 2db2fc81835ea8d0b35f563dd09691b030c920e9
+Author: Boris Ranto <branto@redhat.com>
+Date:   Thu Feb 9 19:17:12 2017 +0100
+
+    ceph-disk: Add unit test for fix command
+
+    This will simulate the command* functions to not actually run anything
+    thus excercising the python code directly. It also checks that the
+    proper (sub-strings) are in the output.
+
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 1ec53dee9a690134936bdc3a09c9a02fecf13a9d)
+
+commit 82764ca8e986c8efc31488f2b30d2a2706f9826c
+Author: Boris Ranto <branto@redhat.com>
+Date:   Tue Jan 31 13:19:33 2017 +0100
+
+    ceph-disk: Add fix subcommand
+
+    This subcommand will fix the SELinux labels and/or file permissions on
+    ceph data (/var/lib/ceph).
+
+    The command is also optimized to run the commands in parallel (per
+    sub-dir in /var/lib/ceph) and do restorecon and chown at the same time
+    to take advantage of the caching mechanisms.
+
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 6d5d30f6ed7538271579cc2ef4e2e364f01a4a6f)
+
+commit 9df2d772ff272386bff6b1657f3f1528fcaf4a8a
+Merge: 6b1782959e 04f8186f37
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon May 8 11:33:59 2017 -0500
+
+    Merge pull request #14983 from liewegas/wip-denc-dump-kraken
+
+    ceph-object-corpus: kraken objects (kraken)
+
+commit 04f8186f37d5add0e68ff47a025f0296377e96d1
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Feb 27 16:10:34 2017 -0500
+
+    test/encoding/readable.sh: join children at finish
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c8a349eb4603be295a6f0e1683105cb93bfcd066)
+
+commit ba249885d1a37e703f37a08958af2f85eb227bd6
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Feb 27 16:10:24 2017 -0500
+
+    test/encoding/readable: fix passing of results to parent
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1325ba672d5b5c8a12722a395ad029c98d1ff236)
+
+commit d97154e61bb615faf566a42a878b2e7a303d99ae
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri May 5 16:48:25 2017 -0400
+
+    messages/MCommand: fix type on decode
+
+    Wow, this has been broken since v0.38, but apparently
+    the message never made it into the object corpus so
+    we never noticed!
+
+    In reality the bug is harmless: decode_message() will
+    set_header which clobbers whatever version the default
+    ctor fills in, so this only affects ceph-dencoder's
+    test.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3018b907c1d0375b5ae20802811591c0a46b61be)
+
+commit b05fa6cd2ff07349d6cc76be0732e627f0290308
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri May 5 16:32:48 2017 -0400
+
+    ceph-object-corpus: kraken objects
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 4ff0cb953e190806f7b9e6791d300bd22ad281d9
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri May 5 16:31:02 2017 -0400
+
+    test/encoding/generate-corpus-objects: fix typo
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b89dff472c86c435fcee767ec9d992b308c4d816)
+
+commit 34e04de108004eccab82d6f791fbffc9ec97ab77
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri May 5 09:19:23 2017 -0400
+
+    test/encoding/generate-corpus-objects.sh: simplify object corpus population
+
+    Script that generates corpus objects.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b38b663a71fdc37a2d5c3341836b6716d576cecc)
+
+commit 260352405e82009132755105d78a885ae327a418
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri May 5 15:19:05 2017 -0400
+
+    qa/workunits/rgw/run-s3tests.sh: run s3-tests
+
+    This works out of the box with a vstart environment and
+
+     RGW=1 ../src/vstart.sh -n -l
+     PATH=bin:$PATH ../qa/workunits/rgw/run-s3tests.sh
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 40fe1d181cf8d31b80db6f8f99394bd66d2affd5)
+
+commit 2b107f91cefbdc6ab173890c3ccf3693768bc76c
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu May 4 18:33:21 2017 -0400
+
+    include/denc: support ENCODE_DUMP
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 97987b4ace737b86a4dc2dce94ce9d330fbe903c)
+
+commit 62bb2086af2f0d63c081a8a0b9d1de8d1c1179a7
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Apr 14 13:21:38 2017 -0400
+
+    osd: fix occasional MOSDMap leak
+
+    _committed_osd_maps() may return early (without putting
+    the ref) on shutdown.
+
+    Fixes: http://tracker.ceph.com/issues/18293
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a74632f964e10a57fb8065aec90dc7b69bddba5b)
+
+commit d51b7553919233d2f18d5fa79a0ff3c09e71ba8a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon May 1 21:06:19 2017 -0400
+
+    cls_rbd: default initialize snapshot namespace for legacy clients
+
+    Creating a snapshot on >=Kraken OSDs using <=Jewel clients can result
+    in an improperly initialized snapshot namespace. As a result, attempting
+    to remove the snapshot using a >=Kraken client will result in an -EINVAL
+    error.
+
+    Fixes: http://tracker.ceph.com/issues/19413
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 03b0b03071f3e04754896664c69f73759ddb907a)
+
+    Conflicts:
+            src/cls/rbd/cls_rbd.h: trivial resolution
+
+commit 6b1782959ee29de6eff9a2d2c81a108485347bbd
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Apr 28 18:03:59 2017 -0400
+
+    mgr/DaemonServer: do not crash on bad authorizer
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 33c211bf9868156e41d7196cbac5dc6bff418248
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Mon Apr 17 14:32:38 2017 -0700
+
+    PendingReleaseNotes: discuss snap trim improvements
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 911a894d64616b2cb5bfe92e3366e8033fb0f59e
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Fri Apr 7 15:45:12 2017 -0700
+
+    PrimaryLogPG: reimplement osd_snap_trim_sleep within the state machine
+
+    Rather than blocking the main op queue, just pause for that amount of
+    time between state machine cycles.
+
+    Also, add osd_snap_trim_sleep to a few of the thrasher yamls.
+
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 2ed7759cfeb03e71f0fbd98fe7c2db2bb741861c)
+
+    Conflicts:
+            src/osd/PrimaryLogPG.cc
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 02a2ef41ae54dac7d903bc5753e6702582d55dd4
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 26 15:41:21 2017 -0800
+
+    rados: check that pool is done trimming before removing it
+
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 4aebf59d906fa3e03d21bdac182f89fe3cd4c802)
+
+commit 95306559db139ad3cda3ee2eac36195582ff9329
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Jan 26 15:05:14 2017 -0800
+
+    osd/: add PG_STATE_SNAPTRIM[_WAIT] to expose snap trim state to user
+
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit c2eac34c86517e410eb4842d8b8085da7d8d7973)
+
+commit 995ef4a6b01635ef09532487f7a4ea5fc90e721d
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Fri Apr 7 15:41:18 2017 -0700
+
+    osd/PrimaryLogPG: limit the number of concurrently trimming pgs
+
+    This patch introduces an AsyncReserver for snap trimming to limit the
+    number of pgs on any single OSD which can be trimming, as with backfill.
+    Unlike backfill, we don't take remote reservations on the assumption
+    that the set of pgs with trimming work to do is already well
+    distributed, so it doesn't seem worth the implementation overhead to get
+    reservations from the peers as well.
+
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 21cc515adfb225ba70f1d80b1b76f0345c214c22)
+
+    Conflicts:
+            src/osd/PrimaryLogPG.cc
+            src/osd/PrimaryLogPG.h
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 49f99c3b4d9051bccb34117075348e3ea58804db
+Merge: 0d6953ec50 e9a10eaccd
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 26 19:53:49 2017 +0200
+
+    Merge pull request #14506 from smithfarm/wip-19119-kraken
+
+    kraken: doc: PendingReleaseNotes: warning about 'osd rm ...' and #13733
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 0d6953ec501f0ec6c9a5a72b532e99554dfe5d09
+Merge: 557fee8d0d 8d0c2297f1
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 26 19:28:58 2017 +0200
+
+    Merge pull request #14692 from smithfarm/wip-fix-hammer-jewel-x
+
+    qa/suites/upgrade: add tiering test to hammer-jewel-x
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 557fee8d0dad8847728820b1d345e2d9ed9ab433
+Merge: e85b09f415 543e4b9060
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Apr 24 22:22:14 2017 +0200
+
+    Merge pull request #14425 from smithfarm/wip-19564-kraken
+
+    kraken: build/ops: ceph-base missing dependency for psmisc in Ubuntu Xenial
+
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit e85b09f4158296dbc559d3b2e0816364ca1c448c
+Merge: d2c17adc14 8c2f590338
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Apr 22 10:55:16 2017 +0200
+
+    Merge pull request #13494 from shinobu-x/wip-18516-kraken
+
+    kraken: build/ops: systemd: Start OSDs after MONs
+
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit d2c17adc140d2358512c4c00e079299ae7f31f74
+Merge: 9aa180b7f1 02afe9855a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Apr 21 17:36:42 2017 +0200
+
+    Merge pull request #14588 from asheplyakov/19618-kraken
+
+    kraken: mon/MonClient: make get_mon_log_message() atomic
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 9aa180b7f13674a3de28115347534540ecf6ae38
+Merge: 8e2e3d8292 914d8a6109
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Apr 21 17:12:29 2017 +0200
+
+    Merge pull request #13543 from shinobu-x/wip-18954-kraken
+
+    kraken: ceph-disk prepare get wrong group name in bluestore
+
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 8e2e3d8292f93812933c049f9489871f249c9dfa
+Merge: e68d3e14b8 d8946abf07
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Apr 21 08:53:25 2017 +0200
+
+    Merge pull request #12746 from SUSE/wip-18387-kraken
+
+    kraken: tests: use ceph-kraken branch for s3tests
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 6b428b77f287b3a2b9b7061e3da3279381f7d3aa
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Tue Apr 11 13:51:47 2017 -0700
+
+    use sudo to check check health
+
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 7af157ad4ce7f7e2b8de97ee10eeaf64b9099bc0)
+
+commit 80c47c87e5c060d45ea704ac31709bb061d79db6
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Wed Mar 29 09:27:20 2017 -0700
+
+    Add reboot case for systemd test
+
+    test systemd units restart after reboot
+
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 7b587304a54d9b21041ffdfbc85fad8d87859c49)
+
+commit a6c2d7feb3248a0bbe0399323bb2fe11e23d15e1
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Wed Mar 29 09:56:11 2017 -0700
+
+    Fix distro's, point to latest version
+
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 1947648669971c1bd1ca189870ed9b25bbd48d3a)
+
+commit 8d0c2297f1dbe95c236357861026263677aea0c5
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Mar 6 15:50:59 2017 +0800
+
+    qa/suites/upgrade: add tiering test to hammer-jewel-x
+
+    Related: http://tracker.ceph.com/issues/19185
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit e8c04a027b5d108b30e63ab759760add57d43198)
+
+commit e68d3e14b8ea42baa30ca4dd836c1651deb99dcd
+Merge: 766a55ac87 1067764418
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 21:12:42 2017 +0200
+
+    Merge pull request #14620 from smithfarm/wip-19659-kraken
+
+    kraken: librbd: corrected resize RPC message backwards compatibility
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 766a55ac87cb7c08f1316cf7f0fd6bca834ad269
+Merge: 01804c78e0 55a15b18e7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 19:06:35 2017 +0200
+
+    Merge pull request #14574 from smithfarm/wip-19620-kraken
+
+    kraken: cephfs: MDS server crashes due to inconsistent metadata.
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 01804c78e0af6fd518e8ad5de2bf39e6743e3039
+Merge: 1866332928 93efc1e948
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 19:05:44 2017 +0200
+
+    Merge pull request #14573 from smithfarm/wip-19483-kraken
+
+    kraken: cephfs: No output for ceph mds rmfailed 0 --yes-i-really-mean-it command
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 18663329283c9df32bd42e1dc41cff10ba2e7826
+Merge: 9ff3560ab4 941a356fa7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 19:04:34 2017 +0200
+
+    Merge pull request #14572 from smithfarm/wip-19335-kraken
+
+    kraken: cephfs: MDS heartbeat timeout during rejoin, when working with large amount of caps/inodes
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 9ff3560ab4c4dfa35983beb845331f8fb261270b
+Merge: 5554f5a82a 473ad4ea2a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 19:03:42 2017 +0200
+
+    Merge pull request #14571 from smithfarm/wip-19045-kraken
+
+    kraken: cephfs: buffer overflow in test LibCephFS.DirLs
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 5554f5a82ad2c78b93651aeef5f37e4ffb4586f1
+Merge: cc49802d76 64616bb2a5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 19:02:54 2017 +0200
+
+    Merge pull request #14570 from smithfarm/wip-18950-kraken
+
+    kraken: cephfs: mds/StrayManager: avoid reusing deleted inode in StrayManager::_purge_stray_logged
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit cc49802d769e3a59664fc5ef80d20171c07693df
+Merge: 0c51569478 2d5d8f2767
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 19:02:02 2017 +0200
+
+    Merge pull request #14569 from smithfarm/wip-18899-kraken
+
+    kraken: cephfs: test_open_inode fails
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 0c51569478ebb75defb6ba059f60007db597016a
+Merge: 63fd2b9884 a7831ccf79
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 19:00:51 2017 +0200
+
+    Merge pull request #14568 from smithfarm/wip-18706-kraken
+
+    kraken: cephfs: fragment space check can cause replayed request fail
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 63fd2b9884eeb7551e08875cb6421a79d22e9674
+Merge: f1affa93d1 eb12e3a752
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 18:59:25 2017 +0200
+
+    Merge pull request #14567 from smithfarm/wip-18700-kraken
+
+    kraken: cephfs: client: fix the cross-quota rename boundary check conditions
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit f1affa93d15f5f37910debe73eee2b4c8774fe2c
+Merge: 2887fb06aa 95335098d5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 18:58:08 2017 +0200
+
+    Merge pull request #14566 from smithfarm/wip-18616-kraken
+
+    kraken: cephfs: segfault in handle_client_caps
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 2887fb06aa25aeb1b7a845b288ba4a556cab74bc
+Merge: 655ab2a76f 91bec618f8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 18:57:08 2017 +0200
+
+    Merge pull request #14565 from smithfarm/wip-18566-kraken
+
+    kraken: cephfs: MDS crashes on missing metadata object
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 655ab2a76f29c3a06e7d7e9bacd3b51b2725952e
+Merge: bb08c04f26 a51e9b5af9
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 18:56:04 2017 +0200
+
+    Merge pull request #14564 from smithfarm/wip-18562-kraken
+
+    kraken: cephfs: test_client_recovery.TestClientRecovery fails
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit bb08c04f26f1d05eedbbdc7cd23f51314edc13cf
+Merge: e3db135268 ba7b6ced0a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 18:55:01 2017 +0200
+
+    Merge pull request #14563 from smithfarm/wip-18552-kraken
+
+    kraken: cephfs: ceph-fuse crash during snapshot tests
+
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit e3db1352688fc68063707882cf0c26f34db4fabf
+Merge: 7fda19ee7e 3bbd3c5dc3
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 18:18:59 2017 +0200
+
+    Merge pull request #14521 from smithfarm/wip-19462-kraken
+
+    kraken: doc: rgw: admin ops: fix the quota section
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 7fda19ee7e8a80f0b34e9d539746c973c8c45707
+Merge: d1f066b7b5 d5398effcf
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 14:52:45 2017 +0200
+
+    Merge pull request #14641 from smithfarm/wip-19693-kraken
+
+    kraken: [test] test_notify.py: rbd.InvalidArgument: error updating features for image test_notify_clone2
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit d1f066b7b58343e48ddaff398be7c799323fac4d
+Merge: d82f0cd250 a0ad2d11ce
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 14:51:45 2017 +0200
+
+    Merge pull request #14533 from smithfarm/wip-18501-kraken
+
+    kraken: rbd-mirror: potential race mirroring cloned image
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit d82f0cd2505dd06507e1a08c30fa20fdbb024847
+Merge: 7638d3d0cd d826600ee2
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 14:50:56 2017 +0200
+
+    Merge pull request #14534 from smithfarm/wip-18549-kraken
+
+    kraken: rbd: 'metadata_set' API operation should not change global config setting
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 7638d3d0cd0b50bc7ae9cb537c7465152cf9e476
+Merge: 2c1d5db604 b9258ccfd8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 14:50:11 2017 +0200
+
+    Merge pull request #14536 from smithfarm/wip-18557-kraken
+
+    kraken: rbd: 'rbd bench-write' will crash if --io-size is 4G
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 2c1d5db6048ed6dd239af0bdbfa79cbc14432fcc
+Merge: 8794053597 e3a230cb19
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 14:49:24 2017 +0200
+
+    Merge pull request #14537 from smithfarm/wip-18601-kraken
+
+    kraken: rbd: Add missing parameter feedback to 'rbd snap limit'
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 87940535979da874b9b44e037b27d869972efae2
+Merge: b457cbd768 bafa2b0c83
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 14:23:08 2017 +0200
+
+    Merge pull request #14538 from smithfarm/wip-18632-kraken
+
+    kraken: rbd: [qa] crash in journal-enabled fsx run
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit b457cbd768b12701532bfa57304e5c25e6b49528
+Merge: 24e6d0e440 3112ab5726
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 14:19:59 2017 +0200
+
+    Merge pull request #14622 from smithfarm/wip-19037-kraken
+
+    kraken: rbd-mirror: deleting a snapshot during sync can result in read errors
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 24e6d0e44080558f956d2c8446cf60f7137649bc
+Merge: 13de280cfe 551ce273a1
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 20 14:18:48 2017 +0200
+
+    Merge pull request #14545 from smithfarm/wip-19324-kraken
+
+    kraken: rbd: [api] temporarily restrict (rbd_)mirror_peer_add from adding multiple peers
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit fa2753245d056f39f0f6996988e193bf4917bc52
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Fri Apr 7 14:33:20 2017 -0700
+
+    osd: pglog: with config, don't assert in the presence of stale divergent_priors
+
+    Fixes: http://tracker.ceph.com/issues/17916
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+
+commit d5398effcfacb6d5436ae32e1ee40872efd5b504
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 19 09:26:31 2017 -0400
+
+    test: rbd master/slave notify test should test active features
+
+    Fixes: http://tracker.ceph.com/issues/19692
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0dcba41cba96566d0b8da54cf0316d523b88ded2)
+
+commit d02d0e5d97fd7d9b3eac1e2ba55776d2c9f7f8ee
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Apr 19 09:12:04 2017 -0400
+
+    qa/suites: client-upgrade/jewel-client-x should test multiple features
+
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7ab93b7779b293ceda325a6d1cdb446cf624a6e3)
+
+commit 13de280cfe7379440185fefcb71a6d7d4a30e65f
+Merge: 5d604d058b b8d988f7f1
+Author: Zack Cerza <zack@cerza.org>
+Date:   Tue Apr 18 14:17:02 2017 -0600
+
+    Merge pull request #14487 from zmc/wip-kraken-workunit
+
+    qa/tasks/workunit: Backport repo fixes from master
+
+commit 5d604d058b0b1f7b7659390093437f109fcb9af7
+Merge: 959248d249 c2232643d0
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Apr 18 22:09:19 2017 +0200
+
+    Merge pull request #14531 from smithfarm/wip-18493-kraken
+
+    kraken: rbd-mirror: sporadic image replayer shut down failure
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 959248d2497a48f98c0a4fd818fcce79c994ca61
+Merge: c195418b4f e91de89b0d
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Apr 18 22:08:34 2017 +0200
+
+    Merge pull request #14532 from smithfarm/wip-18495-kraken
+
+    kraken: rbd: Possible deadlock performing a synchronous API action while refresh in-progress
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 10677644181356f561cc10a19d76833d55917a43
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Apr 18 10:05:53 2017 -0400
+
+    librbd: corrected resize RPC message backwards compatibility
+
+    Commit d1f2c557 incorrectly changed the order of variables within
+    the payload. This resulted in breaking the resize RPC message
+    with older versions of Ceph.
+
+    Fixes: http://tracker.ceph.com/issues/19636
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9db305a99fd945059a173f5ae8ae61744dd28615)
+
+commit c195418b4f240a8f4bf1b1739af9a64ba046c2d2
+Merge: 432bc8b416 81f3d86032
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Apr 18 16:58:39 2017 +0200
+
+    Merge pull request #14095 from shinobu-x/wip-19319-kraken
+
+    kraken: RadosImport::import should return an error if Rados::connect fails
+
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 8f81bb33c7791c46db5e4db1ee45ab02a70580d9
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Apr 5 16:19:57 2017 -0400
+
+    rgw: fix for null version_id in fetch_remote_obj()
+
+    commit 8b43c9781206c22d9aedb4beb8d669bf1e23169f fixed the wrong use of
+    the dest_obj's version, but removed the check for "null" version
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 915370776df5b964c2ee8d9f9329562919eef8d5)
+
+commit ad2b140e01462d3dfe58fd03879f3868e561f165
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Fri Mar 17 16:26:56 2017 +0800
+
+    rgw: version id doesn't work in fetch_remote_obj
+
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit 8b43c9781206c22d9aedb4beb8d669bf1e23169f)
+
+    Conflicts:
+            src/rgw/rgw_rados.cc: trivial: dest_obj.key.instance in master
+              versus dest_obj.get_instance() in Jewel
+
+commit 8d64dd4f29e1b0af51e7938d1aa363e4ff710154
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Apr 13 23:49:50 2017 +0200
+
+    ceph-disk: enable directory backed OSD at boot time
+
+    https://github.com/ceph/ceph/commit/539385b143feee3905dceaf7a8faaced42f2d3c6
+    introduced a regression preventing directory backed OSD from starting at
+    boot time.
+
+    For device backed OSD the boot sequence starts with ceph-disk@.service
+    and proceeds to
+
+        systemctl enable --runtime ceph-osd@.service
+
+    where the --runtime ensure ceph-osd@12 is removed when the machine
+    reboots so that it does not compete with the ceph-disk@/dev/sdb1 unit at
+    boot time.
+
+    However directory backed OSD solely rely on the ceph-osd@.service unit
+    to start at boot time and will therefore fail to boot.
+
+    The --runtime flag is selectively set for device backed OSD only.
+
+    Fixes: http://tracker.ceph.com/issues/19628
+
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit f425a127b7487d2093c8c943f0bcdec3d673d601)
+
+commit 02afe9855a4d04fe56951a9e9ffd3e32537ff77e
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Apr 10 14:53:46 2017 +0800
+
+    mon/MonClient: make get_mon_log_message() atomic
+
+    * LogClient: move reset_session() into get_mon_log_message() and add a
+      "flush" param to the latter. so it can get_mon_log_message()
+      atomically. otherwise another call changing the log queue could sneak
+      in between reset_session() and get_mon_log_message().
+    * MonClient: add a "flush" param to do_send() so we can reset the
+      LogClient session once we are connected to a monitor.
+
+    Fixes: http://tracker.ceph.com/issues/19427
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5215e291da2b527d85e129eda86043490843178e)
+
+    Conflicts:
+            src/mon/MonClient.cc: handle_auth: replaced 'log_client->reset_session();
+            send_log();' sequence with newly introduced 'send_log(true);' like
+            the original patch does
+
+commit 432bc8b4168e70a68e53795dea538882f83a9a8e
+Merge: 8eb10440ce c2cc96e6c1
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sun Apr 16 04:13:12 2017 +0200
+
+    Merge pull request #14323 from shinobu-x/wip-19371-kraken
+
+    kraken: common: monitor creation with IPv6 public network segfaults
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 8eb10440ce144ddab2997ac0b8a868d28de932fe
+Merge: 74907feb3f 1d25327b00
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Apr 15 10:08:15 2017 +0200
+
+    Merge pull request #13284 from shinobu-x/wip-18599-kraken
+
+    kraken: osd: os/bluestore: fix statfs to not include DB partition in free space
+
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 74907feb3fda64f9e9a1f7410ce71683a40f417c
+Merge: fc5c4cd2c0 0e0d149895
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Apr 15 09:57:21 2017 +0200
+
+    Merge pull request #13046 from SUSE/wip-18554-kraken
+
+    kraken: mon: peon wrongly delete routed pg stats op before receive pg stats ack
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit fc5c4cd2c01fa97f65b4bbad0d2cfe6dc7e3f9a9
+Merge: 19b8a9f1f8 7e30b630e2
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Apr 15 00:30:39 2017 +0200
+
+    Merge pull request #14480 from dillaman/wip-19467-kraken
+
+    kraken: librbd: is_exclusive_lock_owner API should ping OSD
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 19b8a9f1f87b2b9ff69eb67b38e1f2e161ea1c1d
+Merge: 91fbfaf3bd d4a43e1fd5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Apr 15 00:28:19 2017 +0200
+
+    Merge pull request #13877 from smithfarm/wip-19178-kraken
+
+    kraken: rgw: anonymous user error code of getting object is not consistent with SWIFT
+
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit 91fbfaf3bdbb9dfcf6704effc271d5038263676d
+Merge: abcf1b1e28 b39bef0961
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Apr 15 00:25:51 2017 +0200
+
+    Merge pull request #13866 from smithfarm/wip-19157-kraken
+
+    kraken: rgw: health check errors out incorrectly
+
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit abcf1b1e28c75feb702b100bbc0ca0ff9c4d14f6
+Merge: cbcb932d3d 47cf8f6699
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Apr 15 00:22:22 2017 +0200
+
+    Merge pull request #13843 from smithfarm/wip-19146-kraken
+
+    kraken: rgw: a few cases where rgw_obj is incorrectly initialized
+
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit cbcb932d3d40118101df350694eacbe2cfb22232
+Merge: 6702056568 03b1ab4b13
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Apr 15 00:20:29 2017 +0200
+
+    Merge pull request #13829 from smithfarm/wip-18898-kraken
+
+    kraken: no http referer info in container metadata dump in swift API
+
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit 670205656847742f1e9d46b152c66f725dfba5dc
+Merge: 841216b781 c8d70ee012
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Apr 15 00:19:00 2017 +0200
+
+    Merge pull request #13780 from smithfarm/wip-18896-kraken
+
+    kraken: should parse the url to http host to compare with the container referer acl
+
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit 841216b781e229205476f633eae75355f87c27ae
+Merge: eb84959bac bfc058d8f9
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Sat Apr 15 00:15:15 2017 +0200
+
+    Merge pull request #12986 from Werkov/rgw-fix-tempurl-url-encoding
+
+    kraken: rgw: Use decoded URI when verifying TempURL
+
+    Reviewed-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+
+commit 55a15b18e7a2188f5ff74b5fa26ef0316ec9c005
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Mar 29 19:38:37 2017 +0100
+
+    tools/cephfs: set dir_layout when injecting inodes
+
+    When we left this as zero, the MDS would interpret it was HASH_LINUX
+    rather than the default HASH_RJENKINS.  Potentially that
+    could cause problems if there perhaps were already dirfrags in
+    the metadata pool that were set up using rjenkins.  Mainly
+    it just seems more appropriate to explicitly set this field
+    rather than hit the fallback behaviour.
+
+    Related: http://tracker.ceph.com/issues/19406
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 7d6d542885bd29b71214f9ca52bd26e9183c5d01)
+
+commit 93efc1e9484ea41f3eb831dcb3d19d63b044ef70
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Mar 9 13:15:46 2017 +0000
+
+    mon: fix hiding mdsmonitor informative strings
+
+    Local `stringstream ss` declarations were hiding
+    the real variable used to feed back to the user.
+
+    Fixes: http://tracker.ceph.com/issues/16709
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 00404ae9bd4cce0518a44d36d2d6a5612f4f9d04)
+
+commit 941a356fa77ffc2f41d5054a8e3f43b46523008e
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Mar 6 11:51:31 2017 +0000
+
+    mds: reset heartbeat in export_remaining_imported_caps
+
+    This loop can be very long.
+
+    Fixes: http://tracker.ceph.com/issues/19118
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 85071f1509beba4a390730e6a3a4332484646d63)
+
+commit 5e19692bb68049ea8fed0c39534fe8abb10a5378
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Mar 6 11:24:50 2017 +0000
+
+    mds: heartbeat_reset in dispatch
+
+    Previously we only heartbeated in tick.  However, our locking is
+    not guaranteed to be fair, so on a super-busy dispatch queue it may be
+    possible for the heartbeat to time out while the tick() function
+    is waiting for mds_lock.
+
+    Fixes: http://tracker.ceph.com/issues/19118
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 819394549af10532419d88742fae3a69d2ea487d)
+
+commit 473ad4ea2a35c74d89605b5579a4ff3866134c2d
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Feb 15 11:45:26 2017 +0800
+
+    test/libcephfs: avoid buffer overflow when testing ceph_getdents()
+
+    The buffer size should be at least "2 * sizeof(struct dirent)".
+    Otherwise, the code that checks dentry '..' overflow.
+
+    Fixes: http://tracker.ceph.com/issues/18941
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit fa6671345b8f3a82dcd232f99e55a982b0a641f1)
+
+commit 64616bb2a5b84f8518ecbb9878a5ba3abaaf87ed
+Author: Zhi Zhang <willzzhang@tencent.com>
+Date:   Fri Feb 10 10:56:46 2017 +0800
+
+    mds/StrayManager: aviod reusing deleted inode in StrayManager::_purge_stray_logged
+
+    Signed-off-by: Zhi Zhang <zhangz.david@outlook.com>
+    (cherry picked from commit 4978e57419482384279d7e784a625f5e5c10961a)
+
+commit 2d5d8f27679d40ff962c8329026b2d75a3fa7c78
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Feb 1 00:38:08 2017 +0000
+
+    tasks/cephfs: switch open vs. write in test_open_inode
+
+    Do the write after opening the file, so that we get good
+    behaviour wrt the change in Mount.open_background that uses
+    file existence to confirm that the open happened.
+
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit a027dba78fc8bc84ae39d7998b386ce21c01e1bf)
+
+commit 284cd5065f871fc7aeb134d0f9dd48ba9d1f12bf
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jan 26 16:48:58 2017 +0000
+
+    qa: fix race in Mount.open_background
+
+    Previously a later remote call could end up executing
+    before the remote python program in open_background
+    had actually got as far as opening the file.
+
+    Fixes: http://tracker.ceph.com/issues/18661
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit c6d91dd91252e703d08b8ac62ac6a47ee82c0bed)
+
+commit a7831ccf795b31705a66be89beef63b0bbe817d3
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jan 25 15:28:23 2017 +0800
+
+    mds: don't purge strays when mds is in clientreplay state
+
+    MDS does not trim log when it's in clientreplay state. If mds hang
+    at clientreplay state (due to bug), purging strays can submit lots
+    of log events and create very large mds log.
+
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 86bbc7fff02668077f27d0924ba3efe6544b77f6)
+
+commit 224745491e5c9e495adce5abbf7731f7fae1ddc5
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jan 25 11:03:45 2017 +0800
+
+    mds: skip fragment space check for replayed request
+
+    when handling replayed request, stray directory can be different
+    from the stray directory used by the original request. The fragment
+    space check for stray directory can fail.
+
+    Fixes: http://tracker.ceph.com/issues/18660
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit afe889cbc5baab196567c2aad01f49fe90901fda)
+
+commit eb12e3a7524fcbc009cabda333a6a958390743bd
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Wed Dec 14 12:09:44 2016 -0800
+
+    client: fix the cross-quota rename boundary check conditions
+
+    We were previously rejecting a rename if either of the involved directories
+    was a quota root, even if the other directory was part of the same quota
+    "tree". What we really want to do is identify the correct quota root
+    (whether local or ancestral) for each directory and compare them. So
+    now we do.
+
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 8e8892aa46accb519faa4bb9fecf66618f1b11b2)
+
+commit 95335098d5fa1a8e308f697a6bcef6ce3e090ac9
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Jan 6 15:42:52 2017 +0800
+
+    mds: fix null pointer dereference in Locker::handle_client_caps
+
+    Locker::handle_client_caps delays processing cap message if the
+    corresponding inode is freezing or frozen. When the message gets
+    processed, client can have already closed the session.
+
+    Fixes: http://tracker.ceph.com/issues/18306
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit e281a0b9c1fdeaf09f1b01f34cecd62e4f49d02e)
+
+commit 91bec618f87d4fc5dc0619d1b980c14ec7527d48
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jan 11 15:50:52 2017 +0800
+
+    qa/tasks: add test_open_ino_errors
+
+    Validate that errors encountered during opening inos are properly
+    propagated
+
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 6526ecc084733b34129aa1f21085fa41fb53b785)
+
+commit 4d5775092d29c2eb3c2d88d1fff93ce5af4779dd
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Jan 3 11:11:12 2017 +0800
+
+    mds: propagate error encountered during opening inode by number
+
+    Fixes: http://tracker.ceph.com/issues/18179
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 2213cc2dcc0e8fb01bcae3863d0d8a4a1fd8873f)
+
+commit a51e9b5af9fbdb5b81ec20e30dfda2222d6ac418
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Jan 9 20:47:37 2017 +0800
+
+    qa/tasks/cephfs: fix kernel force umount
+
+    Fixes: http://tracker.ceph.com/issues/18396
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 4cdeeaac105d7824452a8b578632b7371275b45c)
+
+commit ba7b6ced0a69991bf4f095172575dc536736d40d
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Jan 10 17:16:40 2017 +0800
+
+    client: fix Client::handle_cap_flushsnap_ack() crash
+
+    Struct CapSnap holds a reference to its parent inode. So erasing
+    struct CapSnap from Inode::cap_snaps may drop inode's last reference.
+    The inode gets freed in the middle of erasing struct CapSnap
+
+    Fixes: http://tracker.ceph.com/issues/18460
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 525c52fd491ed1ced385c8047872e3f557f8423f)
+
+commit 551ce273a132d4e7b59b818377568d40cf597f85
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Mar 10 10:56:38 2017 -0500
+
+    rbd: prevent adding multiple mirror peers to a single pool
+
+    The rbd-mirror daemon does not currently support replication
+    from multiple peers. Until that is supported, add a temporary
+    restriction to prevent confusion.
+
+    Fixes: http://tracker.ceph.com/issues/19256
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c0c9d1014d57b3d5b95e7513fcc38d04b9ea5165)
+
+commit 3112ab57269847233c1e006ca05ba02071d579b0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Feb 21 15:33:01 2017 -0500
+
+    rbd-mirror: retry object copy after -ENOENT error
+
+    Fixes: http://tracker.ceph.com/issues/18990
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b4f36d5dc3f4f3cbb23f61cbb945b222248a50df)
+
+commit 46ba56025c283a70b0f384ef38f47542f0885c5b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Feb 21 13:09:39 2017 -0500
+
+    rbd-mirror: object copy should always reference valid snapshots
+
+    If a remote snapshot is deleted while an image sync is in-progress,
+    associate the read request against the most recent, valid remote
+    snapshot for a given snapshot object clone.
+
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9a91efc3047963364944f8be91cee8e8f6afc49a)
+
+commit d1ac70a4d06ed07a66e7886c00c712a26fd72e9d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Feb 21 11:52:00 2017 -0500
+
+    rbd-mirror: replace complex object op tuple with struct
+
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0c181527c0e151784a0f7c466aaa70b0772f91b1)
+
+commit bafa2b0c83d32b97a6dd85e681f2344384c539ac
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jan 20 14:26:43 2017 -0500
+
+    journal: don't hold future lock during assignment
+
+    It's possible that the future raced with its owner and reaches
+    an empty reference count. This was resulting in the future being
+    destructed while its lock was still held.
+
+    Fixes: http://tracker.ceph.com/issues/18618
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0f21ceef8336e35ca16148a9d58f511037911418)
+
+commit e3a230cb19d62bdcd18f3685d384d76e83b42ec5
+Author: tang.jin <tang.jin@istuary.com>
+Date:   Mon Jan 16 22:28:23 2017 +0800
+
+    rbd: add error prompt when input command 'snap set limit' is incomplete
+
+    Signed-off-by: Tang Jin <tang.jin@istuary.com>
+    (cherry picked from commit 8860028f508a9be5a08f512022cfb042021fd19f)
+
+commit b9258ccfd836d49d64c8de3cb614a4f0ce9806aa
+Author: Gaurav Kumar Garg <garg.gaurav52@gmail.com>
+Date:   Tue Jan 10 15:25:13 2017 +0100
+
+    rbd: bench-write should return error if io-size >= 4G
+
+    Currently if user perform bench-write with io-size > 4G
+    then its crashing because currently during memory allocation
+    bufferptr taking size of buffer as a unsigned and io-size > 4G
+    will overflow with unsigned. so during memset operation it will
+    try to set io_size size of memory area pointed by bufferptr,
+    (bufferptr area is:  (4G - io_size)), so it will cause
+    segmentation fault.
+
+    Fix is to return error if io-size >= 4G
+
+    Fixes: http://tracker.ceph.com/issues/18422
+
+    Reported-by:  Jason Dillaman <dillaman@redhat.com>
+    Signed-off-by: Gaurav Kumar Garg <garg.gaurav52@gmail.com>
+    (cherry picked from commit 6ab73e5f420e89e19b52e39dab28fa4c94e00197)
+
+commit d826600ee2c1b50675a0c8642c70ad803bca02a9
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Mon Jan 9 18:31:21 2017 +0100
+
+    librbd: metadata_set API operation should not change global config setting
+
+    Fixes: http://tracker.ceph.com/issues/18465
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 27465b5916b55ac3c2846c74b89f4362ad17ff1e)
+
+commit a0ad2d11cea867eb19d40a121907ec43f37e39a3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Dec 6 15:51:51 2016 -0500
+
+    librbd: delay mirror registration when creating clones
+
+    Fixes: http://tracker.ceph.com/issues/17993
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 778e1126a0343e2221446b8e13b48df5ccac263c)
+
+commit e91de89b0d62ee4869992254ff291c6a23bcafcc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 5 12:12:57 2017 -0500
+
+    librbd: possible deadlock with flush if refresh in-progress
+
+    Fixes: http://tracker.ceph.com/issues/18419
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b95f92a5572d3035c20eba07e76d2c825a9853f7)
+
+commit c2232643d0ee38c8148f13e69065a90c003f46e3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jan 6 15:59:22 2017 -0500
+
+    rbd-mirror: avoid processing new events after stop requested
+
+    Fixes: http://tracker.ceph.com/issues/18441
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d30873b269441815b5fc7de14c7d9a1077a17d8d)
+
+commit faf80bc1dd332f0ccaaa1caf2679c4b2be02288d
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed Apr 5 13:31:08 2017 +0300
+
+    radosgw-admin: use zone id when creating a zone
+
+    Fixes: http://tracker.ceph.com/issues/19498
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 3fea36d635fcba8ca584a1c0ec9f07840009402c)
+
+commit a5fe8124831daaaa67eb89555058d87600d3fff0
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Mar 14 15:43:13 2017 -0400
+
+    qa: rgw task uses period instead of region-map
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit e3e3a71d1f1fb43bb4172ce2dfac9a28ca89df0f)
+
+commit 0e964e477ac1e1a2f6f8c28b8fd0da8e9225360c
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Mar 14 14:18:15 2017 -0400
+
+    rgw-admin: remove deprecated regionmap commands
+
+    Fixes: http://tracker.ceph.com/issues/18725
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 5830c1849a0c0110d17c37784808e456e6dcb7b3)
+
+commit 03e5be4498d31c738f08d534bf8f27381fb8c12b
+Author: liuchang0812 <liuchang0812@gmail.com>
+Date:   Fri Feb 10 18:02:03 2017 +0800
+
+    rgw: don't return skew time in pre-signed url
+
+    Fixes: http://tracker.ceph.com/issues/18828
+
+    Signed-off-by: liuchang0812 <liuchang0812@gmail.com>
+    (cherry picked from commit dd8b348f4aad0124e8a4457117bf3f5f76af7bdb)
+
+commit 595c31f390fda700e6e346c208849d23c74bdf5f
+Author: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+Date:   Fri Feb 24 04:45:04 2017 +0800
+
+    rgw: add the remove-x-delete feature to cancel swift object expiration
+
+    In openstack swift, it also support the feature to cancel the object expiration,
+    which could be found at last point in
+    https://docs.openstack.org/user-guide/cli-swift-set-object-expiration.html. we
+    can remove the object expiration by set 'X-Remove-Delete-At:'.
+
+    This patch also could fix the bug that when we set the object expiration and
+    then upload the same object to the container again. The previous object expiration
+    also works, which is not compatible with the openstack swift.
+
+    Fixes: http://tracker.ceph.com/issues/19074
+    Signed-off-by: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+    (cherry picked from commit 230429ebc4ac9b5840bb93c7e0eeb5edbb949106)
+
+commit 3bbd3c5dc316ebd0fffc1a4cf22f44bd01330f6e
+Author: hrchu <petertc@gmail.com>
+Date:   Wed Mar 29 02:17:04 2017 +0000
+
+    doc: rgw: correct the quota section
+
+    Add the missing option and fix typo.
+
+    Fixes: http://tracker.ceph.com/issues/19397
+
+    Signed-off-by: Chu, Hua-Rong <hrchu@cht.com.tw>
+    (cherry picked from commit 51a88267f0d7f51aeb62092949b66b9f6c062e15)
+
+commit c3d8444890c394077177c555908ed78dc824a587
+Author: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+Date:   Wed Feb 8 15:07:43 2017 +0800
+
+    rgw: fix swift cannot disable object versioning
+
+    we should be able to disable object verioning by removing its X-Versions-Location
+    metadata header by sending an empty key value. this description can be found at
+    No.8 in http://docs.openstack.org/user-guide/cli-swift-set-object-versions.html.
+
+    Fixes: http://tracker.ceph.com/issues/18852
+    Signed-off-by: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+    (cherry picked from commit 17c5a0edd2227703cec867f0f588d4eae36dfe1c)
+
+commit eb84959bac369dbb86674f9e4225ba1c836c104c
+Merge: 85b7bc82e2 dc8ef3508b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 13 18:20:05 2017 +0200
+
+    Merge pull request #13609 from smithfarm/wip-19064-kraken
+
+    kraken: AttributeError: Thrasher instance has no attribute 'ceph_objectstore_tool'
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit e9a10eaccd1d36e74969127fea7a595a21645b6a
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Mar 1 13:18:44 2017 -0600
+
+    PendingReleaseNotes: warning about 'osd rm ...' and #19119
+
+    See http://tracker.ceph.com/issues/19119
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit be96003c464481d8e84825178d600234a0d64d22)
+
+    Conflicts:
+            PendingReleaseNotes
+            - drop "Calculation of recovery priorities has been updated" because
+              that was included in 11.2.0 release notes
+            - do not backport >=12.0.0 release notes
+            - change heading to 11.2.1
+
+commit 85b7bc82e28c6f1da08c6f6829555724d43255d2
+Merge: dd44cfa736 46548c5125
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 13 16:43:50 2017 +0200
+
+    Merge pull request #13790 from shinobu-x/wip-18298-kraken
+
+    kraken: mon: force_create_pg could leave pg stuck in creating state
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit dd44cfa7368b193997bc59114e187c7f51e66cb1
+Merge: d477944262 177141ff17
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 13 16:41:16 2017 +0200
+
+    Merge pull request #13500 from shinobu-x/wip-18587-kraken
+
+     kraken: mon: 'osd crush move ...' doesnt work on osds
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit d4779442620a8f6da5484be6188eeb9f636319c0
+Merge: 37bb087113 33d9a50dec
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 13 16:39:51 2017 +0200
+
+    Merge pull request #13490 from shinobu-x/wip-18820-kraken
+
+    kraken: osd: --flush-journal: sporadic segfaults on exit
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 37bb087113e3c8f8f7a79ee256ae7651a5165bc2
+Merge: 8a6e631b29 d8c0141a75
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 13 16:35:45 2017 +0200
+
+    Merge pull request #13295 from asheplyakov/kraken-bp-18497
+
+    kraken: osd: publish PG stats when backfill-related states change
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 8a6e631b292c56b4a577fd58cd604d31da8ba0bf
+Merge: b7877d1bb5 af2af7015a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 13 16:30:00 2017 +0200
+
+    Merge pull request #13253 from liewegas/wip-enxio-kraken
+
+    kraken: osd: do not send ENXIO on misdirected op by default
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit b7877d1bb5e5a38ad2cae4b0833cbefeed2547cc
+Merge: e1f548e38e b49a71daf5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 13 16:28:08 2017 +0200
+
+    Merge pull request #14322 from shinobu-x/wip-18619-kraken
+
+    kraken: osd: ceph degraded and misplaced status output inaccurate
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 7e30b630e2806c73ea503871599f958b58df7934
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Mar 16 12:28:41 2017 -0400
+
+    librbd: is_exclusive_lock_owner API should ping OSD
+
+    This is required to detect if a peer has been silently blacklisted
+    and is therefore no longer the lock owner.
+
+    Fixes: http://tracker.ceph.com/issues/19287
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e15db05960a284bdf3701256722299d553cfd5aa)
+
+    Conflicts:
+            src/librbd/ManagedLock.[h|cc]: logic moved to ExclusiveLock
+
+commit e1f548e38ef38ad7dc9631ba61e35b52781dfc2f
+Merge: 086a4c8de1 baa42b6d7c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 13 11:11:44 2017 +0200
+
+    Merge pull request #13216 from ovh/bp-osd-updateable-throttles-kraken
+
+    kraken: osd: allow client throttler to be adjusted on-fly, without restart
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 086a4c8de1243c2324d18cbe2d67704218065f83
+Merge: 95d4b4cefd 6d719558eb
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 13 10:45:10 2017 +0200
+
+    Merge pull request #14321 from shinobu-x/wip-kraken-15025
+
+    kraken: osd: New added OSD always down when full flag is set
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 95d4b4cefd0aff88e9c8fe25e44a4b33dfe66451
+Merge: c4e1e706dc 2b7083f8dd
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Thu Apr 13 10:42:31 2017 +0200
+
+    Merge pull request #13091 from dzafman/wip-18624-kraken
+
+    kraken: osd: Revert "PrimaryLogPG::failed_push: update missing as well"
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit c4e1e706dc8b7efd723bffd9b770b8b09c9a2300
+Merge: 4ea954d75e 06093bcb91
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 23:52:06 2017 +0200
+
+    Merge pull request #12790 from SUSE/wip-18418-kraken
+
+    kraken: osd: leveldb corruption leads to Operation not permitted not handled and assert
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 4ea954d75e0ff4969a5a35c77a229d43c66b9b28
+Merge: a4b0d2eeb5 478d0315cb
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 23:03:40 2017 +0200
+
+    Merge pull request #13172 from smithfarm/wip-18713-kraken
+
+    kraken: rgw: radosgw-admin period update reverts deleted zonegroup
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit a4b0d2eeb579b5a347d5a09f97eff8ca5a428101
+Merge: ce6423ac2c 29f55d7dd3
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 23:02:35 2017 +0200
+
+    Merge pull request #13176 from smithfarm/wip-18709-kraken
+
+    kraken: rgw: multisite: sync status reports master is on a different period
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit ce6423ac2c3d92011ab5e2434a3daec82195e085
+Merge: 577e3b4f23 5e15025ffb
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 23:01:42 2017 +0200
+
+    Merge pull request #13864 from smithfarm/wip-19156-kraken
+
+    kraken: rgw: typo in rgw_admin.cc
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 577e3b4f23cca525d49cc02b6873a69ddf7f98e3
+Merge: fca689ec2f c56a3134eb
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 23:00:37 2017 +0200
+
+    Merge pull request #14137 from smithfarm/wip-19331-kraken
+
+    kraken: rgw: upgrade to multisite v2 fails if there is a zone without zone info
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit fca689ec2f98ea40e3bc2ef7d338e73cb551934b
+Merge: 16f9611e05 84447a42e8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 22:59:18 2017 +0200
+
+    Merge pull request #14141 from smithfarm/wip-19342-kraken
+
+    kraken: rgw: "period update" does not remove short_zone_ids of deleted zones
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 16f9611e054e24f0e9324079b07ab2571a699caa
+Merge: 2fec93964c 76ce6841f7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 22:57:16 2017 +0200
+
+    Merge pull request #14142 from smithfarm/wip-19354-kraken
+
+    kraken: rgw: multisite: some 'radosgw-admin data sync' commands hang
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 2fec93964cccd43cedda95f724392f3f22bd47cb
+Merge: 41770bef0d 51e8ba0a06
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 22:56:22 2017 +0200
+
+    Merge pull request #14144 from smithfarm/wip-19356-kraken
+
+    kraken: rgw: when converting region_map we need to use rgw_zone_root_pool
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 41770bef0d640c5cee48ec117e2c90bd3e13f442
+Merge: 278ad230f3 f9e3e9f9ff
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Apr 12 10:08:40 2017 -0700
+
+    Merge pull request #14485 from zmc/wip-upgrade-openstack
+
+    suites/upgrade/hammer-jewel-x: Add volumes
+
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit b8d988f7f19112c3cc2a8379a4ed20418414abba
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Apr 1 23:04:22 2017 +0800
+
+    qa/tasks/workunit.py: use "overrides" as the default settings of workunit
+
+    otherwise the settings in "workunit" tasks are always overridden by the
+    settings in template config. so we'd better follow the way of how
+    "install" task updates itself with the "overrides" settings: it uses the
+    "overrides" as the *defaults*.
+
+    Fixes: http://tracker.ceph.com/issues/19429
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 47080150a17d238f38d9da824d227393ad767aad)
+
+commit 9212c1ec702be1b927a5a10dc1514803a13d85d5
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Mar 30 12:37:01 2017 +0800
+
+    tasks/workunit.py: specify the branch name when cloning a branch
+
+    c1309fb failed to specify a branch when cloning using --depth=1, which
+    by default clones the HEAD. and we can not "git checkout" a specific
+    sha1 if it is not HEAD, after cloning using '--depth=1', so in this
+    change, we dispatch "tag", "branch", "HEAD" using three Refspec classes.
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    (cherry picked from commit 9ca7ccf5f1739f731da8bf31260594aea3a2932d)
+
+commit 665f03a467c1b59eb80782e4ba3bbeb48b5d5efc
+Author: Dan Mick <dan.mick@redhat.com>
+Date:   Tue Mar 28 20:08:13 2017 -0700
+
+    tasks/workunit.py: when cloning, use --depth=1
+
+    Help avoid killing git.ceph.com.  A depth 1 clone takes about
+    7 seconds, whereas a full one takes about 3:40 (much of it
+    waiting for the server to create a huge compressed pack)
+
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    (cherry picked from commit c1309fbef300a062138ac40eb5d3e5081b833072)
+
+commit 63153e957862ac0b46b8a205ed81d319b66ea6f5
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Feb 27 16:29:40 2017 +0800
+
+    qa/tasks/workunit: use ceph.git as an alternative of ceph-ci.git for workunit repo
+
+    if we run upgrade test, where, for example, "jewel" is not in
+    ceph-ci.git repo, we should check ceph.git to clone the workunits.
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit edceabbd47697cb4e7d21798e57a58969a92ce79)
+
+commit 471303c79a552bbd3765a6c47507fc2f2926a29d
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Feb 24 16:43:07 2017 +0800
+
+    qa/tasks/workunit: use the suite repo for cloning workunit
+
+    as "workunits" reside in ceph/qa/workunits, it's more intuitive to
+    respect suite-repo option when cloning workunits.
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 4cf28de4c9a60a4618d101f555f5a2e59b7ab109)
+
+commit 278ad230f35c7742b6b066e693335dceaf291f3d
+Merge: baeac21515 c4f8114c6b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 16:42:37 2017 +0200
+
+    Merge pull request #14098 from shinobu-x/wip-19192-kraken
+
+    kraken: tools: ceph-brag fails to count "in" mds
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit baeac21515b9f472330f72fab6762e4e2535e6ba
+Merge: 29eb1ec09a d91697237a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 16:11:30 2017 +0200
+
+    Merge pull request #14155 from smithfarm/wip-18947-kraken
+
+    kraken: rbd-mirror: additional test stability improvements
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 29eb1ec09ab66cf8d1cee411809ab44a5a5d13b6
+Merge: c2e63ada80 6269f15223
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 16:10:47 2017 +0200
+
+    Merge pull request #14153 from smithfarm/wip-18892-kraken
+
+    kraken: librbd: Incomplete declaration for ContextWQ in librbd/Journal.h
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit c2e63ada80e3baecbaa8f567695ab860257e5cc5
+Merge: 14caf24f1f 3b0a7a8d2a
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 16:09:55 2017 +0200
+
+    Merge pull request #14151 from smithfarm/wip-18822-kraken
+
+    kraken: tests: run-rbd-unit-tests.sh assert in lockdep_will_lock, TestLibRBD.ObjectMapConsistentSnap
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 14caf24f1f3bc94d1a2a081957376474e8c3d66f
+Merge: 041b553a14 85b5817567
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 16:06:58 2017 +0200
+
+    Merge pull request #14149 from smithfarm/wip-18777-kraken
+
+    kraken: rbd: rbd --pool=x rename y z does not work
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 041b553a144cf798679caebcdb16d2ce2104c7ab
+Merge: 1d4617e1f2 24ecaed808
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 15:50:51 2017 +0200
+
+    Merge pull request #14315 from smithfarm/wip-19460-kraken
+
+    kraken: build/ops: spec file mentions non-existent ceph-create-keys systemd unit file, causing ceph-mon units to not be enabled via preset
+
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 1d4617e1f25498a6213cf94c3ff0701b61283ed2
+Merge: c860645bfa fcb86696eb
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 15:42:27 2017 +0200
+
+    Merge pull request #13875 from smithfarm/wip-19172-kraken
+
+    kraken: rgw: S3 create bucket should not do response in json
+
+    Reviewed-by: Abhishek Lekshmanan <abhishek@suse.com>
+
+commit c860645bfaa89ae48c4343208be8613fdc2538ce
+Merge: 2b7c762223 86a9450380
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 15:34:55 2017 +0200
+
+    Merge pull request #13873 from smithfarm/wip-19164-kraken
+
+    kraken: rgw: radosgw-admin: add the 'object stat' command to usage
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 2b7c762223ee628a8e29e216fee4417b5b570edc
+Merge: 2869d35262 d96ae9ead2
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 15:32:54 2017 +0200
+
+    Merge pull request #13245 from smithfarm/wip-18776-kraken
+
+    kraken: rbd: qemu crash triggered by network issues
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 2869d35262bc261783b3418649458d2d6e263f18
+Merge: 9b17d88cf2 85bda6e263
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 15:32:17 2017 +0200
+
+    Merge pull request #13247 from smithfarm/wip-18456-kraken
+
+    kraken: rbd: attempting to remove an image with incompatible features results in partial removal
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 9b17d88cf2ef4df40d26622ffa627d2fb70a557f
+Merge: 4a3f60f04f deb6d2c64c
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 15:31:24 2017 +0200
+
+    Merge pull request #13132 from rjfd/wip-18609-kraken
+
+    kraken: librbd: allow to open an image without opening parent image
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 4a3f60f04fde783fdfbf7ce1ea592ee348be34d3
+Merge: cfed98ae29 16b2fd00ec
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 15:17:09 2017 +0200
+
+    Merge pull request #13185 from smithfarm/wip-18721-kraken
+
+    kraken: build/ops: systemd restarts Ceph Mon to quickly after failing to start
+
+    Reviewed-by: Boris Ranto <branto@redhat.com>
+    Reviewed-by: Wido den Hollander <wido@42on.com>
+
+commit 57f3656231b7780ff11f95a9feaaa28c3d573b35
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Mar 16 12:27:08 2017 -0400
+
+    pybind: fix incorrect exception format strings
+
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 68617455f534a612ade1331f43b032ab524704ae)
+
+commit cfed98ae291f12c119083ba08fe813b9f0bc9eb9
+Merge: 5abdffe950 e40f48dbfc
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 13:55:16 2017 +0200
+
+    Merge pull request #13845 from smithfarm/wip-19147-kraken
+
+    kraken: rgw: DUMPABLE flag is cleared by setuid preventing coredumps
+
+    Reviewed-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit 6e5ac14ab6f53424831e4cf08a3349a12b1ca947
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Fri Feb 17 00:56:34 2017 +0100
+
+    rgw: make sending Content-Length in 204 and 304 controllable
+
+    This commit introduces a new configurable "rgw print prohibited
+    content length" to let operator decide whether RadosGW complies
+    to RFC 7230 (a part of the HTTP specification) or violates it
+    but follows the Swift's behavior.
+
+    Fixes: http://tracker.ceph.com/issues/16602
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit d8e3e64ec97a3c222a56bb6f510e5e23d7858615)
+
+commit 5abdffe950b51d5aff3500fd8dd7454a279da2fc
+Merge: 0517890f58 6f06cf80a7
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 13:43:09 2017 +0200
+
+    Merge pull request #13555 from smithfarm/wip-18707-kraken
+
+    kraken: mds: failed filelock.can_read(-1) assertion in Server::_dir_is_nonempty
+
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 0517890f58106eed3c0348a0a18b24bde7b340ab
+Merge: a48a897710 b464d1a571
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 13:37:49 2017 +0200
+
+    Merge pull request #13488 from shinobu-x/wip-18955-kraken
+
+    kraken: ceph-disk: Fix getting wrong group name when --setgroup in bluestore
+
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit a48a8977100838fc4140d9d59511a6cc21ec0075
+Merge: 59c5511cb1 29c8c3fb86
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 12:46:00 2017 +0200
+
+    Merge pull request #13112 from jcsp/wip-18678
+
+    kraken: mds: finish clientreplay requests before requesting active state
+
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 59c5511cb18286db1b032db8f3459763cc8dbb7c
+Merge: 8ded39b989 dab9fe761e
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:52:42 2017 +0200
+
+    Merge pull request #14048 from asheplyakov/reindex-on-pg-split-kraken
+
+    kraken: osd: pg log split does not rebuild index for parent or child
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 8ded39b9899ea461756650c17e1446682690c6f0
+Merge: 2a2bcc6051 35b13c6687
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:47:12 2017 +0200
+
+    Merge pull request #13116 from smithfarm/wip-18403-kraken
+
+    kraken: mon: cache tiering: base pool last_force_resend not respected (racing read got wrong version)
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 2a2bcc6051ad0d4badfdbabb81bd3d9cfb449234
+Merge: 005e803abe 7475e44214
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:41:21 2017 +0200
+
+    Merge pull request #13487 from shinobu-x/wip-18819-kraken
+
+    kraken: common: possible lockdep false alarm for ThreadPool lock
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 005e803abe5dbe652bbee1c3e353fb38db446800
+Merge: 42e93ae1b5 a379430d19
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 10:36:12 2017 +0200
+
+    Merge pull request #14147 from smithfarm/wip-18769-kraken
+
+    kraken: rbd: [  FAILED  ] TestJournalTrimmer.RemoveObjectsWithOtherClient
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 42e93ae1b5e1e65a8abb3f37469030ed4145fbda
+Merge: fb29415450 ecb459f083
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Wed Apr 12 02:53:06 2017 +0200
+
+    Merge pull request #13026 from SUSE/wip-18606-kraken
+
+    kraken: tools: ceph-disk prepare writes osd log 0 with root owner
+
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit f9e3e9f9ff22901d90e783a28efdfafa12be650f
+Author: Zack Cerza <zack@redhat.com>
+Date:   Tue Apr 11 09:25:50 2017 -0600
+
+    suites/upgrade-hammer-jewel-x: Add volumes
+
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+
+commit 543e4b90603b5992ea5d1c66d89e077375faf14e
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Mar 2 12:41:07 2017 +0100
+
+    build/ops: add psmisc dependency to ceph-base (deb and rpm)
+
+    Fixes: http://tracker.ceph.com/issues/19129
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 769b695465162bc8424abf8e2f259e6765b5bbff)
+
+commit fb29415450eda1ec480600132a937efbe988a1f4
+Merge: 10b441aac7 6b3655fe4b
+Author: Loic Dachary <loic@dachary.org>
+Date:   Sun Apr 9 22:40:21 2017 +0200
+
+    Merge pull request #13497 from shinobu-x/wip-17821-kraken
+
+    kraken: ceph-disk: does not support cluster names different than 'ceph'
+
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 910b9899702cfc08b74df6d988e1981efdb33f88
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Mon Apr 3 13:37:17 2017 +1000
+
+    ceph-disk: Populate mount options when running "list"
+
+    Also tidy up by moving duplicated code into a function
+
+    Fixes: http://tracker.ceph.com/issues/17331
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 7943ab2e01e24f2dfc5b6f1d3ffdc8a49e01af45)
+
+commit 5c7a07677aa142fadbab36d0c1b24aea32ce9ad5
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Tue Mar 28 11:49:08 2017 +0200
+
+    osdc/Objecter: respect epoch barrier in _op_submit()
+
+    Epoch barrier instructs us to avoid sending (i.e. pause) any OSD ops
+    until we see a barrier epoch.  The only thing epoch_barrier check in
+    target_should_be_paused() does is keep already paused ops paused.  We
+    need to actually pause incoming OSD ops in _op_submit().
+
+    Fixes: http://tracker.ceph.com/issues/19396
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit f8e8efc0a53d7bd807cc0c2178aef7c4bed62ab7)
+
+commit c2cc96e6c19c77d0ea13e8a015d0dfa66ba1b304
+Author: Fabian Grünbichler <f.gruenbichler@proxmox.com>
+Date:   Wed Mar 22 16:13:50 2017 +0100
+
+    common: fix segfault in public IPv6 addr picking
+
+    sockaddr is only 16 bytes big, so declaring net as sockaddr
+    and then casting to sockaddr_in6 in case of IPv6 cannot
+    work.
+
+    using sockaddr_storage works for both IPv4 and IPv6, and is
+    used in other code parts as well.
+
+    note that the tests did not find this issue as they declared
+    the bigger structs and casted the references to (sockaddr *)
+
+    Fixes: http://tracker.ceph.com/issues/19371
+    Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
+    (cherry picked from commit ae2ee3d3835fe25b35eeb1a841ee5234cd69eb65)
+
+commit b49a71daf51bf2fdd892cbfc033c0760cbce0464
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Jan 18 08:33:40 2017 -0800
+
+    osd: Calculate degraded and misplaced more accurately
+
+    Calculate num_object_copies based on the larger of pool size,
+    up set size and acting set size.
+
+    Calculate num_objects_degraded as the difference between num_object_copies
+    and all copies found on acting set and backfilling up set OSDs.
+
+    Calculate num_objects_misplaced as all copies on acting set OSDs not in up set
+    less copies that have been backfilled to up set OSDs.
+
+    Fixes: http://tracker.ceph.com/issues/18619
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 8423bc40759cca137f61e7b755411719a84369d4)
+
+commit 6d719558eb8135b3104132e7e70c163169e93876
+Author: Mingxin Liu <mingxin@xsky.com>
+Date:   Mon Mar 13 23:41:58 2017 +0800
+
+    osd: don't share osdmap with objecter when preboot
+
+    Signed-off-by: Mingxin Liu <mingxin@xsky.com>
+    (cherry picked from commit a5a3644eecc49b4eea890c6999fe87536495dcbe)
+
+commit 24ecaed8081f0bfec3324f86423f80e5d29c99b3
+Author: Sébastien Han <seb@redhat.com>
+Date:   Thu Mar 30 09:32:31 2017 +0200
+
+    systemd: remove all occurence of ceph-create-keys
+
+    ceph-create-keys unit file does not exist anymore so there is no need to
+    call it.
+
+    Signed-off-by: Sébastien Han <seb@redhat.com>
+    (cherry picked from commit 205cff12b3fe577a02db119ed3084d5c46ba76ce)
+
+commit b77d780c91f96ce9bb63fa1c09d507d02e555178
+Author: Sébastien Han <seb@redhat.com>
+Date:   Wed Mar 29 15:47:56 2017 +0200
+
+    systemd: remove ceph-create-keys from presets
+
+    ceph-create-keys unit file was removed here:
+
+    * https://github.com/ceph/ceph/commit/8bcb4646b6b9846bb965cdec3ca2a21eb3b26bab
+    * https://github.com/ceph/ceph/commit/dc5fe8d415858358bd0baf5d8dce0a753f5e0cea
+
+    As a consequence the systemctl preset command now fails to run since the
+    unit does not exist anymore. Due to the redirection in /dev/null we
+    don't know what's happening.
+
+    Ultimately the mon unit doesn't get enabled and the mon service won't
+    start after reboot.
+    Removing the old/non-existent unit makes the command succeed now.
+
+    Signed-off-by: Sébastien Han <seb@redhat.com>
+    (cherry picked from commit 151c0ce213186b2196531ce74478ed7671eb9819)
+
+commit 10b441aac71ad2d7a7ee6181c958a8bb0f710213
+Merge: 55444ce3ae df59d6d5f7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Mar 28 21:00:31 2017 -0400
+
+    Merge pull request #14163 from trociny/wip-19368
+
+    kraken: librbd: possible race in ExclusiveLock handle_peer_notification
+
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 55444ce3aea473f5cd7c8f6eeb770b819318a6a4
+Merge: 98a87fa97c 2d668468f6
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Mar 28 15:25:39 2017 -0700
+
+    Merge pull request #14200 from dzafman/wip-18533-again
+
+    Fixes: http://tracker.ceph.com/issues/19391
+
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 2d668468f60dd55a5591aa74bda491545368eee0
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Mar 20 17:28:45 2017 -0700
+
+    filestore, tools: Fix logging of DBObjectMap check() repairs
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 1704f62c0831e6b07138f7dd14a89fef3c9ed2c1)
+
+commit f4945116b70b2ca457565ed313d2e0b52cf13383
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Mar 3 15:04:02 2017 -0800
+
+    osd: Simplify DBObjectMap by no longer creating complete tables
+
+    Bump the version for new maps to 3
+    Make clone less efficient but simpler
+    Add rename operation (use instead of clone/unlink)
+    For now keep code that understands version 2 maps
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 738156a99ed1caf61e5a8230eb8048360056c08e)
+
+    No ghobject_t::operator>() so use Kraken cmp_bitwise() instead
+    Need to use MIN_GHOBJ/MAX_GHOBJ instead of std::min/std::max
+
+commit 06c27d7ec0fa31000232b8f45a0fbeb6702f0180
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 15 16:17:32 2017 -0800
+
+    ceph-osdomap-tool: Fix seg fault with large amount of check error output
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 1dda0411f4fbb14ce1e0062da9f14ec3af505d39)
+
+commit 4810c544b10972e6b276517a2bb4ba2da0b06697
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 15 15:02:33 2017 -0800
+
+    osd: Add automatic repair for DBObjectMap bug
+
+    Add repair command to ceph-osdomap-tool too
+
+    Under some situations the previous rm_keys() code would
+    generated a corrupt complete table.  There is no way to
+    figure out what the table should look like now.  By removing
+    the entries we fix the corruption and aren't much worse off
+    because the corruption caused some deleted keys to re-appear.
+
+    This doesn't breaking the parent/child relationship during
+    repair because some of the keys may still be contained
+    in the parent.
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 4cd3c74c928a32e065ed9543d6c91d8718a6ae3d)
+
+    Conflicts:
+            src/os/filestore/DBObjectMap.h (trivial)
+
+commit 61dad6c15dc7ac94c4bea1fb57823d78ae2db5a1
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 15 14:59:40 2017 -0800
+
+    ceph-osdomap-tool: Fix tool exit status
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 666f14ed90655a2d1bedde8561949625db7a9e6c)
+
+commit a72f923186ac490fe780f7168a6a34c7d933cf96
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Feb 10 15:51:42 2017 -0800
+
+    DBObjectMap: rewrite rm_keys and merge_new_complete
+
+    Leverage the updated in_complete_region and needs_parent to simplify
+    these methods.
+
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit c4dffb68eaafe724f7fdae93a4285a7f8003ea29)
+
+commit 960431a6b8e79f6832a85e462889dd1f26f12624
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Feb 10 15:50:57 2017 -0800
+
+    DBObjectMap: strengthen in_complete_region post condition
+
+    Previously, in_complete_region didn't guarantee anything about
+    where it left complete_iter pointing.  It will be handy for
+    complete_iter to be pointing at the lowest interval which ends
+    after to_test.  Make it so.
+
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 97b35f4d7d4862da4b6f50ecaef0d292a671fd04)
+
+commit a3c3e3ef4e3d20baf28000f11b4ea5bb45e161f9
+Author: Samuel Just <sjust@redhat.com>
+Date:   Fri Feb 10 15:48:57 2017 -0800
+
+    DBObjectMap: fix next_parent()
+
+    The previous implementation assumed that
+    lower_bound(parent_iter->key()) always leaves the iterator
+    on_parent().  There isn't any guarantee, however, that that
+    key isn't present on the child as well.
+
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit 74a7631d0938d7b44894f022224eab10a90d5cec)
+
+commit 86881dd361e21e3e4553b416331ca393e88b7a3e
+Author: Samuel Just <sjust@redhat.com>
+Date:   Thu Feb 9 10:47:59 2017 -0800
+
+    test_object_map: add tests to trigger some bugs related to 18533
+
+    Signed-off-by: Samuel Just <sjust@redhat.com>
+    (cherry picked from commit f131dbcf5bb17107c029f942a57e9bf4432a26ee)
+
+commit 4c7fbea9e4c171ab21f3dcd6ca0b17abb3420979
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Feb 14 12:40:33 2017 -0800
+
+    test: Add ceph_test_object_map to make check tests
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0e97a01bd7291458881ee53cece2d887f6333669)
+
+commit ef5aeab9b22292c366489668ee21ce4b9d2d8572
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 8 18:56:27 2017 -0800
+
+    ceph-osdomap-tool: Add --debug and only show internal logging if enabled
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 5fb2b2d13953979e5da9f571ab8c4b0b510b8368)
+
+commit c5d462f3524f0e5eff1594d7ddf97c3c9c0882a8
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 8 18:55:48 2017 -0800
+
+    osd: DBOjectMap::check: Dump complete mapping when inconsistency found
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit fcf1e17c645e8fad5216c3e59627c817e5c858c7)
+
+commit b37b0298ca9c002033b3ae8ebcaeb7a24690447d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 8 15:38:51 2017 -0800
+
+    test_object_map: Use ASSERT_EQ() for check() so failure doesn't stop testing
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 053a273cbc02d6902a4bb1f11db1ea946498df3a)
+
+commit 8f297204a428b48d2119df0e1a8da41e4f15e4fb
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 8 10:02:40 2017 -0800
+
+    tools: Check for overlaps in internal "complete" table for DBObjectMap
+
+    Changed check to return an error count and fix tool error message
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit e5e8eb962db6187ea19b96ba29ac83469c90b4ea)
+
+    Conflicts:
+            src/os/filestore/DBObjectMap.h (trivial)
+
+commit bc2228a903bbb89a11d7863f9307d6c3b5d4928c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Feb 8 09:40:49 2017 -0800
+
+    tools: Add dump-headers command to ceph-osdomap-tool
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit f4101591ad701a62fe027c4744ca8ea505f44bdc)
+
+    Conflicts:
+            src/os/filestore/DBObjectMap.h (trivial)
+
+commit 97186735783a9758a1695a9938b22c2b4b46f530
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Feb 6 21:09:42 2017 -0800
+
+    tools: Add --oid option to ceph-osdomap-tool
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 2d94889e9ee3359017b1efd560f3557ce03ccee6)
+
+commit c208c91c7f845896b4fe0befd62cc66bee69018f
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Feb 6 21:31:18 2017 -0800
+
+    osd: Remove unnecessary assert and assignment in DBObjectMap
+
+    Fix and add comment(s)
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 937e6a03ea4692cc44d53faa0615f8e808c9eb03)
+
+commit df59d6d5f7deb586cf14a6ef6984e6847db08852
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Mar 28 15:55:11 2017 +0200
+
+    kraken: librbd: possible race in ExclusiveLock handle_peer_notification
+
+    This is a direct commit to kraken -- the master diverged after
+    ManagedLock refactoring and is not affected.
+
+    Fix: http://tracker.ceph.com/issues/19368
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+
+commit d91697237ad77eaf60098536a2cd3538ea9aeabf
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 21 15:31:43 2016 -0500
+
+    qa/workunits/rbd: resolve potential rbd-mirror race conditions
+
+    Fixes: http://tracker.ceph.com/issues/18935
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 63eae97afc1a92412525468263fb8696a243ebac)
+
+commit 6269f152238b77b0d02e4ad865164a531541d6db
+Author: Boris Ranto <branto@redhat.com>
+Date:   Wed Feb 8 23:47:57 2017 +0100
+
+    librbd: Include WorkQueue.h since we use it
+
+    We use m_work_queue of type ContextWQ in handle_update function but we
+    do not include common/WorkQueue.h that defines ContextWQ. This results
+    in dereference of an incomplete type and causes build error in latest
+    Fedora rawhide (future 26).
+
+    Fixes: http://tracker.ceph.com/issues/18862
+
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 480f82847ad1fc7959f1fe5a90761a5a24550993)
+
+commit 3b0a7a8d2aa0dbc7cfce317bd327f88afce2469e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jan 18 20:54:22 2017 -0500
+
+    librbd: avoid possible recursive lock when racing acquire lock
+
+    Fixes: http://tracker.ceph.com/issues/17447
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5e46e8eb664f573bd70ae7c96a6d9a98b0deb09e)
+
+commit 85b5817567c22acf63719a65bd65e77709ffce0c
+Author: Gaurav Kumar Garg <garg.gaurav52@gmail.com>
+Date:   Mon Jan 30 13:03:20 2017 +0100
+
+    rbd: destination pool should be source pool if it is not specified
+
+    Currently if user perform image rename operation and user give pool
+    name as a optional parameter (--pool=<pool_name>) then currently
+    its taking this optional pool name for source pool and making
+    destination pool name default pool name.
+    With this fix if user provide pool name as a optional pool name
+    parameter then it  will consider both soruce and destination pool
+    name as optional parameter pool name.
+
+    Fixes: http://tracker.ceph.com/issues/18326
+
+    Reported-by: М�РК КОРЕ�БЕРГ <socketpair@gmail.com>
+    Signed-off-by: Gaurav Kumar Garg <garg.gaurav52@gmail.com>
+    (cherry picked from commit 01f23aa99fb694da326ab408e75b33c640ce660b)
+
+commit a379430d1921a1e5a330b4bfe0189a9a21dc5f81
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jan 30 17:41:51 2017 -0500
+
+    journal: stop processing removal after error
+
+    Fixed: http://tracker.ceph.com/issues/18738
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 040004f53f7d334fc9452e3f5fd80169cad5f65e)
+
+commit 51e8ba0a06f2b5574f82f0ca63a9e9b73c291b51
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Mar 12 12:11:28 2017 +0200
+
+    rgw: use rgw_zone_root_pool for region_map like is done in hammer
+
+    Fixes: http://tracker.ceph.com/issues/19195
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit c91dd6d9efd148e0fe0f027dde537e977de9aa26)
+
+commit 76ce6841f7066b4902665dcf752568d0b5417652
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Mon Feb 27 15:52:43 2017 +0800
+
+    rgw: use separate http_manager for read_sync_status
+
+    concurrent users of read_sync_status() use different cr managers, when get_resource must
+    use http_manager related to the cr manager.
+
+    Fixes: http://tracker.ceph.com/issues/19236
+
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit c412024889f8995d98096ac863bafee71624bd70)
+
+commit 84447a42e82a21e08f71ea6113216d9a2cb2ab09
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Mar 9 15:24:08 2017 -0500
+
+    rgw: clear old zone short ids on period update
+
+    the short ids of old, removed zones were being kept in the period to
+    guard against hash collisions with new zones
+
+    but for a hash collision to cause a wrong object to sync, that object
+    would have to be uploaded simultaneously to two different zones that had
+    the same short id
+
+    to avoid this, we just have to prevent the period from containing two
+    colliding zones at the same time - we don't have to remember old zone
+    short ids forever
+
+    Fixes: http://tracker.ceph.com/issues/15618
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 9c45633c836c966ab1f75ea2b1ad3fa0a4886600)
+
+commit c56a3134ebe1b531af09b19214a16b37e0e8a6d8
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Mar 9 13:03:24 2017 +0200
+
+    rgw: skip conversion of zones without any zoneparams
+
+    Fixes: http://tracker.ceph.com/issues/19231
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 36cf5a5c8179c6313346b2e29286c537c6fefce8)
+
+commit d0d70359811c306116921cf8f3768a97da16a813
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Mar 9 11:16:26 2017 +0200
+
+    rgw: better debug information for upgrade
+
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit e9f3bf8eab1dd46a92f54b0f7afe1f4c0e4204db)
+
+commit 01f5d8899b8d16e419e1bfdd4e8bbd80a058df93
+Author: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+Date:   Tue Jan 31 18:01:32 2017 +0100
+
+    rgw/rgw_rados.cc: prefer ++operator for non-primitive iterators
+
+    Signed-off-by: Danny Al-Gaaf <danny.al-gaaf@bisect.de>
+    (cherry picked from commit 7086cf9a73f2ec1eb96c0e752beb1b74fca18570)
+
+commit c4f8114c6b86bc1e0afabecf9424dadd30b37006
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Mar 6 11:33:27 2017 +0800
+
+    brag: count the number of mds in fsmap not in mdsmap
+
+    this change was introduced in 4e9b953
+
+    Fixes: http://tracker.ceph.com/issues/19192
+    Signed-off-by: Peng Zhang <zphj1987@gmail.com>
+    (cherry picked from commit 2d25a9c0c760664d3de33ecca0e0272c1031cd46)
+
+commit 81f3d860324da49f71b757e6cc8f32966272cf5c
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Tue Mar 21 12:22:20 2017 +1000
+
+    tools/rados: Check return value of connect
+
+    Fail gracefully if Rados::connect returns an error.
+
+    Fixes: http://tracker.ceph.com/issues/19319
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit c119091ef0844e4a1ddd790a8bfef8f06bb57d58)
+
+commit 076e4b7991e3bd734a41604fa7cb00c7b03d7749
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Oct 20 15:01:01 2016 -0400
+
+    rgw: fix break inside of yield in RGWFetchAllMetaCR
+
+    the yield macro is implemented with for/switch, so the breaks in
+    RGWFetchAllMetaCR weren't being applied to the for loop as expected -
+    so any of these breaks send RGWFetchAllMetaCR into an infinite loop
+
+    removed the yield {} block, so that breaks will apply to the for loop as
+    intended, then added a single yield; statement to allow the
+    entries_index consumer to run one per iteration
+
+    Fixes: http://tracker.ceph.com/issues/17655
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 190bd385a7be52867d65740c410884f5c8cbc21f)
+
+commit 5dcf60702cd1733dbe7f2d1d7dc68a096684bf77
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Mar 3 12:10:40 2017 -0500
+
+    rgw: delete_system_obj() fails on empty object name
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 67401193f871db95a6045915fa59dce8c5dd1012)
+
+commit dea53b3bf267709d2a8daacd961997892ad2d0f4
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Mar 3 11:42:45 2017 -0500
+
+    rgw: if user.email is empty, dont try to delete
+
+    Fixes: http://tracker.ceph.com/issues/18980
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 022ecf0fcc8e44912c8758ee1d9a452dc23cbbce)
+
+commit dab9fe761ec7f46b7da035f710f824516363119c
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Feb 17 12:46:38 2017 -0500
+
+    osd/PGLog: avoid return by value on ginormous log
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a6ead998771753d95382abd082f451e6f67744e4)
+
+commit eac5610896e59bc8e711ed48446aa6967ebb61c8
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Feb 17 14:50:38 2017 -0500
+
+    osd/PGLog: reindex properly on pg log split
+
+    When pg_log_t::split_out_child() runs it builds the list, which means the
+    old indexes are wrong (the point to bad memory), but index() will not
+    rebuild them because ever since b858e869e78927dccebaa350d246bd74af7f1de9
+    we won't rebuild them if they are already built.
+
+    Fix that by calling unindex() before the split.
+
+    Further, the new child log also needs to be indexed.  Fix that too.
+
+    Fixes: http://tracker.ceph.com/issues/18975
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 643ae42cf27f16dd6ed4e1402acc0483bb9fca74)
+
+commit 0227920438d98a8c1d74f0ec7a95bc63fa066acf
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Wed Mar 1 17:19:04 2017 +0100
+
+    osd/OSDMap: don't set weight to IN when OSD is destroyed
+
+    Since commit 4e28f9e63644 ("osd/OSDMap: clear osd_info, osd_xinfo on
+    osd deletion"), weight is set to IN when OSD is deleted.  This changes
+    the result of applying an incremental for clients, not just OSDs.
+    Because CRUSH computations are obviously affected, pre-4e28f9e63644
+    servers disagree with post-4e28f9e63644 clients on object placement,
+    resulting in misdirected requests.
+
+    Fixes: http://tracker.ceph.com/issues/19119
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit a6009d1039a55e2c77f431662b3d6cc5a8e8e63f)
+
+commit d4a43e1fd56cfce1e1b64827a3d98c14ec69e193
+Author: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+Date:   Fri Feb 3 19:06:31 2017 +0800
+
+    rgw: swift: fix anonymous user's error code of getting object
+
+    The openstack swift will return 401 rather than 403 when
+    the anon user has no permission to get objects.
+
+    Fixes: http://tracker.ceph.com/issues/18806
+    Signed-off-by: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+    (cherry picked from commit 318de28e1011914dc7f3701549eb3ef227abbc3c)
+
+commit fcb86696eb4999689c7da71d565e1262beba9dec
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Thu Feb 16 17:40:50 2017 +0100
+
+    doc: rgw: make a note abt system users vs normal users
+
+    Mention that system users don't behave like normal users in context of
+    normal rest operations
+
+    Fixes: http://tracker.ceph.com/issues/18889
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit a47bcf70c9f51a6601b809cba219f5615b204d34)
+
+commit 86a945038059d976e2f7227d9ee327ec2899d15c
+Author: root <root@ceph-node1.homeoffice.wal-mart.com>
+Date:   Tue Feb 7 14:37:36 2017 +0530
+
+    rgw: Let the object stat command be shown in the usage
+
+    Fixes: http://tracker.ceph.com/issues/19013
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+    (cherry picked from commit 0fe76f83d19be098ef54fb0492a376fef3aa9e23)
+
+commit 1f86be6e19dc3e26f62bf57c55d662414247b480
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Feb 20 16:00:01 2017 -0500
+
+    rgw: RGWMetaSyncShardControlCR retries with backoff on all error codes
+
+    RGWBackoffControlCR only treats EBUSY and EAGAIN as 'temporary' error
+    codes, with all other errors being fatal when exit_on_error is set
+
+    to RGWMetaSyncShardControlCR, a 'fatal' error means that no further sync
+    is possible on that shard until the gateway restarts
+
+    this changes RGWMetaSyncShardControlCR to set exit_on_error to false, so
+    that it will continue to retry with backoff no matter what error code it
+    gets
+
+    Fixes: http://tracker.ceph.com/issues/19019
+
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 3e4059557fd6cad5d31014327f60832b36d04a6c)
+
+commit b39bef09611653723a29c989a6012064f17bd586
+Author: root <root@ceph-node1.homeoffice.wal-mart.com>
+Date:   Tue Feb 21 16:33:29 2017 +0530
+
+    rgw: Correct the return codes for the health check feature
+    Fixes: http://tracker.ceph.com/issues/19025
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+
+    (cherry picked from commit 4da2bf310f6d43423554c32e43ebf90ad2c3f3a9)
+
+commit 5e15025ffbf80f65cde891ad47fe1f1b913465cd
+Author: Ronak Jain <ronakjain@outlook.in>
+Date:   Wed Feb 22 12:03:46 2017 +0530
+
+    rgw: Fixes typo in rgw_admin.cc
+
+    Issue: http://tracker.ceph.com/issues/19026
+    Signed-off-by: Ronak Jain <ronakjain@outlook.in>
+    (cherry picked from commit 58837ef6ce8cbcfc2cac29d5f833b2cf62d8737a)
+
+commit e40f48dbfce23cbc2164f8789db4688fbec9c3bb
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Mon Feb 27 13:06:59 2017 +1000
+
+    rgw: set dumpable flag after setuid post ff0e521
+
+    ff0e521 resolved the issue for the other daemons but not for rgw since
+    it calls setuid (via civetweb) after the new code sets PR_SET_DUMPABLE.
+    Add another prctl call before wait_shutdown.
+
+    Fixes: http://tracker.ceph.com/issues/19089
+
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit bc458d39630b599e0e1ca9fe25ad7455fcffdd10)
+
+commit 47cf8f6699ec0690e2a296b3abe289ce5d212e1c
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Feb 27 10:35:01 2017 -0800
+
+    rgw: don't init rgw_obj from rgw_obj_key when it's incorrect to do so
+
+    Fixes: http://tracker.ceph.com/issues/19096
+
+    rgw_obj_key currently deals with the bucket index key, and not
+    representing a (name, instance, ns) tupple. Need to initialize
+    it in two steps.
+
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 392c5d9dae6ba699014ffe6e1e67818fa62d7e41)
+
+commit 03b1ab4b13780a53c7b9017f3a6207dab2ffff5b
+Author: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+Date:   Thu Jan 19 21:00:26 2017 +0800
+
+    rgw: swift: the http referer acl in swift API should be shown
+    The container acl about http referer set should be shown in container metadata dump.
+
+    Fixes: http://tracker.ceph.com/issues/18665
+    Signed-off-by: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+    (cherry picked from commit 3860ddad084c5b8d9d5ec83aeb3220207cd0e5ad)
+
+commit 98a87fa97c9b23e21a05130c72730f5034691310
+Merge: 7e16ea218d e2074cef71
+Author: Sage Weil <sage@newdream.net>
+Date:   Sat Mar 4 11:22:44 2017 -0600
+
+    Merge pull request #13501 from shinobu-x/wip-18371-kraken
+
+     kraken: ceph-disk: error on _bytes2str
+
+commit 46548c51255518e422d544f0d0776068051fdacc
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Mon Nov 14 19:33:56 2016 -0500
+
+    common: Unskew clock
+
+    In preparation to deglobalizing CephContext, remove the CephContext*
+    parameter to ceph_clock_now() and ceph::real_clock::now() that carries
+    a configurable offset.
+
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit 750ad8340c827d2f8896e1251e45f921dddb9f30)
+
+    Conflicts:
+            src/mon/PGMonitor.cc
+
+commit 1120cd74d88b163f7cd1c81ae5670bf6d6153715
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jan 31 12:31:42 2017 -0500
+
+    mon/PGMonitor: fix force_create_pg
+
+    We weren't carefully registering the creating PG.  In
+    particular, the current osd mappings (acting, up, etc.)
+    weren't getting filled in, which meant the PG wasn't
+    (necessarily) mapped to an OSD until the OSDMap updated
+    and we recalculated mappings.
+
+    Fix by sending us through the common code path so that
+    all fields get updated properly.
+
+    Fixes: http://tracker.ceph.com/issues/18298
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9a41a0b7289fa59f4b747a63e152e88af6e8abd5)
+
+    Conflicts:
+            src/mon/PGMonitor.cc
+
+commit 396f3325507cf899433da0a5f4964b81e0c2caa8
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jan 31 12:31:12 2017 -0500
+
+    mon/PGMonitor: clean up some send_pg_creates cruft
+
+    ff1fd4b4b69b52ef5ae734115b1d733aad03b03e already removed most of
+    this; remove the now useless stub and clean up the unneede con
+    check.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b79a85e80a1a660c5a5746270edb86d3f09cd983)
+
+    Conflicts:
+            src/mon/PGMonitor.cc
+
+commit 6196b6edd5ea87c9fabe598a309516e1c0ba73e9
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 16 17:42:26 2016 -0500
+
+    mon/MonCommands: remove send_pg_creates from cmd table
+
+    Forgot this in 6cbdd6750cf330047d52817b9ee9af31a7d318ae
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 15f17bcca0bde2ed15bb45d67659f4ebbd4cb516)
+
+commit c8d70ee0129ce2cc459a2be386531aeee5c6c422
+Author: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+Date:   Thu Jan 19 21:26:17 2017 +0800
+
+    rgw: swift: The http referer should be parsed to compare in swift API
+    The http referer should be parsed to compare with the url set on the container read acl. If we set .r:www.example.com on container read acl, we should parse the hostname 'www.example.com' of the http referer like 'http://www.example.com' from the http request.
+
+    Fixes: http://tracker.ceph.com/issues/18685
+    Signed-off-by: Jing Wenjun <jingwenjun@cmss.chinamobile.com>
+    (cherry picked from commit 941dfad67174ae3ec517e76bf4028c50fb46fe82)
+
+commit 7e16ea218d38703f1903dca002905f040a821590
+Merge: ccc1982472 6adf39b983
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Fri Mar 3 11:14:48 2017 +0800
+
+    Merge pull request #13757 from liewegas/wip-pgp-kraken
+
+    qa/tasks: set pgp = pg num on thrashing finish
+
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 6adf39b98305b6af8d0192b35ec22c5007293d32
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sun Feb 19 13:10:44 2017 +0800
+
+    test: Thrasher: do not update pools_to_fix_pgp_num if nothing happens
+
+    we should not update pools_to_fix_pgp_num if the pool is not expanded or
+    the pg_num is not increased due to pgs being created. this prevent us
+    from fixing the pgp_num after done with thrashing if we actually did
+    nothing when fixing the pgp_num when thrashing, but we removed the pool
+    from pools_to_fix_pgp_num after set_pool_pgpnum() returns.
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit c0f0cde399a80cea617c115417e5390a63ec674e)
+
+commit ab5538fd7e3b727616272edb565e3782d004632a
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sun Feb 12 13:16:57 2017 +0800
+
+    test: Thrasher: update pgp_num of all expanded pools if not yet
+
+    otherwise wait_until_healthy will fail after timeout as seeing warning
+    like:
+
+    HEALTH_WARN pool cephfs_data pg_num 182 > pgp_num 172
+
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 136483a8f940710cbe40804afa7af2eac6728d50)
+
+commit ccc1982472898efe2fcbd6b1cd3cad7afbf2c015
+Merge: ae8eff5f2d 2c9889bb76
+Author: Zack Cerza <zack@cerza.org>
+Date:   Wed Mar 1 15:16:17 2017 -0700
+
+    Merge pull request #13707 from zmc/wip-openstack-volumes-kraken
+
+    qa/suites/ceph-deploy: Drop OpenStack volume count
+
+commit 2c9889bb768198fe292de0c3edb076707e8a50e0
+Author: Zack Cerza <zack@redhat.com>
+Date:   Tue Feb 28 13:07:15 2017 -0700
+
+    qa/suites/ceph-deploy: Drop OpenStack volume count
+
+    Looks like we only need two per node, since there is only one OSD per
+    node, and ceph-deploy wants two disks per OSD to account for the
+    journal.
+
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit 87072e277c9ef259c9ee2ae1f761e252aa216713)
+
+commit ae8eff5f2dddcc064485ab69221c6d716e63925f
+Merge: 9fa9248e92 f9a282bc9d
+Author: Zack Cerza <zack@cerza.org>
+Date:   Mon Feb 27 12:23:10 2017 -0700
+
+    Merge pull request #13673 from zmc/wip-openstack-volumes-kraken
+
+    qa/suites/{ceph-ansible,rest}: OpenStack volumes
+
+commit f9a282bc9dda98a103db2c7ebe63450dad23cc5b
+Author: Zack Cerza <zack@redhat.com>
+Date:   Mon Feb 27 09:14:41 2017 -0700
+
+    qa/suites/rest: Openstack volumes
+
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit 99d942145f4206c00aca30c0bb74f0edc4bac798)
+
+commit e0332944c7647cc269d6b18ca867896d5bea2fa7
+Author: Zack Cerza <zack@redhat.com>
+Date:   Mon Feb 27 09:06:26 2017 -0700
+
+    qa/suites/ceph-ansible: Openstack volumes
+
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit 964b983bdbd412311bce56184d12d1b6d43c7f28)
+
+commit 9fa9248e92f34127ca8b0d3b021b5ac0da0f6609
+Merge: f159208832 b2e3692347
+Author: Brad Hubbard <badone@users.noreply.github.com>
+Date:   Sun Feb 26 09:23:43 2017 +1000
+
+    Merge pull request #13485 from shinobu-x/wip-18644-kraken
+
+    kraken: osd: condition object_info_t encoding on required (not up) features
+
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit f1592088328480c6545435a7031d8f06277a9d0c
+Merge: c59d13d162 2ab8239b09
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 24 17:22:11 2017 -0700
+
+    Merge pull request #13641 from zmc/wip-fs-openstack-kraken
+
+    qa/suites/fs: Add openstack volume configuration
+
+commit 2ab8239b09f5d1215ad828eae7c64fb0f199d663
+Author: Zack Cerza <zack@redhat.com>
+Date:   Fri Feb 24 15:25:19 2017 -0700
+
+    qa/suites/fs: Add openstack volume configuration
+
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit b076d89a3f1fbad7d477913812b2e17529abeacf)
+
+commit c59d13d162a2b2d51fe7c65a56a2a17b7d94aa8f
+Merge: 8f1f2018fb 596cfbbd0f
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 24 14:44:04 2017 -0700
+
+    Merge pull request #13638 from zmc/wip-openstack-volumes-kraken
+
+    qa/suites/{knfs,hadoop,samba}: OpenStack volume configuration
+
+commit 8f1f2018fb06bd56b63cb661a852ba6b44c5e749
+Merge: 1e71356ee4 cec66c4dac
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 24 14:43:34 2017 -0700
+
+    Merge pull request #13635 from zmc/wip-kcephfs-openstack-kraken
+
+    qa/suites/kcephfs: Openstack volume configuration
+
+commit 6b3655fe4bd3c493e6b10dd597d791a5b572df2b
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Feb 22 01:49:12 2017 +0100
+
+    ceph-disk: dmcrypt activate must use the same cluster as prepare
+
+    When dmcrypt is used, the fsid cannot be retrieved from the data
+    partition because it is encrypted. Store the fsid in the lockbox to
+    enable dmcrypt activation using the same logic as regular activation.
+
+    The fsid is used to retrive the cluster name that was used during
+    prepare, reason why activation does not and must not have a --cluster
+    argument.
+
+    Fixes: http://tracker.ceph.com/issues/17821
+
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 7f66672b675abbc0262769d32a38112c781fefac)
+
+    Conflicts:
+            src/ceph-disk/ceph_disk/main.py
+
+commit 596cfbbd0fc521ef66e7e11a7098ea78adeb2565
+Author: Zack Cerza <zack@redhat.com>
+Date:   Fri Feb 24 13:46:54 2017 -0700
+
+    qa/suites/samba: Openstack volume configuration
+
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit e0296d706422ea4dc01d84f8786f6f7104c3d996)
+
+commit 3359050ce87426f3b41afe85c117de2d47ece395
+Author: Zack Cerza <zack@redhat.com>
+Date:   Fri Feb 24 13:45:18 2017 -0700
+
+    qa/suites/hadoop: Openstack volume configuration
+
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit 3fef0a49da2ccfdceba7b98e9096be8305da1111)
+
+commit a2ff395cb66fc20fd9a18e74a6cba5510d144a61
+Author: Zack Cerza <zack@redhat.com>
+Date:   Fri Feb 24 13:44:33 2017 -0700
+
+    qa/suites/knfs: Add openstack volume configuration
+
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit 62c6fd3371adf0f420c12d9c7e2b3a2a0c69256b)
+
+commit cec66c4dacb39600f74927c9f8890805d3f7acdc
+Author: Zack Cerza <zack@redhat.com>
+Date:   Fri Feb 24 13:37:23 2017 -0700
+
+    qa/suites/kcephfs: Openstack volume configuration
+
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit ec6fb28eaf8e2db327e4afc115879a40c7664e07)
+
+commit 1e71356ee44828a0624d77f77b6a22580d1df28b
+Merge: 90c61976e8 7439d4cff2
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 24 12:16:02 2017 -0700
+
+    Merge pull request #13633 from zmc/wip-krbd-openstack-kraken
+
+    qa/suites/krbd: Add openstack volume configuration
+
+commit 90c61976e82c5e23d4d3cbae00eddb47e649bb65
+Merge: 91a2ab03e2 4c55a78a52
+Author: Zack Cerza <zack@cerza.org>
+Date:   Fri Feb 24 12:15:29 2017 -0700
+
+    Merge pull request #13613 from ceph/wip-rgw-openstack-kraken
+
+    qa/suites/rgw: Add openstack volume configuration
+
+commit 91a2ab03e268227164ffec37b693d370485cc004
+Merge: 6c6b185bab 19ca0db392
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 24 10:37:12 2017 -0800
+
+    Merge pull request #13618 from tmuthamizhan/wip-rm-trusty-kraken
+
+    qa: drop ubuntu trusty support
+
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit 7439d4cff231e31dbf7c19591560bc0aa778f68a
+Author: Zack Cerza <zack@redhat.com>
+Date:   Fri Feb 24 11:17:45 2017 -0700
+
+    qa/suites/krbd: Add openstack volume configuration
+
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit 201b4d0d1e92bf95ac6a8f2951b664763030f12d)
+
+commit 4c55a78a52f9e089b97ab8a2feb9876ebdba0516
+Author: Zack Cerza <zack@redhat.com>
+Date:   Thu Feb 23 10:14:05 2017 -0700
+
+    qa/suites/rgw: Add openstack volume configuration
+
+    Without this, OSDs will fail to create on instances whose root fs isn't
+    xfs.
+
+    (cherry picked from commit 8af4c35f9577ef5a88307ea5cbbe2561a473926c)
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+
+commit b2e369234705ae78774261c62081729e6a50e8e5
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Mon Feb 6 12:51:05 2017 +0100
+
+    osd/OSDMap: require OSD features only of OSDs
+
+    Fix up commit 1a5cc32f0a3b ("osd/OSDMap: reflect REQUIRE_*_OSDS flag in
+    required features") -- require_*_osds shouldn't affect older clients.
+
+    Fixes: http://tracker.ceph.com/issues/18831
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit ab558bbf2956157cbde59c155a5180a62ae73d07)
+
+    Conflicts:
+            src/osd/OSDMap.cc
+
+commit 19ca0db392b7bdf9758aae4fb6bb6cb927411568
+Author: Tamil Muthamizhan <tmuthami@redhat.com>
+Date:   Fri Feb 17 21:06:43 2017 +0000
+
+    qa: drop ubuntu trusty support
+
+    ceph-ansible dropped support for OS that doesnt support systemd
+
+    Signed-off-by: Tamil Muthamizhan <tmuthami@redhat.com>
+    (cherry picked from commit 4d4b38eca81f7b57e3d3b31e1c13e7ab0ba5b30f)
+    Signed-off-by: Tamil Muthamizhan <tmuthami@redhat.com>
+
+commit 6c6b185bab1e0b7d7446b97d5d314b4dd60360ff
+Merge: 7d9ef63d65 8ea14ce9ce
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 23 14:55:56 2017 -0800
+
+    Merge pull request #13283 from zmc/wip-smoke-openstack-kraken
+
+    qa/suites/smoke: add openstack requirements
+
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit dc8ef3508b0b3f902772eed947e5861720036dab
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Thu Feb 2 23:23:54 2017 +0100
+
+    tests: Thrasher: eliminate a race between kill_osd and __init__
+
+    If Thrasher.__init__() spawns the do_thrash thread before initializing the
+    ceph_objectstore_tool property, do_thrash races with the rest
+    of Thrasher.__init__() and in some cases do_thrash can call kill_osd() before
+    Trasher.__init__() progresses much further. This can lead to an exception
+    ("AttributeError: Thrasher instance has no attribute 'ceph_objectstore_tool'")
+    being thrown in kill_osd().
+
+    This commit eliminates the race by making sure the ceph_objectstore_tool
+    attribute is initialized before the do_thrash thread is spawned.
+
+    Fixes: http://tracker.ceph.com/issues/18799
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit b519d38fb1967628ad8a1c46fcfb3f984de58790)
+
+commit 7d9ef63d65aad40ce1f2f9f324d68c5fa9d77369
+Merge: 033fdb1406 ab8558e05e
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Feb 21 16:16:26 2017 +0200
+
+    Merge pull request #13201 from dillaman/wip-18703-kraken
+
+    kraken: librbd: prevent self-blacklisting during break lock
+
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 033fdb14064d2a5ae9078944ab5ae8938a583d81
+Merge: f65754e312 cc04659798
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Tue Feb 21 16:15:35 2017 +0200
+
+    Merge pull request #13102 from dillaman/wip-18668
+
+    kraken: test: use librados API to retrieve config params
+
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 6f06cf80a7ff7407af40f46a0aefb4b88f926390
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jan 18 16:23:49 2017 +0800
+
+    mds: fix incorrect assertion in Server::_dir_is_nonempty()
+
+    when filelock is in XLOCKDONE state. client of xlocker can rdlock
+    the filelock. In that case, only client of xlocker can read the lock.
+
+    Fixes: http://tracker.ceph.com/issues/18578
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit fe4ab52b30079a785be053a9fd0197d6990737fe)
+
+commit 914d8a6109f7c2125080104bd4bf463c45833c3c
+Author: craigchi <craig10624@gmail.com>
+Date:   Thu Feb 16 19:21:48 2017 +0800
+
+    ceph-disk: Fix getting wrong group name when --setgroup in bluestore
+
+    ceph-disk prepare --setgroup <GROUP NAME> will be wrong when using with
+    bluestore
+
+    Signed-off-by: craigchi <craig10624@gmail.com>
+    (cherry picked from commit a8c0870e7370a0948e8e7fd53d3376b85bf9c649)
+
+commit 1bc9cfff42237b572e90e976697b1ac5faafad4a
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Feb 14 15:00:09 2017 -0500
+
+    osd/PG: restrict want_acting to up+acting on recovery completion
+
+    On recovery completion we recalculate want_acting to see if we
+    should add recently backfilled osds into acting.  However, at
+    this point we may have gotten infos from others OSDs outside
+    of up/acting that could be used for want_acting.  We currently
+    assert that only up/acting osds are used in
+    PG::RecoveryState::Active::react(const AdvMap&), so we must
+    restrict want_acting to up/acting here.
+
+    We could remove this restriction, but it would mean
+
+    1) checking on every map change that want_acting hasn't been
+    invalidated, and if so, recalculating want_acting and requesting
+    a new pg_temp.  Also, presumably
+
+    2) on each new info, checking whether we can construct a better
+    want_acting, and if so, doing it.
+
+    That would be a good thing, but is a more complicated change.  In
+    reality this case comes up very rarely, so simply make our
+    post-recovery want_acting calculation limit itself to up+acting.
+
+    See 1db67c443d84dc5d1ff53cc820fdfd4a2128b680 for the assertion.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 0f2dee9aa48a00a7f2f809cd4d20e98df771da81)
+
+commit e2074cef713a973f7e8fe23055e6fa27cae50e42
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Jan 3 20:40:00 2017 +0800
+
+    ceph-disk: convert none str to str before printing it
+
+    Error('somethings goes wrong', e) is thrown if exception `e` is caught
+    in ceph-disk, where e is not a string. so we can not just concat it in
+    Error's __str__(). so cast it to str before doing so.
+
+    introduced by d0e29c7
+
+    Fixes: http://tracker.ceph.com/issues/18371
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5e0dd1e7df43a3be589d17878714756a22052d8e)
+
+commit 177141ff1745e93996875b7d8d72af6f00307731
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jan 18 17:02:54 2017 -0600
+
+    mon/OSDMonitor: make 'osd crush move ...' work on osds
+
+    Currently it only allows you to move buckets, which is annoying and much
+    less useful.  To move an OSD you need to use create-or-move, which is
+    harder to use.
+
+    Fixes: http://tracker.ceph.com/issues/18587
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 47956475dea8bb8e07331dd76344a60b776b5158)
+
+commit 8c2f590338dec75dabfb1e9ae0d2c516c91b2636
+Author: Boris Ranto <branto@redhat.com>
+Date:   Wed Jan 25 12:39:40 2017 +0100
+
+    systemd: Start OSDs after MONs
+
+    Currently, we start/stop OSDs and MONs simultaneously. This may cause
+    problems especially when we are shutting down the system. Once the mon
+    goes down it causes a re-election and the MONs can miss the message
+    from the OSD that is going down.
+
+    Resolves: http://tracker.ceph.com/issues/18516
+
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 7f4acf45dd0d86e7d9992a8c30e5876fb57b1914)
+
+commit 33d9a50dec40346511f8480061dadab2a30174cc
+Author: Alexey Sheplyakov <asheplyakov@mirantis.com>
+Date:   Tue Feb 7 16:47:45 2017 +0400
+
+    ceph-osd: --flush-journal: sporadic segfaults on exit
+
+    FileStore holds a number of recources like op thread pool and work
+    queue, key/value DB threads, etc. These should be properly stopped
+    (released) before exiting to avoid segfaults on exit.
+
+    Note: more code paths (mkfs, dump_journal, etc) need similar fixes,
+    these will be submitted as separate patches.
+
+    Fixes: http://tracker.ceph.com/issues/18820
+    Signed-off-by: Alexey Sheplyakov <asheplyakov@mirantis.com>
+    (cherry picked from commit 00184814c156f6194a6ba4b696073ca1c18a3f8f)
+
+commit b464d1a5719ef36410a38cb31e8c4aab802732a6
+Author: craigchi <craig10624@gmail.com>
+Date:   Thu Feb 16 19:21:48 2017 +0800
+
+    ceph-disk: Fix getting wrong group name when --setgroup in bluestore
+
+    ceph-disk prepare --setgroup <GROUP NAME> will be wrong when using with
+    bluestore
+
+    Signed-off-by: craigchi <craig10624@gmail.com>
+    (cherry picked from commit a8c0870e7370a0948e8e7fd53d3376b85bf9c649)
+
+commit 7475e442143de0f0b1919ec8dab8b6d2446eb12b
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Sat Feb 4 15:27:11 2017 +0100
+
+    common: possible lockdep false alarm for ThreadPool lock
+
+    Fixes: http://tracker.ceph.com/issues/18819
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 8677dea4cd462d2141da28623a82b208cbc926f6)
+
+commit f65754e312f11823c5bb7e9fe24655f3fd68bcd2
+Merge: 21b2aa49b3 476b535f85
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 13 08:19:36 2017 +0100
+
+    Merge pull request #13330 from smithfarm/wip-18870-kraken
+
+    kraken: tests: SUSE yaml facets in qa/distros/all are out of date
+
+    Reviewed-by: Abhishek Lekshmanan <abhishek@suse.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 21b2aa49b35bbbbd87ad3b3f27920e79c569f17a
+Merge: 958a4c9cc1 d819ec7198
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 10 15:41:06 2017 -0800
+
+    Merge pull request #12984 from ceph/wip-cherry-pick-4tamil
+
+    qa: Wip cherry pick 4tamil ceph-ansible
+
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit 958a4c9cc135c34f18aa83a62b726bcefbb511b7
+Merge: b2955f4cc5 c968ed34b1
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Thu Feb 9 22:48:01 2017 +0800
+
+    Merge pull request #13239 from smithfarm/wip-18805-kraken
+
+    kraken: tests: ignore bogus ceph-objectstore-tool error in ceph_manager
+
+    Reviewed-by: Kefu Chai <kefu@redhat.com>
+
+commit 476b535f85c5b9a8bcd294e1b1629849efe9e391
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Feb 8 21:23:54 2017 +0100
+
+    tests: drop buildpackages.py
+
+    The buildpackages suite has been moved to teuthology. This cleans up a file
+    that was left behind by https://github.com/ceph/ceph/pull/13297
+
+    Fixes: http://tracker.ceph.com/issues/18846
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 6b7443fb50c117ee7f20d53bbc7530bb0eb7ebd5)
+
+commit ad456bfa2bab14032593c76b5a7f7434381bd1e8
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Feb 8 15:27:00 2017 +0100
+
+    tests: update SUSE yaml facets in qa/distros/all
+
+    Fixes: http://tracker.ceph.com/issues/18856
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 0bd56e871a8549d4b0b1211f09dad2d1120fb606)
+
+commit b2955f4cc5506224d75e64c202392ee9a39fb7bf
+Merge: ce8edcfed6 d24ddec58e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 8 18:50:52 2017 +0100
+
+    Merge pull request #13298 from dachary/wip-18849-kraken
+
+    kraken: tests: remove qa/suites/buildpackages
+
+    Reviewed-by: Nathan Cutler <ncutler@suse.cz>
+
+commit d24ddec58e0086ce6a5f08feb6a96c4022f13a4a
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Feb 7 18:33:29 2017 +0100
+
+    buildpackages: remove because it does not belong
+
+    It should live in teuthology, not in Ceph. And it is currently broken:
+    there is no need to keep it around.
+
+    Fixes: http://tracker.ceph.com/issues/18846
+
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 5a43f8d57925da227c95480501ceec10a29395d8)
+
+commit c968ed34b100baeb3a6a3245ad39c2a2d6fab04e
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Feb 6 18:43:49 2017 +0100
+
+    tests: fix regression in qa/tasks/ceph_master.py
+
+    https://github.com/ceph/ceph/pull/13194 introduced a regression:
+
+    2017-02-06T16:14:23.162 INFO:tasks.thrashosds.thrasher:Traceback (most recent call last):
+      File "/home/teuthworker/src/github.com_ceph_ceph_master/qa/tasks/ceph_manager.py", line 722, in wrapper
+        return func(self)
+      File "/home/teuthworker/src/github.com_ceph_ceph_master/qa/tasks/ceph_manager.py", line 839, in do_thrash
+        self.choose_action()()
+      File "/home/teuthworker/src/github.com_ceph_ceph_master/qa/tasks/ceph_manager.py", line 305, in kill_osd
+        output = proc.stderr.getvalue()
+    AttributeError: 'NoneType' object has no attribute 'getvalue'
+
+    This is because the original patch failed to pass "stderr=StringIO()" to run().
+
+    Fixes: http://tracker.ceph.com/issues/16263
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit db2582e25e390fcaf75952eb59a73dcff643f49c)
+
+commit d8c0141a7506cd38d385270f5a1cf315d808974e
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Dec 30 17:28:59 2016 -0500
+
+    osd/PG: publish PG stats when backfill-related states change
+
+    These frequently get flushed because other updates
+    happen, but we should explicitly ensure that the mon
+    sees these state changes.
+
+    Fixes: http://tracker.ceph.com/issues/18369
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d4adeb7500a113097fdd717ada0231f68badafbb)
+
+commit 1d25327b00f01e987dec022412edb95040d69a42
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 26 14:22:53 2017 -0500
+
+    os/bluestore: fix statfs to not include DB partition in free space
+
+    If we report the DB space as vailable, ceph thinks the OSD can store more
+    data and will not mark the cluster as full as easily.  And in reality, we
+    can't actually store data in this space--only metadata.  Avoid the problem
+    by not reporting it as available.
+
+    Fixes: http://tracker.ceph.com/issues/18599
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c66d5babb1e283869ba0f1f59029bead5ca5f37d)
+
+commit 8ea14ce9ce185204f6906b3919f7fac1419a26a9
+Author: Zack Cerza <zack@redhat.com>
+Date:   Thu Jan 12 14:55:26 2017 -0700
+
+    Add openstack requirements to smoke suite
+
+    Signed-off-by: Zack Cerza <zack@redhat.com>
+    (cherry picked from commit fe9b7552d116b150f178c4cc778fa27cd8d956d1)
+
+commit af2af7015adb0e6617da619ffe89e70e0904fcec
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Feb 3 17:38:05 2017 -0500
+
+    osd: do not send ENXIO on misdirected op by default
+
+    In practice this tends to get bubbled up the stack as an error on
+    the caller, and they usually do not handle it properly.  For example,
+    with librbd, this turns into EIO and break the VM.
+
+    Instead, this will manifest as a hung op on the client.  That is
+    also not ideal, but given that the root cause here is generally a
+    bug, it's not clear what else would be better.
+
+    We already log an error in the cluster log, so teuthology runs will
+    continue to fail.
+
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 923e7f5ce5ed437af15e178299a61029ff48e4a2)
+
+    # Conflicts:
+    #       PendingReleaseNotes
+
+commit 85bda6e263400118830aa40a889bdb695d321b58
+Author: Dongsheng Yang <dongsheng.yang@easystack.cn>
+Date:   Thu Dec 22 21:00:41 2016 -0500
+
+    librbd: don't remove an image w/ incompatible features
+
+    Fixes: http://tracker.ceph.com/issues/18315
+    Signed-off-by: Dongsheng Yang <dongsheng.yang@easystack.cn>
+    (cherry picked from commit f76127b5e617923d14adb62bfb836a635c14f209)
+
+commit d96ae9ead2d2f58180e533a63c640098d5134047
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jan 17 11:55:00 2017 -0500
+
+    osdc: cache should ignore error bhs during trim
+
+    A read error (such as injecting a timeout into an OSD op) might result
+    in a bh in an error state. These should be trimable by the cache.
+
+    Fixes: http://tracker.ceph.com/issues/18436
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5910ed9de9856b5821488a1836487bbbd3d6460e)
+
+commit 9620088542f80efb97960b4d9d51876ff1784aa9
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Jan 31 00:46:22 2017 +0100
+
+    tests: ignore bogus ceph-objectstore-tool error in ceph_manager
+
+    Fixes: http://tracker.ceph.com/issues/16263
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 046e873026c59e733f1844b28ffdc030cbe57b36)
+
+commit baa42b6d7c616e4d3518416fcb86fef5c12a82db
+Author: Piotr Dałek <piotr.dalek@corp.ovh.com>
+Date:   Tue Jan 31 16:07:18 2017 +0100
+
+    OSD: allow client throttler to be adjusted on-fly, without restart
+
+    This patch allows the osd_client_message_cap and
+    osd_client_message_size_cap to be adjusted on-fly, using admin socket
+    functionality.
+
+    Fixes: http://tracker.ceph.com/issues/18791
+    Signed-off-by: Piotr Dałek <piotr.dalek@corp.ovh.com>
+    (cherry picked from commit 64c309d7e18a975931b526e6f5d6f610c3a0d632)
+
+commit ab8558e05e765683691799acc4cfb2280cd537b2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jan 25 14:45:56 2017 -0500
+
+    librbd: prevent self-blacklisting during break lock
+
+    Fixes: http://tracker.ceph.com/issues/18666
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5c590acaec4dd66a9a8c3aa0ec8ab904dd350216)
+
+    Conflicts:
+            src/librbd/exclusive_lock/BreakRequest.cc: managed lock refactor
+            src/test/librbd/exclusive_lock/test_mock_BreakRequest.cc: managed lock refactor
+
+commit 16b2fd00ecf7d7fcc7590f0ca030ff33a32f2196
+Author: Wido den Hollander <wido@42on.com>
+Date:   Mon Jan 23 08:18:27 2017 +0100
+
+    systemd: Restart Mon after 10s in case of failure
+
+    In some situations the IP address the Monitor wants to bind to
+    might not be available yet.
+
+    This might for example be a IPv6 Address which is still performing
+    DAD or waiting for a Router Advertisement to be send by the Router(s).
+
+    Have systemd wait for 10s before starting the Mon and increase the amount
+    of times it does so to 5.
+
+    This allows the system to bring up IP Addresses in the mean time while
+    systemd waits with restarting the Mon.
+
+    Fixes: #18635
+
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit e73eb8cc1e0d45af1f0b7852c551f2ddfb82a520)
+
+commit 29f55d7dd3f35423294fa63cd1e3ccb18e6f7449
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Fri Jan 13 16:32:55 2017 +0100
+
+    rgw_admin: read master log shards from master's current period
+
+    Also make the sync output look similar to the output of data sync
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+
+    (cherry picked from commit cc306c506ca6607223cb89cd388f8e18673c4fe2)
+
+commit 0aebe210079e456c23e857e9c684495813151726
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Fri Jan 13 16:29:47 2017 +0100
+
+    rgw: allow getting master log shards info on specified period
+
+    This is needed for rgw admin's sync status or else we end up always
+    publishing that we're behind since we are always checking against
+    master's first period to sync from
+
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit 063c949d4409a18a22b64791d497e20f7473bc01)
+
+commit 17d68c22da7c3eac336f5ea8ecac13b5e8d75568
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Thu Jan 12 22:09:01 2017 +0100
+
+    rgw_admin: get master's period from store's current period info
+
+    This ensures that we get the current period in contrast to the admin log
+    which gets the master's earliest period.
+
+    Fixes: http://tracker.ceph.com/issues/18064
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit 4ca18df7198a9f0ded8b0100a70b5db7187c3de4)
+
+commit 478d0315cb8457ecfd5cb6f96c82d12e066c7c2f
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Dec 25 12:36:34 2016 +0200
+
+    rgw: clear master_zonegroup when reseting RGWPeriodMap
+
+    Fixes: http://tracker.ceph.com/issues/17239
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit d8f42fe6be659c1d48bf04b30aa54ad616936145)
+
+commit bfc058d8f951d7694104433f709fc73940511b0a
+Author: Michal Koutný <mkoutny@suse.com>
+Date:   Wed Jan 18 20:15:29 2017 +0100
+
+    rgw: Use decoded URI when verifying TempURL
+
+    Instead of calliing url_decode directly, we reuse s->decoded_uri that is
+    initialized in RGWREST::preprocess().
+
+    Fixes: http://tracker.ceph.com/issues/18590
+    Signed-off-by: Michal Koutný <mkoutny@suse.com>
+    (cherry picked from commit 4e1318f4dcbfd64c3ec94f4addf6e38ddd6c013a)
+
+commit deb6d2c64c432869caf8747a75ed2e555acc772b
+Author: Ricardo Dias <rdias@suse.com>
+Date:   Tue Jan 10 15:11:19 2017 +0000
+
+    librbd: allow to open an image without opening parent image
+
+    Fixes: http://tracker.ceph.com/issues/18325
+
+    Signed-off-by: Ricardo Dias <rdias@suse.com>
+    (cherry picked from commit 61af1c25015de087a2423811548d975dd7d430b4)
+
+    Conflicts:
+      src/librbd/exclusive_lock/PostAcquireRequest.cc - deleted, does not
+                                                        exist in kraken
+      src/librbd/librbd.cc - removed rbd_group_* functions that don't exist
+                             in kraken
+
+commit 35b13c6687026f08c100a28f2775d6e772a0e734
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 29 12:08:28 2016 -0500
+
+    mon/OSDMonitor: set last_force_op_resend on overlay pool too
+
+    We currently set the last_force_op_resend field on the
+    base pool when we set or clear the overlay.  Set it on
+    the cache/overlay pool too.  The Objecter should resend
+    even with a change only to the base pool, but the OSD
+    needs to see the change on the overlay pool to correctly
+    discard the op.
+
+    Fixes: http://tracker.ceph.com/issues/18366
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 08c3618771b852840aa88cff1ca98d980d802941)
+
+commit ce8edcfed6cd908779efd229202eab1232d16f1c
+Merge: 16fc6d8a2a 7db1bf762c
+Author: John Spray <jspray@redhat.com>
+Date:   Thu Jan 26 00:33:46 2017 +0100
+
+    Merge pull request #13024 from SUSE/wip-18604-kraken
+
+    kraken: cephfs test failures (ceph.com/qa is broken, should be download.ceph.com/qa)
+
+commit 16fc6d8a2af89bb8a620298729d3951ef32945f7
+Merge: f36efa76d8 ed82de11f5
+Author: John Spray <jspray@redhat.com>
+Date:   Thu Jan 26 00:31:53 2017 +0100
+
+    Merge pull request #12813 from SUSE/wip-18439-kraken
+
+    kraken: fuse: TestVolumeClient.test_evict_client failure creating pidfile
+
+commit f36efa76d83205783ac5931ea678357ebddc62f6
+Merge: 08b560c51b 61ca8fc351
+Author: John Spray <jspray@redhat.com>
+Date:   Thu Jan 26 00:29:55 2017 +0100
+
+    Merge pull request #12951 from jcsp/wip-18361-kraken
+
+    kraken: client: populate metadata during mount
+
+commit 08b560c51b4218697b922cb208da2708f8c50021
+Merge: d50af9e948 8d0eb6e287
+Author: John Spray <jspray@redhat.com>
+Date:   Thu Jan 26 00:29:00 2017 +0100
+
+    Merge pull request #13030 from SUSE/wip-18612-kraken
+
+    kraken: client: segfault on ceph_rmdir path /
+
+commit d50af9e9488501778d1a82d32ba4042ecb5fca54
+Merge: 0fbf923290 8b79964696
+Author: John Spray <jspray@redhat.com>
+Date:   Thu Jan 26 00:27:57 2017 +0100
+
+    Merge pull request #13028 from SUSE/wip-18531-kraken
+
+    kraken: speed up readdir by skipping unwanted dn
+
+commit 0fbf9232903b8e7d7d93c91ea0dafcb4ff8f3c15
+Merge: e4d348b3d8 1ac9886896
+Author: John Spray <jspray@redhat.com>
+Date:   Thu Jan 26 00:25:34 2017 +0100
+
+    Merge pull request #12835 from SUSE/wip-18463-kraken
+
+    kraken: Decode errors on backtrace will crash MDS
+
+commit 29c8c3fb861a7153a13e3f56aaa7772c9116c751
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Jan 10 11:16:47 2017 +0800
+
+    mds: finish clientreplay requests before requesting active state
+
+    All clientreplay requests' finish contexts should be executed
+    before MDCache::export_remaining_imported_caps(). Otherwise
+    MDCache::try_reconnect_cap() may fail to reconnect client caps.
+
+    Fixes: http://tracker.ceph.com/issues/18461
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 80dae314ee90e79d60e2cfee301e43a435c10801)
+
+commit 2b7083f8dd4c8f281f6f8ccbd13f06f08de6bcc1
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Jan 24 16:35:23 2017 -0800
+
+    Revert "PrimaryLogPG::failed_push: update missing as well"
+
+    This reverts commit dd48b972afde2dfa9ab1a6942c7961750222986d.
+
+    Fixes: http://tracker.ceph.com/issues/18659
+
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+
+commit cc046597983bd491cc66081cc33d9046264fe24b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jan 24 09:24:52 2017 -0500
+
+    librbd: improve debug logging for lock / watch state machines
+
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit c5976ac36751e15fd81c67945a2c6d049dc7b316
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jan 23 21:24:41 2017 -0500
+
+    test: use librados API to retrieve config params
+
+    The CephContext object is not ABI-stable, so it is necessary to
+    use the "conf_get" librados methods to safely retrieve a setting.
+
+    Fixes: http://tracker.ceph.com/issues/18617
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8ad40645ac3948f8341e9a80ce8aff8ac5b9ad11)
+
+    Conflicts:
+            src/test/librbd/test_librbd.cc: trivial resolution
+
+commit e4d348b3d850465ae25d7b3cbc8f21fac20ae478
+Merge: 03458fae64 f583485a4e
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Mon Jan 23 20:46:55 2017 +0100
+
+    Merge pull request #13006 from rzarzynski/wip-rgw-18476-kraken
+
+    kraken: rgw: fix handling RGWUserInfo::system in RGWHandler_REST_SWIFT.
+
+    Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
+
+commit f583485a4eacdf489ce00f93cd49dc147bfdb5f9
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue Jan 10 12:09:50 2017 +0100
+
+    rgw: fix handling RGWUserInfo::system in RGWHandler_REST_SWIFT.
+
+    Before this patch the flag was wrongly handled in the Swift API
+    implementation. In rare conditions this might result in setting
+    req_state::system_request.
+
+    This may happen only if both of those conditions are fulfilled:
+     * RadosGW is running in a multi-site configuration (at least
+       one user with the system flag turned on is present),
+     * the "rgw_swift_account_in_url" configurable has been switched
+       to true. The value is false by default and our documentation
+       doesn't actually mention about the option.
+
+    The issue doesn't affect Jewel nor any previous release.
+
+    Fixes: http://tracker.ceph.com/issues/18476
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 8dac93392b6679c3ad9bb28ea66f6bb8c7be511d)
+
+commit 03458fae64df24d58426e5b3274dd1981ef42fc6
+Merge: 61b1beef1d dca2265c41
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Jan 23 19:29:57 2017 +0100
+
+    Merge pull request #13044 from SUSE/wip-18571-kraken
+
+    kraken: Python Swift client commands in Quick Developer Guide don't match configuration in vstart.sh
+
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 7db1bf762c5503933bbbb266e14fc1bd69120dd8
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Jan 17 17:12:46 2017 +0100
+
+    qa: update remaining ceph.com to download.ceph.com
+
+    Fixes: http://tracker.ceph.com/issues/18574
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 549d993d3fd8ffffa280ed4a64aca41d1c6f2da1)
+
+commit 0e0d149895198ee74cff85353eabf19aa4677258
+Author: Mingxin Liu <mingxin@xsky.com>
+Date:   Mon Jan 2 13:20:10 2017 +0800
+
+    mon: do not send duplicated osdmap msg to not sync'ed osd
+
+    prior to this change:
+    a peon may forward the pgstats to leader, and record it locally, but leader will
+    check if osd has the latest map before process, if not, will use a route op to
+    indicate peon to send it, then poen will delete routed op when fininaly send
+    out which make peon cannot send pgstatack when leader has processed the
+    pgstat update. so osd will always track it util reach a threshold block pgstats
+    sending, at worst, reopen mon session.
+    also, both leader and peon will send out the osdmap message to the osd.
+
+    after this change:
+    only the peon will send out the osdmap message. and the pgstatack message
+    will be routed to the osd as expected. so the osd will not keep track of the
+    "acked" pg stats in its queue forever before times out.
+
+    Fixes: http://tracker.ceph.com/issues/18458
+    Signed-off-by: Mingxin Liu <mingxin@xsky.com>
+    (cherry picked from commit 57274488c072ec6912b700288ce5b1ea8372d162)
+
+commit dca2265c41bf035855b30d3279e1ec5726d74ffc
+Author: Ronak Jain <ronakjain@outlook.in>
+Date:   Fri Jan 13 16:57:45 2017 +0530
+
+    Doc: Fixes Python Swift client commands
+
+    Fixes: http://tracker.ceph.com/issues/17746
+    Signed-off-by: Ronak Jain <ronakjain@outlook.in>
+    (cherry picked from commit 8c79959557d60f619adf1a3ed1b5bd1112ceaabb)
+
+commit 8d0eb6e2870593b94cb71f29bcac7a1f422cd101
+Author: Michal Jarzabek <stiopa@gmail.com>
+Date:   Thu Jan 12 21:22:20 2017 +0000
+
+    client/Client.cc: prevent segfaulting
+
+    The segfaulting in the rmdir function is caused by calling
+    filepath::last_dentry() function.
+    last_dentry() function assumes that the bits vector has always at
+    least one element, which is not the case for the the filepath object
+    created with "/" input.
+    This commit also fixes other functions affected by this bug:
+    link, unlink, rename, mkdir, mknod and symlink.
+
+    Fixes: http://tracker.ceph.com/issues/9935
+    Signed-off-by: Michal Jarzabek <stiopa@gmail.com>
+    (cherry picked from commit 6ed7f2364ae5507bab14c60b582929aa7b0ba400)
+
+commit 61b1beef1dc4802c32367fc71968101a09042c15
+Merge: f223e27eeb 6206e1998a
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jan 20 11:14:24 2017 -0600
+
+    Merge pull request #13011 from liewegas/wip-18595-kraken
+
+    os/bluestore: fix Allocator::allocate() int truncation
+
+    Reviewed-by: Igor Fedotov <ifedotov@mirantis.com>
+
+commit 8b7996469652d01143806e1e815894afde3f426b
+Author: Xiaoxi Chen <xiaoxchen@ebay.com>
+Date:   Tue Jan 10 19:11:08 2017 -0700
+
+    mds/server: skip unwanted dn in handle_client_readdir
+
+    We can skip unwanted dn which  < (offset_key, snap) via map.lower_bound, rather than
+    iterate across them.
+
+    Previously we iterate and skip dn which < (offset_key, dn->last), as dn->last >= snap
+     means (offset_key, dn->last) >= (offset_key, snap), and such iterate_and_skip logic
+    still keep, so this commit doesnt change code logic but an optimization.
+
+    Signed-off-by: Xiaoxi Chen <xiaoxchen@ebay.com>
+    (cherry picked from commit 52fe52baf920c672ac7f63a3087dcd31137891b6)
+
+commit ecb459f0837dd05d2802320e7fac3246025e6c13
+Author: Samuel Matzek <smatzek@us.ibm.com>
+Date:   Mon Jan 16 11:11:31 2017 -0600
+
+    Ceph-disk to use correct user in check_journal_req
+
+    The ceph-disk tool calls ceph-osd to check the journal requirements
+    using OSD id 0. This creates a log file for osd-0 on the system
+    using the current user/group for file permissions.  When ceph-disk
+    is run as root this makes the file owned by root which makes
+    the osd daemon for osd.0 unable to write to its own log file.
+
+    This commit changes the journal reqs calls of ceph-osd to pass
+    the ceph user and group so ceph-osd creates the log file with the
+    appropriate permissions.
+
+    Fixes: http://tracker.ceph.com/issues/18538
+
+    Signed-off-by: Samuel Matzek <smatzek@us.ibm.com>
+    (cherry picked from commit bcf7514bf53693ec61e482341787c80494589faf)
+
+commit 6206e1998a1636f6339ef2f1bd2d67a69cd3abf0
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jan 19 19:51:45 2017 -0600
+
+    os/bluestore: fix Allocator::allocate() int truncation
+
+    An allocation of size 0x80000000 gets truncated to 0.  Larger values can
+    break things in similar ways.
+
+    Introduced in 5ab034345d7320fbc86a2133c0c29ec1aca4b71a.
+
+    Fixes: http://tracker.ceph.com/issues/18595
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e19aa8484632ac7d83aa5dc868a1fe4dc167d9b9)
diff --git a/doc/changelog/v12.2.1.txt b/doc/changelog/v12.2.1.txt
new file mode 100644
index 000000000..dda1d4b55
--- /dev/null
+++ b/doc/changelog/v12.2.1.txt
@@ -0,0 +1,3737 @@
+commit 3e7492b9ada8bdc9a5cd0feafd42fbca27f9c38e
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Tue Sep 26 16:27:07 2017 +0000
+
+    12.2.1
+
+commit bf5f5ec7cf0e06125515866acedcc04c393f90b9
+Merge: 63ce514631 d1a6631fc4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 26 11:53:56 2017 -0400
+
+    Merge pull request #17970 from idryomov/wip-pr-17346-luminous
+    
+    luminous: qa/suites/krbd: rbd_xfstests job overhaul
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit d1a6631fc4f7eaf563377f52dcc6fdb5833c0dd1
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Fri Aug 18 17:49:03 2017 +0200
+
+    qa/suites/krbd: new rbd_xfstests job definition
+    
+    ext4 seems to be a better choice for our purposes -- less test churn,
+    rather small and reliable exclude list.
+    
+    All excluded tests but generic/050 fail with no krbd in the mix, most
+    have popped up on linux-ext4 list at least once.
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit 6647c65ebe0688283cc8b9da9f638a4d6ffb75de)
+
+commit 902422c71d2e2322fa05dca2cf8d71e49033005c
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Thu Aug 17 15:35:42 2017 +0200
+
+    qa/tasks/rbd.xfstests: take exclude list from yaml
+    
+    Different filesystems (and further, different configurations of the
+    same filesystem) need different exclude lists.  Hard coding the list in
+    a wrapper script is inflexible.
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit 01914be5a3b35479037054293b85d96596cb870b)
+    
+    Conflicts:
+            qa/run_xfstests_krbd.sh [ trivial, caused by commit
+              3aae5ca6fda8 ("scripts: fix bash path in shebangs") ]
+
+commit 49284c671b335887244fcd06a3667952068cf5f2
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Wed Aug 16 11:47:19 2017 +0200
+
+    qa/run_xfstests.sh: quit building xfstests on test nodes
+    
+    xfstests is a pain to build on trusty, xenial and centos7 with a single
+    script.  It is also very sensitive to dependencies, which again need to
+    be managed on all those distros -- different sets of supported commands
+    and switches, some versions have known bugs, etc.
+    
+    Download a pre-built, statically linked tarball and use it instead.
+    The tarball was generated using xfstests-bld by Ted Ts'o, with a number
+    of tweaks by myself (mostly concerning the build environment).
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit 1a982fea53c30fb246c2ab28822af6a69165ffea)
+
+commit e318dbd19883118885e8eb6c7375d0ca8a3699f6
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Wed Aug 16 11:47:19 2017 +0200
+
+    qa/run_xfstests.sh: drop *_MKFS_OPTIONS variables
+    
+    AFAICT ./check doesn't query EXT4_MKFS_OPTIONS or BTRFS_MKFS_OPTIONS,
+    We don't need anything special for xfs, so remove all of them to avoid
+    confusion.
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit 7a0e45046d556c6624e0225ff006c31d8dbae2a8)
+
+commit b5aeb7921d86452b5b19ac144685b3a199707751
+Author: Ilya Dryomov <idryomov@gmail.com>
+Date:   Wed Aug 16 11:47:19 2017 +0200
+
+    qa/run_xfstests.sh: kill FS_TYPE variable
+    
+    ./check expects FSTYP, so use that throughout.
+    
+    Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
+    (cherry picked from commit ca99f1e8406c14e5899d393ab0bb503061c47b9b)
+
+commit 63ce514631ab0106ebd242f0ece43409dc83f479
+Merge: 9f8d66ebc5 b335066f81
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Sep 25 22:40:05 2017 +0200
+
+    Merge pull request #17945 from batrick/i21540
+    
+    luminous: qa whitelist fixes
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 9f8d66ebc5fb9ad48bdca4c56d54618a7a592415
+Merge: 35a23b861a 1dda4227a5
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Sep 25 22:32:17 2017 +0200
+
+    Merge pull request #17821 from smithfarm/wip-21472-luminous
+    
+    luminous: tests: kcephfs: ignorable MDS cache too large warning
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+
+commit 1dda4227a5c8e2953c645c634e0b77dc6a0515a3
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Mon Aug 28 11:47:29 2017 -0700
+
+    qa: whitelist expected MDS_CACHE_OVERSIZED
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 71f0066f6ec32be18abc0c220a49e203ac3b9786)
+    
+    Conflicts:
+       qa/suites/kcephfs/recovery/tasks/client-limits.yaml - fix
+          already applied by some other commit
+
+commit b335066f81a0f990fb87b84f69c34dda037ba9cf
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Mon Aug 28 11:52:36 2017 -0700
+
+    qa: whitelist expected rstat warning
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit c53d091e5110011866aa85495ff4e5abe0442d71)
+
+commit 35a23b861a037ca0e0bc77952c413b46e95141d4
+Merge: b6f38c4fcd d94b3ca2ae
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Mon Sep 25 15:43:06 2017 +0200
+
+    Merge pull request #17922 from batrick/kcephfs-backports
+    
+    luminous: qa: kcephfs whitelist fixes
+    
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+
+commit b6f38c4fcd0643fd24644bf0d0ad249112727a5e
+Merge: 793b89dc80 16c5e4f0c3
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Sep 22 13:25:59 2017 -0700
+
+    Merge pull request #17855 from smithfarm/wip-21490-luminous
+    
+    luminous: tests: test_rebuild_simple_altpool triggers MDS assertion
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 16c5e4f0c3f365f22e5cd79021f55c2927347a26
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Wed Sep 20 15:44:40 2017 -0700
+
+    qa: get config only on running MDS
+    
+    Fixes: http://tracker.ceph.com/issues/21466
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 8a535d9c72965c79692dccc6ff418b46fb86c3eb)
+
+commit d94b3ca2ae04ff05937cb48a81901187212370a6
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Sep 22 09:44:43 2017 -0700
+
+    qa: whitelist cache too large in client-limits
+    
+    Fixes: http://tracker.ceph.com/issues/21510
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 87039eb30a25158a192ab7099fa81fbb9ce02953)
+
+commit 018331f26c079c5d728a7009db24560e00407feb
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Sep 22 09:42:06 2017 -0700
+
+    qa: whitelist failover log warnings
+    
+    Fixes: http://tracker.ceph.com/issues/21509
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 37a7737712f36422b5f035a0e02bab25ae2da890)
+
+commit dbad9455f956a6e7ebd731b68c50cbb96038542f
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Sep 22 09:38:59 2017 -0700
+
+    qa: ignore evicted client in client-recovery
+    
+    Fixes: http://tracker.ceph.com/issues/21508
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 8ecef57bb10a83587f02ce087b657d303ae7c589)
+
+commit 793b89dc80c82a0f3027210d894dd1b4be6cc12b
+Merge: e8a6c0cfdf 4bb1d89373
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Sep 22 09:53:43 2017 -0700
+
+    Merge PR #17878 into luminous
+    
+    * refs/remotes/upstream/pull/17878/head:
+            qa/cephfs: Fix error in test_filtered_df
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit e8a6c0cfdf1a5ac2505afa7ca1814a9f356647d5
+Merge: 6f6a98619e b0cacf2684
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Sep 22 09:53:41 2017 -0700
+
+    Merge PR #17877 into luminous
+    
+    * refs/remotes/upstream/pull/17877/head:
+            client: Delete onsafe
+            client: add mountedness check inside client_lock
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 6f6a98619e9e121fea20d1604fa97122209a6a72
+Merge: 71b3f31cad 7a84f943ed
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Sep 22 09:53:39 2017 -0700
+
+    Merge PR #17870 into luminous
+    
+    * refs/remotes/upstream/pull/17870/head:
+            mds: reset Formatter on error
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 71b3f31cad17a735ee0c278de04c88338b78b746
+Merge: 26beaa6667 69f0422310
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Sep 22 09:53:37 2017 -0700
+
+    Merge PR #17854 into luminous
+    
+    * refs/remotes/upstream/pull/17854/head:
+            mds: void sending cap import message when inode is frozen
+            client: fix message order check in handle_cap_export()
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 26beaa6667fbf95a371a2cf0d5130d8ff30327d9
+Merge: fc9043a2f5 97ab78548d
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Sep 22 09:53:35 2017 -0700
+
+    Merge PR #17852 into luminous
+    
+    * refs/remotes/upstream/pull/17852/head:
+            mds: check ongoing catter-gather process before capping log
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit fc9043a2f5ee3d641cb5710675fa43c44784fe87
+Merge: 5ae7723f80 13f0c9f534
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Sep 22 09:53:33 2017 -0700
+
+    Merge PR #17822 into luminous
+    
+    * refs/remotes/upstream/pull/17822/head:
+            qa/cephfs: kill mount if it gets evicted by mds
+            qa/cephfs: fix test_evict_client
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Zheng Yan <zyan@redhat.com>
+
+commit 5ae7723f8028a888df93b71252ce3b31fc77da8d
+Merge: 653c332507 8d18150560
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Sep 22 09:44:54 2017 -0700
+
+    Merge pull request #17817 from smithfarm/wip-21464-luminous
+    
+    luminous: qa: ignorable MDS_READ_ONLY warning
+    
+    Reviewed-by: Yan, Zheng <ukernel@gmail.com>
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit 653c332507c6642a4f8167d022ba452c91315c92
+Merge: 8681be5725 105f82d550
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Sep 22 09:41:18 2017 -0700
+
+    Merge pull request #17879 from smithfarm/wip-21449-luminous
+    
+    luminous: tests: test_misc creates metadata pool with dummy object resulting in WRN: POOL_APP_NOT_ENABLED
+    
+    Reviewed-by: Amit Kumar amitkuma@redhat.com
+    Reviewed-by: Yan, Zheng <ukernel@gmail.com>
+
+commit 8681be5725b7a9a1256e338600dd2050e1ddf2f7
+Merge: 09451bc47b f4b4467f8e
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Sep 22 09:40:46 2017 -0700
+
+    Merge pull request #17886 from smithfarm/wip-21486-luminous
+    
+    luminous: tests: [kcephfs] test_client_pin times out waiting for dentry release from kernel
+    
+    Reviewed-by: Amit Kumar amitkuma@redhat.com
+    Reviewed-by: Yan, Zheng <ukernel@gmail.com>
+
+commit 09451bc47bd1d061c022d098d462206f1dac83ac
+Merge: 1700dc4f14 c902ebe7e8
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Sep 22 09:39:26 2017 -0700
+
+    Merge pull request #17887 from smithfarm/wip-21487-luminous
+    
+    luminous: cephfs: MDS rank add/remove log messages say wrong number of ranks
+    
+    Reviewed-by: Amit Kumar amitkuma@redhat.com
+    Reviewed-by: Yan, Zheng <ukernel@gmail.com>
+
+commit 1700dc4f1409bbc8202b420deadfc494aaaaed74
+Merge: a931cf6812 f09786d294
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Sep 22 09:37:42 2017 -0700
+
+    Merge pull request #17888 from smithfarm/wip-21488-luminous
+    
+    luminous: tests: failures from pjd fstest
+    
+    Reviewed-by: Amit Kumar amitkuma@redhat.com
+    Reviewed-by: Yan, Zheng <ukernel@gmail.com>
+
+commit a931cf68120d99661b785cc794e2b6965d621f06
+Merge: 0132a275aa 875f1c2519
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Sep 21 10:56:16 2017 -0500
+
+    Merge pull request #17812 from liewegas/wip-21410-luminous
+    
+    Revert "osd/OSDMap: allow bidirectional swap of pg-upmap-items"
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: xie xingguo <xie.xingguo@zte.com.cn>
+
+commit f09786d2940ca5532f6f59464be4fdf29a4d75b3
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sun Sep 17 16:51:04 2017 +0800
+
+    ceph: do link/rename semantic checks after srcdn is readable
+    
+    For hard link, source inode must not be directory. For rename,
+    types of source/destination inodes must match. If srcdn is replica
+    and we do these checks while it's not readble, it's possible that
+    wrong source inode is used in these checks.
+    
+    Fixes: http://tracker.ceph.com/issues/21383
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 46962b253563a867707e7c5d7887abf2060cc4d7)
+
+commit c902ebe7e89130f7aceaaf664074d74760b30926
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 18 13:16:40 2017 -0400
+
+    mon/MDSMonitor: fix clog msgs on MDS rank add/rm
+    
+    Because these were generated before the actual
+    change, the get_num_in values did not make sense.
+    
+    Fixes: http://tracker.ceph.com/issues/21421
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 511439b515414ae93d791f54a187e44d5fdffa7b)
+
+commit f4b4467f8edc30f33c56e2bc22fb509061d814b6
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Sep 19 14:38:13 2017 +0800
+
+    client: set client_try_dentry_invalidate to false by default
+    
+    By default, ceph-fuse uses side effect of 'dentry invalidation' to
+    trim kernel dcache if it runs on kernel < 3.18. The implemention of
+    kernel function d_invalidate() changed in 3.18 kernel, the method no
+    longer works for upstream kernel >= 3.18.
+    
+    RHEL 3.10 kernel includes backport of patches that change implemention
+    of d_invalidate(). So checking kernel version to decide if 'dentry
+    invalidation' method works is unreliable.
+    
+    Fixes: http://tracker.ceph.com/issues/21423
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 69adaabface27880dd6c8dbfdeeb06cf3f3d346a)
+
+commit 105f82d5502deb3cabb87d8fdd83a98e880fd6e2
+Author: Douglas Fuller <dfuller@redhat.com>
+Date:   Tue Sep 12 13:22:09 2017 -0400
+
+    qa/tasks/cephfs: Whitelist POOL_APP_NOT_ENABLED for test_misc
+    
+    test_misc verifies that ceph fs new will not create a filesystem
+    on a pool that already contains objects. As part of the test, it
+    inserts a dummy object into a pool and then attempts to use it for
+    CephFS. This triggers POOL_APP_NOT_ENABLED. Setting the application
+    metadata for the pool (and having ceph fs new fail because of the
+    existing metadata) would then exercise a different failure case.
+    
+    Signed-off-by: Douglas Fuller <dfuller@redhat.com>
+    (cherry picked from commit eba01c48ca8b3113b1faf353990533a5ae39232c)
+
+commit 4bb1d893739e33fc35ae57ac0290606b5cf8b918
+Author: Douglas Fuller <dfuller@redhat.com>
+Date:   Wed Sep 13 14:00:20 2017 -0400
+
+    qa/cephfs: Fix error in test_filtered_df
+    
+    ceph df accounts for pool size, so there is no need to do it in the test.
+    
+    Fixes: http://tracker.ceph.com/issues/21381
+    Signed-off-by: Douglas Fuller <dfuller@redhat.com>
+    (cherry picked from commit b059cb6290d81df07bd8ec4e1ec9ef0be24892a2)
+
+commit b0cacf2684bfe8c55282bb7defd54053dc7fa2e8
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Thu Sep 7 12:41:40 2017 +0530
+
+    client: Delete onsafe
+    
+    Variable "onsafe" going out of scope leaks the storage it points to. This fixes the Coverity
+    Scan CID 1417473.
+    
+    Signed-off-by: Jos Collin <jcollin@redhat.com>
+    (cherry picked from commit e24a91e9189587c800ef5709c78f1bc0e0ac8419)
+
+commit f146b9f51da11e5698c7c4c9021635961b2790ad
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Fri Aug 25 08:31:47 2017 -0400
+
+    client: add mountedness check inside client_lock
+    
+    Currently we check for mountedness in the high level wrappers, but those
+    checks are lockless. It's possible to have a call that races with
+    ceph_unmount(). It could pass one of the is_mounted() checks in the
+    wrapper, and then block on the client_lock while the unmount is actually
+    running. Eventually it picks up and runs after the unmount returns, with
+    questionable results -- possibly even a crash in some cases.
+    
+    For now, we can explain this away with a simple admonition that
+    applications should ensure that no calls are running when ceph_unmount
+    is called. In the future though, we may need to forcibly shut down the
+    mount when certain events occur (not returning a lease or delegation in
+    time, for instance).
+    
+    Sprinkle in a bunch of "unmounting" checks after taking the client_lock,
+    and simply have the functions return errors (or sensible values in some
+    cases) when the Client is being downed. With that, we ensure that this
+    sort of race can't occur, even when the unmount is not being driven by
+    userland. Note too that in some places I've replaced assertions in the
+    code with error returns, as that's nicer behavior for libraries.
+    
+    Note that this can't replace the ->is_mounted() checks in the lockless
+    wrappers as those are needed to determine whether the client pointer in
+    the ceph_mount_info is still valid. The admonition not to allow
+    ceph_unmount to race with other calls is therefore still necessary.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit efca340596ef4da2b254ff1c64ec4c462d7b95a5)
+
+commit 7a84f943edcdabd013b8ee4eb701e04062ad931a
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Tue Sep 5 14:44:49 2017 -0700
+
+    mds: reset Formatter on error
+    
+    This prevents partial results from concatenating with the error message.
+    
+    Fixes: http://tracker.ceph.com/issues/21252
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 2b2b3d2035a700a65c3c469eedae69ad31023cf6)
+
+commit 847cb76968bb225e4f3713cdfc0a5c38c4047e4e
+Author: Douglas Fuller <dfuller@redhat.com>
+Date:   Wed Jul 12 10:48:42 2017 -0500
+
+    qa/cephfs: Refactor alternate pool test
+    
+    Remove the alternate pool recovery test from test_data_scan. Newer
+    commits will place the test in its own file.
+    
+    Signed-off-by: Douglas Fuller <dfuller@redhat.com>
+    (cherry picked from commit 47318f8ac4391148a1394846dd77c8452b7cf5ae)
+
+commit 3342ba0c6853fda90cf2c8c9bec9d523c9269855
+Author: Douglas Fuller <dfuller@redhat.com>
+Date:   Wed Jul 12 10:45:13 2017 -0500
+
+    qa/cephfs: test CephFS recovery pools
+    
+    Test recovering metadata in to a separate RADOS pool with
+    cephfs_data_scan and friends.
+    
+    Signed-off-by: Douglas Fuller <dfuller@redhat.com>
+    (cherry picked from commit 6af2ae80d327e564ad17f2caeded9c8142becf29)
+
+commit 5ac721394770de88a6218dd3f9bedbdbc9982232
+Author: Douglas Fuller <dfuller@redhat.com>
+Date:   Wed Jul 12 10:43:39 2017 -0500
+
+    qa/cephfs: support CephFS recovery pools
+    
+    Add support for testing recovery of CephFS metadata into an alternate
+    RADOS pool, useful as a disaster recovery mechanism that avoids
+    modifying the metadata in-place.
+    
+    Signed-off-by: Douglas Fuller <dfuller@redhat.com>
+    (cherry picked from commit 8f9a25202093339afb5308051d354d3ae79c6b2d)
+
+commit 0f5f99c1de5484e7563b5c5c9fa481200e1d00c6
+Author: Douglas Fuller <dfuller@redhat.com>
+Date:   Wed Jul 26 13:05:59 2017 -0400
+
+    qa/cephfs: Allow deferred fs creation
+    
+    Permit Filesystem objects to be created and settings modified before
+    calling Filesystem.create().
+    
+    Signed-off-by: Douglas Fuller <dfuller@redhat.com>
+    (cherry picked from commit 5fafc03cb9e6d5f39ce494375ea28c89b896e2f5)
+
+commit 47e485e5ccedce3977b30503e05b743e09fc7c77
+Author: Douglas Fuller <dfuller@redhat.com>
+Date:   Wed Jul 12 10:41:11 2017 -0500
+
+    qa/ceph_test_case: support CephFS recovery pools
+    
+    Add support for testing recovery of CephFS metadata into an alternate
+    RADOS pool, useful as a disaster recovery mechanism that avoids
+    modifying the metadata in-place.
+    
+    Signed-off-by: Douglas Fuller <dfuller@redhat.com>
+    (cherry picked from commit c85562c94a80b8a18975b8d0ee6a7fbd932cf024)
+
+commit 69f042231056071dee16ed49aad95eaa3b0447f6
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Aug 29 11:35:56 2017 +0800
+
+    mds: void sending cap import message when inode is frozen
+    
+    To export an inode to other mds, mds need to:
+    
+    - Freeze the inode (stop issuing caps to clients)
+    - Flush client sessions (ensure client have received all cap messages)
+    - Send cap export message
+    
+    These steps guarantee that clients receive cap import/export messages
+    in proper order (In the case that inode gets exported servel times
+    within a short time)
+    
+    When inode is frozen, mds may have already flushed client sessions.
+    So mds shouldn't send cap import messages.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 48a9e695e58ac15f1e494977e0db7a164bb2fe98)
+
+commit 25def78ceb4cb17c97ae911203a62fb410cd9399
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Aug 28 17:13:31 2017 +0800
+
+    client: fix message order check in handle_cap_export()
+    
+    If importer mds' cap already exists, but cap ID mismatches, client
+    should have received corresponding import message (the imported caps
+    got released later). Because cap ID does not change as long as client
+    holds the caps.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 3c31256a4addcc4eca30c62109e33f77224258c2)
+
+commit 97ab78548d26f980ead8dc5c986304d2cfd3568d
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Sep 21 10:56:07 2017 +0800
+
+    mds: check ongoing catter-gather process before capping log
+    
+    When deactivating mds, MDLog::trim() may start scatter-gather
+    process on mdsdir inode. Locker::scatter_writebehind() submits
+    log entry. So mds should make sure there is no scatter-gather
+    before capping log.
+    
+    Fixes: http://tracker.ceph.com/issues/21467
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+
+commit 0132a275aa86ca80bf1dcacd9905243257317e6c
+Merge: 1befcbebcb bc4095ee44
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Sep 20 08:08:14 2017 -0700
+
+    Merge pull request #17829 from jdurgin/wip-21428-luminous
+    
+    osd/PrimaryLogPG: request osdmap update in the right block
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit bc4095ee44f8c060a43470c29c08767517b4d749
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Sep 20 01:18:35 2017 -0400
+
+    osd/PrimaryLogPG: request osdmap update in the right block
+    
+    Fixes: http://tracker.ceph.com/issues/21428
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit afc6624f768ea3c6e2d155122797db5cce8836f7)
+
+commit 13f0c9f534d0a9df362bb86ab546648fa046c104
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Sep 13 16:34:40 2017 +0800
+
+    qa/cephfs: kill mount if it gets evicted by mds
+    
+    otherwise, teardown() hange at umount
+    
+    Fixes: http://tracker.ceph.com/issues/21275
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 98d86a075242f0c4576814db15cc3f8fd8e06017)
+
+commit 651280281ab1347fb7d0e3114cf87b74398dbc87
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Sep 13 16:15:31 2017 +0800
+
+    qa/cephfs: fix test_evict_client
+    
+    executing mount_a.kill() twice, then executing mount_b.kill_cleanup()
+    twice do not make sense.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 8433ced84733da86d545ccacc06ddea3a0282071)
+
+commit 1befcbebcbcedf398b782adfcc165038d13fc3da
+Merge: ddf84249fa a6ce7bfcf3
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Tue Sep 19 16:45:40 2017 -0700
+
+    Merge pull request #17796 from jdurgin/wip-21428-luminous
+    
+    osd: request new map from PG when needed
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 8d18150560bbf0c41f4d62b17b642449b3610660
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Sep 5 17:09:32 2017 +0800
+
+    qa: whitelist expected MDS_READ_ONLY for kcephfs test
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 584241285f52fc420ebc4a6a6455497a3ed4d45a)
+
+commit a6ce7bfcf3ab00046f220a00ef2c283c3aa0bb31
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Sep 18 22:31:10 2017 -0400
+
+    osd: request new map from PG when needed
+    
+    The fast dispatch refactor in 3cc48278bf0ee5c9535d04b60a661f988c50063b
+    eliminated the osdmap subscription in the ms_fast_dispatch path, which
+    meant ops could reach a PG without having the latest map. In a cluster
+    with few osdmap updates, where the monitor fails to send a new map to
+    an osd (it tries one random osd), this can result in indefinitely
+    blocked requests.
+    
+    Fix this by adding an OSDService mechanism for scheduling a new osdmap
+    subscription request.
+    
+    Fixes: http://tracker.ceph.com/issues/21428
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit dd33360e5a4ed06228af552edadee88e66c5aba9)
+
+commit 875f1c2519c15a94b4f4ccfa3ae0b0b191cfb1dc
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 15 16:52:28 2017 -0400
+
+    Revert "osd/OSDMap: allow bidirectional swap of pg-upmap-items"
+    
+    This reverts commit 09af9b8afb40cc8aa629501582a75e03edf0bf2e.
+    
+    We need to prevent duplicates in the final result.  For example, we
+    can currently take
+     [1,2,3] and apply [(1,2)] and get [2,2,3]
+    or
+     [1,2,3] and apply [(3,2)] and get [1,2,2]
+    
+    The rest of the system is not prepared to handle duplicates in the
+    result set like this.
+    
+    The reverted commit was intended to allow
+    
+     [1,2,3] and [(1,2),(2,1)] to get [2,1,3]
+    
+    to reorder primaries.  First, this bidirectional swap is hard to implement
+    in a way that also prevents dups.  For example,
+     [1,2,3] and [(1,4),(2,3),(3,4)] would give [4,3,4]
+    but would we just drop the last step we'd have [4,3,3] which
+    is also invalid, etc.  Simpler to just not handle bidirectional
+    swaps.  In practice, they are not needed: if you just want to choose
+    a different primary then use primary_affinity, or pg_upmap
+    (not pg_upmap_items).
+    
+    Fixes: http://tracker.ceph.com/issues/21410
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9c73305e3ad11177d58632eba6ece5d2c0e701da)
+
+commit ddf84249fa8a8ec3655c39bac5331ab81c0307b1
+Merge: d324411744 4c16e1b36b
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 22:46:27 2017 +0200
+
+    Merge pull request #17724 from theanalyst/wip-21374-luminous
+    
+    luminous: incorrect erasure-code space in command ceph df
+    
+    Reviewed-By: Josh Durgin <jdurgin@redhat.com>
+
+commit d32441174440b7e639658a9218d7004ebe604b62
+Merge: df73cdcdfe 381ed45b49
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri Sep 15 13:35:36 2017 -0700
+
+    Merge pull request #17745 from tchaikov/wip-luminous-20529
+    
+    luminous: rocksdb,cmake:  build portable binaries
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit df73cdcdfec8263d5f1cd0af98d51928aa7659f3
+Merge: 24c5b9bfb7 e7158e8e41
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:58:05 2017 +0200
+
+    Merge pull request #17635 from smithfarm/wip-21342-luminous
+    
+    luminous: ceph mgr versions shows active mgr as Unknown
+    
+    Reviewed-By: Josh Durgin <jdurgin@redhat.com>
+
+commit 24c5b9bfb72461005585329f82a14b994c8546e4
+Merge: 57e798bc55 b10311dc73
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:57:54 2017 +0200
+
+    Merge pull request #17634 from smithfarm/wip-21341-luminous
+    
+    luminous: mon/OSDMonitor: deleting pool while pgs are being created leads to assert(p != pools.end) in update_creating_pgs()
+    
+    Reviewed-By: Josh Durgin <jdurgin@redhat.com>
+
+commit 57e798bc55ed5d1ae44d2279e2bba349abc88e76
+Merge: 9fc9577df0 fe2fe20100
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:57:13 2017 +0200
+
+    Merge pull request #17624 from tchaikov/wip-luminous-pr-17599
+    
+    luminous: encoding: reset optional<> if it is uninitialized
+    
+    Reviewed-By: Josh Durgin <jdurgin@redhat.com>
+
+commit 9fc9577df0c91f8ee3c55464e57f31fb20060853
+Merge: 0c28640ba0 91dd0fe479
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:55:41 2017 +0200
+
+    Merge pull request #17487 from smithfarm/wip-21231-luminous
+    
+    luminous: core: interval_set: optimize intersect_of insert operations
+    
+    Reviewed-By: Josh Durgin <jdurgin@redhat.com>
+
+commit 0c28640ba09cd1035ae5b4103833ef6bd172ae6c
+Merge: f52c6dcd60 24fe250bb5
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:49:37 2017 +0200
+
+    Merge pull request #17712 from batrick/bp21221
+    
+    luminous: MDCache::try_subtree_merge() may print N^2 lines of debug message
+    
+    Reviewed-By: Yan Zhen <zyan@redhat.com>
+
+commit f52c6dcd60ea352f8762d98e2b2c245abed5c0fc
+Merge: c9f0a6bd2e 9f65be2feb
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:45:04 2017 +0200
+
+    Merge pull request #17623 from liewegas/wip-pr-17572-luminous
+    
+    tools/ceph_objectstore_tool: fix 'dup' unable to duplicate meta PG
+    
+    Reviewed-By: Josh Durgin <jdurgin@redhat.com>
+
+commit c9f0a6bd2ec55b308de28b8ddf1d1658887d263b
+Merge: ece893d1e8 80d848f069
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:43:22 2017 +0200
+
+    Merge pull request #17584 from jdurgin/wip-recovery-delete-encoding-luminous
+    
+    osd/PGBackend, MOSDPGRecoveryDelete[Reply]: handle incompatible encoding
+    
+    Reviewed-By: Sage Weil <sage@redhat.com>
+
+commit ece893d1e8ee7df9f0ee6413628aa2aeda311364
+Merge: dfc8a0fe47 eb6a0bf22e
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:39:49 2017 +0200
+
+    Merge pull request #17686 from ukernel/luminous-21070
+    
+    luminous: mds: fix 'dirfrag end' check in Server::handle_client_readdir
+    
+    Reviewed-By: Patrick Donnelly <pdonelly@redhat.com>
+
+commit dfc8a0fe47b950155eab2bdbe98f42645c3f6140
+Merge: 0be1e868f1 a1be6c9647
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:38:36 2017 +0200
+
+    Merge pull request #17711 from batrick/bp20594
+    
+    luminous: mds: support limiting cache by memory
+    
+    Reviewed-By: Yan Zheng <zyan@redhat.com>
+
+commit 0be1e868f10601df3f293af58a173702db6efed0
+Merge: b0b4d07663 e7e3d8977b
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:36:00 2017 +0200
+
+    Merge pull request #17714 from batrick/bp21222
+    
+    luminous: MDS: standby-replay mds should avoid initiating subtree export
+    
+    Reviewed-By: Yan, Zheng <zyan@redhat.com>
+
+commit b0b4d0766345ffcc552a841be16101f4b80dcf55
+Merge: c69673d955 73ef8de0bf
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:25:56 2017 +0200
+
+    Merge pull request #17570 from theanalyst/wip-luminous-21233
+    
+    luminous: rgw: memory leak in MetadataHandlers
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit c69673d955d784ca580fc283228b881b30065ff6
+Merge: 671740533e ccb3c1edc0
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:25:31 2017 +0200
+
+    Merge pull request #17642 from theanalyst/wip-21350-luminous
+    
+    luminous: rgw: data encryption sometimes fails to follow AWS settings
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 671740533e71ac742259d32b86102fa807ea7fee
+Merge: 87d3d3419e 665c3a6397
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:23:01 2017 +0200
+
+    Merge pull request #17674 from theanalyst/luminous-rgw-reshard-enable
+    
+    luminous: rgw: reshard enable
+    
+    Reviewed-By: Orit Wasserman <owasserm@redhat.com>
+
+commit 87d3d3419ebb9370aeb14496901c55c2a01fdf1e
+Merge: 4bb174db3a 2be6c63f1b
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Sep 15 21:22:05 2017 +0200
+
+    Merge pull request #17479 from smithfarm/wip-21139-luminous
+    
+    luminous: rgw: put lifecycle configuration fails if Prefix is not set
+    
+    Reviewed-By: Daniel Gryniewicz <dang@redhat.com>
+
+commit 381ed45b491fecd9254c2b8ee3da89064cfe8c1a
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Sep 15 10:30:16 2017 +0800
+
+    cmake: do not pass $SIMD_COMPILE_FLAGS to rocksdb cmake
+    
+    which enables SSE42 globally in rocksdb. and we will end up with a
+    binary not portable on non-SSE42 enabled machines.
+    
+    Fixes: http://tracker.ceph.com/issues/20529
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    
+    Conflicts:
+            this change is not cherry-picked from master. because the
+    PR targeting master (https://github.com/ceph/ceph/pull/17388) is
+    still pending on review. and the cmake changes is different if
+    we want to use a recent commit of rocksdb, as it's doing differently
+    in cmake to address the portability issues.
+
+commit 9216086795a6a9c7b33000d44bea8ec9c62d41bb
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Sep 15 10:23:47 2017 +0800
+
+    rocksdb: pick up portable fixes
+    
+    to pick up the the fix to disable SSE42 globally, and only enable it on
+    crc32c. this change is pushed to ceph/rocksdb:luminous.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    Conflicts:
+            this change is not cherry-picked from master, as the master PR
+    (https://github.com/ceph/ceph/pull/17388) is still pending on review.
+    and the latest rocksdb's cmake was revised to address the portability
+    issues. so the fix on ceph side is different if we want to use a
+    recent rocksdb's commit.
+
+commit 73ef8de0bf147a3ca670ab8a4b04ddaaf0ed5bfd
+Author: Luo Kexue <luo.kexue@zte.com.cn>
+Date:   Wed Aug 30 09:32:18 2017 +0800
+
+    rgw: fix resource leak in rgw_bucket.cc and rgw_user.cc
+    
+    Fixes: http://tracker.ceph.com/issues/21214
+    
+    Signed-off-by: Luo Kexue <luo.kexue@zte.com.cn>
+    (cherry picked from commit 1f23976adab74e25ce06941692a881521885c6ee)
+
+commit 34548611d2d0394499e907ff7dd4b43cf53cd2cc
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Fri Aug 25 15:32:56 2017 +0530
+
+    rgw: add missing override in list_keys_init()
+    
+    Added missing override in list_keys_init().
+    
+    Signed-off-by: Jos Collin <jcollin@redhat.com>
+    (cherry picked from commit bfc999f1db96293e6a2188922074031fa0b40ecc)
+
+commit 4bb174db3a3cc18b544d7282516289075763a1f3
+Merge: e8e4f1fd6d 98a53f929e
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Thu Sep 14 11:32:28 2017 +0200
+
+    Merge pull request #17612 from smithfarm/wip-qemu-cleanup
+    
+    build/ops: rpm: do not create librbd.so.1 symlink in /usr/lib64/qemu
+    
+    Reviewed-By: Ken Dreyer <kdreyer@redhat.com>
+
+commit e8e4f1fd6d82540451be2896626e959e1b1bdaa8
+Merge: ad534c0d7a ce2a181834
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Thu Sep 14 11:31:40 2017 +0200
+
+    Merge pull request #17478 from smithfarm/wip-21138-luminous
+    
+    luminous: rgw: object copied from remote src acl permission become full-control issue
+    
+    Reviewed-By: Orit Wasserman <owasserm@redhat.com>
+
+commit ad534c0d7adaa83b3b219592de5a7b1213cb6317
+Merge: f05328733a 07a15984c6
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Thu Sep 14 11:31:23 2017 +0200
+
+    Merge pull request #17445 from smithfarm/wip-21110-luminous
+    
+    luminous: rgw: send data-log list infinitely
+    
+    Reviewed-By: Orit Wasserman <owasserm@redhat.com>
+
+commit 4c16e1b36b60028e9b6877dea7abcfe872b2e753
+Author: liuchang0812 <liuchang0812@gmail.com>
+Date:   Wed Sep 6 11:50:17 2017 +0800
+
+    mon: incorrect MAX AVAIL in "ceph df"
+    
+    Fixes: http://tracker.ceph.com/issues/21243
+    
+    Signed-off-by: liuchang0812 <liuchang0812@gmail.com>
+    (cherry picked from commit 365558571c59dd42cf0934e6c31c7b4bf2c65026)
+
+commit f05328733a04acf454d229f73d79015292bd5fdf
+Merge: 8d292af09f cac29c9618
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Thu Sep 14 10:33:29 2017 +0200
+
+    Merge pull request #17488 from smithfarm/wip-21241-luminous
+    
+    luminous: rgw: usage of --inconsistent-index should require user confirmation and print a warning
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 8d292af09f663a49d935011a7bebec096c77b04e
+Merge: 038dc170c7 24803b76c8
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Thu Sep 14 10:28:16 2017 +0200
+
+    Merge pull request #17484 from smithfarm/wip-21184-luminous
+    
+    luminous: tests: NameError: global name 'name' is not defined
+    
+    Reviewed-By: Orit Wasserman <owasserm@redhat.com>
+
+commit 038dc170c7928a415b0d08536be64b9eebd17511
+Merge: 67e87bff24 39b3b132f2
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Thu Sep 14 10:26:12 2017 +0200
+
+    Merge pull request #17474 from smithfarm/wip-21114-luminous
+    
+    luminous: cephfs: tests: FS_DEGRADED spurious health warnings in some sub-suites
+    
+    Reviewed-By: Patrick Donelly <pdonelly@redhat.com>
+
+commit 67e87bff24424a3e233a9a89bbdef1e1545c0df4
+Merge: ba746cd14d 25a651e100
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Thu Sep 14 10:25:51 2017 +0200
+
+    Merge pull request #17471 from smithfarm/wip-21107-luminous
+    
+    luminous: cephfs: client/mds has wrong check to clear S_ISGID on chown
+    
+    Reviewed-By: Patrick Donelly <pdonelly@redhat.com>
+
+commit e7e3d8977b4e46db7a370880d54094e7b7915826
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Sep 13 11:27:18 2017 +0800
+
+    ceph: fix "stopping mds can't export subtree"
+    
+    caused by commit
+    790c34138f "Make standby-replay mds avoid initiating subtree export"
+    
+    Fixes: http://tracker.ceph.com/issues/21378
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit c294051987de1ff79540060a0563fa02c8f03f91)
+
+commit a0693c8ca4ea54950d44e9b5b2da0219d46ea5a6
+Author: Jianyu Li <joannyli@foxmail.com>
+Date:   Mon Sep 4 21:49:26 2017 +0800
+
+    Make standby-replay mds not migrate empty subtree when trimming cache
+    
+    Signed-off-by: Jianyu Li <joannyli@tencent.com>
+    (cherry picked from commit 178ca58fec2d820f61724d223addac1c4fa67086)
+
+commit 70e814cf60c533fe2e850e451a7607ed5ffe7ed9
+Author: Jianyu Li <joannyli@foxmail.com>
+Date:   Mon Sep 4 18:03:30 2017 +0800
+
+    Make standby-replay mds avoid initiating subtree export
+    
+    Signed-off-by: Jianyu Li <joannyli@tencent.com>
+    (cherry picked from commit 790c34138fbdf995c72e558057b9e64c882b522b)
+
+commit 24fe250bb5fbb8546f5fb280cca10162e3aabb2f
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Mon Sep 4 11:36:00 2017 -0700
+
+    mds: print subtrees only after merge
+    
+    During up:resolve, the MDS tries to merge each subtree with its parent. During
+    testing, QE found that many thousands of subtrees in a directory (made possible
+    using pins) would cause the MDS to spend minutes printing out subtree maps to
+    the debug log. This causes the heartbeat code to consider the MDS as stalled so
+    beacons are no longer sent to the mons resulting in the MDS being removed from
+    the rank.
+    
+    A more complete solution to this problem is to selectively print subtrees
+    relating to the operation (e.g. the subtree and its parents).
+    
+    Fixes: http://tracker.ceph.com/issues/21221
+    Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1485783
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit d0747a37fd06053b2206bb9a952f7ab77f0db2f0)
+
+commit a1be6c96470b843d46754ffda08c524a3c7b9a59
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Sep 13 15:55:46 2017 +0800
+
+    mds: optimize MDCache::rejoin_scour_survivor_replicas()
+    
+    avoid iterating dentries if dirfrag is non-auth
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit d32a2376047bc1363b9f7a2ed09060ec84c78b97)
+
+commit fb3afeb1e7572cda45966fa925102767dcd2f1b7
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Sep 13 15:54:45 2017 +0800
+
+    mds: fix MDSCacheObject::clear_replica_map
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit fd447405bef347da80ed5611331b211021b1628b)
+
+commit 179bf1831c6c4e88eb13ed689293a677ef5d2983
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Mon Sep 11 15:21:52 2017 -0700
+
+    mds: support limiting cache by memory
+    
+    This introduces two config parameters:
+    
+        mds_cache_memory_limit: Sets the soft maximum of the cache to the given
+        byte count. (Like mds_cache_size, this doesn't actually limit the maximum
+        size of the cache. It just dictates the steady-state size.)
+    
+        mds_cache_reservation: This replaces mds_health_cache_threshold everywhere
+        except the Beacon heartbeat sent to the mons. The idea here is to specify a
+        reservation of memory (5% by default) for operations and the MDS tries to
+        always maintain that reservation. So, the MDS will recall caps from clients
+        when it begins dipping into its reservation of memory.
+    
+    mds_cache_size still limits the cache by Inode count but is now by-default 0
+    (i.e. unlimited). The new preferred way of specifying cache limits is by memory
+    size. The default is 1GB.
+    
+    Fixes: http://tracker.ceph.com/issues/20594
+    Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1464976
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 06c94de584e6cd7d347bcdfb79d9fef4fed0d277)
+    
+     Conflicts:
+            PendingReleaseNotes
+            src/mds/MDCache.cc
+
+commit c4d393e6d0b4497e94ab88a8f61436c6271e2ab8
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu Sep 7 21:01:31 2017 -0700
+
+    common: refactor of lru
+    
+    Avoids an unnecessary "max" size of the LRU which was used to calculate the
+    midpoint. Instead, just dynamically move the LRUObjects between top and bottom
+    on-the-fly.
+    
+    This change is necessary for a cache which which does not limit by the number
+    of objects but by some other metric. (In this case, memory.)
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 12d615b3c546211b48ee75921f0e06371dd62dae)
+
+commit 009d3ab0bc3cb5248701a4383e58a86337ee4982
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Tue Sep 12 13:40:17 2017 -0700
+
+    mds: resolve unsigned coercion compiler warning
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 0c2032c2876c3e2cd0d23ab883fc72375731cadf)
+
+commit e25881b1e8c02dd1262c5b2b41eb353b293e1e37
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu Sep 7 20:51:56 2017 -0700
+
+    common: use safer uint64_t for list size
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 0ddd260a32e781ff6bfa49c1a01d6b2d80e5dcd3)
+
+commit f21d2fa7e57e18f368c2c62534eba2cc9d0c80fb
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sun Jul 30 16:09:32 2017 -0700
+
+    common: add bytes2str pretty print function
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 7fff24e10e9cacc94ec320b87e60c7735ff028b6)
+
+commit 97fdc68e029ed1567eff7c5d13a6c1c29bfe5d33
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Tue Sep 12 14:31:18 2017 -0700
+
+    mds: check if waiting is allocated before use
+    
+    This prevents accidental allocation of the map.
+    
+    Also, privatize the variable to protect from this in child classes.
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 055020ce80b1f08d258b4c023bf5465ed7a46034)
+
+commit d50f2c735ac40b9ccfbd8d3423fd817f76de9c3c
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Tue Sep 12 14:29:49 2017 -0700
+
+    mds: go back to compact_map for replicas
+    
+    Zheng observed that an alloc_ptr doesn't really work in this case since any
+    call to get_replicas() will cause the map to be allocated, nullifying the
+    benefit. Use a compact_map until a better solution can be written. (This means
+    that the map will be allocated outside the mempool.)
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 5d67b5cc57ac803e60c2921f554cd0507ed4848e)
+
+commit 2ef222a58c3801eaac5a6d52dda2de1ffe37407b
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu Jul 27 17:21:54 2017 -0700
+
+    mds: use mempool for cache objects
+    
+    The purpose of this is to allow us to track memory usage by cached objects so
+    we can limit cache size based on memory available/allocated to the MDS.
+    
+    This commit is a first step: it adds CInode, CDir, and CDentry to the mempool
+    but not all of the containers in these classes (e.g. std::map). However,
+    MDSCacheObject has been changed to allocate its containers through the mempool
+    by converting compact_* containers to the std versions offered through mempool
+    via the new alloc_ptr.
+    
+    (A compact_* class simply wraps a pointer to the std:: version to reduce memory
+    usage of an object when the container is only occasionally used. The alloc_ptr
+    allows us to achieve the same thing explicitly with only a little handholding:
+    when all entries in the wrapped container are deleted, the caller must call
+    alloc_ptr.release().)
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit e035b64fcb0482c3318656e1680d683814f494fe)
+
+commit f26412883c390b78df132e32d99afa5a14c525f3
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu Jul 27 12:06:23 2017 -0700
+
+    mds: cleanup replica_map access
+    
+    The gymnastics protecting the map failed as the code evolved. Just expose it
+    normally with a getter.
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit d1b6cadd6c1514aeacfbacec253305e1a715c387)
+
+commit 1bf3b1939ccdf30c4645009887c93fcb03d029e8
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu Jul 27 12:10:14 2017 -0700
+
+    common: add alloc_ptr smart pointer
+    
+    This ptr is like a unique_ptr except it allocates the underlying object on
+    access. The idea being that we can save memory if the object is only needed
+    sometimes.
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 5fa557d2713558038af7579de8a3ca56d58b8d90)
+
+commit 44e206f351297fc9092577a9280fabdadd993cc6
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu Jul 27 12:04:04 2017 -0700
+
+    common: add warning on base class use of mempool
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit c0d0fa804ee0fa775ae18f9b7bde2317bb6af52d)
+
+commit 8c82de6a32bcbc8df0f6dadb7aacd7c41bcf77d8
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu Jul 27 12:03:15 2017 -0700
+
+    common: use atomic uin64_t for counter
+    
+    Making this interface thread-safe...
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 59b5931a2f47f90fced779fb89e06868c739c688)
+
+commit ba746cd14ddd70a4f24a734f83ff9d276dd327d1
+Merge: 138acd38a6 21b9659cdd
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Wed Sep 13 23:13:03 2017 +0200
+
+    Merge pull request #17646 from theanalyst/wip-luminous-multifixes
+    
+    luminous: rgw fixes for multisite tests in luminous
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit eb6a0bf22e5777b3977226d831380f5e68ae3739
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Aug 28 11:24:13 2017 +0800
+
+    mds: fix 'dirfrag end' check in Server::handle_client_readdir
+    
+    The 'dirfrag end' check can be wrong when not reading from beginning
+    of dirfrag.
+    
+    Fixes: http://tracker.ceph.com/issues/21070
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 88332c0f2f0a7a99639d1de237d5bb68edddbe70)
+
+commit 138acd38a67bad303fb6804c1f5be0b4e5c4c47f
+Merge: 6753d98d90 e2dda02e02
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 12 16:04:54 2017 -0400
+
+    Merge pull request #17647 from dillaman/wip-21345
+    
+    luminous: qa/workunits/rbd: relax greps to support upgrade formatting change
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 2be6c63f1bf910562ecca8ffd70206b605f1fe07
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Tue Sep 12 11:19:44 2017 +0800
+
+    rgw: LCFilter_S3::to_xml the second "prefix" missed "/"
+    
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit dcef8555e1bf2656aeb5316c4b78f1facf2ae172)
+
+commit 21b9659cdd6c0dae63609ee58915fe1166487bbc
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Aug 30 11:16:25 2017 -0400
+
+    test/rgw: fix test_encrypted_object_sync for 3+ zones
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 8762c4b29051b2b49bfd8c1e0ceaa1aee6417a74)
+
+commit 665c3a6397ed43d4dde0a0ad22483fc964ab61c8
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Tue Sep 12 18:07:01 2017 +0200
+
+    luminous: rgw: re enable dynamic resharding
+    
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+
+commit 6753d98d90c65961e4abbea43f70acb938753f7a
+Merge: 4102a50c0d 64fc7cf105
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Tue Sep 12 08:01:43 2017 -0400
+
+    Merge pull request #17653 from ceph/backport-wip-bz1478598
+    
+    luminous: ceph-volume allow filtering by `uuid`, do not require osd id
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit 4102a50c0ddc969b7f58642f2f67b67e0f671b23
+Merge: 7ca6156003 63c9382eaf
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Sep 12 13:02:35 2017 +0200
+
+    Merge pull request #17485 from smithfarm/wip-21185-luminous
+    
+    luminous: rgw: rgw_file:  incorrect lane lock behavior in evict_block()
+    
+    Reviewed-By: Orit Wasserman <oritwas@redhat.com>
+
+commit 7ca615600393a8337426818fbe0390117752f0ef
+Merge: ab9650b929 e171fa6a6d
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Sep 12 12:40:44 2017 +0200
+
+    Merge pull request #17476 from smithfarm/wip-21135-luminous
+    
+    luminous: rgw: bucket index sporadically reshards to 65521 shards
+    
+    Reviewed-By: Orit Wasserman <owasserm@redhat.com>
+
+commit ab9650b929320320b6f88f31a39575e16d69c05d
+Merge: 31dbc795cd 18442e1585
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Sep 12 11:06:05 2017 +0200
+
+    Merge pull request #17470 from smithfarm/wip-21104-luminous
+    
+    luminous: build/ops: ceph-fuse RPM should require fusermount
+    
+    Reviewed-By: Ken Dreyer <ktdreyer@redhat.com>
+    Reviewed-By: Kefu Chai <kchai@redhat.com>
+
+commit 31dbc795cd1b71f572f99ba8bf5936de19fa9fb8
+Merge: 39385a7512 17626e01d8
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Sep 12 11:05:14 2017 +0200
+
+    Merge pull request #17477 from smithfarm/wip-21137-luminous
+    
+    luminous: mgr: 500 error when attempting to view filesystem data
+    
+    Reviewed-By: John Spray <jcsp@redhat.com>
+
+commit 39385a7512112570c98c95b40c1215f6d4aa19d9
+Merge: 75db027e25 d57399c9da
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon Sep 11 16:09:01 2017 -0500
+
+    Merge pull request #17495 from smithfarm/wip-21235-luminous
+    
+    luminous: tests: thrashosds read error injection doesn't take live_osds into account
+
+commit 75db027e25b07e93f8cdf62316dd51ed6e305df2
+Merge: 975af9c2a4 ca7e48c62e
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon Sep 11 16:07:22 2017 -0500
+
+    Merge pull request #17483 from smithfarm/wip-21183-luminous
+    
+    luminous: mgr: Crash in MonCommandCompletion
+
+commit 975af9c2a48394c04852b142ea9d5ffa98b12db1
+Merge: b755445f84 8e9b52ee88
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon Sep 11 16:06:42 2017 -0500
+
+    Merge pull request #17486 from smithfarm/wip-21188-luminous
+    
+    luminous: pybind: dashboard usage graph getting bigger and bigger
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit b755445f8443b2cd7bb31027996be6a6daef96a4
+Merge: e763746bbf 2a1ae18fa7
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon Sep 11 16:05:24 2017 -0500
+
+    Merge pull request #17350 from liewegas/wip-pr-17334-luminous
+    
+    os/bluestore: reshard-fix wrong shard length
+    
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit e763746bbffd6139e49bb035b2c5763bc28dcd9c
+Merge: 69552ed435 00ed756ae8
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 22:31:14 2017 +0200
+
+    Merge pull request #17473 from smithfarm/wip-21112-luminous
+    
+    luminous: cephfs: get_quota_root sends lookupname op for every buffered write
+    
+    Reviewed-By: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 69552ed4352e25d90574da8e802f2c9572ea2529
+Merge: 71814348c2 a74b3267ee
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 22:29:32 2017 +0200
+
+    Merge pull request #17465 from smithfarm/wip-21045-luminous
+    
+    luminous: rbd: TestMirroringWatcher.ModeUpdated: periodic failure due to injected message failures
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit 71814348c28071e14ebda8ac6bf257e9cbf08a79
+Merge: 2142746fe1 1e9983ac83
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 22:28:59 2017 +0200
+
+    Merge pull request #17565 from theanalyst/wip-21278-luminous
+    
+    luminous: the standbys are not updated via ceph tell mds.* command
+    
+    Reviewed-By: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 2142746fe14f5717062b2e1f9931b36e12896336
+Merge: be9331252a 0ec531240a
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 22:28:06 2017 +0200
+
+    Merge pull request #17566 from theanalyst/wip-21267-luminous
+    
+    luminous: Incorrect grammar in FS message 1 filesystem is have a failed mds daemon
+    
+    Reviewed-By: Patrick Donnelly <pdonnell@redhat.com>
+
+commit be9331252ae966a9d7ec1e5ce9321d8806061d6a
+Merge: cfbd752f6d 22208c8237
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 22:23:50 2017 +0200
+
+    Merge pull request #17557 from theanalyst/wip-21269-luminous
+    
+    luminous: some generic options can not be passed by rbd-nbd
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit cfbd752f6d46fa96307d3529022524279c4896fe
+Merge: b9ec9193d7 3f0ee2aa39
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 21:43:48 2017 +0200
+
+    Merge pull request #17448 from smithfarm/wip-21118-luminous
+    
+    luminous: rgw: need to stream metadata full sync init
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit b9ec9193d7b49e44de3d34674f9dc2b8815f99df
+Merge: 5f0ff20aca 5f0ba1353b
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 21:37:26 2017 +0200
+
+    Merge pull request #17464 from smithfarm/wip-20964-luminous
+    
+    luminous: rbd: switch to new config option getter methods
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit 64fc7cf105ec4cdeacb6e2af3060b914ba47abeb
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Sep 11 12:32:08 2017 -0400
+
+    ceph-volume lvm.trigger just split the osd_id once
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit ecd5af88da2815603283742c661d27e2a48f9f7e)
+
+commit 7cdf1772d69e85646a55202c4faac1d0aaace5c4
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Sep 11 12:30:34 2017 -0400
+
+    ceph-volume tests catch improper parsing of osd uuid
+    
+    When the osd ID existed in the UUID in the same format (e.g. '1-') the
+    parsing would break returning a bogus UUID that is impossible to find
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 5392697121ef150f0edac05df14d116cf0148593)
+
+commit 944e69863347bb11450874980e4f64807d9e6e87
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Fri Sep 8 15:32:15 2017 -0500
+
+    tests: remove 'ceph-' prefix from test node hostnames
+    
+    This updates our Vagrantfile to match the changes in ceph-ansible
+    introduced by https://github.com/ceph/ceph-ansible/commit/298a63c43
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit f41d359ebaec8762ed1a7b8cfd85c23f351a5070)
+
+commit f1b92868fa775ab4c210b3fd02e9a4913485db3a
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Sep 8 12:25:57 2017 -0400
+
+    ceph-volume lvm.api PVolumes should filter by all matching tags, not just one
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit b8613c1d7bf56c49d1b7f1c41a37073b8739bbc3)
+
+commit 54cdf1805a12fd691bd9da0441dab186446aef76
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Sep 8 12:25:31 2017 -0400
+
+    ceph-volume tests pv API should also match all pv_tags
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 120fd6a196045c8511c3eaafec37de43003bf450)
+
+commit 6a1189d1eb74d9c3d7653f4f32c480158d260901
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Sep 8 11:47:09 2017 -0400
+
+    ceph-volume tests prove all volume tags have to match in vgs
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit fe2a7106decb586e83f5b60d6c18c454afe4883e)
+
+commit f4930e0b8327e5dcd6aafaac7cd51c1933b95456
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Sep 8 11:46:34 2017 -0400
+
+    ceph-volume lvm.api all vg tags should match, not just one
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 6d679855aa077e2de3f20316625d478a3697c78f)
+
+commit fafc161e1f6c825930c0848c32fa06b1d5edc849
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Sep 7 17:00:22 2017 -0400
+
+    ceph-volume tests prove all tags have to match in lvs
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 80e401ca970066541ae7255be1caf42883048df4)
+
+commit cdba58264732a648425dd1f78e75a91ea6a5198d
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Sep 7 16:59:53 2017 -0400
+
+    ceph-volume lvm.api all lv tags should match, not just one
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit dbaf30c3e4d921df2a92812383c8805a77eb8f5b)
+
+commit 12d638afea087189a1d3f350dd87185b0ade6617
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Sep 7 16:56:55 2017 -0400
+
+    ceph-volume lvm.activate allow not using osd_id
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 99a8a719695591558da29b3d16a347d5bff8bf3d)
+
+commit a50dc9647282c7b7455774502532f5281ae2aece
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Sep 7 16:54:19 2017 -0400
+
+    ceph-volume tests prove missing args.osd_id is fine
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 8d1a1a885fd65073308c205949bc70b41805f92f)
+
+commit 834d4c0eabf18fe1219f52a70e3bf78235f51f62
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Sep 7 16:51:43 2017 -0400
+
+    ceph-volume tests add is_root and lv fixtures to conftest
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d18d5e139be82de36e1a60340525adbf79fdedd9)
+
+commit 5f0ff20aca1bb25f8758d6533f0aa38b3614c954
+Merge: 90fbef808e 5d229d18ec
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 21:22:44 2017 +0200
+
+    Merge pull request #17446 from smithfarm/wip-21115-luminous
+    
+    luminous: rgw multisite: objects encrypted with SSE-KMS are stored unencrypted in target zone
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 90fbef808eb84f3cdc72172a8e4690f8857c960c
+Merge: a05e676a6a a1a3c07ba4
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 21:22:05 2017 +0200
+
+    Merge pull request #17468 from smithfarm/wip-21052-luminous
+    
+    luminous: build/ops: RHEL 7.3 Selinux denials at OSD start
+    
+    Reviewed-By: Ken Dreyer <ktdreyer@redhat.com>
+
+commit a05e676a6a619327929d1000e22a557e77c1dc6f
+Merge: 16ccc2084c 7c3a82225e
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 21:19:17 2017 +0200
+
+    Merge pull request #17444 from smithfarm/wip-20968-luminous
+    
+    luminous: rgw: bytes_send and bytes_recv in the msg of usage show returning is 0 in master branch
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 16ccc2084c7d477af362e2d99a409f7c929268ac
+Merge: c0c79fd74d 4f7c155c2e
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 21:16:40 2017 +0200
+
+    Merge pull request #17555 from theanalyst/wip-21270-luminous
+    
+    luminous: rgw: shadow objects are sometimes not removed
+    
+    Reviewed-By: Orit Wasserman <owasserm@redhat.com>
+
+commit c0c79fd74df79c0746ee31ef0590dd60d586946c
+Merge: 7b69aefc30 40d46c01b7
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 21:00:36 2017 +0200
+
+    Merge pull request #17558 from theanalyst/wip-21277-luminous
+    
+    luminous: [cls] metadata_list API function does not honor `max_return` parameter.
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit 7b69aefc30d11fd4cc3ea2bb9911761a5d0b7770
+Merge: 893ca2493a 3ac5343841
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 20:59:53 2017 +0200
+
+    Merge pull request #17556 from theanalyst/wip-21265-luminous
+    
+    luminous: [cli] rename of non-existent image results in seg fault
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit 893ca2493aa19b18859064a25034b77642f18fe6
+Merge: 8d180e4386 6468ddf8fc
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 20:59:28 2017 +0200
+
+    Merge pull request #17560 from theanalyst/wip-21288-luminous
+    
+    luminous: [test] various teuthology errors
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit 8d180e4386f6385b2c76976de8e73ca4d93a5c08
+Merge: 06f566fd07 7989da7da6
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 20:58:52 2017 +0200
+
+    Merge pull request #17561 from theanalyst/wip-21289-luminous
+    
+    luminous: [rbd] image-meta list does not return all entries
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit e2dda02e02d25b3785c7355abffdad98a1242482
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Sep 7 08:55:27 2017 -0400
+
+    qa/workunits/rbd: relax greps to support upgrade formatting change
+    
+    Fixes: http://tracker.ceph.com/issues/21181
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 273c84578b06905229a7c6eae7b1a12bbe60de95)
+    
+    Conflicts:
+            qa/workunits/rbd/import_export.sh: trivial resolution
+
+commit 06f566fd073c25c674f0441f83e3ce8f4396e1d5
+Merge: 5e5002d993 9cd6899346
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Mon Sep 11 20:08:14 2017 +0200
+
+    Merge pull request #17336 from tchaikov/wip-luminous-pr-16774
+    
+    luminous: mgr/dashboard: fix rbd's pool sub menu
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit ccb3c1edc0a45dfaadb5c4a4aa6700d0b8c8d92f
+Author: hechuang <hechuang@xsky.com>
+Date:   Thu Jun 29 10:38:23 2017 +0800
+
+    rgw: Data encryption is not follow the AWS agreement
+    
+    Encryption request headers should not be sent for GET requests and HEAD
+    requests if your object uses SSE-KMS/SSE-S3 or you’ll get an HTTP 400
+    BadRequest error.
+    
+    Signed-off-by: hechuang <hechuang@xsky.com>
+    (cherry picked from commit dfe1a7c1e6c470d623c70d83c32cc5b7a921bbc5)
+
+commit e7158e8e41e89eb845ecd7a888a5bfc6a008cc75
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Sep 7 09:44:36 2017 -0400
+
+    mon: fix dropping mgr metadata for active mgr
+    
+    drop_standby() was killing it and it was only getting added
+    back in certain locations.  Instead, make the metadata
+    drop conditional and only do it in the places we're
+    really dropping the daemon, not when we're promoting
+    it to active.
+    
+    Fixes: http://tracker.ceph.com/issues/21260
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 29c6f9adf178f6611a625740f395e397cad9147b)
+
+commit f5ff82d8ff4c45aee345b271ef4a3d1b95f83237
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Sep 7 09:42:29 2017 -0400
+
+    qa/tasks/mgr: regression test for 21260
+    
+    (aka http://tracker.ceph.com/issues/21260)
+    
+    Fixes: http://tracker.ceph.com/issues/21260
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 28cb4bb2e68298a8a307eb2de6121b60db43cb86)
+
+commit b10311dc730edfbd622a5a2ee7fe45293887b61a
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Fri Sep 8 12:48:12 2017 +0100
+
+    mon/OSDMonitor: don't create pgs if pool was deleted
+    
+    A pool may be deleted while in the process of creating pgs. If that
+    happens, we should not queue additional pgs from that pool to be
+    created.
+    
+    Fixes: http://tracker.ceph.com/issues/21309
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit 7a8a88032ec0bf7f1b485419a11646d7441faf2c)
+
+commit f847b164fda0e54a725e4a805124b78a61e545bf
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Thu Sep 7 18:10:18 2017 +0100
+
+    mon/OSDMonitor: fix typo
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit bc5fe2e9099dbb560c2153d3ac85f38b46593a77)
+
+commit fe2fe20100f7f5349d7101bc718f040f5e45bde1
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Sep 8 19:28:09 2017 +0800
+
+    encoding: reset optional<> if it is uninitialized
+    
+    * should reset it, in case we reuse it after initializing it.
+    * initialize the value of `p` using the C++11 style initializer, so it
+      is zero-initialized.
+    * revert 2a83ef3c which disables a warning of:
+     ./include/encoding.h:317:7: warning: 't' may be used uninitialized in
+        this function [-Wmaybe-uninitialized]
+     where the `t` is the temporary variable for initializing the value of
+     `p`.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 0363236dc754180a0215a818d58587943f84fb53)
+
+commit 9f65be2feb0186276e2b91fc58e6573779921970
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Wed Sep 6 17:31:16 2017 +0800
+
+    tools/ceph_objectstore_tool: fix 'dup' unable to duplicate meta PG
+    
+    Recently we plan to bring a Jewel cluster into Luminous.
+    After that is done, which turns out to be a big success,
+    we then try to transform all FileStore osds into BlueStore ones
+    offline but with no luck. The ceph_objectstore_tool keeps complaining:
+    --------------------------------------------------------------------
+    dup from filestore: /var/lib/ceph/osd/ceph-20.old
+          to bluestore: /var/lib/ceph/osd/ceph-20
+    fsid d444b253-337d-4d15-9d63-86ae134ec9ac
+    65 collections
+    1/65 meta
+    cannot get bit count for collection meta: (61) No data available
+    --------------------------------------------------------------------
+    
+    The root cause is that for FileStore Luminous will always try to rewrite
+    pg "bits" as a file attribute on "Load" if that is not available.
+    But since meta pg is never loaded (we skip it during OSD::load_pgs()),
+    we actually never get the chance to do so; hence making the
+    dup method from ceph_objectstore_tool very unhappy since it always
+    expects to see such a attribute from underlying store.
+    
+    Fix the above problem by manually skipping loading the "bits" attribute
+    if underlying OS is FileStore for dup.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    
+    (cherry picked from commit af2c4847e42945a878e2864319185d795b0664a5)
+
+commit 5e5002d99360cc2a3c956aeee43ab4fb25a6eef9
+Merge: ec8fe46855 68f48c7af8
+Author: Sage Weil <sage@newdream.net>
+Date:   Sun Sep 10 14:16:20 2017 -0500
+
+    Merge pull request #17622 from liewegas/wip-more-lum-pi
+    
+    osd: update info only if new_interval
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit ec8fe46855979e999f0e518f73ddd18ba40a23c6
+Merge: 6957c220ed 4683108903
+Author: Sage Weil <sage@newdream.net>
+Date:   Sun Sep 10 14:01:27 2017 -0500
+
+    Merge pull request #17564 from theanalyst/wip-21283-luminous
+    
+    qa/objectstore/bluestore*: less debug output
+
+commit 68f48c7af89fd5d8b224d52c8f60cfd2cdaf2df8
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Sep 4 13:51:34 2017 +0800
+
+    osd: update info only if new_interval
+    
+    refactor OSD::build_initial_pg_history() so we update the info only if a
+    new interval is identified. also, this is consistent with
+    OSD::build_past_intervals_parallel().
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 4ae053439e526fab0451d1502576dce89eb6b44f)
+
+commit 6957c220edc5c1bab7d41fa0d3ca06f02ec268d5
+Merge: 462fc906e1 e2e81403f8
+Author: Sage Weil <sage@newdream.net>
+Date:   Sun Sep 10 13:59:45 2017 -0500
+
+    Merge pull request #17496 from smithfarm/wip-21236-luminous
+    
+    luminous: osd: build_initial_pg_history doesn't update up/acting/etc
+
+commit 462fc906e14bc994667f015f8dd1ece923861c0d
+Merge: 0832adaaa7 a32f6a2e3a
+Author: Sage Weil <sage@newdream.net>
+Date:   Sun Sep 10 13:56:51 2017 -0500
+
+    Merge pull request #17601 from liewegas/wip-21171-luminous
+    
+    os/bluestore: fix deferred write deadlock, aio short return handling
+
+commit 98a53f929ef92ac6c68f34e30df929eca3f323bc
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Aug 29 00:19:24 2017 +0200
+
+    build/ops: rpm: do not create librbd.so.1 symlink in /usr/lib64/qemu
+    
+    As reported by Ademar de Souza Reis Jr <areis@redhat.com>:
+    
+    This symlink should not be necessary anymore. QEMU is properly linked to
+    ceph/librbd these days (look at the qemu-block-rbd sub-package in Fedora and
+    the respective package in RHEL).
+    
+    The symlink was a hack from a time when librbd was distributed and supported by
+    ceph and the rbd driver would be enabled at runtime only when/if the symlink
+    was present.
+    
+    So the right fix is to actually get rid of the symlink altogether and never
+    touch (much less own) /usr/lib64/qemu.
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit be4d6b1a74a0612621c3ad2cedebaa1fa40ffbab)
+
+commit 0832adaaa7e7faaf60ff7a51d18229bfc6eebf76
+Merge: aba1e7cd34 d769b077b1
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Sep 8 14:04:14 2017 -0400
+
+    Merge pull request #17551 from dillaman/wip-21280
+    
+    luminous: test/cls_rbd: mask newer feature bits to support upgrade tests
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 80d848f069b9c5c97a720972f62bbb8eda1d3573
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Tue Sep 5 14:16:57 2017 -0700
+
+    osd/PGBackend, MOSDPGRecoveryDelete[Reply]: handle incompatible encoding
+    
+    Handle a pre-luminous encoding without min_epoch for both these
+    messages.  Bump the header version, so the simple path of inline
+    decoding works for the future.  For 12.2.0 and pre-luminous, we can
+    check the SERVER_LUMINOUS flag on the connection to tell which
+    encoding to use.
+    
+    This is not a cherry-pick from master since all upgrades must stop at
+    luminous, and will use the existing encoding before continuing.
+    
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+
+commit aba1e7cd34bebca60187bed3fc8fcd75a18642b4
+Merge: 28e5f64ec3 ce434ac24f
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 10:58:39 2017 -0500
+
+    Merge pull request #17467 from smithfarm/wip-21046-luminous
+    
+    luminous: common: adding line break at end of some cli results
+
+commit 28e5f64ec3fc946bad8472c342c73c5d3bafb17f
+Merge: 3a270224a5 e2cc9ef48a
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 10:57:33 2017 -0500
+
+    Merge pull request #17472 from smithfarm/wip-21108-luminous
+    
+    luminous: mon: OSDMonitor: osd pool application get support
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 3a270224a5a1e75eddcc74adc8e6effc39674e2d
+Merge: 79750eab64 a656a38d28
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 10:56:56 2017 -0500
+
+    Merge pull request #17475 from smithfarm/wip-21133-luminous
+    
+    luminous: osd: PrimaryLogPG: sparse read won't trigger repair correctly
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 79750eab642eca2b4d6227dfdf28932edcae1256
+Merge: fb1a23d974 0c75157a91
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 10:55:55 2017 -0500
+
+    Merge pull request #17481 from smithfarm/wip-21182-luminous
+    
+    luminous: core: osd crush rule rename not idempotent
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit fb1a23d974cae62ea9effc10f25a03e616f863ac
+Merge: e5885fae2f 927b19ff77
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 10:55:37 2017 -0500
+
+    Merge pull request #17494 from smithfarm/wip-21234-luminous
+    
+    luminous: bluestore: asyn cdeferred_try_submit deadlock
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit e5885fae2ff8215d6c06acf23edb69d02e59ce46
+Merge: e22b3a9410 daacbfd90f
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Sep 8 08:51:33 2017 -0700
+
+    Merge pull request #17469 from smithfarm/wip-21103-luminous
+    
+    luminous: cephfs: client: missing space in some client debug log messages
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit e22b3a941098cd54f4986b8bea8019483fefb214
+Merge: bcd492a062 dd5f9e5333
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Sep 8 08:48:05 2017 -0700
+
+    Merge pull request #17562 from theanalyst/wip-21276-luminous
+    
+    luminous: os/bluestore/BlueFS.cc: 1255: FAILED assert(!log_file->fnode.extents.empty())
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit bcd492a062875c71047faaae58e77568dbaa4c41
+Merge: d48762a858 2c7fbc463c
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 09:28:35 2017 -0500
+
+    Merge pull request #17345 from liewegas/wip-pr-17311-luminous
+    
+    mon: use ceph_clock_now if message is self-generated
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit d48762a858f87177a7834f6ebab6a4be8f8a3d82
+Merge: 7c347e82e9 43ffb7c897
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 09:28:17 2017 -0500
+
+    Merge pull request #17344 from liewegas/wip-pr-17330-luminous
+    
+    mon/OSDMonitor: make 'osd crush class rename' idempotent
+
+commit 7c347e82e9919bc8c72022b3b101e197089428de
+Merge: 92ea7eb529 7b3ab3252f
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 09:27:55 2017 -0500
+
+    Merge pull request #17527 from liewegas/wip-pr-17510-luminous
+    
+    crush: fix fast rule lookup when uniform
+
+commit 92ea7eb529d915a17826dc69f0a266112ad2d0f3
+Merge: 2e60c1711d 81bf6cbfd4
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 09:27:31 2017 -0500
+
+    Merge pull request #17511 from liewegas/wip-stupid-luminous
+    
+    os/bluestore: default to stupid allocator
+    
+    Reviewed-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit 2e60c1711da79e687b5002e40bfe4183ccee5f0a
+Merge: 70c0d2ebf7 ac4cb6dd41
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 09:26:27 2017 -0500
+
+    Merge pull request #17501 from smithfarm/wip-21242-luminous
+    
+    luminous: osd: osd/PG: discard msgs from down peers
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 70c0d2ebf79307ab4e42555672579dc5dfca4592
+Merge: 3ade14114d 8a0d7aa073
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 09:25:37 2017 -0500
+
+    Merge pull request #17500 from smithfarm/wip-21240-luminous
+    
+    luminous: mon: rate limit on health check update logging
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 3ade14114da0798b05b59842a31c9630f0393269
+Merge: 9437427b3e 4c0073d20b
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 09:24:50 2017 -0500
+
+    Merge pull request #17498 from smithfarm/wip-21238-luminous
+    
+    luminous: osd: subscribe to new osdmap while waiting_for_healthy
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 9437427b3e7f544bbd02eeb698198ed6154983f0
+Merge: a06f09ea3f 3806293b1e
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 09:22:49 2017 -0500
+
+    Merge pull request #17497 from smithfarm/wip-21237-luminous
+    
+    luminous: bluestore: osd crash when change option bluestore_csum_type from none to CRC32
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit a06f09ea3f63ffc7874f12a29539201abfce8be8
+Merge: 0fd1035a6f 33dc4e65b2
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 8 09:22:23 2017 -0500
+
+    Merge pull request #17378 from jan--f/wip-21187-luminous
+    
+    luminous: core: osd/PGLog: write only changed dup entries
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit a32f6a2e3a46c1709e5a3e0443d8b45df4bd9c99
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 7 16:28:59 2017 -0400
+
+    os/bluestore/aio: handle short return from io_submit
+    
+    io_submit may return a value less than nr, indicating that only some of
+    the provided iocbs were queued.  If that happens we should loop, not
+    return and silently drop those aios on the floor.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit dc17dfd9ff05b5676488c2b1bca53026b2ca6244)
+
+commit accd582d59efa2ba8349e6ad58775e1ca619d821
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 30 17:18:18 2017 -0400
+
+    os/bluestore: more deferred debugging
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 11ac691dcc86aeed4ccb692a79ade4be04c5ef2f)
+
+commit e9b89c8b011fbbeeb128553b1b12708155a52628
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 29 16:15:52 2017 -0400
+
+    os/bluestore: drop deferred_submit_lock
+    
+    This lock serves no purpose.  It only protects the call to the aio_submit
+    on the OpSequencer running batch, but we are the only caller who will do
+    that submission, and a new batch won't be moved into place until ours is
+    completed and retired (by the completion for the aio we are queueing).
+    
+    More importantly, this fixes a deadlock:
+    
+    - thread A submits aio, but the queue is full, and blocks
+    - thread B blocks taking deferred_submit_lock while holding deferred_lock
+    - aio completion thread blocks trying to take deferred_lock, and thus
+    no further aios are drained.
+    
+    Fixes: http://tracker.ceph.com/issues/21171
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 7a5ef62af194373c3a8c307b05ae6d467e49c9f0)
+
+commit 0fd1035a6f280ebd38e6aabc925afb3d49fd53ea
+Merge: 823585311c c9c168b0ea
+Author: Pan Liu <liupan1111@gmail.com>
+Date:   Fri Sep 8 09:34:36 2017 +0800
+
+    Merge pull request #17534 from liewegas/wip-pr-17459-luminous
+    
+    os/bluestore: add bluestore_prefer_deferred_size_hdd/ssd to tracked keys
+
+commit 0ec531240ad47dc2d11db68a50d7e9debfe67efe
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Aug 28 13:50:46 2017 +0100
+
+    mds: fix "1 filesystem is have a..." message
+    
+    Like "Health check failed: 1 filesystem is have a failed mds daemon (FS_WITH_FAILED_MDS)"
+    
+    Fixes: http://tracker.ceph.com/issues/21153
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 7d21119a47212140d121df99efdb838a3f078a82)
+
+commit 1e9983ac833e8bd74fa82c6a03ad8fbba68747ac
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 5 16:33:24 2017 +0800
+
+    ceph: collect all mds in mdsids()
+    
+    otherwise, only the active mds are returned.
+    
+    Fixes: http://tracker.ceph.com/issues/21230
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit dd702cc94e32f533f0f2e76f9d202d1fa7c6a57f)
+
+commit dabbe79e17269e8061aae60198008292601335f3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 5 16:05:44 2017 +0800
+
+    ceph: always populate `targets` with ids_by_service()
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b682e61ddc190b91da4c6cbaeba3c69d54652afb)
+
+commit 428c9934fdf78d378b2c75ac43759581710472fb
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 5 16:00:59 2017 +0800
+
+    ceph: extract ids_by_service() so it can be reused
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 25639f66910aa8a188d5b4b8fff8803e169da62a)
+
+commit 468310890376a9d6b055e01d1743a8c5753dce35
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 5 17:43:28 2017 -0400
+
+    qa/objectstore/bluestore*: less debug output
+    
+    Let's see if this makes the spurious MON_DOWN failures go away?  (See
+    http://tracker.ceph.com/issues/20910)
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 444f5aa08539cde827e7d93a514367a8ba39b122)
+    
+    Conflicts:
+     qa/objectstore/bluestore-bitmap.yaml
+     Added bluestore-bitmap.yaml to Luminous as well
+
+commit dd5f9e53335892fa04a1d489de1020d5655e1e3f
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 5 15:01:02 2017 -0400
+
+    os/bluestore/BlueFS: prevent _compact_log_async reentry
+    
+    _should_compact_log uses new_log != nullptr to tell whether compaction is
+    already in progress, but we don't set it until we are midway through the
+    process.  Set it at the top of the method to prevent reentry.
+    
+    See 455cc6cea2e5e8bc1126b5047b6003fa583af1b5, which failed to implement
+    this properly.
+    
+    Fixes: http://tracker.ceph.com/issues/21250
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6bd9db304f906e9e4a07ca5ce07ffffdbc1678f0)
+
+commit 7989da7da6b9c2fe1d0e183d8c6d67b9bcb7c097
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Sep 6 16:14:58 2017 -0400
+
+    rbd: support iterating over metadata items when listing
+    
+    Fixes: http://tracker.ceph.com/issues/21179
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9c4f9856b034061aed052192b0ee09b9a9d09bc0)
+
+commit 6468ddf8fc17155229edd6e579db7f1c6405e27d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 5 16:26:52 2017 -0400
+
+    qa/suites/rbd: reduce monitor requirements for mirroring tests
+    
+    Fixes: http://tracker.ceph.com/issues/21251
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 27194c30c50009eaf6391a21837f73b312fe5de6)
+
+commit 0cef781d74bf387a8b29f2197e492579aba42389
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 5 16:20:26 2017 -0400
+
+    qa/suites/rbd: fixed cache override
+    
+    Fixes: http://tracker.ceph.com/issues/21251
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 98061bb3d7ce6309ddb04ea4d7e9d44a7ecd09c6)
+
+commit 8d6ec22730d6538f8bfce3fdc076d360f7cc0e6b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 5 15:59:47 2017 -0400
+
+    qa/suites/rbd: reduce OSD debug log levels
+    
+    The teuthology machines are periodically running out of space
+    due to the aggressive log settings.
+    
+    Fixes: http://tracker.ceph.com/issues/21251
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 763f1877ecda680e62a2b7eff5815bb6b7dd18f2)
+
+commit 40d46c01b77891f64129ad18575f82cb6cf00534
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 5 10:53:47 2017 -0400
+
+    cls/rbd: metadata_list not honoring max_return parameter
+    
+    Fixes: http://tracker.ceph.com/issues/21247
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0130b85732849da8bb3e96f4afb7c023c958aef9)
+
+commit 22208c8237913c5865b4d4cf40ddb083f2ffe48c
+Author: Pan Liu <wanjun.lp@alibaba-inc.com>
+Date:   Sun Sep 3 12:12:15 2017 +0800
+
+    rbd-nbd: fix generic option issue
+    
+    Fixes: http://tracker.ceph.com/issues/20426
+    Signed-off-by: Pan Liu <wanjun.lp@alibaba-inc.com>
+    (cherry picked from commit 16f64a4c01068603736af476f640268d0baa338d)
+
+commit c34be1841d041d318ded0b63e95eab79d63bd0ef
+Author: Pan Liu <wanjun.lp@alibaba-inc.com>
+Date:   Tue Aug 29 23:30:15 2017 +0800
+
+    rbd-nbd: lower down the log level when return -EINVAL in aio_callback.
+    
+    Signed-off-by: Pan Liu <wanjun.lp@alibaba-inc.com>
+    (cherry picked from commit dcd7e5bb82524bf111a6d80d9e0c050ae55b26a1)
+
+commit 3ac5343841805daa21a86be144da94c60a58590a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 5 12:11:45 2017 -0400
+
+    librbd: rename of non-existent image results in seg fault
+    
+    Fixes: http://tracker.ceph.com/issues/21248
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4a75ee43d3b108a9593a010160479fd62884eda8)
+
+commit 4f7c155c2eb241e87160db30de4e24e7d8931188
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Wed Jul 5 16:10:38 2017 -0700
+
+    rgw: add tail tag to track tail instance
+    
+    Use tail_tag instead of idtag. The tail_tag is modified when either
+    creating a new tail, or adding a refcount to the tail (e.g., when
+    copying object). When just modifying the object's head, tail_tag
+    should remain the same. The gc will then use the tail_tag (if
+    exists, idtag if not) as the refcount id.
+    
+    Fixes: http://tracker.ceph.com/issues/20234
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit f71a3e525d9a69fe537428f25930fb207a602afe)
+
+commit d769b077b1a6fa34bfddbef857ae543872df2f75
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 5 21:37:16 2017 -0400
+
+    test/cls_rbd: mask newer feature bits to support upgrade tests
+    
+    Fixes: http://tracker.ceph.com/issues/21217
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit aeb8f29d21625f3570e2f3e6dd26594d1a6df810)
+
+commit c9c168b0ea550fe13181e2e64b53cd8511794d22
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Tue Sep 5 11:45:53 2017 +0800
+
+    os/bluestore: add bluestore_prefer_deferred_size_hdd/ssd to tracked keys
+    
+    To make these two keys can be changed online:
+    
+    ./bin/ceph tell osd.* injectargs "--bluestore_prefer_deferred_size_hdd 0"
+    osd.0: bluestore_prefer_deferred_size_hdd = '0' (not observed, change may require restart)
+    osd.1: bluestore_prefer_deferred_size_hdd = '0' (not observed, change may require restart)
+    osd.2: bluestore_prefer_deferred_size_hdd = '0' (not observed, change may require restart)
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit cd1225634cf4d20e485f7c30d5a0438febc7ddcc)
+
+commit 7b3ab3252f9010590da1cc695919b70e09a3d5e8
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 5 22:25:03 2017 -0400
+
+    crush: fix fast rule lookup when uniform
+    
+    Older clients will search for the first rule with a matching ruleset,
+    type, and size.  The has_uniform_rules bool is only set if we have rule
+    ids and rulesets that line up, but we must also verify that the rest of the
+    mask matches or else we can get a different CRUSH mapping result because
+    the mask might not match and old clients will fail to find a rule and we
+    will find one.  We also can't just check the ruleset as the legacy clients
+    find the *first* (of potentially many) matching rules; hence we only do
+    the fast check if all rulesets == rule id.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f24095e0e9734531dbdbcd4bff1392c463188e8e)
+
+commit 81bf6cbfd4cdea04937bb04eb4902762114e5913
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 5 23:04:15 2017 -0400
+
+    os/bluestore: mark bluestore_allocator as DEV
+    
+    We don't want users changing this without a good reason.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 4b6a4a7ff3e82e015fc8268142d8c5c6437d089f
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 10 11:58:40 2017 -0400
+
+    os/bluestore: put StupidAllocator btree into bluestore_alloc mempool
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6f189d8ab3c549a5c5e01616310c6eb39119fbc1)
+
+commit b943e7c5844d818b0cbc51e0d88fa147c31a44e2
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 10 11:21:32 2017 -0400
+
+    os/bluestore: default 'bluefs_allocator = stupid'
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2b6eea80eacd65cbb1aa3702a3a3b642304c04f5)
+
+commit 3e06813fe92c0ad07ac033aab14d088136b996f9
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 8 12:14:07 2017 -0400
+
+    os/bluestore: default to bluestore_allocator = stupid
+    
+    Performance looks better, and code is simpler.
+    
+    Also fix config option annotations.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3750cf9cafabd3296c22b8b8dff35e44d18b9dd8)
+
+commit ac4cb6dd41adfc549d064756215421c33368dc86
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Aug 24 16:04:54 2017 +0800
+
+    osd/PG: discard msgs from down peers
+    
+    if a repop is replied after a replica goes down in a new osdmap, and
+    before the pg advances to this new osdmap, the repop replies before this
+    repop can be discarded by that replica OSD, because the primary resets the
+    connection to it when handling the new osdmap marking it down, and also
+    resets the messenger sesssion when the replica reconnects. to avoid the
+    out-of-order replies, the messages from that replica should be discarded.
+    
+    Fixes: http://tracker.ceph.com/issues/19605
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit e3fce6be44506168a7a138aab93f6a4d6776397b)
+
+commit 8a0d7aa073eb14fa3324594449968d2755535d6c
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Aug 7 07:52:42 2017 -0400
+
+    mon: rate limit on health check update logging
+    
+    Fixes: http://tracker.ceph.com/issues/20888
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 5b903c7b428c2c7024fdfc1dffacad0b1521b750)
+
+commit 2484a728e2694357de5e50dae2345ce6894d873b
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Sep 1 05:33:16 2017 -0400
+
+    mon: add helper for emitting logs by health status
+    
+    We were starting to repeat our little "if(status==HEALTH_ERR"
+    blocks in too many places.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 618a27868d65aa152ac67cce8057a75c4d61def9)
+
+commit 4c0073d20b72a30e67c97b1527d091d43c67a358
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 24 17:56:13 2017 -0400
+
+    osd: subscribe to new osdmap while waiting_for_healthy
+    
+    If we are sitting around waiting until we are able to ping our "up" peers,
+    we need to be sure that our notion of "up" is still correct and we're not
+    just stuck on an old, stale OSDMap.
+    
+    Fixes: http://tracker.ceph.com/issues/21121
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit fbafa659dae94faba435ca449ee5e77b51108b4b)
+
+commit 3806293b1ecc888f92fdc7f15ef3f25ac37ac29b
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Thu Aug 31 12:42:36 2017 +0800
+
+    os/bluestore: formatting nits
+    
+    So we don't output anything if csum is currently off:
+    
+    _do_alloc_write blob Blob(0x7fad7cc25950 blob([0x42a80000~10000] none/0x1)
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 22740cc9b3e489ae82b4b3071a17793c11abce79)
+
+commit ca667063b77445a2f45cdce04110b3affbdf6bed
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Thu Aug 31 11:42:37 2017 +0800
+
+    os/bluestore: don't re-initialize csum-setting for existing blobs
+    
+    The global checksum setting may change, e.g., from NONE to CRC32,
+    which can cause improper re-initialization of the csum-settings of
+    existing blobs(e.g., partial write/overwrite may turn out to shrink
+    'csum_data').
+    
+    We could develop some complicated solutions but for now let's not
+    bother since the above scenario is rare.
+    
+    Fixes: http://tracker.ceph.com/issues/21175
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 8ec4b4960f9b93ff6d85a02308ccda0bccadb599)
+
+commit e2e81403f8dd2329c0e99c8c8abbfa8351f0401e
+Author: w11979 <wang.wenfeng@h3c.com>
+Date:   Fri Sep 1 10:02:34 2017 -0400
+
+    osd: fix build_initial_pg_history
+    
+    We need to update our info about the previous interval in order to
+    detect interval changes properly.
+    
+    Fixes: http://tracker.ceph.com/issues/21203
+    
+    Signed-off-by: w11979 <wang.wenfeng@h3c.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit afdacc6ed76dd288fdbc0f99857d9ffba23c5132)
+
+commit d57399c9dab0be6905a90b743d6b87ca93700cc6
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 31 14:55:46 2017 -0400
+
+    qa/tasks/ceph_manager: revive osds before doing final rerr reset
+    
+    We assume below that rerrosd is up, but it may not be when we exit the
+    loop.
+    
+    Fixes: http://tracker.ceph.com/issues/21206
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 21027233b2a31b587e7caf95b1d85f5bcf0ae8ec)
+
+commit 927b19ff77b667d6d5b73b5024fd3007196d0002
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 31 16:43:39 2017 -0400
+
+    os/bluestore: separate finisher for deferred_try_submit
+    
+    Reusing finishers[0], which is used for completions back into the OSD,
+    is deadlock-prone: the OSD code might block trying to submit new IO or
+    while waiting for some other bluestore work to complete.
+    
+    Fixes: http://tracker.ceph.com/issues/21207
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2b6a9d1ec0778f66b6e9d9f216df21a00caeba3f)
+
+commit cac29c9618fc5ca8bfcc6a5b4b83ff51443654fa
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed Aug 23 12:13:24 2017 +0300
+
+    rgw-admin: require --yes-i-really-mean-it when using --inconsistent_index
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    Fixes: http://tracker.ceph.com/issues/20777
+    (cherry picked from commit 1cd7ef0eb9bde5a08f8c758ce57c9e298fabad8d)
+
+commit 91dd0fe479c63e0b907ce218fa9f262fa950364f
+Author: Zac Medico <zmedico@gmail.com>
+Date:   Sun Aug 27 05:25:01 2017 -0700
+
+    interval_set: optimize intersect_of for identical spans
+    
+    Optimize comparisons for identical spans of intervals.
+    When this patch is combined with the previous map insert
+    optimization, a benchmark using 400000 identical
+    intervals shows a 7 times performance improvement in
+    comparison to without the patches.
+    
+    Signed-off-by: Zac Medico <zmedico@gmail.com>
+    (cherry picked from commit b6a035666c2765f8895ee9991348dbc025613ed7)
+
+commit 83236f0c476bc7e36c11e934787e69c11283ec13
+Author: Zac Medico <zmedico@gmail.com>
+Date:   Fri Aug 25 09:41:07 2017 -0700
+
+    interval_set: optimize intersect_of insert operations
+    
+    Use the std::map insert method with hint iterator to optimize
+    inserts. This increases performance more than 3.5 times for
+    large numbers of intervals. This will help performance
+    especially in the PGPool::update method, where profiling data
+    has shown that intersection operations are a hot spot. The
+    following benchmark data is for 400000 intervals:
+    
+        4 +-+--+----+----+----+----+----+----+----+----+--+-+
+    P     +    +    +    +    +    +    +    +  *************
+    E     |                             ********            |
+    R 3.5 +-+                       ****                  +-+
+    F     |                   ******                        |
+    O     |                 **                              |
+    R   3 +-+           ****                              +-+
+    M     |          ***                                    |
+    A     |        **                                       |
+    N 2.5 +-+     *                                       +-+
+    C     |     **                                          |
+    E     |     *                                           |
+        2 +-+ **                                          +-+
+    R     |  **                                             |
+    A     | **                                              |
+    T 1.5 +**                                             +-+
+    I     |**                                               |
+    O     +*   +    +    +    +    +    +    +    +    +    +
+        1 +*+--+----+----+----+----+----+----+----+----+--+-+
+          0   0.1  0.2  0.3  0.4  0.5  0.6  0.7  0.8  0.9
+    
+                            SET SIZE RATIO
+    
+    The above chart was generated using benchmark results
+    from the following program:
+    
+    #include <iostream>
+    #include <sys/timeb.h>
+    #include "include/interval_set.h"
+    
+    int main(int argc, char *argv[])
+    {
+      const int interval_count = std::stoi(argv[1]);
+      const int interval_distance = 4;
+      const int interval_size = 2;
+      const int sample_count = 8;
+      const int max_offset = interval_count * interval_distance;
+      interval_set<int> a, b, intersection;
+    
+      for (int i = 0; i < max_offset; i+=interval_distance) {
+        a.insert(i, interval_size);
+      }
+    
+      for (int m = 1; m < 100; m++) {
+        float ratio = 1 / float(m);
+    
+        for (int i = 0; i < max_offset; i+=interval_distance*m) {
+          b.insert(i, interval_size);
+        }
+    
+        struct timeb start, end;
+        int ms = 0;
+        for (int i = 0; i < sample_count; i++) {
+          ftime(&start);
+          intersection.intersection_of(a, b);
+          ftime(&end);
+          ms += (int) (1000.0 * (end.time - start.time)
+            + (end.millitm - start.millitm));
+          intersection.clear();
+        }
+        b.clear();
+    
+        std::cout << ratio << "\t" << ms << std::endl << std::flush;
+      }
+    }
+    
+    Signed-off-by: Zac Medico <zmedico@gmail.com>
+    (cherry picked from commit 32bc0430f70b057d1bba623252e92ab9f279028d)
+
+commit 8e9b52ee884dce49385855eed571ece6cdc9e0d7
+Author: yanyx <yanyx@umcloud.com>
+Date:   Mon Aug 7 18:14:14 2017 +0800
+
+    pybind: fix chart becomes bigger when refresh
+    
+    Fixes: http://tracker.ceph.com/issues/20746
+    Signed-off-by: Yixing Yan <yanyx@umcloud.com>
+    (cherry picked from commit edf93427734ce156f5eaac7a25f4c16a92852b98)
+
+commit 63c9382eafb5d40441ce487c2430c98c214ce84a
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Aug 25 17:55:11 2017 -0400
+
+    rgw_file:  fix LRU lane lock in evict_block()
+    
+    Found by "Supriti Singh" <Supriti.Singh@suse.com>.
+    
+    Fixes http://tracker.ceph.com/issues/21141
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 3301596734adcbda1e8e76a742935efdeb7518fd)
+
+commit 24803b76c804fb1717a4c6ed4f93c63fbd4a8f19
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Aug 28 10:14:03 2017 -0400
+
+    qa: create_cache_pool no longer runs 'pool application enable'
+    
+    Fixes: http://tracker.ceph.com/issues/21155
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 054297450484ff99f95470905cd6d8986c26e2c4)
+
+commit ca7e48c62e6a18c5a627d5e71ed8016e1e41b9a1
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Aug 28 10:08:12 2017 -0400
+
+    mgr: fix crash in MonCommandCompletion
+    
+    Fixes: http://tracker.ceph.com/issues/21157
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 5fa61b2ed078a3cf333a3111d4f43584ce27ba9d)
+
+commit 0c75157a91506dc5edd3c4ae5eb95ac1a0c38aee
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Tue Aug 29 08:21:07 2017 +0800
+
+    mon/OSDMonitor: make 'osd crush rule rename' idempotent
+    
+    Fixes: http://tracker.ceph.com/issues/21162
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit cd2191c3f192986e20beaab84b643dfaf49901f7)
+
+commit b575d00073ccf3f84d84a220f542058471a58472
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Thu Aug 3 15:14:05 2017 +0200
+
+    rgw_lc: add support for optional filter argument and make ID optional
+    
+    Support Filter tag in Lifecycle XML similar to AWS S3, while S3 docs
+    mention that this tag is mandatory, older clients still default to
+    Prefix, and S3 itself seems to relaxed in enforcing the rule, this
+    implementation also follows a similar pattern. Filter optionally
+    supports filtering via (multiple) Object Tags, this is still a TODO. The
+    current implementation of object tags is still as an object xattr, and
+    since the LC processing still iterates over the bucket index which
+    currently doesn't have any info. on tags, this requires some thought
+    into for implementing without a larger performance penalty
+    
+    Fixes: http://tracker.ceph.com/issues/20872
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit 93a858392a2a3bc8c16369dd8c3f6845e99af404)
+
+commit ce2a181834cf60f9d03bbcf849ae72a740bb10a9
+Author: Jiaying Ren <jiaying.ren@umcloud.com>
+Date:   Thu Aug 10 10:57:13 2017 +0800
+
+    rgw: fix obj copied from remote gateway acl full_control issue
+    
+    This part of the code is commit in
+    ea3efca
+    
+    When copy a object to a remote gateway, the check processing which is
+    done by
+    "Bitwise And" between source object's acl elements and
+    grants_headers_def array elements
+    will be done to produce the target object's acl elements.
+    
+    So when the full_control is the first element of grants_headers_def, no
+    matter the
+    source object's acl element permission field is write or read, the
+    result of
+    bitwise and will always be true, then call grants_by_type_add_one_grant
+    with
+    check_perm which is full_control, all of the permission field of the
+    target object's
+    acl elements will be full_control.
+    
+    Fixes: http://tracker.ceph.com/issues/20658
+    
+    Signed-off-by: Enming Zhang <enming.zhang@umcloud.com>
+    (cherry picked from commit 6bb2ed24ad175005e7ebd187166bb8735761493c)
+
+commit 17626e01d832bac0068697947f32202d2f8cc13a
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Aug 10 11:10:09 2017 +0100
+
+    mgr/dashboard: fix FS status on old MDS daemons
+    
+    These may not have the version string.  We already handled
+    this in the equivalent code in the status module, so just
+    do the same thing in dashboard.
+    
+    Fixes: http://tracker.ceph.com/issues/20692
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 800662c2fa3f040992987dc5c635dfbc14f0a0ea)
+
+commit e171fa6a6dd77b2d3490e2d656de149beb5daaab
+Author: Aleksei Gutikov <aleksey.gutikov@synesis.ru>
+Date:   Mon Aug 21 17:04:45 2017 +0300
+
+    rgw: Prevent overflow of stats cached values
+    
+    Fixes: http://tracker.ceph.com/issues/20934
+    Signed-off-by: Aleksei Gutikov <aleksey.gutikov@synesis.ru>
+    (cherry picked from commit 634215eea1ddd4e4f5dc0066c4a2e745cfc20475)
+
+commit a656a38d285acb413d07c5b8b3d0ea90d491ed53
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Thu Aug 24 15:53:23 2017 +0800
+
+    osd/PrimaryLogPG: fix sparse read won't trigger repair correctly
+    
+    Normal reads instead of hole-reads should trigger auto-repair.
+    Also we shall quit on successfully triggering a repair, or else
+    we are at risk of doing the op multiple times.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 4ad25c346215dc2ddc4b1a43948b6fe01316edf2)
+
+commit 39b3b132f24c2e66b7d862ef5b553992dd109cf4
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Aug 21 16:56:20 2017 +0800
+
+    qa: add health whitelist for kcephfs suite
+    
+    Fixes: http://tracker.ceph.com/issues/20892
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit b10989209f41ddb5abe16f816ffd8a3ce118fd02)
+
+commit 00ed756ae866f2f4777c3994f75032824f3ee5ac
+Author: Dan van der Ster <daniel.vanderster@cern.ch>
+Date:   Thu Aug 10 11:33:46 2017 +0200
+
+    client: skip lookupname if writing to unlinked file
+    
+    When a client writes to an unlinked file, get_quota_root tries to
+    resolve the filename via an MDS lookupname op. The op always results in
+    -13 permission denied for path-restricted caps or -2 no such file or
+    directory otherwise. More importantly, the repeated lookupname ops slow
+    down buffered writes significantly.
+    
+    Don't do the lookupname for unlinked files; use the root_ancentor's
+    quota instead.
+    
+    Fixes: http://tracker.ceph.com/issues/20945
+    Backport: jewel, luminous
+    Signed-off-by: Dan van der Ster <daniel.vanderster@cern.ch>
+    (cherry picked from commit 1d617eee2e8c32f7300cb2cb8dd37b3714a4699f)
+
+commit e2cc9ef48a7f7a2014c211a3baa10e7ac1a1a8a7
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Wed Aug 9 21:24:49 2017 +0800
+
+    mon/OSDMonitor: "osd pool application ls" support
+    
+    It would be a pain if we have to call 'ceph osd dump --format=json-pretty'
+    to find out these each time...
+    
+    Demo output:
+    (1) ceph osd pool application get
+    {
+        "cephfs_data_b": {
+            "cephfs": {}
+        },
+        "cephfs_metadata_a": {
+            "cephfs": {}
+        },
+        "test_pool": {
+            "rbd": {
+                "test": "me"
+            }
+        }
+    }
+    
+    (2) ceph osd pool application get test_pool
+    {
+        "rbd": {
+            "test": "me"
+        }
+    }
+    
+    (3) ceph osd pool application get test_pool rbd
+    {
+        "test": "me"
+    }
+    
+    (4) ceph osd pool application get test_pool rbd test
+    me
+    
+    Fixes: http://tracker.ceph.com/issues/20976
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 46586b22882c0c08c099243e189f5a165063da0c)
+
+commit 25a651e100d5854d730f1218154838fa37868686
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Mon Aug 21 13:14:40 2017 -0700
+
+    qa: add chown test clearing S_ISGID and S_ISUID
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 7fd94abd0a2b682c1d7f01699962e27809c061cd)
+
+commit d9a8b3e797252114c930117e6311f924ef0588b3
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Wed Aug 16 10:48:21 2017 -0700
+
+    ceph-fuse: load supplementary groups by default
+    
+    Partially addresses POSIX test failures in [1] due to the config setting being
+    false by default.
+    
+    [1] https://bugzilla.redhat.com/show_bug.cgi?id=1480182#c6
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 08b077338cba1250c36232f91c708d189326bbb6)
+
+commit b9514d48471b40574a47ad433e3573674075e466
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Wed Aug 16 10:20:11 2017 -0700
+
+    client: clear suid/sgid bits on non-zero write
+    
+    According to [1], these bits should be cleared regardless of any exe bits on
+    the file. Also, add the required non-zero write check.
+    
+    [1] http://pubs.opengroup.org/onlinepubs/9699919799/functions/pwrite.html
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 24c9de5d11d5d1145da96a2af52dd22d81e0c89d)
+
+commit e41a8c25345020f47513c43269eb91e3ca6d0852
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Wed Aug 16 10:07:43 2017 -0700
+
+    client: add missing space in debug msg
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 3f806c72dd3b9b28761ee32f5e32001de8362028)
+
+commit f9eb24e74f948d85729432dbb440aa177789df85
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Wed Aug 16 09:52:36 2017 -0700
+
+    cephfs: clear suid/sgid if regular file is exe
+    
+    According to [1], the suid/sgid should be cleared if any of the executable bits
+    are set.
+    
+    Found this while experimenting for [2].
+    
+    [1] http://pubs.opengroup.org/onlinepubs/9699919799/functions/chown.html
+    [2] https://bugzilla.redhat.com/show_bug.cgi?id=1480182
+    
+    Fixes: http://tracker.ceph.com/issues/21004
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit d306d74a60e6d2c79aa4e9a840f3fc12ee8461bc)
+
+commit 469e50646c7d83e1f4e59ea90aeeefbd437976e8
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Wed Aug 16 09:48:09 2017 -0700
+
+    client: refactor clear set uid/gid if -1
+    
+    libfuse already does not set FUSE_SET_ATTR_UID if the chown uid is -1. However,
+    another libcephfs client could call setattr directly with -1 as the uid/gid so we
+    should handle that potential case.
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 6448d4927fb4cfc29392ff58da8775d68e3e875b)
+
+commit 18442e15854f7973c1b478befaff42f08b549505
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Mon Aug 21 13:02:44 2017 -0600
+
+    rpm: ceph-fuse requires fuse package
+    
+    The ceph-fuse(8) man page describes using the "fusermount" tool to mount
+    CephFS. This utility is in the "fuse" package. Prior to this change, the
+    ceph-fuse RPM only depends on "fuse-libs", not "fuse".
+    
+    The Debian packaging has always depended on the full "fuse" (previously
+    "fuse-utils") package, so we should make the RPM packaging do the same.
+    
+    Fixes: http://tracker.ceph.com/issues/21057
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 6fba9cc3f910442d3d09b41f01f454a39bfdbc48)
+
+commit daacbfd90fb4e127dc73142674eacf63d618cdaf
+Author: Zhu Shangzhong <zhu.shangzhong@zte.com.cn>
+Date:   Wed Aug 23 10:20:17 2017 +0800
+
+    client: add blank space between function name and inode number for log
+    
+    Fixes: http://tracker.ceph.com/issues/21069
+    
+    Signed-off-by: Zhu Shangzhong <zhu.shangzhong@zte.com.cn>
+    (cherry picked from commit 0cb4d379fdd2b869bd765e1fcb424a8f8a8f6a04)
+
+commit a1a3c07ba41c2edea2499d1b31baba310e6417e2
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri Jun 9 14:38:28 2017 +0200
+
+    selinux: Allow nvme devices
+    
+    This commit allows nvme devices which use a different label than
+    standard block devices.
+    
+    Fixes: http://tracker.ceph.com/issues/19200
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit a8af61c8da9028adc13f7aea611d5a5fee9dbfd1)
+
+commit ce434ac24f89e3cd3f51fbbe63b37f500913bc7d
+Author: songweibin <song.weibin@zte.com.cn>
+Date:   Thu Aug 17 11:44:43 2017 +0800
+
+    common: add new line break when get command descriptions
+    
+    Signed-off-by: songweibin <song.weibin@zte.com.cn>
+    (cherry picked from commit 19dd4203eb930ffb7a321cb617c580d99ceca9c1)
+
+commit 59a327e0a9a71802c61f8dddf8d217a5a7bfe2f6
+Author: songweibin <song.weibin@zte.com.cn>
+Date:   Wed Aug 9 14:26:32 2017 +0800
+
+    rbd: adding newline operation when only '--format' used
+    
+    Signed-off-by: songweibin <song.weibin@zte.com.cn>
+    (cherry picked from commit 992829d38cb89732f6994642c1bdaa2448e610b7)
+
+commit 94fcad155dd2b1d35264e628309b67977146dfff
+Author: songweibin <song.weibin@zte.com.cn>
+Date:   Tue Aug 1 19:48:04 2017 +0800
+
+    rbd: '--pretty-format' should return error if '--format' not used
+    
+    Signed-off-by: songweibin <song.weibin@zte.com.cn>
+    (cherry picked from commit 47e7549826b55905e7f298717fedf0e8f220940a)
+    
+    Conflicts:
+        src/test/cli/rbd/help.t (master has several new commands compared to
+                                luminous: omit these)
+
+commit a74b3267eef23f17eb90fff180db03c0f2fa61a6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Aug 17 18:39:05 2017 -0400
+
+    test/librbd: TestMirroringWatcher unit tests should ignore duplicates
+    
+    Fixes: http://tracker.ceph.com/issues/21029
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5d3d5ea5aca5807b09050fe8ad2245c4963a0f34)
+
+commit 5f0ba1353b6fe0eec5ed67489a9d1f7f87466d23
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Aug 1 15:03:57 2017 -0400
+
+    rbd: switched from legacy to new-style configuration options
+    
+    Fixes: http://tracker.ceph.com/issues/20737
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7dc82bc08cbfabd424dcd0b83ed618d28db5fe8e)
+
+commit 3f0ee2aa39c7b48380617fe33871c7db5989abcb
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 9 16:23:59 2016 -0800
+
+    rgw: metadata full sync streams metadata keys
+    
+    We don't read the entire list of all metadata entries in a single
+    request.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 4a509a669125a9a2d74de11cce555df2547103d1)
+
+commit a8c156b4761edea882c7cabb981108d2d127f442
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 9 15:43:16 2016 -0800
+
+    rgw: rest list metadata supports max-entries and marker
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit a45fd78d08f4d862c273a13abf787a93ce8e0906)
+
+commit ebd09a4c7aaa8788fe6fbc5ba80b0ea3308ca3e2
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 9 14:34:46 2016 -0800
+
+    rgw-admin: add marker to metadata list
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit ae600c2c3f765f409cb16557870ceb91d6714760)
+
+commit 8d1b349f860a7eaaae0ea281e2f5451ce8fd06f3
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 9 14:30:57 2016 -0800
+
+    rgw: add marker to metadata api keys listing
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 84ece81bc3590b08e5565c5b96bcbb4cb97a4100)
+
+commit 298366deb4473370467e3f345dddadc2d5a87b71
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Fri Dec 9 14:22:57 2016 -0800
+
+    rgw: marker for raw pool listing
+    
+    We now have a marker (cursor) that can be used for raw pool
+    listing.
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit a24b8aeb1e376ce9e5e50535d8ce1a287c99fa90)
+
+commit 5d229d18ec89fbe49a95cb5004be1a18b6376c65
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Aug 16 09:41:43 2017 -0400
+
+    test/rgw: add kms encryption key for teuthology
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit f27ebabe551f62241d1f5b55add9ec6b5d782073)
+
+commit d0c2adc1b71cea08a40805a65cb397b0dc5bedad
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Aug 16 09:01:28 2017 -0400
+
+    test/rgw: add multisite test_encrypted_object_sync
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit acb953a9ed83e5e760d9222c8f9ede6c8c73a341)
+
+commit cc9b83c2a759356142436eae2fc005cc8ce684a9
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jul 26 16:26:25 2017 -0400
+
+    rgw: rgwx-skip-decrypt header bypasses decryption filter on GetObj
+    
+    Fixes: http://tracker.ceph.com/issues/20668
+    Fixes: http://tracker.ceph.com/issues/20671
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit adf7a333017f52682c7319420804433b7d3851a0)
+
+commit b13a1d5ffa00030170bfb0bb944d94f1b011afb3
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jul 26 16:24:13 2017 -0400
+
+    rgw: add skip_decrypt to RGWRESTConn::get_obj()
+    
+    this adds a rgwx-skip-decrypt header to requests from
+    RGWRados::fetch_remote_obj()
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 2a7a5be4b41641cb17fb9127b84ce717ca55d3fe)
+
+commit df4131c7a9bc6b61aadf315d33b0258b52b0fbed
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jul 26 16:23:26 2017 -0400
+
+    rgw: name arguments to RGWRESTConn::get_obj()
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 9bd1aa7facf94eed41c4235722490c91c9dbf399)
+
+commit 07a15984c665b5f4720aa6c69ce9f44a8216d859
+Author: fang.yuxiang <fang.yuxiang@eisoo.com>
+Date:   Wed Aug 9 11:48:53 2017 +0800
+
+    rgw: break sending data-log list infinitely
+    
+    radosgw send data-log list infinitely when opposite end trimmed the data-log and in quiescence.
+    Fixes: http://tracker.ceph.com/issues/20951
+    
+    Signed-off-by: fang yuxiang fang.yuxiang@eisoo.com
+    (cherry picked from commit 40db1fdbc7b24d56b2c7b463238e6b96f2d4bbd9)
+
+commit 7c3a82225e44305fcc5648b0f78719c0331ecca5
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Fri Aug 4 20:01:32 2017 -0400
+
+    Test bytes_sent bugs.
+    
+    Rearrange logic to make it easier to measure accumulation.
+    Instrument the boto request/response loop to count bytes in and out.
+    Accumulate byte counts in usage like structure.
+    Compare actual usage reported by ceph against local usage measured.
+    Report and assert if there are any short-comings.
+    Remove zone placement rule that was newly added at end: tests should be rerunable.
+    
+    Nit: the logic to wait for "delete_obj" is not quite right.
+    
+    Fixes: http://tracker.ceph.com/issues/19870
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    (cherry picked from commit a45ab45f74042617ec382f3d50f4d45d883042f5)
+
+commit 8c9805921edac7257a135873e663aea5c4316d1e
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Sun Jun 18 18:18:39 2017 -0400
+
+    Fix bytes_sent bugs.
+    
+    log bytes sent/received.
+    add cct to bufferingfilter
+    add cct to RGWRestfulIO
+    AccountingFilter - save cct for debugging output
+    implement AccountingFilter::complete_request() - account for bytes reported here.
+    BufferingFilter<T>::complete_request() - ignore counts from send_content_length() complete_header();
+    
+    Code quality note:
+    this patch makes "cct" available for a lot of newly added debug
+    statements.  The debug statements are mostly not very useful (and should
+    go away in the future) - *But* the "cct" logic should be redone and
+    incorporated into some base class (such RestfulClient) so that it is
+    possible to easily add in debug statements such as these in the future.
+    
+    Fixes: http://tracker.ceph.com/issues/19870
+    Signed-off-by: Marcus Watts <mwatts@redhat.com>
+    (cherry picked from commit 0172aab860abe324923a392baf51f956a65b64a5)
+
+commit 823585311c41729309809690a7df5cb83c4ece12
+Merge: 85bc024bd9 e1253e2c24
+Author: Sage Weil <sage@newdream.net>
+Date:   Sat Sep 2 13:47:29 2017 -0500
+
+    Merge pull request #17383 from liewegas/wip-luminous-mon-bits-luminous
+    
+    mon: final luminous mon bits
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit e1253e2c248446cb1ef57777e1796d376f2ae97e
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Sep 2 14:46:47 2017 -0400
+
+    PendingReleaseNotes: must upgrade to 12.2.1 before mimic
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 85bc024bd9496c3cdfd64c5facb841a048179c50
+Merge: 3e35d292bd 6979dd31ed
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri Sep 1 14:37:38 2017 -0700
+
+    Merge pull request #17429 from ceph/wip-luminous-x-luminous
+    
+    qa: add upgrade/luminous-x suite
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 6979dd31ede42d7ab204b13b9346662a049b343a
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Fri Sep 1 14:29:41 2017 -0700
+
+    Initial check in luminous-x suite
+    
+    same from master https://github.com/ceph/ceph/pull/17160
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 3e35d292bdd7d931deb204f5878c0a5dea02b27d
+Merge: fa996769fc 55a519056c
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Fri Sep 1 07:14:15 2017 -0400
+
+    Merge pull request #17408 from ceph/luminous-bz1485011
+    
+    luminous: ceph-volume Persist non-lv devices for journals
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit 55a519056ce2dd2c7e155f6f8a06ee95c428ad72
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Aug 31 10:27:39 2017 -0400
+
+    ceph-volume tests add new ceph_* ansible dev variables required (centos)
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d3abfd42ef8e558a1bbb31e24d0ab3c44ee7f4c4)
+
+commit 00c28d7b3fbc4cf3d04e8de76abe284ed98451aa
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Aug 30 14:46:54 2017 -0400
+
+    ceph-volume tests add new ceph_* ansible dev variables required (xenial)
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit ebb48c58123a1cfcfc6eccdf9f6c2b3a720f1233)
+
+commit 22fbc9d8d0ef3803166c7fcf627f7b41847ad7e3
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Aug 30 14:12:33 2017 -0400
+
+    ceph-volume util create a disk utility for blkid operations
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a19e1288e8ae044f41dff9a1770126aa540cc729)
+
+commit 5ce9597d3579e5f4ee6523b59c42f6b3ef2ece82
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Aug 30 14:11:52 2017 -0400
+
+    ceph-volume lvm.prepare store the blkid uuid of a partition journal
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 5160c5398bbf107bcc826696664c503aa52d0020)
+
+commit 3e8f2cc2a57351c990836792256af899bc233b1a
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Aug 30 14:05:22 2017 -0400
+
+    ceph-volume lvm.activate use the partuuid of a partition to link the journal
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a1080f833b35c5de30d7b577deabf0a0e82fbbf2)
+
+commit a971b0e0c694da4c2cd5eda8d540f4a3965cd80c
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Aug 29 13:34:21 2017 -0400
+
+    ceph-volume tests update the ansible version for functional/tox.ini
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 007655afa3def6788e07d2db10c7cd9cb4ab45b7)
+
+commit 9b20da46761c92f3e39e44383bc0b3e3625ee26c
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Aug 29 13:25:28 2017 -0400
+
+    ceph-volume tests add pv* related unit tests
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 7c22743c9aa1306242fc09ccae5dedf94e025cc4)
+
+commit 459ed5ad1efe0af070e92267657ac4d6a6a15c28
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Aug 29 11:31:56 2017 -0400
+
+    ceph-volume lvm.api include lv_uuid as output fields
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d85656da2531471e11cb90fcc74570c9c2b804e0)
+
+commit 9b04a4cff09e7aaab6fde5d456854847acc4c061
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Aug 29 09:38:08 2017 -0400
+
+    ceph-volume lvm.activate always update the link to the journal
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 6a1de41838cb9dbbdc4a8f59eb07e4340ccb9680)
+
+commit 9c1e84ab1fdaad47cb0e27453b19de8c899a9700
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Aug 29 09:26:35 2017 -0400
+
+    ceph-volume lvm.activate retrieve the journal uuid if journal is a device
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a368fa1ad6d6265c5094eacbe7e72f642b43559d)
+
+commit 7da501a250fb551edb396dde1f19d9737a5812c7
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Aug 28 17:33:39 2017 -0400
+
+    ceph-volume exceptions create a specifc error for multiple pvs
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 73315c49115dbb45f3bb6c9e46955b333602cec2)
+
+commit 38e0caa1bbb2471808b2895604ede20d8a6615fe
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Aug 28 16:52:33 2017 -0400
+
+    ceph-volume lvm.prepare make a journal a pv, use uuids always
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 913cef2d01b91e12c16e42d268043886b62c72fa)
+
+commit 237e42c4ea21a48f04bc092f9c3f9c0d407ec298
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Aug 28 16:51:11 2017 -0400
+
+    ceph-volume lvm.api create the PVolumes class and helpers
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit f2bc125d980485c7b456e0193ec9c66ea0acbe54)
+
+commit 337edc3cd1145cf85dfa3745a8e0ab7c5d26cbf3
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Aug 24 19:28:12 2017 -0400
+
+    ceph-volume lvm.api create the PVolume class
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 1725a5f3589ba2faddc9ce633997e5563fc66a01)
+
+commit eda292b40a2c221d98b8ddf740752cc8dd857cf9
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Aug 24 16:37:48 2017 -0400
+
+    ceph-volume lvm.api add a helper to create pvs
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 00f3fa401871f3108558fea71f6ac03660d00ba1)
+
+commit 4f021d396271c59492f38fe20f7d739be862c15d
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Aug 30 17:22:33 2017 -0400
+
+    qa/suites/rados/thrash/d-luminous/at-end: check for purged_snapdirs
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 807c4c834136e5a3d20b31d50e770430695473ff
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 29 00:01:19 2017 -0400
+
+    mon: set purged_snapdirs OSDMap flag once snapsets have all converted
+    
+    This makes it easier to test whether the upgrade + conversion has
+    completed.  In particular, mimic+ will be able to simply test for this
+    flag without waiting for complete PG stats to know whether it is safe to
+    upgrade beyond luminous.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c3cbaad2a1a4df68fe39394e07c2f3ac9818b9c8)
+
+commit 2f3e2e5578089c6c7498358c166689c697d82020
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Aug 28 15:21:44 2017 -0400
+
+    mon: CEPH_MON_FEATURE_INCOMPAT_LUMINOUS
+    
+    - set this once we have a luminous quorum
+    - require luminous if we have this set locally
+    - include this on mkfs
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5b6c451e3fdaf0785b197fdfacf1c782b932bec9)
+
+commit fa996769fcf1d1eb135c00d98b7b48e370f210d0
+Merge: 935faacb34 dc76083ee4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Aug 30 16:39:09 2017 -0400
+
+    Merge pull request #17381 from dillaman/ritz303-wip-doc-20437
+    
+    luminous: doc/rbd: iSCSI Gateway Documentation
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit dc76083ee451bbaef3e67720d46a6e268b877b60
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Aug 8 19:14:57 2017 -0400
+
+    doc: tweaks for the RBD iSCSI configuration documentation
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b0abab17e3fa96ca973791a3afe077406d066035)
+
+commit 3717fd060b815f8b4f0c2ec3225b28b6bca33601
+Author: Aron Gunn <ritz_303@yahoo.com>
+Date:   Wed Jul 12 17:11:25 2017 -0500
+
+    doc/rbd: Replaced numbered steps with a # for dynamic numbering
+    
+    Fixes: http://tracker.ceph.com/issues/20437
+    
+    Signed-off-by: Aron Gunn <agunn@redhat.com>
+    (cherry picked from commit f109031cb5c8150f042f9754f16bfa5216411eda)
+
+commit 846093201207f1544b386d69c38573b21c69f4db
+Author: Aron Gunn <ritz_303@yahoo.com>
+Date:   Thu Jul 6 17:22:38 2017 -0500
+
+    doc/rbd: Updated the content based on engineering feedback
+    
+    Fixes: http://tracker.ceph.com/issues/20437
+    
+    Signed-off-by: Aron Gunn <agunn@redhat.com>
+    (cherry picked from commit 2a59c82b3d4bbb13bc82cf62fce1d9cb5ba7e6be)
+
+commit 08fffcd3562d299cdb7ba499012a4ea19250433f
+Author: Aron Gunn <ritz_303@yahoo.com>
+Date:   Thu Jul 6 17:22:02 2017 -0500
+
+    doc/rbd: Updated the content based on engineering feedback
+    
+    Fixes: http://tracker.ceph.com/issues/20437
+    
+    Signed-off-by: Aron Gunn <agunn@redhat.com>
+    (cherry picked from commit 1d4f5b458d127065d92652c323c73704cf775754)
+
+commit cee1621875257782d1077a513baf7d264a782795
+Author: Aron Gunn <ritz_303@yahoo.com>
+Date:   Thu Jul 6 17:20:40 2017 -0500
+
+    doc/rbd: Updated the content based on engineering feedback
+    
+    Fixes: http://tracker.ceph.com/issues/20437
+    
+    Signed-off-by: Aron Gunn <agunn@redhat.com>
+    (cherry picked from commit 441820d95890d95ea79bae55afb3ca3b1a8a36e3)
+
+commit bf6086377298ca762d0d901934c6c4ec88b0f456
+Author: Aron Gunn <ritz_303@yahoo.com>
+Date:   Thu Jul 6 17:19:01 2017 -0500
+
+    doc/rbd: Updated the requirements based on engineering feedback
+    
+    Fixes: http://tracker.ceph.com/issues/20437
+    
+    Signed-off-by: Aron Gunn <agunn@redhat.com>
+    (cherry picked from commit c57266bf82c554b93653489a4b355f53b4b86424)
+
+commit 75a3c6d4f11dedee222dc7d07ab1f7a8fe18db14
+Author: Aron Gunn <ritz_303@yahoo.com>
+Date:   Thu Jul 6 12:35:38 2017 -0500
+
+    doc/rbd: Added the iSCSI gateway requirements content
+    
+    Fixes: http://tracker.ceph.com/issues/20437
+    
+    Signed-off-by: Aron Gunn <agunn@redhat.com>
+    (cherry picked from commit 9335742140a2b4e8b408560c600a490b2749bd9d)
+
+commit e05635e22894baf37e115a263b483bf251285aad
+Author: Aron Gunn <ritz_303@yahoo.com>
+Date:   Thu Jul 6 12:34:49 2017 -0500
+
+    doc/rbd: Added the iSCSI gateway overview content
+    
+    Fixes: http://tracker.ceph.com/issues/20437
+    
+    Signed-off-by: Aron Gunn <agunn@redhat.com>
+    (cherry picked from commit d022034bb3a6e1b00cf185cae0b5fa3f4b617a38)
+
+commit 0cd40caa68355fa55b6b9d0fd57c45ae804cfd36
+Author: Aron Gunn <ritz_303@yahoo.com>
+Date:   Thu Jul 6 12:34:10 2017 -0500
+
+    doc/rbd: Added the iSCSI gateway 'gwtop' monitoring content
+    
+    Fixes: http://tracker.ceph.com/issues/20437
+    
+    Signed-off-by: Aron Gunn <agunn@redhat.com>
+    (cherry picked from commit 7e8233429ba158c1a4cfe6e58a7ce24ffe8759c9)
+
+commit 3ef7639370939e1d087693c7dda5c819fd707e23
+Author: Aron Gunn <ritz_303@yahoo.com>
+Date:   Thu Jul 6 12:32:53 2017 -0500
+
+    doc/rbd: Added the iSCSI targets, Ansible and CLI docs
+    
+    Fixes: http://tracker.ceph.com/issues/20437
+    
+    Signed-off-by: Aron Gunn <agunn@redhat.com>
+    (cherry picked from commit cbf654c702e7b7a8e7023c4707c4ea38d584be3e)
+
+commit 1d277117d3aa347177927ff85ce314c2d8233498
+Author: Aron Gunn <ritz_303@yahoo.com>
+Date:   Thu Jul 6 12:31:39 2017 -0500
+
+    doc/rbd: Added the iSCSI initiators, RHEL, Windows, and ESX docs
+    
+    Fixes: http://tracker.ceph.com/issues/20437
+    
+    Signed-off-by: Aron Gunn <agunn@redhat.com>
+    (cherry picked from commit f0e177fa3d8807cfa6ae17f8a0dc759a045623f9)
+
+commit ad0b8eb0944275e023ddfc9140bd32886545f78c
+Author: Aron Gunn <ritz_303@yahoo.com>
+Date:   Thu Jul 6 12:28:06 2017 -0500
+
+    doc/rbd: Updated the "toctree" adding the iSCSI gateway overview doc
+    
+    Fixes: http://tracker.ceph.com/issues/20437
+    
+    Signed-off-by: Aron Gunn <agunn@redhat.com>
+    (cherry picked from commit 4ef8426bff9a32e00093f7c5d2b6b9620252293b)
+
+commit 745f1748769bdef44635c4918deae106bbc28a1f
+Author: Aron Gunn <ritz_303@yahoo.com>
+Date:   Thu Jul 6 12:23:50 2017 -0500
+
+    doc/rbd: Added the unmodified iSCSI initiator images for ESX and Windows 2016
+    
+    Fixes: http://tracker.ceph.com/issues/20437
+    
+    Signed-off-by: Aron Gunn <agunn@redhat.com>
+    (cherry picked from commit 2fd61f8de1587d2bfbbb7e15c952a29566b390a1)
+
+commit 33dc4e65b2862fbadd7eba504247144d4c710a7c
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Aug 24 18:03:36 2017 -0400
+
+    osd/PGLog: write only changed dup entries
+    
+    Writing all of the dup entries whenever one is changed causes a large
+    regression in performance.
+    
+    Instead, keep track of ranges that need to be cleared and written
+    after log merging (dirty_{to,from}_dups) and the earliest dup entry we
+    haven't written yet during normal operation (write_from_dups). This
+    parallels the way we track unpersisted log entries.
+    
+    Check that the correct set of dups is persisted by adding a roundtrip
+    through memstore to each merge_log unit test.
+    
+    Fixes: http://tracker.ceph.com/issues/21026
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 75af72f758aacc1260b8d828d0b8bb7786597fdf)
+
+commit 4c5db08e6600b47cb4e2a3b940ab008bc9b1cb72
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Aug 24 18:34:48 2017 -0400
+
+    Revert "osd: adjust osd_min_pg_log_entries"
+    
+    This reverts commit 1c4df03394312fe67f36448613d8b54cb1a0e2c9.
+    
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 4c8fd9978369448e28355d62b4d1eef3685ea63b)
+
+commit 935faacb342ca2fc264d5a48085e7779757ab667
+Merge: bcf89707d6 8f0945a0f3
+Author: Sage Weil <sage@newdream.net>
+Date:   Wed Aug 30 08:28:33 2017 -0500
+
+    Merge pull request #17359 from tchaikov/wip-luminous-ceph-release-stable
+    
+    luminous: ceph_release: luminous is now 'stable' (12.2.x)
+
+commit 8f0945a0f39e7da629fa94f406bc1314e1b39fc8
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Aug 30 12:23:42 2017 +0800
+
+    ceph_release: luminous is now 'stable' (12.2.x)
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+
+commit 2a1ae18fa71fa7463389b65245eee3279afade54
+Author: chenliuzhong <liuzhong.chen@easystack.cn>
+Date:   Tue Aug 29 02:28:01 2017 -0400
+
+    os/bluestore:reshard-fix wrong shard length
+    
+    change fault_range parameter from 'needs_reshard_end' to 'needs_reshard_end-needs_reshard_begin'
+    be given needs_reshard_end may be out of bounds
+    
+    Signed-off-by: chenliuzhong <liuzhong.chen@easystack.cn>
+    (cherry picked from commit 23bce6a9504955d7faf352242d88d26d4fe7ac96)
+
+commit bcf89707d6c3c3086f001d08c754095b9e7a85f1
+Merge: f726fee338 32ce2a3ae5
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 29 10:48:19 2017 -0400
+
+    Merge tag 'v12.2.0' into luminous
+    
+    v12.2.0
+
+commit 2c7fbc463cf5a981e5966a4a8b7d2a0d1eff3e95
+Author: huangjun <huangjun@xsky.com>
+Date:   Mon Aug 28 22:52:37 2017 +0800
+
+    mon: use ceph_clock_now if message is self-generated
+    
+    Signed-off-by: huangjun <huangjun@xsky.com>
+    (cherry picked from commit 56e8031dac7ee65d54e831e7e372e8771fa0f94a)
+
+commit 43ffb7c897f0fc3b407d09f19954865fad168e03
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Tue Aug 29 09:33:51 2017 +0800
+
+    mon/OSDMonitor: make 'osd crush class rename' idempotent
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 2ee80aead88c90388871ee65d4ed31a2fa47f532)
+
+commit f726fee338e66356f62f9e6d0435df8c94b00cfd
+Merge: 10a5e1911b b713cc831d
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Tue Aug 29 21:33:58 2017 +0800
+
+    Merge pull request #17332 from tchaikov/wip-luminous-pr-17269
+    
+    luminous: mon: fix two stray legacy get_health() callers
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 10a5e1911bf0114a7563ad634b7387fefd90caad
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Mon Aug 28 16:30:20 2017 +0000
+
+    12.2.0
+
+commit 9cd689934677b2b28b67a864a1a9dbe482824adb
+Author: yanyx <yanyx@umcloud.com>
+Date:   Thu Aug 3 14:13:07 2017 +0800
+
+    mgr/dashboard: fix rbd's pool sub menu position
+    
+    Signed-off-by: Yixing Yan <yanyx@umcloud.com>
+    (cherry picked from commit 4e1d539f7cbebdbb0fb29175694413deaa2aaddb)
+
+commit b713cc831da238844a6e42e80e93cb31ee060aad
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Aug 25 18:30:12 2017 -0400
+
+    mon: fix two stray legacy get_health() callers
+    
+    These need to call the new method when appropriate.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 69038a18070d0cd47dc857020cce115514beaa3d)
+
+commit 09e661dcb02db872d79427dfab9a9868bbb93750
+Merge: 9df9e82338 34bafa6134
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon Aug 28 21:41:56 2017 -0500
+
+    Merge pull request #17327 from tchaikov/wip-luminous-pr-17326
+    
+    luminous: doc: fix order of options in osd new
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 34bafa6134e1fc6b0f1e7b0eb43192bc29917769
+Author: Neha Ojha <nojha@redhat.com>
+Date:   Mon Aug 28 15:59:19 2017 -0700
+
+    doc: fix order of options in osd new
+    
+    Signed-off-by: Neha Ojha <nojha@redhat.com>
+    (cherry picked from commit e47463028921f0414e2f64c467f74cf37a1de085)
diff --git a/doc/changelog/v12.2.2.txt b/doc/changelog/v12.2.2.txt
new file mode 100644
index 000000000..7f63ca6e3
--- /dev/null
+++ b/doc/changelog/v12.2.2.txt
@@ -0,0 +1,9540 @@
+commit cf0baeeeeba3b47f9427c6c97e2144b094b7e5ba
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Thu Nov 30 14:59:27 2017 +0000
+
+    12.2.2
+
+commit 83684b91a3c6b31419114b83fc22106146885fb6
+Merge: 6d9f2161b2 b5d02ac0fd
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Thu Nov 30 22:26:46 2017 +0800
+
+    Merge pull request #19240 from tchaikov/wip-22266-luminous
+    
+    luminous: tools/ceph_monstore_tool: rebuild initial mgrmap also
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek@suse.com>
+
+commit 6d9f2161b269b19b0301348dd9b17660420624d5
+Merge: 0b4c3b8f1e b02075a963
+Author: Gregory Farnum <greg@gregs42.com>
+Date:   Wed Nov 29 16:25:14 2017 -0500
+
+    Merge pull request #19230 from tchaikov/wip-22247-luminous
+    
+    luminous: mon/PGMap: Fix %USED calculation
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 0b4c3b8f1ed72c3a0053ed372e1c5c9b0d48de59
+Merge: a57edd4f1a eb99a7ee36
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Wed Nov 29 10:33:34 2017 -0600
+
+    Merge pull request #19239 from ceph/luminous-bz1518264
+    
+    luminous: ceph-volume correctly fallback to bluestore when no objectstore is specified
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit b5d02ac0fd60c26417ad33dc6efcf80bff503fdd
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Nov 29 20:21:33 2017 +0800
+
+    tools/ceph_monstore_tool: rebuild initial mgrmap also
+    
+    Fixes: http://tracker.ceph.com/issues/22266
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit f63d1da4afa3463730ecbc0be29df6375b79fa8f)
+
+commit eb99a7ee36d541939f589a674d9890b14e017dbe
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 28 09:22:19 2017 -0500
+
+    ceph-volume lvm.activate correct detection and fallback of objectstore flags
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit f0ac2dfdbc0a05cff0a5edbedca91274885b7870)
+
+commit 10dbbb0a6e09c6181e81b04765cd368963753bb8
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 28 09:21:34 2017 -0500
+
+    ceph-volume tests.devices.lvm verify objectstore flags in activate
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 8d22856d6c95ac3b45918765b382cae507a8d1a4)
+
+commit a57edd4f1a47f08c0f93d7a3ff57c376b72c9a8a
+Merge: 1b0a5660f7 ee02b5c037
+Author: Sage Weil <sage@newdream.net>
+Date:   Wed Nov 29 08:43:07 2017 -0600
+
+    Merge pull request #19217 from liewegas/wip-p2p-app-warning-luminous
+    
+    upgrade/jewel-x/parallel: debug fuse
+
+commit b02075a963c24c6e1b24d1b076275dcce5350a03
+Author: Xiaoxi Chen <xiaoxchen@ebay.com>
+Date:   Sun Nov 26 22:51:58 2017 -0700
+
+    mon/PGMap: Fix %USED calculation bug.
+    
+    Previous code forgot to multiple raw_used_ratio to calculate
+    used byte.
+    
+    Fixes: http://tracker.ceph.com/issues/22247
+    Signed-off-by: Xiaoxi Chen <xiaoxchen@ebay.com>
+    (cherry picked from commit d10c6c26f9ef7372e2c95da79d23b07ce5f4e0e5)
+
+commit ee02b5c037c67c708335745898328e990ecacc90
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Nov 28 16:48:36 2017 -0600
+
+    qa/suites/upgrade/jewel-x/point-to-point: whitelist more stuff
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit d2a284c782fed3bec4ed38c898351826bca0a8bc
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Nov 28 15:02:18 2017 -0600
+
+    qa/suites/upgrade/jewel-x/parallel: only mount ceph-fuse once
+    
+    Otherwise we do 4 mounts for no good reason.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit b7c6d9799cbfba3cff7bee5fd1de309a8299d911
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Nov 28 15:00:05 2017 -0600
+
+    upgrade/jewel-x/parallel: debug fuse
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 1b0a5660f793714e8cb470c35e084a141135cd8c
+Merge: 4ed747dccf e50af8bb04
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Nov 28 14:48:08 2017 -0600
+
+    Merge pull request #19216 from liewegas/wip-p2p-app-warning-luminous
+    
+    qa/suites/upgrade/jewel-x/point-to-point: fix suppression of pool app warning
+
+commit e50af8bb04bd37bd0e28ddc7935e29bcdc106b05
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Nov 28 14:46:02 2017 -0600
+
+    qa/suites/upgrade/jewel-x/point-to-point: fix suppression of pool app warning
+    
+    It's generated on the mgr.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 4ed747dccfaae15a9a807d3f3db2fc44b8b21fda
+Merge: acb02717f6 6e1e33a88d
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Tue Nov 28 23:47:17 2017 +0800
+
+    Merge pull request #19205 from tchaikov/wip-22136-luminous
+    
+    luminous: qa/ceph-disk: enlarge the simulated SCSI disk
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit acb02717f6e96f96d4128bbebd946238d3c79291
+Merge: 55f5d7d1f1 bb03f6b20b
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Nov 28 14:03:01 2017 +0100
+
+    Merge pull request #19152 from smithfarm/wip-22235-luminous
+    
+    luminous: ceph-disk flake8 test fails on very old, and very new, versions of flake8
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 6e1e33a88d038d5b6a469d7fb4e52182b888010f
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Nov 28 14:42:31 2017 +0800
+
+    qa/ceph-disk: enlarge the simulated SCSI disk
+    
+    100MB will be allocated for journal, and the remaining 100MB is for data
+    device. taking the inode into consideration, there will be approximately
+    87988 kB available for the activated OSD. and it will complain with a
+    "nearfull" state.
+    
+    Fixes: http://tracker.ceph.com/issues/22136
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b3c159e9fb8c31d0acd75f0702080f18959f672e)
+
+commit 55f5d7d1f19696af981007c63a7503b2e5d0a5a8
+Merge: 3bd0007d5f ad2393b8b8
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Tue Nov 28 03:23:09 2017 +0100
+
+    Merge pull request #19173 from smithfarm/wip-22251-luminous
+    
+    luminous: build/ops: macros expanding in spec file comment
+    
+    Reviewed-by: David Disseldorp <ddiss@suse.com>
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 3bd0007d5f1c991a80ff80fc2b7bb983ceee971e
+Merge: 7ce2013dfd 5633a58824
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Nov 27 10:23:57 2017 -0800
+
+    Merge pull request #19179 from ceph/wip-yuriw-BP-19175-luminous
+    
+    qa/suites/rados/rest: move rest_test from qa/suites/rest/
+    
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit 5633a58824f6248b79af74780411746cb7518813
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Nov 27 10:11:51 2017 -0600
+
+    qa/suites/rados/rest: move rest_test from qa/suites/rest/
+    
+    ...and add more health whitelists.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ddb274669ca16b95c452b256e58a2830af10c73d)
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 7ce2013dfda6f06ac47878fd4635ac0862bfbbbd
+Merge: ecec65906a 61e4de133d
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Nov 27 09:20:18 2017 -0800
+
+    Merge pull request #19177 from liewegas/wip-jewel-x-p2p
+    
+    qa/suites/upgrade/jewel-x/point-to-point: skip ec tests when mons may be old
+    
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit 61e4de133d9fa1eddcd858152b7828515e0b280f
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Nov 27 10:28:16 2017 -0600
+
+    qa/suites/upgrade/jewel-x/point-to-point: skip ec tests when mons may be old
+    
+    Early point release mons don't handle legacy ruleset-* ec profiles, new
+    ones do.  Skip the ec tests that may trigger this when we are doing a
+    workload that races with mon upgrades.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit ad2393b8b8dc71a8099fa87924dcb6936a27fd84
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Thu Aug 17 09:19:38 2017 -0600
+
+    rpm: rm macros in comments
+    
+    rpm expands all macros in a .spec file, even those in comments. Drop the
+    percent signs so rpm will not expand these.
+    
+    This change silences rpmlint's warning about macros in comments.
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit 5c1e9f55d3a144c93c9c18b031db3cdc320561ef)
+
+commit bb03f6b20b4963ad5d51c1c6f45f19bc7eaf220b
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Nov 21 11:36:02 2017 +0100
+
+    tests: ceph-disk: ignore E722 in flake8 test
+    
+    Very old, and very new, versions of flake8 treat E722 as an error:
+    
+    flake8 runtests: commands[0] | flake8 --ignore=H105,H405,E127 ceph_disk tests
+    ceph_disk/main.py:1575:9: E722 do not use bare except'
+    ceph_disk/main.py:1582:9: E722 do not use bare except'
+    ceph_disk/main.py:3252:5: E722 do not use bare except'
+    ceph_disk/main.py:3288:21: E722 do not use bare except'
+    ceph_disk/main.py:3296:17: E722 do not use bare except'
+    ceph_disk/main.py:4358:5: E722 do not use bare except'
+    tests/test_main.py:26:1: E722 do not use bare except'
+    ERROR: InvocationError: '/opt/j/ws/mkck/src/ceph-disk/.tox/flake8/bin/flake8 --ignore=H105,H405,E127 ceph_disk tests'
+    
+    Fixes: http://tracker.ceph.com/issues/22207
+    References: https://gitlab.com/pycqa/flake8/issues/361
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 3600cd7bfdca76485d2998c5da1a0dd25816d1ce)
+
+commit ecec65906af60c16fc99f669c1591bb821305a64
+Merge: 613634c1da aac7a85a3a
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Wed Nov 22 13:36:58 2017 +0800
+
+    Merge pull request #19080 from smithfarm/wip-22212-luminous
+    
+    luminous: tests: ceph-disk: silence deprecation warnings
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit aac7a85a3ae9cce38987ce3f51be4c6cb63c8dd2
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Nov 21 21:47:30 2017 +0800
+
+    qa/workunits: silence py warnings for ceph-disk tests
+    
+    ceph-disk now prints "depreacted" warning message when it starts. but
+    the tests parses its stdout and stderr for a json string. so we need to
+    silence the warnings for the tests.
+    
+    Fixes: http://tracker.ceph.com/issues/22154
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d44334f31704487ec3574738e75145872d9932cf)
+
+commit 613634c1da4cea975d6e1da7d258600c454c9400
+Merge: 5a469c0376 94839deb2f
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Mon Nov 20 09:24:41 2017 -0600
+
+    Merge pull request #18989 from ceph/wip-luminous-rm22154
+    
+    luminous: ceph-disk create deprecation warnings
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit 5a469c0376e299e5c60d7d9b3c16640515b11f26
+Merge: 3532f233f6 37d2726751
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Mon Nov 20 22:55:06 2017 +0800
+
+    Merge pull request #19025 from tchaikov/wip-pr-19024-luminous
+    
+    luminous: qa/tasks: prolong revive_osd() timeout to 6 min
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 3532f233f691ac22c32c4533a677b95d0e4c6273
+Merge: 5e519aebe7 0f7332e925
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon Nov 20 07:23:30 2017 -0600
+
+    Merge pull request #18945 from liewegas/wip-22128
+    
+    mon/OSDMonitor: add option to fix up ruleset-* to crush-* for ec profiles
+    
+    Reviewed-by: Joao Eduardo Luis <joao@suse.de>
+
+commit 37d2726751c05a0b5a3c1b739bce49995cc72445
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Nov 20 13:40:56 2017 +0800
+
+    qa/tasks: prolong revive_osd() timeout to 6 min
+    
+    see also #17902
+    
+    Fixes: http://tracker.ceph.com/issues/21474
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 749bbda07522114b99b17a4a01cfcf019520285d)
+
+commit 5e519aebe718bd16d52e974beba8e3bf97b04ae8
+Merge: 9abf091336 96c981ce6b
+Author: Sage Weil <sage@newdream.net>
+Date:   Sat Nov 18 20:34:50 2017 -0600
+
+    Merge pull request #18983 from pdvian/wip-22150-luminous
+    
+    luminous: os/bluestore: fix SharedBlob unregistration
+
+commit 0f7332e9259b320777770f0312233a31103bfeda
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 15 08:55:33 2017 -0600
+
+    mon/OSDMonitor: add option to fix up ruleset-* to crush-* for ec profiles
+    
+    The jewel->luminous upgrade test will fail if we finish the upgrade while
+    a workload setting old-style ec profiles is running.  Add option to
+    automatically fix them up.  Warn to the cluster log when this happens.
+    
+    For now, enable this option to ease upgrades and whitelist the warning.
+    
+    Only include this option in luminous so that we implicitly sunset this
+    compatibility kludge immediately.
+    
+    Fixes: http://tracker.ceph.com/issues/22128
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 9abf0913367a1ad8cf076c79bf3adace8c526853
+Merge: c0e032c153 975229c367
+Author: Sage Weil <sage@newdream.net>
+Date:   Sat Nov 18 09:44:46 2017 -0600
+
+    Merge pull request #18751 from shinobu-x/wip-21701-luminous
+    
+    luminous: ceph-kvstore-tool does not call bluestore's umount when exit
+
+commit c0e032c153a36c00109970395fe42cc2013fdb9a
+Merge: 81b768ab03 4d4aee14a4
+Author: Sage Weil <sage@newdream.net>
+Date:   Sat Nov 18 09:44:19 2017 -0600
+
+    Merge pull request #18750 from shinobu-x/wip-21702-luminous
+    
+    luminous: BlueStore::umount will crash when the BlueStore is opened by start_kv_only()
+
+commit 81b768ab036e66d3f4d36ddc31febc4bbcb143fb
+Merge: f02d7109c5 0fe463b5e1
+Author: Sage Weil <sage@newdream.net>
+Date:   Sat Nov 18 09:41:37 2017 -0600
+
+    Merge pull request #18860 from liewegas/wip-filestore-rocksdb-compression-luminous
+    
+    luminous: os/filestore: disable rocksdb compression
+    
+    Reviewed-by: Douglas Fuller <dfuller@redhat.com>
+
+commit 94839deb2f32e8059206ad48ad711dd2baf2fef5
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 17 13:10:29 2017 -0500
+
+    doc/ceph-volume create a migration section to help users coming from ceph-disk
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit e433efe51481ec63bc9753a0f446fe2b12b5bb6a)
+
+commit b8bf0d047868054135592188c7ebe186181310c5
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 17 13:01:14 2017 -0500
+
+    ceph-disk add deprecation warnings in favor of ceph-volume
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    
+    Fixes: http://tracker.ceph.com/issues/22154
+    (cherry picked from commit c9da92e1ec8d60dc40d86630dab95e4fec4d0d30)
+
+commit f02d7109c5f00cd2165082c4336b3623352be7c5
+Merge: 9515da2777 ebbe033936
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Fri Nov 17 12:19:36 2017 -0500
+
+    Merge pull request #18973 from ceph/backport-18924
+    
+    luminous: ceph-volume: allow using a device or partition for `lvm --data`
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit ebbe033936c424a0f6ba2d102fc271d0b0e9578d
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 17 08:41:24 2017 -0500
+
+    ceph-volume tests.devices.lvm prepare isn't bluestore specific anymore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 2f40dbac386be631ac97e2450117c218d9bf02f9)
+
+commit 96c981ce6b00520a94385228bcf70a90c5ae8ff9
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Nov 7 22:05:10 2017 -0600
+
+    os/bluestore: fix SharedBlob unregistration
+    
+    We use the SharedBlobSet remove() in three cases:
+    
+    - from SharedBlob::put(), we try to remove ourselves from the set, but
+      have to deal with a racing lookup, so the removal is conditional on
+      nref still being 0.
+    - from split_cache(), we move the SharedBlob to another collection
+    - from make_blob_unshared(), we remove the entry when we clear the sbid.
+    
+    The problem is that the condtiional remove() (for the first case) was being
+    used for all three cases, and in the second two cases nref is always != 0,
+    so it doesn't actually happen.  This can lead to a crash during cache
+    shutdown.
+    
+    Fix by making two variants: remove() that is unconditional, and
+    try_remove() that is conditional.
+    
+    Set the sb->coll pointer after because remove() asserts the parent matches
+    where we are unregistering.
+    
+    Fixes: http://tracker.ceph.com/issues/22039
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 78a465b90b1c5187cfec9f25eb58b3bf617ca39b)
+
+commit 9515da277737ad81d48f5c032f627f3cb919f746
+Merge: 5c9b93dadd d19b740359
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Nov 16 13:40:32 2017 -0800
+
+    Merge pull request #18957 from tchaikov/wip-pr-18808-luminous
+    
+    luminous: qa: do not wait for down/out osd for pg convergence
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit ad4668b65cb3e32f1b3afc5ec767004f166856e6
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Nov 16 08:50:34 2017 -0500
+
+    ceph-volume tests.functional create a common playbook directory for setups
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit def89a411f2f4bdb23c4bbd3b71c1310a564f07d)
+
+commit d25a04114c98aa1449809248e8b87ecd1124af3d
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 14 11:01:48 2017 -0500
+
+    doc/ceph-volume lvm prepare can accept devices for --data
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 9bfab348b2ff59c0b939a19c22d4a06bc5236b96)
+    
+    Conflicts:
+            doc/ceph-volume/lvm/prepare.rst
+
+commit 7f7d62a1848e297153f88a8e476585596114e1d5
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 14 08:42:27 2017 -0500
+
+    ceph-volume tests.functional add setup playbook for xenial filestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 210c1e1d4770dde47f0008cadaaec35deb729272)
+
+commit 210e1b3bf507654167349411c5cc85641b1d8680
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 14 08:42:02 2017 -0500
+
+    ceph-volume tests.functional add data device and journal to xenial filestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d0754e25177b9bf3bce695b7991e99f49312d91c)
+
+commit 211895ac0906747bc448b2326ae5c62f3e6ee4a1
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 14 08:40:37 2017 -0500
+
+    ceph-volume tests.functional add setup playbook for xenial bluestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d0f5623963f58fc4d5727a79d8377b2191b0f9dc)
+
+commit 398a1365a12de95bc9810b89a9aeab9040bb07b8
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 14 08:40:18 2017 -0500
+
+    ceph-volume tests.functional add raw data device to xenial bluestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit fbcfb522981cd4039cfe617609c5f160f0b05c52)
+
+commit 0d082d0dbb43cc418bf0786b1f708968050ae114
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 14 08:38:23 2017 -0500
+
+    ceph-volume tests.functional tox.ini addition of setup playbook for partitions
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d074b534643dde37d732b34c36c5089e41848d6d)
+
+commit a167556eaacb444c41299cd28f1efd4327da4747
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 14 08:37:17 2017 -0500
+
+    ceph-volume tests.functional add raw data device to centos7 bluestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 2d416a849ad7b33dde1386da09c81249e6c2299d)
+
+commit 12e180a84f1456dfe7ab940255c6c47ee8ebddf0
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 14 08:36:36 2017 -0500
+
+    ceph-volume tests.functional add setup playbook for centos7 bluestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 161fd373d2f11f64466927e5d615d039822e2ab5)
+
+commit f06ea77590336d439235bd9acc6028b854e8c181
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 14 08:35:59 2017 -0500
+
+    ceph-volume tests.functional add setup playbook for centos7 filestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit b3bb147298f8816651f231f77bc91a3a1b50d96f)
+
+commit f10bfbeefe06e59629a4e9d904e8035bf060920b
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 14 08:35:42 2017 -0500
+
+    ceph-volume tests.functional add raw device to centos7 filestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit c5b8d72570b24dfd0d2f8c1cc8f13812c6c11a69)
+
+commit 968006a271dc82f3dcada4e5de93ecb0e23238c6
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Nov 13 14:29:32 2017 -0500
+
+    ceph-volume lvm.common update --data flag to reflect device acceptance
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 4df7ea8d83cfa02c8864d043b5d5fb2f5409dc96)
+
+commit c8f64af027689a1fb1bb916e6d479519a12b764e
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Nov 13 14:28:32 2017 -0500
+
+    ceph-volume lvm.prepare add example of raw device or partition in help menu
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 6fb8d566aac0e0fa081e6500852855ad177f784f)
+
+commit f930934159edd947024eef1a33e15a30467cb332
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Nov 13 14:26:44 2017 -0500
+
+    ceph-volume lvm.prepare allow partition or raw device as input for --data in filestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 89236ee3f81e50b9b059c10f6d6e7ce048c06474)
+
+commit d19b7403595bef1c3ba73f4a5fb83f5ef2dd3f81
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Nov 8 14:13:54 2017 +0800
+
+    qa: do not wait for down/out osd for pg convergence
+    
+    that osd is not invovlved in the PG state changes.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 7f549af459b40c5bb51a24a3a443ed7a7116915e)
+
+commit 5c9b93dadd8f05b1a146e3a19f7480ae4019815f
+Merge: cc41c393b3 2f380b32da
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Nov 14 08:38:35 2017 -0800
+
+    Merge pull request #18908 from smithfarm/wip-luminous-fixup
+    
+    qa/suites/fs/basic_functional/clusters: more osds
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+
+commit cc41c393b3a76a5f1a18cbda45acc01e2d6fdfc2
+Merge: d69f42e1af 806e49aadb
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Tue Nov 14 11:06:00 2017 -0500
+
+    Merge pull request #18910 from ceph/backport-18882
+    
+    luminous: ceph-volume: add functional tests for simple, rearrange lvm tests
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit 806e49aadb0d832a4bf46fcd681d889d93158232
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 10 13:18:07 2017 -0500
+
+    ceph-volume tests.functional update lvm/tox.ini paths after moving it
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 7d787849556788961155534039886aedfcdb2a88)
+
+commit 724ad01afc0af79fc7f808aae819ff9dc0971b33
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 10 13:14:55 2017 -0500
+
+    ceph-volume tests.functional move top level tox.ini to lvm/tox.ini
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 9b14b96d3bf6c8326cfc91e657161ffab3b8c2b4)
+
+commit 754c237597cc91af4bfe7199f891a9c2a6938ad4
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 10 13:12:51 2017 -0500
+
+    ceph-volume tests.functional move xenial to lvm/xenial
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 00d576d266b63080129e58482bd6fba62534ee97)
+
+commit 762c5f1cc10266d5efa62628c0686b3f8d696bff
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 10 13:11:58 2017 -0500
+
+    ceph-volume tests.functional lvm move xenial/bluestore to xenial/bluestore/create
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d5ec7d1c58141f4593ed9d1e4501c9f3be6d9755)
+
+commit 5b1bde338fbb1e689e3e1305e91f5450ef175edd
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 10 13:09:48 2017 -0500
+
+    ceph-volume tests.functional lvm move xenial/create to xenial/filestore/create
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit c504b8781b3a31475280a032cd75c9e1eaa6d8c9)
+
+commit 5a00604f27acad8cb368495f841ce76cce383414
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 10 13:02:05 2017 -0500
+
+    ceph-volume tests.functional move centos7 to lvm/centos7
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit dc1da3fd0aeafe59a82343fb16fe1fcea66ae3eb)
+
+commit 57d64a86d0758d174c1ca0b432a43194cbe6162c
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 10 12:58:27 2017 -0500
+
+    ceph-volume tests.functional lvm move bluestore to bluestore/create
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit aec1a80a3e1e61720f494980711808bfee783a03)
+
+commit 94a9f3f43d9c3a2ac08be4025e65c47c3767d2c1
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 10 12:54:01 2017 -0500
+
+    ceph-volume tests.functional lvm move create to filestore/create
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d6905f3f620d4c733835e21a275d4127cdcb769b)
+
+commit bc134c20a5409bd6f966d205dd7ba850ff28fc91
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 10 12:42:19 2017 -0500
+
+    ceph-volume tests.functional simple xenial bluestore scenario
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 8f0fdc975951a2a8af31bbaa3abd2b5d807a0add)
+
+commit 004efbfbe4d1eb56190e5e7497a9613776494fae
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 10 12:40:18 2017 -0500
+
+    ceph-volume tests.functional simple xenial filestore scenario
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 7452e1b6f9b82c2ef62bae19c632e81039638ba5)
+
+commit b51744f26f3c6db4d2927a2ac1733f0b80e44010
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 10 12:30:05 2017 -0500
+
+    ceph-volume tests.functional simple tox.ini update for xenial scnearios
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 76ed6a8f2ecab7c58def0453ffed9d8a997e0d5b)
+
+commit 19983b9ba5ed45c8e6fac967fb702fd586132c35
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 10 09:12:09 2017 -0500
+
+    ceph-volume tests.functional simple tox.ini addition for bluestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 02154e445b87a8fdcabbd699a2532833764140b5)
+
+commit 040e7913b028c54aeb9acd854610f6a4f112465d
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 10 09:11:39 2017 -0500
+
+    ceph-volume tests.functional add initial files for simple bluestore activate
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 3320f28e799e1d77d1ea69be9ced603bb69cc5fa)
+
+commit 23f6c3e779c743bc1302d1ad542201434ff9431f
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Nov 9 16:55:30 2017 -0500
+
+    ceph-volume tests.functional add initial files for simple filestore activate
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 9d18bae52ad77be0e64ef105c33a019446b6f1cf)
+
+commit 8f425b678f6c947a8c83f121be4fa46151fd6664
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Nov 9 16:07:53 2017 -0500
+
+    ceph-volume tests.functional create a specific test.yml for simple activate
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 37bde0d3b37a87c456438b0edc0cb9e2dcd39120)
+
+commit 6f1fff00f5210840fe143aac7506ce3773600f05
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Nov 9 15:40:51 2017 -0500
+
+    ceph-volume tests.functional create a separate tox.ini environ for
+    `simple`
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 89757ad12e6938fe2bf997e0f21a5ffac082e30b)
+
+commit d69f42e1afaf8759de1544b3d381671a01cfb55a
+Merge: a7c8c8101d 90976ef6b9
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Mon Nov 13 10:55:24 2017 -0600
+
+    Merge pull request #18907 from ceph/luminous-wip-bz1498200
+    
+    luminous: ceph-volume: support GPT and other deployed OSDs
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit 2f380b32da1b6294b6b0f60622b09b6517f9afdd
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 9 13:44:51 2017 -0500
+
+    qa/suites/fs/basic_functional/clusters: more osds
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 998122c324047c96c16d60d7fbb3a21f4dd63c56)
+
+commit 90976ef6b9815b68df620aac46f2c04117766f99
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Nov 9 10:50:43 2017 -0500
+
+    doc/ceph-volume include the top level systemd.rst in the toctree
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit e2f281c115c15e820588b478214c3600ff97470f)
+
+commit 9a8a00d3a17f03d2b1b3a92662f7288f7d18fe7a
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Nov 9 06:49:49 2017 -0500
+
+    ceph-volume simple.activate add --file in help docstring as an example
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit ff60592897b63581ebce372b3f6ffa0f10d2fe5a)
+
+commit 34194b8d788d91b4f7c969be5f0099a279bc61bf
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Nov 9 06:46:59 2017 -0500
+
+    doc/ceph-volume activate has a --file option for json files
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 29913951b71d68b3f3f9b9fa076f309c0252da36)
+
+commit 582262b57bcc959d1698636000d7a804cab9c611
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 8 17:23:41 2017 -0500
+
+    ceph-volume simple.activate enable the osd when activating
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 366fb8c43af54e9c92f4e95d9a0582c1fc378dbc)
+
+commit a32e72cdb0efb55d185488fb02f44055f9d3dffe
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 8 15:25:05 2017 -0500
+
+    ceph-volume systemd.systemctl always force symlink when masking ceph-disk
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit cb470622fd4f13fafe9cb2f38fb25fd1fbd8d89c)
+
+commit 3af40c35f3cf42be722f9d905cb2d0ba2baf1b1b
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 8 15:21:10 2017 -0500
+
+    ceph-volume simple.activate use the contents of the JSON file, fallback to CLI args
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit c954c913395557bcd89f25b0496c4428e6109494)
+
+commit 161e9657d985aa4ceffad2f56e18edd78ca2a81f
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 8 14:50:16 2017 -0500
+
+    ceph-volume simple.activate only check for missing id and uuid if file is not passed in
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 826c2709dd5ab0fbf6eb419a735d15e718a7712c)
+
+commit 585072afc814c6ac87fd64f15c561c51c1e55070
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 8 14:28:12 2017 -0500
+
+    ceph-volume simple.activate allow to pass a file to activate
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit bd3498f3eabf07637cb936e6c036d1d8d75415b0)
+
+commit 01266c193588066425b44fea4c1729692f999692
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 8 12:49:31 2017 -0500
+
+    ceph-volume tests.util add tests for OSDPath validator
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d7fc26ab4e2cebe485647510119ea300d47db184)
+
+commit 56fdc1c973b41d855563e459d23682990e8940bf
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 8 11:56:36 2017 -0500
+
+    ceph-volume systemd.systemctl masking ceph-disk needs to be done by directly symlinking to dev/null
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit cda47c247b929401a7dce3f1ef70371ea80eeceb)
+
+commit 0a0c289de605b6b74b762b2a0a4ed0f3a04de8ad
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 8 09:47:12 2017 -0500
+
+    ceph-volume systemd.systemctl masking needs to use systemctl mask, not disable
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit bad2f62b34243addc854abf691854b886aecdc19)
+
+commit 32913844b36285aae40f9786e5cb035bfca22f2d
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 8 09:46:20 2017 -0500
+
+    ceph-volume simple.scan if mounted device cannot be detected, raise an error
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 6b2fc4c0ba63e4bd51774ea09cc6542738128621)
+
+commit fe73229fb123fdb45f4ef743d8584f6748318868
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 8 09:17:52 2017 -0500
+
+    ceph-volume simple.activate remove str formatting from logging
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 230faf04a185c0b95c8ad497ddd5501dc70c1409)
+
+commit 50c19ef335305215a636595506eda1b5d16f3687
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 8 09:11:41 2017 -0500
+
+    ceph-volume util.arg_validators always convert the path to an absolute path
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit b5acf4d720b5395ae4ccbf007c94c99610e40089)
+
+commit d27c35c972abba2b230e1cff1fa35437500dcdf6
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 8 09:10:47 2017 -0500
+
+    ceph-volume util.arg_validators require superuser privs to check paths
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 376dad74159532f901d0044f95330fc02f93bd25)
+
+commit 0832ddc52496cd81aa7c2e1e85252b50d2f33044
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Nov 8 09:09:06 2017 -0500
+
+    ceph-volume simple.scan skip directories to scan
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 6bc1e9ee0b8d7bc475f218ae3dd8c6c7fe5a983a)
+
+commit d6071cb75ba1a702c551d073f032336560654e97
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 7 15:08:13 2017 -0500
+
+    ceph-volume tests.devices.simple add activate CLI unit tests
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit cde0d517563b537319d06c758b93e421cdabe59d)
+
+commit 2bff2e7fdbba37280791d3bc9a363e47aa26ab53
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 7 15:07:44 2017 -0500
+
+    ceph-volume tests.devices add help menu unit test
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit dcc7c72196a327f9501045f025761b228b5379df)
+
+commit ec8624005082913faf89a9b001d4abeb7c27a589
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 7 13:42:45 2017 -0500
+
+    doc/ceph-volume systemd describe the interaction with 'simple' activation
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 7071eeaeb5d77c6ab6e7a24f1b630cb0cf8a8e7c)
+
+commit aacd22681d1e7f5a604025b2019b195041151b3f
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 7 13:42:11 2017 -0500
+
+    doc/ceph-volume index initial description of simple
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit f9ee51f9feadb6d0f91b76a2b2ba8a24f1523a77)
+
+commit e9527e65d5da503107e21dc5b695f287c8529049
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 7 13:41:12 2017 -0500
+
+    doc/ceph-volume lvm add back a system.rst with just the lvm portion of it
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit f872cd29b93e02b77172f7b382c4746af9ba845d)
+
+commit ef027a737989dca83c0e5810935bad2280146356
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 7 13:40:44 2017 -0500
+
+    doc/ceph-volume make systemd.rst generic enough to be related to other sub-commands
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit cf4e32a0dc3ab4f55fe7857b51afbc63a638ec66)
+
+commit a83b746eaeece5896770b40949804316c29f6bc1
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 7 13:39:52 2017 -0500
+
+    doc/ceph-volume move lvm/systemd.rst to top level, making it generic
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 0672dd608d4601a936d703ea5ba6edc14a161d8e)
+
+commit bc492490e15d3ea91654cd4dabfb3115f835b9ac
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 7 11:52:17 2017 -0500
+
+    doc/ceph-volume simple.activate initial take on documenting activation
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 80c796bd1c43125f5678710b494b03c486b3fce6)
+
+commit 6301c83097729749fc712846954a3c0f89647f07
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 7 08:35:57 2017 -0500
+
+    doc/ceph-volume lvm update systemd anchor to reflect lvm parent
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 53420a4756a5e4fa6b0f2aa74d6ad1e2919f1954)
+
+commit 5fe4122bcfdc9c8e4f69079e45c907148eccdb7d
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 7 08:35:43 2017 -0500
+
+    doc/ceph-volume lvm.activate update systemd anchor to reflect lvm parent
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 7fc0e7ebcae06f28f6ed08c0db14075ada34e8a7)
+
+commit c98886a99914748d73d0f695808a74850bc9e66e
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Nov 7 07:56:35 2017 -0500
+
+    doc/ceph-volume lvm.systemd update anchor to reflect lvm parent
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 01a8b7e6a45f6fd85bf53d3cab15acc81d0bae1b)
+
+commit 68f89b52d608c3946be9ad92e733fe9472bd35f0
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Nov 6 15:53:39 2017 -0500
+
+    doc/ceph-volume document the scan functionality
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit b30a3eaf17a20a1273724b2cfcaf0a7b36eb08f6)
+
+commit 19d0ca22908fcd3fbe83d39c1b708c5818b34f47
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Nov 6 14:17:20 2017 -0500
+
+    doc/ceph-volume update the index to include the simple sub command and files
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d7f85c64e4b409ea95e30ccb5a1388278843c478)
+
+commit 61c90bacc562e1e2d65b767dca97b08973832131
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Nov 6 11:15:28 2017 -0500
+
+    ceph-volume tests.simple add tests for trigger parsing from systemd
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 0e2620234e82abf85274b3c08b39cdf76430ce48)
+
+commit 80ae5b1fb024827cceb0594a60e3361fd4a1e7a3
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Nov 6 10:36:14 2017 -0500
+
+    ceph-volume tests.simple create etc_path scan tests
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 90ebc4e1ffa7b3273787d708bb2c05180530b5d1)
+
+commit b9ad514b45e0233a17834e19a232bca4467a8140
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Nov 6 10:06:02 2017 -0500
+
+    ceph-volume tests.simple add checks for get_contents in scan
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 647bfe0883d2bb05b4ff7f99894075df3e73d2d5)
+
+commit c9308af91a322f92ab7d1f45a423b2a94f64c429
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Nov 6 09:43:58 2017 -0500
+
+    ceph-volume tests.util add binary verification tests
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 827aa57557561355dd4068e26801b5f1ed62f295)
+
+commit c94f52dffe0ea4b48c9fb316e8c53fd06a81b870
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Nov 6 09:42:53 2017 -0500
+
+    ceph-volume tests create a fixture for temporary files
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 23da3445718c01f71e2a5748fb4fc1c5fdc85ca5)
+
+commit bbb25df806609083efd377befb44d6d3bef3338b
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 3 14:36:38 2017 -0400
+
+    ceph-volume systemd.systemctl helpers to mask ceph-disk
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 3da23fa4caea8c9dc91da3eee5b5acc1860bf242)
+
+commit c971321430fd1a4de7c6fa2687ecf35a5b3ec71c
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 3 14:36:13 2017 -0400
+
+    ceph-volume devices include simple at the module import level
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 027a09b3ea6bac416ff632a69a53393badd51708)
+
+commit 1615d947db50ac37e4ef554acc6debe8fd870962
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 3 14:35:45 2017 -0400
+
+    ceph-volume main add simple to the sub-commands
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 94579c77ffe4d5ba2d457ad3981a2b9ed232a90f)
+
+commit abb77a8d9824eef3bd9c75d17f8df101440c25d1
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 3 14:35:03 2017 -0400
+
+    ceph-volume util add an OSDPath argument validator
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a826fdf4a2ed9ff704fb823d05d3cb706ba89e38)
+
+commit ff2a1833da385d6c0c9087e98cb2b93925b06cab
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 3 14:34:39 2017 -0400
+
+    ceph-volume simple.system add a helper to check if a file is a binary or not
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 7bf8676045b4f966b60cdadf4d5af1c89fca69ae)
+
+commit 84d2a15ce9f1a045ed67f79fa7d1dc99dc1e4b8f
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 3 14:34:06 2017 -0400
+
+    ceph-volume util.system add a context manager for temporary mounting/unmounting
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 6394cdd41943e3cda6867ecc59ab835cb2151707)
+
+commit 3d8ff08c2d46017749e7ecaa4e4d49dda4d55d74
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 3 14:32:42 2017 -0400
+
+    ceph-volume simple.scan initial take on directory/device scanning
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 0e54b9be621cc0969a3bbc972d1870080ccdae77)
+
+commit d701569a6486d7ccccf14b5eadf702ad07db4821
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 3 14:27:23 2017 -0400
+
+    ceph-volume simple.trigger enable systemd triggering for simple setups
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit cbc0c0f775e96a5b0ab5d5bc141acebe4cc48373)
+
+commit 76fe659a19a8a450b4c61f6d2b56e03dc46746ae
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 3 14:26:13 2017 -0400
+
+    ceph-volume simple.activate initial take on activation
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 5a97374605408d18e9be29bcc051ae7b913d1675)
+
+commit 206c7551435983b4cbdaece61b737e4a1f265dbd
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 3 14:23:47 2017 -0400
+
+    ceph-volume simple add __init__ for the module
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 285e5bdd28a42d98418ca90ad0cdec6bd87849db)
+
+commit 8a5edb9a2acc3f0b5477f09edae4f97f5ec7f5b5
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Nov 3 14:23:29 2017 -0400
+
+    ceph-volume simple create the initial, main module
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit f050502425a1993ae8edfdb5785b24129cffeeba)
+
+commit 0fe463b5e1fe62e03b498c32aa708be8c10eb0b8
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Nov 9 20:10:59 2017 -0600
+
+    os/filestore: disable rocksdb compression
+    
+    Experience working with customer escalations suggests that disabling
+    compression improves performance, and the storage overhead is generally
+    not a concern for the metadata and omap data we are storing.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b878ead071b328e5fe7309a2368383e67679e9f7)
+    
+    
+    # Conflicts:
+    #       src/common/options.cc
+
+commit a7c8c8101d4b78b4d6e437620b2c1a38cd752c3f
+Merge: 1071fdcf73 f56de67980
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Thu Nov 9 18:08:27 2017 +0100
+
+    Merge pull request #18801 from linuxbox2/luminous-rgw-readdir2
+    
+    rgw_file: implement variant offset readdir processing -- pullup 2 of 2
+    
+    Reviewed-By: Abhishek Lekshmanan <abhishek@suse.com>
+
+commit 1071fdcf73faa387d0df18489ab7b0359a0c0afb
+Merge: 4db48ca836 8cacd2a568
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Thu Nov 9 22:36:27 2017 +0800
+
+    Merge pull request #18734 from shinobu-x/wip-21648-luminous
+    
+    luminous: mgr[zabbix] float division by zero
+    
+    Reviewed-by: Wido den Hollander <wido@widodh.nl>
+
+commit 4db48ca8369d1dae021131c93a1c8484512aa99b
+Merge: 9cd31820f2 d858570755
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Thu Nov 9 22:35:44 2017 +0800
+
+    Merge pull request #18851 from tchaikov/wip-18759-pr-luminous
+    
+    mon/LogMonitor: "log last" should return up to n entries
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 9cd31820f22d746e680f4ce3b0c780157d24194f
+Merge: 60ff142c8e 8a7f983f33
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Thu Nov 9 22:27:47 2017 +0800
+
+    Merge pull request #18741 from shinobu-x/wip-22019-luminous
+    
+    luminous: tests: "ceph osd create" is not idempotent
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit d858570755d88e9a3708952827a99cc24f09f9ab
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Nov 6 16:01:23 2017 +0800
+
+    mon/LogMonitor: "log last" should return up to n entries
+    
+    limit the # of returned entries to "num", and backoff the start iterator
+    by one if it reaches the rend().
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit de14103f5201da69b360d9b01e249896d59376b3)
+
+commit f56de6798077399f472c8d2df25fb1c3ccaf8090
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Oct 15 21:48:19 2017 -0400
+
+    rgw_file: implement variant offset readdir processing
+    
+    Introduce new rgw_readdir2(...), which in which continues
+    from an arbitrary dirent name, which presumably has been
+    seen in a prior partial enumeration.
+    
+    Add single-file unit test for READDIR cases, librgw_file_marker.cc.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit a2c34c597371809bd8fd40ee1fc6b8e6a5145609)
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 60ff142c8e8ed8fb294f7d13a5461f070aa1ff4f
+Merge: f1776bc788 7efa91c7ce
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Thu Nov 9 10:10:53 2017 +0100
+
+    Merge pull request #18800 from linuxbox2/luminous-rgw_file-fsid-mount
+    
+    Luminous rgw file fsid mount -- pullup 1 of 2
+    
+    Reviewed-By: Abhishek Lekshmanan <abhishek@suse.com>
+
+commit f1776bc7887d2245e632d5a89189294a823ff3b2
+Merge: 2d3c14e5b8 f7dd3be6b7
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Thu Nov 9 10:04:08 2017 +0100
+
+    Merge pull request #18767 from smithfarm/wip-22020-luminous
+    
+    luminous: multisite: race between sync of bucket and bucket instance metadata
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 2d3c14e5b88c773548af61312438a69dcb454726
+Merge: 2bad0ac00c 36e214c67b
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Thu Nov 9 09:56:16 2017 +0100
+
+    Merge pull request #18794 from theanalyst/wip-18709
+    
+    luminous: rgw: Stale bucket index entry remains after object deletion
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 2bad0ac00c898d50c9c036400b4fba09918fb331
+Merge: 9787509155 f81e6dc8ab
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Wed Nov 8 18:19:27 2017 +0800
+
+    Merge pull request #18726 from shinobu-x/wip-luminous-22035
+    
+    luminous: Spurious ceph-mgr failovers during mon elections
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 9787509155df02bd9df21f389968f77d3de2d988
+Merge: 0c3e9e6aab 7e08cdf539
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Wed Nov 8 18:18:39 2017 +0800
+
+    Merge pull request #18738 from shinobu-x/wip-21547-luminous
+    
+    luminous: ceph-mgr gets process called "exe" after respawn
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 0c3e9e6aab5cfbd3d8e998b148e19bab30f3fa0e
+Merge: 628e76fe2a ed4a692254
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Wed Nov 8 18:17:54 2017 +0800
+
+    Merge pull request #18727 from shinobu-x/wip-luminous-22034
+    
+    luminous: key mismatch for mgr after upgrade from jewel to luminous(dev)
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 628e76fe2a939aae4996f43e9e9c9f7d1269e0bb
+Merge: ef5264afc9 be5363a771
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Wed Nov 8 14:44:41 2017 +0800
+
+    Merge pull request #18723 from shinobu-x/wip-luminous-22023
+    
+    luminous: osd: make stat_bytes and stat_bytes_used counters PRIO_USEFUL
+
+commit ef5264afc992d78c1b34c8a96a43216191fa5406
+Merge: a0a2d27c1e 31c56c6e6b
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Wed Nov 8 14:38:10 2017 +0800
+
+    Merge pull request #18737 from shinobu-x/wip-21549-luminous
+    
+    luminous: the dashboard uses absolute links for filesystems and clients
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit a0a2d27c1e40280ae214696dbf5c25567c28ba38
+Merge: 947cae47db 76ab4eba99
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Wed Nov 8 14:37:19 2017 +0800
+
+    Merge pull request #18736 from shinobu-x/wip-21638-luminous
+    
+    luminous: dashboard OSD list has servers and osds in arbitrary order
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 947cae47db9f5dc6d77a85834515670e20b5faf8
+Merge: 2e11e6662a e14de28f9e
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Wed Nov 8 14:36:40 2017 +0800
+
+    Merge pull request #18728 from shinobu-x/wip-luminous-22032
+    
+    luminous: dashboard barfs on nulls where it expects numbers
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 2e11e6662ae167276ece7a96783f02a24fc1f89f
+Merge: 31afe858a9 cd18f84304
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Wed Nov 8 14:34:35 2017 +0800
+
+    Merge pull request #18699 from shinobu-x/wip-luminous-21538
+    
+    luminous: upmap does not respect osd reweights
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 31afe858a96f1d5c67124b92f19cf4542df7e647
+Merge: d9df23cf3a 388b5016d9
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Wed Nov 8 14:30:42 2017 +0800
+
+    Merge pull request #18719 from tchaikov/wip-pr-18373-luminous
+    
+    luminous: osd,mon: add max-pg-per-osd limit
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 388b5016d981c9dd98b6505b2d75b0220cfee772
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Oct 18 16:07:50 2017 +0800
+
+    doc: fix the desc of "osd max pg per osd hard ratio"
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d940f6220346ad21dfc4f351c57389e8430c0257)
+
+commit 04852a400483b6459985be80a7773ef153c04b1e
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Oct 17 12:17:09 2017 +0800
+
+    mon: change "mon_pg_warn_min_per_osd" to uint64_t
+    
+    * this silences a warning from -Wsign-compare.
+    * also switch all its user to the new-style option
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit c3cd0c13a1585388df0f17ab53c170e39c711615)
+
+commit d9df23cf3aa002315e15db46d195be7d5657f9d0
+Merge: 567c29f964 53096755c0
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Nov 7 22:27:15 2017 -0600
+
+    Merge pull request #18702 from liewegas/wip-pr-18670-luminous
+    
+    luminous: qa/tasks/thrashosds: set min_in default to 4
+
+commit 567c29f96461901bd86d4c2d4f84f7b84342837f
+Merge: 77c2b0de60 8cbb2eb937
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Nov 7 22:28:58 2017 +0100
+
+    Merge pull request #18596 from smithfarm/wip-21955-luminous
+    
+    luminous: tests: add EC data pool to testing
+    
+    Reviewed-By: Patrick Donelly <pdonelly@redhat.com>
+
+commit 7efa91c7ced61f0fde81adfc0540a36fafafb10d
+Author: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+Date:   Sat Jul 22 13:12:06 2017 +0800
+
+    rgw_file: introduce rgw_mount2 with a bucket name parameter
+    
+    This explicitly allow mount a specified bucket in librgw.
+    Originally, mounting a bucket is implemented like a sub-directory
+    mount in nfs-ganesha FSAL_RGW with lookup_path.
+    With this change, we allow root_fh to points exactly to the root
+    of a mounted fs instance, which is a bucket or "/" rather than
+    always let root_fh points to "/".
+    
+    Signed-off-by: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+    (cherry picked from commit 860716e17e840df11a7e1d8b6ca0c1ee694b038b)
+
+commit b53c3948a5e48623bb437b2d36718ce3a799aa54
+Author: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+Date:   Fri May 26 17:45:29 2017 +0800
+
+    rgw_file: new fsid for fs instance
+    
+    We use an in-memory fs_inst_counter for fsid currently, but it
+    inevitably cause nfs automount problem across a nfs-ganesha crash.
+    So here we relate the fsid to the s3 uid with a hash.
+    With this, fsid remains consistent across all fs instances and
+    across crashes.
+    
+    We should be able to upgrade from old fsid to new fsid with
+    update_fh().
+    
+    Signed-off-by: Gui Hecheng <guihecheng@cmss.chinamobile.com>
+    (cherry picked from commit 178b547e83a2c0eeb024bbe4ca2ff8d8e7d50572)
+
+commit 77c2b0de6042707b42d6e717e90a525fda8ead76
+Merge: 14e0530f50 555746a09e
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Nov 7 22:07:45 2017 +0100
+
+    Merge pull request #18628 from batrick/i21953
+    
+    luminous: mds: sanitize mdsmap of removed pools
+    
+    Reviewed-By: Sage Weil <sage@redhat.com>
+
+commit 14e0530f50cd5fba46da914cdd09489f13adb2ec
+Merge: bc9f508477 c2237c7c6d
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Nov 7 18:33:21 2017 +0100
+
+    Merge pull request #18626 from dillaman/wip-21968
+    
+    luminous: cls/journal: fixed possible infinite loop in expire_tags
+    
+    Reviewed-By: Mykola Golub <trociny@gmail.com>
+
+commit bc9f5084778607c95dcbc93ea60ce296e8e8ee88
+Merge: 6bc121ddee 1aeec0c032
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Nov 7 18:32:20 2017 +0100
+
+    Merge pull request #18688 from liewegas/wip-balancer-luminous
+    
+    luminous: mgr/balancer: backport crush-compat work from master
+    
+    Reviewed-By: John Spray <jspray@redhat.com>
+
+commit 36e214c67b6982cc7a85e08dedc83ea61e56d6a4
+Author: J. Eric Ivancich <ivancich@redhat.com>
+Date:   Fri Nov 3 09:15:13 2017 -0400
+
+    rgw: fix BZ 1500904, Stale bucket index entry remains after object deletion
+    
+    We have a race condition:
+    
+     1. RGW client #1: requests an object be deleted.
+     2. RGW client #1: sends a prepare op to bucket index OSD #1.
+     3. OSD #1:        prepares the op, adding pending ops to the bucket dir entry
+     4. RGW client #2: sends a list bucket to OSD #1
+     5. RGW client #2: sees that there are pending operations on bucket
+                       dir entry, and calls check_disk_state
+     6. RGW client #2: check_disk_state sees that the object still exists, so it
+                       sends CEPH_RGW_UPDATE to bucket index OSD (#1)
+     7. RGW client #1: sends a delete object to object OSD (#2)
+     8. OSD #2:        deletes the object
+     9. RGW client #2: sends a complete op to bucket index OSD (#1)
+    10. OSD #1:        completes the op
+    11. OSD #1:        receives the CEPH_RGW_UPDATE and updates the bucket index
+                       entry, thereby **RECREATING** it
+    
+    Solution implemented:
+    
+    At step #5 the object's dir entry exists. If we get to beginning of
+    step #11 and the object's dir entry no longer exists, we know that the
+    dir entry was just actively being modified, and ignore the
+    CEPH_RGW_UPDATE operation, thereby NOT recreating it.
+    
+    Signed-off-by: J. Eric Ivancich <ivancich@redhat.com>
+    (cherry picked from commit b33f529e79b74314a2030231e1308ee225717743)
+
+commit 6bc121ddeec4b12c91ae5ac982ad113bda14d2bf
+Merge: ee27efad8b f26f3dc842
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Nov 7 18:21:41 2017 +0100
+
+    Merge pull request #18569 from kmroz/wip-21939-luminous
+    
+    luminous: list bucket which enable versioning get wrong result when user marker
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit ee27efad8ba4d751828dd73012de12ab16c2f78d
+Merge: 601249c6aa a340f1acab
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Nov 7 18:21:03 2017 +0100
+
+    Merge pull request #18591 from cbodley/wip-21938
+    
+    luminous: rgw: fix error handling in ListBucketIndexesCR
+    (approved in irc & standups)
+    
+    Reviewed-By: Abhishek Lekshmanan <abhishek@suse.com>
+
+commit 601249c6aa154d0f9e53332480ac45b5730fb503
+Merge: d2226d5adb 15a52ba832
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Nov 7 18:17:57 2017 +0100
+
+    Merge pull request #18539 from linuxbox2/luminous-baixuyeyu-refcnt
+    
+    luminous: baixuyeyu refcnt
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit d2226d5adbb4752e0180dfa3406e188bd3b37d6e
+Merge: 925c68e9e9 902d467a0e
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Nov 7 18:15:38 2017 +0100
+
+    Merge pull request #18516 from ceph/wip-bp-pr18193-luminous
+    
+    qa/rgw: ignore errors from 'pool application enable'
+    
+    Reviewed-By: Nathan Cutler <ncutler@suse.com>
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 925c68e9e9538a97f98bef1e5a884d31811aa229
+Merge: 2f35da4e60 add8c3db19
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Nov 7 18:14:21 2017 +0100
+
+    Merge pull request #18599 from linuxbox2/lum-rgwfile-21940
+    
+    rgw_file:  set s->obj_size from bytes_written
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 2f35da4e60c2b5fd2f45348587a5b175868d04d7
+Merge: b2961db15d 26c523c3be
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Tue Nov 7 13:07:09 2017 +0100
+
+    Merge pull request #18566 from kmroz/wip-21914-luminous
+    
+    luminous: [rbd-mirror] peer cluster connections should filter out command line optionals
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit 8e3227ad51fa4909a39eb0d05798b48200a03fc5
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Oct 17 12:10:55 2017 +0800
+
+    osd,mon: change "mon_max_pg_per_osd" to uint64_t
+    
+    as it should never be a negative number. if this option is disabled,
+    it's 0.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 0d68197376b6cf48c6679d77ec9f28f279f3da68)
+
+commit ae7823187186310bac117fac437d991398cffd5e
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 26 15:54:14 2017 +0800
+
+    osd: add max-pg-per-osd limit
+    
+    osd will refused to create new pgs, until its pg number is lower
+    than the max-pg-per-osd upper bound setting.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 4c7df944c7f28232873ba681eedce72cdb062ea5)
+
+commit 1db44c5bef752b3a6ce59ea21548b54b77b587d2
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sun Oct 15 20:46:28 2017 +0800
+
+    qa/tasks/ceph: add "create_rbd_pool" option
+    
+    True by default, but we can opt not to create the "rbd" pool.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit ac1e13ce63a6f32ae9ab0db8196c8cc8100d94ed)
+
+commit ed34b00566f9b1b65c09989af44ad24cdb27fbdc
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Oct 12 20:07:46 2017 +0800
+
+    osd: print osd as "osd.$osdid" in log message
+    
+    easier to read this way
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit c6b71ebf6b7b7c54560f331d5f8e4e24f5a60633)
+
+commit df611786d2882e346075f5a32aef46bea3c36d47
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 26 15:50:08 2017 +0800
+
+    mon/OSDMonitor: bypass checks if pg_temp.forced
+    
+    originally, monitor ignores the pg_temp messages sent from replica osds,
+    but to re-trigger peering if replica osd's pg number drops down below
+    the setting, we need to bypass the checks.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit da6f7a810055a0dc9e364aca8e6efbfbc32881b3)
+
+commit aefc9315ed6ebbf209d4496b0fb33c72dc107961
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 26 15:49:11 2017 +0800
+
+    messages/MOSDPGTemp: add `forced` field
+    
+    the monitor is supposed to bypass some checks if `forced` is true.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 6335bf791003e9e33b2f738ab3da06f3c626537a)
+
+commit b2961db15dd7528019df7990f8f7690ae3a95aa9
+Merge: a5899a57e4 4cac43679a
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Tue Nov 7 17:48:34 2017 +0800
+
+    Merge pull request #18775 from tchaikov/wip-upgrade-luminous-x
+    
+    qa: add 1-ceph-install to upgrade:luminous-x
+    
+    Reviewed-by: Yuri Weinstein <yweinste@redhat.com>
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+
+commit 4cac43679a7688cb4765728e1a75e812d0df22de
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Tue Aug 29 09:50:51 2017 -0700
+
+    qa: add 1-ceph-install to upgrade:luminous-x
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+    (cherry picked from commit aff342627cf77e2faae6904e626d9fb2c897f155)
+
+commit 1aeec0c0326678bcdba7ee0a6a1204f460e3ddf7
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Nov 6 21:17:22 2017 -0600
+
+    qa/suites/rados/thrash/d-*/*balancer*: debug osd in mgr
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit f7dd3be6b7804035b47d2e6fb94463f0bc772bbe
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 31 16:56:01 2017 -0400
+
+    rgw: remove placement_rule from cls_user_bucket_entry
+    
+    placement_rule is no longer needed in cls_user_bucket_entry, because the
+    only time that it's needed, we can read it from the bucket instance in
+    RGWRados::update_containers_stats()
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 8e62e3526643da67f5af7daa687120feed469785)
+
+commit 17cc13b87b5378bf03e2b0b52c274772ebaf08ef
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 31 16:29:31 2017 -0400
+
+    rgw: remove placement_rule from rgw_link_bucket()
+    
+    with the fallback in RGWRados::update_containers_stats(), we no longer
+    need to pass placement_rule into the cls_user_bucket_entry
+    
+    this removes the dependency between bucket entrypoint metadata and its
+    bucket instance metadata during multisite sync
+    
+    Fixes: http://tracker.ceph.com/issues/21990
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit dd5d278a4f9b080234cbb77e448208803ec0cf93)
+
+commit 338248a8b376074c0f43b626e9f69fc59ee26487
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 31 16:26:54 2017 -0400
+
+    rgw: take placement_rule from bucket info in update_containers_stats
+    
+    in case the cls_user_bucket_entry doesn't contain a placement_rule, take
+    it from the bucket instance info
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 531195d80d5bac774d58b210307eb19b57d26345)
+
+commit 975229c3674ce99704aef399ecfb770cc8bcf1da
+Author: Chang Liu <liuchang0812@gmail.com>
+Date:   Tue Oct 3 00:12:43 2017 +0800
+
+    tool: ceph-kvstore-tool doesn't umount BlueStore properly
+    
+    Fixes: http://tracker.ceph.com/issues/21625
+    
+    Signed-off-by: Chang Liu <liuchang0812@gmail.com>
+    (cherry picked from commit 81e4560781c4e5a5a033bfd4a4fe4094fc27d964)
+
+commit 4d4aee14a46ed36cdb40c0ff67193c8650b106da
+Author: Chang Liu <liuchang0812@gmail.com>
+Date:   Tue Oct 3 00:01:43 2017 +0800
+
+    os/bluestore: make BlueStore opened by start_kv_only umountable
+    
+    ceph-kvstore-tool use start_kv_only to debug the kvstore. we
+    will get a crash when we try to umount bluestore in kvstore-tool.
+    
+    Fixes: http://tracker.ceph.com/issues/21624
+    
+    Signed-off-by: Chang Liu <liuchang0812@gmail.com>
+    (cherry picked from commit 2754c5acad442e50d97c8daf648cfcadd38222a6)
+
+commit 3f7eadb79f4bb0a2fe25b229c24b652e7dcbaf00
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Nov 4 09:32:51 2017 -0500
+
+    qa/suites/rados/thrash: combine balancer with require-luminous
+    
+    We can only do the balancer stuff if we've done the mkfs-time
+    require luminous because otherwise the mgr commands aren't proxied via
+    the mon.
+    
+    Only do balancer for thrash tests; not for basic, monthrash, verify.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 8a7f983f3322b62cb18421c088e652341125eb1f
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Nov 1 10:45:09 2017 +0800
+
+    qa: stop testing deprecated "ceph osd create"
+    
+    "ceph osd create" is not idempotent, and is considered deprecated.
+    
+    Fixes: http://tracker.ceph.com/issues/21993
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 371d3f40911917704f4992c48236b4c60905bf4e)
+
+commit 7e08cdf53992570d27b47d0028c698b78908ba83
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Sep 15 12:23:39 2017 -0400
+
+    mgr: set explicit thread name
+    
+    This gets used as our process name in some situations
+    when respawning.  This is the same as what commit 4f177bb6b
+    did for the MDS.
+    
+    Fixes: http://tracker.ceph.com/issues/21404
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 44dce0a5664639b2ae0949cc434f1be1176d872d)
+
+commit 31c56c6e6b50b44fef56852e58ab97520ee8614a
+Author: Nick Erdmann <n@nirf.de>
+Date:   Tue Aug 22 16:55:11 2017 +0000
+
+    pybind/mgr/dashboard: updated favicon (old one was 404)
+    
+    Signed-off-by: Nick Erdmann <n@nirf.de>
+    (cherry picked from commit 7a54520f5c3d4db27fe7e2a12760dcd838ba89aa)
+
+commit d822c15af9b838dc2eea08b5f13f1dba785d776e
+Author: Nick Erdmann <n@nirf.de>
+Date:   Tue Aug 22 08:08:45 2017 +0000
+
+    pybind/mgr/dashboard: clean up
+    
+    Signed-off-by: Nick Erdmann <n@nirf.de>
+    (cherry picked from commit f512ac3d7712850f274cd01bb589aa9fbc8c8bef)
+
+commit 1f47cc0b53f7e20e156f74b28502c7705ddb4700
+Author: Nick Erdmann <n@nirf.de>
+Date:   Mon Aug 21 17:21:10 2017 +0000
+
+    pybind/mgr/dashboard: add url_prefix
+    
+    This adds a configuration variable url_prefix to the dashboard that
+    that is prepended to all URLs so you can access the dashboard at
+    http://$IP:$PORT/$PREFIX/. This is necessary if you wish to use a
+    reverse http proxy that forwards to the dashboard under a sub-path.
+    
+    Fixes: http://tracker.ceph.com/issues/20568
+    Signed-off-by: Nick Erdmann <n@nirf.de>
+    (cherry picked from commit 6d285fff0df598d66f46d38bf0a6a2cc9c0dc62f)
+
+commit 76ab4eba99785edf01c8ff9464d997b2ec691b31
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Sep 27 15:13:48 2017 +0100
+
+    mgr/dashboard: sort servers and OSDs in OSD list
+    
+    Fixes: http://tracker.ceph.com/issues/21572
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit f409099fa25b73fe580ee4662aff51636f118ed2)
+
+commit 8cacd2a568fca023a51f3be6e20f90b4ec08c82b
+Author: John Spray <john.spray@redhat.com>
+Date:   Sat Sep 23 15:22:34 2017 +0100
+
+    mgr/zabbix: log exceptions with backtrace
+    
+    Logging just the exception object gave a sometimes-enigmatic
+    single line.  Let's use the logger exception method so that
+    we get a backtrace.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 316dcc8f0b42b85503632d5472c45c9828882f2c)
+
+commit 06706bea9339e26d1a0ab395e9d42d391ee8c03c
+Author: John Spray <john.spray@redhat.com>
+Date:   Sat Sep 23 15:18:18 2017 +0100
+
+    mgr/zabbix: fix div by zero
+    
+    Fixes: http://tracker.ceph.com/issues/21518
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 9c02738b4ea0303f5b3cbfc4748d6791007be834)
+
+commit a5899a57e4f077f4061fc2aa33a61377de7ac025
+Merge: 52f66ef5ba 35de92b259
+Author: Sage Weil <sage@newdream.net>
+Date:   Sat Nov 4 10:52:27 2017 -0500
+
+    Merge pull request #18673 from dzafman/wip-21833
+    
+    osd: build_past_intervals_parallel: Ignore new partially created PGs
+
+commit e14de28f9e524575c633dc98c6ebb9a234c8d1c5
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Sep 27 14:33:58 2017 +0100
+
+    mgr/dashboard: handle null in format_number
+    
+    Fixes: http://tracker.ceph.com/issues/21570
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit f43859d4dca696ba4cac224e0c7e87e4b2aad40f)
+
+commit ed4a692254e71ce7b30566dfd6d16e8a070b3461
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Oct 19 09:28:18 2017 -0400
+
+    mon: don't blow away bootstrap-mgr on upgrades
+    
+    Fixes: http://tracker.ceph.com/issues/20950
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 57229ea2a4369518c7a16b7a09b045b7896f5a70)
+
+commit f81e6dc8abe3535a54cbf683992c95c4eaf4e186
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Oct 12 11:57:50 2017 +0100
+
+    mon: handle monitor lag when killing mgrs
+    
+    Fixes: http://tracker.ceph.com/issues/20629
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 3c3776b30a2da3f5d2ece15c57713c0ce191c778)
+
+commit be5363a771172d3260049b0012519b485e4d728e
+Author: Yao Zongyou <yaozongyou@vip.qq.com>
+Date:   Tue Oct 31 13:23:49 2017 +0800
+
+    osd: make stat_bytes and stat_bytes_used counters PRIO_USEFUL
+    
+    These two counters should be reported to mgr because the dashboard
+    plugin needing them to display osd detailed information.
+    
+    Fixes: http://tracker.ceph.com/issues/21981
+    
+    Signed-off-by: Yao Zongyou <yaozongyou@vip.qq.com>
+    (cherry picked from commit 40bb3c5f05b6f2a917a7e400f88677f41a9629a3)
+
+commit 77766609d808a974aafdff359dadc46af5bd26f4
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Nov 2 16:18:03 2017 -0500
+
+    pybind/mgr/mgr_module: fix calc_pg_upmaps
+    
+    This was fixed in the object cleanup.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ed442164d52c803916ca43a5ed76047089d3da66)
+
+commit 56c27e6be813e6a8b972733468eb099596ff7381
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Nov 2 16:11:26 2017 -0500
+
+    mgr/balancer: enable module by default
+    
+    It will still be "off".
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 26710f0a9b176289237a52e5fa5894342ad163dc)
+
+commit 53096755c02d953867d0fff3e08bc3a62d872ec6
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 1 08:31:31 2017 -0500
+
+    qa/tasks/thrashosds: set min_in default to 4
+    
+    We have EC tests with k=2,m=2, so we need a min of 4.
+    
+    Fixes: http://tracker.ceph.com/issues/21997
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d21809b14ea58dc1f44df844e407ebab5a315062)
+
+commit 52f66ef5ba291fccc265fd7a34a4eae201667153
+Merge: 1b00bdfeef e58656005d
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Nov 3 10:26:44 2017 +0100
+
+    Merge pull request #18446 from theanalyst/wip-21854-luminous
+    
+    luminous: rgw_file: explicit NFSv3 open() emulation
+    
+    Reviewed-By: Abhishek Lekshmanan <abhishek@suse.com>
+    Reviewed-By: Matt Benjamin <mbenjamin@redhat.com>
+
+commit cd18f84304124e1341516194fb73c8bde48db445
+Author: Theofilos Mouratidis <t.mour@cern.ch>
+Date:   Mon Sep 25 16:17:47 2017 +0200
+
+    osd: upmap should respect osd reweights
+    
+    Modify OSDMap::calc_pg_upmaps to take the osd reweight into account when
+    computing the size of each OSD.
+    
+    Signed-off-by: Theofilos Mouratidis <t.mour@cern.ch>
+    Fixes: http://tracker.ceph.com/issues/21538
+    (cherry picked from commit b8561cefa705d31954d47723125376a3c087f4f4)
+
+commit 03a11694feee0d78dcc1f296a77200635a651707
+Author: Theofilos Mouratidis <t.mour@cern.ch>
+Date:   Mon Sep 25 16:15:45 2017 +0200
+
+    test/cli/osdmaptool: test upmap with an out osd
+    
+    Test upmap with one osd marked out.
+    
+    Signed-off-by: Theofilos Mouratidis <t.mour@cern.ch>
+    (cherry picked from commit ff9e024a59d87eda6cbf3be9464ee7ae6f1140d7)
+
+commit 2c7972d3a4928cec039e8637048c26e94c6278d2
+Author: Theofilos Mouratidis <t.mour@cern.ch>
+Date:   Mon Sep 25 16:14:09 2017 +0200
+
+    osdmaptool: add --mark-out option
+    
+    Add a --mark-down <id> option which sets the reweight for a given
+    osd id to 0.0.
+    
+    Signed-off-by: Theofilos Mouratidis <t.mour@cern.ch>
+    (cherry picked from commit bd75a78187143f750f90fb937984eaa6333159d3)
+
+commit 1b00bdfeefd7b724f14634383ae4baaf7a42c996
+Merge: c6052f2706 92d0545781
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Thu Nov 2 21:23:50 2017 +0100
+
+    Merge pull request #18439 from theanalyst/wip-21655-luminous
+    
+    luminous: expose --sync-stats via admin api
+    
+    Reviewed-By: Abhishek Lekshmanan <abhishek@suse.com>
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 282e21bd07c3dfb5d18f338abd3ced4f4eeaa247
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 27 12:53:52 2017 -0500
+
+    mgr/balancer: simplify pool_info tracking
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a59a1df85382d559362c51bb45284b61caaca1d4)
+
+commit e2346f539f477a4b7a4685828876eb89c84ad00d
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 25 22:47:02 2017 -0500
+
+    mgr/balancer: less verbose on 'eval' by default; add 'eval-verbose'
+    
+    The verbose output is helpful for debugging and understanding what is
+    being measured, but most of the time all that you care about is the score.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2c300bc8d7189b464f59c93deb5eb56915df62b8)
+
+commit 64d75fa26d3ceb6ccdbee3b31b3705c4db2fc59f
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Oct 21 10:46:27 2017 -0500
+
+    mgr/balancer: fix pg vs object terminology
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6513e8b0fc2ebcbbbda937ec6d533ea084ef278f)
+
+commit 6082f53391a46076bda1b5d56685d928acd2cd4b
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 24 17:12:54 2017 -0500
+
+    mgr/balancer: restrict to time of day
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3f0b50b50aeb34ed39767289b8475ec5b877cc27)
+
+commit 91b84cb24466f84ce3e189e08f8cba1e2900a4cc
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 5 17:31:06 2017 -0500
+
+    mgr/module: adjust osd_weight min step to .005
+    
+    That should be ~1 PG or less on average; no real sense it taking a
+    step smaller than that!
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 297e2d65abcebe45c6be69a768cf3ea98e1b354d)
+
+commit acbf539d326403b6249ed0e2b0ca5ffc4a52152d
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 5 17:26:16 2017 -0500
+
+    mgr/balancer: if score regresses, take a few more steps
+    
+    To avoid getting stuck in a local minima, take a few additional
+    steps if our score drops and see if it gets better.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 70b503495a0ceb7d1aa4f5fce4403a8eff9a03b5)
+
+commit b9cc962adfd95304cbd56833bb84b827e5eed4d4
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 5 17:25:19 2017 -0500
+
+    mgr/balancer: allow 5% misplaced
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1ad3101818d1855c8eea8386d7682fab2fe9c8b8)
+
+commit 452962767b4f899c8c2589d01a6b8acc3cac9a69
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 5 17:25:06 2017 -0500
+
+    mgr/balancer: more aggressive steps
+    
+    We are smart enough to back off if we misplace too much.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 0dbcf737eaed02b5aa94bc6f27231b4b87f09ce5)
+
+commit 901ae5920c590a2a83f30897b05017e001572e5f
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Oct 1 16:00:06 2017 -0500
+
+    qa/suites/rados/thrash/d-balancer: enable balancer in various modes
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2c9c18d1ec3d33d385adf2a30dc1d3b8e987f9a5)
+
+commit a1ebe97886f7a578bc56a109f10a3d6149954087
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 27 17:05:05 2017 -0400
+
+    mgr/balancer: crush-compat: phase out osd_weights
+    
+    Phase out the osd_weight values as we optimize the crush compat
+    weight set.  Allow a small regression in overall score if it means
+    we are taking a step to remove the osd_weight value.
+    
+    Fix the 'osd reweightn' invocation.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 861c37fd99a76d5f1665a610224778fcbb41e9e1)
+
+commit 72716f058cd132d836c3251b4e6d4b056bb4c0a8
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 27 15:12:54 2017 -0400
+
+    mgr/balancer: crush_compat: cope with 'out' osds
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c2bba5820a0bf2225dfc9ba39cc67c78083aba11)
+
+commit c982848f32ed85639346931e86d331fb52458355
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 27 15:12:36 2017 -0400
+
+    mgr/balancer: stop if we get a perfect score
+    
+    ..and fix the 'worse' threshold (we're just working around floating
+    point imprecision; 1.01 was too coarse).
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f1f3014ffcc67200c20e6fb2b2e6e8f1141a0a1d)
+
+commit 37ed6f218ba4fafd5360b8f62feb321a1feec56f
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 27 11:40:36 2017 -0400
+
+    mgr/balancer: more dead code
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 37bd64405b8a578329b9a903c9a53322ea29caaf)
+
+commit b3dd7abc79fa3dff7768898caf814bcdfb8f8690
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 27 11:27:42 2017 -0400
+
+    mgr/balancer: crush-compat: throttle changes based on max_misplaced
+    
+    Take smaller steps if we overshoot max_misplaced.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit df2e008d3388b657d62ea4547e6b1b7a0251aa04)
+
+commit 25bfd12cf252dbba81f482fe9abe53959f992b4b
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 27 11:19:30 2017 -0400
+
+    mgr/balancer: remove dead code
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 020be37fd06a75ec86e05af966be318c2199da9f)
+
+commit 1aff9dfadc148fb1cf9b60400c35bd11011772b2
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 27 11:19:23 2017 -0400
+
+    mgr/balancer: include pg up mapping in MappingState
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f6f2d253c379a6235e810808e7ce3b83cc4e212c)
+
+commit 9121e73f639bd47310ee29dce69a4c105582a3ab
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 26 18:36:06 2017 -0400
+
+    mgr/balancer: normalize weight-set weights to sum to target weight
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 99fcd161bd222a4f0d2ddde0f1a4caecd1a78882)
+
+commit 619ae53354aa5682aeb7d77d3795a323188dad1a
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 26 18:35:42 2017 -0400
+
+    mgr/balancer: note root id in Eval
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 26a7ec0c8a4c215791369d8650cd17d41317aefe)
+
+commit afcce93591c7e5c53596cec03165033f049a0c63
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 26 18:00:08 2017 -0400
+
+    mgr/balancer: make crush-compat mode work!
+    
+    - it does multiple iterations, like the upmap optimizer.
+    - it decreases the step size if it isn't improving, in the hope
+    that it is overshooting the minimum
+    - debug output is cleaned up a bit (the info level should be
+    genuinely useful)
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d9a31595ba15de5fda104a0154778e3200fc46a0)
+
+commit c6052f2706ad6b7c92f9d6fa381527e8922b30c6
+Merge: 4e1eec777c 56359c367d
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Nov 2 14:54:50 2017 -0500
+
+    Merge pull request #18590 from liewegas/wip-prime-fixes-luminous
+    
+    luminous: ceph-bluestore-tool: prime-osd-dir: update symlinks instead of bailing
+
+commit 4e1eec777cf194cdf9bd4db1e7c4672d5b2adac7
+Merge: 79736e781f 338af16881
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Nov 2 14:54:18 2017 -0500
+
+    Merge pull request #18620 from tchaikov/wip-21527-luminous
+    
+    luminous: mon/mgr: sync "mgr_command_descs","osd_metadata" and "mgr_metadata" prefixes to new mons
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 79736e781fbd36083c639d3f4c22f8f3ae426cce
+Merge: 815d8ab573 9d8e5d763b
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Nov 2 14:53:16 2017 -0500
+
+    Merge pull request #18621 from tchaikov/wip-21534-luminous
+    
+    luminous: mon: update get_store_prefixes implementations
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 815d8ab573215277a5283af1ecabaac84eaf6200
+Merge: c23865f07e 6b6eceee02
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Nov 2 14:52:32 2017 -0500
+
+    Merge pull request #18623 from liewegas/wip-pr-18585-luminous
+    
+    luminous: ceph-bluestore-tool: prime-osd-dir: update symlinks instead of bailing
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit c23865f07e9147f8ac8a8d3837f735558af30dba
+Merge: 24bd1de6af 796e336470
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Nov 2 14:51:58 2017 -0500
+
+    Merge pull request #18625 from tchaikov/wip-ceph-disk-unlock-dmcrypted-partitions-luminous
+    
+    luminous: ceph-disk: unlocks dmcrypted partitions when activating them
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 24bd1de6afaa0d3dab763ae946b03ffb7b333430
+Merge: 240edcfb18 3c3609b7e3
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Nov 2 14:51:32 2017 -0500
+
+    Merge pull request #18650 from b-ranto/wip-allow-getattr-luminous
+    
+    luminous: selinux: Allow getattr on lnk sysfs files
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 240edcfb18475bef84bd5d6345dd365411398773
+Merge: 98edc3b9f2 deb78542ed
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Nov 2 14:51:11 2017 -0500
+
+    Merge pull request #18675 from jcsp/wip-luminous-mgr
+    
+    luminous: bulk backport of ceph-mgr improvements
+
+commit 98edc3b9f2012db277c5796055223f6e69b67e66
+Merge: 96102df540 0c0d8e6f40
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Thu Nov 2 13:39:05 2017 -0400
+
+    Merge pull request #18687 from ceph/backport-18656
+    
+    luminous: "ceph-volume: adds functional testing for bluestore"
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit 96102df5405fc470696df84996853e254c29fd31
+Merge: 172e6c1e0c 2e6b29084a
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Fri Nov 3 01:02:26 2017 +0800
+
+    Merge pull request #18293 from smithfarm/wip-21795-luminous
+    
+    luminous: Ubuntu amd64 client can not discover the ubuntu arm64 ceph cluster
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 0c0d8e6f402ae78ae8110a717188f8b366a67696
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Oct 31 15:25:23 2017 -0500
+
+    ceph-volume tests: adds objecstore as a testing factor
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit f8b114be61d033c853a2e8295b966e30c6a5f721)
+
+commit 9d970edddc1ab453a683bff1dfd19cab46e8cf9a
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Oct 31 15:10:28 2017 -0500
+
+    ceph-volume: enable the centos7-bluestore and xenial-bluestore tests
+    
+    This also updates the ansible and testinfra version to match the current
+    ceph-ansible master branch.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit b49a19a9773599de295197979b498d06ca500cf8)
+
+commit f9554c4e3ef0eea1f4f09828f98c58838390d4fa
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Oct 31 15:09:52 2017 -0500
+
+    ceph-volume: adds the xenial-bluestore testing scenario
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 38372890c9c40e6f536026f61c5ef6e6d23d7111)
+
+commit b7fef119d0730a3cb8f19a461f773e66ce97af0f
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Oct 31 15:05:46 2017 -0500
+
+    ceph-volume: adds the centos7-bluestore testing scenario
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 45e4b0c9d7d4fea1d7bd3481d46edf3b42c40d26)
+
+commit deb78542ed45ac402b4492d5b9ae4a9cdf182724
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Nov 2 07:42:56 2017 -0400
+
+    qa: fix mgr _load_module helper
+    
+    I inadvertently broke this with the latest change
+    to the module ls output.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 4fb3025682d812f86c50fa36532290fc0f8857ae)
+
+commit d4fcb97839a501094db2e1704c9799a107ad100e
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Nov 1 19:10:19 2017 +0000
+
+    mgr: fix up make_unique usage for backport
+    
+    This was getting the definition some other way in master,
+    but in luminous we need to include the backport14 header.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+
+commit f41cbb72f39359d7efc9feb0825cbad61e51c07c
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Oct 20 13:54:29 2017 +0100
+
+    mon: fix up mgr_inactive_grace for backport
+    
+    This was converted to options.cc only, but we cannot
+    backport another commit that removes this legacy health
+    reporting code, so this commit updates the legacy health
+    reporting code for the options.cc style config opt.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+
+commit a80a7a4d7f9ac73b4d3b86f9787dd9de9865530a
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Nov 1 06:35:14 2017 -0400
+
+    mon: include disabled modules in `mgr module ls`
+    
+    Otherwise, when someone wants to see what's possible
+    to do with `mgr module enable` they have to trawl
+    through the whole mgr map dump.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 5861c4c022e825fe28347ba7e61ae86a6f260f24)
+
+commit fad998cb11628db970099cc192277c1824be884a
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Nov 1 06:34:36 2017 -0400
+
+    doc: describe using `mgr module ...` commands
+    
+    ...including the new "mgr services" command.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit d220e1adc337daeac418563dee125f161e85afdf)
+
+commit a46c5733881403f65872ca36d300ab6a4ca9d2cc
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Nov 1 06:28:30 2017 -0400
+
+    doc: describe how to implement standby modules
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit ceb2a91190441b19171e5b5726748e43ee4d1e92)
+
+commit c704b344a4a739e5ba18535a0fd4f05308c89019
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Oct 23 05:37:49 2017 -0400
+
+    qa: fix mgr caps
+    
+    This was still using Kraken era settings
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 9988ebed9530718b523a23d729d499dedab5eb6d)
+
+commit f58cb7035cf8992b8772f3358f2d67787ab585c3
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Oct 19 07:50:19 2017 -0400
+
+    qa: expand mgr testing
+    
+    Some extra coverage of the dashboard, including its standby
+    redirect mode and the publishing of URIs.
+    
+    Also invoking the command_spam mode of the selftest module.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 05e648be6a51d3efa110ad9826bbdd0adcc4dd4d)
+
+commit a067468b3717dbab20f84996e76e2336d5fb5f30
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Oct 17 18:39:17 2017 -0400
+
+    mgr/selftest: extend test and add background spam mode
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit a382c3f1ca98b9fb7300e2d410bb2a1bb10b35ae)
+
+commit 1430e448ff93f73b8e8fba61f591ce4ec8c02380
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Oct 17 18:16:22 2017 -0400
+
+    mgr: drop GIL around set_uri, set_health_checks
+    
+    These didn't need to keep the GIL to go and do their
+    pure C++ parts, and by keeping it they could deadlock
+    while trying to take ActiveMgrModules::lock.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 27ee148e040ebaf512f8e11f814b3a7c8cf21f8b)
+
+commit 2a94381b53241f9ecb98c4cc605ef6547b910ca2
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Oct 17 18:14:43 2017 -0400
+
+    mgr: fix ~MonCommandCompletion
+    
+    This was doing a Py_DECREF outside of the Gil.
+    
+    Fixes: http://tracker.ceph.com/issues/21593
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 58dfa97ba88882fb3540d15e31bcac48a1aef5ef)
+
+commit 045ed0e023ae2d1a567b3426ef2b48980a851b7e
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Oct 16 10:51:34 2017 -0400
+
+    mgr: update for SafeThreadState
+    
+    A bunch of the previous commits were done
+    before this class existed, so updating in
+    one go instead of trying to edit history
+    in fine detail.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 29193a47e6cf8297d9b1ceecc7695f2c85434999)
+
+commit ef67307966575b618614225d6206019781d71647
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Oct 13 11:31:22 2017 -0400
+
+    mgr: refactor PyOSDMap etc implementation
+    
+    Implement real python classes from the C side,
+    rather than exposing only module methods.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 7e61f79f5d56b568103a067d9a1eb87af997ad61)
+
+commit d85f91cf3e2f747cc23218a52d9271dd6e27febb
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 26 18:35:29 2017 -0400
+
+    mgr/PyOSDMap: add CRUSH get_item_weight
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit eacc9021459b31e42232bb958536d594d03b07b3)
+
+commit b1307e074b5fb708420686bf814bd4a7aa76f6d8
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Oct 16 06:33:48 2017 -0400
+
+    mgr: fix py_module_registry shutdown
+    
+    Was calling way too early, which did a
+    Py_Finalize before the modules had been
+    joined.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 0d5b1d70e616d7d1c2d6360375770f5c4754649d)
+
+commit 642a26dfcd9bef06fceb119008f5c3078bbf10e7
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Oct 12 13:14:02 2017 -0400
+
+    mgr: fix thread naming
+    
+    Was passing a reference to a local stringstream into
+    Thread::create, not realising that it was taking a char*
+    reference instead of a copy.  Result was garbage (or usually,
+    all threads having the name of the last one created)
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit bb4e71ed2ebdee1ac5e4b3eee390060e19fea0d8)
+
+commit f9223ad56d8e01c1f6e8690a6a186d02eed8d96d
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Oct 6 11:02:44 2017 -0400
+
+    mgr: cut down duplication between active+standby
+    
+    ...by using PyModuleRunner class from ActivePyModule too.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit df8797320bed7ad9f121477e35d7e3862efd89bd)
+
+commit 728e1e4e136aab5b4d185c738de8df7bcdce18ee
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Oct 4 13:13:25 2017 -0400
+
+    mgr: fix os._exit overrides
+    
+    These would throw an exception when passed
+    a status code.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit e2442c1e20bf4ff12d58af500b34a18cc60d2de1)
+
+commit ca51a1b9d2cf0a3bf0fd7b69b833fa6696e61888
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Aug 24 14:07:37 2017 -0400
+
+    mon/MgrMonitor: reset services map on drop_active
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 7b629ae46599d79ca1929cfc6637b367c6bb9029)
+
+commit 8d4512b5636eb41b2cbb50a42d071580cff728c6
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Aug 22 14:47:10 2017 -0400
+
+    mgr/dashboard: implement standby mode
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 4f7007d1b0226af3f0cc33627ebf5051975657ac)
+
+commit 3953c0b3b7d0012507ac047085a3f863dc4929d3
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Aug 22 11:41:26 2017 -0400
+
+    pybind/mgr: add MgrStandbyModule
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 3048e85cd712b7da77cf6ac55dd6a689d00e47e5)
+
+commit e37df9459d6d9abdbe310153bd53d1c1c4384a6b
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Aug 22 14:42:11 2017 -0400
+
+    mgr: standby modules come up and run now
+    
+    ...they still don't have access to any config though.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit c1471c7501948004096581ee415ab4a1fa2d9379)
+
+commit 977583eba266df834a7e27516cacb750bf7121ec
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Aug 16 10:23:59 2017 -0400
+
+    mgr: enable running modules in standby mode
+    
+    Modules can implement a second, separate class
+    that has access to very little state about the
+    system and can't implement commands.
+    
+    They have just enough information to redirect
+    or forward incoming requests/traffic to the
+    active instance of the module on the active mgr.
+    
+    This enables module authors to create modules
+    that end users can access via any (running) mgr node
+    at any time, rather than having to first work out
+    which mgr node is active.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 25566d1edca638bd15b3ba3326ee7e4d3e573cbb)
+
+commit 6a35a96ea838760945073b756ea7ae13e9a68ccf
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Aug 15 06:53:18 2017 -0400
+
+    mgr: clean up python source file naming
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 70d45a6b93c92baf8d6a3b15765110a5384c5e60)
+
+commit d37bf83f419fd5b34f0cbc8ee5313425e5177bc8
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Aug 14 06:31:18 2017 -0400
+
+    mgr: refactor python module management
+    
+    Separate out the *loading* of modules from
+    the *running* of modules.
+    
+    This is a precursor to enabling modules to run
+    in standby mode.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 9718896c8b844db2f3c07df1d344636da4605e61)
+
+commit b563555edb51b576fcbf94a3b92598a7300aaf09
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jul 27 13:49:27 2017 -0400
+
+    pybind/mgr: use set_uri hook from dashboard+restful modules
+    
+    No more guessing the URL!
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 089e105dd7ec762572ac06794caa7f5543075001)
+
+commit 6dd4d0504de9806fcf7c398a8d66b956b45986d9
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jul 27 11:50:23 2017 -0400
+
+    mgr: enable python modules to advertise their service URI
+    
+    Fixes: http://tracker.ceph.com/issues/17460
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit a0183a63fa791954d14c57632e184858cefe893d)
+
+commit 778322d0913d4d9c70609a3c40d809f62a366020
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jul 27 11:49:45 2017 -0400
+
+    mon/MgrMonitor: store services in map and expose with command
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit c3c3e4e90ba6b09e29879b500f211d607ebabb53)
+
+commit a6b87a8e7c859607d4554833c5f9ad4cb2fea187
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jul 27 11:46:40 2017 -0400
+
+    messages: `services` in MMgrBeacon
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 236841b3b62af92ce0c4852045327fcfbc5c1651)
+
+commit b44cf5f70d0286c9dcdcda8dbb9f014a3386813a
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jul 27 11:45:53 2017 -0400
+
+    mon/MgrMap: store list of services
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 3f703bd91f07b2fe43a16df0083d7b7c23803fd5)
+
+commit 5b677f3f02500b87757b9965d20151e345b8ba3f
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jul 27 06:31:01 2017 -0400
+
+    mgr: carry PyModules ref in MonCommandCompletion
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit e938bf9b9d27e192765c805e5f532c9dd4808b21)
+
+commit b69a656ac76fe38431bcd66c626883e324be1ba4
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Jul 26 12:31:13 2017 -0400
+
+    pybind: update MgrModule for ceph_state->ceph_module
+    
+    & tidy up the places where ceph_state was getting
+    used outside of MgrModule.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 62cb512e4740f1f78f516b4f2179c1123fae1b36)
+
+commit 8b5f302706e91c5c6a88e918a9d4baafd726c211
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Jul 26 07:44:00 2017 -0400
+
+    mgr: refactor python interface
+    
+    Expose a python class instead of a module,
+    so that we have a place to carry our reference
+    to our MgrPyModule* and to PyModules*, rather than
+    passing a handle for the former and using
+    a global pointer for the latter.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 563878ba217491dd0a6fbd588cd56d09e3456c14)
+
+commit 35b4518c4a0c477f0190bfc006434093fe720c05
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Aug 3 06:22:35 2017 -0400
+
+    mgr/dashboard: remove blue highlight on scrubbing pg states
+    
+    This was kind of unnecessary, highlighting a completely normal
+    and healthy situation in a different colour.  The blue was
+    also really hard to read against a grey background.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 99fa1fdf4e1be57792f50907147781d12009b32b)
+
+commit 764e7011d0e69e08aa5f56a5c8c56b8875820b7e
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jul 27 11:42:16 2017 -0400
+
+    mgr/dashboard: clean up fs standby list when empty
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 5e64787c0ae0ac2a365c89bf89dfea425adc17d4)
+
+commit 1d1dce056664a40ea4dc9ebd1dc3826f0e449555
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Aug 30 13:56:39 2017 +0100
+
+    mgr: remove old-style config opt usage
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit ec09a7abc515f802451bf7ef3d22ce8ee6c6c7b3)
+
+commit a0131144036a9ea64c6f8e3289562bf7db18c746
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Aug 30 12:12:40 2017 +0100
+
+    mon: remove old-style mgr config opt usage
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 6af4120d63324150ba19022c41fe4fa8a38cacbb)
+
+commit 526425c63b6a8f2d3f79fdd75c360d7c2ecfeec6
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Aug 30 11:48:25 2017 +0100
+
+    common: populate manager config option metadata
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit eba4c3f2762ae40ba746091e32364c2d68e780d9)
+
+commit 1eee973b90418979c33ca42b0720045f2de670c3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Jul 13 14:49:48 2017 +0800
+
+    common,mds,mgr,mon,osd: store event only if it's added
+    
+    otherwise
+    * we will try to cancel it even it's never been added
+    * we will keep a dangling pointer around. which is, well,
+      scaring.
+    * static analyzer will yell at us:
+      Memory - illegal accesses  (USE_AFTER_FREE)
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 2449b3a5c365987746ada095fde30e3dc63ee0c7)
+
+commit 66aa481be983fdd60beddc7ce621dad3edbf5490
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Oct 3 08:16:10 2017 -0400
+
+    mgr: safety checks on pyThreadState usage
+    
+    Previously relied on the caller of Gil() to
+    pass new_thread=true if they would be
+    calling from a different thread.
+    
+    Enforce this with an assertion, by wrapping
+    PyThreadState in a SafeThreadState class
+    that remembers which POSIX thread
+    it's meant to be used in.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 625e1b5cfb9b8a5843dfe75e97826f70a57d6ebe)
+
+commit 385a6a0e8e6a6dd315ab358dcea7f3c069d9ad4d
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Aug 22 11:38:25 2017 -0400
+
+    mgr: move Gil implementation into .cc
+    
+    The inclusion of Python.h in the .h was awkward
+    for other files including Gil.h.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 23c3a075ee1a27e1b57fcb452a4d6ce53080264e)
+
+commit 241d655e127c71e9d9012d9205005c97d61def8a
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Jul 26 07:21:40 2017 -0400
+
+    mgr: reduce Gil verbosity at level 20
+    
+    Even at 20, it's pretty heavy to be logging
+    every lock acquire/release.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 987612a97529be7e67b89977c4a0cf47906a5ecb)
+
+commit 30db4f5a54eac28e546c43a2ce099aa89d179495
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Wed Oct 11 12:28:19 2017 +0200
+
+    pybind/mgr/prometheus: no ports in osd_metadata
+    
+    Ports might change on a OSD restart and this would create a new metadata
+    metric for this osd.
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 48fec7db4b214fe8ef6a04f8cb53fb8a2fb9c2ca)
+
+commit 80fc65cafaad3bb5499a15edbd97ecf483b4ad82
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Wed Oct 11 10:59:33 2017 +0200
+
+    pybind/mgr/prometheus: add osd_in/out metric; make osd_weight a metric
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit e4c44c1d702ce242f2cb9a58ca7ce1c31fe0a498)
+
+commit db09ea1c75c80dc189042c59e09ebc1c1f8d1249
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Wed Oct 11 20:07:19 2017 +0200
+
+    pybind/mgr_module: move PRIO_* and PERFCOUNTER_* to MgrModule class
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit f69484debade5f4fa2bd3a0d1badc9291cc9d7b7)
+
+commit bbe8fbe9ca98c1b0c8b5b43c0296fd9cc4d09c77
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Oct 9 12:10:22 2017 +0100
+
+    qa/mgr: fix influx/prometheus test names
+    
+    This was a typo: they were swapped around.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit d96a59e74b6984b77c9f3b15f702e3bf45053590)
+
+commit be207ede6028e515a0a632e12330848c93d04a34
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Sep 28 10:50:53 2017 -0400
+
+    doc: flesh out prometheus docs
+    
+    Explain ceph_disk_occupation, importance
+    of instance labels and honor_labels, provide
+    example prometheus configuration yaml.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 5227afed5f33fa9487e1bfa3fd8ce0d82eb4a20f)
+
+commit 8a9e1cb00110ed73f492f1f6cf23b3595756efc4
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Sep 28 10:10:14 2017 -0400
+
+    mgr/prometheus: add ceph_disk_occupation series
+    
+    This is the magic series that enables consumers to
+    easily get the drive stats that go with their
+    OSD stats.
+    
+    Fixes: http://tracker.ceph.com/issues/21594
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 284be75524f7125dc1409b9c05fe47b37484964e)
+
+commit 51d74e41223397abcde006a4ce53e693a2125852
+Author: Benjeman Meekhof <bmeekhof@users.noreply.github.com>
+Date:   Wed Oct 4 10:05:17 2017 -0400
+
+    mgr/influx: Correct name of daemon stat measurement to 'ceph_daemon_stats'
+    
+    Signed-off-by: Benjeman Meekhof <bmeekhof@umich.edu>
+    (cherry picked from commit f9014a1c75c6a3adf414b48a707fd444e65b3024)
+
+commit 2934dda9133a3a8876ab2701b11d4379440a5e41
+Author: Benjeman Meekhof <bmeekhof@users.noreply.github.com>
+Date:   Tue Oct 3 16:30:43 2017 -0400
+
+    mgr/influx: modify module database check to not require admin privileges
+    
+    - existing check tried to list all DB and fails even if DB exists if user is not admin level
+    - still tries to create database if not found and user has privs
+    
+    Signed-off-by: Benjeman Meekhof <bmeekhof@umich.edu>
+    (cherry picked from commit 06d7d37c7b9a8c3f4435eff04b6f4934be5e676f)
+
+commit 8c816b8e0fc1fd3fe227690287bcc6e6fce7c54d
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Tue Oct 10 08:40:31 2017 +0200
+
+    pybind/mgr/prometheus: fix metric type undef -> untyped
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 6306392492d103200b21ea91bce10a315d7c4e16)
+
+commit 2ebab2f19cdd44f4c568eb96388fc4296f0f1814
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 25 11:14:57 2017 -0400
+
+    mgr: respect perf counter prio_adjust in MgrClient
+    
+    This awkwardly involves re-ordering some definitions
+    in perf_counters.h in order to refer to the prio
+    names defined in PerfCountersBuilder.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 88163749b572ffd2bfe0850136fad5dbed2a9180)
+
+commit f8e9c37286d37d43bb768531f9cfbf70f2cb06b6
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 18 09:06:13 2017 -0400
+
+    test: update perfcounters test for priority in output
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 0f531f7871a68db96b2fb66ffdf6fae6935e6107)
+
+commit f073fc4663db17ea44e9c36831d21df1597d193a
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Sep 13 17:16:54 2017 -0400
+
+    qa: add mgr module selftest task
+    
+    The module self test commands give us a chance to
+    catch any other ceph changes that change something
+    that a module was relying on reading.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 99352ceced9d0fe92ddad6b97b1393b41de75d50)
+
+commit 4c22f0f2669d343a8e7e83f0bc2a2dacbe194f34
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Sep 13 10:46:56 2017 -0400
+
+    mgr/prometheus: remove explicit counter list
+    
+    These have had their priorities bumped up to
+    USEFUL, so they'll appear in the default
+    get_all_counters output.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit ad5a31efbea8081f03dd73669e891d03857ef9cc)
+
+commit fec2b3abf9d9abdc12bfeedaf2b22e8b0cc8c9f4
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Sep 13 10:45:21 2017 -0400
+
+    mon: elevate priority of many perf counters
+    
+    We can be quite liberal here, because mons are
+    small in number.  However, we don't want to expose
+    KV database counters at this database from OSDs, so
+    use the prio_adjust mechanism for that.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit ac8320f23dd4c00eb80da0d9837c29744e38bd57)
+
+commit 8b9a18d3a0c40cb0ef10e32018ea1b23ff53f51e
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Sep 13 07:07:50 2017 -0400
+
+    osd: upgrade a bunch of perf counters to PRIO_USEFUL
+    
+    These are broadly the OSD-wide IO stats, which happen
+    to also be the ones that were named in the
+    prometheus plugin until I changed it to be
+    priority-based.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit a1cc4ba2993de62b60fd1e58a9704877a6da5fe4)
+
+commit 278188c76ebdbaaa36b3d185b1378abf7665f3fc
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Sep 13 07:06:24 2017 -0400
+
+    common: PerfCountersBuilder helper for priorities
+    
+    Let the caller set a priority as the defaul, to enable them
+    to create a bunch at a given priority.  This is just a
+    convenience.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 66f61eeda6a2465b5fc0e40a4f1300913db065dc)
+
+commit 3dff5c0f39289e02e882bf86bf3e23e94ee33aa8
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Sep 12 10:27:12 2017 -0400
+
+    mgr/prometheus: add a self-test command
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 76e1ba52b1b95d417cdd04b8fe985acee648f0e9)
+
+commit a6bc96dfe93f7cbcefa8030a3b6830117516931e
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Sep 12 08:05:28 2017 -0400
+
+    mgr/influx: remove file-based config
+    
+    ...and also trim down the configuration to what's really
+    needed.  In general users don't need to pick and choose
+    metrics.  We could add it back if there was a strong
+    motivation.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 6776d4645afc49a4bfb4b62673c91384239037f4)
+
+commit bda26fe0e707ec9e609be613805340dadd17e8f3
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Sep 12 06:51:21 2017 -0400
+
+    mgr/influx: enable self-test without dependencies
+    
+    The idea of self-test commands is that they're self
+    contained and just exercise the module's calls
+    to the Ceph-side.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 125294ab9d6e99aa4c960fea147a4e86624b869e)
+
+commit 9abd779d3321d1cda9c8677600e1e35af46109e7
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Sep 12 06:18:15 2017 -0400
+
+    mgr/influx: revise perf counter handling
+    
+    - Use new get_all_perf_counters path
+    - Consequently get counters for all daemons, not just OSD
+    - Tag stats with ceph_daemon rather than osd_id, as some
+      stats appear from more than one daemon type
+    - Remove summing of perf counters, external TSDB and/or queries
+      can do this.
+    - Remove mgr_id tag: this would change depending on which
+      mgr was active, which is certainly not desirable.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 59b48e7660f4b757804974835027cd08a59843c2)
+
+commit 32f5f0fe515e6890e8aefad6ad8cdc2574ae8a34
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Aug 3 13:00:56 2017 -0400
+
+    mgr: omit module list in beacon logging
+    
+    This is useful in itself, but awkward when dealing
+    with logs generally, because it means that when you
+    grep on the name of a module, you get mostly beacon
+    messages rather than the log messages from the
+    module.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 8d1277fa5c578ce0ea23a70cc58c6cf99921ee25)
+
+commit 10ab4f8b6821e2d6593bc09161c23a9163b93611
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Sep 12 05:42:23 2017 -0400
+
+    mgr: define perf counter constants in mgr_module
+    
+    So that modules can consume perf counter data
+    intelligently without having to hunt around
+    in C land for these constants and redefine them.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 39ab28ed47e869e1466cb3a316a2cb11bdedd23a)
+
+commit 290d15ed912285803a40d02308b921b415c580a2
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 11 09:12:25 2017 -0400
+
+    ceph.in: use PRIO_INTERESTING as daemonperf threshold
+    
+    Using PRIO_USEFUL as the threshold for what goes into
+    time series databases.  I'm claiming that we have
+    more "useful" counters than fit on the screen,
+    so daemonperf's "a screen's worth" threshold
+    should be at the "interesting" level.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 30a74ce343caec2a433cb532ba697fe7013ed05c)
+
+commit 5ee9e15858d5aa2a71a89c13dea9a5db72579726
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 11 09:12:01 2017 -0400
+
+    mon: set some priorities on perf counters
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 29a71c35c39fbe1d4887e3f5ebb93232daab3487)
+
+commit 3bd478794e2626251eda91850974188da208b591
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 4 05:39:11 2017 -0400
+
+    mgr/prometheus: tag stats by daemon name
+    
+    Using osd=0 or similar tags was problematic because
+    daemons of different types have some same-named
+    counters (e.g. MDS and OSD both have objecter
+    perf counters).
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit eb524c272c89f8f99f22969b78caa016db7c671e)
+
+commit af92c011b20eaef073730be2f8d7112a0f7e99f8
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Sep 1 12:02:37 2017 -0400
+
+    mgr/prometheus: use new get_all_perf_counters interface
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 11137aa269271ad15dcf19a8d51ce6f4acb7a98e)
+
+commit 3ce4a950975188051b6a3e6cb366c2fb23f0e88f
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Sep 1 12:01:35 2017 -0400
+
+    common: used fixed size int for perf counter prio
+    
+    ...to avoid any ambiguity in allowed range and
+    make clear how to encode it down the wire.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit ba08fc1008d17aa7a5f285ea2705705ce1a0bda0)
+
+commit d4a0c778747970c231a55baea32f59a1009af4dd
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Sep 1 12:00:59 2017 -0400
+
+    mgr: transmit perf counter prio to the mgr
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit f304f84cfbc22c1a54d152cc38227077bc564a7e)
+
+commit cdcac6e92ff57f1a4e5f5cac676049c6584452c4
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Sep 1 10:46:56 2017 -0400
+
+    common: always include priority in perf counter dump
+    
+    JSON output with inconsistent sets of members is
+    annoying to use on the receiving side.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit e631f1a72735ec618e2f3012ad7b9c5830d6c0eb)
+
+commit 802cd49bae2cce6c55eddd68feb617df893e1046
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Aug 29 11:55:28 2017 -0400
+
+    mgr: add get_all_perf_counters to MgrModule interface
+    
+    This is for use by modules that dump counters
+    in bulk, e.g. to a TSDB.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 9a42d4255d9d968d6162b53b71db292d9d3de2e4)
+
+commit 85a93dcdddc4cd82c8935bfe123cb20f13f8928a
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Fri Aug 11 13:09:24 2017 +0200
+
+    pybind/mgr/prometheus: export cluster-wide pg stats, not per osd
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 13b1236b96d4563e0985cad40d3009b60cc475e7)
+
+commit 1d89c0809f64009d646b929fec675e1aaa6c2b1a
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Fri Aug 11 12:51:47 2017 +0200
+
+    pybind/mgr/prometheus: add more osd metadata
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit e7704fa9cc35549dba526212c2830df589670416)
+
+commit 6a65408c1364669916643329494825aae394b200
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Fri Aug 11 12:05:09 2017 +0200
+
+    pybind/mgr/prometheus: don't get perf counters that are not in schema
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit d4ba07d04477ccae3a89dcdcafbb7e76149dfd1c)
+
+commit 62092680ad966eabcdc6b557e9858fe0562147ee
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Fri Aug 11 12:04:28 2017 +0200
+
+    pybind/mgr/prometheus: add mon and osd perf counters to export
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit fa25d31263a26074225e2a00cb82448066b54069)
+
+commit 752c888533aaa9c7ae5781fafe4a6aa3302059fc
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Thu Aug 10 19:46:07 2017 +0200
+
+    pybind/mgr/prometheus: add index page, export metrics under metrics/
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit d99a506ed37c2d0991d68ecd34ac5fb213a3eea4)
+
+commit b279ae25d95f6c4b8290042fc9f33c4a6f7afac2
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Thu Aug 10 18:19:42 2017 +0200
+
+    pybind/mgr/prometheus: export selected perf_counters
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit f6e2e36ba72caf6347f3bb6a985925d0e35077a2)
+
+commit 4c092524929db048f6863acf1a9f12281bdc1646
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Thu Aug 10 18:18:36 2017 +0200
+
+    pybind/mgr/prometheus: export osd and pool metadata
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 2bea3814699c27baa8f633b56a8800d697685898)
+
+commit e3dafc3dd682a10e2310efdac4615d834933e7d6
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Thu Aug 10 18:15:56 2017 +0200
+
+    pybind/mgr/prometheus: actually emit reported pg counts
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit c288624eed862559b2c86c5dfc85c837716739ab)
+
+commit 1ef5c88ef08ab9950ee8d7d4acda032b5a54f2f5
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Thu Aug 10 18:09:17 2017 +0200
+
+    pybind/mgr/prometheus: no need to wait for notify event
+    
+    If stats or perf counters are not available they won't be emitted.
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit ead0973d7dd12fe985390891c80f1bc15f7b9aec)
+
+commit d0ef1cd2570644451205e5306feca0da94462d1f
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Thu Aug 10 18:07:14 2017 +0200
+
+    pybind/mgr/prometheus: no need to convert perf_schema to ordered_dict
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 5e4b4b5ea2a217731691c1c391c252b08452798a)
+
+commit 7f191ff9dff33cac551ae5f64027d566b17b6d98
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Wed Aug 9 17:22:49 2017 +0200
+
+    pybind/mgr/prometheus: add device_class label to osd metrics
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 76d1918724320b7d6b1120b57b3002bb24099001)
+
+commit b24263387e46ac537a8ac189f9e0e80699518e0b
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Wed Aug 9 16:19:38 2017 +0200
+
+    pybind/mgr/prometheus: add cluster wide metrics; no perf counters for now
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 49b3ff83cd231066d2a8f1809fadbdeb2c0c1f88)
+
+commit 5f0ef3a73a55a3f24997e415a848308afa0263ba
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Fri Aug 4 10:23:11 2017 +0200
+
+    pybind/mgr/prometheus: prefix metrics with 'ceph'; replace :: with _
+    
+    Both follow prometheus best practices. While : is a legal metric
+    character, "Exposed metrics should not contain colons, these are for
+    users to use when aggregating."
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 177afcc7886aa3898d092ebd1e101697bc6539fd)
+
+commit a303218c85498ccf472d2f1b35fd391376fb0faf
+Author: mhdo2 <mhdo2@users.noreply.github.com>
+Date:   Mon Aug 21 12:13:01 2017 -0400
+
+    doc/mgr: add influx plugin docs
+    
+    Signed-off-by: My Do <mhdo@umich.edu>
+    (cherry picked from commit e345fe3c5780976a4e33488b3a75cd24bb2c96c5)
+
+commit de89603e54f550ffc4ff994026f32c5b7d5f6529
+Author: mhdo2 <mhdo2@users.noreply.github.com>
+Date:   Tue Jul 18 18:33:55 2017 -0400
+
+    mgr/influx: added influx plugin
+    
+    Signed-off-by: My Do <mhdo@umich.edu>
+    (cherry picked from commit 68ae26c014d0471cc3f2f979dc8d822b2e50740f)
+
+commit 2fea47d9710ec1e388db044065bc42b8046dae22
+Author: John Spray <john.spray@redhat.com>
+Date:   Sat Sep 23 11:55:55 2017 -0400
+
+    mgr: store declared_types in MgrSession
+    
+    Because we don't (yet) properly prevent multiple sessions
+    from daemons reporting the same name (e.g. rgws), storing
+    it in the DaemonPerfCounters meant that one daemon's report
+    was referring to another daemon's set of reported types.
+    
+    This should always have been a property of the session.
+    
+    The behaviour will still be ugly when multiple daemons
+    are using the same name (stomping on each other's stats/statsu)
+    but it shouldn't crash.
+    
+    Fixes: http://tracker.ceph.com/issues/21197
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit dc415f1ae09a308bd448614934a4c168eb9cf07b)
+
+commit 951d67fc9fee23f3f8f7d7df8fbfb8c29e60f82f
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 18 10:12:00 2017 +0100
+
+    mgr: make pgmap_ready atomic to avoid taking lock
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit d20915741d985e080a723cd6563bc6f4a657276f)
+
+commit 946d1541bd2b14d6f8abbaa201c3521dd117f01b
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Aug 28 07:29:36 2017 -0400
+
+    mgr/DaemonServer: handle MMgrReports in parallel
+    
+    The DaemonStateIndex locking is sufficient to make all
+    the report processing safe: holding DaemonServer::lock
+    through all ms_dispatch was unnecessarily serializing
+    dispatch.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 64af9d3da0fceff9ad0ff668f60d272c46912f34)
+
+commit d70fae092db920e2db77c7c8b044cfb4d9687992
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Aug 24 12:53:24 2017 -0400
+
+    mgr: clean up DaemonStateIndex locking
+    
+    Various things here were dangerously operating
+    outside locks.
+    
+    Additionally switch to a RWLock because this lock
+    will be relatively read-hot when it's taken every time
+    a MMgrReport is handled, to look up the DaemonState
+    for the sender.
+    
+    Fixes: http://tracker.ceph.com/issues/21158
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 806f10847cefe5c7a78fc319b1b130d372197dd3)
+
+commit b86cc9a2e49a4d1b53e83f47876a42b42819efcb
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Aug 31 12:13:23 2017 -0400
+
+    mgr: runtime adjustment of perf counter threshold
+    
+    ceph-mgr has missed out on the `config set` command
+    that the other daemons got recently: add it here
+    and hook it all up to the stats period and threshold
+    settings.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 057b73d641decb9403aba50caae9d139f3a34dd4)
+
+commit c278580d207f0b99daed48afdc689ae77fe73e9b
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Jul 31 09:24:09 2017 -0400
+
+    mgr: apply a threshold to perf counter prios
+    
+    ...so that we can control the level of load
+    we're putting on ceph-mgr with perf counters.  Don't collect
+    anything below PRIO_USEFUL by default.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit bdc775fdd8acdad5c58ff3065a21396f80ce5db4)
+
+commit 11720b96ec8f9ef683710dffa3da4cda2ecf096d
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 8 16:36:23 2017 -0400
+
+    pybind/mgr/balancer: make auto mode work
+    
+    (with upmap at least)
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ef1a3be05671ad31907cf8c4beb64a766359bc66)
+
+commit b6a000f0efef3cd6a143b82ae33cacf660cd8e4c
+Author: Spandan Kumar Sahu <spandankumarsahu@gmail.com>
+Date:   Mon Aug 7 04:01:57 2017 +0530
+
+    src/pybind/mgr/balancer/module.py: improve scoring method
+    
+    * score lies in [0, 1), 0 being perfect distribution
+    * use shifted and scaled cdf of normal distribution
+      to prioritize highly over-weighted device.
+    * consider only over-weighted devices to calculate score
+    
+    Signed-off-by: Spandan Kumar Sahu <spandankumarsahu@gmail.com>
+    (cherry picked from commit c09308c49ca087fb8c5e7d4261b0234190f863d9)
+
+commit 6090ae6c224904516736c5f1a4ae5bcb6d7e6caa
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Aug 4 17:59:20 2017 -0400
+
+    pybind/mgr/balancer: make 'crush-compat' sort of work
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 7a00e02acd1b2ff21dac829de30f80fd69eae602)
+
+commit add3cd36db7c02d52c2fa429c034c114cac526d8
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 3 16:23:08 2017 -0400
+
+    pybind/mgr/balancer: rough framework
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d5e5c68c374e7d5514f89aac2d3df6008d103a76)
+
+commit 0c73e433ab6583fca6eea7678c23b469d643ae04
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jul 27 23:33:06 2017 -0400
+
+    mgr/PyOSDMap: OSDMap.map_pool_pgs_up, CRUSHMap.get_item_name
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a928bf62316c32f37dd1791192fd9a2ddaef0d33)
+
+commit 423947563c8e88f89bef9d71b3116c3302bd40c9
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Jul 23 00:10:56 2017 -0400
+
+    mgr/PyOSDMap: get_crush, find_takes, get_take_weight_osd_map
+    
+    These let us identify distinct CRUSH hierarchies that rules distribute
+    data over, and create relative weight maps for the OSDs they map to.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3b8a276c437cfd599c55a935d141375afda676ff)
+
+commit 073f23734c8058e3efcb440df89adab818510695
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jul 27 10:07:31 2017 -0400
+
+    crush/CrushWrapper: rule_has_take
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ef140de639078b40c05971fb219f7b8c12d83228)
+
+commit df426b5c24e12b1156bccdd8948cbed5977c348a
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jul 22 23:50:27 2017 -0400
+
+    crush/CrushWrapper: refactor get_rule_weight_osd_map to work with roots too
+    
+    Allow us to specify a root node in the hierarchy instead of a rule.
+    This way we can use it in conjunction with find_takes().
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 69454e0570274ff7f252e7f081965dcc9bb04459)
+
+commit 89cac2d5176300838c23a28814257d0f395e39c9
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jul 22 23:17:18 2017 -0400
+
+    pybind/mgr/balancer: do upmap by pool, in random order
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 028a66d43244c15a77e71f3d3e4f41773837ab02)
+
+commit bfdc955ab6ab424b6a46dc1b0022fb3dc13ce157
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jul 11 16:27:08 2017 -0400
+
+    pybind/mgr/balancer: add balancer module
+    
+    - wake up every minute
+    - back off when unknown, inactive, degraded
+    - throttle against misplaced ratio
+    - apply some optimization step
+      - initially implement 'upmap' only
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 0d9685c50f79fbb53dbc8bd98c95900ef6e902b8)
+
+commit c57a55e8d426e6f87d4649a799d9ddee665618e0
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jul 11 16:26:16 2017 -0400
+
+    pybind/mgr/mgr_module: add default arg to get_config
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 39c42ddb9339c1950a3a474e8083db8b24e775a6)
+
+commit 4d686ee875d5bc0125426d348e044988aaca9f91
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Jul 10 23:23:19 2017 -0400
+
+    mgr: add trivial OSDMap wrapper class
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2ef005196ba2eb49c34c32def624938c7a8beb03)
+
+commit 1b721632aa8271d74b319714db0a9fedacb2629b
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Jul 27 10:06:45 2017 -0400
+
+    mgr/PyModules: add 'pg_dump' get
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit bfb9286f4212947183c46543d609b664ea13b489)
+
+commit 68d411cf7b934c724f1f8ebeaa7148daebd79599
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jul 11 16:25:42 2017 -0400
+
+    mgr/PyModules: add 'pg_status' dump
+    
+    This is summary info, same as what's in 'ceph status'.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 85b5b80906d00e098d4b1af1354c60a357022dd2)
+
+commit 35de92b259f50b52e51ab1daf57041553170406d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Nov 1 09:53:34 2017 -0700
+
+    osd: build_past_intervals_parallel: Ignore new partially created PGs
+    
+    Fixes: http://tracker.ceph.com/issues/21833
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+
+commit 172e6c1e0c5bb86db7cc0017acaa674aa893e3a0
+Merge: 2988a39abe 966683fde6
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Wed Nov 1 16:28:25 2017 -0400
+
+    Merge pull request #18674 from ceph/wip-rgw-s3-branch
+    
+    qa/tests: use ceph-luminous branch for s3tests
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 966683fde600837eb9a0f82ca6fd00f52e721809
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Wed Nov 1 10:32:07 2017 -0700
+
+    qa: use ceph-luminous branch for s3tests
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+
+commit 2988a39abedc6a092b8b3c2f0b33a3559a55461b
+Merge: 4dc0a0e38a 8f87fa2d3a
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Nov 1 15:24:55 2017 +0100
+
+    Merge pull request #18412 from kmroz/wip-21659-luminous
+    
+    luminous: mgr: fix crashable DaemonStateIndex::get calls
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 4dc0a0e38a3f1a988c180d47970102df4b326b9e
+Merge: 273e035dbe 72c8583107
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Nov 1 12:33:45 2017 +0100
+
+    Merge pull request #18113 from jcsp/wip-prometheus-port-backport
+    
+    luminous: ceph-mgr: can not change prometheus port for mgr
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 273e035dbe158f162c2ee559744c456c942f1737
+Merge: a7f519913d f463cd84ae
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Oct 31 14:45:27 2017 -0700
+
+    Merge pull request #18655 from ceph/wip-yuriw-bp-PR18634-luminous
+    
+    qa: add "restful" to ceph_mgr_modules in ceph-ansible suite
+
+commit f463cd84ae1fb38817078967a4a40ba4b600f0e5
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Oct 31 11:21:58 2017 +0800
+
+    qa: add "restful" to ceph_mgr_modules in ceph-ansible suite
+    
+    backport of https://github.com/ceph/ceph/pull/18634
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit caf9ee5c60d999951979d0b67afda8d56e1cd91d)
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit a7f519913df3aba83aa3ead9eee6b94aa8f90ffd
+Merge: f2749114c5 b84803d609
+Author: Karol Mroz <kmroz@suse.de>
+Date:   Tue Oct 31 10:26:42 2017 -0700
+
+    Merge pull request #17889 from smithfarm/wip-21372-luminous
+    
+    luminous: core: Improve OSD startup time by only scanning for omap corruption once
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit f2749114c596d4016eb14e4777cb93a11b14b705
+Merge: 528c3b6d32 1236e0f2e9
+Author: Karol Mroz <kmroz@suse.de>
+Date:   Tue Oct 31 10:18:07 2017 -0700
+
+    Merge pull request #18004 from linuxbox2/luminous-rgwf-wr
+    
+    luminous: rgw_file: fix write error when the write offset overlaps.
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 528c3b6d32f9c3319fd429de37bbca578538ed7c
+Merge: ec2d294684 d800747fa1
+Author: Karol Mroz <kmroz@suse.de>
+Date:   Tue Oct 31 09:03:48 2017 -0700
+
+    Merge pull request #18438 from theanalyst/wip-21696-luminous
+    
+    luminous: fix a bug about inconsistent unit of comparison
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 3c3609b7e3ce7339a845f9191a709d12025fbbb2
+Author: Boris Ranto <branto@redhat.com>
+Date:   Thu Sep 21 17:24:07 2017 +0200
+
+    selinux: Allow getattr on lnk sysfs files
+    
+    This showed up during downstream testing for luminous. We are doing
+    getattr on the sysfs lnk files and the current policy does not allow
+    this.
+    
+    Fixes: http://tracker.ceph.com/issues/21523
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 394c26adb97cd150233fe8760355f486d03624a4)
+
+commit ec2d2946840fac65d9588f225adaa6f51c32f6b0
+Merge: 90017e7d93 def3d55eb7
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Tue Oct 31 13:04:00 2017 +0800
+
+    Merge pull request #18410 from kmroz/wip-21732-luminous
+    
+    luminous: qa/suites/rest/basic/tasks/rest_test: whitelisting
+    
+    Reviewed-by: xie xingguo <xie.xingguo@zte.com.cn>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 90017e7d933b52794d32fef69d97d5ba529d3936
+Merge: 4f024c6831 c603faaf17
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Oct 30 20:23:39 2017 -0700
+
+    Merge pull request #18629 from ceph/wip-yuriw-21978-luminous
+    
+    qa/suites/upgrade/jewel-x: Changed typo ('hammer' to 'jewel')
+
+commit c603faaf171fd3074ffe5e1342e8b96cd261e097
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Mon Oct 30 14:52:52 2017 -0700
+
+    Changed typo ('hammer' to 'jewel')
+    
+    Fixes http://tracker.ceph.com/issues/21978
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 555746a09e8bb7bcf99da91f9d00dfab9791d80c
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Oct 27 13:20:07 2017 -0700
+
+    MDSMonitor: wait for readable OSDMap before sanitizing
+    
+    Fixes: http://tracker.ceph.com/issues/21945
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit ca52f3bd93e6c743aa05171108527d877807b426)
+
+commit be13cdc0d4e54ce23c6ecf658100ddd860b1b96f
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Tue Oct 3 12:25:12 2017 -0700
+
+    mds: clean up non-existent data pools in MDSMap
+    
+    Older versions of Ceph weren't strict about preventing pool deletion when the
+    MDSMap referred to to-be-deleted pool. If we are dealing with a cluster
+    upgrade, we should try to gracefully handle that by cleaning out data pools
+    that have been removed.
+    
+    Reproduced this by allowing CephFS pools to be deleted:
+    
+    diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
+    index 85c47c13da6..694b240cb9f 100644
+    --- a/src/mon/OSDMonitor.cc
+    +++ b/src/mon/OSDMonitor.cc
+    @@ -10962,7 +10962,7 @@ int OSDMonitor::_check_remove_pool(int64_t pool_id, const pg_pool_t& pool,
+       FSMap const &pending_fsmap = mon->mdsmon()->get_pending();
+       if (pending_fsmap.pool_in_use(pool_id)) {
+         *ss << "pool '" << poolstr << "' is in use by CephFS";
+    -    return -EBUSY;
+    +    //return -EBUSY;
+       }
+    
+       if (pool.tier_of >= 0) {
+    
+    pdonnell@icewind ~/ceph/build$ bin/ceph osd pool create derp 4 4
+    pool 'derp' created
+    pdonnell@icewind ~/ceph/build$ bin/ceph fs add_data_pool cephfs_a derp
+    added data pool 3 to fsmap
+    pdonnell@icewind ~/ceph/build$ bin/ceph osd pool rm derp derp --yes-i-really-really-mean-it
+    pool 'derp' is in use by CephFSpool 'derp' removed
+    pdonnell@icewind ~/ceph/build$ bin/ceph fs ls
+    ...
+    2017-10-03 12:50:48.409561 7f9e2e05b700 -1 /home/pdonnell/ceph/src/osd/OSDMap.h: In function 'const string& OSDMap::get_pool_name(int64_t) const' thread 7f9e2e05b700 time 2017-10-03 12:50:48.407897
+    /home/pdonnell/ceph/src/osd/OSDMap.h: 1184: FAILED assert(i != pool_name.end())
+    
+     ceph version 12.1.2-2624-g37884a41964 (37884a419640b446fffc1fa4d6074c97339fdd96) mimic (dev)
+      1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0xf5) [0x564ebb5420f5]
+      2: (()+0x41dade) [0x564ebb3cbade]
+      3: (MDSMonitor::preprocess_command(boost::intrusive_ptr<MonOpRequest>)+0x1fb9) [0x564ebb4cd119]
+    
+    Note when testing this fix, use something like this after removing the data pool:
+    
+    pdonnell@icewind ~/ceph/build$ bin/ceph fs set cephfs_a max_mds 2
+    
+    Setting max_mds will cause a new FSMap to be created where MDSMap::sanitize is
+    called; this is simulating the initial load+sanitize of a Hammer legacy MDSMap
+    by the mons.
+    
+    Fixes: http://tracker.ceph.com/issues/21568
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    
+    (cherry picked from commit 7adf0fb819cc98702cd97214192770472eab5d27)
+
+commit 23fa3b726368f036b31e53a0bec8ad4bc654d993
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Tue Oct 3 12:23:03 2017 -0700
+
+    mds: reduce variable scope
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 37884a419640b446fffc1fa4d6074c97339fdd96)
+
+commit 4f024c68319a79bef9d937d7f2363a5b69575773
+Merge: 6166148078 5a0016131b
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Mon Oct 30 14:05:00 2017 -0400
+
+    Merge pull request #18627 from ceph/backport-18513
+    
+    ceph-volume lvm zap backport
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit 5a0016131b1ec900255057876c98fc70e00542f0
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Fri Oct 27 11:29:50 2017 -0500
+
+    ceph-volume: set journal_uuid and journal_device when using a partition
+    
+    This correctly sets the tags when a partition is used for a filestore
+    journal.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 1a8561d38dd7363dc920ae82ec1343b2b75a1ca2)
+
+commit 05522e75f8d98658b5440d3c2f70e93ce65c65ce
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Oct 24 10:13:40 2017 -0500
+
+    docs for ceph-volume lvm zap
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 3e93a31deda96ed6fb23fa19ce3e273de05f7d88)
+
+commit cb1e81447f5c0c4eb8c0fd9fb11d0ceae3e26199
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Mon Oct 23 09:51:43 2017 -0500
+
+    ceph-volume: add tests for ceph-volume lvm zap
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 3d5e391693fc64747a4774287c968f842294eaa6)
+
+commit 33927c10e78842bb4c4ce4c0e4051ccceb8bb9e8
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Mon Oct 23 09:44:07 2017 -0500
+
+    ceph-volume: print success message if zap succeeds
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 6d70762dee0cd9047c291cf9869f666a375e956b)
+
+commit d8381ab509d9f60f6fe62c5692a9374b3187d5c9
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Fri Oct 20 09:58:48 2017 -0500
+
+    ceph-volume: terminal.write only takes one argument
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit a5454eba241abd8cc1b1a660b12a1aec7c3c16e0)
+
+commit 6f2bd88278f2c7dc4cfd156969f98d7cc8c3e763
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Oct 19 16:29:39 2017 -0500
+
+    ceph-volume: the metadata field for the lv path is lv_path not path
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit e3a9113e7025f1e3a3130ec1e2d565f37bf3d2dc)
+
+commit 1dfe7e9854b00850b86168049821447d33298fff
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Oct 19 14:59:26 2017 -0500
+
+    ceph-volume: update help text for ceph-volume lvm zap
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit cbc38c4e4a7dcc24b31bcfa6af73eb8cf04f56ad)
+
+commit c5206435a9bae9d93e6cd9a3c2b98127871ad94e
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Oct 19 14:55:44 2017 -0500
+
+    ceph-volume: remove lvm metadata when zapping an lv
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 5f57fc87205c2d35da9a1f28c72e233ffb3fe4d9)
+
+commit eb53ef67b4e3bf5afb6e414fd42caab67c4cfdf2
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Oct 19 14:55:02 2017 -0500
+
+    ceph-volume: adds Volume.clear_tags
+    
+    Will remove all tags for the current lv
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 3c9401f1618f7fcbea827fe7279e8dfde617d957)
+
+commit 4ee13e0587c5e152fdac18004bc4336f8b90e4af
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Oct 19 12:01:57 2017 -0500
+
+    ceph-volume: zap should leave lvs and partitions intact
+    
+    This will remove filesystems and wipe data from any lv or partition
+    given to 'ceph-volume lvm zap' but still leave it intact for further
+    use.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit caba9fb80b358222192f736bd1b0ab31dca25cec)
+
+commit 8c3ab3459abfebac20bb4ce3ddecf1a7a58cf401
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Oct 19 11:44:11 2017 -0500
+
+    ceph-volume: adds utilites to wipe the fs and clear data when zapping
+    
+    These should eventually move to a disk api.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 2f64d4a0abd41afbcb9eba6a237642502d9a9135)
+
+commit 8a0e7fad57e41e10d51e0e167b671668ee1aa286
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Oct 19 10:35:51 2017 -0500
+
+    ceph-volume: adds tests for api.lvm.remove_lv
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 937b57438147681c2c6e32c6db38d8bea68d4731)
+
+commit c6dd47ab6d1ec6784ec3cef8f65139baeaf4fa3a
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Oct 19 10:35:10 2017 -0500
+
+    ceph-volume: api.lvm.remove_lv should return True when successful
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 7f055aa6deb3904cf0334e214c13a26098b08aa8)
+
+commit 71ae0b8cabc0a696e6762d7f377f9f4b28d2a963
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Oct 18 10:19:25 2017 -0500
+
+    ceph-volume: zap logical volumes
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 1d083a2191315ee94301c2f5f102a0906dd05fa8)
+
+commit 7110922c42ee30fa25dc9dacbfc8d0f9ae2f2a79
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Oct 18 10:19:00 2017 -0500
+
+    ceph-volume: adds a remove_lv command to the lvm api
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit cf98b6971ced0caa29ff4f7ac346df652177fa29)
+
+commit 731610f18b9e4109ce72a4467af209144e4a127d
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Oct 17 14:15:18 2017 -0500
+
+    ceph-volume: stubs out the ceph-volume lvm zap command
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 2e64b797ef6ae91623ffba8ae28d3f8ccc7d7b93)
+
+commit c2237c7c6dbcd5420d8ed87c6f2648c3f4e6edae
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 27 09:58:20 2017 -0400
+
+    qa/suites/rbd: run cls tests for all dependencies
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 27613a63d1f1ee6fa7327cd1c63b875c0e5247f8)
+
+commit 6b75efab390b3019e404f007090e9031328e1301
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 27 09:57:43 2017 -0400
+
+    cls/journal: fixed possible infinite loop in expire_tags
+    
+    Fixes: http://tracker.ceph.com/issues/21956
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9e66dca49591e50b9cab5df311f1dc217eb58fcc)
+
+commit 796e33647022de1fd831b31045242fb8e9d0e4d2
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Aug 30 18:46:49 2017 +0800
+
+    ceph-disk: unlock all partitions when activate
+    
+    should unlock all dmcrypted partitions when activating a device.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit ba2aa0cee9e077d8439ba31228b41beb2d827a04)
+
+commit 3e9aad1a116e02bb126010f310ca2267177b5162
+Author: Felix Winterhalter <felix@audiofair.de>
+Date:   Mon Jul 17 02:04:39 2017 +0200
+
+    ceph-disk activate unlocks bluestore data partition
+    
+    Signed-off-by: Felix Winterhalter <felix@audiofair.de>
+    (cherry picked from commit 1287caf2dbb5ef6e5f243fe1d23633946aef26f9)
+
+commit 6b6eceee027e571d8d3da4af144391f8da85b53e
+Author: Yao Zongyou <yaozongyou@vip.qq.com>
+Date:   Sat Oct 28 18:23:30 2017 +0800
+
+    ceph-bluestore-tool: the link target should not ending with new line
+    
+    Signed-off-by: Yao Zongyou <yaozongyou@vip.qq.com>
+    (cherry picked from commit 39c68d128cf29f8e0a617009d16c5edf471ead47)
+
+commit cc84812507607d0a91233a513088094533a2d6a7
+Author: Yao Zongyou <yaozongyou@vip.qq.com>
+Date:   Sat Oct 28 18:22:27 2017 +0800
+
+    ceph-bluestore-tool: the right action is prime-osd-dir not prime-osd-dev
+    
+    Signed-off-by: Yao Zongyou <yaozongyou@vip.qq.com>
+    (cherry picked from commit d418a04e9fed6bf91cc9acc5ac777eadf664c974)
+
+commit 9d8e5d763b173247f95f0769fd52372a8a7a5371
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 25 10:44:28 2017 +0100
+
+    mon: implement MDSMonitor::get_store_prefixes
+    
+    Fixes: http://tracker.ceph.com/issues/21534
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit a3c317b406c69b9d6a2d7df94806f4c308e6ee6a)
+
+commit 338af1688147babb4e92523972175fe09ebb34b6
+Author: huanwen ren <ren.huanwen@zte.com.cn>
+Date:   Mon Sep 25 15:01:02 2017 +0800
+
+    mon/mgr: sync mgr_command_descs table and mgr_metadata table
+    
+    sync mgr_command_descs table and mgr_metadata table to new mons
+    
+    Fixes: http://tracker.ceph.com/issues/21527
+    
+    Signed-off-by: huanwen ren <ren.huanwen@zte.com.cn>
+    (cherry picked from commit 13f6aa3aaa6de0aeccec67d7c6f3effe43dcae49)
+    
+    Conflict: in master PaxosService::get_store_prefixes(..) is marked const
+    while in luminous, the cleanup commit which added the `const` is not
+    backported yet, so drop the `const` in the backported commit.
+
+commit de07d2954afb012e51d2eaca12c1763e29a19eca
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 25 10:37:18 2017 +0100
+
+    mon/OSDMonitor: tidy prefix definitions
+    
+    We should define them in one place to make it easy
+    when updating get_store_prefixes.
+    
+    Fixes: http://tracker.ceph.com/issues/21534
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 889ac5166ad8fd30678325ddc2da59f45db53f06)
+
+commit 981e552d72c151b6bf2c9bb64b969f9446ef4789
+Author: huanwen ren <ren.huanwen@zte.com.cn>
+Date:   Mon Sep 25 14:55:55 2017 +0800
+
+    mon/osd_metadata: sync osd_metadata table
+    
+    sync osd_metadata table to new mons when add new mons
+    
+    Signed-off-by: huanwen ren <ren.huanwen@zte.com.cn>
+    (cherry picked from commit 755ec735107c75156ae4935f7255bbfe5dc384d9)
+
+commit 3f0ad5f453a259cc53f50b1a9515e8a720da4a1d
+Author: huanwen ren <ren.huanwen@zte.com.cn>
+Date:   Mon Sep 25 14:55:55 2017 +0800
+
+    mon/osd_metadata: sync osd_metadata table
+    
+    sync osd_metadata table to new mons when add new mons
+    
+    Signed-off-by: huanwen ren <ren.huanwen@zte.com.cn>
+    (cherry picked from commit 6c20433bfb049ac6c69f6f7a979006e8b9ea0e29)
+
+commit 61661480780e555fc501aec7c32163596e1e18d3
+Merge: c0ec364ad6 410434b3d2
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sun Oct 29 11:06:28 2017 -0700
+
+    Merge PR #18385 into luminous
+    
+    * refs/pull/18385/head:
+            mds: fix race in PurgeQueue::wait_for_recovery()
+            mds: open purge queue when transitioning out of standby replay
+            mds: always re-probe mds log when standby replay done
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit c0ec364ad6f54da7efbbb6b84ad34ea9782e3df2
+Merge: 5ee57e1b42 695d7ad511
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sun Oct 29 11:06:24 2017 -0700
+
+    Merge PR #18316 into luminous
+    
+    * refs/pull/18316/head:
+            mds: prevent trim count from underflowing
+    
+    Reviewed-by: Zheng Yan <zyan@redhat.com>
+
+commit 5ee57e1b42d93fc0dc77b1221c362753684b280e
+Merge: 84f4bf40da b99acfd601
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sun Oct 29 11:06:20 2017 -0700
+
+    Merge PR #18300 into luminous
+    
+    * refs/pull/18300/head:
+            mds: keep CInode::STATE_QUEUEDEXPORTPIN state when exporting inode
+    
+    Reviewed-by: Zheng Yan <zyan@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 84f4bf40dade5ef55eba139d63187959b98e84e5
+Merge: 2638d5039e 460268b5e9
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sun Oct 29 11:06:16 2017 -0700
+
+    Merge PR #17729 into luminous
+    
+    * refs/pull/17729/head:
+            ceph.in: validate service glob
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 2638d5039e615035563338cd428b44fb23d879ff
+Merge: f6f88c2a9b 2973b6d418
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sun Oct 29 11:00:18 2017 -0700
+
+    Merge PR #18299 into luminous
+    
+    * refs/pull/18299/head:
+            mds: update client metadata for already open session
+    
+    Reviewed-by: Zheng Yan <zyan@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit f6f88c2a9b40cf84658319a07a2bc26858763093
+Merge: 388901f091 f353a1e805
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sun Oct 29 11:00:14 2017 -0700
+
+    Merge PR #18298 into luminous
+    
+    * refs/pull/18298/head:
+            osdc/ObjectCacher: limit memory usage of BufferHead
+    
+    Reviewed-by: Zheng Yan <zyan@redhat.com>
+
+commit 388901f0911b2d45f5b5ca2d653da5c605d99e2d
+Merge: 45cfeae6eb c5ccbf1108
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sun Oct 29 11:00:10 2017 -0700
+
+    Merge PR #18085 into luminous
+    
+    * refs/pull/18085/head:
+            ceph_volume_client: fix setting caps for IDs
+    
+    Reviewed-by: Ramana Raja <rraja@redhat.com>
+
+commit 45cfeae6ebd5a0910c03ff323ecdd5740e569b41
+Merge: ecd5b0066f 1021fe235c
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sun Oct 29 11:00:06 2017 -0700
+
+    Merge PR #18030 into luminous
+    
+    * refs/pull/18030/head:
+            qa: relax cap expected value check
+            mds: improve cap min/max ratio descriptions
+            mds: fix whitespace
+            mds: cap client recall to min caps per client
+            mds: fix conf types
+            mds: fix whitespace
+            doc/cephfs: add client min cache and max cache ratio describe
+            mds: adding tunable features for caps_per_client
+    
+    Reviewed-by: Zheng Yan <zyan@redhat.com>
+
+commit ecd5b0066fc091f34a166989b2e68e165c390dd0
+Merge: 04acd559db d5e583490d
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sun Oct 29 10:55:32 2017 -0700
+
+    Merge PR #17921 into luminous
+    
+    * refs/pull/17921/head:
+            ceph_volume_client: perform snapshot operations in
+    
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+    Reviewed-by: Ramana Raja <rraja@redhat.com>
+
+commit add8c3db195acfda75a1628196e839b0147e7c3f
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Thu Oct 26 18:28:10 2017 -0400
+
+    rgw_file:  set s->obj_size from bytes_written
+    
+    Required to store correct final accounted_size of objects in
+    RGWWriteWriteRequest::write_finish.
+    
+    Fixes: http://tracker.ceph.com/issues/21940
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit b1f528d35aeccabb4e5818aec6feb8e53e562500)
+
+commit 8cbb2eb937cae4e136ef77bc93e6d5346e21c8da
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 9 08:15:21 2017 -0500
+
+    qa/cephfs: test ec data pool
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d0732fc96fbc6849dd51b391d85f765c74cfb593)
+
+commit 04acd559db55c6e8040a2ba65a850e72a04ad2ab
+Merge: d294493c42 8a87d43db2
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Fri Oct 27 11:34:15 2017 -0500
+
+    Merge pull request #18593 from ceph/luminous-wip-bz1499840
+    
+    luminous ceph-volume lvm bluestore support
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit d294493c42ea21e85a1fd36eb8623229a2ff6ea6
+Merge: 58c1b8c4f2 34cd96d342
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 17:30:35 2017 +0200
+
+    Merge pull request #18334 from kmroz/wip-21816-luminous
+    
+    luminous: rgw: fix bilog entries on multipart complete
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 58c1b8c4f28b73d9b7db1cf3fe79ae5062832c88
+Merge: 8742c619b0 d7f6b93af4
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 17:30:07 2017 +0200
+
+    Merge pull request #17861 from smithfarm/wip-21441-luminous
+    
+    luminous: rbd: [cli] mirror getter commands will fail if mirroring has never been enabled
+
+commit 8742c619b03cc5f048dc08f76d67ea64c109b4b3
+Merge: 7c03693d0b f8bfd9458d
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 17:29:55 2017 +0200
+
+    Merge pull request #18416 from kmroz/wip-21855-luminous
+    
+    luminous: librbd: object map batch update might cause OSD suicide timeout
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit 7c03693d0bd1a7108bd2d55eda7b7bd1e4e7f904
+Merge: fc486a26cb 8b3e9917a7
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 17:29:43 2017 +0200
+
+    Merge pull request #18337 from kmroz/wip-21640-luminous
+    
+    luminous: rbd-mirror: forced promotion can result in incorrect status
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit fc486a26cb12466df6269ec4c7717332be2ed92c
+Merge: c944c688e5 4ebd4b3928
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 17:29:22 2017 +0200
+
+    Merge pull request #18336 from kmroz/wip-21639-luminous
+    
+    luminous: librbd: snapshots should be created/removed against data pool
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit c944c688e5a4a7935233e7fb3824b97d9afe44d9
+Merge: bc7e648b18 34af07f97a
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 17:29:03 2017 +0200
+
+    Merge pull request #17860 from smithfarm/wip-21299-luminous
+    
+    luminous: rbd: [rbd-mirror] asok hook names not updated when image is renamed
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit 8a87d43db25a364841d969636bd74b7590c3d563
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 26 15:09:18 2017 -0400
+
+    ceph-volume lvm.activate remove links on activate before priming
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 634b5caaed7c9188e426727e83a0768bdbc51f0d)
+
+commit 869d13037dbb2b8882c4a4a91ec20c9814bcc438
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 26 07:48:24 2017 -0400
+
+    ceph-volume lvm.activate ceph-bluestore-tool needs --cluster too
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit fb36087bc3c7e7bb8358cbf8ad84409fa23fac68)
+
+commit 3baab9f4e286c3b54c179b0f471435c2dbc859b2
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 25 18:57:28 2017 -0400
+
+    ceph-volume lvm.activate consume cluster_name from lvm metadata
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 2b7ff497fe9e0e167ab01bf0d8ce5e26f0da836f)
+
+commit 8f41a983356ee9d182f8247f2da8db15848afae0
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 25 18:57:03 2017 -0400
+
+    ceph-volume lvm.prepare persist cluster_name on lvm metadata
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 6d38ad987e08791be379f6c006c1eb5bf9464415)
+
+commit 1ae30d3130af845ebf563a74a1e2642f5236f1ed
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 25 15:30:43 2017 -0400
+
+    ceph-volume util.prepare bluestore db and wal paths are to devices, not osd dir paths
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit fd616acd6ae0f184c1757e7de38fc746d526908c)
+
+commit fdfbfb8140fc1be3123d1254f4463613dc7c7cb0
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 25 15:30:06 2017 -0400
+
+    ceph-volume lvm.prepare bluestore wal and db links get created with mkfs
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 80698d2309a221d2318b11be9b8f8a95b51416ea)
+
+commit 6ac58f3cd5e292153c82d881a30db1f6c4bfd936
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 25 10:28:43 2017 -0400
+
+    ceph-volume lvm.prepare use wal and db for mkfs
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 4970ab5c804111ce4b2b57817f1e29d58b18a893)
+
+commit c9ecd14ca220e45960650715703cae3e4376e3c0
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 24 16:28:33 2017 -0400
+
+    ceph-volume lvm.prepare ensure tags are set for ceph.type correctly for block and data
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 58dbefd3be7bad51e6c56c50fbfb450e205b3036)
+
+commit 76fb5fec94a9bbfed1163a23faf87e19d5617bd4
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 24 16:26:03 2017 -0400
+
+    ceph-volume lvm.activate check if lv device is mounted at path
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 1ead135bef801280e7cec21283c95c7ba440d84b)
+
+commit 343362027368c4170bd785f1633afeb5ed90f3a2
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 24 15:55:21 2017 -0400
+
+    ceph-volume lvm.prepare default to using bluestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 39dd0d1d10aba08c3e8299eb1702260274a756a7)
+
+commit 0a27fe88266458eead5a011c27cc323f5b7173a5
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 24 15:54:57 2017 -0400
+
+    ceph-volume lvm.create default to using bluestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d3145da6e643dedb2fbed72e7d4d0cddf397c5e3)
+
+commit bd2e0aa78ad1a500935c3134f4d1e6c3413a26ae
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 24 15:54:11 2017 -0400
+
+    ceph-volume lvm.common remove boolean defaults for CLI flags
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a7d5965ab532a621f402bad55d45dfed61d90cc5)
+
+commit c3d474af562ec5b8658bed6362046b0208a2c0eb
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 24 13:29:18 2017 -0400
+
+    ceph-volume lvm.prepare update to use create_osd_path
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit bafb72144666f891773bed95e9a0f757673af2e0)
+
+commit bc7e648b182569865d1c2a2aea6da1e52a3d3cb1
+Merge: 690be9c711 b015ce1161
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:57:48 2017 +0200
+
+    Merge pull request #18138 from kmroz/wip-21684-luminous
+    
+    luminous: rgw: stop/join TokenCache revoke thread only if started.
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 690be9c711b34be1ec4254ba0cedf7d6755f59f6
+Merge: 5727e4a33e 06c31a6cee
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:55:22 2017 +0200
+
+    Merge pull request #17994 from ukernel/luminous-21337
+    
+    luminous: mds: make sure snap inode's last matches its parent dentry's last
+    
+    Reviewed-By: Patrick Donelly <pdonelly@redhat.com>
+
+commit 5727e4a33e0b82e0f9591251572b61458ec2cfae
+Merge: d973d6d7ae 624b3ac505
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:52:43 2017 +0200
+
+    Merge pull request #18431 from theanalyst/wip-21857-luminous
+    
+    luminous: rgw: We cant't get torrents if  objects are encrypted using SSE-C
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 02013a714ec5fa6c2ff1073e432b8f6b9ff41305
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Oct 23 09:44:27 2017 -0400
+
+    ceph-volume lvm.activate only prime-osd-dir when directory is empty
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 9bb6cfd867c2d8e972093d023622f8c2e5b440d4)
+
+commit 28593dbf1caeeec8f7c69dbd06e2ff9759afd072
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Oct 23 09:20:26 2017 -0400
+
+    ceph-volume lvm.activate safeguard against auto detect objectstore flag not being set
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit c9b5f352a8f9e2743e5dca3a0479c8099a59fd12)
+
+commit 8df51c8af5d95d67effece720ec2ff82517155b9
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 15:18:20 2017 -0400
+
+    ceph-volume util.system use strings for comparison in py3
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 1b671a8602b036cb8d53bc40eb70198c13143051)
+
+commit 1f09ee754fec40c4b17ea33829e6449d8749690a
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 15:10:08 2017 -0400
+
+    f ceph-volume tests update changes from the help menu in lvm
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 3ddcd3c2f19cc848a11a2882f150462ad2f26c65)
+
+commit ec7c42bd59dc05cb8bad082d64ffdc28f13d3dc8
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 14:12:02 2017 -0400
+
+    ceph-volume lvm.common --journal-size doesn't accept a volume group
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a8282e9a580bcfa74abdd33032e2ce2851b5cf29)
+
+commit 243098815d1d49570c193767d008cbc4fa0f174b
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 14:04:31 2017 -0400
+
+    ceph-volume lvm.prepare filestore does not require a volume group
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 3653f105927d455b8df143797af0cfe72f8d6834)
+
+commit da5f764a083613e5d74d1401da65c2ba3aff5fb4
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 14:04:05 2017 -0400
+
+    ceph-volume lvm.common volume groups are no longer required
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 526c0aa81ac8eed95c2b4664a00ded2118372fff)
+
+commit eaa9150d4a6bacf62fcf485e8d36430743ac06cf
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 12:07:06 2017 -0400
+
+    doc/ceph-volume activate workflow mention tmpfs in osd dirs
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 8788fe3721034f390240c01493bc1d27d65a73a5)
+
+commit d65af56eaaab728d8c81ddb69b967cb13f177766
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 12:02:10 2017 -0400
+
+    doc/ceph-volume update activation to include bluestore support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 288a3124793d11d71b540555fd03c2f61a79dd16)
+
+commit 6545b1c14e84af5f25fd4bbd405b56f7b5797113
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 11:58:47 2017 -0400
+
+    doc/ceph-volume update create to indicate bluestore support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 61ce8b4b844fcb64fc95373c5515de902a52fc8c)
+
+commit d973d6d7aefae1c34b664f98955baed4c841579b
+Merge: cf621f8b4e 5b5f0b5a33
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:49:03 2017 +0200
+
+    Merge pull request #17858 from smithfarm/wip-21448-luminous
+    
+    luminous: rgw: string_view instance points to expired memory in PrefixableSignatureHelper
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit cf621f8b4e8138284843f6d979aca61651a854bd
+Merge: e0c3a05959 0b3a974be7
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:48:54 2017 +0200
+
+    Merge pull request #18442 from theanalyst/wip-21637-luminous
+    
+    luminous: encryption: PutObj response does not include sse-kms headers
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+    Reviewed-By: Matt Benjamin <mbenjamin@redhat.com>
+
+commit e0c3a05959a70b4cf4213320ba7c990578bf8c19
+Merge: dd7aa70301 14a1dcb1ff
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:48:26 2017 +0200
+
+    Merge pull request #18437 from theanalyst/wip-21698-luminous
+    
+    luminous: radosgw-admin usage show loops indefinitly
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit dd7aa7030144d42958f92af5c7f6804a402f3b6d
+Merge: 3a8f046716 c3a719da27
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:48:12 2017 +0200
+
+    Merge pull request #18434 from theanalyst/wip-21817-luminous
+    
+    luminous: rgw: zone compression type is not validated
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+    Reviewed-By: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 3a8f046716456eb0f51db79f152f3dc67d84c319
+Merge: 0d9b3a01d0 4c18ac1240
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:47:48 2017 +0200
+
+    Merge pull request #18440 from theanalyst/wip-21652-luminous
+    
+    luminous: policy checks missing from Get/SetRequestPayment operations
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 0d9b3a01d0972a2fc5ae9550ba80c38acd9edbeb
+Merge: aa71baeabe a3e2ae4925
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:47:33 2017 +0200
+
+    Merge pull request #18441 from theanalyst/wip-21651-luminous
+    
+    luminous: rgw: avoid logging keystone revocation failures when no keystone is configured
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit aa71baeabef5aa7eb6903593b3dfcc8878227c99
+Merge: 1067293a7e 0b84dd07c4
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:47:13 2017 +0200
+
+    Merge pull request #18443 from theanalyst/wip-21634-luminous
+    
+    luminous:  s3:GetBucketLocation bucket policy fails with 403
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 1067293a7e21fc835447c9e5e1d8416902593ba1
+Merge: f2f8aef89c abefb578c3
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:46:59 2017 +0200
+
+    Merge pull request #18444 from theanalyst/wip-21635-luminous
+    
+    luminous: s3:GetBucketCORS/s3:PutBucketCORS policy fails with 403
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit f2f8aef89c3144ac5d859ea0fe8a7db57cfa8ad6
+Merge: 385d84b8eb 0e65ea466f
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:46:44 2017 +0200
+
+    Merge pull request #18430 from theanalyst/wip-21695-luminous
+    
+    luminous: failed CompleteMultipartUpload request does not release lock
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+    Reviewed-By: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 385d84b8ebb3a4192ce4817aaf6c99b27af0190d
+Merge: 4a74a41c47 f9850d2148
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:46:13 2017 +0200
+
+    Merge pull request #18445 from theanalyst/wip-21633-luminous
+    
+    luminous: s3:GetBucketWebsite/PutBucketWebsite fails with 403
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 4a74a41c473fcda61f156d8bf720bf0e4cf0aec0
+Merge: 14cb83cd07 8b420c4627
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:45:54 2017 +0200
+
+    Merge pull request #18436 from theanalyst/wip-21789-luminous
+    
+    luminous: user creation can overwrite existing user even if different uid is given
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 14cb83cd077258e061cfa09733cf0b9a82f7507a
+Merge: 0339b8b599 fe45086a54
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:45:36 2017 +0200
+
+    Merge pull request #18435 from theanalyst/wip-21790-luminous
+    
+    luminous: RGW: Multipart upload may double the quota
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 0339b8b599f313e6a7b176f0f8b94b8101ab50c3
+Merge: 68cc868d1f 248bd94dab
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:45:19 2017 +0200
+
+    Merge pull request #18432 from theanalyst/wip-21856-luminous
+    
+    disable dynamic resharding in multisite enviorment
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 68cc868d1fdca55308fa1aba0bb708d07170e668
+Merge: f337bef88a 17df0ba8f0
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:44:40 2017 +0200
+
+    Merge pull request #18429 from theanalyst/wip-21792-luminous
+    
+    luminous: encryption: reject requests that don't provide all expected headers
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 29470eac4ca939c824b56f4fbc03c8b7814f9d02
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 11:51:55 2017 -0400
+
+    doc/ceph-volume update prepare with bluestore workflow
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 9f1a5627399e2589fe706b158a32a9fb8642ac23)
+
+commit c90e473ff35d15d309eb92dbdc6b9df2e8bd5ecc
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 10:15:09 2017 -0400
+
+    doc/ceph-volume update filestore to indicate it is not the only/default objectstore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit c93603e5764e021305cfeb1c297ee16318126fe1)
+
+commit 3890c47dddd0d028b0c003cb46a9bec670c445f1
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 10:00:17 2017 -0400
+
+    doc/ceph-volume elaborate on env vars for interval and tries in activation
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 1dc24d1a7065b4e23b1bfa0e7c6d263b5ebc6906)
+
+commit 88240acf19176ae5c139836b7d46848e15ef5454
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 09:33:55 2017 -0400
+
+    ceph-volume lvm.common update cli flags help for bluestore support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit c9693fb3f8386016aba5d7c8f53f5f71f948c88d)
+
+commit a945e7776309d39955918cec7e0c5f1895abc39b
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 09:23:35 2017 -0400
+
+    ceph-volume util.disk st_mode is needed for stat()
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit e3b5663f7968e44ec0cd6db2bfc43acdb233e314)
+
+commit e833916afe22e9c4765344229b5b89cfc9af1b14
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 09:23:02 2017 -0400
+
+    ceph-volume decorators always log to the file with exceptions
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit cb3533a72571251d9f1b6171f783246462726857)
+
+commit 53e9cf6fcd7549b4bfa78d7568d56b30b38e81fd
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 08:49:32 2017 -0400
+
+    ceph-volume tests verify behavior of get_lv_from_argument
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit adabbd462bcb19a6da65211e24dfb2b9b5043ef4)
+
+commit 40cfec281cc2584b2e2987f356109c6c160d4597
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 08:48:01 2017 -0400
+
+    ceph-volume api.lvm return result of get_lv when using arguments
+    
+    If the argument is an absolute path it is fine to just return whatever
+    get_lv finds since it is a "safe" call, it will return a None if nothing
+    is found
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit f209c9d4d50bd7abe0f7f6b9fb29359cb7196718)
+
+commit 7139077aab892c3213815d1dcf71d93df80a32c8
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 08:12:01 2017 -0400
+
+    ceph-volume lvm.listing support bluestore, not only filestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 6a5b89a475c2a36022ca09dbb7f6b94cc3b44881)
+
+commit 654e06f1202596d5cecef9d03134861eed053882
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 08:10:51 2017 -0400
+
+    ceph-volume api.lvm allow full paths to detect lv from arguments
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 8d257101b1556ec5ce805622e9b7b6b74f6b000d)
+
+commit 454d48eb29f4429370b831645a29e0450b5b46b1
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 20 08:10:26 2017 -0400
+
+    ceph-volume tests add support for bluestore list verification
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 4674da9c07f871bf1f25bb782c136fe082095a14)
+
+commit 8d72737f90b0cd1db860678ebb66596e233fd646
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 19 13:20:55 2017 -0400
+
+    ceph-volume tests activate with fsid works for bluestore and filestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 5b9f0853f780db3b0af6b4b7c1ad1d59184069c9)
+
+commit 846aa2492f6db342d9fb4c477c2e13e61e414aaf
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 19 13:14:05 2017 -0400
+
+    ceph-volume tests arg validator can accept absolute paths
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 15586715f506505a57ec5122416541479df74bc8)
+
+commit 2c9ee51a0a7ed5edc900316c19ba27facb9163eb
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 19 12:03:17 2017 -0400
+
+    ceph-volume tests update changes from the help menu in lvm
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 39bdc2e3f621a32bcbb3dfe64f156bbfd7e4f5ee)
+
+commit 5724bb04bb74e17b9f7e96b3a3e434cac720c4ed
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 19 11:46:57 2017 -0400
+
+    ceph-volume tests update to get_lv from get_journal_lv
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 9be1012072f36e21a3f3cef42a42f7f1eb83c64e)
+
+commit 43e25933701605ac8cd7cb3d6fbd4cbf6705565e
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 19 11:30:29 2017 -0400
+
+    ceph-volume tests.util new path/device mount interface
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d5984bad97347e58318793fc05da84832d323113)
+
+commit 9b36a8bf8d471047dc94cdfcac1f84a9802b9fc5
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 19 09:49:25 2017 -0400
+
+    ceph-volume tests update api.lvm tests
+    
+    The create_lv signature changed to require full size description and
+    tags need to be an actual dictionary (vs. keyword args)
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit e8a18637531cde59d73a97d106958489c1d12117)
+
+commit ea57e70e6f826ac17eae0226b9d916e8bff4d021
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 18 16:06:55 2017 -0400
+
+    ceph-volume lvm.activate ceph-bluestore-tool should populate the osd dir
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit f3bb616bcf8a4737a3701ebf812de831785d6341)
+
+commit fe3b61ac0da67d9c80ead24c6ccf79e1507fa132
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 18 15:06:01 2017 -0400
+
+    ceph-volume lvm.activate auto detect objectstore type
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 472aa0b1997a0b1301b699b7ce30dce728aaea0c)
+
+commit 5da9775184e4bc9bf1b83d18eae602c9a510a89f
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 18 15:00:19 2017 -0400
+
+    ceph-volume lvm.trigger enable objectstore auto detection for activate
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit f579ef07026564f1a94b2923a2e4640ade2da503)
+
+commit 307c99e4382895e6b42c193e9b98b9206fd57f9a
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 18 12:00:34 2017 -0400
+
+    ceph-volume lvm.activate add bluestore support for activation
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 5696fed09b65044142b34f89492a6d91d3243e14)
+
+commit 70632b0fac2fa7e411418c90a50b99391379f436
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 18 11:40:38 2017 -0400
+
+    ceph-volume util.system create mappings of mounts
+    
+    This makes it easier for consumers to detect a path, or a device that
+    might be mounted or might have more than one mount, including tmpfs and
+    devtmpfs support, which was just not possible before.
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit f61007cc650f6964ef2ff3ea98a1debf02713198)
+
+commit 839dc1f554a12fb109ca1f992784dd4f4933898a
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Oct 16 06:51:05 2017 -0400
+
+    ceph-volume util.prepare separate filestore vs. bluestore prepare utils
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 85cdc783008ff0e0566c34aaab7cd5f12db1cbc7)
+
+commit deae88983d80cedcdc842d192f8d659b7533d3dc
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Oct 16 06:50:27 2017 -0400
+
+    ceph-volume process allow obfuscating a key or index when logging commands
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit bbcca70d962855d991f0aa8e86a19f1855345668)
+
+commit 18336242b85065d8a2b0a92f22b2e91a81b997ba
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 13 09:22:53 2017 -0400
+
+    ceph-volume util.prepare add other ln helpers for bluestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 555b664882db2cd247087f907bdf4d426fb652f8)
+
+commit 37a1564a41ba8ae6a89eb0903f0fdf6b463d90f0
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 12 16:08:52 2017 -0400
+
+    ceph-volume util.arg_validators allow to pass a group as --data for lvm
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 211c38f591945dee53196068533784c385bae102)
+
+commit afd1588fe06a80f2277082fd7bd15f43d8e601b7
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 12 16:08:16 2017 -0400
+
+    ceph-volume util.disk add utilities for is_device and is_partition
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 6b23fc72240ced158cf5335f6e815dbfd804dfab)
+
+commit 5e8f798d2b8f80b21f3ce4701d22e47992dd0938
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 12 16:05:22 2017 -0400
+
+    ceph-volume api.lvm udpate create_lv for bluestore
+    
+    Require full size notation for LVM (e.g. 50G or 100M) and do not
+    "translate" tags by pre-fixing them with 'ceph'. That is already how the
+    rest of the API works, no need to make that translation in one place
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit ac0d1555d3d9caeef9d4fc3e46bc11b7e437413b)
+
+commit f337bef88a31a5d23420665406a06f761e8d4e6e
+Merge: 3db3ca9938 d2be1e0ccb
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:43:43 2017 +0200
+
+    Merge pull request #17857 from smithfarm/wip-21446-luminous
+    
+    luminous: rgw:multisite: Get bucket location which is located in another zonegroup, will return 301 Moved Permanently
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 3db3ca99380521c79596b06b695eefbb81d57908
+Merge: ced01a5b85 c4d6539374
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:43:00 2017 +0200
+
+    Merge pull request #17856 from smithfarm/wip-21444-luminous
+    
+    luminous: rgw: setxattrs call leads to different mtimes for bucket index and object
+    
+    Reviewed-By: Casey Bodley <cbodley@redhat.com>
+
+commit 3bbd840fe0d34d42c04fac523aee5fc023d8bd3b
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 12 15:42:22 2017 -0400
+
+    ceph-volume lvm.prepare update filestore workflow to use new helpers
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 1e8a79b8f4b9e6360e8e3ace580e46caeedca8c7)
+
+commit ced01a5b85c9ec5657d8f09a447b6308ea597754
+Merge: 2c9a9af424 fe0a41b4a9
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:41:15 2017 +0200
+
+    Merge pull request #18417 from kmroz/wip-21782-luminous
+    
+    luminous: cls/journal: possible infinite loop within tag_list class method
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit 2c9a9af424c618c50334c0efa7f5a5f807aeb4fd
+Merge: a1d286252a 7d2b7e8e31
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:38:04 2017 +0200
+
+    Merge pull request #18287 from smithfarm/wip-retract-assertions-luminous
+    
+    luminous: rgw: Remove assertions in IAM Policy
+    
+    Reviewed-By: Adam Emerson <aemerson@redhat.com>
+    Reviewed-By: Abhishek Lekshmanan <abhishek@suse.com>
+
+commit fb95f3f9fd6122f9b8c3c2215463fc8a1ef62ba2
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 12 14:19:07 2017 -0400
+
+    ceph-volume lvm.prepare initial take on bluestore support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit e4fc3464af472a8dbdf049917eed73519ff82c3b)
+
+commit a1d286252a75c3632a7ecc63a366e5262f9e16cd
+Merge: b151619a45 fd207447d1
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 16:29:02 2017 +0200
+
+    Merge pull request #17859 from smithfarm/wip-21451-luminous
+    
+    luminous: rgw: lc process only schdule the first item of lc objects
+    
+    Reviewed-By: Daniel Gryniewicz <dgryniewicz@redhat.com>
+
+commit 26ef33efc37cdd626c6a7577812bb87947417e01
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Oct 12 14:16:25 2017 -0400
+
+    ceph-volume lvm.common update help flags for bluestore usage
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 17028f3a3c1f9394fa82dc6f2bbd596d96acb037)
+
+commit 005359b92a910b72712edbcaa7b2e27a54d3a093
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 11 14:30:41 2017 -0400
+
+    ceph-volume api.lvm allow to create a vg
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 16d7feb2b5d5eb5fa0bf23a95b60623ae2e1676f)
+
+commit 795e69673f452227afc12d76ed5aa0d6a8af8665
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 10 12:30:06 2017 -0400
+
+    ceph-volume lvm add block CLI flags for bluestore
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 63d2afbc605ac1531bc9f00a1891017bfd42fbce)
+
+commit cbfb082c180e93cd147e164afd1d32ef9234924e
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 10 12:15:14 2017 -0400
+
+    ceph-volume lvm make bluestore the default option
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit dc3ced30747312dedd0e895a75ec15b1925ef1ea)
+
+commit a340f1acab04587f5b2c323abc1a14b8dd7de915
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Oct 9 14:46:47 2017 -0400
+
+    rgw: RGWDataSyncControlCR retries on all errors
+    
+    similar to RGWMetaSyncShardControlCR, we don't want to exit and
+    stop the data sync processor thread on failures. we want to keep
+    retrying with backoff
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 065e67bbd312240f5128c31f5525098c62f3afce)
+    
+    Conflicts:
+            src/rgw/rgw_data_sync.cc (sync tracing)
+
+commit fdec63a28e24e056938139591e5799a0e3693c20
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Oct 9 14:09:40 2017 -0400
+
+    rgw: fix error handling in ListBucketIndexesCR
+    
+    the call to set_state() returns 0, when we want operate() to return the
+    error code instead. use set_cr_error() to do this
+    
+    Fixes: http://tracker.ceph.com/issues/21735
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit ed6340a47f32472bb1f650407dbe739c44bf4bd6)
+
+commit e711cf319fa12ec7b03edda769685353e24cbfcb
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Oct 9 14:08:21 2017 -0400
+
+    rgw: ListBucketIndexesCR spawns entries_index after listing metadata
+    
+    if the metadata listing fails, we won't have to clean up entries_index
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 7f127f54406ef0e08a1265b9bc20433fe3f21523)
+
+commit 56359c367d8267281f714dcecefdb50c598718a6
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 26 13:51:40 2017 -0500
+
+    ceph-bluestore-tool: prime-osd-dir: update symlinks instead of bailing
+    
+    If the symlink points to the right location, do nothing.  If it doesn't,
+    replace it.  If it's not a symlink, bail with EEXIST.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit de8dc42d42218bc1a1779e1bcc5831c567853c8d)
+
+commit b151619a4523b148026e613860a05add6aff3831
+Merge: 84fc30873a 352373ff4a
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Oct 27 15:11:00 2017 +0200
+
+    Merge pull request #18433 from theanalyst/wip-21545-luminous
+    
+    luminous: rgw file write error
+    
+    Reviewed-By: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 84fc30873aa6ed3c1bf0c91307c362d27e9bc72a
+Merge: 2bc24caa00 bb8586b259
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Fri Oct 27 08:03:16 2017 -0500
+
+    Merge pull request #18587 from ceph/luminous-wip-volume-api-change
+    
+    luminous ceph-volume lvm api refactor/move
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit 2bc24caa003ea3c4898202a887a5864078c0f3fd
+Merge: a5548a8247 5f96edfb41
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Oct 27 06:39:58 2017 -0500
+
+    Merge pull request #18568 from liewegas/wip-fix-rop-leak-luminous-2
+    
+    luminous: osd/PG: fix recovery op leak due to recovery preemption
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit bb8586b259726eb0ae72db7c6a238b2d2432fc9b
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 3 16:13:33 2017 -0400
+
+    ceph-volume create an api module to hold common api modules
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit f72368641d7f38b6395afb7ed70a0a9d0794a03a)
+
+commit 3d32fb30193437b57535c79657ee726704f4b037
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 4 06:44:54 2017 -0400
+
+    ceph-volume tests move lvm api tests into its new test module
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit eec0c8a5c27adc9b3fc5e999cbee8165bdb45736)
+
+commit e6145af90c3f1c8c520bb7671194e6d8a13bb5d2
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 4 06:43:36 2017 -0400
+
+    ceph-volume tests refactor lvm api import for test_listing
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 66fd41fd4b35e275196bfb94794f3fda9fd77f76)
+
+commit c63d3ba89489cc42aaa364c99b3d3b4c1bba226c
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 4 06:43:18 2017 -0400
+
+    ceph-volume tests refactor lvm api import for api tests
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 2a9a36bd24bddb35a7618127a0ec57f661a71566)
+
+commit 6efd25344230c0abb51cd82f99310a4b94f2fcd3
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 4 06:42:47 2017 -0400
+
+    ceph-volume tests refactor lvm api import for activate tests
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d81aa8c112312f2fb49d94ce3512b3356b0b8814)
+
+commit 47cd005b672828aedf11aeb67a6099477783f7f0
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Oct 4 06:42:01 2017 -0400
+
+    ceph-volume tests refactor lvm api import in conftest
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a00aff7066fe6a679498ac8a02ec22ef29feced3)
+
+commit 4a6dfa9b6949bdcdba22a7003c451390105d287e
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 3 16:17:08 2017 -0400
+
+    ceph-volume devices.lvm refactor importing api to come from api/lvm.py
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit f4bee6cb42a017abce269ed3926deff04038a47e)
+
+commit 9a20f184676973222c931cecff26ca313e636e1d
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 3 16:14:16 2017 -0400
+
+    ceph-volume move lvm/api.py to api/lvm.py so disk can consume it
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit bb72480d2bf44ff04ea93c98f4a4e59032a28896)
+
+commit f26f3dc842c04ac70e309266aaedbae9d45eeae8
+Author: yuliyang <yuliyang@cmss.chinamobile.com>
+Date:   Sun Sep 24 08:41:04 2017 +0800
+
+    rgw:fix list objects with marker when bucket is enable versioning
+    
+    fix: http://tracker.ceph.com/issues/21500
+    
+    Signed-off-by: yuliyang <yuliyang@cmss.chinamobile.com>
+    (cherry picked from commit bc16b162cf757b9c6ceae30912eb647ef9304f75)
+
+commit a5548a8247cd9b072a2e0357cf6d0358550e7565
+Merge: 724609a8a5 1751a094d7
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 26 16:52:28 2017 -0500
+
+    Merge pull request #18498 from liewegas/wip-recovery-fixes-luminous
+    
+    luminous: osd: fix recovery priority and pg state on recovery->backfill transition
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com.
+
+commit 724609a8a54a7619b48f4608790d81a2a43d4683
+Merge: 50c4cc0610 565b5cd1d6
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Thu Oct 26 16:50:20 2017 -0500
+
+    Merge pull request #18567 from ceph/backport-wip-bz1491250
+    
+    luminous: ceph-volume lvm list
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit 5f96edfb41c4b028732cf12da5ab44322e251b2a
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 26 16:23:41 2017 -0500
+
+    osd/PG: fix recovery op leak due to recovery preemption
+    
+    This was fixed in master in a different patch, but are not yet ready to
+    backport the bits there that came before this.  For now, fix it
+    specifically for luminous.  We can either sort out the conflicts later
+    or revert this and backport the master parts conflict-free.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 50c4cc06101fd1c8a9323cd3a04c92c8a9ab2580
+Merge: a175c0321e 33aa167331
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 26 16:14:43 2017 -0500
+
+    Merge pull request #17805 from tchaikov/wip-luminous-pr-17508
+    
+    luminous: mon: more aggressively convert crush rulesets -> distinct rules
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit a175c0321e745ac095707d5a2d3cd25232272bfc
+Merge: 72b057187a b382db7cfc
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 26 16:14:07 2017 -0500
+
+    Merge pull request #18364 from kmroz/wip-21544-luminous
+    
+    luminous: mon: osd feature checks with 0 up osds
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 72b057187aed9b4e5f3a4ecf82fafa6e9cd2d3fd
+Merge: f6362e3110 b60d8cf0cb
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 26 16:13:31 2017 -0500
+
+    Merge pull request #18413 from kmroz/wip-21693-luminous
+    
+    luminous: osd: additional protection for out-of-bounds EC reads
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit f6362e3110e4e482ff5a676035b862b1b0b666df
+Merge: f36670e190 4c6b1b65f7
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 26 16:11:42 2017 -0500
+
+    Merge pull request #18529 from smithfarm/wip-21918-luminous
+    
+    luminous: Disable messenger logging (debug ms = 0/0) for clients unless overridden.
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit f36670e190ea96589c373d87d30ee8f4b3c6b20b
+Merge: 9a28a5c9d7 cec24f99ab
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 26 16:11:21 2017 -0500
+
+    Merge pull request #18540 from liewegas/wip-pr-17894-luminous
+    
+    luminous: ceph.spec.in,debian/rules: change aio-max-nr to 1048576
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Boris Ranto <branto@redhat.com>
+
+commit 565b5cd1d68437096e283ccb942d8f108c26d3d4
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Wed Sep 13 08:23:40 2017 -0400
+
+    ceph-volume lvm add listing module to main
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit ff72fd0ed52ef2d166007c7769c0140424982dc7)
+
+commit 58095350953140a1fa4b968c2e0a5774a960e739
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 3 11:41:10 2017 -0400
+
+    doc/ceph-volume add lvm/list to the toctree
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d11c80c917f44b33077715b19564aeea30c78e42)
+
+commit 3893947d3e04f049a85245750b841adbbcefeac8
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 3 11:36:06 2017 -0400
+
+    doc/ceph-volume create the reference for the list sub-command
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 9a52939ef8fc4abe720d2525c59997f4f9760902)
+
+commit 0992c3d7c136a58ff38f521fc0978ff036843d27
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 3 11:35:39 2017 -0400
+
+    doc/ceph-volume add create and list to the index
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit ed5fa519aca1335039bfc32bc060af5181265517)
+
+commit 148537cc333d88b75640506c15a1bc221c9e09d6
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 3 08:59:37 2017 -0400
+
+    ceph-volume tests create a unit test module for listing features
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 2c2cf547b07f9510146b86d0746190b6d3fead3d)
+
+commit 285bc06569c8ed06890d23e9537a9d1106825b7f
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Oct 3 08:59:01 2017 -0400
+
+    ceph-volume tests create a small factory fixture
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 5a0b17de81770ef3b851a1ff98232d93c78f49f9)
+
+commit 1e4e241d9c77796874a003799a0546fc185c5241
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Oct 2 14:40:41 2017 -0400
+
+    ceph-volume tests ensure Volume.as_dict gets populated correctly
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit c5b933a96d067fb8c7f753a67123e5662398f964)
+
+commit a0ab94e8d70ae9fc5d741c47a8f0dcfe033f9a5e
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Oct 2 14:31:57 2017 -0400
+
+    ceph-volume lvm.listing initial take on listing command
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 2d80190a039798a637383513f292e6aab62cd3f8)
+
+commit 454c655f2eb0b1e79ca8e1e1d87ff9c0fe4886da
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Oct 2 12:35:13 2017 -0400
+
+    ceph-volume lvm.api be able to return a dictionary repr of a Volume object
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 0cae22750f396d1c1664875edb2742c50a41bffd)
+
+commit 1390fb76d76b76428e6a9210005dbf01fd3b3a1a
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Oct 2 12:12:57 2017 -0400
+
+    ceph-volume process allows to fully mute terminal output on commands
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 233f64a4fd90a19c50d57b849ffaa6a25c835f34)
+
+commit 00e9ec1fa25ac3950f43ec3cf84f8321b04507eb
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Sep 14 07:35:47 2017 -0400
+
+    ceph-volume lvm.api use a helper to get an lv from an arg
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 3c5bbbc5822f02a038f093647c5de58bef2fe8f5)
+
+commit 26c523c3beeb5da7e855be64a053ac9dca7d10e2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Oct 23 14:53:30 2017 -0400
+
+    rbd-mirror: strip environment/CLI overrides for remote cluster
+    
+    Fixes: http://tracker.ceph.com/issues/21894
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 70dc22e03a115b130b9c214030d5996276a1634b)
+
+commit 9a28a5c9d7ae618f5b79d3727ba155b5066a2653
+Merge: 42172a4431 18a99f5f6b
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Oct 25 11:45:38 2017 -0700
+
+    Merge pull request #18456 from liewegas/wip-21882-luminous
+    
+    luminous: messages/MOSDMap: do compat reencode of crush map, too
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 18a99f5f6b4976f87dcd0d4fe7e34fddd90de22b
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 25 07:00:14 2017 -0500
+
+    qa/suites/upgrade/jewel-x: make sure min compat client is hammer for straw2
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit cec24f99ab5e368d2dae7d53628fee09515bd873
+Author: chenliuzhong <liuzhong.chen@easystack.cn>
+Date:   Tue Oct 24 10:54:33 2017 +0800
+
+    ceph.spec.in,debian/rules: change aio-max-nr to 1048576
+    
+    when osd is more than 14 in one host,it report error that aio is not enough.
+    As the default aio-max-nr is 65536, one OSD needs 4096 aios and other programs may use aios.
+    This patch change aio-max-nr to 1048576 when install ceph-osd rpm package and debian package
+    
+    Signed-off-by: chenliuzhong <liuzhong.chen@easystack.cn>
+    (cherry picked from commit 36326dc7104fc2f20f19d51b6f618a029ba072d7)
+
+commit 15a52ba8329831ddf350664e6e6612523f98dcf4
+Author: baixueyu <baixueyu@inspur.com>
+Date:   Wed Oct 18 11:16:20 2017 +0800
+
+    You can find the problem do like this:
+    
+    upload obj to bucket1
+    s3cmd put obj s3://bucket1
+    cp obj from bucket1 to bucket2
+    s3cmd cp s3://bucket1/obj s3://bucket2
+    del obj from bucket1 and bucket2
+    s3cmd del s3://bucket1/obj
+    s3cmd del s3://bucket2/obj
+    you can see the pool 'default.rgw.buckets.data', the data of obj can not deleted.
+    ceph df | grep default.rgw.buckets.data
+    
+    Fixes: http://tracker.ceph.com/issues/21819
+    
+    Signed-off-by: Xueyu Bai baixueyu@inspur.com
+    (cherry picked from commit 2e11545077c485d5c155ab932b6274a8ac863e61)
+
+commit 4c6b1b65f7337809682bf907be94fe946909959a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 19 22:13:36 2017 -0400
+
+    common/common_init: disable ms subsystem log gathering for clients
+    
+    The log gathering causes large performance degradation to clients
+    with high message throughputs. This is hopefully a short-term
+    workaround until per-message logging can be replaced with an
+    efficient data recording system for post-incident analysis
+    use-cases.
+    
+    Fixes: http://tracker.ceph.com/issues/21860
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a3a40413f7908b08c40dec4020034cca4a0c4798)
+
+commit 42172a443183ffe6b36e85770e53fe678db293bf
+Merge: f2afb7a0d5 9f49698fab
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Oct 24 21:36:18 2017 -0500
+
+    Merge pull request #18503 from liewegas/wip-21878-luminous
+    
+    luminous: os/bluestore/BlueFS: fix race with log flush during async log compaction
+    
+    Reviewed-by: Varada Kari <varada.kari@gmail.com>
+
+commit f2afb7a0d590ff55ae11f5bf3b13264a56380ce6
+Merge: d0be2106f9 3ad21109ed
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Oct 24 21:35:29 2017 -0500
+
+    Merge pull request #18501 from liewegas/wip-21766-luminous
+    
+    luminous: os/bluestore: handle compressed extents in blob unsharing checks
+
+commit d0be2106f96377f059ec08800e32e492ec6f82a3
+Merge: d52e59bad7 e6eeb796b4
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Oct 24 21:34:52 2017 -0500
+
+    Merge pull request #18459 from liewegas/wip-pr-18457-luminous
+    
+    luminous: qa/suites/rados/rest/mgr-restful: whitelist more health
+
+commit 902d467a0ec23baebf28a2419a4c0c78e0b1b662
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Oct 9 10:11:14 2017 -0400
+
+    qa/rgw: ignore errors from 'pool application enable'
+    
+    Fixes: http://tracker.ceph.com/issues/21715
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 160240e0c1c216c950b7210a561b27436198de1f)
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 1751a094d76ac84fbcecfc9816904cbb496bd765
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 23 17:11:59 2017 -0500
+
+    osd/PrimaryLogPG: clear DEGRADED at recovery completion even if more backfill
+    
+    We may have log recovery *and* backfill to do, but cease to be degraded
+    as soon as the log recovery portion is done.  If that's the case, clear
+    the DEGRADED bit so that the PG state is not misleading.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3e91fda79d3be771f46394312a6b72ab75660012)
+
+commit d52e59bad700e8add6cd4cf895817aa8fd7a601a
+Merge: 724a51cd44 0048e6a58c
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Tue Oct 24 11:51:19 2017 +0800
+
+    Merge pull request #18491 from liewegas/wip-21573-luminous
+    
+    luminous: buffer: fix ABI breakage by removing list _mempool member
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 9f49698fab5de9ade79d3aba957afe82daa4fecc
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 20 08:51:17 2017 -0500
+
+    os/bluestore/BlueFS: fix race with log flush during async log compaction
+    
+    During async log compaction we rely on _flush-and_sync_log to update the
+    log_writer to jump_to.  However, if racing threads are also trying to flush
+    the log and manage to flush our new log events for us, then our flush will
+    turn into a no-op, and we won't update jump_to correctly at all.  This
+    results in a corrupted log size a bit later one.
+    
+    Fix by ensuring that there are no in-progress flushes before we add our
+    log entries.  Also, add asserts to _flush_and_sync_log to make sure we
+    never bail out early if jump_to is set (which would indicate this or
+    another similar bug is still present).
+    
+    Fixes: http://tracker.ceph.com/issues/21878
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4324c8bc7e66633035c15995e3f82ef91d3a5e8c)
+
+commit 3ad21109ed2df068bb113e8551505b850fbfcb30
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 11 16:48:41 2017 -0500
+
+    os/bluestore: handle compressed extents in blob unsharing checks
+    
+    If the blob is compressed, we aren't mapping to a range within
+    the allocated extents, but rather referencing the entire blob.
+    
+    Fixes: http://tracker.ceph.com/issues/21766
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit eb26cfbfbb2b9b9fe84dfe9c999e2d67fd2661c0)
+
+commit 8f176f31d091cb53e1baa8c317df9e64bef4d95e
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Oct 22 22:46:00 2017 -0500
+
+    osd/PG: on recovery done, requeue for backfill
+    
+    We were keeping our existing recovery reservation slot (with a high
+    priority) and going straight to waiting for backfill reservations on
+    the peers.  This is a problem because the reserver thinks we're doing
+    high priority work when we're actually doing lower-priority backfill.
+    
+    Fix by closing out our recovery reservation and going to the
+    WaitLocalBackfillReserved state, where we'll re-request backfill at the
+    appropriate priority.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 09c50e2fe4fc3e8951a483e4b2693f0ffc008dcf)
+
+commit cc707155ea475836f97791c08c31994fb40e4374
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Oct 22 22:43:18 2017 -0500
+
+    osd/PG: move local_reserver recovery cancel to Recovering state transition
+    
+    This is easier to follow than canceling the reservation in the next state.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit dde007749bb960122f399b4cb8608a8c1ea4eb9c)
+
+commit def29079dc623926fed26a4946d6b9a074dfa2ac
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 20 22:55:29 2017 -0500
+
+    qa/suites/upgrade/jewel-x/parallel: run some jewel after completed upgrade
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 8697eca5ea24c4724f637e4d4f75d3e0d2b95278
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 20 22:52:45 2017 -0500
+
+    qa/suites/upgrade/jewel-x/: set up compat weight-set after cluster upgrade
+    
+    ...but before we run some client workloads.  This will hopefully ensure
+    that older clients will behave.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 89ee508ac51839c3d1f71c5e21443e8f2679c59e
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Oct 21 13:03:17 2017 -0500
+
+    mon/OSDMonitor: add 'osd crush set-all-straw-buckets-to-straw2'
+    
+    This is a common and recommended step that users should take once they
+    have clients that support straw2 (hammer!).  Let's make it easy.
+    
+    This may result in a small amount of data movement, but usually it is
+    very little.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 928770513cbe3a244095e36dcb41c83c0724cfe9)
+
+commit 94a4ca6c8e4d26465cbefd1492ee05f21b08ff1e
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Oct 21 12:59:16 2017 -0500
+
+    crush/CrushWrapper: add bucket_set_alg()
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit edc411ee4cb7b88a970ccfc9d29b3a3ba15d066c)
+
+commit 0048e6a58c7cdf3b3d98df575bc47db8397cd5a9
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 19 16:19:35 2017 -0500
+
+    buffer: remove list _mempool member
+    
+    This broke the C++ ABI by changing the list structure size.  Also, it's
+    not necessary as we can infer the mempool by looking at the other list
+    contents.  We don't (currently) have a need to map an empty list to a
+    particular mempool and have that state stick.
+    
+    Fixes: http://tracker.ceph.com/issues/21573
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9b92d87d37e3892b0096cd728b46154aed1e2d86)
+
+commit 3cd334b6264998490e6fb8321fb7242ff82b36e6
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 19 16:09:10 2017 -0500
+
+    buffer: allow mempool to be passed into raw* ctors and create methods
+    
+    This is more convenient, and also faster than initializing it in
+    buffer_anon and the immediately moving it elsewhere.
+    
+    Drop the optionality of the alignment argument.
+    
+    No users yet.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 481277b8c92368b8cac8fdafe1848720ec1035c2)
+
+commit 724a51cd442d35c950414b86608c48936b01b78b
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 23 12:05:39 2017 -0500
+
+    /etc/sysconfig/ceph: remove jemalloc option
+    
+    This breaks when used with rocksdb, which is now the default.
+    
+    See http://tracker.ceph.com/issues/20557
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 19efccef248c2d66fe3d7fdac8f56d455f2c7b27)
+
+commit 95b6bc9ac41c4985ad97a515c4105351a0e6b94d
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 23 11:16:26 2017 -0500
+
+    etc/default/ceph: remove jemalloc option
+    
+    This breaks when used with rocksdb, which is now the default.
+    
+    See http://tracker.ceph.com/issues/20557
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit fba2055b4e54128db1e9a83f27bb3526d65d4db7)
+
+commit e6eeb796b4d24fcf6797519be06aca9771e73028
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 20 23:14:36 2017 -0500
+
+    qa/suites/rados/rest/mgr-restful: whitelist more health
+    
+    The test is fiddling with OSDs.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ddf3e9162da542af0c5f025957b8304e7359c924)
+
+commit 33aa1673316b77b71e473d1ab71cf61587e425a6
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 5 22:46:48 2017 -0400
+
+    mon/OSDMonitor: improve crush map validation
+    
+    - move into OSDMap method
+    - ensure that rules exist for each pool
+    - ensure pool type matches rule type
+    - ensure rule mask min/max size cover the pool size
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8f8f0b25058ff7319ec9d023f0eceb5832d0ecff)
+
+commit d825de7681ba1cff257ba4108d6c22c81e3a6aab
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Aug 25 11:06:21 2017 +0100
+
+    mon: more forceful renumbering of legacy ruleset IDs
+    
+    Previously, the rules were only modified in the trivial case,
+    so we continued to potentially have CRUSH maps with the
+    legacy ruleset functionality in use.
+    
+    In order to ultimately remove rulesets entirely, we need
+    to do this more aggressively, renumbering all the rules
+    and then updating any pools as needed.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 71d4b2bed54371657693cfb999ade44449be0efd)
+    
+    Conflicts:
+            src/mon/OSDMonitor.cc: the check for multiple rules was removed
+    in master, but not in luminous. once we renumber the legacy ruleset IDs,
+    it's not need to check for and to warn the user at seeing the case where
+    1-to-n mapping from ruleset to rule IDs.
+
+commit f52d1ad23d97e1ab8c70f9b17b75daa0773698b7
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Aug 25 10:42:58 2017 +0100
+
+    osd: s/crush_ruleset_in_use/crush_rule_in_use/
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 675483ecca06686fd4c626bfea33748092aef0c6)
+
+commit 17d73e5a2e592719df8099450c9cd5f46d6fd582
+Merge: d4158133f7 e25a583323
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Oct 20 23:10:49 2017 -0500
+
+    Merge pull request #17773 from xiexingguo/wip-luminous-object-legacy-flag
+    
+    luminous: osd/PrimaryLogPG: _delete_oid - fix incorrect 'legacy' flag
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit d4158133f7586472d01d9c6e669db92ddcd2e73d
+Merge: 84fbb0ff70 8b4fe14ba0
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Oct 20 23:10:06 2017 -0500
+
+    Merge pull request #18362 from kmroz/wip-21543-luminous
+    
+    luminous: qa/tasks: prolong revive_osd() timeout to 6 min
+
+commit 84fbb0ff705df5bc9f164dd5a02bd5a82e6478d7
+Merge: 87ab8d5c80 993d098505
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Oct 20 23:09:52 2017 -0500
+
+    Merge pull request #18388 from tchaikov/wip-pr-18346-luminous
+    
+    os/filestore: print out the error if do_read_entry() fails
+    
+    Reviewed-by: xie xingguo <xie.xingguo@zte.com.cn>
+
+commit 87ab8d5c80374a73ad95f9ed9dd8826c54e64e9f
+Merge: 00daa8462a fd42f2a9cc
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Oct 20 23:09:17 2017 -0500
+
+    Merge pull request #18398 from smithfarm/wip-21783-luminous
+    
+    luminous: cli/crushtools/build.t sometimes fails in jenkins' make check run
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 00daa8462a5a337ecc57bd8664bfcc748befeead
+Merge: 5a4850c8c6 112069c70d
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Oct 20 23:08:58 2017 -0500
+
+    Merge pull request #18401 from liewegas/wip-bluestore-label-luminous
+    
+    luminous: os/bluestore: ceph-bluestore-tool repair updates and prime-osd-dir support
+
+commit dbddf21bd3aac284122fd5fc84abdd4c89b12b3e
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 20 22:32:33 2017 -0500
+
+    messages/MOSDMap: do compat reencode of crush map, too
+    
+    If we are reencoding an incremental, and it embeds a crush map, we need
+    to reencode that in a compatible way too.  This is especially true now
+    because we have the compat crush weight-sets.  Otherwise, a client may
+    learn the crush map through an incremental but not understand choose_args,
+    and not see/understand the alternate weight set.  It will send requests
+    to the wrong OSDs where they will just get dropped.
+    
+    Fixes: http://tracker.ceph.com/issues/21882
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9eaafc66278d59447b29ce6cc378d0cd3ad14ad4)
+
+commit c5ccbf1108532547f54f084b8f06c7f5ddf8a4e8
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Sun Sep 24 20:23:12 2017 +0530
+
+    ceph_volume_client: fix setting caps for IDs
+    
+    ... that have empty OSD and MDS caps. Don't add a ',' at the
+    start of OSD and MDS caps.
+    
+    Fixes: http://tracker.ceph.com/issues/21501
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit baf3b88800c63ef7467abbc7b54e40c0da669f38)
+
+commit b382db7cfce84adc897cd68e4cd7a4eb1b40a9f1
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Wed Sep 20 13:15:30 2017 +1000
+
+    mon/OSDMonitor: mon osd feature checks with 0 up osds
+    
+    get_up_osd_features() returns 0 if no osds are up which causes feature
+    checks to fail.
+    
+    Fixes: http://tracker.ceph.com/issues/21471
+    
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 26ba0ba044846d18f9bdfdbee6e2a9c46882c742)
+    
+    Conflicts:
+            src/mon/MonCommands.h
+              - removed mimic reference
+            src/mon/OSDMonitor.cc
+              - removed mimic-only command path
+
+commit c845db3f8c2a0c2494fbdc9d6cc63c8e0c3b0563
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 19 18:25:56 2017 -0400
+
+    osd/OSDMap: ignore xinfo if features == 0
+    
+    Some old bug (e.g., http://tracker.ceph.com/issues/20751) could
+    result in an UP+EXISTS osd having features==0.  If that happens,
+    we shouldn't crash the mon, which (reasonably) does
+    
+       if (osdmap.get_epoch()) {
+         if (osdmap.get_num_up_osds() > 0) {
+           assert(osdmap.get_up_osd_features() & CEPH_FEATURE_MON_STATEFUL_SUB);
+           check_subs();
+         }
+       }
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8399833f3d221364a8ededb97cd4e1c5a258d924)
+
+commit e58656005d681ee0e35a25b324321cfeea8c7b3a
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Oct 17 17:45:37 2017 -0400
+
+    rgw_file: explicit NFSv3 open() emulation
+    
+    Expect RGW_OPEN_FLAG_V3 in rgw_write(), and if present, attempt
+    a stateless open inline.
+    
+    Required by Ganesha v2.5 and later
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 6ed23b4a0cea7e41b4743e27319737af693947ec)
+
+commit f9850d2148b396032634f7ffa18ae458bd208195
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Thu Sep 28 13:54:32 2017 -0400
+
+    rgw: Check bucket Website operations in policy
+    
+    Add code to check s3:GetBucketWebsite and s3:PutBucketWebsite
+    operations against bucket policy.
+    
+    Fixes: http://tracker.ceph.com/issues/21597
+    Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1493896
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit ceed535957ac186e241fcff26b103cf7efa959b1)
+
+commit abefb578c345a0ddd7100cae987e582b530d3729
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Wed Sep 27 15:42:27 2017 -0400
+
+    rgw: Check bucket CORS operations in policy
+    
+    Add code to check s3:GetCORS and s3:PutCORS operations against bucket
+    policy.
+    
+    Fixes: http://tracker.ceph.com/issues/21578
+    Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1494140
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit 27eb13fe568cc802feaf69131a21db076bcb6746)
+
+commit 0b84dd07c41a6d69d99672218c72a4f69c21b040
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Wed Sep 27 16:08:56 2017 -0400
+
+    rgw: Check bucket GetBucketLocation in policy
+    
+    Add code to check s3:GetBucketLocation against bucket policy.
+    
+    Fixes: http://tracker.ceph.com/issues/21582
+    Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1493934
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit 79188d679edeb6e2f7ca852fdc4224368412cb72)
+
+commit 0b3a974be7bb9cd1d25ee9c2b861f38ea721c66b
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Sep 27 14:26:25 2017 -0400
+
+    rgw: include SSE-KMS headers in encrypted upload response
+    
+    Fixes: http://tracker.ceph.com/issues/21576
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit a5b3db7acb4150d9d27ba42f72658e418dd444c0)
+
+commit a3e2ae4925e7e25c213ef37072d576d1f1ffdc13
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Fri Sep 22 16:11:55 2017 +0200
+
+    rgw: defer constructing keystone engine unless url is configured
+    
+    currently we create a keystone revocation thread even when keystone url
+    is empty, lets defer the construction of keystone unless the urls are
+    configured
+    
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit 104c2f59011e6b515e3845cce529ee44334e76c0)
+
+commit 4c18ac124048f72a10d92aecfe1eadabab0a80b4
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Wed Sep 27 14:35:59 2017 -0400
+
+    rgw: Check bucket versioning operations in policy
+    
+    Add code to check s3:GetBucketVersioning and s3:PutBucketVersioning
+    operations against bucket policy.
+    
+    Fixes: http://tracker.ceph.com/issues/21389
+    Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1490278
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit 16de0fc1c5ede961ebd481f13753214a971c206c)
+
+commit be1ded107ac4e1a1b1515d40553aed3581e0afac
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Thu Sep 14 18:07:15 2017 -0400
+
+    rgw: Check payment operations in policy
+    
+    Add code to check s3:GetBucketRequestPayment and
+    s3:PutBucketRequestPayment operations against bucket policy.
+    
+    Fixes: http://tracker.ceph.com/issues/21389
+    Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1490278
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit f9d1ae1d153319e870c3ccaf7afdc92786cdaa3b)
+
+commit 92d0545781ca2bff39f7bd8e9d63bb8c2d8cb997
+Author: Nathan Johnson <njohnson@ena.com>
+Date:   Thu Sep 7 23:27:40 2017 -0500
+
+    rgw: admin api - add ability to sync user stats from admin api
+    
+    Fixes: http://tracker.ceph.com/issues/21301
+    Signed-off-by: Nathan Johnson <nathan@nathanjohnson.org>
+    (cherry picked from commit 828412d573a0b31387fc9f6a879f9a6083535730)
+
+commit d800747fa18a1872f1d748119bfcc7eff34ae6cf
+Author: gaosibei <gaosb@inspur.com>
+Date:   Tue Sep 26 10:19:22 2017 +0800
+
+    RGW: fix a bug about inconsistent unit of comparison
+    
+    Fixes：http://tracker.ceph.com/issues/21590
+    
+    Signed-off-by: gaosibei <gaosb@inspur.com>
+    (cherry picked from commit cb39f065d6eba87cf1fc2e99334322bf63092df3)
+
+commit 14a1dcb1ff70b2a468cb47e7f239b98e45c63a88
+Author: Mark Kogan <mkogan@redhat.com>
+Date:   Mon Sep 25 09:53:00 2017 +0300
+
+    rgw: update the usage read iterator in truncated scenario
+    Fixes: http://tracker.ceph.com/issues/21196
+    
+    Signed-off-by: Mark Kogan <mkogan@redhat.com>
+    (cherry picked from commit 7306514a813661b77bfcbfc6f534dbabbdd3aa78)
+
+commit 8b420c4627faca27f61f472597c6f67cca2efa2f
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Oct 5 16:39:30 2017 -0400
+
+    rgw: RGWUser::init no longer overwrites user_id
+    
+    if an admin op specifies a user_id and does not find a user with that
+    id, but does find a user based on a later field (email, access key,
+    etc), RGWUser::user_id will be overwritten with the existing user's id
+    
+    when this happens on 'radosgw-admin user create', RGWUser::execute_add()
+    will modify that existing user, instead of trying to create a new user
+    with the given user_id (and failing due to the conflicting email,
+    access key, etc)
+    
+    by preserving the original user_id (when specified), this uid conflict
+    is detected in RGWUser::check_op() and a "user id mismatch" error is
+    returned
+    
+    Fixes: http://tracker.ceph.com/issues/21685
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 4e65c9e9af86e832cb96d57d487771aa28251e7c)
+
+commit fe45086a54d76cd9b913663d4537b36a8a276ed4
+Author: gaosibei <gaosb@inspur.com>
+Date:   Tue Sep 26 10:44:09 2017 +0800
+
+    RGW: Multipart upload may double the quota
+    
+    Fixes: http://tracker.ceph.com/issues/21586
+    
+    Signed-off-by: Sibei Gao <gaosb@inspur.com>
+    (cherry picked from commit 97f95e457fb7f9e36031f41e0f2dd3955daedd87)
+
+commit c3a719da278a830cc1b8101e3d8ef25316da33d7
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Oct 12 13:26:29 2017 -0400
+
+    rgw: 'zone placement' commands validate compression type
+    
+    Fixes: http://tracker.ceph.com/issues/21775
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 33f8ee8b6fa5605edbbfe12d8e097123e509ad9f)
+
+commit 352373ff4a333c9b3869c9f168ee14530067907a
+Author: Yao Zongyou <yaozongyou@vip.qq.com>
+Date:   Sun Sep 24 22:11:22 2017 +0800
+
+    rgw_file: fix write error when the write offset overlaps.
+    
+    Signed-off-by: Yao Zongyou <yaozongyou@vip.qq.com>
+    (cherry picked from commit 872d73f945364002f0fa31762e6976db5b4b3c19)
+
+commit 248bd94dab4359dfea34bfa0cbdd16d9a9b76a02
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Oct 9 13:25:36 2017 +0300
+
+    rgw: disable dynamic resharding in multisite environment
+    
+    Fixes:http://tracker.ceph.com/issues/21725
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit f1ec7a49bde054a19a0a8fd80f37866e1e40379f)
+
+commit 624b3ac5053dbef008d2e9238c6958d07206f6c3
+Author: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+Date:   Tue Sep 26 10:00:59 2017 +0800
+
+    rgw: Torrents are not supported for objects encrypted using SSE-C
+    
+    Fixes: http://tracker.ceph.com/issues/21720
+    
+    Signed-off-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    (cherry picked from commit 9a8ae664da54eb74e74a12cc16a52b0df7df70a0)
+
+commit 0e65ea466f05a2bc8825e51b32901d74082c1b65
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Oct 3 17:48:29 2017 -0400
+
+    rgw: release cls lock if taken in RGWCompleteMultipart
+    
+    Follows Casey's proposal to conditionally release the lock in
+    ::complete(), in order to avoid duplicated code in various early
+    return cases.
+    
+    Fixes: http://tracker.ceph.com/issues/21596
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 704f793f08a02760d23eb5778b738bb07be0e7cf)
+
+commit 17df0ba8f00abbf4b30e0370090251b0c7f6b643
+Author: Enming Zhang <enming.zhang@umcloud.com>
+Date:   Sat Aug 26 04:46:35 2017 +0800
+
+    rgw: encryption add exception handling for from_base64 on bad input
+    
+    If calling from_base64 with charactors that is not base64 encoded,
+    rgw will crash.
+    
+    Signed-off-by: Enming Zhang <enming.zhang@umcloud.com>
+    (cherry picked from commit f0b88e51be86d84793b794f6fe87fdda62004a4d)
+
+commit 46f09e3803ba3c4b463bfa6a2eeb40a0bd7cc0ef
+Author: Enming Zhang <enming.zhang@umcloud.com>
+Date:   Fri Aug 25 19:48:53 2017 +0800
+
+    rgw: encryption fix the issue when not provide encryption mode
+    
+    Now, in RGW, if someone want to upload an object using server-side
+    encryption with providing customer key or kms key id, but not
+    specify the encryption mode in the
+    "x-amz-server-side-encryption-customer-algorithm" or
+    "x-amz-server-side-encryption", the object will be uploaded
+    successfully without encryption.
+    
+    This is not a correct way to deal with it. It is better to
+    return error.
+    
+    Fixes: http://tracker.ceph.com/issues/21581
+    
+    Signed-off-by: Enming Zhang <enming.zhang@umcloud.com>
+    (cherry picked from commit f8c4be8a8943096217d89f7c5abd129fa2414b36)
+
+commit fefbc9fec47cc6858f74b7ea372d39efeb95c61f
+Author: Enming Zhang <enming.zhang@umcloud.com>
+Date:   Fri Aug 25 19:47:21 2017 +0800
+
+    rgw: encryption SSE-KMS add the details of error msg in response
+    
+    Signed-off-by: Enming Zhang <enming.zhang@umcloud.com>
+    (cherry picked from commit 3f16785daf4b3609f2bff7c107a35f908103e32a)
+
+commit 2e493e1f7a1f20d2f33e66816cc72d6b5f193b61
+Author: Enming Zhang <enming.zhang@umcloud.com>
+Date:   Fri Aug 25 19:37:52 2017 +0800
+
+    rgw: encryption SSE-C add the details of error msg in response
+    
+    Signed-off-by: Enming Zhang <enming.zhang@umcloud.com>
+    (cherry picked from commit 9502549ac2133e969a5f268601b92fd4063c1bd9)
+
+commit fe0a41b4a921ac3510b344a6edf6e930a2fa2465
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 12 08:58:03 2017 -0400
+
+    cls/journal: fixed possible infinite loop which could kill the OSD
+    
+    Fixes: http://tracker.ceph.com/issues/21771
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9e7475f8fb08b0ef30cad2c0d680cecf331b44ef)
+
+commit e0d57c544d55f3693f858464f7995b4b30cfa88d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 12 08:56:57 2017 -0400
+
+    test: ceph_test_cls_journal was dropped when converting to cmake
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5d9cfebabac33b645c6199bda63ff4619e6f538f)
+
+commit f8bfd9458d44d96d4ef56a28195a9e36fc0190c6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sun Oct 15 17:26:25 2017 -0400
+
+    librbd: batch large object map updates into multiple chunks
+    
+    Fixes: http://tracker.ceph.com/issues/21797
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 04c5d12acc045731fbf1d0ddce276b5743d9fd89)
+
+commit 6866abc01be5fd92acc918472a604a3af9894b44
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 13 18:55:47 2017 -0400
+
+    test/librbd: initial test cases for trim state machine
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4674b578d8233da3b3c145606ed9c0e4e88a540a)
+
+commit 3b87dea6584010ae05e540825867c9961708ff36
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 13 15:49:00 2017 -0400
+
+    librbd: tweaks to support testing of trim state machine
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3220480d4a22c81cb74f25e528f2bcf3986342c1)
+
+commit ce628329641ac6322d5c59055e8db9eecafbfef4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 13 14:44:40 2017 -0400
+
+    librbd: combine trim state machine object map batch update states
+    
+    The PRE/POST states were previously divided into two halves for handling
+    the copy-up batch and the direct removal batch. This can be simplified by
+    just using a single PRE/POST for the entire deletion region.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 72ce4576fa2b562799a5bc78bd423cfabe097d67)
+    
+    Conflicts:
+            src/librbd/operation/TrimRequest.h
+
+commit 80ff76db5495ae41baf6d4292b6b10c2f789c155
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 13 11:06:38 2017 -0400
+
+    cls/rbd: object map update now utilizes constant-time bit vector operations
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit fc99603ea2b5b268181d38507b8b6fb55ae0a6d0)
+
+commit 5dc46378370da8eede9958a88d6764737d55b09b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 13 11:05:48 2017 -0400
+
+    common/bit_vector: provide constant time iteration of underlying bufferlist
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit daa29f7d2b50bc4b44a108a0f91d45ce90fc16e5)
+
+commit ae4ebaca231e4ae94efc8d445fee792f76e656e1
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 13 11:03:25 2017 -0400
+
+    common/buffer: expose hidden const deference operator
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f35947e3c6661dfe9f951896bc54bc8da158b490)
+
+commit b60d8cf0cbe56500af98b3d3857245e94d904b19
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Oct 2 17:28:04 2017 -0400
+
+    osd: additional protection for out-of-bounds EC reads
+    
+    Fixes: http://tracker.ceph.com/issues/21629
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 881f2a02aecef0cd6bf8ec3e5045b76c8441e832)
+
+commit 8f87fa2d3af44840a3044cd736a2f8da0b602b07
+Author: John Spray <john.spray@redhat.com>
+Date:   Sat Sep 23 13:21:47 2017 -0400
+
+    mgr: fix crashable DaemonStateIndex::get calls
+    
+    This function was recently fixed to return null
+    on missing entries: handle that properly.
+    
+    Fixes: http://tracker.ceph.com/issues/17737
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit ade4827d86bb2bc79466a2da040475651c2aae0d)
+
+commit def3d55eb71e0c62b7fd523865c2ba196342aa1a
+Author: huangjun <huangjun@xsky.com>
+Date:   Fri Oct 6 11:58:50 2017 +0800
+
+    qa/suites/rest/basic/tasks/rest_test: whiltelist OSD_DOWN
+    
+    Fixes: http://tracker.ceph.com/issues/21425
+    Signed-off-by: huangjun <huangjun@xsky.com>
+    (cherry picked from commit 838e12cfc84b386fd4eae09e526b078286f0771e)
+
+commit 20c9e641f7096bf3e208b4d6d28494603eeba891
+Author: huangjun <huangjun@xsky.com>
+Date:   Tue Sep 19 16:13:34 2017 +0800
+
+    qa/suites/rest/basic/tasks/rest_test: more whitelisting
+    
+    Fixes: http://tracker.ceph.com/issues/21425
+    
+    Signed-off-by: huangjun <huangjun@xsky.com>
+    (cherry picked from commit 4d30d02e522039461762d51a3a826a337214ca7e)
+
+commit 5a4850c8c66b75d02fc2597e1ca744c6b0303836
+Merge: 17c6f35703 03e2a68e43
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 19 16:43:39 2017 -0500
+
+    Merge pull request #17692 from smithfarm/wip-21176-luminous
+    
+    build/ops: python-numpy-devel build dependency for SUSE
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 17c6f357030315040f8af95e751921c475b32289
+Merge: 97b30154f8 5aa445897c
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 19 16:42:45 2017 -0500
+
+    Merge pull request #17730 from xiexingguo/wip-pr-17371
+    
+    luminous: mon, osd: per pool space-full flag support
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 97b30154f884f4ea987e3330799bf412b08ba3e4
+Merge: 0a8f1540ba fb7e6a581d
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 19 16:42:07 2017 -0500
+
+    Merge pull request #17779 from smithfarm/wip-rh-74-luminous
+    
+    tests: CentOS 7.4 is now the latest
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 0a8f1540ba08bb79e9ec8f3358490a31be734b2b
+Merge: 5ca7af785a d7cde5f316
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 19 16:41:34 2017 -0500
+
+    Merge pull request #17863 from smithfarm/wip-21343-luminous
+    
+    luminous: mon: DNS SRV default service name not used anymore
+
+commit 5ca7af785a6695121f841909a6118a5103795d99
+Merge: 37d7cfc9f1 49e030da55
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 19 16:40:45 2017 -0500
+
+    Merge pull request #17864 from smithfarm/wip-21438-luminous
+    
+    luminous: core: Daemons(OSD, Mon...) exit abnormally at injectargs command
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 37d7cfc9f136c9604a1e53604a8e0d939eee60db
+Merge: 223b0956b9 077b3601d6
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 19 16:40:18 2017 -0500
+
+    Merge pull request #17866 from smithfarm/wip-21320-luminous
+    
+    luminous: mgr: Quieten scary RuntimeError from restful module on startup
+
+commit 223b0956b93504881a5f06d2589a0b138abf37a9
+Merge: 4f1a7a82a1 82651da2d5
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 19 16:39:53 2017 -0500
+
+    Merge pull request #17951 from smithfarm/wip-21548-luminous
+    
+    luminous: tests: ceph_manager: bad AssertionError: failed to recover before timeout expired
+
+commit 4f1a7a82a1134d6688537653ac07d924275e1f3a
+Merge: 37b13acabe a670921f4f
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 19 16:39:32 2017 -0500
+
+    Merge pull request #18342 from xiexingguo/wip-pr-18318
+    
+    luminous: bluestore: using bluestore_compression_min_blob_size instead of bluestore_compression_max_blob_size to config comp_min_blob_size
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 112069c70df5c7719af9291a8de2b4fcf48d049a
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 16 17:46:03 2017 -0500
+
+    os/bluestore: fsync on _write_bdev_label
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 7aca8af6625156eeaa936629f83ede33d3bbe866)
+
+commit 1e9aa60e3f6acca6649f5d5d51421cc2bd51e55a
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 16 17:45:54 2017 -0500
+
+    ceph-bluestore-tool: implement 'set-label-key' and 'rm-label-key'
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4b468c989f9e8609e7f640cfa4385c454ae6dc17)
+
+commit 170499a1e570bc110be08ca2a38063229fee75dc
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 16 17:42:29 2017 -0500
+
+    os/bluestore: make _write_bdev_label static and public
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 0b0550c967ae0d6ae49641c6859f0b63267e2cd9)
+
+commit 895a277568f902f1c00b9df475ea56b00b115841
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 12 11:40:11 2017 -0500
+
+    doc/man/8/ceph-bluestore-tool: add man page
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 7b91e50dbd289e65f10cb98c5eea9f3171c2d1f1)
+
+commit 24b3b7960e5c5d118aacb63753c794466e801f84
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 13 09:20:53 2017 -0500
+
+    common/options: document 'key', 'keyfile', 'keyring'
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit cdee8b26b9d93189dcd2875038361fb59ed21ba2)
+
+commit 508889d4a7aa93f3bbe13ad8bfc181f3fc9ed2a9
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 12 13:54:06 2017 -0500
+
+    ceph-bluestore-tool: fix show-label to use dev name as key
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4785345ace7b675a0e1c34c457aad1103aab5a1d)
+
+commit 6e17557f288aa7686b8b68c1df31a9a8c26912ac
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 12 08:48:59 2017 -0500
+
+    ceph-bluestore-tool: drop "action ..." print
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5e486c2bd3414f4ee6519574d4d4584618144867)
+
+commit a128aad320e99c396ff17bcaadb48866667b87df
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 12 08:48:46 2017 -0500
+
+    ceph-bluestore-tool: only infer devs if devs.empty()
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2d217367af0df5e18394b1c6179d92db3c0b4448)
+
+commit 24d86603faf4a826684a5e124ed9b447de0b50e8
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 11 17:36:40 2017 -0500
+
+    ceph-bluestore-tool: implement prime-osd-dir
+    
+    This populates an osd dir based on the metadata in a bluestore
+    device label.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 72ee7e29fb77f581a537b1ff2b274c9f3a7de53f)
+
+commit 0f43e6907cfd27f2d6791b896480f2c537115212
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 11 16:33:54 2017 -0500
+
+    os/bluestore: keep meta items for extra bdev paths
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8d07fa5ad0772b4d979dc10695d230074d1f1e69)
+
+commit bb9b70c33a3e013ef78d931fd865ab44c255d772
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 11 17:33:00 2017 -0500
+
+    vstart.sh: specify block files
+    
+    This is mostly just to test ceph-bluestore-tool prime-osd-dir.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d4e0c46d89c2ebe6ae5f9af1c909a09daaea9687)
+
+commit ce83461a60cea6026a7abdd51b2a7a35302392c6
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 11 16:32:16 2017 -0500
+
+    osd: store osd cephx key in "osd_key" meta slot
+    
+    Note that this *only* works with --mkkey (deprecated except for
+    vstart.sh) and --key <key>.  If you write out a full keyring before
+    running --mkfs it doesn't get populated.  Users should pass --key
+    instead of --keyring in that case.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 65b707358e5442eab0fd6ba4d05d9f0a7adc969f)
+
+commit 60d43cb51d7b05cc778cda6b5fcd368db618e3db
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 11 16:17:09 2017 -0500
+
+    ceph-osd: mkkey before mkfs
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 75400ead3061ba1e2820878783fe7b1ddcd9a5fb)
+
+commit faad6afa3441b5ce9455a4de0535baef78b8e7de
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 11 15:53:56 2017 -0500
+
+    os/bluestore: store meta values in main device label
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 33b8e8e86f6bdbee120729561c6cf433d286e2f8)
+
+commit 1027e57d9af374f1fa1dee2a585dfe539dc4f5eb
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Sep 10 15:37:10 2017 -0400
+
+    os/bluestore: fsck: fix error prefixes
+    
+    __func__ is now _fsck instead of fsck
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 7d381af7aed14e08823eca4d0b12de14456ef6a1)
+
+commit 68642c43384d5a30155cd67129267d38d6cc3a06
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 7 12:29:32 2017 -0400
+
+    os/bluestore: allow repair invocation
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 659fcd279ae82c90172fd6ca07a032d75fe50c96)
+
+commit fd42f2a9ccc5016660b9c205f6aa0104ffdd0b10
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Oct 11 17:45:19 2017 +0800
+
+    crushtool: print error message to stderr not dout(1)
+    
+    in hope to fix the mysterious test failure where cli/crushtool/build.t
+    prints nothing when error message is expected.
+    
+    Fixes: http://tracker.ceph.com/issues/21758
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit dc78d40af77ff8d5e4f3d39497e6e3c98c309306)
+
+commit 6902627cb3cb41547be5134aba5c10a39b78163a
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Jul 22 23:51:47 2017 -0400
+
+    crush/CrushWrapper: fix output arg for find_{takes,roots}()
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 60b9cfafc3128cc0cb1f89137221fcc46fcd3802)
+
+commit 993d09850580ca7a697c1a221a8e258f4cd64237
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Oct 17 17:08:59 2017 +0800
+
+    os/filestore: print out the error if do_read_entry() fails
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 42e85fe35d2fe8f3e99bd110021fd5157cf589d7)
+
+commit 410434b3d23c54971f3e2c4df179b139979a33c0
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Oct 19 11:32:38 2017 +0800
+
+    mds: fix race in PurgeQueue::wait_for_recovery()
+    
+    After PurgeQueue::wait_for_recovery() locks the mutex, purge queue
+    may have already been recovered.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+
+commit ee96c34a4d1fc48dec6a1afb4fe98634805c02ce
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Sep 27 19:59:47 2017 +0800
+
+    mds: open purge queue when transitioning out of standby replay
+    
+    MDS opens the purge queue when it starts standby replay. This is
+    wrong because purge queue may change during standby replay.
+    
+    Fixes: http://tracker.ceph.com/issues/19593
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit e515e84f69eeab90ea5c5831f7d3e684e48fb62e)
+
+commit 681cddfa420a0f5f4335564843cdfdb1a4f2371d
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Sep 27 18:49:55 2017 +0800
+
+    mds: always re-probe mds log when standby replay done
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 82c1f5fcfd151aa1e8c39c7f198abec9bf807b65)
+
+commit 37b13acabe3a0cbdcd88c06f49548146d2c69c14
+Merge: 90b6067f6f f30e9a5e6b
+Author: vasukulkarni <vasu.kulkarni@gmail.com>
+Date:   Tue Oct 17 15:47:31 2017 -0700
+
+    Merge pull request #18363 from ceph/wip-yuriw-21822-luminous
+    
+    tests - ceph-ansible vars additions
+
+commit f30e9a5e6bd47129f8a35c1f9067216cfe8a6a70
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Tue Oct 17 13:14:36 2017 -0700
+
+    tests - ceph-ansible vars additions
+    added symlinks for distros
+    
+    Fixes http://tracker.ceph.com/issues/21822
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 90b6067f6f5cbfaf48afb08bd17cb9e5f46a5af9
+Merge: 1718fd969c ab53002186
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Oct 17 16:31:25 2017 -0500
+
+    Merge pull request #17975 from dillaman/wip-systemd-rbd-mirror-luminous
+    
+    luminous: systemd: rbd-mirror does not start on reboot
+    
+    Reviewed-by: Sébastien Han <seb@redhat.com>
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 1718fd969c32cc09cfc208a18455291ab2886e93
+Merge: 345286b775 9d12fa5e6d
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Oct 17 16:30:46 2017 -0500
+
+    Merge pull request #18234 from jdurgin/wip-filestore-rocksdb-luminous
+    
+    filestore: set default readahead and compaction threads for rocksdb
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 345286b775fd269eb9f13cfcb8d0aab533dbfbc6
+Merge: 9223ace029 ab644b89e0
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Oct 17 16:29:38 2017 -0500
+
+    Merge pull request #18236 from tchaikov/wip-21609-luminous
+    
+    luminous: mon,osd: fix "pg ls {forced_backfill, backfilling}"
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 9223ace02909d6551f6126181d11bf00bab1cfe3
+Merge: fdefc8094e 099801e616
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Oct 17 16:29:08 2017 -0500
+
+    Merge pull request #18257 from smithfarm/wip-21699-luminous
+    
+    luminous: mgr status module uses base 10 units
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 8b4fe14ba0e9f8e1b09ae4112dbebd370136bd3f
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Sep 22 10:58:40 2017 +0800
+
+    qa/tasks: prolong revive_osd() timeout to 6 min
+    
+    bluestore_fsck_on_mount and bluestore_fsck_on_mount_deep are enabled by
+    default. and bluestore is used as the default store backend. it takes
+    longer to perform the deep fsck with verbose log. so prolong the
+    revive_osd()'s timeout from 150 sec to 360 sec.
+    
+    Fixes: http://tracker.ceph.com/issues/21474
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 42be200c564184feff1056b7771ce3c1b2cc5527)
+
+commit a670921f4fb2e15f5ea44e9dda238c2eef96897c
+Author: linbing <hawkerous@gmail.com>
+Date:   Mon Oct 16 09:57:23 2017 +0800
+
+    bluestore: using bluestore_compression_min_blob_size instead of bluestore_compression_max_blob_size to config comp_min_blob_size
+    
+    Signed-off-by: linbing <linbing@t2cloud.net>
+    (cherry picked from commit da8dc4d4c60f5fd66f6eb2ba7257c52948f24681)
+
+commit fdefc8094ed639c859db5b4eba858cfe05329e5c
+Merge: bac56cccf9 3a94d69264
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Tue Oct 17 15:19:01 2017 +0800
+
+    Merge pull request #18339 from tchaikov/wip-gen-state-diagram-luminous
+    
+    doc: build with multiple-line state transition function signature
+    
+    Reviewed-By: Nathan Cutler <ncutler@suse.com>
+
+commit bac56cccf9e3fb44a0ed85c151a8b1a0fb5fa4b0
+Merge: 61aca44033 37ddfc61c3
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Tue Oct 17 12:44:04 2017 +0800
+
+    Merge pull request #17835 from rzarzynski/wip-rgw-per-storage-policy-stats-luminous
+    
+    luminous: rgw: add support for Swift's per storage policy statistics
+    
+    Reviewed-by: Matt Benjamin <mbenjami@redhat.com>
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 3a94d6926403ac1f55a9d5723f6b07486f35060b
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 6 08:57:01 2017 -0400
+
+    doc parse two lines for getting the context
+    
+    So that signatures can get parsed when they are split like:
+    
+    PG::RecoveryState::RepWaitBackfillReserved::react(
+      const RemoteReservationCanceled &evt)
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 9e2a52ccec4a5c833dd861ae528d52efbc0f9e5f)
+
+commit ddd88ff365c5399d88890abe1608f12c4a960488
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Oct 6 08:03:36 2017 -0400
+
+    doc raise exceptions with a base class
+    
+    Although valid in very old Python, it is no longer possible to raise
+    "bare" exceptions without a class.
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 347b7b1f193f97a3577a971e074a23366b0c8617)
+
+commit 8b3e9917a70452cc4182aae469aac331314df4e6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 26 15:46:28 2017 -0400
+
+    rbd-mirror: ensure forced-failover cannot result in sync state
+    
+    If the local image already exists and the local image isn't
+    registered in the remote journal, it should be assumed that
+    it's already in the replaying state so it can verify the
+    ancestry to detect split-brains.
+    
+    Fixes: http://tracker.ceph.com/issues/21559
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit bc96c53d947625d12f7b30a45da68a96c09faf70)
+
+commit d7da395e5196eae13b31c018bf503b9e0c169bda
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Sep 26 14:21:42 2017 -0400
+
+    rbd-mirror: forced-promotion should interrupt replay delay to shut down
+    
+    Fixes: http://tracker.ceph.com/issues/21559
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e3b58312572ceb7abc675128dfb231c9b888655e)
+
+commit 4ebd4b39282f715a30a849c01510a281faec2ca6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Sep 29 15:11:38 2017 -0400
+
+    librbd: snapshots should be created/removed against data pool
+    
+    Fixes: http://tracker.ceph.com/issues/21567
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5a3baf1bd852d6c5d0de10a33839658789edc4eb)
+
+commit 34cd96d3425dcea4053ce0514632194dd7b2eaab
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Oct 12 11:25:13 2017 -0400
+
+    cls/rgw: increment header version to avoid overwriting bilog entries
+    
+    Fixes: http://tracker.ceph.com/issues/21772
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 081ba106a910583851b9acdcaf5971e37a439329)
+
+commit 972916bf69192a2c504bf855e2af0480815ecc91
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Oct 11 19:27:55 2017 -0400
+
+    test/rgw: add test_multipart_object_sync
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 1c14d4daf9201d58b1d0038bdd733a16275e037a)
+
+commit 695d7ad51179d4f1f5e4168dec3758e3ffa4fe4d
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sat Oct 14 13:47:53 2017 -0700
+
+    mds: prevent trim count from underflowing
+    
+    Fixes: http://tracker.ceph.com/issues/21807
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 452bdd8b35643ed82d8614efcd4ca22688392eb6)
+
+commit 61aca440334219d5d04df632812dbdf164de3fbc
+Merge: 462b93ecc5 bb92cc93ac
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Oct 15 14:30:57 2017 -0400
+
+    Merge pull request #17834 from rzarzynski/wip-rgw-21148-luminous
+    
+    luminous: rgw: add support for Swift's reversed account listings
+
+commit 462b93ecc5e8caf7b0c35dd83dbbe7e33cc015d9
+Merge: c4ac0d64a9 119995a8f0
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Sun Oct 15 12:48:02 2017 -0400
+
+    Merge pull request #17811 from smithfarm/wip-21456-luminous
+    
+    luminous: rgw: wrong error message is returned when putting container with a name that is too long
+
+commit c4ac0d64a95a6f9ec974446a6971f9d60ab67b9f
+Merge: 5695829ffd 957e221f1f
+Author: Marcus Watts <mwatts@redhat.com>
+Date:   Fri Oct 13 20:42:16 2017 -0400
+
+    Merge pull request #17832 from rzarzynski/wip-rgw-18977-luminous
+    
+    luminous: rgw: list_objects() honors end_marker regardless of namespace.
+
+commit b99acfd6018178ad8cd7db187d6892780f82a276
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Oct 12 12:01:35 2017 +0800
+
+    mds: keep CInode::STATE_QUEUEDEXPORTPIN state when exporting inode
+    
+    Fixes: http://tracker.ceph.com/issues/21768
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 1d160098a93092904026b62326fd9890de48044a)
+
+commit 2973b6d4184cb5c36f8823010ab83a2b8ebb12d0
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Oct 10 17:47:33 2017 +0800
+
+    mds: update client metadata for already open session
+    
+    session opened by Server::prepare_force_open_sessions() has no
+    client metadata.
+    
+    Fixes: http://tracker.ceph.com/issues/21746
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 9d5c92a3367369789836d8669aa421074c69e4f3)
+
+commit f353a1e805f4be3e31d8c3346f0f41fa2619c770
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Oct 9 18:00:38 2017 +0800
+
+    osdc/ObjectCacher: limit memory usage of BufferHead
+    
+    when doing small size random writes, size of data structures that used
+    for tracking dirty data can be larger than the dirty data size.
+    
+    Fixes: http://tracker.ceph.com/issues/21726
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 27db0255992354e15b4285891a33fede6849cf62)
+
+commit 5695829ffd72a03f5611393c6037caf73ff1e813
+Merge: d3804d216d 6d032e6ce3
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri Oct 13 15:29:45 2017 -0700
+
+    Merge pull request #18275 from ceph/wip-yuriw-21776-luminous
+    
+    tests - Removed `distors` and added `supported` to run on all support…
+    
+    Reviewed-by: Vasu Kulkarni <vakulkar@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 6d032e6ce33fe153958a445066a2a7ba00ffd260
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Thu Oct 12 13:36:20 2017 -0700
+
+    tests - Removed `distors` and added `supported` to run on all supported OSs
+    Added `openstack` fragment to run on vps
+    Forced/hard-coded `machine_type=vps`
+    Added `print`s
+    Added only centos and ubuntu latest to support systemd (in distros dir)
+    Added `ceph osd set-require-min-compat-client luminous`
+    
+    Fixes http://tracker.ceph.com/issues/21776
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit d3804d216d21d7e2b12e69eae1c1ef293229a6ca
+Merge: c9b493d2db 28e7d5645e
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Oct 13 14:34:56 2017 -0500
+
+    Merge pull request #18050 from xiexingguo/wip-pr-17610
+    
+    luminous: os/bluestore: set bitmap freelist resolution to min_alloc_size
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit c9b493d2db39765c6ecfb31a3c9e8a38465e6ba0
+Merge: 09d597d34a b435af1137
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Oct 13 14:34:30 2017 -0500
+
+    Merge pull request #18247 from liewegas/wip-seesaw-warning-luminous
+    
+    qa/suites/rados/singleton/all/mon-seesaw: whitelist MON_DOWN
+
+commit 2e6b29084ae505098fe6889d69133a8527c773be
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Sep 1 17:32:22 2017 +0800
+
+    arch/arm: set ceph_arch_aarch64_crc32 only if the build host supports crc32cx
+    
+    HWCAP_CRC32 is defined by the linux kernel source. so it's defined as
+    long as the linux kernel source is new enough. but the compiler on the
+    building host is not necessarily able to build the `crc32cx`
+    instruction. if we happen to have an incapable compiler on a machine with
+    recent linux kernel source, the dummy "ceph_crc32c_aarch64()" will be
+    selected by `ceph_choose_crc32()`. and it always return 0.
+    
+    See-also: http://tracker.ceph.com/issues/19705
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 8a077bd2a93d3e12e20013e76e57a35240f5f481)
+
+commit 7d2b7e8e3113daf40fd56d8258294d45b854ca55
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Tue Oct 10 16:21:48 2017 -0400
+
+    rgw: Remove assertions in IAM Policy
+    
+    A couple of them could be triggered by user input.
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit b3118cabb8060a8cc6a01c4e8264cb18e7b1745a)
+
+commit 09d597d34a1d567dc5350570f044eaa3d74a024f
+Merge: 7fdd9a6dac 37ea1976b3
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 12 14:57:38 2017 -0500
+
+    Merge pull request #18253 from ceph/wip-yuriw-21660_2-luminous_1
+    
+    tests - Added yaml fragmet to cover testing...
+
+commit 7fdd9a6dac4e047ae710dfaad38d5dbe3caa2634
+Merge: 8ad4617f3d 2b7bd94f13
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 12 14:57:23 2017 -0500
+
+    Merge pull request #18254 from ceph/wip-yuriw-21660_3-luminous_1
+    
+    tests - Added yaml fragmet to cover testing...
+
+commit ab644b89e0aad7b0c1a26ed3b9a2f3ea685c7a15
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Oct 11 11:45:33 2017 +0800
+
+    qa: s/backfill/backfilling/
+    
+    it's renamed "backfilling" in 4015343f .
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit e21114274f6a2742748a5a9b965d415241c80f3c)
+
+commit 099801e616c3c9e8501f153501b74d5220383f33
+Author: Yanhu Cao <gmayyyha@gmail.com>
+Date:   Wed Oct 11 09:31:14 2017 +0800
+
+    mgr/status: fix ceph fs status returns error
+    
+    Fixes: http://tracker.ceph.com/issues/21752
+    
+    Signed-off-by: Yanhu Cao <gmayyyha@gmail.com>
+    (cherry picked from commit 4a90daa700142ed1f58f5f2a2cc85f3049bccd64)
+
+commit 0618f19128d0eb04004e65e6eab477b7b1b7fb6a
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Aug 30 20:32:11 2017 +0100
+
+    mgr/status: format byte quantities in base 2 multiples
+    
+    Fixes: http://tracker.ceph.com/issues/21189
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 2a5ae210928f8c9f1e3b72863c277ae5655bb771)
+
+commit 2b7bd94f139251b4373a9662e5add86456566486
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Wed Oct 11 13:12:45 2017 -0700
+
+    tests - Added yaml fragmet to cover testing...
+    ... for http://tracker.ceph.com/issues/21660#note-17
+    
+    for luminous-x (see similar #18200, #18226)
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+    (cherry picked from commit e7fdda31803cd37604a9296a876320bfcb849d34)
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 37ea1976b30e4e03ac535ee353c3a5277c53dbd3
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Tue Oct 10 13:48:33 2017 -0700
+
+    tests - Added yaml fragmet to cover testing...
+    ... for http://tracker.ceph.com/issues/21660#note-17
+    for jewel-x (see similar https://github.com/ceph/ceph/pull/18200)
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+    (cherry picked from commit b552e636ddd8ce2cfb605b4c124440d8dd1e8e47)
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 8ad4617f3dea71984d0d2a6119e9e51ef76779fd
+Merge: edf28e6912 59a5a10904
+Author: Sage Weil <sage@newdream.net>
+Date:   Wed Oct 11 12:20:04 2017 -0500
+
+    Merge pull request #18025 from liewegas/wip-recovery-preemption-luminous
+    
+    luminous: osd: allow recovery preemption
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit edf28e6912c183de4d955f125ef18d8a52594f59
+Merge: 91cb216c01 3354937698
+Author: Sage Weil <sage@newdream.net>
+Date:   Wed Oct 11 12:19:45 2017 -0500
+
+    Merge pull request #18154 from ktdreyer/luminous-gitignore-debian-patches
+    
+    luminous: .gitignore: allow debian .patch files
+
+commit 91cb216c01eca89ff7e338328d64216998effd16
+Merge: 27d3f8c748 b3f9439cfd
+Author: Sage Weil <sage@newdream.net>
+Date:   Wed Oct 11 12:19:08 2017 -0500
+
+    Merge pull request #18189 from tchaikov/wip-retry-oserror-luminous
+    
+    luminous: ceph-disk: retry on OSError
+
+commit 27d3f8c748de33a35c5006513d198bbcbee19b45
+Merge: a33dd01164 f60a942023
+Author: Sage Weil <sage@newdream.net>
+Date:   Wed Oct 11 12:18:27 2017 -0500
+
+    Merge pull request #18227 from liewegas/wip-bluestore-mempool-luminous
+    
+    luminous: os/bluestore: several mempool accounting fixes
+    
+    Reviewed-by: xie xingguo <xie.xingguo@zte.com.cn>
+
+commit b435af1137ede418740d2dcf39114d8476775fd8
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 11 08:24:50 2017 -0500
+
+    qa/suites/rados/singleton/all/mon-seesaw: whitelist MON_DOWN
+    
+    Mgr can get marked down when mon weirdness is happening.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 0a886bc9f3e9805de04c8c60bd3d78b937737b96)
+
+commit 51c6f0def379ba013a82204398a021663f1b5d56
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Oct 10 20:32:08 2017 +0800
+
+    mon/PGMap: let pg_string_state() return boost::optional<>
+    
+    better to be specific, so we don't run into the same problem even we
+    are using the MSB of uint64_t for a pg state bit in future. we could,
+    however use uint64_t(-1) to indicate the pg_string_state()'s failure to
+    parse the state string, because pg_string_state() only translate a
+    single state a time. but it's always better to be explicit than
+    implicit.
+    
+    Fixes: http://tracker.ceph.com/issues/21609
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 260f87a9f1c4e32ff3a596ea341ad07a901f44a9)
+    
+    Conflicts:
+            src/mon/PGMap.cc
+            src/osd/osd_types.cc
+            src/osd/osd_types.h: trivial resolution
+
+commit 9d12fa5e6dad5492491f4718ba7c593077d984a6
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Tue Oct 10 19:54:31 2017 -0400
+
+    common/options: enable multiple rocksdb compaction threads for filestore
+    
+    One of the major benefits of rocksdb over leveldb is multithreaded
+    compaction. The default of 1 thread does not provide much benefit, and
+    is insufficient for heavy rgw workloads.
+    
+    For high-write and delete omap workloads I've seen up to 8 compaction
+    threads be used.  There's little overhead to having a higher max than
+    are needed, so set the default to 8.
+    
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 023fa810aa6b3af305e9027e3f717e54d1bb2712)
+    
+    Conflicts:
+        src/common/options.cc (trivial)
+
+commit 6f2f8cdc6a6fa64928fd2d5459b4974e281d857e
+Author: Mark Nelson <mnelson@redhat.com>
+Date:   Thu Sep 21 17:21:16 2017 -0500
+
+    common/options.cc: Set Filestore rocksdb compaction readahead option.
+    
+    fixes: http://tracker.ceph.com/issues/21505
+    
+    Signed-off-by: Mark Nelson <mnelson@redhat.com>
+    (cherry picked from commit 66567e573836c570040c8d8148c34b0cf9dc9ce2)
+
+commit f60a942023088cbba53a816e6ef846994921cab3
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 28 08:02:50 2017 -0400
+
+    os/bluestore: move several buffer{ptr,list}s into cache_other mempool
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 80c60fcde22cf1269ada45d0914543b8f7d49b3e)
+
+commit 37dd32f449bf2b0aaa3dbcba5cc06da076adbb1e
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 28 08:14:37 2017 -0400
+
+    os/bluestore: put new attrs in correct mempool too
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e7762ea1477e861ebc738aa0588a01c0da883d62)
+
+commit dea2c1e6704c40de11c041e9982e9a9f7d9d4cb3
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 22 08:09:37 2017 -0400
+
+    os/bluestore: put attrs in mempool
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ef6524436fe095f94fce421ce26ed4c64e1424fe)
+
+commit c2ed1f9b1801e43cf1d345da4d4876de5334cc40
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 22 08:08:52 2017 -0400
+
+    buffer: add ptr::[try_]reassign_to_mempool
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 937379d57a0f8097ae713adf714ba0414d36dffe)
+
+commit a33dd01164bb18b24a0909c9419e15fcd2e79446
+Merge: 9e48df69de c0a1168ec2
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Tue Oct 10 13:17:46 2017 -0700
+
+    Merge pull request #18200 from ceph/wip-yuriw-21660-luminous
+    
+    tests - Added yaml fragmet to cover testing
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit c0a1168ec2bacf6d78041f3ae49bfe6fff669913
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Mon Oct 9 12:51:59 2017 -0700
+
+    tests - Added yaml fragmet to cover testing on standanone client.4
+    for http://tracker.ceph.com/issues/21660#note-17
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit b3f9439cfd9141ddde9874c06fd43d346c10c5da
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Oct 7 22:15:11 2017 +0800
+
+    ceph-disk: retry on OSError
+    
+    we are likely to
+    1) create partition, for instance, sdc1
+    2) partprobe sdc
+    3) udevadm settle
+    4) check the device by its path: /dev/sdc1
+    
+    but there is chance that the uevent sent from kernel fails to reach udev
+    before we call "udevadm", hence "/dev/sdc1" does not exist even after
+    "udevadm settle" returns. so we retry in case of OSError here.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 4f82dfb9e761a09484e6ba3bd027da535162783e)
+
+commit 83b3377c6b04d78b20b7c23d145e015d63e471a4
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Oct 7 21:13:47 2017 +0800
+
+    ceph-disk: factor out the retry logic into a decorator
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 02a8620de7f200736900aafc944b2c1ac47d9910)
+
+commit 9e48df69de14d8415eed008b824204313774e2ba
+Merge: a81b0da8e4 41af4ca8f3
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Sun Oct 8 12:05:25 2017 -0400
+
+    Merge pull request #18167 from tchaikov/wip-luminous-sphinx
+    
+    luminous: admin: bump sphinx to 1.6
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit 41af4ca8f3cd3db8dc076c07e1a082a053b91b26
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Sep 11 21:54:33 2017 +0800
+
+    doc: switch to upstream sphinx-ditaa
+    
+    it works with setuptools and is now compatible with py3
+    
+    the py3 branch is created to track the upstream's master branch
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit fda079d5ed4c8a80c677f714a680233a568d512e)
+
+commit e49cae8b03340834e0b44491a4a903c0d61e1f74
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Aug 15 09:12:27 2017 -0400
+
+    admin: bump sphinx to 1.6
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 88a1114cb3f90e52ba718dc832617fa004d63b3c)
+
+commit a81b0da8e4aca275cc3195c04f3e0d485e4a2f31
+Merge: 1e521b5fb4 e4164c43f5
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Fri Oct 6 17:42:30 2017 -0400
+
+    Merge pull request #17914 from liewegas/wip-21498-luminous
+    
+    ceph-disk: fix '--runtime' omission for ceph-osd service
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit 1e521b5fb47de74720d6aab39750c53aa2b2d8e7
+Merge: d550610512 ee5b9ea46c
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Oct 6 15:35:06 2017 -0500
+
+    Merge pull request #18135 from liewegas/wip-vta-luminous
+    
+    luminous: cmake: disable VTA on options.cc
+
+commit 33549376987b18a9bdcf5f56e62268c5791124bc
+Author: Ken Dreyer <kdreyer@redhat.com>
+Date:   Thu Sep 7 11:07:59 2017 -0600
+
+    .gitignore: allow debian .patch files
+    
+    The Ubuntu packaging layout with git-buildpackage assumes a
+    "debian/patches/" directory with several .patch files in it.
+    
+    When upstream's .gitignore tells Git to ignore .patch files, we have to
+    edit that line out downstream. When we forget to do that downstream, it
+    can lead to missing patches and broken downstream builds.
+    
+    Allow patches in the /debian/patches directory so it's easier to
+    maintain an Ubuntu package based on upstream's Git repo.
+    
+    Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
+    (cherry picked from commit c734b0c0296152721b658af7b699a64b3a49d251)
+
+commit d5506105120479c1831ab559b555e2d4ea4c357c
+Merge: d46675a1ec 0dc73f1210
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Oct 6 07:38:53 2017 -0500
+
+    Merge pull request #18140 from liewegas/wip-21660-luminous
+    
+    luminous: src/messages/MOSDMap: reencode OSDMap for older clients
+
+commit d46675a1ecb10f7176265cb1fc5ca971209aecb1
+Merge: 39ad203313 8b2bd38785
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 5 22:12:22 2017 -0500
+
+    Merge pull request #18037 from ajarr/wip-21602-luminous
+    
+    luminous: ceph_volume_client: add get, put, and delete object interfaces
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 39ad2033136e32d76e89e86d3f00bd1b928242ce
+Merge: 473f9d1317 80f6508fc2
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 5 21:03:42 2017 -0500
+
+    Merge pull request #18103 from liewegas/wip-21259-luminous
+    
+    luminous: osd: dump bluestore debug on shutdown if debug option is set
+
+commit 473f9d13173aa751d54b9a322060111d627f71f1
+Merge: c50df6511f 49294965f9
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 5 21:03:28 2017 -0500
+
+    Merge pull request #18112 from liewegas/wip-localpool-min-size-luminous
+    
+    luminous: mgr/localpool: fix min_size, 3x default, crush rule
+    
+    Reviewed-by: Alexander Marangone <amarango@redhat.com>
+
+commit c50df6511fda7d03c3cfa828acd5e56f2ef45d0f
+Merge: 4ff03e5192 16c6dd5129
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 5 21:02:53 2017 -0500
+
+    Merge pull request #18126 from smithfarm/wip-sigpipe-porting-luminous
+    
+    luminous: msg: reimplement sigpipe blocking
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit 4ff03e51928c0d7da9a90d50ec504b30eb9e4b32
+Merge: 766c6f3647 7986d4566a
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 5 21:02:09 2017 -0500
+
+    Merge pull request #18127 from liewegas/wip-21470-luminous
+    
+    luminous: os/bluestore: fix another aio stall/deadlock
+
+commit 766c6f3647b57b482f76ef254c95a651741d9ad5
+Merge: f972702229 3850f8e0b7
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 5 21:01:53 2017 -0500
+
+    Merge pull request #18128 from liewegas/wip-bluefs-allocation-luminous
+    
+    luminous: os/bluestore: make bluefs behave better near enospc
+
+commit f97270222935d1c43a41102c3ada586a7deca190
+Merge: e4c51f62ff f99fe83a37
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 5 21:01:27 2017 -0500
+
+    Merge pull request #18131 from liewegas/wip-ifname-luminous
+    
+    luminous: common/pick_address: add {public,cluster}_network_interface option
+
+commit e4c51f62ffc0f2fcc9214f6b7895210aa79c288b
+Merge: 66af04cd9b f0a6f616c6
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Oct 5 21:01:06 2017 -0500
+
+    Merge pull request #18132 from gregsfortytwo/wip-20416-bitwise-assert-luminous
+    
+    osd: make the PG's SORTBITWISE assert a more generous shutdown
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 0dc73f12105ec271a43fe176c2f2a22849e8475c
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Oct 5 15:26:16 2017 -0500
+
+    src/messages/MOSDMap: reencode OSDMap for older clients
+    
+    We explicitly select which missing bits trigger a reencode.  We
+    already had jewel and earlier covered, but kraken includes all of
+    the previously mentioned bits but not SERVER_LUMINOUS.  This
+    prevents kraken clients from decoding luminous maps.
+    
+    Fixes: http://tracker.ceph.com/issues/21660
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d2664aa34dc4c13da64d1ad187019737080069d7)
+
+commit 3850f8e0b791f2d05906647866b14a390ab2f5da
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 3 11:18:17 2017 -0500
+
+    os/bluestore: add bluestore_bluefs_min_free
+    
+    We need at least ~1GB free so we can write out new SSTs (which are 256MB
+    each).
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 65be614f2bfd65a1f13205a3075c87acc72c4836)
+    
+    (avoided 1_G syntax that luminous doesn't support)
+
+commit b015ce116159ec57d20bcd29306e450a816610f9
+Author: Karol Mroz <kmroz@suse.de>
+Date:   Mon Oct 2 14:01:47 2017 -0700
+
+    rgw: stop/join TokenCache revoke thread only if started.
+    
+    Thread::join triggers an assert otherwise.
+    
+    Fixes http://tracker.ceph.com/issues/21666
+    
+    Signed-off-by: Karol Mroz <kmroz@suse.de>
+    (cherry picked from commit 26f2da083c7dd21b89c1c1e6c498b14e034364a6)
+
+commit ee5b9ea46c8b1e831c60f9cc0b10819b14d5efb8
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Aug 31 18:15:28 2017 +0800
+
+    cmake: disable VTA on options.cc
+    
+    to silence following warning and to avoid compiling this file twice:
+    
+    ceph/src/common/options.cc: In function ‘std::vector<Option> get_global_options()’:
+    ceph/src/common/options.cc:151:21: note: variable tracking
+    size limit exceeded with -fvar-tracking-assignments, retrying without
+     std::vector<Option> get_global_options() {
+                         ^~~~~~~~~~~~~~~~~~
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 4bb56402a6d6f642d54f329aa1e809cbe044e0c6)
+
+commit 59a5a109040884dd2c7c77f3bc84378b54048f1d
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 4 15:28:26 2017 -0500
+
+    osd/PG: separate event for RemoteReservationCanceled
+    
+    Right now we transparently map a RemoteReservationRejected into a
+    *Canceled event because this what peers send over the wire.  Even
+    once new peers start sending and explicit CANCEL, old peers will
+    still do so, so we'll maintain this mapping for a while.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 84d71e6a10b02591b5d5e126b346771871eb1575)
+
+commit 206400997daaec97fe9af7e2e735e1b609006107
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 4 14:55:15 2017 -0500
+
+    osd/PG: separate verb 'Reject' from passive 'rejected'
+    
+    This reduces pg->reject_reservation() callsites from 2 to 1 and
+    makes the state transitions a bit more explicit.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit bf7f101a787652644c27aae4e752fd21f265e866)
+
+commit 8c1716a42def8547f3af446a143715fad24addb5
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Oct 1 15:05:40 2017 -0500
+
+    osd: make note about when we get MBackfillReserve REJECT messages
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6e829a3a7c7aaff78142514fde7703cad48953fd)
+
+commit 260ab6009cdce0d846685e4c7789336db22885f1
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Oct 1 15:04:34 2017 -0500
+
+    osd/PG: handle RecoveryReservationRejected in RepWaitRecoveryReserved
+    
+    This state is analogous to RepWaitBackfillReserved; just like we do there
+    we want to handle the REJECT from the primary by canceling our local
+    remote_reservation.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit ab8f1d291216c07fed7b661896b0a0c1279f50eb)
+
+commit 479b05a8c1659e80028190c7fe90e762084c4c24
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Oct 1 15:03:22 2017 -0500
+
+    osd/PG: ignore RemoteReservationRejected if we are RepNotRecoverying
+    
+    The primary may send us a REJECT (meaning cancel) if recovery/backfill is
+    preempted there.  That can happen even if the recovery isn't reserved or
+    requested here (e.g., because the primary is still waiting for the local
+    reservation).  Just ignore it and remain in RepNotRecovering.
+    
+    Fixes: http://tracker.ceph.com/issues/21613
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1ce235c5dd0b777223f6465e71fb7ee4befd13d8)
+
+commit 2a95156d7be9e6c796d5f0ce7bc0eab89c42230d
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Oct 1 15:01:05 2017 -0500
+
+    osd/PG: cancel local reservation in RemoteReservationRejected handler
+    
+    We can get a RemoteReservationRejected event either because *we* decide
+    to reject, or because we get a REJECT from the primary that means "cancel"
+    (e.g., because recovery/backfill was preempted there).  In both cases we
+    want to cancel our remote_reservation.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f5809afb0f2ca2f1187609e645d6c4c5bd73e39d)
+
+commit 92ff2cbd69c444c6648889a2da7e01be6b00311e
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Oct 1 14:59:31 2017 -0500
+
+    osd/PG: move reject_reservation out of RemoteReservationRejected reaction
+    
+    The RemoteReservationRejected event is also submitted when we are a
+    replica or backfill target and get a MBackfillReserve REJECT message
+    because the primary canceled or was preempted.  In that case, we don't
+    want to send a REJECT back to the primary; we only need to send it in the
+    cases where *we*, locally, decide to reject.  Move the call to those call
+    sites.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 57d18f0e8d9b0428a193c63112a28ddd046337b4)
+
+commit f0a6f616c6c0358c2a7b9eb24fd6ca9c4290be0a
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Fri Sep 29 15:18:26 2017 -0700
+
+    osd: make the PG's SORTBITWISE assert a more generous shutdown
+    
+    We want to stop working if we get activated while sortbitwise is not set
+    on the cluster, but we might have old maps where it wasn't if the flag
+    was changed recently. And doing it in the PG code was a bit silly anyway.
+    
+    Instead check SORTBITWISE in the main OSDMap handling code prior to
+    prepublishing it. Let it go through if we aren't active at the time.
+    
+    Fixes: http://tracker.ceph.com/issues/20416
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 0a691b2b1c19fbc30af5c4046736dacb2fb8bfa4)
+
+commit f99fe83a37241fe1c616ecb854b5196112985861
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 28 17:47:17 2017 -0400
+
+    common/pick_address: add {public,cluster}_network_interface option
+    
+    Add _interfaces option to constrain the choice of IPs in the network
+    list to those on interfaces matching the provided list of interface names.
+    The _interfaces options only work in concert with the _network options,
+    so you must also specify a list of networks if you want to use a specific
+    interface, e.g., by specifying a broad network like "::" or "0.0.0.0/0".
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 95cc790acddb597d6fef2d9e444f0b6e0436f16f)
+
+commit b84803d6092667631527b8b37000cac6447e0b84
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Sep 12 17:17:13 2017 -0700
+
+    osd: Only scan for omap corruption once
+    
+    Before
+            state 2: Can have complete tables (some may be bad)
+            state 3: Never had complete tables
+    After
+            state 2: Can have complete tables (some may be bad)
+            state 3 with legacy: Can have complete tables (bad ones are cleared)
+            state 3: Never had complete tables
+    
+    Once OSDs boot with this change you can't downgrade to a previous release.
+    If someone does downgrade they could have unstable OSDs that hit assert(state.v < 3).
+    The following command run after shutting down the cluster but before downgrading
+    ceph packages would be a way to fix this.
+    
+    ceph-osdomap-tool --omap-path ... --command resetv2
+    
+    Fixes: http://tracker.ceph.com/issues/21328
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 8805ef53424e30fd3f24ee38f5a6bdd9e6dd8641)
+
+commit d0ea152865eeff63cf49e137f5ac342ec6725a6a
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Sep 12 18:06:10 2017 -0700
+
+    tools: Add --backend option to ceph-osdomap-tool default to rocksdb
+    
+    Fix hard-coded "leveldb" backend.  The command is broken in Luminous
+    now that "rocksdb" is the default.
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit de43493990923bcdd20f88e9d04647e5ba709feb)
+
+commit 94090379a13639ce0a8619dfe178cc62acdd8fc4
+Author: Luo Kexue <luo.kexue@zte.com.cn>
+Date:   Mon Aug 28 09:24:10 2017 +0800
+
+    osd, mds, tools: drop the invalid comment and some unused variables
+    
+    Signed-off-by: Luo Kexue <luo.kexue@zte.com.cn>
+    (cherry picked from commit 2e93424167de15e91394169f4395f5f446e710e2)
+
+commit 382ff40fb17db2c657318038495ae7140f95b7e6
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Sep 12 23:14:15 2017 -0700
+
+    tools: Add the ability to reset state to v2
+    
+    Available for testing and user downgrade.
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 3e4d68640cc43efc0cf10ea8119b3aa583b7f36b)
+
+commit dc2c1d9d2fdd5449dd7c3213e1d2b5cbd928d4d2
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Sep 12 22:12:52 2017 -0700
+
+    tools: Show DB state information
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 3214882a95f9f70d9f6d28f4e403ee16324530f9)
+
+commit 36675e34b610ed6afa90bdb93da6e49025e4d9e1
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 3 10:41:33 2017 -0500
+
+    os/bluestore/BlueFS: crash on enospc
+    
+    We were passing this error back to rocksdb, which would then crash (much)
+    later with a corrupted SST file!  No good.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 569e924812bd7cc61e6e755f265890dbd5c95c31)
+
+commit 16c6dd5129f54a6f5eec7fe16719a179c176548e
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Tue Oct 3 15:54:06 2017 -0700
+
+    msgr: add a mechanism for Solaris to avoid dying on SIGPIPE
+    
+    This is fairly clean: we define an RAII object in the Messenger.h on
+    Solaris, and "declare" it with a macro in the implementations. There's
+    no code duplication and on Linux it's just entirely compiled out.
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit cba20a95e816aaf9f0971b29b14b0be8c524b59d)
+    
+    Conflicts:
+        src/msg/async/PosixStack.cc - luminous #include "common/simple_spin.h" is
+            missing in master (trivial resolution)
+
+commit 7986d4566a1f6d8e22a9acd19608530ce64cbfa1
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 3 16:48:37 2017 -0500
+
+    os/bluestore: use normal Context for async deferred_try_submit
+    
+    I'm not quite sure why the FunctionContext did not ever execute on the
+    finisher thread (perhaps the [&] captured some state on the stack that it
+    shouldn't have?).  In any case, using a traditional Context here appears
+    to resolve the problem (of the async deferred_try_submit() never executing,
+    leading to a bluestore stall/deadlock).
+    
+    Fixes: http://tracker.ceph.com/issues/21470
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 67ec75805787ed63b35f8d70478a7a2cd785df06)
+
+commit 9ab0f64bcf74f74838f6066d554f728902c31715
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 29 13:47:19 2017 -0500
+
+    os/bluestore: wake kv thread when blocking on deferred_bytes
+    
+    We need to wake the kv thread whenever setting deferred_aggressive to
+    ensure that txns with deferred io that have committed but haven't submitted
+    their deferred writes get submitted.  This aligns us with the other
+    users of deferred_aggressive (e.g., _osr_drain_all).
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 633b17007701d87903fe1d10b19c44210a3326eb)
+
+commit 0251cfcd17111334d3f13dc1e9d52a9aa9302354
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Tue Oct 3 15:06:25 2017 -0700
+
+    Revert "SIGPIPE suppression for platforms without SO_NOSIGPIPE or MSG_NOSIGNAL"
+    
+    This reverts commit 131deb39769c1187c334ee84f552d3be01f1751b. It added
+    code that shouldn't have entered the project repo.
+    
+    Conflicts:
+            src/msg/async/AsyncConnection.cc
+            src/msg/async/AsyncConnection.h
+            src/msg/simple/Pipe.cc
+            src/msg/simple/Pipe.h
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 21bdb730932196a48407be0effab8813a78c4b1b)
+
+commit 72c858310796848128bbccfa328794d504471e5a
+Author: wujian <wujian3659@163.com>
+Date:   Fri Sep 15 11:23:44 2017 +0800
+
+    ceph-mgr: can not change prometheus port for mgr
+    
+    Signed-off-by: wujian <wujian3659@163.com>
+    (cherry picked from commit 6cc0338ed15a0402471cfd05acfcf00d07517d1a)
+
+commit 49294965f9f92946382949264344c2aa0b6d70bd
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Oct 4 08:25:38 2017 -0500
+
+    mgr/localpool: fix rule selection
+    
+    The 'osd pool create' arg parsing is broken; the rule name for
+    'ceph osd pool create $name $numpgs replicated $rulename' is passed
+    via the erasure_code_profile param.  Too many req=false options
+    without a way to disambiguate them.
+    
+    Work around it by passing both 'rule' and 'erasure_code_profile'
+    keys, so that if/when the hack in OSDMonitor.cc is removed it will
+    still work.  Blech.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 7623513935525498640defa2064c291fd69a2b76)
+
+commit cf6f90c5e1e17a1f44f6ed80422b5f1c543362ca
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 2 17:11:46 2017 -0500
+
+    mgr/localpool: optionally adjust min_size too
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit af72a8932c60a52aad76f1cc94bfbb31048215cc)
+
+commit d18b7b8652e38c679993241ac232bb7310db7e22
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 2 17:11:38 2017 -0500
+
+    mgr/localpool: default to 3x
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9624923f19dc649a8f182f28e9580a3045964d9e)
+
+commit 80f6508fc25018dd9c70d05f99c40325062d129e
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 21 15:19:47 2017 -0400
+
+    osd: make shutdown debug conditional (and off by default)
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6169cf64423c7554dc0630d90d8dee27437bf05f)
+
+commit ca177e93d4e52d50a49b76af516a8009153197e7
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 7 17:12:42 2017 -0400
+
+    osd: debug_bluestore on shutdown
+    
+    ...just like we do with filestore etc.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 354dccda322b7ac8844b321f603e049128bf4779)
+
+commit 4d931ad0d349a6974f9f7053792e6f826b67169a
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 7 17:11:30 2017 -0400
+
+    os/bluestore: dump stray cache content on shutdown
+    
+    Tracking down http://tracker.ceph.com/issues/21259.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 28d9b6b0e92cf51996a12a43c81f7ac2abcaecaa)
+
+commit 66af04cd9b61d482362361cb3b12af2d8ffa462c
+Merge: 57b5f58f5e b064ed172a
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Oct 3 13:41:21 2017 -0500
+
+    Merge pull request #17734 from liewegas/wip-21089-onmount
+    
+    luminous: os/bluestore: replace 21089 repair with something online (instead of fsck)
+    
+    Reviewed-by: xie xingguo <xie.xingguo@zte.com.cn>
+
+commit 57b5f58f5e8be116deed59c5c97636eb730c3dc8
+Merge: ae9de1673a a069e8ae61
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Oct 3 13:40:56 2017 -0500
+
+    Merge pull request #17814 from liewegas/wip-pg-limits-luminous
+    
+    luminous: mon: cluster limit on pgs
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit a069e8ae61a452cefe5673f0035251c00857fee5
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 14 16:01:14 2017 -0400
+
+    doc/rados/operations/health-checks: fix TOO_MANY_PGS discussion
+    
+    Fiddling with pgp_num doesn't help with TOO_MANY_PGS.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 027672b777402381f6736e517ed287b38bb17abb)
+
+commit 7320ee7293a34e4ea0ab5254373f51b93249b91d
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 14 16:00:31 2017 -0400
+
+    mon: rename mon_pg_warn_max_per_osd -> mon_max_pg_per_osd
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 986b86fbebf9e06f9f841da8ded0bedb310fc69b)
+
+commit 37ddfc61c3376afeccfe7a8029412b9a38401bfc
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 22 12:03:57 2017 -0400
+
+    rgw: dont reuse stale RGWObjectCtx for get_bucket_info()
+    
+    if the earlier call to store->get_bucket_entrypoint_info() failed with
+    ENOENT, the obj_ctx will cache exists=false.
+    put_bucket_entrypoint_info() doesn't invalidate that, so this call to
+    get_bucket_info() was reading from a stale cache and failing with ENOENT
+    
+    Fixes: http://tracker.ceph.com/issues/21506
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 3f4f760a85f162262c5b419e9cf78966c7299f0a)
+
+commit 28e7d5645e5fdf5471a2a809bd232521f2d33814
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 20 12:38:46 2017 -0400
+
+    os/bluestore: ignore 0x2000~2000 extent oddity from luminous upgrade
+    
+    Luminous does a block_size granularity freelist, and assumes that
+    0~ROUND_UP_TO(SUPER_RESERVED,block_size) is used.  Current master uses
+    min_alloc_size granularity and changes that assumption to
+    0~ROUND_UP_TO(SUPER_RESERVED,min_alloc_size).  That means if master
+    fsck's a luminous-created bluestore, it will think 0x2000~2000 is used
+    (current baked-in min_alloc_size-based assumption) but the old freelist
+    says it is free (old mkfs assumption).  The disparity is harmless since
+    the extent is below min_alloc_size, so ignore it.
+    
+    Fixes: http://tracker.ceph.com/issues/21408
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4959ad310a96208565b47c035ab5d5dded1b8ff5)
+
+commit ae9de1673a7b4aa032cbde7c5646c4d945ff618a
+Merge: eec0edaaa0 3a037dda42
+Author: Sage Weil <sage@newdream.net>
+Date:   Sun Oct 1 15:56:10 2017 -0500
+
+    Merge pull request #17867 from smithfarm/wip-21443-luminous
+    
+    luminous: mgr: Prometheus crash when update
+    
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit eec0edaaa031400e7728081c306a1901cb981486
+Merge: 98e729a977 ddca5dbe04
+Author: Sage Weil <sage@newdream.net>
+Date:   Sun Oct 1 15:55:59 2017 -0500
+
+    Merge pull request #17868 from smithfarm/wip-21452-luminous
+    
+    luminous: mgr: prometheus module generates invalid output when counter names contain non-alphanum characters
+    
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 98e729a9773acb5200357b3178cf04883383cf4a
+Merge: 87e8829605 faf3fec1a6
+Author: Sage Weil <sage@newdream.net>
+Date:   Sun Oct 1 15:55:30 2017 -0500
+
+    Merge pull request #17896 from liewegas/wip-bluestore-tool-luminous
+    
+    ceph-objectstore-tool and ceph-bluestore-tool: backports from master
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 87e8829605adb2609cd4867f3d23489d84e657fb
+Merge: a48b2e41e7 1fc6a51a41
+Author: Sage Weil <sage@newdream.net>
+Date:   Sun Oct 1 15:55:11 2017 -0500
+
+    Merge pull request #17930 from jcsp/wip-luminous-health-warn
+    
+    mon: show legacy health warning in `status` output
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit a48b2e41e78f9190b7f94d11af2a04c919380aae
+Merge: ced74c00d6 4f28ef0016
+Author: Sage Weil <sage@newdream.net>
+Date:   Sun Oct 1 15:54:47 2017 -0500
+
+    Merge pull request #18031 from xiexingguo/wip-pr-17987
+    
+    luminous: osd/PrimaryLogPG: kick off recovery on backoffing a degraded object
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit ced74c00d6de9fbf84ecb4d3064fbaec01ec827f
+Merge: 242635d938 f9d0442ca5
+Author: Sage Weil <sage@newdream.net>
+Date:   Sun Oct 1 15:54:17 2017 -0500
+
+    Merge pull request #18033 from xiexingguo/wip-pr-17698
+    
+    luminous: os/bluestore: allocate entire write in one go
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 242635d938fcd46d337b7727f0f3f578122b41ad
+Merge: 9d8acf6f3f 86d143722e
+Author: Sage Weil <sage@newdream.net>
+Date:   Sun Oct 1 15:53:56 2017 -0500
+
+    Merge pull request #18038 from jecluis/wip-21300-luminous
+    
+    luminous: mon/MgrMonitor: handle cmd descs to/from disk in the absence of active mgr
+
+commit 1021fe235ca756337e3a3ff4804680c1e93b0270
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Sep 29 08:48:14 2017 -0700
+
+    qa: relax cap expected value check
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit b37c7f7db7f0d76e524aabdad6dd0f16548268a6)
+
+commit 633b114aca65548b91a898db7bbbef784b76180b
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 8 18:08:51 2017 -0400
+
+    os/bluestore: use min_alloc_size for freelist resolution
+    
+    For HDD with min_alloc_size=64k, this is a 16x reduction in allocation
+    metadata!
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6b8e4d512604095fb8a209229d4633ac19b499de)
+    
+    Conflicts:
+    Slightly conflict with 834542c4027a8cc4d23eff089827d328919372d2, which
+    drops the literal description of apply().
+
+commit 3decf1c0de9f54b09afc954b13c2c8fac1011de9
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 8 18:08:07 2017 -0400
+
+    os/bluestore: align bluefs_extents to min_alloc_size
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 0c777efdcb2ee5a6322f0eb277e681d0f086e0b6)
+
+commit 529c02db4da9b45cc13bedaa20509a896217fd83
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 8 18:07:38 2017 -0400
+
+    os/bluestore/FreelistManager: create: accept min alloc size
+    
+    Accept a block size other than bdev_block_size.  Let's call it, oh, I don't
+    know, min_alloc_size.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 52453d4ca223c8819f8e35f2c0b691803e74537f)
+
+commit 37929849863c60531bcdbf80124e751504ac710f
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 8 18:06:05 2017 -0400
+
+    os/bluestore: mkfs: choose min_alloc_size earlier
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3efde01586776b23bbac1e663ae5baf6500acee4)
+
+commit efcf874295db0a68c43a9fd0a3a20cc3dcdf050a
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 8 18:05:29 2017 -0400
+
+    os/bluestore: require that bluefs_alloc_size be multiple of min_alloc_size
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5b47ac59816894e983a98d3da8b5415d569c6663)
+
+commit 58c52baeaed4a0a6dd5aed076db60988845431d0
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 27 17:42:07 2017 -0400
+
+    qa/suites/rados/singleton/all/recovery-preemption: add test
+    
+    This mirrors what I was testing locally.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d7b29acb1952d0b3dfd19326fd3418cfbd66ef3c)
+
+commit 2e28f3e5ae3a4140e802f915b8db5dc434b6a3c2
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 21 12:37:13 2017 -0400
+
+    osd/PG: handle racy preemption
+    
+    If we finish recovery/backfill and go active, but also get
+    preempted at the same time, we can ignore the event.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit d8c3756d26abbaf326f634a494dcb12fe373f68d)
+
+commit ba106b6d32fb3138b7a61fe613b211e02f1e44aa
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 19 15:26:40 2017 -0500
+
+    osd/PG: allow local recovery reservations to be preempted
+    
+    If a PG has a higher recovery priority and a lower-priority item is in
+    progress, allow it to be preempted.  This triggers the RecoveryCancel
+    or BackfillCancel event with a 0 delay, which means it will immediately
+    re-request a reservation (and presumably wait).
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit a8534cce1a1661375a93aff2314633bce63695ed)
+
+commit 33ea286d0cedf44b7b3c70bc9f35c7f88fcc0361
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 19 15:25:05 2017 -0500
+
+    common/AsyncReserver: support preemption
+    
+    If an (optional) preemption context is provided, use that to preempt
+    and existing reservation and grant a higher-priority one.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit dbc002eaa90e952df1acf295a630443ac3ada418)
+
+commit dad61658d1f48f1089e265d7441bf1349fd42f1f
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 21 12:37:49 2017 -0400
+
+    common/AsyncReserver: get a cct
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 08d2c8875bba7c41d1f9eeec586f6880f8be99b2)
+
+commit 1b0875ca6d9ab91f59ffe2356113ed2cc643a276
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 21 10:30:57 2017 -0400
+
+    osd: PG_STATE_BACKFILL -> PG_STATE_BACKFILLING
+    
+    Match user-facing string
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 31a34946f7aa0ad2b9848cbc808d209394a9ad44)
+    
+    - add update to mon/PGMap.cc
+
+commit 9d8acf6f3f85dd1410629cc680474e38f3be9838
+Merge: 62ff5e4c98 b54c7ba225
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 29 07:11:31 2017 -0500
+
+    Merge pull request #17862 from smithfarm/wip-21307-luminous
+    
+    luminous: mon: Client client.admin marked osd.2 out, after it was down for 1504627577 seconds
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 62ff5e4c981a9e0d9a92bf3ac85210d98b0db038
+Merge: 453858d4b7 4c9d86bd86
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 29 07:10:55 2017 -0500
+
+    Merge pull request #17865 from smithfarm/wip-21465-luminous
+    
+    luminous: OSD metadata 'backend_filestore_dev_node' is unknown even for simple deployment
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 453858d4b777ee81e5af4f443afc07e6af8b1a60
+Merge: 2e85e4f0c4 430a140d38
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 29 07:10:26 2017 -0500
+
+    Merge pull request #17936 from liewegas/wip-ruleset-errors-luminous
+    
+    mon/OSDMonitor: error out if setting ruleset-* ec profile property
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 2e85e4f0c4f196dce8115e0a3aecc506caf1c7d1
+Merge: 1c228c95dc 481da57cca
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 29 07:10:06 2017 -0500
+
+    Merge pull request #17946 from liewegas/wip-mgr-localpool-luminous
+    
+    luminous: mgr/localpool: module to automagically create localized pools
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 1c228c95dc142c3ca142f039e606e25cb500e53a
+Merge: 9915a2f600 9aafc53ed4
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Sep 29 07:09:27 2017 -0500
+
+    Merge pull request #17998 from liewegas/wip-pr-17978-luminous
+    
+    rbdmap: fix umount when multiple mounts use the same RBD
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 86d143722e1f16a19a56ed7a163d1dc9327c0c90
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Wed Sep 20 17:53:40 2017 +0100
+
+    mon/MgrMonitor: read cmd descs if empty on update_from_paxos()
+    
+    If the MgrMonitor's `command_descs` is empty, the monitor will not send
+    the mgr commands to clients on `get_descriptions`. This, in turn, has
+    the clients sending the commands to the monitors, which will have no
+    idea how to handle them.
+    
+    Therefore, make sure to read the `command_descs` from disk if the vector
+    is empty.
+    
+    Fixes: http://tracker.ceph.com/issues/21300
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+    (cherry picked from commit 3d06079bae0fbc096d6c3639807d9be3597e841a)
+
+commit 8b2bd387859bdc2287ba6dcaba27d8b234984a5a
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Wed Sep 13 19:53:43 2017 +0530
+
+    pybind/ceph_volume_client: add get, put, and delete object interfaces
+    
+    Wrap low-level rados APIs to allow ceph_volume_client to get, put, and
+    delete objects. The interfaces would allow OpenStack Manila's
+    cephfs driver to store config data in a shared storage to implement
+    highly available Manila deployments. Restrict  write(put) and
+    read(get) object sizes to 'osd_max_size' config setting.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit d1bd171d6b6eb00c47168f38cec1a30f9c9f02bd)
+
+commit 9421b40f1748c50b7f92abe8615c0cd9208fdbdc
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Mon Sep 18 20:16:30 2017 +0530
+
+    pybind/ceph_volume_client: remove 'compat_version'
+    
+    ... class attribute of the 'CephFSVolumeClient' class. It was supposed
+    to record the earliest version of CephFSVolumeClient that the current
+    version is compatible with. It's not useful data to be stored as a
+    class attribute.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 894a734aa5a56b20d3f68bb3ad644b370f193934)
+
+commit dd23023329b2382afef23ee02d7ef9b14baaf07b
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Mon Sep 18 20:09:55 2017 +0530
+
+    pybind/ceph_volume_client: set the version
+    
+    ... of on-disk structures to be same as the class attribute 'version'
+    of the CephFSVolumeClient class.
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit 8267c2ba09841095f20a8833c155185529e64f46)
+
+commit 9ee3f7ad21049d0692804d4405b4bf0f8c795ae3
+Author: Joao Eduardo Luis <joao@suse.de>
+Date:   Wed Sep 27 17:55:17 2017 +0100
+
+    mon/MgrMonitor: populate on-disk cmd descs if empty on upgrade
+    
+    During kraken, when we first introduced the mgrs, we wouldn't populate
+    the on-disk command descriptions on create_initial(). Therefore, if we
+    are upgrading from a cluster that never had a mgr, we may end up
+    crashing because we have no cmd descs to load from disk.
+    
+    Fixes: http://tracker.ceph.com/issues/21300
+    
+    Signed-off-by: Joao Eduardo Luis <joao@suse.de>
+
+commit f9d0442ca50078224b72f9876d2fed41fd409be6
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Aug 10 16:44:59 2017 -0400
+
+    os/bluestore: allocate entire write in one go
+    
+    On the first pass through the writes, compress data and calculate a final
+    amount of space we need to allocate.  On the second pass, assign the
+    extents to blobs and queue the writes.
+    
+    This allows us to do a single allocation for all blobs, which will lead
+    to less fragmentation and a much better write pattern.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e200f358499af8e3acb6ac4f675cc167433b53ec)
+
+commit 4f28ef00162692557654d84bc1c0988998a6f2c4
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Wed Sep 27 16:05:56 2017 +0800
+
+    osd/PrimaryLogPG: kick off recovery on backoffing a degraded object
+    
+    As we are now blocking frontend ops from accessing that very object!
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit db20328b456d70d6728fd27f17da6f2f3546e84b)
+
+commit 711892dceaa7e4598bffff0edcff45554bc06ffc
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu Sep 28 16:01:00 2017 -0700
+
+    mds: improve cap min/max ratio descriptions
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit dcf97d17356f59c8870a36a99a77f879a20df348)
+
+commit bef8ded7b82fc6896398bd334bdbd4fc63ebfe51
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Thu Sep 28 15:56:26 2017 -0700
+
+    mds: fix whitespace
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit ab69d682937a0a13246d28cfa2bbbf8c1b7ced38)
+
+commit a5c96d4f7a5081242c6abf6c722adfa131030a55
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Wed Sep 27 10:39:01 2017 -0700
+
+    mds: cap client recall to min caps per client
+    
+    Fixes: http://tracker.ceph.com/issues/21575
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 538834171fe4524b4bb7cffdcb08c5b13fe7689f)
+
+commit 913c72d96a6c405b4e4fe57abcfbf58d53a66e63
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Wed Sep 27 09:29:39 2017 -0700
+
+    mds: fix conf types
+    
+    This correct an assertion failure.
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit bfc490249566356cff785f2f89dc03d69af322da)
+
+commit 91084538cdfe80230bf346ee9cb6e8bfe080d6d8
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Wed Sep 27 09:29:23 2017 -0700
+
+    mds: fix whitespace
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 8a5d71bf4517e6a3c194c3e63e0283747b75d4b1)
+
+commit 5b1306142feb820a6c4f6e180ba68e428985b3f1
+Author: ren.huanwen <ren.huanwen@zte.com.cn>
+Date:   Mon Sep 18 15:51:29 2017 +0800
+
+    doc/cephfs: add client min cache and max cache ratio describe
+    
+    Signed-off-by: ren.huanwen <ren.huanwen@zte.com.cn>
+    (cherry picked from commit d879ff91e76f66f90364038920fc2c62dd18c449)
+
+commit a39ab633cb97fd4fa0e1ef3e0e2e56709f980857
+Author: ren.huanwen <ren.huanwen@zte.com.cn>
+Date:   Mon Sep 18 15:14:19 2017 +0800
+
+    mds: adding tunable features for caps_per_client
+    
+    Sometimes we need to raise or lower the value of "max_caps_per_client" and
+    "min_caps_per_client" to improve recall_client_state efficiency
+    
+    Signed-off-by: ren.huanwen <ren.huanwen@zte.com.cn>
+    (cherry picked from commit a07b376628505832f5528bd7f2b0faeabe9a5f5d)
+
+commit 4ff28ab144b3a0adfea8ad8ddee750243c824d14
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 21 10:28:59 2017 -0400
+
+    osd/osd_types: make BACKFILL <-> "backfilling" for parser
+    
+    We render BACKFILL as "backfilling"; make sure parse works that
+    way too.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6fa40e44dc579a009edff7be95d3fd37defbc7f6)
+
+commit 2a9cfa2ad256df11bc1c3d95720ef89d11813846
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 21 10:28:08 2017 -0400
+
+    osd/osd_types: remove weird BACKFILL state hack
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5bcfaf4bd40fee9ea4bfd5ed086b51facc5fe78e)
+
+commit 429804b6bd193611f7474174e45ce94d784fd184
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 19 15:47:23 2017 -0500
+
+    osd/PG: Cancel{Recovery,Backfill} -> Defer{Recovery,Backfill}
+    
+    "Defer" is more accurate here; we aren't canceling anything, just
+    rescheduling the work.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 2e45497a20a0b61975fbf0cf851d417f31b35489)
+
+commit 4cbd62ecdc42aa91045f7f5c7ce5dab0cb59c16e
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 19 14:53:31 2017 -0500
+
+    osd/PG: specify delay in Cancel{Recovery,Backfill}
+    
+    For now it is always the retry interval, but later perhaps not!
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 597dfd11728b55ad181316ac10be7155e37a6ba9)
+
+commit 86778de01166902612b3173f1d0687f13027a21f
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 19 14:49:05 2017 -0500
+
+    osd/PG: make some trivial events TrivialEvent
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3eadfa087f3ff5338f87306fd384d1f4749b8005)
+
+commit 6c529092e8f4bd1e2a4a4069bf9f319d49430ccb
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 19 14:44:50 2017 -0500
+
+    osd/PG: set {backfill,recovery}_wait when canceling backfill/recovery
+    
+    The only caller currently is when we get as far as we can with backfill
+    or recovery but still have unfound objects.  In this case, go back into
+    the *_wait state instead of appearing as though we are still doing
+    something.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4b216771bd31e5712471e13ad32ee8a1e519eb30)
+
+commit 06c31a6ceecf1c33e1456c746169961802903001
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Sep 27 22:19:06 2017 +0800
+
+    mds: make sure snap inode's last matches its parent dentry's last
+    
+    This patch is for luminous only. The issue has been fixed in master
+    branch in another way (change is much bigger)
+    
+    Fixes: http://tracker.ceph.com/issues/21337
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+
+commit 1236e0f2e9e8724569cf5fc7d7d525c7fca6ad77
+Author: Yao Zongyou <yaozongyou@vip.qq.com>
+Date:   Sun Sep 24 22:11:22 2017 +0800
+
+    rgw_file: fix write error when the write offset overlaps.
+    
+    Fixes: http://tracker.ceph.com/issues/21455
+    
+    Signed-off-by: Yao Zongyou <yaozongyou@vip.qq.com>
+    (cherry picked from commit 872d73f945364002f0fa31762e6976db5b4b3c19)
+
+commit 9aafc53ed4edca8bebb96bc36a0b37ce32c79a75
+Author: Alexandre Marangone <a.marangone@gmail.com>
+Date:   Tue Sep 26 11:35:04 2017 -0700
+
+    rbdmap: fix umount when multiple mounts use the same RBD
+    
+    When a Kubernetes Pod consumes a RBD it is mounted two
+    times on the same host. When the host shutdown umount will
+    fail leading to a hung system
+    
+    Signed-off-by: Alexandre Marangone <amarango@redhat.com>
+    (cherry picked from commit 40825daecedb2a3481021e4d36a367c339eb9b62)
+
+commit 9915a2f600788c37fb42f39a766ce93b9ae4d816
+Merge: 3e7492b9ad 6b5f212b91
+Author: vasukulkarni <vasu.kulkarni@gmail.com>
+Date:   Wed Sep 27 09:50:44 2017 -0700
+
+    Merge pull request #17953 from ceph/wip-ceph-ansible-bport
+    
+    tests - ceph-ansible backports
+
+commit 6b5f212b9170dbd99779527b41be233b9d40b5e4
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Thu Sep 21 15:34:31 2017 -0700
+
+    use ovh instead of vps
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 56d1511bdfaee5a2a0d85bd606348632bfcbf09e)
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit ab5300218699c947f380b0437f931d559bdbf074
+Author: Sébastien Han <seb@redhat.com>
+Date:   Tue Sep 26 14:05:37 2017 +0200
+
+    rbd-mirorr: does not start on reboot
+    
+    The current systemd unit file misses 'PartOf=ceph-rbd-mirror.target',
+    which results in the unit not starting after reboot.
+    If you have ceph-rbd-mirror@rbd-mirror.ceph-rbd-mirror0, it won't start
+    after reboot even if enabled.
+    Adding 'PartOf=ceph-rbd-mirror.target' will enable
+    ceph-rbd-mirror.target when ceph-rbd-mirror@rbd-mirror.ceph-rbd-mirror0
+    gets enabled.
+    
+    Signed-off-by: Sébastien Han <seb@redhat.com>
+    (cherry picked from commit e6cd9570ba210c5f4501b6f4fa443245a22c4d6c)
+
+commit 481da57cca2f43f3cb14beaaa22eff3b2f62c875
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Sep 26 12:10:28 2017 +0800
+
+    doc/mgr: add "local pool" plugin to toc
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 0b831e7886daad5836ca3cb58c01c123f99a1cde)
+
+commit 315e336f38d1bb0e821dbc1943447cc61f203e2c
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Tue Sep 19 17:09:08 2017 -0700
+
+    qa/tests: skip test that use public bucket landsat-pds from AWS
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit e11a1f773d8795dd9bd9819aaf229c6598e22ca6)
+
+commit fc7db58fc5ed071436d5f0f6cc9dcd61e3276aea
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Tue Sep 19 17:06:03 2017 -0700
+
+    qa/tests: update required ceph-ansbile vars
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 184d5e6bad46451b3a8c5787f56148566bb42f20)
+
+commit 4342d80354c941d47598897ca7ec6a6b4928edf6
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Tue Sep 19 17:03:46 2017 -0700
+
+    qa/tests: Fix ceph-ansible upstream vars
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 077904a0b5dffe9d64feade94cf30ffc92f1c0e5)
+
+commit a3d43ad8f3e685a028e943b5aee9ec4720cba60b
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Sun Sep 10 12:42:16 2017 -0700
+
+    Fix get_system_type failure due to invalid remote name
+    
+    recent changes caused the remote name to be invalid, fix the
+    arg passed to get_system_type
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 9fe58d5ed6bd2338271a9e003f51d80c2e440e23)
+
+commit ea8ddc95a8336f37065edfd7f061601c5c4cac4b
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Fri Sep 8 10:00:49 2017 -0700
+
+    Stop the mgr on node which is not client, this will ensure
+    the client.0's mgr is active mgr for workunit to work.
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit b1fd615e9f507d4243744b34887366086c1890ac)
+
+commit 69bbaf5173c09afbf02b1894843570d403ad4235
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Thu Sep 7 17:10:04 2017 -0700
+
+    Test ceph-mgr RESTful api
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit e5b5a1b8c8d0e750d29fe41a16252dcb0adf449f)
+
+commit 581656e3874822f958400c235241cddb07c4aca3
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Thu Sep 7 16:54:51 2017 -0700
+
+    Add additional variables as required by ceph-ansible for upstream
+    installation.
+    
+    possible revert if those are made default
+    issue: https://github.com/ceph/ceph-ansible/issues/1834
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit e963bfd6ca0ea1525e37dc35bf3309548c7a3831)
+
+commit 559a47997c7f6c1e8e86e2fd06759ded0e7cccd9
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Fri Aug 18 11:35:54 2017 -0700
+
+    Rename folders to fix task order
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 9cc00c5c1a6e99323a4655aad5c093b8d87609f6)
+
+commit 655b0733970c6798af087fdb886bf914711fef1b
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Fri Aug 18 11:09:50 2017 -0700
+
+    use bluestore with dmcrypt option
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 1041c803f1f8f7be0edd0e78df352152fa3c4e6f)
+
+commit 244689b36a2beb6dbd9beed429e5b46f4b714bc3
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Fri Aug 18 11:08:00 2017 -0700
+
+    Add dmcrypt option
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit f6de5d9f9e2a3666116311c146a8f2a45f3b8dbb)
+
+commit d2ddcc0324a66b1cb67a640df987e5bc1a6f4887
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Fri Aug 18 11:05:01 2017 -0700
+
+    Separate the main task from options
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 60d00e0eadda3a7a7cfbac2acdc81a4d55a27570)
+
+commit 05f7ce97a8b94b5abcaddd7d125398c7d0118bbc
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Fri Aug 18 10:47:22 2017 -0700
+
+    Catchup with recent changes with ceph-ansible
+    
+    Adds osd_scenario and ceph_stable_release variables
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 0395b844884a63a2cb7480dcc40762df6915b403)
+
+commit 58ffed6fb9e8695deb53edd7754aed56c73403bd
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Thu Aug 24 11:02:20 2017 -0700
+
+    Add workaround for http://tracker.ceph.com/issues/20950
+    
+    mgr bootstrap key differs on disk, rewrite the new key
+    using auth get.
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 5c43c5972ddb04724c8659a1b8d71cda1e0acdfb)
+
+commit c2284be89df724bbf1446b28cdf712722d7581a1
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Thu Aug 24 09:24:52 2017 -0700
+
+    Add kraken to luminous upgrade case
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 71ea4987e0d55cc49e90dfbaf9722769cc892f3f)
+
+commit 6cc150023f1dc2450bbf83ebdba8360c5693645c
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Mon Aug 7 12:19:41 2017 -0700
+
+    Add kernel tar/untar and systemd task
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 526d4aef1cb73ffa6f73e1353dc06eed6cccb9f9)
+
+commit 39eb7cb0bd77925d422c68aaed86fbd39bdfc80e
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Tue Aug 1 16:32:42 2017 -0700
+
+    Restart services after upgrade
+    
+    This seems to be an issue and should be fixed in our systemd scripts,
+    need to discuss more on this.
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 2116f76a1f1fbb32ca5844c1d051e22b1b657e0f)
+
+commit 566dada1f6a73b8be128167a425becea7ba110fe
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Fri Jul 28 23:11:41 2017 -0700
+
+    since the roles are mapped inside ceph-deploy, store the roles that
+    are mapped and use the new mapped role for upgrades during later
+    stage.
+    
+    eg: mon.a is mapped to mon.mira002 during install, store this mapping
+    and durig upgrade map it back to appropriate name to find the hostname
+    with that role
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 8a2b9a08095dc97e0e440cc900c650d5efb55a4f)
+
+commit 093a13268dff5b1114294456e83e0e24bfd7be6d
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Fri Jul 28 17:48:09 2017 -0700
+
+    Add doc string for missing ceph-deploy-branch
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit e7a0a4f87f2fbea0c9e27341c952245f1ad34ab8)
+
+commit df415e81887b06918731d7d3b7677ecab1ee2db5
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Thu Jul 27 15:24:25 2017 -0700
+
+    use mon.a as ceph-admin node, the previous get_first_mon returns
+    a sorted(mons)[0] as first mon.
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 407ce2613d37252e48c2a648b94b065ab070ec46)
+
+commit 1128ae099cc350f29b243321bc2815aaf7bbfe80
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Thu Jul 27 11:17:31 2017 -0700
+
+    use elif instead of else to handle cases for jewel install
+    
+    jewel needs neither filestore or bluestore as an option, so provide none
+    when running with jewel branch.
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit f3ec7cabf66c76d712f77a4dfefe96f374507982)
+
+commit 7db1550a55002d28d899aee6ec695604045edabb
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Wed Jul 26 19:18:11 2017 -0700
+
+    Jewel to luminous upgrade scenario using ceph-deploy
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 9c85c76a90dcf46ec2bcc6840f4fc57ed86583b5)
+
+commit ab219d2e985cd1c6a7fd17b5ced4db25cb0b7427
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Wed Jul 26 18:56:08 2017 -0700
+
+    Add upgrade functionality using ceph-deploy
+    
+    This is to test for customer like upgrade scenarios and to find
+    any issues that may be related to systemd, packaging etc
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 9a73127c16d3395934d003024964ed1373d81083)
+
+commit a145127bde2e2fb96a1fd2b821ed2b647e8ae223
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Wed Jul 26 15:01:15 2017 -0700
+
+    Fix old flake8 issues
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit b409c3d163194b0b65e26b0fbc93def8e17a10b4)
+
+commit 28b3c242bdbfc655534cdd037966ed1dcddc978a
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Wed Jul 26 14:58:50 2017 -0700
+
+    Add option to skip mgr install for old release
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit a9be0c387ed2ae09227dad833c8647af110bd2de)
+
+commit 82651da2d504913f2145085df8a89484308f3eba
+Author: huangjun <huangjun@xsky.com>
+Date:   Wed Sep 20 00:04:04 2017 +0800
+
+    qa/ceph_manager: check pg state again before timedout
+    
+    Pg state maybe all in active+clean when no recovering going on,
+    so check it again before timedout.
+    
+    Fixes: http://tracker.ceph.com/issues/21294
+    
+    Signed-off-by: huangjun <huangjun@xsky.com>
+    (cherry picked from commit fa40add7f0123dfeac30986f3d53cdfa77736a87)
+
+commit 4017a26162706f1b6bed3f7b37174c8ca47b54ad
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Sep 24 14:05:14 2017 -0400
+
+    qa/workunits/mgr/test_localpool: simple test for localpool mode
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6383fa5b3004a606c15efd2be1b7ce5d57f05086)
+
+commit 45a804f25bc42f03b0596b1ba394fb6be575d392
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 6 15:34:50 2017 -0400
+
+    pybind/mgr/localpool: module to automagically create localized pools
+    
+    By default, this will create a pool per rack, 3x replication, with a host
+    failure domain.  Those parameters can be customized via mgr config-key
+    options.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1a0f42b70a4c9fa68dc47f2f521d0f1e8f5bb220)
+
+commit 430a140d38ba6b9e07c53d5a4a5c6232bef4fb0e
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 20 16:42:01 2017 -0400
+
+    mon/OSDMonitor: error out if setting ruleset-* ec profile property
+    
+    We change ruleset -> crush back in dc7a2aaf7a34b1e6af0c7b79dc44a69974c1da23.
+    If someone tries to use the old property, error out early, instead of
+    silently not doing the thing they thought they told us to do.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6376d75eda648789b5e316a1ac2883708db7128e)
+
+commit 1fc6a51a416761d9ccd172937cda1a6b1c11faad
+Author: John Spray <john.spray@redhat.com>
+Date:   Sat Sep 23 13:48:36 2017 +0100
+
+    mon: show legacy health warning in `status` output
+    
+    Previously you only got the text of this if you were
+    either looking at "health detail" or if you had
+    already set the preluminous_compat setting (in which
+    case you presumably were already aware so the message
+    isn't doing much).
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+
+commit d5e583490d946242bdfd26e863dd0aac27c12e3c
+Author: Ramana Raja <rraja@redhat.com>
+Date:   Wed Sep 20 20:27:22 2017 +0530
+
+    ceph_volume_client: perform snapshot operations in
+    
+    ... client configured snap directory name, instead of in hard-coded
+    '.snap' directory.
+    
+    Fixes: http://tracker.ceph.com/issues/21476
+    
+    Signed-off-by: Ramana Raja <rraja@redhat.com>
+    (cherry picked from commit f4fc1722594ed007706b54901fb07a2a443d1b96)
+
+commit e4164c43f5ec808c0420036d2a647a5a7d38a436
+Author: Carl Xiong <xiongc05@gmail.com>
+Date:   Fri Sep 22 11:55:33 2017 +0800
+
+    ceph-disk: fix '--runtime' omission for ceph-osd service
+    
+    f425a127b introduces a regression that ceph-disk omits "--runtime" when
+    enabling ceph-osd@$ID.service units for device-backed OSDs.
+    
+    Fixes: http://tracker.ceph.com/issues/21498
+    
+    Signed-off-by: Carl Xiong <cxiong@suse.com>
+    (cherry picked from commit a385b5b0c35106c9b44d81655983b2f7566b21cd)
+
+commit faf3fec1a628c9c4f4791e33d7795ef719267cbc
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 13 18:33:21 2017 -0400
+
+    ceph-bluestore-tool: better default logging; --log-file and --log-level options
+    
+    - keep derr going to stderr (so we see 'fsck error' messages)
+    - hide the rest
+    - add friendly --log-file option
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9bd98b42fea1791935280f5ad19e5420d5303e03)
+
+commit 2184e3077caa9de5f21cc901d26f6ecfb76de9e1
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 7 18:27:20 2017 -0400
+
+    ceph-bluestore-tool: add 'bluefs-bdev-expand' to expand wal or db usage
+    
+    If you are using the wal or db devices, this will expand bluefs's usage to
+    include the entire block device.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit aa4baad529835c1999ff9cc1a2f509c52a0cc699)
+
+commit af9a43a47b1dd0d8bb9946d4b4a0ab9b5c39e427
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 7 18:20:27 2017 -0400
+
+    ceph-bluestore-tool: add 'bluefs-bdev-sizes' command
+    
+    Show bdev sizes vs owned extents.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9e492f2a2217ee94670e9e0d6cf28e4b7bff5ad0)
+
+commit 6a6678c52ac4aba7fd313ddc2cb3c013819fe28f
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 7 18:12:21 2017 -0400
+
+    ceph-bluestore-tool: factor out bluefs mount
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 86db2d7b09de4362d90fba834d61978860d73309)
+
+commit 53f9941f28ac1435bf14f48d369b8ea03336c9d2
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 7 12:27:35 2017 -0400
+
+    os/ObjectStore: add repair interface
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c7b7a1f04f78fa62890c567d0ca53874c8d75eb7)
+
+commit edd4d98fefe63ad31bf96fa61716b4ee079c0498
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Sep 8 17:53:07 2017 -0700
+
+    ceph-objectstore-tool: Make pg removal require --force
+    
+    Add new export-remove to combine the 2 operations
+    
+    Fixes: http://tracker.ceph.com/issues/21272
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 3bb20f6d750915d176c7a34367dac966a20faa76)
+
+commit bbe0f34cc81a8db873ef23cca0cdea8651411b86
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Sep 8 17:09:48 2017 -0700
+
+    ceph-objectstore-tool: Better messages for bad --journal-path
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 49ca1fff7fc4360d2f3a9cac60c0ba651cbc4750)
+
+commit fd7d53bacb10d5b2094f1f579c96899d8a0a4b4f
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Sep 6 20:41:50 2017 -0700
+
+    test: Fix ceph-objectstore-tool test for standalone and latest code
+    
+    vstart.sh now defaults to bluestore, so specify filestore
+    Set environment for run-standalone.sh and cmake build
+    Create td/cot_dir as test directory
+    Crush output format change
+    Change dir into test directory
+    Give a little time after pool creation
+    Check for core files as ceph-helpers.sh does
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 3ac219df2d635a08be52df3ad44cf0683de46af2)
+
+commit 3e20e0f3071188f52b7a7c3f9e36b09305431ed0
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Sep 6 20:40:37 2017 -0700
+
+    test: Move ceph-objectstore-tool test to standalone
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 495c32fd31b311d22f0d4509e896916bd2671615)
+
+commit ddca5dbe04a2647218fabff6098691edd6b7a157
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Aug 28 19:55:06 2017 +0100
+
+    mgr/prometheus: only turn - into _minus at end
+    
+    ...of perf counter name.
+    
+    So that mds_mem_dir- becomes mds_mem_dir_minus, but
+    throttle-filestore_bytes becomes throttle_filestore_bytes.
+    
+    At some point once this is all settled we should
+    probably just change Ceph's internal perf counter
+    naming to satisfy the major TSDB naming rules.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 29ac9270d43a225bb2336adaaf813c12e12a715a)
+
+commit 8375251d150397c0097718f6e93ed38e9e89cfbb
+Author: jermudgeon <jhaustin@gmail.com>
+Date:   Sun Aug 27 21:26:28 2017 -0800
+
+    mgr/prometheus: Fix for MDS metrics
+    
+    MDS metrics come in these forms:
+    
+    mds_mem_dir #Directories
+    mds_mem_dir+ #Directories opened
+    mds_mem_dir- #Directories closed
+    
+    In this case, continuing the trend of replacing all illegal characters with '_' results in…
+    
+    mds_mem_dir #Directories
+    mds_mem_dir_ #Directories opened
+    mds_mem_dir_ #Directories closed
+    
+    which is palpably a bad idea.
+    
+    Suggested replacement for '+' = '_plus' seems fine, and a perusal of all metrics indicate that only MDS metrics end in '-' or '+' at this time.
+    
+    Replacing '-' with '_minus' is probably less good for the general case, if anyone has a better idea…
+    
+    I suppose another alternative would be to change MDS metrics so they don't use 'illegal' characters, but this also seems cumbersome and would break more third parties.
+    
+    Fixes: http://tracker.ceph.com/issues/20899
+    Signed-off-by: Jeremy H Austin <jhaustin@gmail.com>
+    (cherry picked from commit d719cd04b294e90ab9d440ba7d033826c069a2de)
+
+commit 3a037dda4273cea289641e93a006b2857147ee73
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Sep 8 11:33:02 2017 -0400
+
+    mgr: fix py calls for dne service perf counters
+    
+    Fixes: http://tracker.ceph.com/issues/21253
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit f9a4ca07acecd15986cbce61a6e118a6cb05af29)
+
+commit 077b3601d6bf68167d1857b79bb3782e23bc8ccc
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Jul 27 11:48:42 2017 -0400
+
+    mgr/restful: cleaner message when not configured
+    
+    RuntimeError+backtrace prints should be for
+    genuinely exceptional exceptions.  For these
+    known cases, we should just print the message cleanly.
+    
+    http://tracker.ceph.com/issues/21292
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit f7a1d57dd0d6c29b8c57a5de624da33777403fe5)
+
+commit 4c9d86bd86954cbee25a3ea2a60af063114f7790
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Aug 8 13:59:41 2017 -0400
+
+    os/filestore: fix device/partition metadata detection
+    
+    The UUID thing (a) relies on partition labels to work, which isn't
+    always true (and won't be true for ceph-volume going forward), and
+    (b) reportedly doesn't work anyway.  The fd-based helper works
+    just fine (even for vstart).
+    
+    Fixes: http://tracker.ceph.com/issues/20944
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e62862be0b68a17763e19ed8453436c457294505)
+
+commit 49e030da55030068c35790f751ea6631a324e5c2
+Author: Yan Jun <yan.jun8@zte.com.cn>
+Date:   Wed Sep 13 18:15:33 2017 +0800
+
+    test/config: regulate parameter order of ASSERT_EQ
+    
+    Signed-off-by: Yan Jun <yan.jun8@zte.com.cn>
+    (cherry picked from commit ccaba817dcd2c84c75cf0f784d90b78527c9f92b)
+
+commit 85ab5c86aafa92a36078133d7da07db54f064f4b
+Author: Yan Jun <yan.jun8@zte.com.cn>
+Date:   Tue Sep 12 11:20:34 2017 +0800
+
+    common: fix daemon abnormal exit at parsing invalid arguments
+    
+    Signed-off-by: Yan Jun <yan.jun8@zte.com.cn>
+    (cherry picked from commit 3f075372d1400806ae32e0e9210b9c15316dfe66)
+
+commit d7cde5f316242caa86ef62690104228aafc58a42
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Sep 7 13:12:15 2017 +0800
+
+    mon,monmap: use new style config opts
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 3124eb2148f5c6703c75b9e556ddb773898a6e55)
+
+commit fe039e41d78f7d74c39cdc9034cadccf632e9ac2
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Sep 7 12:47:36 2017 +0800
+
+    mon,option: set default value for mon_dns_srv_name
+    
+    Fixes: http://tracker.ceph.com/issues/21204
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 50207769f0c187cc8caf2bed3a1c94df409718e5)
+
+commit b54c7ba225314cc3e67707ec8dbac4a7f8381929
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Sep 6 12:57:40 2017 -0400
+
+    mon: fix `osd out` clog message
+    
+    This was printing the absolute time instead of the period.
+    
+    Fixes: http://tracker.ceph.com/issues/21249
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit f07480fec729d66f4cdc19a1c3bde3d185df1289)
+
+commit d7f6b93af4ce1115a1bde5883421c8ad860aa53f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Sep 8 16:43:58 2017 -0400
+
+    rbd: mirror "get" actions now have cleaner error messages
+    
+    Fixes: http://tracker.ceph.com/issues/21319
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 053be9847ff0052348aa259520d641923e57537d)
+
+commit bb0874c9db50bbfbf84396a4f1308282102a1694
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Sep 8 15:41:36 2017 -0400
+
+    cls/rbd: avoid recursively listing the watchers on rbd_mirroring object
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 79be496ac098859304efcabc7b5eb8afca6eca91)
+
+commit 34af07f97a67ead7d00b4352aea6300ac737ad9b
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Fri Aug 18 20:08:12 2017 +0200
+
+    rbd-mirror: potential lockdep issue
+    
+    (cycle between ImageReplayerAdminSocketHook and ImageReplayer locks)
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 226b1a3be16915c79e16106d42260687683e7a92)
+
+commit 925f223b7a57eae382c168d4bc4f05f50dd7bf6b
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Fri Aug 11 14:27:41 2017 +0200
+
+    rbd-mirror: update asok hook name on image rename
+    
+    Fixes: http://tracker.ceph.com/issues/20860
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 9ddb5da3377029d11cb1089102d7146e9ee2537a)
+
+commit fd207447d1acebbf7a62c0d03b3ec9025ac19fbd
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Thu Aug 17 16:02:02 2017 +0800
+
+    rgw: fix lc process only schdule the first item of lc objects
+    
+    If more than two bucket hash to the same lc object, only the first bucket will do lifecycle operation. Other buckets lifecycle will never be schduled.
+    
+    Fixes: http://tracker.ceph.com/issues/21022
+    
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit 9561dc0a903d9021f64857762e73887ffe46c28e)
+
+commit 5b5f0b5a33c638ec6f52d6936c60e5efd76eacab
+Author: Radoslaw Zarzynski <rzarzyns@redhat.com>
+Date:   Wed Aug 23 21:31:11 2017 +0200
+
+    rgw: fix accessing expired memory in PrefixableSignatureHelper.
+    
+    Fixes: http://tracker.ceph.com/issues/21085
+    Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
+    (cherry picked from commit 9260d39ceb1ef39a32c8f1742c3069bb83d628f8)
+
+commit d2be1e0ccb64021f770f28a6097f5468d375dbec
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Fri Aug 25 15:05:52 2017 +0800
+
+    rgw: return bucket's location no matter which zonegroup it located in.
+    
+    Get bucket location which is created in another zonegroup, will return "301 Moved Permanently".
+    
+    Fixes: http://tracker.ceph.com/issues/21125
+    
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit 71d2eca3e00de19419fb07bf9346f5863b20d40f)
+
+commit c4d6539374a02378209688dfa829caa39af0a214
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Thu Aug 31 13:39:13 2017 +0200
+
+    rgw: rgw_rados: set_attrs now sets the same time for BI & object
+    
+    `RGWRados::set_attrs()` used to set slightly different mtimes for the
+    object and in the bucket dir entry as we do an object write and set the
+    time at bucket index as the time at that point. Fix this by using the
+    setting the same mtime for the object as well.
+    
+    Fixes: http://tracker.ceph.com/issues/21200
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit 87e8e89fe11fa805739a2b85d1b009424641fa4e)
+
+commit dfe42384fbce90ec4c315e10ebd6b4fc4234827b
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed Dec 28 19:22:32 2016 +0100
+
+    rgw: calculate and print Swift's X-Account-Storage-Policy-* headers.
+    
+    Fixes: http://tracker.ceph.com/issues/17932
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 5e5c0677ba69f0d162df1da9a22b1fb56028baa4)
+
+commit 6e4c63fb136d958e0b4fe279419978a322043201
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed Dec 28 17:14:35 2016 +0100
+
+    rgw: bucket linking stores also the info about a placement rule.
+    
+    Fixes: http://tracker.ceph.com/issues/17932
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit de7a8679af93457c10bf514c17a7cfc5327e745c)
+
+commit 933f840094c3614466904f0fce32c2346a8f1709
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed Dec 28 17:12:50 2016 +0100
+
+    rgw: convey placement rule in RGWBucketEnt and cls_user_bucket_entry.
+    
+    Fixes: http://tracker.ceph.com/issues/17932
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 5eca89f08d29ac6aa7b55a3b4fb5b4183bdc0496)
+
+commit 5a1d0c42854fb35d44e63859d81074867f1a3295
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed Dec 28 15:54:42 2016 +0100
+
+    rgw: clean-up around and implement the move semantics in RGWBucketEnt.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
+    (cherry picked from commit 545dabe31a9ac231db4b8e3ce674ddfd9c91ee20)
+
+commit bb92cc93ac32c1f98d7cfae778d684b94dd2de58
+Author: Radoslaw Zarzynski <rzarzyns@redhat.com>
+Date:   Tue Aug 29 19:16:20 2017 +0200
+
+    rgw: enforce the std::move semantic across the path of RGWUserBuckets.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
+    (cherry picked from commit 4a7a4344840c873189331f22c8143ba031622d20)
+
+commit 395e9a48d32db7dfe0419316772886b939a44233
+Author: Radoslaw Zarzynski <rzarzyns@redhat.com>
+Date:   Tue Aug 29 19:19:59 2017 +0200
+
+    rgw: {end_}marker params are handled during Swift's reversed account listing.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
+    (cherry picked from commit 2f1f7a179d6a031c77929359313fc1fcc9f75bef)
+
+commit 8582664267b2e72a836e27df859910481371dee0
+Author: Radoslaw Zarzynski <rzarzyns@redhat.com>
+Date:   Mon Aug 28 21:54:56 2017 +0200
+
+    rgw: add basic support for Swift's reversed account listings.
+    
+    Fixes: http://tracker.ceph.com/issues/21148
+    Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
+    (cherry picked from commit 870f54806b7c1f570d2e82d3d2aa308b2178c6f0)
+
+commit d61de49cf270fedb206530a9d3cc1b4970f2b484
+Author: Radoslaw Zarzynski <rzarzyns@redhat.com>
+Date:   Mon Aug 28 21:53:28 2017 +0200
+
+    rgw: abstract partial data processing in RGWListBuckets.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
+    (cherry picked from commit a842dd09d584ff7636ec972646caff218df59c4b)
+
+commit 957e221f1ffc73726187aba99e9e103291b0ba6f
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Wed May 24 16:50:26 2017 +0200
+
+    rgw: list_objects() honors end_marker regardless of namespace.
+    
+    This patch fixes a regression related to handling of the end_marker
+    parameter during Swift's container listing operation. It has been
+    introduced in a5d1fa0587184f43c69d8e03114b58d43f320781 and causes
+    Tempest's test_list_container_contents_with_end_marker to fail.
+    
+    Fixes: http://tracker.ceph.com/issues/18977
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 09db1e412ef0853e7f345f813c8d452ec8354c36)
+
+commit 004240eea79b02eb830f3c9e72de8e0216ff7557
+Author: Sage Weil <sage@redhat.com>
+Date:   Sat Sep 9 22:47:06 2017 -0400
+
+    qa/standalong/mon/osd-pool-create: fewer pgs in test
+    
+    This runs afoul of the new max pg per osd limit.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c9ffeeebebe84111ce3be51c04170804bf2dbc74)
+
+commit 54ba2dda2034dd80594f17261451832744d2c7bf
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 8 08:08:30 2017 -0400
+
+    mon/OSDMonitor: assume a minimum cluster size of 3
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 1010761e6ad392fcaa647ec0d1b3d071535adfb3)
+
+commit f581731eaeb0f0c625f62928bf3724f8e705132c
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 1 14:45:12 2017 -0400
+
+    mon/OSDMonitor: prevent pg_num from exceeding mon_pg_warn_max_per_osd
+    
+    Check total pg count for the cluster vs osd count and max pgs per osd
+    before allowing pool creation, pg_num change, or pool size change.
+    
+    "in" OSDs are the ones we distribute data too, so this should be the right
+    count to use.  (Whether they happen to be up or down at the moment is
+    incidental.)
+    
+    If the user really wants to create the pool, they can change the
+    configurable limit.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 3ea2e518d27e6c06182c2cb3d9c0b9a0dab8dd22)
+
+commit acb2f280f03324e2ddb9d359e18ff2f0a3f897e6
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 6 11:41:07 2017 -0400
+
+    common/options: reduce mon_pg_warn_max_per_osd to 200
+    
+    This is 2x the recommended target (100 per OSD).
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b7fa440a9802005c167f854e1d19f9dd4fa8dfff)
+
+commit 119995a8f0ed70fb3e9efb5511576c44cfd6dbaa
+Author: Radoslaw Zarzynski <rzarzyns@redhat.com>
+Date:   Tue Sep 19 14:46:53 2017 +0200
+
+    rgw: rename the configurables for metadata limits to start with rgw_.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
+    (cherry picked from commit 9b06985caec27fc102769e46799ca6608e80eb85)
+
+commit 6080248f12e05a7478af1f8f31747da1aa9b881c
+Author: Radoslaw Zarzynski <rzarzyns@redhat.com>
+Date:   Tue Aug 29 12:16:30 2017 +0200
+
+    rgw: return proper message when deleting non-empty Swift's container.
+    
+    The bug that is fixed in this patch has been responsible for failing
+    the Tempest's test_delete_non_empty_container test case.
+    
+    The investigation has been made by: Marcus Watts <mwatts@redhat.com>.
+    
+    Fixes: http://tracker.ceph.com/issues/21169
+    Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
+    (cherry picked from commit 308c8d3ab7e3e4978beccba2c27beb95e75cde22)
+
+commit 71e37b1a10796ddfe786a2d389b0497ce1dc1bf4
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Sun Nov 20 21:49:57 2016 +0100
+
+    rgw: seed::get_torrent_file returns errors in the usual way.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 21ad80a1c237f8d243efaacd8218b29719a0a6b0)
+
+commit 429f5e1d8b7ec114289ba45e7c896e264b15ec7c
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue May 30 16:36:06 2017 +0200
+
+    rgw: add support for max_meta_count of Swift API's /info.
+    
+    Fixes: http://tracker.ceph.com/issues/17934
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 9e53c14a3398dfe55f6b816bb57c4b1b6d01c8ec)
+
+commit 70e150cdfa108f469d85a27ad1a93952f71347e0
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue May 30 22:19:33 2017 +0200
+
+    rgw: Swift API returns 400 Bad Request on too long container names.
+    
+    Fixes: http://tracker.ceph.com/issues/17935
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 1845e41292696da9e20ecd87b9260b44ee312aed)
+
+commit 02c208a6b1ae335ab15c0fab3d9d6dba803037fa
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Mon May 29 19:08:21 2017 +0200
+
+    rgw: honor custom rgw_err::message in Swift's error handling.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit e8516147d2926631fed98a793606bcbb0e6db9e1)
+
+commit 912132b272dbf534e7594b40a78c5efcd51c9466
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Mon May 29 18:02:23 2017 +0200
+
+    rgw: add support for max_meta_value_length of Swift API's /info.
+    
+    Fixes: http://tracker.ceph.com/issues/17936
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 06b1f25021924e0c55da0bc22216f31344735dfe)
+
+commit cd219153978974a3711093912ca329ae5d1d997c
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Mon Nov 21 19:34:48 2016 +0100
+
+    rgw: refactor rgw_get_request_metadata to reduce the number of dynallocs.
+    
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 64b792326350c4c09ed5f829d082d4050bfb0408)
+
+commit 4b82d1e25bf8f7be1915e7467cb822d5beea96a1
+Author: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+Date:   Tue May 30 22:19:58 2017 +0200
+
+    rgw: add support for max_meta_name_length of Swift API's /info.
+    
+    Fixes: http://tracker.ceph.com/issues/17938
+    Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>
+    (cherry picked from commit 593d656dffd477943b8bd15153004d905b4b0d73)
+
+commit fb7e6a581d063db10bb7e0cce4ae358068c7a2bd
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Sep 18 18:01:17 2017 +0200
+
+    tests: CentOS 7.4 is now the latest
+    
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 2311b64025cdb6131035aaf01e7c97486da12e15)
+
+commit e25a583323a225dcfab0567c872247b33bf36f8f
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Mon Sep 18 14:44:36 2017 +0800
+
+    osd/PrimaryLogPG: _delete_oid - fix incorrect 'legacy' flag
+    
+    For pre-Luminous created objects, we shall default 'legacy' flag
+    to true, so we can still create a snapdir object properly if necessary
+    for Luminous backward compatibility.
+    
+    Note that this patch is not going to land on master first
+    (and hence can not be cherry-picked from master) because it will
+    finally be deprecated by https://github.com/ceph/ceph/pull/17579,
+    in which we are going to kill the snapdir object completely for Mimic.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+
+commit 5aa445897c293cae407454fadbbc47116a2cbe76
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Sat Sep 16 10:04:56 2017 +0800
+
+    qa: fix pool-quota related tests
+    
+    https://github.com/ceph/ceph/pull/17371 introduces support of
+    per-pool space-full flag, which turns out to set both
+    full and full_no_quota flags now if a pool is currently running out
+    of quota.
+    
+    Actually this test is fragile as long as we keep appending new flags
+    at pool granularity, but let's not bother with that complexity now.
+    
+    Fixes: http://tracker.ceph.com/issues/21409
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 5dd3458eec7d6c56a71ee6f0fcdd45aa15a58ec7)
+
+commit 00b44ecf4415710150e0646174174fbde0167f42
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Mon Aug 28 15:51:28 2017 +0800
+
+    mon, osd: per pool space-full flag support
+    
+    The newly introduced 'device-class' can be used to separate
+    different type of devices into different pools, e.g, hdd-pool
+    for backup data and all-flash-pool for DB applications.
+    
+    However, if any osd of the cluster is currently running out
+    of space (exceeding the predefined 'full' threshold), Ceph
+    will mark the whole cluster as full and prevent writes to all pools,
+    which turns out to be very wrong.
+    
+    This patch instead makes the space 'full' control at pool granularity,
+    which exactly leverages the pool quota logic but shall solve
+    the above problem.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit b4ca5ae462c6f12ca48b787529938862646282cd)
+    
+    Conflicts:
+    slight confilicts in src/osd/OSDMap.h because
+    e71626732452951ed9e6c489f15618424d15acf2 removed get_full_osd_util().
+
+commit b064ed172a309aad4e8850110be4311466bf5894
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 14 10:02:33 2017 -0400
+
+    os/bluestore: repair 21089 on freelist init
+    
+    Fix up the size inconsistency on freelist init.  This way it will always
+    happen after an upgrade... and before the user moves to something
+    post-luminous.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 1bf89aa3ce2f1fdb7979e536f6788c8dae275d47
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 14 10:01:25 2017 -0400
+
+    os/bluestore: fsck: remove fsck repair for 21089
+    
+    This requires the user to run fsck, which they likely won't do.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit e9f143c681ab1a79b9c726cab55282d71ecfe1c7
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 13 23:03:07 2017 -0400
+
+    os/bluestore/KernelDevice: hack to inject bad device size
+    
+    This conditionally reintroduces bug http://tracker.ceph.com/issues/21089,
+    fixed by f6f1ae3724d593d3709d982c973ec18a25a47b6e, to aid in testing
+    repair.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 460268b5e9c9a77aae134049aa320977f02fc911
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Wed Aug 30 15:28:11 2017 -0700
+
+    ceph.in: validate service glob
+    
+    Fixes: http://tracker.ceph.com/issues/21191
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 59bee1e7274934cebe0dcefd9fcedb6886a5e2d0)
+
+commit 03e2a68e431eaa737b9e66c4d3b289e58d6cf19b
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Aug 30 10:17:31 2017 +0200
+
+    build/ops: python-numpy-devel build dependency for SUSE
+    
+    Fixes: http://tracker.ceph.com/issues/21176
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 1586f2ca9ab94af85682945a3c7c7ebbd82c6e03)
diff --git a/doc/changelog/v12.2.3.txt b/doc/changelog/v12.2.3.txt
new file mode 100644
index 000000000..c8974f7db
--- /dev/null
+++ b/doc/changelog/v12.2.3.txt
@@ -0,0 +1,10247 @@
+commit 2dab17a455c09584f2a85e6b10888337d1ec8949
+Author: Jenkins Build Slave User <ceph-release-team@redhat.com>
+Date:   Mon Feb 19 23:14:46 2018 +0000
+
+    12.2.3
+
+commit c0429254cec6fdfa1089f3dc205a7a6199d2e40a
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Feb 20 06:40:14 2018 +0800
+
+    cmake: disable DOWNLOAD_NO_PROGRESS if cmake ver is lower than 3.1
+    
+    see https://cmake.org/cmake/help/v3.1/module/ExternalProject.html and
+    https://cmake.org/cmake/help/v3.0/module/ExternalProject.html. the
+    former has DOWNLOAD_NO_PROGRESS, while the latter does not.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d051bf0441c933e27fb0f54bc6ed999d1e8eca89)
+
+commit 74c89b975f786ca2bf6de7310f191e92faeaca6c
+Merge: 76df814ac4 7e33d67f1b
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Fri Feb 16 08:06:00 2018 -0500
+
+    Merge pull request #20452 from ceph/backport-wip-rm22785
+    
+    luminous ceph-volume consume mount/format options from ceph.conf
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit 7e33d67f1b143eff9c63af5d44d2d2dd480ed2f3
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 16 06:51:12 2018 -0500
+
+    ceph-volume tests remove unused import
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit ad49283b2ee1e4a1a8a54d0c71039a2afc778b29)
+
+commit 76df814ac485df9f72e58081f8d6bb17151cc18b
+Merge: 3e86319b8f cdc27208aa
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Fri Feb 16 11:15:57 2018 +0100
+
+    Merge pull request #20451 from ceph/wip-yuri-PR20053-luminous
+    
+    qa/tests: Applied PR 20053 to stress-split tests
+    
+    Reviewed-By: Jason Dillaman <jdillaman@redhat.com>
+
+commit cdc27208aa170061ef798b9457d6e85294e9238b
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Thu Feb 15 10:26:46 2018 -0800
+
+    qa/tests: Applied PR 20053 to stress-split tests
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 18a0ee570ffb456053e1712881ee61f8b2072f87
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Feb 12 16:47:36 2018 -0500
+
+    ceph-volume util.prepare extend flags when found
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 287c952bc8c3422c6eed72a1e0b22612d0728314)
+
+commit cb5da5fffe625aa6b930bdf4a77869716b77caff
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Feb 12 16:47:17 2018 -0500
+
+    ceph-volume util.constants update mount flags to include rw, normalize on list
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 457a5173d2e08192273c22f1abe52a64773188b4)
+
+commit 33f7da3f165be7bbc4a56613078d95d6fa6e79cb
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Feb 12 16:46:47 2018 -0500
+
+    ceph-volume configuration normalize underscore vs. whitespace named options
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit b80e3bf1175a65181d1f2ed38b6a416c6b730d82)
+
+commit 93e41ef6434daea37df3322e87e0c6e367de1741
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Feb 12 16:46:06 2018 -0500
+
+    ceph-volume tests check for mount/format usage from ceph.conf
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 676c93fcf3041de07e8125d36d8dc7ad474db9b1)
+
+commit c586a24e91ca627449452103d562b6be1fbfcb4d
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Feb 12 16:44:45 2018 -0500
+
+    ceph-volume tests add ceph.conf stub for custom conf files
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a8eecc64716564479308fafba5159e271c42feac)
+
+commit 3e86319b8fa5f85a078441b88935f812c8039580
+Merge: dbe215d69d 599869d7f2
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Wed Feb 14 16:59:11 2018 +0100
+
+    Merge pull request #20429 from ceph/backport-wip-rm22988
+    
+    luminous ceph-volume use realpath when checking mounts
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit 599869d7f2df5d207ba9afa8915ce4980ff65137
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Feb 13 13:22:41 2018 -0500
+
+    ceph-volume tests verify realpath usage in mounts
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 16eabbc3abb7d03a0808963a9aae6b2ffb3a875b)
+
+commit 565c044d556eaf741d79e7abbf5f472ee4366b44
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Feb 13 12:59:48 2018 -0500
+
+    ceph-volume util.system optionally use realpath on devices
+    
+    To check a mounted device it is needed to verify on a combination of
+    realpath and plain devices against realpath and plain paths. In LVM, two
+    different paths might refer to the same devices
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 53e691e1d5122ff533e69ae08c73e18c1ed74765)
+
+commit dbe215d69d8ccf0956f197cef88f50f6518bbfa0
+Merge: 2fc8d6c4ad dd94ac120e
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Wed Feb 14 14:13:59 2018 +0100
+
+    Merge pull request #20438 from ceph/backport-wip-rm22876
+    
+    luminous: ceph-volume: lvm zap will unmount osd paths used by zapped devices
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit dd94ac120ed337f0080dc776f3234eeaade46b54
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Mon Feb 12 15:23:08 2018 -0600
+
+    ceph-volume: use terminal.MultiLogger in `lvm zap`
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 3f2e7074ee42ff8248afaed2aa9bfd919096fc38)
+
+commit ec84f39a0573887e005fdb0bfe7d4e5e9457c51b
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Feb 7 09:06:45 2018 -0600
+
+    ceph-volume: use api helper methods for pvs and lvs in lvm zap
+    
+    This is just a cleaner way of doing this instead of using PVolumes
+    and Volumes directly.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit c8a6da4e1bb645ca5ee8cb8f5d861584fe70210d)
+
+commit 19e02e6c433f718ce638624a803105525168cb52
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Feb 7 08:35:47 2018 -0600
+
+    ceph-volume: fix documentation typos for lvm zap
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 5552cefc4ce6d6d819f43671cec83ca23b119665)
+
+commit 1de5592d4a13a0c0475f723fc02fb8e1c70b482e
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Feb 1 14:52:46 2018 -0600
+
+    ceph-volume: when zapping unmount osd directories
+    
+    If you zap an lv, device or partition and it's currently mounted as a
+    ceph osd directory then it will be unmounted so the zap can complete.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit d4639ac116407f71ba6bff06a3202c1490d57dee)
+
+commit 13f42da7ae449dde84a94f93ac22f8bf0a8c7f79
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Feb 1 14:29:30 2018 -0600
+
+    ceph-volume: adds a util.system.unmount function
+    
+    This function will unmount the given path
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 6909baef542f3cd9b7d97113a6152280e72013f6)
+
+commit 2fc8d6c4adfe32586b9d177ed6ab7b46c6b2b110
+Merge: 22c6b1d3d6 24f423fa0b
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Tue Feb 13 16:43:14 2018 -0500
+
+    Merge pull request #20407 from cbodley/wip-qa-multisite-trim-luminous
+    
+    luminous: qa/rgw: disable log trim in multisite suite
+    
+    Reviewed-by: Yuri Weinstein <yuri.weinstein@gmail.com>
+
+commit 24f423fa0b937fc8df8f7c48fdc1d83510d0a44f
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Dec 11 16:14:05 2017 -0500
+
+    qa/rgw: disable log trim in multisite suite
+    
+    the multisite tests run manual trim operations with radosgw-admin, which
+    can race with internal log trimming to produce tests failures
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit f8909bb6a6320822da7a4c4dab242f84732ebf5d)
+
+commit 22c6b1d3d6bbc7ee466fddd3af13bcdb7f0c9a55
+Merge: fda9e4a3e2 01f79c086a
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Thu Feb 8 18:56:12 2018 +0100
+
+    Merge pull request #20368 from ceph/backport-20367
+    
+    luminous: ceph-volume: adds custom cluster name support to simple
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit fda9e4a3e2ab4dd91430d9197792ed15e8ad256d
+Merge: faefe0b68f 29c89c4964
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Thu Feb 8 18:08:49 2018 +0100
+
+    Merge pull request #20357 from pdvian/wip-22938-luminous
+    
+    luminous: rgw: unlink deleted bucket from bucket's owner
+    
+    Reviewed-By: Orit Wasserman <owasserm@redhat.com>
+    Reviewed-By: Abhishek Lekshmanan <abhishek@suse.com>
+
+commit 01f79c086a2d6e1fd10f3d6283c6f12ef03991a2
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Feb 8 07:09:57 2018 -0600
+
+    ceph-volume: use a custom cluster name in simple functional tests
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 7f1dc6b3ab885253d28c223bda7e6e1232a84609)
+
+commit 8d50005d3efddde777916df214ace36178f4741f
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Feb 7 14:42:24 2018 -0600
+
+    ceph-volume: set conf.cluster name during simple activate
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit b1cee74fa2225fe10a024c7291bd23aadcc23441)
+
+commit faefe0b68fa748e8cb50ffb97b34d39ace65f35c
+Merge: 13f400f801 bf3ee8b22b
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Feb 8 12:43:44 2018 +0200
+
+    Merge pull request #20338 from cbodley/wip-22930
+    
+    luminous: rgw: fix for empty query string in beast frontend
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 29c89c496424dbaf66ab8771d1e9c9578eb092a2
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jan 18 14:53:35 2018 -0500
+
+    rgw: unlink deleted bucket from bucket's owner
+    
+    if a bucket is deleted by an admin/system user instead of its
+    owner, the unlink would fail and the deleted bucket remained
+    visible to the original owner
+    
+    Fixes: http://tracker.ceph.com/issues/22248
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 4701e85a3ea72852a61a106724fa22497f3d5789)
+
+commit 13f400f80182c4fe0ebe8f9be19fa63898003f0b
+Merge: 609f2a88e0 b8b3eee73d
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Feb 7 14:49:36 2018 -0800
+
+    Merge pull request #20204 from tchaikov/wip-pr-18614-luminous
+    
+    luminous: osd,mgr: report pending creating pgs to mgr
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 609f2a88e0e8b0cb40ac7853d474a175f4c9699e
+Merge: 2cef7159fe d93eef3127
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Feb 7 14:45:33 2018 -0800
+
+    Merge pull request #20249 from pdvian/wip-22864-luminous
+    
+    luminous: mds: fix scrub crash
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 2cef7159fe89a428256910d90dbfb0d42924d0cf
+Merge: 8bca8fb027 7756a44529
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Feb 7 14:45:07 2018 -0800
+
+    Merge pull request #20256 from pdvian/wip-22860-luminous
+    
+    luminous: osdc: "FAILED assert(bh->last_write_tid > tid)" in powercycle-wip-yuri-master-1.19.18-distro-basic-smithi
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 8bca8fb0274c5f3690262400a92eb23bb87a4012
+Merge: 16d85f9673 042144a281
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Feb 7 14:44:39 2018 -0800
+
+    Merge pull request #20299 from pdvian/wip-22859-luminous
+    
+    luminous: mds: set higher priority for some perf counters
+    
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 16d85f9673931596017aeb3eb23132b5f3ef3500
+Merge: 43a4e40916 5282a60c8f
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Feb 7 14:44:03 2018 -0800
+
+    Merge pull request #20300 from pdvian/wip-22867-luminous
+    
+    luminous: MDS : Avoid the assert failure when the inode for the cap_export from other…
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 43a4e40916159f347bca12ff02d2b64947d2a4e4
+Merge: 1133c9bba1 c63fc964ce
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Feb 7 14:41:45 2018 -0800
+
+    Merge pull request #20339 from batrick/i22242
+    
+    luminous: mds: limit size of subtree migration
+    
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit 1133c9bba1750c8f861293f44ed9e99f679fd2cb
+Merge: a8cdfea4b3 87991643b3
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Feb 7 14:41:14 2018 -0800
+
+    Merge pull request #20340 from batrick/i22240
+    
+    luminous: mds: prevent filelock from being stuck at XSYN state
+    
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit a8cdfea4b34e46a59fdf023cbeb57b9dfc0db2ee
+Merge: e2e6d30f3f 655d266944
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Feb 7 14:40:33 2018 -0800
+
+    Merge pull request #20341 from batrick/i22089
+    
+    luminous: mds: don't report repaired backtraces in damagetable, write back after repair, clean up scrub log
+    
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit e2e6d30f3f00af38c49aa9845eaac940680f37cd
+Merge: 33ebb80c91 6caae86b7c
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Feb 7 14:38:58 2018 -0800
+
+    Merge pull request #20337 from liewegas/wip-bluestore-aio-read
+    
+    luminous: os/bluestore: propagate read EIO errors up the stack
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 33ebb80c915c66dfc4d64b2ce85f6fc690f4326a
+Merge: b99521accd fc5807003b
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Wed Feb 7 19:02:31 2018 +0100
+
+    Merge pull request #20350 from ceph/backport-20264
+    
+    luminous: ceph-volume dmcrypt support for simple
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit fc5807003b01e6d009c392e0577765582db53d64
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Feb 6 11:36:03 2018 -0600
+
+    ceph-volume: do not test custom cluster names with simple
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 63046e0d7bd05a42477dd1fbd4af11f81046b08d)
+
+commit c1eb7b9915a61d93a5b09ff588a5906e35a2b208
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Feb 6 10:46:44 2018 -0600
+
+    ceph-volume: sleep 2 minutes after reboot in simple tests
+    
+    This will allow for some time for OSD services to start back up after
+    reboot. If not we see race conditions where tests fail because they
+    were run before the OSD finished startup.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 85b319a2c577c1fa86c6ccbbac61f015dd24ad21)
+
+commit a7ecc1c91a9492337736c9674b4242317a4e526a
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Mon Feb 5 14:18:28 2018 -0600
+
+    ceph-volume: adds the simple dmcrypt_plain and dmcrypt_luks tests
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 3052010a80a1783b6ebd629a603638ab5806f8fd)
+
+commit aab4961b5e597614480d23efcc88008cb0d600c4
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Mon Feb 5 13:53:13 2018 -0600
+
+    ceph-volume: fix ceph-volume simple scan help menu test
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 2ee70bf2d1958c0655ddb5b7cc4d2ed3f4bc58b8)
+
+commit c1035af7a6a669c5f88ef7f30942a2b9ed347c1c
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 09:11:09 2018 -0500
+
+    ceph-volume tests.functional add simple xenial filestore dmcrypt plain support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 00b14177bdbe6d66424001b1c35ca8fa7b46977f)
+
+commit 55def6eb6c463c08cd1c24839b6891395425372a
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 09:10:56 2018 -0500
+
+    ceph-volume tests.functional add simple xenial filestore dmcrypt luks support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 0b2f869ccbb30c27bd19f2a05bb893957912142e)
+
+commit 591796d073d6032a39a7e7b70cd56965fc8320f0
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 09:10:36 2018 -0500
+
+    ceph-volume tests.functional add simple xenial bluestore dmcrypt plain support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 0174f9e1646f4b51c712506f629b48c329f20ea1)
+
+commit 31665eac381cf78cd4fdc8993b75dc291121a095
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 09:10:12 2018 -0500
+
+    ceph-volume tests.functional add simple centos7 filestore dmcrypt plain support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 4c26162002709ccdd1b48c05126befdab7f269da)
+
+commit 4de4d69bf0e260c38177b32ba4f199dd13b9983f
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 09:09:57 2018 -0500
+
+    ceph-volume tests.functional add simple centos7 filestore dmcrypt luks support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a911d7c6a99e2ea235400614568d9cd362470cac)
+
+commit eb97511c3fb9a33df50bf35ace515a068211df7c
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 09:09:41 2018 -0500
+
+    ceph-volume tests.functional add simple centos7 bluestore dmcrypt plain support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 8e0213ab053e80038bdae8d1b06d448fd397e3bc)
+
+commit badd73df5d30f034c5ceee37fdb479bba17fffa6
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 09:09:21 2018 -0500
+
+    ceph-volume tests.functional add simple centos7 bluestore dmcrypt luks support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 76151c461bfe1e5ae4dc277c9d9357efb81e8c92)
+
+commit 5d6aa430a51a481cae400b8fa9d8ee3fbc5fa392
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 09:08:23 2018 -0500
+
+    ceph-volume tests.functional add simple bluestore dmcrypt luks support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d6e24022e8dfe845823fa0e87f3fba028c4b25b5)
+
+commit 8fcd52e88a54959dde41333ed440b88d78a021a3
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 08:36:45 2018 -0500
+
+    doc/ceph-volume lvm prepare fully supports encryption now
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 0b2075759e8788b2c4ae46f94fdee953cc82b21e)
+
+commit 476eac136a7aee3485b91b81108bc1d05d0d6698
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 08:36:02 2018 -0500
+
+    doc/ceph-volume scan update for encryption support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit e0d3bb82255a937cd937d33f966979fb541d1443)
+
+commit 141279aeff3892f1778fddb373d2e2180e402950
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 08:28:40 2018 -0500
+
+    doc/ceph-volume remove notice that dmcrypt is not supported
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a7d11ca27df1d22c8109cad5bf7be8fd23395c71)
+
+commit 2f7d03418783b33eaf0cab1fd22ff95f94861d3e
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 08:26:58 2018 -0500
+
+    doc/man/ceph-volume add simple documentation
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 9524021aa15280d98f2b04d1fd66552733adf8a2)
+
+commit 1adafb88b6e458527759a13afe2362aa79c4adaa
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 08:15:36 2018 -0500
+
+    ceph-volume simple.scan update help menu to indicate device support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 09f35cb249674eda044b8fceea6f402ffac7731d)
+
+commit d015b1933920a855a6a2cafdda863162b804ecbe
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 08:13:12 2018 -0500
+
+    ceph-volume tests for keyring parsing
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit b90c1a86843d8f0c8c7fb68fe059bdd72f3730fd)
+
+commit 77d40d0713385dcb54b8aa1d0d5a2e1f8386609c
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 08:12:51 2018 -0500
+
+    ceph-volume tests for validate_devices
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit bece7af0523b04d39c26011990cb1fe323aecdd8)
+
+commit 70b5a325ffe93cb55eaf0f6debfc4f5dd305c24b
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Feb 2 08:12:05 2018 -0500
+
+    ceph-volume simple.activate b64decode keys for activation as well
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 636ebc127797c8020cfb7aff09955b72beac8bdf)
+
+commit baf12acff49e0e2129e0a7b63d10a497929f4992
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Feb 1 16:17:19 2018 -0500
+
+    ceph-volume tests add validation for lsblk parsers
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 03102e4417194e16deb8b738fcdaf87e699e6073)
+
+commit 44cbcdee151adc302bd77ca2377032ed1bdb1bfd
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Feb 1 15:51:41 2018 -0500
+
+    ceph-volume tests validate parsing of cryptsetup
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit cce6a12f4a0d8d9a3d7dcb509b42ed2536d9257f)
+
+commit 941fbbf29ba893e0fe7657669e969c7d784e08d3
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Feb 1 15:51:12 2018 -0500
+
+    ceph-volume tests add a stub for process.call
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit b80b61b90f8539b363c03c96e63f31c38c707fc8)
+
+commit 2ce659749487f60c4012207aaae6665e889f52dc
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Feb 1 15:35:11 2018 -0500
+
+    ceph-volume util.encryption add notes about extra b64decode call for ceph-disk
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit c2367ef4a4db1e9c62e33dc211474f652d6bc6af)
+
+commit 36d81bdf2ab24b52db2808ca8cf91ebd864eddb1
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Feb 1 15:34:09 2018 -0500
+
+    ceph-volume util.encryption parse legacy encrypted with dirs too
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 32cb810f8ab7e4a710bfb2590a065dea41369fce)
+
+commit de3013287fe9c6d879cc4fbca6c06437e5989e21
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Feb 1 15:32:54 2018 -0500
+
+    ceph-volume terminal create a logger to get terminal+log messages in one call
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit f6dd0ff15f0aeda9f660e923153f1d3dae68e583)
+
+commit c25ba9528e66ab7522d2452fc14c935924767839
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Feb 1 15:32:16 2018 -0500
+
+    ceph-volume simple.activate support dmcrypted devices for both plain and luks
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 5ece73b533ee4b577c0ce61e3685a880e1de4851)
+
+commit 8a19b31c162ae08913136895144303d8999600dd
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Feb 1 15:31:28 2018 -0500
+
+    ceph-volume simple.scan parse the keyring out of the keyring file
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a1124362816cbaaad7255b7ffc3698b68601ffe5)
+    
+    Conflicts:
+            src/ceph-volume/ceph_volume/devices/simple/scan.py
+
+commit 1d94a5f8e2d9d5e314463cbbb7cf7fc704d3ffc2
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Jan 30 16:23:06 2018 -0500
+
+    ceph-volume simple.scan support dmcrypt OSDs when scanning
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 6c87d66d76ee1fc70f8dc6211d89dbd489c6ed3a)
+
+commit c3aa6e6b90a73d4754ae28270b650ca9a79809eb
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Jan 30 09:35:53 2018 -0500
+
+    ceph-volume util.encryption add a utilty to open plain encrypted devices
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit e5759e4fc9485e92a2faec277c8a920295751844)
+
+commit b2bd65505df108420ee79c93d8da6ffdfa7f7675
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Jan 30 09:34:53 2018 -0500
+
+    ceph-volume util.system tmp mounts can now remove a dmcrypt mapper
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 60e8699650a491013d51239b9dc58160946d3bf9)
+
+commit 520200ceea0070a1a14976d06befc72217ed438f
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Jan 30 09:22:19 2018 -0500
+
+    ceph-volume util.system do not 'translate' using realpath
+    
+    Using realpath always means that device mapper paths, like for dmcrypt
+    or LVM will get mangled and will return something like `/dev/dm-1` which
+    is not useful for anything in ceph-volume
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 469d01dc1aee8d6528944ef0acec58df868a9da7)
+
+commit abf21f28612c03ce82465b1bb816d477b07da702
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Jan 30 09:19:06 2018 -0500
+
+    ceph-volume util.disk add a PART_ENTRY_TYPE detection utility
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit cdb812438d6064089bac964cf38817d877afca38)
+
+commit c177b491758db2e7d81de100a9e52821f38b748a
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Jan 26 10:44:14 2018 -0500
+
+    ceph-volume util.encryption add helpers for legacy devices
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d59b087e50e4e2076c59f6316cc89eb2a070aa52)
+
+commit cc9840140e7a7fe385fcd2cbaeef7351cfd79ba8
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Jan 26 10:25:31 2018 -0500
+
+    ceph-volume util.constants add ceph-disk partition labels
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 548a74bd239abe6b1b7a5a27fb2779800cfa7ac1)
+
+commit 69d142adb8d70f1b5c9f55d72395313fd1e80048
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Jan 25 11:04:15 2018 -0500
+
+    ceph-volume util.disk support PKNAME and absolute paths in lsblk
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit b90044c2ffff38d0dc36d598d237fcb066efe400)
+
+commit b99521accd04a9761fea8d9a927442ae51bf18a5
+Merge: 62337485fa 40a16ac6cc
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Wed Feb 7 08:19:29 2018 -0500
+
+    Merge pull request #20323 from ceph/backport-20203
+    
+    luminous: ceph-volume: fix usage of the --osd-id flag
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit 62337485faf13bd1caa02adec877fd9a5de1bd78
+Merge: 3d84e35378 b47837100f
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Wed Feb 7 08:18:32 2018 -0500
+
+    Merge pull request #20322 from ceph/backport-20059
+    
+    luminous: doc/ceph-volume OSD use the fsid file, not the osd_fsid
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit 3d84e3537826d1dcffe8a82195b93a8488d71b8b
+Merge: 1c847a1fc8 a26b4ae765
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Feb 6 14:21:16 2018 -0800
+
+    Merge pull request #20162 from tchaikov/wip-pr-20130-luminous
+    
+    mgr: balancer: fixed mistype "AttributeError: 'Logger' object has no ...
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 1c847a1fc88bb23126c77ac09a38dde29a5500e4
+Merge: 6b5ae18b13 03569abb40
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Feb 6 14:20:18 2018 -0800
+
+    Merge pull request #20325 from jcsp/wip-22851-luminous
+    
+    luminous: pybind/mgr/dashboard: fix duplicated slash in html href
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 6b5ae18b133a260ef50f39652b64fd8ea208722a
+Merge: ac0e193f40 91673b1edf
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Feb 6 14:19:14 2018 -0800
+
+    Merge pull request #20326 from theanalyst/wip-22892-luminous
+    
+     luminous: _read_bdev_label unable to decode label at offset
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 655d266944374eea2a846f182cc00193ef4cf4b8
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sun Nov 19 20:25:22 2017 +0800
+
+    mds: properly execute scrub finish context
+    
+    Bug was introduced by commit 7e52729699 (mds: flush after scrub repairs)
+    
+    Fixes: http://tracker.ceph.com/issues/22058
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit ff71fbdcbb87ffc9baa02319ff0f336c173c5616)
+
+commit 798cc37c7a2acb4af182bdc7b214c4c7445645d0
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Oct 25 09:39:15 2017 -0400
+
+    mds: clean up clog damage message when repairing
+    
+    If we've repaired then we don't need to be at WRN
+    and we don't need to advise the user to look up
+    the damage detail.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 3160ef2c0a66324621a95926bed0416bbd537353)
+
+commit 6e544ac967b533f47ce2708b1757cd048ea2354c
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Oct 25 06:30:57 2017 -0400
+
+    mds: flush after scrub repairs
+    
+    Otherwise, if we restart the MDS right after the scrub,
+    then scrub again, it will see the same inconsistency when
+    it looks at the on-disk state.
+    
+    This involves adapting the use of ScrubHeader to be something
+    writeable during scrub to record whether repair happened.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 7e52729699e87dd75022433153450fd4ba71b40e)
+
+commit f19f1e1cbc5b23eeb028c015c77ffe32306aba64
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Oct 25 05:24:52 2017 -0400
+
+    mds: don't report repaired backtraces in damagetable
+    
+    Fixes: http://tracker.ceph.com/issues/18743
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 444382c3370bafa21564bb1e6a5c80f3e4825f6a)
+
+commit 162e25ec04e5288608718dcbd9d57b8234c17fb9
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Oct 25 05:24:03 2017 -0400
+
+    mds: cleaner scrub complete log msg
+    
+    People often are not using a tag with their scrub,
+    let's not have the ugly "with tag ''" in that case.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 5fd00a6e1b914880ab9f401faf591e73024f18ef)
+
+commit 87991643b38b1a0e526b599b65bb010e66c08374
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Nov 7 12:39:49 2017 +0800
+
+    mds: prevent filelock from being stuck at XSYN state
+    
+    Lock:file_eval() does not properly handle the condition: filelock
+    is in XSYN state, both loner_cap and want_loner_cap are >= 0, and
+    loner_cap != want_loner_cap.
+    
+    To set loner to the wanted one, mds need to revoke caps from the old
+    loner. Changing lock state to MIX can do the job.
+    
+    Fixes: http://tracker.ceph.com/issues/22008
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com
+    (cherry picked from commit 67a428741fc82eb5f4120042197acf0cec34213a)
+
+commit 4fbde822b85fb96e79f800df0ab24d508887665b
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Nov 7 10:50:14 2017 +0800
+
+    mds: implement LOCK_XSYN_LOCK and LOCK_XSYN_MIX
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit c7261e1a6d95d1ed98a1d9066905aa75af4c2886)
+
+commit f354347debffec93910e0db0be4e165c00778d83
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Nov 7 10:37:44 2017 +0800
+
+    mds: cleanup set/drop loner code
+    
+    integrate functionalities of try_drop_loner and try_set_loner into
+    CInode::choose_ideal_loner
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit d4cbf3aeeada57a88b794d3d6d5fb1cd891e31cd)
+
+commit ac0e193f402a90202fafa3f1bfff4f99697babba
+Merge: bd75db92c6 06a1c3d1e4
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Feb 6 09:40:01 2018 -0800
+
+    Merge pull request #19157 from pdvian/wip-22237-luminous
+    
+    luminous: mds: don't delay processing completed requests in replay queue
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit bd75db92c630fc6636a357697f01784c9b2aa98e
+Merge: 4974d42eea 024e11dbfc
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Feb 6 09:37:50 2018 -0800
+
+    Merge pull request #20327 from theanalyst/wip-22922-luminous
+    
+    luminous: rgw: bucket resharding should not update bucket ACL or user stats
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit c63fc964cef1d45fd0a4de9c6715bcd16badc597
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Nov 3 14:14:07 2017 +0800
+
+    mds: limit size of MExportDir message
+    
+    Only export port of subtree if whole subtree tree is too large for
+    sinlge MExportDir message. The un-exported portion are added to
+    export queue.
+    
+    Fixes: http://tracker.ceph.com/issues/21892
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit ca5c987860821a7784db1c6a74d1c661cdf427cc)
+
+commit a7fd4f95d2d4462f4e382cd85f0b011786319ff9
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Nov 3 16:13:32 2017 +0800
+
+    mds: optimize MDCache::try_subtree_merge
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 1f27ebbef95b39f7fa3c94d5dc9f191a58c9f9b4)
+
+commit 45f4b1a141fe3c3caafaece0e82f2f35d0586030
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Nov 3 12:03:44 2017 +0800
+
+    mds: optimize import/export state access
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit fb9277c14314aaa6a97ee23da9057ddc3c750408)
+
+commit bf3ee8b22bf1c2242189107cc662a28f5ea7c219
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jan 25 12:09:20 2018 -0500
+
+    rgw: fix for empty query string in beast frontend
+    
+    when the target does not contain a ?, the QUERY_STRING was being set to
+    the same value as REQUEST_URI. this QUERY_STRING is included in the
+    signature, and caused SignatureDoesNotMatch failures
+    
+    Fixes: http://tracker.ceph.com/issues/22797
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit ab9e79684ac7ae33b78522df6732b99271d10016)
+
+commit 6caae86b7c7591b1e2902a9002b7649fe99022e5
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Fri Sep 15 22:28:40 2017 +0800
+
+    os/bluestore: propagate read-EIO for aio
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 301912603789d78e3560da84df1e337edd046e46)
+
+commit e57274ee2a51e100579dadae88eb3c61e4fb1642
+Author: Pan Liu <wanjun.lp@alibaba-inc.com>
+Date:   Wed Sep 6 18:09:55 2017 +0800
+
+    os/bluestore: report error and quit correctly when disk error happens.
+    
+    Signed-off-by: Pan Liu <wanjun.lp@alibaba-inc.com>
+    (cherry picked from commit ed3175895fa92929089f7507e7d173f557d61e27)
+
+commit 68180fafd60681a2ea385f0785463792586f01d5
+Author: Pan Liu <wanjun.lp@alibaba-inc.com>
+Date:   Wed Sep 6 18:06:06 2017 +0800
+
+    os/filestore: use __func__ instead of hardcode function name
+    
+    Signed-off-by: Pan Liu <wanjun.lp@alibaba-inc.com>
+    (cherry picked from commit fdb4048a92fad40beab1350a68b29681bd4b6131)
+
+commit 9b93ff70a329e3a66b918d8194f851a2c8a5e3d1
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Fri Sep 15 14:30:47 2017 +0800
+
+    os/bluestore: add "bluestore_read_eio" counter
+    
+    To track down how many EIO errors have been propagated to
+    high level callers.
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit d2d989fcc6a194b9374aeea64e45969b08990262)
+
+commit 902c01792228beb2a539095fec5f24bd293814b5
+Author: xie xingguo <xie.xingguo@zte.com.cn>
+Date:   Fri Sep 15 10:30:04 2017 +0800
+
+    os/bluestore: propagate read-EIO to high level callers
+    
+    E.g., we can let auto-repair to properly handle this
+    instead of crashing the whole osd.
+    
+    Observe this once on one of our test cluster:
+    
+    /clove/vm/clove/ceph/rpmbuild/BUILD/ceph-12.2.0/src/os/bluestore/BlueStore.cc: 6604: FAILED assert(r == 0)
+    
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit a51d9e979fbbe49917712889efb102806479fad2)
+
+commit 024e11dbfcb7077ce2956969f2242bb9bc814d9d
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Jan 21 10:12:43 2018 +0200
+
+    rgw: bucket resharding should not update bucket ACL or user stats
+    
+    bucket link op resets bucket ACL using rgw_link_bucket instead
+    
+    Fixes: http://tracker.ceph.com/issues/22124
+    Fixes: http://tracker.ceph.com/issues/22742
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 1223baf76b45559f171c67c5e737f8e5f7d843e0)
+
+commit 91673b1edfd3d4023d77eb95801db6acf7f900f9
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jan 23 21:07:08 2018 -0600
+
+    os/bluestore: change bdev parse error to ENOENT
+    
+    If there is not a valid label, then the label is not found.  This is a
+    more reasonable error code than "Invalid argumnet".
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5cd98b0dfdc7758761ff789b12faca2266a7e128)
+
+commit 9c083ea16b02c4daef444629439247a415c557c7
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jan 23 21:06:27 2018 -0600
+
+    os/bluestore: make bdev label parse error less noisy
+    
+    This happens during the normal initialization of a new bluestore osd and it
+    is confusing for users.  Make it less noisy.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 14c498dd22d1e89d30a0896e89fe83e8adf6ac76)
+
+commit 40a16ac6cc1550a7d8631c10cbc523a089d143a2
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Jan 31 15:45:44 2018 -0600
+
+    ceph-volume: adds tests for zap --destroy and --osd-id
+    
+    These are not added for dmcrypt right now because there is a bug related
+    to zapping encrypted disks that needs fixed first.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 8d38932c211c589b6171d9e17b100869c23890ac)
+
+commit 97592c255a6df1caeb97e41f88160e17c6e3878b
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Jan 31 09:04:09 2018 -0600
+
+    ceph-volume: no need to return osd_id from util.prepare.check_id
+    
+    Now that osd_id is passed to util.prepare.create_id it doesn't
+    make any sense to return osd_id from check_id anymore as it's
+    not being used.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit f88f2d3bc1f5dcae1e9b2be62dab20942374fc29)
+
+commit c01a2bf515c08ae0a058fdf0fd2ca5f469ed5880
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Jan 30 15:00:04 2018 -0600
+
+    ceph-volume: when reusing an osd ID you must give that id to 'osd new'
+    
+    If you do not then auth will not be created for the new OSD and the
+    daemon will not be able to start.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 43f699fcbc9f30875f70295e086a4e423008b266)
+
+commit 7d0c8833a9b91a6a9da41f4b97938c8dbd244a99
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Jan 30 11:39:05 2018 -0600
+
+    ceph-volume: prepare_* methods do not need to recreate fsid or osd_id
+    
+    The prepare_filestore and prepare_bluestore methods do not need to
+    attempt to recreate osd_id or fsid because this happens in prepare
+    already.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 7c809166aed82298b1fe1581f0e28174a535574f)
+
+commit eab0d1e6cd760e3ec854febae3988a8024d136d4
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Jan 30 11:32:53 2018 -0600
+
+    ceph_volume: modify util.prepare.check_id to handle stdout as a list
+    
+    process.call returns stdout as a list, not a string
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit a08fc639613b3d9bcb3a09875e535287dd58870a)
+
+commit d670b7b3fcb191b8e9f415f261b029689087d116
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Jan 30 09:22:58 2018 -0600
+
+    ceph-volume: tests for util.prepare.check_id
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 0d27fff319f1c4705648fdaff18ee47441f26a24)
+
+commit 55c0eda3f31f1327578daa11435720d1e513a250
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Jan 30 09:21:47 2018 -0600
+
+    ceph-volume: return osd_id from util.prepare.check_id if it exists
+    
+    This also changes this so the osd_id is returned as a string so
+    an ID of 0 would evaluate to True.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 699aa65878ca18f2e2cd70d7444cad18b4a3fd98)
+
+commit 5b04509b61dcfa4b920ef16d62a93934df1c363b
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Mon Jan 29 10:48:09 2018 -0600
+
+    ceph-volume: check to see if an OSD id exists before reusing it
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 548eadeec6fb42a18c3e84dbbd8164862f8836e6)
+
+commit 7e8dbc50129b7729036974bf3f20cfbfad2794bb
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Mon Jan 29 10:43:04 2018 -0600
+
+    ceph-volume: adds a prepare util for checking OSD ID existance
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 4ac99de6d98b646bf160230584f6532775cdc9cb)
+
+commit 03569abb408e603e4799f97eac333d118ddf23c7
+Author: Shengjing Zhu <zhsj@umcloud.com>
+Date:   Thu Feb 1 16:47:26 2018 +0800
+
+    pybind/mgr/dashboard: fix duplicated slash in html href
+    
+    The url_prefix val in html templates is expected not containing '/'
+    in the end. Otherwise the href will be rendered as '//static/', which
+    causes browser treating it as http://static/.
+    
+    Fixes: http://tracker.ceph.com/issues/22851
+    Signed-off-by: Shengjing Zhu <zhsj@umcloud.com>
+    (cherry picked from commit 24fe1cae636bf00e3d5923c7e826992d42d92946)
+
+commit b47837100f3eae46a2c983f0299e40bf7944c9f8
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Jan 22 15:16:40 2018 -0500
+
+    doc/ceph-volume OSD use the fsid file, not the osd_fsid
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a5f26c622b0f11b7d8179da22d18f719b9febe0a)
+
+commit 4974d42eea073bd1f2f27cdd3b5a35be789450a7
+Merge: 239b1ae5e1 f4d5757fb5
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Mon Feb 5 16:08:27 2018 +0100
+
+    Merge pull request #20241 from ceph/backport-wip-rm22619
+    
+    luminous ceph-volume: dmcrypt support for lvm
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit 5282a60c8fa50db6fff957673dbd73ec1c22a59a
+Author: Jianyu Li <joannyli@foxmail.com>
+Date:   Fri Jan 5 00:54:30 2018 +0800
+
+    Avoid the assert failure when the inode for the cap_export from other MDS happened not in MDCache
+    
+    Signed-off-by: Jianyu Li <joannyli@tencent.com>
+    (cherry picked from commit 82b9b838ae0bd17247f6296f9d7001331f50663c)
+
+commit 042144a281073c4cd3e80e969f7eaf7f2b289de5
+Author: Shangzhong Zhu <zhu.shangzhong@zte.com.cn>
+Date:   Tue Jan 23 20:35:36 2018 +0800
+
+    mds: set higher priority for some perf counters
+    Fixes: http://tracker.ceph.com/issues/22776
+    
+    Signed-off-by: Shangzhong Zhu <zhu.shangzhong@zte.com.cn>
+    (cherry picked from commit 05c496d444b388ec505df4598abd3fc6309ed99f)
+
+commit 239b1ae5e19c16e976c2045fef5ad65f1f727278
+Merge: 982bd23865 7a8e915483
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sat Feb 3 11:31:34 2018 -0800
+
+    Merge PR #18930 into luminous
+    
+    * refs/pull/18930/head:
+            mds: additional damage handling case in EImportStart
+
+commit 982bd2386543be41eb3d67f9207971d285b08807
+Merge: b8392683a1 127236f676
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Fri Feb 2 15:47:11 2018 -0800
+
+    Merge pull request #20098 from ovh/bp-luminous-snaptrimq-visibility
+    
+    luminous: osd, pg, mgr: make snap trim queue problems visible
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit b8392683a1cc9ab489d3c77f8edac395469b2ce8
+Merge: 9083f7db62 5cec46600d
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 15:10:18 2018 -0800
+
+    Merge pull request #19267 from tchaikov/wip-ceph-disk-fsid-luminous
+    
+    luminous: ceph_disk: allow "no fsid" on activate
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 9083f7db62b49037d1b6d8c1a092f0028b5882fb
+Merge: 119c9f0363 bd558d7e72
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 15:09:49 2018 -0800
+
+    Merge pull request #19387 from ceph/wip-cd-luminous-upgrade
+    
+    luminous: tests: ceph-deploy: use wip branch for final upgrade step
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 119c9f03637bcb2e7236b8010fb038f0c3235bad
+Merge: 5b9c671928 5a40ef2244
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 15:08:49 2018 -0800
+
+    Merge pull request #19610 from ukernel/luminous-22306
+    
+    luminous: osdc/Journaler: add 'stopping' check to various finish callbacks
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 5b9c6719287aa9e2783cdbec9be95955f877f3bb
+Merge: faf8e1cc56 44643cae8d
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 15:08:10 2018 -0800
+
+    Merge pull request #19967 from smithfarm/wip-22707-luminous
+    
+    luminous: ceph_objectstore_tool: no flush before collection_empty() calls; ObjectStore/StoreTest.SimpleAttrTest/2 fails
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit faf8e1cc5654abea8efcc7d05804fdd2db8f5803
+Merge: b7728d23be 654678ab2d
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 15:04:29 2018 -0800
+
+    Merge pull request #19047 from shinobu-x/wip-22193-luminous
+    
+    luminous: bluestore: OSD crash on boot with assert caused by Bluefs on flush write
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit b7728d23bec0fb8af5d10d8af6c48d325331818c
+Merge: f185e67496 e56106cfc1
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 15:01:50 2018 -0800
+
+    Merge pull request #19388 from ceph/wip-cv-luminous-bp
+    
+    luminous: qa: ceph-volume updates
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit f185e67496721fe04cef4d99d097242a8277b2bd
+Merge: 9ed49bdb3e e621e9d7f1
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 15:01:04 2018 -0800
+
+    Merge pull request #19487 from smithfarm/wip-22389-luminous
+    
+    luminous: ceph-objectstore-tool: Add option dump-import to examine an export
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 9ed49bdb3ed0953e348bcde7cc73d39022b5a759
+Merge: 2d92e6a4b8 5e5d5c8196
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 15:00:26 2018 -0800
+
+    Merge pull request #19501 from shinobu-x/wip-22399-luminous
+    
+    luminous: Manager daemon x is unresponsive. No standby daemons available
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 2d92e6a4b89d5bd12aef5f7049f94a63ca1228af
+Merge: e579ed377d 6cc4a33933
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:59:49 2018 -0800
+
+    Merge pull request #19865 from smithfarm/wip-22634-luminous
+    
+    luminous: build/ops: ceph-mgr dashboard has dependency on python-jinja2
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit e579ed377daf35679c551a254ea0770806499e03
+Merge: 595a5984db d2a68571a8
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:59:08 2018 -0800
+
+    Merge pull request #19966 from smithfarm/wip-22706-luminous
+    
+    luminous: tests: force backfill test can conflict with pool removal
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 595a5984db75025018ab0f740e1e2de7eebf7931
+Merge: 241573d06c cd9e6788cb
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:58:23 2018 -0800
+
+    Merge pull request #20116 from smithfarm/wip-22266-luminous
+    
+    tools/ceph_monstore_tool: include mgrmap in initial paxos epoch
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 241573d06ca41144e1674f418ab7ae923485e20d
+Merge: 2d6afdca94 f454c293d1
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:57:20 2018 -0800
+
+    Merge pull request #20215 from smithfarm/wip-22701-luminous
+    
+    luminous: build/ops: ceph-volume fails when centos7 image doesn't have lvm2 installed
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit 2d6afdca9449c94d81aa1c10266894c94f192f2b
+Merge: fcfd31a5eb f2f00cac71
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:56:30 2018 -0800
+
+    Merge pull request #20219 from tchaikov/wip-pr-20128-luminous
+    
+    luminous: common/pick_address: wrong prefix_len in pick_iface()
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit fcfd31a5ebc7cad1a1a4304a007cf48374a4f90c
+Merge: 77c4502c8d 9d617b4ecc
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:38:23 2018 -0800
+
+    Merge pull request #19849 from liewegas/wip-monlog-luminous
+    
+    mon: allow cluster and debug logs to both go to stderr with different prefixes
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Joao Eduardo Luis <joao@suse.de>
+
+commit 77c4502c8d566cb1e63d8f42b0088f790922e61e
+Merge: f24fb875a8 f59380e159
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:36:55 2018 -0800
+
+    Merge pull request #19888 from pdvian/wip-22633-luminous
+    
+    luminous: OSD crushes with FAILED assert(used_blocks.size() > count) during the first start after upgrade 12.2.1 -> 12.2.2
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit f24fb875a8313d0965dce7dad05d988f6df118d7
+Merge: db33f288c0 8b13643b32
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:36:03 2018 -0800
+
+    Merge pull request #19928 from jdurgin/wip-luminous-omap-recovery
+    
+    config: lower default omap entries recovered at once
+    
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit db33f288c036bf0ebdf540eb607745d53ad4589c
+Merge: 9379ed806a d5e2e43de8
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:35:38 2018 -0800
+
+    Merge pull request #19938 from liewegas/wip-list-size-luminous
+    
+    luminous: common: compute SimpleLRU's size with contents.size() instead of lru.size()
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 9379ed806ae419871cd535ce067e7e7adc3eec8a
+Merge: 5879a59eb6 e9f5612bd7
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:34:27 2018 -0800
+
+    Merge pull request #19947 from tchaikov/wip-pr-18191-luminous
+    
+    qa/standalone/osd/osd-mark-down: create pool to get updated osdmap faster
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 5879a59eb65b6d5277658a8a130d57be00c60d4b
+Merge: 8f212ed733 8e3edae0c8
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:33:48 2018 -0800
+
+    Merge pull request #19969 from smithfarm/wip-22691-luminous
+    
+    luminous: ceph-base symbols not stripped in debs
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 8f212ed733baa63f0d971f00e23b8b92cd416fe8
+Merge: d9542bcf02 dc96e3c1f8
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:33:13 2018 -0800
+
+    Merge pull request #19976 from pdvian/wip-22690-luminous
+    
+    luminous: tests: increase osd count for ec testing
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit d9542bcf02dd3574d27eca04fb7ab799d06b8452
+Merge: 29ffc82d0b 91be5317da
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:32:39 2018 -0800
+
+    Merge pull request #19977 from pdvian/wip-22692-luminous
+    
+    luminous: common: compute SimpleLRU's size with contents.size() instead of lru.…
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 29ffc82d0bdb34a3c3e6b89df87b95b3bc6361e5
+Merge: 6b80c14af1 07a122a3fd
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Feb 2 14:31:56 2018 -0800
+
+    Merge pull request #19995 from pdvian/wip-22698-luminous
+    
+    luminous: common/throttle: start using 64-bit values
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit f4d5757fb574411e8e47055f2aaaf54bce076501
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Jan 22 14:21:37 2018 -0500
+
+    doc/ceph-volume add lvm/encryption to the index
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 163ee6a61235e97c341b2817114d98b576697096)
+
+commit 4d237d7418a7c1a6f7bce80d978ee5fae8e2d22a
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Jan 22 12:11:32 2018 -0500
+
+    doc/man/ceph-volume add dmcrypt support
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit babcdf2fa59baab6bc2cd8a1ac190b8d02fa3fce)
+
+commit 5a7b50f121aee3fc13474b42e11b5d19a36d7764
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Jan 22 12:06:25 2018 -0500
+
+    doc/man/ceph-volume add docs for zap sub-command
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit ad73be2c2a6ee273d37b706626974d6900742256)
+
+commit 3c8abd24b2801a8e76f4b327a1b029cb0eecfdd7
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Jan 22 12:03:18 2018 -0500
+
+    doc/man/ceph-volume add docs for list sub-command
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 498d7a06a1f17c769c11da7ba9f83bcf2c0e433d)
+
+commit e51d7dd4da3a156250489f33deddcbe8c208d2aa
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Jan 22 11:41:32 2018 -0500
+
+    doc/dev/ceph-volume update internal LVM metadata information
+    
+    Includes dmcrypt information, and updated key names to reflect current
+    code
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 46babe80b638eb607e2af6f0c8bb77e675d4d63b)
+
+commit 8b938c42f76b95425476c7ee2df275fa721d8de9
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Jan 19 11:12:52 2018 -0500
+
+    ceph-volume tests.functional add xenial filestore tests for dmcrypt
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit f0f0c1f8c4dc433d8978de134c120328842a5d7f)
+
+commit 5c405a05fcb07b8abd81a11c152f197e3596db78
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Jan 19 11:12:40 2018 -0500
+
+    ceph-volume tests.functional add xenial bluestore tests for dmcrypt
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit be0dfe76535fa135168763275513d7fafba6fe4c)
+
+commit 558b5c62fcc1dd6ac7627e11523c60b0bfab3787
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Jan 19 11:08:25 2018 -0500
+
+    ceph-volume tests.functional add dmcrypt support in tox.ini
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d1ef5c487ba1ea868fd85b8c425986a2cf435519)
+
+commit 95c279b4363a3630422f15a3bd5ad33bd10204e3
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Jan 19 11:03:58 2018 -0500
+
+    ceph-volume tests.functional add centos7 filestore tests for dmcrypt
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 2477ac805e172a10ae3214151a78b4601cf88021)
+
+commit 9b9ed02de83fa4397f776427fa37a21fdb16f4b9
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Jan 19 10:45:32 2018 -0500
+
+    ceph-volume tests.functional add centos7 bluestore tests for dmcrypt
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 18972322567e158a56d0e71ca4f2afbed413c855)
+
+commit 1c560cd8053ab5e6b921b3c1365cb70a1c71e811
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Jan 19 09:58:36 2018 -0500
+
+    ceph-volume tests add util.prepare unit tests
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a774eae43a089798f895507f18b0c5d68a49fbb4)
+
+commit 43fb4561d8c44ddd5d246e601671d91606a09ca3
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Jan 19 09:58:18 2018 -0500
+
+    ceph-volume tests add fixture helpers for call and run
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d51973a64d9f18e0dfde7601b3ffbc57571ea653)
+
+commit 22c10490150426a45c832a59cf6613d331f7cb19
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Jan 18 16:37:00 2018 -0500
+
+    ceph-volume util allow prepare module to work with encryption keys
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a36dcdf9cfbc3ca012c5cb022e0d7dbcd95984f4)
+
+commit b5f674daff7113a7c17b10cd213734140545b018
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Jan 18 16:22:06 2018 -0500
+
+    ceph-volume lvm.activate allow encryption setups for luks
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 3476e87cec353b100cf8c117d74708f51dd6de2c)
+
+commit a26321213b8d77084345af306d7ee8a73b238255
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Jan 18 16:19:26 2018 -0500
+
+    ceph-volume lvm.prepare allow encryption setups for luks
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit be3300455e479b03ac072d28bb05ccc7eaddb2e9)
+
+commit fde4817d87132d5dade973fba36743f8b3c37230
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Jan 18 14:46:42 2018 -0500
+
+    ceph-volume process allow silencing output on file logging for sensitive data
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 0e2190c7a22b61aeecdb60b6c8bfe5ba6aa873a4)
+
+commit f357d5db65d427c1184169e588c475742ac1a45d
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Jan 18 14:29:53 2018 -0500
+
+    ceph-volume util create a helper module for encryption
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit dc34a32675a870da98931adffacb2d3db4f74c6c)
+
+commit 38c8a384b22da1de36b567d2b3816ba936083216
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Jan 16 09:02:31 2018 -0500
+
+    ceph-volume lvm.common add dmcrypt flag
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 88ff873e9a60bd811356f7e15c8ef7f826d09337)
+
+commit 65a38c7855f3e1ce7788d25d02ac7ca779b917a0
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Jan 11 09:02:03 2018 -0500
+
+    doc/ceph-volume create an encryption doc explaining dmcrypt workflows
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 477670a0912c9e01fe59155bf135eaf92bdc3318)
+
+commit 6b80c14af1f36dbb0bd5c73d9935d41bf7d53350
+Merge: a219bf4727 0aa31d29d5
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Feb 2 13:19:51 2018 -0600
+
+    Merge pull request #20244 from ceph/backport-19276
+    
+    luminous: ceph-volume: do not use --key during mkfs
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 7756a44529dd27e0960393e098104c7c1808bb36
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Jan 25 12:08:47 2018 +0800
+
+    osdc/ObjectCacher: don't merge TX buffer heads
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 0b1e515d1c5dddf3c11eddbe6d53167d2fb6aaab)
+
+commit 43b37b83df897b57c10cea2ae307897f1f7fb96c
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Jan 25 10:25:25 2018 +0800
+
+    osdc/ObjectCacher: fix off-by-one error in bh_write_commit()
+    
+    Fixes: https://tracker.ceph.com/issues/22741
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit cc4c6dcc2ffdc482f1915db569949e674691c372)
+
+commit d93eef31277d30020b016d74e8d6b7e93e850fd9
+Author: dongdong tao <tdd21151186@gmail.com>
+Date:   Fri Jan 19 00:08:44 2018 +0800
+
+    mds: fix scrub crash
+    
+    Fixes: http://tracker.ceph.com/issues/22730
+    Signed-off-by: dongdong tao <tdd21151186@gmail.com>
+    (cherry picked from commit e741c1a6e16cb5179d623ef491f66d31b0ffdfd6)
+
+commit a219bf47277d923db18dd221660a04bb6bac3580
+Merge: e62f21ca34 f57a80f0a8
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Feb 1 15:40:51 2018 -0800
+
+    Merge pull request #20247 from ceph/revert-19552-wip-22452-luminous
+    
+    Revert " luminous: msg/async: unregister connection failed when racing happened"
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit f57a80f0a8f4ad4d7d482c05ac8b25fd160f55df
+Author: Sage Weil <sage@newdream.net>
+Date:   Thu Feb 1 17:28:06 2018 -0600
+
+    Revert " luminous: msg/async: unregister connection failed when racing happened"
+
+commit e62f21ca347750d3289bc1f66f5066a09c3445f8
+Merge: 419ed62db3 82c9e3dbe1
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 15:10:13 2018 -0800
+
+    Merge pull request #19187 from liewegas/wip-trim-mgrmap-luminous
+    
+    mon/MgrMonitor: limit mgrmap history
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 419ed62db3d8a6aed94c1598e6dadaddbac7b9ab
+Merge: 11c3f8f3dc 3bfb493fa0
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 15:09:29 2018 -0800
+
+    Merge pull request #19269 from tchaikov/wip-pr-19202-luminous
+    
+    luminous: cmake,common/RWLock: check for libpthread extensions
+    
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 11c3f8f3dc179d34f3197ce7ffd07731b5e652c9
+Merge: 4106dbddea c22f06b3c5
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 15:08:45 2018 -0800
+
+    Merge pull request #19499 from shinobu-x/wip-22402-luminous
+    
+    luminous: osd: replica read can trigger cache promotion
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 4106dbddeab069e816219c1fc775852eab8fd1ff
+Merge: 6dcfa33858 9a9ed272a7
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 15:08:07 2018 -0800
+
+    Merge pull request #19552 from shinobu-x/wip-22452-luminous
+    
+     luminous: msg/async: unregister connection failed when racing happened
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 6dcfa3385836357f556e5d45e97a0b05eda5df58
+Merge: 589342d932 47f74bcd13
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 15:07:36 2018 -0800
+
+    Merge pull request #19628 from shinobu-x/wip-22501-luminous
+    
+     luminous: tests: do not configure ec data pool with memstore
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 589342d932d3e7310167842dd22c1303bf1156d5
+Merge: 37ce36b6d0 ce54fd9b99
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 15:06:56 2018 -0800
+
+    Merge pull request #19630 from shinobu-x/wip-22507-luminous
+    
+     luminous: bluestore: do not crash on over-large objects
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 37ce36b6d0e630b0c527ead826d9761fd4cb9622
+Merge: bb9964ae21 6791e67faf
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 15:06:17 2018 -0800
+
+    Merge pull request #19742 from tchaikov/wip-luminous-pr-19649-and-19650
+    
+    luminous: common/dns, erasure-code: fix mem leaks
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit bb9964ae2163eeff35538e4df897317c1e477fa8
+Merge: 6f0167b181 ac064e1fc8
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 15:02:15 2018 -0800
+
+    Merge pull request #20049 from liewegas/wip-zero-length-luminous
+    
+    os: fix 0-length zero semantics, test
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 6f0167b181a5850b9c40128b682ab210b217824a
+Merge: 742b7ba11f a0c0e637fe
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 15:01:35 2018 -0800
+
+    Merge pull request #20068 from smithfarm/wip-22761-luminous
+    
+    luminous: osd: do not check out-of-date osdmap for DESTROYED flag on start
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 742b7ba11fae67fe806306284b9ae76fcb8160a0
+Merge: c8082bda93 238ddccbbc
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 15:01:11 2018 -0800
+
+    Merge pull request #20069 from smithfarm/wip-22770-luminous
+    
+    luminous: tools: ceph-objectstore-tool set-size should clear data-digest
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit c8082bda938d6db1c4def4617875ab99429d78aa
+Merge: df3088dbfd 8950736b8e
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 15:00:41 2018 -0800
+
+    Merge pull request #20150 from pdvian/wip-22807-luminous
+    
+    luminous: mon: do not use per_pool_sum_delta to show recovery summary
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 0aa31d29d5eb3b2071dbdb678fc3f1dc303a64ea
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sat Dec 2 00:43:21 2017 +0800
+
+    ceph-volume: process: disable stdin param of run()
+    
+    we cannot use process.communicate() to feed the Popen with input,
+    because, upon return of process.communicate() the stdout,stderr are
+    closed. see https://docs.python.org/2/library/subprocess.html#subprocess.Popen.communicate .
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 6df444cf33a251f5dcb536606b44af696cdb4ed5)
+    
+    Conflicts:
+            src/ceph-volume/ceph_volume/process.py
+
+commit a67d46b63228517cd9ce37617a7e0f5f79ab16f1
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Nov 30 08:33:21 2017 -0600
+
+    ceph-volume: use --keyfile instead of --key
+    
+    We do not want the key to show up on the command line (it may appear in
+    the process list or sudo log file).
+    
+    Fixes: http://tracker.ceph.com/issues/22283
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 46abd50f8bcbd347d577581ae42156b718556158)
+
+commit 5717860f38cc79e165bd5bf791064177f8a4200d
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Nov 30 08:32:47 2017 -0600
+
+    osd: accept 'keyfile' as well as 'key' during mkfs
+    
+    Also, don't print the raw key to the log.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    Signed-off-by: Kefu Chai <kefu@redhat.com>
+    (cherry picked from commit 18aff37ee3a2c2d3016f08c98161a5acc41ff8e5)
+
+commit df3088dbfd021393106a5fba93bd722a5e0f540a
+Merge: d43ec551af cf6799e4b5
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:15:14 2018 -0800
+
+    Merge pull request #19370 from shinobu-x/wip-22339-luminous
+    
+    luminous: client: quit on failed remount during dentry invalidate test #19370
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit d43ec551af92b7c205f80c1c6617a9b258a98697
+Merge: 685e526cc5 4943899005
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:14:47 2018 -0800
+
+    Merge pull request #19585 from ukernel/luminous-22460
+    
+    luminous: mds: handle client session messages when mds is stopping
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 685e526cc5d601f21292f13f56cfdcafd5c3c9b4
+Merge: 5ca708cd9a d0d66c73b0
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:14:18 2018 -0800
+
+    Merge pull request #19626 from shinobu-x/wip-22499-luminous
+    
+     luminous: cephfs-journal-tool: tool would miss to report some invalid range
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 5ca708cd9ac00e63557648ba14b4ef6b369dbaa1
+Merge: b5c189b84e 6eea75200f
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:13:48 2018 -0800
+
+    Merge pull request #19627 from shinobu-x/wip-22500-luminous
+    
+     luminous: cephfs: potential adjust failure in lru_expire
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit b5c189b84e7a98ece7c0d485ee7ec87bb2cdc0cf
+Merge: a6189cc37f 08edb16311
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:13:16 2018 -0800
+
+    Merge pull request #19776 from ukernel/luminous-22492
+    
+    luminous: mds: respect mds_client_writeable_range_max_inc_objs config
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit a6189cc37f169ecdce05576fd88362f7b2871927
+Merge: b4bc724549 dd6b6626cc
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:12:48 2018 -0800
+
+    Merge pull request #19829 from pdvian/wip-22573-luminous
+    
+    luminous: vstart_runner: fixes for recent cephfs changes
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit b4bc7245497d2550c9fc2fd047ac9a185c334b7b
+Merge: 66189e1656 98e3e2a7c8
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:12:20 2018 -0800
+
+    Merge pull request #19830 from shinobu-x/wip-22579-luminous
+    
+    luminous: mds: check for CEPH_OSDMAP_FULL is now wrong; cluster full flag is obsolete
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 66189e16560485cdea9b97b46e47371cf134b249
+Merge: b35c39f6db 5dcd2a5977
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:11:52 2018 -0800
+
+    Merge pull request #19871 from smithfarm/wip-21948-luminous
+    
+    luminous: mon: MDSMonitor: reject misconfigured mds_blacklist_interval
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit b35c39f6db246463045a5eb6b5c477b99fdeab77
+Merge: 243f187c02 5fd01ca888
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:11:28 2018 -0800
+
+    Merge pull request #19959 from smithfarm/wip-22694-luminous
+    
+    luminous: mds: fix dump last_sent
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 243f187c023962f17fa7715c196c45e712ea51a1
+Merge: b1c27aedb9 80ea9ed3b3
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:11:02 2018 -0800
+
+    Merge pull request #19962 from smithfarm/wip-22580-luminous
+    
+    luminous: tests: full flag not set on osdmap for tasks.cephfs.test_full
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit b1c27aedb92d79f507b4fa0bc514b2d173d87296
+Merge: f60c2df92d cc76ab1eef
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:10:29 2018 -0800
+
+    Merge pull request #19968 from ukernel/luminous-22699
+    
+    luminous: ceph-fuse: ::rmdir() uses a deleted memory structure of dentry leads …
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit f60c2df92d021f4eb82da5e74de06464818e30c8
+Merge: 69c87bedcc 5353569eea
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:09:07 2018 -0800
+
+    Merge pull request #19982 from ukernel/luminous-22719
+    
+    luminous: mds: handle 'inode gets queued for recovery multiple times'
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 69c87bedcc0814e10db6d38b50f836be6c0528c9
+Merge: d81ce5c0ed 2260f03b36
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:05:39 2018 -0800
+
+    Merge pull request #20042 from tchaikov/wip-pr-19998-luminous
+    
+    luminous: fix broken use of streamstream::rdbuf()
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit d81ce5c0ed368d2ec01d539b282966e25b4138e7
+Merge: 2b0599d308 3ccf7e0bc6
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Feb 1 13:04:16 2018 -0800
+
+    Merge pull request #20106 from smithfarm/wip-22502-luminous
+    
+    luminous: osd: Pool Compression type option doesn't apply to new OSDs
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 2b0599d308ecf7e7d637caf1932eef8020ff49fc
+Merge: 2ac9bbdd2e b12e00097b
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Thu Feb 1 15:07:48 2018 -0500
+
+    Merge pull request #20240 from ceph/backport-20010
+    
+    luminous: ceph-volume: adds a --destroy flag to ceph-volume lvm zap
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit b12e00097b7dc9ad322abc5f4e53d7367b3ee37c
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Jan 18 14:54:55 2018 -0600
+
+    ceph-volume: clean up docstrings for lvm zap
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 57471fe7e74c94351e713ff99cbcbaa7dc6a0f52)
+
+commit d06e7d52d27f19ec1ab9d39af4f45733fbe21246
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Jan 18 14:41:53 2018 -0600
+
+    ceph-volume: print a message to terminal if --destroy is skipped
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit d2529ee5dd9a4dc6a81eeb2d5e54c0bdd6beeb2f)
+
+commit aba99e70d9f3f555b7d7e183d2ea9e506ca2ca40
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Jan 18 14:34:22 2018 -0600
+
+    ceph-volume: no need to set PVolume.vg_name explicitly
+    
+    This will be set automatically in __init__ by looping
+    through kw.items and creating properties on PVolume
+    with them.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 129182645c4a7e5e42500d3ef2083cfdfb0137e4)
+
+commit 039d1875e4e5ca52e2d54c35ff2dbfea25202d49
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Jan 18 14:31:20 2018 -0600
+
+    ceph-volume: switch remove_vg and remove_pv to process.run
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 5a1be9a73919e13fe603861588bcc20872a4c133)
+
+commit 5f98bc964b2ef05110363c70507048c85e33fa8c
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Jan 18 14:25:46 2018 -0600
+
+    ceph-volume: allow passing a fail_msg param to process.run
+    
+    This will allow the user to provide a more detailed error message
+    on why the command failed.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 7204851042fa42799e1003ef57d57a512d4a7f28)
+    
+    Conflicts:
+            src/ceph-volume/ceph_volume/process.py
+
+commit 2ac9bbdd2ee98f0b07f61d507fbf56f5bfbe213e
+Merge: 3f0d1f7166 ccc6b55463
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Thu Feb 1 15:01:04 2018 -0500
+
+    Merge pull request #20239 from ceph/backport-19949
+    
+    luminous: ceph-volume: adds --crush-device-class flag for lvm prepare and create #19949
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit 8d7552bfff302728669b51df41c2334e2520e2e8
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Jan 18 12:04:38 2018 -0600
+
+    ceph-volume: tests for lvm zap --destroy
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit fafa183d8f13fa769eaab5f865f6592b89225901)
+
+commit 4f3ebfe4e1f008230bad5df7b42e6c103c75e79d
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Jan 18 11:10:48 2018 -0600
+
+    ceph-volume: also remove pvs with lvm zap --destroy
+    
+    Leaving the pv around doesn't keep us from redeploying on
+    that device because we also wipefs, but explicitly destroying
+    it makes the output read better.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit cb748ccb2264aea93dd4717a46d7f833c5b1eccd)
+
+commit 2a42491c65bc49bbcd7ecd86ca61878dde5dc73d
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Jan 18 11:10:15 2018 -0600
+
+    ceph-volume: adds an api.lvm.remove_pv method
+    
+    Used to remove physical volumes
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 0811c02b118c270b2bec87cd26a877b26bc780b1)
+
+commit 8fd6f29ee3988d3158467e9cf447e63c7d3485e5
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Jan 18 11:05:32 2018 -0600
+
+    ceph-volume: remove the -a flag from api.lvm.get_pvs
+    
+    When we run `pvs -a` it returns all pvs, even ones that
+    have been recently deleted. Because of this pvs that have just
+    been zapped with `lvm zap` show up and can cause issues if a device
+    is zapped more than once.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 10a349742cae909d1ebf0a0b3b49c88774929f53)
+
+commit 4d23258e19b56d04f2d6f108ad808b01bc38b831
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Jan 18 08:26:07 2018 -0600
+
+    ceph-volume: docs for the --destroy flag of lvm zap
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 708cb43c503e962304211041570d6ca660bca1fb)
+
+commit 89a0aa94c9b5becee205a1803541ceb2aad961ad
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Jan 17 15:11:19 2018 -0600
+
+    ceph-volume: adds the --destroy flag to ceph-volume lvm zap
+    
+    If you use the --destroy flag and are zapping a raw device
+    or parition then zap will destroy any vgs or lvs it finds on that
+    device.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit aea326b772395c076e0d75f7c2f591a46aef7f57)
+
+commit 3002953e4f506850339c676a85dc5402def7423d
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Jan 17 13:12:17 2018 -0600
+
+    ceph-volume: adds an api.lvm.remove_vg method
+    
+    This method can be used to remove volume groups
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 50a6a543a956d133b428cf36ad629c2131be5490)
+
+commit 9146305907b1c8d9f0162b44c76fd6133b7cb094
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Jan 17 12:58:50 2018 -0600
+
+    ceph-volume: expose vg_name in api.lvm.get_api_pvs
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit b7f24ec451e167d81a83c1e21597a9fece4f0684)
+
+commit 3f0d1f7166e20defae5ed8783f7f49f7a8b461d1
+Merge: 5250230360 4bd607b2ae
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Thu Feb 1 14:55:57 2018 -0500
+
+    Merge pull request #20238 from ceph/backport-19875
+    
+    luminous: ceph-volume: adds success messages for lvm prepare/activate/create
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit ccc6b55463335895b5896f4e772f6c0e412d7522
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Jan 17 11:44:10 2018 -0600
+
+    ceph-volume: print the lvm activate success message correctly
+    
+    Move the terminal.success to the methods that have access to the
+    correct osd_id so that they do not print None for the osd_id
+    in the log message.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit c9f7689a7a71a6b2321301b84599f319e8ae8eba)
+
+commit 0e47c46fef88b3dc4ee9c7849da67f24db375044
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Mon Jan 15 11:55:44 2018 -0600
+
+    ceph-volume: docs for --crush-device-class
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 21f459437bd1208812504a20e5a50f7a47bfa979)
+
+commit 73e714357d898a06b7bb172691a0dbe360f0f535
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Jan 11 13:11:31 2018 -0600
+
+    ceph-volume: adds crush_device_class to json metadata for 'osd new'
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 3404d8bba503f3f74f411b11108d39cb626d3bed)
+
+commit 5d9b454522128c0ba0ee1c93485accc48e91e7a4
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Thu Jan 11 13:04:07 2018 -0600
+
+    ceph-volume: adds crush_device_class to the functional tests
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 36a388d72acb049b5ee25ceea6a8d341aa58e31f)
+
+commit 00efa1a1beac442fb008f2ccb2e26939eab2cc10
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Jan 10 12:19:54 2018 -0600
+
+    ceph-volume: add crush_device_class to lvm metadata
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 11da2185635ce14103471d851f1e8c84199d3745)
+
+commit 48591335c3903cb5446208da9940752fb8b390c7
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Jan 10 09:29:15 2018 -0600
+
+    ceph-volume: adds a --crush-device-class flag to lvm create and prepare
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit bf468bc3737c15a8f15ec19ebf81c42672e28aaf)
+
+commit 4bd607b2ae7519414e8dd5c92e7522c8c5024060
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Jan 9 12:21:35 2018 -0600
+
+    ceph-volume: adds success message to ceph-volume simple activate
+    
+    This used to only show the message if systemd was not being used, now
+    it will show this always if the command is successful.
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 87c6ed14e7e0b528a1fe107e2e531908cdea0f47)
+
+commit ad73312ff0d376b8e055936e2f9c6d02ec90d0f4
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Jan 9 09:52:23 2018 -0600
+
+    ceph-volume: add success message for ceph-volume lvm create
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 0185d93cc1be1e7a6d3a803631ff92df9787754a)
+
+commit 4709380f6b23d7bb5e6c0b98f7745d2495f2af07
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Jan 9 09:48:09 2018 -0600
+
+    ceph-volume: add success message for ceph-volume lvm activate
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit d30928d7c7d7cfb7e465ec2e7e46dc48407436f0)
+
+commit ce5df11e06c85e2d00ee1bdd98a0386212effce6
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Tue Jan 9 09:45:40 2018 -0600
+
+    ceph-volume: add success message for ceph-volume lvm prepare
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit 75c460f6e1864f15e1d23146c5740ce43c000e42)
+
+commit 5250230360a7d91d6cee1a425e183c8220684889
+Merge: 4e9092b194 25fbed91ec
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Thu Feb 1 20:43:41 2018 +0100
+
+    Merge pull request #20237 from ceph/backport-wip-rm22281
+    
+    luminous: ceph-volume rollback on failed OSD prepare/create
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit 25fbed91ec23260cba944411929055a75d151f96
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Dec 5 15:31:55 2017 -0500
+
+    ceph-volume lvm.create rollback osd creation when prepare or activate fails
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 07be6fe0ab7981ebdc34779a8ccd6220d640a549)
+
+commit d6598635b89eb3dd11389c4fae322b8f3934b5c8
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Dec 5 10:22:05 2017 -0500
+
+    ceph-volume lvm.create fallback to bluestore when objectstore is not specified
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 6781d6498e01bfc89189c235540e5c2a2e6bc8ca)
+
+commit c49473556746a0e7f0fe9b7a25e0a2cd056281d5
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Dec 5 10:21:20 2017 -0500
+
+    ceph-volume lvm.prepare fallback to bluestore when objectstore is not specified
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 9b17ad2f8fe617d219fdb2f46dd2675669f32f08)
+
+commit 14e16a236ae6c5fad0f9e25a70a2c2d74eee5546
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Dec 5 10:11:03 2017 -0500
+
+    ceph-volume lvm.prepare rollback osd when prepare fails and an osd id was generated
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 579d12806b2ffc693a4e70ba3eaf663fa281d35e)
+
+commit 2d23836024ee796f9212bb44ef9f1fb18d77eef4
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Dec 5 10:08:35 2017 -0500
+
+    ceph-volume lvm.common create a rollback_osd utility to cleanup failed osd prepare/create calls
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit e533792e0dbafc160238c0f5783f4430e2705571)
+
+commit 4e9092b1949d635599a1d47ab8ccedfa69d77db5
+Merge: 041dd60315 26b7337583
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Feb 1 12:30:26 2018 -0500
+
+    Merge pull request #19483 from smithfarm/wip-22375-luminous
+    
+    luminous: build/ops: luminous build fails with --without-radosgw
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 041dd603153c5270ebeb4091b537ae88b5941f2d
+Merge: c727db5bb9 445583a922
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Feb 1 12:10:34 2018 -0500
+
+    Merge pull request #20211 from dillaman/wip-22198-luminous
+    
+    luminous: librbd: compare and write against a clone can result in failure
+    
+    Reviewed-by: Mykola Golub <mgolub@suse.com>
+
+commit 445583a922c9eecf1eba60df4f9da49fb7bc8926
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Nov 15 08:24:50 2017 -0500
+
+    test/librbd: disable compare and write if skipping partial discard
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4e8b0b60a72fe1e1ac10a2beb46b77d4b58c91c4)
+
+commit e41f815e6b4e96ba7155202a2ce3e046370c0d56
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Nov 9 12:10:30 2017 -0500
+
+    librbd: refactor object write request state machines
+    
+    Fixes: http://tracker.ceph.com/issues/20789
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0e643fb926f0484bb4f860740c7f85b692de6737)
+
+commit 11f21becff13e1f27f758500816cb9643cd010d0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Nov 9 10:24:08 2017 -0500
+
+    librbd: consolidate all object discard-related logic into single state machine
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 318797f59adcedcc386e01f1975011e0086434ac)
+
+commit 4f77368ab08df27417cc5c5f3c3200a5f68733ca
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Nov 8 12:31:28 2017 -0500
+
+    librbd: simplify interface between object IO and copyup state machines
+    
+    The initial copyup was not receiving a write hint and the
+    code for hints was duplicated multiple times. Additionally,
+    the object map state should match the last executed write op.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7117aa4ecaf0b619b527b2783fa1d09b11f3dd55)
+    
+    Conflicts:
+            src/librbd/io/ObjectRequest.h: trivial resolution
+
+commit 7807083ea27ccb4b60dbc050e9d135464756fdb1
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Nov 9 15:15:58 2017 -0500
+
+    librbd: copyup state machine needs to handle empty write ops
+    
+    The compare-and-write object operation cannot be executed
+    concurrently within a copyup operation since the object might not
+    exist yet (if not performing a deep-copy).
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f6db9b8027b6978b4d28fcf9f0389c88f9109e75)
+
+commit 487c41e71460610ff3833a6cc6f3c21ab345a2d4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Nov 7 14:36:10 2017 -0500
+
+    librbd: object cacher should re-use read state machine
+    
+    This adds support for sparse-reads and ensures all object reads
+    utilize a single, tested code path.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e79c0cf2009c69681ed4c70f52ade2f5fd570567)
+
+commit b12810056dd74d0b6367123a4a7be6599b93f727
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Nov 7 12:24:44 2017 -0500
+
+    librbd: refactor io::ObjectReadRequest
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 62c3282b875f7345d7b71228d1e35e0cc87d44fa)
+
+commit f7b21684671b0d3df6db2bca9507921181a008e4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Nov 7 14:17:25 2017 -0500
+
+    librbd: default template parameter for C_SparseReadRequest
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ab0e558961ed9b63c00ad1d09ee0f9f79fd62295)
+
+commit 7b20cc9b519707e047bee6e6166daea232b35f5c
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Nov 7 13:42:19 2017 -0500
+
+    librbd: reduce lock scope when assembling read results
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 284feb6a32795c64b1c5ec3b3bf0d959e72dd940)
+
+commit ba3b6099945ae6e4d3781eee13027a6bcea2e61f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Nov 7 13:33:12 2017 -0500
+
+    librbd: ObjectReadRequest does not require buffer extents
+    
+    The ReadResult wrapper requires the buffer extents to know
+    how to properly reconstruct the out buffer.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 35ce1c2d39302011adbf494ada6248495681a436)
+
+commit b77b7218c4087555997872661fb0d08f737406a4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 6 18:35:42 2017 -0500
+
+    librbd: templatize IO object request state machines
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 654a78fa54520370f5466f9d282d4e9f8365ad48)
+
+commit 1fe2fe8d899930feecf6deee01d594691cab3f90
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Nov 10 21:45:13 2017 -0500
+
+    test/librados_test_stub: mocked remainder of IO ops used by librbd
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c4e5613e4a74316ba9667654e887a835b240609a)
+
+commit 36f7a543252761f356aa60acc8d499142fed4966
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Nov 10 16:26:19 2017 -0500
+
+    test/librados_test_stub: simulate set_alloc_hint creating a non-existent file
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4b851ce0fa534ca043aca835057b9d68660971a4)
+
+commit 1581b9e3d3cd5f3c90e3883709d5db947501bd40
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Wed Aug 23 09:46:42 2017 +0200
+
+    test/librados_test_stub: pass snap context to zero op
+    
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit 78ca0fde22076e8b156feb8dfd60aaee316a1895)
+
+commit c727db5bb921bd68cbc7e23c9c49c50c401a8669
+Merge: a3647582de 092e6cf29a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Feb 1 11:15:26 2018 -0500
+
+    Merge pull request #20209 from dillaman/wip-22033-luminous
+    
+    luminous: mgr/dashboard: added iSCSI IOPS/throughput metrics
+    
+    Reviewed-by: Mykola Golub <mgolub@suse.com>
+
+commit a3647582deb09ead251562afa6e92d970473f564
+Merge: 2f887e044e 436d89b9d5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Feb 1 11:14:48 2018 -0500
+
+    Merge pull request #20210 from dillaman/wip-22169-luminous
+    
+    luminous: librbd: set deleted parent pointer to null
+    
+    Reviewed-by: Mykola Golub <mgolub@suse.com>
+
+commit 2f887e044ec10bb018ab8d6284be1a367769fbaf
+Merge: 402b8e0885 c1c43110ef
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Feb 1 11:13:36 2018 -0500
+
+    Merge pull request #20208 from dillaman/wip-21920-luminous
+    
+    luminous: librbd: default to sparse-reads for any IO operation over 64K
+    
+    Reviewed-by: Mykola Golub <mgolub@suse.com>
+
+commit 402b8e08854c3436f92b9c8af01a429caee79da8
+Merge: d2dbebf90b c22e49f321
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Feb 1 11:12:42 2018 -0500
+
+    Merge pull request #20207 from dillaman/wip-21793-luminous
+    
+    luminous: rbd-mirror: primary image should register in remote, non-primary image's journal
+    
+    Reviewed-by: Mykola Golub <mgolub@suse.com>
+
+commit d2dbebf90b52ec4b23bcb1dd7bcd49e49c6414d9
+Merge: eaedbcd0e1 1a0e9498a6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Feb 1 11:12:11 2018 -0500
+
+    Merge pull request #20206 from dillaman/wip-21694-luminous
+    
+    luminous: librbd: journal should ignore -EILSEQ errors from compare-and-write
+    
+    Reviewed-by: Mykola Golub <mgolub@suse.com>
+
+commit eaedbcd0e11ac833c833840a8a132d16f6f39cba
+Merge: 672ef6752b 2691ada5ac
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Feb 1 11:11:39 2018 -0500
+
+    Merge pull request #20205 from dillaman/wip-22577-luminous
+    
+    luminous: qa/workunits/rbd: simplify split-brain test to avoid potential race
+    
+    Reviewed-by: Mykola Golub <mgolub@suse.com>
+
+commit 672ef6752bc1dd754cc74dabb0dd64b0ac5096f6
+Merge: 381c044d77 cfd05be6ac
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Feb 1 11:11:04 2018 -0500
+
+    Merge pull request #20153 from pdvian/wip-22809-luminous
+    
+    luminous: librbd: fix snap create/rm may taking long time
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 381c044d7726983a459103bcd0b29f08ea0944c5
+Merge: b272904f57 37067a6f12
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Feb 1 11:09:15 2018 -0500
+
+    Merge pull request #20135 from dillaman/wip-22806-luminous
+    
+    luminous: librbd: force removal of a snapshot cannot ignore dependent children
+    
+    Reviewed-by: Mykola Golub <mgolub@suse.com>
+
+commit b272904f57eb1daa8769737bed268fccb4c9070f
+Merge: cf8fd69d16 2e69cd92d0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Feb 1 11:00:25 2018 -0500
+
+    Merge pull request #19123 from pdvian/wip-22174-luminous
+    
+    luminous: possible deadlock in various maintenance operations
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit f2f00cac71c5ae1b6b19706b5a77dfa1bbf11ff7
+Author: Gu Zhongyan <guzhongyan@360.cn>
+Date:   Fri Jan 26 16:19:50 2018 +0800
+
+    common/pick_address: wrong prefix_len in pick_iface()
+    With prefix_len initialized as zero, mask in netmask_ipv4/6()
+    will always be zero, so find_ip_in_subnet() always return the
+    first interface.
+    Set prefix_len to the right value to fetch the right interface.
+    
+    Signed-off-by: Gu Zhongyan guzhongyan@360.cn
+    Signed-off-by: yupeng chen chenyupeng-it@360.cn
+    (cherry picked from commit b0d8043a72af22648c058382c46a132264736aaa)
+
+commit 06a1c3d1e4cdb0f267dfc1189534ef747ac38521
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Nov 20 08:29:12 2017 +0800
+
+    mds: don't delay processing completed requests in replay queue
+    
+    Completed requests can also be in replay queue, they need to be processed
+    (treated as lookup request) in clientreplay stage. MDS will be stuck at
+    clientreplay stage if it delays processing requests of this type.
+    
+    the bug was introduced by commit 0afbc033
+    (mds: don't rdlock locks in replica object while auth mds is recovering)
+    
+    Fixes: http://tracker.ceph.com/issues/22163
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 959fad28b2a1b6ad1e5f8597d04ceb9b1761b98a)
+
+commit f454c293d178f108fa0d795d8f2d4487942c71b7
+Author: Theofilos Mouratidis <t.mour@cern.ch>
+Date:   Thu Dec 14 17:43:59 2017 +0100
+
+    ceph-volume: Require lvm2, move to osd package
+    
+    Fixes: http://tracker.ceph.com/issues/22443
+    
+    Signed-off-by: Theofilos Mouratidis <t.mour@cern.ch>
+    (cherry picked from commit 02bc369e052125f50c7d3a7fe9b311215291c84d)
+
+commit 93f13ab24cf2c54a183e90e23a4bddb4fe028da3
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Tue Nov 21 21:32:57 2017 +0100
+
+    build/ops: rpm: fix systemd macros for ceph-volume@.service
+    
+    Fixes: http://tracker.ceph.com/issues/22217
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit d1f2df37dd03d77132e711423aacda46768d1e02)
+
+commit e621e9d7f1447f796b3f80475e38bc7de3f9b6a6
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Dec 8 18:48:21 2017 -0800
+
+    test: ceph_objectstore_tool.py: Perform dump-import
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit c4602c9ac8f7819abc89c8668fbdc4572341ac5e)
+
+commit 24b0482add051f0e0bf5e40490a001540b29e074
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Dec 5 20:57:47 2017 -0800
+
+    ceph-objectstore-tool: Add option "dump-import" to examine an export
+    
+    Fixes: http://tracker.ceph.com/issues/22086
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit a8b8d541ddcc8085a9e7e301948665f3d6c17f0c)
+    
+    Conflicts: (trivial resolution: list of subcommands is slightly different in
+                luminous, compared to master)
+            qa/standalone/special/ceph_objectstore_tool.py
+            src/tools/ceph_objectstore_tool.cc
+
+commit a5282f55df1fcc55ad7ec140bde87001f2145d58
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Dec 5 21:04:16 2017 -0800
+
+    ceph-objectstore-tool: Remove unused map section from log dump
+    
+    Caused by: 44cb6d30640954c66fc03d4ce58b583ffd614951
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b20eeada312c95713cf2ea032fd7437d2b282558)
+
+commit 99d740322e231b3c1ddc8a8c6517baca71c14a01
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Dec 5 18:25:19 2017 -0800
+
+    ceph-objectstore-tool: Fix output of section type number
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 508328784516136fe189581bfbfe5a791db8e9fd)
+
+commit faef2777ffe0c6548616f1691c72470096e96fb4
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Dec 5 17:53:01 2017 -0800
+
+    ceph-objectstore-tool: Improve ceph-objectstore-tool usage output
+    
+    Remove unused argument test-align
+    
+    Caused by: 1c8731c31eb53e17d8f363906438963ab5688fe0
+    
+    Positional arguments are described by usage() because
+    the parsing code creates descriptions like "--object" for the
+    object positional argument.
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit c6896cccc35bc9a159bbe8c3bbb6a0c9ab08fcac)
+
+commit cf8fd69d16b786caa178ca412c509425766856b4
+Merge: 427bad3aae dbf54df879
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Jan 31 14:10:45 2018 -0800
+
+    Merge pull request #17875 from smithfarm/wip-21359-luminous
+    
+    luminous: cephfs: racy is_mounted() checks in libcephfs
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit cf6799e4b5323641502e6da9f546c9a6cee3cb21
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Wed Jan 31 22:57:42 2018 +0100
+
+    client: quit on failed remount during dentry invalidate test
+    
+    Fixes: http://tracker.ceph.com/issues/22269
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 5b2b1d14468c290c56ee6c95ea557c99464e0098)
+    
+    Conflicts:
+      PendingReleaseNotes: trivial conflict
+            src/ceph_fuse.cc
+            src/client/Client.cc: i chosed to pick code from 5b2b1d1 because, to fixthe issue, we need to call _do_remount which was introduced in 5b2b1d1.
+
+commit 427bad3aae9b427fe7308e91e965e57a29964918
+Merge: 9e0eea2be2 d4f5dc2780
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Jan 31 13:21:50 2018 -0800
+
+    Merge pull request #20121 from batrick/i21252p2
+    
+    luminous: mds: fix return value of MDCache::dump_cache
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 5dcd2a59770887a3560d24712904c57f39dd3742
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Oct 18 11:34:53 2017 +0100
+
+    mon: s/mds_blacklist_interval/mon_mds_blacklist_interval/
+    
+    We can rename LEVEL_DEV options at will.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit ed7fc5370c45deb4c80a36e9166e6dd84d4a2b99)
+
+commit d8f9aec74259620bafcde7fa673eea52777d8348
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Oct 17 23:06:07 2017 +0100
+
+    mon: tidy up mds_blacklist_interval
+    
+    Remove the misleading reference to this from the client
+    eviction page, it was never the right option to mention
+    there (my mistake).
+    
+    Demote the option from LEVEL_ADVANCED to LEVEL_DEV as it
+    is hard to imagine a good reason for the user to change it.
+    
+    Set a hard minimum of one hour, to make it harder to
+    corrupt` a system by setting it close to zero.
+    
+    Remove the legacy definition of the field while we're at it.
+    
+    Fixes: http://tracker.ceph.com/issues/21821
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 0d68d3513e03cc2400facf421e08c8a92cb2886a)
+    
+    Conflicts:
+        doc/cephfs/eviction.rst (trivial resolution of conflict resulting from
+            0e43f0f01cd800fee4cd800f1545405b449fa55b being merged to luminous)
+
+commit 9e0eea2be2d135c3c55fb402c64db171a70022a1
+Merge: 6ef48aeb55 075e3b7506
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jan 31 13:27:40 2018 -0500
+
+    Merge pull request #20213 from dillaman/wip-21868-luminous
+    
+    luminous: doc/rbd: tweaks for the LIO iSCSI gateway
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 075e3b75063b0aaf04adb9f451651ab2290c0d15
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Nov 3 10:09:38 2017 -0400
+
+    doc/rbd: tweaks to the Windows iSCSI initiator directions
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 997bb80ba231c4a21237485ac6780d573be992c2)
+
+commit ba26353aae43cf01d9ee66e2d23d8c0ba9a0c9d9
+Author: Ashish Singh <assingh@redhat.com>
+Date:   Fri Oct 20 20:45:35 2017 +0530
+
+    doc: Added CHAP cofiguration instructions for iSCSI
+    
+    Added details to specify the CHAP username and password while
+    discovering/login the iSCSI target.
+    
+    Signed-off-by: Ashish Singh <assingh@redhat.com>
+    (cherry picked from commit 2a5755e0767108aefa6ccfb8a85e5d63e8c04fe8)
+
+commit a3101dd478baedbc701d548c2752bdba0556f4a2
+Author: Mike Christie <mchristi@redhat.com>
+Date:   Mon Jan 22 17:08:55 2018 -0600
+
+    doc: fix ceph-iscsi-config version number
+    
+    Signed-off-by: Mike Christie <mchristi@redhat.com>
+    (cherry picked from commit 3ab71020a5e420a333cb4a6b0be67ade185c1593)
+
+commit 3560c6de1892998cf848f799f312fea258dc8b01
+Author: Mike Christie <mchristi@redhat.com>
+Date:   Sat Jan 20 16:06:42 2018 -0600
+
+    doc: add ceph iscsi test kernel info
+    
+    Signed-off-by: Mike Christie <mchristi@redhat.com>
+    (cherry picked from commit 5c30318660eebc882326d87eec39b4fbe4cc85d7)
+
+commit 4bc150c8280fe94f1c4525d2e58ba55cd1229e82
+Author: Mike Christie <mchristi@redhat.com>
+Date:   Tue Jan 16 12:46:09 2018 -0600
+
+    doc: add rbd pool requirement for gwcli
+    
+    Signed-off-by: Mike Christie <mchristi@redhat.com>
+    (cherry picked from commit b56d994f3addcd9c6a7848dec9be93568cdb2d0e)
+
+commit 30c93cdb409e64baa9f11f4e23717c5529f09b20
+Author: Mike Christie <mchristi@redhat.com>
+Date:   Mon Jan 15 12:59:37 2018 -0600
+
+    doc: add daemon-reload for rbd-target-api setup
+    
+    Signed-off-by: Mike Christie <mchristi@redhat.com>
+    (cherry picked from commit 4ca0d508bb9b68c791abc5799d0195f4baace455)
+
+commit 46bf788c310031739016cf07a5be8c23170e6f90
+Author: Mike Christie <mchristi@redhat.com>
+Date:   Tue Jan 16 12:04:28 2018 -0600
+
+    doc: add gwcli manual install instructions
+    
+    Signed-off-by: Mike Christie <mchristi@redhat.com>
+    (cherry picked from commit f45aafed1c7a5393e712f9fbfd7479fc444ad3d8)
+
+commit efdb585141bfebfaab38dc0049905c6c3a5baae1
+Author: Mike Christie <mchristi@redhat.com>
+Date:   Thu Jan 11 14:54:06 2018 -0600
+
+    doc: separate gwcli install from post install setup steps
+    
+    Signed-off-by: Mike Christie <mchristi@redhat.com>
+    (cherry picked from commit f80f083d0a982d7e89c77e3a6f8d71c2871a8f2c)
+
+commit f399a3399cb210ef76fa17c8a31178a2327aaa25
+Author: Mike Christie <mchristi@redhat.com>
+Date:   Thu Jan 11 14:53:16 2018 -0600
+
+    doc: add note to clarify trusted_ip_list use
+    
+    Signed-off-by: Mike Christie <mchristi@redhat.com>
+    (cherry picked from commit 3cc367dc6ebca18e884d7c75d2cd1fa7907b7139)
+
+commit 2eda81b9245c340be76decd4728ccb5a3d950ac9
+Author: Mike Christie <mchristi@redhat.com>
+Date:   Wed Jan 10 22:14:18 2018 -0600
+
+    doc: update esx instructions
+    
+    Signed-off-by: Mike Christie <mchristi@redhat.com>
+    (cherry picked from commit 3806f9750d29c6d930a7c8a003902e79b99766f6)
+    
+    Conflicts:
+            doc/rbd/iscsi-initiator-esx.rst: trivial resolution
+
+commit f4b1f3974b26b52d0cfd54f6c1237bccf5cb9921
+Author: Mike Christie <mchristi@redhat.com>
+Date:   Wed Jan 10 20:33:03 2018 -0600
+
+    doc: add warning about SCSI PGRs/reservations
+    
+    Signed-off-by: Mike Christie <mchristi@redhat.com>
+    (cherry picked from commit 62661a2e23ce98432f3c037fa9a8155a633ee182)
+
+commit 723d5fabfabfff2ad517f67a8fd1bdf1c2380bfc
+Author: Mike Christie <mchristi@redhat.com>
+Date:   Wed Jan 10 18:51:51 2018 -0600
+
+    doc: add esx web based client images
+    
+    Signed-off-by: Mike Christie <mchristi@redhat.com>
+    (cherry picked from commit 78e12135eeb5829cad65cebd3c2be9df4d582b1f)
+
+commit 0f937ed3d67103e5874bd8862d36ab53177f4677
+Author: Mike Christie <mchristi@redhat.com>
+Date:   Thu Jan 11 19:36:06 2018 -0600
+
+    doc: remove windows based vsphere client images
+    
+    Signed-off-by: Mike Christie <mchristi@redhat.com>
+    (cherry picked from commit 426b562fdc1b373b72de2d69c4851be22ec12008)
+
+commit f8ab982b72f2a8f27bc034945685dc55ef9222bf
+Author: Mike Christie <mchristi@redhat.com>
+Date:   Thu Jan 11 19:35:22 2018 -0600
+
+    doc: fix gwcli examples
+    
+    Signed-off-by: Mike Christie <mchristi@redhat.com>
+    (cherry picked from commit 480f6a82a2b9dcd0179951ea16432d8d32f2c96d)
+
+commit 335fa90e0db06a2b6b74ac756f0d935ea4e67648
+Author: Mike Christie <mchristi@redhat.com>
+Date:   Wed Jan 10 16:38:13 2018 -0600
+
+    doc: Update ceph iscsi kernel/distro requirements.
+    
+    Signed-off-by: Mike Christie <mchristi@redhat.com>
+    (cherry picked from commit f436e993d359dfdad26b7d569d555a3b3a5b798d)
+
+commit 0363593523c1b95233a23529ae1a8910088f0fd2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Oct 11 12:02:55 2017 -0400
+
+    doc/rbd: tweaks for the LIO iSCSI gateway
+    
+    Fixes: http://tracker.ceph.com/issues/21763
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4f227e108d40271d5d6b2b1e88659d86cbc6c914)
+
+commit 092e6cf29a33d49ed1abddb2558695bd6de4ae5b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 31 11:05:47 2017 -0400
+
+    mgr/dashboard: added iSCSI IOPS/throughput metrics
+    
+    Fixes: http://tracker.ceph.com/issues/21391
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e55ef24c9140c23c832a2205a2154e68452545a3)
+
+commit 451ae9ed812481aedb372ddb97d0e01e1c711ca7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 31 10:41:36 2017 -0400
+
+    mgr/dashboard: include A/O start relative timestamp for iSCSI
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit e62f186613a1de64a155a852b21ec9033c6e73ea)
+
+commit 076ee1a247a6a65e88f2b62b1742be3197f81387
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 17 15:40:55 2017 -0400
+
+    librbd: track image open and lock acquire time via perf counter
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8583270fd2d0930c1fccd2285378de0844b1d928)
+
+commit 30ff1fbec70cb7140ffc66a4cb25060795550fea
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 17 15:21:42 2017 -0400
+
+    librbd: export read and writes performance counters for daemons
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7a9d10a4e61ed168678f38f6a224859e0df44099)
+
+commit fad78966d49f01b6c3d3fa3aba20f314d1e758b6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 17 15:49:22 2017 -0400
+
+    librbd: moved performance counter enums to common location
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 2224e5ca1dacef1e3f8ec4854f08b4abe86f5d75)
+    
+    Conflicts:
+            src/librbd/internal.h: trivial resolution
+
+commit 436d89b9d538b7521993b0fd6d0aeca253827775
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sat Nov 18 08:30:27 2017 -0500
+
+    librbd: set deleted parent pointer to null
+    
+    Fixes: http://tracker.ceph.com/issues/22158
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7b53256bb825383fc604a96d71bcd51c36668cc4)
+
+commit b8b3eee73d79345e28404281040f7bdd3c36d171
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Nov 23 12:56:31 2017 +0800
+
+    qa: silence SLOW_OPS,PENDING_CREATING_PGS warnings
+    
+    this is an intermediate step to deprecate REQUEST_SLOW warnings.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 4a1f2a5c78cc77eafe99f087d10c0d3daa7fca7b)
+    
+    Conflicts:
+            qa/suites/rados/singleton/all/thrash-eio.yaml
+            qa/suites/smoke/basic/tasks/mon_thrash.yaml
+            qa/suites/smoke/basic/tasks/rados_bench.yaml
+            qa/suites/smoke/basic/tasks/rados_cache_snaps.yaml
+            qa/suites/smoke/basic/tasks/rados_ec_snaps.yaml
+            qa/suites/smoke/basic/tasks/rbd_fsx.yaml: do not cherry-pick the
+    SLOW_OPS related changes.
+
+commit 030d7e8c840386c366ca794ec86e14ddab44d6d5
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Nov 1 23:17:57 2017 +0800
+
+    mgr: summarize osd metrics in MMgrReport and sent it to mon
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 7e7978732d20c506eca581f2b153ede70ceefa3d)
+    
+    Conflicts:
+            src/mgr/OSDHealthMetricCollector.cc: only the change reporting
+    pending-creating-pgs is cherry-picked.
+
+commit 687967459b0a929fb33cc689401475c6354ab391
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Nov 9 21:37:11 2017 +0800
+
+    mon/health_check: s/std::list/list/
+    
+    so it's self-contained, and .cc files including it are not obliged to `using namespace std`.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 416082c1ce6c357943f4a6b9e081a3211b1110c0)
+
+commit bb5ee06baac34aff59578575e14fe2a6bb6ae0ab
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Oct 23 12:29:31 2017 +0800
+
+    osd: send health-checks to mgr
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit f4b74125e44fe78154fb377fa06fc08b3325859d)
+    
+    Conflicts:
+            src/osd/OSD.cc
+            src/osd/OSD.h: only the changes related to reporting
+    pending-creating-pgs are cherry-picked. because we want to minimize the
+    impact to luminous. and to remove the slow ops from cluster log is not
+    in the scope of this backport. also, with_unique_lock was introduced
+    after luminous is branched, so use the plain lock_guard<> instead.
+            src/osd/OSDHealthMetric.h: because denc.h does not support
+    denc of enum types in luminous. so we need to case from/to osd_metric
+    when necessary.
+
+commit 2472ec3b956b104717fa23ca5d0dc29cfbc43955
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 17 14:57:08 2017 -0400
+
+    librbd: track the child of an image in-memory
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ce2ae1d7c0f8818c86cbc746921e8eb6b0f89913)
+
+commit c22e49f3216c44bc26e029b05602a4b06e92c49d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 27 16:02:47 2017 -0400
+
+    rbd-mirror: journal debug message should not be logged as error
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 625222edbce66c98b86fc87fa3bd12ea40db17a2)
+
+commit 08ed96a7ddc1e587854a7da7edd6c350cc0a1fe5
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 27 16:02:20 2017 -0400
+
+    rbd-mirror: avoid attempting to decode an empty client data buffer
+    
+    Fixes: http://tracker.ceph.com/issues/21961
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1e1d15eb0c14a6be82a6d75ab0088a6f0f4f8125)
+
+commit c1c43110ef683b7a1f8dbb7136ad44b31943fe5a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 19 13:52:56 2017 -0400
+
+    librbd: default to sparse-reads for any IO operation over 64K
+    
+    Testing BlueStore against both HDDs and OSDs with fully allocated
+    and sparse-allocated objects shows a performance improvement with
+    sparse-read between 32K and 64K.
+    
+    Fixes: http://tracker.ceph.com/issues/21849
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 251658471eab6d8cd968d678922bab437f72a9c7)
+
+commit cec71d94ffd0b86caee5524d2346b1d66f19869e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 19 14:32:05 2017 -0400
+
+    librbd: templatize io::CopyupRequest and io::ObjectRequest
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit dd5452004047c31e0f3eb813d5dd0b50bd68e76f)
+
+commit ba74ae952cbc032689833ab60bb8c59df99a675e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 5 16:38:32 2017 -0400
+
+    rbd-mirror: removed duplicate client registration logic
+    
+    The client will now be registered before the bootstrap state machine
+    is invoked.
+    
+    Fixes: http://tracker.ceph.com/issues/21561
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d774d4702fbb9ff0b88797392b6673964d6b8c7c)
+
+commit f7d43cb0dc690dd0210090276514eaf257aa22d4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 5 15:56:19 2017 -0400
+
+    rbd-mirror: local primary images should still register to remote images
+    
+    Fixes: http://tracker.ceph.com/issues/21561
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d74b95dc10a5073ee7980583d94628ded7850ebb)
+
+commit 1a0e9498a6bd32987051fb02b3264bdb212a6310
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 3 15:42:50 2017 -0400
+
+    librbd: journal should ignore -EILSEQ errors from compare-and-write
+    
+    Fixes: http://tracker.ceph.com/issues/21628
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c7a9b045808ba741f6aadd2c2e65bd84135347c4)
+
+commit 2691ada5ac3179d874381b44c1482c3dc85cce31
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Dec 19 16:29:18 2017 -0500
+
+    qa/workunits/rbd: simplify split-brain test to avoid potential race
+    
+    Fixes: http://tracker.ceph.com/issues/22485
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 8c2ee6dd0f4c77b9fbbc24a673841b04d3e175d6)
+    
+    Conflicts:
+            qa/workunits/rbd/rbd_mirror.sh: trivial resolution
+
+commit 69e8a0ab28c53303030423ac795577564827dbaa
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Sun Oct 29 19:07:54 2017 +0800
+
+    osd: remember is_primary when adding pending creating pgs
+    
+    so mgr can avoid calculating the same PG from different OSDs.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 486515ae884b91571a9066296680fff78f93f66d)
+
+commit 530283690eb8b5016ad5ed92f9f44cdcb977e847
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Oct 24 22:41:30 2017 +0800
+
+    mgr/MgrClient: send health_checks using MMgrReport
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 6fed1d21be1adf1464b5834787c95fae205a91b5)
+
+commit 6ef48aeb555790a6d5a8ef025a7ecf84dfe07de3
+Merge: 0489bddb5d bb3d091b46
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 31 12:09:08 2018 +0000
+
+    Merge pull request #19555 from shinobu-x/wip-22455-luminous
+    
+     luminous: balancer crush-compat sends "foo" command
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 0489bddb5db768d4034bbcfdbd75151cc1df8278
+Merge: 8bbf1653ff f2a202775f
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 31 12:06:59 2018 +0000
+
+    Merge pull request #20156 from pdvian/wip-22811-luminous
+    
+    luminous: mon: fix mgr using auth_client_required policy
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 8bbf1653ff4fabdac6bc8e79c9913fed5c2727ab
+Merge: 1e3229418a 0aa3c5c577
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 31 12:01:45 2018 +0000
+
+    Merge pull request #19481 from jan--f/wip-22421-luminous
+    
+    luminous: mon: reenable timer to send digest when paxos is temporarily inactive
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 1e3229418aa47cc1648121134e18d8f491c6cdac
+Merge: 2d64d41a39 c5cb0cab9f
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 31 11:59:04 2018 +0000
+
+    Merge pull request #19553 from shinobu-x/wip-22453-luminous
+    
+     luminous: mgr/balancer/upmap_max_iterations must be cast to integer
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 2d64d41a394b8dea69436c9b9c06d2a8aa3466f1
+Merge: d351505965 40954a9c27
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 31 11:49:41 2018 +0000
+
+    Merge pull request #19624 from shinobu-x/wip-22496-luminous
+    
+     luminous: KeyError: ('name',) in balancer rm
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit d351505965654e1f8c4525ca7e9cfebd308828fc
+Merge: ea85e979b2 b5892a13b9
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 31 11:48:50 2018 +0000
+
+    Merge pull request #19834 from pdvian/wip-22585-luminous
+    
+    luminous: pybind/mgr/prometheus: fix metric type undef -> untyped
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit ea85e979b239127ee80373949dfa88a9c317d03f
+Merge: 81712618b4 1b2b589a12
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 31 11:47:05 2018 +0000
+
+    Merge pull request #20084 from pdvian/wip-22760-luminous
+    
+    luminous: mgr: prometheus: added osd commit/apply latency metrics (#22718).
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 81712618b4ce36511674bdfb0720b4d7bb454c06
+Merge: fab5aa92cb bb21f23731
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 31 11:46:03 2018 +0000
+
+    Merge pull request #20089 from pdvian/wip-22768-luminous
+    
+    luminous: mgr: disconnect unregistered service daemon when report received
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit fab5aa92cb2883b2aab88ffec769a57cc36fe115
+Merge: 7eb2423bef 91bfcc73f6
+Author: John Spray <jspray@redhat.com>
+Date:   Wed Jan 31 11:42:58 2018 +0000
+
+    Merge pull request #20182 from jcsp/wip-22831
+    
+    luminous: pybind/mgr/dashboard: fix reverse proxy support
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 7eb2423befadb1bd04ba68ebdbb7a4734daff9e2
+Merge: 3512fc40cb 52e59854c2
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:47:21 2018 -0800
+
+    Merge pull request #17869 from smithfarm/wip-21479-luminous
+    
+    luminous: mgr: Services reported with blank hostname
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 3512fc40cb75f97ca66df1682c5873c5110a3b78
+Merge: 25cf00a740 4db6b35548
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:46:34 2018 -0800
+
+    Merge pull request #18832 from shinobu-x/wip-22075-luminous
+    
+    luminous: mgr tests don't indicate failure if exception thrown from serve()
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 25cf00a740573d329a35cd050b2723070d8e0bad
+Merge: ef1038691d eeb12253d4
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:46:04 2018 -0800
+
+    Merge pull request #18842 from shinobu-x/wip-21863-luminous
+    
+    luminous: ceph-conf: dump parsed config in plain text or as json
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit ef1038691da028df20b4e22a6a628bf3e6a72586
+Merge: a3091bdd20 de1d25c2dd
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:45:40 2018 -0800
+
+    Merge pull request #19002 from tchaikov/wip-pr-18976-luminous
+    
+    luminous: udev: Fix typo in udev OSD rules file
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit a3091bdd20a64c02dc423052015e46d42240adf3
+Merge: 5563a65058 1bdd81e095
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:44:46 2018 -0800
+
+    Merge pull request #19023 from pdvian/wip-22164-luminous
+    
+    luminous: cluster [ERR] Unhandled exception from module 'balancer' while running on mgr.x: 'NoneType' object has no attribute 'iteritems'" in cluster log
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 5563a650587375640ecefca6643570d5366ce420
+Merge: b0b09df8aa c21e1c7831
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:44:16 2018 -0800
+
+    Merge pull request #19031 from pdvian/wip-22167-luminous
+    
+    luminous: Various odd clog messages for mons
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit b0b09df8aa5150fcad9bfd2f200bd74b71f5faaf
+Merge: a35101135f 6f983c14d9
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:42:52 2018 -0800
+
+    Merge pull request #19039 from jan--f/wip-22199-luminous
+    
+    luminous: tools/crushtool: skip device id if no name exists
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit a35101135f6fd8bb83e4a44baf7abe9e2c5b83d6
+Merge: 5f175b5bea 488228e919
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:41:50 2018 -0800
+
+    Merge pull request #19046 from shinobu-x/wip-22189-luminous
+    
+    luminous: osdc/Objecter: objecter op_send_bytes perf counter always 0
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 5f175b5bea82c848a7c2bc9aef4860e5e8672310
+Merge: a1636c348e d5f2e16600
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:41:25 2018 -0800
+
+    Merge pull request #19059 from pdvian/wip-22176-luminous
+    
+    luminous: osd: subscribe osdmaps if any pending pgs
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit a1636c348ebb1e8ec8a79e60cc22193701da84c2
+Merge: 5c79793917 540f425f1a
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:40:44 2018 -0800
+
+    Merge pull request #19083 from smithfarm/wip-22213-luminous
+    
+    luminous: On pg repair the primary is not favored as was intended
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+
+commit 5c7979391775529b5650afe18b25315bfc1e2c4a
+Merge: 525fe577b8 ed24c8ce8f
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:40:04 2018 -0800
+
+    Merge pull request #19084 from shinobu-x/wip-22216-luminous
+    
+     luminous: "osd status" command exception if OSD not in pgmap stats
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 525fe577b803552eae568d765f34b4e9c98ea8c7
+Merge: 91d1a7bed8 8f67e4553a
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:39:36 2018 -0800
+
+    Merge pull request #19118 from tangwenjun3/wip-backport-19030
+    
+    luminous: os/bluestore: fix the allocate in bluefs
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: xie xingguo <xie.xingguo@zte.com.cn>
+    Reviewed-by: Shinobu Kinjo <skinjo@redhat.com>
+
+commit 91d1a7bed8254f54290ca089ccc39a41a5da962e
+Merge: 2c2e60681d bdb9d385a5
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:33:43 2018 -0800
+
+    Merge pull request #19133 from smithfarm/wip-22194-luminous
+    
+    luminous: Default kernel.pid_max is easily exceeded during recovery on high OSD-count system
+    
+    Reviewed-by: David Disseldorp <ddiss@suse.de>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 2c2e60681d93b0aa3b1b8d8da8d1f02b12bdc8ed
+Merge: 3937ae421a 7ffe8c41d3
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:30:53 2018 -0800
+
+    Merge pull request #19631 from shinobu-x/wip-22509-luminous
+    
+     luminous: osd: "sudo cp /var/lib/ceph/osd/ceph-0/fsid ..." fails
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 3937ae421a47d781af11860788f999fcaaeeade4
+Merge: 74064cb9e4 0a347506af
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:30:11 2018 -0800
+
+    Merge pull request #19839 from tchaikov/wip-22558-luminous
+    
+    luminous: rpm: adjust ceph-{osdomap,kvstore,monstore}-tool feature move
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 74064cb9e4d2731919374a28c030cda034d24907
+Merge: f5dcf47a10 c86675dadc
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:28:05 2018 -0800
+
+    Merge pull request #18838 from shinobu-x/wip-21973-luminous
+    
+    luminous: [test] UpdateFeatures RPC message should be included in test_notify.py
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit f5dcf47a109c52bb4db7385d9ae6fd9ae53a5102
+Merge: 40ca9d22de 8cb21e6ea6
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:27:34 2018 -0800
+
+    Merge pull request #18840 from shinobu-x/wip-21970-luminous
+    
+    luminous: [journal] tags are not being expired if no other clients are registered
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 40ca9d22de212ac111cd1d744106e21aa13d5615
+Merge: f7778d4700 74a6592d7f
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:26:21 2018 -0800
+
+    Merge pull request #18949 from jcsp/wip-luminous-rgw-config-desc
+    
+    luminous: options.cc: document rgw config options
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit f7778d4700e654c62b49158b30c541d6bf879523
+Merge: c421d533a2 2d9aafe1bc
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 30 16:25:18 2018 -0800
+
+    Merge pull request #19051 from shinobu-x/wip-22185-luminous
+    
+    luminous: rbd: abort in listing mapped nbd devices when running in a container
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 91bfcc73f68a869703eb2f6a5ee5f5962b9b727b
+Author: Nick Erdmann <n@nirf.de>
+Date:   Mon Jan 22 12:33:32 2018 +0000
+
+    pybind/mgr/dashboard: fix reverse proxy support
+    
+    This fixes http redirection for reverse http proxies
+    
+    Fixes: http://tracker.ceph.com/issues/22557
+    Signed-off-by: Nick Erdmann <n@nirf.de>
+    (cherry picked from commit 95e1963cb5327f0699081c4c4b0b355d109ff0e3)
+
+commit f0cae0bffced632fcc9f0dfec5d1c85424d9a42c
+Author: Nick Erdmann <n@nirf.de>
+Date:   Fri Jan 26 13:40:27 2018 +0000
+
+    pybind/mgr: fix typo
+    
+    Signed-off-by: Nick Erdmann <n@nirf.de>
+    (cherry picked from commit fdd64bb2dac33085b47c737032a525d89319c866)
+
+commit ba2c6015ebe0d2fd55e38f1a1472810551e8d0c1
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Jan 15 11:49:08 2018 +0800
+
+    pybind/mgr/mgr_module: add default param for MgrStandbyModule.get_config()
+    
+    this matches its counterpart of MgrModule.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit bc7de8b124c32a2ee6910a3bac6a06f1c566a2ac)
+
+commit c421d533a258c8950205f59a032a3bdf9fd298fb
+Merge: d146d83076 660abda211
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Tue Jan 30 09:29:15 2018 -0500
+
+    Merge pull request #20062 from cbodley/wip-luminous-rgw-bilog-trim
+    
+    luminous: rgw multisite: automated trimming for bucket index logs
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit 2e69cd92d07ca7f7957f67d40d46a04b94ccb0b4
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Nov 14 12:07:36 2017 -0500
+
+    librbd: added missing locks for snap rollback and rename
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0ccd26f9989b892be2ab620bcf7f4fbf14980e8e)
+
+commit 7df6f0602cef58ee9185814ca3363d5448bb739d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 13 13:28:06 2017 -0500
+
+    librbd: possible deadlock with synchronous maintenance operations
+    
+    Fixes: http://tracker.ceph.com/issues/22120
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 90b7ecd8a8aac9a5c282d44004752ade0c195331)
+
+commit d146d830762414934375afe257c69a25f4e533f3
+Merge: 20cfb9bda3 67fbd3830f
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon Jan 29 10:22:14 2018 -0600
+
+    Merge pull request #19848 from cbodley/wip-luminous-boost-166
+    
+    luminous: backport beast frontend and boost 1.66 update
+
+commit 20cfb9bda35dea6ccd83f526a88a38edef210412
+Merge: 9981b51339 7dabbe9131
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon Jan 29 10:21:07 2018 -0600
+
+    Merge pull request #19500 from shinobu-x/wip-22401-luminous
+    
+    luminous: rgw: make HTTP dechunking compatible with Amazon S3
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 9981b513395e261b472ff516b17df419cfec714d
+Merge: abeb3c56c0 fc791b980d
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jan 29 08:12:39 2018 -0800
+
+    Merge pull request #17816 from smithfarm/wip-hadoop-s3a
+    
+    luminous: tests: qa/rgw: hadoop-s3a suite targets centos_latest
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit abeb3c56c074f9024565ed8d48dc0d89b4337084
+Merge: 8fe88c45e3 0d18c24fce
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jan 29 08:10:27 2018 -0800
+
+    Merge pull request #18972 from linuxbox2/luminous-22084
+    
+    luminous: rgw: Fix swift object expiry not deleting objects
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 8fe88c45e3eb9d985ee20cf49679f5db08faf3db
+Merge: d55eda8267 fe5c8b34a7
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jan 29 08:09:56 2018 -0800
+
+    Merge pull request #19086 from shinobu-x/wip-22210-luminous
+    
+    luminous: radosgw-admin zonegroup get and zone get should return defaults when there is no realm
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit d55eda826741653537f64c6cccd7464d7b82efa2
+Merge: 7e84974d2d 14434beda6
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jan 29 08:09:25 2018 -0800
+
+    Merge pull request #19088 from pdvian/wip-22177-luminous
+    
+    luminous: rgw: check going_down() when lifecycle processing
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 7e84974d2d5b653776eb89c623130ef651108a0d
+Merge: 0f1f98ad44 e3bb21529b
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jan 29 08:08:15 2018 -0800
+
+    Merge pull request #19636 from shinobu-x/wip-22506-luminous
+    
+    luminous: rgw usage trim only trims a few entries
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 0f1f98ad449b73568077513196f0d8dc43a9b07f
+Merge: d51dbcaf03 ad0ea9fa53
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jan 29 08:06:43 2018 -0800
+
+    Merge pull request #19784 from linuxbox2/luminous-20201
+    
+    luminous: RGW: S3 POST policy should not require Content-Type
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+
+commit d51dbcaf03a8c8a6c7a8330fb2f23a03d75a411a
+Merge: 1708c17f28 84fff7aca5
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jan 29 08:05:58 2018 -0800
+
+    Merge pull request #19799 from smithfarm/wip-22581-luminous
+    
+    luminous: rgw: multisite: 'radosgw-admin sync error list' contains temporary EBUSY errors
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 1708c17f28dfc57462f9b05c1c9b6c73312193e5
+Merge: 7c3e468476 2adc133b86
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jan 29 08:03:10 2018 -0800
+
+    Merge pull request #19867 from smithfarm/wip-22591-luminous
+    
+    luminous: rgw: refuses upload when Content-Type missing from POST policy
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 7c3e46847667ad104b3b718b7f4460b2fc9f763f
+Merge: 915a8e1454 4318ac5e3f
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jan 29 08:02:23 2018 -0800
+
+    Merge pull request #20051 from ceph/revert-19538-wip-22434-luminous
+    
+    Revert "luminous: rgw: user stats increased after bucket reshard"
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 915a8e1454982e6b996e6757d13523fb4083cd2d
+Merge: 9882489f8a e0b8512419
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jan 29 08:01:56 2018 -0800
+
+    Merge pull request #20056 from linuxbox2/luminous-rgw-latchfix
+    
+    luminous: rgw_file: alternate fix deadlock on lru eviction
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 9882489f8ae81104ae621cf9633f61b469640a8e
+Merge: 440fdfa438 35d2baf47b
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jan 29 07:57:29 2018 -0800
+
+    Merge pull request #20073 from smithfarm/wip-22767-luminous
+    
+    luminous: rgw: librgw: fix shutdown error with resources uncleaned
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 440fdfa438bf881a3e5e2c8d4379284dc70eb673
+Merge: f83a246ba5 e3fbaa92ce
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jan 29 07:56:53 2018 -0800
+
+    Merge pull request #20075 from smithfarm/wip-22773-luminous
+    
+    luminous: rgw: file deadlock on lru evicting
+    
+    Reviewed-by: Matt Benjamin <mbenjami@redhat.com>
+
+commit f83a246ba550a7725438823c296cc18e56dea7ff
+Merge: e50835f80c b219b854b7
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Mon Jan 29 07:56:04 2018 -0800
+
+    Merge pull request #20107 from dreamhost/wip-22792
+    
+    luminous: rgw: When a system object is created exclusively, do not distribute the
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit a26b4ae76521e4cd12a65e69c735676454ce6e4c
+Author: Konstantin Shalygin <k0ste@k0ste.ru>
+Date:   Fri Jan 26 18:31:20 2018 +0700
+
+    mgr: balancer: fixed mistype "AttributeError: 'Logger' object has no attribute 'err'"
+    
+    Signed-off-by: Konstantin Shalygin <k0ste@k0ste.ru>
+    (cherry picked from commit 2062e84c7a33fc5170740f2d60d07ddf62085457)
+
+commit f2a202775fdf1132891221fa58700b276da7429b
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Jan 23 07:43:12 2018 -0500
+
+    mgr: apply auth_service_required to client conns
+    
+    Previously was using auth_cluster_required for all
+    connections, which meant that if someone had
+    disabled client cephx, they'd get BADAUTHORIZER
+    from their CLI when it tried to load mgr
+    command descriptions.
+    
+    Disabling cephx on the admin CLI is odd, but the mon
+    tolerates it so the mgr should too.
+    
+    Fixes: https://tracker.ceph.com/issues/22096
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 86ee30c33a06a13e6bb360b80dd1e41ba928147b)
+
+commit 2852a5f533a5cf044517fa288e8076ac493b1da0
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Jan 22 13:42:20 2018 +0000
+
+    mon: fix mgr using auth_client_required policy
+    
+    This caused mgr daemons to fail to authenticate
+    when auth_client_required was set to something
+    different to auth_cluster_required.
+    
+    Fixes: https://tracker.ceph.com/issues/22096
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 1e06fe003e50d360b5c5ce9824bba044b6406ec5)
+
+commit cfd05be6ace8fb3145068c464bc0a73d487836a1
+Author: Song Shun <song.shun3@zte.com.cn>
+Date:   Wed Jan 17 09:32:45 2018 +0800
+
+    librbd: fix snap create/rm may taking long time
+    
+      fix snap create/rm may taking long time
+      http://tracker.ceph.com/issues/22716
+    
+    Signed-off-by: Song Shun <song.shun3@zte.com.cn>
+    (cherry picked from commit d04ed348a1b20e5ea5bedada2462cb41f0f1d85a)
+    
+    Conflicts:
+            test_mock_SnapshotRemoveRequest.cc: define om as a ObjectMap<ImageCtx> obj
+
+commit 8950736b8e3c2837f201b4eba4fdebcc97fbd9be
+Author: Chang Liu <liuchang0812@gmail.com>
+Date:   Tue Jan 23 15:39:56 2018 +0800
+
+    mon: use pool_sum as param name in PGMapDigest::recovery_summary
+    
+    the param name of delta_sum of PGMapDigest::recovery_summary() is misleading too.
+    delta_sum fits the recovery_rate_summary() well, but not recovery_summary().
+    
+    Signed-off-by: Chang Liu <liuchang0812@gmail.com>
+    (cherry picked from commit 73ff298b898ca76e3f9b615449b4e4c0f2d0c017)
+
+commit e94900c7caa2bd920684073704bc688249bc0591
+Author: Chang Liu <liuchang0812@gmail.com>
+Date:   Thu Jan 18 22:50:25 2018 +0800
+
+    mon: do not use per_pool_sum_delta to show recovery summary
+    
+    Fixes: http://tracker.ceph.com/issues/22727
+    
+    Signed-off-by: Chang Liu <liuchang0812@gmail.com>
+    (cherry picked from commit 5981ddc1992190431743ce1f0b834fb8899b6811)
+
+commit 540f425f1a86dabf746ac4bdd14c54b4ec76f664
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Oct 20 19:50:32 2017 -0700
+
+    osd: Scrub of object with only object error(s) are not logged
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 1ad05b1068ddd5d3312af45af1a60587200ddcd7)
+
+commit 4c2a8f1dd08882eca83512152c8570f497081229
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Oct 20 17:34:49 2017 -0700
+
+    osd: Primary shard needs to be at the front to be preferred
+    
+    Caused by: cd0d8b0714d8684cf61b4650e170027ef46f489b
+    
+    Fixes: http://tracker.ceph.com/issues/21907
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b7d2f27da1a68046a688749f95093a88e6362b28)
+
+commit 17c64595a2f3481ab60ffe6e453a6dc879f1e01e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Oct 19 12:56:58 2017 -0700
+
+    ceph-osdomap-tool: Add compact comand option
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b1f397f4fc3313ee0325722156759bfe32286c6b)
+
+commit 152272ce49cf2b32e4b3dbba5aa28019e9a183e0
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 18 18:07:41 2017 -0700
+
+    osd: Fixes for mark_unfound_lost
+    
+    Let recovery requeue operations and release backoff
+    For "delete" mark object context, if present, to non-existent
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0c106f1a46815bb769e7a503b1adb265480b6779)
+
+commit 339e99005691ebdcdac45ba249add4ce4d6a8125
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 18 16:07:16 2017 -0700
+
+    ceph-objectstore-tool: Fix debug output
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 090c773fc9b7f3316ed88edd24b85a703224ddc5)
+
+commit f23035fb17fb816c1b0f41ccf4072a14f50b1fa1
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 18 16:04:59 2017 -0700
+
+    test: Remove bogus check in ceph_objectstore_tool.py
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit f918b1fac1f3cdb4cf316fb4c7afc90de49ce06e)
+
+commit dbf54df87941f538e73b672131018ce4baba338c
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Fri Aug 25 08:31:47 2017 -0400
+
+    client: reset unmounting flag to false when starting a new mount
+    
+    Once the "unmounting" flag goes to true, it never flips back to false.
+    We don't have a lot of checks for "unmounting" in the code, but they're
+    in some subtle places today and I plan to add more later.
+    
+    It's not clear to me whether it's possible (or advisable) to reuse a
+    Client that has previously been unmounted. It most certainly won't work
+    as expected today, and the checks I'll be adding will break it for sure.
+    
+    Make sure we clear the unmounting flag when we go to re-mount the
+    Client.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 3bc61f6c23a9ada5e2250f66e4df04a66dcfbacf)
+
+commit 20164df443795c4bde6bfdbb9eeda34e915c7557
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Fri Aug 25 08:31:47 2017 -0400
+
+    client: rework Client::get_local_osd() return codes
+    
+    It currently returns -1 when there isn't one, but the C wrapper can
+    also return -ENOTCONN if the cmount isn't mounted. Change it to return
+    -ENXIO (No such device or address) in the case of "no local osd".
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 4285b970cb57c88deae0317be24afcf21a95fdc0)
+
+commit 12e93809aa1adbb7c3b112b4ba6b3a4374429163
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Fri Aug 25 08:31:47 2017 -0400
+
+    client: remove misleading comment in get_cap_ref
+    
+    That may have been true at one time, but it's not always a single bit now.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 93f8cc1f71ffdc8dddca7efb56d82548a66ee3fe)
+
+commit e50835f80c63563a419ae47b05f72885e61e528e
+Merge: bba828b2d7 4efada35fd
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Jan 26 08:15:17 2018 -0500
+
+    Merge pull request #20136 from dillaman/wip-22815-luminous
+    
+    luminous: qa/workunits/rbd: switch devstack to pike release
+    
+    Reviewed-by: Mykola Golub <mgolub@suse.com>
+
+commit 4efada35fd6b2d9b24cd89ac1dc3aeb7c823cfbc
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jan 24 11:28:19 2018 -0500
+
+    qa/workunits/rbd: switch devstack to pike release
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 7ff1e0bc2193acd009a74db81cf8e8cc8639ad12)
+
+commit 37067a6f1275ea62d40f398c7b5d8e76f08f527a
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jan 24 14:40:56 2018 -0500
+
+    librbd: force removal of a snapshot cannot ignore dependent children
+    
+    Fixes: http://tracker.ceph.com/issues/22791
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit fcc58ecfeba6bbc72588e69dba35779f94d28ba5)
+
+commit bba828b2d761b1459b32588accb6838ceeb0cb40
+Merge: f473d3e444 4dda1b6ead
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 12:03:21 2018 -0800
+
+    Merge pull request #19105 from batrick/i22228
+    
+    luminous: client: anchor Inode while trimming caps
+    
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+    Reviewed-by: Yuri Weinstein <yweins@redhat.com>
+
+commit f473d3e4446110f219f69a577a4e5392ec4b422d
+Merge: 646a2c4dd4 ee06a16ee8
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 12:02:32 2018 -0800
+
+    Merge pull request #19360 from ukernel/luminous-22219
+    
+    luminous: mds: ignore export pin for unlinked directory
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 646a2c4dd4e02a2f995b9b59d514cdef19065478
+Merge: 1b1ce1326e 1643d5a4f8
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 12:01:35 2018 -0800
+
+    Merge pull request #19646 from pdvian/wip-22503-luminous
+    
+    luminous: mds: properly eval locks after importing inode
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 1b1ce1326e991b2e2d65826848fb594d238b6815
+Merge: 09baff4bbb 80de33b1ca
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Jan 25 11:46:32 2018 -0800
+
+    Merge pull request #20055 from dzafman/wip-22724
+    
+    luminous: miscounting degraded objects and PG stuck in recovery_unfound
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 80de33b1ca215046290160fcc2781f3b29cb5c35
+Merge: 7522ddd978 09baff4bbb
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Jan 25 11:44:18 2018 -0800
+
+    Merge branch 'luminous' into wip-22724
+
+commit 09baff4bbbaa80d57ecb130b7921f19e9df03e8f
+Merge: d7874dc154 0040c7bfe1
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Thu Jan 25 11:36:22 2018 -0800
+
+    Merge pull request #20081 from dzafman/wip-recovering-luminous
+    
+    luminous: osd/ReplicatedPG.cc: recover_replicas: object added to missing set for backfill, but is not in recovering, error!
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit d7874dc1543b26ea0ac86f93d70588ec2e394fd9
+Merge: 2ecc4c326e a170e67760
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 11:12:20 2018 -0800
+
+    Merge pull request #18008 from ukernel/luminous-21584
+    
+    luminous: mds: fix CDir::log_mark_dirty()
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Jos Collin <jcollin@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit 2ecc4c326edaef0ab95ee43e69ddbc37b428840a
+Merge: 6097c4f10e ddba907279
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 11:11:17 2018 -0800
+
+    Merge pull request #18782 from ukernel/luminous-21985
+    
+    luminous: mds: fix MDS_FEATURE_INCOMPAT_FILE_LAYOUT_V2 definition
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 6097c4f10e5353ee73b40b7defb4c828aeb7b071
+Merge: b18280dbd5 ab103b9fe3
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 11:09:39 2018 -0800
+
+    Merge pull request #18783 from ukernel/luminous-21975
+    
+    luminous: mds: trim 'N' log segments according to how many log segments are there
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit b18280dbd572768b0b84edce034a2419ae181a0f
+Merge: 443bfb698f d73deb017c
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 11:08:27 2018 -0800
+
+    Merge pull request #18831 from shinobu-x/wip-22076-luminous
+    
+    luminous: cephfs: "ceph tell mds" commands result in "File exists" errors on client admin socket
+    
+    Reviewed-by: Jos Collin <jcollin@redhat.com>
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 443bfb698f96c2e339a793a2950aab7dcd03b232
+Merge: 0e762253c6 efd4147fa8
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 11:07:27 2018 -0800
+
+    Merge pull request #18835 from shinobu-x/wip-22074-luminous
+    
+    luminous: mds: don't check gid when none specified in auth caps
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 0e762253c6709631c984cfddd2a066bc87034255
+Merge: 1412d885aa 1410b5a5f5
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 11:06:52 2018 -0800
+
+    Merge pull request #18869 from shinobu-x/wip-21952-luminous
+    
+    luminous: mds: no assertion on inode being purging in find_ino_peers()
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 1412d885aab6a0c76566960e22109920f7580803
+Merge: dbc7cf62ea e485b89ffd
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 11:06:17 2018 -0800
+
+    Merge pull request #18871 from shinobu-x/wip-21947-luminous
+    
+    luminous: mds: preserve order of requests during recovery of multimds cluster
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit dbc7cf62ea5b46775c1b142523e9cb01d65a9dad
+Merge: 4a0835d0a4 1ec93753a3
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 11:05:28 2018 -0800
+
+    Merge pull request #18912 from shinobu-x/wip-22077-luminous
+    
+    luminous: src/mds/MDCache.cc: 7421: FAILED assert(CInode::count() == inode_map.size() + snap_inode_map.size())
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 4a0835d0a476aa7686ec449f2d8c7cb0b2ced4c3
+Merge: 6fa7c82bca bd3e7795ca
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 11:04:50 2018 -0800
+
+    Merge pull request #19055 from shinobu-x/wip-22192-luminous
+    
+    luminous: MDSMonitor: monitor gives constant "is now active in filesystem cephfs as rank" cluster log info messages
+    
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+
+commit 6fa7c82bca25a0d28fd2104850aac3770d0243c7
+Merge: cdfe41dde0 c514d3d427
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 11:02:59 2018 -0800
+
+    Merge pull request #19326 from ukernel/luminous-22263
+    
+    luminous: mds: handle client reconnect gather race
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit cdfe41dde0e4ae2bc5d8d46a27595532db299a62
+Merge: dc68871f08 35bb3d4d17
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 11:00:27 2018 -0800
+
+    Merge pull request #19480 from smithfarm/wip-22407-luminous
+    
+    luminous: cephfs: client: implement delegation support in userland cephfs
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Jeff Layton <jlayton@redhat.com>
+
+commit dc68871f0850971c41434b1618e7662b436e4d18
+Merge: 75d7ce3cb3 8d6c2c2dea
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 10:58:46 2018 -0800
+
+    Merge pull request #19775 from ukernel/luminous-19578
+    
+    luminous: mds: track dirty dentries in separate list
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 75d7ce3cb3cea53340bd7738d1b9567f3c5e88d1
+Merge: b2e38487d7 e97399d2ee
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 10:58:18 2018 -0800
+
+    Merge pull request #19827 from pdvian/wip-22587-luminous
+    
+    luminous: mds: reduce debugging level for balancer messages
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit b2e38487d78327c2c092e786842028e61177edbc
+Merge: af9499dba0 55f4642be2
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 10:57:05 2018 -0800
+
+    Merge pull request #20082 from smithfarm/wip-21525-luminous
+    
+    luminous: cephfs: client: dual client segfault with racing ceph_shutdown
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit af9499dba0e1822b8cd56f3d533fdc0be88e0065
+Merge: 832f228dc8 dc0e0250d1
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 10:55:53 2018 -0800
+
+    Merge pull request #20085 from pdvian/wip-22763-luminous
+    
+    luminous: cephfs-journal-tool: add "set pool_id" option
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 832f228dc845129c95bbb688f70322ee636f7e74
+Merge: e7113bca11 4c211d0d6c
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 10:55:19 2018 -0800
+
+    Merge pull request #20086 from pdvian/wip-22765-luminous
+    
+    luminous: client: avoid recursive lock in ll_get_vino
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 26b733758360f144008a4b48272bad45a7ac7673
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Dec 5 09:58:04 2017 -0500
+
+    ceph-dencoder: moved RBD types outside of RGW preprocessor guard
+    
+    Fixes: http://tracker.ceph.com/issues/22321
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b00ac12f4045cd39ac9a436cf32d233d16bc3485)
+
+commit d4f5dc27803d0f356c15ee2a4de4c22dbf943fe3
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Sep 6 17:06:44 2017 +0800
+
+    mds: fix return value of MDCache::dump_cache
+    
+    previous commit "mds: track snap inodes through sorted map" makes
+    MDCache::dump_cache return 1 on success.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit f519fca9dd958121a289676edf5175fb8be9894f)
+    
+    Backport note: the original cause was commit
+    46c829d23aa5379a973d2330b0551a5b92598133. The fix was part of snapshot fixes
+    for Mimic in f519fca9dd958121a289676edf5175fb8be9894f which was not backported.
+
+commit fc791b980d2e9835e66fec8b39264be370d6a740
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Sep 18 13:29:40 2017 -0400
+
+    qa/rgw: hadoop suite targets centos_latest
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit a97d5bea42a40909aa4e9672505b9ad994cec184)
+    
+    Conflicts:
+        qa/suites/rgw/hadoop-s3a/s3a-hadoop.yaml (changes already backported
+            via 6b5f212b9170dbd99779527b41be233b9d40b5e4)
+
+commit 470c5f23a0b85fb8a71c98566ecb0f95e376ceef
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Sep 18 13:28:22 2017 -0400
+
+    qa/rgw: use overrides for hadoop version
+    
+    instead of having two separate tasks, add v27.yaml and v28.yaml with
+    overrides for hadoop_version
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 4a953a16d7b80d343899de39a8cf55dc761847de)
+
+commit e7113bca11803206dd81b8c3a5e457c4941c845a
+Merge: 60c74451c3 22ff147734
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 09:11:10 2018 -0800
+
+    Merge pull request #18019 from ukernel/luminous-21091
+    
+    luminous: mds: fix StrayManager::truncate()
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 60c74451c36d1e20a79bcb25edc58a070c74f934
+Merge: cea6b6e392 de05d2c819
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 25 12:03:54 2018 -0500
+
+    Merge pull request #19485 from dillaman/wip-21788-luminous
+    
+    luminous: librbd: refresh image after applying new/removing old metadata
+    
+    Reviewed-by: Mykola Golub <mgolub@suse.com>
+
+commit cea6b6e392dccd3793d6db2eeed50dd9969c80e8
+Merge: fa97a4793b 369dcb36c7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 25 11:44:39 2018 -0500
+
+    Merge pull request #18834 from shinobu-x/wip-22073-luminous
+    
+    luminous: [api] compare-and-write methods not properly advertised
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit fa97a4793bfb8ee092fc10b4961652465f5d686b
+Merge: 2d0a2df9b8 818c354a43
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 25 11:27:18 2018 -0500
+
+    Merge pull request #19058 from shinobu-x/wip-22190-luminous
+    
+    luminous: class rbd.Image discard----OSError: [errno 2147483648] error discarding region
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 2d0a2df9b82412b1fd64ee8a6ce13a60ea59c56e
+Merge: 5bfd5cb595 2070fcf7d4
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 08:18:13 2018 -0800
+
+    Merge pull request #19066 from pdvian/wip-22172-luminous
+    
+    luminous: [rbd-nbd] Fedora does not register resize events
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 5bfd5cb5953d2cc0524632b79ee07821b01b7fae
+Merge: 043a7a19e0 17ff1f1fd2
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 08:17:38 2018 -0800
+
+    Merge pull request #19107 from pdvian/wip-22208-luminous
+    
+    luminous: rbd: disk usage on empty pool no longer returns an error message
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 043a7a19e07b821693fce5d42317dbf9d825993e
+Merge: 588e575936 c3c3bb0bc9
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 08:17:01 2018 -0800
+
+    Merge pull request #19305 from pdvian/wip-21700-luminous
+    
+    luminous: rbd-mirror: Allow a different data-pool to be used on the secondary cluster
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 588e5759360fdeaf00ac6c3c901cf3d20a269191
+Merge: ab6b45d687 08a2358f85
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 08:16:26 2018 -0800
+
+    Merge pull request #19447 from smithfarm/wip-21646-luminous
+    
+    luminous: librbd: Image-meta should be dynamically refreshed
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit ab6b45d687b4a66e25833875c81d2f2dab8d2d95
+Merge: 43333e0d5f 5e8c4f83f4
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 08:15:50 2018 -0800
+
+    Merge pull request #19479 from smithfarm/wip-22376-luminous
+    
+    luminous: rbd: Python RBD metadata_get does not work
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 43333e0d5fd2949441d99e9683e6399afd67e5e3
+Merge: 3ac673215f 235032ec64
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 08:14:53 2018 -0800
+
+    Merge pull request #19484 from dillaman/wip-21644-luminous
+    
+    luminous: rbd-mirror: sync image metadata when transfering remote image
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 3ac673215f3310c12762ad1bc3e32948b220e741
+Merge: 9560a8b2dc 1507015169
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 08:13:37 2018 -0800
+
+    Merge pull request #19503 from shinobu-x/wip-22395-luminous
+    
+    luminous: librbd: cannot clone all image-metas if we have more than 64 key/value pairs
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 9560a8b2dcb13cf774a94d97b7d3bc274f77c22c
+Merge: 9f48a65b50 feb4d7ba37
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 08:12:33 2018 -0800
+
+    Merge pull request #19504 from shinobu-x/wip-22393-luminous
+    
+    luminous: librbd: cannot copy all image-metas if we have more than 64 key/value pairs
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 9f48a65b50538c3925fc162f5e282c0a414c0a14
+Merge: 49d5128b05 2b9bbf605e
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 08:11:57 2018 -0800
+
+    Merge pull request #19554 from shinobu-x/wip-22454-luminous
+    
+     luminous: cluster resource agent ocf:ceph:rbd - wrong permissions
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 49d5128b055930859d4d8716ea94100e5cbc3878
+Merge: 5051aa8646 3d0f4fa752
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 08:11:22 2018 -0800
+
+    Merge pull request #19625 from shinobu-x/wip-22497-luminous
+    
+     luminous: [rbd-mirror] new pools might not be detected
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 5051aa8646f9240043261e184baa5fd05bcedd23
+Merge: 147d57b3a5 a15eb7df1e
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 08:10:37 2018 -0800
+
+    Merge pull request #19800 from smithfarm/wip-21641-luminous
+    
+    luminous: rbd ls -l crashes with SIGABRT
+    
+    Reviewed-by: Mykola Golub <mgolub@mirantis.com>
+
+commit 52e59854c21d4e29e49a16c6d2f73470e68c73cd
+Author: Chang Liu <liuchang0812@gmail.com>
+Date:   Sat Oct 21 23:35:47 2017 +0800
+
+    mgr: request daemon's metadata when receiving a report message from an unknown server
+    
+    Fixes: http://tracker.ceph.com/issues/21687
+    
+    Signed-off-by: Chang Liu <liuchang0812@gmail.com>
+    (cherry picked from commit a39813837ca84434e57afa47138973268ff366d3)
+
+commit e97ad14cdc22c0da907fabc48d62698c94f6ba66
+Author: liuchang0812 <liuchang0812@gmail.com>
+Date:   Thu Aug 24 10:56:29 2017 +0800
+
+    mgr: kill MgrSession when MMgrReport come from daemon without metadata info
+    
+    Signed-off-by: liuchang0812 <liuchang0812@gmail.com>
+    (cherry picked from commit 29080134339e5e64d50af1db9fe50df7ea55c1d0)
+
+commit 147d57b3a5e7c9dde57d8f7ccd4f947c92d31ad7
+Merge: a69f543b70 88c987cdb5
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 08:09:49 2018 -0800
+
+    Merge pull request #19802 from smithfarm/wip-21690-luminous
+    
+    luminous: tests: rbd_mirror_helpers.sh request_resync_image function saves image id to wrong variable
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit a69f543b70e554c88022dec020bd78d48da0445d
+Merge: 7ba84945e7 17aa16dc6b
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Thu Jan 25 08:09:02 2018 -0800
+
+    Merge pull request #19853 from pdvian/wip-22593-luminous
+    
+    luminous: rbd: librbd: filter out potential race with image rename
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 4318ac5e3f36b9885953f03d30c6b6a72b62d7e6
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Mon Jan 22 17:18:43 2018 +0200
+
+    Revert "luminous: rgw: user stats increased after bucket reshard"
+    
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+
+commit cd9e6788cbb8ead60c82d15c574c5a00f4c1d222
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Jan 4 13:06:52 2018 +0800
+
+    tools/ceph_monstore_tool: include mgrmap in initial paxos epoch
+    
+    before this change, the "mgr" map is not included paxos, thus why the
+    peon mon fails to have the mgr map after it is sync'ed with its leader.
+    and hence there is chance that ceph-mgr is unable to get a mgr map from
+    the monitor it is connected to.
+    
+    Fixes: http://tracker.ceph.com/issues/22266
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit f3f2a8bad4da7fda485eba7412d40b7b5758452d)
+
+commit b219b854b708c532bacb37ee8c8253c80586fef6
+Author: J. Eric Ivancich <ivancich@redhat.com>
+Date:   Wed Dec 6 16:36:28 2017 -0500
+
+    When a system object is created exclusively, do not distribute the
+    cache information to the other rados gateways and instead let them
+    lazily retrieve the information when they need it.
+    
+    Signed-off-by: J. Eric Ivancich <ivancich@redhat.com>
+    (cherry picked from commit 647ce3387312fc683660c1f3c7571c577379be1c)
+    Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
+
+commit 7ba84945e7598333dca5f75dcbb7771f6b85fa13
+Merge: 9297805666 9a7e1f6a44
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Jan 24 13:19:46 2018 -0800
+
+    Merge pull request #18722 from shinobu-x/wip-luminous-21946
+    
+    luminous: mds: set PRIO_USEFUL on num_sessions counter
+    
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 92978056660f60e878a4a2aca377e2721c4d08e5
+Merge: ae19fa733f 948ca443fc
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Jan 24 13:18:41 2018 -0800
+
+    Merge pull request #18730 from shinobu-x/wip-luminous-22030
+    
+    luminous: List of filesystems does not get refreshed after a filesystem deletion
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit ae19fa733f2fe6dc1feab1650851d559abb48791
+Merge: fbb14a0e7b fd956acfd6
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Jan 24 13:18:04 2018 -0800
+
+    Merge pull request #18754 from shinobu-x/wip-21636-luminous
+    
+    luminous: ceph-monstore-tool --readable mode doesn't understand FSMap, MgrMap
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 3ccf7e0bc6ca51d0d1e54f5bb7558f9f7aa199f3
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Jan 22 13:02:40 2018 +0800
+
+    osd/PG: print a space after __func__ in log message
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit e7bdf589db2010ea82aa33d60e9c54c14c7a78f0)
+
+commit 32c0eab8749e813b44ffe5ecefa3e59c61cb90a7
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Jan 22 13:08:48 2018 +0800
+
+    osd: update store with options after pg is created
+    
+    Fixes: http://tracker.ceph.com/issues/22419
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 43b2e512e61319f14f7390d063d66f7a2a568b4d)
+    
+    Conflicts:
+        src/osd/PG.cc (omitted immediately preceding write_if_dirty() call
+            because fa7818aefa92b4e9e067945c33f96d520e885f4e not backported to
+            luminous)
+
+commit fbb14a0e7bd76053f7347afb1c0460e4e984b768
+Merge: 06b9f828cf 7d6984b554
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Wed Jan 24 17:07:06 2018 +0100
+
+    Merge pull request #20053 from dillaman/wip-22740-luminous
+    
+    luminous: qa/suites/upgrade: disable broken kraken upgrade cls_rbd test
+    
+    Reviewed-By: Yuri Weinstein <yuriw@redhat.com>
+
+commit 06b9f828cfef1843e43852e2e95efcb6d0b6c4d7
+Merge: 1898fb6f38 767ad8518d
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Jan 24 07:05:13 2018 -0800
+
+    Merge pull request #18564 from kmroz/wip-21688-luminous
+    
+    luminous: Possible deadlock in 'list_children' when refresh is required
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 1898fb6f38e5eaea16ff0b1d6007bfc605337136
+Merge: b0a841fd7c 1d4435f0cd
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Jan 24 07:04:38 2018 -0800
+
+    Merge pull request #18698 from shinobu-x/wip-luminous-21808
+    
+    luminous: rbd: fix crash during map
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit b0a841fd7cc356b073bc38f4b66d55e741e5de68
+Merge: 61ec339e6c c6bc756304
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Wed Jan 24 07:03:43 2018 -0800
+
+    Merge pull request #19038 from pdvian/wip-22183-luminous
+    
+    luminous: rgw: set sync_from_all as true when no value is seen
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 61ec339e6cba3935f531886684d25cd5708fb67a
+Merge: 2d93e37609 34450ed6c0
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Wed Jan 24 15:23:54 2018 +0100
+
+    Merge pull request #19446 from smithfarm/wip-22404-luminous
+    
+    luminous: doc: crush_ruleset is invalid command in luminous
+    
+    Reviewed-By: Abhishek Lekshmanan <abhishek@suse.com>
+
+commit 2d93e37609a757f2d3e6e4870a8054eb36626ed8
+Merge: c93fa099ee acccae56c4
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Wed Jan 24 08:46:21 2018 -0500
+
+    Merge pull request #18747 from shinobu-x/wip-21794-luminous
+    
+    luminous: backoff causes out of order op
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 127236f6766c8539dbcf6a7957b4f3a04e7807db
+Author: Piotr Dałek <piotr.dalek@corp.ovh.com>
+Date:   Wed Dec 13 16:13:33 2017 +0100
+
+    options, Mon: monitor the snap trim queues
+    
+    If new option "mon osd snap trim queue warn on" is set to value larger
+    than 0 (32768 by default), cluster will go into HEALTH_WARN state
+    once any pg has a snap trim queue larger than that value. This can
+    be used as an indicator of snaptrimmer not keeping up and disk space
+    not being reclaimed fast enough. Warning message will tell how many
+    pgs are affected.
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@corp.ovh.com>
+    (cherry picked from commit 8412a65e0e6f610fb39430d8f65e561b6dbda13f)
+
+commit ca4413dee9a148c5ffda235793b02499a3975e88
+Author: Piotr Dałek <piotr.dalek@corp.ovh.com>
+Date:   Wed Dec 13 15:23:55 2017 +0100
+
+    osd, PG: share the snaptrimq.size() within pg_stat_t
+    
+    That way it will be unnecessary to go through all pgs separately
+    to find pgs with excessively long snap trim queues. And we don't need
+    to share snap trim queues itself, which may be large by itself.
+    As snap trim queues tend to be short and anything above 50 000
+    I consider absurdly large, the snaptrimq_len is capped at 2^32 to
+    save space in pg_stat_t.
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@corp.ovh.com>
+    (cherry picked from commit 022d246f17b31d309525395831c84e9856533adb)
+    
+    Conflicts:
+            src/osd/osd_types.cc
+         - needed a fix for snaptrimq_len that was placed after two other
+    new fields
+
+commit faeb81cd4dc24dc087877644ee27b937d50b4108
+Author: Piotr Dałek <piotr.dalek@corp.ovh.com>
+Date:   Wed Dec 13 10:14:30 2017 +0100
+
+    osd/PrimaryLogPG: dump snap_trimq size
+    
+    One can just parse the snap_trimq string, but that's much more
+    expensive than just reading an unsigned int.
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@corp.ovh.com>
+    (cherry picked from commit dc7781cf17d11cb09067656cb25c0d710ab60d71)
+
+commit c93fa099ee173a0fea14bc8b2c36c2161fa0124b
+Merge: 6c5684c880 b97227d34e
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Wed Jan 24 11:23:49 2018 +0800
+
+    Merge pull request #20087 from dzafman/wip-misc-luminous
+    
+    qa/standalone: Minor fixes for ceph_objectstore_tool.py test
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit 0040c7bfe1a47b73954c046cf460e2e96c66263a
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 11 20:16:44 2017 -0700
+
+    test: Cleanup test-erasure-eio.sh code
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 69b5fc54feb59f8b0a26a3ca3e925980c91b5b75)
+
+commit 325eec1ba9c9b03d3ad6bcf56572be64d6c16c98
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Oct 9 14:15:51 2017 -0700
+
+    test: Add replicated recovery/backfill test
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit c2572bee3c6256419e0b265e9e2829e7f3afb76d)
+
+commit 3068ea1e93f1806b74b78c636693f7afcff95aea
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Oct 9 14:03:23 2017 -0700
+
+    osd: Better recovery/backfill restart for mark_unfound_lost delete/revert
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 689bff354a6bfa1cf47b0d0a04fa9f1b2ef68f75)
+
+commit 25a901baf7f5d8130b7ed3d6913608105ccd4de6
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Oct 9 08:19:21 2017 -0700
+
+    osd, mon: Add new pg states recovery_unfound and backfill_unfound
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 7f8b0ce9e681f727d8217e3ed74a1a3355f364f3)
+    
+    Conflicts:
+            src/osd/PG.h (trivial)
+            src/osd/osd_types.h (trivial)
+
+commit e40e2aa53b192a86abaa95fc74f05b5575e16d62
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Oct 9 08:17:29 2017 -0700
+
+    osd: Add new UnfoundBackfill and UnfoundRecovery pg transitions
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit bb2bcb95f51abc206e005e44ef383ee45b8f2209)
+    
+    Conflicts:
+            src/osd/PG.cc (trivial, no MBackfillReserve::CANCEL)
+            src/osd/PG.h (trivial)
+    
+    fixup! osd: Add new UnfoundBackfill and UnfoundRecovery pg transitions
+
+commit bb21f23731027df30844236573648bca300b523e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Nov 30 10:05:14 2017 -0500
+
+    mgr: disconnect unregistered service daemon when report received
+    
+    This will allow the service daemon to reconnect and re-register
+    itself as a service daemon without requiring the mgr client to
+    subscribe to MgrStats and detect its removal.
+    
+    Fixes: http://tracker.ceph.com/issues/22286
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit c97cca13ab066ed2306708986ff387d903b2486a)
+
+commit b97227d34e097e282aa51fe88a7549fe12caaad6
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Sep 18 14:07:09 2017 -0700
+
+    test: Fix ceph-objectstore-tool usage check
+    
+    Caused by: c7b7a1f04f78fa62890c567d0ca53874c8d75eb7
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 0364ae104afc15effadfe968b5f60221704d933c)
+
+commit fe956597f2a3f2e9e0b4c2eb80547d4e575f6ee3
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Jan 23 18:16:32 2018 -0800
+
+    test: ceph_objectstore_tool.py bad --filestore option to vstart
+    
+    Luminous doesn't have --filestore option
+    
+    Caused by: fd7d53bacb10d5b2094f1f579c96899d8a0a4b4f
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+
+commit 4c211d0d6ce76e592eb9b70445fde6f572fdb4f2
+Author: dongdong tao <tdd21151186@gmail.com>
+Date:   Mon Jan 8 16:00:31 2018 +0800
+
+    client: avoid recursive lock in ll_get_vino
+    
+    Fixes: http://tracker.ceph.com/issues/22629
+    Signed-off-by: dongdong tao <tdd21151186@gmail.com>
+    (cherry picked from commit fdfbe40c3f16c14dc9c3352f3eea66d34138ee9d)
+
+commit dc0e0250d1d5a261b2ee4a29eaa33246d7a7ebcb
+Author: dongdong tao <tdd21151186@gmail.com>
+Date:   Tue Jan 9 17:07:11 2018 +0800
+
+    cephfs-journal-tool: add usage help info for header set <field>
+    
+    Signed-off-by: dongdong tao <tdd21151186@gmail.com>
+    (cherry picked from commit 2b06bff950a78b15aca083b20eaa812b4214e8d4)
+
+commit 61a64511f99c2d8f30ab046fa72e518fba727058
+Author: dongdong tao <tdd21151186@gmail.com>
+Date:   Tue Jan 9 16:54:40 2018 +0800
+
+    mds: add error log info
+    
+    Signed-off-by: dongdong tao <tdd21151186@gmail.com>
+    (cherry picked from commit a7c2ecd6391d22c5312a9e568ff742cff252749e)
+
+commit 4d07eb8ba724c5bfcc7f8f12e6213282f497fb58
+Author: dongdong tao <tdd21151186@gmail.com>
+Date:   Tue Jan 9 16:31:07 2018 +0800
+
+    cephfs-journal-tool: add "set pool_id" option
+    
+    Fixes: http://tracker.ceph.com/issues/22631
+    
+    Signed-off-by: dongdong tao <tdd21151186@gmail.com>
+    (cherry picked from commit b3a5e313117028b23133f3e0d72e0b6b7159a178)
+
+commit 1b2b589a12c2e70c09c44e9f490819c833e1e69d
+Author: Konstantin Shalygin <k0ste@k0ste.ru>
+Date:   Wed Jan 17 13:40:05 2018 +0700
+
+    mgr: prometheus: added osd commit/apply latency metrics (#22718).
+    
+    Signed-off-by: Konstantin Shalygin <k0ste@k0ste.ru>
+    (cherry picked from commit 1adf4325455ccdbf03da63046c510bb041ac8520)
+
+commit 6c5684c880882e71e262aec4c8d7d4478edb6c98
+Merge: f7bfda8fce 7012cf4b60
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 23 15:10:49 2018 -0800
+
+    Merge pull request #18742 from shinobu-x/wip-21924-luminous
+    
+    luminous: ceph_test_objectstore fails ObjectStore/StoreTest.Synthetic/1 (filestore) buffer content mismatch
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit f7bfda8fce31ebf186907400c50f8a13eb64088a
+Merge: 6bd8b907ae dfebcce5a4
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 23 15:10:19 2018 -0800
+
+    Merge pull request #18744 from shinobu-x/wip-21922-luminous
+    
+    luminous: Objecter::C_ObjectOperation_sparse_read throws/catches exceptions on -ENOENT
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 6bd8b907aec6414fb04cbfd24ed1d01d66331109
+Merge: f5e1d2b160 c281456346
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 23 15:09:41 2018 -0800
+
+    Merge pull request #18745 from shinobu-x/wip-21921-luminous
+    
+    luminous: Objecter::_send_op unnecessarily constructs costly hobject_t
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 55f4642be2bbdc758378cb58a2d369c14e37bfb1
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Wed Oct 11 11:16:39 2017 -0400
+
+    test: make the LibCephFS.ShutdownRacer test even more thrashy
+    
+    Have each thread do the startup and shutdown in a loop for a specified
+    number of times.
+    
+    Tracker: http://tracker.ceph.com/issues/21512
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit f877e365a42f8f34af6fc3382593ef09101a50d0)
+
+commit 963d6fb26565799a08e01577cf15adc0a2f1a956
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Wed Oct 11 11:16:38 2017 -0400
+
+    lockdep: free_ids and lock_ref hashes must be truly global
+    
+    It's possible for the teardown of g_lockdep_ceph_ctx to occur, followed
+    by a new context being registered as the lockdep context. When that
+    occurs, we can end up reusing lock id's that were previously handed out
+    to consumers. We need for those IDs to be persistent across lockdep
+    enablement and disablement.
+    
+    Make both the free_ids table, and the lock_refs map persistent across
+    lockdep_unregister_ceph_context and lockdep_register_ceph_context cycles.
+    Entries in those tables will only be deleted by the destruction of the
+    associated mutex.
+    
+    When lockdep_unregister is called, do the refcounting like we normally
+    would, but only clear out the state when the lockid is registered
+    in the lock_names hash.
+    
+    Finally, we do still need to handle the case where g_lockdep has gone
+    false even when there are outstanding references after the decrement.
+    Only log the message if that's not the case.
+    
+    With this, we can deal with the case of multiple clients enabling and
+    disabling lockdep in an unsynchronized way.
+    
+    Tracker: http://tracker.ceph.com/issues/21512
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 82fe4e94bcb706434f9215bc8405ff60770cf14f)
+
+commit 42bc9058a77ff427dd1dc8a4ba8098c54782a918
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Wed Oct 11 11:16:38 2017 -0400
+
+    common: add a clear_g_str_vec() function to clear g_str_vec
+    
+    Prefix str_vec and str_vec_lock with "g_" to make it clear that they are
+    truly global values. Add a new clear_g_str_vec function to allow it to
+    be explicitly cleaned out by callers that need that functionality
+    (mostly testcase for now).
+    
+    Tracker: http://tracker.ceph.com/issues/21512
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit f72ace0a5666db49bb390ff4ec6808e6e55a779f)
+
+commit f5e1d2b1608c17449002f9ab61e0021cf25b256f
+Merge: 2e0800bd68 5234ef2bca
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 23 15:08:59 2018 -0800
+
+    Merge pull request #18746 from shinobu-x/wip-21916-luminous
+    
+    luminous: msg/async/AsyncConnection.cc: 1835: FAILED assert(state == STATE_CLOSED)
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 8dec6175ab1167bab01af2594ac5d687072f3af2
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Wed Oct 11 11:16:38 2017 -0400
+
+    common: make it safe to call env_to_vec multiple times
+    
+    After it has been called once and we have outstanding CephContexts with
+    pointers into str_vec, we can't call get_str_vec on it again.
+    
+    Add a static local mutex to protect access to str_vec.
+    
+    Tracker: http://tracker.ceph.com/issues/21512
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 90e898de5f4b8d22f1a6d0e2aedf9e8c50cf72d5)
+
+commit eeb62a53c057bdb55e2da014dc7633312ab648b5
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Thu Sep 14 09:28:34 2017 -0400
+
+    lockdep: fix Mutex tests to disable lockdep properly
+    
+    ...and make g_lockdep a bool.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 0cd0bd778a6149d3e2fe657060e223be3675aed7)
+
+commit 864a2c5a2a40806067fa09a1390c3af14c8dd0ef
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Thu Sep 14 09:28:33 2017 -0400
+
+    client: test shutdown race
+    
+    Spawn threads that bring up a bunch of ceph_mounts with individual
+    CephContext objects, and then tear them down in parallel.
+    
+    Tracker: http://tracker.ceph.com/issues/20988
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 8252f3168a1bc90add7f7515c549e9eecffa54b7)
+
+commit 8db3ecd6c759251fa8105aeb676e74f9c9302027
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Thu Sep 14 12:22:52 2017 -0400
+
+    client: fix signed/unsigned comparison compiler warning
+    
+    The build says:
+    
+    src/client/Client.cc: In member function ‘void Client::trim_caps(MetaSession*, int)’:
+    src/client/Client.cc:4121:22: warning: comparison between signed and unsigned integer expressions [-Wsign-compare]
+       if (s->caps.size() > max)
+           ~~~~~~~~~~~~~~~^~~~~
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit e057b6770b5545f327990cbaf17e7a391e3e8a50)
+
+commit 49ac68c3f95a0bdccb7a6c2f1706a3784ffc706a
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Thu Sep 14 09:28:34 2017 -0400
+
+    lockdep: fix races with concurrent lockdep teardown
+    
+    If the cct is unregistered while other threads are flogging mutexes,
+    then we can hit all sorts of bugs. Ensure that we handle that
+    situation sanely, by checking that g_lockdep is still set after
+    we take the lockdep_mutex.
+    
+    Also, remove an assertion from lockdep_unregister, and just turn it into
+    an immediate return. It's possible to have a call to
+    lockdep_unregister_ceph_context, and then a call to
+    lockdep_register_ceph_context while a mutex is being held by another
+    task.
+    
+    In that case, it's possible the lock does not exist in the map
+    when we go to unregister it. That's not a bug though, just a natural
+    consequence of that series of actions.
+    
+    Tracker: http://tracker.ceph.com/issues/20988
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 75f41a95782a7ee83a243d91963e8d591402f8a6)
+
+commit b66ba62822bc7935d60a4424cedbfe7a845631c6
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Thu Sep 14 09:28:34 2017 -0400
+
+    lockdep: don't take lockdep_mutex twice for new lock registrations
+    
+    We can do it under the same mutex, which should be more efficient.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 01863bb6fb62ea89aa3e21e43bf4dc4f3da9cfcb)
+
+commit 2e0800bd68234a32edb1f23c33402252fa6d200a
+Merge: ea771cbd92 8941606f5e
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 23 15:07:07 2018 -0800
+
+    Merge pull request #18749 from shinobu-x/wip-21785-luminous
+    
+    luminous: OSDMap cache assert on shutdown
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit ea771cbd926656db371ca6fa08ed172c76a88195
+Merge: b03323c80c b36bd48204
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 23 15:06:19 2018 -0800
+
+    Merge pull request #18753 from shinobu-x/wip-21697-luminous
+    
+    luminous: OSDService::recovery_need_sleep read+updated without locking
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit b03323c80c4759bece12748268f16b8abe1197a6
+Merge: a0b489712d 5c9d212f94
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 23 14:52:13 2018 -0800
+
+    Merge pull request #18715 from shinobu-x/wip-luminous-21964
+    
+    luminous: ceph_test_cls_log failures related to cls_cxx_subop_version()
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit a0b489712dbcc95c2f8cec8f8988acb532947385
+Merge: c8d526b8a8 b6f50a463a
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 23 14:51:37 2018 -0800
+
+    Merge pull request #18732 from shinobu-x/wip-luminous-21875
+    
+    luminous: ceph-mgr spuriously reloading OSD metadata on map changes
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit c8d526b8a8b0eb5fa545c25210e98625232cb8a8
+Merge: 029594a080 f0914619a5
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Tue Jan 23 14:50:34 2018 -0800
+
+    Merge pull request #18735 from shinobu-x/wip-22029-luminous
+    
+    luminous: restarting active ceph-mgr cause glitches in bps and iops metrics
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit a09a42a4a850ee6f5e64fc0dc79995dbbfbdf13d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Oct 2 13:51:17 2017 -0700
+
+    test: Test case that reproduces tracker 18162
+    
+    recover_replicas: object added to missing set for backfill, but is not in recovering, error!
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b9de5eec267627c8bc5ff0759ddea6c4a8aa7bce)
+
+commit 23f7c556d9ad903ad928e6193b50f5a61106429a
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Oct 3 18:32:20 2017 -0700
+
+    osd: Better handle failure to get enough EC shards to backfill
+    
+    Fixes: http://tracker.ceph.com/issues/18162
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 6a02bfef3d44a13589c1a90bec29ff0ac64f97aa)
+
+commit 081fe72398633b66b93bc961cfd7d8599d5d0cba
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Sep 21 20:35:27 2017 -0700
+
+    osd: For recovery get all possible shards to read on errors
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 390d12f71a1bd6e07f3516b1c73e467e9960725d)
+
+commit 5bed56a620f8c1c333f900f72d818cdec0f6381a
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Sep 27 14:29:25 2017 -0700
+
+    test: Use feature to get last array element
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 43e3206de2a20bf17ad9f2b16b7e486804097bf2)
+
+commit 62e9c05af9c7b1de44ebe4e055e84a419e0542b8
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Sep 21 17:48:28 2017 -0700
+
+    osd: Allow recovery to send additional reads
+    
+    For now it doesn't include non-acting OSDs
+    Added test for this case
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 1235810c2ad08ccb7ef5946686eb2b85798f5bca)
+
+commit bfd0029910e33840e6ef5d87bb48b222c57bad03
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Sep 21 14:49:17 2017 -0700
+
+    test: Allow modified options to existing setup functions
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit f92aa6c82449152901a4663c523e9ba43363eca3)
+
+commit 39bb3cf8e688a4fd4bc66295c39c66554cb8e089
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Sep 12 15:09:14 2017 -0700
+
+    osd: CLEANUP: Change getattr_maybe_cache() to reflect how it is being used
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 143111a112ec64724226f83ed285f66ee52204b2)
+
+commit a7a2de03ffcbb829c6ce6f95c5e20fb14ad1a3af
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Sep 12 15:03:33 2017 -0700
+
+    test: Add a removal test for erasure code read
+    
+    Test feature: http://tracker.ceph.com/issues/14513
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 50e08b0a5d7dc30a164313c1480b6cc829b6ec65)
+
+commit 7522ddd97845b8c2a15ad81a30b46a69e6f1444c
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Oct 23 16:07:48 2017 -0500
+
+    mon/PGMap: 'unclean' does not imply damaged
+    
+    Everything (that I can think of) that would lead to a PG being unclean is
+    already reported via another health message.  And there are cases where a
+    PG is unclean (e.g., because it is backfilling) where we are not degraded.
+    
+    Fix by ignoring this flag in the health checks.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 89e4cb90af3c193fe436390b5e3288701e6fc4af)
+
+commit 67fbd3830f2681c49b65b0ac93eccd58c219114c
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Jan 23 10:49:31 2018 -0500
+
+    boost: include more specific asio headers
+    
+    when building with gcc5 and boost 1.66, this #include <boost/asio.hpp> fails to compile:
+    
+    In file included from build/boost/include/boost/asio/impl/write.hpp:25:0,
+                     from build/boost/include/boost/asio/write.hpp:927,
+                     from build/boost/include/boost/asio/buffered_write_stream.hpp:29,
+                     from build/boost/include/boost/asio/buffered_stream.hpp:22,
+                     from build/boost/include/boost/asio.hpp:41,
+                     from src/common/Graylog.h:7,
+                     from src/common/LogClient.cc:20:
+    build/boost/include/boost/asio/detail/consuming_buffers.hpp: In member function ‘boost::asio::detail::consuming_buffers<Buffer, Buffers, Buffer_Iterator>::prepared_buffers_type boost::asio::detail::consuming_buffers<Buffer, Buffers, Buffer_Iterator>::prepare(std::size_t)’:
+    build/boost/include/boost/asio/detail/consuming_buffers.hpp:105:50: error: parse error in template argument list
+         while (next != end && max_size > 0 && result.count < result.max_buffers)
+                                                      ^
+    src/CMakeFiles/common-objs.dir/build.make:135: recipe for target 'src/CMakeFiles/common-objs.dir/common/LogClient.cc.o' failed
+    
+    see also: https://svn.boost.org/trac10/ticket/13368
+    
+    this commit targets the luminous branch specifically because the issues
+    doesn't reproduce on master (as we require gcc7+)
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+
+commit 660abda21163bf4297b325ea2ac42a28661d5f62
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 24 17:55:34 2017 -0400
+
+    rgw: more documentation for bilog trim config
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 4d4442168c15775f8346ebc6295a6de912250b8f)
+
+commit 2ed35c9f922409ad5e5e1e199687d25977b35cf6
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 24 16:54:59 2017 -0400
+
+    rgw: fix notify timeout for BucketTrimWatcher
+    
+    from seconds to msec
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit ada2d108e627d519794e1a6c08af23cecbc5be79)
+
+commit 3f06276fc80696f693cc754b49351d5f16b76818
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 24 16:51:39 2017 -0400
+
+    rgw: curly brace style for bilog trim classes
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit a0c438eefcf0c99094efa0d98902cc4fca5957f0)
+
+commit 6fc4c7a12345b59cf3e14546006bbe7941eb98a3
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 24 14:45:12 2017 -0400
+
+    rgw: BucketTrimWatcher checks handle in stop
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 6e4d9735036a68575696e53b6732e4680a0226d6)
+
+commit 24f7fd2cf9efedf86b214c4d5f45113eed0a2f80
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 24 12:23:41 2017 -0400
+
+    rgw: hold cr reference in MetadataListCR callback
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 3a1474031e4266100c5ccaf082756bdf0607a2be)
+
+commit 32ebc0e546903a3129defcee76541ed613bba195
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 24 12:23:08 2017 -0400
+
+    rgw: add comment for bucket in BucketCounter
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit c6d946b3e1fbb10da43273e62c8584ffe3afbcdf)
+
+commit d1620fef15d5dff8ebdfe26af0b37f0f24a6046a
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 24 11:08:24 2017 -0400
+
+    rgw: move shard marker helper into BucketIndexShardsManager
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 4be7039964a2d17069b3604a4c994743145a013e)
+
+commit 668c2833ace4af4cb805e4d5ae38b22b6e03473a
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 10 15:05:01 2017 -0400
+
+    qa/rgw: add kwargs for debug output
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit fa8a0713de6b5b8ea9c9400f3ed467d87940612a)
+
+commit f1bcf84000c43772cc9e89040356fef6c3b464cc
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Sep 20 10:01:17 2017 -0400
+
+    test/rgw: add test_bucket_index_log_trim()
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit d9dc58cf2c6f64562843cc1d00df82d99379e640)
+
+commit 96baf2d00fe111b131f4fd9fa96e56442ee8b359
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 15 15:54:44 2017 -0400
+
+    rgw: add TrimComplete to watch/notify api
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 916f5995c92acd0c5ba66fde6031fed7cb91cb95)
+
+commit 7c430a08afaf6700cf4e64b7f7333e6ceafab2be
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 15 14:48:43 2017 -0400
+
+    rgw: RGWDataSyncSingleEntryCR calls BucketChangeObserver
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 1c50d727b5df574e28d90cd99abe15db1742e4b1)
+    
+    Conflicts: sync tracing not backported
+            src/rgw/rgw_data_sync.cc
+            src/rgw/rgw_data_sync.h
+
+commit 07a122a3fda26c7b65ecf0879f24a8380568b47f
+Author: Igor Fedotov <ifedotov@suse.com>
+Date:   Wed Jan 3 14:16:19 2018 +0300
+
+    common/throttle: start using 64-bit values
+    
+    Signed-off-by: Igor Fedotov <ifedotov@suse.com>
+    (cherry picked from commit fa37ed1a48fd804ac199509bd78c470480ecbb22)
+    
+    Conflicts:
+            src/common/Throttle.cc : Resolved in _reset_max and put func
+            src/common/Throttle.h : Retained only count var changes
+
+commit e3fbaa92cedb101f79c7e0712017662f6d2ec1cf
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Jan 19 13:05:27 2018 -0500
+
+    rgw_file: alternate fix deadlock on lru eviction
+    
+    This change is an alternate fix for two problems found and fixed
+    by Yao Zongyou <yaozongyou@vip.qq.com>.
+    
+    The deadlock can be avoided just by not taking it in the recycle
+    case, which invariantly holds the lock.
+    
+    The invalidation of the insert iterator by the recyle-path unlink
+    we'd like to handle as a condition in order to preserve the cached
+    insertion point optimization we get in the common case.  (The
+    original behavior was, indeed, incorrect.)
+    
+    Based on feedback from Yao, removed the RGWFileHandle dtor version
+    of the unlink check, which I think happened twice.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 3cf0880f86b8f7911139c4e3d672cf47420c8f49)
+
+commit 35d2baf47b2ad29c961da2f861b4d71b65835b00
+Author: Tao Chen <chentao@umcloud.com>
+Date:   Fri Dec 1 16:42:44 2017 +0800
+
+    librgw: fix shutdown err with resources uncleaned
+    
+    Fixed: http://tracker.ceph.com/issues/22296
+    
+    Signed-off-by: Tao Chen <chentao@umcloud.com>
+    (cherry picked from commit 0fa2be059e649684a4fe15a456d564acde8e7831)
+
+commit 238ddccbbc4ae83e1cd49d2522bf1d43d62ae4f9
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Nov 9 16:11:14 2017 -0800
+
+    ceph-objectstore-tool: Fix set-size to clear data_digest if changing disk size
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 7ca3ce93099de9412012b7a88121f9fced34f4b8)
+
+commit a0c0e637fe276aa1b6223522ca00a810e83d5084
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jan 17 10:38:29 2018 -0600
+
+    osd: only exit if *latest* map(s) say we are destroyed
+    
+    It's possible our current map is older, we were destroyed then, but in
+    newer maps our osd was recreated.  This happens when the oldest map after
+    a recreated osd happens to land on an epoch where the osd was marked
+    destroyed.
+    
+    Fix by only exiting if one of the newest maps says we are (still)
+    destroyed.
+    
+    Fixes: http://tracker.ceph.com/issues/22673
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4e4d6466aab259eb0436989401b80b6c3cf87a7c)
+
+commit 21aef40b727ddfd7fba4e4c341ddd1cf2e21c404
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 15 14:09:14 2017 -0400
+
+    rgw: start BucketTrimManager in RGWRados
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 74aedaef0757f56a77671d6de59628538cfa0bbb)
+
+commit 0d86081fcd74c85d2834aa7e7279a1fdf26e0c04
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Sep 7 16:27:53 2017 -0400
+
+    rgw: add BucketTrimInstanceCR
+    
+    fetches bucket sync status from each peer, calculates the min markers
+    for each shard, and trims the bilog shards. calls the TrimObserver on
+    success
+    
+    Fixes: http://tracker.ceph.com/issues/18229
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 35cf54e1059bd0a46e99fecc191c117f656c7b02)
+
+commit da126d8fe2637f09688e343edb7ddaf4d0f60b37
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Sep 7 16:24:13 2017 -0400
+
+    rgw: add HTTPManager to BucketTrimManager
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 04400bc60b1cf9efbc49761cba8311e0a95dfc9e)
+
+commit 6b41f30976c292dce92a5a19f0949778c55aed56
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Sep 7 16:17:46 2017 -0400
+
+    rgw: RGWGetBucketInstanceInfoCR takes rgw_bucket or metadata key
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 689badbe7fb468d4aec14e2f86bfc4cce9c103a2)
+
+commit aba2e0b64c0bc9e8dde20296a258b0d9850b5f72
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Sep 14 16:50:47 2017 -0400
+
+    rgw: add RGWRadosBILogTrimCR
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 0dea44dad3a572cd966d22dd345c1e226c073f6e)
+
+commit 1efe55f46ea366233a8aaeb73ea58add944dfea0
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Sep 14 16:51:13 2017 -0400
+
+    rgw: add RGWBucketInfo overload for BucketShard::init
+    
+    for use by callers that have already read the bucket instance info
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit f0caa1ca8a1d502128fd45237c1e7c13118a9711)
+
+commit 8dc99bbd6931dc47a6197c07e6a6dde383edc0cb
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Sep 7 15:26:35 2017 -0400
+
+    rgw: add /admin/log rest api for bucket sync status
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit fab595ab9bfe7f477ab03efee66d1a335d91f7cb)
+
+commit 001f55774cdef304f1d4ec5a7b4a424ad697d0ae
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Sep 7 16:17:00 2017 -0400
+
+    rgw: add json decoders for bucket sync status
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 28c6ac62b877f0eafc94facb77255bfe81f48c38)
+
+commit 2c71e5cdda0eb2bfe871653a40b95627546f7612
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Sep 12 12:48:56 2017 -0400
+
+    rgw: add rgw_bucket_sync_status() to bypass manager
+    
+    RGWBucketSyncStatusManager::init() is doing a lot of extra work that's
+    not needed to serve the rest api (spawning an http manager thread,
+    fetching the bucket instance info, etc)
+    
+    uses RGWShardCollectCR to limit the number of concurrent reads to 16
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 403d139c673180ce00eab420a2fdebdacf366dac)
+
+commit b26e997d0a57f9845ed1bc192e0b92c4307da07f
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Sep 6 12:59:03 2017 -0400
+
+    rgw: add 'radosgw-admin bilog autotrim'
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 4f736f1ec74f74e8dc1ae4ffb5b71274351c66f0)
+
+commit f65ad4a3fa9637c5eb69422d95386dd7ed2a3888
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Sep 7 12:48:47 2017 -0400
+
+    rgw: add configure_bucket_trim()
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit d29f96ae3e1b7d65b2f513340e8c2c42cf6de9f1)
+
+commit b72f8aefc277b607d6207eac1586de85e02a8f4d
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 1 11:06:30 2017 -0400
+
+    rgw: BucketTrimManager implements BucketTrimObserver
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 7be4eab8a339e9e083352a44ad09272da717c73e)
+
+commit e7b3a636a76daa60dc12e1aee02b6a7ccfb9a779
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 1 12:40:58 2017 -0400
+
+    rgw: collect cold buckets for trim
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 06a22a134f9af92753fa206eb64025472ec94f40)
+
+commit d13f6a10fa3d5a92ed1a676cece453537af1a374
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 1 12:37:56 2017 -0400
+
+    rgw: add BucketTrimStatus
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 2c07d7dd0e6c358bcdba409747ebf13f846a77e1)
+
+commit fbc356a2df53368c2fb72d34f6d59faae2eda7a0
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Sep 7 12:12:43 2017 -0400
+
+    rgw: add MetadataListCR to loop over bucket instances
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit c9d50860b735809e343a0831ba3d346b0b37973c)
+
+commit f746e86bcade6edf1374578b3b64d2f5b9594b88
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 1 12:31:44 2017 -0400
+
+    rgw: add BucketTrimCR to spawn trim for active buckets
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 82c059530871af054c48de0e6a091b1f74f2eb12)
+
+commit 8434b7bd3e85352b24f8dff08f805556115c58e8
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 1 12:22:47 2017 -0400
+
+    rgw: add BucketTrimPollCR for interval and lease logic
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 129fc99d5208279029ff1722d21f0ad24c37db62)
+
+commit 1d8dbaebca536cbf0341d4d0284dcbad191e64ad
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 1 11:29:55 2017 -0400
+
+    rgw: add TrimCounters api to BucketTrimWatcher
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 5bcf109eac30780cfa9ae5d524d2bde638651f40)
+
+commit e34da7e6b35ed29560189e8ccf7d2c8f46861df5
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 1 11:26:01 2017 -0400
+
+    rgw: add BucketTrimWatcher to serve watch/notify apis
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit f96d9a8e22195bfb3347b5add7a4385895d36f9c)
+
+commit fb7e2c0e6377a1f3bd3f32a0fe81a76e7ac53ea2
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 1 10:57:41 2017 -0400
+
+    rgw: BucketTrimManager implements BucketChangeObserver
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit b4249cc432b5b74e5c8f545bdc7daddbc9e60d23)
+
+commit 92c9fd9521aac1fed4f2bf6d991a62280351c9a6
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Aug 24 10:01:36 2017 -0400
+
+    common: introduce BoundedKeyCounter and unit test
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit e9a5ec9f64dd6cd163a855335b846181c3ac83d2)
+    
+    Conflicts:
+            src/test/common/CMakeLists.txt (add_ceph_unittest)
+
+commit 3aba3f5975c7abaac104732155f567e2ec2685b7
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Sep 1 10:51:57 2017 -0400
+
+    rgw: add skeleton for BucketTrimManager
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 4309adb36be8dff737ab2196f59743c316b12bca)
+    
+    Conflicts:
+            src/rgw/CMakeLists.txt (sync tracing not backported)
+
+commit 82c1b7d75a2b4486e8c41440deafa6326641e73e
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Aug 30 16:19:36 2017 -0400
+
+    rgw: introduce RGWRadosNotifyCR for aio_notify
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 964d966969bf757570bffb650ebb4ef515a6592e)
+
+commit 7308044ac932ada012b22ce9b0ab65ce70d9bec9
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Aug 29 15:51:56 2017 -0400
+
+    rgw: MetadataManager interface takes const string refs
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit b3a5c5a0569f95a5a755ea0dc03ec2c1ea4bdc4d)
+
+commit 7d6984b554158f8d5c3bae0223bfdf740ab9926d
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Jan 22 10:34:57 2018 -0500
+
+    qa/suites/upgrade: disable broken kraken upgrade cls_rbd test
+    
+    Fixes: https://tracker.ceph.com/issues/22740
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit e0b85124193882fb1eb04fec0ab2d907523a2dde
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Fri Jan 19 13:05:27 2018 -0500
+
+    rgw_file: alternate fix deadlock on lru eviction
+    
+    This change is an alternate fix for two problems found and fixed
+    by Yao Zongyou <yaozongyou@vip.qq.com>.
+    
+    The deadlock can be avoided just by not taking it in the recycle
+    case, which invariantly holds the lock.
+    
+    The invalidation of the insert iterator by the recyle-path unlink
+    we'd like to handle as a condition in order to preserve the cached
+    insertion point optimization we get in the common case.  (The
+    original behavior was, indeed, incorrect.)
+    
+    Based on feedback from Yao, removed the RGWFileHandle dtor version
+    of the unlink check, which I think happened twice.
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit 3cf0880f86b8f7911139c4e3d672cf47420c8f49)
+
+commit e8822fe18f68eefd8043e36dd803bda1c2aa4f0c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Jan 16 10:44:29 2018 -0800
+
+    tests: recovery-unfound-found test needs to account for correct misplaced calculations
+    
+    The test expected HEALTH_OK when in a state with misplaced objects therefore HEALTH_WARN
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 9f103f013c08bb215f58247f75d33f4f95bb3719)
+
+commit 40ecfe282e3f6584cb7f61ffbe99ea7cde7dfa18
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Jan 10 13:30:41 2018 -0800
+
+    osd: Don't start recovery for missing until active pg state set
+    
+    I was seeing recovery hang when it is started before _activate_committed()
+    The state machine passes into "Active" but this transitions to activating
+    pg state and after commmitted into "active" pg state.
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 64047e1bac2e775a06423a03cfab69b88462538c)
+
+commit 02660d92390b4e4ba3ce46ccc1eba7bf0644460b
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Nov 16 14:26:27 2017 -0600
+
+    osd/PG: restart recovery if NotRecovering and unfound found
+    
+    If we are in recovery_unfound state waiting for unfound objects, and we
+    find them, we need to restart the recovery reservation process so that we
+    can recover.  Do this by queueing DoRecover() event instead of calling
+    queue_recovery() (which won't do anything since we're not in
+    recoverying|backfilling pg states).
+    
+    Make the parent Active state ignore DoRecovery so that if we are already
+    in some phase of recovery/backfill the event gets ignored.  It is already
+    handled by the other important substates that care, like Clean (for
+    repair's benefit).
+    
+    I'm not sure why states like Activating are paying attention tot his vevent...
+    
+    Fixes: http://tracker.ceph.com/issues/22145
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 4cfe31c63b519f2dce22f061c9951c302f6efb1e)
+    
+    Conflicts:
+            src/osd/PG.h (trivial, not all events exist in luminous)
+
+commit 673df31fe2fa6244b4327e646e47605aa482aa43
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Nov 16 15:20:01 2017 -0600
+
+    qa/suites/rados: test for recovery_unfound bug
+    
+    See http://tracker.ceph.com/issues/22145
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 25b7965f8803ae68759973f4afe515d3da9c0f3f)
+
+commit f4f25becde3d83ea10a02e2fb19ec8bfe1d792c9
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Nov 16 15:11:45 2017 -0600
+
+    osd/PG: document state hierarchy
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e2a75c91dd21f362d0b7fb6acf6a360beefd4168)
+
+commit 23a08fce7db373cfed8e69b379f82654110e436a
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Dec 12 09:22:37 2017 -0600
+
+    osd/PG: include primary in PG operator<< for ec pools
+    
+    Otherwise it is confusing!
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 0d98f262c0894b11b77deb3a4bf494f20be486c3)
+
+commit 029594a080b5f02c47da4f08ab4b99c77ddb75a5
+Merge: 1a69f3ed7c 747d05c298
+Author: Sage Weil <sage@newdream.net>
+Date:   Mon Jan 22 09:02:32 2018 -0600
+
+    Merge pull request #19257 from ifed01/wip-ifed-rocksdb-fix
+    
+    luminous: rocksdb: fixes early metadata spill over to slow device in
+
+commit ac064e1fc846a5a0db0062e6acfca3e0c623137c
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 24 15:20:49 2017 -0500
+
+    ceph_test_objectstore: do not change model for 0-length zero
+    
+    We almost fixed this in the pr merged at 2dbbb351e2e0e3703880023bf51f55790f763e04
+    but missed this piece.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 390779d1aa9b46648e4f5390dc431c255c70385d)
+
+commit 6a691bb79db67ebe58e2d5179deaa1d7f17e391d
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 6 15:29:32 2017 -0500
+
+    os/bluestore: 0-length zero should not change object size
+    
+    Fixes: http://tracker.ceph.com/issues/21712
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b588eaf2b0fdf06c94104d5a542bd571499f2b85)
+
+commit 445f9b52face0d44fb386bbb9a9704ad803322fa
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 6 15:29:15 2017 -0500
+
+    os/filestore: make 0-length zero avoid touching object length
+    
+    Fixes: http://tracker.ceph.com/issues/21712
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 979b7c9cd2e4919b8c7d5ed2418019310959f4bf)
+
+commit a146e49c51d3ed0d82a293bd5b809e4e97961732
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 6 15:28:49 2017 -0500
+
+    os/ObjectStore: 0-length zero does not change length of object
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9ad1f4f10ff7bfe32d0a37361640fe5c65e56699)
+
+commit 2260f03b360e8226bf5875c5840dabc07080d174
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jan 17 11:23:39 2018 -0600
+
+    mon/OSDMonitor: squelch 'id' arg errors
+    
+    Several commands use 'id' but for other types (e.g., vector<int64_t>).
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 0580f0125bc014e5d97b8317a1bd53a3495f03bf)
+    
+    Conflicts:
+            src/mon/OSDMonitor.cc: pass g_ceph_context instead of cct to
+    cmd_getval() to be consistent with other callsites of this function. we
+    can do the s/g_ceph_context/cct/ in another commit, if it turns out to
+    be a critical fix.
+
+commit e07090345249bba6d3720bff62ab13641b1a9c38
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jan 17 11:22:55 2018 -0600
+
+    use stringsream::str(), not rdbuf()
+    
+    Fixes: http://tracker.ceph.com/issues/22715
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 918361e92518056bf89c3d19a03e9398c7a74b97)
+
+commit 1a69f3ed7c4b13940848a32bbf474f8b6b77f3bf
+Merge: 7a4b42185b 58d6d043b2
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Jan 19 15:10:19 2018 -0600
+
+    Merge pull request #19071 from smithfarm/wip-22181-luminous
+    
+    luminous: rgw segfaults after running radosgw-admin data sync init
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+
+commit 7a4b42185b8948fc3d078ace811cb3fc9178a396
+Merge: 6d56c2fdef 23b3c9de3d
+Author: Sage Weil <sage@newdream.net>
+Date:   Fri Jan 19 15:09:45 2018 -0600
+
+    Merge pull request #19077 from smithfarm/wip-22171-luminous
+    
+    luminous: rgw: log keystone errors at a higher level
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 6d56c2fdef336f8ce1034a67cd18758690055827
+Merge: 63a05442c3 5b9f1281a3
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Fri Jan 19 16:08:03 2018 -0500
+
+    Merge pull request #19810 from adamemerson/wip-luminous-21901
+    
+    luminous: rgw: bucket policy evaluation logical error
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 63a05442c33d191786ff781dba8fd47b5297b0c2
+Merge: e24edea8d9 9db61c8b14
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Fri Jan 19 16:06:08 2018 -0500
+
+    Merge pull request #19847 from adamemerson/wip-22541-luminous
+    
+    luminous: rgw: put bucket policy panics RGW process
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit e24edea8d94639f6fb2da5c75dea397b2e06e258
+Merge: ca5ae981f6 c49417d8b0
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Jan 19 13:03:58 2018 -0800
+
+    Merge pull request #19785 from linuxbox2/luminous-22410
+    
+    luminous: rgw: fix chained cache invalidation to prevent cache size growth
+    
+    Reviewed-by: Adam Emerson <aemerson@redhat.com>
+
+commit ca5ae981f65da0a2f8213cd3ea52cb37db432e96
+Merge: f2c5146f40 34864fe7ba
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Jan 19 13:02:34 2018 -0800
+
+    Merge pull request #19788 from adamemerson/wip-luminous-cache
+    
+    luminous: rgw: Random 500 errors in Swift PutObject (needs cache fixes)
+    
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+
+commit f2c5146f409ab4f51b17ae311ad8090ea3ccf5b7
+Merge: e002a82db8 b9bf9b1255
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Fri Jan 19 16:01:40 2018 -0500
+
+    Merge pull request #19053 from shinobu-x/wip-22187-luminous
+    
+    luminous: rgw: add cors header rule check in cors option request
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit e002a82db83f20812f6aa1c70165f1275c4efff7
+Merge: ccd4b53800 5bf8d71001
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Jan 19 13:01:10 2018 -0800
+
+    Merge pull request #19819 from adamemerson/wip-backport-22601
+    
+    luminous: rgw: S3 API Policy Conditions IpAddress and NotIpAddress do not work
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit ccd4b538003acee14dfb9f72f5c39a38ab6ec12b
+Merge: 21ff007c65 274fef45c2
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Fri Jan 19 16:00:56 2018 -0500
+
+    Merge pull request #19050 from shinobu-x/wip-22184-luminous
+    
+    luminous: Dynamic bucket indexing, resharding and tenants seems to be broken
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 21ff007c656327fe4bfcdb723bb97fdb64e26985
+Merge: 599d6a1681 90b56a2903
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Fri Jan 19 16:00:38 2018 -0500
+
+    Merge pull request #19085 from shinobu-x/wip-22215-luminous
+    
+    luminous: rgw: bucket index object not deleted after radosgw-admin bucket rm --purge-objects --bypass-gc
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 599d6a16814965d721259bac94ed92fdb1964ce5
+Merge: 7a686ddb09 a88e48e7d9
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Fri Jan 19 15:54:24 2018 -0500
+
+    Merge pull request #18870 from shinobu-x/wip-21949-luminous
+    
+    luminous: rgw: null instance mtime incorrect when enable versioning
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 7a686ddb093b4198ba44501ab0d5993d868c907f
+Merge: c143de468a 860625046d
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Fri Jan 19 15:54:05 2018 -0500
+
+    Merge pull request #18764 from smithfarm/wip-22017-luminous
+    
+    luminous: rgw: Segmentation fault when starting radosgw after reverting .rgw.root
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit c143de468a9a279e95e37d65354733bdb3895a20
+Merge: 92a1908c88 cb38378e5d
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Fri Jan 19 15:53:53 2018 -0500
+
+    Merge pull request #18765 from smithfarm/wip-22024-luminous
+    
+    luminous: RGWCrashError: RGW will crash if a putting lc config request does not include an ID tag in the request xml
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 92a1908c88bb862c13363122851acc5739ea6bc1
+Merge: 11a3dae295 ed87de0622
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Fri Jan 19 15:53:40 2018 -0500
+
+    Merge pull request #18766 from smithfarm/wip-22021-luminous
+    
+    luminous: rgw: modify s3 type subuser access permission fail
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 11a3dae29509ceb00bcc93da94b9873cc457229f
+Merge: 7ae70b8b6a dbd70f0b82
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Fri Jan 19 15:53:28 2018 -0500
+
+    Merge pull request #18867 from shinobu-x/wip-22027-luminous
+    
+    luminous: multisite: destination zone does not compress synced objects
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 7ae70b8b6a3efc89895e8fdf74e929a1d41eaf1c
+Merge: 262bd71520 2dc009e68e
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Jan 19 12:48:23 2018 -0800
+
+    Merge pull request #18868 from shinobu-x/wip-22026-luminous
+    
+    luminous: Policy parser may or may not dereference uninitialized boost::optional sometimes
+    
+    Reviewed-by: Adam Emerson <aemerson@redhat.com>
+
+commit 262bd71520169871cbf5cf050bf4dd6eeee4f7da
+Merge: 14ca29d5cf eb0c60f7d7
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Fri Jan 19 15:46:58 2018 -0500
+
+    Merge pull request #19538 from shinobu-x/wip-22434-luminous
+    
+    luminous: rgw: user stats increased after bucket reshard
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit 14ca29d5cfc6c088c39bb47b26d2a9533f552f48
+Merge: b26aeac2c4 4085e87b84
+Author: Casey Bodley <cbodley@users.noreply.github.com>
+Date:   Fri Jan 19 15:46:21 2018 -0500
+
+    Merge pull request #19489 from dplyakin/luminous-get-website-error-code
+    
+    luminous: rgw: fix GET website response error code
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+
+commit b26aeac2c462e47d91f9847a25e79ae5e6b9b0e2
+Merge: 370abb546d 05b60db8ef
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Jan 19 12:44:55 2018 -0800
+
+    Merge pull request #19090 from pdvian/wip-22179-luminous
+    
+    luminous: rgw: Fix swift object expiry not deleting objects
+    
+    Reviewed-by: Casey Bodley <cbodley@redhat.com>
+    Reviewed-by: Amit Kumar <amitkuma@redhat.com>
+
+commit 370abb546d871d8cb72f61bc7d74a547b90e4112
+Merge: 85558065d4 b013f7fdb5
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Jan 19 12:43:07 2018 -0800
+
+    Merge pull request #19434 from cbodley/wip-luminous-pr-16807
+    
+    luminous: rgw: revert PR #16807
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+
+commit 85558065d46dde2d1d26b2b159b09e77f73de432
+Merge: eaedb0f06c 27a1a7f6fa
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Jan 19 12:41:22 2018 -0800
+
+    Merge pull request #19502 from shinobu-x/wip-22397-luminous
+    
+    luminous: rgw: radosgw-admin reshard command argument error.
+    
+    Reviewed-by: Orit Wasserman <owasserm@redhat.com>
+
+commit eaedb0f06cbf1287c636cb0a3394179edd049506
+Merge: ad11a8e9b2 72de713bf6
+Author: Yuri Weinstein <yuri.weinstein@gmail.com>
+Date:   Fri Jan 19 12:40:45 2018 -0800
+
+    Merge pull request #19506 from shinobu-x/wip-22388-luminous
+    
+    luminous: rgw: 501 is returned When init multipart is using V4 signature and chunk encoding
+    
+    Reviewed-by: Radoslaw Zarzynski <rzarzyns@redhat.com
+
+commit e066b69178b7b9a98d8e88bed9bd41cdec08c4c1
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Nov 28 17:51:52 2017 -0800
+
+    test: Verify stat calculations during backfill
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 88ce0c1a91178c25c75e9b627c2a2b64b0969648)
+
+commit 98cf928941303ed96b9b8e51e16c3e99b12812e3
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Nov 17 17:01:56 2017 -0800
+
+    test: Verify stat calculations during recovery
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit f5af1af6d3727f915dfcf7ea8a243b4a52b43859)
+
+commit f7bc7475b072724104003f12f439fa7502754598
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Jan 11 14:22:54 2018 -0800
+
+    ceph-helpers.sh: Add flush_pg_stats() to wait_for_clean() to make it reliable
+    
+    osd-scrub-repair.sh: Fixes for omap keys landing on different OSDs due to flush
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit aeba36a660cd4ab59669ee15bf92cb9640df0371)
+
+commit 82b1a41e10641dc9e8ccc90f7dd2f00e366eec92
+Author: David Zafman <dzafman@redhat.com>
+Date:   Mon Dec 4 13:02:04 2017 -0800
+
+    qa: Ignore degraded PGs when injecting random eio errors
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit c77941f593755f79e4a800545d7ff437bbe0562d)
+
+commit 7065a781a42c0ab94155f92b0719e429105a8072
+Author: David Zafman <dzafman@redhat.com>
+Date:   Sat Nov 18 10:16:53 2017 -0800
+
+    osd: Improve the way insufficient targets is handled to be compatible with EC
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 91d1487ecfb497905a20556baf699ce8dde1cc30)
+    
+    Conflicts:
+            src/osd/PG.cc (trivial)
+
+commit 6a652944e376e9f319996a87d0c7a253eed019c1
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Oct 31 18:15:53 2017 -0700
+
+    osd: Improve pg degraded state setting based on _update_calc_stats() degraded count
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 74f9e70903723bdad190bc3f71a2ca2109bfe4f7)
+    
+    Conflicts:
+            src/osd/PG.cc (trivial, ignore change to code not present)
+            src/osd/PG.h (trivial, white space diff)
+
+commit 1afeb8bc5e3291d517beed781b8698bba3280b2c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Oct 26 15:36:54 2017 -0700
+
+    osd: Handling when recovery sources have missing
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit b769b98686e98b16a74318346ddf0cee88ac7476)
+
+commit b683ba9fe9d01b1010c2fbbb48a4d654a5831f5b
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 25 21:44:14 2017 -0700
+
+    osd: Base pg degraded state on num_degraded_objects
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 5ffde0002f605401fb99535ff86afa0eb0404c67)
+
+commit 15bd330eac78e00d20b0dab6a5ac3dbc1fc9897d
+Author: David Zafman <dzafman@redhat.com>
+Date:   Wed Oct 25 09:37:00 2017 -0700
+
+    osd: Rewrite _update_calc_stats() to make it cleaner and more accurate
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit e1075f107372c97580b984c74666027fc2e71daf)
+    
+    Conflicts:
+            src/osd/PG.cc (trivial)
+
+commit 9142833bec8adf3d4e0355c796d5c5f5c5fa115c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Tue Nov 14 13:00:06 2017 -0800
+
+    osd: cleanup: Remove unused const vars
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 9a66a6f517ea0c9c7963b2f3127f5efb78aea346)
+
+commit 50cdacb9b9bfc658b8825b269cafdb48210b5003
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Oct 26 10:36:32 2017 -0700
+
+    osd: cleanup: Fix log message
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 01ac714aa3522dde130b48d5901e72533aa423b1)
+
+commit ad11a8e9b2d4e53389fe91748e75baacd47dbbf5
+Merge: abb91dc4f1 ad9db7eea8
+Author: John Spray <jspray@redhat.com>
+Date:   Thu Jan 18 10:02:00 2018 +0000
+
+    Merge pull request #19929 from zmc/wip-luminous-prom-pg_deep
+    
+    luminous: mgr/prometheus: add missing 'deep' state to PG_STATES in ceph-mgr pro…
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit abb91dc4f194696ea2b91a37d159ece5ccbbc9ce
+Merge: 2b7410003e 124b4e3465
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jan 17 20:13:36 2018 -0500
+
+    Merge pull request #19943 from dillaman/wip-22676-luminous
+    
+    luminous: cls/rbd: remove incompatible group features from partial implementation
+    
+    Reviewed-by: Mykola Golub <mgolub@suse.com>
+
+commit 124b4e3465ddf8751011dbd920851e251e1d2c57
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jan 17 17:21:47 2018 -0500
+
+    qa/suites: filter out cls_rbd tests against unsupported methods
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 2b7410003e332e8dd87ee5a367e712439d0744be
+Merge: cc6a3561cc 9f9db135a0
+Author: Abhishek L <abhishek.lekshmanan@gmail.com>
+Date:   Wed Jan 17 23:12:37 2018 +0100
+
+    Merge pull request #18865 from shinobu-x/wip-21631-luminous
+    
+    luminous: doc: remove region from "INSTALL CEPH OBJECT GATEWAY"
+    
+    Reviewed-By: Abhishek Lekshmanan <abhishek@suse.com>
+
+commit cadd8426b358647505e950017d9a146e6f7f6696
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Sat Jan 13 16:15:48 2018 -0500
+
+    cls/rbd: remove incompatible group features from partial implementation
+    
+    Fixes: https://tracker.ceph.com/issues/22676
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 5353569eea13e1ef6ddd752c8ebfdc367c6e7245
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jan 10 13:04:55 2018 +0800
+
+    mds: handle 'inode gets queued for recovery multiple times'
+    
+    Fixes: http://tracker.ceph.com/issues/22647
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 06bbf939ac7af9969765d50fad275c24b204a188)
+
+commit 4063b807daa9e02be7fc9829f2338d13364baeee
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Jan 10 11:52:35 2018 +0800
+
+    mds: use elist to track recover queue items
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 74f2a87887e81dc1d5895976982efb51b0a5e97e)
+    
+     Conflicts:
+            src/mds/CInode.cc
+
+commit 91be5317da05dea8f8cf9695b2059a72c4c151c4
+Author: Xuehan Xu <xuxuehan@360.cn>
+Date:   Sat Jan 6 10:40:33 2018 +0800
+
+    common: compute SimpleLRU's size with contents.size() instead of lru.size()
+    
+    As libstdc++ earlier than version 5 implement the list::size() as a O(n) operation,
+    this should be needed to avoid regression of various ceph component's performance.
+    
+    Signed-off-by: Xuehan Xu <xuxuehan@360.cn>
+    (cherry picked from commit 7e0a27a5c8b7d12d378de4d700ed7a95af7860c3)
+
+commit dc96e3c1f876ecb7fccb3238f8641d79b0dcc51d
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Tue Jan 9 16:48:18 2018 -0800
+
+    qa: increase osd count for ec testing
+    
+    Missing in d0732fc96fbc6849dd51b391d85f765c74cfb593.
+    
+    Fixes: http://tracker.ceph.com/issues/22646
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit f8c275b8b6c59c2f269990ec81e503a54c0dc7c1)
+
+commit 1142dacc6467af2643c317fa74fdeed5064c0565
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Tue Jan 9 16:47:25 2018 -0800
+
+    qa: add missing openstack configs
+    
+    First introduced in: ec6fb28eaf8e2db327e4afc115879a40c7664e07
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit e8a0f1c554cad7529e49dcacfd8506c7e33f949a)
+
+commit 8e3edae0c8b472a5fc9db119173de21a3374dffd
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jan 9 09:40:06 2018 -0600
+
+    debian: add -dbg packages for python-{rados,rgw,rbd,cephfs}
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b5bd43323fd96aa975e0500261b65a46067b68fa)
+
+commit 98eaf4435c92ae158a45df2dfef5c70506548fed
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Jan 9 08:37:15 2018 -0600
+
+    debian/rules: strip ceph-base libraries
+    
+    This includes rados classes and ec plugins.
+    
+    Fixes: http://tracker.ceph.com/issues/22640
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c3a7895178751051f8a0f2d0afae577cfe7badbd)
+
+commit cc76ab1eefab661b794d2852836441fe6e7a761b
+Author: YunfeiGuan <yunfeiguan@xtaotech.com>
+Date:   Wed Jan 3 11:43:07 2018 +0800
+
+    ceph-fuse: ::rmdir() uses a deleted memory structure of dentry leads a core
+    
+    we should add the dentry ref immediately after "get_or_create" in
+    case of the ref be put to zero.
+    
+    Fixes: http://tracker.ceph.com/issues/22536
+    Signed-off-by: YunfeiGuan <yunfeiguan@xtaotech.com>
+    (cherry picked from commit 65fcccc04a6239a9c7cbf0192de14f963ebf1079)
+
+commit 44643cae8da8068c3b9ab086b9dfc367f653e043
+Author: Igor Fedotov <ifedotov@suse.com>
+Date:   Wed Jan 3 16:48:51 2018 +0300
+
+    objectstore/store_test: fix lack of flush prior to collection_empty() call
+    
+    Fixes: http://tracker.ceph.com/issues/22409
+    
+    Signed-off-by: Igor Fedotov <ifedotov@suse.com>
+    (cherry picked from commit f669fcdfd5e5facd1b702ed113cac6f1d56bef5b)
+
+commit d2a68571a834e0c8ee8495fd82cde5d85b872098
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jan 3 08:37:12 2018 -0600
+
+    mgr/DaemonServer: fix error string
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 78b7f10d298070515418f5a71931d6235498c87f)
+
+commit adc47b95958b96322a649e6e2f5d6d4e2d58701e
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Jan 3 08:37:02 2018 -0600
+
+    qa/tasks/ceph_manager: tolerate failure to force backfill/recoery
+    
+    The pool may have been deleted out from underneath us.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8651e15c93ce419529b82c4c19feef1ab2b647c8)
+
+commit 80ea9ed3b30bcea3e1b561a160525280238af4bf
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Mon Dec 18 21:29:11 2017 -0800
+
+    qa: check pool full flags
+    
+    Cluster-wide flag removed in b4ca5ae462c6f12ca48b787529938862646282cd.
+    
+    Fixes: http://tracker.ceph.com/issues/22475
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 6e046dfc90e0a119ceb13935dc6d15efb2845184)
+
+commit 5fd01ca888b9f8b0077d07f18e62e8a4d108a38e
+Author: dongdong tao <tdd21151186@gmail.com>
+Date:   Wed Jan 3 21:35:16 2018 +0800
+
+    mds: fix dump last_sent
+    
+    issue: http://tracker.ceph.com/issues/22562
+    
+    Signed-off-by: dongdong tao <tdd21151186@gmail.com>
+    (cherry picked from commit 3d3df18bdb21aff25d1f1111718eccbb6640b5a6)
+
+commit e56106cfc1056e2479919ab8cc721cbf75791ca5
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Mon Jan 15 13:57:51 2018 -0800
+
+    create 4 lv's by default for ceph-volume tests
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+
+commit ad9db7eea86f3518faa54820ac62c6e34dc7cc7c
+Author: Peter Woodman <peter@shortbus.org>
+Date:   Sat Nov 11 16:32:37 2017 -0800
+
+    mgr/prometheus: add missing 'deep' state to PG_STATES in ceph-mgr prometheus plugin
+    
+    without this the endpoint throws 500s when any PG is doing a deep scrub.
+    
+    Signed-off-by: Peter Woodman <peter@shortbus.org>
+    Fixes: http://tracker.ceph.com/issues/22116
+    (cherry picked from commit 8c93c0eeaff406af0878916ef5b9ce958896d586)
+
+commit a1ca7f7d93acd5f7fa58b25371b9bcfb7855ef25
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Mon Nov 13 14:42:56 2017 +0100
+
+    pybind/mgr/prometheus: don't crash when encountering an unknown PG state
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 67a1b66b06bf74274c2cf1d4b5c900f2c56a990d)
+
+commit cc6a3561cc8ec65e05451073d0dfdd6efb4b9700
+Merge: 905b73442b 1961e32ce9
+Author: John Spray <jspray@redhat.com>
+Date:   Mon Jan 15 14:42:55 2018 +0000
+
+    Merge pull request #19925 from jcsp/wip-dashboard-trim-luminous2
+    
+    luminous: mgr: remove unused static files from dashboard module
+    
+    Reviewed-by: Sebastien Han <shan@redhat.com>
+
+commit e9f5612bd789ca17a01c6e51794e2f65e40ecaed
+Author: huangjun <huangjun@xsky.com>
+Date:   Mon Oct 9 22:05:21 2017 +0800
+
+    qa/standalone/osd/osd-mark-down: create pool to get updated osdmap faster
+    
+    Mon send osdmap to random osds after we mark osd down, the down osd
+    may use more than $sleep time to get updated osdmap if there is no
+    osd ping between osds. So create pool after setup cluster.
+    
+    Signed-off-by: huangjun <huangjun@xsky.com>
+    (cherry picked from commit ee618a38a9ed06b3ea4d02e46cdeae6afb376b82)
+
+commit d5e2e43de82cb1bf02b32bfea5a8e4c9bb495494
+Author: Xuehan Xu <xuxuehan@360.cn>
+Date:   Sat Jan 6 10:40:33 2018 +0800
+
+    common: compute SimpleLRU's size with contents.size() instead of lru.size()
+    
+    As libstdc++ earlier than version 5 implement the list::size() as a O(n) operation,
+    this should be needed to avoid regression of various ceph component's performance.
+    
+    Signed-off-by: Xuehan Xu <xuxuehan@360.cn>
+    (cherry picked from commit 7e0a27a5c8b7d12d378de4d700ed7a95af7860c3)
+
+commit 8b13643b324fe1be39cb45777c11652da2a80c61
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Jan 10 21:39:28 2018 -0500
+
+    config: lower default omap entries recovered at once
+    
+    For large omap DBs, reading 64k leads to heartbeat timeouts.  There
+    are numerous callchains leading to this recovery step, many of which
+    do not have heartbeat handles, so for an easily backported version
+    just change the default number of entries read. DBs approaching 100GB
+    may require an even lower setting, but this should be good enough for
+    most clusters, without sacrificing recovery speed.
+    
+    Fixes: http://tracker.ceph.com/issues/21897
+    Signed-off-by: Josh Durgin <jdurgin@redhat.com>
+    (cherry picked from commit 72c2076f2c14778982fb944ffade3f071a727d1a)
+
+commit 1961e32ce997917e819b40b7f8ec21fd14d7d174
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Aug 2 15:11:07 2017 +0100
+
+    mgr/dashboard: remove unneeded bits of AdminLTE
+    
+    AdminLTE is delivered in a handy all-in format,
+    but we don't need all this stuff at runtime.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 38df5150ee026c6dc9ec763f757713c7e4d99cdb)
+
+commit 492c170a6874f0fe12bef21ae702f5f6314b2107
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Aug 2 15:15:04 2017 +0100
+
+    mgr/dashboard: remove non-minified bootstrap
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 5ce51ebbf047e373eac624e49e2d2bcfb46f09ff)
+
+commit f081fde52e523f10f2e57133c3a86e6fa9596714
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Aug 2 15:54:17 2017 +0100
+
+    mgr/dashboard: remove un-minified datatables source
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 888f0f934e59fe28fdd97bca044702ce445bfe2e)
+
+commit 47194cdcca4e065616f74415de62e7f562eb7eb0
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 18 06:10:31 2017 -0400
+
+    mgr/dashboard: don't include remote js/css
+    
+    This will help anyone running away from the internet.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 199d7ff26959961f200b86062f6290c9c17cba9a)
+
+commit 8e4cc4f9991bc2ee5fcf01d0b1f6ff9334c6c853
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Sep 18 06:13:45 2017 -0400
+
+    mgr/dashboard: re-arrange static files
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 85a87da7aa46266abd8d4c42218bcbccce386316)
+
+commit 905b73442b8b16a898fcc8168a8ef4da792424bc
+Merge: 0e57572c31 723b0f2050
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Thu Jan 11 14:31:26 2018 +0800
+
+    Merge pull request #19880 from jcsp/wip-doc-dashboard-standby-luminous
+    
+    doc: update mgr/dashboard doc about standbys
+    
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit f59380e1598981468cdafe9f575bace41c9a61b3
+Author: Igor Fedotov <ifedotov@suse.com>
+Date:   Sat Dec 30 02:05:13 2017 +0300
+
+    os/bluestore: add asserts for fsck's used_blocks bitmap access as
+    boost doesn't always handle out-of-range access properly.
+    
+    Signed-off-by: Igor Fedotov <ifedotov@suse.com>
+    (cherry picked from commit 121c40286a60a63cda32c7111aaade2043ff18df)
+
+commit f919061a53ef8a7e4e01e841428f4bbb02e640d7
+Author: Igor Fedotov <ifedotov@suse.com>
+Date:   Fri Dec 29 20:59:16 2017 +0300
+
+    os/bluestore: refactor FreeListManager to get clearer view on the number
+    of alloc units it tracks.
+    This also fixes out-of-range access for fsck's used_blocks bitmap that
+    might happen when checking stores created prior to v12.2.2
+    Fixes http://tracker.ceph.com/issues/22535
+    
+    Signed-off-by: Igor Fedotov <ifedotov@suse.com>
+    (cherry picked from commit f64c236f278732b0fa211cd9e93c4f9d5a77a356)
+    
+    Conflicts:
+            src/os/bluestore/BlueStore.cc: Removed argument 'what' from apply.
+
+commit 723b0f2050dcb06f2772c7b1f3e5f94e68a4b2e0
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Jan 9 22:23:33 2018 +0000
+
+    doc: update mgr/dashboard doc about standbys
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit fa14812cbb584b61cb2ce687ae4c14225eecd9fe)
+
+commit 2adc133b86a50a6c79d59f81b245b6bfd5cf1e14
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Oct 31 18:31:13 2017 -0400
+
+    RGW: S3 POST policy should not require Content-Type
+    
+    In the current implementation, putting a key in the acting
+    RGWPolicyEnv makes it required in RGWPolicyEnv::match_policy_vars.
+    
+    I'm not sure this is the intent, but in any case, add it to the env
+    only if sent.
+    
+    Fixes: http://tracker.ceph.com/issues/20201
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit af29276c32f7009a78bd8b90d8f7d19026859c74)
+
+commit 6cc4a33933109bd9cd333034a538b2cc3621f56a
+Author: John Spray <john.spray@redhat.com>
+Date:   Tue Dec 19 11:41:08 2017 +0000
+
+    packaging: explicit jinja2 dependency for dashboard
+    
+    We were getting this via python-flask, when mgr was
+    installed on nodes with ceph-mon, but for correctness
+    in general we should depend on it from ceph-mgr (it
+    is imported by the dashboard module).
+    
+    Fixes: http://tracker.ceph.com/issues/22457
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 87399bea8321a7a9ab554085be3a3d5e0563f27f)
+
+commit 0e57572c311df7597c0e7c235dfe195ae04b7cfd
+Merge: 0706e9b842 2042881c67
+Author: John Spray <jspray@redhat.com>
+Date:   Tue Jan 9 11:13:04 2018 +0000
+
+    Merge pull request #19844 from jcsp/wip-22615
+    
+    luminous: mgr/dashboard: Fix PG status coloring
+    
+    Reviewed-by: Wido den Hollander <wido@42on.com>
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 0706e9b842cb6027f70adea8d8cc996763fe642b
+Merge: 3de49aa945 0f0df7db74
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Tue Jan 9 09:16:58 2018 +0000
+
+    Merge pull request #19858 from joscollin/wip-luminous-doc-misc-fixes
+    
+    luminous: doc: misc fixes for CephFS best practices
+    
+    Reviewed-by: Abhishek Lekshmanan <abhishek.lekshmanan@gmail.com>
+
+commit 0f0df7db74069fef8b526c849002f82f9d8087dd
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Fri Jan 5 11:51:00 2018 +0530
+
+    doc: misc fixes
+    
+    misc fixes for best-practices.
+    
+    Signed-off-by: Jos Collin <jcollin@redhat.com>
+    (cherry picked from commit ecacd1078db5020e6d65436f9d56cc9ed0016666)
+
+commit 2e7c40232ff12393a1afe10b8ff1669f13047b14
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Fri Jan 5 11:24:23 2018 +0530
+
+    doc: fix heading
+    
+    Fixed 'Which kernel version?' heading not having brown color background.
+    
+    Signed-off-by: Jos Collin <jcollin@redhat.com>
+    (cherry picked from commit a068fd842f431a54f9db2d437a71cdc11907a8e1)
+
+commit 17aa16dc6b993373539cecc2a4e110c0455dcbb6
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Dec 20 16:55:06 2017 -0500
+
+    librbd: filter out potential race with image rename
+    
+    Fixes: http://tracker.ceph.com/issues/18435
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 001d2fc35c170f17e706aeae541fb86dac960319)
+
+commit 9d617b4ecceda094040f4fe4db490588e6684a18
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 7 16:40:04 2017 -0600
+
+    mon/LogMonitor: add mon_cluster_log_to_stderr
+    
+    Optionally send cluster log messages to stderr (prefixed by the
+    channel).
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 49b327e8657695a359ea63e427b5de2e4525f480)
+
+commit acc76d713f11f52c39f6ba5868ca1e502c1985b2
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Dec 7 16:39:30 2017 -0600
+
+    log: add log_stderr_prefix option
+    
+    Allows you to set a prefix for debug log messages send to stderr (e.g.,
+    "debug ").
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit c142ae3f1530067f35b8a1e1bac9ddeea4d81c09)
+
+commit 473f12d4fd4c0ee747f7a1822f513ad1f87cdfb0
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Sun Nov 26 15:46:26 2017 -0500
+
+    cmake: add WITH_BOOST_CONTEXT option
+    
+    adds a more specific option for this boost::context dependency, which was
+    previously only used by the radosgw beast frontend. see
+    http://tracker.ceph.com/issues/20048 for more background
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit b589b73c70bc4c644b7a040d9aee5083881254b9)
+    
+    Conflicts:
+            src/include/config-h.in.cmake (missing HAVE_GETENTROPY from
+                      https://github.com/ceph/ceph/pull/17972)
+
+commit b1ad4b1e9e27d2b6e0cfe2b05e7693aa4c298eb5
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Dec 18 22:23:53 2017 -0500
+
+    cmake: remove Beast submodule and include path
+    
+    the beast library is included in boost 1.66
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit d85f42dd697365ad6c6be80af2e31a6415f55d5e)
+    
+    Conflicts:
+            src/rgw/CMakeLists.txt (missing SYSTEM flag from
+                     https://github.com/ceph/ceph/pull/18711)
+
+commit eb6f089631225ab4779184a3d20c7a1cbde44853
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Dec 5 14:43:06 2017 -0500
+
+    rgw: update beast frontend for boost 1.66
+    
+    Fixes: http://tracker.ceph.com/issues/22600
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit a66a6449296d39e57724fa733ac3069fecfdcdaa)
+
+commit b1bfa824b096189c8e5ee8e42f9bf400c44059d7
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Dec 18 13:03:35 2017 -0500
+
+    cmake: update minimum boost version to 1.66
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 0e47a339dface3807ab0061f9ce62fe1d2654d29)
+
+commit 1d2e15ac08f43ad2125ba0892f3beb1c632eda20
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Nov 27 14:16:33 2017 -0500
+
+    submodule: update Beast to ceph/ceph-master branch
+    
+    pulls beast submodule up from v116 tag to v124, with an additional bug
+    fix for async_read_some()
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit efe90765957371a78b36305727dc3eb34252ad91)
+
+commit acb49cfc8f2cc79b0bf38ab0a95c07398a9c5061
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Nov 9 23:20:40 2017 -0500
+
+    rgw: RGWEnv::set() takes std::string
+    
+    the beast frontend will either pass a string_ref or a string_view,
+    depending on the boost version. we can't overload RGWEnv::set() for
+    both, because a call to env.set("literal") would be ambiguous
+    
+    both string_ref and string_view have a to_string() member function, so
+    RGWEnv::set() now takes a std::string by value and moves it into the
+    map. this involes a single string copy, whether we pass a temporary
+    std::string (in beast) or a const char* (in civetweb)
+    
+    Fixes: http://tracker.ceph.com/issues/22101
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 6fbd3f358a17debea8c04f976946d5f245576f31)
+
+commit 0f08d3d9a53f1cb1f1ea13181a3c96a14f7b65dd
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Oct 18 20:22:11 2017 -0400
+
+    rgw: fix for pause in beast frontend
+    
+    pause_for_new_config() was only stopping the listener on pause, but
+    existing keepalive connections would keep trying to read. this prevented
+    the frontend thread calls to io_service::run() from returning to
+    complete the pause
+    
+    Fixes: http://tracker.ceph.com/issues/21831
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit c64ac823adfb6d717570d3d3e82b89ea73dc3a77)
+
+commit 3d79dadf4aba3f15352fbf9750ff69afe0adaf9e
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Sep 26 14:35:56 2017 -0400
+
+    qa/rgw: add beast frontend to some rgw suites
+    
+    added a qa/rgw_frontend directory for civetweb.yaml and the new
+    beast.yaml. the rgw suites for multifs and singleton now symlink
+    rgw_frontend/civetweb.yaml. the multisite, tempest and verify suites
+    symlink rgw_frontend to test both. this doubles the number of jobs in
+    those suites
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 607b72f553b3d4c1ad6feba9ee7199a4d91272f1)
+
+commit 53ff61f95db75a892da0c085e9eaf9e3425e94d2
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Oct 11 09:01:35 2017 -0400
+
+    rgw: ask beast parser about keepalive/close
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 5d7f154a77eb2cf82b6e4e49d20687c8ed839113)
+
+commit c3bb0bae2a9878ec8c3302168f82a22aed5a983d
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Sep 25 11:13:08 2017 -0400
+
+    rgw: remove boost::coroutine and context deps
+    
+    the beast frontend no longer uses stackful coroutines, so these
+    dependencies aren't necessary
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit b904b575042d6598a9cf74d23beecd4884e097d2)
+
+commit 30b4d2bb9385dfc7837f7941df5a7a3c507c8b72
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Jul 17 10:12:07 2017 -0400
+
+    rgw: beast frontend discards unread body before next header
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit c6bc7e1b0cbaf553f000aaa2893fa1486fc02a75)
+
+commit c897636ff13750acd446cd7a37b4b5c83e4d4dde
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Jul 12 11:39:26 2017 -0400
+
+    rgw: beast frontend uses callbacks instead of coroutines
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 7767d8d88aecac8e88ed4c87a187c7a2ed67cf48)
+
+commit 0a024c978b72402e4f8792c60c017d3ccbc60050
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Jul 11 16:12:02 2017 -0400
+
+    rgw: set header/body size limits on beast parser
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 1f72684eb4c6fb9fd021240ae55812894afc19e9)
+
+commit 1bc2728da84992c2a737202f04ad358470140a34
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Thu Jul 6 16:31:23 2017 -0400
+
+    rgw: update beast frontend/submodule to v116
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 746c218c620d7681f6c9d769631ee1ac0d2b5987)
+
+commit 2dd6512cdd9ed2548ceae42d5785c140b1ac2a35
+Author: Jiaying Ren <jiaying.ren@umcloud.com>
+Date:   Thu Aug 10 15:21:32 2017 +0800
+
+    rgw: fix opslog uri as per Amazon s3
+    
+    According to s3[1], current Request-URI opslog entry are missing:
+    
+    + request method
+    + query string
+    + http version number
+    
+    [1] http://docs.aws.amazon.com/AmazonS3/latest/dev/LogFormat.html
+    
+    Fixes: http://tracker.ceph.com/issues/20971
+    
+    Reported-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    Signed-off-by: Jiaying Ren <jiaying.ren@umcloud.com>
+    (cherry picked from commit 96bb12a158ab899ec219d88e140499a4e27b4ba5)
+
+commit e7bd12a38716d17a7d6b752d6730d5913916cded
+Author: Jiaying Ren <jiaying.ren@umcloud.com>
+Date:   Mon Aug 7 17:30:27 2017 +0800
+
+    rgw: fix opslog can't record referrer when using curl as client
+    
+    Fixes: http://tracker.ceph.com/issues/20935
+    
+    Reported-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    Signed-off-by: Jiaying Ren <jiaying.ren@umcloud.com>
+    (cherry picked from commit 23323b7f725dc1e99e4a81512b57d342dab9a3fe)
+
+commit 9db61c8b14a915c8f99dfe896610a2431f277f68
+Author: Bingyin Zhang <zhangbingyin@cloudin.cn>
+Date:   Tue Dec 26 17:06:44 2017 +0800
+
+    rgw: put bucket policy panics RGW process
+    
+    Fixes: http://tracker.ceph.com/issues/22541
+    Signed-off-by: Bingyin Zhang <zhangbingyin@cloudin.cn>
+    (cherry picked from commit f05a044cad18c16ebc2c9a177f38b4bdc76cfd66)
+
+commit 3de49aa94560625909794153b4ab965aeeba4b37
+Merge: 907a72465e ca980a63c3
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Mon Jan 8 17:22:30 2018 +0000
+
+    Merge pull request #19505 from shinobu-x/wip-22392-luminous
+    
+    luminous: mds: tell session ls returns vanila EINVAL when MDS is not active
+    
+    Reviewed-by: Jos Collin <jcollin@redhat.com>
+
+commit 907a72465e0f3546724d9686b3cafe5d7208cf9f
+Merge: 70755a02e6 567bd5acad
+Author: John Spray <jspray@redhat.com>
+Date:   Mon Jan 8 16:18:12 2018 +0000
+
+    Merge pull request #19831 from pdvian/wip-22576-luminous
+    
+    luminous: qa: configure zabbix properly before selftest
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+    Reviewed-by: Wido den Hollander <wido@42on.com>
+
+commit 2042881c6702a9d660b57385a908e9da7df4cd0f
+Author: Wido den Hollander <wido@42on.com>
+Date:   Mon Dec 11 11:33:16 2017 +0100
+
+    mgr/dashboard: Fix PG status coloring
+    
+    This was broken in the 12.2.2 release and PGs would not get a
+    color anymore.
+    
+    This commit also makes sure pools/PGs are marked as red when
+    one or more PGs are inactive, inconsistent, incomplete, down, etc.
+    
+    Signed-off-by: Wido den Hollander <wido@42on.com>
+    (cherry picked from commit 450aa2133d05916822083969260ee07622591e33)
+
+commit 0a347506af4b45decf604bea4144ac1eafc2d34e
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Jan 4 16:08:25 2018 +0800
+
+    rpm: adjust ceph-{osdomap,kvstore,monstore}-tool feature move
+    
+    this is the rpm's counterpart of debian/control changes related to the
+    ceph-{osdomap,kvstore,monstore}-tool feature move. see #19328 and #19356.
+    the commit introducing this move is 6dba25e. and
+    
+    $ git describe 6dba25e
+    v12.2.2-8-g6dba25e39d
+    
+    so the first release that have this change is 12.2.2-8. in other words,
+    ceph-{base,osd,mon} >= 12.2.2.8 cannot co-exist with ceph-test < 12.2.2-8
+    in the same system. so we let ceph-test Requires ceph-common with the
+    same version. and since ceph-{osd,mon} Requires ceph-base with the same
+    version, and ceph-base Requires ceph-common with the same version, so by
+    tiering ceph-test with ceph-common with the same version, we enforce
+    this restriction.
+    
+    Fixes: http://tracker.ceph.com/issues/22558
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit b07aa210aa0ede54ffc3dbe49e334bd51a8f6342)
+
+commit 70755a02e66eb82fefc7d3a0114fe5a98a7f1dc2
+Merge: c859483fee 4a79dd9065
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Jan 8 12:15:39 2018 +0200
+
+    Merge pull request #19787 from linuxbox2/luminous-21984
+    
+    rgw: fix rewrite a versioning object create a new object bug
+
+commit b5892a13b9e10f1d5cc5e3c4e869bac06c3926ca
+Author: Ilya Margolin <ilya@ulani.de>
+Date:   Thu Dec 14 17:07:01 2017 +0100
+
+    pybind/mgr/prometheus: fix metric type undef -> untyped
+    
+    Fixes "Prometheus exporter can't get metrics after update to 12.2.2"
+    
+    Fixes: http://tracker.ceph.com/issues/22313
+    Signed-off-by: Ilya Margolin <listen@ulani.de>
+    (cherry picked from commit 58669bedf530e1ccaf4d6c4bf018eeec43b78647)
+
+commit 567bd5acad2d1d5598a63835b9ce874e697cf2a8
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Dec 21 08:27:45 2017 -0500
+
+    qa: configure zabbix properly before selftest
+    
+    Even though the selftest routine doesn't care about
+    the settings, we should set them to avoid emitting
+    nasty log/health messages when enabling the module.
+    
+    Fixes: http://tracker.ceph.com/issues/22514
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit c64c9ff00d2df2177135bcd2735bd7aeac625643)
+
+commit 98e3e2a7c840f985baa4ae82a9ce943e85b99657
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Tue Dec 19 11:37:14 2017 -0800
+
+    mds: check metadata pool not cluster is full
+    
+    CEPH_OSDMAP_FULL flag was obsoleted by
+    b4ca5ae462c6f12ca48b787529938862646282cd. So, check if the metadata pool is
+    full instead which is a decent proxy (as metadata operations can still
+    proceed). However, the data pool may still be full which would result in some
+    operations still not completing (like inode backtrace updates).
+    
+    Fixes: http://tracker.ceph.com/issues/22483
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit d678415bb03afd1a67edaa0eac031a6a9cf3fbf9)
+
+commit dd6b6626ccc97666a9de24fcc770f9b7ee9b3a31
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Wed Dec 13 18:07:00 2017 -0800
+
+    vstart_runner: ignore failed dentry invalidation
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit cd1e5f53f0a732e4398efadbbf5b6ce914ddd647)
+
+commit c5c0435efcdce7e2e87cf95a0e13642b40b9a42f
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Wed Dec 13 17:50:32 2017 -0800
+
+    vstart_runner: set ec_profile attribute
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 0f507d88559e7060e86a4a3ea9fcffb02c013bec)
+
+commit e97399d2eefb13ff8083ce2d9c9686e69c30e488
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Dec 15 14:23:01 2017 -0800
+
+    mds: reduce debugging level for balancer messages
+    
+    Fixes: http://tracker.ceph.com/issues/21853
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 435babe4a44c73d094eeddb5e621ebd29a62aff8)
+
+commit 5bf8d71001bf929813fd5e3bb101c60630ab6c5d
+Author: John Gibson <jgibson@mitre.org>
+Date:   Tue Sep 19 09:17:22 2017 -0400
+
+    rgw: Policies now properly evaluate the X-Forwarded-For header.
+    
+    Signed-off-by: John Gibson <jgibson@mitre.org>
+    (cherry picked from commit 5f7d9c4ff6c78f65d074dbdf8a181cb9ae09851e)
+
+commit ac3e81b8ad887cc84e553d7dd24aa002ea536269
+Author: John Gibson <jgibson@mitre.org>
+Date:   Tue Sep 19 10:55:12 2017 -0400
+
+    rgw: Test of proper parsing of the X-Forwarded-For header for policies.
+    
+    Signed-off-by: John Gibson <jgibson@mitre.org>
+    (cherry picked from commit e02facbf5c7b06b4d1d628ddb83ee74935493def)
+
+commit 65b515b9d0c119dde57f57d1d7c75d81e175113b
+Author: John Gibson <jgibson@mitre.org>
+Date:   Sun Dec 24 15:49:50 2017 -0500
+
+    rgw: Bucket IP address policy evaluation now uses rgw_remote_addr_param.
+    
+    Previously bucket policy ip address restrictions were only being evaluated
+    against the REMOTE_ADDR environment variable and ignoring the header
+    specified by the rgw_remote_addr_param configuration option. This rendered
+    ip-based bucket policies worthless when running behind a reverse proxy.
+    
+    Signed-off-by: John Gibson <jgibson@mitre.org>
+    (cherry picked from commit c4c24ca986f17c68b75f76fc48ad489002fcf87e)
+
+commit a75908b8098ef39ed0193616e9d4fb723d73883c
+Author: John Gibson <jgibson@mitre.org>
+Date:   Thu Sep 14 10:12:59 2017 -0400
+
+    rgw: rgw_iam test harness can now stringify() the fake identity.
+    
+    Signed-off-by: John Gibson <jgibson@mitre.org>
+    (cherry picked from commit 557bdb617e6eb78f9f0e6d6ba35b79b318a6c275)
+
+commit 1d8f684a3cf11c5fc422462790b5a87d5ab5b587
+Author: John Gibson <jgibson@mitre.org>
+Date:   Sun Dec 24 15:48:00 2017 -0500
+
+    rgw: Fixed several bugs in policies related to IPv6 addresses.
+    
+    The IPv6 conversion was not properly converting the address to host byte
+    order.
+    The text conversion of IPv6 addresses was using raw byte values instead of
+    the converted number. The portions of the addresses were grouped by bytes
+    instead of 16-bit words. The prefix length was erroneously being rendered
+    in hex.
+    
+    http://tracker.ceph.com/issues/20991
+    
+    Signed-off-by: John Gibson <jgibson@mitre.org>
+    (cherry picked from commit a25ca37401d1e8dc4349201b9f64aa6990bea0d5)
+
+commit 9f7aeb4bd52bbdaef4ffd0127b38c3084e95a706
+Author: John Gibson <jgibson@mitre.org>
+Date:   Sun Dec 24 15:44:54 2017 -0500
+
+    rgw: Fixed several bugs in policies related to IP Addresses.
+    
+    Comparisons of two individual IP addresses caused an assertion error.
+    The text conversion of IPv4 addresses was using raw byte values instead of
+    the converted number.
+    NotIpAddress condition now works with multiple values.
+    
+    http://tracker.ceph.com/issues/20991
+    
+    Signed-off-by: John Gibson <jgibson@mitre.org>
+    (cherry picked from commit ca21596c95aa3871d14ac9112840d3b1101a8635)
+
+commit 5fbb50c1880b0dafc64892e0c23d188c2eccbec7
+Author: John Gibson <jgibson@mitre.org>
+Date:   Tue Dec 26 16:40:31 2017 -0500
+
+    rgw: Added tests for S3 Policy IP Address feature.
+    
+    Signed-off-by: John Gibson <jgibson@mitre.org>
+    (cherry picked from commit 3d260f26e6826182b4de520878b548a28a96e81b)
+
+commit e4a5cf95e645d118a781b65c4ca6e5c37dd7eb6c
+Author: John Gibson <jgibson@mitre.org>
+Date:   Thu Aug 10 13:39:35 2017 -0400
+
+    rgw: Added support for testing the wildcard principal in policies.
+    
+    Signed-off-by: John Gibson <jgibson@mitre.org>
+    (cherry picked from commit e83b647d44153a7775647693fca1848e592ee107)
+
+commit 7cca4a6c1034ce32c41b73bace538965c5e77a8c
+Author: yuliyang <yuliyang@cmss.chinamobile.com>
+Date:   Mon Nov 27 14:32:44 2017 +0800
+
+    rgw: implement ipv4 aws:SourceIp condition for bucket policy
+    
+    Signed-off-by: yuliyang <yuliyang@cmss.chinamobile.com>
+    (cherry picked from commit 2fb445b6f7c1e997e83b1c7da2a1fecdde164d35)
+
+commit e49bf0889e6144b6683d8cb4554e38efe29b7a8a
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Nov 3 10:42:37 2017 -0400
+
+    rgw: simplify use of map::emplace in iam
+    
+    the piecewise_construct overloads of map::emplace() are only needed when
+    there's ambiguity around which arguments go to which constructor
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 9787fe6b8dc545419b0f3375c06a57a414ae8aa6)
+
+commit 8eb36333bc8e709cfb9e4958821548f813e105da
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Thu Jul 27 17:36:37 2017 +0200
+
+    rgw: policy: support for s3 conditionals in ListBucket
+    
+    This adds support for s3:prefix,delimeter & maxkeys identifiers when
+    specified as conditionals in policy.
+    
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit a7184ca8c507b66ef01687bd85528342ea3bf934)
+
+commit 3f8ab0b8940c332b95c517d02e4aa658f394cd6d
+Author: Jiaying Ren <jiaying.ren@umcloud.com>
+Date:   Mon Aug 7 15:55:19 2017 +0800
+
+    rgw: fix opslog can not record remote_addr
+    
+    Fixes: http://tracker.ceph.com/issues/20931
+    
+    Reported-by: Zhang Shaowen <zhangshaowen@cmss.chinamobile.com>
+    Signed-off-by: Jiaying Ren <jiaying.ren@umcloud.com>
+    (cherry picked from commit 650d30d64e084df12f4cfcb3d1edaefb5d475bb4)
+
+commit 5b9f1281a3bfe3e75bd4ecd2d3983543396d119d
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Fri Oct 27 22:48:18 2017 -0400
+
+    rgw: Fix evaluation of bucket management permissions
+    
+    I spent some more time looking through the documentation of how work
+    is evaluated, and the examples on
+    
+    <http://docs.aws.amazon.com/AmazonS3/latest/dev/
+    access-control-auth-workflow-bucket-operation.html>
+    
+    Have convinced me that the behavior that was requested is more correct
+    than what we were doing before.
+    
+    Fixes: http://tracker.ceph.com/issues/21901
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit 343a25aa2134b6fdddeca6c9dfbaefde2dc9c66a)
+
+commit febd39cf67eb026cf577d72440f5d5b61c98c3ad
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Fri Oct 27 20:17:52 2017 -0400
+
+    rgw: Refactor checking of some ops
+    
+    Since some operations check the user against the bucket owner in the
+    absence of a policy, rather than open-coding that everywhere, act like
+    a proper computer scientist and abstract it.
+    
+    Fixes: http://tracker.ceph.com/issues/21896
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit 8818a0cb5e699135976e057061fb8e9d99850cd3)
+
+commit c859483fee526656b6c6cd4d9d0c975cff57b565
+Merge: bccb4839ee ce1fe2480c
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Fri Jan 5 17:07:00 2018 +0000
+
+    Merge pull request #19449 from smithfarm/wip-22398-luminous
+    
+    luminous: doc: man page for mount.fuse.ceph
+    
+    Reviewed-by: Jos Collin <jcollin@redhat.com>
+
+commit 34864fe7ba839e0d593437b6e62f6812f1c1a375
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Wed Dec 20 17:06:32 2017 -0500
+
+    rgw: Plumb refresh logic into object cache
+    
+    Now when we force a refetch of bucket info it will actually go to the
+    OSD rather than simply using the objects in the object cache.
+    
+    Fixes: http://tracker.ceph.com/issues/22517
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit d997f657750faf920170843e62deacab70008d8b)
+
+commit 5ceb7cb998b505d2fe0cd98b534a918424e4d809
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Tue Dec 19 16:47:09 2017 -0500
+
+    rgw: Add expiration in the object cache
+    
+    We had it in the chained caches, but it doesn't do much good if
+    they just fetch objects out of the object cache.
+    
+    Fixes: http://tracker.ceph.com/issues/22517
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit 82a7e6ca31b416a7f0e41b5fda4c403d1d6be947)
+
+commit 767fec1e22247ec1c6010c308d51ba77dd31cb39
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Tue Dec 19 12:53:05 2017 -0500
+
+    rgw: retry CORS put/delete operations on ECANCELLED
+    
+    Fixes: http://tracker.ceph.com/issues/22517
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit bff7e61ca5a66b301ec49c1cf9054d1b74535832)
+
+commit bfd4091f7faf43ebc4eb81c8215e8b3ec2d4a8c4
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Fri Nov 17 17:15:26 2017 -0500
+
+    rgw: Expire entries in bucket info cache
+    
+    To bound the degree to which an RGW instance can go out to lunch if
+    the watch/notify breaks down, force refresh of any cache entry over a
+    certain age.
+    
+    Fifteen minutes by default, and expiration can be turned off entirely.
+    
+    This is separate from the LRU. The LRU removes entries based on the
+    last time of access. This expiration patch forces refresh based on the
+    last time they were updated.
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit 4489cb58a15647a31ac0546d70400af5668404cb)
+    Fixes: http://tracker.ceph.com/issues/22517
+
+commit ce1fe2480c0cbef7337f870400bf441ef13e9544
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Fri Jan 5 19:37:31 2018 +0530
+
+    doc: fix typo
+    
+    Signed-off-by: Jos Collin <jcollin@redhat.com>
+    (cherry picked from commit a925bb4520797376b4b169bbcfe613cd1fa36429)
+
+commit fa86f78983e990b10a24d6c25526014e47370a71
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Fri Jan 5 12:41:37 2018 +0530
+
+    doc: add mount.fuse.ceph to index
+    
+    Added mount.fuse.ceph to index.rst
+    
+    Fixes: http://tracker.ceph.com/issues/22595
+    Signed-off-by: Jos Collin <jcollin@redhat.com>
+    (cherry picked from commit db23f4ce30cd521b5b8909edc8c26dc87aeecd98)
+
+commit e21b3af6a378cfffe58df3d56a9f9884adba8a36
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Mon Nov 27 16:16:35 2017 +0530
+
+    doc: doc for mount.fuse.ceph
+    
+    Created doc for mount.fuse.ceph.
+    
+    Fixes: http://tracker.ceph.com/issues/21539
+    Signed-off-by: Jos Collin <jcollin@redhat.com>
+    (cherry picked from commit 6c39818eaf39358cab8dd3dce579c932fba0d05d)
+
+commit 88c987cdb5f02ddc445389b40f9473e8fc14b8af
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 3 13:25:33 2017 -0400
+
+    qa/workunits/rbd: fixed variable name for resync image id
+    
+    Fixes: http://tracker.ceph.com/issues/21663
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 22816ed3f48b0a08a6d4e0cf0b0e14dabbb9ce69)
+
+commit a15eb7df1e3b6d273b4366e7afefd3c2ade4775e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Sep 27 09:40:08 2017 -0400
+
+    librbd: hold cache_lock while clearing cache nonexistence flags
+    
+    When transitioning from a snapshot that had an associated parent
+    to a snapshot where the parent was flattened and removed, the cache
+    was being referenced without holding the required lock.
+    
+    Fixes: http://tracker.ceph.com/issues/21558
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 16ef97830cde30efb96f7aee69834b3a5c2d5248)
+
+commit 84fff7aca5d7387566304206eb4cfbb934c2d536
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Dec 18 11:42:21 2017 -0500
+
+    rgw: dont log EBUSY errors in 'sync error list'
+    
+    these temporary errors get retried automatically, so no admin
+    intervention is required. logging them only serves to waste space in
+    omap and obscure the more serious sync errors
+    
+    Fixes: http://tracker.ceph.com/issues/22473
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit ca4510bc76ad7beed1128539aa9e424d29dd8585)
+    
+    Conflicts:
+        src/rgw/rgw_data_sync.cc ("multisite log tracing" feature - see
+            https://github.com/ceph/ceph/pull/16492 - is not being backported to
+            luminous)
+
+commit e51e1fa45e6a1740b94b56977bc927d302474f5c
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Fri Nov 17 16:16:38 2017 -0500
+
+    rgw: Handle stale bucket info in RGWDeleteBucketPolicy
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit e397b7e6d0c49d625fb2b2363311e6486f2045fe)
+    Fixes: http://tracker.ceph.com/issues/22517
+
+commit 08bf9c07e012a365d40901088feede4c4c200f84
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Fri Nov 17 16:15:04 2017 -0500
+
+    rgw: Handle stale bucket info in RGWPutBucketPolicy
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit 1738b4f6b726b462abb436f78026c1577b55f05e)
+    Fixes: http://tracker.ceph.com/issues/22517
+
+commit 3f2fa364fb7f8140d20ad7de13a69cd2cc31ec77
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Fri Nov 17 16:05:06 2017 -0500
+
+    rgw: Handle stale bucket info in RGWDeleteBucketWebsite
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit f4d274248e43cb38ff2b27782c010b2c35b12b2b)
+    Fixes: http://tracker.ceph.com/issues/22517
+
+commit b266755161b6769af0a08bf647c771a00a7aa50d
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Fri Nov 17 16:03:13 2017 -0500
+
+    rgw: Handle stale bucket info in RGWSetBucketWebsite
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit b2b7385f194def1025a8947bab876c9856b06400)
+    Fixes: http://tracker.ceph.com/issues/22517
+
+commit 672eea5d88099a590ddf847ad080214944c780cb
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Fri Nov 17 15:59:44 2017 -0500
+
+    rgw: Handle stale bucket info in RGWSetBucketVersioning
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit a0a1e7c2ef992b8758bcfb20d893730c1b202475)
+    Fixes: http://tracker.ceph.com/issues/22517
+
+commit fc271e4e3c44e520d8a530ebfc758a438c110737
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Fri Nov 17 15:53:05 2017 -0500
+
+    rgw: Handle stale bucket info in RGWPutMetadataBucket
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit ebb86301b20098e15824f469001f6153b27965f5)
+    Fixes: http://tracker.ceph.com/issues/22517
+
+commit dea29460ed513f965de2dcac9e29180b06d56b58
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Fri Nov 17 15:51:42 2017 -0500
+
+    rgw: Add retry_raced_bucket_write
+    
+    If the OSD informs us that our bucket info is out of date when we need
+    to write, we should have a way to update it.
+    
+    This template function allows us to wrap relevant sections of code so
+    they'll be retried against new bucket info on -ECANCELED.
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit 1a3fcc70c0747791aa423cd0aa7d2596eaf3d73c)
+    Fixes: http://tracker.ceph.com/issues/22517
+
+commit 4a79dd90657c6635f39de555fcc33daa584ecbd7
+Author: Enming Zhang <enming.zhang@umcloud.com>
+Date:   Tue Oct 31 15:21:21 2017 +0800
+
+    rgw: fix rewrite a versioning object create a new object bug
+    
+    Fixes: http://tracker.ceph.com/issues/21984
+    
+    Signed-off-by: Enming Zhang <enming.zhang@umcloud.com>
+    (cherry picked from commit 700a0292362128cb29586a64ef8215a07d96736b)
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+
+commit 2e3a89f019a776ad5731e49cd16b51b3f8e2e807
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Thu Nov 16 14:42:58 2017 -0500
+
+    rgw: Add try_refresh_bucket_info function
+    
+    Sometimes operations fail with -ECANCELED. This means we got raced. If
+    this happens we should update our bucket info from cache and try again.
+    
+    Some user reports suggest that our cache may be getting and staying
+    out of sync. This is a bug and should be fixed, but it would also be
+    nice if we were robust enough to notice the problem and refresh.
+    
+    So in that case, we invalidate the cache and fetch direct from the
+    OSD, putting a warning in the log.
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit 9114e5e50995f0c7d2be5c24aa4712d89cd89f48)
+    Fixes: http://tracker.ceph.com/issues/22517
+
+commit c49417d8b0f614a895bdc6f19b00868d694f8ef8
+Author: Mark Kogan <mkogan@redhat.com>
+Date:   Tue Dec 12 10:34:05 2017 -0500
+
+    rgw: fix chained cache invalidation to prevent cache size growth
+    above the rgw_cache_lru_size limit
+    
+    Fixes: http://tracker.ceph.com/issues/22410
+    
+    Signed-off-by: Mark Kogan <mkogan@redhat.com>
+    (cherry picked from commit a6a1b664d313a54ad9d2f64b859296b1352b1ce4)
+
+commit ad0ea9fa530d641fdadcae29059660446b428588
+Author: Matt Benjamin <mbenjamin@redhat.com>
+Date:   Tue Oct 31 18:31:13 2017 -0400
+
+    RGW: S3 POST policy should not require Content-Type
+    
+    In the current implementation, putting a key in the acting
+    RGWPolicyEnv makes it required in RGWPolicyEnv::match_policy_vars.
+    
+    I'm not sure this is the intent, but in any case, add it to the env
+    only if sent.
+    
+    Fixes: http://tracker.ceph.com/issues/20201
+    
+    Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
+    (cherry picked from commit af29276c32f7009a78bd8b90d8f7d19026859c74)
+
+commit 08edb1631191ad67266a1a8a66b1d147ed0abf9f
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Dec 20 09:56:50 2017 +0800
+
+    mds: respect mds_client_writeable_range_max_inc_objs config
+    
+    get_layout_size_increment() return layout.get_period(). A period
+    contain layout.stripe_count objects.
+    
+    The config is for limiting number of objects need to probe when
+    recovering a file.
+    
+    Fixes: http://tracker.ceph.com/issues/22492
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit ee11b6cd9ce97820c4f53ed98380bbd63fab891f)
+    
+     Conflicts:
+            src/mds/Locker.cc
+
+commit 8d6c2c2dea78ed485d3d63fd39085ca1e82c7d71
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Dec 18 16:48:51 2017 +0800
+
+    mds: track dirty dentries in separate list
+    
+    this should improve performance of large directory
+    
+    Fixes: http://tracker.ceph.com/issues/19578
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 49559663825ff742bde386355be864e03e646ffa)
+
+commit bccb4839ee8fde4636001976df1890b7a0f1f0a0
+Merge: 840cc7b174 c1c539f1ef
+Author: Sage Weil <sage@newdream.net>
+Date:   Wed Jan 3 11:07:39 2018 -0600
+
+    Merge pull request #19243 from liewegas/wip-fix-statfs-luminous
+    
+    mon/Monitor: fix statfs handling before luminous switchover happens
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+    Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
+commit 840cc7b1747fe12e42e7e182afd8298d1f8684b0
+Merge: d94942cdfe 2928e2cf3a
+Author: vasukulkarni <vasu.kulkarni@gmail.com>
+Date:   Tue Jan 2 14:23:35 2018 -0800
+
+    Merge pull request #19746 from ceph/wip-use-installer0-luminous2
+    
+    qa/tests - Added options to use both cases: mon.a and installer.0
+
+commit 2928e2cf3ab02357b0eea1ae9afe332bc240af06
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Tue Jan 2 10:12:54 2018 -0800
+
+    qa/tests - Added options to use both cases: mon.a and installer.0
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+    (cherry picked from commit 10fc85089c3bb64ced8c3a0ea17987e9ec5f46a2)
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit 322966a68940637a8575f9e06f07b0475ed271ca
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Wed Dec 20 13:05:22 2017 -0800
+
+    qa/tests: run ceph-ansible task on installer.0 role/node
+    
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+    (cherry picked from commit 3b2a26d919c173c887fd193f186ea56c33fcd9ae)
+
+commit 6791e67faf352b31c9e6d4e191b82ba8fac7adda
+Author: Yao Zongyou <yaozongyou@vip.qq.com>
+Date:   Fri Dec 22 16:44:31 2017 +0800
+
+    erasure-code: use jerasure_free_schedule to properly free a schedule
+    
+    Signed-off-by: Yao Zongyou <yaozongyou@vip.qq.com>
+    (cherry picked from commit 8113fa522d223a2e6708c345a5f216cbe4a5e758)
+
+commit e950f5dbf07ed8d7fb37047be797f3cfe60a1fe1
+Author: Yao Zongyou <yaozongyou@vip.qq.com>
+Date:   Sat Dec 23 12:48:47 2017 +0800
+
+    common/dns_resolve: fix memory leak
+    
+    Signed-off-by: Yao Zongyou <yaozongyou@vip.qq.com>
+    (cherry picked from commit d7c3b5940fae5a474a27b7eee72582223e1b0504)
+
+commit d94942cdfe8fc9fccf9fd79ece8bc828d608e713
+Merge: c4905daa94 eb9b1c0413
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Jan 2 09:17:16 2018 -0600
+
+    Merge pull request #19741 from joscollin/luminous
+    
+    luminous: doc: update Blacklisting and OSD epoch barrier
+
+commit eb9b1c0413aadd7f81d55a0917e840de62af67ff
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Tue Jan 2 16:17:39 2018 +0530
+
+    doc: fix grammar mistake
+    
+    Fixed grammar mistake in a sentence.
+    
+    Signed-off-by: Jos Collin <jcollin@redhat.com>
+
+commit 0e43f0f01cd800fee4cd800f1545405b449fa55b
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Tue Jan 2 16:11:30 2018 +0530
+
+    doc: update Blacklisting and OSD epoch barrier
+    
+    Updated missing OSD epoch barrier section, which is referenced by http://docs.ceph.com/docs/luminous/cephfs/full/#hammer-and-later.
+    
+    Fixes: http://tracker.ceph.com/issues/22552
+    Signed-off-by: Jos Collin <jcollin@redhat.com>
+
+commit c4905daa94d8b2fbad32a4e39c3e3076380363c5
+Merge: 7aa78f01c9 5ec37f1f49
+Author: Sage Weil <sage@newdream.net>
+Date:   Sat Dec 30 14:12:19 2017 -0600
+
+    Merge pull request #19721 from tchaikov/wip-luminous-boost-un-submodule
+    
+    luminous: boost un-submodule
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 5ec37f1f49acf8fbfcb7667be43b139a68d3aa96
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Nov 24 13:56:02 2017 +0800
+
+    make-dist: exclude unused bits in boost
+    
+    the docs, examples and tests are not used. so drop them. we could go
+    further by removing unused components in boost. but that'd be an issue
+    if somebody added a component in CMakeLists but forgets to update this
+    script. also, we need to remove boost/$component and lib/$component to
+    achieve this goal. this also introduces extra complicity. so leave it
+    for another change.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 65f91227a6b052ad60b01276d7d72dea07cbb1a4)
+
+commit 5eb42af8ce5b96b41b7d8c80cd2cc95b7eff541d
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Nov 24 10:27:19 2017 +0800
+
+    make-dist: repackage boost in the correct path
+    
+    before this change, boost is put into ./src. after this change, it is
+    put into $outfile/src. i.e. something like
+    ceph-12.1.2-4592-gf5f2ced624/src .
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 2218efa005ab871f26402b5dd33b2e599897c11f)
+
+commit 9e49711c2293761bd8b501ef1bc948bbf1e08066
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Thu Sep 28 15:28:53 2017 +1000
+
+    make-dist,cmake: Try multiple URLs to download boost before failing
+    
+    Remove SPOF during boost download for make-dist and later cmake > 3.7
+    
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 598556f98b65a03ad40ad076623b9cc8b507810e)
+
+commit 7929ff7f7edeff6df27adfdf6da5b4a89e36c4e5
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Sep 26 17:15:17 2017 -0400
+
+    make-dist,cmake: move boost tarball location to download.ceph.com
+    
+    Sourceforge is down.  Also, we can hammer our servers instead of
+    theirs.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 19987549e0ce526e353e24adddc7df8b1f165aab)
+
+commit b798225cf69da9d156e46ebe4fec463f79430d70
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Sep 1 01:00:04 2017 +0800
+
+    make-dist: download and repackage boost
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit f04436b189376cf7b53d710ab2f2bf01d47482ad)
+
+commit 3c6f83bb7c4b544118626fa92f7e9e5a407a1973
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue May 30 23:58:44 2017 +0800
+
+    boost: remove boost submodule
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 6b23056cf8d0d1bbd269bc96eef33dc27c25fd9e)
+
+commit 7aa78f01c95a3eac32e27c2c90ffe6d867cb89dc
+Merge: f519145585 84d60ae507
+Author: Nathan Cutler <presnypreklad@gmail.com>
+Date:   Fri Dec 29 18:27:43 2017 +0100
+
+    Merge pull request #19680 from tchaikov/wip-luminous-22220
+    
+    luminous: install-deps.sh: revert gcc to the one shipped by distro
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Nathan Cutler <ncutler@suse.com>
+
+commit f519145585040612801416b3f23097494d46667b
+Merge: 88b4534a0f 96b924fc75
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Tue Dec 26 01:27:04 2017 +0800
+
+    Merge pull request #19048 from shinobu-x/wip-22196-luminous
+    
+    luminous: mgr[zabbix] float division by zero (osd['kb'] = 0)
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 84d60ae507db8c290f9b8fab302bdaaacce247e1
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Dec 22 22:42:16 2017 +0800
+
+    install-deps.sh: update g++ symlink also
+    
+    we need to update g++ symlink also, if it points to the wrong version
+    
+    http://tracker.ceph.com/issues/22220
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    
+    Conflicts: the libboost issue does not affect master. as master builds
+         boost from source. so, it's not cherry-picked from master.
+    (cherry picked from commit 248a157635b46d3cf23e37ae263c62b0dc4e0e59)
+
+commit 54abba8cf87414f0d5444b1de78fd35fa379a461
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Dec 14 21:01:43 2017 +0800
+
+    install-deps.sh: readlink /usr/bin/gcc not /usr/bin/x86_64-linux-gnu-gcc
+    
+    See: http://tracker.ceph.com/issues/22220
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 5be6ff11e536cb492dd50dedf8a04fb9acc1222e)
+
+commit 2d377d351120d06382ae303e10a330de143f0a85
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Dec 13 13:36:54 2017 +0800
+
+    install-deps.sh: point gcc to the one shipped by distro
+    
+    to define a struct in a method is legal in C++11, but it causes internal
+    compiler error due to https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82155
+    if we are using GCC-7. so we need to either workaround in our source
+    code by moving the struct definition out of the member method or revert
+    to a GCC without this bug. but if we go with the first route, the jewel
+    build still fails, because GCC-7 starts to use the new CXX11 ABI, which
+    is not compatible with the libboost we use in jewel. the libboost was
+    still built with the old ABI for backward compatibility. so let's just
+    fix the install-deps.sh to point gcc to the origin one.
+    
+    See: http://tracker.ceph.com/issues/22220
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    
+    Conflicts: the libboost issue does not affect master. as master builds
+     boost from source. so, it's not cherry-picked from master.
+    (cherry picked from commit ccc4dea90e483ea8bf6bee0721ef929e7f48ff5a)
+
+commit 1643d5a4f8f726ce27ac9f5cc8160c34c07c09bb
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Sat Dec 9 14:09:24 2017 +0800
+
+    mds: properly eval locks after importing inode
+    
+    We should call Locker:eval() for all imported inodes who have non-zero
+    'wanted caps'. MDS does not properly handle following case.
+    
+    - client open a inode for read, it send a cap message to MDS.a (the cap
+      message updates 'wanted caps')
+    - MDS.a receive the cap message, the inode is non-auth and is ambiguous
+      auth. MDS.a can not request 'wanted caps' from auth mds.
+    - MDS.a finishes importing the inode from. But no caps are imported and
+      mds_caps_wanted map is empty.
+    
+    The bug can cause read hang.
+    
+    Fixes: http://tracker.ceph.com/issues/22357
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit debb556076c5685f0d8bb8029a95684b9e552eb8)
+
+commit e3bb21529b8e5a8f483cf8dab4d0012279e0373b
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Tue Nov 28 14:09:31 2017 +0100
+
+    test/cls: add a basic test for rgw usage log
+    
+    A basic test for cls_rgw for usage that sets 512 usage log entries,
+    reads them and deletes them
+    
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit ca5be9fdf88ae2f35122ee0bafb5575731b872f6)
+
+commit 7810567fa37308bfeadf63e65f08adc36bfb886e
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Tue Nov 21 16:38:27 2017 +0100
+
+    cls/rgw: trim all usage entries in cls_rgw
+    
+    Currently trim usage will only trim upto 128 omap entries, since we need
+    to run this in a loop until we're done, actually make the cls return
+    -ENODATA so that we know when to stop the loop (inspired by a similar
+    call in cls_log) this involves the following changes
+    
+    * return -ENODATA when iterate entries goes through and the value of
+      iter (which is set as the last value of key when succeeded)
+    * use IoCtx for calling the loop from within cls rather than in rgw
+    * drop the goto call in rgw_rados since we can return once we're done
+      processing
+    
+    Fixes: http://tracker.ceph.com/issues/22234
+    
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit b548a3f3443452210d92cad574bcb73ba6d2ce42)
+
+commit 7ffe8c41d3c095ebaf635aceaf2ee39d8eb58001
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Tue Dec 12 19:58:01 2017 -0800
+
+    memstore: write fsid to fsid not fs_fsid
+    
+    Fixes: http://tracker.ceph.com/issues/20736
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit e8c6882cc9c28b07ae1d5dc1169920c3ee821ed5)
+
+commit ce54fd9b995a5e28fbba462e5843c17ee58fea29
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Nov 20 09:30:50 2017 -0600
+
+    os/bluestore: prevent mount if osd_max_object_size >= 4G
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit f284bbb0593eafc25013c01c8aa7b8bdfc39d906)
+
+commit 47f74bcd136c3f9d0034ed1414442fdb03824bf8
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Mon Dec 18 18:41:31 2017 -0800
+
+    qa: don't configure ec data pool with memstore
+    
+    Fixes: http://tracker.ceph.com/issues/22436
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit b2284f23b813399613f8540efd2d426b3c6f9839)
+
+commit 6eea75200fed82a2c5513795a79834527090957c
+Author: root <tdd108128@163.com>
+Date:   Fri Dec 1 15:26:00 2017 +0800
+
+    cephfs: potential adjust failure in lru_expire
+    
+    Fix: the first adjust is no needed,it will never take real effect.
+         the second 'adjust' may never get the chance to be executed
+         suppose we can reach the second 'adjust', it will crash because the bottom list is empty now.
+    
+    Fixes: http://tracker.ceph.com/issues/22458
+    
+    Signed-off-by: dongdong tao <tdd21151186@gmail.com>
+    (cherry picked from commit 590c39eab02e64de7393c35ae7a9efb6ce626770)
+
+commit d0d66c73b02065dcdaf70905a69fc37c0f7a6e6e
+Author: root <tdd108128@163.com>
+Date:   Sun Dec 10 15:35:21 2017 +0800
+
+    cephfs-journal-tool: tool would miss to report some invalid range
+    
+    Fixes: http://tracker.ceph.com/issues/22459
+    Signed-off-by: dongdong tao <tdd21151186@gmail.com>
+    (cherry picked from commit 7708bff40cdf059a362902156a6d0660ba21fb14)
+
+commit 3d0f4fa752751a54ee2a88faa0abbb96a55fea11
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Dec 15 14:51:46 2017 -0500
+
+    rbd-mirror: cluster watcher should ensure it has latest OSD map
+    
+    Fixes: http://tracker.ceph.com/issues/22461
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9eb247961ae8bfe63c4025032ad532b7c3e3a1e7)
+
+commit 40954a9c27cb3813c8f15c2f89d7223069b9031c
+Author: Dan van der Ster <daniel.vanderster@cern.ch>
+Date:   Mon Dec 18 13:53:20 2017 +0100
+
+    mgr/balancer: fix KeyError in balancer rm
+    
+    Fix the typo in the plan name which leads to a KeyError in balancer
+    rm.
+    
+    Signed-off-by: Dan van der Ster <daniel.vanderster@cern.ch>
+    Fixes: http://tracker.ceph.com/issues/22470
+    (cherry picked from commit 4f581e738ee7c9d519fc4113f6bb0d0d1ce827ff)
+
+commit 5a40ef22449481bca0223f48be8823a3c9c2b198
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Dec 11 09:06:07 2017 +0800
+
+    osdc/Journaler: introduce STATE_STOPPING state
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+
+commit 05397e405f552bb720e12b9c0557b4c9234ffaa0
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Dec 11 08:43:57 2017 +0800
+
+    osdc/Journaler: add 'stopping' check to various finish callbacks
+    
+    These callbacks are executed by finisher. When they are being executed,
+    Journaler can be in stopping state.
+    
+    Fixes: http://tracker.ceph.com/issues/22360
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 7ffa87e6a2ab8fb2c64588411c6c2ebff2f91f93)
+
+commit 4943899005267250f42909e2a91a8a54e9479da4
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Nov 29 18:44:30 2017 +0800
+
+    mds: handle client session messages when mds is stopping
+    
+    handle session messages except CEPH_SESSION_REQUEST_OPEN. The problem
+    I found is that mds ignores CEPH_SESSION_REQUEST_RENEWCAPS, which
+    causes client sessions to become stale. Locker::revoke_stale_caps()
+    increases client caps' sequence number. This causes clients to warn
+    about caps' sequence number mismatch when handle caps import/export
+    message.
+    
+    mds should handle CEPH_SESSION_FLUSHMSG_ACK message too. Because
+    one step of exporting subtree is flushing session messages.
+    
+    Fixes:  http://tracker.ceph.com/issues/22460
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 559fdcaec28c8459c5d801efc3842e604324f91f)
+
+commit 88b4534a0fab6f7020874ff8903b2f2eb9d06eb2
+Merge: 5bef6fa349 d223a8f1b9
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Mon Dec 18 13:25:15 2017 -0800
+
+    Merge pull request #18771 from ceph/wip-yuriw-22048-luminous
+    
+    tests - Initial checkin for luminous point-to-point upgrade
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit bb3d091b463dce78f147e9f1333bd667cf2836ed
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Dec 13 13:08:46 2017 +0000
+
+    mgr/balancer: don't use 'foo' tags on commands
+    
+    This looks weird in logs when code elsewhere logs
+    that it didn't handle a command with tag 'foo'
+    
+    Fixes: http://tracker.ceph.com/issues/22361
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 432d07a0fe3e9992c21539898ce5be6c6b12a745)
+
+commit 2b9bbf605e776640b07ec52d15606f6c04fe39f8
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Wed Dec 13 18:15:27 2017 +0100
+
+    build/ops: rpm: set permissions 0755 on rbd resource agent
+    
+    Fixes: http://tracker.ceph.com/issues/22362
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 37e4bc3141ee36a4152834bd10e570bad7389807)
+
+commit c5cb0cab9f307f5bd1b58ce94b0a9e1fb510db9e
+Author: Dan van der Ster <daniel.vanderster@cern.ch>
+Date:   Wed Dec 13 17:02:44 2017 +0100
+
+    mgr/balancer: cast config vals to int or float
+    
+    upmap_max_iterations and other config vals need to be numeric.
+    Cast them appropriately.
+    
+    Signed-off-by: Dan van der Ster <daniel.vanderster@cern.ch>
+    Fixes: http://tracker.ceph.com/issues/22429
+    (cherry picked from commit d7713e6f823dd62a33a27996e1da8e29f3f7b7c5)
+
+commit 9a9ed272a76a60d9dafebbcc8ff1356ff4d7a9a6
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Wed Nov 15 18:33:17 2017 +0800
+
+    msg/async/AsyncConnection: unregister connection when racing happened
+    
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit 5216309c25522e9e4a3c3a03ceb927079de91e9b)
+
+commit 5bef6fa349207cfc88356a8a24c9ab2557179132
+Merge: d268faf20b e134317b81
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Fri Dec 15 20:52:14 2017 +0800
+
+    Merge pull request #19522 from ceph/wip-ceph-disk-deprecation-revert
+    
+    Revert "ceph-disk add deprecation warnings in favor of ceph-volume"
+    
+    Reviewed-By: Nathan Cutler <ncutler@suse.com>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit eb0c60f7d73a89dd7e77f4edcda83b81fa20e784
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Thu Nov 30 12:09:10 2017 +0200
+
+    rgw: reshard should not update stats when linking new bucket instance
+    
+    Fixes: http://tracker.ceph.com/issues/22124
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit fc95210f05367d08c679ec77d23bee6467d01980)
+    
+    Conflicts:
+            src/rgw/rgw_bucket.h: i chosed to pick code from commit fc95210 because conflicts were caused by exsiting parameter and newly added parameter which is used in a method.
+
+commit d223a8f1b91c73ee558a2d28f8af812e9583353b
+Author: Yuri Weinstein <yweinste@redhat.com>
+Date:   Mon Nov 6 12:22:27 2017 -0800
+
+    tests - Initial checkin for luminous point-to-point upgrade
+    
+    Fixes http://tracker.ceph.com/issues/22048
+    Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
+
+commit d268faf20bc30748d6efe949dc81c12812955f39
+Merge: d242b1c767 fb3b176a80
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Thu Dec 14 22:03:23 2017 +0100
+
+    Merge pull request #19532 from ceph/luminous-rm22297
+    
+    luminous ceph-volume handle inline comments in the ceph.conf file
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit d242b1c7677df993eeee64fbd2a9b128ff4ddcd4
+Merge: dd24747b44 1018f803bd
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Thu Dec 14 21:50:22 2017 +0100
+
+    Merge pull request #19530 from ceph/luminous-rm22326
+    
+    luminous ceph-volume: warn on missing ceph.conf file
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit fb3b176a80b0ada18445bd725d7670c9492509fc
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Dec 4 08:22:23 2017 -0500
+
+    ceph-volume configuration allow inlined comments for # and ;
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit aec5f5042a684952c2b07922695e7675e13645ec)
+
+commit 8fad755dd5ab1de78c636cb93dce5864cc9f07a2
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Dec 4 08:21:29 2017 -0500
+
+    ceph-volume tests verify that INI comments can be inlined
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit fe6c4c0d6055c46baadd89723c0a2b78b5ffb7f8)
+
+commit dd24747b44bd70ef82ff81efe835d7eceefb4d22
+Merge: e3f0c349b0 8748c8fb84
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Thu Dec 14 13:48:53 2017 -0500
+
+    Merge pull request #19525 from ceph/backport-19363
+    
+    luminous: ceph-volume: removed the explicit use of sudo
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit 1018f803bde2f748ea5b15928d587f80f9bc938d
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Dec 5 13:59:22 2017 -0500
+
+    ceph-volume test logging ignored ceph.conf file
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 69482d1d8059c3b264532f7623c9e6592c57b7c3)
+
+commit 72dfb268fab89bd2fa93e7d9af0894498df6c45f
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Tue Dec 5 13:58:53 2017 -0500
+
+    ceph-volume main warn on inability to load ceph.conf, don't raise
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 44c768db0da0163340b52643122b66ecebbefe23)
+
+commit e3f0c349b0ea34378bcbc183c1ea366aa484c5c4
+Merge: 4909437230 2b06a7f413
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Thu Dec 14 18:30:52 2017 +0100
+
+    Merge pull request #19528 from ceph/luminous-rm22305
+    
+    luminous ceph-volume should be able to handle multiple LVM (VG/LV) tags
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit 4909437230d8ba9f38205c4ee6ed94a4b0b549e8
+Merge: d87dbedc8f 21924133a9
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Thu Dec 14 18:26:30 2017 +0100
+
+    Merge pull request #19527 from ceph/luminous-rm22299
+    
+    luminous ceph-volume Format correctly when vg/lv cannot be used
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit d87dbedc8f83fa4c1d4ec5698e53b27e852de788
+Merge: c7383d20c4 775369d5c8
+Author: Andrew Schoen <andrew.schoen@gmail.com>
+Date:   Thu Dec 14 18:22:22 2017 +0100
+
+    Merge pull request #19526 from ceph/luminous-rm22280
+    
+    luminous ceph-volume: handle leading whitespace/tabs in ceph.conf
+    
+    Reviewed-by: Andrew Schoen <aschoen@redhat.com>
+
+commit 2b06a7f413796e2b42faca1172ee97f60adf2f1b
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Dec 4 09:56:00 2017 -0500
+
+    ceph-volume tests.api ensure that we can handle non-ceph tags in lvs
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit c98731ecede09d8d15dca99c57e331212747060e)
+
+commit 47d46678449c13a1aa45f0ae08cecac6397950a1
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Mon Dec 4 09:54:35 2017 -0500
+
+    ceph-volume api.lvm only consider 'ceph.' tags in logical volumes
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit a7e7df14602a10190a2830262a67202e7ad38f49)
+
+commit 21924133a9c8f954d86ca9da531867d09dd6298b
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Dec 1 13:53:39 2017 -0500
+
+    ceph-volume tests verify proper formatting of RuntimeError on vg/lv error
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit d7320313a15ad8bb948e501382269c8f33a8cb70)
+
+commit 9119839d6ae002fa602901340ef28271833ce08d
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Dec 1 13:53:06 2017 -0500
+
+    ceph-volume lvm.prepare correctly format argument for vg/lv error
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 7c0b00615d46a9451cb3a7731594cd7864eabc07)
+
+commit 775369d5c8832cbbb9815aeb0de8c36f33aabfd8
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Nov 30 08:58:37 2017 -0500
+
+    ceph-volume test leading whitespace is handled in configurations
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit ec7f8a27b8c51e1e1d02a2a13d9219bbb1a58c26)
+
+commit 89c9183aad044f2811dafa7e072a1db2f3412d31
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Nov 30 08:58:04 2017 -0500
+
+    ceph-volume trim tabbed/whitespaced configuration files when loading them
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+    (cherry picked from commit 9706e8cc9eb53d0f01fc78978625c1d18bf4667e)
+
+commit 8748c8fb84c23ab747aaed8acae5e5f8ce88f659
+Author: Andrew Schoen <aschoen@redhat.com>
+Date:   Wed Dec 6 10:40:55 2017 -0600
+
+    ceph-volume: removed the explicit use of sudo
+    
+    This will allow users to run ceph-volume on systems that do
+    not have sudo installed.
+    
+    Fixes: http://tracker.ceph.com/issues/22282
+    
+    Signed-off-by: Andrew Schoen <aschoen@redhat.com>
+    (cherry picked from commit aee71a3f5169043f750fd768e7fea9c74309e12d)
+
+commit e134317b8162efd3a6fa25c1c803e43228535f38
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Thu Dec 14 08:28:58 2017 -0500
+
+    Revert "ceph-disk add deprecation warnings in favor of ceph-volume"
+    
+    This reverts commit b8bf0d047868054135592188c7ebe186181310c5.
+    
+    Deprecation warnings for ceph-disk will no longer be present in any
+    Luminous release beyond 12.2.2 - but are still present in master and any
+    newer release.
+    
+    Signed-off-by: Alfredo Deza <adeza@redhat.com>
+
+commit 7dabbe91310cb0f97b9d6d09f697f74be2a8ac80
+Author: Radoslaw Zarzynski <rzarzyns@redhat.com>
+Date:   Thu Aug 17 15:05:41 2017 -0400
+
+    rgw: incorporate the Transfer-Encoding fix for CivetWeb.
+    
+    This commit updates the version of CivetWeb used for RadosGW to
+    get the fix for handling clients that send Content-Length together
+    with Transfer-Encoding. The current handling differs from S3 and
+    thus affects AWSv4.
+    
+    Fixes: http://tracker.ceph.com/issues/21015
+    Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
+    (cherry picked from commit c9e1daeffb10f36a191960328977f09ef2c05c19)
+
+commit 72de713bf697c4e444adf80fe76df87873119f1a
+Author: Jeegn Chen <jeegnchen@gmail.com>
+Date:   Thu Nov 16 09:12:24 2017 +0800
+
+    rgw: AWS v4 authorization work when INIT_MULTIPART is chunked
+    
+    Add RGW_OP_INIT_MULTIPART as a the single chunk special case
+    like RGW_OP_COMPLETE_MULTIPART.
+    
+    Fixes: http://tracker.ceph.com/issues/22129
+    Signed-off-by: Jeegn Chen <jeegnchen@gmail.com>
+    (cherry picked from commit c8827e5017ce81377cb174ad74cc2f44c3368457)
+
+commit ca980a63c3125a356cff79eba9df06fd2b37d731
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Wed Nov 22 17:20:58 2017 +0530
+
+    pybind: return error message when ceph_mds_command() returns error
+    
+    Returned the error message when ceph_mds_command() returns error.
+    
+    Signed-off-by: Jos Collin <jcollin@redhat.com>
+    (cherry picked from commit 941b58c968f6b0e359f279c8bc9e7decf51e75d1)
+
+commit 42bb91a258512c34ea402f34f130518069db3835
+Author: Jos Collin <jcollin@redhat.com>
+Date:   Wed Nov 22 17:12:43 2017 +0530
+
+    mds: Fix error message when mds not active
+    
+    Fix error message for mds not active state. Also fixed the 'unrecognized command' logic by avoiding the new stringstream local object.
+    
+    Signed-off-by: Jos Collin <jcollin@redhat.com>
+    (cherry picked from commit 10f93cf5828afbc8ff25ef81b2437ef5c0362396)
+
+commit feb4d7ba37260599952c076c23033288e671f30e
+Author: PCzhangPC <pengcheng.zhang@easystack.cn>
+Date:   Fri Oct 20 17:07:41 2017 +0800
+
+    test:add a test case in test_librbd
+    
+    Signed-off-by: PCzhangPC <pengcheng.zhang@easystack.cn>
+    (cherry picked from commit be5ad8764f65f43b6844eaa99c591a31c4d3af82)
+
+commit 13ef77c450a7f108233a6fa5f15d72393db33feb
+Author: PCzhangPC <pengcheng.zhang@easystack.cn>
+Date:   Mon Oct 16 22:46:40 2017 +0800
+
+    rbd:can not copy all image-metas if we have more than 64 key/value pairs
+    
+    Signed-off-by: PCzhangPC <pengcheng.zhang@easystack.cn>
+    (cherry picked from commit 71178643b2361557f31094e2490d4124b27cd7ff)
+
+commit 1507015169f062cbaa9b3b1e652bb34ba5d586b5
+Author: PCzhangPC <pengcheng.zhang@easystack.cn>
+Date:   Sat Oct 28 14:29:45 2017 +0800
+
+    test_librbd:add a test case of 70 key/val pairs in TestClone2
+    
+    Signed-off-by: PCzhangPC <pengcheng.zhang@easystack.cn>
+    (cherry picked from commit 85713bdcc3870728dce02f87ee3cfd36257ae533)
+
+commit c4715235c2b3d28c3aaa4584f58d3a49a89daf79
+Author: PCzhangPC <pengcheng.zhang@easystack.cn>
+Date:   Sat Oct 21 11:31:02 2017 +0800
+
+    test_librbd:add a test case of 70 key/val pairs in TestClone
+    
+    Signed-off-by: PCzhangPC <pengcheng.zhang@easystack.cn>
+    (cherry picked from commit 3e08577a296bc4ce2c1592849108926b45e9b541)
+
+commit fecfd013a8d2e251057840ee1fd112e20af45a5e
+Author: PCzhangPC <pengcheng.zhang@easystack.cn>
+Date:   Fri Oct 20 14:21:09 2017 +0800
+
+    librbd: cannot clone all image-metas if we have more than 64 key/value pairs
+    
+    Signed-off-by: PCzhangPC <pengcheng.zhang@easystack.cn>
+    (cherry picked from commit ccc56384032be7f6bd48e28b6825b3ce589c7cf7)
+
+commit 27a1a7f6fa6e5368fe1b3b56acfd81dbc49f85c6
+Author: Yao Zongyou <yaozongyou@vip.qq.com>
+Date:   Mon Oct 9 17:19:45 2017 +0800
+
+    rgw: fix command argument error for radosgw-admin.
+    
+    Fixes: http://tracker.ceph.com/issues/21723
+    Signed-off-by: Yao Zongyou <yaozongyou@vip.qq.com>
+    (cherry picked from commit bba3a89591683e6129a514af3144f1c0e8e1cc05)
+
+commit 5e5d5c8196cf99137009be049cca159aca9396ea
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 29 10:29:52 2017 -0600
+
+    qa/suites/rados/thrash: extend mgr beacon grace when many msgr failures injected
+    
+    Fixes: http://tracker.ceph.com/issues/21147
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 431d1482ffef80f9e52826a0425cae899c5dfbda)
+
+commit c22f06b3c5ee51f7bc6f4d1a0340ed7f63c106a9
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 15 10:17:07 2017 -0400
+
+    osd/PrimaryLogPG: move cache_mode==none check to top
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 5a26ac4d0f643f90d1f513ee33081faa9b0a7e7d)
+
+commit 21880df6bf27465637c97e504b5a6dcf2ea31d9f
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Aug 7 18:42:57 2017 -0400
+
+    osd/PrimaryLogPG: send requests to primary on cache miss
+    
+    If a client has {BALANCE,LOCALIZE}_READS and sends a request to a
+    replica, but the object isn't in the cache, send them back to the
+    primary.  Otherwise we might do something rash (like trigger a
+    promotion from a replica).
+    
+    Fixes: http://tracker.ceph.com/issues/20919
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 741a8720996b74434a036b143209111ce5203cbc)
+
+commit 4085e87b84a788a4285b5f18f1cd64556c08038d
+Author: Dmitry Plyakin <dplyakin@gmail.com>
+Date:   Wed Nov 29 16:03:02 2017 +0300
+
+    rgw: fix GET website response error code
+    
+    Change NoSuchKey error code to NoSuchWebsiteConfiguration, when bucket doesn't have website configuration.
+    
+    Fixes: http://tracker.ceph.com/issues/22272
+    Signed-off-by: Dmitry Plyakin <dplyakin@gmail.com>
+    (cherry picked from commit 56344f0e147e1781bb359bfde6878511b077487f)
+
+commit de05d2c81913e608ca9089b40df971f339ed0ba9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 6 15:03:34 2017 -0400
+
+    librbd: refresh image after applying new/removing old metadata
+    
+    Fixes: http://tracker.ceph.com/issues/21711
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ad01a883c5d9139d3ad3176a39a538bb8b247388)
+
+commit 235032ec642a7da077b04b26147cfd86e3ce3d86
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Sep 28 14:00:29 2017 -0400
+
+    rbd-mirror: sync image metadata when transfering remote image
+    
+    Fixes: http://tracker.ceph.com/issues/21535
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 9aa5dfb8ec46e8babd0f167dcc0e234a7c60a50c)
+
+commit 35bb3d4d17db84eac03496d22969b69ee6111b2b
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Tue Oct 24 08:49:27 2017 -0400
+
+    mds: fold mds_revoke_cap_timeout into mds_session_timeout
+    
+    Right now, we have two different timeout settings -- one for when the
+    client is just not responding at all (mds_session_timeout), and one for
+    when the client is otherwise responding but isn't returning caps in a
+    timely fashion (mds_cap_revoke_timeout).
+    
+    The default settings on them are equivalent (60s), but only the
+    mds_session_timeout is communicated via the mdsmap. The
+    mds_cap_revoke_timeout is known only to the MDS. Neither timeout results
+    in anything other than warnings in the current codebase.
+    
+    There is also a third setting (mds_session_autoclose) that is also
+    communicated via the MDSmap. Exceeding that value (default of 300s)
+    could eventually result in the client being blacklisted from the
+    cluster. The code to implement that doesn't exist yet, however.
+    
+    The current codebase doesn't do any real sanity checking of these
+    timeouts, so the potential for admins to get them wrong is rather high.
+    It's hard to concoct a use-case where we'd want to warn about these
+    events at different intervals.
+    
+    Simplify this by just removing the mds_cap_revoke_timeout setting, and
+    replace its use in the code with the mds_session_timeout. With that, the
+    client can at least determine when warnings might start showing up in
+    the MDS' logs.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 3321cc7b375a5e0ea1da4ab197ab447639ca4db3)
+
+commit 9981a1db90984b18431017d348f0a3e7ffb76f61
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Wed Oct 18 07:27:49 2017 -0400
+
+    client: add new delegation testcases
+    
+    Test basic acquire/break functionality from both other clients
+    and the same client, for different conflicting opens, as well as
+    changes to the namespace.
+    
+    Then test delegation timeout behavior. Open file, take delegation in
+    main thread. Spawn another thread to open the file again, breaking
+    delegation. Have main thread ignore it, and wait for the spawned thread
+    to be joined. Once it is, ensure that subsequent access of the cmount
+    returns -ENOTCONN.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit d9c6a9129eb65031eb1fc6f769cfe59bf3bb1cff)
+
+commit 0aa3c5c577ce4011b727123b45e89648bba32596
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Fri Dec 8 16:13:19 2017 +0100
+
+    mon: reenable timer to send digest when paxos is temporarily inactive
+    
+    Fixes: http://tracker.ceph.com/issues/22142
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 0457043d2f0040629ec425a356b1bb1f96cf0332)
+
+commit 9490a1551dcb09b9d05e5e49632dfb3277f13981
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Tue Nov 14 07:26:56 2017 -0500
+
+    client: add delegation support for cephfs
+    
+    Add the ability for ceph userland clients to request a delegation for a
+    cephfs open file. With this, userland will get a callback if there is a
+    request for access that conflicts with the delegation. The client is
+    then expected to return the delegation at which point the conflicting
+    access will succeed.
+    
+    Handing out a delegation means that we're trusting the application to
+    give back the caps when they are needed elsewhere. If it fails to
+    uphold its end of the bargain, we need to take steps to forcibly revoke
+    them, so that other clients can make progress.
+    
+    When that occurs, shut down the mount such that further calls return
+    -ENOTCONN. That should prevent the application from holding on to caps
+    indefinitely, and ensure that it will no longer do damage. It should
+    also prevent the client from ending up blacklisted.
+    
+    Since this is an implicit agreement between ceph and the application,
+    we require that the application call ceph_set_deleg_timeout() to set
+    the delegation return timeout before we hand out any delegations. This
+    helps ensure that the various application and ceph cluster timeouts
+    are all in agreement.
+    
+    An open call on ceph can easily return without any caps being granted to
+    the client. This is not generally a problem on ceph since it will wait
+    on caps before doing any operations on that open Fh, but it's less than
+    ideal when there is a delegation outstanding.
+    
+    Both NFS and SMB servers can grant deny locks, and at least in the case
+    of NFS, those can be cached such that the server is unaware of them if
+    the client holds a delegation.
+    
+    Most applications that use delegations will also want to ensure that
+    delegations are broken before allowing an open to proceed. A non-zero
+    delegation return timeout also cues the client to wait on a minimal set
+    of caps after an open return to ensure that this is the case.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit fad99776195e299b37071892e459ad8abf2fddfc)
+    
+    Conflicts:
+        src/client/Inode.cc (luminous does not have
+            a5d97ad257d6fae8f66200513805e1778cb0c8ba so we preserve the following
+            lines at the end of the Inode dtor:
+              delete fcntl_locks;
+              delete flock_locks;
+            )
+
+commit 3a613f6966ef3f32eb5f26ec5ebc5772a4dc2bf0
+Author: Jeff Layton <jlayton@redhat.com>
+Date:   Thu Oct 12 08:29:28 2017 -0400
+
+    common: remove data_dir_option from common_preinit and global_pre_init
+    
+    No one ever passes anything in there.
+    
+    Signed-off-by: Jeff Layton <jlayton@redhat.com>
+    (cherry picked from commit 85f2315bc59106b8fba6099c5e79209ea9b96efa)
+
+commit 5e8c4f83f4f161a595e75340cecbdd41b1803f39
+Author: Mykola Golub <to.my.trociny@gmail.com>
+Date:   Tue Dec 5 15:48:58 2017 +0200
+
+    pybind/rbd: raise KeyError when metadata does not exist
+    
+    Signed-off-by: Mykola Golub <to.my.trociny@gmail.com>
+    (cherry picked from commit 7a1b13f8849ce7bd325316bafad5bab660c77825)
+
+commit 2d7bea6148756fd8f3f7e124148d296d4290691f
+Author: Mykola Golub <to.my.trociny@gmail.com>
+Date:   Tue Dec 5 15:48:23 2017 +0200
+
+    pybind/rbd: fix metadata functions error handling
+    
+    Fixes: http://tracker.ceph.com/issues/22306
+    Signed-off-by: Mykola Golub <to.my.trociny@gmail.com>
+    (cherry picked from commit 2e6872b28eccd10b8bafdadaf3e6049123792022)
+
+commit c7383d20c4258984de2fb524f2751207aa150d78
+Merge: b3fd46810c 5bd076ef41
+Author: Sage Weil <sage@newdream.net>
+Date:   Tue Dec 12 09:04:20 2017 -0600
+
+    Merge pull request #19042 from liewegas/wip-22128
+    
+    mon/OSDMonitor: fix ruleset-* to crush-* fixup
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+    Reviewed-by: xie xingguo <xie.xingguo@zte.com.cn>
+
+commit 08a2358f8520efe03eac435d1f8b2b30e936a021
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Sep 29 12:22:57 2017 -0400
+
+    librbd: avoid dynamically refreshing non-atomic configuration settings
+    
+    Fixes: http://tracker.ceph.com/issues/21529
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit ede691323d94dc04a30f81aca5576a3d6d1930af)
+
+commit caf17a876e75cd448947b7b729a8e39689a304df
+Author: Dongsheng Yang <dongsheng.yang@easystack.cn>
+Date:   Fri Aug 11 17:44:19 2017 +0800
+
+    librbd: notify watcher when updating image metadata
+    
+    Signed-off-by: Dongsheng Yang <dongsheng.yang@easystack.cn>
+    (cherry picked from commit b10d26dfa84627b2622d405d272b1133bb773245)
+    
+    Conflicts:
+        src/librbd/image/RefreshRequest.h (luminous does not have
+            233482cf708aa7b4723c529d387c4605bdf0722f so we leave the following variables
+            uninitialized:
+              uint8_t m_order;
+              uint64_t m_size;
+              uint64_t m_features;
+              uint64_t m_incompatible_features;
+              uint64_t m_flags;
+            )
+
+commit 34450ed6c0a324ed86fd47770662554e7cdfeae2
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Mon Dec 11 17:09:25 2017 +0100
+
+    doc: globally change CRUSH ruleset to CRUSH rule
+    
+    Since kraken, Ceph enforces a 1:1 correspondence between CRUSH ruleset and
+    CRUSH rule, so effectively ruleset and rule are the same thing, although
+    the term "ruleset" still survives - notably in the CRUSH rule itself, where it
+    effectively denotes the number of the rule.
+    
+    This commit updates the documentation to more faithfully reflect the current
+    state of the code.
+    
+    Fixes: http://tracker.ceph.com/issues/20559
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit aea9fa01ae630f4ffd2744f577723f5305c718b4)
+    
+    Conflicts: (trivial resolution)
+            doc/man/8/ceph.rst
+            doc/rados/configuration/pool-pg-config-ref.rst
+            doc/rados/operations/pools.rst
+
+commit b013f7fdb5b57725f01fc5305d94d32d7599f732
+Author: fang.yuxiang <fang.yuxiang@eisoo.com>
+Date:   Fri Aug 4 11:19:00 2017 +0800
+
+    rgw: revert PR #8765
+    
+    As talk with cbodley in PR(#16716), we shouldn't not mix such dangerous
+    configuration reload with periodically SIGHUP for logs zip.
+    
+    Signed-off-by: fang yuxiang fang.yuxiang@eisoo.com
+    (cherry picked from commit 2f30bbb492db3df0522888905d497174e36d0596)
+
+commit b3fd46810c01be00ebf26b9893d8bab107ee93f2
+Merge: 092ea5174c f6fb266699
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Fri Dec 8 10:54:12 2017 +0800
+
+    Merge pull request #19355 from tchaikov/wip-18589-luminous
+    
+    luminous: build/ops: move ceph-*-tool binaries out of ceph-test subpackage
+    
+    Reviewed-by: Ken Dreyer <kdreyer@redhat.com>
+
+commit 1d6644d091ab6c3aa64879e0ae68aa697877f32d
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Wed Dec 6 12:13:40 2017 -0800
+
+    qa/tests: Add debug info when creating ceph volumes
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 9f1a1e0543da6be15b75c3168603cc84a62bd157)
+
+commit 0616bcba07b24ff6aced3048753f8e6535aedf06
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Fri Dec 1 14:16:45 2017 -0800
+
+    qa/tests: add tests for ceph-volume
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 95fb43b54d843e4a7bedaad72b4a71c3489f0dea)
+
+commit bccaa6e627f32725698b544499cab44f3ed72807
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Fri Dec 1 14:11:55 2017 -0800
+
+    qa/tests: update tests to use new ceph-volume syntax
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 3ecaebd0d884033c55a501d218ab426385d92357)
+
+commit bd558d7e729005aece735fedadeb3325efae235d
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Thu Dec 7 17:56:55 2017 -0800
+
+    qa/tests: default to wip branch for final upgrade.
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+
+commit 3f28b195d3f391f1a9e36073209f2c895dee89f9
+Author: Vasu Kulkarni <vasu@redhat.com>
+Date:   Thu Dec 7 17:47:01 2017 -0800
+
+    qa/tests: when no branch is specified, use wip branch under test for upgrade
+    
+    Signed-off-by: Vasu Kulkarni <vasu@redhat.com>
+    (cherry picked from commit 307e52109411e237e34b824da78fa5b175b16a9d)
+
+commit f6fb266699f7a39005cb76a2c878bdff5b3a7ff0
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Dec 6 10:19:09 2017 +0800
+
+    debian/control: adjust ceph-{osdomap,kvstore,monstore}-tool feature move
+    
+    this is a follow-up of #19328. we need to get this change into 12.2.3.
+    so better off do the switch somewhere after 12.2.2 which has been
+    tagged, and before 12.2.3, which is not tagged yet.
+    
+    please note, this is not targetting master, because i want to make
+    sure the change number (the <num> in << 12.2.2-<num>) is correct. it
+    does not hurt if it's not, as long as it is ">> 12.2.2", so the replace
+    machinery in 12.2.3 works, and it covers the releases where the
+    ceph-{osdomap,kvstore,monstore}-tool are not move yet. but why don't
+    make it more right?
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 57bb57f76d733d6e6dea4fc0682058e4e6abe7df)
+
+commit ee06a16ee8c8434f75adf05c04f94607a5ec090f
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Nov 22 12:33:09 2017 +0800
+
+    mds: remove useless check in Migrator::export_dir
+    
+    There is an assert(dest != mds->get_nodeid()) at very beginning of
+    the function. There also is a check for if 'dest' mds is active.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 7afe92f644c97ae1282837bc75cde0b4a8054d0f)
+
+commit 443fa3546593f667b01cca7466e3692223c4f226
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Nov 22 11:49:43 2017 +0800
+
+    mds: ignore export pin for unlinked directory
+    
+    Otherwise, stray directory inode may have pinned subtree dirfrag.
+    The subtree dirfrag prevents stray inode from getting purged.
+    
+    Fixes: http://tracker.ceph.com/issues/22219
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit b7cee60467582d7398bbe85e051a722e30c0897e)
+
+commit d57d0945dbb6ec4b46e1749fe7989b48794c3544
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Dec 4 21:25:16 2017 -0600
+
+    debian/control: adjust ceph-{osdomap,kvstore,monstore}-tool feature move
+    
+    The backport didn't make 12.2.2, but it will be in 12.2.3.
+    
+    Fixes: http://tracker.ceph.com/issues/22319
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e0c814266fe653311673e07bd7c7dbb51d866f82)
+
+commit 00503d1880e829024cd1d5e10cca0d388718dedb
+Author: liuchang0812 <liuchang0812@gmail.com>
+Date:   Fri Aug 18 21:52:52 2017 +0800
+
+    doc: add ceph-kvstore-tool's man
+    
+    Signed-off-by: liuchang0812 <liuchang0812@gmail.com>
+    (cherry picked from commit 0667db4eb61338887743d17c93a61c0e89868b08)
+
+commit 92093824d5deddad4dfe4219fee3c95fe4c38f55
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Oct 23 20:15:16 2017 +0800
+
+    debian: fix package relationships after d3ac8d18
+    
+    d3ac8d18 moves ceph-client-debug from ceph-test to ceph-base without
+    updating the package relationships between the two involved packages.
+    which results in:
+    
+    dpkg: error processing archive /var/cache/apt/archives/ceph-test_12.2.1-241-g43e027b-1trusty_amd64.deb (--unpack):
+     trying to overwrite '/usr/bin/ceph-client-debug', which is also in package ceph-base 10.2.10-14-gcbaddae-1trusty
+    dpkg-deb: error: subprocess paste was killed by signal (Broken pipe)
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit ed988fc660c5da3cb97e48e82ac5b7ad8f3bcd41)
+
+commit a566ab942b35af11f2cd0fd269f48a4e9fdc3a66
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Oct 23 15:26:35 2017 +0800
+
+    debian: fix package relationships after 40caf6a6
+    
+    we have issues when running upgrade tests:
+    
+    dpkg: error processing archive /var/cache/apt/archives/ceph-osd_13.0.0-2201-g6cc0b41-1trusty_amd64.deb (--unpack):
+    trying to overwrite '/usr/bin/ceph-osdomap-tool', which is also in package ceph-test 10.2.10-14-gcbaddae-1trusty
+    
+    in 40caf6a6, we moves some tools from ceph-test out into ceph-osd,
+    ceph-mon and ceph-base respectively. but didn't update the relationships
+    between these packages accordingly. this causes the upgrade failure.
+    
+    see https://www.debian.org/doc/debian-policy/#document-ch-relationships
+    for more details on "Breaks" and "Conflicts".
+    
+    the reason why the package version to be replaced/conflicted is 12.2.2
+    is that: i assume that this change will be backported to luminous, and
+    the next release of it will be 12.2.2 .
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 50dad805d9f635f2d8085237e94ee8fd0948dd3c)
+
+commit 09fd3c730804e094624cbbbd3e6c4481d1a3a86e
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Oct 13 10:34:52 2017 +0200
+
+    build/ops: deb: move ceph-*-tool binaries out of ceph-test subpackage
+    
+    ceph-osdomap-tool into ceph-osd subpackage
+    ceph-monstore-tool into ceph-mon subpackage
+    ceph-kvstore-tool into the ceph-base subpackage
+    
+    Fixes: http://tracker.ceph.com/issues/21762
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit 40caf6a6d85fbde930b74df57a62381f6225c3fd)
+
+commit 6dba25e39d07aeb7fe073fc77f3af054c3481ce0
+Author: Nathan Cutler <ncutler@suse.com>
+Date:   Fri Oct 13 10:34:52 2017 +0200
+
+    build/ops: rpm: move ceph-*-tool binaries out of ceph-test subpackage
+    
+    ceph-osdomap-tool into ceph-osd subpackage
+    ceph-monstore-tool into ceph-mon subpackage
+    ceph-kvstore-tool into the ceph-base subpackage
+    
+    Fixes: http://tracker.ceph.com/issues/21762
+    Signed-off-by: Nathan Cutler <ncutler@suse.com>
+    (cherry picked from commit d7b493a7108a68302bc0f48337bf3c253a720266)
+
+commit 092ea5174c069f060bcccc1fa044d5e67da3938c
+Merge: 117e91ef29 753279aac6
+Author: Alfredo Deza <alfredo@deza.pe>
+Date:   Tue Dec 5 07:01:35 2017 -0500
+
+    Merge pull request #19299 from tchaikov/wip-pr-19196-luminous
+    
+    luminous: ceph-disk: fix signed integer is greater than maximum when call major
+    
+    Reviewed-by: Alfredo Deza <adeza@redhat.com>
+
+commit c514d3d427176eb957c3cd57b99b7ed590468ed7
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Nov 28 17:06:47 2017 +0800
+
+    mds: handle client reconnect gather race
+    
+    Fixes: http://tracker.ceph.com/issues/22263
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit e4ecf26a8ef622f8290129a682afb1c64b3e7e00)
+
+commit c3c3bb0bc90c980214ff3247823dfbf06046ff95
+Author: Adam Wolfe Gordon <awg@digitalocean.com>
+Date:   Fri Sep 29 15:32:38 2017 +0000
+
+    doc: Update rbd-mirror docs to reflect data pool selection changes
+    
+    Signed-off-by: Adam Wolfe Gordon <awg@digitalocean.com>
+    (cherry picked from commit 57745b94394ac65436a17249733a22c463d51c0c)
+
+commit b634fdaaf0b0f998a62c52ec59a40a4709a41331
+Author: Adam Wolfe Gordon <awg@digitalocean.com>
+Date:   Tue Sep 26 20:30:06 2017 +0000
+
+    rbd-mirror: Improve data pool selection when creating images
+    
+    Previously we used the source image's data pool name
+    unconditionally. There were two problems with that:
+    
+    1. If a pool with the same name didn't exist locally, creation of the
+       local image would fail.
+    2. If the local pool had a default data pool configured it would be
+       ignored.
+    
+    Change local image creation so it uses the default pool if configured,
+    and uses the remote pool name only if a pool with that name exists
+    locally. If neither of those is true, leave the data pool unset.
+    
+    Signed-off-by: Adam Wolfe Gordon <awg@digitalocean.com>
+    (cherry picked from commit 2e239c05518e67b3db80500dbdd3fc5dde30e443)
+
+commit 753279aac6beeacda7fb7c7d15129dff1a27d214
+Author: Song Shun <song.shun3@zte.com.cn>
+Date:   Tue Nov 28 11:28:43 2017 +0800
+
+    ceph-disk: fix signed integer is greater than maximum when call major
+      fix signed integer is greater than maximum when call os.major
+      using python 2.7.5 in Centos 7
+    
+    Signed-off-by: Song Shun <song.shun3@zte.com.cn>
+    (cherry picked from commit f77934b19939796d7ab52daf4dac44846a2ad162)
+
+commit 117e91ef2986f65b059203e23d6efe689a182fb5
+Merge: cf0baeeeeb ce90e0b764
+Author: Alfredo Deza <adeza@redhat.com>
+Date:   Fri Dec 1 09:57:44 2017 -0500
+
+    Merge remote-tracking branch 'gh/luminous' into luminous
+
+commit ce90e0b7648587c4dc8bf354abbfaa15b63b50b1
+Merge: 9c23617d6d 6d02b2a1a2
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Fri Dec 1 13:48:29 2017 +0800
+
+    Merge pull request #19001 from tchaikov/wip-pr-18848-luminous
+    
+    luminous: mgr/dashboard: fix audit log loading
+    
+    Reviewed-by: John Spray <john.spray@redhat.com>
+
+commit 9c23617d6d3e3f6d6d21de0156e17f9b7cb5f0e9
+Merge: 83684b91a3 77258ed181
+Author: Kefu Chai <tchaikov@gmail.com>
+Date:   Fri Dec 1 13:47:17 2017 +0800
+
+    Merge pull request #19270 from tchaikov/wip-pr-19225-luminous
+    
+    luminous: qa/suites/rados/singleton: more whitelist
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 77258ed181dbe04e56da249a71edd3220baf954a
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Nov 29 13:51:49 2017 +0800
+
+    qa/suites/rados/singleton: more whitelist
+    
+    * SLOW_OPS is normal in a cluster with flattering OSDs
+    * so is OBJECT_MISPLACED.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 0e987665fe480530a0d1664392604713828bfa5c)
+
+commit 3bfb493fa0c61f33ac930320f7b69ce4a96c2eb6
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Nov 28 15:21:55 2017 +0800
+
+    cmake,common/RWLock: check for libpthread extensions
+    
+    pthread_rwlockattr_setkind_np() is a GNU extension of libpthread. and
+    Tianshan Qu pointed out, we cannot use
+    ifdef(PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP) to detect the
+    availability of this function, because it's an enum not a macro. so,
+    like other *_np() extensions, we check this one also using cmake at
+    the configure phase.
+    
+    Reported-by: Tianshan Qu <tianshan@xsky.com>
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+
+commit 5cec46600de8b60385b43cba1e7dc4d1f9a380ac
+Author: Dan Mick <dan.mick@redhat.com>
+Date:   Wed Nov 29 15:29:17 2017 -0800
+
+    ceph_disk: allow "no fsid" on activate
+    
+    The intent was to allow "no fsid" configurations when only one
+    conf file named 'ceph.conf' was present, but the code has a bug
+    in that ceph-osd --show-config-value will return a default all-0
+    uuid.  Treat 'all-0' as we were treating 'None'.
+    
+    Signed-off-by: Dan Mick <dan.mick@redhat.com>
+    (cherry picked from commit 4d010fb83d3cad50953cdf010d1bb20c62588b3c)
+
+commit 7e8dc9763039bf784030480f60b2554f6ff7ffd5
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Nov 30 19:32:05 2017 +0800
+
+    ceph-disk: silence deprecate warnings while testing
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit ecd4f69c12f57b0d2eacd18bd4bef30f1ce24dd2)
+
+commit d0b0e03ef37a20760e702805fdc95862ec3daeba
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Nov 30 19:31:09 2017 +0800
+
+    ceph-disk/tests: mock get_fsid
+    
+    should offer a valid fsid for the ceph-disk under testing.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit cc49d1d2b99dcc49c60db8ea1ee90caf58650aa4)
+
+commit 747d05c298b13c1d4350b90beb8cec8828262332
+Author: Igor Fedotov <ifedotov@suse.com>
+Date:   Thu Nov 30 16:05:20 2017 +0300
+
+    luminous: rocksdb: fixes early metadata spill over to slow device in
+    bluefs.
+    
+    Fixes http://tracker.ceph.com/issues/22264
+    
+    Signed-off-by: Igor Fedotov <ifedotov@suse.com>
+
+commit c1c539f1efc8f2fe448bf3c4cbbed8f6401ab24d
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 29 15:20:59 2017 -0600
+
+    mon/Monitor: fix statfs handling before luminous switchover happens
+    
+    After the mons are luminous but before we switch over to using the
+    MgrStatMonitor's new info, the version on mgrstat will generally be <<
+    than that of pgmon, and the client will send that version with the
+    request.  This means that the statfs message will perpetually appear to be
+    in the future and fail the is_readable() check.
+    
+    Fix this with any ugly hack that resets the version to 1 if we haven't
+    completed the luminous upgrade yet.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 82c9e3dbe1b6ff04feae76a089d1a821c3ed819f
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Nov 27 16:45:25 2017 -0600
+
+    mon/MgrMonitor: limit mgrmap history
+    
+    Keep 500 by default (like we do osdmaps and mdsmaps).
+    
+    Fixes: http://tracker.ceph.com/issues/22257
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 00fa8b10990f945f3e875c9850eedeb65af3fd2e)
+    
+    [removed const from get_trim_to() for luminous]
+
+commit bdb9d385a5e1675ac2832300e3cfa1ba6b746ce2
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Mon Nov 20 15:42:09 2017 +0800
+
+    cmake: only create sysctl file on linux
+    
+    and check 64bit platform by using the sizeof(void*)
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 89a48189ea7b9bd58db1bc95ccd7b3d9fb9b1bb3)
+
+commit acccd2332cd0bc0f8b8f00276ce25a9582c083d7
+Author: David Disseldorp <ddiss@suse.de>
+Date:   Tue Nov 14 16:32:39 2017 +0100
+
+    sysctl.d: set kernel.pid_max=4194304 on 64-bit systems
+    
+    For CONFIG_BASE_FULL Linux kernels, the maximum number of proc/thread
+    IDs is set to 32768 by default. This default limit can be quite easily
+    hit during recovery on nodes with high OSD counts.
+    To avoid hitting the pid_max default limit, attempt to configure it to
+    4194304, which corresponds to the maximum limit possible on 64-bit
+    CONFIG_BASE_FULL kernels.
+    
+    Fixes: http://tracker.ceph.com/issues/21929
+    
+    Signed-off-by: David Disseldorp <ddiss@suse.de>
+    (cherry picked from commit 14a0c2a727dd71560c3cf18171378ccde6e14699)
+
+commit 718528304063c7ea03d7777c3eaf6e26a0900f59
+Author: David Disseldorp <ddiss@suse.de>
+Date:   Tue Nov 14 15:55:39 2017 +0100
+
+    sysctl: relocate 90-ceph-osd.conf under etc/sysctl/
+    
+    Signed-off-by: David Disseldorp <ddiss@suse.de>
+    (cherry picked from commit 4d0b70068894af2d9c2c4b6fea1451a47864bfd8)
+
+commit 8f67e4553ac67ff062f3d9d1187a3bd67f02f020
+Author: tangwenjun <tang.wenjun3@zte.com.cn>
+Date:   Mon Nov 20 15:28:29 2017 +0800
+
+    os/bluestore: fix the allocate in bluefs
+    
+    when bluefs succeed to reserve but failed to alloc in db space,
+    
+    it will cause a assert, just because of the space fragmentation.
+    
+    in this situation, it could not use slow device space,
+    
+    and it would happen in stupid or avl allocator.
+    
+    Signed-off-by: tangwenjun <tang.wenjun3@zte.com.cn>
+    (cherry picked from commit d4f868ae7b9df723a9f785cbe6caee1f718e5e33)
+
+commit 17ff1f1fd2bfa744c141741af9dc81761218da63
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 20 11:03:17 2017 -0500
+
+    rbd: disk usage on empty pool no longer returns an error message
+    
+    Fixes: http://tracker.ceph.com/issues/22200
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f4528122d66d1bc7dd07a64d1af7b65fe53f7b0b)
+
+commit 4dda1b6ead6b1f04f996403881f61e9b7d94dba0
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Sun Nov 19 15:08:18 2017 -0800
+
+    client: anchor Inode while trimming caps
+    
+    This prevents the Inode from being deleted until after cap trimming is
+    finished. In particular, this prevents remove_all_caps from being called which
+    screws up the traversal of caps in trim_caps.
+    
+    Fixes: http://tracker.ceph.com/issues/22157
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 1439337e949c9fcb7d15eb38c22d19eb57d3d0f2)
+
+commit 05b60db8ef1999bb7a75f7cbbc721f64e67cec48
+Author: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+Date:   Wed Nov 8 21:35:54 2017 +0530
+
+    rgw: Fix swift object expiry not deleting objects
+    
+    In cls_timeindex_list() though `to_index` has expired for a timespan, the marker is set for a subsequent index during the time boundary check.
+    This marker is further returned to RGWObjectExpirer::process_single_shard(), where this out_marker is trimmed from the respective shard,
+    resulting in a lost removal hint and a leaked object.
+    
+    Fixes: http://tracker.ceph.com/issues/22084
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+    (cherry picked from commit 70adfaae5073d2680a9722526a6a19795dd18780)
+
+commit fe5c8b34a7dbaa1585de54668abeae005dc56388
+Author: lvshanchun <lvshanchun@gmail.com>
+Date:   Wed Nov 1 15:52:48 2017 -0400
+
+    radosgw-admin zonegroup get and zone get return defaults when there is no realm
+    
+    Fixs: http://tracker.ceph.com/issues/21615
+    Signed-off-by: lvshanchun <lvshanchun@gmail.com>
+    (cherry picked from commit 2c1653ca379d0bf78dfcfe931d895a16b9bfd21f)
+
+commit 14434beda69510108a37ae14d8d5318c93d78da4
+Author: Yao Zongyou <yaozongyou@vip.qq.com>
+Date:   Thu Nov 9 20:40:15 2017 +0800
+
+    rgw: check going_down() when lifecycle processing
+    
+    Fixes: http://tracker.ceph.com/issues/22099
+    
+    Signed-off-by: Yao Zongyou <yaozongyou@vip.qq.com>
+    (cherry picked from commit b761989033ac0722a1504238648a806c8004b8e3)
+
+commit 90b56a2903db892bcd736d9f9eeca3a8847b353b
+Author: Aleksei Gutikov <aleksey.gutikov@synesis.ru>
+Date:   Tue Nov 14 15:45:39 2017 +0300
+
+    rgw: fix radosgw-admin bucket rm with --purge-objects and --bypass-gc
+    
+    Call RGWRados::delete_bucket() from rgw_remove_bucket_bypass_gc()
+    instead of partial copy of code of RGWRados::delete_bucket().
+    
+    Fix updating user stats after radosgw-admin bucket rm --purge-objects and --bypass-gc
+    Due to rgw_user(const std::string& s) was called incorrect version of rgw_bucket_sync_user_stats().
+    
+    Fixes: http://tracker.ceph.com/issues/22122
+    Fixes: http://tracker.ceph.com/issues/19959
+    Signed-off-by: Aleksei Gutikov <aleksey.gutikov@synesis.ru>
+    (cherry picked from commit db42e385d26ee4d1ef94b900102b705d6a794029)
+
+commit ed24c8ce8f74a9b66d3da3dc2ae3ab5c4a4c86e6
+Author: Yanhu Cao <gmayyyha@gmail.com>
+Date:   Fri Oct 27 13:28:03 2017 +0800
+
+    mgr: 'osd status' command return add state
+    
+    Signed-off-by: Yanhu Cao <gmayyyha@gmail.com>
+    (cherry picked from commit f3475c9fdf5b32443df01c1a735c7289066c7508)
+
+commit 85667ad225882cdf4af58166b0d302e69b2d17c0
+Author: Yanhu Cao <gmayyyha@gmail.com>
+Date:   Mon Oct 9 13:47:11 2017 +0800
+
+    mgr: fix "osd status" command exception if OSD not in pgmap stats
+    
+    Fixes: http://tracker.ceph.com/issues/21707
+    
+    Signed-off-by: Yanhu Cao <gmayyyha@gmail.com>
+    (cherry picked from commit ce3953248a04c065643938898029cfbf9c843ea3)
+
+commit 23b3c9de3d3a950eb64ff9a4b77da145f7d1d865
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Fri Nov 17 14:48:17 2017 +0100
+
+    rgw: keystone: bump up logging when error is received
+    
+    When keystone engine is used and there is an error (often due to
+    misconfigured urls etc) these do not show up at the final logs as we
+    just return  -1 at the final stage. Adding logs at level 5 for these
+    cases, as they can help identify a possible misconfigured url or admin
+    token. Eg:
+    
+    ```
+    5 Failed keystone auth from http://localhost:5000/v3/v3/auth/tokens with 404
+    ```
+    
+    Fixes: http://tracker.ceph.com/issues/22151
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit 347c5148a57de9007d6c1293add6bf27004fce41)
+
+commit 58d6d043b20bd2ce185c560814315f82fc33c399
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Fri Nov 10 14:27:59 2017 -0500
+
+    rgw: set num_shards on 'radosgw-admin data sync init'
+    
+    Fixes: http://tracker.ceph.com/issues/22083
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 90e860c25b87670810426e94b04b515af1e4f154)
+
+commit 239a2f8eb0ad5ba3869d0168cfa5bed81479a9d7
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Wed Nov 8 10:21:55 2017 -0500
+
+    dencoder/rgw: expose rgw sync status types
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 662b02489c9394a359834cd3d3527bce5b903100)
+
+commit 442120898b8abf22fcd66e06eb8607481653c890
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Thu Nov 9 15:50:56 2017 +0100
+
+    rgw: data sync: set num_shards when building full maps
+    
+    When radosgw-admin data sync init is called on a cluster, the next run
+    of rgw crashes as when it processes ListBucketIndexesCR, num_shards
+    isn't set which is later referenced in ListBucketIndexesCR. Setting the
+    n sync_info.num_shards correctly to handle this case
+    
+    Fixes: http://tracker.ceph.com/issues/22083
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit 4015a57c473d896164b0617be93777d0947a9576)
+
+commit 2070fcf7d4260e3a13219ce708a46b2c550ccc14
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Nov 15 10:35:16 2017 -0500
+
+    librbd: invalidating the cache shouldn't hold write lock
+    
+    This can cause deadlock when readahead is in-progress since neither
+    can make forward progress.
+    
+    Fixes: http://tracker.ceph.com/issues/22131
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6a335481d20c6a765c84d561a01fb52172eccba4)
+
+commit 3e91197ccd0df2bb0560e057861a2f0639da02bd
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Nov 15 10:34:32 2017 -0500
+
+    rbd-nbd: rescan partition table after image resize event
+    
+    Fixes: http://tracker.ceph.com/issues/22131
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit db13e4757451b463e7fb65c43247e033f24d45e5)
+
+commit 818c354a43ca83be10daa1f0cd251635b57371b9
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Nov 15 09:09:15 2017 -0500
+
+    librbd: prevent overflow of discard API result code
+    
+    Prevent discard/writesame lengths larger than 2GB.
+    
+    Fixes: http://tracker.ceph.com/issues/21966
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3effd324db181e625665be33b5c6529dca723cc5)
+    
+    Conflicts:
+            PendingReleaseNotes
+            src/librbd/librbd.cc: i chosed to pick code from 3effd32 in order to prevent discard/writesame against lengths larger than 2GB. `len` used in picked code was already in luminous. so it's safe to pick.
+
+commit 5bd076ef41fd8bdd366654507f2e7c287d1f95dc
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Nov 20 14:35:48 2017 -0600
+
+    qa/suites/upgrade/jewel-x: move mon_warn_on_pool_no_app to global
+    
+    Check runs on mgr.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit bd3e7795ca9315aa4d550eeef42884f6f0516029
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Fri Oct 27 12:36:06 2017 -0700
+
+    MDSMonitor: only clog changes to active
+    
+    Otherwise we get constant INFO messages that an MDS is active.
+    
+    Fixes: http://tracker.ceph.com/issues/21959
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 997a688d0809221500e0e3dd30e12ccec6c19780)
+
+commit b9bf9b1255d3b3f8d581fcc54e7c39912517f6d0
+Author: yuliyang <yuliyang@cmss.chinamobile.com>
+Date:   Thu Oct 26 14:35:56 2017 +0800
+
+    rgw: add cors header rule check in cors option request
+    
+    fix http://tracker.ceph.com/issues/22002
+    
+    Signed-off-by: yuliyang <yuliyang@cmss.chinamobile.com>
+    (cherry picked from commit 72e8fc52fb078fa1443e1ba5321718882969aa78)
+
+commit 2d9aafe1bc07cfa65e1bd3d5ea28b575a15eaf53
+Author: Li Wang <laurence.liwang@gmail.com>
+Date:   Wed Nov 1 09:21:29 2017 +0000
+
+    rbd-nbd: fix unused nbd device search bug in container
+    
+    In some container scenarios, the host may choose to
+    map a specific nbd device, for example, /dev/nbd6 into the
+    container, in that case, the nbd device available in the
+    container is not numbered from 0. The current unused
+    nbd device search function will return no result.
+    This patch fixes it.
+    
+    Fixes: http://tracker.ceph.com/issues/22012
+    
+    Signed-off-by: Li Wang <laurence.liwang@gmail.com>
+    Reviewed-by: Yunchuan Wen <yunchuan.wen@kylin-cloud.com>
+    (cherry picked from commit be0f9581f9727187ca03232e0b368e7da7a60609)
+
+commit 210b826ff06fd145f730e2b5303e6472d82fd8e5
+Author: Li Wang <laurence.liwang@gmail.com>
+Date:   Wed Nov 1 06:31:10 2017 +0000
+
+    rbd-nbd: not abort in listing mapped nbd devices
+    
+    In some container scenarios, the file '/sys/block/nbd0/pid'
+    in the container records the host pid of the process
+    which opened the device, therefore the file '/proc/pid/cmdline'
+    corresponding to the pid in the container does not exist,
+    in that case, 'rbd-nbd list-mapped' will cause the
+    following assertion failure, this patch fixes it.
+    
+    src/tools/rbd_nbd/rbd-nbd.cc: In function 'int get_mapped_info(int, Config*)'
+    /src/tools/rbd_nbd/rbd-nbd.cc: 834: FAILED assert(ifs.is_open())
+     ceph version 13.0.0-1632-gf9cfe84 (f9cfe843a1c749d145b6c3b81a0519cc6536eb28) mimic (dev)
+     1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x102) [0x7fc6b1054ba2]
+     2: (()+0x15c58) [0x560cd4697c58]
+     3: (main()+0x9) [0x560cd46937f9]
+     4: (__libc_start_main()+0xf0) [0x7fc6b003f830]
+     5: (_start()+0x29) [0x560cd46938f9]
+     NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
+    Aborted (core dumped)
+    
+    Fixes: http://tracker.ceph.com/issues/22011
+    
+    Signed-off-by: Li Wang <laurence.liwang@gmail.com>
+    (cherry picked from commit 07d5b81b57f17696f4f693225fcb5a1f7060e9d9)
+
+commit 274fef45c2ca619aaad3ec14e8c6d80d22832a85
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed Nov 8 12:01:38 2017 +0200
+
+    rgw: allow tenant without user id for reshard commands
+    
+    Fixes: http://tracker.ceph.com/issues/22046
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 0a2142e83b58fa8e238bcb748d1cb97bdba674c5)
+
+commit 58870ed9111fe3f3e05eadde1fecc8eae7b0eda2
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Tue Nov 7 15:37:34 2017 +0200
+
+    rgw: use tenant when updating the reshard log
+    
+    Fixes: http://tracker.ceph.com/issues/22046
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 055c0d34394b8274a7e48d4f2e3913c059099898)
+
+commit 96b924fc750008e66e1cea29a67f1e8c7008a3ed
+Author: Ilja Slepnev <islepnev@gmail.com>
+Date:   Tue Oct 24 20:27:59 2017 +0300
+
+    mgr/zabbix: ignore osd with 0 kb capacity
+    
+    Fixes: http://tracker.ceph.com/issues/21904
+    Signed-off-by: Ilja Slepnev <islepnev@gmail.com>
+    (cherry picked from commit e532949556453668158d0e261e59e0fae48e461f)
+
+commit 654678ab2de4b4909e1ad12384e790105709ec1a
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Fri Nov 10 22:47:41 2017 +0800
+
+    common/buffers: add function parameter to limit buffers size.
+    
+    Fixes: http://tracker.ceph.com/issues/21932
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit fba63cebba215b36ce850f47bd8e706edaa8023e)
+
+commit 488228e919980d1aba3313bf5f1601e30bfdc03d
+Author: Jianpeng Ma <jianpeng.ma@intel.com>
+Date:   Thu Nov 9 01:00:52 2017 +0800
+
+    osdc/Objecter: record correctly value for l_osdc_op_send_bytes.
+    
+    Fixes: http://tracker.ceph.com/issues/21982
+    Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>
+    (cherry picked from commit c5c7ad56c0d5c758874de0f5ea66504963bbb75e)
+
+commit 134818228b5c0fa54e934c842038de0b9ad63620
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Nov 20 08:55:59 2017 -0600
+
+    mon/OSDMonitor: fix mon_fixup_legacy_erasure_code_profiles
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+
+commit 6f983c14d94f51b458730a8ef1d1723d8aa8c1e2
+Author: Jan Fajerski <jfajerski@suse.com>
+Date:   Wed Nov 8 10:30:09 2017 +0100
+
+    tools/crushtool: skip device id if no name exists
+    
+    When an OSD with an id < max_id is removed, i.e. the osd ids are not
+    continuous, crushtool decompile prints bogus info. Skip any device ids
+    without a name.
+    
+    Signed-off-by: Jan Fajerski <jfajerski@suse.com>
+    (cherry picked from commit 85737f94571208f21c972c11530ab56bfaededa2)
+
+commit d5f2e1660055d296640eebcfcaf13435dff212ef
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Nov 14 18:42:02 2017 +0800
+
+    osd: subscribe osdmaps if any pending pgs
+    
+    subscribe from monitor continously for new osdmaps so osd is able to get
+    the osdmap instructing it to delete PGs even if nobody is poking it.
+    
+    Fixes: http://tracker.ceph.com/issues/22113
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 4037ab44b30ab23b2e464b24a82fee6fff9a2d08)
+
+commit c6bc756304ff567f39e163e2140d6bb00e8adc6a
+Author: Abhishek Lekshmanan <abhishek@suse.com>
+Date:   Tue Nov 14 17:39:16 2017 +0100
+
+    rgw: set sync_from_all as true when no value is seen
+    
+    In order to support jewel-luminous multisite scenarios where when a
+    jewel cluster is the master, the sync-from-all json field wouldn't be
+    set leading to the secondary not syncing data as this value can't be
+    overriden to true, so defaulting the value of sync_from_all to true when
+    we don't encounter it in the json
+    
+    Fixes: http://tracker.ceph.com/issues/22062
+    Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
+    (cherry picked from commit e0116fa42020ff5c0d78101b9b3b15e62236b4b8)
+
+commit c21e1c7831e43ed5d07924589b24ce826bb86467
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Nov 8 11:54:44 2017 -0500
+
+    mon: clean up cluster logging on mon events
+    
+    These changes come from observing the output
+    when killing a mon and watching the survivors
+    form a new quorum.
+    
+    Fixes: http://tracker.ceph.com/issues/22082
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 294fe62feeb155e1ec06d508d92435de98cce90a)
+
+commit 6875bdaf5fcc000c4a4615e3f6a03a530bdd74da
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Nov 8 11:53:45 2017 -0500
+
+    mon: monmap log on active should be debug
+    
+    This isn't even a change to the map, just something
+    that got echoed every time a new mon was the leader.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit c9eb2793e6074521d161632e20954295527d3653)
+
+commit 1bdd81e095a4bd628c278a3e6abd7faf62ba5399
+Author: Sage Weil <sage@redhat.com>
+Date:   Sun Nov 12 19:49:02 2017 -0600
+
+    mgr/balancer: skip CRUSH_ITEM_NONE
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit e2cbb4bc7a2badc3ff89fb4bf4d678d8e9f94652)
+
+commit 3efc4c8d542060cd43fa509022063323d1eecaac
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 8 17:37:29 2017 -0600
+
+    qa/suites/rados: stop testing firefly tunables
+    
+    We can't mix the balancer compat-set testing with firefly tunables because
+    it requires that all buckets be straw2.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 6455954d29e180e24dcadfabce685699477e2736)
+
+commit 5f8a59a48e2399bf41c7a0b6ce9662f6b150ba44
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Nov 8 17:35:35 2017 -0600
+
+    mgr/balancer: fail a bit more gracefully if we can't create a compat weight-set
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 476b2335fd08568d142d17848bfc58dc6e5c86d2)
+
+commit 6d02b2a1a276c98dd3f9881d3e113f1db14fee04
+Author: John Spray <john.spray@redhat.com>
+Date:   Thu Nov 9 07:33:10 2017 -0500
+
+    mgr/dashboard: fix audit log loading
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 85e1dadb287ee5216985b16b4a144dfe148d6a60)
+
+commit de1d25c2dd31ac731225ee2a3c3d2805513881b4
+Author: Mitch Birti <yahooguntu@users.noreply.github.com>
+Date:   Thu Nov 16 16:27:20 2017 -0600
+
+    udev: Fix typo in udev OSD rules file
+    
+    The rule for lockbox partitions had an invalid UUID for the "change" action.
+    
+    Signed-off-by: Mitch Birti <yahooguntu@gmail.com>
+    (cherry picked from commit 515417294c2548694c2101639c8bbf7f88e716aa)
+
+commit a170e67760ee3e36d6f5c0393710b0d58fd94005
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Aug 22 09:59:12 2017 +0800
+
+    mds: fix CDir::log_mark_dirty()
+    
+    the 'will dirty' check is wrong because we don't always project fnode.
+    
+    Fixes: http://tracker.ceph.com/issues/21584
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 461bbc7e89a9ed440478f30145158b4989c137d0)
+
+commit 0d18c24fcedd82fa0203ce37bac9099bdd455f35
+Author: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+Date:   Wed Nov 8 21:35:54 2017 +0530
+
+    rgw: Fix swift object expiry not deleting objects
+    
+    In cls_timeindex_list() though `to_index` has expired for a timespan, the marker is set for a subsequent index during the time boundary check.
+    This marker is further returned to RGWObjectExpirer::process_single_shard(), where this out_marker is trimmed from the respective shard,
+    resulting in a lost removal hint and a leaked object.
+    
+    Fixes: http://tracker.ceph.com/issues/22084
+    Signed-off-by: Pavan Rallabhandi <PRallabhandi@walmartlabs.com>
+    (cherry picked from commit 70adfaae5073d2680a9722526a6a19795dd18780)
+
+commit 74a6592d7fc8a4e1d4000025615b72e7297285a1
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Tue Sep 26 13:04:26 2017 -0700
+
+    options.cc: document rgw config options
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 79e58a346e743b95b4435fdc72f2fd7ac01e3f1e)
+
+commit 7acb9a8da2c843ac4fb577c3e5eb6c699604da9e
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Fri Sep 29 10:51:54 2017 +0800
+
+    common/options: use user-defined literals for sizes
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 6788ea0e9a51217cd56b282c40533a8971a36530)
+
+commit ecf102618ec34afee96093812e68382e6b9c4872
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Wed Aug 23 14:35:19 2017 +0800
+
+    common/options: use user-defined literals for default intervals
+    
+    for better readablility
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 3704fe283b606737533d62a082337381b95bdbd9)
+
+commit bc94035a30360a8c2353afb5f0355e0675abbdeb
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Thu Aug 24 17:08:00 2017 +0800
+
+    common/options: use user-defined literals for default sizes
+    
+    for better readablity.
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit d9b38a1d6172a43e7ebb755be5cf044f19d7035d)
+
+commit 7a8e915483397d740e3be3815ece23f4cd3c6b7a
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Oct 11 11:36:36 2017 +0100
+
+    mds: additional damage handling case in EImportStart
+    
+    Fixes: http://tracker.ceph.com/issues/21759
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit b4c662c2cf47f7b6f952e776d0212156861eaffb)
+
+commit 1ec93753a37a24a7e72566848ccc0baab1df84ce
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Aug 2 17:26:56 2017 +0800
+
+    mds: track snap inodes through sorted map
+    
+    Current mds track both head inodes and snap inodes through unsorted
+    map. The unsorted map makes finding snap inode that follows a given
+    snapid difficult. Currnt MDCache::pick_inode_snap() use snap set to
+    guess snap inode's last. The method isn't reliable because snap set
+    may change after creating the snap inode. For example:
+    
+    MDS cows inode[2,head] with snap set[5,6], which results inode[2,6]
+    and inode[7,head].
+    
+    Later mds wants to find snap inode that follows snapid 2. But the
+    snap set become [5], mds can't find snap inode [2,5].
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 7b9eae62c8c654ff82684451c222257d2c93be64)
+    
+    Conflicts:
+            src/mds/MDCache.cc: when i cherry-picked 3ca602e, there were conflicts in same file. to fix conflicts, i need to introduce snap_inode_map because it's not in luminous. to intoroduce this, i need to cherry-pick 7b9eae62. After cherry-picking 7b9eae62, there were conflicts. i picked code from head because it's introduced by 3ca602e which is required to fix http://tracker.ceph.com/issues/21928.
+
+commit 85791328fc563932673e2d6ad8701179fee0c3a3
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Oct 26 16:21:41 2017 +0800
+
+    mds: fix inode count check in MDCache::check_memory_usage()
+    
+    Fixes: http://tracker.ceph.com/issues/21928
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 3ca602e3bdaa211ae2446d4bbf62431e14d41bef)
+    
+    Conflicts:
+            src/mds/MDCache.cc
+
+commit e485b89ffd5754b5c6356ccd018fe602d7e55e07
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Oct 20 17:03:10 2017 +0800
+
+    mds: disable early reply for dir layout and quota related requests
+    
+    These requests impacts whole subtree tree, replaying them when
+    mds recovers may break order of requests in multimds cluster.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit edd76fc88377b398b5f2d274684d10f6ed3314cd)
+
+commit cb8eff43b1abd8c268df9e57906d677ff4be8d95
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Oct 18 20:58:15 2017 +0800
+
+    mds: don't rdlock locks in replica object while auth mds is recovering
+    
+    Auth mds may take xlock on the lock and change the object when replaying
+    unsafe requests. To guarantee new requests and replayed unsafe requests
+    (on auth mds) get processed in proper order, we shouldn't rdlock locks in
+    replica object while auth mds of the object is recovering
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 0afbc0338e1b9f32340eaa74899d8d43ac8608fe)
+
+commit f1be92ee34399617cad31a489ec49e993c6eaff6
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Wed Oct 18 20:35:33 2017 +0800
+
+    mds: move ScatterLock::state_flags into SimpleLock
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit da6e50890de8b682eb5756d0e4e08583908b3778)
+
+commit ebd7a6670fbb9724bf72f7b0587e5c18ade03a48
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Aug 11 17:52:19 2017 +0800
+
+    mds: make mksnap/setlayout wait for unsafe requests on other mds
+    
+    This guarantees replayed unsafe requests (on other mds) and mksnap/setlayout
+    get processed in proper order.
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit dee3711bd745704fba224dc6f90e88c04cacff58)
+
+commit a88e48e7d9cba77db8cb5aeb9512733d62a08382
+Author: lu.shasha <lu.shasha@eisoo.com>
+Date:   Tue Oct 10 15:51:04 2017 +0800
+
+    rgw: don't change rados object's mtime when update olh
+    
+    null instance and olh share rados object. Null instance using the rados object mtime as its mtime. When olh updated, the rados object mtime will change.
+    Thus the null instance will be incorrect. When list objects in bucket, the mtime is correct, down the null instance the last modified time is incorrect.
+    So when update olh, using the previous mtime, thus the null instance mtime will be correct.
+    
+    Fixes: http://tracker.ceph.com/issues/21743
+    
+    Signed-off-by: Shasha Lu <lu.shasha@eisoo.com>
+    (cherry picked from commit 3189edc28ea6d3c117c2e2f0664f45c69f67ab50)
+
+commit 1410b5a5f5505ba7df0fdf313eefee4306807f3a
+Author: Zhi Zhang <willzzhang@tencent.com>
+Date:   Mon Oct 9 14:33:05 2017 +0800
+
+    mds: no assertion on inode being purging in find_ino_peers()
+    
+    Signed-off-by: Zhi Zhang <zhangz.david@outlook.com>
+    (cherry picked from commit e55b2dbfcacb3b5afddba2d2b24c0386a0a48ebb)
+
+commit 2dc009e68ef9e850aebcc55634a1505212786309
+Author: Adam C. Emerson <aemerson@redhat.com>
+Date:   Fri Oct 27 15:57:18 2017 -0400
+
+    rgw: Fix dereference of empty optional
+    
+    Due to the lack of a return, there was a case where an invalid ARN
+    could cause a dereference of an uninitialized boost::optional.
+    
+    As a bit of defensive programming, restructure a couple functions to
+    make that kind of error impossible by ensuring the optional is only in
+    scope when it is initialized and relying less in early return on
+    error.
+    
+    Fixes: http://tracker.ceph.com/issues/21962
+    
+    Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
+    (cherry picked from commit 5249139be7a2748eabbf898cf340989875bfa509)
+
+commit dbd70f0b82a8eb512614b68d8de0c861da9e9d47
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Oct 23 14:23:32 2017 -0400
+
+    rgw: fix extra_data_len handling in PutObj filters
+    
+    the RGWPutObj_Compress filter relies on a starting offset of 0 to ensure
+    that we only compress entire objects
+    
+    for RGWRados::fetch_remote_obj(), we also fetch object metadata. the
+    replies come back with a 'Rgwx-Embedded-Metadata-Len' header, which
+    specifies how many bytes of object metadata are at the front of the
+    request body. when this is present, the offsets passed from
+    RGWRadosPutObj down to the RGWPutObjDataProcessor filters are offsets
+    into the http response body, rather than logical offsets into the object
+    data itself
+    
+    this commit adds a transformation to RGWRadosPutObj so that only that
+    logical offset is visible to the RGWPutObjDataProcessor
+    
+    Fixes: http://tracker.ceph.com/issues/21895
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit ac7ffe6ef7faef4e251c970ce2efd04616c0459c)
+
+commit 9f9db135a058c3134a96a845900bb795d8e708fc
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Sun Oct 1 08:40:27 2017 +0300
+
+    doc: replace region with zonegroup in configure bucket sharding section
+    
+    Fixes: http://tracker.ceph.com/issues/21610
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 8a1034bccdd514a3eaa6abfdc62c5e3d1e46d5cf)
+
+commit eeb12253d4caec2194970a84a9440d7ccf372d3d
+Author: Piotr Dałek <piotr.dalek@corp.ovh.com>
+Date:   Tue Oct 17 09:48:15 2017 +0200
+
+    tools/ceph-conf: dump parsed config in plain text or as json
+    
+    This is useful for finding differences between ceph.conf on disk
+    and in osd/mon memory.
+    
+    Signed-off-by: Piotr Dałek <piotr.dalek@corp.ovh.com>
+    (cherry picked from commit 951434f3a40fd697e5eae8fbcbf779abceb3ba34)
+
+commit 8cb21e6ea6fa0abe50d0c5d2b2070963c3ee3de2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 27 16:45:54 2017 -0400
+
+    cls/journal: ensure tags are properly expired
+    
+    Previously, if only the local image was using the journal or if
+    a disconnected peer was attached, the tag entries could not be
+    expired even if unreferenced.
+    
+    Fixes: http://tracker.ceph.com/issues/21960
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 19fa1c7f5b2809e9a223b7b196dfc031e97a5dcd)
+
+commit c86675dadc8d97c60cb94f32713c9819941607b2
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 26 10:57:20 2017 -0400
+
+    test/librbd: added update_features RPC message to test_notify
+    
+    Fixes: http://tracker.ceph.com/issues/21936
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b003ff320036437b42bd8f31c0096162a3c47fca)
+
+commit efd4147fa895ea86b65c807aee7991c016c4d7aa
+Author: Douglas Fuller <dfuller@redhat.com>
+Date:   Thu Nov 2 16:30:17 2017 -0400
+
+    cephfs: Do not check auth gid when not specified
+    
+    For auth caps that omit the gid, do not check for a gid match.
+    
+    Fixes: http://tracker.ceph.com/issues/22009
+    Signed-off-by: Douglas Fuller <dfuller@redhat.com>
+    (cherry picked from commit 0e2cfdf507ab3ec8459bc6ea9b73b7a1285274d0)
+
+commit 369dcb36c70ec6c9ab974eb6b7542068ed9fcbda
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Nov 3 12:03:49 2017 -0400
+
+    librbd: added preprocessor macro for detecting compare-and-write support
+    
+    Fixes: http://tracker.ceph.com/issues/22036
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 95d716cc7098cb04063649584e0774f49a580e1d)
+
+commit 4db6b35548fdbf5cd2a84413933170547abf9a01
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Nov 1 12:08:03 2017 -0400
+
+    mgr: emit cluster log message on serve() exception
+    
+    Fixes: http://tracker.ceph.com/issues/21999
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit f4763c32fa84f57c93c559d742203fb3fd2985a8)
+
+commit d73deb017ce9ad2525a2bda54553c01f386f0b35
+Author: Patrick Donnelly <pdonnell@redhat.com>
+Date:   Tue Oct 31 11:55:29 2017 -0700
+
+    ceph.in: pass RADOS inst to LibCephFS
+    
+    This avoids multiple instances of the admin socket and other redundancies.
+    
+    Fixes: http://tracker.ceph.com/issues/21967
+    Fixes: http://tracker.ceph.com/issues/21406
+    
+    Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
+    (cherry picked from commit 76a950b2ec473741ed3cffca5c198e8c56f32a1c)
+
+commit ab103b9fe36ad0dd7e470c4c2cf701963cc8bf3c
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Oct 30 20:03:29 2017 +0800
+
+    mds: trim 'N' log segments according to how many log segments are there
+    
+    Config 'mds_log_max_expiring' is 20 by default. It means that at most
+    20 log segments get trimmed in each tick. For busy cluster, this can
+    cause mds behind on trimming log segments.
+    
+    Fixes: fixes: http://tracker.ceph.com/issues/21975
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 7de37382ace36b6b724b07dcd58178137c49e6b8)
+
+commit ddba907279719631903e3a20543056d81d176a1b
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Oct 31 16:56:51 2017 +0800
+
+    mds: fix MDS_FEATURE_INCOMPAT_FILE_LAYOUT_V2 definition
+    
+    Fixes: http://tracker.ceph.com/issues/21985
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 6c1543dfc55d6db8493535b9b62a30236cf8c638)
+
+commit e0186c9e259898286b720e0e671e708f580cbf25
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Tue Oct 31 16:37:07 2017 +0800
+
+    mds: remove useless incompat feature definination in FSMap.h
+    
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 990088ef16f901588169cd20ed99d0415ec1f7f4)
+
+commit cb38378e5dc4821c232b15a99b30aeeb7b9f9744
+Author: Enming Zhang <enming.zhang@umcloud.com>
+Date:   Wed Oct 25 21:10:56 2017 +0800
+
+    rgw: loadgen fix rgw crash issue
+    
+    When generating random bucket name and object name during
+    loadgen processing, RGW will crash. The reason is calling
+    "gen_rand_alphanumeric" with "cct = NULL".
+    
+    Fixes: http://tracker.ceph.com/issues/22006
+    
+    Signed-off-by: Enming Zhang <enming.zhang@umcloud.com>
+    (cherry picked from commit 7d021782e29c74eca36a9bb6ca59930f87093115)
+
+commit ed87de0622cdb1ac93915967949d581ed62e31d9
+Author: yuliyang <yuliyang@cmss.chinamobile.com>
+Date:   Tue Oct 31 14:20:14 2017 +0800
+
+    rgw: modify_s3_type_subuser_access_permissions_fail_through_admin_rest_api
+    
+    fix: http://tracker.ceph.com/issues/21983
+    
+    Signed-off-by: yuliyang <yuliyang@cmss.chinamobile.com>
+    (cherry picked from commit bd1385d4766ee496bf87587dd40086ee3d510672)
+
+commit 03d1cd996cf189d10a23c839aead4075cbdb888f
+Author: Enming Zhang <enming.zhang@umcloud.com>
+Date:   Wed Oct 25 18:56:08 2017 +0800
+
+    rgw: lc fix rgw crash when lc configuration xml not including ID
+    
+    When a putting lc config request does not include an ID tag in lc
+    configuration xml, RGW should generate a random ID for the lc
+    configuration.
+    
+    At present RGW will crash when generate a random ID for
+    lc configuration.
+    
+    Fixes: http://tracker.ceph.com/issues/21980
+    
+    Signed-off-by: Enming Zhang <enming.zhang@umcloud.com>
+    (cherry picked from commit 3d2f63de494f8cd065d3af3cfa49e111a294622c)
+
+commit 5e44bedf293beffb6823ae866373314a3675c431
+Author: Enming Zhang <enming.zhang@umcloud.com>
+Date:   Wed Oct 25 02:36:47 2017 +0800
+
+    rgw: lc support Content-MD5 in request headers
+    
+    According to AWS S3, this header must be used
+    as a message integrity check to verify that
+    the request body was not corrupted in transit.
+    
+    Content-MD5 is the base64-encoded 128-bit MD5
+    digest of the data
+    
+    Signed-off-by: Enming Zhang <enming.zhang@umcloud.com>
+    (cherry picked from commit 9c0abe6d638c92f1f29798afa846f8a80fa64814)
+
+commit 860625046d4a0b79e01b0bbcc5027fbbbfbad07a
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed Nov 1 17:49:46 2017 +0200
+
+    rgw: add missing current_history initialization
+    
+    Fixes: http://tracker.ceph.com/issues/21996
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit cc3246cf46ba26604e4b3bef1f8fe0c9990bdeb6)
+
+commit 32ca69b0d9fec21f1ecb94707850eb9d3322c435
+Author: Orit Wasserman <owasserm@redhat.com>
+Date:   Wed Nov 1 12:15:35 2017 +0200
+
+    rgw: init oldest period after setting run_sync_thread
+    
+    Fixes: http://tracker.ceph.com/issues/21996
+    Signed-off-by: Orit Wasserman <owasserm@redhat.com>
+    (cherry picked from commit 8b301679161d1405fdae379ff93c33d4d637698d)
+
+commit f0914619a5cc0d9108c4c6efd0f3464ec4fd0fff
+Author: Kefu Chai <kchai@redhat.com>
+Date:   Tue Oct 31 18:42:51 2017 +0800
+
+    mon/PGMap: use new-style options
+    
+    Signed-off-by: Kefu Chai <kchai@redhat.com>
+    (cherry picked from commit 7a23097c34c35c1bf6ec09e86ed3acbb0807068c)
+    
+    Conflicts:
+            src/common/legacy_config_opts.h
+            src/mon/PGMap.cc: in 729a089, PGMap::get_health() is removed
+    from master branch, but in luminous, this function is still around for
+    backward compatibility. but that function is still using the old-style
+    options. in this change, i choose to keep some of the old-style options
+    removed in the cherry-picked cleanup to appease PGMap::get_health(),
+    instead of migrating all of them to the new-style because of less
+    efforts.
+
+commit fd956acfd674b32a7518cb4699a500fc4c845172
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Sep 27 17:11:24 2017 -0400
+
+    tools: update monstore tool for fsmap, mgrmap
+    
+    Fixes: http://tracker.ceph.com/issues/21577
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 0ad7bd0b43b473211de30ed9f2050059e069e4d3)
+
+commit 0ebc4cc18f974975a27c77212ea0c38bcbdcb67c
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Sep 27 14:59:26 2017 -0400
+
+    tools: handle decode errors in monstore tool
+    
+    Print a single line message instead of dumping
+    a backtrace.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 61ceafc8bba3747b492f8b0f779793813a682ecb)
+
+commit b36bd48204f80b47e3d90189fce366caad474650
+Author: Neha Ojha <nojha@redhat.com>
+Date:   Thu Sep 28 09:50:24 2017 -0700
+
+    osd: hold lock while accessing recovery_needs_sleep
+    
+    Signed-off-by: Neha Ojha <nojha@redhat.com>
+    (cherry picked from commit 4bf8d13273b66cc89a94e55259b6fcb106061bd6)
+
+commit 8941606f5e559850dd968ea2966c2364c09b976a
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Mon Oct 9 14:16:05 2017 -0700
+
+    common: by default, do not assert on leaks in the shared_cache code
+    
+    Update the standard qa suite cluster configs so that we continue
+    asserting in our nightlies, but users don't hit this.
+    
+    Fixes: http://tracker.ceph.com/issues/21737
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 165b61a7df2731602e9e28c5107a873444bf0507)
+
+commit acccae56c4b67bbb0a788e8ca7b248e9b8826285
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Sep 15 16:03:38 2017 -0400
+
+    osd: fix waiting_for_peered vs flushing
+    
+    on_flush() requeues waiting_for_peered, but we flush twice on the
+    primary during peering, and we don't want to requeue the first one
+    (when we have the master pg log merged).
+    
+    Fix by moving waiting_for_peered to waiting_for_flush if we aren't
+    already flush on _activate_committed.  If we get an op and are
+    peered but not flushed, queue ops there.  (We can simplify this
+    check a bit since pgbackend inactive message handling doesn't care
+    about flushed or not flushed.)  When flushed, we requeue
+    waiting_for_flush.
+    
+    The waiting_for_flush, waiting_for_peered, and waiting_for_active
+    lists are all mutually exclusive, so this mostly serves to
+    clarify what we are waiting for (not to keep items separate). And
+    it means that on_flushed() will only requeue things that were
+    waiting for it specifically.
+    
+    Fixes: http://tracker.ceph.com/issues/21407
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 8f7dc8b0d8135b20322c09a5909ad43d4c83b0ea)
+
+commit 5234ef2bcaa905861e7c15a9774b82467b2e84e7
+Author: Haomai Wang <haomai@xsky.com>
+Date:   Mon Oct 23 12:38:56 2017 +0800
+
+    msg/async/AsyncConnection: state will be NONE if replacing by another one
+    
+    Fixes: http://tracker.ceph.com/issues/21883
+    Signed-off-by: Haomai Wang <haomai@xsky.com>
+    (cherry picked from commit f2eb981377f4061debfa67cbd88ca2921a9dfb27)
+
+commit c281456346f0c65e3fa4bb8ca0baf42f4d16448e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 20 09:32:14 2017 -0400
+
+    osdc/Objecter: delay initialization of hobject_t in _send_op
+    
+    Fixes: http://tracker.ceph.com/issues/21845
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit eca4fafbcfa922d9e2e98564e69596f6c0b1b93c)
+
+commit dfebcce5a445793267ec8979bcb56ec04954d154
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Oct 19 22:24:31 2017 -0400
+
+    osdc/Objecter: skip sparse-read result decode if bufferlist is empty
+    
+    If the OSD does not execute sub-ops due to errors encountered prior to
+    the sub-op, the sub-op result remains zeroed with empty out data.
+    Attempting to decode the empty bufferlist results in large exception
+    handling CPU overhead.
+    
+    Fixes: http://tracker.ceph.com/issues/21844
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit dc9b309d03074862daad9ef05ef643da870f6722)
+
+commit 7012cf4b60c34dc7f41db9acbdeeeab20e8cd80a
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 24 15:20:49 2017 -0500
+
+    ceph_test_objectstore: do not change model for 0-length zero
+    
+    We almost fixed this in the pr merged at 2dbbb351e2e0e3703880023bf51f55790f763e04
+    but missed this piece.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 390779d1aa9b46648e4f5390dc431c255c70385d)
+
+commit 51b380a0a74f0335a04d1121042449909549c976
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 6 15:29:32 2017 -0500
+
+    os/bluestore: 0-length zero should not change object size
+    
+    Fixes: http://tracker.ceph.com/issues/21712
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b588eaf2b0fdf06c94104d5a542bd571499f2b85)
+
+commit 45d70b1329a8c635d2ac0ea5ec171cbea0f6ce02
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 6 15:29:15 2017 -0500
+
+    os/filestore: make 0-length zero avoid touching object length
+    
+    Fixes: http://tracker.ceph.com/issues/21712
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 979b7c9cd2e4919b8c7d5ed2418019310959f4bf)
+
+commit 0d921686080d0d1ce81c55a0e8926951fd9ffa58
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Oct 6 15:28:49 2017 -0500
+
+    os/ObjectStore: 0-length zero does not change length of object
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 9ad1f4f10ff7bfe32d0a37361640fe5c65e56699)
+
+commit bacebf0fd7f841185236c1dc6754955da1dfb9b2
+Author: Aleksei Gutikov <aleksey.gutikov@synesis.ru>
+Date:   Mon Oct 16 17:31:12 2017 +0300
+
+    mgr: skip first non-zero incremental in PGMap::apply_incremental()
+    
+    After initialization of PGMap instance PGMap::stamp is zero
+    and this cause huge first delta.
+    Also after mgr restart first non-zero value coming to PGMap::apply_incremental()
+    is current pg_sum value so it produces unreasonably huge pg_sum_delta.
+    This patch introduces a workaround to save pg_sum and not update pg_sum_delta
+    by first non-zero incremental.
+    
+    Signed-off-by: Aleksei Gutikov <aleksey.gutikov@synesis.ru>
+    Fixes: http://tracker.ceph.com/issues/21773
+    (cherry picked from commit 013a3803c931ac2e5c390dcd209e1dd0a58418b9)
+    
+    Conflicts:
+            src/mon/PGMap.cc
+
+commit b6f50a463a2757b5822ad5143c4ba8eb86e8d27e
+Author: Yanhu Cao <gmayyyha@gmail.com>
+Date:   Wed Aug 9 10:22:19 2017 +0800
+
+    mgr/Mgr: implement completion of osd MetadataUpdate
+    
+    Fixes: http://tracker.ceph.com/issues/21159
+    
+    Signed-off-by: Yanhu Cao <gmayyyha@gmail.com>
+    (cherry picked from commit cb26eead3f195dd51ae79ec9e41700cdde674648)
+
+commit 948ca443fcdf876436a66cc0188cb7e99e1cb068
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Sep 29 07:02:21 2017 -0400
+
+    mgr/dashboard: redirect away if viewed filesystem is removed
+    
+    Previously this would sit there with stale data in the browser
+    if you were viewing a filesystem and then deleted it in the
+    background.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 83800cd7c11e229e291c699ed45bc65c7fcd9357)
+
+commit 2ed0f0918f4d2d02c57e0ed65c6d790aa185ad8d
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Sep 29 07:01:48 2017 -0400
+
+    mgr/dashboard: 404 instead of 500 on missing filesystem
+    
+    Still not the most beautiful but lets reserve 500s
+    for real errors.
+    
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 7e62cb703c64dbe177cc69d904ed4888b74cdb86)
+
+commit ed2dcad99b9d2ebb7d8d3b836b4761b0c10555f5
+Author: John Spray <john.spray@redhat.com>
+Date:   Fri Sep 29 06:43:24 2017 -0400
+
+    mds: fix FSMap copy constructor
+    
+    This was confusing ceph-mgr, which did a fsmap = new_fsmap
+    and found that old filesystems were never disappearing
+    after being removed.
+    
+    Fixes: http://tracker.ceph.com/issues/21599
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 516896d3c93ace4e610960f0f2b7a9824df3f7f2)
+
+commit 9a7e1f6a448e57f5ae24844d8a7edb0fd4963933
+Author: John Spray <john.spray@redhat.com>
+Date:   Wed Oct 25 09:41:56 2017 -0400
+
+    mds: set PRIO_USEFUL on num_sessions counter
+    
+    This is used by dashboard and status modules,
+    so let's make sure we're sending it to the mgr.
+    
+    Fixes: http://tracker.ceph.com/issues/21927
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 1d0a7d1f35e1c1be472b0555323a9d2ac2d2d777)
+
+commit 5c9d212f94160715a44dca79f18be1c8b2f7ce55
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Tue Oct 31 12:26:07 2017 -0400
+
+    cmake/cls: add install() for ceph_test_cls_log
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 9e94244db2be48bc6ed198a093d7bbb8ae8a72b1)
+
+commit b228d35e1df2302215d751ed35988a34f1238bef
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Mon Oct 30 15:31:03 2017 -0400
+
+    qa: add ceph_test_cls_log to cls workunit
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 83d6b493f71db9417c7b2b7db5f1da9c2d71eec3)
+
+commit 9f09b1bb1f032d626992fa9b63ac613ee0969f3d
+Author: Casey Bodley <cbodley@redhat.com>
+Date:   Sat Oct 28 16:28:27 2017 -0400
+
+    osd: add processed_subop_count for cls_cxx_subop_version()
+    
+    cls_log_add() relies on cls_cxx_subop_version() to generate unique keys
+    for log entries with the same timestamp. because cls calls back into
+    do_osd_ops(), resetting current_osd_subop_num means that cls_log_add()
+    will keep seeing the same subop version and generating the same keys.
+    this causes the following failure in ceph_test_cls_log:
+    
+    [ RUN      ] cls_rgw.test_log_add_same_time
+    /home/cbodley/ceph/src/test/cls_log/test_cls_log.cc:144: Failure
+          Expected: 10
+    To be equal to: (int)entries.size()
+          Which is: 1
+    [  FAILED  ] cls_rgw.test_log_add_same_time (1180 ms)
+    
+    Fixes: http://tracker.ceph.com/issues/21964
+    
+    Signed-off-by: Casey Bodley <cbodley@redhat.com>
+    (cherry picked from commit 95faac9ae2cd38b1ca25c90f90bd4a53fe6af4d4)
+
+commit 1d4435f0cda503b9742f1cad72930bf09bfce265
+Author: Peter Keresztes Schmidt <carbenium@outlook.com>
+Date:   Sun Oct 15 06:36:54 2017 +0200
+
+    rbd: fix crash during map
+    
+    Currently the iterator isn't advanced after the erase call leading to a
+    second call on the iterator, which crashes due to a double free.
+    Since C++11 the map::erase function returns an iterator pointing to the
+    next element. Use the return value to set the iterator after erasing.
+    
+    Fixes: http://tracker.ceph.com/issues/21808
+    
+    Signed-off-by: Peter Keresztes Schmidt <carbenium@outlook.com>
+    (cherry picked from commit 9e49c4124422e58dd40dfb6038425430d3845412)
+
+commit 767ad8518d670e396b1a6a9d9574c5b8479673d1
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Oct 4 10:46:46 2017 -0400
+
+    librbd: list_children should not attempt to refresh image
+    
+    The snap_lock is being held when this method is invoked, which can
+    result in a deadlock.
+    
+    Fixes: http://tracker.ceph.com/issues/21670
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4c585d826f38ff97d3a484a30eca0588c79396b4)
+
+commit 22ff1477348f2fbfe94ef6a08b09687916734aa6
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Aug 24 17:06:30 2017 +0800
+
+    mds: fix StrayManager::truncate()
+    
+    old code does not set PurgeItem::action
+    
+    Fixes: http://tracker.ceph.com/issues/21091
+    Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
+    (cherry picked from commit 45fd51254523f5707e5a67dd7c6ba6011f80e179)
diff --git a/doc/changelog/v9.2.1.txt b/doc/changelog/v9.2.1.txt
new file mode 100644
index 000000000..cdd9e95d5
--- /dev/null
+++ b/doc/changelog/v9.2.1.txt
@@ -0,0 +1,1637 @@
+commit 752b6a3020c3de74e07d2a8b4c5e48dab5a6b6fd (tag: refs/tags/v9.2.1)
+Author: Jenkins Build Slave User <jenkins-build@jenkins-slave-wheezy.localdomain>
+Date:   Wed Feb 24 22:07:26 2016 +0000
+
+    9.2.1
+
+commit 71f380a81c6870466e11a74a597f847494ba23e9
+Merge: 8350215 b0c9176
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 11 22:32:53 2016 +0700
+
+    Merge pull request #7484 from dillaman/wip-14610-infernalis
+    
+    librbd: flattening an rbd image with active IO can lead to hang
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 83502159f5b94139784b908dd4e4bdf1d420e62a
+Merge: f5e33cd 46ed443
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 11 22:32:41 2016 +0700
+
+    Merge pull request #7406 from dillaman/wip-14542-infernalis
+    
+    librbd: ImageWatcher shouldn't block the notification thread
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit f5e33cd348532b4ed911f49240a5da0c6859a9ad
+Merge: a474c09 389ed34
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 11 22:32:25 2016 +0700
+
+    Merge pull request #6981 from dillaman/wip-14062-infernalis
+    
+    librbd: fix merge-diff for >2GB diff-files
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit b0c91768d4ac81baaf9b5edec92adad91ade9b64
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Dec 23 12:06:50 2015 -0500
+
+    librbd: ensure librados callbacks are flushed prior to destroying image
+    
+    Fixes: #14092
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 98157ab3274bd960e4487e34f5a83e9c921a6ac8)
+
+commit f3987367920368cbce865533cf11a1eb207fb9c7
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jul 30 22:31:55 2015 -0400
+
+    librbd: simplify IO flush handling
+    
+    Add a new convenience method to ImageCtx for handling flush
+    requests and cleanup flush handling with dealing with the cache.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (based on commit ee7c6f73992d3b09c6b401fbb782b2151f2399c7)
+
+commit da772fa88db47703d491ebaa9bd709e62858b3b0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Aug 14 13:28:13 2015 -0400
+
+    WorkQueue: PointerWQ drain no longer waits for other queues
+    
+    If another (independent) queue was processing, drain could
+    block waiting.  Instead, allow drain to exit quickly if
+    no items are being processed and the queue is empty for
+    the current WQ.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit b118d7df1e34387b6e5649a5b205cf061598d0d4)
+
+commit 5593e57c1b1c198e95a636ffdda4922259582a5f
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Feb 2 10:54:53 2016 -0500
+
+    test: new librbd flatten test case
+    
+    AIO operations after a flatten operation were previously
+    hanging during the close of the parent image.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5b3a4d2cbca51e5c6795ba7d1189920c7d9af806)
+
+commit 46ed443bf674c50a04705597f1caa4882c1ca5b3
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 28 14:38:20 2016 -0500
+
+    librbd: ImageWatcher shouldn't block the notification thread
+    
+    Blocking the notification thread will also result in librados async
+    callbacks becoming blocked (since they use the same thread).
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 6f94bde44500cc4592ac9a842cbb150b8cabf96b)
+    
+    Conflicts:
+        src/librbd/ImageWatcher.[cc|h]: fewer RPC messages and synchronous
+                                        snapshot actions
+
+commit 2be722a3e944436deff7a8ae513e22b02fd31237
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 28 14:35:54 2016 -0500
+
+    librados_test_stub: watch/notify now behaves similar to librados
+    
+    Notifications are executed via the same librados AIO callback
+    thread, so it's now possible to catch deadlock.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 0a3822f1559ba3fe3def6a65883b9c6c7c5a33fe)
+
+commit 8990ee87c9d943f01de68e772da213dc7aad9011
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Thu Jan 28 12:40:18 2016 -0500
+
+    tests: simulate writeback flush during snap create
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 5e564ea9f869b987f3ada2465edfbe5edf9f6435)
+
+commit 389ed3462f6377d48d87a0e0952061585b0ea106
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Dec 18 15:22:13 2015 -0500
+
+    librbd: fix merge-diff for >2GB diff-files
+    
+    Fixes: #14062
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (derived from commit 68125dd01349edf93cfa1af5028c2d438b5ae089)
+
+commit a474c096a3084ef9f8453e76cebff49e4649d4ea
+Merge: 4c04050 0f50439
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Feb 11 13:59:30 2016 +0700
+
+    Merge pull request #6629 from Abhishekvrshny/wip-13733-infernalis
+    
+    rbd: misdirected op in rbd balance-reads test
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 4c04050379e5d8d8512dc51df6b0ecbd1e37dcd7
+Merge: 1aa60f6 29d30ec
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 10 13:00:23 2016 +0700
+
+    Merge pull request #7431 from Abhishekvrshny/wip-14067-infernalis
+    
+    infernalis : Ceph file system is not freeing space
+    
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+
+commit 1aa60f6680195a557ce81eeabc6794b4c84d87b1
+Merge: 4dc9791 7fd56ea
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 10 12:59:30 2016 +0700
+
+    Merge pull request #7429 from Abhishekvrshny/wip-14490-infernalis
+    
+    infernalis: fsx failed to compile
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 4dc97912b6003069bc1582caf18bbc702a8864be
+Merge: 2bfc75a a2644ed
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 10 12:57:48 2016 +0700
+
+    Merge pull request #6853 from Abhishekvrshny/wip-13889-infernalis
+    
+    infernalis: Segmentation fault accessing file using fuse mount
+    
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+
+commit 2bfc75a0c32685f1151c923a2d84a29549b89ec9
+Merge: e71d062 6c68971
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 10 12:57:09 2016 +0700
+
+    Merge pull request #6752 from ukernel/infernalis-11482
+    
+    mds: fix client capabilities during reconnect (client.XXXX isn't responding to mclientcaps warning)
+    
+    Reviewed-by: Yan, Zheng <zyan@redhat.com>
+
+commit e71d0629caaf0d639efdc9f081bcc2aa8ce156e1
+Merge: 6552eeb 58b43e6
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Feb 10 12:53:34 2016 +0700
+
+    Merge pull request #6628 from Abhishekvrshny/wip-13792-infernalis
+    
+    rbd-replay-* moved from ceph-test-dbg to ceph-common-dbg as well
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 6552eeb2c687e5c7eb8ffbed942f6d99b1c969f9
+Merge: e507a5f 7a44714
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Feb 9 11:58:19 2016 +0700
+
+    Merge pull request #7079 from Abhishekvrshny/wip-14199-infernalis
+    
+    infernalis: [ FAILED ] TestLibRBD.SnapRemoveViaLockOwner
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit e507a5f90f5ec857359a985b28c64d3cb6366bc4
+Merge: 930ef94 83bba36
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Feb 9 11:58:01 2016 +0700
+
+    Merge pull request #7080 from Abhishekvrshny/wip-14142-infernalis
+    
+    infernalis: Verify self-managed snapshot functionality on image create
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 930ef9462ad1d5a92225f2b83ef797b37e22aeba
+Merge: 1061db6 8175e16
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Feb 9 11:57:37 2016 +0700
+
+    Merge pull request #7428 from Abhishekvrshny/wip-14321-infernalis
+    
+    infernalis: cls_rbd: object_map_save should enable checksums
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 1061db6ba16955ef2f3fa49dfed461872b970614
+Merge: 2de6016 2c5b90a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 8 22:23:18 2016 +0700
+
+    Merge pull request #7427 from Abhishekvrshny/wip-14465-infernalis
+    
+    infernalis: rbd-replay does not check for EOF and goes to endless loop
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 2de60165dd76ece7dc84326e2721d61cde05e004
+Merge: a8d147e f870ba9
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 8 22:23:01 2016 +0700
+
+    Merge pull request #7426 from Abhishekvrshny/wip-14552-infernalis
+    
+    infernalis: rbd: TaskFinisher::cancel should remove event from SafeTimer
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit a8d147e813d157855f65e271569cedb0ac95a72d
+Merge: 77a26d8 fde4ec6
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Feb 8 08:49:41 2016 -0500
+
+    Merge pull request #6397 from SUSE/wip-13615-infernalis
+    
+    OSD::build_past_intervals_parallel() shall reset primary and up_primary when begin a new past_interval.
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 77a26d8e7bdb4f351c90a61e8ac87b251294a461
+Merge: cea356a af2ced5
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Feb 8 08:49:17 2016 -0500
+
+    Merge pull request #6840 from SUSE/wip-13791-infernalis
+    
+    Objecter: potential null pointer access when do pool_snap_list.
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit cea356a959ed410a6094140c9d03852e968ab7a8
+Merge: 5469e99 5ef86d7
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Feb 8 08:48:49 2016 -0500
+
+    Merge pull request #6851 from Abhishekvrshny/wip-14018-infernalis
+    
+    infernalis: osd/PG.cc: 288: FAILED assert(info.last_epoch_started >= info.history.last_epoch_started)
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 5469e9904ccc170d282d061308f6694e20ef8946
+Merge: ba4583f 7ac5b15
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Feb 8 08:48:25 2016 -0500
+
+    Merge pull request #6849 from Abhishekvrshny/wip-13979-infernalis
+    
+    osd: call on_new_interval on newly split child PG
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit ba4583f3a25a0d01d31e82cdfcd2be414a5648ae
+Merge: fbd96c8 b211686
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Feb 8 08:48:03 2016 -0500
+
+    Merge pull request #6907 from Abhishekvrshny/wip-13929-infernalis
+    
+    infernalis: Ceph Pools' MAX AVAIL is 0 if some OSDs' weight is 0
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit fbd96c8a49548a4cd6598c700111ac8b1be7b1d3
+Merge: cef8fc3 73564ac
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Feb 8 08:47:36 2016 -0500
+
+    Merge pull request #7421 from Abhishekvrshny/wip-14494-infernalis
+    
+    infernalis: pgs stuck inconsistent after infernalis upgrade
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit cef8fc305e2c52a09b4af9e5a54c3c4117325453
+Merge: 15c27ce 2647c20
+Author: Sage Weil <sage@redhat.com>
+Date:   Mon Feb 8 08:46:25 2016 -0500
+
+    Merge pull request #6627 from Abhishekvrshny/wip-13771-infernalis
+    
+    Objecter: pool op callback may hang forever.
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 15c27ce7328baf92ca904e1b92aa8e21b8452263
+Merge: d1da031 2b6d817
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 8 18:18:07 2016 +0700
+
+    Merge pull request #7543 from SUSE/wip-14676-infernalis
+    
+    infernalis: rgw: radosgw-admin --help doesn't show the orphans find command
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit d1da03131acf735946c3a0deab72d8f7da9e44d8
+Merge: 22e0c6a 1665f81
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 8 11:22:28 2016 +0700
+
+    Merge pull request #6993 from badone/wip-13993-infernalis
+    
+    log: Log.cc: Assign LOG_DEBUG priority to syslog calls
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 22e0c6a7388af5e3b3a085ef7f3e2ad162d2c0f8
+Merge: 489a13d e102e5a
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 8 11:21:11 2016 +0700
+
+    Merge pull request #6882 from dachary/wip-13988-reuse-osd-id-infernalis
+    
+    tests: verify it is possible to reuse an OSD id
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 489a13d4dd4483937dd7bd7798cb6e7d1dbd240c
+Merge: de01ba4 7d90c01
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 8 11:18:35 2016 +0700
+
+    Merge pull request #6852 from Abhishekvrshny/wip-14013-infernalis
+    
+    infernalis: systemd/ceph-disk@.service assumes /bin/flock
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit de01ba4ea2df67e5b732d6e71cdf324642becbd2
+Merge: b5427f4 5d1e6ae
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 8 11:16:23 2016 +0700
+
+    Merge pull request #6846 from Abhishekvrshny/wip-13638-infernalis
+    
+    FileStore: potential memory leak if getattrs fails.
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit b5427f466eb8507dc865d80d0e3293652452f558
+Merge: 5dbd998 ac31f7f
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 8 11:14:18 2016 +0700
+
+    Merge pull request #6836 from SUSE/wip-13891-infernalis
+    
+    infernalis: auth/cephx: large amounts of log are produced by osd
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 5dbd9984e09f3c520c69a406f98577bfdcd9b220
+Merge: 2016468 c968a40
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 8 11:12:47 2016 +0700
+
+    Merge pull request #6833 from SUSE/wip-13935-infernalis
+    
+    infernalis: Ceph daemon failed to start, because the service name was already used.
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 201646882e2c05d2e2436f62b7bfb50a6466aa54
+Merge: b09670f edde13c
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 8 11:12:00 2016 +0700
+
+    Merge pull request #6694 from xiexingguo/xxg-wip-13869
+    
+    osd: fix race condition during send_failures
+    
+    Reviewed-by: David Zafman <dzafman@redhat.com>
+    Reviewed-by: Sage Weil <sage@redhat.com>
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+    Reviewed-by: Kefu Chai <kchai@redhat.com>
+
+commit b09670fea2ff844333c654fb73046dd1596fb2e3
+Merge: 59df52d 4d2a135
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 8 11:09:30 2016 +0700
+
+    Merge pull request #6626 from Abhishekvrshny/wip-13655-infernalis
+    
+    crush: crash if we see CRUSH_ITEM_NONE in early rule step
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 59df52df1697ee8dccb39430940e683a7bdab275
+Merge: f8f36b6 0eb2ddb
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Feb 8 11:06:41 2016 +0700
+
+    Merge pull request #6449 from dachary/wip-13671-infernalis
+    
+    tests: testprofile must be removed before it is re-created
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit 2b6d817187265686d21de76db14870ff22d549de
+Author: Yehuda Sadeh <yehuda@redhat.com>
+Date:   Mon Feb 1 16:33:55 2016 -0800
+
+    rgw-admin: document orphans commands in usage
+    
+    Fixes: #14516
+    
+    Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
+    (cherry picked from commit 105a76bf542e05b739d5a03ca8ae55432350f107)
+
+commit f8f36b6877c0d0d356d1c53a171d5a1a0be423b0
+Merge: 7ffe6ff 399b7c5
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Feb 4 16:23:51 2016 -0500
+
+    Merge pull request #6880 from dachary/wip-14044-infernalis
+    
+    infernalis: ceph-disk list fails on /dev/cciss!c0d0
+
+commit 7ffe6ff4e26db538335b8069ce4fd14fe4cdd76c
+Merge: a8fd5ba 8707b0b
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jan 29 09:05:14 2016 -0500
+
+    Merge pull request #6392 from SUSE/wip-13589-infernalis
+    
+    mon: should not set isvalid = true when cephx_verify_authorizer retur…
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit a8fd5ba7764b941aeb8a8ec2867a7e212159c84d
+Merge: 1296c2b 1bfb6f7
+Author: Sage Weil <sage@redhat.com>
+Date:   Fri Jan 29 08:55:45 2016 -0500
+
+    Merge pull request #6500 from SUSE/wip-13678-infernalis
+    
+    systemd: no rbdmap systemd unit file
+
+commit 29d30ecd4667567a58a7f92641a8a7618e88fd44
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Thu Nov 12 21:57:27 2015 +0800
+
+    mds: properly set STATE_STRAY/STATE_ORPHAN for stray dentry/inode
+    
+    Fixes: #13777
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 460c74a0b872336a7279f0b40b17ed672b6e15a1)
+
+commit 5f54671e0715781146044e2d38ba79ad03634a64
+Author: John Spray <john.spray@redhat.com>
+Date:   Mon Nov 16 10:57:56 2015 +0000
+
+    mon: don't require OSD W for MRemoveSnaps
+    
+    Use ability to execute "osd pool rmsnap" command
+    as a signal that the client should be permitted
+    to send MRemoveSnaps too.
+    
+    Note that we don't also require the W ability,
+    unlike Monitor::_allowed_command -- this is slightly
+    more permissive handling, but anyone crafting caps
+    that explicitly permit "osd pool rmsnap" needs to
+    know what they are doing.
+    
+    Fixes: #13777
+    Signed-off-by: John Spray <john.spray@redhat.com>
+    (cherry picked from commit 0b474c52abd3d528c041544f73b1d27d7d1b1320)
+
+commit 7fd56eaf215753156bdb6481835f399ad303faf0
+Author: Greg Farnum <gfarnum@redhat.com>
+Date:   Wed Jan 13 13:17:53 2016 -0800
+
+    fsx: checkout old version until it compiles properly on miras
+    
+    I sent a patch to xfstests upstream at
+    http://article.gmane.org/gmane.comp.file-systems.fstests/1665, but
+    until that's fixed we need a version that works in our test lab.
+    
+    Signed-off-by: Greg Farnum <gfarnum@redhat.com>
+    (cherry picked from commit 7d52372ae74878ebd001036ff0a7aad525eb15b6)
+
+commit 8175e163fceb2bdc49ca291e183c08575e6c5d18
+Author: Douglas Fuller <dfuller@redhat.com>
+Date:   Thu Jan 7 11:01:19 2016 -0800
+
+    cls_rbd: enable object map checksums for object_map_save
+    
+    object_map_save disables CRCs when an object map footer isn't provided.
+    Unconditionally re-enable object map CRCs before re-encoding the new object
+    map.
+    
+    Fixes: #14280
+    Signed-off-by: Douglas Fuller <dfuller@redhat.com>
+    (cherry picked from commit d5c02f3ed26edec095d45d7a7f26ff26d1b5aacc)
+
+commit 2c5b90a268ebcdcd8c806ef72c08184cc4e0417a
+Author: Mykola Golub <mgolub@mirantis.com>
+Date:   Thu Jan 21 13:45:42 2016 +0200
+
+    rbd-replay: handle EOF gracefully
+    
+    Fixes: #14452
+    Signed-off-by: Mykola Golub <mgolub@mirantis.com>
+    (cherry picked from commit c59b84c3e2c9bbda68219e4d2288a889dd9ca6cb)
+
+commit f870ba9ab41cac127d182a52f435c7adc628de71
+Author: Douglas Fuller <dfuller@redhat.com>
+Date:   Fri Jan 22 11:18:40 2016 -0800
+
+    rbd: remove canceled tasks from timer thread
+    
+    When canceling scheduled tasks using the timer thread, TaskFinisher::cancel
+    does not call SafeTimer::cancel_event, so events fire anyway. Add this call.
+    
+    Fixes: #14476
+    Signed-off-by: Douglas Fuller <dfuller@redhat.com>
+    (cherry picked from commit 2aa0f318c862dbe3027d74d345671506605778eb)
+
+commit 73564ac22319d158d4bade9ab1a3832a9669d099
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Sep 24 11:38:41 2015 -0400
+
+    osd/PG: For performance start scrub scan at pool to skip temp objects
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 05d79faa512210b0f0a91640d18db33b887a6e73)
+
+commit 2480fca93beb700aa9a963f5b568ab6bf8915e0f
+Author: David Zafman <dzafman@redhat.com>
+Date:   Fri Dec 18 09:08:19 2015 -0800
+
+    osd/OSD: clear_temp_objects() include removal of Hammer temp objects
+    
+    Fixes: #13862
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 10b4a0825d9917b6fdd0d6450640238b78ba05d4)
+
+commit 9ff90acc3efdb1c5bdd23c8658e8692f2879231e
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Dec 17 18:04:08 2015 -0800
+
+    osd: Improve log message which isn't about a particular shard
+    
+    Remove redundant dout()
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit e85907fcc582922925609f595f68c597a88c39dc)
+
+commit 1296c2baef3412f462ee2124af747a892ea8b7a9
+Merge: 9dc5e6c d4c41d0
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Jan 13 17:15:41 2016 -0800
+
+    Merge pull request #7225 from dillaman/wip-13810-infernalis
+    
+    tests: notification slave needs to wait for master
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit d4c41d041b7b176723ccd548102a9d2edf91956e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Jan 13 12:44:01 2016 -0500
+
+    tests: notification slave needs to wait for master
+    
+    If the slave instance starts before the master, race
+    conditions are possible.
+    
+    Fixes: #13810
+    Backport: infernalis, hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3992d6fe67bbf82322cedc1582406caaf6d4de60)
+
+commit e102e5a0e261f407913c5d47991c2549f6664ffc
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Dec 10 15:20:32 2015 +0100
+
+    tests: verify it is possible to reuse an OSD id
+    
+    When an OSD id is removed via ceph osd rm, it will be reused by the next
+    ceph osd create command. Verify that and OSD reusing such an id
+    successfully comes up.
+    
+    http://tracker.ceph.com/issues/13988 Refs: #13988
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 7324615bdb829f77928fa10d4e988c6422945937)
+
+commit 399b7c5409d9801aa7f38533dd8b7c19c726a594
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Jan 5 17:33:45 2016 +0100
+
+    ceph-disk: list accepts absolute dev names
+    
+    The ceph-disk list subcommand now accepts /dev/sda as well as sda.
+    The filtering is done on the full list of devices instead of restricting
+    the number of devices explored. Always obtaining the full list of
+    devices makes things simpler when trying to match a dmcrypted device to
+    the corresponding raw device.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 591d581c84cfd72d7c655ac88b0911a318b96e95)
+    
+    Conflicts:
+    	src/ceph-disk: as part of the implementation of deactivate /
+    	destroy in master, the prototype of list_device was changed
+            to take a list of paths instead of the all arguments (args).
+
+commit e222d7bce218b732025ce094ed2d1bb31af0d6f6
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Jan 5 14:25:51 2016 +0100
+
+    ceph-disk: display OSD details when listing dmcrypt devices
+    
+    The details about a device that mapped via dmcrypt are directly
+    available. Do not try to fetch them from the device entry describing the
+    devicemapper entry.
+    
+    http://tracker.ceph.com/issues/14230 Fixes: #14230
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 7aab4ed6f108ddc7bc90300f1999a38f30da3a57)
+    
+    Conflicts:
+    	src/ceph-disk: an incorrect attempt was made to fix the same
+                           problem. It was not backported and does not
+                           need to be. It is entirely contained in the
+                           code block removed and is the reason for the
+                           conflict.
+
+commit 87db693eb7d52ad297e8d265ffffa0d7e1125478
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Dec 9 16:52:10 2015 +0100
+
+    tests: limit ceph-disk unit tests to test dir
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 499c80db606fe3926a8a603e03fdba6967d66003)
+
+commit 2c0d7ef2c72b7570a1304cad927a0b612ea3d4a8
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Jan 5 17:38:59 2016 +0100
+
+    ceph-disk: factorize duplicated dmcrypt mapping
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 35a0c94c4cd3a57cfc382c64eaa9cfb9306dd2e6)
+
+commit 22881b1686dc040d8af01c66f59a935065bb680a
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Jan 5 17:42:11 2016 +0100
+
+    ceph-disk: fix regression in cciss devices names
+    
+    The cciss driver has device paths such as /dev/cciss/c0d1 with a
+    matching /sys/block/cciss!c0d1. The general case is that whenever a
+    device name is found in /sys/block, the / is replaced by the !.
+    
+    When refactoring the ceph-disk list subcommand, this conversion was
+    overlooked in a few places. All explicit concatenation of /dev with a
+    device name are replaced with a call to get_dev_name which does the same
+    but also converts all ! in /.
+    
+    http://tracker.ceph.com/issues/13970 Fixes: #13970
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit a2fd3a535e66b3a2b694cda9c6add33383ccfa4a)
+    
+    Conflicts:
+    	src/ceph-disk : trivial resolution
+
+commit 9dc5e6cbf85681d0fb9ab249c571672456eb6556
+Merge: 64fab4f 5798d1e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Thu Jan 7 15:06:32 2016 +0100
+
+    Merge pull request #7001 from dachary/wip-14145-infernalis
+    
+    infernalis: ceph-disk: use blkid instead of sgdisk -i
+    
+    
+    On CentOS 7.1 and other operating systems with a version of udev greater or equal to 214,
+    running ceph-disk prepare triggered unexpected removal and addition of partitions on
+    the disk being prepared. That created problems ranging from the OSD not being activated
+    to failures because /dev/sdb1 does not exist although it should.
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit 5798d1e76c380182472d08ec6f62ebe0f1a9897e
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Jan 6 23:36:57 2016 +0100
+
+    tests: ceph-disk cryptsetup close must try harder
+    
+    Similar to how it's done in dmcrpyt_unmap in master (
+    132e56615805cba0395898cf165b32b88600d633 ), the infernalis tests helper
+    that were deprecated by the addition of the deactivate / destroy
+    ceph-disk subcommand must try cryptsetup close a few times in some
+    contexts.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+
+commit 379bbe0613e8dbe2f4d38f4f9ee23a10ed39a64d
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Sat Dec 19 00:53:03 2015 +0100
+
+    ceph-disk: protect deactivate with activate lock
+    
+    When ceph-disk prepares the disk, it triggers udev events and each of
+    them ceph-disk activate. If systemctl stop ceph-osd@2 happens while
+    there still are ceph-disk activate in flight, the systemctl stop may be
+    cancelled by the systemctl enable issued by one of the pending ceph-disk
+    activate.
+    
+    This only matters in a test environment where disks are destroyed
+    shortly after they are activated.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 6395bf856b4d4511f0758174ef915ebcafbe3777)
+    
+    Conflicts:
+    
+            src/ceph-disk: ceph-disk deactivate does not exist in ceph-disk
+                on infernalis. But the same feature is implemented in
+                ceph-test-disk.py for test purposes and has the same
+                problem. The patch is adapted to ceph-test-disk.py.
+
+commit 4da2f9494dbd72e84d381cc12125c61931a27628
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Jan 6 11:15:19 2016 +0100
+
+    ceph-disk: retry cryptsetup remove
+    
+    Retry a cryptsetup remove ten times. After the ceph-osd terminates, the
+    device is released asyncrhonously and an attempt to cryptsetup remove
+    will may fail because it is considered busy. Although a few attempts are
+    made before giving up, the number of attempts / the duration of the
+    attempts cannot be controlled with a cryptsetup option. The workaround
+    is to increase this by trying a few times.
+    
+    If cryptsetup remove fails for a reason that is unrelated to timeout,
+    the error will be repeated a few times. There is no undesirable side
+    effect. It will not hide a problem.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 132e56615805cba0395898cf165b32b88600d633)
+
+commit 676a4b42267133a9242a81503f7dd0eaeb0afd33
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Dec 18 17:03:21 2015 +0100
+
+    ceph-disk: use blkid instead of sgdisk -i
+    
+    sgdisk -i 1 /dev/vdb opens /dev/vdb in write mode which indirectly
+    triggers a BLKRRPART ioctl from udev (starting version 214 and up) when
+    the device is closed (see below for the udev release note). The
+    implementation of this ioctl by the kernel (even old kernels) removes
+    all partitions and adds them again (similar to what partprobe does
+    explicitly).
+    
+    The side effects of partitions disappearing while ceph-disk is running
+    are devastating.
+    
+    sgdisk is replaced by blkid which only opens the device in read mode and
+    will not trigger this unexpected behavior.
+    
+    The problem does not show on Ubuntu 14.04 because it is running udev <
+    214 but shows on CentOS 7 which is running udev > 214.
+    
+    git clone git://anonscm.debian.org/pkg-systemd/systemd.git
+    systemd/NEWS:
+    CHANGES WITH 214:
+    
+            * As an experimental feature, udev now tries to lock the
+              disk device node (flock(LOCK_SH|LOCK_NB)) while it
+              executes events for the disk or any of its partitions.
+              Applications like partitioning programs can lock the
+              disk device node (flock(LOCK_EX)) and claim temporary
+              device ownership that way; udev will entirely skip all event
+              handling for this disk and its partitions. If the disk
+              was opened for writing, the close will trigger a partition
+              table rescan in udev's "watch" facility, and if needed
+              synthesize "change" events for the disk and all its partitions.
+              This is now unconditionally enabled, and if it turns out to
+              cause major problems, we might turn it on only for specific
+              devices, or might need to disable it entirely. Device Mapper
+              devices are excluded from this logic.
+    
+    http://tracker.ceph.com/issues/14080 Fixes: #14080
+    
+    Signed-off-by: Ilya Dryomov <idryomov@redhat.com>
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 9dce05a8cdfc564c5162885bbb67a04ad7b95c5a)
+
+commit bb8d53636d3a433bb7752e3a291a86a8af058215
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Dec 16 15:57:03 2015 +0100
+
+    ceph-disk: dereference symlinks in destroy and zap
+    
+    The behavior of partprobe or sgdisk may be subtly different if given a
+    symbolic link to a device instead of an actual device. The debug output
+    is also more confusing when the symlink shows instead of the device it
+    points to.
+    
+    Always dereference the symlink before running destroy and zap.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit fe71647bc9bd0f9ddc6d470ee7bee1e6b0983e2b)
+    
+    Conflicts:
+            src/ceph-disk
+              trivial, because destroy is not implemented
+              in infernalis
+
+commit 74c27423f4925c7a31d6675851ffa3d2f5265ef1
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Dec 16 12:33:25 2015 +0100
+
+    ceph-disk: increase partprobe / udevadm settle timeouts
+    
+    The default of 120 seconds may be exceeded when the disk is very slow
+    which can happen in cloud environments. Increase it to 600 seconds
+    instead.
+    
+    The partprobe command may fail for the same reason but it does not have
+    a timeout parameter. Instead, try a few times before failing.
+    
+    The udevadm settle guarding partprobe are not necessary because
+    partprobe already does the same. However, partprobe does not provide a
+    way to control the timeout. Having a udevadm settle after another is
+    going to be a noop most of the time and not add any delay. It matters
+    when the udevadm settle run by partprobe fails with a timeout because
+    partprobe will silentely ignores the failure.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 730b5d62d3cda7de4076bafa6e9e35f1eb8e2190)
+
+commit 263eb579ac7179a19157004ed2de294085dc4a63
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Dec 16 12:36:47 2015 +0100
+
+    tests: ceph-disk workunit increase verbosity
+    
+    So that reading the teuthology log is enough in most cases to figure out
+    the cause of the error.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit fd7fe8c4977658f66651dad5efb0d816ae71b38b)
+    
+    Conflicts:
+    	qa/workunits/ceph-disk/ceph-disk-test.py:
+              trivial, because destroy/deactivate are not implemented
+              in infernalis. The existing destroy_osd function
+              has to be modified so the id returned by sh() does
+              not have a trailing newline.
+
+commit 936dda76ce11baa9f2b7cce882b655550188a731
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Dec 16 12:30:20 2015 +0100
+
+    ceph-disk: log parted output
+    
+    Should parted output fail to parse, it is useful to get the full output
+    when running in verbose mode.
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit f5d36b9ac299e9f6d52cc32d540cc1c3342de6e7)
+
+commit db26cc4b7fbd5b6f2a3da58fb7ac7736fc058065
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Wed Dec 16 12:29:17 2015 +0100
+
+    ceph-disk: do not discard stderr
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 5fa35ba10e10b56262757afc43929ab8ee4164f2)
+    
+    Conflicts:
+    	src/ceph-disk : trivial, because destroy/deactivate
+            are not implemented in infernalis
+
+commit 83bba36afbae58ebe7cea2b0fb8ff51edb475fff
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Dec 14 17:49:55 2015 -0500
+
+    tests: new integration test for validating new RBD pools
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 00cfe4efacd664032f700afe9701d41bacf8700a)
+
+commit 96cb5e2888e698879f3d3663dfa6f556e983a555
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Dec 14 17:41:49 2015 -0500
+
+    librbd: optionally validate RBD pool configuration (snapshot support)
+    
+    Fixes: #13633
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 1fea4dadc60e13518e9ee55d136fbc4e9d3a621e)
+
+commit 7a44714b66218cf39aa6e1673059a13729c7c39e
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Dec 23 13:26:39 2015 -0500
+
+    librbd: properly handle replay of snap remove RPC message
+    
+    Fixes: #14164
+    Backport: infernalis
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit bc309d9d7612f005a3d50ecf099ddf9b706a1bf6)
+
+commit 64fab4f080c5b7d79b51b8db8c71428b507efec5
+Merge: f480cea 7676343
+Author: Josh Durgin <jdurgin@redhat.com>
+Date:   Wed Dec 23 10:47:30 2015 -0800
+
+    Merge pull request #7038 from dillaman/wip-14121-infernalis
+    
+    tests: rebuild exclusive lock test should acquire exclusive lock
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 7676343c781e9c15e18c5960d221a29840835332
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Wed Dec 23 10:31:07 2015 -0500
+
+    tests: rebuild exclusive lock test should acquire exclusive lock
+    
+    Starting with Jewel, the object map will not be loaded until the
+    exclusive lock is acquired since it might be updated by the
+    lock owner.
+    
+    Fixes: #14121
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 1665f8187b16c9b47238743a7a94f0850160eb71
+Author: Brad Hubbard <bhubbard@redhat.com>
+Date:   Mon Dec 7 11:31:28 2015 +1000
+
+    log: Log.cc: Assign LOG_DEBUG priority to syslog calls
+    
+    Fixes: #13993
+    Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
+    (cherry picked from commit 8e93f3f45db681f82633ca695a7dc4e7bd030584)
+
+commit b211686dddf35d990c07ea50b868b98ac3abf8d3
+Author: Chengyuan Li <chengyli@ebay.com>
+Date:   Thu Nov 19 22:29:39 2015 -0700
+
+    mon/PGMonitor: MAX AVAIL is 0 if some OSDs' weight is 0
+    
+    In get_rule_avail(), even p->second is 0, it's possible to be used
+    as divisor and quotient is infinity, then is converted to an integer
+    which is negative value.
+    So we should check p->second value before calculation.
+    
+    It fixes BUG #13840.
+    
+    Signed-off-by: Chengyuan Li <chengyli@ebay.com>
+    (cherry picked from commit 18713e60edd1fe16ab571f7c83e6de026db483ca)
+
+commit f480cea217008fa7b1e476d30dcb13023e6431d1
+Merge: d3aaeae 34d0b69
+Author: Abhishek Varshney <Abhishekvrshny@users.noreply.github.com>
+Date:   Wed Dec 9 11:22:26 2015 +0530
+
+    Merge pull request #6395 from SUSE/wip-13593-infernalis
+    
+    Ceph-fuse won't start correctly when the option log_max_new in ceph.conf set to zero
+    
+    Reviewed-by: Greg Farnum <gfarnum@redhat.com>
+
+commit d3aaeae1884a26b6a069c6926ebe9cea62625850
+Merge: 476af49 ebc5e4e
+Author: Loic Dachary <loic@dachary.org>
+Date:   Wed Dec 9 00:06:33 2015 +0100
+
+    Merge pull request #6828 from dachary/wip-ceph-disk-augeas
+    
+    tests: ceph-disk workunit uses configobj
+    
+    Reviewed-by: Loic Dachary <ldachary@redhat.com>
+
+commit ebc5e4ee46dce9ab17208cdebc31ae5cc8c7e828
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Oct 22 01:48:31 2015 +0200
+
+    tests: ceph-disk workunit uses the ceph task
+    
+    The ceph-disk workunit deploy keys that are not deployed by default by
+    the ceph teuthology task.
+    
+    The OSD created by the ceph task are removed from the default
+    bucket (via osd rm) so they do not interfere with the tests.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 163de5b0f8f46695ab41b3f2288e9b5c1feaedab)
+
+commit 0afaeb243448819dfce0de2778c27015a46e195a
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Thu Oct 22 00:21:49 2015 +0200
+
+    tests: ceph-disk workunit uses configobj
+    
+    Instead of using augtool to modify the configuration file, use
+    configobj. It is also used by the install teuthology task. The .ini
+    lens (puppet lens really) is unable to read ini files created by
+    configobj.
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit f4906a124cc194dccd855679a04a5c7ffc125a44)
+
+commit a2644ed52c4b8a075919866af9d333fa1f703644
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Mon Nov 9 11:37:02 2015 +0800
+
+    client: use null snapc to check pool permission
+    
+    snap inodes' ->snaprealm can be NULL, so dereferencing it in
+    check_pool_perm() can cause segment fault. The pool permission
+    check does not write any data, so it's safe to use null snapc.
+    
+    Fixes: #13714
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit fad3772fb7731272d47cbfd9e81f22f5df3701a2)
+
+commit 476af491ba03843e953dd257ad972f09c64dd816
+Merge: af86215 54c8af8
+Author: Loic Dachary <loic@dachary.org>
+Date:   Tue Dec 8 09:34:39 2015 +0100
+
+    Merge pull request #6845 from dachary/wip-14019-infernalis
+    
+    infernalis: libunwind package missing on CentOS 7
+    
+    Reviewed-by: Brad Hubbard <bhubbard@redhat.com>
+
+commit 7d90c018e5cfec982f96ce6dd45b6f89218f441d
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Dec 4 21:11:09 2015 +0100
+
+    build/ops: systemd ceph-disk unit must not assume /bin/flock
+    
+    The flock command may be installed elsewhere, depending on the
+    system. Let the PATH search figure that out.
+    
+    http://tracker.ceph.com/issues/13975 Fixes: #13975
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit c8f7d44c935bd097db7d131b785bdab78a7a650c)
+
+commit 5ef86d78f86c8feeae5bade408ce03bd3b86573c
+Author: David Zafman <dzafman@redhat.com>
+Date:   Thu Dec 3 14:52:24 2015 -0800
+
+    osd: Test osd_find_best_info_ignore_history_les config in another assert
+    
+    Signed-off-by: David Zafman <dzafman@redhat.com>
+    (cherry picked from commit 02a9a41f151a3d968bf8066749658659dc6e3ac4)
+
+commit 7ac5b151224f618e2f81387f74b1b7c3159c2b43
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Dec 2 14:50:28 2015 -0500
+
+    osd: call on_new_interval on newly split child PG
+    
+    We must call on_new_interval() on any interval change *and* on the
+    creation of the PG.  Currently we call it from PG::init() and
+    PG::start_peering_interval().  However, PG::split_into() did not
+    do so for the child PG, which meant that the new child feature
+    bits were not properly initialized and the bitwise/nibblewise
+    debug bit was not correctly set.  That, in turn, could lead to
+    various misbehaviors, the most obvious of which is scrub errors
+    due to the sort order mismatch.
+    
+    Fixes: #13962
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit fb120d7b2da5715e7f7d1baa65bfa70d2e5d807a)
+
+commit 5d1e6ae408ee0d9e2972b3de9f81b6a95e86a0a3
+Author: xiexingguo <258156334@qq.com>
+Date:   Mon Oct 26 18:38:01 2015 +0800
+
+    FileStore: potential memory leak if _fgetattrs fails
+    
+    Memory leak happens if _fgetattrs encounters some error and simply returns.
+    Fixes: #13597
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    
+    (cherry picked from commit ace7dd096b58a88e25ce16f011aed09269f2a2b4)
+
+commit 54c8af80c6e253d13993931d55720c8da99a2b2c
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Dec 8 08:02:56 2015 +0100
+
+    build/ops: enable CR in CentOS 7
+    
+    To get libunwind from the CR repositories until CentOS 7.2.1511 is released.
+    
+    http://tracker.ceph.com/issues/13997 Fixes: #13997
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit 247ee6084b58861da601d349bdba739b252d96de)
+
+commit af2ced594699f95f2dcafd82fa15ca480fa844eb
+Author: xiexingguo <258156334@qq.com>
+Date:   Mon Nov 2 21:46:11 2015 +0800
+
+    Objecter: remove redundant result-check of _calc_target in _map_session.
+    
+    Result-code check is currently redundant since _calc_target never returns a negative value.
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    
+    (cherry picked from commit 5a6117e667024f51e65847f73f7589467b6cb762)
+
+commit 43dede3144f98de16f8373f376e716f2f73b6f82
+Author: xiexingguo <258156334@qq.com>
+Date:   Thu Oct 29 17:32:50 2015 +0800
+
+    Objecter: potential null pointer access when do pool_snap_list.
+    
+    Objecter: potential null pointer access when do pool_snap_list. Shall check pool existence first.
+    Fixes: #13639
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    (cherry picked from commit 865541605b6c32f03e188ec33d079b44be42fa4a)
+
+commit ac31f7f8ac168e27b5a33093832aab3bfc97f667
+Author: qiankunzheng <zheng.qiankun@h3c.com>
+Date:   Thu Nov 5 07:29:49 2015 -0500
+
+    auth/cephx: large amounts of log are produced by osd
+    if the auth of osd is deleted when the osd is running, the osd will produce large amounts of log.
+    
+    Fixes:#13610
+    Signed-off-by: Qiankun Zheng <zheng.qiankun@h3c.com>
+    (cherry picked from commit 102f0b19326836e3b0754b4d32da89eb2bc0b03c)
+
+commit c968a40dfb0a0412d1feb7f8f085c8293b42bf4d
+Author: wangchaunhong <root@A22832429.(none)>
+Date:   Tue Oct 20 18:40:23 2015 +0800
+
+    init-ceph: fix systemd-run cant't start ceph daemon sometimes
+    
+    Fixes: #13474
+    Signed-off-by: Chuanhong Wang <wang.chuanhong@zte.com.cn>
+    (cherry picked from commit 2f36909e1e08bac993e77d1781a777b386335669)
+
+commit 0eb2ddb0adf39ef167012e85317e3cfdddf34598
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Tue Nov 3 00:21:51 2015 +0100
+
+    tests: test/librados/test.cc must create profile
+    
+    Now that the create_one_ec_pool function removes the testprofile each
+    time it is called, it must create the testprofile erasure code profile
+    again for the test to use.
+    
+    http://tracker.ceph.com/issues/13664 Refs: #13664
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit a60342942b5a42ee04d59af77a6b904ce62eefc4)
+
+commit 26adf7a3536d7fe5b3a04866e37a4e6afac08509
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Nov 2 20:24:51 2015 +0100
+
+    tests: destroy testprofile before creating one
+    
+    The testprofile erasure code profile is destroyed before creating a new
+    one so that it does not fail when another testprofile erasure code
+    profile already exists with different parameters.
+    
+    This must be done when creating erasure coded pools with the C++
+    interface, in the same way it's done with the C interface.
+    
+    http://tracker.ceph.com/issues/13664 Fixes: #13664
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit 47abab9a6f182aa0abe5047c04402850379bcd6d)
+
+commit c3619f4c1ba16e3d0959c78865ee4218e911da39
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Mon Nov 2 20:23:52 2015 +0100
+
+    tests: add destroy_ec_profile{,_pp} helpers
+    
+    Signed-off-by: Loic Dachary <ldachary@redhat.com>
+    (cherry picked from commit ab46d79bc09fc711fa35302f49eecac81a98519b)
+
+commit 1bfb6f767357fcd7ea424fc17b799fb2c171cf80
+Author: Boris Ranto <branto@redhat.com>
+Date:   Mon Nov 2 14:07:47 2015 +0100
+
+    rbdmap: systemd support
+    
+    Fixes: #13374
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit 9224ac2ad25f7d017916f58b642c0ea25305c3e5)
+
+commit 4eded389eea931fd9cdb0b9700c022088e8825ec
+Author: Boris Ranto <branto@redhat.com>
+Date:   Fri Oct 30 18:33:36 2015 +0100
+
+    rbdmap: Move do_map and do_unmap shell functions to rbdmap script
+    
+    This patch creates rbdmap shell script that is called from init-rbdmap
+    init script. The patch also renames src/rbdmap configuration file to
+    src/etc-rbdmap so that rbdmap shell script can be installed via build
+    system directly. Finally, the patch accomodates these changes in spec
+    file and build system.
+    
+    Fixes: #13374
+    Signed-off-by: Boris Ranto <branto@redhat.com>
+    (cherry picked from commit c0980af3c72f01e6f99fd1e7e91c446934d6d856)
+    
+    Conflicts:
+    	src/init-rbdmap
+                Trivial resolution.
+
+commit 6c68971d1cfc27d30f92a83119e922180b389b43
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Oct 30 21:04:04 2015 +0800
+
+    mds: consider client's flushing caps when choosing lock states
+    
+    Client may flush and drop caps at the same time. If client need to
+    send cap reconnect before the caps get flushed. The issued caps in
+    the cap reconnect does not include the flushing caps. When choosing
+    lock states, MDS only consider the issued caps in cap reconnect, it
+    may choose wrong states.
+    
+    Fixes: #11482
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit ce9a596dcaf95dd4af0a3a9e28871462a6bcb930)
+
+commit 3fcf905dd65d6db514d403e2829b039df7bd9645
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Oct 30 17:28:02 2015 +0800
+
+    client: cancel revoking caps when reconnecting the mds
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 9ab61b2779740d9038b169da0de616b89b07b813)
+
+commit 7e8706bada034ce428716963a5e3151723f83126
+Author: Yan, Zheng <zyan@redhat.com>
+Date:   Fri Oct 30 17:21:11 2015 +0800
+
+    mds: choose EXCL state for filelock when client has Fb capability
+    
+    Signed-off-by: Yan, Zheng <zyan@redhat.com>
+    (cherry picked from commit 655ae79dd266dfd488aff67642aceab4ab383d01)
+
+commit af86215e0b178340055ff6ec7c8ec3b3c0bd1acf
+Merge: 899ff61 497c1f0
+Author: Abhishek Varshney <Abhishekvrshny@users.noreply.github.com>
+Date:   Tue Dec 1 17:44:24 2015 +0530
+
+    Merge pull request #6634 from Abhishekvrshny/wip-13761-infernalis
+    
+    unknown argument --quiet in udevadm settle
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 899ff6160a289dfeb5c287afa8d9b168e11382ed
+Merge: f2b323a fdb5dbd
+Author: Abhishek Varshney <Abhishekvrshny@users.noreply.github.com>
+Date:   Mon Nov 30 21:56:40 2015 +0530
+
+    Merge pull request #6650 from Abhishekvrshny/wip-13830-infernalis
+    
+    init script reload doesn't work on EL7
+    
+    Reviewed-by: Boris Ranto <branto@redhat.com>
+
+commit f2b323a69bb54ee99743bf53043d6a4e21e2a0bb
+Merge: 5c02e2b effaf26
+Author: Abhishek Varshney <Abhishekvrshny@users.noreply.github.com>
+Date:   Mon Nov 30 21:55:55 2015 +0530
+
+    Merge pull request #6477 from SUSE/wip-13705-infernalis
+    
+    rbd : enable feature objectmap
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit 5c02e2bd78782f1ea19836806d3773748be66ed1
+Merge: ed8ac03 c2e7fe9
+Author: Abhishek Varshney <Abhishekvrshny@users.noreply.github.com>
+Date:   Mon Nov 30 21:55:22 2015 +0530
+
+    Merge pull request #6474 from SUSE/wip-13619-infernalis
+    
+    rbd clone issue
+    
+    Reviewed-by: Jason Dillaman <dillaman@redhat.com>
+
+commit ed8ac030a979ca9fb2cbf340c316c4432e54cd0d
+Merge: 1551c1b ad84753
+Author: Abhishek Varshney <Abhishekvrshny@users.noreply.github.com>
+Date:   Mon Nov 30 21:54:44 2015 +0530
+
+    Merge pull request #6633 from Abhishekvrshny/wip-13759-infernalis
+    
+    rbd: pure virtual method called
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit 1551c1be297cfc73457819c7c983b113ac9c86f6
+Merge: b8000d6 83c3880
+Author: Abhishek Varshney <Abhishekvrshny@users.noreply.github.com>
+Date:   Mon Nov 30 21:54:18 2015 +0530
+
+    Merge pull request #6632 from Abhishekvrshny/wip-13756-infernalis
+    
+    QEMU hangs after creating snapshot and stopping VM
+    
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit b8000d6f489af3e13c4996b46363c3884ccfd0de
+Merge: fc6f21a 74cb593
+Author: Abhishek Varshney <Abhishekvrshny@users.noreply.github.com>
+Date:   Mon Nov 30 21:50:40 2015 +0530
+
+    Merge pull request #6630 from Abhishekvrshny/wip-13754-infernalis
+    
+    Avoid re-writing old-format image header on resize
+    Reviewed-by: Josh Durgin <jdurgin@redhat.com>
+
+commit fc6f21a1d3ff34b4ff522ef03b368b3ad4570a1c
+Merge: bb2ecea a2c6a21
+Author: Loic Dachary <loic@dachary.org>
+Date:   Mon Nov 30 15:01:23 2015 +0100
+
+    Merge pull request #6396 from SUSE/wip-13342-infernalis
+    
+    ceph upstart script rbdmap.conf incorrectly processes parameters
+    
+    Reviewed-by: Sage Weil <sage@redhat.com>
+
+commit edde13ce55d0789523a3101e5f435fc61bef5d69
+Author: Sage Weil <sage@redhat.com>
+Date:   Thu Sep 17 21:42:53 2015 -0400
+
+    osd: fix send_failures() locking
+    
+    It is unsafe to check failure_queue.empty() without the lock.
+    Fixes: #13869
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit b3ca828ae8ebc9068073494c46faf3e8e1443ada)
+
+commit 0f5043975dbde75036b9f9e8be757494b8cdc73b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 23 14:17:06 2015 -0400
+
+    osd: move misdirected op check from OSD thread to PG thread
+    
+    The op message isn't fully decoded until after the start of the PG
+    op handling.  Therefore, checking for misdirected ops needs to wait
+    until after the RWM flags have been initialized.
+    
+    Fixes: #13491
+    Backport: infernalis
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 56ba90f5e49d499ea9f6bda9665031ce90abdc32)
+    
+    Conflicts:
+    	src/osd/ReplicatedPG.cc
+    		In ReplicatedPG::do_op, resolved conflicts caused by afcfb050930e2e4086d631119022fd2dee9feb21 and 818d790f7d424520bc96c651571f2c86e94caf1e
+
+commit a2c6a212fbb8002fa95270c0f37c02496447624f
+Author: Sage Weil <sage@redhat.com>
+Date:   Wed Sep 30 08:29:05 2015 -0400
+
+    init-rbdmap: fix CMDPARAMS
+    
+    Fixes: #13214
+    Reported-by: Wyllys Ingersoll <wyllys.ingersoll@keepertech.com>
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 02113ac449cd7631f1c9a3840c94bbf253c052bd)
+
+commit fdb5dbd744a6cfa4f7497e3a98aed13ab8daf4d9
+Author: Herve Rousseau <hroussea@cern.ch>
+Date:   Fri Nov 6 09:52:28 2015 +0100
+
+    rgw: fix reload on non Debian systems.
+    
+    When using reload in non-debian systems, /bin/sh's kill is used to send the HUP signal to the radosgw process.
+    This kill version doesn't understand -SIGHUP as a valid signal, using -HUP does work.
+    
+    Fix: #13709
+    Backport: hammer
+    Signed-off-by: Hervé Rousseau <hroussea@cern.ch>
+    (cherry picked from commit 1b000abac3a02d1e788bf25eead4b6873133f5d2)
+
+commit 497c1f06a2e221631626ffb66bece310406a7491
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 27 10:13:27 2015 -0400
+
+    krbd: remove deprecated --quiet param from udevadm
+    
+    This parameter has been removed since systemd 213, so this
+    effects Fedora 21+, Debian Jessie, and potentially future
+    releases of RHEL 7.
+    
+    Fixes: #13560
+    Backport: hammer, infernalis
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 4300f2a9fe29627eea580564ff2d576de3647467)
+
+commit d0536f9d2f5f54791daa476a9b951fcef9488a50
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Oct 27 10:12:34 2015 -0400
+
+    run_cmd: close parent process console file descriptors
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit f46f7dc94139c0bafe10361622416d7dc343d31f)
+
+commit ad84753ad2d2bfb9ef572a7d21cacb47be6eec1b
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Tue Jul 7 12:11:13 2015 -0400
+
+    WorkQueue: new PointerWQ base class for ContextWQ
+    
+    The existing work queues do not properly function if added to a running
+    thread pool.  librbd uses a singleton thread pool which requires
+    dynamically adding/removing work queues as images are opened and closed.
+    
+    Fixes: #13636
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 3e78b18b09d75626ca2599bac3b9f9c9889507a5)
+
+commit 83c38802a2be16faf7be4ac5ac3c589de8b2dbb0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 9 11:22:24 2015 -0500
+
+    librbd: fixed deadlock while attempting to flush AIO requests
+    
+    In-flight AIO requests might force a flush if a snapshot was created
+    out-of-band.  The flush completion was previously invoked asynchronously,
+    potentially via the same thread worker handling the AIO request. This
+    resulted in the flush operation deadlocking since it can't complete.
+    
+    Fixes: #13726
+    Backport: infernalis, hammer
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit bfeb90e5fe24347648c72345881fd3d932243c98)
+
+commit 0b21b3f0dc2a9676f3fb2b19141ce75324f53c72
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 9 10:48:10 2015 -0500
+
+    tests: new test case to catch deadlock on RBD image refresh
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit a9729d9553e7fb925509cad8d388cf52a9fede9c)
+
+commit 74cb593a3d29a5549cbe89a974aaa4814d35b8e0
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Mon Nov 2 16:50:19 2015 -0500
+
+    librbd: resize should only update image size within header
+    
+    Previously, the whole RBD image format 1 header struct was
+    re-written to disk on a resize operation.
+    
+    Fixes: #13674
+    Backport: infernalis, hammer, firefly
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit d5be20b6d4646284571568ab28cbf45b0729390b)
+
+commit 6750cab2d6f2c5f928f9c2d2c356047f93e9ce30
+Author: Jason Dillaman <dillaman@redhat.com>
+Date:   Fri Oct 23 14:15:54 2015 -0400
+
+    osd: ensure op rwm flags are checked before they are initialized
+    
+    These flags are now lazy-initialized within the PG thread at the
+    start of op processing instead of once the message is received.
+    
+    Signed-off-by: Jason Dillaman <dillaman@redhat.com>
+    (cherry picked from commit 652856321b0f14293cd885c865552d5b7e787f9a)
+
+commit 58b43e656cd9278667af2b7ac4fee72f50a03caa
+Author: Loic Dachary <ldachary@redhat.com>
+Date:   Fri Nov 13 19:10:28 2015 +0100
+
+    build/ops: rbd-replay moved from ceph-test-dbg to ceph-common-dbg
+    
+    http://tracker.ceph.com/issues/13785 Fixes: #13785
+    
+    Signed-off-by: Loic Dachary <loic@dachary.org>
+    (cherry picked from commit a0204dc47acc0c50223e449fd23a2fc97bfd111a)
+
+commit 2647c20cee7ed5738c711a0c435e35938c7739bd
+Author: xiexingguo <258156334@qq.com>
+Date:   Thu Oct 29 20:04:11 2015 +0800
+
+    Objecter: pool_op callback may hang forever.
+    
+    pool_op callback may hang forever due to osdmap update during reply handling.
+    Fixes: #13642
+    Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>
+    
+    (cherry picked from commit 00c6fa9e31975a935ed2bb33a099e2b4f02ad7f2)
+
+commit 4d2a135c042039e0c48302ffd667dc88d024eed9
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 13 09:55:01 2015 -0400
+
+    crush/mapper: ensure take bucket value is valid
+    
+    Ensure that the take argument is a valid bucket ID before indexing the
+    buckets array.
+    
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 93ec538e8a667699876b72459b8ad78966d89c61)
+
+commit ecb6aa232180e3226d56bc8fe0cb643c90f5101b
+Author: Sage Weil <sage@redhat.com>
+Date:   Tue Oct 27 20:55:26 2015 -0400
+
+    crush/mapper: ensure bucket id is valid before indexing buckets array
+    
+    We were indexing the buckets array without verifying the index was within
+    the [0,max_buckets) range.  This could happen because a multistep rule
+    does not have enough buckets and has CRUSH_ITEM_NONE
+    for an intermediate result, which would feed in CRUSH_ITEM_NONE and
+    make us crash.
+    
+    Fixes: #13477
+    Signed-off-by: Sage Weil <sage@redhat.com>
+    (cherry picked from commit 976a24a326da8931e689ee22fce35feab5b67b76)
+
+commit effaf26acccf81675b74277eb0a75368668b0677
+Author: xinxin shu <xinxin.shu@intel.com>
+Date:   Wed Oct 21 19:01:21 2015 +0800
+
+    librbd : fix enable objectmap feature issue
+    
+    Fixes: #13558
+    
+    Signed-off-by: xinxin shu <xinxin.shu@intel.com>
+    (cherry picked from commit b0536ebab4e1f34e1ed87fe5efbb00d0f7b48abb)
+
+commit c2e7fe98738cd31c0886d594920c50a53cbf9bf4
+Author: xinxin shu <xinxin.shu@intel.com>
+Date:   Wed Oct 21 14:56:17 2015 +0800
+
+    rbd: fix clone issue when we specify image feature
+    
+    Fixes: #13553
+    
+    Signed-off-by: xinxin shu <xinxin.shu@intel.com>
+    (cherry picked from commit da48dbb8f8c9417343d2ca7819c58b7c46ef7ad0)
+
+commit fde4ec62888ba0aee97ce714bb15d191625b736f
+Author: xiexingguo <258156334@qq.com>
+Date:   Tue Oct 13 14:04:20 2015 +0800
+
+    OSD:shall reset primary and up_primary fields when beginning a new past_interval.
+    
+    Shall reset primary and up_primary fields when we start over a new past_interval in OSD::build_past_intervals_parallel().
+    Fixes: #13471
+    Signed-off-by: xie.xingguo@zte.com.cn
+    (cherry picked from commit 65064ca05bc7f8b6ef424806d1fd14b87add62a4)
+
+commit 34d0b692cf1b9c0c2ea508b81d252b901fe1f52b
+Author: wenjunhuang <wenjunhuang@tencent.com>
+Date:   Sat Oct 10 14:30:56 2015 +0800
+
+    ceph-fuse.cc: While starting ceph-fuse, start the log thread first
+    
+    http://tracker.ceph.com/issues/13443 Fixes: #13443
+    
+    Signed-off-by: Wenjun Huang <wenjunhuang@tencent.com>
+    (cherry picked from commit f2763085754462610730a23bb5652237714abc2a)
+
+commit 8707b0b083e7a7732d1e3563b00d3adf3e09807b
+Author: yangruifeng <yangruifeng.09209@h3c.com>
+Date:   Mon Oct 19 08:08:12 2015 -0400
+
+    mon: should not set isvalid = true when cephx_verify_authorizer return false
+    
+    Fixes: #13525
+    Signed-off-by: Ruifeng Yang <yangruifeng.09209@h3c.com>
+    (cherry picked from commit c7f75b8f7c0a773148ec16141941efd00ee76626)
diff --git a/doc/conf.py b/doc/conf.py
new file mode 100644
index 000000000..490e886cd
--- /dev/null
+++ b/doc/conf.py
@@ -0,0 +1,231 @@
+import fileinput
+import os
+import shutil
+import sys
+
+import yaml
+
+
+top_level = \
+    os.path.dirname(
+        os.path.dirname(
+            os.path.abspath(__file__)))
+
+
+def parse_ceph_release():
+    with open(os.path.join(top_level, 'src/ceph_release')) as f:
+        lines = f.readlines()
+        assert(len(lines) == 3)
+        # 16, pacific, dev
+        version, codename, status = [line.strip() for line in lines]
+        return version, codename, status
+
+
+def latest_stable_release():
+    with open(os.path.join(top_level, 'doc/releases/releases.yml')) as input:
+        releases = yaml.safe_load(input)['releases']
+        # get the first release
+        return next(iter(releases.keys()))
+
+
+def is_release_eol(codename):
+    with open(os.path.join(top_level, 'doc/releases/releases.yml')) as input:
+        releases = yaml.safe_load(input)['releases']
+        return 'actual_eol' in releases.get(codename, {})
+
+
+# project information
+project = 'Ceph'
+copyright = ('2016, Ceph authors and contributors. '
+             'Licensed under Creative Commons Attribution Share Alike 3.0 '
+             '(CC-BY-SA-3.0)')
+version, codename, release = parse_ceph_release()
+pygments_style = 'sphinx'
+
+# HTML output options
+html_theme = 'ceph'
+html_theme_path = ['_themes']
+html_title = "Ceph Documentation"
+html_logo = 'logo.png'
+html_context = {'is_release_eol': is_release_eol(codename)}
+html_favicon = 'favicon.ico'
+html_show_sphinx = False
+html_static_path = ["_static"]
+html_sidebars = {
+    '**': ['smarttoc.html', 'searchbox.html']
+    }
+
+html_css_files = ['css/custom.css']
+
+# general configuration
+templates_path = ['_templates']
+source_suffix = '.rst'
+exclude_patterns = ['**/.#*',
+                    '**/*~',
+                    'start/quick-common.rst',
+                    '**/*.inc.rst']
+if tags.has('man'):             # noqa: F821
+    master_doc = 'man_index'
+    exclude_patterns += ['index.rst',
+                         'architecture.rst',
+                         'glossary.rst',
+                         'release*.rst',
+                         'api/*',
+                         'cephadm/*',
+                         'cephfs/*',
+                         'dev/*',
+                         'governance.rst',
+                         'foundation.rst',
+                         'install/*',
+                         'mon/*',
+                         'rados/*',
+                         'mgr/*',
+                         'ceph-volume/*',
+                         'radosgw/*',
+                         'rbd/*',
+                         'start/*',
+                         'releases/*']
+else:
+    master_doc = 'index'
+    exclude_patterns += ['man_index.rst']
+
+build_with_rtd = os.environ.get('READTHEDOCS') == 'True'
+
+sys.path.insert(0, os.path.abspath('_ext'))
+
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.graphviz',
+    'sphinx.ext.mathjax',
+    'sphinx.ext.todo',
+    'sphinx-prompt',
+    'sphinx_autodoc_typehints',
+    'sphinx_substitution_extensions',
+    'breathe',
+    'ceph_commands',
+    'ceph_releases',
+    'sphinxcontrib.openapi'
+    ]
+
+ditaa = shutil.which("ditaa")
+if ditaa is not None:
+    # in case we don't have binfmt_misc enabled or jar is not registered
+    ditaa_args = ['-jar', ditaa]
+    ditaa = 'java'
+    extensions += ['sphinxcontrib.ditaa']
+else:
+    extensions += ['plantweb.directive']
+    plantweb_defaults = {
+        'engine': 'ditaa'
+    }
+
+if build_with_rtd:
+    extensions += ['sphinx_search.extension']
+
+# sphinx.ext.todo options
+todo_include_todos = True
+
+# sphinx_substitution_extensions options
+rst_prolog = f"""
+.. |stable-release| replace:: {latest_stable_release()}
+"""
+
+# breath options
+breathe_default_project = "Ceph"
+# see $(top_srcdir)/Doxyfile
+
+breathe_build_directory = os.path.join(top_level, "build-doc")
+breathe_projects = {"Ceph": os.path.join(top_level, breathe_build_directory)}
+breathe_projects_source = {
+    "Ceph": (os.path.join(top_level, "src/include/rados"),
+             ["rados_types.h", "librados.h"])
+}
+breathe_domain_by_extension = {'py': 'py',
+                               'c': 'c', 'h': 'c',
+                               'cc': 'cxx', 'hpp': 'cxx'}
+breathe_doxygen_config_options = {
+    'EXPAND_ONLY_PREDEF': 'YES',
+    'MACRO_EXPANSION': 'YES',
+    'PREDEFINED': 'CEPH_RADOS_API= ',
+    'WARN_IF_UNDOCUMENTED': 'NO',
+}
+
+# edit_on_github options
+# the docs are rendered with github links pointing to master. the javascript
+# snippet in _static/ceph.js rewrites the edit links when a page is loaded, to
+# point to the correct branch.
+edit_on_github_project = 'ceph/ceph'
+edit_on_github_branch = 'master'
+
+# graphviz options
+graphviz_output_format = 'svg'
+
+def generate_state_diagram(input_paths, output_path):
+    sys.path.append(os.path.join(top_level, 'doc', 'scripts'))
+    from gen_state_diagram import do_filter, StateMachineRenderer
+    inputs = [os.path.join(top_level, fn) for fn in input_paths]
+    output = os.path.join(top_level, output_path)
+
+    def process(app):
+        with fileinput.input(files=inputs) as f:
+            input = do_filter(f)
+            render = StateMachineRenderer()
+            render.read_input(input)
+            with open(output, 'w') as dot_output:
+                render.emit_dot(dot_output)
+
+    return process
+
+
+# mocking ceph_module offered by ceph-mgr. `ceph_module` is required by
+# mgr.mgr_module
+class Dummy(object):
+    def __getattr__(self, _):
+        return lambda *args, **kwargs: None
+
+
+class Mock(object):
+    __all__ = []
+
+    def __init__(self, *args, **kwargs):
+        pass
+
+    def __call__(self, *args, **kwargs):
+        return Mock()
+
+    @classmethod
+    def __getattr__(cls, name):
+        mock = type(name, (Dummy,), {})
+        mock.__module__ = __name__
+        return mock
+
+
+# autodoc options
+sys.modules['ceph_module'] = Mock()
+
+if build_with_rtd:
+    autodoc_mock_imports = ['ceph']
+    pybinds = ['pybind/mgr',
+               'python-common']
+else:
+    pybinds = ['pybind',
+               'pybind/mgr',
+               'python-common']
+
+for c in pybinds:
+    pybind = os.path.join(top_level, 'src', c)
+    if pybind not in sys.path:
+        sys.path.insert(0, pybind)
+
+
+# handles edit-on-github and old version warning display
+def setup(app):
+    app.add_js_file('js/ceph.js')
+    if ditaa is None:
+        # add "ditaa" as an alias of "diagram"
+        from plantweb.directive import DiagramDirective
+        app.add_directive('ditaa', DiagramDirective)
+    app.connect('builder-inited',
+                generate_state_diagram(['src/osd/PeeringState.h',
+                                        'src/osd/PeeringState.cc'],
+                                       'doc/dev/peering_graph.generated.dot'))
diff --git a/doc/dev/PlanningImplementation.txt b/doc/dev/PlanningImplementation.txt
new file mode 100644
index 000000000..871eb5f37
--- /dev/null
+++ b/doc/dev/PlanningImplementation.txt
@@ -0,0 +1,43 @@
+ <big>About this Document</big>
+This document contains planning and implementation procedures for Ceph.  The audience for this document includes technical support personnel, installation engineers, system administrators, and quality assurance.  
+<B>Prerequisites<b>
+Users of this document must be familiar with Linux command line options.  They must also be familiar with the overall Ceph product. 
+Before You Begin
+Before implementing a new Ceph System, first answer the questions in the Ceph Getting Started Guide to determine your configuration needs.  Once you have determined your hardware and configuration needs, the following decisions must be made:
+•	Determine what level of technical support you need.  Pick from the Ceph Technical Support options in the next section.
+•	Determine how much and what level of training your organization needs.
+Ceph Technical Support Options
+The Ceph Technical support model provides 4 tiers of technical support options:
+1st – This option is for brand new customers that need installation, configuration, and setup on their production environment.  
+2nd – This level of support requires a trouble ticket to be generated on a case by case basis as customer difficulties arise.  Customers can choose between two maintenance options; they can either purchase a yearly maintenance contract, or pay for each trouble resolution as it occurs.
+3rd – This option comes with our bundled packages for customers who have also purchased our hosting plans.  In this case, the customer is a service provider. The Help Desk can generally provide this level of incident resolution. (NEED MORE INFO)
+4th – This level of support requires a Service Level Agreement (SLA) between the customer and Dreamhost.  This level is used for handling the most difficult or advanced problems.
+Planning a Ceph Cluster Configuration
+The following section contains guidelines for planning the deployment for a Ceph cluster configuration.  A Ceph cluster consists of the following core components:
+•	Monitors – These must be an odd number, such as one, three, or five.  Three is the preferred configuration.
+•	Object Storage Devices (OSD) – used as storage nodes
+•	Metadata Servers (MDS)
+For redundancy, you should employ several of these components. 
+Monitors 
+The monitors handle central cluster management, configuration, and state.  
+Hardware Requirements: 
+•	A few gigs of local disk space 
+•	A fixed network address 
+ Warning: Never configure 2 monitors per cluster.  If you do, they will both have to be up all of the time, which will greatly degrade system performance. 
+Object Storage Devices 
+The OSDs store the actual data on the disks. A minimum of two is required. 
+Hardware Requirements: 
+•	As many disks as possible for faster performance and scalability
+•	An SSD or NVRAM for a journal, or a RAID controller with a battery-backed NVRAM. 
+•	Ample RAM for better file system caching 
+•	Fast network 
+ Metadata Servers 
+The metadata server daemon commands act as a distributed, coherent cache of file system metadata. They do not store data locally; all metadata is stored on disk via the storage nodes. 
+Metadata servers can be added into the cluster on an as-needed basis.  The load is automatically balanced. The max_mds parameter controls how many cmds instances are active. Any additional running instances are put in standby mode and can be activated if one of the active daemons becomes unresponsive. 
+Hardware Requirements: 
+•	Large amount of  RAM 
+•	Fast CPU 
+•	Fast (low latency) network 
+•	At least two servers for redundancy and load balancing
+TIPS: If you have just a few nodes, put cmon, cmds, and cosd on the same node.  For moderate node configurations, put cmon and cmds together, and cosd on the disk nodes.  For large node configurations, put cmon, cmds, and cosd each on their own dedicated machine. 
+
diff --git a/doc/dev/blkin.rst b/doc/dev/blkin.rst
new file mode 100644
index 000000000..00a5748fa
--- /dev/null
+++ b/doc/dev/blkin.rst
@@ -0,0 +1,169 @@
+=========================
+ Tracing Ceph With Blkin
+=========================
+
+Ceph can use Blkin, a library created by Marios Kogias and others,
+which enables tracking a specific request from the time it enters
+the system at higher levels till it is finally served by RADOS.
+
+In general, Blkin implements the Dapper_ tracing semantics
+in order to show the causal relationships between the different
+processing phases that an IO request may trigger. The goal is an
+end-to-end visualisation of the request's route in the system,
+accompanied by information concerning latencies in each processing
+phase. Thanks to LTTng this can happen with a minimal overhead and
+in realtime. The LTTng traces can then be visualized with Twitter's
+Zipkin_.
+
+.. _Dapper: http://static.googleusercontent.com/media/research.google.com/el//pubs/archive/36356.pdf
+.. _Zipkin: https://zipkin.io/
+
+
+Installing Blkin
+================
+
+You can install Markos Kogias' upstream Blkin_ by hand.::
+
+  cd blkin/
+  make && make install
+
+or build distribution packages using DistroReadyBlkin_, which also comes with
+pkgconfig support. If you choose the latter, then you must generate the
+configure and make files first.::
+
+  cd blkin
+  autoreconf -i
+
+.. _Blkin: https://github.com/marioskogias/blkin
+.. _DistroReadyBlkin: https://github.com/agshew/blkin
+
+
+Configuring Ceph with Blkin
+===========================
+
+If you built and installed Blkin by hand, rather than building and
+installing packages, then set these variables before configuring
+Ceph.::
+
+  export BLKIN_CFLAGS=-Iblkin/
+  export BLKIN_LIBS=-lzipkin-cpp
+
+Blkin support in Ceph is disabled by default, so you may
+want to configure with something like::
+
+  ./do_cmake -DWITH_BLKIN=ON
+
+Config option for blkin must be set to true in ceph.conf to get
+traces from rbd through OSDC and OSD::
+
+  rbd_blkin_trace_all = true
+
+
+Testing Blkin
+=============
+
+It's easy to test Ceph's Blkin tracing. Let's assume you don't have
+Ceph already running, and you compiled Ceph with Blkin support but
+you didn't install it. Then launch Ceph with the ``vstart.sh`` script
+in Ceph's src directory so you can see the possible tracepoints.::
+
+  cd src
+  OSD=3 MON=3 RGW=1 ./vstart.sh -n
+  lttng list --userspace
+
+You'll see something like the following:::
+
+  UST events:
+  -------------
+  PID: 8987 - Name: ./ceph-osd
+        zipkin:timestamp (loglevel: TRACE_WARNING (4)) (type: tracepoint)
+        zipkin:keyval_integer (loglevel: TRACE_WARNING (4)) (type: tracepoint)
+        zipkin:keyval_string (loglevel: TRACE_WARNING (4)) (type: tracepoint)
+        lttng_ust_tracelog:TRACE_DEBUG (loglevel: TRACE_DEBUG (14)) (type: tracepoint)
+
+  PID: 8407 - Name: ./ceph-mon
+        zipkin:timestamp (loglevel: TRACE_WARNING (4)) (type: tracepoint)
+        zipkin:keyval_integer (loglevel: TRACE_WARNING (4)) (type: tracepoint)
+        zipkin:keyval_string (loglevel: TRACE_WARNING (4)) (type: tracepoint)
+        lttng_ust_tracelog:TRACE_DEBUG (loglevel: TRACE_DEBUG (14)) (type: tracepoint)
+
+  ...
+
+Next, stop Ceph so that the tracepoints can be enabled.::
+
+  ./stop.sh
+
+Start up an LTTng session and enable the tracepoints.::
+
+  lttng create blkin-test
+  lttng enable-event --userspace zipkin:timestamp
+  lttng enable-event --userspace zipkin:keyval_integer
+  lttng enable-event --userspace zipkin:keyval_string
+  lttng start
+
+Then start up Ceph again.::
+
+  OSD=3 MON=3 RGW=1 ./vstart.sh -n
+
+You may want to check that ceph is up.::
+
+  ./ceph status
+
+Now put something in using rados, check that it made it, get it back, and remove it.::
+
+  ./ceph osd pool create test-blkin
+  ./rados put test-object-1 ./vstart.sh --pool=test-blkin
+  ./rados -p test-blkin ls
+  ./ceph osd map test-blkin test-object-1
+  ./rados get test-object-1 ./vstart-copy.sh --pool=test-blkin
+  md5sum vstart*
+  ./rados rm test-object-1 --pool=test-blkin
+
+You could also use the example in ``examples/librados/`` or ``rados bench``.
+
+Then stop the LTTng session and see what was collected.::
+
+  lttng stop
+  lttng view
+
+You'll see something like:::
+
+  [15:33:08.884275486] (+0.000225472) ubuntu zipkin:timestamp: { cpu_id = 53 }, { trace_name = "op", service_name = "Objecter", port_no = 0, ip = "0.0.0.0", trace_id = 5485970765435202833, span_id = 5485970765435202833, parent_span_id = 0, event = "osd op reply" }
+  [15:33:08.884614135] (+0.000002839) ubuntu zipkin:keyval_integer: { cpu_id = 10 }, { trace_name = "", service_name = "Messenger", port_no = 6805, ip = "0.0.0.0", trace_id = 7381732770245808782, span_id = 7387710183742669839, parent_span_id = 1205040135881905799, key = "tid", val = 2 }
+  [15:33:08.884616431] (+0.000002296) ubuntu zipkin:keyval_string: { cpu_id = 10 }, { trace_name = "", service_name = "Messenger", port_no = 6805, ip = "0.0.0.0", trace_id = 7381732770245808782, span_id = 7387710183742669839, parent_span_id = 1205040135881905799, key = "entity type", val = "client" }
+
+
+Install  Zipkin
+===============
+One of the points of using Blkin is so that you can look at the traces
+using Zipkin. Users should run Zipkin as a tracepoints collector and
+also a web service. The executable jar runs a collector on port 9410 and
+the web interface on port 9411
+
+Download Zipkin Package::
+
+  git clone https://github.com/openzipkin/zipkin && cd zipkin
+  wget -O zipkin.jar 'https://search.maven.org/remote_content?g=io.zipkin.java&a=zipkin-server&v=LATEST&c=exec'
+  java -jar zipkin.jar
+
+
+Show Ceph's Blkin Traces in Zipkin-web
+======================================
+Download babeltrace-zipkin project. This project takes the traces
+generated with blkin and sends them to a Zipkin collector using scribe::
+
+  git clone https://github.com/vears91/babeltrace-zipkin
+  cd babeltrace-zipkin
+
+Send lttng data to Zipkin::
+
+  python3 babeltrace_zipkin.py ${lttng-traces-dir}/${blkin-test}/ust/uid/0/64-bit/ -p ${zipkin-collector-port(9410 by default)} -s ${zipkin-collector-ip}
+
+Example::
+
+  python3 babeltrace_zipkin.py ~/lttng-traces-dir/blkin-test-20150225-160222/ust/uid/0/64-bit/ -p 9410 -s 127.0.0.1
+
+Check Ceph traces on webpage::
+
+  Browse http://${zipkin-collector-ip}:9411
+  Click "Find traces"
diff --git a/doc/dev/bluestore.rst b/doc/dev/bluestore.rst
new file mode 100644
index 000000000..91d71d037
--- /dev/null
+++ b/doc/dev/bluestore.rst
@@ -0,0 +1,85 @@
+===================
+BlueStore Internals
+===================
+
+
+Small write strategies
+----------------------
+
+* *U*: Uncompressed write of a complete, new blob.
+
+  - write to new blob
+  - kv commit
+
+* *P*: Uncompressed partial write to unused region of an existing
+  blob.
+
+  - write to unused chunk(s) of existing blob
+  - kv commit
+
+* *W*: WAL overwrite: commit intent to overwrite, then overwrite
+  async.  Must be chunk_size = MAX(block_size, csum_block_size)
+  aligned.
+
+  - kv commit
+  - wal overwrite (chunk-aligned) of existing blob
+
+* *N*: Uncompressed partial write to a new blob.  Initially sparsely
+  utilized.  Future writes will either be *P* or *W*.
+
+  - write into a new (sparse) blob
+  - kv commit
+
+* *R+W*: Read partial chunk, then to WAL overwrite.
+
+  - read (out to chunk boundaries)
+  - kv commit
+  - wal overwrite (chunk-aligned) of existing blob
+
+* *C*: Compress data, write to new blob.
+
+  - compress and write to new blob
+  - kv commit
+
+Possible future modes
+---------------------
+
+* *F*: Fragment lextent space by writing small piece of data into a
+  piecemeal blob (that collects random, noncontiguous bits of data we
+  need to write).
+
+  - write to a piecemeal blob (min_alloc_size or larger, but we use just one block of it)
+  - kv commit
+
+* *X*: WAL read/modify/write on a single block (like legacy
+  bluestore).  No checksum.
+
+  - kv commit
+  - wal read/modify/write
+
+Mapping
+-------
+
+This very roughly maps the type of write onto what we do when we
+encounter a given blob.  In practice it's a bit more complicated since there
+might be several blobs to consider (e.g., we might be able to *W* into one or
+*P* into another), but it should communicate a rough idea of strategy.
+
++--------------------------+--------+--------------+-------------+--------------+---------------+
+|                          | raw    | raw (cached) | csum (4 KB) | csum (16 KB) | comp (128 KB) |
++--------------------------+--------+--------------+-------------+--------------+---------------+
+| 128+ KB (over)write      | U      | U            | U           | U            | C             |
++--------------------------+--------+--------------+-------------+--------------+---------------+
+| 64 KB (over)write        | U      | U            | U           | U            | U or C        |
++--------------------------+--------+--------------+-------------+--------------+---------------+
+| 4 KB overwrite           | W      | P | W        | P | W       | P | R+W      | P | N (F?)    |
++--------------------------+--------+--------------+-------------+--------------+---------------+
+| 100 byte overwrite       | R+W    | P | W        | P | R+W     | P | R+W      | P | N (F?)    |
++--------------------------+--------+--------------+-------------+--------------+---------------+
+| 100 byte append          | R+W    | P | W        | P | R+W     | P | R+W      | P | N (F?)    |
++--------------------------+--------+--------------+-------------+--------------+---------------+
++--------------------------+--------+--------------+-------------+--------------+---------------+
+| 4 KB clone overwrite     | P | N  | P | N        | P | N       | P | N        | N (F?)        |
++--------------------------+--------+--------------+-------------+--------------+---------------+
+| 100 byte clone overwrite | P | N  | P | N        | P | N       | P | N        | N (F?)        |
++--------------------------+--------+--------------+-------------+--------------+---------------+
diff --git a/doc/dev/cache-pool.rst b/doc/dev/cache-pool.rst
new file mode 100644
index 000000000..7dc71c828
--- /dev/null
+++ b/doc/dev/cache-pool.rst
@@ -0,0 +1,200 @@
+Cache pool
+==========
+
+Purpose
+-------
+
+Use a pool of fast storage devices (probably SSDs) and use it as a
+cache for an existing slower and larger pool.
+
+Use a replicated pool as a front-end to service most I/O, and destage
+cold data to a separate erasure coded pool that does not currently (and
+cannot efficiently) handle the workload.
+
+We should be able to create and add a cache pool to an existing pool
+of data, and later remove it, without disrupting service or migrating
+data around.
+
+Use cases
+---------
+
+Read-write pool, writeback
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We have an existing data pool and put a fast cache pool "in front" of
+it.  Writes will go to the cache pool and immediately ack.  We flush
+them back to the data pool based on the defined policy.
+
+Read-only pool, weak consistency
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We have an existing data pool and add one or more read-only cache
+pools.  We copy data to the cache pool(s) on read.  Writes are
+forwarded to the original data pool.  Stale data is expired from the
+cache pools based on the defined policy.
+
+This is likely only useful for specific applications with specific
+data access patterns.  It may be a match for rgw, for example.
+
+
+Interface
+---------
+
+Set up a read/write cache pool foo-hot for pool foo::
+
+ ceph osd tier add foo foo-hot
+ ceph osd tier cache-mode foo-hot writeback
+
+Direct all traffic for foo to foo-hot::
+
+ ceph osd tier set-overlay foo foo-hot
+
+Set the target size and enable the tiering agent for foo-hot::
+
+ ceph osd pool set foo-hot hit_set_type bloom
+ ceph osd pool set foo-hot hit_set_count 1
+ ceph osd pool set foo-hot hit_set_period 3600   # 1 hour
+ ceph osd pool set foo-hot target_max_bytes 1000000000000  # 1 TB
+ ceph osd pool set foo-hot min_read_recency_for_promote 1
+ ceph osd pool set foo-hot min_write_recency_for_promote 1
+
+Drain the cache in preparation for turning it off::
+
+ ceph osd tier cache-mode foo-hot forward
+ rados -p foo-hot cache-flush-evict-all
+
+When cache pool is finally empty, disable it::
+
+ ceph osd tier remove-overlay foo
+ ceph osd tier remove foo foo-hot
+
+Read-only pools with lazy consistency::
+
+ ceph osd tier add foo foo-east
+ ceph osd tier cache-mode foo-east readonly
+ ceph osd tier add foo foo-west
+ ceph osd tier cache-mode foo-west readonly
+
+
+
+Tiering agent
+-------------
+
+The tiering policy is defined as properties on the cache pool itself.
+
+HitSet metadata
+~~~~~~~~~~~~~~~
+
+First, the agent requires HitSet information to be tracked on the
+cache pool in order to determine which objects in the pool are being
+accessed.  This is enabled with::
+
+ ceph osd pool set foo-hot hit_set_type bloom
+ ceph osd pool set foo-hot hit_set_count 1
+ ceph osd pool set foo-hot hit_set_period 3600   # 1 hour
+
+The supported HitSet types include 'bloom' (a bloom filter, the
+default), 'explicit_hash', and 'explicit_object'.  The latter two
+explicitly enumerate accessed objects and are less memory efficient.
+They are there primarily for debugging and to demonstrate pluggability
+for the infrastructure.  For the bloom filter type, you can additionally
+define the false positive probability for the bloom filter (default is 0.05)::
+
+ ceph osd pool set foo-hot hit_set_fpp 0.15
+
+The hit_set_count and hit_set_period define how much time each HitSet
+should cover, and how many such HitSets to store.  Binning accesses
+over time allows Ceph to independently determine whether an object was
+accessed at least once and whether it was accessed more than once over
+some time period ("age" vs "temperature").
+
+The ``min_read_recency_for_promote`` defines how many HitSets to check for the
+existence of an object when handling a read operation. The checking result is
+used to decide whether to promote the object asynchronously. Its value should be
+between 0 and ``hit_set_count``. If it's set to 0, the object is always promoted.
+If it's set to 1, the current HitSet is checked. And if this object is in the
+current HitSet, it's promoted. Otherwise not. For the other values, the exact
+number of archive HitSets are checked. The object is promoted if the object is
+found in any of the most recent ``min_read_recency_for_promote`` HitSets.
+
+A similar parameter can be set for the write operation, which is
+``min_write_recency_for_promote``. ::
+
+ ceph osd pool set {cachepool} min_read_recency_for_promote 1
+ ceph osd pool set {cachepool} min_write_recency_for_promote 1
+
+Note that the longer the ``hit_set_period`` and the higher the
+``min_read_recency_for_promote``/``min_write_recency_for_promote`` the more RAM
+will be consumed by the ceph-osd process. In particular, when the agent is active
+to flush or evict cache objects, all hit_set_count HitSets are loaded into RAM.
+
+Cache mode
+~~~~~~~~~~
+
+The most important policy is the cache mode:
+
+ ceph osd pool set foo-hot cache-mode writeback
+
+The supported modes are 'none', 'writeback', 'forward', and
+'readonly'.  Most installations want 'writeback', which will write
+into the cache tier and only later flush updates back to the base
+tier.  Similarly, any object that is read will be promoted into the
+cache tier.
+
+The 'forward' mode is intended for when the cache is being disabled
+and needs to be drained.  No new objects will be promoted or written
+to the cache pool unless they are already present.  A background
+operation can then do something like::
+
+  rados -p foo-hot cache-try-flush-evict-all
+  rados -p foo-hot cache-flush-evict-all
+
+to force all data to be flushed back to the base tier.
+
+The 'readonly' mode is intended for read-only workloads that do not
+require consistency to be enforced by the storage system.  Writes will
+be forwarded to the base tier, but objects that are read will get
+promoted to the cache.  No attempt is made by Ceph to ensure that the
+contents of the cache tier(s) are consistent in the presence of object
+updates.
+
+Cache sizing
+~~~~~~~~~~~~
+
+The agent performs two basic functions: flushing (writing 'dirty'
+cache objects back to the base tier) and evicting (removing cold and
+clean objects from the cache).
+
+The thresholds at which Ceph will flush or evict objects is specified
+relative to a 'target size' of the pool.  For example::
+
+ ceph osd pool set foo-hot cache_target_dirty_ratio .4
+ ceph osd pool set foo-hot cache_target_dirty_high_ratio .6
+ ceph osd pool set foo-hot cache_target_full_ratio .8
+
+will begin flushing dirty objects when 40% of the pool is dirty and begin
+evicting clean objects when we reach 80% of the target size.
+
+The target size can be specified either in terms of objects or bytes::
+
+ ceph osd pool set foo-hot target_max_bytes 1000000000000  # 1 TB
+ ceph osd pool set foo-hot target_max_objects 1000000       # 1 million objects
+
+Note that if both limits are specified, Ceph will begin flushing or
+evicting when either threshold is triggered.
+
+Other tunables
+~~~~~~~~~~~~~~
+
+You can specify a minimum object age before a recently updated object is
+flushed to the base tier::
+
+ ceph osd pool set foo-hot cache_min_flush_age 600   # 10 minutes
+
+You can specify the minimum age of an object before it will be evicted from
+the cache tier::
+
+ ceph osd pool set foo-hot cache_min_evict_age 1800   # 30 minutes
+
+
+
diff --git a/doc/dev/ceph-volume/index.rst b/doc/dev/ceph-volume/index.rst
new file mode 100644
index 000000000..5feef8089
--- /dev/null
+++ b/doc/dev/ceph-volume/index.rst
@@ -0,0 +1,14 @@
+===================================
+ceph-volume developer documentation
+===================================
+
+.. rubric:: Contents
+
+.. toctree::
+   :maxdepth: 1
+
+
+   plugins
+   lvm
+   zfs
+   systemd
diff --git a/doc/dev/ceph-volume/lvm.rst b/doc/dev/ceph-volume/lvm.rst
new file mode 100644
index 000000000..f2df6d850
--- /dev/null
+++ b/doc/dev/ceph-volume/lvm.rst
@@ -0,0 +1,179 @@
+
+.. _ceph-volume-lvm-api:
+
+LVM
+===
+The backend of ``ceph-volume lvm`` is LVM, it relies heavily on the usage of
+tags, which is a way for LVM to allow extending its volume metadata. These
+values can later be queried against devices and it is how they get discovered
+later.
+
+.. warning:: These APIs are not meant to be public, but are documented so that
+             it is clear what the tool is doing behind the scenes. Do not alter
+             any of these values.
+
+
+.. _ceph-volume-lvm-tag-api:
+
+Tag API
+-------
+The process of identifying logical volumes as part of Ceph relies on applying
+tags on all volumes. It follows a naming convention for the namespace that
+looks like::
+
+    ceph.<tag name>=<tag value>
+
+All tags are prefixed by the ``ceph`` keyword to claim ownership of that
+namespace and make it easily identifiable. This is how the OSD ID would be used
+in the context of lvm tags::
+
+    ceph.osd_id=0
+
+
+.. _ceph-volume-lvm-tags:
+
+Metadata
+--------
+The following describes all the metadata from Ceph OSDs that is stored on an
+LVM volume:
+
+
+``type``
+--------
+Describes if the device is an OSD or Journal, with the ability to expand to
+other types when supported (for example a lockbox)
+
+Example::
+
+    ceph.type=osd
+
+
+``cluster_fsid``
+----------------
+Example::
+
+    ceph.cluster_fsid=7146B649-AE00-4157-9F5D-1DBFF1D52C26
+
+
+``data_device``
+---------------
+Example::
+
+    ceph.data_device=/dev/ceph/data-0
+
+
+``data_uuid``
+-------------
+Example::
+
+    ceph.data_uuid=B76418EB-0024-401C-8955-AE6919D45CC3
+
+
+``journal_device``
+------------------
+Example::
+
+    ceph.journal_device=/dev/ceph/journal-0
+
+
+``journal_uuid``
+----------------
+Example::
+
+    ceph.journal_uuid=2070E121-C544-4F40-9571-0B7F35C6CB2B
+
+
+``encrypted``
+-------------
+Example for enabled encryption with ``luks``::
+
+    ceph.encrypted=1
+
+When encryption is not supported or simply disabled::
+
+    ceph.encrypted=0
+
+
+``osd_fsid``
+------------
+Example::
+
+    ceph.osd_fsid=88ab9018-f84b-4d62-90b4-ce7c076728ff
+
+
+``osd_id``
+----------
+Example::
+
+    ceph.osd_id=1
+
+
+``block_device``
+----------------
+Just used on :term:`bluestore` backends. Captures the path to the logical
+volume path.
+
+Example::
+
+    ceph.block_device=/dev/mapper/vg-block-0
+
+
+``block_uuid``
+--------------
+Just used on :term:`bluestore` backends. Captures either the logical volume UUID or
+the partition UUID.
+
+Example::
+
+    ceph.block_uuid=E5F041BB-AAD4-48A8-B3BF-31F7AFD7D73E
+
+
+``db_device``
+-------------
+Just used on :term:`bluestore` backends. Captures the path to the logical
+volume path.
+
+Example::
+
+    ceph.db_device=/dev/mapper/vg-db-0
+
+
+``db_uuid``
+-----------
+Just used on :term:`bluestore` backends. Captures either the logical volume UUID or
+the partition UUID.
+
+Example::
+
+    ceph.db_uuid=F9D02CF1-31AB-4910-90A3-6A6302375525
+
+
+``wal_device``
+--------------
+Just used on :term:`bluestore` backends. Captures the path to the logical
+volume path.
+
+Example::
+
+    ceph.wal_device=/dev/mapper/vg-wal-0
+
+
+``wal_uuid``
+------------
+Just used on :term:`bluestore` backends. Captures either the logical volume UUID or
+the partition UUID.
+
+Example::
+
+    ceph.wal_uuid=A58D1C68-0D6E-4CB3-8E99-B261AD47CC39
+
+
+``vdo``
+-------
+A VDO-enabled device is detected when device is getting prepared, and then
+stored for later checks when activating. This affects mount options by
+appending the ``discard`` mount flag, regardless of mount flags being used.
+
+Example for an enabled VDO device::
+
+    ceph.vdo=1
diff --git a/doc/dev/ceph-volume/plugins.rst b/doc/dev/ceph-volume/plugins.rst
new file mode 100644
index 000000000..95bc761e2
--- /dev/null
+++ b/doc/dev/ceph-volume/plugins.rst
@@ -0,0 +1,65 @@
+.. _ceph-volume-plugins:
+
+Plugins
+=======
+``ceph-volume`` started initially to provide support for using ``lvm`` as
+the underlying system for an OSD. It is included as part of the tool but it is
+treated like a plugin.
+
+This modularity, allows for other device or device-like technologies to be able
+to consume and re-use the utilities and workflows provided.
+
+Adding Plugins
+--------------
+As a Python tool, plugins ``setuptools`` entry points. For a new plugin to be
+available, it should have an entry similar to this in its ``setup.py`` file:
+
+.. code-block:: python
+
+    setup(
+        ...
+        entry_points = dict(
+            ceph_volume_handlers = [
+                'my_command = my_package.my_module:MyClass',
+            ],
+        ),
+
+The ``MyClass`` should be a class that accepts ``sys.argv`` as its argument,
+``ceph-volume`` will pass that in at instantiation and call them ``main``
+method.
+
+This is how a plugin for ``ZFS`` could look like for example:
+
+.. code-block:: python
+
+    class ZFS(object):
+
+        help_menu = 'Deploy OSDs with ZFS'
+        _help = """
+    Use ZFS as the underlying technology for OSDs
+
+    --verbose   Increase the verbosity level
+        """
+
+        def __init__(self, argv):
+            self.argv = argv
+
+        def main(self):
+            parser = argparse.ArgumentParser()
+            args = parser.parse_args(self.argv)
+            ...
+
+And its entry point (via ``setuptools``) in ``setup.py`` would looke like:
+
+.. code-block:: python
+
+        entry_points = {
+            'ceph_volume_handlers': [
+                'zfs = ceph_volume_zfs.zfs:ZFS',
+            ],
+        },
+
+After installation, the ``zfs`` subcommand would be listed and could be used
+as::
+
+    ceph-volume zfs
diff --git a/doc/dev/ceph-volume/systemd.rst b/doc/dev/ceph-volume/systemd.rst
new file mode 100644
index 000000000..8553430ee
--- /dev/null
+++ b/doc/dev/ceph-volume/systemd.rst
@@ -0,0 +1,37 @@
+.. _ceph-volume-systemd-api:
+
+systemd
+=======
+The workflow to *"activate"* an OSD is by relying on systemd unit files and its
+ability to persist information as a suffix to the instance name.
+
+``ceph-volume`` exposes the following convention for unit files::
+
+    ceph-volume@<sub command>-<extra metadata>
+
+For example, this is how enabling an OSD could look like for the 
+:ref:`ceph-volume-lvm` sub command::
+
+    systemctl enable ceph-volume@lvm-0-8715BEB4-15C5-49DE-BA6F-401086EC7B41
+
+
+These 3 pieces of persisted information are needed by the sub-command so that
+it understands what OSD it needs to activate.
+
+Since ``lvm`` is not the only subcommand that will be supported, this
+is how it will allow other device types to be defined.
+
+At some point for example, for plain disks, it could be::
+
+    systemctl enable ceph-volume@disk-0-8715BEB4-15C5-49DE-BA6F-401086EC7B41
+
+At startup, the systemd unit will execute a helper script that will parse the
+suffix and will end up calling ``ceph-volume`` back. Using the previous
+example for lvm, that call will look like::
+
+    ceph-volume lvm activate 0 8715BEB4-15C5-49DE-BA6F-401086EC7B41
+
+
+.. warning:: These workflows are not meant to be public, but are documented so that
+             it is clear what the tool is doing behind the scenes. Do not alter
+             any of these values.
diff --git a/doc/dev/ceph-volume/zfs.rst b/doc/dev/ceph-volume/zfs.rst
new file mode 100644
index 000000000..18de7652a
--- /dev/null
+++ b/doc/dev/ceph-volume/zfs.rst
@@ -0,0 +1,176 @@
+
+.. _ceph-volume-zfs-api:
+
+ZFS
+===
+The backend of ``ceph-volume zfs`` is ZFS, it relies heavily on the usage of
+tags, which is a way for ZFS to allow extending its volume metadata. These
+values can later be queried against devices and it is how they get discovered
+later.
+
+Currently this interface is only usable when running on FreeBSD.
+
+.. warning:: These APIs are not meant to be public, but are documented so that
+             it is clear what the tool is doing behind the scenes. Do not alter
+             any of these values.
+
+
+.. _ceph-volume-zfs-tag-api:
+
+Tag API
+-------
+The process of identifying filesystems, volumes and pools as part of Ceph relies
+on applying tags on all volumes. It follows a naming convention for the
+namespace that looks like::
+
+    ceph.<tag name>=<tag value>
+
+All tags are prefixed by the ``ceph`` keyword to claim ownership of that
+namespace and make it easily identifiable. This is how the OSD ID would be used
+in the context of zfs tags::
+
+    ceph.osd_id=0
+
+Tags on filesystems are stored as property.
+Tags on a zpool are stored in the comment property as a concatenated list 
+separated by ``;`` 
+
+.. _ceph-volume-zfs-tags:
+
+Metadata
+--------
+The following describes all the metadata from Ceph OSDs that is stored on a
+ZFS filesystem, volume, pool:
+
+
+``type``
+--------
+Describes if the device is an OSD or Journal, with the ability to expand to
+other types when supported 
+
+Example::
+
+    ceph.type=osd
+
+
+``cluster_fsid``
+----------------
+Example::
+
+    ceph.cluster_fsid=7146B649-AE00-4157-9F5D-1DBFF1D52C26
+
+
+``data_device``
+---------------
+Example::
+
+    ceph.data_device=/dev/ceph/data-0
+
+
+``data_uuid``
+-------------
+Example::
+
+    ceph.data_uuid=B76418EB-0024-401C-8955-AE6919D45CC3
+
+
+``journal_device``
+------------------
+Example::
+
+    ceph.journal_device=/dev/ceph/journal-0
+
+
+``journal_uuid``
+----------------
+Example::
+
+    ceph.journal_uuid=2070E121-C544-4F40-9571-0B7F35C6CB2B
+
+
+``osd_fsid``
+------------
+Example::
+
+    ceph.osd_fsid=88ab9018-f84b-4d62-90b4-ce7c076728ff
+
+
+``osd_id``
+----------
+Example::
+
+    ceph.osd_id=1
+
+
+``block_device``
+----------------
+Just used on :term:`bluestore` backends. Captures the path to the logical
+volume path.
+
+Example::
+
+    ceph.block_device=/dev/gpt/block-0
+
+
+``block_uuid``
+--------------
+Just used on :term:`bluestore` backends. Captures either the logical volume UUID or
+the partition UUID.
+
+Example::
+
+    ceph.block_uuid=E5F041BB-AAD4-48A8-B3BF-31F7AFD7D73E
+
+
+``db_device``
+-------------
+Just used on :term:`bluestore` backends. Captures the path to the logical
+volume path.
+
+Example::
+
+    ceph.db_device=/dev/gpt/db-0
+
+
+``db_uuid``
+-----------
+Just used on :term:`bluestore` backends. Captures either the logical volume UUID or
+the partition UUID.
+
+Example::
+
+    ceph.db_uuid=F9D02CF1-31AB-4910-90A3-6A6302375525
+
+
+``wal_device``
+--------------
+Just used on :term:`bluestore` backends. Captures the path to the logical
+volume path.
+
+Example::
+
+    ceph.wal_device=/dev/gpt/wal-0
+
+
+``wal_uuid``
+------------
+Just used on :term:`bluestore` backends. Captures either the logical volume UUID or
+the partition UUID.
+
+Example::
+
+    ceph.wal_uuid=A58D1C68-0D6E-4CB3-8E99-B261AD47CC39
+
+
+``compression``
+---------------
+A compression-enabled device can always be set using the native zfs settings on
+a volume or filesystem. This will/can be activated during creation of the volume
+of filesystem. 
+When activated by ``ceph-volume zfs`` this tag will be created.
+Compression manually set AFTER ``ceph-volume`` will go unnoticed, unless this 
+tag is also manually set.
+
+Example for an enabled compression device::
+
+    ceph.vdo=1
diff --git a/doc/dev/ceph_krb_auth.rst b/doc/dev/ceph_krb_auth.rst
new file mode 100644
index 000000000..627b9bd8e
--- /dev/null
+++ b/doc/dev/ceph_krb_auth.rst
@@ -0,0 +1,1094 @@
+===============================================================================
+A Detailed Documentation on How to Set up Ceph Kerberos Authentication
+===============================================================================
+
+This document provides details on the Kerberos authorization protocol. This is
+the 1st draft and we will try to keep it updated along with code changes that
+might take place.
+
+Several free implementations of this protocol are available (MIT, Heimdal,
+MS...), covering a wide range of operating systems. The Massachusetts
+Institute of Technology (MIT), where Kerberos was originally developed,
+continues to develop their Kerberos package and it is the implementation we
+chose to work with. `MIT Kerberos <http://web.mit.edu/Kerberos/>`_.
+
+Please, provide feedback to Daniel Oliveira (doliveira@suse.com)
+
+*Last update: Dec 3, 2018*
+
+|
+
+Background
+----------
+
+Before we get into *Kerberos details*, let us define a few terms so we can
+understand what to expect from it, *what it can and can't do*:
+
+Directory Services
+    A directory service is a customizable information store that functions as
+    a single point from which users can locate resources and services
+    distributed throughout the network. This customizable information store
+    also gives administrators a single point for managing its objects and their
+    attributes. Although this information store appears as a single point to
+    the users of the network, it is actually most often stored in a distributed
+    form. A directory service consists of at least one *Directory Server and a
+    Directory Client* and are implemented based on *X.500 standards*.
+
+    *OpenLDAP, 389 Directory Server, MS Active Directory, NetIQ eDirectory* are
+    some good examples.
+
+    A directory service is often characterized as a *write-once-read-many-times
+    service*, meaning the data that would normally be stored in an directory
+    service would not be expected to change on every access.
+
+    The database that forms a directory service *is not designed for
+    transactional data*.
+
+|
+
+LDAP (Lightweight Directory Access Protocol v3)
+    LDAP is a set of LDAP Protocol Exchanges *(not an implementation of a
+    server)* that defines the method by which data is accessed. LDAPv3 is a
+    standard defined by the IETF in RFC 2251 and describes how data is
+    represented in the Directory Service (the Data Model or DIT).
+
+    Finally, it defines how data is loaded into (imported) and saved from
+    (exported) a directory service (using LDIF). LDAP does not define how data
+    is stored or manipulated. Data Store is an 'automagic' process as far as
+    the standard is concerned and is generally handled by back-end modules.
+
+    No Directory Service implementation has all the features of LDAP v3
+    protocol implemented. All Directory Server implementations have their
+    different problems and/or anomalies, and features that may not return
+    results as another Directory Server implementation would.
+
+|
+
+Authentication
+    Authentication is about validating credentials (like User Name/ID and
+    password) to verify the identity. The system determines whether one is what
+    they say they are using their credentials.
+
+    Usually, authentication is done by a username and password, and sometimes
+    in conjunction with *(single, two, or multi) factors of authentication*,
+    which refers to the various ways to be authenticated.
+
+|
+
+Authorization
+    Authorization occurs after the identity is successfully authenticated by
+    the system, which ultimately gives one full permission to access the
+    resources such as information, files, databases, and so forth, almost
+    anything. It determines the ability to access the system and up to what
+    extent (what kind of permissions/rights are given and to where/what).
+
+|
+
+Auditing
+    Auditing takes the results from both *authentication and authorization* and
+    records them into an audit log. The audit log records records all actions
+    taking by/during the authentication and authorization for later review by
+    the administrators. While authentication and authorization are preventive
+    systems (in which unauthorized access is prevented), auditing is a reactive
+    system (in which it gives detailed log of how/when/where someone accessed
+    the environment).
+
+|
+
+Kerberos (KRB v5)
+    Kerberos is a network *authentication protocol*. It is designed to provide
+    strong authentication for client/server applications by using secret-key
+    cryptography (symmetric key). A free implementation of this protocol is
+    available from the MIT. However, Kerberos is available in many commercial
+    products as well.
+
+    It was designed to provide secure authentication to services over an
+    insecure network. Kerberos uses tickets to authenticate a user, or service
+    application and never transmits passwords over the network in the clear.
+    So both client and server can prove their identity without sending any
+    unencrypted secrets over the network.
+
+    Kerberos can be used for single sign-on (SSO). The idea behind SSO is
+    simple, we want to login just once and be able to use any service that we
+    are entitled to, without having to login on each of those services.
+
+|
+
+Simple Authentication and Security Layer (SASL)
+    SASL **(RFC 4422)** is a framework that helps developers to implement
+    different authentication mechanisms (implementing a series of challenges
+    and responses), allowing both clients and servers to negotiate a mutually
+    acceptable mechanism for each connection, instead of hard-coding them.
+
+    Examples of SASL mechanisms:
+
+        * ANONYMOUS **(RFC 4505)**
+
+            - For guest access, meaning *unauthenticated*
+
+        * CRAM-MD5 **(RFC 2195)**
+
+            - Simple challenge-response scheme based on *HMAC-MD5*.
+              It does not establish any security layer. *Less secure than
+              DIGEST-MD5 and GSSAPI.*
+
+        * DIGEST-MD5 **(RFC 2831)**
+
+            - HTTP Digest compatible *(partially)* challenge-response scheme
+              based upon MD5, offering a *data security layer*. It is preferred
+              over PLAIN text passwords, protecting against plain text attacks.
+              It is a mandatory authentication method for LDAPv3 servers.
+
+        * EXTERNAL **(RFCs 4422, 5246, 4301, 2119)**
+
+            - Where *authentication is implicit* in the context (i.e; for
+              protocols using IPsec or TLS [TLS/SSL to performing certificate-
+              based authentication] already). This method uses public keys for
+              strong authentication.
+
+        * GS2 **(RFC 5801)**
+
+            - Family of mechanisms supports arbitrary GSS-API mechanisms in
+              SASL
+
+        * NTLM (MS Proprietary)
+
+            - MS Windows NT LAN Manager authentication mechanism
+
+        * OAuth 1.0/2.0 **(RFCs 5849, 6749, 7628)**
+
+            - Authentication protocol for delegated resource access
+
+        * OTP **(RFC 2444)**
+
+            - One-time password mechanism *(obsoletes the SKEY mechanism)*
+
+        * PLAIN **(RFC 4616)**
+
+            - Simple Cleartext password mechanism **(RFC 4616)**. This is not a
+              preferred mechanism for most applications because of its relative
+              lack of strength.
+
+        * SCRAM **(RFCs 5802, 7677)**
+
+            - Modern challenge-response scheme based mechanism with channel
+              binding support
+
+|
+
+Generic Security Services Application Program Interface (GSSAPI)
+    GSSAPI **(RFCs 2078, 2743, 2744, 4121, 4752)** is widely used by protocol
+    implementers as a way to implement Kerberos v5 support in their
+    applications. It provides a generic interface and message format that can
+    encapsulate authentication exchanges from any authentication method that
+    has a GSSAPI-compliant library.
+
+    It does not define a protocol, authentication, or security mechanism
+    itself; it instead makes it easier for application programmers to support
+    multiple authentication mechanisms by providing a uniform, generic API for
+    security services. It is a set of functions that include both an API and a
+    methodology for approaching authentication, aiming to insulate application
+    protocols from the specifics of security protocols as much as possible.
+
+    *Microsoft Windows Kerberos* implementation does not include GSSAPI support
+    but instead includes a *Microsoft-specific API*, the *Security Support
+    Provider Interface (SSPI)*. In Windows, an SSPI client can communicate with
+    a *GSSAPI server*.
+
+    *Most applications that support GSSAPI also support Kerberos v5.*
+
+|
+
+Simple and Protected GSSAPI Negotiation Mechanism (SPNEGO)
+    As we can see, GSSAPI solves the problem of providing a single API to
+    different authentication mechanisms. However, it does not solve the problem
+    of negotiating which mechanism to use. In fact for GSSAPI to work, the two
+    applications communicating with each other must know in advance what
+    authentication mechanism they plan to use, which usually is not a problem
+    if only one mechanism is supported (meaning Kerberos v5).
+
+    However, if there are multiple mechanisms to choose from, a method is
+    needed to securely negotiate an authentication mechanism that is mutually
+    supported between both client and server; which is where
+    *SPNEGO (RFC 2478, 4178)* makes a difference.
+
+    *SPNEGO* provides a framework for two parties that are engaged in
+    authentication to select from a set of possible authentication mechanisms,
+    in a manner that preserves the opaque nature of the security protocols to
+    the application protocol that uses it.
+
+    It is a security protocol that uses a *GSSAPI authentication mechanism* and
+    negotiates among several available authentication mechanisms in an
+    implementation, selecting one for use to satisfy the authentication needs
+    of the application protocol.
+
+    It is a *meta protocol* that travels entirely in other application
+    protocols; it is never used directly without an application protocol.
+
+|
+
+*Why is this important and why do we care? Like, at all?*
+
+    Having this background information in mind, we can easily describe things
+    like:
+
+        1. *Ceph Kerberos authentication* is based totally on MIT *Kerberos*
+        implementation using *GSSAPI*.
+
+        2. At the moment we are still using *Kerberos default backend
+        database*, however we plan on adding LDAP as a backend which would
+        provide us with *authentication with GSSAPI (KRB5)* and *authorization
+        with LDAP (LDAPv3)*, via *SASL mechanism*.
+
+|
+
+Before We Start
+---------------
+
+We assume the environment already has some external services up and running
+properly:
+
+    * Kerberos needs to be properly configured, which also means (for both
+      every server and KDC):
+
+        - Time Synchronization (either using `NTP <http://www.ntp.org/>`_  or `chrony <https://chrony.tuxfamily.org/>`_).
+
+            + Not only Kerberos, but also Ceph depends and relies on time
+              synchronization.
+
+        - DNS resolution
+
+            + Both *(forward and reverse)* zones, with *fully qualified domain
+              name (fqdn)* ``(hostname + domain.name)``
+
+            + KDC discover can be set up to use DNS ``(srv resources)`` as
+              service location protocol *(RFCs 2052, 2782)*, as well as *host
+              or domain* to the *appropriate realm* ``(txt record)``.
+
+            + Even though these DNS entries/settings are not required to run a
+              ``Kerberos realm``, they certainly help to eliminate the need for
+              manual configuration on all clients.
+
+            + This is extremely important, once most of the Kerberos issues are
+              usually related to name resolution. Kerberos is very picky when
+              checking on systems names and host lookups.
+
+    * Whenever possible, in order to avoid a *single point of failure*, set up
+      a *backup, secondary, or slave*, for every piece/part in the
+      infrastructure ``(ntp, dns, and kdc servers)``.
+
+
+Also, the following *Kerberos terminology* is important:
+
+    * Ticket
+
+        - Tickets or Credentials, are a set of information that can be used to
+          verify the client's identity. Kerberos tickets may be stored in a
+          file, or they may exist only in memory.
+
+        - The first ticket obtained is a ticket-granting ticket (TGT), which
+          allows the clients to obtain additional tickets. These additional
+          tickets give the client permission for specific services. The
+          requesting and granting of these additional tickets happens
+          transparently.
+
+            + The TGT, which expires at a specified time, permits the client to
+              obtain additional tickets, which give permission for specific
+              services. The requesting and granting of these additional tickets
+              is user-transparent.
+
+    * Key Distribution Center (KDC).
+
+        - The KDC creates a ticket-granting ticket (TGT) for the client,
+          encrypts it using the client's password as the key, and sends the
+          encrypted TGT back to the client. The client then attempts to decrypt
+          the TGT, using its password. If the client successfully decrypts the
+          TGT (i.e., if the client gave the correct password), it keeps the
+          decrypted TGT, which indicates proof of the client's identity.
+
+        -  The KDC is comprised of three components:
+
+            + Kerberos database, which stores all the information about the
+              principals and the realm they belong to, among other things.
+            + Authentication service (AS)
+            + Ticket-granting service (TGS)
+
+    * Client
+
+        - Either a *user, host or a service* who sends a request for a ticket.
+
+    * Principal
+
+        - It is a unique identity to which Kerberos can assign tickets.
+          Principals can have an arbitrary number of components. Each component
+          is separated by a component separator, generally ``/``. The last
+          component is the *realm*, separated from the rest of the principal by
+          the realm separator, generally ``@``.
+
+        - If there is no realm component in the principal, then it will be
+          assumed that the principal is in the default realm for the context in
+          which it is being used.
+
+        - Usually, a principal is divided into three parts:
+
+            + The ``primary``, the ``instance``, and the ``realm``
+
+            + The format of a typical Kerberos V5 principal is
+              ``primary/instance@REALM``.
+
+            + The ``primary`` is the first part of the principal. In the case
+              of a user, it's the same as the ``username``. For a host, the
+              primary is the word ``host``. For Ceph, will use ``ceph`` as a
+              primary name which makes it easier to organize and identify Ceph
+              related principals.
+
+            + The ``instance`` is an optional string that qualifies the
+              primary. The instance is separated from the primary by a slash
+              ``/``. In the case of a user, the instance is usually ``null``,
+              but a user might also have an additional principal, with an
+              instance called ``admin``, which one uses to administrate a
+              database.
+
+              The principal ``johndoe@MYDOMAIN.COM`` is completely separate
+              from the principal ``johndoe/admin@MYDOMAIN.COM``, with a
+              separate password, and separate permissions. In the case of a
+              host, the instance is the fully qualified hostname,
+              i.e., ``osd1.MYDOMAIN.COM``.
+
+            + The ``realm`` is the Kerberos realm. Usually, the Kerberos realm
+              is the domain name, in *upper-case letters*. For example, the
+              machine ``osd1.MYDOMAIN.COM`` would be in the realm
+              ``MYDOMAIN.COM``.
+
+    * Keytab
+
+        - A keytab file stores the actual encryption key that can be used in
+          lieu of a password challenge for a given principal. Creating keytab
+          files are useful for noninteractive principals, such as *Service
+          Principal Names*, which are often associated with long-running
+          processes like Ceph daemons. A keytab file does not have to be a
+          "1:1 mapping" to a single principal. Multiple different principal
+          keys can be stored in a single keytab file:
+
+            + The keytab file allows a user/service to authenticate without
+              knowledge of the password. Due to this, *keytabs should be
+              protected* with appropriate controls to prevent unauthorized
+              users from authenticating with it.
+
+            + The default client keytab file is ``/etc/krb5.keytab``
+
+|
+
+The 'Ceph side' of the things
+------------------------------
+
+In order to configure connections (from Ceph nodes) to the KDC:
+
+1. Login to the Kerberos client (Ceph server nodes) and confirm it is properly
+   configured, by checking and editing ``/etc/krb5.conf`` file properly:  ::
+
+    /etc/krb5.conf
+    [libdefaults]
+        dns_canonicalize_hostname = false
+        rdns = false
+        forwardable = true
+        dns_lookup_realm = true
+        dns_lookup_kdc = true
+        allow_weak_crypto = false
+        default_realm = MYDOMAIN.COM
+        default_ccache_name = KEYRING:persistent:%{uid}
+    [realms]
+        MYDOMAIN.COM = {
+            kdc = kerberos.mydomain.com
+            admin_server = kerberos.mydomain.com
+            ...
+        }
+    ...
+
+
+2. Login to the *KDC Server* and confirm it is properly configured to
+   authenticate to the Kerberos realm in question:
+
+    a. Kerberos related DNS RRs:  ::
+
+        /var/lib/named/master/mydomain.com
+        kerberos                IN A        192.168.10.21
+        kerberos-slave          IN A        192.168.10.22
+        _kerberos               IN TXT      "MYDOMAIN.COM"
+        _kerberos._udp          IN SRV      1 0 88 kerberos
+        _kerberos._tcp          IN SRV      1 0 88 kerberos
+        _kerberos._udp          IN SRV      20 0 88 kerberos-slave
+        _kerberos-master._udp   IN SRV      0 0 88 kerberos
+        _kerberos-adm._tcp      IN SRV      0 0 749 kerberos
+        _kpasswd._udp           IN SRV      0 0 464 kerberos
+        ...
+
+
+    b. KDC configuration file:  ::
+
+        /var/lib/kerberos/krb5kdc/kdc.conf
+        [kdcdefaults]
+                kdc_ports = 750,88
+        [realms]
+                MYDOMAIN.COM = {
+                    acl_file = /var/lib/kerberos/krb5kdc/kadm5.acl
+                    admin_keytab = FILE:/var/lib/kerberos/krb5kdc/kadm5.keytab
+                    default_principal_flags = +postdateable +forwardable +renewable +proxiable
+                                                            +dup-skey -preauth -hwauth +service
+                                                            +tgt-based +allow-tickets -pwchange
+                                                            -pwservice
+                    dict_file = /var/lib/kerberos/krb5kdc/kadm5.dict
+                    key_stash_file = /var/lib/kerberos/krb5kdc/.k5.MYDOMAIN.COM
+                    kdc_ports = 750,88
+                    max_life = 0d 10h 0m 0s
+                    max_renewable_life = 7d 0h 0m 0s
+                }
+        ...
+
+
+3. Still on the KDC Server, run the Kerberos administration utility;
+   ``kadmin.local`` so we can list all the principals already created.  ::
+
+    kadmin.local:  listprincs
+    K/M@MYDOMAIN.COM
+    krbtgt/MYDOMAIN.COM@MYDOMAIN.COM
+    kadmin/admin@MYDOMAIN.COM
+    kadmin/changepw@MYDOMAIN.COM
+    kadmin/history@MYDOMAIN.COM
+    kadmin/kerberos.mydomain.com@MYDOMAIN.COM
+    root/admin@MYDOMAIN.COM
+    ...
+
+
+4. Add a *principal for each Ceph cluster node* we want to be authenticated by
+   Kerberos:
+
+    a. Adding principals:  ::
+
+        kadmin.local:  addprinc -randkey ceph/ceph-mon1
+        Principal "ceph/ceph-mon1@MYDOMAIN.COM" created.
+        kadmin.local:  addprinc -randkey ceph/ceph-osd1
+        Principal "ceph/ceph-osd1@MYDOMAIN.COM" created.
+        kadmin.local:  addprinc -randkey ceph/ceph-osd2
+        Principal "ceph/ceph-osd2@MYDOMAIN.COM" created.
+        kadmin.local:  addprinc -randkey ceph/ceph-osd3
+        Principal "ceph/ceph-osd3@MYDOMAIN.COM" created.
+        kadmin.local:  addprinc -randkey ceph/ceph-osd4
+        Principal "ceph/ceph-osd4@MYDOMAIN.COM" created.
+        kadmin.local:  listprincs
+        K/M@MYDOMAIN.COM
+        krbtgt/MYDOMAIN.COM@MYDOMAIN.COM
+        kadmin/admin@MYDOMAIN.COM
+        kadmin/changepw@MYDOMAIN.COM
+        kadmin/history@MYDOMAIN.COM
+        kadmin/kerberos.mydomain.com@MYDOMAIN.COM
+        root/admin@MYDOMAIN.COM
+        ceph/ceph-mon1@MYDOMAIN.COM
+        ceph/ceph-osd1@MYDOMAIN.COM
+        ceph/ceph-osd2@MYDOMAIN.COM
+        ceph/ceph-osd3@MYDOMAIN.COM
+        ceph/ceph-osd4@MYDOMAIN.COM
+        ...
+
+
+    b. This follows the same idea if we are creating a *user principal*  ::
+
+        kadmin.local:  addprinc johndoe
+        WARNING: no policy specified for johndoe@MYDOMAIN.COM; defaulting to no policy
+        Enter password for principal "johndoe@MYDOMAIN.COM":
+        Re-enter password for principal "johndoe@MYDOMAIN.COM":
+        Principal "johndoe@MYDOMAIN.COM" created.
+        ...
+
+
+5. Create a *keytab file* for each Ceph cluster node:
+
+    As the default client keytab file is ``/etc/krb5.keytab``, we will want to
+    use a different file name, so we especify which *keytab file to create* and
+    which *principal to export keys* from:  ::
+
+        kadmin.local:  ktadd -k /etc/gss_client_mon1.ktab ceph/ceph-mon1
+        Entry for principal ceph/ceph-mon1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/etc/gss_client_mon1.ktab.
+        Entry for principal ceph/ceph-mon1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/etc/gss_client_mon1.ktab.
+        Entry for principal ceph/ceph-mon1 with kvno 2, encryption type des3-cbc-sha1 added to keytab WRFILE:/etc/gss_client_mon1.ktab.
+        Entry for principal ceph/ceph-mon1 with kvno 2, encryption type arcfour-hmac added to keytab WRFILE:/etc/gss_client_mon1.ktab.
+        kadmin.local:  ktadd -k /etc/gss_client_osd1.ktab ceph/ceph-osd1
+        Entry for principal ceph/ceph-osd1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/etc/gss_client_osd1.ktab.
+        Entry for principal ceph/ceph-osd1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/etc/gss_client_osd1.ktab.
+        Entry for principal ceph/ceph-osd1 with kvno 2, encryption type des3-cbc-sha1 added to keytab WRFILE:/etc/gss_client_osd1.ktab.
+        Entry for principal ceph/ceph-osd1 with kvno 2, encryption type arcfour-hmac added to keytab WRFILE:/etc/gss_client_osd1.ktab.
+        kadmin.local:  ktadd -k /etc/gss_client_osd2.ktab ceph/ceph-osd2
+        Entry for principal ceph/ceph-osd2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/etc/gss_client_osd2.ktab.
+        Entry for principal ceph/ceph-osd2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/etc/gss_client_osd2.ktab.
+        Entry for principal ceph/ceph-osd2 with kvno 2, encryption type des3-cbc-sha1 added to keytab WRFILE:/etc/gss_client_osd2.ktab.
+        Entry for principal ceph/ceph-osd2 with kvno 2, encryption type arcfour-hmac added to keytab WRFILE:/etc/gss_client_osd2.ktab.
+        kadmin.local:  ktadd -k /etc/gss_client_osd3.ktab ceph/ceph-osd3
+        Entry for principal ceph/ceph-osd3 with kvno 3, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/etc/gss_client_osd3.ktab.
+        Entry for principal ceph/ceph-osd3 with kvno 3, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/etc/gss_client_osd3.ktab.
+        Entry for principal ceph/ceph-osd3 with kvno 3, encryption type des3-cbc-sha1 added to keytab WRFILE:/etc/gss_client_osd3.ktab.
+        Entry for principal ceph/ceph-osd3 with kvno 3, encryption type arcfour-hmac added to keytab WRFILE:/etc/gss_client_osd3.ktab.
+        kadmin.local:  ktadd -k /etc/gss_client_osd4.ktab ceph/ceph-osd4
+        Entry for principal ceph/ceph-osd4 with kvno 4, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/etc/gss_client_osd4.ktab.
+        Entry for principal ceph/ceph-osd4 with kvno 4, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/etc/gss_client_osd4.ktab.
+        Entry for principal ceph/ceph-osd4 with kvno 4, encryption type des3-cbc-sha1 added to keytab WRFILE:/etc/gss_client_osd4.ktab.
+        Entry for principal ceph/ceph-osd4 with kvno 4, encryption type arcfour-hmac added to keytab WRFILE:/etc/gss_client_osd4.ktab.
+
+        # ls -1 /etc/gss_client_*
+        /etc/gss_client_mon1.ktab
+        /etc/gss_client_osd1.ktab
+        /etc/gss_client_osd2.ktab
+        /etc/gss_client_osd3.ktab
+        /etc/gss_client_osd4.ktab
+
+
+    We can also check these newly created keytab client files by:  ::
+
+        # klist -kte /etc/gss_client_mon1.ktab
+        Keytab name: FILE:/etc/gss_client_mon1.ktab
+        KVNO Timestamp           Principal
+        ---- ------------------- ------------------------------------------------------
+           2 10/8/2018 14:35:30 ceph/ceph-mon1@MYDOMAIN.COM (aes256-cts-hmac-sha1-96)
+           2 10/8/2018 14:35:31 ceph/ceph-mon1@MYDOMAIN.COM (aes128-cts-hmac-sha1-96)
+           2 10/8/2018 14:35:31 ceph/ceph-mon1@MYDOMAIN.COM (des3-cbc-sha1)
+           2 10/8/2018 14:35:31 ceph/ceph-mon1@MYDOMAIN.COM (arcfour-hmac)
+        ...
+
+
+6. A new *set parameter* was added in Ceph, ``gss ktab client file`` which
+   points to the keytab file related to the Ceph node *(or principal)* in
+   question.
+
+    By default it points to ``/var/lib/ceph/$name/gss_client_$name.ktab``. So,
+    in the case of a Ceph server ``osd1.mydomain.com``, the location and name
+    of the keytab file should be: ``/var/lib/ceph/osd1/gss_client_osd1.ktab``
+
+    Therefore, we need to ``scp`` each of these newly created keytab files from
+    the KDC to their respective Ceph cluster nodes (i.e):
+    ``# for node in mon1 osd1 osd2 osd3 osd4; do scp /etc/gss_client_$node*.ktab root@ceph-$node:/var/lib/ceph/$node/; done``
+
+    Or whatever other way one feels comfortable with, as long as each keytab
+    client file gets copied over to the proper location.
+
+    At this point, even *without using any keytab client file* we should be
+    already able to authenticate a *user principal*:  ::
+
+        # kdestroy -A && kinit -f johndoe && klist -f
+        Password for johndoe@MYDOMAIN.COM:
+        Ticket cache: KEYRING:persistent:0:0
+        Default principal: johndoe@MYDOMAIN.COM
+
+        Valid starting       Expires              Service principal
+        10/10/2018 15:32:01  10/11/2018 07:32:01  krbtgt/MYDOMAIN.COM@MYDOMAIN.COM
+            renew until 10/11/2018 15:32:01, Flags: FRI
+        ...
+
+
+    Given that the *keytab client file* is/should already be copied and available at the
+    Kerberos client (Ceph cluster node), we should be able to athenticate using it before
+    going forward:  ::
+
+        # kdestroy -A && kinit -k -t /etc/gss_client_mon1.ktab -f 'ceph/ceph-mon1@MYDOMAIN.COM' && klist -f
+        Ticket cache: KEYRING:persistent:0:0
+        Default principal: ceph/ceph-mon1@MYDOMAIN.COM
+
+        Valid starting       Expires              Service principal
+        10/10/2018 15:54:25  10/11/2018 07:54:25  krbtgt/MYDOMAIN.COM@MYDOMAIN.COM
+            renew until 10/11/2018 15:54:25, Flags: FRI
+        ...
+
+
+7. The default client keytab is used, if it is present and readable, to
+   automatically obtain initial credentials for GSSAPI client applications. The
+   principal name of the first entry in the client keytab is used by default
+   when obtaining initial credentials:
+
+    a. The ``KRB5_CLIENT_KTNAME environment`` variable.
+    b. The ``default_client_keytab_name`` profile variable in ``[libdefaults]``.
+    c. The hardcoded default, ``DEFCKTNAME``.
+
+    So, what we do is to internally, set the environment variable
+    ``KRB5_CLIENT_KTNAME`` to the same location as ``gss_ktab_client_file``,
+    so ``/var/lib/ceph/osd1/gss_client_osd1.ktab``, and change the ``ceph.conf``
+    file to add the new authentication method.  ::
+
+        /etc/ceph/ceph.conf
+        [global]
+            ...
+            auth cluster required = gss
+            auth service required = gss
+            auth client required = gss
+            gss ktab client file = /{$my_new_location}/{$my_new_ktab_client_file.keytab}
+            ...
+
+
+8. With that the GSSAPIs will then be able to read the keytab file and using
+   the process of name and service resolution *(provided by the DNS)*, able to
+   request a *TGT* as follows:
+
+    a. User/Client sends principal identity and credentials to the KDC Server
+       (TGT request).
+    b. KDC checks its internal database for the principal in question.
+    c. a TGT is created and wrapped by the KDC, using the principal's key
+       (TGT + Key).
+    d. The newly created TGT, is decrypted and stored in the credentials
+       cache.
+    e. At this point, Kerberos/GSSAPI aware applications (and/or services) are
+       able to check the list of active TGT in the keytab file.
+
+|
+|
+
+** *For Ceph Developers Only* **
+=================================
+
+We certainly could have used straight native ``KRB5 APIs`` (instead of
+``GSSAPIs``), but we wanted a more portable option as regards network security,
+which is the hallmark of the ``GSS`` *(Generic Security Standard)* ``-API``.
+It does not actually provide security services itself.
+
+Rather, it is a framework that provides security services to callers in a
+generic way.  ::
+
+    +---------------------------------+
+    |        Application              |
+    +---------------------------------+
+    | Protocol (RPC, Etc. [Optional]) |
+    +---------------------------------+
+    |         GSS-API                 |
+    +---------------------------------+
+    |   Security Mechs (Krb v5, Etc)  |
+    +---------------------------------+
+
+
+The GSS-API does two main things:
+
+    1. It creates a security context in which data can be passed between
+       applications. A context can be thought of as a sort of *"state of trust"*
+       between two applications.
+
+       Applications that share a context know who each other are and thus can
+       permit data transfers between them as long as the context lasts.
+
+    2. It applies one or more types of protection, known as *"security services"*,
+       to the data to be transmitted.
+
+
+GSS-API provides several types of portability for applications:
+
+    a. **Mechanism independence.** GSS-API provides a generic interface to the
+       mechanisms for which it has been implemented. By specifying a default
+       security mechanism, an application does not need to know which mechanism
+       it is using (for example, Kerberos v5), or even what type of mechanism
+       it uses. As an example, when an application forwards a user's credential
+       to a server, it does not need to know if that credential has a Kerberos
+       format or the format used by some other mechanism, nor how the
+       credentials are stored by the mechanism and accessed by the application.
+       (If necessary, an application can specify a particular mechanism to use)
+
+    b. **Protocol independence.** The GSS-API is independent of any
+       communications protocol or protocol suite. It can be used with
+       applications that use, for example, sockets, RCP, or TCP/IP.
+       RPCSEC_GSS "RPCSEC_GSS Layer" is an additional layer that smoothly
+       integrates GSS-API with RPC.
+
+    c. **Platform independence.** The GSS-API is completely oblivious to the
+       type of operating system on which an application is running.
+
+    d. **Quality of Protection independence.** Quality of Protection (QOP) is
+       the name given to the type of algorithm used in encrypting data or
+       generating cryptographic tags; the GSS-API allows a programmer to ignore
+       QOP, using a default provided by the GSS-API.
+       (On the other hand, an application can specify the QOP if necessary.)
+
+    The basic security offered by the GSS-API is authentication. Authentication
+    is the verification of an identity: *if you are authenticated, it means
+    that you are recognized to be who you say you are.*
+
+    The GSS-API provides for two additional security services, if supported by the
+    underlying mechanisms:
+
+    1. **Integrity:** It's not always sufficient to know that an application
+       sending you data is who it claims to be. The data itself could have
+       become corrupted or compromised.
+
+       The GSS-API provides for data to be accompanied by a cryptographic tag,
+       known as an ``Message Integrity Code (MIC)``, to prove that the data
+       that arrives at your doorstep is the same as the data that the sender
+       transmitted. This verification of the data's validity is known as
+       *"integrity"*.
+
+    2. **Confidentiality:** Both authentication and integrity, however, leave
+       the data itself alone, so if it's somehow intercepted, others can read
+       it.
+
+       The GSS-API therefore allows data to be encrypted, if underlying
+       mechanisms support it. This encryption of data is known as *"confidentiality"*.
+
+|
+
+Mechanisms Available With GSS-API:
+
+    The current implementation of the GSS-API works only with the Kerberos v5 security
+    mechanism.  ::
+
+        Mechanism Name          Object Identifier       Shared Library  Kernel Module
+        ----------------------  ----------------------  --------------  --------------
+        diffie_hellman_640_0    1.3.6.4.1.42.2.26.2.4   dh640-0.so.1
+        diffie_hellman_1024_0   1.3.6.4.1.42.2.26.2.5   dh1024-0.so.1
+        SPNEGO                  1.3.6.1.5.5.2
+        iakerb                  1.3.6.1.5.2.5
+        SCRAM-SHA-1             1.3.6.1.5.5.14
+        SCRAM-SHA-256           1.3.6.1.5.5.18
+        GSS-EAP (arc)           1.3.6.1.5.5.15.1.1.*
+        kerberos_v5             1.2.840.113554.1.2.2    gl/mech_krb5.so gl_kmech_krb5
+
+        Therefore:
+            Kerberos Version 5 GSS-API Mechanism
+            OID {1.2.840.113554.1.2.2}
+
+            Kerberos Version 5 GSS-API Mechanism
+            Simple and Protected GSS-API Negotiation Mechanism
+            OID {1.3.6.1.5.5.2}
+
+
+    There are two different formats:
+
+        1. The first, ``{ 1 2 3 4 }``, is officially mandated by the GSS-API
+           specs. ``gss_str_to_oid()`` expects this first format.
+
+        2. The second, ``1.2.3.4``, is more widely used but is not an official
+           standard format.
+
+    Although the GSS-API makes protecting data simple, it does not do certain
+    things, in order to maximize its generic nature. These include:
+
+        a. Provide security credentials for a user or application. These must
+           be provided by the underlying security mechanism(s). The GSS-API
+           does allow applications to acquire credentials, either automatically
+           or explicitly.
+
+        b. Transfer data between applications. It is the application's
+           responsibility to handle the transfer of all data between peers,
+           whether it is security-related or "plain" data.
+
+        c. Distinguish between different types of transmitted data (for
+           example, to know or determine that a data packet is plain data and
+           not GSS-API related).
+
+        d. Indicate status due to remote (asynchronous) errors.
+
+        e. Automatically protect information sent between processes of a
+           multiprocess program.
+
+        f. Allocate string buffers ("Strings and Similar Data") to be passed to
+           GSS-API functions.
+
+        g. Deallocate GSS-API data spaces. These must be explicitly deallocated
+           with functions such as ``gss_release_buffer()`` and
+           ``gss_delete_name()``.
+
+|
+
+These are the basic steps in using the GSS-API:
+
+    1. Each application, sender and recipient, acquires credentials explicitly,
+       if credentials have not been acquired automatically.
+
+    2. The sender initiates a security context and the recipient accepts it.
+
+    3. The sender applies security protection to the message (data) it wants to
+       transmit. This means that it either encrypts the message or stamps it
+       with an identification tag. The sender transmits the protected message.
+       (The sender can choose not to apply either security protection, in which
+       case the message has only the default GSS-API security service
+       associated with it. That is authentication, in which the recipient knows
+       that the sender is who it claims to be.)
+
+    4. The recipient decrypts the message (if needed) and verifies it
+       (if appropriate).
+
+    5. (Optional) The recipient returns an identification tag to the sender for
+       confirmation.
+
+    6. Both applications destroy the shared security context. If necessary,
+       they can also deallocate any *"leftover"* GSS-API data.
+
+    Applications that use the GSS-API should include the file ``gssapi.h``.
+
+    Good References:
+        - `rfc1964 <https://tools.ietf.org/html/rfc1964>`_.
+        - `rfc2743 <https://tools.ietf.org/html/rfc2743>`_.
+        - `rfc2744 <https://tools.ietf.org/html/rfc2744>`_.
+        - `rfc4178 <https://tools.ietf.org/html/rfc4178>`_.
+        - `rfc6649 <https://tools.ietf.org/html/rfc6649>`_.
+        - `MIT Kerberos Documentation <https://web.mit.edu/kerberos/krb5-latest/doc/appdev/gssapi.html>`_.
+
+|
+
+** *Kerberos Server Setup* **
+------------------------------
+
+First and foremost, ``this is not a recommendation for a production
+environment``. We are not covering ``Master/Slave replication cluster`` or
+anything production environment related (*ntp/chrony, dns, pam/nss, sssd, etc*).
+
+Also, on the server side there might be different dependencies and/or
+configuration steps needed, depending on which backend database will be used.
+``LDAP as a backend database`` is a good example of that.
+
+On the client side there are different steps depending on which client backend
+configuration will be used. For example ``PAM/NSS`` or ``SSSD`` (along with
+LDAP for identity service, [and Kerberos for authentication service]) which is
+the best suited option for joining ``MS Active Directory domains``, and doing
+``User Logon Management``.
+
+By no means we intend to cover every possible scenario/combination here. These
+steps are for a simple *get a (MIT) Kerberos Server up and running*.
+
+Please, note that *rpm packages might have slightly different names*, as well
+as the locations for the binaries and/or configuration files, depending on
+which Linux distro we are referring to.
+
+Finally, keep in mind that some Linux distros will have their own ``wizards``,
+which can perform the basic needed configuration:  ::
+
+    SUSE:
+        Kerberos server:
+            yast2 auth-server
+
+        Kerberos client:
+            pam/nss: yast2 ldapkrb
+            sssd: yast2 auth-client
+
+
+However, we are going through the ``manual configuration``.
+
+
+In order to get a new MIT KDC Server running:
+
+1. Install the KDC server by:
+
+    a. Install the needed packages:  ::
+
+        SUSE: zypper install krb5 krb5-server krb5-client
+            Additionally:
+                for development: krb5-devel
+                if using 'sssd': sssd-krb5 sssd-krb5-common
+
+        REDHAT: yum install krb5-server krb5-libs krb5-workstation
+            Additionally: 'Needs to be checked'
+
+
+    b. Edit the KDC Server configuration file:  ::
+
+        /var/lib/kerberos/krb5kdc/kdc.conf
+        [kdcdefaults]
+                kdc_ports = 750,88
+        [realms]
+                MYDOMAIN.COM = {
+                    acl_file = /var/lib/kerberos/krb5kdc/kadm5.acl
+                    admin_keytab = FILE:/var/lib/kerberos/krb5kdc/kadm5.keytab
+                    default_principal_flags = +postdateable +forwardable +renewable +proxiable
+                                                            +dup-skey -preauth -hwauth +service
+                                                            +tgt-based +allow-tickets -pwchange
+                                                            -pwservice
+                    dict_file = /var/lib/kerberos/krb5kdc/kadm5.dict
+                    key_stash_file = /var/lib/kerberos/krb5kdc/.k5.MYDOMAIN.COM
+                    kdc_ports = 750,88
+                    max_life = 0d 10h 0m 0s
+                    max_renewable_life = 7d 0h 0m 0s
+                }
+        ...
+
+
+    c. Edit the Kerberos Client configuration file:  ::
+
+        /etc/krb5.conf
+        [libdefaults]
+            dns_canonicalize_hostname = false
+            rdns = false
+            forwardable = true
+            dns_lookup_realm = true     //--> if using DNS/DNSMasq
+            dns_lookup_kdc = true       //--> if using DNS/DNSMasq
+            allow_weak_crypto = false
+            default_realm = MYDOMAIN.COM
+            default_ccache_name = KEYRING:persistent:%{uid}
+
+        [realms]
+            MYDOMAIN.COM = {
+                kdc = kerberos.mydomain.com
+                admin_server = kerberos.mydomain.com
+                ...
+            }
+        ...
+
+
+2. Create the Kerberos database:  ::
+
+    SUSE: kdb5_util create -s
+
+    REDHAT: kdb5_util create -s
+
+
+3. Enable and Start both 'KDC and KDC admin' servers:  ::
+
+    SUSE: systemctl enable/start krb5kdc
+          systemctl enable/start kadmind
+
+    REDHAT: systemctl enable/start krb5kdc
+            systemctl enable/start kadmin
+
+
+4. Create a Kerberos Administrator
+    Kerberos principals can be created either locally on the KDC server itself
+    or through the network, using an 'admin principal'. On the KDC server,
+    using ``kadmin.local``:
+
+    a. List the existing principals:  ::
+
+        kadmin.local:  listprincs
+        K/M@MYDOMAIN.COM
+        krbtgt/MYDOMAIN.COM@MYDOMAIN.COM
+        kadmin/admin@MYDOMAIN.COM
+        kadmin/changepw@MYDOMAIN.COM
+        kadmin/history@MYDOMAIN.COM
+        kadmin/kerberos.mydomain.com@MYDOMAIN.COM
+        root/admin@MYDOMAIN.COM
+        ...
+
+
+    b. In case we don't have a built-in 'admin principal', we then create one
+    (whatever ``principal name``, we are using ``root``, once by default
+    ``kinit`` tries to authenticate using the same system login user name,
+    unless a ``principal`` is passed as an argument ``kinit principal``):  ::
+
+        # kadmin.local -q "addprinc root/admin"
+        Authenticating as principal root/admin@MYDOMAIN.COM with password.
+        WARNING: no policy specified for root/admin@MYDOMAIN.COM; defaulting to no policy
+        Enter password for principal "root/admin@MYDOMAIN.COM":
+
+
+    c. Confirm the newly created 'admin principal' has the needed permissions
+       in the KDC ACL (if ACLs are changed, ``kadmind`` needs to be restarted):  ::
+
+        SUSE: /var/lib/kerberos/krb5kdc/kadm5.acl
+        REDHAT: /var/kerberos/krb5kdc/kadm5.acl
+
+        ###############################################################################
+        #Kerberos_principal      permissions     [target_principal]      [restrictions]
+        ###############################################################################
+        #
+        */admin@MYDOMAIN.COM     *
+
+
+    d. Create a simple 'user principal' (same steps as by *The 'Ceph side' of
+       the things*; 4a):  ::
+
+        kadmin.local:  addprinc johndoe
+        WARNING: no policy specified for johndoe@MYDOMAIN.COM; defaulting to no policy
+        Enter password for principal "johndoe@MYDOMAIN.COM":
+        Re-enter password for principal "johndoe@MYDOMAIN.COM":
+        Principal "johndoe@MYDOMAIN.COM" created.
+
+
+    e. Confirm the newly created 'user principal' is able to authenticate (same
+       steps as by *The 'Ceph side' of the things*; 6):  ::
+
+        # kdestroy -A && kinit -f johndoe && klist -f
+        Password for johndoe@MYDOMAIN.COM:
+        Ticket cache: KEYRING:persistent:0:0
+        Default principal: johndoe@MYDOMAIN.COM
+
+        Valid starting       Expires              Service principal
+        11/16/2018 13:11:16  11/16/2018 23:11:16  krbtgt/MYDOMAIN.COM@MYDOMAIN.COM
+                renew until 11/17/2018 13:11:16, Flags: FRI
+        ...
+
+
+5. At this point, we should have a *simple (MIT) Kerberos Server up and running*:
+
+     a. Considering we will want to work with keytab files, for both 'user and
+        service' principals, refer to The *'Ceph side' of the things* starting
+        at step 4.
+
+     b. Make sure you are comfortable with following and their ``manpages``:  ::
+
+            krb5.conf       -> Krb client config file
+            kdc.conf        -> KDC server config file
+
+            krb5kdc         -> KDC server daemon
+            kadmind         -> KDC administration daemon
+
+            kadmin          -> Krb administration tool
+            kdb5_util       -> Krb low-level database administration tool
+
+            kinit           -> Obtain and cache Kerberos ticket-granting ticket tool
+            klist           -> List cached Kerberos tickets tool
+            kdestroy        -> Destroy Kerberos tickets tool
+
+
+6. Name Resolution
+    As mentioned earlier, Kerberos *relies heavly on name resolution*. Most of
+    the Kerberos issues are usually related to name resolution, since Kerberos
+    is *very picky* on both *systems names* and *host lookups*.
+
+    a. As described in *The 'Ceph side' of the things*; step 2a, DNS RRs
+       greatly improves service location and host/domain resolution, by using
+       ``(srv resources)`` and ``(txt record)`` respectively (as per
+       *Before We Start*; *DNS resolution*).  ::
+
+        /var/lib/named/master/mydomain.com
+        kerberos                IN A        192.168.10.21
+        kerberos-slave          IN A        192.168.10.22
+        _kerberos               IN TXT      "MYDOMAIN.COM"
+        _kerberos._udp          IN SRV      1 0 88 kerberos
+        _kerberos._tcp          IN SRV      1 0 88 kerberos
+        _kerberos._udp          IN SRV      20 0 88 kerberos-slave
+        _kerberos-master._udp   IN SRV      0 0 88 kerberos
+        _kerberos-adm._tcp      IN SRV      0 0 749 kerberos
+        _kpasswd._udp           IN SRV      0 0 464 kerberos
+        ...
+
+
+    b. For a small network or development environment, where a *DNS server is
+       not available*, we have the option to use ``DNSMasq``, an
+       ease-to-configure lightweight DNS server (along with some other
+       capabilities).
+
+       These records can be added to ``/etc/dnsmasq.conf`` (in addition to the
+       needed 'host records'):  ::
+
+        /etc/dnsmasq.conf
+        ...
+        txt-record=_kerberos.mydomain.com,"MYDOMAIN.COM"
+        srv-host=_kerberos._udp.mydomain.com,kerberos.mydomain.com,88,1
+        srv-host=_kerberos._udp.mydomain.com,kerberos-2.mydomain.com,88,20
+        srv-host=_kerberos-master._udp.mydomain.com,kerberos.mydomain.com,88,0
+        srv-host=_kerberos-adm._tcp.mydomain.com,kerberos.mydomain.com,749,0
+        srv-host=_kpasswd._udp.mydomain.com,kerberos.mydomain.com,464,0
+        srv-host=_kerberos._tcp.mydomain.com,kerberos.mydomain.com,88,1
+        ...
+
+
+    c. After 'b)' is all set, and ``dnsmasq`` service up and running, we can
+       test it using:  ::
+
+        # nslookup kerberos
+        Server:     192.168.10.1
+        Address:    192.168.10.1#53
+
+        Name:   kerberos.mydomain.com
+        Address: 192.168.10.21
+
+        # host -t SRV _kerberos._tcp.mydomain.com
+        _kerberos._tcp.mydomain.com has SRV record 1 0 88 kerberos.mydomain.com.
+
+        # host -t SRV {each srv-host record}
+        # host -t TXT _kerberos.mydomain.com
+        _kerberos.mydomain.com descriptive text "MYDOMAIN.COM"
+        ...
+
+
+    f. As long as ``name resolution`` is working properly, either ``dnsmasq``
+       or ``named``, Kerberos should be able to find the needed service
+       records.
diff --git a/doc/dev/cephadm/cephadm-exporter.rst b/doc/dev/cephadm/cephadm-exporter.rst
new file mode 100644
index 000000000..bc41fcaeb
--- /dev/null
+++ b/doc/dev/cephadm/cephadm-exporter.rst
@@ -0,0 +1,306 @@
+================
+cephadm Exporter
+================
+
+There are a number of long running tasks that the cephadm 'binary' runs which can take several seconds
+to run. This latency represents a scalability challenge to the Ceph orchestrator management plane.
+
+To address this, cephadm needs to be able to run some of these longer running tasks asynchronously - this
+frees up processing on the mgr by offloading tasks to each host, reduces latency and improves scalability.
+
+This document describes the implementation requirements and design for an 'exporter' feature
+
+
+Requirements
+============
+The exporter should address these functional and non-functional requirements;
+
+* run as a normal systemd unit
+* utilise the same filesystem schema as other services deployed with cephadm
+* require only python3 standard library modules (no external dependencies)
+* use encryption to protect the data flowing from a host to Ceph mgr
+* execute data gathering tasks as background threads
+* be easily extended to include more data gathering tasks
+* monitor itself for the health of the data gathering threads
+* cache metadata to respond to queries quickly
+* respond to a metadata query in <30ms to support large Ceph clusters (000's nodes)
+* provide CLI interaction to enable the exporter to be deployed either at bootstrap time, or once the
+  cluster has been deployed.
+* be deployed as a normal orchestrator service (similar to the node-exporter)
+
+High Level Design
+=================
+
+This section will focus on the exporter logic **only**.
+
+.. code::
+
+    Establish a metadata cache object (tasks will be represented by separate attributes)
+    Create a thread for each data gathering task; host, ceph-volume and list_daemons
+        each thread updates it's own attribute within the cache object
+    Start a server instance passing requests to a specific request handler
+        the request handler only interacts with the cache object
+        the request handler passes metadata back to the caller
+    Main Loop
+        Leave the loop if a 'stop' request is received
+        check thread health
+            if a thread that was active, is now inactive
+                update the cache marking the task as inactive
+                update the cache with an error message for that task
+        wait for n secs
+        
+
+In the initial exporter implementation, the exporter has been implemented as a RESTful API.
+
+
+Security
+========
+
+The cephadm 'binary' only supports standard python3 features, which has meant the RESTful API has been
+developed using the http module, which itself is not intended for production use. However, the implementation
+is not complex (based only on HTTPServer and BaseHHTPRequestHandler) and only supports the GET method - so the
+security risk is perceived as low.
+
+Current mgr to host interactions occurs within an ssh connection, so the goal of the exporter is to adopt a similar
+security model.
+
+The initial REST API is implemented with the following features;
+
+* generic self-signed, or user provided SSL crt/key to encrypt traffic between the mgr and the host
+* 'token' based authentication of the request
+
+All exporter instances will use the **same** crt/key to secure the link from the mgr to the host(s), in the same way
+that the ssh access uses the same public key and port for each host connection.
+
+.. note:: Since the same SSL configuration is used on every exporter, when you supply your own settings you must
+  ensure that the CN or SAN components of the distinguished name are either **not** used or created using wildcard naming.
+
+The crt, key and token files are all defined with restrictive permissions (600), to help mitigate against the risk of exposure
+to any other user on the Ceph cluster node(s).
+
+Administrator Interaction
+=========================
+Several new commands are required to configure the exporter, and additional parameters should be added to the bootstrap
+process to allow the exporter to be deployed automatically for new clusters.
+
+
+Enhancements to the 'bootstrap' process
+---------------------------------------
+bootstrap should support additional parameters to automatically configure exporter daemons across hosts
+
+``--with-exporter``
+
+By using this flag, you're telling the bootstrap process to include the cephadm-exporter service within the 
+cluster. If you do not provide a specific configuration (SSL, token, port) to use, defaults would be applied.
+
+``--exporter-config``
+
+With the --exporter-config option, you may pass your own SSL, token and port information. The file must be in 
+JSON format and contain the following fields; crt, key, token and port. The JSON content should be validated, and any
+errors detected passed back to the user during the argument parsing phase (before any changes are done).
+
+
+Additional ceph commands
+------------------------
+::
+
+# ceph cephadm generate-exporter-config
+
+This command will create generate a default configuration consisting of; a self signed certificate, a randomly generated
+32 character token and the default port of 9443 for the REST API.
+::
+
+# ceph cephadm set-exporter-config -i <config.json>
+
+Use a JSON file to define the crt, key, token and port for the REST API. The crt, key and token are validated by
+the mgr/cephadm module prior storing the values in the KV store. Invalid or missing entries should be reported to the 
+user.
+::
+
+# ceph cephadm clear-exporter-config
+
+Clear the current configuration (removes the associated keys from the KV store)
+::
+
+# ceph cephadm get-exporter-config
+
+Show the current exporter configuration, in JSON format
+
+
+.. note:: If the service is already deployed any attempt to change or clear the configuration will
+    be denied. In order to change settings you must remove the service, apply the required configuration
+    and re-apply (``ceph orch apply cephadm-exporter``)
+
+
+
+New Ceph Configuration Keys
+===========================
+The exporter configuration is persisted to the monitor's KV store, with the following keys:
+
+| mgr/cephadm/exporter_config
+| mgr/cephadm/exporter_enabled
+
+
+
+RESTful API
+===========
+The primary goal of the exporter is the provision of metadata from the host to the mgr. This interaction takes
+place over a simple GET interface. Although only the GET method is supported, the API provides multiple URLs to
+provide different views on the metadata that has been gathered.
+
+.. csv-table:: Supported URL endpoints
+    :header: "URL", "Purpose"
+
+    "/v1/metadata", "show all metadata including health of all threads"
+    "/v1/metadata/health", "only report on the health of the data gathering threads"
+    "/v1/metadata/disks", "show the disk output (ceph-volume inventory data)"
+    "/v1/metadata/host", "show host related metadata from the gather-facts command"
+    "/v1/metatdata/daemons", "show the status of all ceph cluster related daemons on the host"
+
+Return Codes
+------------
+The following HTTP return codes are generated by the API
+
+.. csv-table:: Supported HTTP Responses
+    :header: "Status Code", "Meaning"
+
+    "200", "OK"
+    "204", "the thread associated with this request is no longer active, no data is returned"
+    "206", "some threads have stopped, so some content is missing"
+    "401", "request is not authorised - check your token is correct"
+    "404", "URL is malformed, not found"
+    "500", "all threads have stopped - unable to provide any metadata for the host"
+
+
+Deployment
+==========
+During the initial phases of the exporter implementation, deployment is regarded as optional but is available
+to new clusters and existing clusters that have the feature (Pacific and above).
+
+* new clusters : use the ``--with-exporter`` option
+* existing clusters : you'll need to set the configuration and deploy the service manually
+
+.. code::
+
+    # ceph cephadm generate-exporter-config
+    # ceph orch apply cephadm-exporter
+
+If you choose to remove the cephadm-exporter service, you may simply 
+
+.. code::
+
+    # ceph orch rm cephadm-exporter
+
+This will remove the daemons, and the exporter releated settings stored in the KV store.
+
+
+Management
+==========
+Once the exporter is deployed, you can use the following snippet to extract the host's metadata.
+
+.. code-block:: python
+
+    import ssl
+    import json
+    import sys
+    import tempfile
+    import time
+    from urllib.request import Request, urlopen
+
+    # CHANGE THIS V
+    hostname = "rh8-1.storage.lab"
+    
+    print("Reading config.json")
+    try:
+        with open('./config.json', 'r') as f:
+            raw=f.read()
+    except FileNotFoundError as e:
+        print("You must first create a config.json file using the cephadm get-exporter-config command")
+        sys.exit(1)
+
+    cfg = json.loads(raw)
+    with tempfile.NamedTemporaryFile(buffering=0) as t:
+        print("creating a temporary local crt file from the json")
+        t.write(cfg['crt'].encode('utf-8'))
+
+        ctx = ssl.create_default_context()
+        ctx.check_hostname = False
+        ctx.load_verify_locations(t.name)
+        hdrs={"Authorization":f"Bearer {cfg['token']}"}
+        print("Issuing call to gather metadata")
+        req=Request(f"https://{hostname}:9443/v1/metadata",headers=hdrs)
+        s_time = time.time()
+        r = urlopen(req,context=ctx)
+        print(r.status)
+        print("call complete")
+        # assert r.status == 200
+        if r.status in [200, 206]:
+
+            raw=r.read()  # bytes string
+            js=json.loads(raw.decode())
+            print(json.dumps(js, indent=2))
+        elapsed = time.time() - s_time
+        print(f"Elapsed secs : {elapsed}")
+
+
+.. note:: the above example uses python3, and assumes that you've extracted the config using the ``get-exporter-config`` command.
+
+
+Implementation Specific Details
+===============================
+
+In the same way as a typical container based deployment, the exporter is deployed to a directory under ``/var/lib/ceph/<fsid>``. The 
+cephadm binary is stored in this cluster folder, and the daemon's configuration and systemd settings are stored
+under ``/var/lib/ceph/<fsid>/cephadm-exporter.<id>/``.
+
+.. code::
+
+    [root@rh8-1 cephadm-exporter.rh8-1]# pwd
+    /var/lib/ceph/cb576f70-2f72-11eb-b141-525400da3eb7/cephadm-exporter.rh8-1
+    [root@rh8-1 cephadm-exporter.rh8-1]# ls -al 
+    total 24
+    drwx------. 2 root root  100 Nov 25 18:10 .
+    drwx------. 8 root root  160 Nov 25 23:19 ..
+    -rw-------. 1 root root 1046 Nov 25 18:10 crt
+    -rw-------. 1 root root 1704 Nov 25 18:10 key
+    -rw-------. 1 root root   64 Nov 25 18:10 token
+    -rw-------. 1 root root   38 Nov 25 18:10 unit.configured
+    -rw-------. 1 root root   48 Nov 25 18:10 unit.created
+    -rw-r--r--. 1 root root  157 Nov 25 18:10 unit.run
+
+
+In order to respond to requests quickly, the CephadmDaemon uses a cache object (CephadmCache) to hold the results
+of the cephadm commands.
+
+The exporter doesn't introduce any new data gathering capability - instead it merely calls the existing cephadm commands.
+
+The CephadmDaemon class creates a local HTTP server(uses ThreadingMixIn), secured with TLS and uses the CephadmDaemonHandler
+to handle the requests. The request handler inspects the request header and looks for a valid Bearer token - if this is invalid
+or missing the caller receives a 401 Unauthorized error.
+
+The 'run' method of the CephadmDaemon class, places the scrape_* methods into different threads with each thread supporting
+a different refresh interval. Each thread then periodically issues it's cephadm command, and places the output
+in the cache object.
+
+In addition to the command output, each thread also maintains it's own timestamp record in the cache so the caller can 
+very easily determine the age of the data it's received.
+
+If the underlying cephadm command execution hits an exception, the thread passes control to a _handle_thread_exception method.
+Here the exception is logged to the daemon's log file and the exception details are added to the cache, providing visibility
+of the problem to the caller.
+
+Although each thread is effectively given it's own URL endpoint (host, disks, daemons), the recommended way to gather data from
+the host is to simply use the ``/v1/metadata`` endpoint. This will provide all of the data, and indicate whether any of the
+threads have failed.
+
+The run method uses "signal" to establish a reload hook, but in the initial implementation this doesn't take any action and simply
+logs that a reload was received.
+
+
+Future Work
+===========
+
+#. Consider the potential of adding a restart policy for threads
+#. Once the exporter is fully integrated into mgr/cephadm, the goal would be to make the exporter the 
+   default means of data gathering. However, until then the exporter will remain as an opt-in 'feature
+   preview'.
diff --git a/doc/dev/cephadm/compliance-check.rst b/doc/dev/cephadm/compliance-check.rst
new file mode 100644
index 000000000..eea462445
--- /dev/null
+++ b/doc/dev/cephadm/compliance-check.rst
@@ -0,0 +1,121 @@
+================
+Compliance Check
+================
+
+The stability and reliability of a Ceph cluster is dependent not just upon the Ceph daemons, but
+also the OS and hardware that Ceph is installed on. This document is intended to promote a design 
+discussion for providing a "compliance" feature within mgr/cephadm, which would be responsible for
+identifying common platform-related issues that could impact Ceph stability and operation.
+
+The ultimate goal of these checks is to identify issues early and raise a healthcheck WARN
+event, to alert the Administrator to the issue.
+
+Prerequisites
+=============
+In order to effectively analyse the hosts that Ceph is deployed to, this feature requires a cache
+of host-related metadata. The metadata is already available from cephadm's HostFacts class and the
+``gather-facts`` cephadm command. For the purposes of this document, we will assume that this
+data is available within the mgr/cephadm "cache" structure.
+
+Some checks will require that the host status is also populated e.g. ONLINE, OFFLINE, MAINTENANCE
+
+Administrator Interaction
+=========================
+Not all users will require this feature, and must be able to 'opt out'. For this reason,
+mgr/cephadm must provide controls, such as the following;
+
+.. code-block::
+
+   ceph cephadm compliance enable | disable | status [--format json]
+   ceph cephadm compliance ls [--format json]
+   ceph cephadm compliance enable-check <name>
+   ceph cephadm compliance disable-check <name>
+   ceph cephadm compliance set-check-interval <int>
+   ceph cephadm compliance get-check-interval
+
+The status option would show the enabled/disabled state of the feature, along with the
+check-interval.
+
+The ``ls`` subcommand would show all checks in the following format;
+
+``check-name status description``
+
+Proposed Integration
+====================
+The compliance checks are not required to run all the time, but instead should run at discrete
+intervals. The interval would be configurable under via the :code:`set-check-interval`
+subcommand (default would be every 12 hours)
+
+
+mgr/cephadm currently executes an event driven (time based) serve loop to act on deploy/remove and
+reconcile activity. In order to execute the compliance checks, the compliance check code would be 
+called from this main serve loop - when the :code:`set-check-interval` is met.
+
+
+Proposed Checks
+===============
+All checks would push any errors to a list, so multiple issues can be escalated to the Admin at
+the same time. The list below provides a description of each check, with the text following the 
+name indicating a shortname version *(the shortname is the reference for command Interaction
+when enabling or disabling a check)*
+
+
+OS Consistency (OS)
+___________________
+* all hosts must use same vendor
+* all hosts must be on the same major release (this check would only be applicable to distributions that
+  offer a long-term-support strategy (RHEL, CentOS, SLES, Ubuntu etc)
+
+
+*src: gather-facts output*
+
+Linux Kernel Security Mode (LSM)
+________________________________
+* All hosts should have a consistent SELINUX/AppArmor configuration
+
+*src: gather-facts output*
+
+Services Check (SERVICES)
+_________________________
+Hosts that are in an ONLINE state should adhere to the following;
+
+* all daemons (systemd units) should be enabled
+* all daemons should be running (not dead)
+
+*src: list_daemons output*
+
+Support Status (SUPPORT)
+________________________
+If support status has been detected, it should be consistent across all hosts. At this point
+support status is available only for Red Hat machines.
+
+*src: gather-facts output*
+
+Network : MTU (MTU)
+________________________________
+All network interfaces on the same Ceph network (public/cluster) should have the same MTU
+
+*src: gather-facts output*
+
+Network : LinkSpeed (LINKSPEED)
+____________________________________________
+All network interfaces on the same Ceph network (public/cluster) should have the same Linkspeed
+
+*src: gather-facts output*
+
+Network : Consistency (INTERFACE)
+______________________________________________
+All hosts with OSDs should have consistent network configuration - eg. if some hosts do
+not separate cluster/public traffic but others do, that is an anomaly that would generate a
+compliance check warning.
+
+*src: gather-facts output*
+
+Notification Strategy
+=====================
+If any of the checks fail, mgr/cephadm would raise a WARN level alert
+
+Futures
+=======
+The checks highlighted here serve only as a starting point, and we should expect to expand
+on the checks over time.
diff --git a/doc/dev/cephadm/developing-cephadm.rst b/doc/dev/cephadm/developing-cephadm.rst
new file mode 100644
index 000000000..d9f81c2c0
--- /dev/null
+++ b/doc/dev/cephadm/developing-cephadm.rst
@@ -0,0 +1,263 @@
+=======================
+Developing with cephadm
+=======================
+
+There are several ways to develop with cephadm.  Which you use depends
+on what you're trying to accomplish.
+
+vstart --cephadm
+================
+
+- Start a cluster with vstart, with cephadm configured
+- Manage any additional daemons with cephadm
+- Requires compiled ceph binaries
+
+In this case, the mon and manager at a minimum are running in the usual
+vstart way, not managed by cephadm.  But cephadm is enabled and the local
+host is added, so you can deploy additional daemons or add additional hosts.
+
+This works well for developing cephadm itself, because any mgr/cephadm
+or cephadm/cephadm code changes can be applied by kicking ceph-mgr
+with ``ceph mgr fail x``.  (When the mgr (re)starts, it loads the
+cephadm/cephadm script into memory.)
+
+::
+
+   MON=1 MGR=1 OSD=0 MDS=0 ../src/vstart.sh -d -n -x --cephadm
+
+- ``~/.ssh/id_dsa[.pub]`` is used as the cluster key.  It is assumed that
+  this key is authorized to ssh with no passphrase to root@`hostname`.
+- cephadm does not try to manage any daemons started by vstart.sh (any
+  nonzero number in the environment variables).  No service spec is defined
+  for mon or mgr.
+- You'll see health warnings from cephadm about stray daemons--that's because
+  the vstart-launched daemons aren't controlled by cephadm.
+- The default image is ``quay.io/ceph-ci/ceph:master``, but you can change
+  this by passing ``-o container_image=...`` or ``ceph config set global container_image ...``.
+
+
+cstart and cpatch
+=================
+
+The ``cstart.sh`` script will launch a cluster using cephadm and put the
+conf and keyring in your build dir, so that the ``bin/ceph ...`` CLI works
+(just like with vstart).  The ``ckill.sh`` script will tear it down.
+
+- A unique but stable fsid is stored in ``fsid`` (in the build dir).
+- The mon port is random, just like with vstart.
+- The container image is ``quay.io/ceph-ci/ceph:$tag`` where $tag is
+  the first 8 chars of the fsid.
+- If the container image doesn't exist yet when you run cstart for the
+  first time, it is built with cpatch.
+
+There are a few advantages here:
+
+- The cluster is a "normal" cephadm cluster that looks and behaves
+  just like a user's cluster would.  In contrast, vstart and teuthology
+  clusters tend to be special in subtle (and not-so-subtle) ways (e.g.
+  having the ``lockdep`` turned on).
+
+To start a test cluster::
+
+  sudo ../src/cstart.sh
+
+The last line of the output will be a line you can cut+paste to update
+the container image.  For instance::
+
+  sudo ../src/script/cpatch -t quay.io/ceph-ci/ceph:8f509f4e
+
+By default, cpatch will patch everything it can think of from the local
+build dir into the container image.  If you are working on a specific
+part of the system, though, can you get away with smaller changes so that
+cpatch runs faster.  For instance::
+
+  sudo ../src/script/cpatch -t quay.io/ceph-ci/ceph:8f509f4e --py
+
+will update the mgr modules (minus the dashboard).  Or::
+
+  sudo ../src/script/cpatch -t quay.io/ceph-ci/ceph:8f509f4e --core
+
+will do most binaries and libraries.  Pass ``-h`` to cpatch for all options.
+
+Once the container is updated, you can refresh/restart daemons by bouncing
+them with::
+
+  sudo systemctl restart ceph-`cat fsid`.target
+
+When you're done, you can tear down the cluster with::
+
+  sudo ../src/ckill.sh   # or,
+  sudo ../src/cephadm/cephadm rm-cluster --force --fsid `cat fsid`
+
+cephadm bootstrap --shared_ceph_folder
+======================================
+
+Cephadm can also be used directly without compiled ceph binaries.
+
+Run cephadm like so::
+
+  sudo ./cephadm bootstrap --mon-ip 127.0.0.1 \
+    --ssh-private-key /home/<user>/.ssh/id_rsa \
+    --skip-mon-network \
+    --skip-monitoring-stack --single-host-defaults \
+    --skip-dashboard \ 
+    --shared_ceph_folder /home/<user>/path/to/ceph/
+
+- ``~/.ssh/id_rsa`` is used as the cluster key.  It is assumed that
+  this key is authorized to ssh with no passphrase to root@`hostname`.
+
+Source code changes made in the ``pybind/mgr/`` directory then
+require a daemon restart to take effect. 
+
+Note regarding network calls from CLI handlers
+==============================================
+
+Executing any cephadm CLI commands like ``ceph orch ls`` will block the
+mon command handler thread within the MGR, thus preventing any concurrent
+CLI calls. Note that pressing ``^C`` will not resolve this situation,
+as *only* the client will be aborted, but not execution of the command
+within the orchestrator manager module itself. This means, cephadm will
+be completely unresponsive until the execution of the CLI handler is
+fully completed. Note that even ``ceph orch ps`` will not respond while
+another handler is executing.
+
+This means we should do very few synchronous calls to remote hosts.
+As a guideline, cephadm should do at most ``O(1)`` network calls in CLI handlers.
+Everything else should be done asynchronously in other threads, like ``serve()``.
+
+Note regarding different variables used in the code
+===================================================
+
+* a ``service_type`` is something like mon, mgr, alertmanager etc defined 
+  in ``ServiceSpec``
+* a ``service_id`` is the name of the service. Some services don't have 
+  names.
+* a ``service_name`` is ``<service_type>.<service_id>``
+* a ``daemon_type`` is the same as the service_type, except for ingress,
+  which has the haproxy and keepalived daemon types.
+* a ``daemon_id`` is typically ``<service_id>.<hostname>.<random-string>``. 
+  (Not the case for e.g. OSDs. OSDs are always called OSD.N)
+* a ``daemon_name`` is ``<daemon_type>.<daemon_id>``
+
+Kcli: a virtualization management tool to make easy orchestrators development
+=============================================================================
+`Kcli <https://github.com/karmab/kcli>`_ is meant to interact with existing
+virtualization providers (libvirt, KubeVirt, oVirt, OpenStack, VMware vSphere,
+GCP and AWS) and to easily deploy and customize VMs from cloud images.
+
+It allows you to setup an environment with several vms with your preferred
+configuration( memory, cpus, disks) and OS flavor.
+
+main advantages:
+----------------
+  - Is fast. Typically you can have a completely new Ceph cluster ready to debug
+    and develop orchestrator features in less than 5 minutes.
+  - Is a "near production" lab. The lab created with kcli is very near of "real"
+    clusters in QE labs or even in production. So easy to test "real things" in
+    almost "real environment"
+  - Is safe and isolated. Do not depend of the things you have installed in your
+    machine. And the vms are isolated from your environment.
+  - Easy to work "dev" environment. For "not compilated" software pieces,
+    for example any mgr module. It is an environment that allow you to test your
+    changes interactively.
+
+Installation:
+-------------
+Complete documentation in `kcli installation <https://kcli.readthedocs.io/en/latest/#installation>`_
+but we strongly suggest to use the container image approach.
+
+So things to do:
+  - 1. Review `requeriments <https://kcli.readthedocs.io/en/latest/#libvirt-hypervisor-requisites>`_
+    and install/configure whatever you need to meet them.
+  - 2. get the kcli image and create one alias for executing the kcli command
+    ::
+
+        # podman pull quay.io/karmab/kcli
+        # alias kcli='podman run --net host -it --rm --security-opt label=disable -v $HOME/.ssh:/root/.ssh -v $HOME/.kcli:/root/.kcli -v /var/lib/libvirt/images:/var/lib/libvirt/images -v /var/run/libvirt:/var/run/libvirt -v $PWD:/workdir -v /var/tmp:/ignitiondir quay.io/karmab/kcli'
+
+.. note:: /var/lib/libvirt/images can be customized.... be sure that you are
+   using this folder for your OS images
+
+.. note:: Once you have used your kcli tool to create and use different labs, we
+   suggest you to "save" and use your own kcli image.
+   Why?: kcli is alive and it changes (and for the moment only exists one tag ...
+   latest). Because we have more than enough with the current functionality, and
+   what we want is overall stability,
+   we suggest to store the kcli image you are using in a safe place and update
+   your kcli alias to use your own image.
+
+Test your kcli installation:
+----------------------------
+See the kcli `basic usage workflow <https://kcli.readthedocs.io/en/latest/#basic-workflow>`_
+
+Create a Ceph lab cluster
+-------------------------
+In order to make easy this task we are going to use a kcli plan.
+
+A kcli plan is a file where you can define the different settings you want to
+have in a set of vms.
+You can define hardware parameters (cpu, memory, disks ..), operating system and
+it also allows you to automate the installation and configuration of any
+software you want to have.
+
+There is a `repository <https://github.com/karmab/kcli-plans>`_ with a collection of
+plans that can be used for different purposes. And we have predefined plans to
+install Ceph clusters using Ceph ansible or cephadm, lets create our first Ceph
+cluster using cephadm::
+
+# kcli2 create plan -u https://github.com/karmab/kcli-plans/blob/master/ceph/ceph_cluster.yml
+
+This will create a set of three vms using the plan file pointed by the url.
+After a few minutes (depend of your laptop power), lets examine the cluster:
+
+* Take a look to the vms created::
+
+  # kcli list vms
+
+* Enter in the bootstrap node::
+
+  # kcli ssh ceph-node-00
+
+* Take a look to the ceph cluster installed::
+
+  [centos@ceph-node-00 ~]$ sudo -i
+  [root@ceph-node-00 ~]# cephadm version
+  [root@ceph-node-00 ~]# cephadm shell
+  [ceph: root@ceph-node-00 /]# ceph orch host ls
+
+Create a Ceph cluster to make easy developing in mgr modules (Orchestrators and Dashboard)
+------------------------------------------------------------------------------------------
+The cephadm kcli plan (and cephadm) are prepared to do that.
+
+The idea behind this method is to replace several python mgr folders in each of
+the ceph daemons with the source code folders in your host machine.
+This "trick" will allow you to make changes in any orchestrator or dashboard
+module and test them intermediately. (only needed to disable/enable the mgr module)
+
+So in order to create a ceph cluster for development purposes you must use the
+same cephadm plan but with a new parameter pointing your Ceph source code folder::
+
+  # kcli create plan -u https://github.com/karmab/kcli-plans/blob/master/ceph/ceph_cluster.yml -P ceph_dev_folder=/home/mycodefolder/ceph
+
+Ceph Dashboard development
+--------------------------
+Ceph dashboard module is not going to be loaded if previously you have not
+generated the frontend bundle.
+
+For now, in order load properly the Ceph Dashboardmodule and to apply frontend
+changes you have to run "ng build" on your laptop::
+
+  # Start local frontend build with watcher (in background):
+  sudo dnf install -y nodejs
+  cd <path-to-your-ceph-repo>
+  cd src/pybind/mgr/dashboard/frontend
+  sudo chown -R <your-user>:root dist node_modules
+  NG_CLI_ANALYTICS=false npm ci
+  npm run build -- --deleteOutputPath=false --watch &
+
+After saving your changes, the frontend bundle will be built again.
+When completed, you'll see::
+
+  "Localized bundle generation complete."
+
+Then you can reload your Dashboard browser tab.
diff --git a/doc/dev/cephadm/host-maintenance.rst b/doc/dev/cephadm/host-maintenance.rst
new file mode 100644
index 000000000..2b84ec7bd
--- /dev/null
+++ b/doc/dev/cephadm/host-maintenance.rst
@@ -0,0 +1,104 @@
+================
+Host Maintenance
+================
+
+All hosts that support Ceph daemons need to support maintenance activity, whether the host
+is physical or virtual. This means that management workflows should provide
+a simple and consistent way to support this operational requirement. This document defines
+the maintenance strategy that could be implemented in cephadm and mgr/cephadm.
+
+
+High Level Design
+=================
+Placing a host into maintenance, adopts the following workflow;
+
+#. confirm that the removal of the host does not impact data availability (the following
+   steps will assume it is safe to proceed)
+
+   * orch host ok-to-stop <host> would be used here
+
+#. if the host has osd daemons, apply noout to the host subtree to prevent data migration
+   from triggering during the planned maintenance slot.
+#. Stop the ceph target (all daemons stop)
+#. Disable the ceph target on that host, to prevent a reboot from automatically starting
+   ceph services again)
+
+
+Exiting Maintenance, is basically the reverse of the above sequence
+
+Admin Interaction
+=================
+The ceph orch command will be extended to support maintenance. 
+
+.. code-block::
+
+    ceph orch host maintenance enter <host> [ --force ]
+    ceph orch host maintenance exit <host>
+
+.. note:: In addition, the host's status should be updated to reflect whether it
+   is in maintenance or not. 
+
+The 'check' Option 
+__________________
+The orch host ok-to-stop command focuses on ceph daemons (mon, osd, mds), which
+provides the first check. However, a ceph cluster also uses other types of daemons
+for monitoring, management and non-native protocol support which means the 
+logic will need to consider service impact too. The 'check' option provides 
+this additional layer to alert the user of service impact to *secondary*
+daemons.
+
+The list below shows some of these additional daemons.
+
+* mgr (not included in ok-to-stop checks) 
+* prometheus, grafana, alertmanager
+* rgw
+* haproxy
+* iscsi gateways
+* ganesha gateways
+
+By using the --check option first, the Admin can choose whether to proceed. This 
+workflow is obviously optional for the CLI user, but could be integrated into the
+UI workflow to help less experienced Administators manage the cluster.
+
+By adopting this two-phase approach, a UI based workflow would look something
+like this.
+
+#. User selects a host to place into maintenance
+ 
+   * orchestrator checks for data **and** service impact
+#. If potential impact is shown, the next steps depend on the impact type
+   
+   * **data availability** : maintenance is denied, informing the user of the issue
+   * **service availability** : user is provided a list of affected services and 
+     asked to confirm
+
+
+Components Impacted
+===================
+Implementing this capability will require changes to the following; 
+
+* cephadm
+
+  * Add maintenance subcommand with the following 'verbs'; enter, exit, check
+
+* mgr/cephadm
+
+  * add methods to CephadmOrchestrator for enter/exit and check
+  * data gathering would be skipped for hosts in a maintenance state
+
+* mgr/orchestrator
+
+  * add CLI commands to OrchestratorCli which expose the enter/exit and check interaction
+
+
+Ideas for Future Work
+=====================
+#. When a host is placed into maintenance, the time of the event could be persisted. This
+   would allow the orchestrator layer to establish a maintenance window for the task and 
+   alert if the maintenance window has been exceeded.
+#. The maintenance process could support plugins to allow other integration tasks to be
+   initiated as part of the transition to and from maintenance. This plugin capability could
+   support actions like; 
+   
+   * alert suppression to 3rd party monitoring framework(s)
+   * service level reporting, to record outage windows
diff --git a/doc/dev/cephadm/index.rst b/doc/dev/cephadm/index.rst
new file mode 100644
index 000000000..a09baffdb
--- /dev/null
+++ b/doc/dev/cephadm/index.rst
@@ -0,0 +1,15 @@
+===================================
+CEPHADM Developer Documentation
+===================================
+
+.. rubric:: Contents
+
+.. toctree::
+   :maxdepth: 1
+
+
+   developing-cephadm
+   host-maintenance
+   compliance-check
+   cephadm-exporter
+   scalability-notes
diff --git a/doc/dev/cephadm/scalability-notes.rst b/doc/dev/cephadm/scalability-notes.rst
new file mode 100644
index 000000000..157153cb3
--- /dev/null
+++ b/doc/dev/cephadm/scalability-notes.rst
@@ -0,0 +1,95 @@
+#############################################
+ Notes and Thoughts on Cephadm's scalability
+#############################################
+
+*********************
+ About this document
+*********************
+
+This document does NOT define a specific proposal or some future work.
+Instead it merely lists a few thoughts that MIGHT be relevant for future
+cephadm enhacements.
+
+*******
+ Intro
+*******
+
+Current situation:
+
+Cephadm manages all registered hosts. This means that it periodically
+scrapes data from each host to identify changes on the host like:
+
+-  disk added/removed
+-  daemon added/removed
+-  host network/firewall etc has changed
+
+Currently, cephadm scrapes each host (up to 10 in parallel) every 6
+minutes, unless a refresh is forced manually.
+
+Refreshes for disks (ceph-volume), daemons (podman/docker), etc, happen
+in sequence.
+
+With the cephadm exporter, we have now reduced the time to scan hosts
+considerably, but the question remains:
+
+Is the cephadm-exporter sufficient to solve all future scalability
+issues?
+
+***********************************************
+ Considerations of cephadm-exporter's REST API
+***********************************************
+
+The cephadm-exporter uses HTTP to serve an endpoint to the hosts
+metadata. We MIGHT encounter some issues with this approach, which need
+to be mitigated at some point.
+
+-  With the cephadm-exporter we use SSH and HTTP to connect to each
+   host. Having two distinct transport layers feels odd, and we might
+   want to consider reducing it to only a single protocol.
+
+-  The current approach of delivering ``bin/cephadm`` to the host doesn't
+   allow the use of external dependencies. This means that we're stuck
+   with the built-in HTTP server lib, which isn't great for providing a
+   good developer experience. ``bin/cephadm`` needs to be packaged and
+   distributed (one way or the other) for us to make use of a better
+   http server library.
+
+************************
+ MON's config-key store
+************************
+
+After the ``mgr/cephadm`` queried metadata from each host, cephadm stores
+the data within the mon's k-v store.
+
+If each host would be allowed to write their own metadata to the store,
+``mgr/cephadm`` would no longer be required to gather the data.
+
+Some questions arise:
+
+-  ``mgr/cephadm`` now needs to query data from the config-key store,
+   instead of relying on cached data.
+
+-  cephadm knows three different types of data: (1) Data that is
+   critical and needs to be stored in the config-key store. (2) Data
+   that can be kept in memory only. (3) Data that can be stored in
+   RADOS pool. How can we apply this idea to those different types of
+   data.
+
+*******************************
+ Increase the worker pool size
+*******************************
+
+``mgr/cephadm`` is currently able to scrape 10 nodes at the same time.
+
+The scrape of a individual host takes the same amount of time persists.
+We'd just reduce the overall execution time.
+
+At best we can reach O(hosts) + O(daemons).
+
+*************************
+ Backwards compatibility
+*************************
+
+Any changes need to be backwards compatible or completely isolated from
+any existing functionality. There are running cephadm clusters out there
+that require an upgrade path.
diff --git a/doc/dev/cephfs-mirroring.rst b/doc/dev/cephfs-mirroring.rst
new file mode 100644
index 000000000..3ca487c03
--- /dev/null
+++ b/doc/dev/cephfs-mirroring.rst
@@ -0,0 +1,403 @@
+================
+CephFS Mirroring
+================
+
+CephFS supports asynchronous replication of snapshots to a remote CephFS file system via
+`cephfs-mirror` tool. Snapshots are synchronized by mirroring snapshot data followed by
+creating a snapshot with the same name (for a given directory on the remote file system) as
+the snapshot being synchronized.
+
+Requirements
+------------
+
+The primary (local) and secondary (remote) Ceph clusters version should be Pacific or later.
+
+Key Idea
+--------
+
+For a given snapshot pair in a directory, `cephfs-mirror` daemon will rely on readdir diff
+to identify changes in a directory tree. The diffs are applied to directory in the remote
+file system thereby only synchronizing files that have changed between two snapshots.
+
+This feature is tracked here: https://tracker.ceph.com/issues/47034.
+
+Currently, snapshot data is synchronized by bulk copying to the remote filesystem.
+
+.. note:: Synchronizing hardlinks is not supported -- hardlinked files get synchronized
+          as separate files.
+
+Creating Users
+--------------
+
+Start by creating a user (on the primary/local cluster) for the mirror daemon. This user
+requires write capability on the metadata pool to create RADOS objects (index objects)
+for watch/notify operation and read capability on the data pool(s).
+
+  $ ceph auth get-or-create client.mirror mon 'profile cephfs-mirror' mds 'allow r' osd 'allow rw tag cephfs metadata=*, allow r tag cephfs data=*' mgr 'allow r'
+
+Create a user for each file system peer (on the secondary/remote cluster). This user needs
+to have full capabilities on the MDS (to take snapshots) and the OSDs::
+
+  $ ceph fs authorize <fs_name> client.mirror_remote / rwps
+
+This user should be used (as part of peer specification) when adding a peer.
+
+Starting Mirror Daemon
+----------------------
+
+Mirror daemon should be spawned using `systemctl(1)` unit files::
+
+  $ systemctl enable cephfs-mirror@mirror
+  $ systemctl start cephfs-mirror@mirror
+
+`cephfs-mirror` daemon can be run in foreground using::
+
+  $ cephfs-mirror --id mirror --cluster site-a -f
+
+.. note:: User used here is `mirror` as created in the `Creating Users` section.
+
+Mirroring Design
+----------------
+
+CephFS supports asynchronous replication of snapshots to a remote CephFS file system
+via `cephfs-mirror` tool. For a given directory, snapshots are synchronized by transferring
+snapshot data to the remote file system and creating a snapshot with the same name as the
+snapshot being synchronized.
+
+Snapshot Synchronization Order
+------------------------------
+
+Although the order in which snapshots get chosen for synchronization does not matter,
+snapshots are picked based on creation order (using snap-id).
+
+Snapshot Incarnation
+--------------------
+
+A snapshot may be deleted and recreated (with the same name) with different contents.
+An "old" snapshot could have been synchronized (earlier) and the recreation of the
+snapshot could have been done when mirroring was disabled. Using snapshot names to
+infer the point-of-continuation would result in the "new" snapshot (incarnation)
+never getting picked up for synchronization.
+
+Snapshots on the secondary file system stores the snap-id of the snapshot it was
+synchronized from. This metadata is stored in `SnapInfo` structure on the MDS.
+
+Interfaces
+----------
+
+`Mirroring` module (manager plugin) provides interfaces for managing directory snapshot
+mirroring. Manager interfaces are (mostly) wrappers around monitor commands for managing
+file system mirroring and is the recommended control interface.
+
+Mirroring Module and Interface
+------------------------------
+
+Mirroring module provides interface for managing directory snapshot mirroring. The module
+is implemented as a Ceph Manager plugin. Mirroring module does not manage spawning (and
+terminating) the mirror daemons. Right now the preferred way would be to start/stop
+mirror daemons via `systemctl(1)`. Going forward, deploying mirror daemons would be
+managed by `cephadm` (Tracker: http://tracker.ceph.com/issues/47261).
+
+The manager module is responsible for assigning directories to mirror daemons for
+synchronization. Multiple mirror daemons can be spawned to achieve concurrency in
+directory snapshot synchronization. When mirror daemons are spawned (or terminated)
+, the mirroring module discovers the modified set of mirror daemons and rebalances
+the directory assignment amongst the new set thus providing high-availability.
+
+.. note:: Multiple mirror daemons is currently untested. Only a single mirror daemon
+          is recommended.
+
+Mirroring module is disabled by default. To enable mirroring use::
+
+  $ ceph mgr module enable mirroring
+
+Mirroring module provides a family of commands to control mirroring of directory
+snapshots. To add or remove directories, mirroring needs to be enabled for a given
+file system. To enable mirroring use::
+
+  $ ceph fs snapshot mirror enable <fs_name>
+
+.. note:: Mirroring module commands use `fs snapshot mirror` prefix as compared to
+          the monitor commands which `fs mirror` prefix. Make sure to use module
+          commands.
+
+To disable mirroring, use::
+
+  $ ceph fs snapshot mirror disable <fs_name>
+
+Once mirroring is enabled, add a peer to which directory snapshots are to be mirrored.
+Peers follow `<client>@<cluster>` specification and get assigned a unique-id (UUID)
+when added. See `Creating Users` section on how to create Ceph users for mirroring.
+
+To add a peer use::
+
+  $ ceph fs snapshot mirror peer_add <fs_name> <remote_cluster_spec> [<remote_fs_name>] [<remote_mon_host>] [<cephx_key>]
+
+`<remote_fs_name>` is optional, and default to `<fs_name>` (on the remote cluster).
+
+This requires the remote cluster ceph configuration and user keyring to be available in
+the primary cluster. See `Bootstrap Peers` section to avoid this. `peer_add` additionally
+supports passing the remote cluster monitor address and the user key. However, bootstrapping
+a peer is the recommended way to add a peer.
+
+.. note:: Only a single peer is supported right now.
+
+To remove a peer use::
+
+  $ ceph fs snapshot mirror peer_remove <fs_name> <peer_uuid>
+
+.. note:: See `Mirror Daemon Status` section on how to figure out Peer UUID.
+
+To list file system mirror peers use::
+
+  $ ceph fs snapshot mirror peer_list <fs_name>
+
+To configure a directory for mirroring, use::
+
+  $ ceph fs snapshot mirror add <fs_name> <path>
+
+To stop a mirroring directory snapshots use::
+
+  $ ceph fs snapshot mirror remove <fs_name> <path>
+
+Only absolute directory paths are allowed. Also, paths are normalized by the mirroring
+module, therfore, `/a/b/../b` is equivalent to `/a/b`.
+
+  $ mkdir -p /d0/d1/d2
+  $ ceph fs snapshot mirror add cephfs /d0/d1/d2
+  {}
+  $ ceph fs snapshot mirror add cephfs /d0/d1/../d1/d2
+  Error EEXIST: directory /d0/d1/d2 is already tracked
+
+Once a directory is added for mirroring, its subdirectory or ancestor directories are
+disallowed to be added for mirorring::
+
+  $ ceph fs snapshot mirror add cephfs /d0/d1
+  Error EINVAL: /d0/d1 is a ancestor of tracked path /d0/d1/d2
+  $ ceph fs snapshot mirror add cephfs /d0/d1/d2/d3
+  Error EINVAL: /d0/d1/d2/d3 is a subtree of tracked path /d0/d1/d2
+
+Commands to check directory mapping (to mirror daemons) and directory distribution are
+detailed in `Mirror Daemon Status` section.
+
+Bootstrap Peers
+---------------
+
+Adding a peer (via `peer_add`) requires the peer cluster configuration and user keyring
+to be available in the primary cluster (manager host and hosts running the mirror daemon).
+This can be avoided by bootstrapping and importing a peer token. Peer bootstrap involves
+creating a bootstrap token on the peer cluster via::
+
+  $ ceph fs snapshot mirror peer_bootstrap create <fs_name> <client_entity> <site-name>
+
+e.g.::
+
+  $ ceph fs snapshot mirror peer_bootstrap create backup_fs client.mirror_remote site-remote
+  {"token": "eyJmc2lkIjogIjBkZjE3MjE3LWRmY2QtNDAzMC05MDc5LTM2Nzk4NTVkNDJlZiIsICJmaWxlc3lzdGVtIjogImJhY2t1cF9mcyIsICJ1c2VyIjogImNsaWVudC5taXJyb3JfcGVlcl9ib290c3RyYXAiLCAic2l0ZV9uYW1lIjogInNpdGUtcmVtb3RlIiwgImtleSI6ICJBUUFhcDBCZ0xtRmpOeEFBVnNyZXozai9YYUV0T2UrbUJEZlJDZz09IiwgIm1vbl9ob3N0IjogIlt2MjoxOTIuMTY4LjAuNTo0MDkxOCx2MToxOTIuMTY4LjAuNTo0MDkxOV0ifQ=="}
+
+`site-name` refers to a user-defined string to identify the remote filesystem. In context
+of `peer_add` interface, `site-name` is the passed in `cluster` name from `remote_cluster_spec`.
+
+Import the bootstrap token in the primary cluster via::
+
+  $ ceph fs snapshot mirror peer_bootstrap import <fs_name> <token>
+
+e.g.::
+
+  $ ceph fs snapshot mirror peer_bootstrap import cephfs eyJmc2lkIjogIjBkZjE3MjE3LWRmY2QtNDAzMC05MDc5LTM2Nzk4NTVkNDJlZiIsICJmaWxlc3lzdGVtIjogImJhY2t1cF9mcyIsICJ1c2VyIjogImNsaWVudC5taXJyb3JfcGVlcl9ib290c3RyYXAiLCAic2l0ZV9uYW1lIjogInNpdGUtcmVtb3RlIiwgImtleSI6ICJBUUFhcDBCZ0xtRmpOeEFBVnNyZXozai9YYUV0T2UrbUJEZlJDZz09IiwgIm1vbl9ob3N0IjogIlt2MjoxOTIuMTY4LjAuNTo0MDkxOCx2MToxOTIuMTY4LjAuNTo0MDkxOV0ifQ==
+
+Mirror Daemon Status
+--------------------
+
+Mirror daemons get asynchronously notified about changes in file system mirroring status
+and/or peer updates.
+
+CephFS mirroring module provides `mirror daemon status` interface to check mirror daemon
+status::
+
+  $ ceph fs snapshot mirror daemon status
+
+E.g::
+
+  $ ceph fs snapshot mirror daemon status | jq
+  [
+    {
+      "daemon_id": 284167,
+      "filesystems": [
+        {
+          "filesystem_id": 1,
+          "name": "a",
+          "directory_count": 1,
+          "peers": [
+            {
+              "uuid": "02117353-8cd1-44db-976b-eb20609aa160",
+              "remote": {
+                "client_name": "client.mirror_remote",
+                "cluster_name": "ceph",
+                "fs_name": "backup_fs"
+              },
+              "stats": {
+                "failure_count": 1,
+                "recovery_count": 0
+              }
+            }
+          ]
+        }
+      ]
+    }
+  ]
+
+An entry per mirror daemon instance is displayed along with information such as configured
+peers and basic stats. For more detailed stats, use the admin socket interface as detailed
+below.
+
+CephFS mirror daemons provide admin socket commands for querying mirror status. To check
+available commands for mirror status use::
+
+  $ ceph --admin-daemon /path/to/mirror/daemon/admin/socket help
+  {
+      ....
+      ....
+      "fs mirror status cephfs@360": "get filesystem mirror status",
+      ....
+      ....
+  }
+
+Commands with `fs mirror status` prefix provide mirror status for mirror enabled
+file systems. Note that `cephfs@360` is of format `filesystem-name@filesystem-id`.
+This format is required since mirror daemons get asynchronously notified regarding
+file system mirror status (A file system can be deleted and recreated with the same
+name).
+
+Right now, the command provides minimal information regarding mirror status::
+
+  $ ceph --admin-daemon /var/run/ceph/cephfs-mirror.asok fs mirror status cephfs@360
+  {
+    "rados_inst": "192.168.0.5:0/1476644347",
+    "peers": {
+        "a2dc7784-e7a1-4723-b103-03ee8d8768f8": {
+            "remote": {
+                "client_name": "client.mirror_remote",
+                "cluster_name": "site-a",
+                "fs_name": "backup_fs"
+            }
+        }
+    },
+    "snap_dirs": {
+        "dir_count": 1
+    }
+  }
+
+`Peers` section in the command output above shows the peer information such as unique
+peer-id (UUID) and specification. The peer-id is required to remove an existing peer
+as mentioned in the `Mirror Module and Interface` section.
+
+Command with `fs mirror peer status` prefix provide peer synchronization status. This
+command is of format `filesystem-name@filesystem-id peer-uuid`::
+
+  $ ceph --admin-daemon /var/run/ceph/cephfs-mirror.asok fs mirror peer status cephfs@360 a2dc7784-e7a1-4723-b103-03ee8d8768f8
+  {
+    "/d0": {
+        "state": "idle",
+        "last_synced_snap": {
+            "id": 120,
+            "name": "snap1",
+            "sync_duration": 0.079997898999999997,
+            "sync_time_stamp": "274900.558797s"
+        },
+        "snaps_synced": 2,
+        "snaps_deleted": 0,
+        "snaps_renamed": 0
+    }
+  }
+
+Synchronization stats such as `snaps_synced`, `snaps_deleted` and `snaps_renamed` are reset
+on daemon restart and/or when a directory is reassigned to another mirror daemon (when
+multiple mirror daemons are deployed).
+
+A directory can be in one of the following states::
+
+  - `idle`: The directory is currently not being synchronized
+  - `syncing`: The directory is currently being synchronized
+  - `failed`: The directory has hit upper limit of consecutive failures
+
+When a directory hits a configured number of consecutive synchronization failures, the
+mirror daemon marks it as `failed`. Synchronization for these directories are retried.
+By default, the number of consecutive failures before a directory is marked as failed
+is controlled by `cephfs_mirror_max_consecutive_failures_per_directory` configuration
+option (default: 10) and the retry interval for failed directories is controlled via
+`cephfs_mirror_retry_failed_directories_interval` configuration option (default: 60s).
+
+E.g., adding a regular file for synchronization would result in failed status::
+
+  $ ceph fs snapshot mirror add cephfs /f0
+  $ ceph --admin-daemon /var/run/ceph/cephfs-mirror.asok fs mirror peer status cephfs@360 a2dc7784-e7a1-4723-b103-03ee8d8768f8
+  {
+    "/d0": {
+        "state": "idle",
+        "last_synced_snap": {
+            "id": 120,
+            "name": "snap1",
+            "sync_duration": 0.079997898999999997,
+            "sync_time_stamp": "274900.558797s"
+        },
+        "snaps_synced": 2,
+        "snaps_deleted": 0,
+        "snaps_renamed": 0
+    },
+    "/f0": {
+        "state": "failed",
+        "snaps_synced": 0,
+        "snaps_deleted": 0,
+        "snaps_renamed": 0
+    }
+  }
+
+This allows a user to add a non-existent directory for synchronization. The mirror daemon
+would mark the directory as failed and retry (less frequently). When the directory comes
+to existence, the mirror daemons would unmark the failed state upon successfull snapshot
+synchronization.
+
+When mirroring is disabled, the respective `fs mirror status` command for the file system
+will not show up in command help.
+
+Mirroring module provides a couple of commands to display directory mapping and distribution
+information. To check which mirror daemon a directory has been mapped to use::
+
+  $ ceph fs snapshot mirror dirmap cephfs /d0/d1/d2
+  {
+    "instance_id": "404148",
+    "last_shuffled": 1601284516.10986,
+    "state": "mapped"
+  }
+
+.. note:: `instance_id` is the RAODS instance-id associated with a mirror daemon.
+
+Other information such as `state` and `last_shuffled` are interesting when running
+multiple mirror daemons.
+
+When no mirror daemons are running the above command shows::
+
+  $ ceph fs snapshot mirror dirmap cephfs /d0/d1/d2
+  {
+    "reason": "no mirror daemons running",
+    "state": "stalled"
+  }
+
+Signifying that no mirror daemons are running and mirroring is stalled.
+
+Re-adding Peers
+---------------
+
+When re-adding (reassigning) a peer to a file system in another cluster, ensure that
+all mirror daemons have stopped synchronization to the peer. This can be checked
+via `fs mirror status` admin socket command (the `Peer UUID` should not show up
+in the command output). Also, it is recommended to purge synchronized directories
+from the peer  before re-adding it to another file system (especially those directories
+which might exist in the new primary file system). This is not required if re-adding
+a peer to the same primary file system it was earlier synchronized from.
+
+Feature Status
+--------------
+
+`cephfs-mirror` daemon is built by default (follows `WITH_CEPHFS` CMake rule).
diff --git a/doc/dev/cephfs-reclaim.rst b/doc/dev/cephfs-reclaim.rst
new file mode 100644
index 000000000..ce56f5eaa
--- /dev/null
+++ b/doc/dev/cephfs-reclaim.rst
@@ -0,0 +1,104 @@
+CephFS Reclaim Interface
+========================
+
+Introduction
+------------
+NFS servers typically do not track ephemeral state on stable storage. If
+the NFS server is restarted, then it will be resurrected with no
+ephemeral state, and the NFS clients are expected to send requests to
+reclaim what state they held during a grace period.
+
+In order to support this use-case, libcephfs has grown several functions
+that allow a client that has been stopped and restarted to destroy or
+reclaim state held by a previous incarnation of itself. This allows the
+client to reacquire state held by its previous incarnation, and to avoid
+the long wait for the old session to time out before releasing the state
+previously held.
+
+As soon as an NFS server running over cephfs goes down, it's racing
+against its MDS session timeout. If the Ceph session times out before
+the NFS grace period is started, then conflicting state could be
+acquired by another client. This mechanism also allows us to increase
+the timeout for these clients, to ensure that the server has a long
+window of time to be restarted.
+
+Setting the UUID
+----------------
+In order to properly reset or reclaim against the old session, we need a
+way to identify the old session. This done by setting a unique opaque
+value on the session using **ceph_set_uuid()**. The uuid value can be
+any string and is treated as opaque by the client.
+
+Setting the uuid directly can only be done on a new session, prior to
+mounting. When reclaim is performed the current session will inherit the
+old session's uuid.
+
+Starting Reclaim
+----------------
+After calling ceph_create and ceph_init on the resulting struct
+ceph_mount_info, the client should then issue ceph_start_reclaim,
+passing in the uuid of the previous incarnation of the client with any
+flags.
+
+CEPH_RECLAIM_RESET
+   This flag indicates that we do not intend to do any sort of reclaim
+   against the old session indicated by the given uuid, and that it
+   should just be discarded. Any state held by the previous client
+   should be released immediately.
+
+Finishing Reclaim
+-----------------
+After the Ceph client has completed all of its reclaim operations, the
+client should issue ceph_finish_reclaim to indicate that the reclaim is
+now complete.
+
+Setting Session Timeout (Optional)
+----------------------------------
+When a client dies and is restarted, and we need to preserve its state,
+we are effectively racing against the session expiration clock. In this
+situation we generally want a longer timeout since we expect to
+eventually kill off the old session manually.
+
+Example 1: Reset Old Session
+----------------------------
+This example just kills off the MDS session held by a previous instance
+of itself. An NFS server can start a grace period and then ask the MDS
+to tear down the old session. This allows clients to start reclaim
+immediately.
+
+(Note: error handling omitted for clarity)
+
+.. code-block:: c
+
+	struct ceph_mount_info *cmount;
+	const char *uuid = "foobarbaz";
+
+	/* Set up a new cephfs session, but don't mount it yet. */
+	rc = ceph_create(&cmount);
+	rc = ceph_init(&cmount);
+
+	/*
+	 * Set the timeout to 5 minutes to lengthen the window of time for
+	 * the server to restart, should it crash.
+	 */
+	ceph_set_session_timeout(cmount, 300);
+
+	/*
+	 * Start reclaim vs. session with old uuid. Before calling this,
+	 * all NFS servers that could acquire conflicting state _must_ be
+	 * enforcing their grace period locally.
+	 */
+	rc = ceph_start_reclaim(cmount, uuid, CEPH_RECLAIM_RESET);
+
+	/* Declare reclaim complete */
+	rc = ceph_finish_reclaim(cmount);
+
+	/* Set uuid held by new session */
+	ceph_set_uuid(cmount, nodeid);
+
+	/*
+	 * Now mount up the file system and do normal open/lock operations to
+	 * satisfy reclaim requests.
+	 */
+	ceph_mount(cmount, rootpath);
+	...
diff --git a/doc/dev/cephfs-snapshots.rst b/doc/dev/cephfs-snapshots.rst
new file mode 100644
index 000000000..423240b97
--- /dev/null
+++ b/doc/dev/cephfs-snapshots.rst
@@ -0,0 +1,133 @@
+CephFS Snapshots
+================
+
+CephFS supports snapshots, generally created by invoking mkdir within the
+``.snap`` directory. Note this is a hidden, special directory, not visible
+during a directory listing.
+
+Overview
+-----------
+
+Generally, snapshots do what they sound like: they create an immutable view
+of the file system at the point in time they're taken. There are some headline
+features that make CephFS snapshots different from what you might expect:
+
+* Arbitrary subtrees. Snapshots are created within any directory you choose,
+  and cover all data in the file system under that directory.
+* Asynchronous. If you create a snapshot, buffered data is flushed out lazily,
+  including from other clients. As a result, "creating" the snapshot is
+  very fast.
+
+Important Data Structures
+-------------------------
+* SnapRealm: A `SnapRealm` is created whenever you create a snapshot at a new
+  point in the hierarchy (or, when a snapshotted inode is move outside of its
+  parent snapshot). SnapRealms contain an `sr_t srnode`, and `inodes_with_caps`
+  that are part of the snapshot. Clients also have a SnapRealm concept that
+  maintains less data but is used to associate a `SnapContext` with each open
+  file for writing.
+* sr_t: An `sr_t` is the on-disk snapshot metadata. It is part of the containing
+  directory and contains sequence counters, timestamps, the list of associated
+  snapshot IDs, and `past_parent_snaps`.
+* SnapServer: SnapServer manages snapshot ID allocation, snapshot deletion and
+  tracks list of effective snapshots in the file system. A file system only has
+  one instance of snapserver.
+* SnapClient: SnapClient is used to communicate with snapserver, each MDS rank
+  has its own snapclient instance. SnapClient also caches effective snapshots
+  locally.
+
+Creating a snapshot
+-------------------
+CephFS snapshot feature is enabled by default on new file system. To enable it
+on existing file systems, use command below.
+
+.. code::
+
+       $ ceph fs set <fs_name> allow_new_snaps true
+
+When snapshots are enabled, all directories in CephFS will have a special
+``.snap`` directory. (You may configure a different name with the ``client
+snapdir`` setting if you wish.)
+
+To create a CephFS snapshot, create a subdirectory under
+``.snap`` with a name of your choice. For example, to create a snapshot on
+directory "/1/2/3/", invoke ``mkdir /1/2/3/.snap/my-snapshot-name`` .
+
+This is transmitted to the MDS Server as a
+CEPH_MDS_OP_MKSNAP-tagged `MClientRequest`, and initially handled in
+Server::handle_client_mksnap(). It allocates a `snapid` from the `SnapServer`,
+projects a new inode with the new SnapRealm, and commits it to the MDLog as
+usual. When committed, it invokes
+`MDCache::do_realm_invalidate_and_update_notify()`, which notifies all clients
+with caps on files under "/1/2/3/", about the new SnapRealm. When clients get
+the notifications, they update client-side SnapRealm hierarchy, link files
+under "/1/2/3/" to the new SnapRealm and generate a `SnapContext` for the
+new SnapRealm.
+
+Note that this *is not* a synchronous part of the snapshot creation!
+
+Updating a snapshot
+-------------------
+If you delete a snapshot, a similar process is followed. If you remove an inode
+out of its parent SnapRealm, the rename code creates a new SnapRealm for the
+renamed inode (if SnapRealm does not already exist), saves IDs of snapshots that
+are effective on the original parent SnapRealm into `past_parent_snaps` of the
+new SnapRealm, then follows a process similar to creating snapshot.
+
+Generating a SnapContext
+------------------------
+A RADOS `SnapContext` consists of a snapshot sequence ID (`snapid`) and all
+the snapshot IDs that an object is already part of. To generate that list, we
+combine `snapids` associated with the SnapRealm and all valid `snapids` in
+`past_parent_snaps`. Stale `snapids` are filtered out by SnapClient's cached
+effective snapshots.
+
+Storing snapshot data
+---------------------
+File data is stored in RADOS "self-managed" snapshots. Clients are careful to
+use the correct `SnapContext` when writing file data to the OSDs.
+
+Storing snapshot metadata
+-------------------------
+Snapshotted dentries (and their inodes) are stored in-line as part of the
+directory they were in at the time of the snapshot. *All dentries* include a
+`first` and `last` snapid for which they are valid. (Non-snapshotted dentries
+will have their `last` set to CEPH_NOSNAP).
+
+Snapshot writeback
+------------------
+There is a great deal of code to handle writeback efficiently. When a Client
+receives an `MClientSnap` message, it updates the local `SnapRealm`
+representation and its links to specific `Inodes`, and generates a `CapSnap`
+for the `Inode`. The `CapSnap` is flushed out as part of capability writeback,
+and if there is dirty data the `CapSnap` is used to block fresh data writes
+until the snapshot is completely flushed to the OSDs.
+
+In the MDS, we generate snapshot-representing dentries as part of the regular
+process for flushing them. Dentries with outstanding `CapSnap` data is kept
+pinned and in the journal.
+
+Deleting snapshots
+------------------
+Snapshots are deleted by invoking "rmdir" on the ".snap" directory they are
+rooted in. (Attempts to delete a directory which roots snapshots *will fail*;
+you must delete the snapshots first.) Once deleted, they are entered into the
+`OSDMap` list of deleted snapshots and the file data is removed by the OSDs.
+Metadata is cleaned up as the directory objects are read in and written back
+out again.
+
+Hard links
+----------
+Inode with multiple hard links is moved to a dummy global SnapRealm. The
+dummy SnapRealm covers all snapshots in the file system. The inode's data
+will be preserved for any new snapshot. These preserved data will cover
+snapshots on any linkage of the inode.
+
+Multi-FS
+---------
+Snapshots and multiple file systems don't interact well. Specifically, each
+MDS cluster allocates `snapids` independently; if you have multiple file systems
+sharing a single pool (via namespaces), their snapshots *will* collide and
+deleting one will result in missing file data for others. (This may even be
+invisible, not throwing errors to the user.) If each FS gets its own
+pool things probably work, but this isn't tested and may not be true.
diff --git a/doc/dev/cephx.rst b/doc/dev/cephx.rst
new file mode 100644
index 000000000..27e501f99
--- /dev/null
+++ b/doc/dev/cephx.rst
@@ -0,0 +1,406 @@
+=====
+Cephx
+=====
+
+.. _cephx:
+
+Intro
+-----
+
+The protocol design looks a lot like kerberos.  The authorizer "KDC"
+role is served by the monitor, who has a database of shared secrets
+for each entity.  Clients and non-monitor daemons all start by
+authenticating with the monitor to obtain tickets, mostly referreed to
+in the code as authorizers.  These tickets provide both
+*authentication* and *authorization* in that they include a
+description of the *capabilities* for the entity, a concise structured
+description of what actions are allowed, that can be interpreted and
+enforced by the service daemons.
+
+Other references
+----------------
+
+- A write-up from 2012 on cephx as it existed at that time by Peter
+  Reiher: :ref:`cephx_2012_peter`
+
+Terms
+-----
+
+- *monitor(s)*: central authorization authority
+- *service*: the set of all daemons of a particular type (e.g., all
+  OSDs, all MDSs)
+- *client*: an entity or principal that is accessing the service
+- *entity name*: the string identifier for a principal
+  (e.g. client.admin, osd.123)
+- *ticket*: a bit of data that cryptographically asserts identify and
+  authorization
+
+- *principal*: a client or daemon, identified by a unique entity_name,
+  that shares a secret with the monitor.
+- *principal_secret*: principal secret, a shared secret (16 bytes)
+  known by the principal and the monitor
+- *mon_secret*: monitor secret, a shared secret known by all monitors
+- *service_secret*: a rotating secret known by all members of a
+  service class (e.g., all OSDs)
+
+- *auth ticket*: a ticket proving identity to the monitors
+- *service ticket*: a ticket proving identify and authorization to a
+  service
+
+  
+Terminology
+-----------
+
+``{foo, bar}^secret`` denotes encryption by secret.
+
+
+Context
+-------
+
+The authentication messages described here are specific to the cephx
+auth implementation.  The messages are transferred by the Messenger
+protocol or by MAuth messages, depending on the version of the
+messenger protocol.  See also :ref:`msgr2-protocol`.
+
+An initial (messenger) handshake negotiates an authentication method
+to be used (cephx vs none or krb or whatever) and an assertion of what
+entity the client or daemon is attempting to authenticate as.
+
+Phase I: obtaining auth ticket
+------------------------------
+
+The cephx exchange begins with the monitor knowing who the client
+claims to be, and an initial cephx message from the monitor to the
+client/principal.::
+
+  a->p : 
+    CephxServerChallenge {
+      u64 server_challenge     # random (by server)
+    }
+
+The client responds by adding its own challenge, and calculating a
+value derived from both challenges and its shared key
+principal_secret.::
+
+  p->a :
+    CephxRequestHeader {
+      u16 CEPHX_GET_AUTH_SESSION_KEY
+    }
+    CephXAuthenticate {
+      u8 2                     # 2 means nautilus+
+      u64 client_challenge     # random (by client)
+      u64 key = {client_challenge ^ server_challenge}^principal_secret   # (roughly)
+      blob old_ticket          # old ticket, if we are reconnecting or renewing
+      u32 other_keys           # bit mask of service keys we want
+    }
+
+Prior to nautilus,::
+
+    CephXAuthenticate {
+      u8 1                     # 2 means nautilus+
+      u64 client_challenge     # random (by client)
+      u64 key = {client_challenge + server_challenge}^principal_secret   # (roughly)
+      blob old_ticket          # old ticket, if we are reconnecting or renewing
+    }
+
+The monitor looks up principal_secret in database, and verifies the
+key is correct.  If old_ticket is present, verify it is valid, and we
+can reuse the same global_id.  (Otherwise, a new global_id is assigned
+by the monitor.)::
+
+  a->p :
+    CephxReplyHeader {
+      u16 CEPHX_GET_AUTH_SESSION_KEY
+      s32 result (0)
+    }
+    u8 encoding_version = 1
+    u32 num_tickets ( = 1)
+    ticket_info           # (N = 1)
+
+plus (for Nautilus and later)::
+
+    u32 connection_secret_len      # in bytes
+    connection_secret^session_key
+    u32 other_keys_len             # bytes of other keys (encoded)
+    other_keys {
+      u8 encoding_version = 1
+      u32 num_tickets
+      service_ticket_info * N      # for each service ticket
+    }
+
+where::
+
+    ticket_info {
+      u32 service_id       # CEPH_ENTITY_TYPE_AUTH
+      u8 msg_version (1)
+      {CephXServiceTicket service_ticket}^principal_secret
+      {CephxTicketBlob ticket_blob}^existing session_key   # if we are renewing a ticket,
+      CephxTicketBlob ticket_blob                          # otherwise
+    }
+
+    service_ticket_info {
+      u32 service_id       # CEPH_ENTITY_TYPE_{OSD,MDS,MGR}
+      u8 msg_version (1)
+      {CephXServiceTicket service_ticket}^principal_secret
+      CephxTicketBlob ticket_blob
+    }
+
+    CephxServiceTicket {
+      CryptoKey session_key      # freshly generated (even if old_ticket is present)
+      utime_t expiration         # now + auth_mon_ticket_ttl
+    }
+
+    CephxTicketBlob {
+      u64 secret_id             # which service ticket encrypted this; -1 == monsecret, otherwise service's rotating key id
+      {CephXServiceTicketInfo ticket}^mon_secret
+    }
+
+    CephxServiceTicketInfo {
+      CryptoKey session_key     # same session_key as above
+      AuthTicket ticket
+    }
+
+    AuthTicket {
+      EntityName name           # client's identity, as proven by its possession of principal_secret
+      u64 global_id             # newly assigned, or from old_ticket
+      utime_t created, renew_after, expires
+      AuthCapsInfo       # what client is allowed to do
+      u32 flags = 0      # unused
+    }
+
+So: for each ticket, principal gets a part that it decrypts with its
+secret to get the session_key (CephxServiceTicket).  And the
+CephxTicketBlob is opaque (secured by the mon secret) but can be used
+later to prove who we are and what we can do (see CephxAuthorizer
+below).
+
+For Nautilus+, we also include the service tickets.
+
+The client can infer that the monitor is authentic because it can
+decrypt the service_ticket with its secret (i.e., the server has its
+secret key).
+
+
+Phase II: Obtaining service tickets (pre-nautilus)
+--------------------------------------------------
+
+Now the client needs the keys used to talk to non-monitors (osd, mds,
+mgr).::
+
+  p->a :
+    CephxRequestHeader {
+      u16 CEPHX_GET_PRINCIPAL_SESSION_KEY
+    }
+    CephxAuthorizer authorizer      
+    CephxServiceTicketRequest {
+      u32 keys    # bitmask of CEPH_ENTITY_TYPE_NAME (MGR, OSD, MDS, etc)
+    }
+
+where::
+
+    CephxAuthorizer {
+      u8 AUTH_MODE_AUTHORIZER (1)
+      u64 global_id
+      u32 service_id    # CEPH_ENTITY_TYPE_*
+      CephxTicketBlob auth_ticket
+      {CephxAuthorize msg}^session_key
+    }
+
+    CephxAuthorize msg {
+      u8 2
+      u64 nonce                         # random from client
+      bool have_challenge = false       # not used here
+      u64 server_challenge_plus_one = 0 # not used here
+    }
+
+The monitor validates the authorizer by decrypting the auth_ticket
+with ``mon_secret`` and confirming that it says this principal is who
+they say they are in the CephxAuthorizer fields.  Note that the nonce
+random bytes aren't used here (the field exists for Phase III below).
+
+Assuming all is well, the authorizer can generate service tickets
+based on the CEPH_ENTITY_TYPE_* bits in the ``keys`` bitmask.
+
+The response looks like::
+
+    CephxResponseHeader {
+      u16 CEPHX_GET_PRINCIPAL_SESSION_KEY
+      s32 result (= 0)
+    }
+    u8 encoding_version = 1
+    u32 num_tickets
+    ticket_info * N
+  
+Where, as above,::
+
+    ticket_info {
+      u32 service_id      # CEPH_ENTITY_TYPE_{OSD,MGR,MDS}
+      u8 msg_version (1)
+      {CephXServiceTicket service_ticket}^principal_secret
+      CephxTicketBlob ticket_blob
+    }
+
+    CephxServiceTicket {
+      CryptoKey session_key
+      utime_t expiration
+    }
+
+    CephxTicketBlob {
+      u64 secret_id       # which version of the (rotating) service ticket encrypted this
+      {CephXServiceTicketInfo ticket}^rotating_service_secret
+    }
+
+    CephxServiceTicketInfo {
+      CryptoKey session_key
+      AuthTicket ticket
+    }
+
+    AuthTicket {
+      EntityName name
+      u64 global_id
+      utime_t created, renew_after, expires
+      AuthCapsInfo       # what you are allowed to do
+      u32 flags = 0      # unused
+    }
+
+This concludes the authentication exchange with the monitor.  The
+client or daemon now has tickets to talk to the mon and all other
+daemons of interest.
+
+
+Phase III: Opening a connection to a service
+--------------------------------------------
+
+When a connection is opened, an "authorizer" payload is sent::
+
+  p->s :
+    CephxAuthorizer {
+      u8 AUTH_MODE_AUTHORIZER (1)
+      u64 global_id
+      u32 service_id    # CEPH_ENTITY_TYPE_*
+      CephxTicketBlob auth_ticket
+      {CephxAuthorize msg}^session_key
+    }
+
+    CephxAuthorize msg {
+      u8 2
+      u64 nonce               # random from client
+      bool have_challenge = false
+      u64 server_challenge_plus_one = 0
+    }
+
+Note that prior to the Luminous v12.2.6 or Mimic v13.2.2 releases, the
+CephxAuthorize msg did not contain a challenge, and consisted only
+of::
+
+    CephxAuthorize msg {
+      u8 1
+      u64 nonce               # random from client
+    }
+    
+The server will inspect the auth_ticket CephxTicketBlob (by decrypting
+it with its current rotating service key).  If it is a pre-v12.2.6 or
+pre-v13.2.2 client, the server immediately replies with::
+
+  s->p :
+    {CephxAuthorizeReply reply}^session_key
+
+where::
+
+    CephxAuthorizeReply {
+      u64 nonce_plus_one
+    }
+
+Otherwise, the server will respond with a challenge (to prevent replay
+attacks)::
+
+  s->p :
+    {CephxAuthorizeChallenge challenge}^session_key
+
+where::
+
+    CephxAuthorizeChallenge {
+      u64 server_challenge        # random from server
+    }
+
+The client decrypts and updates its CephxAuthorize msg accordingly,
+resending most of the same information as before::
+
+  p->s :
+    CephxAuthorizer {
+      u8 AUTH_MODE_AUTHORIZER (1)
+      u64 global_id
+      u32 service_id    # CEPH_ENTITY_TYPE_*
+      CephxTicketBlob auth_ticket
+      {CephxAuthorize msg}^session_key
+    }
+
+where::
+
+    CephxAuthorize msg {
+      u8 2
+      u64 nonce                        # (new) random from client
+      bool have_challenge = true
+      u64 server_challenge_plus_one    # server_challenge + 1
+    }
+
+The server validates the ticket as before, and then also verifies the
+msg nonce has it's challenge + 1, confirming this is a live
+authentication attempt (not a replay).
+
+Finally, the server responds with a reply that proves its authenticity
+to the client.  It also includes some entropy to use for encryption of
+the session, if it is needed for the mode.::
+
+  s->p :
+    {CephxAuthorizeReply reply}^session_key
+
+where::
+
+    CephxAuthorizeReply {
+      u64 nonce_plus_one
+      u32 connection_secret_length
+      connection secret
+    }
+
+Prior to nautilus, there is no connection secret::
+
+    CephxAuthorizeReply {
+      u64 nonce_plus_one
+    }
+
+The client decrypts and confirms that the server incremented nonce
+properly and that this is thus a live authentication request and not a
+replay.
+
+
+Rotating service secrets
+------------------------
+
+Daemons make use of a rotating secret for their tickets instead of a
+fixed secret in order to limit the severity of a compromised daemon.
+If a daemon's secret key is compromised by an attacker, that daemon
+and its key can be removed from the monitor's database, but the
+attacker may also have obtained a copy of the service secret shared by
+all daemons.  To mitigate this, service keys rotate periodically so
+that after a period of time (auth_service_ticket_ttl) the key the
+attacker obtained will no longer be valid.::
+
+  p->a :
+    CephxRequestHeader {
+      u16 CEPHX_GET_ROTATING_KEY
+    }
+
+  a->p :
+    CephxReplyHeader {
+      u16 CEPHX_GET_ROTATING_KEY
+      s32 result = 0
+    }
+    {CryptoKey service_key}^principal_secret
+
+That is, the new rotating key is simply protected by the daemon's
+rotating secret.
+
+Note that, as an implementation detail, the services keep the current
+key and the prior key on hand so that the can continue to validate
+requests while the key is being rotated.
diff --git a/doc/dev/cephx_protocol.rst b/doc/dev/cephx_protocol.rst
new file mode 100644
index 000000000..7b8c17876
--- /dev/null
+++ b/doc/dev/cephx_protocol.rst
@@ -0,0 +1,341 @@
+.. _cephx_2012_peter:
+
+============================================================
+A Detailed Description of the Cephx Authentication Protocol
+============================================================
+
+Peter Reiher
+7/13/12
+
+This document provides deeper detail on the Cephx authorization protocol whose high level flow 
+is described in the memo by Yehuda (12/19/09).  Because this memo discusses details of 
+routines called and variables used, it represents a snapshot.  The code might be changed 
+subsequent to the creation of this document, and the document is not likely to be updated in
+lockstep.  With luck, code comments will indicate major changes in the way the protocol is
+implemented.
+
+Introduction
+-------------
+
+The basic idea of the protocol is based on Kerberos.  A client wishes to obtain something from 
+a server.  The server will only offer the requested service to authorized clients.  Rather 
+than requiring each server to deal with authentication and authorization issues, the system 
+uses an authorization server.  Thus, the client must first communicate with the authorization 
+server to authenticate itself and to obtain credentials that will grant it access to the
+service it wants.
+
+Authorization is not the same as authentication.  Authentication provides evidence that some 
+party is who it claims to be.  Authorization provides evidence that a particular party is
+allowed to do something.  Generally, secure authorization implies secure authentication 
+(since without authentication, you may authorize something for an imposter), but the reverse 
+is not necessarily true.  One can authenticate without authorizing.  The purpose 
+of this protocol is to authorize.
+
+The basic approach is to use symmetric cryptography throughout.  Each client C has its own
+secret key, known only to itself and the authorization server A.  Each server S has its own
+secret key, known only to itself and the authorization server A.  Authorization information 
+will be passed in tickets, encrypted with the secret key of the entity that offers the service.
+There will be a ticket that A gives to C, which permits C to ask A for other tickets.  This 
+ticket will be encrypted with A's key, since A is the one who needs to check it.  There will 
+later be tickets that A issues that allow C to communicate with S to ask for service.  These 
+tickets will be encrypted with S's key, since S needs to check them.   Since we wish to provide 
+security of the communications, as well, session keys are set up along with the tickets.  
+Currently, those session keys are only used for authentication purposes during this protocol 
+and the handshake between the client C and the server S, when the client provides its service 
+ticket.  They could be used for authentication or secrecy throughout, with some changes to 
+the system.
+
+Several parties need to prove something to each other if this protocol is to achieve its 
+desired security effects.
+
+1.  The client C must prove to the authenticator A that it really is C.  Since everything
+is being done via messages, the client must also prove that the message proving authenticity
+is fresh, and is not being replayed by an attacker.
+
+2.  The authenticator A must prove to client C that it really is the authenticator.  Again,
+proof that replay is not occurring is also required.
+
+3.  A and C must securely share a session key to be used for distribution of later
+authorization material between them.  Again, no replay is allowable, and the key must be
+known only to A and C.
+
+4.  A must receive evidence from C that allows A to look up C's authorized operations with
+server S.  
+
+5.  C must receive a ticket from A that will prove to S that C can perform its authorized
+operations.   This ticket must be usable only by C.
+
+6.  C must receive from A a session key to protect the communications between C and S.  The
+session key must be fresh and not the result of a replay.
+
+Getting Started With Authorization
+-----------------------------------
+
+When the client first needs to get service, it contacts the monitor.  At the moment, it has 
+no tickets.  Therefore, it uses the "unknown" protocol to talk to the monitor.  This protocol 
+is specified as ``CEPH_AUTH_UNKNOWN``.  The monitor also takes on the authentication server 
+role, A.  The remainder of the communications will use the cephx protocol (most of whose code 
+will be found in files in ``auth/cephx``).  This protocol is responsible for creating and 
+communicating the tickets spoken of above.  
+
+Currently, this document does not follow the pre-cephx protocol flow.  It starts up at the 
+point where the client has contacted the server and is ready to start the cephx protocol itself.
+
+Once we are in the cephx protocol, we can get the tickets.  First, C needs a ticket that 
+allows secure communications with A.  This ticket can then be used to obtain other tickets. 
+This is phase I of the protocol, and consists of a send from C to A and a response from A to C.
+Then, C needs a ticket to allow it to talk to S to get services.  This is phase II of the 
+protocol, and consists of a send from C to A and a response from A to C.
+
+Phase I:
+--------
+
+The client is set up to know that it needs certain things, using a variable called ``need``, 
+which is part of the ``AuthClientHandler`` class, which the ``CephxClientHandler`` inherits 
+from.  At this point, one thing that's encoded in the ``need`` variable is 
+``CEPH_ENTITY_TYPE_AUTH``, indicating that we need to start the authentication protocol 
+from scratch.  Since we're always talking to the same authorization server, if we've gone 
+through this step of the protocol before (and the resulting ticket/session hasn't timed out), 
+we can skip this step and just ask for client tickets.  But it must be done initially, and 
+we'll assume that we are in that state.
+
+The message C sends to A in phase I is build in ``CephxClientHandler::build_request()`` (in 
+``auth/cephx/CephxClientHandler.cc``).  This routine is used for more than one purpose.  
+In this case, we first call ``validate_tickets()`` (from routine 
+``CephXTicektManager::validate_tickets()`` which lives in ``auth/cephx/CephxProtocol.h``).  
+This code runs through the list of possible tickets to determine what we need, setting values 
+in the ``need`` flag as necessary.  Then we call ``ticket.get_handler()``.  This routine 
+(in ``CephxProtocol.h``) finds a ticket of the specified type (a ticket to perform 
+authorization) in the ticket map, creates a ticket handler object for it,  and puts the 
+handler into the right place in the map.  Then we hit specialized code to deal with individual 
+cases.  The case here is when we still need to authenticate to A (the 
+``if (need & CEPH_ENTITY_TYPE_AUTH)`` branch).
+
+We now create a message of type ``CEPHX_GET_AUTH_SESSION_KEY``.  We need to authenticate
+this message with C's secret key, so we fetch that from the local key repository.  We create
+a random challenge, whose purpose is to prevent replays.  We encrypt that challenge using
+``cephx_calc_client_server_challenge()``.  We already
+have a server challenge (a similar set of random bytes, but created by the server and sent to
+the client) from our pre-cephx stage.  We take both challenges and our secret key and 
+produce a combined encrypted challenge value, which goes into ``req.key``.
+
+If we have an old ticket, we store it in ``req.old_ticket``.  We're about to get a new one.
+
+The entire ``req`` structure, including the old ticket and the cryptographic hash of the two 
+challenges, gets put into the message.  Then we return from this function, and the 
+message is sent.
+
+We now switch over to the authenticator side, A.  The server receives the message that was 
+sent, of type ``CEPH_GET_AUTH_SESSION_KEY``.  The message gets handled in ``prep_auth()``,
+in ``mon/AuthMonitor.cc``, which calls ``handle_request()`` is ``CephxServiceHandler.cc`` to 
+do most of the work.  This routine, also, handles multiple cases.  
+
+The control flow is determined by the ``request_type`` in the ``cephx_header`` associated 
+with the message.  Our case here is ``CEPH_GET_AUTH_SESSION_KEY``.  We need the
+secret key A shares with C, so we call ``get_secret()`` from out local key repository to get 
+it. (It's called a ``key_server`` in the code, but it's not really a separate machine or
+processing entity. It's more like the place where locally used keys are kept.)  We should
+have set up a server challenge already with this client, so we make sure
+we really do have one.  (This variable is specific to a ``CephxServiceHandler``, so there 
+is a different one for each such structure we create, presumably one per client A is 
+dealing with.)  If there is no challenge, we'll need to start over, since we need to 
+check the client's crypto hash, which depends on a server challenge, in part.
+
+We now call the same routine the client used to calculate the hash, based on the same values: 
+the client challenge (which is in the incoming message), the server challenge (which we saved), 
+and the client's key (which we just obtained).  We check to see if the client sent the same 
+thing we expected.  If so, we know we're talking to the right client.  We know the session is 
+fresh, because it used the challenge we sent it to calculate its crypto hash.  So we can
+give it an authentication ticket.
+
+We fetch C's ``eauth`` structure.  This contains an ID, a key, and a set of caps (capabilities).
+
+The client sent us its old ticket in the message, if it had one.  If
+so, we set a flag, ``should_enc_ticket``, to true and set the global
+ID to the global ID in that old ticket.  If the attempt to decode its
+old ticket fails (most probably because it didn't have one),
+``should_enc_ticket`` remains false.  Now we set up the new ticket,
+filling in timestamps, the name of C, and the global ID provided in the
+method call (unless there was an old ticket).  We need a new session
+key to help the client communicate securely with us, not using its
+permanent key.  We set the service ID to ``CEPH_ENTITY_TYPE_AUTH``,
+which will tell the client C what to do with the message we send it.
+We build a cephx response header and call
+``cephx_build_service_ticket_reply()``.
+
+``cephx_build_service_ticket_reply()`` is in ``auth/cephx/CephxProtocol.cc``.  This 
+routine will build up the response message.   Much of it copies data from its parameters to 
+a message structure.  Part of that information (the session key and the validity period) 
+gets encrypted with C's permanent key.  If the ``should_encrypt_ticket`` flag is set, 
+encrypt it using the old ticket's key.  Otherwise, there was no old ticket key, so the 
+new ticket is not encrypted.  (It is, of course, already encrypted with A's permanent key.)  
+Presumably the point of this second encryption is to expose less material encrypted with 
+permanent keys.
+
+Then we call the key server's ``get_service_caps()`` routine on the entity name, with a 
+flag ``CEPH_ENTITY_TYPE_MON``, and capabilities, which will be filled in by this routine.  
+The use of that constant flag means we're going to get the client's caps for A, not for some 
+other data server.  The ticket here is to access the authorizer A, not the service S.  The 
+result of this call is that the caps variable  (a parameter to the routine we're in) is 
+filled in with the monitor capabilities that will allow C to  access A's authorization services.
+
+``handle_request()`` itself does not send the response message.  It builds up the 
+``result_bl``, which basically holds that message's contents, and the capabilities structure, 
+but it doesn't send the message.  We go back to ``prep_auth()``, in ``mon/AuthMonitor.cc``, 
+for that.    This routine does some fiddling around with the caps structure that just got 
+filled in.  There's a global ID that comes up as a result of this fiddling that is put into 
+the reply message.  The reply message is built here (mostly from the ``response_bl`` buffer) 
+and sent off.
+
+This completes Phase I of the protocol.  At this point, C has authenticated itself to A, and A has generated a new session key and ticket allowing C to obtain server tickets from A.
+
+Phase II
+--------
+
+This phase starts when C receives the message from A containing a new ticket and session key.
+The goal of this phase is to provide C with a session key and ticket allowing it to
+communicate with S.
+
+The message A sent to C is dispatched to ``build_request()`` in ``CephxClientHandler.cc``, 
+the same routine that was used early in Phase I to build the first message in the protocol.  
+This time, when ``validate_tickets()`` is called, the ``need`` variable will not contain 
+``CEPH_ENTITY_TYPE_AUTH``, so a different branch through the bulk of the routine will be 
+used.  This is the branch indicated by ``if (need)``.  We have a ticket for the authorizer, 
+but we still need service tickets.
+
+We must send another message to A to obtain the tickets (and session key) for the server 
+S.  We set the ``request_type`` of the message to ``CEPHX_GET_PRINCIPAL_SESSION_KEY`` and 
+call ``ticket_handler.build_authorizer()`` to obtain an authorizer.  This routine is in 
+``CephxProtocol.cc``.  We set the key for this authorizer to be the session key we just got 
+from A,and create a new nonce.  We put the global ID, the service ID, and the ticket into a 
+message buffer that is part of the authorizer.  Then we create a new ``CephXAuthorize`` 
+structure.  The nonce we just created goes there.  We encrypt this ``CephXAuthorize`` 
+structure with the current session key and stuff it into the authorizer's buffer.  We 
+return the authorizer.
+
+Back in ``build_request()``, we take the part of the authorizer that was just built (its 
+buffer, not the session key or anything else) and shove it into the buffer we're creating 
+for the message that will go to A.  Then we delete the authorizer.  We put the requirements 
+for what we want in ``req.keys``, and we put ``req`` into the buffer.  Then we return, and 
+the message gets sent.
+
+The authorizer A receives this message which is of type ``CEPHX_GET_PRINCIPAL_SESSION_KEY``.
+The message gets handled in ``prep_auth()``, in ``mon/AuthMonitor.cc``, which again calls 
+``handle_request()`` in ``CephxServiceHandler.cc`` to do most of the work.  
+
+In this case, ``handle_request()`` will take the ``CEPHX_GET_PRINCIPAL_SESSION_KEY`` case. 
+It will call ``cephx_verify_authorizer()`` in ``CephxProtocol.cc``.  Here, we will grab 
+a bunch of data out of the input buffer, including the global and service IDs and the ticket 
+for A.   The ticket contains a ``secret_id``, indicating which key is being used for it.     
+If the secret ID pulled out of the ticket was -1, the ticket does not specify which secret 
+key A should use.  In this case, A should use the key for the specific entity that C wants
+to contact, rather than a rotating key shared by all server entities of the same type.
+To get that key, A must consult the key repository to find the right key.   Otherwise, 
+there's already a structure obtained from the key repository to hold the necessary secret.  
+Server secrets rotate on a time expiration basis (key rotation is not covered in this
+document), so run through that structure to find its current secret.  Either way, A now 
+knows the secret key used to create this ticket.  Now decrypt the encrypted part of the 
+ticket, using this key.  It should be a ticket for A.  
+
+The ticket also contains a session key that C should have used to encrypt other parts of 
+this message.  Use that session key to decrypt the rest of the message.  
+
+Create a ``CephXAuthorizeReply`` to hold our reply.  Extract the nonce (which was in the stuff 
+we just decrypted), add 1 to it, and put the result in the reply.  Encrypt the reply and 
+put it in the buffer provided in the call to ``cephx_verify_authorizer()`` and return 
+to ``handle_request()``.  This will be used to prove to C that A (rather than an attacker) 
+created this response.
+
+Having verified that the message is valid and from C, now we need to build it a ticket for S.
+We need to know what S it wants to communicate with and what services it wants.  Pull the
+ticket request that describes those things out of its message.  Now run through the ticket
+request to see what it wanted.  (He could potentially be asking for multiple different
+services in the same request, but we will assume it's just one, for this discussion.)  Once we 
+know which service ID it's after, call ``build_session_auth_info()``.
+
+``build_session_auth_info()`` is in ``CephxKeyServer.cc``.  It checks to see if the 
+secret for the ``service_ID`` of S is available and puts it into the subfield of one of 
+the parameters, and calls the similarly named ``_build_session_auth_info()``, located in 
+the same file.      This routine loads up the new ``auth_info`` structure with the 
+ID of S, a ticket, and some timestamps for that ticket.  It generates a new session key 
+and puts it in the structure.   It then calls ``get_caps()`` to fill in the 
+``info.ticket`` caps field.  ``get_caps()`` is also in ``CephxKeyServer.cc``.  It fills the 
+``caps_info`` structure it is provided with caps for S allowed to C.
+
+Once ``build_session_auth_info()`` returns, A has a list of the capabilities allowed to 
+C for S.  We put a validity period based on the current TTL for this context into the info 
+structure, and put it into the ``info_vec`` structure we are preparing in response to the 
+message.  
+
+Now call ``build_cephx_response_header()``, also in ``CephxServiceHandler.cc``.   Fill in 
+the ``request_type``, which is ``CEPHX_GET_PRINCIPAL_SESSION_KEY``, a status of 0, 
+and the result buffer.  
+
+Now call ``cephx_build_service_ticket_reply()``, which is in ``CephxProtocol.cc``.  The 
+same routine was used towards the end of A's handling of its response in phase I.  Here, 
+the session key (now a session key to talk to S, not A) and the validity period for that 
+key will be encrypted with the existing session key shared between C and A.  
+The ``should_encrypt_ticket`` parameter is false here, and no key is provided for that 
+encryption.  The ticket in question, destined for S once C sends it there, is already 
+encrypted with S's secret.  So, essentially, this routine will put ID information, 
+the encrypted session key, and the ticket allowing C to talk to S into the buffer to 
+be sent to C.
+
+After this routine returns, we exit from ``handle_request()``, going back to ``prep_auth()`` 
+and ultimately to the underlying message send code.  
+
+The client receives this message. The nonce is checked as the message passes through
+``Pipe::connect()``, which is in ``msg/SimpleMessager.cc``.  In a lengthy ``while(1)`` loop in
+the middle of this routine, it gets an authorizer.  If the get was successful, eventually
+it will call ``verify_reply()``, which checks the nonce.  ``connect()`` never explicitly
+checks to see if it got an authorizer, which would suggest that failure to provide an
+authorizer would allow an attacker to skip checking of the nonce.  However, in many places,
+if there is no authorizer, important connection fields will get set to zero, which will
+ultimately cause the connection to fail to provide data.  It would be worth testing, but
+it looks like failure to provide an authorizer, which contains the nonce, would not be helpful
+to an attacker.
+
+The message eventually makes its way through to ``handle_response()``, in 
+``CephxClientHandler.cc``.    In this routine, we call ``get_handler()`` to get a ticket 
+handler to hold the ticket we have just received.  This routine is embedded in the definition 
+for a ``CephXTicketManager`` structure.  It takes a type (``CEPH_ENTITY_TYPE_AUTH``, in 
+this case) and looks through the ``tickets_map`` to find that type.  There should be one, and 
+it should have the session key of the session between C and A in its entry.  This key will 
+be used to decrypt the information provided by A, particularly the new session key allowing 
+C to talk to S.
+
+We then call ``verify_service_ticket_reply()``, in ``CephxProtocol.cc``.  This routine 
+needs to determine if the ticket is OK and also obtain the session key associated with this 
+ticket.  It decrypts the encrypted portion of the message buffer, using the session key 
+shared with A.  This ticket was not encrypted (well, not twice - tickets are always encrypted, 
+but sometimes double encrypted, which this one isn't).  So it can be stored in a service 
+ticket buffer directly.  We now grab the ticket out of that buffer.  
+
+The stuff we decrypted with the session key shared between C and A included the new session 
+key.  That's our current session key for this ticket, so set it.  Check validity and 
+set the expiration times.  Now return true, if we got this far.  
+
+Back in ``handle_response()``, we now call ``validate_tickets()`` to adjust what we think 
+we need, since we now have a ticket we didn't have before.  If we've taken care of 
+everything we need, we'll return 0.
+
+This ends phase II of the protocol.  We have now successfully set up a ticket and session key 
+for client C to talk to server S.  S will know that C is who it claims to be, since A will
+verify it.  C will know it is S it's talking to, again because A verified it.  The only
+copies of the session key for C and S to communicate were sent encrypted under the permanent
+keys of C and S, respectively, so no other party (excepting A, who is trusted by all) knows
+that session key.  The ticket will securely indicate to S what C is allowed to do, attested 
+to by A.  The nonces passed back and forth between A and C ensure that they have not been 
+subject to a replay attack.  C has not yet actually talked to S, but it is ready to.
+
+Much of the security here falls apart if one of the permanent keys is compromised.  Compromise
+of C's key means that the attacker can pose as C and obtain all of C's privileges, and can
+eavesdrop on C's legitimate conversations.  He can also pretend to be A, but only in 
+conversations with C.  Since it does not (by hypothesis) have keys for any services, he
+cannot generate any new tickets for services, though it can replay old tickets and session
+keys until S's permanent key is changed or the old tickets time out. 
+
+Compromise of S's key means that the attacker can pose as S to anyone, and can eavesdrop on 
+any user's conversation with S.  Unless some client's key is also compromised, the attacker
+cannot generate new fake client tickets for S, since doing so requires it to authenticate
+himself as A, using the client key it doesn't know.
diff --git a/doc/dev/config-key.rst b/doc/dev/config-key.rst
new file mode 100644
index 000000000..d7b79db2f
--- /dev/null
+++ b/doc/dev/config-key.rst
@@ -0,0 +1,68 @@
+===================
+ config-key layout
+===================
+
+*config-key* is a general-purpose key/value storage service offered by
+the mons.  Generally speaking, you can put whatever you want there.
+Current in-tree users should be captured here with their key layout
+schema.
+
+OSD dm-crypt keys
+=================
+
+Key::
+
+  dm-crypt/osd/$OSD_UUID/luks = <json string>
+
+The JSON payload has the form::
+
+  { "dm-crypt": <secret> }
+
+where the secret is a base64 encoded LUKS key.
+
+Created by the 'osd new' command (see OSDMonitor.cc).
+
+Consumed by ceph-volume, and similar tools. Normally access to the
+dm-crypt/osd/$OSD_UUID prefix is allowed by a client.osd-lockbox.$OSD_UUID
+cephx key, such that only the appropriate host can retrieve the LUKS key (which
+in turn decrypts the actual raw key, also stored on the device itself).
+
+
+ceph-mgr modules
+================
+
+The convention for keys is::
+
+  mgr/$MODULE/$option = $value
+
+or::
+
+  mgr/$MODULE/$MGRID/$option = $value
+
+For example,::
+
+  mgr/dashboard/server_port = 80
+  mgr/dashboard/foo/server_addr = 1.2.3.4
+  mgr/dashboard/bar/server_addr = 1.2.3.5
+
+
+Configuration
+=============
+
+Configuration options for clients and daemons are also stored in config-key.
+
+Keys take the form::
+
+  config/$option = $value
+  config/$type/$option = $value
+  config/$type.$id/$option = $value
+  config/$type.$id/$mask[/$mask2...]/$option = $value
+
+Where
+
+* `type` is a daemon type (`osd`, `mon`, `mds`, `mgr`, `client`)
+* `id` is a daemon id (e.g., `0`, `foo`), such that `$type.$id` is something like `osd.123` or `mds.foo`)
+* `mask` restricts who the option applies to, and can take two forms:
+
+  #. `$crush_type:$crush_value`.  For example, `rack:foorack`
+  #. `class:$classname`, in reference to CRUSH device classes (e.g., `ssd`)
diff --git a/doc/dev/config.rst b/doc/dev/config.rst
new file mode 100644
index 000000000..5b620b2fe
--- /dev/null
+++ b/doc/dev/config.rst
@@ -0,0 +1,166 @@
+=================================
+ Configuration Management System
+=================================
+
+The configuration management system exists to provide every daemon with the
+proper configuration information. The configuration can be viewed as a set of
+key-value pairs.
+
+How can the configuration be set? Well, there are several sources:
+ - the ceph configuration file, usually named ceph.conf
+ - command line arguments::
+    --debug-ms=1
+    --debug-pg=10
+    etc.
+ - arguments injected at runtime using "injectargs" or "config set"
+
+
+The Configuration File
+======================
+
+Most configuration settings originate in the Ceph configuration file.
+
+How do we find the configuration file? Well, in order, we check:
+ - the default locations
+ - the environment variable CEPH_CONF
+ - the command line argument -c
+
+Each stanza of the configuration file describes the key-value pairs that will be in
+effect for a particular subset of the daemons. The "global" stanza applies to
+everything. The "mon", "osd", and "mds" stanzas specify settings to take effect
+for all monitors, all OSDs, and all mds servers, respectively.  A stanza of the
+form mon.$name, osd.$name, or mds.$name gives settings for the monitor, OSD, or
+MDS of that name, respectively. Configuration values that appear later in the
+file win over earlier ones.
+
+A sample configuration file can be found in src/sample.ceph.conf.
+
+
+Metavariables
+=============
+
+The configuration system allows any configuration value to be
+substituted into another value using the ``$varname`` syntax, similar
+to how bash shell expansion works.
+
+A few additional special metavariables are also defined:
+ - $host: expands to the current hostname
+ - $type: expands to one of "mds", "osd", "mon", or "client"
+ - $id: expands to the daemon identifier. For ``osd.0``, this would be ``0``; for ``mds.a``, it would be ``a``; for ``client.admin``, it would be ``admin``.
+ - $num: same as $id
+ - $name: expands to $type.$id
+
+
+Reading configuration values
+====================================================
+
+There are two ways for Ceph code to get configuration values. One way is to
+read it directly from a variable named "g_conf," or equivalently,
+"g_ceph_ctx->_conf." The other is to register an observer that will be called
+every time the relevant configuration values changes. This observer will be
+called soon after the initial configuration is read, and every time after that
+when one of the relevant values changes. Each observer tracks a set of keys
+and is invoked only when one of the relevant keys changes.
+
+The interface to implement is found in common/config_obs.h.
+
+The observer method should be preferred in new code because
+ - It is more flexible, allowing the code to do whatever reinitialization needs
+   to be done to implement the new configuration value.
+ - It is the only way to create a std::string configuration variable that can
+   be changed by injectargs.
+ - Even for int-valued configuration options, changing the values in one thread
+   while another thread is reading them can lead to subtle and
+   impossible-to-diagnose bugs.
+
+For these reasons, reading directly from g_conf should be considered deprecated
+and not done in new code.  Do not ever alter g_conf.
+
+Changing configuration values
+====================================================
+
+Configuration values can be changed by calling ``g_conf()->set_val``. After changing
+the configuration, you should call ``g_conf()->apply_changes`` to re-run all the
+affected configuration observers. For convenience, you can call
+``g_conf()->set_val_or_die`` to make a configuration change which you think should
+never fail.
+
+Injectargs, parse_argv, and parse_env are three other functions which modify
+the configuration. Just like with set_val, you should call apply_changes after
+calling these functions to make sure your changes get applied.
+
+
+Defining config options
+=======================
+
+New-style config options are defined in common/options.cc. All new config
+options should go here (and not into legacy_config_opts.h).
+
+Levels
+------
+
+The Option constructor takes a "level" value:
+
+* *LEVEL_BASIC* is for basic config options that a normal operator is likely to adjust.
+* *LEVEL_ADVANCED* is for options that an operator *can* adjust, but should not touch unless they understand what they are doing. Adjusting advanced options poorly can lead to problems (performance or even data loss) if done incorrectly.
+* *LEVEL_DEV* is for options in place for use by developers only, either for testing purposes, or to describe constants that no user should adjust but we prefer not to compile into the code.
+
+Description and long description
+--------------------------------
+
+Short description of the option. Sentence fragment. e.g.::
+
+  .set_description("Default checksum algorithm to use")
+
+The long description is complete sentences, perhaps even multiple
+paragraphs, and may include other detailed information or notes.::
+
+  .set_long_description("crc32c, xxhash32, and xxhash64 are available.  The _16 and _8 variants use only a subset of the bits for more compact (but less reliable) checksumming.")
+
+Default values
+--------------
+
+There is a default value for every config option. In some cases, there may
+also be a *daemon default* that only applies to code that declares itself
+as a daemon (in this case, the regular default only applies to non-daemons).
+
+Safety
+------
+
+If an option can be safely changed at runtime::
+
+  .set_safe()
+
+Service
+-------
+
+Service is a component name, like "common", "osd", "rgw", "mds", etc. It may
+be a list of components, like::
+
+  .add_service("mon mds osd mgr")
+
+For example, the rocksdb options affect both the osd and mon.
+
+Tags
+----
+
+Tags identify options across services that relate in some way. Example include;
+
+  - network -- options affecting network configuration
+  - mkfs -- options that only matter at mkfs time
+
+Enums
+-----
+
+For options with a defined set of allowed values::
+
+  .set_enum_allowed({"none", "crc32c", "crc32c_16", "crc32c_8", "xxhash32", "xxhash64"})
+
+Flags
+-----
+
+* **RUNTIME**: the value can be updated at runtime
+* **NO_MON_UPDATE**: Daemons/clients do not pull this value from the monitor config database.  We disallow setting this option via 'ceph config set ...'.  This option should be configured via ceph.conf or via the command line.
+* **STARTUP**: option takes effect only during daemon startup
+* **CLUSTER_CREATE**: option only affects cluster creation
+* **CREATE**: option only affects daemon creation
diff --git a/doc/dev/context.rst b/doc/dev/context.rst
new file mode 100644
index 000000000..1a2b2cbfb
--- /dev/null
+++ b/doc/dev/context.rst
@@ -0,0 +1,20 @@
+=============
+ CephContext
+=============
+
+A CephContext represents a single view of the Ceph cluster. It comes complete
+with a configuration, a set of performance counters (PerfCounters), and a
+heartbeat map. You can find more information about CephContext in
+src/common/ceph_context.h.
+
+Generally, you will have only one CephContext in your application, called
+g_ceph_context. However, in library code, it is possible that the library user
+will initialize multiple CephContexts. For example, this would happen if he
+called rados_create more than once.
+
+A ceph context is required to issue log messages. Why is this? Well, without
+the CephContext, we would not know which log messages were disabled and which
+were enabled.  The dout() macro implicitly references g_ceph_context, so it
+can't be used in library code.  It is fine to use dout and derr in daemons, but
+in library code, you must use ldout and lderr, and pass in your own CephContext
+object. The compiler will enforce this restriction.
diff --git a/doc/dev/continuous-integration.rst b/doc/dev/continuous-integration.rst
new file mode 100644
index 000000000..5c2f15823
--- /dev/null
+++ b/doc/dev/continuous-integration.rst
@@ -0,0 +1,285 @@
+Continuous Integration Architecture
+===================================
+
+In Ceph, we rely on multiple CI pipelines in our development. Most of these pipelines
+are centered around Jenkins. And their configurations are generated using `Jenkins Job Builder`_.
+
+.. _Jenkins Job Builder: https://docs.openstack.org/infra/jenkins-job-builder/
+
+Let's take the ``make check`` performed by Jenkins as an example.
+
+ceph-pull-requests
+------------------
+
+``ceph-pull-requests`` is a jenkins job which gets triggered by a GitHub pull
+request or a trigger phrase like::
+
+    jenkins test make check
+
+There are multiple parties involved in this jenkins job:
+
+.. graphviz::
+
+   digraph  {
+     rankdir="LR";
+     github [
+      label="<git> git_repo | <webhooks> webhooks | <api> api";
+      shape=record;
+      href="https://github.com/ceph/ceph";
+     ];
+     subgraph cluster_lab {
+       label="Sepia Lab";
+       href="https://wiki.sepia.ceph.com/doku.php";
+       shape=circle;
+       apt_mirror [
+         href="http://apt-mirror.front.sepia.ceph.com";
+       ];
+       shaman [
+         href="https://shaman.ceph.com";
+       ];
+       chacra [
+         peripheries=3;
+         href="https://chacra.ceph.com";
+       ];
+       subgraph cluster_jenkins {
+         label="jenkins";
+         href="https://jenkins.ceph.com";
+         jenkins_controller [ label = "controller" ];
+         jenkins_agents [ label = "agents", peripheries=3 ];
+       };
+     };
+     {
+       rank=same;
+       package_repos [ peripheries=3 ];
+       pypi;
+       npm;
+     }
+     github:webhooks -> jenkins_controller [ label = "notify", color = "crimson" ];
+     jenkins_controller -> jenkins_agents [ label = "schedule jobs" ];
+     jenkins_agents -> github:git [ label = "git pull" ];
+     jenkins_agents -> shaman [ label = "query for chacra repo URL" ];
+     jenkins_agents -> chacra [ label = "pull build dependencies" ];
+     jenkins_agents -> package_repos [ label = "pull build dependencies" ];
+     jenkins_agents -> pypi [ label = "pull Python packages" ];
+     jenkins_agents -> npm [ label = "pull JavaScript packages" ];
+     jenkins_agents -> apt_mirror [ label = "pull build dependencies" ];
+     jenkins_agents -> github:api [ label = "update", color = "crimson" ];
+   }
+
+Where
+
+Sepia Lab
+   `Sepia Lab`_ is a test lab used by the Ceph project. This lab offers
+   the storage and computing resources required by our CI infra.
+
+Jenkins agents
+   are a set of machines which perform the CI jobs. In this case, they
+
+   #. pull the git repo from GitHub and
+   #. rebase the pull request against the latest master
+   #. set necessary environment variables
+   #. run ``run-make-check.sh``
+
+Chacra
+   is a server offering RESTful API allowing the clients to store and
+   retrieve binary packages. It also creates the repo for uploaded
+   packages automatically. Once a certain repo is created on chacra, the
+   configured shaman server is updated as well, then we can query shaman
+   for the corresponding repo address. Chacra not only hosts Ceph packages,
+   it also hosts quite a few other packages like various build dependencies.
+
+Shaman
+   is a server offering RESTful API allowing the clients to query the
+   information of repos hosted by chacra nodes. Shaman is also known
+   for its `Web UI`_. But please note, shaman does not build the
+   packages, it just offers information on the builds.
+
+As the following shows, `chacra`_ manages multiple projects whose metadata
+are stored in a database. These metadata are exposed via Shaman as a web
+service. `chacractl`_ is a utility to interact with the `chacra`_ service.
+
+.. graphviz::
+
+   digraph  {
+     libboost [
+       shape=cylinder;
+     ];
+     libzbd [
+       shape=cylinder;
+     ];
+     other_repos [
+       label="...";
+       shape=cylinder;
+     ];
+     postgresql [
+       shape=cylinder;
+       style=filled;
+     ]
+     shaman -> postgresql;
+     chacra -> postgresql;
+     chacractl -> chacra;
+     chacra -> libboost;
+     chacra -> libzbd;
+     chacra -> other_repos;
+   }
+
+.. _Sepia Lab: https://wiki.sepia.ceph.com/doku.php
+.. _Web UI: https://shaman.ceph.com
+
+build dependencies
+------------------
+
+Just like lots of other software projects, Ceph has both build-time and
+run-time dependencies. Most of time, we are inclined to use the packages
+prebuilt by the distro. But there are cases where
+
+- the necessary dependencies are either missing in the distro, or
+- their versions are too old, or
+- they are packaged without some important feature enabled.
+- we want to ensure that the version of a certain runtime dependency is
+  identical to the one we tested in our lab.
+
+No matter what the reason is, we either need to build them from source, or
+to package them as binary packages instead of using the ones shipped by the
+distro. Quite a few build-time dependencies are included as git submodules,
+but in order to avoid rebuilding these dependencies repeatedly, we pre-built
+some of them and uploaded them to our own repos. So, when performing
+``make check``, the building hosts in our CI just pull them from our internal
+repos hosting these packages instead of building them.
+
+So far, following packages are prebuilt for ubuntu focal, and then uploaded to
+`chacra`_:
+
+libboost
+    packages `boost`_. The packages' names are changed from ``libboost-*`` to
+    ``ceph-libboost-*``, and they are instead installed into ``/opt/ceph``, so
+    they don't interfere with the official ``libboost`` packages shipped by
+    distro. Its build scripts are hosted at https://github.com/ceph/ceph-boost.
+    See https://github.com/ceph/ceph-boost/commit/2a8ae02932b2a1fd6a68072da8ca0df2b99b805c
+    for an example of how to bump the version number. The commands used to
+    build 1.79 on a vanilla Ubuntu Focal OS are below.
+
+    .. prompt:: bash $
+
+       sudo apt install debhelper dctrl-tools chrpath libbz2-dev libicu-dev bison \
+         flex docbook-to-man help2man xsltproc doxygen dh-python python3-all-dev graphviz
+       wget http://download.ceph.com/qa/boost_1_79_0.tar.bz2
+       git clone https://github.com/ceph/ceph-boost
+       tar xjf boost_1_79_0.tar.bz2
+       cp -ra ceph-boost/debian boost_1_79_0/
+       pushd boost_1_79_0
+       export DEB_BUILD_OPTIONS='parallel=6 nodoc'
+       dpkg-buildpackage -us -uc -b
+       popd
+       BOOST_SHA=$(git ls-remote https://github.com/ceph/ceph-boost main | awk '{ print $1 }')
+       ls *.deb | chacractl binary create \
+         libboost/master/$BOOST_SHA/ubuntu/focal/amd64/flavors/default
+
+libzbd
+    packages `libzbd`_ . The upstream libzbd includes debian packaging already.
+
+libpmem
+    packages `pmdk`_ . Please note, ``ndctl`` is one of the build dependencies of
+    pmdk, for an updated debian packaging, please see
+    https://github.com/ceph/ceph-ndctl .
+
+.. note::
+
+   please ensure that the package version and the release number of the
+   packaging are properly updated when updating/upgrading the packaging,
+   otherwise it would be difficult to tell which version of the package
+   is installed. We check the package version before trying to upgrade
+   it in ``install-deps.sh``.
+
+.. _boost: https://www.boost.org
+.. _libzbd: https://github.com/westerndigitalcorporation/libzbd
+.. _pmdk: https://github.com/pmem/pmdk
+
+But in addition to these libraries, ``ceph-mgr-dashboard``'s frontend uses lots of
+JavaScript packages. Quite a few of them are not packaged by distros. Not to
+mention the trouble of testing different combination of versions of these
+packages. So we decided to include these JavaScript packages in our dist tarball
+using ``make-dist``.
+
+Also, because our downstream might not want to use the prepackaged binaries when
+redistributing the precompiled Ceph packages, we also need to include these
+libraries in our dist tarball. They are
+
+- boost
+- liburing
+- pmdk
+
+``make-dist`` is a script used by our CI pipeline to create dist tarball so the
+tarball can be used to build the Ceph packages in a clean room environment. When
+we need to upgrade these third party libraries, we should
+
+- update the CMake script
+- rebuild the prebuilt packages and
+- update this script to reflect the change.
+
+Uploading Dependencies
+----------------------
+
+To ensure that prebuilt packages are available by the jenkins agents, we need to
+upload them to either ``apt-mirror.front.sepia.ceph.com`` or `chacra`_. To upload
+packages to the former would require the help of our lab administrator, so if we
+want to maintain the package repositories on regular basis, a better choice would be
+to manage them using `chacractl`_. `chacra`_ represents packages repositories using
+a resource hierarchy, like::
+
+  <project>/<branch>/<ref>/<distro>/<distro-version>/<arch>
+
+In which:
+
+project
+    in general, it is used for denoting a set of related packages. For instance,
+    ``libboost``.
+
+branch
+    branch of project. This mirrors the concept of a Git repo.
+
+ref
+    a unique id of a given version of a set packages. This id is used to reference
+    the set packages under the ``<project>/<branch>``. It is a good practice to
+    version the packaging recipes, like the ``debian`` directory for building DEB
+    packages and the ``spec`` for building RPM packages, and use the SHA1 of the
+    packaging recipe for the ``ref``. But you could also use a random string for
+    ``ref``, like the tag name of the built source tree.
+
+distro
+    the distro name for which the packages are built. Currently, following distros are
+    supported:
+
+    - centos
+    - debian
+    - fedora
+    - rhel
+    - ubuntu
+
+distro-version
+    the version of the distro. For instance, if a package is built on ubuntu focal,
+    the ``distro-version`` should be ``20.04``.
+
+arch
+    the architecture of the packages. It could be:
+
+    - arm64
+    - amd64
+    - noarch
+
+So, for example, we can upload the prebuilt boost packages to chacra like
+
+.. prompt:: bash $
+
+   ls *.deb | chacractl binary create \
+     libboost/master/099c0fd56b4a54457e288a2eff8fffdc0d416f7a/ubuntu/focal/amd64/flavors/default
+
+.. _chacra: https://github.com/ceph/chacra
+.. _chacractl: https://github.com/ceph/chacractl
+
+Update ``install-deps.sh``
+--------------------------
+
+We also need to update ``install-deps.sh`` to point the built script to the new
+repo. Please refer to the `script <https://github.com/ceph/ceph/blob/master/install-deps.sh>`_,
+for more details.
diff --git a/doc/dev/corpus.rst b/doc/dev/corpus.rst
new file mode 100644
index 000000000..4005f70c0
--- /dev/null
+++ b/doc/dev/corpus.rst
@@ -0,0 +1,100 @@
+
+Corpus structure
+================
+
+ceph.git/ceph-object-corpus is a submodule.::
+
+ bin/   # misc scripts
+ archive/$version/objects/$type/$hash  # a sample of encoded objects from a specific version
+
+You can also mark known or deliberate incompatibilities between versions with::
+
+ archive/$version/forward_incompat/$type
+
+The presence of a file indicates that new versions of code cannot
+decode old objects across that ``$version`` (this is normally the case).
+
+
+How to generate an object corpus
+--------------------------------
+
+.. highlight:: shell
+
+We can generate an object corpus for a particular version of ceph using the
+script of ``script/gen-corpus.sh``, or by following the instructions below:
+
+#. Checkout a clean repo (best not to do this where you normally work)::
+
+	git clone ceph.git
+	cd ceph
+	git submodule update --init --recursive
+
+#. Build with flag to dump objects to ``/tmp/foo``::
+
+	rm -rf /tmp/foo ; mkdir /tmp/foo
+	do_cmake.sh -DCMAKE_CXX_FLAGS="-DENCODE_DUMP_PATH=/tmp/foo"
+	cd build
+	make
+
+#. Start via vstart::
+
+	cd build
+	MON=3 MGR=2 OSD=3 MDS=3 RGW=1 ../src/vstart.sh -n -x
+
+#. Use as much functionality of the cluster as you can, to exercise as many object encoder methods as possible::
+
+	bin/ceph osd pool create mypool
+	bin/rados -p mypool bench 10 write -b 123
+	bin/ceph osd out 0
+	bin/ceph osd in 0
+	bin/init-ceph restart osd.1
+	for f in ../qa/workunits/cls/*.sh ; do PATH="bin:$PATH" $f ; done
+	PATH="bin:$PATH" ../qa/workunits/rados/test.sh
+	bin/ceph_test_librbd
+	bin/ceph_test_libcephfs
+	bin/init-ceph restart mds.a
+	../qa/workunits/rgw/run-s3tests.sh
+
+#. Stop::
+
+	../src/stop.sh
+
+#. Import the corpus (this will take a few minutes)::
+
+	../src/test/encoding/import.sh /tmp/foo `bin/ceph-dencoder version` ../ceph-object-corpus/archive
+	../src/test/encoding/import-generated.sh ../ceph-object-corpus/archive
+
+#. Prune it!  There will be a bazillion copies of various objects, and we only want a representative sample.::
+
+	pushd ../ceph-object-corpus
+	bin/prune-archive.sh
+	popd
+
+#. Verify the tests pass::
+
+	ctest -R readable.sh
+
+#. Commit it to the corpus repo and push::
+
+	pushd ../ceph-object-corpus
+	git checkout -b wip-new
+	git add archive/`../build/bin/ceph-dencoder version`
+	git commit -m `../build/bin/ceph-dencoder version`
+	git remote add cc git@github.com:ceph/ceph-object-corpus.git
+	git push cc wip-new
+	popd
+
+#. Go test it out::
+
+	cd my/regular/tree
+	cd ceph-object-corpus
+	git fetch origin
+	git checkout wip-new
+	cd ../build
+	ctest -R readable.sh
+
+#. If everything looks good, update the submodule master branch, and commit the submodule in ceph.git.
+
+
+
+
diff --git a/doc/dev/cpu-profiler.rst b/doc/dev/cpu-profiler.rst
new file mode 100644
index 000000000..5b06a4791
--- /dev/null
+++ b/doc/dev/cpu-profiler.rst
@@ -0,0 +1,54 @@
+=====================
+ Installing Oprofile
+=====================
+
+The easiest way to profile Ceph's CPU consumption is to use the `oprofile`_
+system-wide profiler.
+
+.. _oprofile: http://oprofile.sourceforge.net/about/
+
+Installation
+============
+
+If you are using a Debian/Ubuntu distribution, you can install ``oprofile`` by
+executing the following::
+
+	sudo apt-get install oprofile oprofile-gui
+	
+
+Compiling Ceph for Profiling
+============================
+
+To compile Ceph for profiling, first clean everything. :: 
+
+	make distclean
+	
+Then, export the following settings so that you can see callgraph output. :: 
+
+	export CFLAGS="-fno-omit-frame-pointer -O2 -g"
+
+Finally, compile Ceph. :: 
+
+	./autogen.sh
+	./configure
+	make
+
+You can use ``make -j`` to execute multiple jobs depending upon your system. For
+example::
+
+	make -j4
+
+
+Ceph Configuration 
+==================
+
+Ensure that you disable ``lockdep``. Consider setting logging to 
+levels appropriate for a production cluster. See `Ceph Logging and Debugging`_ 
+for details.
+
+.. _Ceph Logging and Debugging: ../../rados/troubleshooting/log-and-debug
+
+See the `CPU Profiling`_ section of the RADOS Troubleshooting documentation for details on using Oprofile.
+
+
+.. _CPU Profiling: ../../rados/troubleshooting/cpu-profiling
\ No newline at end of file
diff --git a/doc/dev/crimson/crimson.rst b/doc/dev/crimson/crimson.rst
new file mode 100644
index 000000000..954b88a37
--- /dev/null
+++ b/doc/dev/crimson/crimson.rst
@@ -0,0 +1,284 @@
+=======
+crimson
+=======
+
+Crimson is the code name of crimson-osd, which is the next generation ceph-osd.
+It targets fast networking devices, fast storage devices by leveraging state of
+the art technologies like DPDK and SPDK, for better performance. And it will
+keep the support of HDDs and low-end SSDs via BlueStore. Crismon will try to
+be backward compatible with classic OSD.
+
+.. highlight:: console
+
+Building Crimson
+================
+
+Crismon is not enabled by default. To enable it::
+
+  $ WITH_SEASTAR=true ./install-deps.sh
+  $ mkdir build && cd build
+  $ cmake -DWITH_SEASTAR=ON ..
+
+Please note, `ASan`_ is enabled by default if crimson is built from a source
+cloned using git.
+
+Also, Seastar uses its own lockless allocator which does not play well with
+the alien threads. So, to use alienstore / bluestore backend, you might want to
+pass ``-DSeastar_CXX_FLAGS=-DSEASTAR_DEFAULT_ALLOCATOR`` to ``cmake`` when
+configuring this project to use the libc allocator, like::
+
+  $ cmake -DWITH_SEASTAR=ON -DSeastar_CXX_FLAGS=-DSEASTAR_DEFAULT_ALLOCATOR ..
+
+.. _ASan: https://github.com/google/sanitizers/wiki/AddressSanitizer
+
+Running Crimson
+===============
+
+As you might expect, crimson is not featurewise on par with its predecessor yet.
+
+object store backend
+--------------------
+
+At the moment ``crimson-osd`` offers two object store backends:
+
+- CyanStore: CyanStore is modeled after memstore in classic OSD.
+- AlienStore: AlienStore is short for Alienized BlueStore.
+
+Seastore is still under active development.
+
+daemonize
+---------
+
+Unlike ``ceph-osd``, ``crimson-osd`` does daemonize itself even if the
+``daemonize`` option is enabled. Because, to read this option, ``crimson-osd``
+needs to ready its config sharded service, but this sharded service lives
+in the seastar reactor. If we fork a child process and exit the parent after
+starting the Seastar engine, that will leave us with a single thread which is
+the replica of the thread calls `fork()`_. This would unnecessarily complicate
+the code, if we would have tackled this problem in crimson.
+
+Since a lot of GNU/Linux distros are using systemd nowadays, which is able to
+daemonize the application, there is no need to daemonize by ourselves. For
+those who are using sysvinit, they can use ``start-stop-daemon`` for daemonizing
+``crimson-osd``. If this is not acceptable, we can whip up a helper utility
+to do the trick.
+
+
+.. _fork(): http://pubs.opengroup.org/onlinepubs/9699919799/functions/fork.html
+
+logging
+-------
+
+Currently, ``crimson-osd`` uses the logging utility offered by Seastar. see
+``src/common/dout.h`` for the mapping between different logging levels to
+the severity levels in Seastar. For instance, the messages sent to ``derr``
+will be printed using ``logger::error()``, and the messages with debug level
+over ``20`` will be printed using ``logger::trace()``.
+
++---------+---------+
+| ceph    | seastar |
++---------+---------+
+| < 0     | error   |
++---------+---------+
+|   0     | warn    |
++---------+---------+
+| [1, 5)  | info    |
++---------+---------+
+| [5, 20] | debug   |
++---------+---------+
+| >  20   | trace   |
++---------+---------+
+
+Please note, ``crimson-osd``
+does not send the logging message to specified ``log_file``. It writes
+the logging messages to stdout and/or syslog. Again, this behavior can be
+changed using ``--log-to-stdout`` and ``--log-to-syslog`` command line
+options. By default, ``log-to-stdout`` is enabled, and the latter disabled.
+
+
+vstart.sh
+---------
+
+To facilitate the development of crimson, following options would be handy when
+using ``vstart.sh``,
+
+``--crimson``
+    start ``crimson-osd`` instead of ``ceph-osd``
+
+``--nodaemon``
+    do not daemonize the service
+
+``--redirect-output``
+    redirect the stdout and stderr of service to ``out/$type.$num.stdout``.
+
+``--osd-args``
+    pass extra command line options to crimson-osd or ceph-osd. It's quite
+    useful for passing Seastar options to crimson-osd. For instance, you could
+    use ``--osd-args "--memory 2G"`` to set the memory to use. Please refer
+    the output of::
+
+      crimson-osd --help-seastar
+
+    for more Seastar specific command line options.
+
+``--memstore``
+    use the CyanStore as the object store backend.
+
+``--bluestore``
+    use the AlienStore as the object store backend. This is the default setting,
+    if not specified otherwise.
+
+So, a typical command to start a single-crimson-node cluster is::
+
+  $  MGR=1 MON=1 OSD=1 MDS=0 RGW=0 ../src/vstart.sh -n -x \
+    --without-dashboard --memstore \
+    --crimson --nodaemon --redirect-output \
+    --osd-args "--memory 4G"
+
+Where we assign 4 GiB memory, a single thread running on core-0 to crimson-osd.
+
+You could stop the vstart cluster using::
+
+  $ ../src/stop.sh --crimson
+
+
+CBT Based Testing
+=================
+
+We can use `cbt`_ for performing perf tests::
+
+  $ git checkout master
+  $ make crimson-osd
+  $ ../src/script/run-cbt.sh --cbt ~/dev/cbt -a /tmp/baseline ../src/test/crimson/cbt/radosbench_4K_read.yaml
+  $ git checkout yet-another-pr
+  $ make crimson-osd
+  $ ../src/script/run-cbt.sh --cbt ~/dev/cbt -a /tmp/yap ../src/test/crimson/cbt/radosbench_4K_read.yaml
+  $ ~/dev/cbt/compare.py -b /tmp/baseline -a /tmp/yap -v
+  19:48:23 - INFO     - cbt      - prefill/gen8/0: bandwidth: (or (greater) (near 0.05)):: 0.183165/0.186155  => accepted
+  19:48:23 - INFO     - cbt      - prefill/gen8/0: iops_avg: (or (greater) (near 0.05)):: 46.0/47.0  => accepted
+  19:48:23 - WARNING  - cbt      - prefill/gen8/0: iops_stddev: (or (less) (near 0.05)):: 10.4403/6.65833  => rejected
+  19:48:23 - INFO     - cbt      - prefill/gen8/0: latency_avg: (or (less) (near 0.05)):: 0.340868/0.333712  => accepted
+  19:48:23 - INFO     - cbt      - prefill/gen8/1: bandwidth: (or (greater) (near 0.05)):: 0.190447/0.177619  => accepted
+  19:48:23 - INFO     - cbt      - prefill/gen8/1: iops_avg: (or (greater) (near 0.05)):: 48.0/45.0  => accepted
+  19:48:23 - INFO     - cbt      - prefill/gen8/1: iops_stddev: (or (less) (near 0.05)):: 6.1101/9.81495  => accepted
+  19:48:23 - INFO     - cbt      - prefill/gen8/1: latency_avg: (or (less) (near 0.05)):: 0.325163/0.350251  => accepted
+  19:48:23 - INFO     - cbt      - seq/gen8/0: bandwidth: (or (greater) (near 0.05)):: 1.24654/1.22336  => accepted
+  19:48:23 - INFO     - cbt      - seq/gen8/0: iops_avg: (or (greater) (near 0.05)):: 319.0/313.0  => accepted
+  19:48:23 - INFO     - cbt      - seq/gen8/0: iops_stddev: (or (less) (near 0.05)):: 0.0/0.0  => accepted
+  19:48:23 - INFO     - cbt      - seq/gen8/0: latency_avg: (or (less) (near 0.05)):: 0.0497733/0.0509029  => accepted
+  19:48:23 - INFO     - cbt      - seq/gen8/1: bandwidth: (or (greater) (near 0.05)):: 1.22717/1.11372  => accepted
+  19:48:23 - INFO     - cbt      - seq/gen8/1: iops_avg: (or (greater) (near 0.05)):: 314.0/285.0  => accepted
+  19:48:23 - INFO     - cbt      - seq/gen8/1: iops_stddev: (or (less) (near 0.05)):: 0.0/0.0  => accepted
+  19:48:23 - INFO     - cbt      - seq/gen8/1: latency_avg: (or (less) (near 0.05)):: 0.0508262/0.0557337  => accepted
+  19:48:23 - WARNING  - cbt      - 1 tests failed out of 16
+
+Where we compile and run the same test against two branches. One is ``master``, another is ``yet-another-pr`` branch.
+And then we compare the test results. Along with every test case, a set of rules is defined to check if we have
+performance regressions when comparing two set of test results. If a possible regression is found, the rule and
+corresponding test results are highlighted.
+
+.. _cbt: https://github.com/ceph/cbt
+
+Hacking Crimson
+===============
+
+
+Seastar Documents
+-----------------
+
+See `Seastar Tutorial <https://github.com/scylladb/seastar/blob/master/doc/tutorial.md>`_ .
+Or build a browsable version and start an HTTP server::
+
+  $ cd seastar
+  $ ./configure.py --mode debug
+  $ ninja -C build/debug docs
+  $ python3 -m http.server -d build/debug/doc/html
+
+You might want to install ``pandoc`` and other dependencies beforehand.
+
+Debugging Crimson
+=================
+
+Debugging with GDB
+------------------
+
+The `tips`_ for debugging Scylla also apply to Crimson.
+
+.. _tips: https://github.com/scylladb/scylla/blob/master/docs/debugging.md#tips-and-tricks
+
+Human-readable backtraces with addr2line
+----------------------------------------
+
+When a seastar application crashes, it leaves us with a serial of addresses, like::
+
+  Segmentation fault.
+  Backtrace:
+    0x00000000108254aa
+    0x00000000107f74b9
+    0x00000000105366cc
+    0x000000001053682c
+    0x00000000105d2c2e
+    0x0000000010629b96
+    0x0000000010629c31
+    0x00002a02ebd8272f
+    0x00000000105d93ee
+    0x00000000103eff59
+    0x000000000d9c1d0a
+    /lib/x86_64-linux-gnu/libc.so.6+0x000000000002409a
+    0x000000000d833ac9
+  Segmentation fault
+
+``seastar-addr2line`` offered by Seastar can be used to decipher these
+addresses. After running the script, it will be waiting for input from stdin,
+so we need to copy and paste the above addresses, then send the EOF by inputting
+``control-D`` in the terminal::
+
+  $ ../src/seastar/scripts/seastar-addr2line -e bin/crimson-osd
+
+    0x00000000108254aa
+    0x00000000107f74b9
+    0x00000000105366cc
+    0x000000001053682c
+    0x00000000105d2c2e
+    0x0000000010629b96
+    0x0000000010629c31
+    0x00002a02ebd8272f
+    0x00000000105d93ee
+    0x00000000103eff59
+    0x000000000d9c1d0a
+    0x00000000108254aa
+  [Backtrace #0]
+  seastar::backtrace_buffer::append_backtrace() at /home/kefu/dev/ceph/build/../src/seastar/src/core/reactor.cc:1136
+  seastar::print_with_backtrace(seastar::backtrace_buffer&) at /home/kefu/dev/ceph/build/../src/seastar/src/core/reactor.cc:1157
+  seastar::print_with_backtrace(char const*) at /home/kefu/dev/ceph/build/../src/seastar/src/core/reactor.cc:1164
+  seastar::sigsegv_action() at /home/kefu/dev/ceph/build/../src/seastar/src/core/reactor.cc:5119
+  seastar::install_oneshot_signal_handler<11, &seastar::sigsegv_action>()::{lambda(int, siginfo_t*, void*)#1}::operator()(int, siginfo_t*, void*) const at /home/kefu/dev/ceph/build/../src/seastar/src/core/reactor.cc:5105
+  seastar::install_oneshot_signal_handler<11, &seastar::sigsegv_action>()::{lambda(int, siginfo_t*, void*)#1}::_FUN(int, siginfo_t*, void*) at /home/kefu/dev/ceph/build/../src/seastar/src/core/reactor.cc:5101
+  ?? ??:0
+  seastar::smp::configure(boost::program_options::variables_map, seastar::reactor_config) at /home/kefu/dev/ceph/build/../src/seastar/src/core/reactor.cc:5418
+  seastar::app_template::run_deprecated(int, char**, std::function<void ()>&&) at /home/kefu/dev/ceph/build/../src/seastar/src/core/app-template.cc:173 (discriminator 5)
+  main at /home/kefu/dev/ceph/build/../src/crimson/osd/main.cc:131 (discriminator 1)
+
+Please note, ``seastar-addr2line`` is able to extract the addresses from
+the input, so you can also paste the log messages like::
+
+  2020-07-22T11:37:04.500 INFO:teuthology.orchestra.run.smithi061.stderr:Backtrace:
+  2020-07-22T11:37:04.500 INFO:teuthology.orchestra.run.smithi061.stderr:  0x0000000000e78dbc
+  2020-07-22T11:37:04.501 INFO:teuthology.orchestra.run.smithi061.stderr:  0x0000000000e3e7f0
+  2020-07-22T11:37:04.501 INFO:teuthology.orchestra.run.smithi061.stderr:  0x0000000000e3e8b8
+  2020-07-22T11:37:04.501 INFO:teuthology.orchestra.run.smithi061.stderr:  0x0000000000e3e985
+  2020-07-22T11:37:04.501 INFO:teuthology.orchestra.run.smithi061.stderr:  /lib64/libpthread.so.0+0x0000000000012dbf
+
+Unlike classic OSD, crimson does not print a human-readable backtrace when it
+handles fatal signals like `SIGSEGV` or `SIGABRT`. And it is more complicated
+when it comes to a stripped binary. So before planting a signal handler for
+those signals in crimson, we could to use `script/ceph-debug-docker.sh` to parse
+the addresses in the backtrace::
+
+  # assuming you are under the source tree of ceph
+  $ ./src/script/ceph-debug-docker.sh  --flavor crimson master:27e237c137c330ebb82627166927b7681b20d0aa centos:8
+  ....
+  [root@3deb50a8ad51 ~]# wget -q https://raw.githubusercontent.com/scylladb/seastar/master/scripts/seastar-addr2line
+  [root@3deb50a8ad51 ~]# dnf install -q -y file
+  [root@3deb50a8ad51 ~]# python3 seastar-addr2line -e /usr/bin/crimson-osd
+  # paste the backtrace here
diff --git a/doc/dev/crimson/error-handling.rst b/doc/dev/crimson/error-handling.rst
new file mode 100644
index 000000000..43017457d
--- /dev/null
+++ b/doc/dev/crimson/error-handling.rst
@@ -0,0 +1,158 @@
+==============
+error handling
+==============
+
+
+In Seastar, a ``future`` represents a value not yet available but that can become
+available later. ``future`` can have one of following states:
+
+* unavailable: value is not available yet,
+* value,
+* failed: an exception was thrown when computing the value. This exception has
+  been captured and stored in the ``future`` instance via ``std::exception_ptr``.
+
+In the last case, the exception can be processed using ``future::handle_exception()`` or
+``future::handle_exception_type()``. Seastar even provides ``future::or_terminate()`` to
+terminate the program if the future fails.
+
+But in Crimson, quite a few errors are not serious enough to fail the program entirely.
+For instance, if we try to look up an object by its object id, and that operation could
+fail because the object does not exist or it is corrupted, we need to recover that object
+for fulfilling the request instead of terminating the process.
+
+In other words, these errors are expected. Moreover, the performance of the unhappy path
+should also be on par with that of the happy path. Also, we want to have a way to ensure
+that all expected errors are handled. It should be something like the statical analysis
+performed by compiler to spit a warning if any enum value is not handled in a ``switch-case``
+statement.
+
+Unfortunately, ``seastar::future`` is not able to satisfy these two requirements.
+
+* Seastar imposes re-throwing an exception to dispatch between different types of
+  exceptions. This is not very performant nor even scalable as locking in the language's
+  runtime can occur.
+* Seastar does not encode the expected exception type in the type of the returned
+  ``seastar::future``. Only the type of the value is encoded. This imposes huge
+  mental load on programmers as ensuring that all intended errors are indeed handled
+  requires manual code audit.
+
+.. highlight:: c++
+
+So, "errorator" is created. It is a wrapper around the vanilla ``seastar::future``.
+It addresses the performance and scalability issues while embedding the information
+about all expected types-of-errors to the type-of-future.::
+
+  using ertr = crimson::errorator<crimson::ct_error::enoent,
+                                  crimson::ct_error::einval>;
+
+In above example we defined an errorator that allows for two error types:
+
+* ``crimson::ct_error::enoent`` and
+* ``crimson::ct_error::einval``.
+
+These (and other ones in the ``crimson::ct_error`` namespace) are basically
+unthrowable wrappers over ``std::error_code`` to exclude accidental throwing
+and ensure signaling errors in a way that enables compile-time checking.
+
+The most fundamental thing in an errorator is a descendant of ``seastar::future``
+which can be used as e.g. function's return type::
+
+  static ertr::future<int> foo(int bar) {
+    if (bar == 42) {
+      return crimson::ct_error::einval::make();
+    } else {
+      return ertr::make_ready_future(bar);
+    }
+  }
+
+It's worth to note that returning an error that is not a part the errorator's error set
+would result in a compile-time error::
+
+  static ertr::future<int> foo(int bar) {
+    // Oops, input_output_error is not allowed in `ertr`.  static_assert() will
+    // terminate the compilation. This behaviour is absolutely fundamental for
+    // callers -- to figure out about all possible errors they need to worry
+    // about is enough to just take a look on the function's signature; reading
+    // through its implementation is not necessary anymore!
+    return crimson::ct_error::input_output_error::make();
+  }
+
+The errorator concept goes further. It not only provides callers with the information
+about all potential errors embedded in the function's type; it also ensures at the caller
+site that all these errors are handled. As the reader probably know, the main method
+in ``seastar::future`` is ``then()``. On errorated future it is available but only if errorator's
+error set is empty (literally: ``errorator<>::future``); otherwise callers have
+to use ``safe_then()`` instead::
+
+  seastar::future<> baz() {
+    return foo(42).safe_then(
+      [] (const int bar) {
+        std::cout << "the optimistic path! got bar=" << bar << std::endl
+        return ertr::now();
+      },
+      ertr::all_same_way(const std::error_code& err) {
+        // handling errors removes them from errorator's error set
+        std::cout << "the error path! got err=" << err << std::endl;
+        return ertr::now();
+      }).then([] {
+        // as all errors have been handled, errorator's error set became
+        // empty and the future instance returned from `safe_then()` has
+        // `then()` available!
+        return seastar::now();
+      });
+  }
+
+In the above example ``ertr::all_same_way`` has been used to handle all errors in the same
+manner. This is not obligatory -- a caller can handle each of them separately. Moreover,
+it can provide a handler for only a subset of errors. The price for that is the availability
+of ``then()``::
+
+  using einval_ertr = crimson::errorator<crimson::ct_error::einval>;
+
+  // we can't return seastar::future<> (aka errorator<>::future<>) as handling
+  // as this level deals only with enoent leaving einval without a handler.
+  // handling it becomes a responsibility of a caller of `baz()`.
+  einval_ertr::future<> baz() {
+    return foo(42).safe_then(
+      [] (const int bar) {
+        std::cout << "the optimistic path! got bar=" << bar << std::endl
+        return ertr::now();
+      },
+      // provide a handler only for crimson::ct_error::enoent.
+      // crimson::ct_error::einval stays unhandled!
+      crimson::ct_error::enoent::handle([] {
+        std::cout << "the enoent error path!" << std::endl;
+        return ertr::now();
+      }));
+    // .safe_then() above returned `errorator<crimson::ct_error::einval>::future<>`
+    // which lacks `then()`.
+  }
+
+That is, handling errors removes them from errorated future's error set. This works
+in the opposite direction too -- returning new errors in ``safe_then()`` appends them
+the error set. Of course, this set must be compliant with error set in the ``baz()``'s
+signature::
+
+  using broader_ertr = crimson::errorator<crimson::ct_error::enoent,
+                                          crimson::ct_error::einval,
+                                          crimson::ct_error::input_output_error>;
+
+  broader_ertr::future<> baz() {
+    return foo(42).safe_then(
+      [] (const int bar) {
+        std::cout << "oops, the optimistic path generates a new error!";
+        return crimson::ct_error::input_output_error::make();
+      },
+      // we have a special handler to delegate the handling up. For conveience,
+      // the same behaviour is available as single argument-taking variant of
+      // `safe_then()`.
+      ertr::pass_further{});
+  }
+
+As it can be seen, handling and signaling errors in ``safe_then()`` is basically
+an operation on the error set checked at compile-time.
+
+More details can be found in `the slides from ceph::errorator<> throw/catch-free,
+compile time-checked exceptions for seastar::future<>
+<https://www.slideshare.net/ScyllaDB/cepherrorator-throwcatchfree-compile-timechecked-exceptions-for-seastarfuture>`_
+presented at the Seastar Summit 2019.
diff --git a/doc/dev/crimson/index.rst b/doc/dev/crimson/index.rst
new file mode 100644
index 000000000..55f071825
--- /dev/null
+++ b/doc/dev/crimson/index.rst
@@ -0,0 +1,11 @@
+===============================
+Crimson developer documentation
+===============================
+
+.. rubric:: Contents
+
+.. toctree::
+   :glob:
+
+   *
+
diff --git a/doc/dev/crimson/poseidonstore.rst b/doc/dev/crimson/poseidonstore.rst
new file mode 100644
index 000000000..3fbefd04b
--- /dev/null
+++ b/doc/dev/crimson/poseidonstore.rst
@@ -0,0 +1,586 @@
+===============
+ PoseidonStore
+===============
+
+Key concepts and goals
+======================
+
+* As one of the pluggable backend stores for Crimson, PoseidonStore targets only
+  high-end NVMe SSDs (not concerned with ZNS devices).
+* Designed entirely for low CPU consumption
+
+  - Hybrid update strategies for different data types (in-place, out-of-place) to
+    minimize CPU consumption by reducing host-side GC.
+  - Remove a black-box component like RocksDB and a file abstraction layer in BlueStore
+    to avoid unnecessary overheads (e.g., data copy and serialization/deserialization)
+  - Utilize NVMe feature (atomic large write command, Atomic Write Unit Normal).
+    Make use of io_uring, new kernel asynchronous I/O interface, to selectively use the interrupt
+    driven mode for CPU efficiency (or polled mode for low latency).
+* Sharded data/processing model
+
+Background
+----------
+
+Both in-place and out-of-place update strategies have their pros and cons.
+
+* Log-structured store
+
+  Log-structured based storage system is a typical example that adopts an update-out-of-place approach.
+  It never modifies the written data. Writes always go to the end of the log. It enables I/O sequentializing.
+
+  * Pros
+
+    - Without a doubt, one sequential write is enough to store the data
+    - It naturally supports transaction (this is no overwrite, so the store can rollback
+      previous stable state)
+    - Flash friendly (it mitigates GC burden on SSDs)
+  * Cons
+
+    - There is host-side GC that induces overheads
+
+      - I/O amplification (host-side)
+      - More host-CPU consumption
+
+    - Slow metadata lookup
+    - Space overhead (live and unused data co-exist)
+
+* In-place update store
+
+  The update-in-place strategy has been used widely for conventional file systems such as ext4 and xfs.
+  Once a block has been placed in a given disk location, it doesn't move.
+  Thus, writes go to the corresponding location in the disk.
+
+  * Pros
+
+    - Less host-CPU consumption (No host-side GC is required)
+    - Fast lookup
+    - No additional space for log-structured, but there is internal fragmentation
+  * Cons
+
+    - More writes occur to record the data (metadata and data section are separated)
+    - It cannot support transaction. Some form of WAL required to ensure update atomicity
+      in the general case
+    - Flash unfriendly (Give more burdens on SSDs due to device-level GC)
+
+Motivation and Key idea
+-----------------------
+
+In modern distributed storage systems, a server node can be equipped with multiple
+NVMe storage devices. In fact, ten or more NVMe SSDs could be attached on a server.
+As a result, it is hard to achieve NVMe SSD's full performance due to the limited CPU resources
+available in a server node. In such environments, CPU tends to become a performance bottleneck.
+Thus, now we should focus on minimizing host-CPU consumption, which is the same as the Crimson's objective.
+
+Towards an object store highly optimized for CPU consumption, three design choices have been made.
+
+* **PoseidonStore does not have a black-box component like RocksDB in BlueStore.**
+
+  Thus, it can avoid unnecessary data copy and serialization/deserialization overheads.
+  Moreover, we can remove an unnecessary file abstraction layer, which was required to run RocksDB.
+  Object data and metadata is now directly mapped to the disk blocks.
+  Eliminating all these overheads will reduce CPU consumption (e.g., pre-allocation, NVME atomic feature).
+
+* **PoseidonStore uses hybrid update strategies for different data size, similar to BlueStore.**
+
+  As we discussed, both in-place and out-of-place update strategies have their pros and cons.
+  Since CPU is only bottlenecked under small I/O workloads, we chose update-in-place for small I/Os to mininize CPU consumption
+  while choosing update-out-of-place for large I/O to avoid double write. Double write for small data may be better than host-GC overhead
+  in terms of CPU consumption in the long run. Although it leaves GC entirely up to SSDs,
+
+* **PoseidonStore makes use of io_uring, new kernel asynchronous I/O interface to exploit interrupt-driven I/O.**
+
+  User-space driven I/O solutions like SPDK provide high I/O performance by avoiding syscalls and enabling zero-copy
+  access from the application. However, it does not support interrupt-driven I/O, which is only possible with kernel-space driven I/O.
+  Polling is good for low-latency but bad for CPU efficiency. On the other hand, interrupt is good for CPU efficiency and bad for
+  low-latency (but not that bad as I/O size increases). Note that network acceleration solutions like DPDK also excessively consume
+  CPU resources for polling. Using polling both for network and storage processing aggravates CPU consumption.
+  Since network is typically much faster and has a higher priority than storage, polling should be applied only to network processing.
+
+high-end NVMe SSD has enough powers to handle more works. Also, SSD lifespan is not a practical concern these days
+(there is enough program-erase cycle limit [#f1]_). On the other hand, for large I/O workloads, the host can afford process host-GC.
+Also, the host can garbage collect invalid objects more effectively when their size is large
+
+Observation
+-----------
+
+Two data types in Ceph
+
+* Data (object data)
+
+  - The cost of double write is high
+  - The best mehod to store this data is in-place update
+
+    - At least two operations required to store the data: 1) data and 2) location of
+      data. Nevertheless, a constant number of operations would be better than out-of-place
+      even if it aggravates WAF in SSDs
+
+* Metadata or small data (e.g., object_info_t, snapset, pg_log, and collection)
+
+  - Multiple small-sized metadata entries for an object
+  - The best solution to store this data is WAL + Using cache
+
+    - The efficient way to store metadata is to merge all metadata related to data
+      and store it though a single write operation even though it requires background
+      flush to update the data partition
+
+
+Design
+======
+.. ditaa::
+
+  +-WAL partition-|----------------------Data partition-------------------------------+
+  | Sharded partition                                                                 |
+  +-----------------------------------------------------------------------------------+
+  | WAL -> |      | Super block | Freelist info | Onode radix tree info| Data blocks  |
+  +-----------------------------------------------------------------------------------+
+  | Sharded partition 2
+  +-----------------------------------------------------------------------------------+
+  | WAL -> |      | Super block | Freelist info | Onode radix tree info| Data blocks  |
+  +-----------------------------------------------------------------------------------+
+  | Sharded partition N
+  +-----------------------------------------------------------------------------------+
+  | WAL -> |      | Super block | Freelist info | Onode radix tree info| Data blocks  |
+  +-----------------------------------------------------------------------------------+
+  | Global information (in reverse order)
+  +-----------------------------------------------------------------------------------+
+  | Global WAL -> | | SB | Freelist |                                                 |
+  +-----------------------------------------------------------------------------------+
+
+
+* WAL
+
+  - Log, metadata and small data are stored in the WAL partition
+  - Space within the WAL partition is continually reused in a circular manner
+  - Flush data to trim WAL as necessary
+* Disk layout
+
+  - Data blocks are metadata blocks or data blocks
+  - Freelist manages the root of free space B+tree
+  - Super block contains management info for a data partition
+  - Onode radix tree info contains the root of onode radix tree
+
+
+I/O procedure
+-------------
+* Write
+
+  For incoming writes, data is handled differently depending on the request size;
+  data is either written twice (WAL) or written in a log-structured manner.
+
+  #. If Request Size ≤ Threshold (similar to minimum allocation size in BlueStore)
+
+     Write data and metadata to [WAL] —flush—> Write them to [Data section (in-place)] and
+     [Metadata section], respectively.
+
+     Since the CPU becomes the bottleneck for small I/O workloads, in-place update scheme is used.
+     Double write for small data may be better than host-GC overhead in terms of CPU consumption
+     in the long run
+  #. Else if Request Size > Threshold
+
+     Append data to [Data section (log-structure)] —> Write the corresponding metadata to [WAL]
+     —flush—> Write the metadata to [Metadata section]
+
+  For large I/O workloads, the host can afford process host-GC
+  Also, the host can garbage collect invalid objects more effectively when their size is large
+
+  Note that Threshold can be configured to a very large number so that only the scenario (1) occurs.
+  With this design, we can control the overall I/O procedure with the optimizations for crimson
+  as described above.
+
+  * Detailed flow
+
+    We make use of a NVMe write command which provides atomicity guarantees (Atomic Write Unit Power Fail)
+    For example, 512 Kbytes of data can be atomically written at once without fsync().
+
+    * stage 1
+
+      - if the data is small
+        WAL (written) --> | TxBegin A | Log Entry | TxEnd A |
+        Append a log entry that contains pg_log, snapset, object_infot_t and block allocation
+        using NVMe atomic write command on the WAL
+      - if the data is large
+        Data partition (written) --> | Data blocks |
+    * stage 2
+
+      - if the data is small
+        No need.
+      - if the data is large
+        Then, append the metadata to WAL.
+        WAL --> | TxBegin A | Log Entry | TxEnd A |
+
+* Read
+
+  - Use the cached object metadata to find out the data location
+  - If not cached, need to search WAL after checkpoint and Object meta partition to find the
+    latest meta data
+
+* Flush (WAL --> Data partition)
+
+  - Flush WAL entries that have been committed. There are two conditions
+    (1. the size of WAL is close to full, 2. a signal to flush).
+    We can mitigate the overhead of frequent flush via batching processing, but it leads to
+    delaying completion.
+
+
+Crash consistency
+------------------
+
+* Large case
+
+  #. Crash occurs right after writing Data blocks
+
+     - Data partition --> | Data blocks |
+     - We don't need to care this case. Data is not alloacted yet in reality. The blocks will be reused.
+  #. Crash occurs right after WAL
+
+     - Data partition --> | Data blocks |
+     - WAL --> | TxBegin A | Log Entry | TxEnd A |
+     - Write procedure is completed, so there is no data loss or inconsistent state
+
+* Small case
+
+  #. Crash occurs right after writing WAL
+
+     - WAL --> | TxBegin A | Log Entry| TxEnd A |
+     - All data has been written
+
+
+Comparison
+----------
+
+* Best case (pre-allocation)
+
+  - Only need writes on both WAL and Data partition without updating object metadata (for the location).
+* Worst case
+
+  - At least three writes are required additionally on WAL, object metadata, and data blocks.
+  - If the flush from WAL to the data parition occurs frequently, radix tree onode structure needs to be update
+    in many times. To minimize such overhead, we can make use of batch processing to minimize the update on the tree
+    (the data related to the object has a locality because it will have the same parent node, so updates can be minimized)
+
+* WAL needs to be flushed if the WAL is close to full or a signal to flush.
+
+  - The premise behind this design is OSD can manage the latest metadata as a single copy. So,
+    appended entries are not to be read
+* Either best of the worst case does not produce severe I/O amplification (it produce I/Os, but I/O rate is constant)
+  unlike LSM-tree DB (the proposed design is similar to LSM-tree which has only level-0)
+
+
+Detailed Design
+===============
+
+* Onode lookup
+
+  * Radix tree
+    Our design is entirely based on the prefix tree. Ceph already makes use of the characteristic of OID's prefix to split or search
+    the OID (e.g., pool id + hash + oid). So, the prefix tree fits well to store or search the object. Our scheme is designed
+    to lookup the prefix tree efficiently.
+
+  * Sharded partition
+    A few bits (leftmost bits of the hash) of the OID determine a sharded partition where the object is located.
+    For example, if the number of partitions is configured as four, The entire space of the hash in hobject_t
+    can be divided into four domains (0x0xxx ~ 0x3xxx, 0x4xxx ~ 0x7xxx, 0x8xxx ~ 0xBxxx and 0xCxxx ~ 0xFxxx).
+
+  * Ondisk onode
+
+    .. code-block:: c
+
+            stuct onode {
+              extent_tree block_maps;
+              b+_tree omaps;
+              map xattrs;
+            }
+
+    onode contains the radix tree nodes for lookup, which means we can search for objects using tree node information in onode.
+    Also, if the data size is small, the onode can embed the data and xattrs.
+    The onode is fixed size (256 or 512 byte). On the other hands, omaps and block_maps are variable-length by using pointers in the onode.
+
+    .. ditaa::
+
+           +----------------+------------+--------+
+           | on\-disk onode | block_maps | omaps  |
+           +----------+-----+------------+--------+
+                      |           ^         ^
+                      |           |         |
+                      +-----------+---------+
+
+
+  * Lookup
+    The location of the root of onode tree is specified on Onode radix tree info, so we can find out where the object
+    is located by using the root of prefix tree. For example, shared partition is determined by OID as described above.
+    Using the rest of the OID's bits and radix tree, lookup procedure find outs the location of the onode.
+    The extent tree (block_maps) contains where data chunks locate, so we finally figure out the data location.
+
+
+* Allocation
+
+  * Sharded partitions
+
+    The entire disk space is divided into several  data chunks called sharded partition (SP).
+    Each SP has its own data structures to manage the partition.
+
+  * Data allocation
+
+    As we explained above, the management infos (e.g., super block, freelist info, onode radix tree info) are pre-allocated
+    in each shared partition. Given OID, we can map any data in Data block section to the extent tree in the onode.
+    Blocks can be allocated by searching the free space tracking data structure (we explain below).
+
+    ::
+
+            +-----------------------------------+
+            | onode radix tree root node block  |
+            |          (Per-SP Meta)            |
+            |                                   |
+            |           # of records            |
+            |    left_sibling / right_sibling   |
+            | +--------------------------------+|
+            | | keys[# of records]             ||
+            | | +-----------------------------+||
+            | | |    start onode ID           |||
+            | | |           ...               |||
+            | | +-----------------------------+||
+            | +--------------------------------||
+            | +--------------------------------+|
+            | | ptrs[# of records]             ||
+            | | +-----------------------------+||
+            | | |       SP block number       |||
+            | | |           ...               |||
+            | | +-----------------------------+||
+            | +--------------------------------+|
+            +-----------------------------------+
+
+  * Free space tracking
+    The freespace is tracked on a per-SP basis. We can use extent-based B+tree in XFS for free space tracking.
+    The freelist info contains the root of free space B+tree. Granularity is a data block in Data blocks partition.
+    The data block is the smallest and fixed size unit of data.
+
+    ::
+
+             +-----------------------------------+
+             | Free space B+tree root node block |
+             |          (Per-SP Meta)            |
+             |                                   |
+             |           # of records            |
+             |    left_sibling / right_sibling   |
+             | +--------------------------------+|
+             | | keys[# of records]             ||
+             | | +-----------------------------+||
+             | | |   startblock / blockcount   |||
+             | | |           ...               |||
+             | | +-----------------------------+||
+             | +--------------------------------||
+             | +--------------------------------+|
+             | | ptrs[# of records]             ||
+             | | +-----------------------------+||
+             | | |       SP block number       |||
+             | | |           ...               |||
+             | | +-----------------------------+||
+             | +--------------------------------+|
+             +-----------------------------------+
+
+* Omap and xattr
+  In this design, omap and xattr data is tracked by b+tree in onode. The onode only has the root node of b+tree.
+  The root node contains entires which indicate where the key onode exists.
+  So, if we know the onode, omap can be found via omap b+tree.
+
+* Fragmentation
+
+  - Internal fragmentation
+
+    We pack different types of data/metadata in a single block as many as possible to reduce internal fragmentation.
+    Extent-based B+tree may help reduce this further by allocating contiguous blocks that best fit for the object
+
+  - External fragmentation
+
+    Frequent object create/delete may lead to external fragmentation
+    In this case, we need cleaning work (GC-like) to address this.
+    For this, we are referring the NetApp’s Continuous Segment Cleaning, which seems similar to the SeaStore’s approach
+    Countering Fragmentation in an Enterprise Storage System (NetApp, ACM TOS, 2020)
+
+.. ditaa::
+
+
+       +---------------+-------------------+-------------+
+       | Freelist info | Onode radix tree  | Data blocks +-------+
+       +---------------+---------+---------+-+-----------+       |
+                                 |           |                   |
+            +--------------------+           |                   |
+            |                                |                   |
+            |        OID                     |                   |
+            |                                |                   |
+        +---+---+                            |                   |
+        | Root  |                            |                   |
+        +---+---+                            |                   |
+            |                                |                   |
+            v                                |                   |
+       /-----------------------------\       |                   |
+       |          Radix tree         |       |                   v
+       +---------+---------+---------+       |      /---------------\
+       | onode   | ...     | ...     |       |      | Num Chunk     |
+       +---------+---------+---------+       |      |               |
+    +--+ onode   | ...     | ...     |       |      | <Offset, len> |
+    |  +---------+---------+---------+       |      | <Offset, len> +-------+
+    |                                        |      | ...           |       |
+    |                                        |      +---------------+       |
+    |                                        |      ^                       |
+    |                                        |      |                       |
+    |                                        |      |                       |
+    |                                        |      |                       |
+    |  /---------------\  /-------------\    |      |                       v
+    +->| onode         |  | onode       |<---+      |       /------------+------------\
+       +---------------+  +-------------+           |       | Block0     | Block1     |
+       | OID           |  | OID         |           |       +------------+------------+
+       | Omaps         |  | Omaps       |           |       | Data       | Data       |
+       | Data Extent   |  | Data Extent +-----------+       +------------+------------+
+       +---------------+  +-------------+
+
+WAL
+---
+Each SP has a WAL.
+The datas written to the WAL are metadata updates, free space update and small data.
+Note that only data smaller than the predefined threshold needs to be written to the WAL.
+The larger data is written to the unallocated free space and its onode's extent_tree is updated accordingly
+(also on-disk extent tree). We statically allocate WAL partition aside from data partition pre-configured.
+
+
+Partition and Reactor thread
+----------------------------
+In early stage development, PoseidonStore will employ static allocation of partition. The number of sharded partitions
+is fixed and the size of each partition also should be configured before running cluster.
+But, the number of partitions can grow as below. We leave this as a future work.
+Also, each reactor thread has a static set of SPs.
+
+.. ditaa::
+
+   +------+------+-------------+------------------+
+   | SP 1 | SP N | -->     <-- | global partition |
+   +------+------+-------------+------------------+
+
+
+
+Cache
+-----
+There are mainly two cache data structures; onode cache and block cache.
+It looks like below.
+
+#. Onode cache:
+   lru_map <OID, OnodeRef>;
+#. Block cache (data and omap):
+   Data cache --> lru_map <paddr, value>
+
+To fill the onode data structure, the target onode needs to be retrieved using the prefix tree.
+Block cache is used for caching a block contents. For a transaction, all the updates to blocks
+(including object meta block, data block) are first performed in the in-memory block cache.
+After writing a transaction to the WAL, the dirty blocks are flushed to their respective locations in the
+respective partitions.
+PoseidonStore can configure cache size for each type. Simple LRU cache eviction strategy can be used for both.
+
+
+Sharded partitions (with cross-SP transaction)
+----------------------------------------------
+The entire disk space is divided into a number of chunks called sharded partitions (SP).
+The prefixes of the parent collection ID (original collection ID before collection splitting. That is, hobject.hash)
+is used to map any collections to SPs.
+We can use BlueStore's approach for collection splitting, changing the number of significant bits for the collection prefixes.
+Because the prefixes of the parent collection ID do not change even after collection splitting, the mapping between
+the collection and SP are maintained.
+The number of SPs may be configured to match the number of CPUs allocated for each disk so that each SP can hold
+a number of objects large enough for cross-SP transaction not to occur.
+
+In case of need of cross-SP transaction, we could use the global WAL. The coordinator thread (mainly manages global partition) handles
+cross-SP transaction via acquire the source SP and target SP locks before processing the cross-SP transaction.
+Source and target probably are blocked.
+
+For the load unbalanced situation,
+Poseidonstore can create partitions to make full use of entire space efficiently and provide load balaning.
+
+
+CoW/Clone
+---------
+As for CoW/Clone, a clone has its own onode like other normal objects.
+
+Although each clone has its own onode, data blocks should be shared between the original object and clones
+if there are no changes on them to minimize the space overhead.
+To do so, the reference count for the data blocks is needed to manage those shared data blocks.
+
+To deal with the data blocks which has the reference count, poseidon store makes use of shared_blob
+which maintains the referenced data block.
+
+As shown the figure as below,
+the shared_blob tracks the data blocks shared between other onodes by using a reference count.
+The shared_blobs are managed by shared_blob_list in the superblock.
+
+
+.. ditaa::
+
+
+    /----------\         /----------\
+    | Object A |         | Object B |
+    +----------+         +----------+
+    | Extent   |         | Extent   |
+    +---+--+---+         +--+----+--+
+        |  |                |    |
+        |  |     +----------+    |
+        |  |     |               |
+        |  +---------------+     |
+        |        |         |     |
+        v        v         v     v
+    +---------------+---------------+
+    | Data block 1  | Data block 2  |
+    +-------+-------+------+--------+
+            |              |
+            v              v
+    /---------------+---------------\
+    | shared_blob 1 | shared_blob 2 |
+    +---------------+---------------+ shared_blob_list
+    | refcount      | refcount      |
+    +---------------+---------------+
+
+Plans
+=====
+
+All PRs should contain unit tests to verify its minimal functionality.
+
+* WAL and block cache implementation
+
+  As a first step, we are going to build the WAL including the I/O procedure to read/write the WAL.
+  With WAL development, the block cache needs to be developed together.
+  Besides, we are going to add an I/O library to read/write from/to the NVMe storage to
+  utilize NVMe feature and the asynchronous interface.
+
+* Radix tree and onode
+
+  First, submit a PR against this file with a more detailed on disk layout and lookup strategy for the onode radix tree.
+  Follow up with implementation based on the above design once design PR is merged.
+  The second PR will be the implementation regarding radix tree which is the key structure to look up
+  objects.
+
+* Extent tree
+
+  This PR is the extent tree to manage data blocks in the onode. We build the extent tree, and
+  demonstrate how it works when looking up the object.
+
+* B+tree for omap
+
+  We will put together a simple key/value interface for omap. This probably will be a separate PR.
+
+* CoW/Clone
+
+  To support CoW/Clone, shared_blob and shared_blob_list will be added.
+
+* Integration to Crimson as to I/O interfaces
+
+  At this stage, interfaces for interacting with Crimson such as queue_transaction(), read(), clone_range(), etc.
+  should work right.
+
+* Configuration
+
+  We will define Poseidon store configuration in detail.
+
+* Stress test environment and integration to teuthology
+
+  We will add stress tests and teuthology suites.
+
+.. rubric:: Footnotes
+
+.. [#f1] Stathis Maneas, Kaveh Mahdaviani, Tim Emami, Bianca Schroeder: A Study of SSD Reliability in Large Scale Enterprise Storage Deployments. FAST 2020: 137-149
diff --git a/doc/dev/cxx.rst b/doc/dev/cxx.rst
new file mode 100644
index 000000000..e8ab9d4de
--- /dev/null
+++ b/doc/dev/cxx.rst
@@ -0,0 +1,27 @@
+C++17 and libstdc++ ABI
+=======================
+
+Ceph has switched over to C++17 in mimic. To build Ceph on old distros without
+GCC-7, it is required to install GCC-7 from additionary repos. On RHEL/CentOS,
+we are using devtoolset-7_ from SCLs_ for building Ceph. But devltoolset-7 is
+always using the old ABI_ even if ``_GLIBCXX_USE_CXX11_ABI=1`` is defined. So,
+on RHEL/CentOS, the old implementations of ``std::string`` and ``std::list``
+are still used. In other words, ``std::string`` is still copy-on-write, and
+``std::list::size()`` is still O(n) on these distros. But on Ubuntu Xenial,
+Ceph is built using the new ABI. So, because we are still using libstdc++ and
+devtoolset for building packages on RHEL/CentOS, please do not rely on the
+behavior of the new ABI or the old one.
+
+For those who argue that "GCC supports dual ABI!", here comes the long story.
+The problem is in the system shared library and ``libstdc++_nonshared.a`` model.
+If some symbol is exported from the system shared library, we must use that, and
+cannot override it. Also, the dual ABI support requires several of the system
+shared library symbols to behave differently (e.g. for locale facets, need
+to register twice as many, one set for old ABI, another for new ABI). So, this
+leaves us with no options but to stick with the old ABI, if we want to enable
+the built binaries to run on old distros where only the libstdc++ with the old
+ABI is available.
+
+.. _ABI: https://gcc.gnu.org/onlinedocs/libstdc++/manual/using_dual_abi.html
+.. _devtoolset-7: https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7/
+.. _SCLs: https://www.softwarecollections.org/
diff --git a/doc/dev/dashboard/ui_goals.rst b/doc/dev/dashboard/ui_goals.rst
new file mode 100644
index 000000000..4e68ec1f5
--- /dev/null
+++ b/doc/dev/dashboard/ui_goals.rst
@@ -0,0 +1,78 @@
+===========================
+Ceph Dashboard Design Goals
+===========================
+
+.. note:: This document is intended to provide a focal point for discussing the overall design
+   principles for mgr/dashboard
+
+Introduction
+============
+
+Most distributed storage architectures are inherently complex and can present a management challenge
+to Operations teams who are typically stretched across multiple product and platform disciplines. In
+general terms, the complexity of any solution can have a direct bearing on the operational costs
+incurred to manage it. The answer is simple...make it simple :)
+
+This document is intended to highlight Ceph Dashboard design goals which may help to  
+
+* reduce complexity
+* increase productivity
+* improve time-to-value
+* increase observability
+
+ 
+Understanding the Persona of the Target User
+============================================
+
+Ceph has historically been administered from the CLI. The CLI has always and will always offer the
+richest, most flexible way to install and manage a Ceph cluster. Administrators who require and
+demand this level of control are unlikely to adopt a UI for anything more than a technical curiosity.
+
+The relevance of the UI is therefore more critical for a new SysAdmin, where it can help technology
+adoption and reduce the operational friction that is normally experienced when implementing a new
+solution.
+
+Understanding the target user persona is therefore a fundamental first step in design. Attempting to
+design a UI that meets the requirements of a 'seasoned' Ceph Administrator or Developer, and a
+relatively new SysAdmin is unlikely to satisfy either user group.
+
+Design Principles
+=================
+
+Key Principles
+______________
+
+
+#. **Clarity and consistency**. The UI should ensure the data shown is unambiguous and consistent across
+   different views
+#. **Data timeliness**. Data displayed in the UI must be timely. State information **must** be reasonably
+   recent for it to be relevant and acted upon with confidence. In addition, the age of the data should
+   be shown as age (e.g. 20s ago) rather than UTC timestamps to make it more immediately consumable by
+   the Administrator.
+#. **Automate through workflows**. If the admin has to follow a 'recipe' to perform a task, the goal of
+   the dashboard UI should be to implement the flow.
+#. **Provide a natural next step**. The UI **is** the *expert system*, so instead of expecting the user
+   to know where they go next, the UI should lead them. This means linking components together to
+   establish a flow and deeper integration between the alertmanager implementation and the dashboard
+   elements enabling an Admin to efficiently step from alert to affected component.
+#. **Platform visibility**. The platform (OS and hardware configuration) is a fundamental component of the
+   solution, so providing platform level insights can help deliver a more holistic view of the Ceph cluster.
+#. **Jargon Busting**. Jargon is an unavoidable component of most systems. However, a good system will
+   include inline help to support new and infrequent users of the UI.
+
+
+Common Pitfalls
+_______________
+
+* Don't re-implement CLI commands in the UI. The sysadmin will likely use the CLI primitives in scripts
+  to automate tasks, so by simply adding a CLI feature we miss the workflow and add complexity, which
+  potentially 'bloats' the UI.
+* Don't think like a developer...try and adopt the mindset of an Administrator, who only works with the
+  Ceph cluster part-time - this is the reality for today's Operations teams.
+
+
+Focus On User Experience
+========================
+Ultimately, the goal must be to move away from pushing complexity onto the GUI user through multi-step
+workflows like iSCSI configuration or setting specific cluster flags in defined sequences. Simplicity
+should be the goal for the UI...let's leave the complexity to the CLI.
diff --git a/doc/dev/delayed-delete.rst b/doc/dev/delayed-delete.rst
new file mode 100644
index 000000000..31f3e6b97
--- /dev/null
+++ b/doc/dev/delayed-delete.rst
@@ -0,0 +1,13 @@
+=========================
+ CephFS delayed deletion
+=========================
+
+The deletion of a file does not immediately remove its data. Each of the file's
+underlying objects must be removed independently. If these objects were removed 
+immediately, the client would have to send ``size_of_file / stripe_size *
+replication_count`` messages. This would consume significant bandwith and would
+slow the client unacceptably. If snapshots exist, their existence can prevent
+the deletion of objects associated with them.
+
+In these cases, such files are (1) marked as deleted on the MDS and (2) deleted
+lazily.
diff --git a/doc/dev/dev_cluster_deployement.rst b/doc/dev/dev_cluster_deployement.rst
new file mode 100644
index 000000000..69185e7f0
--- /dev/null
+++ b/doc/dev/dev_cluster_deployement.rst
@@ -0,0 +1,169 @@
+=================================
+ Deploying a development cluster
+=================================
+
+In order to develop on ceph, a Ceph utility,
+*vstart.sh*, allows you to deploy fake local cluster for development purpose.
+
+Usage
+=====
+
+It allows to deploy a fake local cluster on your machine for development purpose. It starts rgw, mon, osd and/or mds, or all of them if not specified.
+
+To start your development cluster, type the following::
+
+	vstart.sh [OPTIONS]...
+
+In order to stop the cluster, you can type::
+
+	./stop.sh
+
+Options
+=======
+
+.. option:: -b, --bluestore
+
+    Use bluestore as the objectstore backend for osds.
+
+.. option:: --cache <pool>
+
+    Set a cache-tier for the specified pool.
+
+.. option:: -d, --debug
+
+    Launch in debug mode.
+
+.. option:: -e
+
+    Create an erasure pool.
+
+.. option:: -f, --filestore
+
+    Use filestore as the osd objectstore backend.
+
+.. option:: --hitset <pool> <hit_set_type>
+
+    Enable hitset tracking.
+
+.. option:: -i ip_address
+
+    Bind to the specified *ip_address* instead of guessing and resolve from hostname.
+
+.. option:: -k
+
+    Keep old configuration files instead of overwritting theses.
+
+.. option:: -K, --kstore
+
+    Use kstore as the osd objectstore backend.
+
+.. option:: -l, --localhost
+
+    Use localhost instead of hostname.
+
+.. option:: -m ip[:port]
+
+    Specifies monitor *ip* address and *port*.
+
+.. option:: --memstore
+
+    Use memstore as the objectstore backend for osds
+
+.. option:: --multimds <count>
+
+    Allow multimds with maximum active count.
+
+.. option:: -n, --new
+
+    Create a new cluster.
+
+.. option:: -N, --not-new
+
+    Reuse existing cluster config (default).
+
+.. option:: --nodaemon
+
+    Use ceph-run as wrapper for mon/osd/mds.
+
+.. option:: --nolockdep
+
+    Disable lockdep
+
+.. option:: -o <config>
+
+    Add *config* to all sections in the ceph configuration.
+
+.. option:: --rgw_port <port>
+
+    Specify ceph rgw http listen port.
+
+.. option:: --rgw_frontend <frontend>
+
+    Specify the rgw frontend configuration (default is civetweb).
+
+.. option:: --rgw_compression <compression_type>
+
+    Specify the rgw compression plugin (default is disabled).
+
+.. option:: --smallmds
+
+    Configure mds with small limit cache size.
+
+.. option:: --short
+
+    Short object names only; necessary for ext4 dev
+
+.. option:: --valgrind[_{osd,mds,mon}] 'valgrind_toolname [args...]'
+
+    Launch the osd/mds/mon/all the ceph binaries using valgrind with the specified tool and arguments.
+
+.. option:: --without-dashboard
+
+    Do not run using mgr dashboard.
+
+.. option:: -x
+
+    Enable cephx (on by default).
+
+.. option:: -X
+
+    Disable cephx.
+
+
+Environment variables
+=====================
+
+{OSD,MDS,MON,RGW}
+
+Theses environment variables will contains the number of instances of the desired ceph process you want to start.
+
+Example: ::
+
+	OSD=3 MON=3 RGW=1 vstart.sh
+
+
+============================================================
+ Deploying multiple development clusters on the same machine
+============================================================
+
+In order to bring up multiple ceph clusters on the same machine, *mstart.sh* a
+small wrapper around the above *vstart* can help.
+
+Usage
+=====
+
+To start multiple clusters, you would run mstart for each cluster you would want
+to deploy, and it will start monitors, rgws for each cluster on different ports
+allowing you to run multiple mons, rgws etc. on the same cluster. Invoke it in
+the following way::
+
+  mstart.sh <cluster-name> <vstart options>
+
+For eg::
+
+  ./mstart.sh cluster1 -n
+
+
+For stopping the cluster, you do::
+
+  ./mstop.sh <cluster-name>
diff --git a/doc/dev/developer_guide/basic-workflow.rst b/doc/dev/developer_guide/basic-workflow.rst
new file mode 100644
index 000000000..5917b56be
--- /dev/null
+++ b/doc/dev/developer_guide/basic-workflow.rst
@@ -0,0 +1,515 @@
+.. _basic workflow dev guide:
+
+Basic Workflow
+==============
+
+The following chart illustrates the basic Ceph development workflow:
+
+.. ditaa::
+
+            Upstream Code                       Your Local Environment
+
+           /----------\        git clone           /-------------\
+           |   Ceph   | -------------------------> | ceph/main   |
+           \----------/                            \-------------/
+                ^                                    |
+                |                                    | git branch fix_1
+                | git merge                          |
+                |                                    v
+           /----------------\  git commit --amend   /-------------\
+           |  make check    |---------------------> | ceph/fix_1  |
+           | ceph--qa--suite|                       \-------------/
+           \----------------/                        |
+                ^                                    | fix changes
+                |                                    | test changes
+                | review                             | git commit
+                |                                    |
+                |                                    v
+           /--------------\                        /-------------\
+           |   github     |<---------------------- | ceph/fix_1  |
+           | pull request |         git push       \-------------/
+           \--------------/
+
+This page assumes that you are a new contributor with an idea for a bugfix or
+an enhancement, but you do not know how to proceed. Watch the `Getting Started
+with Ceph Development <https://www.youtube.com/watch?v=t5UIehZ1oLs>`_ video for
+a practical summary of this workflow.
+
+Updating the tracker
+--------------------
+
+Find the :ref:`issue-tracker` (Redmine) number of the bug you intend to fix. If
+no tracker issue exists, create one. There is only one case in which you do not
+have to create a Redmine tracker issue: the case of minor documentation changes.
+
+Simple documentation cleanup does not require a corresponding tracker issue.
+Major documenatation changes do require a tracker issue. Major documentation
+changes include adding new documentation chapters or files, and making 
+substantial changes to the structure or content of the documentation.
+
+A (Redmine) tracker ticket explains the issue (bug) to other Ceph developers to
+keep them informed as the bug nears resolution. Provide a useful, clear title
+and include detailed information in the description. When composing the title
+of the ticket, ask yourself "If I need to search for this ticket two years from
+now, which keywords am I likely to search for?" Then include those keywords in
+the title.
+
+If your tracker permissions are elevated, assign the bug to yourself by setting
+the ``Assignee`` field. If your tracker permissions have not been elevated,
+just add a comment with a short message that says "I am working on this issue".
+
+Ceph Workflow Overview
+----------------------
+
+Three repositories are involved in the Ceph workflow. They are:
+
+1. The upstream repository (ceph/ceph)
+2. Your fork of the upstream repository (your_github_id/ceph)
+3. Your local working copy of the repository (on your workstation)
+
+The procedure for making changes to the Ceph repository is as follows:
+
+#. Configure your local environment
+
+   #. :ref:`Create a fork<forking>` of the "upstream Ceph"
+      repository.
+
+   #. :ref:`Clone the fork<cloning>` to your local filesystem.
+
+#. Fix the bug
+
+   #. :ref:`Synchronize local main with upstream main<synchronizing>`.
+         
+   #. :ref:`Create a bugfix branch<bugfix_branch>` in your local working copy.
+         
+   #. :ref:`Make alterations to the local working copy of the repository in your
+      local filesystem<fixing_bug_locally>`.
+   
+   #. :ref:`Push the changes in your local working copy to your fork<push_changes>`.
+
+#. Create a Pull Request to push the change upstream
+
+   #. Create a Pull Request that asks for your changes to be added into the
+      "upstream Ceph" repository.
+
+Preparing Your Local Working Copy of the Ceph Repository 
+--------------------------------------------------------
+
+The procedures in this section, "Preparing Your Local Working Copy of the Ceph
+Repository", must be followed only when you are first setting up your local
+environment. If this is your first time working with the Ceph project, then
+these commands are necessary and are the first commands that you should run.
+
+.. _forking:
+
+Creating a Fork of the Ceph Repository
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+See the `GitHub documentation
+<https://help.github.com/articles/fork-a-repo/#platform-linux>`_ for
+detailed instructions on forking. In short, if your GitHub username is
+"mygithubaccount", your fork of the upstream repo will appear at
+``https://github.com/mygithubaccount/ceph``. 
+
+.. _cloning:
+
+Cloning Your Fork  
+^^^^^^^^^^^^^^^^^
+
+After you have created your fork, clone it by running the following command:
+
+.. prompt:: bash $
+
+   git clone https://github.com/mygithubaccount/ceph
+
+You must fork the Ceph repository before you clone it.  If you fail to fork,
+you cannot open a `GitHub pull request
+<https://docs.github.com/en/free-pro-team@latest/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request>`_.
+
+For more information on using GitHub, refer to `GitHub Help
+<https://help.github.com/>`_.
+
+Configuring Your Local Environment
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The commands in this section configure your local git environment so that it
+generates "Signed-off-by:" tags. These commands also set up your local
+environment so that it can stay synchronized with the upstream repository.
+
+These commands are necessary only during the initial setup of your local
+working copy. Another way to say that is "These commands are necessary
+only the first time that you are working with the Ceph repository. They are,
+however, unavoidable, and if you fail to run them then you will not be able
+to work on the Ceph repository.".
+
+1. Configure your local git environment with your name and email address.  
+
+   .. note::
+      These commands will work only from within the ``ceph/`` directory
+      that was created when you cloned your fork.
+
+   .. prompt:: bash $
+
+      git config user.name "FIRST_NAME LAST_NAME"
+      git config user.email "MY_NAME@example.com"
+
+2. Add the upstream repo as a "remote" and fetch it:
+
+   .. prompt:: bash $
+
+      git remote add ceph https://github.com/ceph/ceph.git
+      git fetch ceph
+
+   These commands fetch all the branches and commits from ``ceph/ceph.git`` to
+   the local git repo as ``remotes/ceph/$BRANCH_NAME`` and can be referenced as
+   ``ceph/$BRANCH_NAME`` in local git commands.
+
+Fixing the Bug
+--------------
+
+.. _synchronizing:
+
+Synchronizing Local Main with Upstream Main
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In your local working copy, there is a copy of the ``main`` branch in
+``remotes/origin/main``. This is called "local main". This copy of the
+main branch (https://github.com/your_github_id/ceph.git) is "frozen in time"
+at the moment that you cloned it, but the upstream repo
+(https://github.com/ceph/ceph.git, typically abbreviated to ``ceph/ceph.git``)
+that it was forked from is not frozen in time: the upstream repo is still being
+updated by other contributors. 
+
+Because upstream main is continually receiving updates from other
+contributors, your fork will drift farther and farther from the state of the
+upstream repo when you cloned it.
+
+Keep your fork's ``main`` branch synchronized with upstream main to reduce drift
+between your fork's main branch and the upstream main branch.
+
+Here are the commands for keeping your fork synchronized with the
+upstream repository:
+
+.. prompt:: bash $
+
+   git fetch ceph
+   git checkout main 
+   git reset --hard ceph/main
+   git push -u origin main
+
+Follow this procedure often to keep your local ``main`` in sync with upstream
+``main``.
+
+If the command ``git status`` returns a line that reads "Untracked files", see
+:ref:`the procedure on updating submodules <update-submodules>`.
+
+.. _bugfix_branch:
+
+Creating a Bugfix branch
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Create a branch for your bugfix:
+
+.. prompt:: bash $
+
+   git checkout main 
+   git checkout -b fix_1
+   git push -u origin fix_1
+
+The first command (git checkout main) makes sure that the bugfix branch
+"fix_1" is created from the most recent state of the main branch of the
+upstream repository. 
+
+The second command (git checkout -b fix_1) creates a "bugfix branch" called
+"fix_1" in your local working copy of the repository. The changes that you make
+in order to fix the bug will be commited to this branch.
+
+The third command (git push -u origin fix_1) pushes the bugfix branch from
+your local working repository to your fork of the upstream repository.
+
+.. _fixing_bug_locally:
+
+Fixing the bug in the local working copy
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+#. **Updating the tracker**
+
+   In the `Ceph issue tracker <https://tracker.ceph.com>`_, change the status
+   of the tracker issue to "In progress".  This communicates to other Ceph
+   contributors that you have begun working on a fix, which helps to avoid
+   duplication of effort. If you don't have permission to change that field,
+   just comment that you are working on the issue. 
+
+#. **Fixing the bug itself**
+
+   This guide cannot tell you how to fix the bug that you have chosen to fix.
+   This guide assumes that you know what required improvement, and that you
+   know what to do to provide that improvement.
+
+   It might be that your fix is simple and requires only minimal testing. But
+   that's unlikely. It is more likely that the process of fixing your bug will
+   be iterative and will involve trial, error, skill, and patience. 
+
+   For a detailed discussion of the tools available for validating bugfixes,
+   see the chapters on testing.
+
+Pushing the Fix to Your Fork
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   
+You have finished work on the bugfix. You have tested the bugfix, and you
+believe that it works. 
+   
+#. Commit the changes to your local working copy.
+
+   Commit the changes to the `fix_1` branch of your local working copy by using
+   the ``--signoff`` option (here represented as the `s` portion of the `-as`
+   flag): 
+
+   .. prompt:: bash $
+
+      git commit -as
+
+   .. _push_changes:
+
+#. Push the changes to your fork:
+
+   Push the changes from the `fix_1` branch of your local working copy to the
+   `fix_1` branch of your fork of the upstream repository:
+
+   .. prompt:: bash $
+
+      git push origin fix_1
+   
+   .. note::
+
+      In the command ``git push origin fix_1``, ``origin`` is the name of your
+      fork of the upstream Ceph repository, and can be thought of as a nickname
+      for ``git@github.com:username/ceph.git``, where ``username`` is your
+      GitHub username.
+
+      It is possible that ``origin`` is not the name of your fork. Discover the
+      name of your fork by running ``git remote -v``, as shown here:
+
+      .. code-block:: bash
+
+         $ git remote -v
+         ceph	https://github.com/ceph/ceph.git (fetch)
+         ceph	https://github.com/ceph/ceph.git (push)
+         origin	git@github.com:username/ceph.git (fetch)
+         origin	git@github.com:username/ceph.git (push)
+
+      The line::
+     
+         origin git@github.com:username/ceph.git (fetch) 
+      
+      and the line:: 
+        
+         origin git@github.com:username/ceph.git (push) 
+         
+      provide the information that "origin" is the name of your fork of the
+      Ceph repository.
+
+
+Opening a GitHub pull request
+-----------------------------
+
+After you have pushed the bugfix to your fork, open a GitHub pull request
+(PR). This makes your bugfix visible to the community of Ceph contributors.
+They will review it. They may perform additional testing on your bugfix, and
+they might request changes to the bugfix.
+
+Be prepared to receive suggestions and constructive criticism in the form of
+comments within the PR. 
+
+If you don't know how to create and manage pull requests, read `this GitHub
+pull request tutorial`_.
+
+.. _`this GitHub pull request tutorial`:
+   https://help.github.com/articles/using-pull-requests/
+
+To learn what constitutes a "good" pull request, see
+the `Git Commit Good Practice`_ article at the `OpenStack Project Wiki`_.
+
+.. _`Git Commit Good Practice`: https://wiki.openstack.org/wiki/GitCommitMessages
+.. _`OpenStack Project Wiki`: https://wiki.openstack.org/wiki/Main_Page
+
+See also our own `Submitting Patches
+<https://github.com/ceph/ceph/blob/main/SubmittingPatches.rst>`_ document.
+
+After your pull request (PR) has been opened, update the :ref:`issue-tracker`
+by adding a comment directing other contributors to your PR. The comment can be
+as simple as this::
+
+    *PR*: https://github.com/ceph/ceph/pull/$NUMBER_OF_YOUR_PULL_REQUEST
+
+Understanding Automated PR validation
+-------------------------------------
+
+When you create or update your PR, the Ceph project's `Continuous Integration
+(CI) <https://en.wikipedia.org/wiki/Continuous_integration>`_ infrastructure
+automatically tests it. At the time of this writing (May 2022), the automated
+CI testing included many tests. These five are among them:
+
+#. a test to check that the commits are properly signed (see :ref:`submitting-patches`): 
+#. a test to check that the documentation builds
+#. a test to check that the submodules are unmodified
+#. a test to check that the API is in order
+#. a :ref:`make check<make-check>` test
+
+Additional tests may be run depending on which files your PR modifies.
+
+The :ref:`make check<make-check>` test builds the PR and runs it through a
+battery of tests. These tests run on servers that are operated by the Ceph
+Continuous Integration (CI) team. When the tests have completed their run, the
+result is shown on GitHub in the pull request itself.
+
+Test your modifications before you open a PR.  Refer to the chapters
+on testing for details.
+
+Notes on PR make check test
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The GitHub :ref:`make check<make-check>` test is driven by a Jenkins instance.
+
+Jenkins merges your PR branch into the latest version of the base branch before
+it starts any tests. This means that you don't have to rebase the PR in order
+to pick up any fixes.
+
+You can trigger PR tests at any time by adding a comment to the PR - the
+comment should contain the string "test this please". Since a human who is
+subscribed to the PR might interpret that as a request for him or her to test
+the PR, you must address Jenkins directly. For example, write "jenkins retest
+this please". If you need to run only one of the tests, you can request it with
+a command like "jenkins test signed". A list of these requests is automatically
+added to the end of each new PR's description, so check there to find the
+single test you need.
+
+If there is a build failure and you aren't sure what caused it, check the
+:ref:`make check<make-check>` log. To access the make check log, click the
+"details" (next to the :ref:`make check<make-check>` test in the PR) link to
+enter the Jenkins web GUI. Then click "Console Output" (on the left).
+
+Jenkins is configured to search logs for strings that are known to have been
+associated with :ref:`make check<make-check>` failures in the past. However,
+there is no guarantee that these known strings are associated with any given
+:ref:`make check<make-check>` failure. You'll have to read through the log to
+determine the cause of your specific failure.
+
+Integration tests AKA ceph-qa-suite
+-----------------------------------
+
+Since Ceph is complex, it may be necessary to test your fix to
+see how it behaves on real clusters running on physical or virtual
+hardware. Tests designed for this purpose live in the `ceph/qa
+sub-directory`_ and are run via the `teuthology framework`_.
+
+.. _`ceph/qa sub-directory`: https://github.com/ceph/ceph/tree/main/qa/
+.. _`teuthology repository`: https://github.com/ceph/teuthology
+.. _`teuthology framework`: https://github.com/ceph/teuthology
+
+The Ceph community has access to the `Sepia lab
+<https://wiki.sepia.ceph.com/doku.php>`_ where :ref:`testing-integration-tests` can be
+run on physical hardware. Other developers may add tags like "needs-qa" to your
+PR. This allows PRs that need testing to be merged into a single branch and
+tested all at the same time. Since teuthology suites can take hours (even
+days in some cases) to run, this can save a lot of time.
+
+To request access to the Sepia lab, start `here <https://wiki.sepia.ceph.com/doku.php?id=vpnaccess>`_.
+
+Integration testing is discussed in more detail in the :ref:`testing-integration-tests`
+chapter.
+
+Code review
+-----------
+
+Once your bugfix has been thoroughly tested, or even during this process,
+it will be subjected to code review by other developers. This typically
+takes the form of comments in the PR itself, but can be supplemented
+by discussions on :ref:`irc` and the :ref:`mailing-list`.
+
+Amending your PR
+----------------
+
+While your PR is going through testing and `Code Review`_, you can
+modify it at any time by editing files in your local branch.
+
+After updates are committed locally (to the ``fix_1`` branch in our
+example), they need to be pushed to GitHub so they appear in the PR.
+
+Modifying the PR is done by adding commits to the ``fix_1`` branch upon
+which it is based, often followed by rebasing to modify the branch's git
+history. See `this tutorial
+<https://www.atlassian.com/git/tutorials/rewriting-history>`_ for a good
+introduction to rebasing. When you are done with your modifications, you
+will need to force push your branch with:
+
+.. prompt:: bash $
+
+   git push --force origin fix_1
+
+Why do we take these extra steps instead of simply adding additional commits
+the PR?  It is best practice for a PR to consist of a single commit; this
+makes for clean history, eases peer review of your changes, and facilitates
+merges.  In rare circumstances it also makes it easier to cleanly revert
+changes.
+
+Merging
+-------
+
+The bugfix process completes when a project lead merges your PR.
+
+When this happens, it is a signal for you (or the lead who merged the PR)
+to change the :ref:`issue-tracker` status to "Resolved". Some issues may be
+flagged for backporting, in which case the status should be changed to
+"Pending Backport" (see the :ref:`backporting` chapter for details).
+
+See also :ref:`merging` for more information on merging.
+
+Proper Merge Commit Format
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This is the most basic form of a merge commit::
+
+       doc/component: title of the commit 
+
+       Reviewed-by: Reviewer Name <rname@example.com>
+
+This consists of two parts:
+
+#. The title of the commit / PR to be merged.
+#. The name and email address of the reviewer. Enclose the reviewer's email 
+   address in angle brackets.
+
+Using .githubmap to Find a Reviewer's Email Address
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+If you cannot find the email address of the reviewer on his or her GitHub
+page, you can look it up in the **.githubmap** file, which can be found in
+the repository at **/ceph/.githubmap**.
+
+Using "git log" to find a Reviewer's Email Address
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+If you cannot find a reviewer's email address by using the above methods, you
+can search the git log for their email address. Reviewers are likely to have
+committed something before.  If they have made previous contributions, the git
+log will probably contain their email address.
+
+Use the following command
+
+.. prompt:: bash [branch-under-review]$
+
+   git log
+
+Using ptl-tool to Generate Merge Commits
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Another method of generating merge commits involves using Patrick Donnelly's
+**ptl-tool** pull commits. This tool can be found at
+**/ceph/src/script/ptl-tool.py**.  Merge commits that have been generated by
+the **ptl-tool** have the following form::
+
+     Merge PR #36257 into main 
+     * refs/pull/36257/head:
+             client: move client_lock to _unmount()
+             client: add timer_lock support
+     Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
+
diff --git a/doc/dev/developer_guide/dash-devel.rst b/doc/dev/developer_guide/dash-devel.rst
new file mode 100644
index 000000000..29974555a
--- /dev/null
+++ b/doc/dev/developer_guide/dash-devel.rst
@@ -0,0 +1,2590 @@
+.. _dashdevel:
+
+Ceph Dashboard Developer Documentation
+======================================
+
+.. contents:: Table of Contents
+
+Feature Design
+--------------
+
+To promote collaboration on new Ceph Dashboard features, the first step is
+the definition of a design document. These documents then form the basis of
+implementation scope and permit wider participation in the evolution of the
+Ceph Dashboard UI.
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Design Documents:
+
+    UI Design Goals <../dashboard/ui_goals>
+
+
+Preliminary Steps
+-----------------
+
+The following documentation chapters expect a running Ceph cluster and at
+least a running ``dashboard`` manager module (with few exceptions). This
+chapter gives an introduction on how to set up such a system for development,
+without the need to set up a full-blown production environment. All options
+introduced in this chapter are based on a so called ``vstart`` environment.
+
+.. note::
+
+  Every ``vstart`` environment needs Ceph `to be compiled`_ from its Github
+  repository, though Docker environments simplify that step by providing a
+  shell script that contains those instructions.
+
+  One exception to this rule are the `build-free`_ capabilities of
+  `ceph-dev`_. See below for more information.
+
+.. _to be compiled: https://docs.ceph.com/docs/master/install/build-ceph/
+
+vstart
+~~~~~~
+
+"vstart" is actually a shell script in the ``src/`` directory of the Ceph
+repository (``src/vstart.sh``). It is used to start a single node Ceph
+cluster on the machine where it is executed. Several required and some
+optional Ceph internal services are started automatically when it is used to
+start a Ceph cluster. vstart is the basis for the three most commonly used
+development environments in Ceph Dashboard.
+
+You can read more about vstart in `Deploying a development cluster`_.
+Additional information for developers can also be found in the `Developer
+Guide`_.
+
+.. _Deploying a development cluster: https://docs.ceph.com/docs/master/dev/dev_cluster_deployement/
+.. _Developer Guide: https://docs.ceph.com/docs/master/dev/quick_guide/
+
+Host-based vs Docker-based Development Environments
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This document introduces you to three different development environments, all
+based on vstart. Those are:
+
+- vstart running on your host system
+
+- vstart running in a Docker environment
+
+  * ceph-dev-docker_
+  * ceph-dev_
+
+  Besides their independent development branches and sometimes slightly
+  different approaches, they also differ with respect to their underlying
+  operating systems.
+
+  ========= ======================  ========
+  Release   ceph-dev-docker         ceph-dev
+  ========= ======================  ========
+  Mimic     openSUSE Leap 15        CentOS 7
+  Nautilus  openSUSE Leap 15        CentOS 7
+  Octopus   openSUSE Leap 15.2      CentOS 8
+  --------- ----------------------  --------
+  Master    openSUSE Tumbleweed     CentOS 8
+  ========= ======================  ========
+
+.. note::
+
+  Independently of which of these environments you will choose, you need to
+  compile Ceph in that environment. If you compiled Ceph on your host system,
+  you would have to recompile it on Docker to be able to switch to a Docker
+  based solution. The same is true vice versa. If you previously used a
+  Docker development environment and compiled Ceph there and you now want to
+  switch to your host system, you will also need to recompile Ceph (or
+  compile Ceph using another separate repository).
+
+  `ceph-dev`_ is an exception to this rule as one of the options it provides
+  is `build-free`_. This is accomplished through a Ceph installation using
+  RPM system packages. You will still be able to work with a local Github
+  repository like you are used to.
+
+
+Development environment on your host system
+...........................................
+
+- No need to learn or have experience with Docker, jump in right away.
+
+- Limited amount of scripts to support automation (like Ceph compilation).
+
+- No pre-configured easy-to-start services (Prometheus, Grafana, etc).
+
+- Limited amount of host operating systems supported, depending on which
+  Ceph version is supposed to be used.
+
+- Dependencies need to be installed on your host.
+
+- You might find yourself in the situation where you need to upgrade your
+  host operating system (for instance due to a change of the GCC version used
+  to compile Ceph).
+
+
+Development environments based on Docker
+........................................
+
+- Some overhead in learning Docker if you are not used to it yet.
+
+- Both Docker projects provide you with scripts that help you getting started
+  and automate recurring tasks.
+
+- Both Docker environments come with partly pre-configured external services
+  which can be used to attach to or complement Ceph Dashboard features, like
+
+  - Prometheus
+  - Grafana
+  - Node-Exporter
+  - Shibboleth
+  - HAProxy
+
+- Works independently of the operating system you use on your host.
+
+
+.. _build-free: https://github.com/rhcs-dashboard/ceph-dev#quick-install-rpm-based
+
+vstart on your host system
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The vstart script is usually called from your `build/` directory like so:
+
+.. code::
+
+  ../src/vstart.sh -n -d
+
+In this case ``-n`` ensures that a new vstart cluster is created and that a
+possibly previously created cluster isn't re-used. ``-d`` enables debug
+messages in log files. There are several more options to chose from. You can
+get a list using the ``--help`` argument.
+
+At the end of the output of vstart, there should be information about the
+dashboard and its URLs::
+
+  vstart cluster complete. Use stop.sh to stop. See out/* (e.g. 'tail -f out/????') for debug output.
+
+  dashboard urls: https://192.168.178.84:41259, https://192.168.178.84:43259, https://192.168.178.84:45259
+    w/ user/pass: admin / admin
+  restful urls: https://192.168.178.84:42259, https://192.168.178.84:44259, https://192.168.178.84:46259
+    w/ user/pass: admin / 598da51f-8cd1-4161-a970-b2944d5ad200
+
+During development (especially in backend development), you also want to
+check on occasions if the dashboard manager module is still running. To do so
+you can call `./bin/ceph mgr services` manually. It will list all the URLs of
+successfully enabled services. Only URLs of services which are available over
+HTTP(S) will be listed there. Ceph Dashboard is one of these services. It
+should look similar to the following output:
+
+.. code::
+
+  $ ./bin/ceph mgr services
+  {
+      "dashboard": "https://home:41931/",
+      "restful": "https://home:42931/"
+  }
+
+By default, this environment uses a randomly chosen port for Ceph Dashboard
+and you need to use this command to find out which one it has become.
+
+Docker
+~~~~~~
+
+Docker development environments usually ship with a lot of useful scripts.
+``ceph-dev-docker`` for instance contains a file called `start-ceph.sh`,
+which cleans up log files, always starts a Rados Gateway service, sets some
+Ceph Dashboard configuration options and automatically runs a frontend proxy,
+all before or after starting up your vstart cluster.
+
+Instructions on how to use those environments are contained in their
+respective repository README files.
+
+- ceph-dev-docker_
+- ceph-dev_
+
+.. _ceph-dev-docker: https://github.com/ricardoasmarques/ceph-dev-docker
+.. _ceph-dev: https://github.com/rhcs-dashboard/ceph-dev
+
+Frontend Development
+--------------------
+
+Before you can start the dashboard from within a development environment, you
+will need to generate the frontend code and either use a compiled and running
+Ceph cluster (e.g. started by ``vstart.sh``) or the standalone development web
+server.
+
+The build process is based on `Node.js <https://nodejs.org/>`_ and requires the
+`Node Package Manager <https://www.npmjs.com/>`_ ``npm`` to be installed.
+
+Prerequisites
+~~~~~~~~~~~~~
+
+ * Node 12.18.2 or higher
+ * NPM 6.13.4 or higher
+
+nodeenv:
+  During Ceph's build we create a virtualenv with ``node`` and ``npm``
+  installed, which can be used as an alternative to installing node/npm in your
+  system.
+
+  If you want to use the node installed in the virtualenv you just need to
+  activate the virtualenv before you run any npm commands. To activate it run
+  ``. build/src/pybind/mgr/dashboard/node-env/bin/activate``.
+
+  Once you finish, you can simply run ``deactivate`` and exit the virtualenv.
+
+Angular CLI:
+  If you do not have the `Angular CLI <https://github.com/angular/angular-cli>`_
+  installed globally, then you need to execute ``ng`` commands with an
+  additional ``npm run`` before it.
+
+Package installation
+~~~~~~~~~~~~~~~~~~~~
+
+Run ``npm ci`` in directory ``src/pybind/mgr/dashboard/frontend`` to
+install the required packages locally.
+
+Adding or updating packages
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Run the following commands to add/update a package::
+
+  npm install <PACKAGE_NAME>
+  npm ci
+
+Setting up a Development Server
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Create the ``proxy.conf.json`` file based on ``proxy.conf.json.sample``.
+
+Run ``npm start`` for a dev server.
+Navigate to ``http://localhost:4200/``. The app will automatically
+reload if you change any of the source files.
+
+Code Scaffolding
+~~~~~~~~~~~~~~~~
+
+Run ``ng generate component component-name`` to generate a new
+component. You can also use
+``ng generate directive|pipe|service|class|guard|interface|enum|module``.
+
+Build the Project
+~~~~~~~~~~~~~~~~~
+
+Run ``npm run build`` to build the project. The build artifacts will be
+stored in the ``dist/`` directory. Use the ``--prod`` flag for a
+production build (``npm run build -- --prod``). Navigate to ``https://localhost:8443``.
+
+Build the Code Documentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Run ``npm run doc-build`` to generate code docs in the ``documentation/``
+directory. To make them accessible locally for a web browser, run
+``npm run doc-serve`` and they will become available at ``http://localhost:8444``.
+With ``npm run compodoc -- <opts>`` you may
+`fully configure it <https://compodoc.app/guides/usage.html>`_.
+
+Code linting and formatting
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We use the following tools to lint and format the code in all our TS, SCSS and
+HTML files:
+
+- `codelyzer <http://codelyzer.com/>`_
+- `html-linter <https://github.com/chinchiheather/html-linter>`_
+- `htmllint-cli <https://github.com/htmllint/htmllint-cli>`_
+- `Prettier <https://prettier.io/>`_
+- `TSLint <https://palantir.github.io/tslint/>`_
+- `stylelint <https://stylelint.io/>`_
+
+We added 2 npm scripts to help run these tools:
+
+- ``npm run lint``, will check frontend files against all linters
+- ``npm run fix``, will try to fix all the detected linting errors
+
+Ceph Dashboard and Bootstrap
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Currently we are using Bootstrap on the Ceph Dashboard as a CSS framework. This means that most of our SCSS and HTML
+code can make use of all the utilities and other advantages Bootstrap is offering. In the past we often have used our
+own custom styles and this lead to more and more variables with a single use and double defined variables which
+sometimes are forgotten to be removed or it led to styling be inconsistent because people forgot to change a color or to
+adjust a custom SCSS class.
+
+To get the current version of Bootstrap used inside Ceph please refer to the ``package.json`` and search for:
+
+- ``bootstrap``: For the Bootstrap version used.
+- ``@ng-bootstrap``: For the version of the Angular bindings which we are using.
+
+So for the future please do the following when visiting a component:
+
+- Does this HTML/SCSS code use custom code? - If yes: Is it needed? --> Clean it up before changing the things you want
+  to fix or change.
+- If you are creating a new component: Please make use of Bootstrap as much as reasonably possible! Don't try to
+  reinvent the wheel.
+- If possible please look up if Bootstrap has guidelines on how to extend it properly to do achieve what you want to
+  achieve.
+
+The more bootstrap alike our code is the easier it is to theme, to maintain and the less bugs we will have. Also since
+Bootstrap is a framework which tries to have usability and user experience in mind we increase both points
+exponentially. The biggest benefit of all is that there is less code for us to maintain which makes it easier to read
+for beginners and even more easy for people how are already familiar with the code.
+
+Writing Unit Tests
+~~~~~~~~~~~~~~~~~~
+
+To write unit tests most efficient we have a small collection of tools,
+we use within test suites.
+
+Those tools can be found under
+``src/pybind/mgr/dashboard/frontend/src/testing/``, especially take
+a look at ``unit-test-helper.ts``.
+
+There you will be able to find:
+
+``configureTestBed`` that replaces the initial ``TestBed``
+methods. It takes the same arguments as ``TestBed.configureTestingModule``.
+Using it will run your tests a lot faster in development, as it doesn't
+recreate everything from scratch on every test. To use the default behaviour
+pass ``true`` as the second argument.
+
+``PermissionHelper`` to help determine if
+the correct actions are shown based on the current permissions and selection
+in a list.
+
+``FormHelper`` which makes testing a form a lot easier
+with a few simple methods. It allows you to set a control or multiple
+controls, expect if a control is valid or has an error or just do both with
+one method. Additional you can expect a template element or multiple elements
+to be visible in the rendered template.
+
+Running Unit Tests
+~~~~~~~~~~~~~~~~~~
+
+Run ``npm run test`` to execute the unit tests via `Jest
+<https://facebook.github.io/jest/>`_.
+
+If you get errors on all tests, it could be because `Jest
+<https://facebook.github.io/jest/>`__ or something else was updated.
+There are a few ways how you can try to resolve this:
+
+- Remove all modules with ``rm -rf dist node_modules`` and run ``npm install``
+  again in order to reinstall them
+- Clear the cache of jest by running ``npx jest --clearCache``
+
+Running End-to-End (E2E) Tests
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We use `Cypress <https://www.cypress.io/>`__ to run our frontend E2E tests.
+
+E2E Prerequisites
+.................
+
+You need to previously build the frontend.
+
+In some environments, depending on your user permissions and the CYPRESS_CACHE_FOLDER,
+you might need to run ``npm ci`` with the ``--unsafe-perm`` flag.
+
+You might need to install additional packages to be able to run Cypress.
+Please run ``npx cypress verify`` to verify it.
+
+run-frontend-e2e-tests.sh
+.........................
+
+Our ``run-frontend-e2e-tests.sh`` script is the go to solution when you wish to
+do a full scale e2e run.
+It will verify if everything needed is installed, start a new vstart cluster
+and run the full test suite.
+
+Start all frontend E2E tests by running::
+
+  $ ./run-frontend-e2e-tests.sh
+
+Report:
+  You can follow the e2e report on the terminal and you can find the screenshots
+  of failed test cases by opening the following directory::
+
+    src/pybind/mgr/dashboard/frontend/cypress/screenshots/
+
+Device:
+  You can force the script to use a specific device with the ``-d`` flag::
+
+    $ ./run-frontend-e2e-tests.sh -d <chrome|chromium|electron|docker>
+
+Remote:
+  By default this script will stop and start a new vstart cluster.
+  If you want to run the tests outside the ceph environment, you will need to
+  manually define the dashboard url using ``-r`` and, optionally, credentials
+  (``-u``, ``-p``)::
+
+    $ ./run-frontend-e2e-tests.sh -r <DASHBOARD_URL> -u <E2E_LOGIN_USER> -p <E2E_LOGIN_PWD>
+
+Note:
+  When using docker, as your device, you might need to run the script with sudo
+  permissions.
+
+run-cephadm-e2e-tests.sh
+.........................
+
+``run-cephadm-e2e-tests.sh`` runs a subset of E2E tests to verify that the Dashboard and cephadm as
+Orchestrator backend behave correctly.
+
+Prerequisites: you need to install `KCLI
+<https://kcli.readthedocs.io/en/latest/>`_ and Node.js in your local machine.
+
+Configure KCLI plan requirements::
+
+  $ sudo chown -R $(id -un) /var/lib/libvirt/images
+  $ mkdir -p /var/lib/libvirt/images/ceph-dashboard dashboard
+  $ kcli create pool -p /var/lib/libvirt/images/ceph-dashboard dashboard
+  $ kcli create network -c 192.168.100.0/24 dashboard
+
+Note:
+  This script is aimed to be run as jenkins job so the cleanup is triggered only in a jenkins
+  environment. In local, the user will shutdown the cluster when desired (i.e. after debugging).
+
+Start E2E tests by running::
+
+  $ cd <your/ceph/repo/dir>
+  $ sudo chown -R $(id -un) src/pybind/mgr/dashboard/frontend/{dist,node_modules,src/environments}
+  $ ./src/pybind/mgr/dashboard/ci/cephadm/run-cephadm-e2e-tests.sh
+
+Note:
+  In fedora 35, there can occur a permission error when trying to mount the shared_folders. This can be
+  fixed by running::
+
+    $ sudo setfacl -R -m u:qemu:rwx <abs-path-to-your-user-home>
+
+  or also by setting the appropriate permission to your $HOME directory
+
+You can also start a cluster in development mode (so the frontend build starts in watch mode and you
+only have to reload the page for the changes to be reflected) by running::
+
+  $ ./src/pybind/mgr/dashboard/ci/cephadm/start-cluster.sh --dev-mode
+
+Note:
+  Add ``--expanded`` if you need a cluster ready to deploy services (one with enough monitor
+  daemons spread across different hosts and enough OSDs).
+
+Test your changes by running:
+
+  $ ./src/pybind/mgr/dashboard/ci/cephadm/run-cephadm-e2e-tests.sh
+
+Shutdown the cluster by running:
+
+  $ kcli delete plan -y ceph
+  $ # In development mode, also kill the npm build watch process (e.g., pkill -f "ng build")
+
+Other running options
+.....................
+
+During active development, it is not recommended to run the previous script,
+as it is not prepared for constant file changes.
+Instead you should use one of the following commands:
+
+- ``npm run e2e`` - This will run ``ng serve`` and open the Cypress Test Runner.
+- ``npm run e2e:ci`` - This will run ``ng serve`` and run the Cypress Test Runner once.
+- ``npx cypress run`` - This calls cypress directly and will run the Cypress Test Runner.
+  You need to have a running frontend server.
+- ``npx cypress open`` - This calls cypress directly and will open the Cypress Test Runner.
+  You need to have a running frontend server.
+
+Calling Cypress directly has the advantage that you can use any of the available
+`flags <https://docs.cypress.io/guides/guides/command-line.html#cypress-run>`__
+to customize your test run and you don't need to start a frontend server each time.
+
+Using one of the ``open`` commands, will open a cypress application where you
+can see all the test files you have and run each individually.
+This is going to be run in watch mode, so if you make any changes to test files,
+it will retrigger the test run.
+This cannot be used inside docker, as it requires X11 environment to be able to open.
+
+By default Cypress will look for the web page at ``https://localhost:4200/``.
+If you are serving it in a different URL you will need to configure it by
+exporting the environment variable CYPRESS_BASE_URL with the new value.
+E.g.: ``CYPRESS_BASE_URL=https://localhost:41076/ npx cypress open``
+
+CYPRESS_CACHE_FOLDER
+.....................
+
+When installing cypress via npm, a binary of the cypress app will also be
+downloaded and stored in a cache folder.
+This removes the need to download it every time you run ``npm ci`` or even when
+using cypress in a separate project.
+
+By default Cypress uses ~/.cache to store the binary.
+To prevent changes to the user home directory, we have changed this folder to
+``/ceph/build/src/pybind/mgr/dashboard/cypress``, so when you build ceph or run
+``run-frontend-e2e-tests.sh`` this is the directory Cypress will use.
+
+When using any other command to install or run cypress,
+it will go back to the default directory. It is recommended that you export the
+CYPRESS_CACHE_FOLDER environment variable with a fixed directory, so you always
+use the same directory no matter which command you use.
+
+
+Writing End-to-End Tests
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+The PagerHelper class
+.....................
+
+The ``PageHelper`` class is supposed to be used for general purpose code that
+can be used on various pages or suites.
+
+Examples are
+
+- ``navigateTo()`` - Navigates to a specific page and waits for it to load
+- ``getFirstTableCell()`` - returns the first table cell. You can also pass a
+  string with the desired content and it will return the first cell that
+  contains it.
+- ``getTabsCount()`` - returns the amount of tabs
+
+Every method that could be useful on several pages belongs there. Also, methods
+which enhance the derived classes of the PageHelper belong there. A good
+example for such a case is the ``restrictTo()`` decorator. It ensures that a
+method implemented in a subclass of PageHelper is called on the correct page.
+It will also show a developer-friendly warning if this is not the case.
+
+Subclasses of PageHelper
+........................
+
+Helper Methods
+""""""""""""""
+
+In order to make code reusable which is specific for a particular suite, make
+sure to put it in a derived class of the ``PageHelper``. For instance, when
+talking about the pool suite, such methods would be ``create()``, ``exist()``
+and ``delete()``. These methods are specific to a pool but are useful for other
+suites.
+
+Methods that return HTML elements which can only be found on a specific page,
+should be either implemented in the helper methods of the subclass of PageHelper
+or as own methods of the subclass of PageHelper.
+
+Using PageHelpers
+"""""""""""""""""
+
+In any suite, an instance of the specific ``Helper`` class should be
+instantiated and called directly.
+
+.. code:: TypeScript
+
+  const pools = new PoolPageHelper();
+
+  it('should create a pool', () => {
+    pools.exist(poolName, false);
+    pools.navigateTo('create');
+    pools.create(poolName, 8);
+    pools.exist(poolName, true);
+  });
+
+Code Style
+..........
+
+Please refer to the official `Cypress Core Concepts
+<https://docs.cypress.io/guides/core-concepts/introduction-to-cypress.html#Cypress-Can-Be-Simple-Sometimes>`__
+for a better insight on how to write and structure tests.
+
+``describe()`` vs ``it()``
+""""""""""""""""""""""""""
+
+Both ``describe()`` and ``it()`` are function blocks, meaning that any
+executable code necessary for the test can be contained in either block.
+However, Typescript scoping rules still apply, therefore any variables declared
+in a ``describe`` are available to the ``it()`` blocks inside of it.
+
+``describe()`` typically are containers for tests, allowing you to break tests
+into multiple parts. Likewise, any setup that must be made before your tests are
+run can be initialized within the ``describe()`` block. Here is an example:
+
+.. code:: TypeScript
+
+  describe('create, edit & delete image test', () => {
+    const poolName = 'e2e_images_pool';
+
+    before(() => {
+      cy.login();
+      pools.navigateTo('create');
+      pools.create(poolName, 8, 'rbd');
+      pools.exist(poolName, true);
+    });
+
+    beforeEach(() => {
+      cy.login();
+      images.navigateTo();
+    });
+
+    //...
+
+  });
+
+As shown, we can initiate the variable ``poolName`` as well as run commands
+before our test suite begins (creating a pool). ``describe()`` block messages
+should include what the test suite is.
+
+``it()`` blocks typically are parts of an overarching test. They contain the
+functionality of the test suite, each performing individual roles.
+Here is an example:
+
+.. code:: TypeScript
+
+  describe('create, edit & delete image test', () => {
+    //...
+
+    it('should create image', () => {
+      images.createImage(imageName, poolName, '1');
+      images.getFirstTableCell(imageName).should('exist');
+    });
+
+    it('should edit image', () => {
+      images.editImage(imageName, poolName, newImageName, '2');
+      images.getFirstTableCell(newImageName).should('exist');
+    });
+
+    //...
+  });
+
+As shown from the previous example, our ``describe()`` test suite is to create,
+edit and delete an image. Therefore, each ``it()`` completes one of these steps,
+one for creating, one for editing, and so on. Likewise, every ``it()`` blocks
+message should be in lowercase and written so long as "it" can be the prefix of
+the message. For example, ``it('edits the test image' () => ...)`` vs.
+``it('image edit test' () => ...)``. As shown, the first example makes
+grammatical sense with ``it()`` as the prefix whereas the second message does
+not. ``it()`` should describe what the individual test is doing and what it
+expects to happen.
+
+Differences between Frontend Unit Tests and End-to-End (E2E) Tests / FAQ
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+General introduction about testing and E2E/unit tests
+
+
+What are E2E/unit tests designed for?
+.....................................
+
+E2E test:
+
+It requires a fully functional system and tests the interaction of all components
+of the application (Ceph, back-end, front-end).
+E2E tests are designed to mimic the behavior of the user when interacting with the application
+- for example when it comes to workflows like creating/editing/deleting an item.
+Also the tests should verify that certain items are displayed as a user would see them
+when clicking through the UI (for example a menu entry or a pool that has been
+created during a test and the pool and its properties should be displayed in the table).
+
+Angular Unit Tests:
+
+Unit tests, as the name suggests, are tests for smaller units of the code.
+Those tests are designed for testing all kinds of Angular components (e.g. services, pipes etc.).
+They do not require a connection to the backend, hence those tests are independent of it.
+The expected data of the backend is mocked in the frontend and by using this data
+the functionality of the frontend can be tested without having to have real data from the backend.
+As previously mentioned, data is either mocked or, in a simple case, contains a static input,
+a function call and an expected static output.
+More complex examples include the state of a component (attributes of the component class),
+that define how the output changes according to the given input.
+
+Which E2E/unit tests are considered to be valid?
+................................................
+
+This is not easy to answer, but new tests that are written in the same way as already existing
+dashboard tests should generally be considered valid.
+Unit tests should focus on the component to be tested.
+This is either an Angular component, directive, service, pipe, etc.
+
+E2E tests should focus on testing the functionality of the whole application.
+Approximately a third of the overall E2E tests should verify the correctness
+of user visible elements.
+
+How should an E2E/unit test look like?
+......................................
+
+Unit tests should focus on the described purpose
+and shouldn't try to test other things in the same `it` block.
+
+E2E tests should contain a description that either verifies
+the correctness of a user visible element or a complete process
+like for example the creation/validation/deletion of a pool.
+
+What should an E2E/unit test cover?
+...................................
+
+E2E tests should mostly, but not exclusively, cover interaction with the backend.
+This way the interaction with the backend is utilized to write integration tests.
+
+A unit test should mostly cover critical or complex functionality
+of a component (Angular Components, Services, Pipes, Directives, etc).
+
+What should an E2E/unit test NOT cover?
+.......................................
+
+Avoid duplicate testing: do not write E2E tests for what's already
+been covered as frontend-unit tests and vice versa.
+It may not be possible to completely avoid an overlap.
+
+Unit tests should not be used to extensively click through components and E2E tests
+shouldn't be used to extensively test a single component of Angular.
+
+Best practices/guideline
+........................
+
+As a general guideline we try to follow the 70/20/10 approach - 70% unit tests,
+20% integration tests and 10% end-to-end tests.
+For further information please refer to `this document
+<https://testing.googleblog.com/2015/04/just-say-no-to-more-end-to-end-tests.html>`__
+and the included "Testing Pyramid".
+
+Further Help
+~~~~~~~~~~~~
+
+To get more help on the Angular CLI use ``ng help`` or go check out the
+`Angular CLI
+README <https://github.com/angular/angular-cli/blob/master/README.md>`__.
+
+Example of a Generator
+~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+    # Create module 'Core'
+    src/app> ng generate module core -m=app --routing
+
+    # Create module 'Auth' under module 'Core'
+    src/app/core> ng generate module auth -m=core --routing
+    or, alternatively:
+    src/app> ng generate module core/auth -m=core --routing
+
+    # Create component 'Login' under module 'Auth'
+    src/app/core/auth> ng generate component login -m=core/auth
+    or, alternatively:
+    src/app> ng generate component core/auth/login -m=core/auth
+
+Frontend Typescript Code Style Guide Recommendations
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Group the imports based on its source and separate them with a blank
+line.
+
+The source groups can be either from Angular, external or internal.
+
+Example:
+
+.. code:: javascript
+
+    import { Component } from '@angular/core';
+    import { Router } from '@angular/router';
+
+    import { ToastrManager } from 'ngx-toastr';
+
+    import { Credentials } from '../../../shared/models/credentials.model';
+    import { HostService } from './services/host.service';
+
+Frontend components
+~~~~~~~~~~~~~~~~~~~
+
+There are several components that can be reused on different pages.
+This components are declared on the components module:
+`src/pybind/mgr/dashboard/frontend/src/app/shared/components`.
+
+Helper
+~~~~~~
+
+This component should be used to provide additional information to the user.
+
+Example:
+
+.. code:: html
+
+    <cd-helper>
+      Some <strong>helper</strong> html text
+    </cd-helper>
+
+Terminology and wording
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Instead of using the Ceph component names, the approach
+suggested is to use the logical/generic names (Block over RBD, Filesystem over
+CephFS, Object over RGW). Nevertheless, as Ceph-Dashboard cannot completely hide
+the Ceph internals, some Ceph-specific names might remain visible.
+
+Regarding the wording for action labels and other textual elements (form titles,
+buttons, etc.), the chosen approach is to follow `these guidelines
+<https://www.patternfly.org/styles/terminology-and-wording/#terminology-and-wording-for-action-labels>`_.
+As a rule of thumb, 'Create' and 'Delete' are the proper wording for most forms,
+instead of 'Add' and 'Remove', unless some already created item is either added
+or removed to/from a set of items (e.g.: 'Add permission' to a user vs. 'Create
+(new) permission').
+
+In order to enforce the use of this wording, a service ``ActionLabelsI18n`` has
+been created, which provides translated labels for use in UI elements.
+
+Frontend branding
+~~~~~~~~~~~~~~~~~
+
+Every vendor can customize the 'Ceph dashboard' to his needs. No matter if
+logo, HTML-Template or TypeScript, every file inside the frontend folder can be
+replaced.
+
+To replace files, open ``./frontend/angular.json`` and scroll to the section
+``fileReplacements`` inside the production configuration. Here you can add the
+files you wish to brand. We recommend to place the branded version of a file in
+the same directory as the original one and to add a ``.brand`` to the file
+name, right in front of the file extension. A ``fileReplacement`` could for
+example look like this:
+
+.. code:: javascript
+
+    {
+      "replace": "src/app/core/auth/login/login.component.html",
+      "with": "src/app/core/auth/login/login.component.brand.html"
+    }
+
+To serve or build the branded user interface run:
+
+    $ npm run start -- --prod
+
+or
+
+    $ npm run build -- --prod
+
+Unfortunately it's currently not possible to use multiple configurations when
+serving or building the UI at the same time. That means a configuration just
+for the branding ``fileReplacements`` is not an option, because you want to use
+the production configuration anyway
+(https://github.com/angular/angular-cli/issues/10612).
+Furthermore it's also not possible to use glob expressions for
+``fileReplacements``. As long as the feature hasn't been implemented, you have
+to add the file replacements manually to the angular.json file
+(https://github.com/angular/angular-cli/issues/12354).
+
+Nevertheless you should stick to the suggested naming scheme because it makes
+it easier for you to use glob expressions once it's supported in the future.
+
+To change the variable defaults or add your own ones you can overwrite them in
+``./frontend/src/styles/vendor/_variables.scss``.
+Just reassign the variable you want to change, for example ``$color-primary: teal;``
+To overwrite or extend the default CSS, you can add your own styles in
+``./frontend/src/styles/vendor/_style-overrides.scss``.
+
+UI Style Guide
+~~~~~~~~~~~~~~
+
+The style guide is created to document Ceph Dashboard standards and maintain
+consistency across the project. Its an effort to make it easier for
+contributors to process designing and deciding mockups and designs for
+Dashboard.
+
+The development environment for Ceph Dashboard has live reloading enabled so
+any changes made in UI are reflected in open browser windows. Ceph Dashboard
+uses Bootstrap as the main third-party CSS library.
+
+Avoid duplication of code. Be consistent with the existing UI by reusing
+existing SCSS declarations as much as possible.
+
+Always check for existing code similar to what you want to write.
+You should always try to keep the same look-and-feel as the existing code.
+
+Colors
+......
+
+All the colors used in Ceph Dashboard UI are listed in
+`frontend/src/styles/defaults/_bootstrap-defaults.scss`. If using new color
+always define color variables in the `_bootstrap-defaults.scss` and
+use the variable instead of hard coded color values so that changes to the
+color are reflected in similar UI elements.
+
+The main color for the Ceph Dashboard is `$primary`. The primary color is
+used in navigation components and as the `$border-color` for input components of
+form.
+
+The secondary color is `$secondary` and is the background color for Ceph
+Dashboard.
+
+Buttons
+.......
+
+Buttons are used for performing actions such as: “Submit�, “Edit, “Create" and
+“Update�.
+
+**Forms:** When using to submit forms anywhere in the Dashboard, the main action
+button should use the `cd-submit-button` component and the secondary button should
+use `cd-back-button` component. The text on the action button should be same as the
+form title and follow a title case. The text on the secondary button should be
+`Cancel`. `Perform action` button should always be on right while `Cancel`
+button should always be on left.
+
+**Modals**: The main action button should use the `cd-submit-button` component and
+the secondary button should use `cd-back-button` component. The text on the action
+button should follow a title case and correspond to the action to be performed.
+The text on the secondary button should be `Close`.
+
+**Disclosure Button:** Disclosure buttons should be used to allow users to
+display and hide additional content in the interface.
+
+**Action Button**: Use the action button to perform actions such as edit or update
+a component. All action button should have an icon corresponding to the actions they
+perform and button text should follow title case. The button color should be the
+same as the form's main button color.
+
+**Drop Down Buttons:** Use dropdown buttons to display predefined lists of
+actions. All drop down buttons have icons corresponding to the action they
+perform.
+
+Links
+.....
+
+Use text hyperlinks as navigation to guide users to a new page in the application
+or to anchor users to a section within a page. The color of the hyperlinks
+should be `$primary`.
+
+Forms
+.....
+
+Mark invalid form fields with red outline and show a meaningful error message.
+Use red as font color for message and be as specific as possible.
+`This field is required.` should be the exact error message for required fields.
+Mark valid forms with a green outline and a green tick at the end of the form.
+Sections should not have a bigger header than the parent.
+
+Modals
+......
+
+Blur any interface elements in the background to bring the modal content into
+focus. The heading of the modal should reflect the action it can perform and
+should be clearly mentioned at the top of the modal. Use `cd-back-button`
+component in the footer for closing the modal.
+
+Icons
+.....
+
+We use `Fork Awesome <https://forkaweso.me/Fork-Awesome/>`_ classes for icons.
+We have a list of used icons in `src/app/shared/enum/icons.enum.ts`, these
+should be referenced in the HTML, so its easier to change them later. When
+icons are next to text, they should be center-aligned horizontally. If icons
+are stacked, they should also be center-aligned vertically. Use small icons
+with buttons. For notifications use large icons.
+
+Navigation
+..........
+
+For local navigation use tabs. For overall navigation use expandable vertical
+navigation to collapse and expand items as needed.
+
+Alerts and notifications
+........................
+
+Default notification should have `text-info` color. Success notification should
+have `text-success` color. Failure notification should have `text-danger` color.
+
+Error Handling
+~~~~~~~~~~~~~~
+
+For handling front-end errors, there is a generic Error Component which can be
+found in ``./src/pybind/mgr/dashboard/frontend/src/app/core/error``. For
+reporting a new error, you can simply extend the ``DashboardError`` class
+in ``error.ts`` file and add specific header and message for the new error. Some
+generic error classes are already in place such as ``DashboardNotFoundError``
+and ``DashboardForbiddenError`` which can be called and reused in different
+scenarios.
+
+For example - ``throw new DashboardNotFoundError()``.
+
+I18N
+----
+
+How to extract messages from source code?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To extract the I18N messages from the templates and the TypeScript files just
+run the following command in ``src/pybind/mgr/dashboard/frontend``::
+
+  $ npm run i18n:extract
+
+This will extract all marked messages from the HTML templates first and then
+add all marked strings from the TypeScript files to the translation template.
+Since the extraction from TypeScript files is still not supported by Angular
+itself, we are using the
+`ngx-translator <https://github.com/ngx-translate/i18n-polyfill>`_ extractor to
+parse the TypeScript files.
+
+When the command ran successfully, it should have created or updated the file
+``src/locale/messages.xlf``.
+
+The file isn't tracked by git, you can just use it to start with the
+translation offline or add/update the resource files on transifex.
+
+Supported languages
+~~~~~~~~~~~~~~~~~~~
+
+All our supported languages should be registered in both exports in
+``supported-languages.enum.ts`` and have a corresponding test in
+``language-selector.component.spec.ts``.
+
+The ``SupportedLanguages`` enum will provide the list for the default language selection.
+
+Translating process
+~~~~~~~~~~~~~~~~~~~
+
+To facilitate the translation process of the dashboard we are using a web tool
+called `transifex <https://www.transifex.com/>`_.
+
+If you wish to help translating to any language just go to our `transifex
+project page <https://www.transifex.com/ceph/ceph-dashboard/>`_, join the
+project and you can start translating immediately.
+
+All translations will then be reviewed and later pushed upstream.
+
+Updating translated messages
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Any time there are new messages translated and reviewed in a specific language
+we should update the translation file upstream.
+
+To do that, check the settings in the i18n config file
+``src/pybind/mgr/dashboard/frontend/i18n.config.json``:: and make sure that the
+organization is *ceph*, the project is *ceph-dashboard* and the resource is
+the one you want to pull from and push to e.g. *Master:master*. To find a list
+of available resources visit `<https://www.transifex.com/ceph/ceph-dashboard/content/>`_.
+
+After you checked the config go to the directory ``src/pybind/mgr/dashboard/frontend`` and run::
+
+  $ npm run i18n
+
+This command will extract all marked messages from the HTML templates and
+TypeScript files. Once the source file has been created it will push it to
+transifex and pull the latest translations. It will also fill all the
+untranslated strings with the source string.
+The tool will ask you for an api token, unless you added it by running:
+
+  $ npm run i18n:token
+
+To create a transifex api token visit `<https://www.transifex.com/user/settings/api/>`_.
+
+After the command ran successfully, build the UI and check if everything is
+working as expected. You also might want to run the frontend tests.
+
+Suggestions
+~~~~~~~~~~~
+
+Strings need to start and end in the same line as the element:
+
+.. code-block:: html
+
+  <!-- avoid -->
+  <span i18n>
+    Foo
+  </span>
+
+  <!-- recommended -->
+  <span i18n>Foo</span>
+
+
+  <!-- avoid -->
+  <span i18n>
+    Foo bar baz.
+    Foo bar baz.
+  </span>
+
+  <!-- recommended -->
+  <span i18n>Foo bar baz.
+    Foo bar baz.</span>
+
+Isolated interpolations should not be translated:
+
+.. code-block:: html
+
+  <!-- avoid -->
+  <span i18n>{{ foo }}</span>
+
+  <!-- recommended -->
+  <span>{{ foo }}</span>
+
+Interpolations used in a sentence should be kept in the translation:
+
+.. code-block:: html
+
+  <!-- recommended -->
+  <span i18n>There are {{ x }} OSDs.</span>
+
+Remove elements that are outside the context of the translation:
+
+.. code-block:: html
+
+  <!-- avoid -->
+  <label i18n>
+    Profile
+    <span class="required"></span>
+  </label>
+
+  <!-- recommended -->
+  <label>
+    <ng-container i18n>Profile<ng-container>
+    <span class="required"></span>
+  </label>
+
+Keep elements that affect the sentence:
+
+.. code-block:: html
+
+  <!-- recommended -->
+  <span i18n>Profile <b>foo</b> will be removed.</span>
+
+Backend Development
+-------------------
+
+The Python backend code of this module requires a number of Python modules to be
+installed. They are listed in file ``requirements.txt``. Using `pip
+<https://pypi.python.org/pypi/pip>`_ you may install all required dependencies
+by issuing ``pip install -r requirements.txt`` in directory
+``src/pybind/mgr/dashboard``.
+
+If you're using the `ceph-dev-docker development environment
+<https://github.com/ricardoasmarques/ceph-dev-docker/>`_, simply run
+``./install_deps.sh`` from the toplevel directory to install them.
+
+Unit Testing
+~~~~~~~~~~~~
+
+In dashboard we have two different kinds of backend tests:
+
+1. Unit tests based on ``tox``
+2. API tests based on Teuthology.
+
+Unit tests based on tox
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+We included a ``tox`` configuration file that will run the unit tests under
+Python 3, as well as linting tools to guarantee the uniformity of code.
+
+You need to install ``tox`` and ``coverage`` before running it. To install the
+packages in your system, either install it via your operating system's package
+management tools, e.g. by running ``dnf install python-tox python-coverage`` on
+Fedora Linux.
+
+Alternatively, you can use Python's native package installation method::
+
+  $ pip install tox
+  $ pip install coverage
+
+To run the tests, run ``src/script/run_tox.sh`` in the dashboard directory (where
+``tox.ini`` is located)::
+
+  ## Run Python 3 tests+lint commands:
+  $ ../../../script/run_tox.sh --tox-env py3,lint,check
+
+  ## Run Python 3 arbitrary command (e.g. 1 single test):
+  $ ../../../script/run_tox.sh --tox-env py3 "" tests/test_rgw_client.py::RgwClientTest::test_ssl_verify
+
+You can also run tox instead of ``run_tox.sh``::
+
+  ## Run Python 3 tests command:
+  $ tox -e py3
+
+  ## Run Python 3 arbitrary command (e.g. 1 single test):
+  $ tox -e py3 tests/test_rgw_client.py::RgwClientTest::test_ssl_verify
+
+Python files can be automatically fixed and formatted according to PEP8
+standards by using ``run_tox.sh --tox-env fix`` or ``tox -e fix``.
+
+We also collect coverage information from the backend code when you run tests. You can check the
+coverage information provided by the tox output, or by running the following
+command after tox has finished successfully::
+
+  $ coverage html
+
+This command will create a directory ``htmlcov`` with an HTML representation of
+the code coverage of the backend.
+
+API tests based on Teuthology
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+How to run existing API tests:
+  To run the API tests against a real Ceph cluster, we leverage the Teuthology
+  framework. This has the advantage of catching bugs originated from changes in
+  the internal Ceph code.
+
+  Our ``run-backend-api-tests.sh`` script will start a ``vstart`` Ceph cluster
+  before running the Teuthology tests, and then it stops the cluster after the
+  tests are run. Of course this implies that you have built/compiled Ceph
+  previously.
+
+  Start all dashboard tests by running::
+
+    $ ./run-backend-api-tests.sh
+
+  Or, start one or multiple specific tests by specifying the test name::
+
+    $ ./run-backend-api-tests.sh tasks.mgr.dashboard.test_pool.PoolTest
+
+  Or, ``source`` the script and run the tests manually::
+
+    $ source run-backend-api-tests.sh
+    $ run_teuthology_tests [tests]...
+    $ cleanup_teuthology
+
+How to write your own tests:
+  There are two possible ways to write your own API tests:
+
+  The first is by extending one of the existing test classes in the
+  ``qa/tasks/mgr/dashboard`` directory.
+
+  The second way is by adding your own API test module if you're creating a new
+  controller for example. To do so you'll just need to add the file containing
+  your new test class to the ``qa/tasks/mgr/dashboard`` directory and implement
+  all your tests here.
+
+  .. note:: Don't forget to add the path of the newly created module to
+    ``modules`` section in ``qa/suites/rados/mgr/tasks/dashboard.yaml``.
+
+  Short example: Let's assume you created a new controller called
+  ``my_new_controller.py`` and the related test module
+  ``test_my_new_controller.py``. You'll need to add
+  ``tasks.mgr.dashboard.test_my_new_controller`` to the ``modules`` section in
+  the ``dashboard.yaml`` file.
+
+  Also, if you're removing test modules please keep in mind to remove the
+  related section. Otherwise the Teuthology test run will fail.
+
+  Please run your API tests on your dev environment (as explained above)
+  before submitting a pull request. Also make sure that a full QA run in
+  Teuthology/sepia lab (based on your changes) has completed successfully
+  before it gets merged. You don't need to schedule the QA run yourself, just
+  add the 'needs-qa' label to your pull request as soon as you think it's ready
+  for merging (e.g. make check was successful, the pull request is approved and
+  all comments have been addressed). One of the developers who has access to
+  Teuthology/the sepia lab will take care of it and report the result back to
+  you.
+
+
+How to add a new controller?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A controller is a Python class that extends from the ``BaseController`` class
+and is decorated with either the ``@Controller``, ``@ApiController`` or
+``@UiApiController`` decorators. The Python class must be stored inside a Python
+file located under the ``controllers`` directory. The Dashboard module will
+automatically load your new controller upon start.
+
+``@ApiController`` and ``@UiApiController`` are both specializations of the
+``@Controller`` decorator.
+
+The ``@ApiController`` should be used for controllers that provide an API-like
+REST interface and the ``@UiApiController`` should be used for endpoints consumed
+by the UI but that are not part of the 'public' API. For any other kinds of
+controllers the ``@Controller`` decorator should be used.
+
+A controller has a URL prefix path associated that is specified in the
+controller decorator, and all endpoints exposed by the controller will share
+the same URL prefix path.
+
+A controller's endpoint is exposed by implementing a method on the controller
+class decorated with the ``@Endpoint`` decorator.
+
+For example create a file ``ping.py`` under ``controllers`` directory with the
+following code:
+
+.. code-block:: python
+
+  from ..tools import Controller, ApiController, UiApiController, BaseController, Endpoint
+
+  @Controller('/ping')
+  class Ping(BaseController):
+    @Endpoint()
+    def hello(self):
+      return {'msg': "Hello"}
+
+  @ApiController('/ping')
+  class ApiPing(BaseController):
+    @Endpoint()
+    def hello(self):
+      return {'msg': "Hello"}
+
+  @UiApiController('/ping')
+  class UiApiPing(BaseController):
+    @Endpoint()
+    def hello(self):
+      return {'msg': "Hello"}
+
+The ``hello`` endpoint of the ``Ping`` controller can be reached by the
+following URL: https://mgr_hostname:8443/ping/hello using HTTP GET requests.
+As you can see the controller URL path ``/ping`` is concatenated to the
+method name ``hello`` to generate the endpoint's URL.
+
+In the case of the ``ApiPing`` controller, the ``hello`` endpoint can be
+reached by the following URL: https://mgr_hostname:8443/api/ping/hello using a
+HTTP GET request.
+The API controller URL path ``/ping`` is prefixed by the ``/api`` path and then
+concatenated to the method name ``hello`` to generate the endpoint's URL.
+Internally, the ``@ApiController`` is actually calling the ``@Controller``
+decorator by passing an additional decorator parameter called ``base_url``::
+
+  @ApiController('/ping') <=> @Controller('/ping', base_url="/api")
+
+``UiApiPing`` works in a similar way than the ``ApiPing``, but the URL will be
+prefixed by ``/ui-api``: https://mgr_hostname:8443/ui-api/ping/hello. ``UiApiPing`` is
+also a ``@Controller`` extension::
+
+  @UiApiController('/ping') <=> @Controller('/ping', base_url="/ui-api")
+
+The ``@Endpoint`` decorator also supports many parameters to customize the
+endpoint:
+
+* ``method="GET"``: the HTTP method allowed to access this endpoint.
+* ``path="/<method_name>"``: the URL path of the endpoint, excluding the
+  controller URL path prefix.
+* ``path_params=[]``: list of method parameter names that correspond to URL
+  path parameters. Can only be used when ``method in ['POST', 'PUT']``.
+* ``query_params=[]``: list of method parameter names that correspond to URL
+  query parameters.
+* ``json_response=True``: indicates if the endpoint response should be
+  serialized in JSON format.
+* ``proxy=False``: indicates if the endpoint should be used as a proxy.
+
+An endpoint method may have parameters declared. Depending on the HTTP method
+defined for the endpoint the method parameters might be considered either
+path parameters, query parameters, or body parameters.
+
+For ``GET`` and ``DELETE`` methods, the method's non-optional parameters are
+considered path parameters by default. Optional parameters are considered
+query parameters. By specifying the ``query_parameters`` in the endpoint
+decorator it is possible to make a non-optional parameter to be a query
+parameter.
+
+For ``POST`` and ``PUT`` methods, all method parameters are considered
+body parameters by default. To override this default, one can use the
+``path_params`` and ``query_params`` to specify which method parameters are
+path and query parameters respectively.
+Body parameters are decoded from the request body, either from a form format, or
+from a dictionary in JSON format.
+
+Let's use an example to better understand the possible ways to customize an
+endpoint:
+
+.. code-block:: python
+
+  from ..tools import Controller, BaseController, Endpoint
+
+  @Controller('/ping')
+  class Ping(BaseController):
+
+    # URL: /ping/{key}?opt1=...&opt2=...
+    @Endpoint(path="/", query_params=['opt1'])
+    def index(self, key, opt1, opt2=None):
+      """..."""
+
+    # URL: /ping/{key}?opt1=...&opt2=...
+    @Endpoint(query_params=['opt1'])
+    def __call__(self, key, opt1, opt2=None):
+      """..."""
+
+    # URL: /ping/post/{key1}/{key2}
+    @Endpoint('POST', path_params=['key1', 'key2'])
+    def post(self, key1, key2, data1, data2=None):
+      """..."""
+
+
+In the above example we see how the ``path`` option can be used to override the
+generated endpoint URL in order to not use the method's name in the URL. In the
+``index`` method we set the ``path`` to ``"/"`` to generate an endpoint that is
+accessible by the root URL of the controller.
+
+An alternative approach to generate an endpoint that is accessible through just
+the controller's path URL is by using the ``__call__`` method, as we show in
+the above example.
+
+From the third method you can see that the path parameters are collected from
+the URL by parsing the list of values separated by slashes ``/`` that come
+after the URL path ``/ping`` for ``index`` method case, and ``/ping/post`` for
+the ``post`` method case.
+
+Defining path parameters in endpoints's URLs using python methods's parameters
+is very easy but it is still a bit strict with respect to the position of these
+parameters in the URL structure.
+Sometimes we may want to explicitly define a URL scheme that
+contains path parameters mixed with static parts of the URL.
+Our controller infrastructure also supports the declaration of URL paths with
+explicit path parameters at both the controller level and method level.
+
+Consider the following example:
+
+.. code-block:: python
+
+  from ..tools import Controller, BaseController, Endpoint
+
+  @Controller('/ping/{node}/stats')
+  class Ping(BaseController):
+
+    # URL: /ping/{node}/stats/{date}/latency?unit=...
+    @Endpoint(path="/{date}/latency")
+    def latency(self, node, date, unit="ms"):
+      """ ..."""
+
+In this example we explicitly declare a path parameter ``{node}`` in the
+controller URL path, and a path parameter ``{date}`` in the ``latency``
+method. The endpoint for the ``latency`` method is then accessible through
+the URL: https://mgr_hostname:8443/ping/{node}/stats/{date}/latency .
+
+For a full set of examples on how to use the ``@Endpoint``
+decorator please check the unit test file: ``tests/test_controllers.py``.
+There you will find many examples of how to customize endpoint methods.
+
+
+Implementing Proxy Controller
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Sometimes you might need to relay some requests from the Dashboard frontend
+directly to an external service.
+For that purpose we provide a decorator called ``@Proxy``.
+(As a concrete example, check the ``controllers/rgw.py`` file where we
+implemented an RGW Admin Ops proxy.)
+
+
+The ``@Proxy`` decorator is a wrapper of the ``@Endpoint`` decorator that
+already customizes the endpoint for working as a proxy.
+A proxy endpoint works by capturing the URL path that follows the controller
+URL prefix path, and does not do any decoding of the request body.
+
+Example:
+
+.. code-block:: python
+
+  from ..tools import Controller, BaseController, Proxy
+
+  @Controller('/foo/proxy')
+  class FooServiceProxy(BaseController):
+
+    @Proxy()
+    def proxy(self, path, **params):
+      """
+      if requested URL is "/foo/proxy/access/service?opt=1"
+      then path is "access/service" and params is {'opt': '1'}
+      """
+
+
+How does the RESTController work?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We also provide a simple mechanism to create REST based controllers using the
+``RESTController`` class. Any class which inherits from ``RESTController`` will,
+by default, return JSON.
+
+The ``RESTController`` is basically an additional abstraction layer which eases
+and unifies the work with collections. A collection is just an array of objects
+with a specific type. ``RESTController`` enables some default mappings of
+request types and given parameters to specific method names. This may sound
+complicated at first, but it's fairly easy. Lets have look at the following
+example:
+
+.. code-block:: python
+
+  import cherrypy
+  from ..tools import ApiController, RESTController
+
+  @ApiController('ping')
+  class Ping(RESTController):
+    def list(self):
+      return {"msg": "Hello"}
+
+    def get(self, id):
+      return self.objects[id]
+
+In this case, the ``list`` method is automatically used for all requests to
+``api/ping`` where no additional argument is given and where the request type
+is ``GET``. If the request is given an additional argument, the ID in our
+case, it won't map to ``list`` anymore but to ``get`` and return the element
+with the given ID (assuming that ``self.objects`` has been filled before). The
+same applies to other request types:
+
++--------------+------------+----------------+-------------+
+| Request type | Arguments  | Method         | Status Code |
++==============+============+================+=============+
+| GET          | No         | list           | 200         |
++--------------+------------+----------------+-------------+
+| PUT          | No         | bulk_set       | 200         |
++--------------+------------+----------------+-------------+
+| POST         | No         | create         | 201         |
++--------------+------------+----------------+-------------+
+| DELETE       | No         | bulk_delete    | 204         |
++--------------+------------+----------------+-------------+
+| GET          | Yes        | get            | 200         |
++--------------+------------+----------------+-------------+
+| PUT          | Yes        | set            | 200         |
++--------------+------------+----------------+-------------+
+| DELETE       | Yes        | delete         | 204         |
++--------------+------------+----------------+-------------+
+
+To use a custom endpoint for the above listed methods, you can
+use ``@RESTController.MethodMap``
+
+.. code-block:: python
+
+  import cherrypy
+  from ..tools import ApiController, RESTController
+
+    @RESTController.MethodMap(version='0.1')
+    def create(self):
+      return {"msg": "Hello"}
+
+This decorator supports three parameters to customize the
+endpoint:
+
+* ``resource"``: resource id.
+* ``status=200``: set the HTTP status response code
+* ``version``: version
+
+How to use a custom API endpoint in a RESTController?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you don't have any access restriction you can use ``@Endpoint``. If you
+have set a permission scope to restrict access to your endpoints,
+``@Endpoint`` will fail, as it doesn't know which permission property should be
+used. To use a custom endpoint inside a restricted ``RESTController`` use
+``@RESTController.Collection`` instead. You can also choose
+``@RESTController.Resource`` if you have set a ``RESOURCE_ID`` in your
+``RESTController`` class.
+
+.. code-block:: python
+
+  import cherrypy
+  from ..tools import ApiController, RESTController
+
+  @ApiController('ping', Scope.Ping)
+  class Ping(RESTController):
+    RESOURCE_ID = 'ping'
+
+    @RESTController.Resource('GET')
+    def some_get_endpoint(self):
+      return {"msg": "Hello"}
+
+    @RESTController.Collection('POST')
+    def some_post_endpoint(self, **data):
+      return {"msg": data}
+
+Both decorators also support five parameters to customize the
+endpoint:
+
+* ``method="GET"``: the HTTP method allowed to access this endpoint.
+* ``path="/<method_name>"``: the URL path of the endpoint, excluding the
+  controller URL path prefix.
+* ``status=200``: set the HTTP status response code
+* ``query_params=[]``: list of method parameter names that correspond to URL
+  query parameters.
+* ``version``: version
+
+How to restrict access to a controller?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+All controllers require authentication by default.
+If you require that the controller can be accessed without authentication,
+then you can add the parameter ``secure=False`` to the controller decorator.
+
+Example:
+
+.. code-block:: python
+
+  import cherrypy
+  from . import ApiController, RESTController
+
+
+  @ApiController('ping', secure=False)
+  class Ping(RESTController):
+    def list(self):
+      return {"msg": "Hello"}
+
+How to create a dedicated UI endpoint which uses the 'public' API?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Sometimes we want to combine multiple calls into one single call
+to save bandwidth or for other performance reasons.
+In order to achieve that, we first have to create an ``@UiApiController`` which
+is used for endpoints consumed by the UI but that are not part of the
+'public' API. Let the ui class inherit from the REST controller class.
+Now you can use all methods from the api controller.
+
+Example:
+
+.. code-block:: python
+
+  import cherrypy
+  from . import UiApiController, ApiController, RESTController
+
+
+  @ApiController('ping', secure=False)  # /api/ping
+  class Ping(RESTController):
+    def list(self):
+      return self._list()
+
+    def _list(self):  # To not get in conflict with the JSON wrapper
+      return [1,2,3]
+
+
+  @UiApiController('ping', secure=False)  # /ui-api/ping
+  class PingUi(Ping):
+    def list(self):
+      return self._list() + [4, 5, 6]
+
+How to access the manager module instance from a controller?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We provide the manager module instance as a global variable that can be
+imported in any module.
+
+Example:
+
+.. code-block:: python
+
+  import logging
+  import cherrypy
+  from .. import mgr
+  from ..tools import ApiController, RESTController
+
+  logger = logging.getLogger(__name__)
+
+  @ApiController('servers')
+  class Servers(RESTController):
+    def list(self):
+      logger.debug('Listing available servers')
+      return {'servers': mgr.list_servers()}
+
+
+How to write a unit test for a controller?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We provide a test helper class called ``ControllerTestCase`` to easily create
+unit tests for your controller.
+
+If we want to write a unit test for the above ``Ping`` controller, create a
+``test_ping.py`` file under the ``tests`` directory with the following code:
+
+.. code-block:: python
+
+  from .helper import ControllerTestCase
+  from .controllers.ping import Ping
+
+
+  class PingTest(ControllerTestCase):
+      @classmethod
+      def setup_test(cls):
+          cp_config = {'tools.authenticate.on': True}
+          cls.setup_controllers([Ping], cp_config=cp_config)
+
+      def test_ping(self):
+          self._get("/api/ping")
+          self.assertStatus(200)
+          self.assertJsonBody({'msg': 'Hello'})
+
+The ``ControllerTestCase`` class starts by initializing a CherryPy webserver.
+Then it will call the ``setup_test()`` class method where we can explicitly
+load the controllers that we want to test. In the above example we are only
+loading the ``Ping`` controller. We can also provide ``cp_config`` in order to
+update the controller's cherrypy config (e.g. enable authentication as shown in the example).
+
+How to update or create new dashboards in grafana?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We are using ``jsonnet`` and ``grafonnet-lib`` to write code for the grafana dashboards.
+All the dashboards are written inside ``grafana_dashboards.jsonnet`` file in the
+monitoring/grafana/dashboards/jsonnet directory.
+
+We generate the dashboard json files directly from this jsonnet file by running this
+command in the grafana/dashboards directory:
+``jsonnet -m . jsonnet/grafana_dashboards.jsonnet``.
+(For the above command to succeed we need ``jsonnet`` package installed and ``grafonnet-lib``
+directory cloned in our machine. Please refer - 
+``https://grafana.github.io/grafonnet-lib/getting-started/`` in case you have some trouble.)
+
+To update an existing grafana dashboard or to create a new one, we need to update
+the ``grafana_dashboards.jsonnet`` file and generate the new/updated json files using the
+above mentioned command. For people who are not familiar with grafonnet or jsonnet implementation
+can follow this doc - ``https://grafana.github.io/grafonnet-lib/``.
+
+Example grafana dashboard in jsonnet format:
+
+To specify the grafana dashboard properties such as title, uid etc we can create a local function -
+
+::
+
+    local dashboardSchema(title, uid, time_from, refresh, schemaVersion, tags,timezone, timepicker)
+
+To add a graph panel we can spcify the graph schema in a local function such as -
+
+::
+
+    local graphPanelSchema(title, nullPointMode, stack, formatY1, formatY2, labelY1, labelY2, min, fill, datasource)
+
+and then use these functions inside the dashboard definition like -
+
+::
+
+    {
+        radosgw-sync-overview.json: //json file name to be generated
+
+        dashboardSchema(
+          'RGW Sync Overview', 'rgw-sync-overview', 'now-1h', '15s', .., .., ..
+        )
+
+        .addPanels([
+          graphPanelSchema(
+            'Replication (throughput) from Source Zone', 'Bps', null, .., .., ..)
+        ])
+    }
+
+The valid grafonnet-lib attributes can be found here - ``https://grafana.github.io/grafonnet-lib/api-docs/``.
+  
+
+How to listen for manager notifications in a controller?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The manager notifies the modules of several types of cluster events, such
+as cluster logging event, etc...
+
+Each module has a "global" handler function called ``notify`` that the manager
+calls to notify the module. But this handler function must not block or spend
+too much time processing the event notification.
+For this reason we provide a notification queue that controllers can register
+themselves with to receive cluster notifications.
+
+The example below represents a controller that implements a very simple live
+log viewer page:
+
+.. code-block:: python
+
+  from __future__ import absolute_import
+
+  import collections
+
+  import cherrypy
+
+  from ..tools import ApiController, BaseController, NotificationQueue
+
+
+  @ApiController('livelog')
+  class LiveLog(BaseController):
+      log_buffer = collections.deque(maxlen=1000)
+
+      def __init__(self):
+          super(LiveLog, self).__init__()
+          NotificationQueue.register(self.log, 'clog')
+
+      def log(self, log_struct):
+          self.log_buffer.appendleft(log_struct)
+
+      @cherrypy.expose
+      def default(self):
+          ret = '<html><meta http-equiv="refresh" content="2" /><body>'
+          for l in self.log_buffer:
+              ret += "{}<br>".format(l)
+          ret += "</body></html>"
+          return ret
+
+As you can see above, the ``NotificationQueue`` class provides a register
+method that receives the function as its first argument, and receives the
+"notification type" as the second argument.
+You can omit the second argument of the ``register`` method, and in that case
+you are registering to listen all notifications of any type.
+
+Here is an list of notification types (these might change in the future) that
+can be used:
+
+* ``clog``: cluster log notifications
+* ``command``: notification when a command issued by ``MgrModule.send_command``
+  completes
+* ``perf_schema_update``: perf counters schema update
+* ``mon_map``: monitor map update
+* ``fs_map``: cephfs map update
+* ``osd_map``: OSD map update
+* ``service_map``: services (RGW, RBD-Mirror, etc.) map update
+* ``mon_status``: monitor status regular update
+* ``health``: health status regular update
+* ``pg_summary``: regular update of PG status information
+
+
+How to write a unit test when a controller accesses a Ceph module?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Consider the following example that implements a controller that retrieves the
+list of RBD images of the ``rbd`` pool:
+
+.. code-block:: python
+
+  import rbd
+  from .. import mgr
+  from ..tools import ApiController, RESTController
+
+
+  @ApiController('rbdimages')
+  class RbdImages(RESTController):
+      def __init__(self):
+          self.ioctx = mgr.rados.open_ioctx('rbd')
+          self.rbd = rbd.RBD()
+
+      def list(self):
+          return [{'name': n} for n in self.rbd.list(self.ioctx)]
+
+In the example above, we want to mock the return value of the ``rbd.list``
+function, so that we can test the JSON response of the controller.
+
+The unit test code will look like the following:
+
+.. code-block:: python
+
+  import mock
+  from .helper import ControllerTestCase
+
+
+  class RbdImagesTest(ControllerTestCase):
+      @mock.patch('rbd.RBD.list')
+      def test_list(self, rbd_list_mock):
+          rbd_list_mock.return_value = ['img1', 'img2']
+          self._get('/api/rbdimages')
+          self.assertJsonBody([{'name': 'img1'}, {'name': 'img2'}])
+
+
+
+How to add a new configuration setting?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you need to store some configuration setting for a new feature, we already
+provide an easy mechanism for you to specify/use the new config setting.
+
+For instance, if you want to add a new configuration setting to hold the
+email address of the dashboard admin, just add a setting name as a class
+attribute to the ``Options`` class in the ``settings.py`` file::
+
+  # ...
+  class Options(object):
+    # ...
+
+    ADMIN_EMAIL_ADDRESS = ('admin@admin.com', str)
+
+The value of the class attribute is a pair composed by the default value for that
+setting, and the python type of the value.
+
+By declaring the ``ADMIN_EMAIL_ADDRESS`` class attribute, when you restart the
+dashboard module, you will automatically gain two additional CLI commands to
+get and set that setting::
+
+  $ ceph dashboard get-admin-email-address
+  $ ceph dashboard set-admin-email-address <value>
+
+To access, or modify the config setting value from your Python code, either
+inside a controller or anywhere else, you just need to import the ``Settings``
+class and access it like this:
+
+.. code-block:: python
+
+  from settings import Settings
+
+  # ...
+  tmp_var = Settings.ADMIN_EMAIL_ADDRESS
+
+  # ....
+  Settings.ADMIN_EMAIL_ADDRESS = 'myemail@admin.com'
+
+The settings management implementation will make sure that if you change a
+setting value from the Python code you will see that change when accessing
+that setting from the CLI and vice-versa.
+
+
+How to run a controller read-write operation asynchronously?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Some controllers might need to execute operations that alter the state of the
+Ceph cluster. These operations might take some time to execute and to maintain
+a good user experience in the Web UI, we need to run those operations
+asynchronously and return immediately to frontend some information that the
+operations are running in the background.
+
+To help in the development of the above scenario we added the support for
+asynchronous tasks. To trigger the execution of an asynchronous task we must
+use the following class method of the ``TaskManager`` class::
+
+  from ..tools import TaskManager
+  # ...
+  TaskManager.run(name, metadata, func, args, kwargs)
+
+* ``name`` is a string that can be used to group tasks. For instance
+  for RBD image creation tasks we could specify ``"rbd/create"`` as the
+  name, or similarly ``"rbd/remove"`` for RBD image removal tasks.
+
+* ``metadata`` is a dictionary where we can store key-value pairs that
+  characterize the task. For instance, when creating a task for creating
+  RBD images we can specify the metadata argument as
+  ``{'pool_name': "rbd", image_name': "test-img"}``.
+
+* ``func`` is the python function that implements the operation code, which
+  will be executed asynchronously.
+
+* ``args`` and ``kwargs`` are the positional and named arguments that will be
+  passed to ``func`` when the task manager starts its execution.
+
+The ``TaskManager.run`` method triggers the asynchronous execution of function
+``func`` and returns a ``Task`` object.
+The ``Task`` provides the public method ``Task.wait(timeout)``, which can be
+used to wait for the task to complete up to a timeout defined in seconds and
+provided as an argument. If no argument is provided the ``wait`` method
+blocks until the task is finished.
+
+The ``Task.wait`` is very useful for tasks that usually are fast to execute but
+that sometimes may take a long time to run.
+The return value of the ``Task.wait`` method is a pair ``(state, value)``
+where ``state`` is a string with following possible values:
+
+* ``VALUE_DONE = "done"``
+* ``VALUE_EXECUTING = "executing"``
+
+The ``value`` will store the result of the execution of function ``func`` if
+``state == VALUE_DONE``. If ``state == VALUE_EXECUTING`` then
+``value == None``.
+
+The pair ``(name, metadata)`` should unequivocally identify the task being
+run, which means that if you try to trigger a new task that matches the same
+``(name, metadata)`` pair of the currently running task, then the new task
+is not created and you get the task object of the current running task.
+
+For instance, consider the following example:
+
+.. code-block:: python
+
+  task1 = TaskManager.run("dummy/task", {'attr': 2}, func)
+  task2 = TaskManager.run("dummy/task", {'attr': 2}, func)
+
+If the second call to ``TaskManager.run`` executes while the first task is
+still executing then it will return the same task object:
+``assert task1 == task2``.
+
+
+How to get the list of executing and finished asynchronous tasks?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The list of executing and finished tasks is included in the ``Summary``
+controller, which is already polled every 5 seconds by the dashboard frontend.
+But we also provide a dedicated controller to get the same list of executing
+and finished tasks.
+
+The ``Task`` controller exposes the ``/api/task`` endpoint that returns the
+list of executing and finished tasks. This endpoint accepts the ``name``
+parameter that accepts a glob expression as its value.
+For instance, an HTTP GET request of the URL ``/api/task?name=rbd/*``
+will return all executing and finished tasks which name starts with ``rbd/``.
+
+To prevent the finished tasks list from growing unbounded, we will always
+maintain the 10 most recent finished tasks, and the remaining older finished
+tasks will be removed when reaching a TTL of 1 minute. The TTL is calculated
+using the timestamp when the task finished its execution. After a minute, when
+the finished task information is retrieved, either by the summary controller or
+by the task controller, it is automatically deleted from the list and it will
+not be included in further task queries.
+
+Each executing task is represented by the following dictionary::
+
+  {
+    'name': "name",  # str
+    'metadata': { },  # dict
+    'begin_time': "2018-03-14T15:31:38.423605Z",  # str (ISO 8601 format)
+    'progress': 0  # int (percentage)
+  }
+
+Each finished task is represented by the following dictionary::
+
+  {
+    'name': "name",  # str
+    'metadata': { },  # dict
+    'begin_time': "2018-03-14T15:31:38.423605Z",  # str (ISO 8601 format)
+    'end_time': "2018-03-14T15:31:39.423605Z",  # str (ISO 8601 format)
+    'duration': 0.0,  # float
+    'progress': 0  # int (percentage)
+    'success': True,  # bool
+    'ret_value': None,  # object, populated only if 'success' == True
+    'exception': None,  # str, populated only if 'success' == False
+  }
+
+
+How to use asynchronous APIs with asynchronous tasks?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The ``TaskManager.run`` method as described in a previous section, is well
+suited for calling blocking functions, as it runs the function inside a newly
+created thread. But sometimes we want to call some function of an API that is
+already asynchronous by nature.
+
+For these cases we want to avoid creating a new thread for just running a
+non-blocking function, and want to leverage the asynchronous nature of the
+function. The ``TaskManager.run`` is already prepared to be used with
+non-blocking functions by passing an object of the type ``TaskExecutor`` as an
+additional parameter called ``executor``. The full method signature of
+``TaskManager.run``::
+
+  TaskManager.run(name, metadata, func, args=None, kwargs=None, executor=None)
+
+
+The ``TaskExecutor`` class is responsible for code that executes a given task
+function, and defines three methods that can be overridden by
+subclasses::
+
+  def init(self, task)
+  def start(self)
+  def finish(self, ret_value, exception)
+
+The ``init`` method is called before the running the task function, and
+receives the task object (of class ``Task``).
+
+The ``start`` method runs the task function. The default implementation is to
+run the task function in the current thread context.
+
+The ``finish`` method should be called when the task function finishes with
+either the ``ret_value`` populated with the result of the execution, or with
+an exception object in the case that execution raised an exception.
+
+To leverage the asynchronous nature of a non-blocking function, the developer
+should implement a custom executor by creating a subclass of the
+``TaskExecutor`` class, and provide an instance of the custom executor class
+as the ``executor`` parameter of the ``TaskManager.run``.
+
+To better understand the expressive power of executors, we write a full example
+of use a custom executor to execute the ``MgrModule.send_command`` asynchronous
+function:
+
+.. code-block:: python
+
+  import json
+  from mgr_module import CommandResult
+  from .. import mgr
+  from ..tools import ApiController, RESTController, NotificationQueue, \
+                      TaskManager, TaskExecutor
+
+
+  class SendCommandExecutor(TaskExecutor):
+      def __init__(self):
+          super(SendCommandExecutor, self).__init__()
+          self.tag = None
+          self.result = None
+
+      def init(self, task):
+          super(SendCommandExecutor, self).init(task)
+
+          # we need to listen for 'command' events to know when the command
+          # finishes
+          NotificationQueue.register(self._handler, 'command')
+
+          # store the CommandResult object to retrieve the results
+          self.result = self.task.fn_args[0]
+          if len(self.task.fn_args) > 4:
+              # the user specified a tag for the command, so let's use it
+              self.tag = self.task.fn_args[4]
+          else:
+              # let's generate a unique tag for the command
+              self.tag = 'send_command_{}'.format(id(self))
+              self.task.fn_args.append(self.tag)
+
+      def _handler(self, data):
+          if data == self.tag:
+              # the command has finished, notifying the task with the result
+              self.finish(self.result.wait(), None)
+              # deregister listener to avoid memory leaks
+              NotificationQueue.deregister(self._handler, 'command')
+
+
+  @ApiController('test')
+  class Test(RESTController):
+
+      def _run_task(self, osd_id):
+          task = TaskManager.run("test/task", {}, mgr.send_command,
+                                 [CommandResult(''), 'osd', osd_id,
+                                  json.dumps({'prefix': 'perf histogram dump'})],
+                                 executor=SendCommandExecutor())
+          return task.wait(1.0)
+
+      def get(self, osd_id):
+          status, value = self._run_task(osd_id)
+          return {'status': status, 'value': value}
+
+
+The above ``SendCommandExecutor`` executor class can be used for any call to
+``MgrModule.send_command``. This means that we should need just one custom
+executor class implementation for each non-blocking API that we use in our
+controllers.
+
+The default executor, used when no executor object is passed to
+``TaskManager.run``, is the ``ThreadedExecutor``. You can check its
+implementation in the ``tools.py`` file.
+
+
+How to update the execution progress of an asynchronous task?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The asynchronous tasks infrastructure provides support for updating the
+execution progress of an executing task.
+The progress can be updated from within the code the task is executing, which
+usually is the place where we have the progress information available.
+
+To update the progress from within the task code, the ``TaskManager`` class
+provides a method to retrieve the current task object::
+
+  TaskManager.current_task()
+
+The above method is only available when using the default executor
+``ThreadedExecutor`` for executing the task.
+The ``current_task()`` method returns the current ``Task`` object. The
+``Task`` object provides two public methods to update the execution progress
+value: the ``set_progress(percentage)``, and the ``inc_progress(delta)``
+methods.
+
+The ``set_progress`` method receives as argument an integer value representing
+the absolute percentage that we want to set to the task.
+
+The ``inc_progress`` method receives as argument an integer value representing
+the delta we want to increment to the current execution progress percentage.
+
+Take the following example of a controller that triggers a new task and
+updates its progress:
+
+.. code-block:: python
+
+  from __future__ import absolute_import
+  import random
+  import time
+  import cherrypy
+  from ..tools import TaskManager, ApiController, BaseController
+
+
+  @ApiController('dummy_task')
+  class DummyTask(BaseController):
+      def _dummy(self):
+          top = random.randrange(100)
+          for i in range(top):
+              TaskManager.current_task().set_progress(i*100/top)
+              # or TaskManager.current_task().inc_progress(100/top)
+              time.sleep(1)
+          return "finished"
+
+      @cherrypy.expose
+      @cherrypy.tools.json_out()
+      def default(self):
+          task = TaskManager.run("dummy/task", {}, self._dummy)
+          return task.wait(5)  # wait for five seconds
+
+
+How to deal with asynchronous tasks in the front-end?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+All executing and most recently finished asynchronous tasks are displayed on
+"Background-Tasks" and if finished on "Recent-Notifications" in the menu bar.
+For each task a operation name for three states (running, success and failure),
+a function that tells who is involved and error descriptions, if any, have to
+be provided. This can be  achieved by appending
+``TaskManagerMessageService.messages``.  This has to be done to achieve
+consistency among all tasks and states.
+
+Operation Object
+  Ensures consistency among all tasks. It consists of three verbs for each
+  different state f.e.
+  ``{running: 'Creating', failure: 'create', success: 'Created'}``.
+
+#. Put running operations in present participle f.e. ``'Updating'``.
+#. Failed messages always start with ``'Failed to '`` and should be continued
+   with the operation in present tense f.e. ``'update'``.
+#. Put successful operations in past tense f.e. ``'Updated'``.
+
+Involves Function
+  Ensures consistency among all messages of a task, it resembles who's
+  involved by the operation. It's a function that returns a string which
+  takes the metadata from the task to return f.e.
+  ``"RBD 'somePool/someImage'"``.
+
+Both combined create the following messages:
+
+* Failure => ``"Failed to create RBD 'somePool/someImage'"``
+* Running => ``"Creating RBD 'somePool/someImage'"``
+* Success => ``"Created RBD 'somePool/someImage'"``
+
+For automatic task handling use ``TaskWrapperService.wrapTaskAroundCall``.
+
+If for some reason ``wrapTaskAroundCall`` is not working for you,
+you have to subscribe to your asynchronous task manually through
+``TaskManagerService.subscribe``, and provide it with a callback,
+in case of a success to notify the user. A notification can
+be triggered with ``NotificationService.notifyTask``. It will use
+``TaskManagerMessageService.messages`` to display a message based on the state
+of a task.
+
+Notifications of API errors are handled by ``ApiInterceptorService``.
+
+Usage example:
+
+.. code-block:: javascript
+
+  export class TaskManagerMessageService {
+    // ...
+    messages = {
+      // Messages for task 'rbd/create'
+      'rbd/create': new TaskManagerMessage(
+        // Message prefixes
+        ['create', 'Creating', 'Created'],
+        // Message suffix
+        (metadata) => `RBD '${metadata.pool_name}/${metadata.image_name}'`,
+        (metadata) => ({
+          // Error code and description
+          '17': `Name is already used by RBD '${metadata.pool_name}/${
+                 metadata.image_name}'.`
+        })
+      ),
+      // ...
+    };
+    // ...
+  }
+
+  export class RBDFormComponent {
+    // ...
+    createAction() {
+      const request = this.createRequest();
+      // Subscribes to 'call' with submitted 'task' and handles notifications
+      return this.taskWrapper.wrapTaskAroundCall({
+        task: new FinishedTask('rbd/create', {
+          pool_name: request.pool_name,
+          image_name: request.name
+        }),
+        call: this.rbdService.create(request)
+      });
+    }
+    // ...
+  }
+
+
+REST API documentation
+~~~~~~~~~~~~~~~~~~~~~~
+Ceph-Dashboard provides two types of documentation for the **Ceph RESTful API**:
+
+* **Static documentation**: available at :ref:`mgr-ceph-api`. This comes from a versioned specification located at ``src/pybind/mgr/dashboard/openapi.yaml``.
+* **Interactive documentation**: available from a running Ceph-Dashboard instance (top-right ``?`` icon > API Docs).
+
+If changes are made to the ``controllers/`` directory, it's very likely that
+they will result in changes to the generated OpenAPI specification. For that
+reason, a checker has been implemented to block unintended changes. This check
+is automatically triggered by the Pull Request CI (``make check``) and can be
+also manually invoked: ``tox -e openapi-check``.
+
+If that checker failed, it means that the current Pull Request is modifying the
+Ceph API and therefore:
+
+#. The versioned OpenAPI specification should be updated explicitly: ``tox -e openapi-fix``.
+#. The team @ceph/api will be requested for reviews (this is automated via Github CODEOWNERS), in order to asses the impact of changes.
+
+Additionally, Sphinx documentation can be generated from the OpenAPI
+specification with ``tox -e openapi-doc``.
+
+The Ceph RESTful OpenAPI specification is dynamically generated from the
+``Controllers`` in ``controllers/`` directory.  However, by default it is not
+very detailed, so there are two decorators that can and should be used to add
+more information:
+
+* ``@EndpointDoc()`` for documentation of endpoints. It has four optional arguments
+  (explained below): ``description``, ``group``, ``parameters`` and
+  ``responses``.
+* ``@ControllerDoc()`` for documentation of controller or group associated with
+  the endpoints. It only takes the two first arguments: ``description`` and
+  ``group``.
+
+
+``description``: A a string with a short (1-2 sentences) description of the object.
+
+
+``group``: By default, an endpoint is grouped together with other endpoints
+within the same controller class. ``group`` is a string that can be used to
+assign an endpoint or all endpoints in a class to another controller or a
+conceived group name.
+
+
+``parameters``: A dict used to describe path, query or request body parameters.
+By default, all parameters for an endpoint are listed on the Swagger UI page,
+including information of whether the parameter is optional/required and default
+values. However, there will be no description of the parameter and the parameter
+type will only be displayed in some cases.
+When adding information, each parameters should be described as in the example
+below. Note that the parameter type should be expressed as a built-in python
+type and not as a string. Allowed values are ``str``, ``int``, ``bool``, ``float``.
+
+.. code-block:: python
+
+ @EndpointDoc(parameters={'my_string': (str, 'Description of my_string')})
+ def method(my_string): pass
+
+For body parameters, more complex cases are possible. If the parameter is a
+dictionary, the type should be replaced with a ``dict`` containing its nested
+parameters. When describing nested parameters, the same format as other
+parameters is used. However, all nested parameters are set as required by default.
+If the nested parameter is optional this must be specified as for ``item2`` in
+the example below. If a nested parameters is set to optional, it is also
+possible to specify the default value (this will not be provided automatically
+for nested parameters).
+
+.. code-block:: python
+
+  @EndpointDoc(parameters={
+    'my_dictionary': ({
+      'item1': (str, 'Description of item1'),
+      'item2': (str, 'Description of item2', True),  # item2 is optional
+      'item3': (str, 'Description of item3', True, 'foo'),  # item3 is optional with 'foo' as default value
+  }, 'Description of my_dictionary')})
+  def method(my_dictionary): pass
+
+If the parameter is a ``list`` of primitive types, the type should be
+surrounded with square brackets.
+
+.. code-block:: python
+
+  @EndpointDoc(parameters={'my_list': ([int], 'Description of my_list')})
+  def method(my_list): pass
+
+If the parameter is a ``list`` with nested parameters, the nested parameters
+should be placed in a dictionary and surrounded with square brackets.
+
+.. code-block:: python
+
+  @EndpointDoc(parameters={
+    'my_list': ([{
+      'list_item': (str, 'Description of list_item'),
+      'list_item2': (str, 'Description of list_item2')
+  }], 'Description of my_list')})
+  def method(my_list): pass
+
+
+``responses``: A dict used for describing responses. Rules for describing
+responses are the same as for request body parameters, with one difference:
+responses also needs to be assigned to the related response code as in the
+example below:
+
+.. code-block:: python
+
+  @EndpointDoc(responses={
+    '400':{'my_response': (str, 'Description of my_response')}})
+  def method(): pass
+
+
+Error Handling in Python
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Good error handling is a key requirement in creating a good user experience
+and providing a good API.
+
+Dashboard code should not duplicate C++ code. Thus, if error handling in C++
+is sufficient to provide good feedback, a new wrapper to catch these errors
+is not necessary. On the other hand, input validation is the best place to
+catch errors and generate the best error messages. If required, generate
+errors as soon as possible.
+
+The backend provides few standard ways of returning errors.
+
+First, there is a generic Internal Server Error::
+
+    Status Code: 500
+    {
+        "version": <cherrypy version, e.g. 13.1.0>,
+        "detail": "The server encountered an unexpected condition which prevented it from fulfilling the request.",
+    }
+
+
+For errors generated by the backend, we provide a standard error
+format::
+
+    Status Code: 400
+    {
+        "detail": str(e),     # E.g. "[errno -42] <some error message>"
+        "component": "rbd",   # this can be null to represent a global error code
+        "code": "3",          # Or a error name, e.g. "code": "some_error_key"
+    }
+
+
+In case, the API Endpoints uses @ViewCache to temporarily cache results,
+the error looks like so::
+
+    Status Code 400
+    {
+        "detail": str(e),     # E.g. "[errno -42] <some error message>"
+        "component": "rbd",   # this can be null to represent a global error code
+        "code": "3",          # Or a error name, e.g. "code": "some_error_key"
+        'status': 3,          # Indicating the @ViewCache error status
+    }
+
+In case, the API Endpoints uses a task the error looks like so::
+
+    Status Code 400
+    {
+        "detail": str(e),     # E.g. "[errno -42] <some error message>"
+        "component": "rbd",   # this can be null to represent a global error code
+        "code": "3",          # Or a error name, e.g. "code": "some_error_key"
+        "task": {             # Information about the task itself
+            "name": "taskname",
+            "metadata": {...}
+        }
+    }
+
+
+Our WebUI should show errors generated by the API to the user. Especially
+field-related errors in wizards and dialogs or show non-intrusive notifications.
+
+Handling exceptions in Python should be an exception. In general, we
+should have few exception handlers in our project. Per default, propagate
+errors to the API, as it will take care of all exceptions anyway. In general,
+log the exception by adding ``logger.exception()`` with a description to the
+handler.
+
+We need to distinguish between user errors from internal errors and
+programming errors. Using different exception types will ease the
+task for the API layer and for the user interface:
+
+Standard Python errors, like ``SystemError``, ``ValueError`` or ``KeyError``
+will end up as internal server errors in the API.
+
+In general, do not ``return`` error responses in the REST API. They will be
+returned by the  error handler. Instead, raise the appropriate exception.
+
+Plug-ins
+~~~~~~~~
+
+New functionality can be provided by means of a plug-in architecture. Among the
+benefits this approach brings in, loosely coupled development is one of the most
+notable. As the Ceph Dashboard grows in feature richness, its code-base becomes
+more and more complex. The hook-based nature of a plug-in architecture allows to
+extend functionality in a controlled manner, and isolate the scope of the
+changes.
+
+Ceph Dashboard relies on `Pluggy <https://pluggy.readthedocs.io>`_ to provide
+for plug-ing support. On top of pluggy, an interface-based approach has been
+implemented, with some safety checks (method override and abstract method
+checks).
+
+In order to create a new plugin, the following steps are required:
+
+#. Add a new file under ``src/pybind/mgr/dashboard/plugins``.
+#. Import the ``PLUGIN_MANAGER`` instance and the ``Interfaces``.
+#. Create a class extending the desired interfaces. The plug-in library will
+   check if all the methods of the interfaces have been properly overridden.
+#. Register the plugin in the ``PLUGIN_MANAGER`` instance.
+#. Import the plug-in from within the Ceph Dashboard ``module.py`` (currently no
+   dynamic loading is implemented).
+
+The available Mixins (helpers) are:
+
+- ``CanMgr``: provides the plug-in with access to the ``mgr`` instance under ``self.mgr``.
+
+The available Interfaces are:
+
+- ``Initializable``: requires overriding ``init()`` hook. This method is run at
+  the very beginning of the dashboard module, right after all imports have been
+  performed.
+- ``Setupable``: requires overriding ``setup()`` hook. This method is run in the
+  Ceph Dashboard ``serve()`` method, right after CherryPy has been configured,
+  but before it is started. It's a placeholder for the plug-in initialization
+  logic.
+- ``HasOptions``: requires overriding ``get_options()`` hook by returning a list
+  of ``Options()``. The options returned here are added to the
+  ``MODULE_OPTIONS``.
+- ``HasCommands``: requires overriding ``register_commands()`` hook by defining
+  the commands the plug-in can handle and decorating them with ``@CLICommand``.
+  The commands can be optionally returned, so that they can be invoked
+  externally (which makes unit testing easier).
+- ``HasControllers``: requires overriding ``get_controllers()`` hook by defining
+  and returning the controllers as usual.
+- ``FilterRequest.BeforeHandler``: requires overriding
+  ``filter_request_before_handler()`` hook. This method receives a
+  ``cherrypy.request`` object for processing. A usual implementation of this
+  method will allow some requests to pass or will raise a ``cherrypy.HTTPError``
+  based on the ``request`` metadata and other conditions.
+
+New interfaces and hooks should be added as soon as they are required to
+implement new functionality. The above list only comprises the hooks needed for
+the existing plugins.
+
+A sample plugin implementation would look like this:
+
+.. code-block:: python
+
+  # src/pybind/mgr/dashboard/plugins/mute.py
+
+  from . import PLUGIN_MANAGER as PM
+  from . import interfaces as I
+
+  from mgr_module import CLICommand, Option
+  import cherrypy
+
+  @PM.add_plugin
+  class Mute(I.CanMgr, I.Setupable, I.HasOptions, I.HasCommands,
+                       I.FilterRequest.BeforeHandler, I.HasControllers):
+    @PM.add_hook
+    def get_options(self):
+      return [Option('mute', default=False, type='bool')]
+
+    @PM.add_hook
+    def setup(self):
+      self.mute = self.mgr.get_module_option('mute')
+
+    @PM.add_hook
+    def register_commands(self):
+      @CLICommand("dashboard mute")
+      def _(mgr):
+        self.mute = True
+        self.mgr.set_module_option('mute', True)
+        return 0
+
+    @PM.add_hook
+    def filter_request_before_handler(self, request):
+      if self.mute:
+        raise cherrypy.HTTPError(500, "I'm muted :-x")
+
+    @PM.add_hook
+    def get_controllers(self):
+      from ..controllers import ApiController, RESTController
+
+      @ApiController('/mute')
+      class MuteController(RESTController):
+        def get(_):
+          return self.mute
+
+      return [MuteController]
+
+
+Additionally, a helper for creating plugins ``SimplePlugin`` is provided. It
+facilitates the basic tasks (Options, Commands, and common Mixins). The previous
+plugin could be rewritten like this:
+
+.. code-block:: python
+
+  from . import PLUGIN_MANAGER as PM
+  from . import interfaces as I
+  from .plugin import SimplePlugin as SP
+
+  import cherrypy
+
+  @PM.add_plugin
+  class Mute(SP, I.Setupable, I.FilterRequest.BeforeHandler, I.HasControllers):
+    OPTIONS = [
+        SP.Option('mute', default=False, type='bool')
+    ]
+
+    def shut_up(self):
+      self.set_option('mute', True)
+      self.mute = True
+      return 0
+
+    COMMANDS = [
+        SP.Command("dashboard mute", handler=shut_up)
+    ]
+
+    @PM.add_hook
+    def setup(self):
+      self.mute = self.get_option('mute')
+
+    @PM.add_hook
+    def filter_request_before_handler(self, request):
+      if self.mute:
+        raise cherrypy.HTTPError(500, "I'm muted :-x")
+
+    @PM.add_hook
+    def get_controllers(self):
+      from ..controllers import ApiController, RESTController
+
+      @ApiController('/mute')
+      class MuteController(RESTController):
+        def get(_):
+          return self.mute
+
+      return [MuteController]
diff --git a/doc/dev/developer_guide/essentials.rst b/doc/dev/developer_guide/essentials.rst
new file mode 100644
index 000000000..2fe7a13cd
--- /dev/null
+++ b/doc/dev/developer_guide/essentials.rst
@@ -0,0 +1,338 @@
+Essentials (tl;dr)
+==================
+
+This chapter presents essential information that every Ceph developer needs
+to know.
+
+Leads
+-----
+
+The Ceph project is led by Sage Weil. In addition, each major project
+component has its own lead. The following table shows all the leads and
+their nicks on `GitHub`_:
+
+.. _github: https://github.com/
+
+========= ================ =============
+Scope     Lead             GitHub nick
+========= ================ =============
+Ceph      Sage Weil        liewegas
+RADOS     Neha Ojha        neha-ojha
+RGW       Yehuda Sadeh     yehudasa
+RGW       Matt Benjamin    mattbenjamin
+RBD       Jason Dillaman   dillaman
+CephFS    Patrick Donnelly batrick
+Dashboard Lenz Grimmer     LenzGr
+MON       Joao Luis        jecluis
+Build/Ops Ken Dreyer       ktdreyer
+Docs      Zac Dover        zdover23
+========= ================ =============
+
+The Ceph-specific acronyms in the table are explained in
+:doc:`/architecture`.
+
+History
+-------
+
+See the `History chapter of the Wikipedia article`_.
+
+.. _`History chapter of the Wikipedia article`: https://en.wikipedia.org/wiki/Ceph_%28software%29#History
+
+Licensing
+---------
+
+Ceph is free software.
+
+Unless stated otherwise, the Ceph source code is distributed under the
+terms of the LGPL2.1 or LGPL3.0. For full details, see the file
+`COPYING`_ in the top-level directory of the source-code tree.
+
+.. _`COPYING`:
+  https://github.com/ceph/ceph/blob/master/COPYING
+
+Source code repositories
+------------------------
+
+The source code of Ceph lives on `GitHub`_ in a number of repositories below
+the `Ceph "organization"`_.
+
+.. _`Ceph "organization"`: https://github.com/ceph
+
+A working knowledge of git_ is essential to make a meaningful contribution to the project as a developer.
+
+.. _git: https://git-scm.com/doc
+
+Although the `Ceph "organization"`_ includes several software repositories,
+this document covers only one: https://github.com/ceph/ceph.
+
+Redmine issue tracker
+---------------------
+
+Although `GitHub`_ is used for code, Ceph-related issues (Bugs, Features,
+Backports, Documentation, etc.) are tracked at http://tracker.ceph.com,
+which is powered by `Redmine`_.
+
+.. _Redmine: http://www.redmine.org
+
+The tracker has a Ceph project with a number of subprojects loosely
+corresponding to the various architectural components (see
+:doc:`/architecture`).
+
+Mere `registration`_ in the tracker automatically grants permissions
+sufficient to open new issues and comment on existing ones.
+
+.. _registration: http://tracker.ceph.com/account/register
+
+To report a bug or propose a new feature, `jump to the Ceph project`_ and
+click on `New issue`_.
+
+.. _`jump to the Ceph project`: http://tracker.ceph.com/projects/ceph
+.. _`New issue`: http://tracker.ceph.com/projects/ceph/issues/new
+
+.. _mailing-list:
+
+Mailing lists
+-------------
+
+Ceph Development Mailing List
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+The ``dev@ceph.io`` list is for discussion about the development of Ceph,
+its interoperability with other technology, and the operations of the
+project itself.
+
+The email discussion list for Ceph development is open to all. Subscribe by
+sending a message to ``dev-request@ceph.io`` with the following line in the
+body of the message::
+
+    subscribe ceph-devel
+
+
+Ceph Client Patch Review Mailing List
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+The ``ceph-devel@vger.kernel.org`` list is for discussion and patch review
+for the Linux kernel Ceph client component. Note that this list used to
+be an all-encompassing list for developers. When searching the archives, 
+remember that this list contains the generic devel-ceph archives before mid-2018.
+
+Subscribe to the list covering the Linux kernel Ceph client component by sending
+a message to ``majordomo@vger.kernel.org`` with the following line in the body
+of the message::
+
+    subscribe ceph-devel
+
+
+Other Ceph Mailing Lists
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+There are also `other Ceph-related mailing lists`_.
+
+.. _`other Ceph-related mailing lists`: https://ceph.com/irc/
+
+.. _irc:
+
+
+IRC
+---
+
+In addition to mailing lists, the Ceph community also communicates in real time
+using `Internet Relay Chat`_.
+
+.. _`Internet Relay Chat`: http://www.irchelp.org/
+
+The Ceph community gathers in the #ceph channel of the Open and Free Technology
+Community (OFTC) IRC network.
+
+Created in 1988, Internet Relay Chat (IRC) is a relay-based, real-time chat
+protocol. It is mainly designed for group (many-to-many) communication in
+discussion forums called channels, but also allows one-to-one communication via
+private message. On IRC you can talk to many other members using Ceph, on
+topics ranging from idle chit-chat to support questions. Though a channel might
+have many people in it at any one time, they might not always be at their
+keyboard; so if no-one responds, just wait around and someone will hopefully
+answer soon enough.
+
+Registration
+^^^^^^^^^^^^
+
+If you intend to use the IRC service on a continued basis, you are advised to
+register an account. Registering gives you a unique IRC identity and allows you
+to access channels where unregistered users have been locked out for technical
+reasons.
+
+See ``the official OFTC (Open and Free Technology Community) documentation's
+registration instructions
+<https://www.oftc.net/Services/#register-your-account>`` to learn how to
+register your IRC account.
+
+Channels
+~~~~~~~~
+
+To connect to the OFTC IRC network, download an IRC client and configure it to
+connect to ``irc.oftc.net``. Then join one or more of the channels. Discussions
+inside #ceph are logged and archives are available online.
+
+Here are the real-time discussion channels for the Ceph community:
+
+  -  #ceph
+  -  #ceph-devel
+  -  #cephfs
+  -  #ceph-dashboard
+  -  #ceph-orchestrators
+  -  #sepia
+
+.. _submitting-patches:
+
+Submitting patches
+------------------
+
+The canonical instructions for submitting patches are contained in the
+file `CONTRIBUTING.rst`_ in the top-level directory of the source-code
+tree. There may be some overlap between this guide and that file.
+
+.. _`CONTRIBUTING.rst`:
+  https://github.com/ceph/ceph/blob/main/CONTRIBUTING.rst
+
+All newcomers are encouraged to read that file carefully.
+
+Building from source
+--------------------
+
+See instructions at :doc:`/install/build-ceph`.
+
+Using ccache to speed up local builds
+-------------------------------------
+`ccache`_ can make the process of rebuilding the ceph source tree faster. 
+
+Before you use `ccache`_ to speed up your rebuilds of the ceph source tree,
+make sure that your source tree is clean and will produce no build failures.
+When you have a clean source tree, you can confidently use `ccache`_, secure in
+the knowledge that you're not using a dirty tree.
+
+Old build artifacts can cause build failures. You might introduce these
+artifacts unknowingly when switching from one branch to another. If you see
+build errors when you attempt a local build, follow the procedure below to
+clean your source tree.
+
+Cleaning the Source Tree
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. prompt:: bash $
+
+  make clean
+  
+.. note:: The following commands will remove everything in the source tree 
+          that isn't tracked by git. Make sure to back up your log files 
+          and configuration options before running these commands.
+
+.. prompt:: bash $
+
+   git clean -fdx; git submodule foreach git clean -fdx
+
+Building Ceph with ccache
+^^^^^^^^^^^^^^^^^^^^^^^^^
+``ccache`` is available as a package in most distros. To build ceph with
+ccache, run the following command.
+
+.. prompt:: bash $
+
+  cmake -DWITH_CCACHE=ON ..
+
+Using ccache to Speed Up Build Times
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+``ccache`` can be used for speeding up all builds of the system. For more
+details, refer to the `run modes`_ section of the ccache manual. The default
+settings of ``ccache`` can be displayed with the ``ccache -s`` command.
+
+.. note:: We recommend overriding the ``max_size``. The default is 10G.
+          Use a larger value, like 25G. Refer to the `configuration`_ section
+          of the ccache manual for more information.
+
+To further increase the cache hit rate and reduce compile times in a
+development environment, set the version information and build timestamps to
+fixed values. This makes it unnecessary to rebuild the binaries that contain
+this information.
+
+This can be achieved by adding the following settings to the ``ccache``
+configuration file ``ccache.conf``::
+
+  sloppiness = time_macros
+  run_second_cpp = true
+
+Now, set the environment variable ``SOURCE_DATE_EPOCH`` to a fixed value (a
+UNIX timestamp) and set ``ENABLE_GIT_VERSION`` to ``OFF`` when running
+``cmake``:
+
+.. prompt:: bash $
+
+  export SOURCE_DATE_EPOCH=946684800
+  cmake -DWITH_CCACHE=ON -DENABLE_GIT_VERSION=OFF ..
+
+.. note:: Binaries produced with these build options are not suitable for
+  production or debugging purposes, as they do not contain the correct build
+  time and git version information.
+
+.. _`ccache`: https://ccache.samba.org/
+.. _`run modes`: https://ccache.samba.org/manual.html#_run_modes
+.. _`configuration`: https://ccache.samba.org/manual.html#_configuration
+
+Development-mode cluster
+------------------------
+
+See :doc:`/dev/quick_guide`.
+
+Kubernetes/Rook development cluster
+-----------------------------------
+
+See :ref:`kubernetes-dev`
+
+.. _backporting:
+
+Backporting
+-----------
+
+All bugfixes should be merged to the ``main`` branch before being
+backported. To flag a bugfix for backporting, make sure it has a
+`tracker issue`_ associated with it and set the ``Backport`` field to a
+comma-separated list of previous releases (e.g. "hammer,jewel") that you think
+need the backport.
+The rest (including the actual backporting) will be taken care of by the
+`Stable Releases and Backports`_ team.
+
+.. _`tracker issue`: http://tracker.ceph.com/
+.. _`Stable Releases and Backports`: http://tracker.ceph.com/projects/ceph-releases/wiki
+
+Dependabot
+----------
+
+Dependabot is a GitHub bot that scans the dependencies in the repositories for
+security vulnerabilities (CVEs). If a fix is available for a discovered CVE,
+Dependabot creates a pull request to update the dependency.
+
+Dependabot also indicates the compatibility score of the upgrade. This score is
+based on the number of CI failures that occur in other GitHub repositories
+where the fix was applied. 
+
+With some configuration, Dependabot can perform non-security updates (for
+example, it can upgrade to the latest minor version or patch version).
+
+Dependabot supports `several languages and package managers
+<https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/about-dependabot-version-updates#supported-repositories-and-ecosystems>`_.
+As of July 2022, the Ceph project receives alerts only from pip (based on the
+`requirements.txt` files) and npm (`package*.json`). It is possible to extend
+these alerts to git submodules, Golang, and Java. As of July 2022, there is no
+support for C++ package managers such as vcpkg, conan, C++20 modules.
+
+Many of the dependencies discovered by Dependabot will best be updated
+elsewhere than the Ceph Github repository (distribution packages, for example,
+will be a better place to update some of the dependencies). Nonetheless, the
+list of new and existing vulnerabilities generated by Dependabot will be
+useful.
+
+`Here is an example of a Dependabot pull request.
+<https://github.com/ceph/ceph/pull/46998>`_
+
+Guidance for use of cluster log
+-------------------------------
+
+If your patches emit messages to the Ceph cluster log, please consult
+this: :doc:`/dev/logging`.
diff --git a/doc/dev/developer_guide/index.rst b/doc/dev/developer_guide/index.rst
new file mode 100644
index 000000000..30d5e5e22
--- /dev/null
+++ b/doc/dev/developer_guide/index.rst
@@ -0,0 +1,25 @@
+============================================
+Contributing to Ceph: A Guide for Developers
+============================================
+
+:Author: Loic Dachary
+:Author: Nathan Cutler
+:License: Creative Commons Attribution Share Alike 3.0 (CC-BY-SA-3.0)
+
+.. note:: You may also be interested in the :doc:`/dev/internals` documentation.
+
+.. toctree::
+   :maxdepth: 1
+
+   Introduction <intro>
+   Essentials <essentials>
+   What is Merged and When <merging>
+   Issue tracker <issue-tracker>
+   Basic workflow <basic-workflow>
+   Tests: Unit Tests <tests-unit-tests>
+   Tests: Integration Tests <tests-integration-tests>
+   Running Tests Locally <running-tests-locally>
+   Running Integration Tests using Teuthology <running-tests-using-teuth>
+   Running Tests in the Cloud <running-tests-in-cloud>
+   Ceph Dashboard Developer Documentation (formerly HACKING.rst) <dash-devel>
+   cephadm Developer Documentation  <../cephadm/index>
diff --git a/doc/dev/developer_guide/intro.rst b/doc/dev/developer_guide/intro.rst
new file mode 100644
index 000000000..67b449c55
--- /dev/null
+++ b/doc/dev/developer_guide/intro.rst
@@ -0,0 +1,25 @@
+Introduction
+============
+
+This guide has two aims. First, it should lower the barrier to entry for
+software developers who wish to get involved in the Ceph project. Second,
+it should serve as a reference for Ceph developers.
+
+We assume that readers are already familiar with Ceph (the distributed
+object store and file system designed to provide excellent performance,
+reliability and scalability). If not, please refer to the `project website`_
+and especially the `publications list`_. Another way to learn about what's
+happening in Ceph is to check out our `youtube channel`_ , where we post Tech
+Talks, Code walk-throughs and Ceph Developer Monthly recordings.
+
+.. _`project website`: https://ceph.com
+.. _`publications list`: https://ceph.com/publications/
+.. _`youtube channel`: https://www.youtube.com/c/CephStorage
+
+Since this document is to be consumed by developers, who are assumed to
+have Internet access, topics covered elsewhere, either within the Ceph
+documentation or elsewhere on the web, are treated by linking. If you
+notice that a link is broken or if you know of a better link, please
+`report it as a bug`_.
+
+.. _`report it as a bug`: http://tracker.ceph.com/projects/ceph/issues/new
diff --git a/doc/dev/developer_guide/issue-tracker.rst b/doc/dev/developer_guide/issue-tracker.rst
new file mode 100644
index 000000000..eae68f3f0
--- /dev/null
+++ b/doc/dev/developer_guide/issue-tracker.rst
@@ -0,0 +1,39 @@
+.. _issue-tracker:
+
+Issue Tracker
+=============
+
+See `Redmine Issue Tracker`_ for a brief introduction to the Ceph Issue
+Tracker.
+
+Ceph developers use the issue tracker to
+
+1. keep track of issues - bugs, fix requests, feature requests, backport
+requests, etc.
+
+2. communicate with other developers and keep them informed as work
+on the issues progresses.
+
+Issue tracker conventions
+-------------------------
+
+When you start working on an existing issue, it's nice to let the other
+developers know this - to avoid duplication of labor. Typically, this is
+done by changing the :code:`Assignee` field (to yourself) and changing the
+:code:`Status` to *In progress*. Newcomers to the Ceph community typically do
+not have sufficient privileges to update these fields, however: they can
+simply update the issue with a brief note.
+
+.. table:: Meanings of some commonly used statuses
+
+   ================ ===========================================
+   Status           Meaning
+   ================ ===========================================
+   New              Initial status
+   In Progress      Somebody is working on it
+   Need Review      Pull request is open with a fix
+   Pending Backport Fix has been merged, backport(s) pending
+   Resolved         Fix and backports (if any) have been merged
+   ================ ===========================================
+
+.. _Redmine issue tracker: https://tracker.ceph.com
diff --git a/doc/dev/developer_guide/merging.rst b/doc/dev/developer_guide/merging.rst
new file mode 100644
index 000000000..36e10fc84
--- /dev/null
+++ b/doc/dev/developer_guide/merging.rst
@@ -0,0 +1,138 @@
+.. _merging:
+
+Commit merging:  scope and cadence
+==================================
+
+Commits are merged into branches according to criteria specific to each phase
+of the Ceph release lifecycle. This chapter codifies these criteria.
+
+Development releases (i.e. x.0.z)
+---------------------------------
+
+What ?
+^^^^^^
+
+* Features
+* Bug fixes
+
+Where ?
+^^^^^^^
+
+Features are merged to the *main* branch. Bug fixes should be merged to the
+corresponding named branch (e.g. *nautilus* for 14.0.z, *pacific* for 16.0.z,
+etc.). However, this is not mandatory - bug fixes and documentation
+enhancements can be merged to the *main* branch as well, since the *main*
+branch is itself occasionally merged to the named branch during the development
+releases phase. In either case, if a bug fix is important it can also be
+flagged for backport to one or more previous stable releases.
+
+When ?
+^^^^^^
+
+After each stable release, candidate branches for previous releases enter
+phase 2 (see below).  For example: the *jewel* named branch was created when
+the *infernalis* release candidates entered phase 2. From this point on,
+*main* was no longer associated with *infernalis*. After he named branch of
+the next stable release is created, *main* will be occasionally merged into
+it.
+
+Branch merges
+^^^^^^^^^^^^^
+
+* The latest stable release branch is merged periodically into main.
+* The main branch is merged periodically into the branch of the stable release.
+* The main is merged into the stable release branch
+  immediately after each development (x.0.z) release.
+
+Stable release candidates (i.e. x.1.z) phase 1
+----------------------------------------------
+
+What ?
+^^^^^^
+
+* Bug fixes only
+
+Where ?
+^^^^^^^
+
+The stable release branch (e.g. *jewel* for 10.0.z, *luminous*
+for 12.0.z, etc.) or *main*.  Bug fixes should be merged to the named
+branch corresponding to the stable release candidate (e.g. *jewel* for
+10.1.z) or to *main*. During this phase, all commits to *main* will be
+merged to the named branch, and vice versa. In other words, it makes
+no difference whether a commit is merged to the named branch or to
+*main* - it will make it into the next release candidate either way.
+
+When ?
+^^^^^^
+
+After the first stable release candidate is published, i.e. after the
+x.1.0 tag is set in the release branch.
+
+Branch merges
+^^^^^^^^^^^^^
+
+* The stable release branch is merged periodically into *main*.
+* The *main* branch is merged periodically into the stable release branch.
+* The *main* branch is merged into the stable release branch
+  immediately after each x.1.z release candidate.
+
+Stable release candidates (i.e. x.1.z) phase 2
+----------------------------------------------
+
+What ?
+^^^^^^
+
+* Bug fixes only
+
+Where ?
+^^^^^^^
+
+The stable release branch (e.g. *mimic* for 13.0.z, *octopus* for 15.0.z
+,etc.). During this phase, all commits to the named branch will be merged into
+*main*. Cherry-picking to the named branch during release candidate phase 2
+is performed manually since the official backporting process begins only when
+the release is pronounced "stable".
+
+When ?
+^^^^^^
+
+After Sage Weil announces that it is time for phase 2 to happen.
+
+Branch merges
+^^^^^^^^^^^^^
+
+* The stable release branch is occasionally merged into main.
+
+Stable releases (i.e. x.2.z)
+----------------------------
+
+What ?
+^^^^^^
+
+* Bug fixes
+* Features are sometime accepted
+* Commits should be cherry-picked from *main* when possible
+* Commits that are not cherry-picked from *main* must pertain to a bug unique to
+  the stable release
+* See also the `backport HOWTO`_ document
+
+.. _`backport HOWTO`:
+  http://tracker.ceph.com/projects/ceph-releases/wiki/HOWTO#HOWTO
+
+Where ?
+^^^^^^^
+
+The stable release branch (*hammer* for 0.94.x, *infernalis* for 9.2.x,
+etc.)
+
+When ?
+^^^^^^
+
+After the stable release is published, i.e. after the "vx.2.0" tag is set in
+the release branch.
+
+Branch merges
+^^^^^^^^^^^^^
+
+Never
diff --git a/doc/dev/developer_guide/running-tests-in-cloud.rst b/doc/dev/developer_guide/running-tests-in-cloud.rst
new file mode 100644
index 000000000..60118aefd
--- /dev/null
+++ b/doc/dev/developer_guide/running-tests-in-cloud.rst
@@ -0,0 +1,289 @@
+Running Tests in the Cloud
+==========================
+
+In this chapter, we will explain in detail how use an OpenStack
+tenant as an environment for Ceph `integration testing`_.
+
+Assumptions and caveat
+----------------------
+
+We assume that:
+
+1. you are the only person using the tenant
+2. you have the credentials
+3. the tenant supports the ``nova`` and ``cinder`` APIs
+
+Caveat: be aware that, as of this writing (July 2016), testing in
+OpenStack clouds is a new feature. Things may not work as advertised.
+If you run into trouble, ask for help on `IRC`_ or the `Mailing list`_, or
+open a bug report at the `ceph-workbench bug tracker`_.
+
+.. _`ceph-workbench bug tracker`: http://ceph-workbench.dachary.org/root/ceph-workbench/issues
+
+Prepare tenant
+--------------
+
+If you have not tried to use ``ceph-workbench`` with this tenant before,
+proceed to the next step.
+
+To start with a clean slate, login to your tenant via the Horizon dashboard
+and:
+
+* terminate the ``teuthology`` and ``packages-repository`` instances, if any
+* delete the ``teuthology`` and ``teuthology-worker`` security groups, if any
+* delete the ``teuthology`` and ``teuthology-myself`` key pairs, if any
+
+Also do the above if you ever get key-related errors ("invalid key", etc.)
+when trying to schedule suites.
+
+Getting ceph-workbench
+----------------------
+
+Since testing in the cloud is done using the ``ceph-workbench ceph-qa-suite``
+tool, you will need to install that first. It is designed
+to be installed via Docker, so if you don't have Docker running on your
+development machine, take care of that first. You can follow `the official
+tutorial <https://docs.docker.com/engine/installation/>`_ to install if
+you have not installed yet.
+
+Once Docker is up and running, install ``ceph-workbench`` by following the
+`Installation instructions in the ceph-workbench documentation
+<http://ceph-workbench.readthedocs.io/en/latest/#installation>`_.
+
+Linking ceph-workbench with your OpenStack tenant
+-------------------------------------------------
+
+Before you can trigger your first teuthology suite, you will need to link
+``ceph-workbench`` with your OpenStack account.
+
+First, download a ``openrc.sh`` file by clicking on the "Download OpenStack
+RC File" button, which can be found in the "API Access" tab of the "Access
+& Security" dialog of the OpenStack Horizon dashboard.
+
+Second, create a ``~/.ceph-workbench`` directory, set its permissions to
+700, and move the ``openrc.sh`` file into it. Make sure that the filename
+is exactly ``~/.ceph-workbench/openrc.sh``.
+
+Third, edit the file so it does not ask for your OpenStack password
+interactively. Comment out the relevant lines and replace them with
+something like::
+
+.. prompt:: bash $
+
+    export OS_PASSWORD="aiVeth0aejee3eep8rogho3eep7Pha6ek"
+
+When ``ceph-workbench ceph-qa-suite`` connects to your OpenStack tenant for
+the first time, it will generate two keypairs: ``teuthology-myself`` and
+``teuthology``.
+
+.. If this is not the first time you have tried to use
+.. ``ceph-workbench ceph-qa-suite`` with this tenant, make sure to delete any
+.. stale keypairs with these names!
+
+Run the dummy suite
+-------------------
+
+You are now ready to take your OpenStack teuthology setup for a test
+drive
+
+.. prompt:: bash $
+
+   ceph-workbench ceph-qa-suite --suite dummy
+
+Be forewarned that the first run of ``ceph-workbench ceph-qa-suite`` on a
+pristine tenant will take a long time to complete because it downloads a VM
+image and during this time the command may not produce any output.
+
+The images are cached in OpenStack, so they are only downloaded once.
+Subsequent runs of the same command will complete faster.
+
+Although ``dummy`` suite does not run any tests, in all other respects it
+behaves just like a teuthology suite and produces some of the same
+artifacts.
+
+The last bit of output should look something like this::
+
+  pulpito web interface: http://149.202.168.201:8081/
+  ssh access           : ssh -i /home/smithfarm/.ceph-workbench/teuthology-myself.pem ubuntu@149.202.168.201 # logs in /usr/share/nginx/html
+
+What this means is that ``ceph-workbench ceph-qa-suite`` triggered the test
+suite run. It does not mean that the suite run has completed. To monitor
+progress of the run, check the Pulpito web interface URL periodically, or
+if you are impatient, ssh to the teuthology machine using the ssh command
+shown and do
+
+.. prompt:: bash $
+
+   tail -f /var/log/teuthology.*
+
+The `/usr/share/nginx/html` directory contains the complete logs of the
+test suite. If we had provided the ``--upload`` option to the
+``ceph-workbench ceph-qa-suite`` command, these logs would have been
+uploaded to http://teuthology-logs.public.ceph.com.
+
+Run a standalone test
+---------------------
+
+The standalone test explained in `Reading a standalone test`_ can be run
+with the following command
+
+.. prompt:: bash $
+
+   ceph-workbench ceph-qa-suite --suite rados/singleton/all/admin-socket.yaml
+
+This will run the suite shown on the current ``master`` branch of
+``ceph/ceph.git``. You can specify a different branch with the ``--ceph``
+option, and even a different git repo with the ``--ceph-git-url`` option. (Run
+``ceph-workbench ceph-qa-suite --help`` for an up-to-date list of available
+options.)
+
+The first run of a suite will also take a long time, because ceph packages
+have to be built, first. Again, the packages so built are cached and
+``ceph-workbench ceph-qa-suite`` will not build identical packages a second
+time.
+
+Interrupt a running suite
+-------------------------
+
+Teuthology suites take time to run. From time to time one may wish to
+interrupt a running suite. One obvious way to do this is::
+
+.. prompt:: bash $
+
+   ceph-workbench ceph-qa-suite --teardown
+
+This destroys all VMs created by ``ceph-workbench ceph-qa-suite`` and
+returns the OpenStack tenant to a "clean slate".
+
+Sometimes you may wish to interrupt the running suite, but keep the logs,
+the teuthology VM, the packages-repository VM, etc. To do this, you can
+``ssh`` to the teuthology VM (using the ``ssh access`` command reported
+when you triggered the suite -- see `Run the dummy suite`_) and, once
+there
+
+.. prompt:: bash $
+
+   sudo /etc/init.d/teuthology restart
+
+This will keep the teuthology machine, the logs and the packages-repository
+instance but nuke everything else.
+
+Upload logs to archive server
+-----------------------------
+
+Since the teuthology instance in OpenStack is only semi-permanent, with
+limited space for storing logs, ``teuthology-openstack`` provides an
+``--upload`` option which, if included in the ``ceph-workbench ceph-qa-suite``
+command, will cause logs from all failed jobs to be uploaded to the log
+archive server maintained by the Ceph project. The logs will appear at the
+URL::
+
+    http://teuthology-logs.public.ceph.com/$RUN
+
+where ``$RUN`` is the name of the run. It will be a string like this::
+
+    ubuntu-2016-07-23_16:08:12-rados-hammer-backports---basic-openstack
+
+Even if you don't providing the ``--upload`` option, however, all the logs can
+still be found on the teuthology machine in the directory
+``/usr/share/nginx/html``.
+
+Provision VMs ad hoc
+--------------------
+
+From the teuthology VM, it is possible to provision machines on an "ad hoc"
+basis, to use however you like. The magic incantation is::
+
+.. prompt:: bash $
+
+    teuthology-lock --lock-many $NUMBER_OF_MACHINES \
+        --os-type $OPERATING_SYSTEM \
+        --os-version $OS_VERSION \
+        --machine-type openstack \
+        --owner $EMAIL_ADDRESS
+
+The command must be issued from the ``~/teuthology`` directory. The possible
+values for ``OPERATING_SYSTEM`` AND ``OS_VERSION`` can be found by examining
+the contents of the directory ``teuthology/openstack/``. For example
+
+.. prompt:: bash $
+
+    teuthology-lock --lock-many 1 --os-type ubuntu --os-version 16.04 \
+        --machine-type openstack --owner foo@example.com
+
+When you are finished with the machine, find it in the list of machines
+
+.. prompt:: bash $
+
+    openstack server list
+
+to determine the name or ID, and then terminate it with
+
+.. prompt:: bash $
+
+    openstack server delete $NAME_OR_ID
+
+Deploy a cluster for manual testing
+-----------------------------------
+
+The `teuthology framework`_ and ``ceph-workbench ceph-qa-suite`` are
+versatile tools that automatically provision Ceph clusters in the cloud and
+run various tests on them in an automated fashion. This enables a single
+engineer, in a matter of hours, to perform thousands of tests that would
+keep dozens of human testers occupied for days or weeks if conducted
+manually.
+
+However, there are times when the automated tests do not cover a particular
+scenario and manual testing is desired. It turns out that it is simple to
+adapt a test to stop and wait after the Ceph installation phase, and the
+engineer can then ssh into the running cluster. Simply add the following
+snippet in the desired place within the test YAML and schedule a run with the
+test::
+
+    tasks:
+    - exec:
+        client.0:
+          - sleep 1000000000 # forever
+
+(Make sure you have a ``client.0`` defined in your ``roles`` stanza or adapt
+accordingly.)
+
+The same effect can be achieved using the ``interactive`` task::
+
+    tasks:
+    - interactive
+
+By following the test log, you can determine when the test cluster has entered
+the "sleep forever" condition. At that point, you can ssh to the teuthology
+machine and from there to one of the target VMs (OpenStack) or teuthology
+worker machines machine (Sepia) where the test cluster is running.
+
+The VMs (or "instances" in OpenStack terminology) created by
+``ceph-workbench ceph-qa-suite`` are named as follows:
+
+``teuthology`` - the teuthology machine
+
+``packages-repository`` - VM where packages are stored
+
+``ceph-*`` - VM where packages are built
+
+``target*`` - machines where tests are run
+
+The VMs named ``target*`` are used by tests. If you are monitoring the
+teuthology log for a given test, the hostnames of these target machines can
+be found out by searching for the string ``Locked targets``::
+
+    2016-03-20T11:39:06.166 INFO:teuthology.task.internal:Locked targets:
+      target149202171058.teuthology: null
+      target149202171059.teuthology: null
+
+The IP addresses of the target machines can be found by running ``openstack
+server list`` on the teuthology machine, but the target VM hostnames (e.g.
+``target149202171058.teuthology``) are resolvable within the teuthology
+cluster.
+
+.. _Integration testing: ../tests-integration-tests
+.. _IRC:  ../essentials/#irc
+.. _Mailing List: ../essentials/#mailing-list
+.. _Reading A Standalone Test: ../testing-integration-tests/#reading-a-standalone-test
+.. _teuthology framework: https://github.com/ceph/teuthology
diff --git a/doc/dev/developer_guide/running-tests-locally.rst b/doc/dev/developer_guide/running-tests-locally.rst
new file mode 100644
index 000000000..b786c12e8
--- /dev/null
+++ b/doc/dev/developer_guide/running-tests-locally.rst
@@ -0,0 +1,138 @@
+Running Unit Tests
+==================
+
+How to run s3-tests locally
+---------------------------
+
+RGW code can be tested by building Ceph locally from source, starting a vstart
+cluster, and running the "s3-tests" suite against it.
+
+The following instructions should work on jewel and above.
+
+Step 1 - build Ceph
+^^^^^^^^^^^^^^^^^^^
+
+Refer to :doc:`/install/build-ceph`.
+
+You can do step 2 separately while it is building.
+
+Step 2 - vstart
+^^^^^^^^^^^^^^^
+
+When the build completes, and still in the top-level directory of the git
+clone where you built Ceph, do the following, for cmake builds::
+
+    cd build/
+    RGW=1 ../src/vstart.sh -n
+
+This will produce a lot of output as the vstart cluster is started up. At the
+end you should see a message like::
+
+    started.  stop.sh to stop.  see out/* (e.g. 'tail -f out/????') for debug output.
+
+This means the cluster is running.
+
+
+Step 3 - run s3-tests
+^^^^^^^^^^^^^^^^^^^^^
+
+.. highlight:: console
+
+To run the s3tests suite do the following::
+
+   $ ../qa/workunits/rgw/run-s3tests.sh
+
+
+Running test using vstart_runner.py
+-----------------------------------
+CephFS and Ceph Manager code is be tested using `vstart_runner.py`_.
+
+Running your first test
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+The Python tests in Ceph repository can be executed on your local machine
+using `vstart_runner.py`_. To do that, you'd need `teuthology`_ installed::
+
+    $ virtualenv --python=python3 venv
+    $ source venv/bin/activate
+    $ pip install 'setuptools >= 12'
+    $ pip install git+https://github.com/ceph/teuthology#egg=teuthology[test]
+    $ deactivate
+
+The above steps installs teuthology in a virtual environment. Before running
+a test locally, build Ceph successfully from the source (refer
+:doc:`/install/build-ceph`) and do::
+
+    $ cd build
+    $ ../src/vstart.sh -n -d -l
+    $ source ~/path/to/teuthology/venv/bin/activate
+
+To run a specific test, say `test_reconnect_timeout`_ from
+`TestClientRecovery`_ in ``qa/tasks/cephfs/test_client_recovery``, you can
+do::
+
+    $ python ../qa/tasks/vstart_runner.py tasks.cephfs.test_client_recovery.TestClientRecovery.test_reconnect_timeout
+
+The above command runs vstart_runner.py and passes the test to be executed as
+an argument to vstart_runner.py. In a similar way, you can also run the group
+of tests in the following manner::
+
+    $ # run all tests in class TestClientRecovery
+    $ python ../qa/tasks/vstart_runner.py tasks.cephfs.test_client_recovery.TestClientRecovery
+    $ # run  all tests in test_client_recovery.py
+    $ python ../qa/tasks/vstart_runner.py tasks.cephfs.test_client_recovery
+
+Based on the argument passed, vstart_runner.py collects tests and executes as
+it would execute a single test.
+
+vstart_runner.py can take the following options -
+
+--clear-old-log             deletes old log file before running the test
+--create                    create Ceph cluster before running a test
+--create-cluster-only       creates the cluster and quits; tests can be issued
+                            later
+--interactive               drops a Python shell when a test fails
+--log-ps-output             logs ps output; might be useful while debugging
+--teardown                  tears Ceph cluster down after test(s) has finished
+                            runnng
+--kclient                   use the kernel cephfs client instead of FUSE
+--brxnet=<net/mask>         specify a new net/mask for the mount clients' network
+                            namespace container (Default: 192.168.0.0/16)
+
+.. note:: If using the FUSE client, ensure that the fuse package is installed
+          and enabled on the system and that ``user_allow_other`` is added
+          to ``/etc/fuse.conf``.
+
+.. note:: If using the kernel client, the user must have the ability to run
+          commands with passwordless sudo access. A failure on the kernel
+          client may crash the host, so it's recommended to use this
+          functionality within a virtual machine.
+
+Internal working of vstart_runner.py -
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+vstart_runner.py primarily does three things -
+
+* collects and runs the tests
+    vstart_runner.py setups/teardowns the cluster and collects and runs the
+    test. This is implemented using methods ``scan_tests()``, ``load_tests()``
+    and ``exec_test()``. This is where all the options that vstart_runner.py
+    takes are implemented along with other features like logging and copying
+    the traceback to the bottom of the log.
+
+* provides an interface for issuing and testing shell commands
+    The tests are written assuming that the cluster exists on remote machines.
+    vstart_runner.py provides an interface to run the same tests with the
+    cluster that exists within the local machine. This is done using the class
+    ``LocalRemote``. Class ``LocalRemoteProcess`` can manage the process that
+    executes the commands from ``LocalRemote``, class ``LocalDaemon`` provides
+    an interface to handle Ceph daemons and class ``LocalFuseMount`` can
+    create and handle FUSE mounts.
+
+* provides an interface to operate Ceph cluster
+    ``LocalCephManager`` provides methods to run Ceph cluster commands with
+    and without admin socket and ``LocalCephCluster`` provides methods to set
+    or clear ``ceph.conf``.
+
+.. _test_reconnect_timeout: https://github.com/ceph/ceph/blob/master/qa/tasks/cephfs/test_client_recovery.py#L133
+.. _TestClientRecovery: https://github.com/ceph/ceph/blob/master/qa/tasks/cephfs/test_client_recovery.py#L86
+.. _teuthology: https://github.com/ceph/teuthology
+.. _vstart_runner.py: https://github.com/ceph/ceph/blob/master/qa/tasks/vstart_runner.py
diff --git a/doc/dev/developer_guide/running-tests-using-teuth.rst b/doc/dev/developer_guide/running-tests-using-teuth.rst
new file mode 100644
index 000000000..492b7790e
--- /dev/null
+++ b/doc/dev/developer_guide/running-tests-using-teuth.rst
@@ -0,0 +1,183 @@
+Running Integration Tests using Teuthology
+==========================================
+
+Getting binaries
+----------------
+To run integration tests using teuthology, you need to have Ceph binaries
+built for your branch. Follow these steps to initiate the build process -
+
+#. Push the branch to `ceph-ci`_ repository. This triggers the process of
+   building the binaries.
+
+#. To confirm that the build process has been initiated, spot the branch name
+   at `Shaman`_. Little after the build process has been initiated, the single
+   entry with your branch name would multiply, each new entry for a different
+   combination of distro and flavour.
+
+#. Wait until the packages are built and uploaded, and the repository offering
+   them are created. This is marked by colouring the entries for the branch
+   name green. Preferably, wait until each entry is coloured green. Usually,
+   it takes around 2-3 hours depending on the availability of the machines.
+
+.. note:: Branch to be pushed on ceph-ci can be any branch, it shouldn't
+   necessarily be a PR branch.
+
+.. note:: In case you are pushing master or any other standard branch, check
+   `Shaman`_ beforehand since it already might have builds ready for it.
+
+Triggering Tests
+----------------
+After building is complete, proceed to trigger tests -
+
+#. Log in to the teuthology machine::
+
+       ssh <username>@teuthology.front.sepia.ceph.com
+
+   This would require Sepia lab access. To know how to request it, see: https://ceph.github.io/sepia/adding_users/
+
+#. Next, get teuthology installed. Run the first set of commands in
+   `Running Your First Test`_ for that. After that, activate the virtual
+   environment in which teuthology is installed.
+
+#. Run the ``teuthology-suite`` command::
+
+        teuthology-suite -v -m smithi -c wip-devname-feature-x -s fs -p 110 --filter "cephfs-shell"
+
+   Following are the options used in above command with their meanings -
+        -v          verbose
+        -m          machine name
+        -c          branch name, the branch that was pushed on ceph-ci
+        -s          test-suite name
+        -p          higher the number, lower the priority of the job
+        --filter    filter tests in given suite that needs to run, the arg to
+                    filter should be the test you want to run
+
+.. note:: The priority number present in the command above is just a
+   placeholder. It might be highly inappropriate for the jobs you may want to
+   trigger. See `Testing Priority`_ section to pick a priority number.
+
+.. note:: Don't skip passing a priority number, the default value is 1000
+   which way too high; the job probably might never run.
+
+#. Wait for the tests to run. ``teuthology-suite`` prints a link to the
+   `Pulpito`_ page created for the tests triggered.
+
+Other frequently used/useful options are ``-d`` (or ``--distro``),
+``--distroversion``, ``--filter-out``, ``--timeout``, ``flavor``, ``-rerun``,
+``-l`` (for limiting number of jobs) , ``-n`` (for how many times job would
+run) and ``-e`` (for email notifications). Run ``teuthology-suite --help``
+to read description of these and every other options available.
+
+Testing QA changes (without re-building binaires)
+-------------------------------------------------
+While writing a PR you might need to test your PR repeatedly using teuthology.
+If you are making non-QA changes, you need to follow the standard process of
+triggering builds, waiting for it to finish and then triggering tests and
+wait for the result. But if changes you made are purely changes in qa/,
+you don't need rebuild the binaries. Instead you can test binaries built for
+the ceph-ci branch and instruct ``teuthology-suite`` command to use a separate
+branch for running tests. The separate branch can be passed to the command
+by using ``--suite-repo`` and ``--suite-branch``. Pass the link to the GitHub
+fork where your PR branch exists to the first option and pass the PR branch
+name to the second option.
+
+For example, if you want to make changes in ``qa/`` after testing ``branch-x``
+(of which has ceph-ci branch is ``wip-username-branch-x``) by running
+following command::
+
+    teuthology-suite -v -m smithi -c wip-username-branch-x -s fs -p 50 --filter cephfs-shell
+
+You can make the modifications locally, update the PR branch and then
+trigger tests from your PR branch as follows::
+
+    teuthology-suite -v -m smithi -c wip-username-branch-x -s fs -p 50 --filter cephfs-shell --suite-repo https://github.com/username/ceph --suite-branch branch-x
+
+You can verify if the tests were run using this branch by looking at values
+for the keys ``suite_branch``, ``suite_repo`` and ``suite_sha1`` in the job
+config printed at the very beginning of the teuthology job.
+
+About Suites and Filters
+------------------------
+See `Suites Inventory`_ for a list of suites of integration tests present
+right now. Alternatively, each directory under ``qa/suites`` in Ceph
+repository is an integration test suite, so looking within that directory
+to decide an appropriate argument for ``-s`` also works.
+
+For picking an argument for ``--filter``, look within
+``qa/suites/<suite-name>/<subsuite-name>/tasks`` to get keywords for filtering
+tests. Each YAML file in there can trigger a bunch of tests; using the name of
+the file, without the extension part of the file name, as an argument to the
+``--filter`` will trigger those tests. For example, the sample command above
+uses ``cephfs-shell`` since there's a file named ``cephfs-shell.yaml`` in
+``qa/suites/fs/basic_functional/tasks/``. In case, the file name doesn't hint
+what bunch of tests it would trigger, look at the contents of the file for
+``modules`` attribute. For ``cephfs-shell.yaml`` the ``modules`` attribute
+is ``tasks.cephfs.test_cephfs_shell`` which means it'll trigger all tests in
+``qa/tasks/cephfs/test_cephfs_shell.py``.
+
+Killing Tests
+-------------
+Sometimes a teuthology job might not complete running for several minutes or
+even hours after tests that were trigged have completed running and other
+times wrong set of tests can be triggered is filter wasn't chosen carefully.
+To save resource it's better to termniate such a job. Following is the command
+to terminate a job::
+
+        teuthology-kill -r teuthology-2019-12-10_05:00:03-smoke-master-testing-basic-smithi
+
+Let's call the argument passed to ``-r`` as test ID. It can be found
+easily in the link to the Pulpito page for the tests you triggered. For
+example, for the above test ID, the link is - http://pulpito.front.sepia.ceph.com/teuthology-2019-12-10_05:00:03-smoke-master-testing-basic-smithi/
+
+Re-running Tests
+----------------
+Pass ``--rerun`` option, with test ID as an argument to it, to
+``teuthology-suite`` command::
+
+    teuthology-suite -v -m smithi -c wip-rishabh-fs-test_cephfs_shell-fix -p 50 --rerun teuthology-2019-12-10_05:00:03-smoke-master-testing-basic-smithi
+
+The meaning of rest of the options is already covered in `Triggering Tests`
+section.
+
+Teuthology Archives
+-------------------
+Once the tests have finished running, the log for the job can be obtained by
+clicking on job ID at the Pulpito page for your tests. It's more convenient to
+download the log and then view it rather than viewing it in an internet
+browser since these logs can easily be upto size of 1 GB. What's much more
+easier is to log in to the teuthology machine again
+(``teuthology.front.sepia.ceph.com``), and access the following path::
+
+    /ceph/teuthology-archive/<test-id>/<job-id>/teuthology.log
+
+For example, for above test ID path is::
+
+    /ceph/teuthology-archive/teuthology-2019-12-10_05:00:03-smoke-master-testing-basic-smithi/4588482/teuthology.log
+
+This way the log remotely can be viewed remotely without having to wait too
+much.
+
+Naming the ceph-ci branch
+-------------------------
+There are no hard conventions (except for the case of stable branch; see
+next paragraph) for how the branch pushed on ceph-ci is named. But, to make
+builds and tests easily identitifiable on Shaman and Pulpito respectively,
+prepend it with your name. For example branch ``feature-x`` can be named
+``wip-yourname-feature-x`` while pushing on ceph-ci.
+
+In case you are using one of the stable branches (e.g.  nautilis, mimic,
+etc.), include the name of that stable branch in your ceph-ci branch name.
+For example, ``feature-x`` PR branch should be named as
+``wip-feature-x-nautilus``. *This is not just a matter of convention but this,
+more essentially, builds your branch in the correct environment.*
+
+Delete the branch from ceph-ci, once it's not required anymore. If you are
+logged in at GitHub, all your branches on ceph-ci can be easily found here -
+https://github.com/ceph/ceph-ci/branches.
+
+.. _ceph-ci: https://github.com/ceph/ceph-ci
+.. _Pulpito: http://pulpito.front.sepia.ceph.com/
+.. _Running Your First Test: ../running-tests-locally/#running-your-first-test
+.. _Shaman: https://shaman.ceph.com/builds/ceph/
+.. _Suites Inventory: ../tests-integration-tests/#suites-inventory
+.. _Testing Priority: ../tests-integration-tests/#testing-priority
diff --git a/doc/dev/developer_guide/tests-integration-tests.rst b/doc/dev/developer_guide/tests-integration-tests.rst
new file mode 100644
index 000000000..c8e6dbcd4
--- /dev/null
+++ b/doc/dev/developer_guide/tests-integration-tests.rst
@@ -0,0 +1,522 @@
+.. _testing-integration-tests:
+
+Testing - Integration Tests
+===========================
+
+Ceph has two types of tests: :ref:`make check <make-check>` tests and integration tests.
+When a test requires multiple machines, root access or lasts for a
+longer time (for example, to simulate a realistic Ceph deployment), it
+is deemed to be an integration test. Integration tests are organized into
+"suites", which are defined in the `ceph/qa sub-directory`_ and run with
+the ``teuthology-suite`` command.
+
+The ``teuthology-suite`` command is part of the `teuthology framework`_.
+In the sections that follow we attempt to provide a detailed introduction
+to that framework from the perspective of a beginning Ceph developer.
+
+Teuthology consumes packages
+----------------------------
+
+It may take some time to understand the significance of this fact, but it
+is `very` significant. It means that automated tests can be conducted on
+multiple platforms using the same packages (RPM, DEB) that can be
+installed on any machine running those platforms.
+
+Teuthology has a `list of platforms that it supports
+<https://github.com/ceph/ceph/tree/master/qa/distros/supported>`_ (as
+of September 2020 the list consisted of "RHEL/CentOS 8" and "Ubuntu 18.04").  It
+expects to be provided pre-built Ceph packages for these platforms.
+Teuthology deploys these platforms on machines (bare-metal or
+cloud-provisioned), installs the packages on them, and deploys Ceph
+clusters on them - all as called for by the test.
+
+The Nightlies
+-------------
+
+A number of integration tests are run on a regular basis in the `Sepia
+lab`_ against the official Ceph repositories (on the ``master`` development
+branch and the stable branches). Traditionally, these tests are called "the
+nightlies" because the Ceph core developers used to live and work in
+the same time zone and from their perspective the tests were run overnight.
+
+The results of the nightlies are published at http://pulpito.ceph.com/. The
+developer nick shows in the
+test results URL and in the first column of the Pulpito dashboard.  The
+results are also reported on the `ceph-qa mailing list
+<https://ceph.com/irc/>`_ for analysis.
+
+Testing Priority
+----------------
+
+The ``teuthology-suite`` command includes an almost mandatory option ``-p <N>``
+which specifies the priority of the jobs submitted to the queue. The lower
+the value of ``N``, the higher the priority. The option is almost mandatory
+because the default is ``1000`` which matches the priority of the nightlies.
+Nightlies are often half-finished and cancelled due to the volume of testing
+done so your jobs may never finish. Therefore, it is common to select a
+priority less than 1000.
+
+Job priority should be selected based on the following recommendations:
+
+* **Priority < 10:** Use this if the sky is falling and some group of tests
+  must be run ASAP.
+
+* **10 <= Priority < 50:** Use this if your tests are urgent and blocking
+  other important development.
+
+* **50 <= Priority < 75:** Use this if you are testing a particular
+  feature/fix and running fewer than about 25 jobs. This range can also be
+  used for urgent release testing.
+
+* **75 <= Priority < 100:** Tech Leads will regularly schedule integration
+  tests with this priority to verify pull requests against master.
+
+* **100 <= Priority < 150:** This priority is to be used for QE validation of
+  point releases.
+
+* **150 <= Priority < 200:** Use this priority for 100 jobs or fewer of a
+  particular feature/fix that you'd like results on in a day or so.
+
+* **200 <= Priority < 1000:** Use this priority for large test runs that can
+  be done over the course of a week.
+
+In case you don't know how many jobs would be triggered by
+``teuthology-suite`` command, use ``--dry-run`` to get a count first and then
+issue ``teuthology-suite`` command again, this time without ``--dry-run`` and
+with ``-p`` and an appropriate number as an argument to it.
+
+To skip the priority check, use ``--force-priority``. In order to be sensitive
+to the runs of other developers who also need to do testing, please use it in
+emergency only.
+
+Suites Inventory
+----------------
+
+The ``suites`` directory of the `ceph/qa sub-directory`_ contains
+all the integration tests, for all the Ceph components.
+
+`ceph-deploy <https://github.com/ceph/ceph/tree/master/qa/suites/ceph-deploy>`_
+  install a Ceph cluster with ``ceph-deploy`` (:ref:`ceph-deploy man page <ceph-deploy>`)
+
+`dummy <https://github.com/ceph/ceph/tree/master/qa/suites/dummy>`_
+  get a machine, do nothing and return success (commonly used to
+  verify the :ref:`testing-integration-tests` infrastructure works as expected)
+
+`fs <https://github.com/ceph/ceph/tree/master/qa/suites/fs>`_
+  test CephFS mounted using kernel and FUSE clients, also with multiple MDSs.
+
+`krbd <https://github.com/ceph/ceph/tree/master/qa/suites/krbd>`_
+  test the RBD kernel module
+
+`powercycle <https://github.com/ceph/ceph/tree/master/qa/suites/powercycle>`_
+  verify the Ceph cluster behaves when machines are powered off
+  and on again
+
+`rados <https://github.com/ceph/ceph/tree/master/qa/suites/rados>`_
+  run Ceph clusters including OSDs and MONs, under various conditions of
+  stress
+
+`rbd <https://github.com/ceph/ceph/tree/master/qa/suites/rbd>`_
+  run RBD tests using actual Ceph clusters, with and without qemu
+
+`rgw <https://github.com/ceph/ceph/tree/master/qa/suites/rgw>`_
+  run RGW tests using actual Ceph clusters
+
+`smoke <https://github.com/ceph/ceph/tree/master/qa/suites/smoke>`_
+  run tests that exercise the Ceph API with an actual Ceph cluster
+
+`teuthology <https://github.com/ceph/ceph/tree/master/qa/suites/teuthology>`_
+  verify that teuthology can run integration tests, with and without OpenStack
+
+`upgrade <https://github.com/ceph/ceph/tree/master/qa/suites/upgrade>`_
+  for various versions of Ceph, verify that upgrades can happen
+  without disrupting an ongoing workload
+
+.. _`ceph-deploy man page`: ../../man/8/ceph-deploy
+
+teuthology-describe-tests
+-------------------------
+
+In February 2016, a new feature called ``teuthology-describe-tests`` was
+added to the `teuthology framework`_ to facilitate documentation and better
+understanding of integration tests (`feature announcement
+<http://article.gmane.org/gmane.comp.file-systems.ceph.devel/29287>`_).
+
+The upshot is that tests can be documented by embedding ``meta:``
+annotations in the yaml files used to define the tests. The results can be
+seen in the `ceph-qa-suite wiki
+<http://tracker.ceph.com/projects/ceph-qa-suite/wiki/>`_.
+
+Since this is a new feature, many yaml files have yet to be annotated.
+Developers are encouraged to improve the documentation, in terms of both
+coverage and quality.
+
+How integration tests are run
+-----------------------------
+
+Given that - as a new Ceph developer - you will typically not have access
+to the `Sepia lab`_, you may rightly ask how you can run the integration
+tests in your own environment.
+
+One option is to set up a teuthology cluster on bare metal. Though this is
+a non-trivial task, it `is` possible. Here are `some notes
+<http://docs.ceph.com/teuthology/docs/LAB_SETUP.html>`_ to get you started
+if you decide to go this route.
+
+If you have access to an OpenStack tenant, you have another option: the
+`teuthology framework`_ has an OpenStack backend, which is documented `here
+<https://github.com/dachary/teuthology/tree/openstack#openstack-backend>`__.
+This OpenStack backend can build packages from a given git commit or
+branch, provision VMs, install the packages and run integration tests
+on those VMs. This process is controlled using a tool called
+``ceph-workbench ceph-qa-suite``. This tool also automates publishing of
+test results at http://teuthology-logs.public.ceph.com.
+
+Running integration tests on your code contributions and publishing the
+results allows reviewers to verify that changes to the code base do not
+cause regressions, or to analyze test failures when they do occur.
+
+Every teuthology cluster, whether bare-metal or cloud-provisioned, has a
+so-called "teuthology machine" from which tests suites are triggered using the
+``teuthology-suite`` command.
+
+A detailed and up-to-date description of each `teuthology-suite`_ option is
+available by running the following command on the teuthology machine
+
+.. prompt:: bash $
+
+   teuthology-suite --help
+
+.. _teuthology-suite: http://docs.ceph.com/teuthology/docs/teuthology.suite.html
+
+How integration tests are defined
+---------------------------------
+
+Integration tests are defined by yaml files found in the ``suites``
+subdirectory of the `ceph/qa sub-directory`_ and implemented by python
+code found in the ``tasks`` subdirectory. Some tests ("standalone tests")
+are defined in a single yaml file, while other tests are defined by a
+directory tree containing yaml files that are combined, at runtime, into a
+larger yaml file.
+
+Reading a standalone test
+-------------------------
+
+Let us first examine a standalone test, or "singleton".
+
+Here is a commented example using the integration test
+`rados/singleton/all/admin-socket.yaml
+<https://github.com/ceph/ceph/blob/master/qa/suites/rados/singleton/all/admin-socket.yaml>`_
+
+.. code-block:: yaml
+
+      roles:
+      - - mon.a
+        - osd.0
+        - osd.1
+      tasks:
+      - install:
+      - ceph:
+      - admin_socket:
+          osd.0:
+            version:
+            git_version:
+            help:
+            config show:
+            config set filestore_dump_file /tmp/foo:
+            perf dump:
+            perf schema:
+
+The ``roles`` array determines the composition of the cluster (how
+many MONs, OSDs, etc.) on which this test is designed to run, as well
+as how these roles will be distributed over the machines in the
+testing cluster. In this case, there is only one element in the
+top-level array: therefore, only one machine is allocated to the
+test. The nested array declares that this machine shall run a MON with
+id ``a`` (that is the ``mon.a`` in the list of roles) and two OSDs
+(``osd.0`` and ``osd.1``).
+
+The body of the test is in the ``tasks`` array: each element is
+evaluated in order, causing the corresponding python file found in the
+``tasks`` subdirectory of the `teuthology repository`_ or
+`ceph/qa sub-directory`_ to be run. "Running" in this case means calling
+the ``task()`` function defined in that file.
+
+In this case, the `install
+<https://github.com/ceph/teuthology/blob/master/teuthology/task/install/__init__.py>`_
+task comes first. It installs the Ceph packages on each machine (as
+defined by the ``roles`` array). A full description of the ``install``
+task is `found in the python file
+<https://github.com/ceph/teuthology/blob/master/teuthology/task/install/__init__.py>`_
+(search for "def task").
+
+The ``ceph`` task, which is documented `here
+<https://github.com/ceph/ceph/blob/master/qa/tasks/ceph.py>`__ (again,
+search for "def task"), starts OSDs and MONs (and possibly MDSs as well)
+as required by the ``roles`` array. In this example, it will start one MON
+(``mon.a``) and two OSDs (``osd.0`` and ``osd.1``), all on the same
+machine. Control moves to the next task when the Ceph cluster reaches
+``HEALTH_OK`` state.
+
+The next task is ``admin_socket`` (`source code
+<https://github.com/ceph/ceph/blob/master/qa/tasks/admin_socket.py>`_).
+The parameter of the ``admin_socket`` task (and any other task) is a
+structure which is interpreted as documented in the task. In this example
+the parameter is a set of commands to be sent to the admin socket of
+``osd.0``. The task verifies that each of them returns on success (i.e.
+exit code zero).
+
+This test can be run with
+
+.. prompt:: bash $
+
+   teuthology-suite --machine-type smithi --suite rados/singleton/all/admin-socket.yaml fs/ext4.yaml
+
+Test descriptions
+-----------------
+
+Each test has a "test description", which is similar to a directory path,
+but not the same. In the case of a standalone test, like the one in
+`Reading a standalone test`_, the test description is identical to the
+relative path (starting from the ``suites/`` directory of the
+`ceph/qa sub-directory`_) of the yaml file defining the test.
+
+Much more commonly, tests are defined not by a single yaml file, but by a
+`directory tree of yaml files`. At runtime, the tree is walked and all yaml
+files (facets) are combined into larger yaml "programs" that define the
+tests. A full listing of the yaml defining the test is included at the
+beginning of every test log.
+
+In these cases, the description of each test consists of the
+subdirectory under `suites/
+<https://github.com/ceph/ceph/tree/master/qa/suites>`_ containing the
+yaml facets, followed by an expression in curly braces (``{}``) consisting of
+a list of yaml facets in order of concatenation. For instance the
+test description::
+
+  ceph-deploy/basic/{distros/centos_7.0.yaml tasks/ceph-deploy.yaml}
+
+signifies the concatenation of two files:
+
+* ceph-deploy/basic/distros/centos_7.0.yaml
+* ceph-deploy/basic/tasks/ceph-deploy.yaml
+
+How tests are built from directories
+------------------------------------
+
+As noted in the previous section, most tests are not defined in a single
+yaml file, but rather as a `combination` of files collected from a
+directory tree within the ``suites/`` subdirectory of the `ceph/qa sub-directory`_.
+
+The set of all tests defined by a given subdirectory of ``suites/`` is
+called an "integration test suite", or a "teuthology suite".
+
+Combination of yaml facets is controlled by special files (``%`` and
+``+``) that are placed within the directory tree and can be thought of as
+operators.  The ``%`` file is the "convolution" operator and ``+``
+signifies concatenation.
+
+Convolution operator
+^^^^^^^^^^^^^^^^^^^^
+
+The convolution operator, implemented as an empty file called ``%``, tells
+teuthology to construct a test matrix from yaml facets found in
+subdirectories below the directory containing the operator.
+
+For example, the `ceph-deploy suite
+<https://github.com/ceph/ceph/tree/master/qa/suites/ceph-deploy/>`_ is
+defined by the ``suites/ceph-deploy/`` tree, which consists of the files and
+subdirectories in the following structure
+
+.. code-block:: none
+
+  qa/suites/ceph-deploy
+  ├── %
+  ├── distros
+  │   ├── centos_latest.yaml
+  │   └── ubuntu_latest.yaml
+  └── tasks
+      ├── ceph-admin-commands.yaml
+      └── rbd_import_export.yaml
+
+This is interpreted as a 2x1 matrix consisting of two tests:
+
+1. ceph-deploy/basic/{distros/centos_7.0.yaml tasks/ceph-deploy.yaml}
+2. ceph-deploy/basic/{distros/ubuntu_16.04.yaml tasks/ceph-deploy.yaml}
+
+i.e. the concatenation of centos_7.0.yaml and ceph-deploy.yaml and
+the concatenation of ubuntu_16.04.yaml and ceph-deploy.yaml, respectively.
+In human terms, this means that the task found in ``ceph-deploy.yaml`` is
+intended to run on both CentOS 7.0 and Ubuntu 16.04.
+
+Without the file percent, the ``ceph-deploy`` tree would be interpreted as
+three standalone tests:
+
+* ceph-deploy/basic/distros/centos_7.0.yaml
+* ceph-deploy/basic/distros/ubuntu_16.04.yaml
+* ceph-deploy/basic/tasks/ceph-deploy.yaml
+
+(which would of course be wrong in this case).
+
+Referring to the `ceph/qa sub-directory`_, you will notice that the
+``centos_7.0.yaml`` and ``ubuntu_16.04.yaml`` files in the
+``suites/ceph-deploy/basic/distros/`` directory are implemented as symlinks.
+By using symlinks instead of copying, a single file can appear in multiple
+suites. This eases the maintenance of the test framework as a whole.
+
+All the tests generated from the ``suites/ceph-deploy/`` directory tree
+(also known as the "ceph-deploy suite") can be run with
+
+.. prompt:: bash $
+
+   teuthology-suite --machine-type smithi --suite ceph-deploy
+
+An individual test from the `ceph-deploy suite`_ can be run by adding the
+``--filter`` option
+
+.. prompt:: bash $
+
+   teuthology-suite \
+      --machine-type smithi \
+      --suite ceph-deploy/basic \
+      --filter 'ceph-deploy/basic/{distros/ubuntu_16.04.yaml tasks/ceph-deploy.yaml}'
+
+.. note:: To run a standalone test like the one in `Reading a standalone
+   test`_, ``--suite`` alone is sufficient. If you want to run a single
+   test from a suite that is defined as a directory tree, ``--suite`` must
+   be combined with ``--filter``. This is because the ``--suite`` option
+   understands POSIX relative paths only.
+
+Concatenation operator
+^^^^^^^^^^^^^^^^^^^^^^
+
+For even greater flexibility in sharing yaml files between suites, the
+special file plus (``+``) can be used to concatenate files within a
+directory. For instance, consider the `suites/rbd/thrash
+<https://github.com/ceph/ceph/tree/master/qa/suites/rbd/thrash>`_
+tree
+
+.. code-block:: none
+
+  qa/suites/rbd/thrash
+  ├── %
+  ├── clusters
+  │   ├── +
+  │   ├── fixed-2.yaml
+  │   └── openstack.yaml
+  └── workloads
+      ├── rbd_api_tests_copy_on_read.yaml
+      ├── rbd_api_tests.yaml
+      └── rbd_fsx_rate_limit.yaml
+
+This creates two tests:
+
+* rbd/thrash/{clusters/fixed-2.yaml clusters/openstack.yaml workloads/rbd_api_tests_copy_on_read.yaml}
+* rbd/thrash/{clusters/fixed-2.yaml clusters/openstack.yaml workloads/rbd_api_tests.yaml}
+
+Because the ``clusters/`` subdirectory contains the special file plus
+(``+``), all the other files in that subdirectory (``fixed-2.yaml`` and
+``openstack.yaml`` in this case) are concatenated together
+and treated as a single file. Without the special file plus, they would
+have been convolved with the files from the workloads directory to create
+a 2x2 matrix:
+
+* rbd/thrash/{clusters/openstack.yaml workloads/rbd_api_tests_copy_on_read.yaml}
+* rbd/thrash/{clusters/openstack.yaml workloads/rbd_api_tests.yaml}
+* rbd/thrash/{clusters/fixed-2.yaml workloads/rbd_api_tests_copy_on_read.yaml}
+* rbd/thrash/{clusters/fixed-2.yaml workloads/rbd_api_tests.yaml}
+
+The ``clusters/fixed-2.yaml`` file is shared among many suites to
+define the following ``roles``
+
+.. code-block:: yaml
+
+  roles:
+  - [mon.a, mon.c, osd.0, osd.1, osd.2, client.0]
+  - [mon.b, osd.3, osd.4, osd.5, client.1]
+
+The ``rbd/thrash`` suite as defined above, consisting of two tests,
+can be run with
+
+.. prompt:: bash $
+
+   teuthology-suite --machine-type smithi --suite rbd/thrash
+
+A single test from the rbd/thrash suite can be run by adding the
+``--filter`` option
+
+.. prompt:: bash $
+
+   teuthology-suite \
+      --machine-type smithi \
+      --suite rbd/thrash \
+      --filter 'rbd/thrash/{clusters/fixed-2.yaml clusters/openstack.yaml workloads/rbd_api_tests_copy_on_read.yaml}'
+
+Filtering tests by their description
+------------------------------------
+
+When a few jobs fail and need to be run again, the ``--filter`` option
+can be used to select tests with a matching description. For instance, if the
+``rados`` suite fails the `all/peer.yaml <https://github.com/ceph/ceph/blob/master/qa/suites/rados/singleton/all/peer.yaml>`_ test, the following will only
+run the tests that contain this file
+
+.. prompt:: bash $
+
+   teuthology-suite --machine-type smithi --suite rados --filter all/peer.yaml
+
+The ``--filter-out`` option does the opposite (it matches tests that do `not`
+contain a given string), and can be combined with the ``--filter`` option.
+
+Both ``--filter`` and ``--filter-out`` take a comma-separated list of strings
+(which means the comma character is implicitly forbidden in filenames found in
+the `ceph/qa sub-directory`_). For instance
+
+.. prompt:: bash $
+
+  teuthology-suite --machine-type smithi --suite rados --filter all/peer.yaml,all/rest-api.yaml
+
+will run tests that contain either
+`all/peer.yaml <https://github.com/ceph/ceph/blob/master/qa/suites/rados/singleton/all/peer.yaml>`_
+or
+`all/rest-api.yaml <https://github.com/ceph/ceph/blob/master/qa/suites/rados/singleton/all/rest-api.yaml>`_
+
+Each string is looked up anywhere in the test description and has to
+be an exact match: they are not regular expressions.
+
+Reducing the number of tests
+----------------------------
+
+The ``rados`` suite generates tens or even hundreds of thousands of tests out
+of a few hundred files. This happens because teuthology constructs test
+matrices from subdirectories wherever it encounters a file named ``%``. For
+instance, all tests in the `rados/basic suite
+<https://github.com/ceph/ceph/tree/master/qa/suites/rados/basic>`_ run with
+different messenger types: ``simple``, ``async`` and ``random``, because they
+are combined (via the special file ``%``) with the `msgr directory
+<https://github.com/ceph/ceph/tree/master/qa/suites/rados/basic/msgr>`_
+
+All integration tests are required to be run before a Ceph release is
+published. When merely verifying whether a contribution can be merged without
+risking a trivial regression, it is enough to run a subset. The ``--subset``
+option can be used to reduce the number of tests that are triggered. For
+instance
+
+.. prompt:: bash $
+
+   teuthology-suite --machine-type smithi --suite rados --subset 0/4000
+
+will run as few tests as possible. The tradeoff in this case is that
+not all combinations of test variations will together,
+but no matter how small a ratio is provided in the ``--subset``,
+teuthology will still ensure that all files in the suite are in at
+least one test. Understanding the actual logic that drives this
+requires reading the teuthology source code.
+
+The ``--limit`` option only runs the first ``N`` tests in the suite:
+this is rarely useful, however, because there is no way to control which
+test will be first.
+
+.. _ceph/qa sub-directory: https://github.com/ceph/ceph/tree/master/qa
+.. _Sepia Lab: https://wiki.sepia.ceph.com/doku.php
+.. _teuthology repository: https://github.com/ceph/teuthology
+.. _teuthology framework: https://github.com/ceph/teuthology
diff --git a/doc/dev/developer_guide/tests-unit-tests.rst b/doc/dev/developer_guide/tests-unit-tests.rst
new file mode 100644
index 000000000..72d724d98
--- /dev/null
+++ b/doc/dev/developer_guide/tests-unit-tests.rst
@@ -0,0 +1,177 @@
+Testing - unit tests
+====================
+
+The Ceph GitHub repository has two types of tests: unit tests (also called
+``make check`` tests) and integration tests. Strictly speaking, the
+``make check`` tests are not "unit tests", but rather tests that can be run
+easily on a single build machine after compiling Ceph from source, whereas
+integration tests require package installation and multi-machine clusters to
+run.
+
+.. _make-check:
+
+What does "make check" mean?
+----------------------------
+
+After compiling Ceph, the code can be run through a battery of tests. For
+historical reasons, this is often referred to as ``make check`` even though
+the actual command used to run the tests is now ``ctest``. To be included in
+this group of tests, a test must:
+
+* bind ports that do not conflict with other tests
+* not require root access
+* not require more than one machine to run
+* complete within a few minutes
+
+For the sake of simplicity, this class of tests is referred to as "make
+check tests" or "unit tests". This is meant to distinguish these tests from
+the more complex "integration tests" that are run via the `teuthology
+framework`_.
+
+While it is possible to run ``ctest`` directly, it can be tricky to correctly
+set up your environment for it. Fortunately, there is a script that makes it
+easy to run the unit tests on your code. This script can be run from the
+top-level directory of the Ceph source tree by invoking:
+
+  .. prompt:: bash $
+
+     ./run-make-check.sh
+
+You will need a minimum of 8GB of RAM and 32GB of free drive space for this
+command to complete successfully on x86_64 architectures; other architectures
+may have different requirements. Depending on your hardware, it can take from
+twenty minutes to three hours to complete.
+
+
+How unit tests are declared
+---------------------------
+
+Unit tests are declared in the ``CMakeLists.txt`` file, which is found in the
+``./src`` directory. The ``add_ceph_test`` and ``add_ceph_unittest`` CMake
+functions are used to declare unit tests.  ``add_ceph_test`` and
+``add_ceph_unittest`` are themselves defined in
+``./cmake/modules/AddCephTest.cmake``. 
+
+Some unit tests are scripts and other unit tests are binaries that are
+compiled during the build process.  
+
+* ``add_ceph_test`` function - used to declare unit test scripts 
+* ``add_ceph_unittest`` function - used for unit test binaries
+
+Unit testing of CLI tools
+-------------------------
+Some of the CLI tools are tested using special files ending with the extension
+``.t`` and stored under ``./src/test/cli``. These tests are run using a tool
+called `cram`_ via a shell script called ``./src/test/run-cli-tests``.
+`cram`_ tests that are not suitable for ``make check`` can also be run by
+teuthology using the `cram task`_.
+
+.. _`cram`: https://bitheap.org/cram/
+.. _`cram task`: https://github.com/ceph/ceph/blob/master/qa/tasks/cram.py
+
+Tox-based testing of Python modules
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Some of the Python modules in Ceph use `tox <https://tox.readthedocs.io/en/latest/>`_ 
+to run their unit tests.
+
+Most of these Python modules can be found in the directory ``./src/pybind/``.
+
+Currently (December 2020) the following modules use **tox**:
+
+* Cephadm (``./src/cephadm/tox.ini``)
+* Ceph Manager Python API (``./src/pybind/mgr``)
+
+  * ``./src/pybind/mgr/tox.ini``
+    
+  * ``./src/pybind/mgr/dashboard/tox.ini``
+
+  * ``./src/pybind/tox.ini``
+
+* Dashboard (``./src/pybind/mgr/dashboard``)
+* Python common (``./src/python-common/tox.ini``)
+* CephFS (``./src/tools/cephfs/tox.ini``)
+* ceph-volume
+
+  * ``./src/ceph-volume/tox.ini``
+
+  * ``./src/ceph-volume/plugin/zfs/tox.ini``
+
+  * ``./src/ceph-volume/ceph_volume/tests/functional/batch/tox.ini``
+
+  * ``./src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini``
+
+  * ``./src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini``
+
+Configuring Tox environments and tasks 
+""""""""""""""""""""""""""""""""""""""
+Most tox configurations support multiple environments and tasks. 
+
+The list of environments and tasks that are supported is in the ``tox.ini``
+file, under ``envlist``. For example, here are the first three lines of
+``./src/cephadm/tox.ini``::
+
+   [tox]
+   envlist = py3, mypy
+   skipsdist=true
+
+In this example, the ``Python 3`` and ``mypy`` environments are specified.
+
+The list of environments can be retrieved with the following command:
+
+  .. prompt:: bash $
+
+     tox --list
+
+Or:
+
+  .. prompt:: bash $
+
+     tox -l
+
+Running Tox
+"""""""""""
+To run **tox**, just execute ``tox`` in the directory containing
+``tox.ini``.  If you do not specify any environments (for example, ``-e
+$env1,$env2``), then ``tox`` will run all environments. Jenkins will run
+``tox`` by executing ``./src/script/run_tox.sh``.
+
+Here are some examples from Ceph Dashboard that show how to specify different
+environments and run options::
+
+  ## Run Python 2+3 tests+lint commands:
+  $ tox -e py27,py3,lint,check
+
+  ## Run Python 3 tests+lint commands:
+  $ tox -e py3,lint,check
+
+  ## To run it as Jenkins would:  
+  $ ../../../script/run_tox.sh --tox-env py3,lint,check
+
+Manager core unit tests
+"""""""""""""""""""""""
+
+Currently only doctests_ inside ``mgr_util.py`` are run.
+
+To add more files to be tested inside the core of the manager, open the
+``tox.ini`` file and add the files to be tested  at the end of the line that
+includes ``mgr_util.py``.
+
+.. _doctests: https://docs.python.org/3/library/doctest.html
+
+Unit test caveats
+-----------------
+
+#. Unlike the various Ceph daemons and ``ceph-fuse``, the unit tests are
+   linked against the default memory allocator (glibc) unless they are
+   explicitly linked against something else. This enables tools such as
+   **valgrind** to be used in the tests.
+
+#. Google Test unit testing library hides the client output from the shell.
+   In order to debug the client after setting the desired debug level
+   (e.g ``ceph config set client debug_rbd 20``), the debug log file can
+   be found at ``build/out/client.admin.<pid>.log``.
+   This can also be handy when examining teuthology failed unit test
+   jobs, the job's debug level can be set at the relevant yaml file.
+
+.. _make check:
+.. _teuthology framework: https://github.com/ceph/teuthology
diff --git a/doc/dev/development-workflow.rst b/doc/dev/development-workflow.rst
new file mode 100644
index 000000000..dfcab929d
--- /dev/null
+++ b/doc/dev/development-workflow.rst
@@ -0,0 +1,248 @@
+=====================
+Development workflows
+=====================
+
+This page explains the workflows a developer is expected to follow to
+implement the goals that are part of the Ceph release cycle. It does not
+go into technical details and is designed to provide a high level view
+instead. Each chapter is about a given goal such as ``Merging bug
+fixes or features`` or ``Publishing point releases and backporting``.
+
+A key aspect of all workflows is that none of them blocks another. For
+instance, a bug fix can be backported and merged to a stable branch
+while the next point release is being published. For that specific
+example to work, a branch should be created to avoid any
+interference. In practice it is not necessary for Ceph because:
+
+* there are few people involved
+* the frequency of backports is not too high
+* the reviewers, who know a release is being published, are unlikely
+  to merge anything that may cause issues
+
+This ad-hoc approach implies the workflows are changed on a regular
+basis to adapt. For instance, ``quality engineers`` were not involved
+in the workflow to publish ``dumpling`` point releases. The number of
+commits being backported to ``firefly`` made it impractical for developers
+tasked to write code or fix bugs to also run and verify the full suite
+of integration tests. Inserting ``quality engineers`` makes it
+possible for someone to participate in the workflow by analyzing test
+results.
+
+The workflows are not enforced when they impose an overhead that does
+not make sense. For instance, if the release notes for a point release
+were not written prior to checking all integration tests, they can be
+committed to the stable branch and the result sent for publication
+without going through another run of integration tests.
+
+Release Cycle
+=============
+
+::
+
+    Ceph              hammer                             infernalis
+    Developer          CDS                                  CDS 
+    Summit              |                                    |
+                        |                                    |
+    development         |                                    |
+    release             |  v0.88  v0.89  v0.90   ...         |  v9.0.0
+                   --v--^----^--v---^------^--v-     ---v----^----^---  2015       
+                     |          |             |         |
+    stable         giant        |             |      hammer
+    release        v0.87        |             |      v0.94
+                                |             |          
+    point                    firefly       dumpling
+    release                  v0.80.8       v0.67.12
+
+
+Four times a year, the development roadmap is discussed online during
+the `Ceph Developer Summit <http://tracker.ceph.com/projects/ceph/wiki/Planning#Ceph-Developer-Summit>`_. A
+new stable release (hammer, infernalis, jewel ...) is published at the same
+frequency.  Every other release (firefly, hammer, jewel...) is a `Long Term
+Stable (LTS) <../../releases>`_.  See `Understanding the release cycle
+<../../releases#understanding-the-release-cycle>`_ for more information.
+
+Merging bug fixes or features
+=============================
+
+The development branch is ``master`` and the workflow followed by all
+developers can be summarized as follows:
+
+* The developer prepares a series of commits
+* The developer submits the series of commits via a pull request
+* A reviewer is assigned the pull request
+* When the pull request looks good to the reviewer, it is merged into
+  an integration branch by the tester
+* After a successful run of integration tests, the pull request is
+  merged by the tester
+
+The ``developer`` is the author of a series of commits. The
+``reviewer`` is responsible for providing feedback to the developer on
+a regular basis and the developer is invited to ping the reviewer if
+nothing happened after a week. After the ``reviewer`` is satisfied
+with the pull request, (s)he passes it to the ``tester``. The
+``tester`` is responsible for running teuthology integration tests on
+the pull request. If nothing happens within a month the ``reviewer`` is
+invited to ping the ``tester``.
+
+Resolving bug reports and implementing features
+===============================================
+
+All bug reports and feature requests are in the `issue tracker
+<http://tracker.ceph.com>`_ and the workflow can be summarized as
+follows:
+
+* The reporter creates the issue with priority ``Normal``
+* A developer may pick the issue right away
+* During a bi-weekly bug scrub, the team goes over all new issue and
+  assign them a priority
+* The bugs with higher priority are worked on first
+
+Each ``team`` is responsible for a project, managed by :ref:`leads <governance>`.
+
+The ``developer`` assigned to an issue is responsible for it. The
+status of an open issue can be:
+
+* ``New``: it is unclear if the issue needs work.
+* ``Verified``: the bug can be reproduced or showed up multiple times
+* ``In Progress``: the developer is working on it this week
+* ``Pending Backport``: the fix needs to be backported to the stable
+  releases listed in the backport field
+
+For each ``Pending Backport`` issue, there exists at least one issue
+in the ``Backport`` tracker to record the work done to cherry pick the
+necessary commits from the master branch to the target stable branch.
+See `the backporter manual
+<http://tracker.ceph.com/projects/ceph-releases/wiki/HOWTO>`_ for more
+information.
+
+Running and interpreting teuthology integration tests
+=====================================================
+
+The :doc:`/dev/sepia` runs `teuthology
+<https://github.com/ceph/teuthology/>`_ integration tests `on a regular basis <http://tracker.ceph.com/projects/ceph-releases/wiki/HOWTO_monitor_the_automated_tests_AKA_nightlies#Automated-tests-AKA-nightlies>`_ and the
+results are posted on `pulpito <http://pulpito.ceph.com/>`_ and the
+`ceph-qa mailing list <https://ceph.com/irc/>`_.
+
+* The job failures are `analyzed by quality engineers and developers
+  <http://tracker.ceph.com/projects/ceph-releases/wiki/HOWTO_monitor_the_automated_tests_AKA_nightlies#List-of-suites-and-watchers>`_
+* If the cause is environmental (e.g. network connectivity), an issue
+  is created in the `sepia lab project
+  <http://tracker.ceph.com/projects/lab/issues/new>`_
+* If the bug is known, a pulpito URL to the failed job is added to the issue
+* If the bug is new, an issue is created
+
+The ``quality engineer`` is either a developer or a member of the QE
+team. There is at least one integration test suite per project:
+
+* `rgw <https://github.com/ceph/ceph/tree/master/qa/suites/rgw>`_ suite
+* `CephFS <https://github.com/ceph/ceph/tree/master/qa/suites/fs>`_ suite
+* `rados <https://github.com/ceph/ceph/tree/master/qa/suites/rados>`_ suite
+* `rbd <https://github.com/ceph/ceph/tree/master/qa/suites/rbd>`_ suite
+
+and many others such as
+
+* `upgrade <https://github.com/ceph/ceph/tree/master/qa/suites/upgrade>`_ suites
+* `power-cyle <https://github.com/ceph/ceph/tree/master/qa/suites/powercycle>`_ suite
+* ...
+
+Preparing a new release
+=======================
+
+A release is prepared in a dedicated branch, different from the
+``master`` branch.
+
+* For a stable releases it is the branch matching the release code
+  name (dumpling, firefly, etc.)
+* For a development release it is the ``next`` branch
+
+The workflow expected of all developers to stabilize the release
+candidate is the same as the normal development workflow with the
+following differences:
+
+* The pull requests must target the stable branch or next instead of
+  master
+* The reviewer rejects pull requests that are not bug fixes
+* The ``Backport`` issues matching a teuthology test failure and set
+  with priority ``Urgent`` must be fixed before the release
+
+Cutting a new stable release
+============================
+
+A new stable release can be cut when:
+
+* all ``Backport`` issues with priority ``Urgent`` are fixed
+* integration and upgrade tests run successfully
+
+Publishing a new stable release implies a risk of regression or
+discovering new bugs during the upgrade, no matter how carefully it is
+tested. The decision to cut a release must take this into account: it
+may not be wise to publish a stable release that only fixes a few
+minor bugs. For instance if only one commit has been backported to a
+stable release that is not a LTS, it is better to wait until there are
+more.
+
+When a stable release is to be retired, it may be safer to
+recommend an upgrade to the next LTS release instead of
+proposing a new point release to fix a problem. For instance, the
+``dumpling`` v0.67.11 release has bugs related to backfilling which have
+been fixed in ``firefly`` v0.80.x. A backport fixing these backfilling
+bugs has been tested in the draft point release ``dumpling`` v0.67.12 but
+they are large enough to introduce a risk of regression. As ``dumpling``
+is to be retired, users suffering from this bug can
+upgrade to ``firefly`` to fix it. Unless users manifest themselves and ask
+for ``dumpling`` v0.67.12, this draft release may never be published.
+
+* The ``Ceph lead`` decides a new stable release must be published
+* The ``release master`` gets approval from all leads
+* The ``release master`` writes and commits the release notes
+* The ``release master`` informs the ``quality engineer`` that the
+  branch is ready for testing
+* The ``quality engineer`` runs additional integration tests
+* If the ``quality engineer`` discovers new bugs that require an
+  ``Urgent Backport``, the release goes back to being prepared, it
+  was not ready after all
+* The ``quality engineer`` informs the ``publisher`` that the branch
+  is ready for release
+* The ``publisher`` `creates the packages and sets the release tag
+  <../release-process>`_
+
+The person responsible for each role is:
+
+* Sage Weil is the ``Ceph lead``
+* Sage Weil is the ``release master`` for major stable releases
+  (``firefly`` 0.80, ``hammer`` 0.94 etc.)
+* Loic Dachary is the ``release master`` for stable point releases
+  (``firefly`` 0.80.10, ``hammer`` 0.94.1 etc.)
+* Yuri Weinstein is the ``quality engineer``
+* Alfredo Deza is the ``publisher``
+
+Cutting a new development release
+=================================
+
+The publication workflow of a development release is the same as
+preparing a new release and cutting it, with the following
+differences:
+
+* The ``next`` branch is reset to the tip of ``master`` after
+  publication
+* The ``quality engineer`` is not required to run additional tests,
+  the ``release master`` directly informs the ``publisher`` that the
+  release is ready to be published.
+
+Publishing point releases and backporting
+=========================================
+
+The publication workflow of the point releases is the same as
+preparing a new release and cutting it, with the following
+differences:
+
+* The ``backport`` field of each issue contains the code name of the
+  stable release
+* There is exactly one issue in the ``Backport`` tracker for each
+  stable release to which the issue is backported
+* All commits are cherry-picked with ``git cherry-pick -x`` to
+  reference the original commit
+
+See `the backporter manual
+<http://tracker.ceph.com/projects/ceph-releases/wiki/HOWTO>`_ for more
+information.
diff --git a/doc/dev/documenting.rst b/doc/dev/documenting.rst
new file mode 100644
index 000000000..c1a11890e
--- /dev/null
+++ b/doc/dev/documenting.rst
@@ -0,0 +1,144 @@
+==================
+ Documenting Ceph
+==================
+
+User documentation
+==================
+
+The documentation on docs.ceph.com is generated from the restructuredText
+sources in ``/doc/`` in the Ceph git repository.
+
+Please make sure that your changes are written in a way that is intended
+for end users of the software, unless you are making additions in
+``/doc/dev/``, which is the section for developers.
+
+All pull requests that modify user-facing functionality must
+include corresponding updates to documentation: see 
+`Submitting Patches`_ for more detail.
+
+Check your .rst syntax is working as expected by using the "View"
+button in the github user interface when looking at a diff on
+an .rst file, or build the docs locally using the ``admin/build-doc``
+script.
+
+For more information about the Ceph documentation, see 
+:doc:`/start/documenting-ceph`.
+
+Code Documentation
+==================
+
+C and C++ can be documented with Doxygen_, using the subset of Doxygen
+markup supported by Breathe_.
+
+.. _Doxygen: http://www.doxygen.nl/
+.. _Breathe: https://github.com/michaeljones/breathe
+
+The general format for function documentation is::
+
+  /**
+   * Short description
+   *
+   * Detailed description when necessary
+   *
+   * preconditons, postconditions, warnings, bugs or other notes
+   *
+   * parameter reference
+   * return value (if non-void)
+   */
+
+This should be in the header where the function is declared, and
+functions should be grouped into logical categories. The `librados C
+API`_ provides a complete example. It is pulled into Sphinx by
+`librados.rst`_, which is rendered at :doc:`/rados/api/librados`.
+
+To generate the doxygen documentation in HTML format use:
+
+::
+
+   # make doxygen
+
+HTML output will be under: ``build-doc/doxygen/html`` 
+
+.. _`librados C API`: https://github.com/ceph/ceph/blob/master/src/include/rados/librados.h
+.. _`librados.rst`: https://github.com/ceph/ceph/raw/master/doc/rados/api/librados.rst
+
+Drawing diagrams
+================
+
+Graphviz
+--------
+
+You can use Graphviz_, as explained in the `Graphviz extension documentation`_.
+
+.. _Graphviz: http://graphviz.org/
+.. _`Graphviz extension documentation`: https://www.sphinx-doc.org/en/master/usage/extensions/graphviz.html
+
+.. graphviz::
+
+   digraph "example" {
+     foo -> bar;
+     bar -> baz;
+     bar -> th
+   }
+
+Most of the time, you'll want to put the actual DOT source in a
+separate file, like this::
+
+  .. graphviz:: myfile.dot
+
+See the `Dot User's Manual <https://www.graphviz.org/pdf/dotguide.pdf>`_ by
+Emden R. Gansner, Eleftherios Koutsofios, and Stephen North for examples of
+digraphs. This is especially useful if this is your first time encountering
+GraphViz.
+
+Ditaa
+-----
+
+You can use Ditaa_:
+
+.. _Ditaa: http://ditaa.sourceforge.net/
+
+.. ditaa::
+
+   +--------------+   /=----\
+   | hello, world |-->| hi! |
+   +--------------+   \-----/
+
+
+Blockdiag
+---------
+
+If a use arises, we can integrate Blockdiag_. It is a Graphviz-style
+declarative language for drawing things, and includes:
+
+- `block diagrams`_: boxes and arrows (automatic layout, as opposed to
+  Ditaa_)
+- `sequence diagrams`_: timelines and messages between them
+- `activity diagrams`_: subsystems and activities in them
+- `network diagrams`_: hosts, LANs, IP addresses etc (with `Cisco
+  icons`_ if wanted)
+
+.. _Blockdiag: http://blockdiag.com/en/
+.. _`Cisco icons`: https://pypi.org/project/blockdiagcontrib-cisco/
+.. _`block diagrams`: http://blockdiag.com/en/blockdiag/
+.. _`sequence diagrams`: http://blockdiag.com/en/seqdiag/index.html
+.. _`activity diagrams`: http://blockdiag.com/en/actdiag/index.html
+.. _`network diagrams`: http://blockdiag.com/en/nwdiag/
+
+
+Inkscape
+--------
+
+You can use Inkscape to generate scalable vector graphics.
+https://inkscape.org/en/ for restructedText documents.
+
+If you generate diagrams with Inkscape, you should
+commit both the Scalable Vector Graphics (SVG) file and export a
+Portable Network Graphic (PNG) file. Reference the PNG file.
+
+By committing the SVG file, others will be able to update the
+SVG diagrams using Inkscape.
+
+HTML5 will support SVG inline.
+
+.. _`Submitting Patches`: https://github.com/ceph/ceph/blob/master/SubmittingPatches.rst
diff --git a/doc/dev/encoding.rst b/doc/dev/encoding.rst
new file mode 100644
index 000000000..398c85d95
--- /dev/null
+++ b/doc/dev/encoding.rst
@@ -0,0 +1,95 @@
+
+Serialization (encode/decode)
+=============================
+
+When a structure is sent over the network or written to disk, it is
+encoded into a string of bytes.  Serializable structures have
+``encode`` and ``decode`` methods that write and read from ``bufferlist``
+objects representing byte strings.
+
+Adding a field to a structure
+-----------------------------
+
+You can see examples of this all over the Ceph code, but here's an
+example:
+
+::
+
+    class AcmeClass
+    {
+        int member1;
+        std::string member2;
+
+        void encode(bufferlist &bl)
+        {
+            ENCODE_START(1, 1, bl);
+            ::encode(member1, bl);
+            ::encode(member2, bl);
+            ENCODE_FINISH(bl);
+        }
+
+        void decode(bufferlist::iterator &bl)
+        {
+            DECODE_START(1, bl);
+            ::decode(member1, bl);
+            ::decode(member2, bl);
+            DECODE_FINISH(bl);
+        }
+    };
+
+The ``ENCODE_START`` macro writes a header that specifies a *version* and
+a *compat_version* (both initially 1).  The message version is incremented
+whenever a change is made to the encoding.  The compat_version is incremented
+only if the change will break existing decoders -- decoders are tolerant
+of trailing bytes, so changes that add fields at the end of the structure
+do not require incrementing compat_version.
+
+The ``DECODE_START`` macro takes an argument specifying the most recent
+message version that the code can handle.  This is compared with the
+compat_version encoded in the message, and if the message is too new then
+an exception will be thrown.  Because changes to compat_verison are rare,
+this isn't usually something to worry about when adding fields.
+
+In practice, changes to encoding usually involve simply adding the desired fields
+at the end of the ``encode`` and ``decode`` functions, and incrementing
+the versions in ``ENCODE_START`` and ``DECODE_START``.  For example, here's how
+to add a third field to ``AcmeClass``:
+
+::
+
+    class AcmeClass
+    {
+        int member1;
+        std::string member2;
+        std::vector<std::string> member3;
+
+        void encode(bufferlist &bl)
+        {
+            ENCODE_START(2, 1, bl);
+            ::encode(member1, bl);
+            ::encode(member2, bl);
+            ::encode(member3, bl);
+            ENCODE_FINISH(bl);
+        }
+
+        void decode(bufferlist::iterator &bl)
+        {
+            DECODE_START(2, bl);
+            ::decode(member1, bl);
+            ::decode(member2, bl);
+            if (struct_v >= 2) {
+                ::decode(member3, bl);
+            }
+            DECODE_FINISH(bl);
+        }
+    };
+
+Note that the compat_version did not change because the encoded message
+will still be decodable by versions of the code that only understand
+version 1 -- they will just ignore the trailing bytes where we encode ``member3``.
+
+In the ``decode`` function, decoding the new field is conditional: this is
+because we might still be passed older-versioned messages that do not
+have the field.  The ``struct_v`` variable is a local set by the ``DECODE_START``
+macro.
+
diff --git a/doc/dev/erasure-coded-pool.rst b/doc/dev/erasure-coded-pool.rst
new file mode 100644
index 000000000..8ad697702
--- /dev/null
+++ b/doc/dev/erasure-coded-pool.rst
@@ -0,0 +1,135 @@
+Erasure Coded pool
+==================
+
+Purpose
+-------
+
+Erasure-coded pools require less storage space compared to replicated
+pools.  The erasure-coding support has higher computational requirements and
+only supports a subset of the operations allowed on an object (for instance,
+partial write is not supported).
+
+Use cases
+---------
+
+Cold storage
+~~~~~~~~~~~~
+
+An erasure-coded pool is created to store a large number of 1GB
+objects (imaging, genomics, etc.) and 10% of them are read per
+month. New objects are added every day and the objects are not
+modified after being written. On average there is one write for 10,000
+reads.
+
+A replicated pool is created and set as a cache tier for the
+erasure coded pool. An agent demotes objects (i.e. moves them from the
+replicated pool to the erasure-coded pool) if they have not been
+accessed in a week.
+
+The erasure-coded pool CRUSH rule targets hardware designed for
+cold storage with high latency and slow access time. The replicated
+pool CRUSH rule targets faster hardware to provide better response
+times.
+
+Cheap multidatacenter storage
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Ten datacenters are connected with dedicated network links. Each
+datacenter contains the same amount of storage with no power-supply
+backup and no air-cooling system.
+
+An erasure-coded pool is created with a CRUSH rule that will
+ensure no data loss if at most three datacenters fail
+simultaneously. The overhead is 50% with erasure code configured to
+split data in six (k=6) and create three coding chunks (m=3). With
+replication the overhead would be 400% (four replicas).
+
+Interface
+---------
+
+Set up an erasure-coded pool::
+
+ $ ceph osd pool create ecpool erasure
+
+Set up an erasure-coded pool and the associated CRUSH rule ``ecrule``::
+
+ $ ceph osd crush rule create-erasure ecrule
+ $ ceph osd pool create ecpool erasure default ecrule
+
+Set the CRUSH failure domain to osd (instead of host, which is the default)::
+
+ $ ceph osd erasure-code-profile set myprofile \
+     crush-failure-domain=osd
+ $ ceph osd erasure-code-profile get myprofile
+ k=2
+ m=2
+ plugin=jerasure
+ technique=reed_sol_van
+ crush-failure-domain=osd
+ $ ceph osd pool create ecpool erasure myprofile
+
+Control the parameters of the erasure code plugin::
+
+ $ ceph osd erasure-code-profile set myprofile \
+     k=3 m=2
+ $ ceph osd erasure-code-profile get myprofile
+ k=3
+ m=2
+ plugin=jerasure
+ technique=reed_sol_van
+ $ ceph osd pool create ecpool erasure myprofile
+
+Choose an alternate erasure code plugin::
+
+ $ ceph osd erasure-code-profile set myprofile \
+     plugin=example technique=xor
+ $ ceph osd erasure-code-profile get myprofile
+ k=2
+ m=2
+ plugin=example
+ technique=xor
+ $ ceph osd pool create ecpool 12 12 erasure \
+     myprofile
+
+Display the default erasure code profile::
+
+  $ ceph osd erasure-code-profile ls
+  default
+  $ ceph osd erasure-code-profile get default
+  k=2
+  m=2
+  plugin=jerasure
+  technique=reed_sol_van
+
+Create a profile to set the data to be distributed on six OSDs (k+m=6) and sustain the loss of three OSDs (m=3) without losing data::
+
+  $ ceph osd erasure-code-profile set myprofile k=3 m=3
+  $ ceph osd erasure-code-profile get myprofile
+  k=3
+  m=3
+  plugin=jerasure
+  technique=reed_sol_van
+  $ ceph osd erasure-code-profile ls
+  default
+  myprofile
+
+Remove a profile that is no longer in use (otherwise it will fail with EBUSY)::
+
+  $ ceph osd erasure-code-profile ls
+  default
+  myprofile
+  $ ceph osd erasure-code-profile rm myprofile
+  $ ceph osd erasure-code-profile ls
+  default
+
+Set the rule to ssd (instead of default)::
+
+ $ ceph osd erasure-code-profile set myprofile \
+     crush-root=ssd
+ $ ceph osd erasure-code-profile get myprofile
+ k=2
+ m=2
+ plugin=jerasure
+ technique=reed_sol_van
+ crush-root=ssd
+
diff --git a/doc/dev/file-striping.rst b/doc/dev/file-striping.rst
new file mode 100644
index 000000000..405c9718d
--- /dev/null
+++ b/doc/dev/file-striping.rst
@@ -0,0 +1,161 @@
+File striping
+=============
+
+The text below describes how files from Ceph file system clients are
+stored across objects stored in RADOS.
+
+ceph_file_layout
+----------------
+
+Ceph distributes (stripes) the data for a given file across a number
+of underlying objects.  The way file data is mapped to those objects
+is defined by the ceph_file_layout structure.  The data distribution
+is a modified RAID 0, where data is striped across a set of objects up
+to a (per-file) fixed size, at which point another set of objects
+holds the file's data.  The second set also holds no more than the
+fixed amount of data, and then another set is used, and so on.
+
+Defining some terminology will go a long way toward explaining the
+way file data is laid out across Ceph objects.
+
+- file
+    A collection of contiguous data, named from the perspective of
+    the Ceph client (i.e., a file on a Linux system using Ceph
+    storage).  The data for a file is divided into fixed-size
+    "stripe units," which are stored in ceph "objects."
+- stripe unit
+    The size (in bytes) of a block of data used in the RAID 0
+    distribution of a file.  All stripe units for a file have equal
+    size.  The last stripe unit is typically incomplete--i.e. it
+    represents the data at the end of the file as well as unused
+    "space" beyond it up to the end of the fixed stripe unit size.
+- stripe count
+    The number of consecutive stripe units that constitute a RAID 0
+    "stripe" of file data.
+- stripe
+    A contiguous range of file data, RAID 0 striped across "stripe
+    count" objects in fixed-size "stripe unit" blocks.
+- object
+    A collection of data maintained by Ceph storage.  Objects are
+    used to hold portions of Ceph client files.
+- object set
+    A set of objects that together represent a contiguous portion of
+    a file.
+
+Three fields in the ceph_file_layout structure define this mapping::
+
+	u32 fl_stripe_unit;
+	u32 fl_stripe_count;
+	u32 fl_object_size;
+
+(They are actually maintained in their on-disk format, __le32.)
+
+The role of the first two fields should be clear from the
+definitions above.
+
+The third field is the maximum size (in bytes) of an object used to
+back file data.  The object size is a multiple of the stripe unit.
+
+A file's data is blocked into stripe units, and consecutive stripe
+units are stored on objects in an object set.  The number of objects
+in a set is the same as the stripe count.  No object storing file
+data will exceed the file's designated object size, so after some
+fixed number of complete stripes, a new object set is used to store
+subsequent file data.
+
+Note that by default, Ceph uses a simple striping strategy in which
+object_size equals stripe_unit and stripe_count is 1.  This simply
+puts one stripe_unit in each object.  
+
+Here's a more complex example::
+
+    file size = 1 trillion = 1000000000000 bytes
+
+    fl_stripe_unit = 64KB = 65536 bytes
+    fl_stripe_count = 5 stripe units per stripe
+    fl_object_size = 64GB = 68719476736 bytes
+
+This means::
+
+    file stripe size = 64KB * 5 = 320KB = 327680 bytes
+    each object holds 64GB / 64KB = 1048576 stripe units
+    file object set size = 64GB * 5 = 320GB = 343597383680 bytes
+	(also 1048576 stripe units * 327680 bytes per stripe unit)
+
+So the file's 1 trillion bytes can be divided into complete object
+sets, then complete stripes, then complete stripe units, and finally
+a single incomplete stripe unit::
+
+    - 1 trillion bytes / 320GB per object set = 2 complete object sets
+        (with 312805232640 bytes remaining)
+    - 312805232640 bytes / 320KB per stripe = 954605 complete stripes
+        (with 266240 bytes remaining)
+    - 266240 bytes / 64KB per stripe unit = 4 complete stripe units
+        (with 4096 bytes remaining)
+    - and the final incomplete stripe unit holds those 4096 bytes.
+
+The ASCII art below attempts to capture this::
+
+     _________   _________   _________   _________   _________
+    /object  0\ /object  1\ /object  2\ /object  3\ /object  4\
+    +=========+ +=========+ +=========+ +=========+ +=========+
+    |  stripe | |  stripe | |  stripe | |  stripe | |  stripe |
+  o |   unit  | |   unit  | |   unit  | |   unit  | |   unit  | stripe 0
+  b |     0   | |     1   | |     2   | |     3   | |     4   |
+  j |---------| |---------| |---------| |---------| |---------|
+  e |  stripe | |  stripe | |  stripe | |  stripe | |  stripe |
+  c |   unit  | |   unit  | |   unit  | |   unit  | |   unit  | stripe 1
+  t |     5   | |     6   | |     7   | |     8   | |     9   |
+    |---------| |---------| |---------| |---------| |---------|
+  s |     .   | |     .   | |     .   | |     .   | |     .   |
+  e       .           .           .           .           .
+  t |     .   | |     .   | |     .   | |     .   | |     .   |
+    |---------| |---------| |---------| |---------| |---------|
+  0 |  stripe | |  stripe | |  stripe | |  stripe | |  stripe | stripe
+    |   unit  | |   unit  | |   unit  | |   unit  | |   unit  | 1048575
+    | 5242875 | | 5242876 | | 5242877 | | 5242878 | | 5242879 |
+    \=========/ \=========/ \=========/ \=========/ \=========/
+
+     _________   _________   _________   _________   _________
+    /object  5\ /object  6\ /object  7\ /object  8\ /object  9\
+    +=========+ +=========+ +=========+ +=========+ +=========+
+    |  stripe | |  stripe | |  stripe | |  stripe | |  stripe | stripe
+  o |   unit  | |   unit  | |   unit  | |   unit  | |   unit  | 1048576
+  b | 5242880 | | 5242881 | | 5242882 | | 5242883 | | 5242884 |
+  j |---------| |---------| |---------| |---------| |---------|
+  e |  stripe | |  stripe | |  stripe | |  stripe | |  stripe | stripe
+  c |   unit  | |   unit  | |   unit  | |   unit  | |   unit  | 1048577
+  t | 5242885 | | 5242886 | | 5242887 | | 5242888 | | 5242889 |
+    |---------| |---------| |---------| |---------| |---------|
+  s |     .   | |     .   | |     .   | |     .   | |     .   |
+  e       .           .           .           .           .
+  t |     .   | |     .   | |     .   | |     .   | |     .   |
+    |---------| |---------| |---------| |---------| |---------|
+  1 |  stripe | |  stripe | |  stripe | |  stripe | |  stripe | stripe
+    |   unit  | |   unit  | |   unit  | |   unit  | |   unit  | 2097151
+    | 10485755| | 10485756| | 10485757| | 10485758| | 10485759|
+    \=========/ \=========/ \=========/ \=========/ \=========/
+
+     _________   _________   _________   _________   _________
+    /object 10\ /object 11\ /object 12\ /object 13\ /object 14\
+    +=========+ +=========+ +=========+ +=========+ +=========+
+    |  stripe | |  stripe | |  stripe | |  stripe | |  stripe | stripe
+  o |   unit  | |   unit  | |   unit  | |   unit  | |   unit  | 2097152
+  b | 10485760| | 10485761| | 10485762| | 10485763| | 10485764|
+  j |---------| |---------| |---------| |---------| |---------|
+  e |  stripe | |  stripe | |  stripe | |  stripe | |  stripe | stripe
+  c |   unit  | |   unit  | |   unit  | |   unit  | |   unit  | 2097153
+  t | 10485765| | 10485766| | 10485767| | 10485768| | 10485769|
+    |---------| |---------| |---------| |---------| |---------|
+  s |     .   | |     .   | |     .   | |     .   | |     .   |
+  e       .           .           .           .           .
+  t |     .   | |     .   | |     .   | |     .   | |     .   |
+    |---------| |---------| |---------| |---------| |---------|
+  2 |  stripe | |  stripe | |  stripe | |  stripe | |  stripe | stripe
+    |   unit  | |   unit  | |   unit  | |   unit  | |   unit  | 3051756
+    | 15258780| | 15258781| | 15258782| | 15258783| | 15258784|
+    |---------| |---------| |---------| |---------| |---------|
+    |  stripe | |  stripe | |  stripe | |  stripe | | (partial| (partial
+    |   unit  | |   unit  | |   unit  | |   unit  | |  stripe | stripe
+    | 15258785| | 15258786| | 15258787| | 15258788| |  unit)  | 3051757)
+    \=========/ \=========/ \=========/ \=========/ \=========/
diff --git a/doc/dev/freebsd.rst b/doc/dev/freebsd.rst
new file mode 100644
index 000000000..b1645b873
--- /dev/null
+++ b/doc/dev/freebsd.rst
@@ -0,0 +1,53 @@
+==============================
+FreeBSD Implementation details
+==============================
+
+
+Disk layout
+-----------
+
+Current implementation works on ZFS pools
+
+* created in /var/lib/ceph
+* One ZFS pool per OSD, like::
+
+    gpart create -s GPT ada1
+    gpart add -t freebsd-zfs -l osd1 ada1
+    zpool create -o mountpoint=/var/lib/ceph/osd/osd.1 osd
+
+* Maybe add some cache and log (ZIL)? Assuming that ada2 is an SSD::
+
+    gpart create -s GPT ada2
+    gpart add -t freebsd-zfs -l osd1-log -s 1G ada2
+    zpool add osd1 log gpt/osd1-log
+    gpart add -t freebsd-zfs -l osd1-cache -s 10G ada2
+    zpool add osd1 log gpt/osd1-cache
+
+* Note: *UFS2 does not allow large xattribs*
+
+
+Configuration
+-------------
+
+As per FreeBSD default parts of extra software go into ``/usr/local/``. Which
+means that for ``/etc/ceph.conf`` the default location is
+``/usr/local/etc/ceph/ceph.conf``. Smartest thing to do is to create a softlink
+from ``/etc/ceph`` to ``/usr/local/etc/ceph``::
+
+  ln -s /usr/local/etc/ceph /etc/ceph
+
+A sample file is provided in ``/usr/local/share/doc/ceph/sample.ceph.conf``
+
+
+MON creation
+------------
+
+Monitors are created by following the manual creation steps on::
+
+  https://docs.ceph.com/en/latest/install/manual-freebsd-deployment/
+
+
+OSD creation
+------------
+
+OSDs can be manually created only, see :ref:`freebsd_adding_osds`
diff --git a/doc/dev/generatedocs.rst b/doc/dev/generatedocs.rst
new file mode 100644
index 000000000..8632eb176
--- /dev/null
+++ b/doc/dev/generatedocs.rst
@@ -0,0 +1,83 @@
+Building Ceph Documentation
+===========================
+
+Ceph utilizes Python's Sphinx documentation tool. For details on
+the Sphinx documentation tool, refer to `The Sphinx Documentation Tool <https://www.sphinx-doc.org/en/master/>`_.
+
+To build the Ceph documentation set, you must:
+
+1. Clone the Ceph repository
+2. Install the required tools
+3. Build the documents
+4. Demo the documents (Optional)
+
+.. highlight:: bash
+
+Clone the Ceph Repository
+-------------------------
+
+To clone the Ceph repository, you must have ``git`` installed
+on your local host. To install ``git``, execute::
+
+	sudo apt-get install git
+
+To clone the Ceph repository, execute::
+
+	git clone git://github.com/ceph/ceph
+
+You should have a full copy of the Ceph repository.
+
+
+Install the Required Tools
+--------------------------
+
+To build the Ceph documentation, some dependencies are required.
+To know what packages are needed, you can launch this command::
+
+    cd ceph
+    admin/build-doc
+
+If dependencies are missing, the command above will fail
+with a message that suggests you a command to install all
+missing dependencies.
+
+
+Build the Documents
+-------------------
+
+Once you have installed all the dependencies, execute the build (the
+same command as above)::
+
+	cd ceph
+	admin/build-doc
+
+Once you build the documentation set, you may navigate to the source directory to view it::
+
+	cd build-doc/output
+
+There should be an ``html`` directory and a ``man`` directory containing documentation
+in HTML and manpage formats respectively.
+
+``admin/build-doc`` takes a long time to prepare the environment and build the document.
+But you can just rebuild the document on changes using::
+
+  admin/build-doc livehtml
+
+This feature uses ``sphinx-autobuild`` under the hood. You can also pass options to it. For
+instance, to open the browser after building the documentation::
+
+  admin/build-doc livehtml -- --open-browser
+
+Please see `sphinx-autobuild <https://pypi.org/project/sphinx-autobuild/>`_ for more details.
+
+Demo the Documents
+-------------------
+
+Once you build the documentation, as described above, you can demo the rendered documents
+by running ``serve-doc``::
+
+	cd ceph
+	admin/serve-doc
+
+This will serve the ``build-doc/output/html`` directory over port 8080 via 
+Python's ``SimpleHTTPServer`` module.
diff --git a/doc/dev/iana.rst b/doc/dev/iana.rst
new file mode 100644
index 000000000..d2daf3395
--- /dev/null
+++ b/doc/dev/iana.rst
@@ -0,0 +1,16 @@
+IANA Numbers
+============
+
+Private Enterprise Number (PEN) Assignment
+------------------------------------------
+
+50495
+
+Organization ``Ceph``.
+
+Port number (monitor)
+---------------------
+
+3300
+
+That's 0xce4, or ce4h, or (sort of) "ceph."
diff --git a/doc/dev/internals.rst b/doc/dev/internals.rst
new file mode 100644
index 000000000..a894394c9
--- /dev/null
+++ b/doc/dev/internals.rst
@@ -0,0 +1,52 @@
+================
+ Ceph Internals
+================
+
+.. note:: If you're looking for how to use Ceph as a library from your
+   own software, please see :doc:`/api/index`.
+
+You can start a development mode Ceph cluster, after compiling the source, with::
+
+	cd build
+	OSD=3 MON=3 MGR=3 ../src/vstart.sh -n -x
+	# check that it's there
+	bin/ceph health
+
+.. rubric:: Mailing list
+
+The ``dev@ceph.io`` list is for discussion about the development of Ceph,
+its interoperability with other technology, and the operations of the
+project itself.  Subscribe by sending a message to ``dev-request@ceph.io``
+with the line::
+
+ subscribe ceph-devel
+
+in the body of the message.
+
+The ceph-devel@vger.kernel.org list is for discussion
+and patch review for the Linux kernel Ceph client component.
+Subscribe by sending a message to ``majordomo@vger.kernel.org`` with the line::
+
+ subscribe ceph-devel
+
+in the body of the message.
+
+.. raw:: html
+
+   <!---
+
+.. rubric:: Contents
+
+.. toctree::
+   :glob:
+
+   *
+   osd_internals/index*
+   mds_internals/index*
+   radosgw/index*
+   ceph-volume/index*
+   crimson/index*
+
+.. raw:: html
+
+   --->
diff --git a/doc/dev/kubernetes.rst b/doc/dev/kubernetes.rst
new file mode 100644
index 000000000..75b100b24
--- /dev/null
+++ b/doc/dev/kubernetes.rst
@@ -0,0 +1,228 @@
+
+.. _kubernetes-dev:
+
+=======================================
+Hacking on Ceph in Kubernetes with Rook
+=======================================
+
+.. warning::
+
+    This is *not* official user documentation for setting up production
+    Ceph clusters with Kubernetes.  It is aimed at developers who want
+    to hack on Ceph in Kubernetes.
+
+This guide is aimed at Ceph developers getting started with running
+in a Kubernetes environment. It assumes that you may be hacking on Rook,
+Ceph or both, so everything is built from source.
+
+TL;DR for hacking on MGR modules
+================================
+
+Make your changes to the Python code base and then from Ceph's
+``build`` directory, run::
+
+    ../src/script/kubejacker/kubejacker.sh '192.168.122.1:5000'
+
+where ``'192.168.122.1:5000'`` is a local docker registry and
+Rook's ``CephCluster`` CR uses ``image: 192.168.122.1:5000/ceph/ceph:latest``.
+
+1. Build a kubernetes cluster
+=============================
+
+Before installing Ceph/Rook, make sure you've got a working kubernetes
+cluster with some nodes added (i.e. ``kubectl get nodes`` shows you something).
+The rest of this guide assumes that your development workstation has network
+access to your kubernetes cluster, such that ``kubectl`` works from your
+workstation.
+
+`There are many ways <https://kubernetes.io/docs/setup/>`_
+to build a kubernetes cluster: here we include some tips/pointers on where
+to get started.
+
+`kubic-terraform-kvm <https://github.com/kubic-project/kubic-terraform-kvm>`_
+might also be an option.
+
+Or `Host your own <https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/>`_ with
+``kubeadm``.
+
+Some Tips
+---------
+
+Here are some tips for a smoother ride with
+
+``kubeadm``:
+
+- If you have previously added any yum/deb repos for kubernetes packages,
+  disable them before trying to use the packages.cloud.google.com repository.
+  If you don't, you'll get quite confusing conflicts.
+- Even if your distro already has docker, make sure you're installing it
+  a version from docker.com that is within the range mentioned in the
+  kubeadm install instructions. Especially, note that the docker in CentOS 7, 8
+  will *not* work.
+
+``minikube``:
+
+- Start up minikube by passing local docker registry address::
+  ``minikube start --driver=docker --insecure-registry='192.168.122.1:5000'``
+
+Hosted elsewhere
+----------------
+
+If you do not have any servers to hand, you might try a pure
+container provider such as Google Compute Engine. Your mileage may
+vary when it comes to what kinds of storage devices are visible
+to your kubernetes cluster.
+
+Make sure you check how much it's costing you before you spin up a big cluster!
+
+
+2. Run a docker registry
+========================
+
+Run this somewhere accessible from both your workstation and your
+kubernetes cluster (i.e. so that ``docker push/pull`` just works everywhere).
+This is likely to be the same host you're using as your kubernetes master.
+
+1. Install the ``docker-distribution`` package.
+2. If you want to configure the port, edit ``/etc/docker-distribution/registry/config.yml``
+3. Enable the registry service:
+
+::
+
+    systemctl enable docker-distribution
+    systemctl start docker-distribution
+
+You may need to mark the registry as **insecure**.
+
+3. Build Rook
+=============
+
+.. note::
+
+    Building Rook is **not required** to make changes to Ceph.
+
+Install Go if you don't already have it.
+
+Download the Rook source code:
+
+::
+
+    go get github.com/rook/rook
+
+    # Ignore this warning, as Rook is not a conventional go package
+    can't load package: package github.com/rook/rook: no Go files in /home/jspray/go/src/github.com/rook/rook
+
+You will now have a Rook source tree in ~/go/src/github.com/rook/rook -- you may
+be tempted to clone it elsewhere, but your life will be easier if you
+leave it in your GOPATH.
+
+Run ``make`` in the root of your Rook tree to build its binaries and containers:
+
+::
+
+    make
+    ...
+    === saving image build-9204c79b/ceph-amd64
+    === docker build build-9204c79b/ceph-toolbox-base-amd64
+    sha256:653bb4f8d26d6178570f146fe637278957e9371014ea9fce79d8935d108f1eaa
+    === docker build build-9204c79b/ceph-toolbox-amd64
+    sha256:445d97b71e6f8de68ca1c40793058db0b7dd1ebb5d05789694307fd567e13863
+    === caching image build-9204c79b/ceph-toolbox-base-amd64
+
+You can use ``docker image ls`` to see the resulting built images.  The
+images you care about are the ones with tags ending "ceph-amd64" (used
+for the Rook operator and Ceph daemons) and "ceph-toolbox-amd64" (used
+for the "toolbox" container where the CLI is run).
+
+4. Build Ceph
+=============
+
+.. note::
+
+    Building Ceph is **not required** to make changes to MGR modules
+    written in Python.
+
+
+The Rook containers and the Ceph containers are independent now. Note that
+Rook's Ceph client libraries need to communicate with the Ceph cluster,
+therefore a compatible major version is required.
+
+You can run a Registry docker container with access to your Ceph source
+tree using a command like:
+
+::
+
+    docker run -i -v /my/ceph/src:/my/ceph/src -p 192.168.122.1:5000:5000 -t --name registry registry:2
+
+
+Once you have built Ceph, you can inject the resulting binaries into
+the Rook container image using the ``kubejacker.sh`` script (run from
+your build directory but from *outside* your build container).
+
+5. Run Kubejacker
+=================
+
+``kubejacker`` needs access to your docker registry. Execute the script
+to build a docker image containing your latest Ceph binaries:
+
+::
+
+    build$ ../src/script/kubejacker/kubejacker.sh "<host>:<port>"
+
+
+Now you've got your freshly built Rook and freshly built Ceph into
+a single container image, ready to run.  Next time you change something
+in Ceph, you can re-run this to update your image and restart your
+kubernetes containers. If you change something in Rook, then re-run the Rook
+build, and the Ceph build too.
+
+5. Run a Rook cluster
+=====================
+
+Please refer to `Rook's documentation <https://rook.io/docs/rook/master/ceph-quickstart.html>`_
+for setting up a Rook operator, a Ceph cluster and the toolbox.
+
+The Rook source tree includes example .yaml files in
+``cluster/examples/kubernetes/ceph/``. Copy these into
+a working directory, and edit as necessary to configure
+the setup you want:
+
+- Ensure that ``spec.cephVersion.image`` points to your docker registry::
+
+    spec:
+      cephVersion:
+        allowUnsupported: true
+        image: 192.168.122.1:5000/ceph/ceph:latest
+
+Then, load the configuration into the kubernetes API using ``kubectl``:
+
+::
+
+    kubectl apply -f ./cluster-test.yaml
+
+Use ``kubectl -n rook-ceph get pods`` to check the operator
+pod the Ceph daemons and toolbox are is coming up.
+
+Once everything is up and running,
+you should be able to open a shell in the toolbox container and
+run ``ceph status``.
+
+If your mon services start but the rest don't, it could be that they're
+unable to form a quorum due to a Kubernetes networking issue: check that
+containers in your Kubernetes cluster can ping containers on other nodes.
+
+Cheat sheet
+===========
+
+Open a shell in your toolbox container::
+
+    kubectl -n rook-ceph exec -it $(kubectl -n rook-ceph get pod -l "app=rook-ceph-tools" -o jsonpath="{.items[0].metadata.name}") -- bash
+
+Inspect the Rook operator container's logs::
+
+    kubectl -n rook-ceph logs -l app=rook-ceph-operator
+
+Inspect the ceph-mgr container's logs::
+
+    kubectl -n rook-ceph logs -l app=rook-ceph-mgr
+
diff --git a/doc/dev/libs.rst b/doc/dev/libs.rst
new file mode 100644
index 000000000..203dd38b0
--- /dev/null
+++ b/doc/dev/libs.rst
@@ -0,0 +1,18 @@
+======================
+ Library architecture
+======================
+
+Ceph is structured into libraries which are built and then combined together to
+make executables and other libraries.
+
+- libcommon: a collection of utilities which are available to nearly every ceph
+  library and executable. In general, libcommon should not contain global
+  variables, because it is intended to be linked into libraries such as
+  libcephfs.so.
+
+- libglobal: a collection of utilities focused on the needs of Ceph daemon
+  programs. In here you will find pidfile management functions, signal
+  handlers, and so forth.
+
+.. todo:: document other libraries
+
diff --git a/doc/dev/logging.rst b/doc/dev/logging.rst
new file mode 100644
index 000000000..67d3de141
--- /dev/null
+++ b/doc/dev/logging.rst
@@ -0,0 +1,106 @@
+
+Use of the cluster log
+======================
+
+(Note: none of this applies to the local "dout" logging.  This is about
+the cluster log that we send through the mon daemons)
+
+Severity
+--------
+
+Use ERR for situations where the cluster cannot do its job for some reason.
+For example: we tried to do a write, but it returned an error, or we tried
+to read something, but it's corrupt so we can't, or we scrubbed a PG but
+the data was inconsistent so we can't recover.
+
+Use WRN for incidents that the cluster can handle, but have some abnormal/negative
+aspect, such as a temporary degradation of service, or an unexpected internal
+value.  For example, a metadata error that can be auto-fixed, or a slow operation.
+
+Use INFO for ordinary cluster operations that do not indicate a fault in
+Ceph.  It is especially important that INFO level messages are clearly
+worded and do not cause confusion or alarm.
+
+Frequency
+---------
+
+It is important that messages of all severities are not excessively
+frequent.  Consumers may be using a rotating log buffer that contains
+messages of all severities, so even DEBUG messages could interfere
+with proper display of the latest INFO messages if the DEBUG messages
+are too frequent.
+
+Remember that if you have a bad state (as opposed to event), that is
+what health checks are for -- do not spam the cluster log to indicate
+a continuing unhealthy state.
+
+Do not emit cluster log messages for events that scale with
+the number of clients or level of activity on the system, or for
+events that occur regularly in normal operation.  For example, it
+would be inappropriate to emit a INFO message about every
+new client that connects (scales with #clients), or to emit and INFO
+message about every CephFS subtree migration (occurs regularly).
+
+Language and formatting
+-----------------------
+
+(Note: these guidelines matter much less for DEBUG-level messages than
+ for INFO and above.  Concentrate your efforts on making INFO/WRN/ERR
+ messages as readable as possible.)
+
+Use the passive voice.  For example, use "Object xyz could not be read", rather
+than "I could not read the object xyz".
+
+Print long/big identifiers, such as inode numbers, as hex, prefixed
+with an 0x so that the user can tell it is hex.  We do this because
+the 0x makes it unambiguous (no equivalent for decimal), and because
+the hex form is more likely to fit on the screen.
+
+Print size quantities as a human readable MB/GB/etc, including the unit
+at the end of the number.  Exception: if you are specifying an offset,
+where precision is essential to the meaning, then you can specify
+the value in bytes (but print it as hex).
+
+Make a good faith effort to fit your message on a single line.  It does
+not have to be guaranteed, but it should at least usually be
+the case.  That means, generally, no printing of lists unless there
+are only a few items in the list.
+
+Use nouns that are meaningful to the user, and defined in the
+documentation.  Common acronyms are OK -- don't waste screen space
+typing "Rados Object Gateway" instead of RGW.  Do not use internal
+class names like "MDCache" or "Objecter".  It is okay to mention
+internal structures if they are the direct subject of the message,
+for example in a corruption, but use plain English.
+Example: instead of "Objecter requests" say "OSD client requests"
+Example: it is okay to mention internal structure in the context
+of "Corrupt session table" (but don't say "Corrupt SessionTable")
+
+Where possible, describe the consequence for system availability, rather
+than only describing the underlying state.  For example, rather than
+saying "MDS myfs.0 is replaying", say that "myfs is degraded, waiting
+for myfs.0 to finish starting".
+
+While common acronyms are fine, don't randomly truncate words.  It's not
+"dir ino", it's "directory inode".
+
+If you're logging something that "should never happen", i.e. a situation
+where it would be an assertion, but we're helpfully not crashing, then
+make that clear in the language -- this is probably not a situation
+that the user can remediate themselves.
+
+Avoid UNIX/programmer jargon.  Instead of "errno", just say "error" (or
+preferably give something more descriptive than the number!)
+
+Do not mention cluster map epochs unless they are essential to
+the meaning of the message.  For example, "OSDMap epoch 123 is corrupt"
+would be okay (the epoch is the point of the message), but saying "OSD
+123 is down in OSDMap epoch 456" would not be (the osdmap and epoch
+concepts are an implementation detail, the down-ness of the OSD
+is the real message).  Feel free to send additional detail to
+the daemon's local log (via `dout`/`derr`).
+
+If you log a problem that may go away in the future, make sure you
+also log when it goes away.  Whatever priority you logged the original
+message at, log the "going away" message at INFO.
+
diff --git a/doc/dev/logs.rst b/doc/dev/logs.rst
new file mode 100644
index 000000000..7e703e541
--- /dev/null
+++ b/doc/dev/logs.rst
@@ -0,0 +1,55 @@
+============
+ Debug logs
+============
+
+The main debugging tool for Ceph is the dout and derr logging functions.
+Collectively, these are referred to as "dout logging."
+
+Dout has several log faculties, which can be set at various log
+levels using the configuration management system. So it is possible to enable
+debugging just for the messenger, by setting debug_ms to 10, for example.
+
+The dout macro avoids even generating log messages which are not going to be
+used, by enclosing them in an "if" statement. What this means is that if you
+have the debug level set at 0, and you run this code::
+
+	dout(20) << "myfoo() = " << myfoo() << dendl;
+
+
+myfoo() will not be called here.
+
+Unfortunately, the performance of debug logging is relatively low. This is
+because there is a single, process-wide mutex which every debug output
+statement takes, and every debug output statement leads to a write() system
+call or a call to syslog(). There is also a computational overhead to using C++
+streams to consider. So you will need to be parsimonious in your logging to get
+the best performance.
+
+Sometimes, enabling logging can hide race conditions and other bugs by changing
+the timing of events. Keep this in mind when debugging.
+
+Performance counters
+====================
+
+Ceph daemons use performance counters to track key statistics like number of
+inodes pinned. Performance counters are essentially sets of integers and floats
+which can be set, incremented, and read using the PerfCounters API.
+
+A PerfCounters object is usually associated with a single subsystem.  It
+contains multiple counters. This object is thread-safe because it is protected
+by an internal mutex. You can create multiple PerfCounters objects.
+
+Currently, three types of performance counters are supported: u64 counters,
+float counters, and long-run floating-point average counters. These are created
+by PerfCountersBuilder::add_u64, PerfCountersBuilder::add_fl, and
+PerfCountersBuilder::add_fl_avg, respectively. u64 and float counters simply
+provide a single value which can be updated, incremented, and read atomically.
+floating-pointer average counters provide two values: the current total, and
+the number of times the total has been changed. This is intended to provide a
+long-run average value.
+
+Performance counter information can be read in JSON format from the
+administrative socket (admin_sock). This is implemented as a UNIX domain
+socket. The Ceph performance counter plugin for collectd shows an example of how
+to access this information. Another example can be found in the unit tests for
+the administrative sockets.
diff --git a/doc/dev/macos.rst b/doc/dev/macos.rst
new file mode 100644
index 000000000..4f966be3d
--- /dev/null
+++ b/doc/dev/macos.rst
@@ -0,0 +1,50 @@
+build on MacOS
+==============
+
+Since we've switched to C++ 17, and the default clang shipped with Xcode 9 does not support all the C++ 17 language features, it's suggested to install clang using brew::
+
+  brew install llvm
+
+and install all the necessary bits::
+
+  brew install snappy ccache cmake pkg-config
+  pip install cython
+
+install FUSE if you want to build the FUSE support::
+
+  brew cask install osxfuse
+
+then, under the source directory of Ceph::
+
+  mkdir build
+  cd build
+  export PKG_CONFIG_PATH=/usr/local/Cellar/nss/3.48/lib/pkgconfig:/usr/local/Cellar/openssl/1.0.2t/lib/pkgconfig
+  cmake .. -DBOOST_J=4 \
+    -DCMAKE_C_COMPILER=/usr/local/opt/llvm/bin/clang \
+    -DCMAKE_CXX_COMPILER=/usr/local/opt/llvm/bin/clang++ \
+    -DCMAKE_EXE_LINKER_FLAGS="-L/usr/local/opt/llvm/lib" \
+    -DENABLE_GIT_VERSION=OFF \
+    -DSNAPPY_ROOT_DIR=/usr/local/Cellar/snappy/1.1.7_1 \
+    -DWITH_BABELTRACE=OFF \
+    -DWITH_BLUESTORE=OFF \
+    -DWITH_CCACHE=OFF \
+    -DWITH_CEPHFS=OFF \
+    -DWITH_KRBD=OFF \
+    -DWITH_LIBCEPHFS=OFF \
+    -DWITH_LTTNG=OFF \
+    -DWITH_LZ4=OFF \
+    -DWITH_MANPAGE=ON \
+    -DWITH_MGR=OFF \
+    -DWITH_MGR_DASHBOARD_FRONTEND=OFF \
+    -DWITH_RADOSGW=OFF \
+    -DWITH_RDMA=OFF \
+    -DWITH_SPDK=OFF \
+    -DWITH_SYSTEMD=OFF \
+    -DWITH_TESTS=OFF \
+    -DWITH_XFS=OFF
+
+The paths to ``nss`` and ``snappy`` might vary if newer versions of the packages are installed.
+
+Also, please consider using boost v1.69 to address the bug of https://github.com/boostorg/atomic/issues/15.
+
+Currently, the most practical uses for Ceph on MacOS might be FUSE and some other librados based applications.
diff --git a/doc/dev/mds_internals/data-structures.rst b/doc/dev/mds_internals/data-structures.rst
new file mode 100644
index 000000000..c77175a16
--- /dev/null
+++ b/doc/dev/mds_internals/data-structures.rst
@@ -0,0 +1,44 @@
+MDS internal data structures
+==============================
+
+*CInode*
+  CInode contains the metadata of a file, there is one CInode for each file.
+  The CInode stores information like who owns the file, how big the file is.
+
+*CDentry*
+  CDentry is the glue that holds inodes and files together by relating inode to
+  file/directory names. A CDentry links to at most one CInode (it may not link
+  to any CInode). A CInode may be linked by multiple CDentries.
+
+*CDir*
+  CDir only exists for directory inode, it's used to link CDentries under the
+  directory. A CInode can have multiple CDir when the directory is fragmented.
+
+These data structures are linked together as::
+
+  CInode
+  CDir
+   |   \
+   |      \
+   |         \
+  CDentry   CDentry
+  CInode    CInode
+  CDir      CDir
+   |         |  \
+   |         |     \
+   |         |        \
+  CDentry   CDentry  CDentry
+  CInode    CInode   CInode
+
+As this doc is being written, size of CInode is about 1400 bytes, size of CDentry
+is about 400 bytes, size of CDir is about 700 bytes. These data structures are
+quite large. Please be careful if you want to add new fields to them.
+
+*OpenFileTable*
+  Open file table tracks open files and their ancestor directories. Recovering
+  MDS can easily get open files' paths, significantly reducing the time of
+  loading inodes for open files. Each entry in the table corresponds to an inode,
+  it records linkage information (parent inode and dentry name) of the inode. MDS
+  can constructs the inode's path by recursively lookup parent inode's linkage.
+  Open file table is stored in omap of RADOS objects, table entries correspond to
+  KV pairs in omap.
diff --git a/doc/dev/mds_internals/exports.rst b/doc/dev/mds_internals/exports.rst
new file mode 100644
index 000000000..c5b0e3915
--- /dev/null
+++ b/doc/dev/mds_internals/exports.rst
@@ -0,0 +1,76 @@
+
+===============
+Subtree exports
+===============
+
+Normal Migration
+----------------
+
+The exporter begins by doing some checks in export_dir() to verify
+that it is permissible to export the subtree at this time.  In
+particular, the cluster must not be degraded, the subtree root may not
+be freezing or frozen (\ie already exporting, or nested beneath
+something that is exporting), and the path must be pinned (\ie not
+conflicted with a rename).  If these conditions are met, the subtree
+freeze is initiated, and the exporter is committed to the subtree
+migration, barring an intervening failure of the importer or itself.
+
+The MExportDirDiscover serves simply to ensure that the base directory
+being exported is open on the destination node.  It is pinned by the
+importer to prevent it from being trimmed.  This occurs before the
+exporter completes the freeze of the subtree to ensure that the
+importer is able to replicate the necessary metadata.  When the
+exporter receives the MExportDirDiscoverAck, it allows the freeze to proceed.
+
+The MExportDirPrep message then follows to populate a spanning tree that
+includes all dirs, inodes, and dentries necessary to reach any nested
+exports within the exported region.  This replicates metadata as well,
+but it is pushed out by the exporter, avoiding deadlock with the
+regular discover and replication process.  The importer is responsible
+for opening the bounding directories from any third parties before
+acknowledging.  This ensures that the importer has correct dir_auth
+information about where authority is delegated for all points nested
+within the subtree being migrated.  While processing the MExportDirPrep,
+the importer freezes the entire subtree region to prevent any new
+replication or cache expiration.
+
+The warning stage occurs only if the base subtree directory is open by
+nodes other than the importer and exporter.  If so, then a
+MExportDirNotify message informs any bystanders that the authority for
+the region is temporarily ambiguous.  In particular, bystanders who
+are trimming items from their cache must send MCacheExpire messages to
+both the old and new authorities.  This is necessary to ensure that
+the surviving authority reliably receives all expirations even if the
+importer or exporter fails.  While the subtree is frozen (on both the
+importer and exporter), expirations will not be immediately processed;
+instead, they will be queued until the region is unfrozen and it can
+be determined that the node is or is not authoritative for the region.
+
+The MExportDir message sends the actual subtree metadata to the importer.
+Upon receipt, the importer inserts the data into its cache, logs a
+copy in the EImportStart, and replies with an MExportDirAck.  The exporter
+can now log an EExport, which ultimately specifies that
+the export was a success.  In the presence of failures, it is the
+existence  of the EExport that disambiguates authority during recovery.
+
+Once logged, the exporter will send an MExportDirNotify to any
+bystanders, informing them that the authority is no longer ambiguous
+and cache expirations should be sent only to the new authority (the
+importer).  Once these are acknowledged, implicitly flushing the
+bystander to exporter message streams of any stray expiration notices,
+the exporter unfreezes the subtree, cleans up its state, and sends a
+final MExportDirFinish to the importer.  Upon receipt, the importer logs
+an EImportFinish(true), unfreezes its subtree, and cleans up its
+state.
+
+
+PARTIAL FAILURE RECOVERY
+
+
+
+RECOVERY FROM JOURNAL
+
+
+
+
+
diff --git a/doc/dev/mds_internals/index.rst b/doc/dev/mds_internals/index.rst
new file mode 100644
index 000000000..c8c82ad10
--- /dev/null
+++ b/doc/dev/mds_internals/index.rst
@@ -0,0 +1,10 @@
+==============================
+MDS developer documentation
+==============================
+
+.. rubric:: Contents
+
+.. toctree::
+   :glob:
+
+   *
diff --git a/doc/dev/messenger.rst b/doc/dev/messenger.rst
new file mode 100644
index 000000000..729538913
--- /dev/null
+++ b/doc/dev/messenger.rst
@@ -0,0 +1,33 @@
+============================
+ Messenger notes
+============================
+
+Messenger is the Ceph network layer implementation. Currently Ceph supports
+one messenger type: "async".
+
+ceph_perf_msgr
+==============
+
+ceph_perf_msgr is used to do benchmark for messenger module only and can help
+to find the bottleneck or time consuming within messenger moduleIt just like
+"iperf", we need to start server-side program firstly:
+
+# ./ceph_perf_msgr_server 172.16.30.181:10001 1 0
+
+The first argument is ip:port pair which is telling the destination address the
+client need to specified. The second argument configures the server threads. The
+third argument tells the "think time"(us) when dispatching messages. After Giant,
+CEPH_OSD_OP message which is the actual client read/write io request is fast
+dispatched without queueing to Dispatcher, in order to achieve better performance.
+So CEPH_OSD_OP message will be processed inline, "think time" is used by mock
+this "inline process" process.
+
+# ./ceph_perf_msgr_client 172.16.30.181:10001 1 32 10000 10 4096 
+
+The first argument is specified the server ip:port, and the second argument is
+used to specify client threads. The third argument specify the concurrency(the
+max inflight messages for each client thread), the fourth argument specify the
+io numbers will be issued to server per client thread. The fifth argument is
+used to indicate the "think time" for client thread when receiving messages,
+this is also used to mock the client fast dispatch process. The last argument
+specify the message data length to issue.
diff --git a/doc/dev/mon-bootstrap.rst b/doc/dev/mon-bootstrap.rst
new file mode 100644
index 000000000..2b67b4707
--- /dev/null
+++ b/doc/dev/mon-bootstrap.rst
@@ -0,0 +1,212 @@
+===================
+ Monitor bootstrap
+===================
+
+Terminology:
+
+* ``cluster``: a set of monitors
+* ``quorum``: an active set of monitors consisting of a majority of the cluster
+
+In order to initialize a new monitor, it must always be fed:
+
+#. a logical name
+#. secret keys
+#. a cluster fsid (uuid)
+
+In addition, a monitor needs to know two things:
+
+#. what address to bind to
+#. who its peers are (if any)
+
+There are a range of ways to do both.
+
+Logical id
+==========
+
+The logical id should be unique across the cluster.  It will be
+appended to ``mon.`` to logically describe the monitor in the Ceph
+cluster.  For example, if the logical id is ``foo``, the monitor's
+name will be ``mon.foo``.
+
+For most users, there is no more than one monitor per host, which
+makes the short hostname logical choice.
+
+Secret keys
+===========
+
+The ``mon.`` secret key is stored a ``keyring`` file in the ``mon data`` directory.  It can be generated
+with a command like::
+
+        ceph-authtool --create-keyring /path/to/keyring --gen-key -n mon.
+
+When creating a new monitor cluster, the keyring should also contain a ``client.admin`` key that can be used
+to administer the system::
+
+        ceph-authtool /path/to/keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow'
+
+The resulting keyring is fed to ``ceph-mon --mkfs`` with the ``--keyring <keyring>`` command-line argument.
+
+Cluster fsid
+============
+
+The cluster fsid is a normal uuid, like that generated by the ``uuidgen`` command.  It
+can be provided to the monitor in two ways:
+
+#. via the ``--fsid <uuid>`` command-line argument (or config file option)
+#. via a monmap provided to the new monitor via the ``--monmap <path>`` command-line argument.
+
+Monitor address
+===============
+
+The monitor address can be provided in several ways.
+
+#. via the ``--public-addr <ip[:port]>`` command-line option (or config file option)
+#. via the ``--public-network <cidr>`` command-line option (or config file option)
+#. via the monmap provided via ``--monmap <path>``, if it includes a monitor with our name
+#. via the bootstrap monmap (provided via ``--inject-monmap <path>`` or generated from ``--mon-host <list>``) if it includes a monitor with no name (``noname-<something>``) and an address configured on the local host.
+
+Peers
+=====
+
+The monitor peers are provided in several ways:
+
+#. via the initial monmap, provided via ``--monmap <filename>``
+#. via the bootstrap monmap generated from ``--mon-host <list>``
+#. via the bootstrap monmap generated from ``[mon.*]`` sections with the deprecated ``mon addr`` options in the config file (note that this method is *not* recommended and does not support binding to both v1 and v2 protocol addresses)
+#. dynamically via the admin socket
+
+However, these methods are not completely interchangeable because of
+the complexity of creating a new monitor cluster without danger of
+races.
+
+Cluster creation
+================
+
+There are three basic approaches to creating a cluster:
+
+#. Create a new cluster by specifying the monitor names and addresses ahead of time.
+#. Create a new cluster by specifying the monitor names ahead of time, and dynamically setting the addresses as ``ceph-mon`` daemons configure themselves.
+#. Create a new cluster by specifying the monitor addresses ahead of time.
+
+
+Names and addresses
+-------------------
+
+Generate a monmap using ``monmaptool`` with the names and addresses of the initial
+monitors.  The generated monmap will also include a cluster fsid.  Feed that monmap
+to each monitor daemon::
+
+        ceph-mon --mkfs -i <name> --monmap <initial_monmap> --keyring <initial_keyring>
+
+When the daemons start, they will know exactly who they and their peers are.
+
+
+Addresses only
+--------------
+
+The initial monitor addresses can be specified with the ``mon host`` configuration value,
+either via a config file or the command-line argument.  This method has the advantage that
+a single global config file for the cluster can have a line like::
+
+     mon host = a.foo.com, b.foo.com, c.foo.com
+
+and will also serve to inform any ceph clients or daemons who the monitors are.
+
+The ``ceph-mon`` daemons will need to be fed the initial keyring and cluster fsid to 
+initialize themselves:
+
+     ceph-mon --mkfs -i <name> --fsid <uuid> --keyring <initial_keyring>
+
+When the daemons first start up, they will share their names with each other and form a
+new cluster.
+
+Names only
+----------
+
+In dynamic "cloud" environments, the cluster creator may not (yet)
+know what the addresses of the monitors are going to be.  Instead,
+they may want machines to configure and start themselves in parallel
+and, as they come up, form a new cluster on their own.  The problem is
+that the monitor cluster relies on strict majorities to keep itself
+consistent, and in order to "create" a new cluster, it needs to know
+what the *initial* set of monitors will be.
+
+This can be done with the ``mon initial members`` config option, which
+should list the ids of the initial monitors that are allowed to create
+the cluster::
+
+     mon initial members = foo, bar, baz
+
+The monitors can then be initialized by providing the other pieces of
+information (they keyring, cluster fsid, and a way of determining
+their own address).  For example::
+
+     ceph-mon --mkfs -i <name> --mon-initial-hosts 'foo,bar,baz' --keyring <initial_keyring> --public-addr <ip>
+
+When these daemons are started, they will know their own address, but
+not their peers.  They can learn those addresses via the admin socket::
+
+     ceph daemon mon.<id> add_bootstrap_peer_hint <peer ip>
+
+Once they learn enough of their peers from the initial member set,
+they will be able to create the cluster.
+
+
+Cluster expansion
+=================
+
+Cluster expansion is slightly less demanding than creation, because
+the creation of the initial quorum is not an issue and there is no
+worry about creating separately independent clusters.
+
+New nodes can be forced to join an existing cluster in two ways:
+
+#. by providing no initial monitor peers addresses, and feeding them dynamically.
+#. by specifying the ``mon initial members`` config option to prevent the new nodes from forming a new, independent cluster, and feeding some existing monitors via any available method.
+
+Initially peerless expansion
+----------------------------
+
+Create a new monitor and give it no peer addresses other than its own.  For
+example::
+
+     ceph-mon --mkfs -i <myid> --fsid <fsid> --keyring <mon secret key> --public-addr <ip>
+
+Once the daemon starts, you can give it one or more peer addresses (preferably a bare IP address with no port; the mon will set the addr types and ports for you) to join with::
+
+     ceph daemon mon.<id> add_bootstrap_peer_hint <peer ip>
+
+Alternatively, you can explicitly specify the addrvec_t with::
+
+     ceph daemon mon.<id> add_bootstrap_peer_hintv <peer addrvec>
+
+For example,::
+
+     ceph daemon mon.new add_bootstrap_peer_hintv v2:1.2.3.4:3300,v1:1.2.3.4:6789
+
+This monitor will never participate in cluster creation; it can only
+join an existing cluster.
+
+Note that the address(es) specified should match exactly the addresses
+the new monitor is binding too.  If, for example, the new mon binds to
+only a v2 address but a v2 and v1 address are provided, there is some
+possibility of confusion in the mons.
+
+Expanding with initial members
+------------------------------
+
+You can feed the new monitor some peer addresses initially and avoid badness by also
+setting ``mon initial members``.  For example::
+
+     ceph-mon --mkfs -i <myid> --fsid <fsid> --keyring <mon secret key> --public-addr <ip> --mon-host foo,bar,baz
+
+When the daemon is started, ``mon initial members`` must be set via the command line or config file::
+
+     ceph-mon -i <myid> --mon-initial-members foo,bar,baz
+
+to prevent any risk of split-brain.
+  
+
+
+
+
diff --git a/doc/dev/mon-elections.rst b/doc/dev/mon-elections.rst
new file mode 100644
index 000000000..86cfc3803
--- /dev/null
+++ b/doc/dev/mon-elections.rst
@@ -0,0 +1,130 @@
+=================
+Monitor Elections
+=================
+
+The Original Algorithm
+======================
+Historically, monitor leader elections have been very simple: the lowest-ranked
+monitor wins!
+
+This is accomplished using a low-state "Elector" module (though it has now
+been split into an Elector that handles message-passing, and an ElectionLogic
+that makes the voting choices). It tracks the election epoch and not much
+else. Odd epochs are elections; even epochs have a leader and let the monitor
+do its ongoing work. When a timeout occurs or the monitor asks for a
+new election, we bump the epoch and send out Propose messages to all known
+monitors.
+In general, if we receive an old message we either drop it or trigger a new
+election (if we think the sender is newly-booted and needs to join quorum). If
+we receive a message from a newer epoch, we bump up our epoch to match and
+either Defer to the Proposer or else bump the epoch again and Propose
+ourselves if we expect to win over them. When we receive a Propose within
+our current epoch, we either Defer to the sender or ignore them (we ignore them
+if they are of a higher rank than us, or higher than the rank we have already
+deferred to).
+(Note that if we have the highest rank it is possible for us to defer to every
+other monitor in sequence within the same election epoch!)
+
+This resolves under normal circumstances because all monitors agree on the
+priority voting order, and epochs are only bumped when a monitor isn't
+participating or sees a possible conflict with the known proposers.
+
+The Problems
+==============
+The original algorithm didn't work at all under a variety of netsplit
+conditions. This didn't manifest often in practice but has become
+important as the community and commercial vendors move Ceph into
+spaces requiring the use of "stretch clusters".
+
+The New Algorithms
+==================
+We still default to the original ("classic") election algorithm, but
+support letting users change to new ones via the CLI. These
+algorithms are implemented as different functions and switch statements
+within the ElectionLogic class.
+
+The first algorithm is very simple: "disallow" lets you add monitors
+to a list of disallowed leaders.
+The second, "connectivity", incorporates connection score ratings
+and elects the monitor with the best score.
+
+Algorithm: disallow
+===================
+If a monitor is in the disallowed list, it always defers to another
+monitor, no matter the rank. Otherwise, it is the same as the classic
+algorithm is.
+Since changing the disallowed list requires a paxos update, monitors
+in an election together should always have the same set. This means
+the election order is constant and static across the full monitor set
+and elections resolve trivially (assuming a connected network).
+
+This algorithm really just exists as a demo and stepping-stone to
+the more advanced connectivity mode, but it may have utility in asymmetric
+networks and clusters.
+
+Algorithm: connectivity
+=======================
+This algorithm takes as input scores for each connection
+(both ways, discussed in the next section) and attempts to elect the monitor
+with the highest total score. We keep the same basic message-passing flow as the
+classic algorithm, in which elections are driven by reacting to Propose messages.
+But this has several challenges since unlike ranks, scores are not static (and
+might change during an election!). To guarantee an election epoch does not
+produce multiple leaders, we must maintain two key invariants:
+* Monitors must maintain static scores during an election epoch
+* Any deferral must be transitive -- if A defers to B and then to C,
+B had better defer to C as well!
+
+We handle these very explicitly: by branching a copy stable_peer_tracker
+of our peer_tracker scoring object whenever starting an election (or
+bumping the epoch), and by refusing to defer to a monitor if it won't
+be deferred to by our current leader choice. (All Propose messages include
+a copy of the scores the leader is working from, so peers can evaluate them.)
+
+Of course, those modifications can easily block. To guarantee forward progress,
+we make several further adjustments:
+* If we want to defer to a new peer, but have already deferred to a peer
+whose scores don't allow that, we bump the election epoch and start()
+the election over again.
+* All election messages include the scores the sender is aware of.
+
+This guarantees we will resolve the election as long as the network is
+reasonably stable (even if disconnected): As long as all score "views"
+result in the same deferral order, an election will complete normally. And by
+broadly sharing scores across the full set of monitors, monitors rapidly
+converge on the global newest state.
+
+This algorithm has one further important feature compared to the classic and
+disallowed handlers: it can ignore out-of-quorum peers. Normally, whenever
+a monitor B receives a Propose from an out-of-quorum peer C, B will itself trigger
+a new election to give C an opportunity to join. But because the
+highest-scoring monitor A may be netsplit from C, this is not desirable. So in
+the connectivity election algorithm, B only "forwards" Propose messages when B's
+scores indicate the cluster would choose a leader other than A.
+
+Connection Scoring
+==================
+We implement scoring within the ConnectionTracker class, which is
+driven by the Elector and provided to ElectionLogic as a resource. Elector
+is responsible for sending out MMonPing messages, and for reporting the
+results in to the ConnectionTracker as calls to report_[live|dead]_connection
+with the relevant peer and the time units the call counts for. (These time units
+are seconds in the monitor, but the ConnectionTracker is agnostic and our unit
+tests count simple time steps.)
+
+We configure a "half life" and each report updates the peer's current status
+(alive or dead) and its total score. The new score is current_score * (1 - units_alive / (2 * half_life)) + (units_alive / (2 * half_life)). (For a dead report, we of course
+subtract the new delta, rather than adding it).
+
+We can further encode and decode the ConnectionTracker for wire transmission,
+and receive_peer_report()s of a full ConnectionTracker (containing all
+known scores) or a ConnectionReport (representing a single peer's scores)
+to slurp up the scores from peers. These scores are of course all versioned so
+we are in no danger of accidentally going backwards in time.
+We can query an individual connection score (if the connection is down, it's 0)
+or the total score of a specific monitor, which is the connection score from all
+other monitors going in to that one.
+
+By default, we consider pings failed after 2 seconds (mon_elector_ping_timeout)
+and ping live connections every second (mon_elector_ping_divisor). The halflife
+is 12 hours (mon_con_tracker_score_halflife).
diff --git a/doc/dev/mon-on-disk-formats.rst b/doc/dev/mon-on-disk-formats.rst
new file mode 100644
index 000000000..e48d39a41
--- /dev/null
+++ b/doc/dev/mon-on-disk-formats.rst
@@ -0,0 +1,91 @@
+##############
+ON-DISK FORMAT
+##############
+
+
+************
+UPGRADE PATH
+************
+
+On-disk formats, or even data structure formats, may be changed during an
+upgrade. Services wishing to do so, may so do it via the
+`PaxosService::upgrade_format()` call path. There is no formalized, unified
+format versioning; the `PaxosService` class keeps track of its
+`format_version` through a key in the store, assumed an `unsigned int`,  but
+it will be the service's responsibility to give meaning to those versions.
+
+AUTH MONITOR
+============
+
+versions
+--------
+
+versions are represented with a single `unsigned int`. By default, the value
+zero represents the absence of a formal upgraded format. The first format
+version was introduced in Dumpling; clusters upgrading to Dumpling saw their
+format version being increased from zero to one.::
+
+  0 to 1 - introduced in v0.65, dev release for v0.67 dumpling
+  1 to 2 - introduced in v12.0.2, dev release for luminous
+  2 to 3 - introduced in mimic
+
+  0 - all clusters pre-dumpling
+  1 - all clusters dumpling+ and pre-luminous
+  2 - all clusters luminous+ and pre-mimic
+  3 - all clusters mimic+
+
+  version 1: introduces new-style monitor caps (i.e., profiles)
+  version 2: introduces mgr caps and bootstrap-mgr key
+  version 3: creates all bootstrap and admin keys if they don't yet exist
+
+callstack
+---------
+
+format_version set on `PaxosService::refresh()`:::
+
+  - initially called from Monitor::refresh_from_paxos
+    - initially called from Monitor::init_paxos()
+      - initially called from Monitor::preinit()
+
+AuthMonitor::upgrade_format() called by `PaxosService::_active()`:::
+
+  - called from C_Committed callback, from PaxosService::propose_pending()
+  - called from C_Active callback, from PaxosService::_active()
+  - called from PaxosService::election_finished()
+
+  - on a freshly deployed cluster, upgrade_format() will be first called
+    *after* create_initial().
+  - on an existing cluster, upgrade_format() will be called after the first
+    election.
+
+  - upgrade_format() is irrelevant on a freshly deployed cluster, as there is
+    no format to upgrade at this point.
+
+boil down
+---------
+
+* if `format_version >= current_version` then format is uptodate, return.
+* if `features doesn't contain LUMINOUS` then `current_version = 1`
+* else if `features doesn't contain MIMIC` then `current_version = 2`
+* else `current_version = 3`
+
+if `format_version == 0`:::
+
+  - upgrade to format version 1
+    - move to new-style monitor caps (i.e., profiles):
+      - set daemon profiles for existing entities
+      - set profile for existing bootstrap keys
+
+if `format_version == 1`:::
+
+  - upgrade to format version 2
+    - for existing entities:
+      - add new cap for mgr
+    - for existing 'mgr' entities, fix 'mon' caps due to bug from kraken
+      setting 'allow \*', and set 'allow profile mgr' instead.
+    - add bootstrap-mgr key.
+
+if `format_version == 2`:::
+
+  - upgrade to format version 3
+    - create all bootstrap keys if they don't currently exist
diff --git a/doc/dev/mon-osdmap-prune.rst b/doc/dev/mon-osdmap-prune.rst
new file mode 100644
index 000000000..6ff059b84
--- /dev/null
+++ b/doc/dev/mon-osdmap-prune.rst
@@ -0,0 +1,415 @@
+===========================
+FULL OSDMAP VERSION PRUNING
+===========================
+
+For each incremental osdmap epoch, the monitor will keep a full osdmap
+epoch in the store.
+
+While this is great when serving osdmap requests from clients, allowing
+us to fulfill their request without having to recompute the full osdmap
+from a myriad of incrementals, it can also become a burden once we start
+keeping an unbounded number of osdmaps.
+
+The monitors will attempt to keep a bounded number of osdmaps in the store.
+This number is defined (and configurable) via ``mon_min_osdmap_epochs``, and
+defaults to 500 epochs. Generally speaking, we will remove older osdmap
+epochs once we go over this limit.
+
+However, there are a few constraints to removing osdmaps. These are all
+defined in ``OSDMonitor::get_trim_to()``.
+
+In the event one of these conditions is not met, we may go over the bounds
+defined by ``mon_min_osdmap_epochs``. And if the cluster does not meet the
+trim criteria for some time (e.g., unclean pgs), the monitor may start
+keeping a lot of osdmaps. This can start putting pressure on the underlying
+key/value store, as well as on the available disk space.
+
+One way to mitigate this problem would be to stop keeping full osdmap
+epochs on disk. We would have to rebuild osdmaps on-demand, or grab them
+from cache if they had been recently served. We would still have to keep
+at least one osdmap, and apply all incrementals on top of either this
+oldest map epoch kept in the store or a more recent map grabbed from cache.
+While this would be feasible, it seems like a lot of cpu (and potentially
+IO) would be going into rebuilding osdmaps.
+
+Additionally, this would prevent the aforementioned problem going forward,
+but would do nothing for stores currently in a state that would truly
+benefit from not keeping osdmaps.
+
+This brings us to full osdmap pruning.
+
+Instead of not keeping full osdmap epochs, we are going to prune some of
+them when we have too many.
+
+Deciding whether we have too many will be dictated by a configurable option
+``mon_osdmap_full_prune_min`` (default: 10000). The pruning algorithm will be
+engaged once we go over this threshold.
+
+We will not remove all ``mon_osdmap_full_prune_min`` full osdmap epochs
+though. Instead, we are going to poke some holes in the sequence of full
+maps. By default, we will keep one full osdmap per 10 maps since the last
+map kept; i.e., if we keep epoch 1, we will also keep epoch 10 and remove
+full map epochs 2 to 9. The size of this interval is configurable with
+``mon_osdmap_full_prune_interval``.
+
+Essentially, we are proposing to keep ~10% of the full maps, but we will
+always honour the minimum number of osdmap epochs, as defined by
+``mon_min_osdmap_epochs``, and these won't be used for the count of the
+minimum versions to prune. For instance, if we have on-disk versions
+[1..50000], we would allow the pruning algorithm to operate only over
+osdmap epochs [1..49500); but, if have on-disk versions [1..10200], we
+won't be pruning because the algorithm would only operate on versions
+[1..9700), and this interval contains less versions than the minimum
+required by ``mon_osdmap_full_prune_min``.
+
+
+ALGORITHM
+=========
+
+Say we have 50,000 osdmap epochs in the store, and we're using the
+defaults for all configurable options.
+
+::
+
+    -----------------------------------------------------------
+    |1|2|..|10|11|..|100|..|1000|..|10000|10001|..|49999|50000|
+    -----------------------------------------------------------
+     ^ first                                            last ^
+
+We will prune when all the following constraints are met:
+
+1. number of versions is greater than ``mon_min_osdmap_epochs``;
+
+2. the number of versions between ``first`` and ``prune_to`` is greater (or
+   equal) than ``mon_osdmap_full_prune_min``, with ``prune_to`` being equal to
+   ``last`` minus ``mon_min_osdmap_epochs``.
+
+If any of these conditions fails, we will *not* prune any maps.
+
+Furthermore, if it is known that we have been pruning, but since then we
+are no longer satisfying at least one of the above constraints, we will
+not continue to prune. In essence, we only prune full osdmaps if the
+number of epochs in the store so warrants it.
+
+As pruning will create gaps in the sequence of full maps, we need to keep
+track of the intervals of missing maps. We do this by keeping a manifest of
+pinned maps -- i.e., a list of maps that, by being pinned, are not to be
+pruned.
+
+While pinned maps are not removed from the store, maps between two consecutive
+pinned maps will; and the number of maps to be removed will be dictated by the
+configurable option ``mon_osdmap_full_prune_interval``. The algorithm makes an
+effort to keep pinned maps apart by as many maps as defined by this option,
+but in the event of corner cases it may allow smaller intervals. Additionally,
+as this is a configurable option that is read any time a prune iteration
+occurs, there is the possibility this interval will change if the user changes
+this config option.
+
+Pinning maps is performed lazily: we will be pinning maps as we are removing
+maps. This grants us more flexibility to change the prune interval while
+pruning is happening, but also simplifies considerably the algorithm, as well
+as the information we need to keep in the manifest. Below we show a simplified
+version of the algorithm:::
+
+    manifest.pin(first)
+    last_to_prune = last - mon_min_osdmap_epochs
+
+    while manifest.get_last_pinned() + prune_interval < last_to_prune AND
+          last_to_prune - first > mon_min_osdmap_epochs AND
+          last_to_prune - first > mon_osdmap_full_prune_min AND
+          num_pruned < mon_osdmap_full_prune_txsize:
+      
+      last_pinned = manifest.get_last_pinned()
+      new_pinned = last_pinned + prune_interval
+      manifest.pin(new_pinned)
+      for e in (last_pinned .. new_pinned):
+        store.erase(e)
+        ++num_pruned
+
+In essence, the algorithm ensures that the first version in the store is
+*always* pinned. After all, we need a starting point when rebuilding maps, and
+we can't simply remove the earliest map we have; otherwise we would be unable
+to rebuild maps for the very first pruned interval.
+
+Once we have at least one pinned map, each iteration of the algorithm can
+simply base itself on the manifest's last pinned map (which we can obtain by
+reading the element at the tail of the manifest's pinned maps list).
+
+We'll next need to determine the interval of maps to be removed: all the maps
+from ``last_pinned`` up to ``new_pinned``, which in turn is nothing more than
+``last_pinned`` plus ``mon_osdmap_full_prune_interval``. We know that all maps
+between these two values, ``last_pinned`` and ``new_pinned`` can be removed,
+considering ``new_pinned`` has been pinned.
+
+The algorithm ceases to execute as soon as one of the two initial
+preconditions is not met, or if we do not meet two additional conditions that
+have no weight on the algorithm's correctness:
+
+1. We will stop if we are not able to create a new pruning interval properly
+   aligned with ``mon_osdmap_full_prune_interval`` that is lower than
+   ``last_pruned``. There is no particular technical reason why we enforce
+   this requirement, besides allowing us to keep the intervals with an
+   expected size, and preventing small, irregular intervals that would be
+   bound to happen eventually (e.g., pruning continues over the course of
+   several iterations, removing one or two or three maps each time).
+
+2. We will stop once we know that we have pruned more than a certain number of
+   maps. This value is defined by ``mon_osdmap_full_prune_txsize``, and
+   ensures we don't spend an unbounded number of cycles pruning maps. We don't
+   enforce this value religiously (deletes do not cost much), but we make an
+   effort to honor it.
+
+We could do the removal in one go, but we have no idea how long that would
+take. Therefore, we will perform several iterations, removing at most
+``mon_osdmap_full_prune_txsize`` osdmaps per iteration.
+
+In the end, our on-disk map sequence will look similar to::
+
+    ------------------------------------------
+    |1|10|20|30|..|49500|49501|..|49999|50000|
+    ------------------------------------------
+     ^ first                           last ^
+
+
+Because we are not pruning all versions in one go, we need to keep state
+about how far along on our pruning we are. With that in mind, we have
+created a data structure, ``osdmap_manifest_t``, that holds the set of pinned
+maps:::
+
+    struct osdmap_manifest_t:
+        set<version_t> pinned;
+
+Given we are only pinning maps while we are pruning, we don't need to keep
+track of additional state about the last pruned version. We know as a matter
+of fact that we have pruned all the intermediate maps between any two
+consecutive pinned maps.
+
+The question one could ask, though, is how can we be sure we pruned all the
+intermediate maps if, for instance, the monitor crashes. To ensure we are
+protected against such an event, we always write the osdmap manifest to disk
+on the same transaction that is deleting the maps. This way we have the
+guarantee that, if the monitor crashes, we will read the latest version of the
+manifest: either containing the newly pinned maps, meaning we also pruned the
+in-between maps; or we will find the previous version of the osdmap manifest,
+which will not contain the maps we were pinning at the time we crashed, given
+the transaction on which we would be writing the updated osdmap manifest was
+not applied (alongside with the maps removal).
+
+The osdmap manifest will be written to the store each time we prune, with an
+updated list of pinned maps. It is written in the transaction effectively
+pruning the maps, so we guarantee the manifest is always up to date. As a
+consequence of this criteria, the first time we will write the osdmap manifest
+is the first time we prune. If an osdmap manifest does not exist, we can be
+certain we do not hold pruned map intervals.
+
+We will rely on the manifest to ascertain whether we have pruned maps
+intervals. In theory, this will always be the on-disk osdmap manifest, but we
+make sure to read the on-disk osdmap manifest each time we update from paxos;
+this way we always ensure having an up to date in-memory osdmap manifest.
+
+Once we finish pruning maps, we will keep the manifest in the store, to
+allow us to easily find which maps have been pinned (instead of checking
+the store until we find a map). This has the added benefit of allowing us to
+quickly figure out which is the next interval we need to prune (i.e., last
+pinned plus the prune interval). This doesn't however mean we will forever
+keep the osdmap manifest: the osdmap manifest will no longer be required once
+the monitor trims osdmaps and the earliest available epoch in the store is
+greater than the last map we pruned.
+
+The same conditions from ``OSDMonitor::get_trim_to()`` that force the monitor
+to keep a lot of osdmaps, thus requiring us to prune, may eventually change
+and allow the monitor to remove some of its oldest maps.
+
+MAP TRIMMING
+------------
+
+If the monitor trims maps, we must then adjust the osdmap manifest to
+reflect our pruning status, or remove the manifest entirely if it no longer
+makes sense to keep it. For instance, take the map sequence from before, but
+let us assume we did not finish pruning all the maps.::
+
+    -------------------------------------------------------------
+    |1|10|20|30|..|490|500|501|502|..|49500|49501|..|49999|50000|
+    -------------------------------------------------------------
+     ^ first            ^ pinned.last()                   last ^
+
+    pinned = {1, 10, 20, ..., 490, 500}
+
+Now let us assume that the monitor will trim up to epoch 501. This means
+removing all maps prior to epoch 501, and updating the ``first_committed``
+pointer to ``501``. Given removing all those maps would invalidate our
+existing pruning efforts, we can consider our pruning has finished and drop
+our osdmap manifest. Doing so also simplifies starting a new prune, if all
+the starting conditions are met once we refreshed our state from the
+store.
+
+We would then have the following map sequence: ::
+
+    ---------------------------------------
+    |501|502|..|49500|49501|..|49999|50000|
+    ---------------------------------------
+     ^ first                        last ^
+
+However, imagine a slightly more convoluted scenario: the monitor will trim
+up to epoch 491. In this case, epoch 491 has been previously pruned from the
+store.
+
+Given we will always need to have the oldest known map in the store, before
+we trim we will have to check whether that map is in the prune interval
+(i.e., if said map epoch belongs to ``[ pinned.first()..pinned.last() )``).
+If so, we need to check if this is a pinned map, in which case we don't have
+much to be concerned aside from removing lower epochs from the manifest's
+pinned list. On the other hand, if the map being trimmed to is not a pinned
+map, we will need to rebuild said map and pin it, and only then will we remove
+the pinned maps prior to the map's epoch. 
+
+In this case, we would end up with the following sequence:::
+
+    -----------------------------------------------
+    |491|500|501|502|..|49500|49501|..|49999|50000|
+    -----------------------------------------------
+     ^   ^- pinned.last()                   last ^
+     `- first
+
+There is still an edge case that we should mention. Consider that we are
+going to trim up to epoch 499, which is the very last pruned epoch.
+
+Much like the scenario above, we would end up writing osdmap epoch 499 to
+the store; but what should we do about pinned maps and pruning?
+
+The simplest solution is to drop the osdmap manifest. After all, given we
+are trimming to the last pruned map, and we are rebuilding this map, we can
+guarantee that all maps greater than e 499 are sequential (because we have
+not pruned any of them). In essence, dropping the osdmap manifest in this
+case is essentially the same as if we were trimming over the last pruned
+epoch: we can prune again later if we meet the required conditions.
+
+And, with this, we have fully dwelled into full osdmap pruning. Later in this
+document one can find detailed `REQUIREMENTS, CONDITIONS & INVARIANTS` for the
+whole algorithm, from pruning to trimming. Additionally, the next section
+details several additional checks to guarantee the sanity of our configuration
+options. Enjoy.
+
+
+CONFIGURATION OPTIONS SANITY CHECKS
+-----------------------------------
+
+We perform additional checks before pruning to ensure all configuration
+options involved are sane:
+
+1. If ``mon_osdmap_full_prune_interval`` is zero we will not prune; we
+   require an actual positive number, greater than one, to be able to prune
+   maps. If the interval is one, we would not actually be pruning any maps, as
+   the interval between pinned maps would essentially be a single epoch. This
+   means we would have zero maps in-between pinned maps, hence no maps would
+   ever be pruned.
+
+2. If ``mon_osdmap_full_prune_min`` is zero we will not prune; we require a
+   positive, greater than zero, value so we know the threshold over which we
+   should prune. We don't want to guess.
+
+3. If ``mon_osdmap_full_prune_interval`` is greater than
+   ``mon_osdmap_full_prune_min`` we will not prune, as it is impossible to
+   ascertain a proper prune interval.
+
+4. If ``mon_osdmap_full_prune_txsize`` is lower than
+   ``mon_osdmap_full_prune_interval`` we will not prune; we require a
+   ``txsize`` with a value at least equal than ``interval``, and (depending on
+   the value of the latter) ideally higher.
+
+
+REQUIREMENTS, CONDITIONS & INVARIANTS
+-------------------------------------
+
+REQUIREMENTS
+~~~~~~~~~~~~
+
+* All monitors in the quorum need to support pruning.
+
+* Once pruning has been enabled, monitors not supporting pruning will not be
+  allowed in the quorum, nor will be allowed to synchronize.
+
+* Removing the osdmap manifest results in disabling the pruning feature quorum
+  requirement. This means that monitors not supporting pruning will be allowed
+  to synchronize and join the quorum, granted they support any other features
+  required.
+
+
+CONDITIONS & INVARIANTS
+~~~~~~~~~~~~~~~~~~~~~~~
+
+* Pruning has never happened, or we have trimmed past its previous
+  intervals:::
+
+    invariant: first_committed > 1
+
+    condition: pinned.empty() AND !store.exists(manifest)
+
+
+* Pruning has happened at least once:::
+
+    invariant: first_committed > 0
+    invariant: !pinned.empty())
+    invariant: pinned.first() == first_committed
+    invariant: pinned.last() < last_committed
+
+      precond: pinned.last() < prune_to AND
+               pinned.last() + prune_interval < prune_to
+
+     postcond: pinned.size() > old_pinned.size() AND
+               (for each v in [pinned.first()..pinned.last()]:
+                 if pinned.count(v) > 0: store.exists_full(v)
+                 else: !store.exists_full(v)
+               )
+
+
+* Pruning has finished:::
+
+    invariant: first_committed > 0
+    invariant: !pinned.empty()
+    invariant: pinned.first() == first_committed
+    invariant: pinned.last() < last_committed
+
+    condition: pinned.last() == prune_to OR
+               pinned.last() + prune_interval < prune_to
+
+
+* Pruning intervals can be trimmed:::
+
+    precond:   OSDMonitor::get_trim_to() > 0
+
+    condition: !pinned.empty()
+
+    invariant: pinned.first() == first_committed
+    invariant: pinned.last() < last_committed
+    invariant: pinned.first() <= OSDMonitor::get_trim_to()
+    invariant: pinned.last() >= OSDMonitor::get_trim_to()
+
+* Trim pruned intervals:::
+
+    invariant: !pinned.empty()
+    invariant: pinned.first() == first_committed
+    invariant: pinned.last() < last_committed
+    invariant: pinned.first() <= OSDMonitor::get_trim_to()
+    invariant: pinned.last() >= OSDMonitor::get_trim_to()
+
+    postcond:  pinned.empty() OR
+               (pinned.first() == OSDMonitor::get_trim_to() AND
+                pinned.last() > pinned.first() AND
+                (for each v in [0..pinned.first()]:
+                  !store.exists(v) AND
+                  !store.exists_full(v)
+                ) AND
+                (for each m in [pinned.first()..pinned.last()]:
+                  if pinned.count(m) > 0: store.exists_full(m)
+                  else: !store.exists_full(m) AND store.exists(m)
+                )
+               )
+    postcond:  !pinned.empty() OR
+               (!store.exists(manifest) AND
+                (for each v in [pinned.first()..pinned.last()]:
+                  !store.exists(v) AND
+                  !store.exists_full(v)
+                )
+               )
+
diff --git a/doc/dev/msgr2.rst b/doc/dev/msgr2.rst
new file mode 100644
index 000000000..585dc34d2
--- /dev/null
+++ b/doc/dev/msgr2.rst
@@ -0,0 +1,840 @@
+.. _msgr2-protocol:
+
+msgr2 protocol (msgr2.0 and msgr2.1)
+====================================
+
+This is a revision of the legacy Ceph on-wire protocol that was
+implemented by the SimpleMessenger.  It addresses performance and
+security issues.
+
+Goals
+-----
+
+This protocol revision has several goals relative to the original protocol:
+
+* *Flexible handshaking*.  The original protocol did not have a
+  sufficiently flexible protocol negotiation that allows for features
+  that were not required.
+* *Encryption*.  We will incorporate encryption over the wire.
+* *Performance*.  We would like to provide for protocol features
+  (e.g., padding) that keep computation and memory copies out of the
+  fast path where possible.
+* *Signing*.  We will allow for traffic to be signed (but not
+  necessarily encrypted).  This is not implemented.
+
+Definitions
+-----------
+
+* *client* (C): the party initiating a (TCP) connection
+* *server* (S): the party accepting a (TCP) connection
+* *connection*: an instance of a (TCP) connection between two processes.
+* *entity*: a ceph entity instantiation, e.g. 'osd.0'.  each entity
+  has one or more unique entity_addr_t's by virtue of the 'nonce'
+  field, which is typically a pid or random value.
+* *session*: a stateful session between two entities in which message
+  exchange is ordered and lossless.  A session might span multiple
+  connections if there is an interruption (TCP connection disconnect).
+* *frame*: a discrete message sent between the peers.  Each frame
+  consists of a tag (type code), payload, and (if signing
+  or encryption is enabled) some other fields.  See below for the
+  structure.
+* *tag*: a type code associated with a frame.  The tag
+  determines the structure of the payload.
+
+Phases
+------
+
+A connection has four distinct phases:
+
+#. banner
+#. authentication frame exchange
+#. message flow handshake frame exchange
+#. message frame exchange
+
+Banner
+------
+
+Both the client and server, upon connecting, send a banner::
+
+  "ceph v2\n"
+  __le16 banner payload length
+  banner payload
+
+A banner payload has the form::
+
+  __le64 peer_supported_features
+  __le64 peer_required_features
+
+This is a new, distinct feature bit namespace (CEPH_MSGR2_*).
+Currently, only CEPH_MSGR2_FEATURE_REVISION_1 is defined. It is
+supported but not required, so that msgr2.0 and msgr2.1 peers
+can talk to each other.
+
+If the remote party advertises required features we don't support, we
+can disconnect.
+
+
+.. ditaa::
+
+           +---------+        +--------+
+           | Client  |        | Server |
+           +---------+        +--------+
+                | send banner     |
+                |----+       +----|
+                |    |       |    |
+                |    +-------+--->|
+                | send banner|    |
+                |<-----------+    |
+                |                 |
+
+Frame format
+------------
+
+After the banners are exchanged, all further communication happens
+in frames.  The exact format of the frame depends on the connection
+mode (msgr2.0-crc, msgr2.0-secure, msgr2.1-crc or msgr2.1-secure).
+All connections start in crc mode (either msgr2.0-crc or msgr2.1-crc,
+depending on peer_supported_features from the banner).
+
+Each frame has a 32-byte preamble::
+
+  __u8 tag
+  __u8 number of segments
+  {
+    __le32 segment length
+    __le16 segment alignment
+  } * 4
+  reserved (2 bytes)
+  __le32 preamble crc
+
+An empty frame has one empty segment.  A non-empty frame can have
+between one and four segments, all segments except the last may be
+empty.
+
+If there are less than four segments, unused (trailing) segment
+length and segment alignment fields are zeroed.
+
+The reserved bytes are zeroed.
+
+The preamble checksum is CRC32-C.  It covers everything up to
+itself (28 bytes) and is calculated and verified irrespective of
+the connection mode (i.e. even if the frame is encrypted).
+
+### msgr2.0-crc mode
+
+A msgr2.0-crc frame has the form::
+
+  preamble (32 bytes)
+  {
+    segment payload
+  } * number of segments
+  epilogue (17 bytes)
+
+where epilogue is::
+
+  __u8 late_flags
+  {
+    __le32 segment crc
+  } * 4
+
+late_flags is used for frame abortion.  After transmitting the
+preamble and the first segment, the sender can fill the remaining
+segments with zeros and set a flag to indicate that the receiver must
+drop the frame.  This allows the sender to avoid extra buffering
+when a frame that is being put on the wire is revoked (i.e. yanked
+out of the messenger): payload buffers can be unpinned and handed
+back to the user immediately, without making a copy or blocking
+until the whole frame is transmitted.  Currently this is used only
+by the kernel client, see ceph_msg_revoke().
+
+The segment checksum is CRC32-C.  For "used" empty segments, it is
+set to (__le32)-1.  For unused (trailing) segments, it is zeroed.
+
+The crcs are calculated just to protect against bit errors.
+No authenticity guarantees are provided, unlike in msgr1 which
+attempted to provide some authenticity guarantee by optionally
+signing segment lengths and crcs with the session key.
+
+Issues:
+
+1. As part of introducing a structure for a generic frame with
+   variable number of segments suitable for both control and
+   message frames, msgr2.0 moved the crc of the first segment of
+   the message frame (ceph_msg_header2) into the epilogue.
+
+   As a result, ceph_msg_header2 can no longer be safely
+   interpreted before the whole frame is read off the wire.
+   This is a regression from msgr1, because in order to scatter
+   the payload directly into user-provided buffers and thus avoid
+   extra buffering and copying when receiving message frames,
+   ceph_msg_header2 must be available in advance -- it stores
+   the transaction id which the user buffers are keyed on.
+   The implementation has to choose between forgoing this
+   optimization or acting on an unverified segment.
+
+2. late_flags is not covered by any crc.  Since it stores the
+   abort flag, a single bit flip can result in a completed frame
+   being dropped (causing the sender to hang waiting for a reply)
+   or, worse, in an aborted frame with garbage segment payloads
+   being dispatched.
+
+   This was the case with msgr1 and got carried over to msgr2.0.
+
+### msgr2.1-crc mode
+
+Differences from msgr2.0-crc:
+
+1. The crc of the first segment is stored at the end of the
+   first segment, not in the epilogue.  The epilogue stores up to
+   three crcs, not up to four.
+
+   If the first segment is empty, (__le32)-1 crc is not generated.
+
+2. The epilogue is generated only if the frame has more than one
+   segment (i.e. at least one of second to fourth segments is not
+   empty).  Rationale: If the frame has only one segment, it cannot
+   be aborted and there are no crcs to store in the epilogue.
+
+3. Unchecksummed late_flags is replaced with late_status which
+   builds in bit error detection by using a 4-bit nibble per flag
+   and two code words that are Hamming Distance = 4 apart (and not
+   all zeros or ones).  This comes at the expense of having only
+   one reserved flag, of course.
+
+Some example frames:
+
+* A 0+0+0+0 frame (empty, no epilogue)::
+
+    preamble (32 bytes)
+
+* A 20+0+0+0 frame (no epilogue)::
+
+    preamble (32 bytes)
+    segment1 payload (20 bytes)
+    __le32 segment1 crc
+
+* A 0+70+0+0 frame::
+
+    preamble (32 bytes)
+    segment2 payload (70 bytes)
+    epilogue (13 bytes)
+
+* A 20+70+0+350 frame::
+
+    preamble (32 bytes)
+    segment1 payload (20 bytes)
+    __le32 segment1 crc
+    segment2 payload (70 bytes)
+    segment4 payload (350 bytes)
+    epilogue (13 bytes)
+
+where epilogue is::
+
+  __u8 late_status
+  {
+    __le32 segment crc
+  } * 3
+
+Hello
+-----
+
+* TAG_HELLO: client->server and server->client::
+
+    __u8 entity_type
+    entity_addr_t peer_socket_address
+
+  - We immediately share our entity type and the address of the peer (which can be useful
+    for detecting our effective IP address, especially in the presence of NAT).
+
+
+Authentication
+--------------
+
+* TAG_AUTH_REQUEST: client->server::
+
+    __le32 method;  // CEPH_AUTH_{NONE, CEPHX, ...}
+    __le32 num_preferred_modes;
+    list<__le32> mode  // CEPH_CON_MODE_*
+    method specific payload
+
+* TAG_AUTH_BAD_METHOD server -> client: reject client-selected auth method::
+
+    __le32 method
+    __le32 negative error result code
+    __le32 num_methods
+    list<__le32> allowed_methods // CEPH_AUTH_{NONE, CEPHX, ...}
+    __le32 num_modes
+    list<__le32> allowed_modes   // CEPH_CON_MODE_*
+
+  - Returns the attempted auth method, and error code (-EOPNOTSUPP if
+    the method is unsupported), and the list of allowed authentication
+    methods.
+
+* TAG_AUTH_REPLY_MORE: server->client::
+
+    __le32 len;
+    method specific payload
+
+* TAG_AUTH_REQUEST_MORE: client->server::
+
+    __le32 len;
+    method specific payload
+
+* TAG_AUTH_DONE: (server->client)::
+
+    __le64 global_id
+    __le32 connection mode // CEPH_CON_MODE_*
+    method specific payload
+
+  - The server is the one to decide authentication has completed and what
+    the final connection mode will be.
+
+
+Example of authentication phase interaction when the client uses an
+allowed authentication method:
+
+.. ditaa::
+
+           +---------+        +--------+
+           | Client  |        | Server |
+           +---------+        +--------+
+                | auth request    |
+                |---------------->|
+                |<----------------|
+                |        auth more|
+                |                 |
+                |auth more        |
+                |---------------->|
+                |<----------------|
+                |        auth done|
+
+
+Example of authentication phase interaction when the client uses a forbidden
+authentication method as the first attempt:
+
+.. ditaa::
+
+           +---------+        +--------+
+           | Client  |        | Server |
+           +---------+        +--------+
+                | auth request    |
+                |---------------->|
+                |<----------------|
+                |   bad method    |
+                |                 |
+                | auth request    |
+                |---------------->|
+                |<----------------|
+                |        auth more|
+                |                 |
+                | auth more       |
+                |---------------->|
+                |<----------------|
+                |        auth done|
+
+
+Post-auth frame format
+----------------------
+
+Depending on the negotiated connection mode from TAG_AUTH_DONE, the
+connection either stays in crc mode or switches to the corresponding
+secure mode (msgr2.0-secure or msgr2.1-secure).
+
+### msgr2.0-secure mode
+
+A msgr2.0-secure frame has the form::
+
+  {
+    preamble (32 bytes)
+    {
+      segment payload
+      zero padding (out to 16 bytes)
+    } * number of segments
+    epilogue (16 bytes)
+  } ^ AES-128-GCM cipher
+  auth tag (16 bytes)
+
+where epilogue is::
+
+    __u8 late_flags
+    zero padding (15 bytes)
+
+late_flags has the same meaning as in msgr2.0-crc mode.
+
+Each segment and the epilogue are zero padded out to 16 bytes.
+Technically, GCM doesn't require any padding because Counter mode
+(the C in GCM) essentially turns a block cipher into a stream cipher.
+But, if the overall input length is not a multiple of 16 bytes, some
+implicit zero padding would occur internally because GHASH function
+used by GCM for generating auth tags only works on 16-byte blocks.
+
+Issues:
+
+1. The sender encrypts the whole frame using a single nonce
+   and generating a single auth tag.  Because segment lengths are
+   stored in the preamble, the receiver has no choice but to decrypt
+   and interpret the preamble without verifying the auth tag -- it
+   can't even tell how much to read off the wire to get the auth tag
+   otherwise!  This creates a decryption oracle, which, in conjunction
+   with Counter mode malleability, could lead to recovery of sensitive
+   information.
+
+   This issue extends to the first segment of the message frame as
+   well.  As in msgr2.0-crc mode, ceph_msg_header2 cannot be safely
+   interpreted before the whole frame is read off the wire.
+
+2. Deterministic nonce construction with a 4-byte counter field
+   followed by an 8-byte fixed field is used.  The initial values are
+   taken from the connection secret -- a random byte string generated
+   during the authentication phase.  Because the counter field is
+   only four bytes long, it can wrap and then repeat in under a day,
+   leading to GCM nonce reuse and therefore a potential complete
+   loss of both authenticity and confidentiality for the connection.
+   This was addressed by disconnecting before the counter repeats
+   (CVE-2020-1759).
+
+### msgr2.1-secure mode
+
+Differences from msgr2.0-secure:
+
+1. The preamble, the first segment and the rest of the frame are
+   encrypted separately, using separate nonces and generating
+   separate auth tags.  This gets rid of unverified plaintext use
+   and keeps msgr2.1-secure mode close to msgr2.1-crc mode, allowing
+   the implementation to receive message frames in a similar fashion
+   (little to no buffering, same scatter/gather logic, etc).
+
+   In order to reduce the number of en/decryption operations per
+   frame, the preamble is grown by a fixed size inline buffer (48
+   bytes) that the first segment is inlined into, either fully or
+   partially.  The preamble auth tag covers both the preamble and the
+   inline buffer, so if the first segment is small enough to be fully
+   inlined, it becomes available after a single decryption operation.
+
+2. As in msgr2.1-crc mode, the epilogue is generated only if the
+   frame has more than one segment.  The rationale is even stronger,
+   as it would require an extra en/decryption operation.
+
+3. For consistency with msgr2.1-crc mode, late_flags is replaced
+   with late_status (the built-in bit error detection isn't really
+   needed in secure mode).
+
+4. In accordance with `NIST Recommendation for GCM`_, deterministic
+   nonce construction with a 4-byte fixed field followed by an 8-byte
+   counter field is used.  An 8-byte counter field should never repeat
+   but the nonce reuse protection put in place for msgr2.0-secure mode
+   is still there.
+
+   The initial values are the same as in msgr2.0-secure mode.
+
+   .. _`NIST Recommendation for GCM`: https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38d.pdf
+
+As in msgr2.0-secure mode, each segment is zero padded out to
+16 bytes.  If the first segment is fully inlined, its padding goes
+to the inline buffer.  Otherwise, the padding is on the remainder.
+The corollary to this is that the inline buffer is consumed in
+16-byte chunks.
+
+The unused portion of the inline buffer is zeroed.
+
+Some example frames:
+
+* A 0+0+0+0 frame (empty, nothing to inline, no epilogue)::
+
+    {
+      preamble (32 bytes)
+      zero padding (48 bytes)
+    } ^ AES-128-GCM cipher
+    auth tag (16 bytes)
+
+* A 20+0+0+0 frame (first segment fully inlined, no epilogue)::
+
+    {
+      preamble (32 bytes)
+      segment1 payload (20 bytes)
+      zero padding (28 bytes)
+    } ^ AES-128-GCM cipher
+    auth tag (16 bytes)
+
+* A 0+70+0+0 frame (nothing to inline)::
+
+    {
+      preamble (32 bytes)
+      zero padding (48 bytes)
+    } ^ AES-128-GCM cipher
+    auth tag (16 bytes)
+    {
+      segment2 payload (70 bytes)
+      zero padding (10 bytes)
+      epilogue (16 bytes)
+    } ^ AES-128-GCM cipher
+    auth tag (16 bytes)
+
+* A 20+70+0+350 frame (first segment fully inlined)::
+
+    {
+      preamble (32 bytes)
+      segment1 payload (20 bytes)
+      zero padding (28 bytes)
+    } ^ AES-128-GCM cipher
+    auth tag (16 bytes)
+    {
+      segment2 payload (70 bytes)
+      zero padding (10 bytes)
+      segment4 payload (350 bytes)
+      zero padding (2 bytes)
+      epilogue (16 bytes)
+    } ^ AES-128-GCM cipher
+    auth tag (16 bytes)
+
+* A 105+0+0+0 frame (first segment partially inlined, no epilogue)::
+
+    {
+      preamble (32 bytes)
+      segment1 payload (48 bytes)
+    } ^ AES-128-GCM cipher
+    auth tag (16 bytes)
+    {
+      segment1 payload remainder (57 bytes)
+      zero padding (7 bytes)
+    } ^ AES-128-GCM cipher
+    auth tag (16 bytes)
+
+* A 105+70+0+350 frame (first segment partially inlined)::
+
+    {
+      preamble (32 bytes)
+      segment1 payload (48 bytes)
+    } ^ AES-128-GCM cipher
+    auth tag (16 bytes)
+    {
+      segment1 payload remainder (57 bytes)
+      zero padding (7 bytes)
+    } ^ AES-128-GCM cipher
+    auth tag (16 bytes)
+    {
+      segment2 payload (70 bytes)
+      zero padding (10 bytes)
+      segment4 payload (350 bytes)
+      zero padding (2 bytes)
+      epilogue (16 bytes)
+    } ^ AES-128-GCM cipher
+    auth tag (16 bytes)
+
+where epilogue is::
+
+    __u8 late_status
+    zero padding (15 bytes)
+
+late_status has the same meaning as in msgr2.1-crc mode.
+
+Message flow handshake
+----------------------
+
+In this phase the peers identify each other and (if desired) reconnect to
+an established session.
+
+* TAG_CLIENT_IDENT (client->server): identify ourselves::
+
+    __le32 num_addrs
+    entity_addrvec_t*num_addrs entity addrs
+    entity_addr_t target entity addr
+    __le64 gid (numeric part of osd.0, client.123456, ...)
+    __le64 global_seq
+    __le64 features supported (CEPH_FEATURE_* bitmask)
+    __le64 features required (CEPH_FEATURE_* bitmask)
+    __le64 flags (CEPH_MSG_CONNECT_* bitmask)
+    __le64 cookie
+
+  - client will send first, server will reply with same.  if this is a
+    new session, the client and server can proceed to the message exchange.
+  - the target addr is who the client is trying to connect *to*, so
+    that the server side can close the connection if the client is
+    talking to the wrong daemon.
+  - type.gid (entity_name_t) is set here, by combinging the type shared in the hello
+    frame with the gid here.  this means we don't need it
+    in the header of every message.  it also means that we can't send
+    messages "from" other entity_name_t's.  the current
+    implementations set this at the top of _send_message etc so this
+    shouldn't break any existing functionality.  implementation will
+    likely want to mask this against what the authenticated credential
+    allows.
+  - cookie is the client coookie used to identify a session, and can be used
+    to reconnect to an existing session.
+  - we've dropped the 'protocol_version' field from msgr1
+
+* TAG_IDENT_MISSING_FEATURES (server->client): complain about a TAG_IDENT
+  with too few features::
+
+    __le64 features we require that the peer didn't advertise
+
+* TAG_SERVER_IDENT (server->client): accept client ident and identify server::
+
+    __le32 num_addrs
+    entity_addrvec_t*num_addrs entity addrs
+    __le64 gid (numeric part of osd.0, client.123456, ...)
+    __le64 global_seq
+    __le64 features supported (CEPH_FEATURE_* bitmask)
+    __le64 features required (CEPH_FEATURE_* bitmask)
+    __le64 flags (CEPH_MSG_CONNECT_* bitmask)
+    __le64 cookie
+
+  - The server cookie can be used by the client if it is later disconnected
+    and wants to reconnect and resume the session.
+
+* TAG_RECONNECT (client->server): reconnect to an established session::
+
+    __le32 num_addrs
+    entity_addr_t * num_addrs
+    __le64 client_cookie
+    __le64 server_cookie
+    __le64 global_seq
+    __le64 connect_seq
+    __le64 msg_seq (the last msg seq received)
+
+* TAG_RECONNECT_OK (server->client): acknowledge a reconnect attempt::
+
+    __le64 msg_seq (last msg seq received)
+
+  - once the client receives this, the client can proceed to message exchange.
+  - once the server sends this, the server can proceed to message exchange.
+
+* TAG_RECONNECT_RETRY_SESSION (server only): fail reconnect due to stale connect_seq
+
+* TAG_RECONNECT_RETRY_GLOBAL (server only): fail reconnect due to stale global_seq
+
+* TAG_RECONNECT_WAIT (server only): fail reconnect due to connect race.
+
+  - Indicates that the server is already connecting to the client, and
+    that direction should win the race.  The client should wait for that
+    connection to complete.
+
+* TAG_RESET_SESSION (server only): ask client to reset session::
+
+      __u8 full
+
+  - full flag indicates whether peer should do a full reset, i.e., drop
+    message queue.
+
+
+Example of failure scenarios:
+
+* First client's client_ident message is lost, and then client reconnects.
+
+.. ditaa::
+
+           +---------+           +--------+
+           | Client  |           | Server |
+           +---------+           +--------+
+                |                     |
+    c_cookie(a) | client_ident(a)     |
+                |-------------X       |
+                |                     |
+                | client_ident(a)     |
+                |-------------------->|
+                |<--------------------|
+                |     server_ident(b) | s_cookie(b)
+                |                     |
+                | session established |
+                |                     |
+
+
+* Server's server_ident message is lost, and then client reconnects.
+
+.. ditaa::
+
+           +---------+           +--------+
+           | Client  |           | Server |
+           +---------+           +--------+
+                |                     |
+    c_cookie(a) | client_ident(a)     |
+                |-------------------->|
+                |        X------------|
+                |     server_ident(b) | s_cookie(b)
+                |                     |
+                |                     |
+                | client_ident(a)     |
+                |-------------------->|
+                |<--------------------|
+                |     server_ident(c) | s_cookie(c)
+                |                     |
+                | session established |
+                |                     |
+
+
+* Server's server_ident message is lost, and then server reconnects.
+
+.. ditaa::
+
+           +---------+           +--------+
+           | Client  |           | Server |
+           +---------+           +--------+
+                |                     |
+    c_cookie(a) | client_ident(a)     |
+                |-------------------->|
+                |        X------------|
+                |     server_ident(b) | s_cookie(b)
+                |                     |
+                |                     |
+                |     reconnect(a, b) |
+                |<--------------------|
+                |-------------------->|
+                | reset_session(F)    |
+                |                     |
+                |     client_ident(a) | c_cookie(a)
+                |<--------------------|
+                |-------------------->|
+    s_cookie(c) | server_ident(c)     |
+                |                     |
+
+
+* Connection failure after session is established, and then client reconnects.
+
+.. ditaa::
+
+           +---------+           +--------+
+           | Client  |           | Server |
+           +---------+           +--------+
+                |                     |
+    c_cookie(a) | session established | s_cookie(b)
+                |<------------------->|
+                |        X------------|
+                |                     |
+                | reconnect(a, b)     |
+                |-------------------->|
+                |<--------------------|
+                |        reconnect_ok |
+                |                     |
+
+
+* Connection failure after session is established because server reset,
+  and then client reconnects.
+
+.. ditaa::
+
+           +---------+           +--------+
+           | Client  |           | Server |
+           +---------+           +--------+
+                |                     |
+    c_cookie(a) | session established | s_cookie(b)
+                |<------------------->|
+                |        X------------| reset
+                |                     |
+                | reconnect(a, b)     |
+                |-------------------->|
+                |<--------------------|
+                |  reset_session(RC*) |
+                |                     |
+    c_cookie(c) | client_ident(c)     |
+                |-------------------->|
+                |<--------------------|
+                |     server_ident(d) | s_cookie(d)
+                |                     |
+
+RC* means that the reset session full flag depends on the policy.resetcheck
+of the connection.
+
+
+* Connection failure after session is established because client reset,
+  and then client reconnects.
+
+.. ditaa::
+
+           +---------+           +--------+
+           | Client  |           | Server |
+           +---------+           +--------+
+                |                     |
+    c_cookie(a) | session established | s_cookie(b)
+                |<------------------->|
+          reset |        X------------|
+                |                     |
+    c_cookie(c) | client_ident(c)     |
+                |-------------------->|
+                |<--------------------| reset if policy.resetcheck
+                |     server_ident(d) | s_cookie(d)
+                |                     |
+
+
+Message exchange
+----------------
+
+Once a session is established, we can exchange messages.
+
+* TAG_MSG: a message::
+
+    ceph_msg_header2
+    front
+    middle
+    data_pre_padding
+    data
+
+  - The ceph_msg_header2 is modified from ceph_msg_header:
+      * include an ack_seq.  This avoids the need for a TAG_ACK
+        message most of the time.
+      * remove the src field, which we now get from the message flow
+        handshake (TAG_IDENT).
+      * specifies the data_pre_padding length, which can be used to
+        adjust the alignment of the data payload.  (NOTE: is this is
+        useful?)
+
+* TAG_ACK: acknowledge receipt of message(s)::
+
+    __le64 seq
+
+  - This is only used for stateful sessions.
+
+* TAG_KEEPALIVE2: check for connection liveness::
+
+    ceph_timespec stamp
+
+  - Time stamp is local to sender.
+
+* TAG_KEEPALIVE2_ACK: reply to a keepalive2::
+
+    ceph_timestamp stamp
+
+  - Time stamp is from the TAG_KEEPALIVE2 we are responding to.
+
+* TAG_CLOSE: terminate a connection
+
+  Indicates that a connection should be terminated. This is equivalent
+  to a hangup or reset (i.e., should trigger ms_handle_reset).  It
+  isn't strictly necessary or useful as we could just disconnect the
+  TCP connection.
+
+
+Example of protocol interaction (WIP)
+_____________________________________
+
+
+.. ditaa::
+
+           +---------+        +--------+
+           | Client  |        | Server |
+           +---------+        +--------+
+                | send banner       |
+                |----+       +------|
+                |    |       |      |
+                |    +-------+----->|
+                | send banner|      |
+                |<-----------+      |
+                |                   |
+                |  send new stream  |
+                |------------------>|
+                | auth request      |
+                |------------------>|
+                |<------------------|
+                |   bad method      |
+                |                   |
+                | auth request      |
+                |------------------>|
+                |<------------------|
+                |         auth more |
+                |                   |
+                | auth more         |
+                |------------------>|
+                |<------------------|
+                |         auth done |
+                |                   |
+
+
diff --git a/doc/dev/network-encoding.rst b/doc/dev/network-encoding.rst
new file mode 100644
index 000000000..d59b0ee9e
--- /dev/null
+++ b/doc/dev/network-encoding.rst
@@ -0,0 +1,214 @@
+==================
+ Network Encoding
+==================
+
+This describes the encoding used to serialize data.  It doesn't cover specific
+objects/messages but focuses on the base types.
+
+The types are not self documenting in any way.  They can not be decoded unless
+you know what they are.
+
+Conventions
+===========
+
+Integers
+--------
+
+The integer types used will be named ``{signed}{size}{endian}``.  For example
+``u16le`` is an unsigned 16 bit integer encoded in little endian byte order
+while ``s64be`` is a signed 64 bit integer in big endian.  Additionally ``u8``
+and ``s8`` will represent signed and unsigned bytes respectively.  Signed
+integers use two's complement encoding.
+
+Complex Types
+-------------
+
+This document will use a c-like syntax for describing structures.  The
+structure represents the data that will go over the wire.  There will be no
+padding between the elements and the elements will be sent in the order they
+appear.  For example::
+
+	struct foo {
+		u8    tag;
+		u32le data;
+	}
+
+When encoding the values ``0x05`` and ``0x12345678`` respectively will appear on
+the wire as ``05 78 56 34 12``.
+
+Variable Arrays
+---------------
+
+Unlike c, length arrays can be used anywhere in structures and will be inline in
+the protocol.  Furthermore the length may be described using an earlier item in
+the structure.
+
+::
+	
+	struct blob {
+		u32le size;
+		u8    data[size];
+		u32le checksum;
+	}
+
+This structure is encoded as a 32 bit size, followed by ``size`` data bytes,
+then a 32 bit checksum.
+
+Primitive Aliases
+-----------------
+
+These types are just aliases for primitive types.
+
+::
+	
+	// From /src/include/types.h
+	
+	typedef u32le epoch_t;
+	typedef u32le ceph_seq_t;
+	typedef u64le ceph_tid_t;
+	typedef u64le version_t;
+
+
+Structures
+==========
+
+These are the way structures are encoded.  Note that these structures don't
+actually exist in the source but are the way that different types are encoded.
+
+Optional
+--------
+
+Optionals are represented as a presence byte, followed by the item if it exists.
+
+::
+	
+	struct ceph_optional<T> {
+		u8 present;
+		T  element[present? 1 : 0]; // Only if present is non-zero.
+	}
+
+Optionals are used to encode ``boost::optional``.
+
+Pair
+----
+
+Pairs are simply the first item followed by the second.
+
+::
+	
+	struct ceph_pair<A,B> {
+		A a;
+		B b;
+	}
+
+Pairs are used to encode ``std::pair``.
+
+Triple
+------
+
+Triples are simply the tree elements one after another.
+
+::
+	
+	struct ceph_triple<A,B,C> {
+		A a;
+		B b;
+		C c;
+	}
+
+Triples are used to encode ``ceph::triple``.
+
+
+List
+----
+
+Lists are represented as an element count followed by that many elements.
+
+::
+	
+	struct ceph_list<T> {
+		u32le length;
+		T     elements[length];
+	}
+
+.. note::
+	The size of the elements in the list are not necessarily uniform.
+
+Lists are used to encode ``std::list``, ``std::vector``, ``std::deque``,
+``std::set`` and ``ceph::unordered_set``.
+
+Blob
+----
+
+A Blob is simply a list of bytes.
+
+::
+	
+	struct ceph_string {
+		ceph_list<u8>;
+	}
+	
+	// AKA
+	
+	struct ceph_string {
+		u32le size;
+		u8    data[size];
+	}
+
+Blobs are used to encode ``std::string``, ``const char *`` and ``bufferlist``.
+
+.. note::
+	The content of a Blob is arbitrary binary data.
+
+Map
+---
+
+Maps are a list of pairs.
+
+::
+	
+	struct ceph_map<K,V> {
+		ceph_list<ceph_pair<K,V>>;
+	}
+	
+	// AKA
+	
+	struct ceph_map<K,V> {
+		u32le length;
+		ceph_pair<K,V> entries[length];
+	}
+
+Maps are used to encode ``std::map``, ``std::multimap`` and
+``ceph::unordered_map``.
+
+Complex Types
+=============
+
+These aren't hard to find in the source but the common ones are listed here for
+convenience.
+
+utime_t
+-------
+
+::
+	
+	// From /src/include/utime.h
+	struct utime_t {
+		u32le tv_sec;  // Seconds since epoch.
+		u32le tv_nsec; // Nanoseconds since the last second.
+	}
+
+ceph_entity_name
+----------------
+
+::
+	
+	// From /src/include/msgr.h
+	struct ceph_entity_name {
+		u8    type; // CEPH_ENTITY_TYPE_*
+		u64le num;
+	}
+	
+	// CEPH_ENTITY_TYPE_* defined in /src/include/msgr.h
+
+.. vi: textwidth=80 noexpandtab
diff --git a/doc/dev/network-protocol.rst b/doc/dev/network-protocol.rst
new file mode 100644
index 000000000..f6fb1738d
--- /dev/null
+++ b/doc/dev/network-protocol.rst
@@ -0,0 +1,197 @@
+==================
+ Network Protocol
+==================
+
+This file describes the network protocol used by Ceph.  In order to understand
+the way the structures are defined it is recommended to read the introduction
+of :doc:`/dev/network-encoding` first.
+
+Hello
+=====
+
+The protocol starts with a handshake that confirms that both nodes are talking
+ceph and shares some basic information.
+
+Banner
+------
+
+The first action is the server sending banner to the client.  The banner is
+defined in ``CEPH_BANNER`` from ``src/include/msgr.h``.  This is followed by
+the server's then client's address each encoded as a ``entity_addr_t``.
+
+Once the client verifies that the servers banner matches its own it replies with
+its banner and its address.
+
+Connect
+-------
+
+Once the banners have been verified and the addresses exchanged the connection
+negotiation begins.  First the client sends a ``ceph_msg_connect`` structure
+with its information.
+
+::
+	
+	// From src/include/msgr.h
+	struct ceph_msg_connect {
+		u64le features;            // Supported features (CEPH_FEATURE_*)
+		u32le host_type;           // CEPH_ENTITY_TYPE_*
+		u32le global_seq;          // Number of connections initiated by this host.
+		u32le connect_seq;         // Number of connections initiated in this session.
+		u32le protocol_version;
+		u32le authorizer_protocol;
+		u32le authorizer_len;
+		u8    flags;               // CEPH_MSG_CONNECT_*
+		u8    authorizer[authorizer_len];
+	}
+
+Connect Reply
+-------------
+
+Once the connect has been sent the connection has effectively been opened,
+however the first message the server sends must be a connect reply message.
+
+::
+	
+	struct ceph_msg_connect_reply {
+		u8    tag; // Tag indicating response code.
+		u64le features;
+		u32le global_seq;
+		u32le connect_seq;
+		u32le protocol_version;
+		u32le authorizer_len;
+		u8    flags;
+		u8    authorizer[authorizer_len];
+	}
+
+MSGR Protocol
+=============
+
+This is a low level protocol over which messages are delivered.  The messages
+at this level consist of a tag byte, identifying the type of message, followed
+by the message data.
+
+::
+	
+	// Virtual structure.
+	struct {
+		u8 tag; // CEPH_MSGR_TAG_*
+		u8 data[]; // Length depends on tag and data.
+	}
+
+The length of ``data`` is determined by the tag byte and depending on the
+message type via information in the ``data`` array itself.
+
+.. note::
+	There is no way to determine the length of the message if you do not
+	understand the type of message.
+
+The message tags are defined in ``src/include/msgr.h`` and the current ones
+are listed below along with the data they include.  Note that the defined
+structures don't exist in the source and are merely for representing the
+protocol.
+
+CEPH_MSGR_TAG_CLOSE (0x06)
+--------------------------
+
+::
+	
+	struct ceph_msgr_close {
+		u8 tag = 0x06;
+		u8 data[0]; // No data.
+	}
+
+The close message indicates that the connection is being closed.
+
+CEPH_MSGR_TAG_MSG (0x07)
+------------------------
+
+::
+	
+	struct ceph_msgr_msg {
+		u8 tag = 0x07;
+		ceph_msg_header header;
+		u8 front [header.front_len ];
+		u8 middle[header.middle_len];
+		u8 data  [header.data_len  ];
+		ceph_msg_footer footer;
+	}
+	
+	// From src/include/msgr.h
+	struct ceph_msg_header {
+		u64le seq;       // Sequence number.
+		u64le tid;       // Transaction ID.
+		u16le type;      // Message type (CEPH_MSG_* or MSG_*).
+		u16le priority;  // Priority (higher is more important).
+		u16le version;   // Version of message encoding.
+		
+		u32le front_len;  // The size of the front section.
+		u32le middle_len; // The size of the middle section.
+		u32le data_len;   // The size of the data section.
+		u16le data_off;   // The way data should be aligned by the receiver.
+		
+		ceph_entity_name src; // Information about the sender.
+		
+		u16le compat_version; // Oldest compatible encoding version.
+		u16le reserved;       // Unused.
+		u32le crc;            // CRC of header.
+	}
+	
+	// From src/include/msgr.h
+	struct ceph_msg_footer {
+		u32le front_crc;  // Checksums of the various sections.
+		u32le middle_crc; //
+		u32le data_crc;   //
+		u64le sig; // Crypographic signature.
+		u8    flags;
+	}
+
+Messages are the business logic of Ceph.  They are what is used to send data and
+requests between nodes.  The message header contains the length of the message
+so unknown messages can be handled gracefully.
+
+There are two names for the message type constants ``CEPH_MSG_*`` and ``MSG_*``.
+The only difference between the two is that the first are considered "public"
+while the second is for internal use only.  There is no protocol-level
+difference.
+
+CEPH_MSGR_TAG_ACK (0x08)
+------------------------
+
+::
+	
+	struct ceph_msgr_ack {
+		u8    tag = 0x08;
+		u64le seq; // The sequence number of the message being acknowledged.
+	}
+
+CEPH_MSGR_TAG_KEEPALIVE (0x09)
+------------------------------
+
+::
+	
+	struct ceph_msgr_keepalive {
+		u8 tag = 0x09;
+		u8 data[0]; // No data.
+	}
+
+CEPH_MSGR_TAG_KEEPALIVE2 (0x0E)
+-------------------------------
+
+::
+	
+	struct ceph_msgr_keepalive2 {
+		u8      tag = 0x0E;
+		utime_t timestamp;
+	}
+
+CEPH_MSGR_TAG_KEEPALIVE2_ACK (0x0F)
+-----------------------------------
+
+::
+	
+	struct ceph_msgr_keepalive2_ack {
+		u8      tag = 0x0F;
+		utime_t timestamp;
+	}
+
+.. vi: textwidth=80 noexpandtab
diff --git a/doc/dev/object-store.rst b/doc/dev/object-store.rst
new file mode 100644
index 000000000..2d9a7d8fa
--- /dev/null
+++ b/doc/dev/object-store.rst
@@ -0,0 +1,67 @@
+====================================
+ Object Store Architecture Overview
+====================================
+
+.. graphviz::
+
+   digraph object_store {
+    size="7,7";
+    node [color=lightblue2, style=filled, fontname="Serif"];
+
+    "testrados" -> "librados"
+    "testradospp" -> "librados"
+
+    "rbd" -> "librados"
+
+    "radostool" -> "librados"
+
+    "radosgw-admin" -> "radosgw"
+
+    "radosgw" -> "librados"
+
+    "radosacl" -> "librados"
+
+    "librados" -> "objecter"
+
+    "ObjectCacher" -> "Filer"
+
+    "dumpjournal" -> "Journaler"
+
+    "Journaler" -> "Filer"
+
+    "SyntheticClient" -> "Filer"
+    "SyntheticClient" -> "objecter"
+
+    "Filer" -> "objecter"
+
+    "objecter" -> "OSDMap"
+
+    "ceph-osd" -> "PG"
+    "ceph-osd" -> "ObjectStore"
+
+    "crushtool" -> "CrushWrapper"
+
+    "OSDMap" -> "CrushWrapper"
+
+    "OSDMapTool" -> "OSDMap"
+
+    "PG" -> "PrimaryLogPG"
+    "PG" -> "ObjectStore"
+    "PG" -> "OSDMap"
+
+    "PrimaryLogPG" -> "ObjectStore"
+    "PrimaryLogPG" -> "OSDMap"
+
+    "ObjectStore" -> "FileStore"
+    "ObjectStore" -> "BlueStore"
+
+    "BlueStore" -> "rocksdb"
+
+    "FileStore" -> "xfs"
+    "FileStore" -> "btrfs"
+    "FileStore" -> "ext4"
+  }
+
+
+.. todo:: write more here
+
diff --git a/doc/dev/osd-class-path.rst b/doc/dev/osd-class-path.rst
new file mode 100644
index 000000000..d0e54acac
--- /dev/null
+++ b/doc/dev/osd-class-path.rst
@@ -0,0 +1,16 @@
+=======================
+ OSD class path issues
+=======================
+
+::
+
+  2011-12-05 17:41:00.994075 7ffe8b5c3760 librbd: failed to assign a block name for image
+  create error: error 5: Input/output error
+
+This usually happens because your OSDs can't find ``cls_rbd.so``. They
+search for it in ``osd_class_dir``, which may not be set correctly by
+default (http://tracker.ceph.com/issues/1722).
+
+Most likely it's looking in ``/usr/lib/rados-classes`` instead of
+``/usr/lib64/rados-classes`` - change ``osd_class_dir`` in your
+``ceph.conf`` and restart the OSDs to fix it.
diff --git a/doc/dev/osd_internals/async_recovery.rst b/doc/dev/osd_internals/async_recovery.rst
new file mode 100644
index 000000000..ab0a036f1
--- /dev/null
+++ b/doc/dev/osd_internals/async_recovery.rst
@@ -0,0 +1,53 @@
+=====================
+Asynchronous Recovery
+=====================
+
+Ceph Placement Groups (PGs) maintain a log of write transactions to
+facilitate speedy recovery of data. During recovery, each of these PG logs
+is used to determine which content in each OSD is missing or outdated.
+This obviates the need to scan all RADOS objects.
+See :ref:`Log Based PG <log-based-pg>` for more details on this process.
+
+Prior to the Nautilus release this recovery process was synchronous: it
+blocked writes to a RADOS object until it was recovered. In contrast,
+backfill could allow writes to proceed (assuming enough up-to-date replicas
+were available) by temporarily assigning a different acting set, and
+backfilling an OSD outside of the acting set. In some circumstances
+this ends up being significantly better for availability, e.g. if the
+PG log contains 3000 writes to disjoint objects.  When the PG log contains
+thousands of entries, it could actually be faster (though not as safe) to
+trade backfill for recovery by deleting and redeploying the containing
+OSD than to iterate through the PG log.  Recovering several megabytes
+of RADOS object data (or even worse, several megabytes of omap keys,
+notably RGW bucket indexes) can drastically increase latency for a small
+update, and combined with requests spread across many degraded objects
+it is a recipe for slow requests.
+
+To avoid this we can perform recovery in the background on an OSD
+out-of-band of the live acting set, similar to backfill, but still using
+the PG log to determine what needs to be done. This is known as *asynchronous
+recovery*.
+
+The threashold for performing asynchronous recovery instead of synchronous
+recovery is not a clear-cut. There are a few criteria which
+need to be met for asynchronous recovery:
+
+* Try to keep ``min_size`` replicas available
+* Use the approximate magnitude of the difference in length of
+  logs combined with historical missing objects to estimate the cost of
+  recovery
+* Use the parameter ``osd_async_recovery_min_cost`` to determine
+  when asynchronous recovery is appropriate
+
+With the existing peering process, when we choose the acting set we
+have not fetched the PG log from each peer; we have only the bounds of
+it and other metadata from their ``pg_info_t``. It would be more expensive
+to fetch and examine every log at this point, so we only consider an
+approximate check for log length for now. In Nautilus, we improved
+the accounting of missing objects, so post-Nautilus this information
+is also used to determine the cost of recovery.
+
+While async recovery is occurring, writes to members of the acting set
+may proceed, but we need to send their log entries to the async
+recovery targets (just like we do for backfill OSDs) so that they
+can completely catch up.
diff --git a/doc/dev/osd_internals/backfill_reservation.rst b/doc/dev/osd_internals/backfill_reservation.rst
new file mode 100644
index 000000000..3c380dcf6
--- /dev/null
+++ b/doc/dev/osd_internals/backfill_reservation.rst
@@ -0,0 +1,93 @@
+====================
+Backfill Reservation
+====================
+
+When a new OSD joins a cluster all PGs with it in their acting sets must
+eventually backfill.  If all of these backfills happen simultaneously
+they will present excessive load on the OSD:  the "thundering herd"
+effect.
+
+The ``osd_max_backfills`` tunable limits the number of outgoing or
+incoming backfills that are active on a given OSD. Note that this limit is
+applied separately to incoming and to outgoing backfill operations.
+Thus there can be as many as ``osd_max_backfills * 2`` backfill operations
+in flight on each OSD.  This subtlety is often missed, and Ceph
+operators can be puzzled as to why more ops are observed than expected.
+
+Each ``OSDService`` now has two AsyncReserver instances: one for backfills going
+from the OSD (``local_reserver``) and one for backfills going to the OSD
+(``remote_reserver``).  An ``AsyncReserver`` (``common/AsyncReserver.h``)
+manages a queue by priority of waiting items and a set of current reservation
+holders.  When a slot frees up, the ``AsyncReserver`` queues the ``Context*``
+associated with the next item on the highest priority queue in the finisher
+provided to the constructor.
+
+For a primary to initiate a backfill it must first obtain a reservation from
+its own ``local_reserver``.  Then it must obtain a reservation from the backfill
+target's ``remote_reserver`` via a ``MBackfillReserve`` message. This process is
+managed by sub-states of ``Active`` and ``ReplicaActive`` (see the sub-states
+of ``Active`` in PG.h).  The reservations are dropped either on the ``Backfilled``
+event, which is sent on the primary before calling ``recovery_complete``
+and on the replica on receipt of the ``BackfillComplete`` progress message),
+or upon leaving ``Active`` or ``ReplicaActive``.
+
+It's important to always grab the local reservation before the remote
+reservation in order to prevent a circular dependency.
+
+We minimize the risk of data loss by prioritizing the order in
+which PGs are recovered.  Admins can override the default order by using
+``force-recovery`` or ``force-backfill``. A ``force-recovery`` with op
+priority ``255`` will start before a ``force-backfill`` op at priority ``254``.
+
+If recovery is needed because a PG is below ``min_size`` a base priority of
+``220`` is used. This is incremented by the number of OSDs short of the pool's
+``min_size`` as well as a value relative to the pool's ``recovery_priority``.
+The resultant priority is capped at ``253`` so that it does not confound forced
+ops as described above. Under ordinary circumstances a recovery op is
+prioritized at ``180`` plus a value relative to the pool's ``recovery_priority``.
+The resultant priority is capped at ``219``.
+
+If backfill is needed because the number of acting OSDs is less than
+the pool's ``min_size``, a priority of ``220`` is used.  The number of OSDs
+short of the pool's ``min_size`` is added as well as a value relative to
+the pool's ``recovery_priority``.  The total priority is limited to ``253``.
+
+If backfill is needed because a PG is undersized,
+a priority of ``140`` is used.  The number of OSDs below the size of the pool is
+added as well as a value relative to the pool's ``recovery_priority``.  The
+resultant priority is capped at ``179``.  If a backfill op is
+needed because a PG is degraded, a priority of ``140`` is used.  A value
+relative to the pool's ``recovery_priority`` is added.  The resultant priority
+is capped at ``179`` .  Under ordinary circumstances a
+backfill op priority of ``100`` is used.  A value relative to the pool's
+``recovery_priority`` is added.  The total priority is capped at ``139``.
+
+.. list-table:: Backfill and Recovery op priorities
+   :widths: 20 20 20
+   :header-rows: 1
+
+   * - Description
+     - Base priority
+     - Maximum priority
+   * - Backfill
+     - 100
+     - 139
+   * - Degraded Backfill
+     - 140
+     - 179
+   * - Recovery
+     - 180
+     - 219
+   * - Inactive Recovery
+     - 220
+     - 253
+   * - Inactive Backfill
+     - 220
+     - 253
+   * - force-backfill
+     - 254
+     -
+   * - force-recovery
+     - 255
+     -
+
diff --git a/doc/dev/osd_internals/erasure_coding.rst b/doc/dev/osd_internals/erasure_coding.rst
new file mode 100644
index 000000000..40064961b
--- /dev/null
+++ b/doc/dev/osd_internals/erasure_coding.rst
@@ -0,0 +1,87 @@
+==============================
+Erasure Coded Placement Groups
+==============================
+
+Glossary
+--------
+
+*chunk* 
+   When the encoding function is called, it returns chunks of the same
+   size as each other. There are two kinds of chunks: (1) *data
+   chunks*, which can be concatenated to reconstruct the original
+   object, and (2) *coding chunks*, which can be used to rebuild a
+   lost chunk.
+
+*chunk rank*
+   The index of a chunk, as determined by the encoding function. The
+   rank of the first chunk is 0, the rank of the second chunk is 1,
+   and so on.
+
+*K*
+   The number of data chunks into which an object is divided. For
+   example, if *K* = 2, then a 10KB object is divided into two objects
+   of 5KB each.
+
+*M* 
+   The number of coding chunks computed by the encoding function. *M*
+   is equal to the number of OSDs that can be missing from the cluster
+   without the cluster suffering data loss. For example, if there are
+   two coding chunks, then two OSDs can be missing without data loss.
+
+*N*
+   The number of data chunks plus the number of coding chunks: that
+   is, *K* + *M*.
+
+*rate*
+   The proportion of the total chunks containing useful information:
+   that is, *K* divided by *N*. For example, suppose that *K* = 9 and
+   *M* = 3. This would mean that *N* = 12 (because *K* + *M* = 9 + 3).
+   Therefore, the *rate* (*K* / *N*) would be 9 / 12 = 0.75. In other
+   words, 75% of the chunks would contain useful information.
+
+*shard* (also called *strip*)
+   An ordered sequence of chunks of the same rank from the same object. For a
+   given placement group, each OSD contains shards of the same rank. In the
+   special case in which an object is encoded with only one call to the
+   encoding function, the term *chunk* may be used instead of *shard* because
+   the shard is made of a single chunk. The chunks in a shard are ordered
+   according to the rank of the stripe (see *stripe* below) they belong to.
+
+
+*stripe* 
+   If an object is so large that encoding it requires more than one
+   call to the encoding function, each of these calls creates a set of
+   chunks called a *stripe*.
+
+The definitions are illustrated as follows (PG stands for placement group):
+::
+ 
+                 OSD 40                       OSD 33
+       +-------------------------+ +-------------------------+
+       |      shard 0 - PG 10    | |      shard 1 - PG 10    |
+       |+------ object O -------+| |+------ object O -------+|
+       ||+---------------------+|| ||+---------------------+||
+ stripe|||    chunk  0         ||| |||    chunk  1         ||| ...
+   0   |||    stripe 0         ||| |||    stripe 0         ||| 
+       ||+---------------------+|| ||+---------------------+||
+       ||+---------------------+|| ||+---------------------+||
+ stripe|||    chunk  0         ||| |||    chunk  1         ||| ...
+   1   |||    stripe 1         ||| |||    stripe 1         |||
+       ||+---------------------+|| ||+---------------------+||
+       ||+---------------------+|| ||+---------------------+||
+ stripe|||    chunk  0         ||| |||    chunk  1         ||| ...
+   2   |||    stripe 2         ||| |||    stripe 2         |||
+       ||+---------------------+|| ||+---------------------+||
+       |+-----------------------+| |+-----------------------+|
+       |         ...             | |         ...             |
+       +-------------------------+ +-------------------------+
+
+Table of contents
+-----------------
+
+.. toctree::
+   :maxdepth: 1
+
+   Developer notes <erasure_coding/developer_notes>
+   Jerasure plugin <erasure_coding/jerasure>
+   High level design document <erasure_coding/ecbackend>
diff --git a/doc/dev/osd_internals/erasure_coding/developer_notes.rst b/doc/dev/osd_internals/erasure_coding/developer_notes.rst
new file mode 100644
index 000000000..586b4b71b
--- /dev/null
+++ b/doc/dev/osd_internals/erasure_coding/developer_notes.rst
@@ -0,0 +1,223 @@
+============================
+Erasure Code developer notes
+============================
+
+Introduction
+------------
+
+Each chapter of this document explains an aspect of the implementation
+of the erasure code within Ceph. It is mostly based on examples being
+explained to demonstrate how things work. 
+
+Reading and writing encoded chunks from and to OSDs
+---------------------------------------------------
+
+An erasure coded pool stores each object as K+M chunks. It is divided
+into K data chunks and M coding chunks. The pool is configured to have
+a size of K+M so that each chunk is stored in an OSD in the acting
+set. The rank of the chunk is stored as an attribute of the object.
+
+Let's say an erasure coded pool is created to use five OSDs ( K+M =
+5 ) and sustain the loss of two of them ( M = 2 ).
+
+When the object *NYAN* containing *ABCDEFGHI* is written to it, the
+erasure encoding function splits the content in three data chunks,
+simply by dividing the content in three : the first contains *ABC*,
+the second *DEF* and the last *GHI*. The content will be padded if the
+content length is not a multiple of K. The function also creates two
+coding chunks : the fourth with *YXY* and the fifth with *GQC*. Each
+chunk is stored in an OSD in the acting set. The chunks are stored in
+objects that have the same name ( *NYAN* ) but reside on different
+OSDs. The order in which the chunks were created must be preserved and
+is stored as an attribute of the object ( shard_t ), in addition to its
+name. Chunk *1* contains *ABC* and is stored on *OSD5* while chunk *4*
+contains *YXY* and is stored on *OSD3*.
+
+::
+ 
+                             +-------------------+
+                        name |        NYAN       |
+                             +-------------------+
+                     content |      ABCDEFGHI    |
+                             +--------+----------+
+                                      |
+                                      |
+                                      v
+                               +------+------+
+               +---------------+ encode(3,2) +-----------+
+               |               +--+--+---+---+           |
+               |                  |  |   |               |
+               |          +-------+  |   +-----+         |
+               |          |          |         |         |
+            +--v---+   +--v---+   +--v---+  +--v---+  +--v---+
+      name  | NYAN |   | NYAN |   | NYAN |  | NYAN |  | NYAN |
+            +------+   +------+   +------+  +------+  +------+
+     shard  |  1   |   |  2   |   |  3   |  |  4   |  |  5   |
+            +------+   +------+   +------+  +------+  +------+
+   content  | ABC  |   | DEF  |   | GHI  |  | YXY  |  | QGC  |
+            +--+---+   +--+---+   +--+---+  +--+---+  +--+---+
+               |          |          |         |         |
+               |          |          |         |         |
+               |          |       +--+---+     |         |
+               |          |       | OSD1 |     |         |
+               |          |       +------+     |         |
+               |          |       +------+     |         |
+               |          +------>| OSD2 |     |         |
+               |                  +------+     |         |
+               |                  +------+     |         |
+               |                  | OSD3 |<----+         |
+               |                  +------+               |
+               |                  +------+               |
+               |                  | OSD4 |<--------------+
+               |                  +------+
+               |                  +------+
+               +----------------->| OSD5 |
+                                  +------+
+
+
+
+
+When the object *NYAN* is read from the erasure coded pool, the
+decoding function reads three chunks : chunk *1* containing *ABC*,
+chunk *3* containing *GHI* and chunk *4* containing *YXY* and rebuild
+the original content of the object *ABCDEFGHI*. The decoding function
+is informed that the chunks *2* and *5* are missing ( they are called
+*erasures* ). The chunk *5* could not be read because the *OSD4* is
+*out*. 
+
+The decoding function could be called as soon as three chunks are
+read : *OSD2* was the slowest and its chunk does not need to be taken into
+account. This optimization is not implemented in Firefly.
+
+::
+ 
+                             +-------------------+
+                        name |        NYAN       |
+                             +-------------------+
+                     content |      ABCDEFGHI    |
+                             +--------+----------+
+                                      ^
+                                      |
+                                      |
+                               +------+------+
+                               | decode(3,2) |
+                               | erasures 2,5|
+               +-------------->|             |
+               |               +-------------+
+               |                     ^   ^
+               |                     |   +-----+
+               |                     |         |
+            +--+---+   +------+   +--+---+  +--+---+
+      name  | NYAN |   | NYAN |   | NYAN |  | NYAN |
+            +------+   +------+   +------+  +------+
+     shard  |  1   |   |  2   |   |  3   |  |  4   |
+            +------+   +------+   +------+  +------+
+   content  | ABC  |   | DEF  |   | GHI  |  | YXY  |
+            +--+---+   +--+---+   +--+---+  +--+---+
+               ^          .          ^         ^
+               |    TOO   .          |         |
+               |    SLOW  .       +--+---+     |
+               |          ^       | OSD1 |     |
+               |          |       +------+     |
+               |          |       +------+     |
+               |          +-------| OSD2 |     |
+               |                  +------+     |
+               |                  +------+     |
+               |                  | OSD3 |-----+
+               |                  +------+
+               |                  +------+
+               |                  | OSD4 | OUT
+               |                  +------+
+               |                  +------+
+               +------------------| OSD5 |
+                                  +------+
+
+
+Erasure code library
+--------------------
+
+Using `Reed-Solomon <https://en.wikipedia.org/wiki/Reed_Solomon>`_,
+with parameters K+M, object O is encoded by dividing it into chunks O1,
+O2, ...  OM and computing coding chunks P1, P2, ... PK. Any K chunks
+out of the available K+M chunks can be used to obtain the original
+object.  If data chunk O2 or coding chunk P2 are lost, they can be
+repaired using any K chunks out of the K+M chunks. If more than M
+chunks are lost, it is not possible to recover the object.
+
+Reading the original content of object O can be a simple
+concatenation of O1, O2, ... OM, because the plugins are using
+`systematic codes
+<https://en.wikipedia.org/wiki/Systematic_code>`_. Otherwise the chunks
+must be given to the erasure code library *decode* method to retrieve
+the content of the object.
+
+Performance depend on the parameters to the encoding functions and
+is also influenced by the packet sizes used when calling the encoding
+functions ( for Cauchy or Liberation for instance ): smaller packets
+means more calls and more overhead.
+
+Although Reed-Solomon is provided as a default, Ceph uses it via an
+`abstract API <https://github.com/ceph/ceph/blob/v0.78/src/erasure-code/ErasureCodeInterface.h>`_ designed to
+allow each pool to choose the plugin that implements it using
+key=value pairs stored in an `erasure code profile`_. 
+
+.. _erasure code profile: ../../../erasure-coded-pool
+
+::
+ 
+ $ ceph osd erasure-code-profile set myprofile \
+     crush-failure-domain=osd
+ $ ceph osd erasure-code-profile get myprofile
+ directory=/usr/lib/ceph/erasure-code
+ k=2
+ m=1
+ plugin=jerasure
+ technique=reed_sol_van
+ crush-failure-domain=osd
+ $ ceph osd pool create ecpool erasure myprofile
+
+The *plugin* is dynamically loaded from *directory*  and expected to
+implement the *int __erasure_code_init(char *plugin_name, char *directory)* function 
+which is responsible for registering an object derived from *ErasureCodePlugin* 
+in the registry. The `ErasureCodePluginExample <https://github.com/ceph/ceph/blob/v0.78/src/test/erasure-code/ErasureCodePluginExample.cc>`_ plugin reads:
+
+::
+ 
+  ErasureCodePluginRegistry &instance = 
+                             ErasureCodePluginRegistry::instance();
+  instance.add(plugin_name, new ErasureCodePluginExample());
+
+The *ErasureCodePlugin* derived object must provide a factory method
+from which the concrete implementation of the *ErasureCodeInterface*
+object can be generated. The `ErasureCodePluginExample plugin <https://github.com/ceph/ceph/blob/v0.78/src/test/erasure-code/ErasureCodePluginExample.cc>`_ reads:
+
+::
+ 
+  virtual int factory(const map<std::string,std::string> &parameters,
+                      ErasureCodeInterfaceRef *erasure_code) {
+    *erasure_code = ErasureCodeInterfaceRef(new ErasureCodeExample(parameters));
+    return 0;
+  } 
+
+The *parameters* argument is the list of *key=value* pairs that were
+set in the erasure code profile, before the pool was created.
+
+::
+ 
+  ceph osd erasure-code-profile set myprofile \
+     directory=<dir>         \ # mandatory
+     plugin=jerasure         \ # mandatory
+     m=10                    \ # optional and plugin dependent
+     k=3                     \ # optional and plugin dependent
+     technique=reed_sol_van  \ # optional and plugin dependent
+
+Notes
+-----
+
+If the objects are large, it may be impractical to encode and decode
+them in memory. However, when using *RBD* a 1TB device is divided in
+many individual 4MB objects and *RGW* does the same.
+
+Encoding and decoding is implemented in the OSD. Although it could be
+implemented client side for read write, the OSD must be able to encode
+and decode on its own when scrubbing.
diff --git a/doc/dev/osd_internals/erasure_coding/ecbackend.rst b/doc/dev/osd_internals/erasure_coding/ecbackend.rst
new file mode 100644
index 000000000..624ec217e
--- /dev/null
+++ b/doc/dev/osd_internals/erasure_coding/ecbackend.rst
@@ -0,0 +1,207 @@
+=================================
+ECBackend Implementation Strategy
+=================================
+
+Misc initial design notes
+=========================
+
+The initial (and still true for ec pools without the hacky ec
+overwrites debug flag enabled) design for ec pools restricted
+EC pools to operations which can be easily rolled back:
+
+- CEPH_OSD_OP_APPEND: We can roll back an append locally by
+  including the previous object size as part of the PG log event.
+- CEPH_OSD_OP_DELETE: The possibility of rolling back a delete
+  requires that we retain the deleted object until all replicas have
+  persisted the deletion event.  ErasureCoded backend will therefore
+  need to store objects with the version at which they were created
+  included in the key provided to the filestore.  Old versions of an
+  object can be pruned when all replicas have committed up to the log
+  event deleting the object.
+- CEPH_OSD_OP_(SET|RM)ATTR: If we include the prior value of the attr
+  to be set or removed, we can roll back these operations locally.
+
+Log entries contain a structure explaining how to locally undo the
+operation represented by the operation
+(see osd_types.h:TransactionInfo::LocalRollBack).
+
+PGTemp and Crush
+----------------
+
+Primaries are able to request a temp acting set mapping in order to
+allow an up-to-date OSD to serve requests while a new primary is
+backfilled (and for other reasons).  An erasure coded pg needs to be
+able to designate a primary for these reasons without putting it in
+the first position of the acting set.  It also needs to be able to
+leave holes in the requested acting set.
+
+Core Changes:
+
+- OSDMap::pg_to_*_osds needs to separately return a primary.  For most
+  cases, this can continue to be acting[0].
+- MOSDPGTemp (and related OSD structures) needs to be able to specify
+  a primary as well as an acting set.
+- Much of the existing code base assumes that acting[0] is the primary
+  and that all elements of acting are valid.  This needs to be cleaned
+  up since the acting set may contain holes.
+
+Distinguished acting set positions
+----------------------------------
+
+With the replicated strategy, all replicas of a PG are
+interchangeable.  With erasure coding, different positions in the
+acting set have different pieces of the erasure coding scheme and are
+not interchangeable.  Worse, crush might cause chunk 2 to be written
+to an OSD which happens already to contain an (old) copy of chunk 4.
+This means that the OSD and PG messages need to work in terms of a
+type like pair<shard_t, pg_t> in order to distinguish different pg
+chunks on a single OSD.
+
+Because the mapping of object name to object in the filestore must
+be 1-to-1, we must ensure that the objects in chunk 2 and the objects
+in chunk 4 have different names.  To that end, the objectstore must
+include the chunk id in the object key.
+
+Core changes:
+
+- The objectstore `ghobject_t needs to also include a chunk id
+  <https://github.com/ceph/ceph/blob/firefly/src/common/hobject.h#L241>`_ making it more like
+  tuple<hobject_t, gen_t, shard_t>.
+- coll_t needs to include a shard_t.
+- The OSD pg_map and similar pg mappings need to work in terms of a
+  spg_t (essentially
+  pair<pg_t, shard_t>).  Similarly, pg->pg messages need to include
+  a shard_t
+- For client->PG messages, the OSD will need a way to know which PG
+  chunk should get the message since the OSD may contain both a
+  primary and non-primary chunk for the same pg
+
+Object Classes
+--------------
+
+Reads from object classes will return ENOTSUP on ec pools by invoking
+a special SYNC read.
+
+Scrub
+-----
+
+The main catch, however, for ec pools is that sending a crc32 of the
+stored chunk on a replica isn't particularly helpful since the chunks
+on different replicas presumably store different data.  Because we
+don't support overwrites except via DELETE, however, we have the
+option of maintaining a crc32 on each chunk through each append.
+Thus, each replica instead simply computes a crc32 of its own stored
+chunk and compares it with the locally stored checksum.  The replica
+then reports to the primary whether the checksums match.
+
+With overwrites, all scrubs are disabled for now until we work out
+what to do (see doc/dev/osd_internals/erasure_coding/proposals.rst).
+
+Crush
+-----
+
+If crush is unable to generate a replacement for a down member of an
+acting set, the acting set should have a hole at that position rather
+than shifting the other elements of the acting set out of position.
+
+=========
+ECBackend
+=========
+
+MAIN OPERATION OVERVIEW
+=======================
+
+A RADOS put operation can span
+multiple stripes of a single object. There must be code that
+tessellates the application level write into a set of per-stripe write
+operations -- some whole-stripes and up to two partial
+stripes. Without loss of generality, for the remainder of this
+document we will focus exclusively on writing a single stripe (whole
+or partial). We will use the symbol "W" to represent the number of
+blocks within a stripe that are being written, i.e., W <= K.
+
+There are three data flows for handling a write into an EC stripe. The
+choice of which of the three data flows to choose is based on the size
+of the write operation and the arithmetic properties of the selected
+parity-generation algorithm.
+
+(1) whole stripe is written/overwritten
+(2) a read-modify-write operation is performed.
+
+WHOLE STRIPE WRITE
+------------------
+
+This is the simple case, and is already performed in the existing code
+(for appends, that is). The primary receives all of the data for the
+stripe in the RADOS request, computes the appropriate parity blocks
+and send the data and parity blocks to their destination shards which
+write them. This is essentially the current EC code.
+
+READ-MODIFY-WRITE
+-----------------
+
+The primary determines which of the K-W blocks are to be unmodified,
+and reads them from the shards. Once all of the data is received it is
+combined with the received new data and new parity blocks are
+computed. The modified blocks are sent to their respective shards and
+written. The RADOS operation is acknowledged.
+
+OSD Object Write and Consistency
+--------------------------------
+
+Regardless of the algorithm chosen above, writing of the data is a two
+phase process: commit and rollforward. The primary sends the log
+entries with the operation described (see
+osd_types.h:TransactionInfo::(LocalRollForward|LocalRollBack).  
+In all cases, the "commit" is performed in place, possibly leaving some
+information required for a rollback in a write-aside object.  The
+rollforward phase occurs once all acting set replicas have committed
+the commit (sorry, overloaded term) and removes the rollback information.
+
+In the case of overwrites of exsting stripes, the rollback information
+has the form of a sparse object containing the old values of the
+overwritten extents populated using clone_range.  This is essentially
+a place-holder implementation, in real life, bluestore will have an
+efficient primitive for this.
+
+The rollforward part can be delayed since we report the operation as
+committed once all replicas have committed.  Currently, whenever we
+send a write, we also indicate that all previously committed
+operations should be rolled forward (see
+ECBackend::try_reads_to_commit).  If there aren't any in the pipeline
+when we arrive at the waiting_rollforward queue, we start a dummy
+write to move things along (see the Pipeline section later on and
+ECBackend::try_finish_rmw).
+
+ExtentCache
+-----------
+
+It's pretty important to be able to pipeline writes on the same
+object.  For this reason, there is a cache of extents written by
+cacheable operations.  Each extent remains pinned until the operations
+referring to it are committed.  The pipeline prevents rmw operations
+from running until uncacheable transactions (clones, etc) are flushed
+from the pipeline.
+
+See ExtentCache.h for a detailed explanation of how the cache
+states correspond to the higher level invariants about the conditions
+under which cuncurrent operations can refer to the same object.
+
+Pipeline
+--------
+
+Reading src/osd/ExtentCache.h should have given a good idea of how
+operations might overlap.  There are several states involved in
+processing a write operation and an important invariant which
+isn't enforced by PrimaryLogPG at a higher level which need to be
+managed by ECBackend.  The important invariant is that we can't
+have uncacheable and rmw operations running at the same time
+on the same object.  For simplicity, we simply enforce that any
+operation which contains an rmw operation must wait until
+all in-progress uncacheable operations complete.
+
+There are improvements to be made here in the future.
+
+For more details, see ECBackend::waiting_* and
+ECBackend::try_<from>_to_<to>.
+
diff --git a/doc/dev/osd_internals/erasure_coding/jerasure.rst b/doc/dev/osd_internals/erasure_coding/jerasure.rst
new file mode 100644
index 000000000..27669a0b2
--- /dev/null
+++ b/doc/dev/osd_internals/erasure_coding/jerasure.rst
@@ -0,0 +1,33 @@
+===============
+jerasure plugin
+===============
+
+Introduction
+------------
+
+The parameters interpreted by the jerasure plugin are:
+
+::
+ 
+  ceph osd erasure-code-profile set myprofile \
+     directory=<dir>         \ # plugin directory absolute path
+     plugin=jerasure         \ # plugin name (only jerasure)
+     k=<k>                   \ # data chunks (default 2)
+     m=<m>                   \ # coding chunks (default 2)
+     technique=<technique>   \ # coding technique
+
+The coding techniques can be chosen among *reed_sol_van*,
+*reed_sol_r6_op*, *cauchy_orig*, *cauchy_good*, *liberation*,
+*blaum_roth* and *liber8tion*.
+
+The *src/erasure-code/jerasure* directory contains the
+implementation. It is a wrapper around the code found at
+`https://github.com/ceph/jerasure <https://github.com/ceph/jerasure>`_
+and `https://github.com/ceph/gf-complete
+<https://github.com/ceph/gf-complete>`_ , pinned to the latest stable
+version in *.gitmodules*. These repositories are copies of the
+upstream repositories `http://jerasure.org/jerasure/jerasure
+<http://jerasure.org/jerasure/jerasure>`_ and
+`http://jerasure.org/jerasure/gf-complete
+<http://jerasure.org/jerasure/gf-complete>`_ . The difference
+between the two, if any, should match pull requests against upstream.
diff --git a/doc/dev/osd_internals/erasure_coding/proposals.rst b/doc/dev/osd_internals/erasure_coding/proposals.rst
new file mode 100644
index 000000000..d048ce8a1
--- /dev/null
+++ b/doc/dev/osd_internals/erasure_coding/proposals.rst
@@ -0,0 +1,385 @@
+:orphan:
+
+=================================
+Proposed Next Steps for ECBackend
+=================================
+
+PARITY-DELTA-WRITE
+------------------
+
+RMW operations current require 4 network hops (2 round trips).  In
+principle, for some codes, we can reduce this to 3 by sending the
+update to the replicas holding the data blocks and having them
+compute a delta to forward onto the parity blocks.
+
+The primary reads the current values of the "W" blocks and then uses
+the new values of the "W" blocks to compute parity-deltas for each of
+the parity blocks.  The W blocks and the parity delta-blocks are sent
+to their respective shards.
+
+The choice of whether to use a read-modify-write or a
+parity-delta-write is complex policy issue that is TBD in the details
+and is likely to be heavily dependent on the computational costs
+associated with a parity-delta vs. a regular parity-generation
+operation. However, it is believed that the parity-delta scheme is
+likely to be the preferred choice, when available.
+
+The internal interface to the erasure coding library plug-ins needs to
+be extended to support the ability to query if parity-delta
+computation is possible for a selected algorithm as well as an
+interface to the actual parity-delta computation algorithm when
+available.
+
+Stripe Cache
+------------
+
+It may be a good idea to extend the current ExtentCache usage to
+cache some data past when the pinning operation releases it.
+One application pattern that is important to optimize is the small
+block sequential write operation (think of the journal of a journaling
+file system or a database transaction log). Regardless of the chosen
+redundancy algorithm, it is advantageous for the primary to
+retain/buffer recently read/written portions of a stripe in order to
+reduce network traffic. The dynamic contents of this cache may be used
+in the determination of whether a read-modify-write or a
+parity-delta-write is performed. The sizing of this cache is TBD, but
+we should plan on allowing at least a few full stripes per active
+client. Limiting the cache occupancy on a per-client basis will reduce
+the noisy neighbor problem.
+
+Recovery and Rollback Details
+=============================
+
+Implementing a Rollback-able Prepare Operation
+----------------------------------------------
+
+The prepare operation is implemented at each OSD through a simulation
+of a versioning or copy-on-write capability for modifying a portion of
+an object.
+
+When a prepare operation is performed, the new data is written into a
+temporary object. The PG log for the
+operation will contain a reference to the temporary object so that it
+can be located for recovery purposes as well as a record of all of the
+shards which are involved in the operation. 
+
+In order to avoid fragmentation (and hence, future read performance),
+creation of the temporary object needs special attention. The name of
+the temporary object affects its location within the KV store. Right
+now its unclear whether it's desirable for the name to locate near the
+base object or whether a separate subset of keyspace should be used
+for temporary objects. Sam believes that colocation with the base
+object is preferred (he suggests using the generation counter of the
+ghobject for temporaries).  Whereas Allen believes that using a
+separate subset of keyspace is desirable since these keys are
+ephemeral and we don't want to actually colocate them with the base
+object keys. Perhaps some modeling here can help resolve this
+issue. The data of the temporary object wants to be located as close
+to the data of the base object as possible. This may be best performed
+by adding a new ObjectStore creation primitive that takes the base
+object as an additional parameter that is a hint to the allocator.
+
+Sam: I think that the short lived thing may be a red herring.  We'll
+be updating the donor and primary objects atomically, so it seems like
+we'd want them adjacent in the key space, regardless of the donor's
+lifecycle.
+
+The apply operation moves the data from the temporary object into the
+correct position within the base object and deletes the associated
+temporary object. This operation is done using a specialized
+ObjectStore primitive. In the current ObjectStore interface, this can
+be done using the clonerange function followed by a delete, but can be
+done more efficiently with a specialized move primitive.
+Implementation of the specialized primitive on FileStore can be done
+by copying the data. Some file systems have extensions that might also
+be able to implement this operation (like a defrag API that swaps
+chunks between files). It is expected that NewStore will be able to
+support this efficiently and natively (It has been noted that this
+sequence requires that temporary object allocations, which tend to be
+small, be efficiently converted into blocks for main objects and that
+blocks that were formerly inside of main objects must be reusable with
+minimal overhead)
+
+The prepare and apply operations can be separated arbitrarily in
+time. If a read operation accesses an object that has been altered by
+a prepare operation (but without a corresponding apply operation) it
+must return the data after the prepare operation. This is done by
+creating an in-memory database of objects which have had a prepare
+operation without a corresponding apply operation. All read operations
+must consult this in-memory data structure in order to get the correct
+data. It should explicitly recognized that it is likely that there
+will be multiple prepare operations against a single base object and
+the code must handle this case correctly. This code is implemented as
+a layer between ObjectStore and all existing readers.  Annoyingly,
+we'll want to trash this state when the interval changes, so the first
+thing that needs to happen after activation is that the primary and
+replicas apply up to last_update so that the empty cache will be
+correct.
+
+During peering, it is now obvious that an unapplied prepare operation
+can easily be rolled back simply by deleting the associated temporary
+object and removing that entry from the in-memory data structure.
+
+Partial Application Peering/Recovery modifications
+--------------------------------------------------
+
+Some writes will be small enough to not require updating all of the
+shards holding data blocks.  For write amplification minization
+reasons, it would be best to avoid writing to those shards at all,
+and delay even sending the log entries until the next write which
+actually hits that shard.
+
+The delaying (buffering) of the transmission of the prepare and apply
+operations for witnessing OSDs creates new situations that peering
+must handle. In particular the logic for determining the authoritative
+last_update value (and hence the selection of the OSD which has the
+authoritative log) must be modified to account for the valid but
+missing (i.e., delayed/buffered) pglog entries to which the
+authoritative OSD was only a witness to.
+
+Because a partial write might complete without persisting a log entry
+on every replica, we have to do a bit more work to determine an
+authoritative last_update.  The constraint (as with a replicated PG)
+is that last_update >= the most recent log entry for which a commit
+was sent to the client (call this actual_last_update).  Secondarily,
+we want last_update to be as small as possible since any log entry
+past actual_last_update (we do not apply a log entry until we have
+sent the commit to the client) must be able to be rolled back.  Thus,
+the smaller a last_update we choose, the less recovery will need to
+happen (we can always roll back, but rolling a replica forward may
+require an object rebuild).  Thus, we will set last_update to 1 before
+the oldest log entry we can prove cannot have been committed.  In
+current master, this is simply the last_update of the shortest log
+from that interval (because that log did not persist any entry past
+that point -- a precondition for sending a commit to the client).  For
+this design, we must consider the possibility that any log is missing
+at its head log entries in which it did not participate.  Thus, we
+must determine the most recent interval in which we went active
+(essentially, this is what find_best_info currently does).  We then
+pull the log from each live osd from that interval back to the minimum
+last_update among them.  Then, we extend all logs from the
+authoritative interval until each hits an entry in which it should
+have participated, but did not record.  The shortest of these extended
+logs must therefore contain any log entry for which we sent a commit
+to the client -- and the last entry gives us our last_update.
+
+Deep scrub support
+------------------
+
+The simple answer here is probably our best bet.  EC pools can't use
+the omap namespace at all right now.  The simplest solution would be
+to take a prefix of the omap space and pack N M byte L bit checksums
+into each key/value.  The prefixing seems like a sensible precaution
+against eventually wanting to store something else in the omap space.
+It seems like any write will need to read at least the blocks
+containing the modified range.  However, with a code able to compute
+parity deltas, we may not need to read a whole stripe.  Even without
+that, we don't want to have to write to blocks not participating in
+the write.  Thus, each shard should store checksums only for itself.
+It seems like you'd be able to store checksums for all shards on the
+parity blocks, but there may not be distinguished parity blocks which
+are modified on all writes (LRC or shec provide two examples).  L
+should probably have a fixed number of options (16, 32, 64?) and be
+configurable per-pool at pool creation.  N, M should be likewise be
+configurable at pool creation with sensible defaults.
+
+We need to handle online upgrade.  I think the right answer is that
+the first overwrite to an object with an append only checksum
+removes the append only checksum and writes in whatever stripe
+checksums actually got written.  The next deep scrub then writes
+out the full checksum omap entries.
+
+RADOS Client Acknowledgement Generation Optimization
+====================================================
+
+Now that the recovery scheme is understood, we can discuss the
+generation of the RADOS operation acknowledgement (ACK) by the
+primary ("sufficient" from above). It is NOT required that the primary
+wait for all shards to complete their respective prepare
+operations. Using our example where the RADOS operations writes only
+"W" chunks of the stripe, the primary will generate and send W+M
+prepare operations (possibly including a send-to-self). The primary
+need only wait for enough shards to be written to ensure recovery of
+the data, Thus after writing W + M chunks you can afford the lost of M
+chunks. Hence the primary can generate the RADOS ACK after W+M-M => W
+of those prepare operations are completed.
+
+Inconsistent object_info_t versions
+===================================
+
+A natural consequence of only writing the blocks which actually
+changed is that we don't want to update the object_info_t of the
+objects which didn't.  I actually think it would pose a problem to do
+so: pg ghobject namespaces are generally large, and unless the osd is
+seeing a bunch of overwrites on a small set of objects, I'd expect
+each write to be far enough apart in the backing ghobject_t->data
+mapping to each constitute a random metadata update.  Thus, we have to
+accept that not every shard will have the current version in its
+object_info_t.  We can't even bound how old the version on a
+particular shard will happen to be.  In particular, the primary does
+not necessarily have the current version.  One could argue that the
+parity shards would always have the current version, but not every
+code necessarily has designated parity shards which see every write
+(certainly LRC, iirc shec, and even with a more pedestrian code, it
+might be desirable to rotate the shards based on object hash).  Even
+if you chose to designate a shard as witnessing all writes, the pg
+might be degraded with that particular shard missing.  This is a bit
+tricky, currently reads and writes implicitly return the most recent
+version of the object written.  On reads, we'd have to read K shards
+to answer that question.  We can get around that by adding a "don't
+tell me the current version" flag.  Writes are more problematic: we
+need an object_info from the most recent write in order to form the
+new object_info and log_entry.
+
+A truly terrifying option would be to eliminate version and
+prior_version entirely from the object_info_t.  There are a few
+specific purposes it serves:
+
+#. On OSD startup, we prime the missing set by scanning backwards
+   from last_update to last_complete comparing the stored object's
+   object_info_t to the version of most recent log entry.
+#. During backfill, we compare versions between primary and target
+   to avoid some pushes. We use it elsewhere as well
+#. While pushing and pulling objects, we verify the version.
+#. We return it on reads and writes and allow the librados user to
+   assert it atomically on writesto allow the user to deal with write
+   races (used extensively by rbd).
+
+Case (3) isn't actually essential, just convenient.  Oh well.  (4)
+is more annoying. Writes are easy since we know the version.  Reads
+are tricky because we may not need to read from all of the replicas.
+Simplest solution is to add a flag to rados operations to just not
+return the user version on read.  We can also just not support the
+user version assert on ec for now (I think?  Only user is rgw bucket
+indices iirc, and those will always be on replicated because they use
+omap).
+
+We can avoid (1) by maintaining the missing set explicitly.  It's
+already possible for there to be a missing object without a
+corresponding log entry (Consider the case where the most recent write
+is to an object which has not been updated in weeks.  If that write
+becomes divergent, the written object needs to be marked missing based
+on the prior_version which is not in the log.)  THe PGLog already has
+a way of handling those edge cases (see divergent_priors).  We'd
+simply expand that to contain the entire missing set and maintain it
+atomically with the log and the objects.  This isn't really an
+unreasonable option, the additional keys would be fewer than the
+existing log keys + divergent_priors and aren't updated in the fast
+write path anyway.
+
+The second case is a bit trickier.  It's really an optimization for
+the case where a pg became not in the acting set long enough for the
+logs to no longer overlap but not long enough for the PG to have
+healed and removed the old copy.  Unfortunately, this describes the
+case where a node was taken down for maintenance with noout set. It's
+probably not acceptable to re-backfill the whole OSD in such a case,
+so we need to be able to quickly determine whether a particular shard
+is up to date given a valid acting set of other shards.
+
+Let ordinary writes which do not change the object size not touch the
+object_info at all.  That means that the object_info version won't
+match the pg log entry version.  Include in the pg_log_entry_t the
+current object_info version as well as which shards participated (as
+mentioned above).  In addition to the object_info_t attr, record on
+each shard s a vector recording for each other shard s' the most
+recent write which spanned both s and s'.  Operationally, we maintain
+an attr on each shard containing that vector.  A write touching S
+updates the version stamp entry for each shard in S on each shard in
+S's attribute (and leaves the rest alone).  If we have a valid acting
+set during backfill, we must have a witness of every write which
+completed -- so taking the max of each entry over all of the acting
+set shards must give us the current version for each shard.  During
+recovery, we set the attribute on the recovery target to that max
+vector (Question: with LRC, we may not need to touch much of the
+acting set to recover a particular shard -- can we just use the max of
+the shards we used to recovery, or do we need to grab the version
+vector from the rest of the acting set as well?  I'm not sure, not a
+big deal anyway, I think).
+
+The above lets us perform blind writes without knowing the current
+object version (log entry version, that is) while still allowing us to
+avoid backfilling up to date objects.  The only catch is that our
+backfill scans will can all replicas, not just the primary and the
+backfill targets.
+
+It would be worth adding into scrub the ability to check the
+consistency of the gathered version vectors -- probably by just
+taking 3 random valid subsets and verifying that they generate
+the same authoritative version vector.
+
+Implementation Strategy
+=======================
+
+It goes without saying that it would be unwise to attempt to do all of
+this in one massive PR.  It's also not a good idea to merge code which
+isn't being tested.  To that end, it's worth thinking a bit about
+which bits can be tested on their own (perhaps with a bit of temporary
+scaffolding).
+
+We can implement the overwrite friendly checksumming scheme easily
+enough with the current implementation.  We'll want to enable it on a
+per-pool basis (probably using a flag which we'll later repurpose for
+actual overwrite support).  We can enable it in some of the ec
+thrashing tests in the suite.  We can also add a simple test
+validating the behavior of turning it on for an existing ec pool
+(later, we'll want to be able to convert append-only ec pools to
+overwrite ec pools, so that test will simply be expanded as we go).
+The flag should be gated by the experimental feature flag since we
+won't want to support this as a valid configuration -- testing only.
+We need to upgrade append only ones in place during deep scrub.
+
+Similarly, we can implement the unstable extent cache with the current
+implementation, it even lets us cut out the readable ack the replicas
+send to the primary after the commit which lets it release the lock.
+Same deal, implement, gate with experimental flag, add to some of the
+automated tests.  I don't really see a reason not to use the same flag
+as above.
+
+We can certainly implement the move-range primitive with unit tests
+before there are any users.  Adding coverage to the existing
+objectstore tests would suffice here.
+
+Explicit missing set can be implemented now, same deal as above --
+might as well even use the same feature bit.
+
+The TPC protocol outlined above can actually be implemented an append
+only EC pool.  Same deal as above, can even use the same feature bit.
+
+The RADOS flag to suppress the read op user version return can be
+implemented immediately.  Mostly just needs unit tests.
+
+The version vector problem is an interesting one.  For append only EC
+pools, it would be pointless since all writes increase the size and
+therefore update the object_info.  We could do it for replicated pools
+though.  It's a bit silly since all "shards" see all writes, but it
+would still let us implement and partially test the augmented backfill
+code as well as the extra pg log entry fields -- this depends on the
+explicit pg log entry branch having already merged.  It's not entirely
+clear to me that this one is worth doing separately.  It's enough code
+that I'd really prefer to get it done independently, but it's also a
+fair amount of scaffolding that will be later discarded.
+
+PGLog entries need to be able to record the participants and log
+comparison needs to be modified to extend logs with entries they
+wouldn't have witnessed.  This logic should be abstracted behind
+PGLog so it can be unittested -- that would let us test it somewhat
+before the actual ec overwrites code merges.
+
+Whatever needs to happen to the ec plugin interface can probably be
+done independently of the rest of this (pending resolution of
+questions below).
+
+The actual nuts and bolts of performing the ec overwrite it seems to
+me can't be productively tested (and therefore implemented) until the
+above are complete, so best to get all of the supporting code in
+first.
+
+Open Questions
+==============
+
+Is there a code we should be using that would let us compute a parity
+delta without rereading and reencoding the full stripe?  If so, is it
+the kind of thing we need to design for now, or can it be reasonably
+put off?
+
+What needs to happen to the EC plugin interface?
diff --git a/doc/dev/osd_internals/index.rst b/doc/dev/osd_internals/index.rst
new file mode 100644
index 000000000..7e82914aa
--- /dev/null
+++ b/doc/dev/osd_internals/index.rst
@@ -0,0 +1,10 @@
+==============================
+OSD developer documentation
+==============================
+
+.. rubric:: Contents
+
+.. toctree::
+   :glob:
+
+   *
diff --git a/doc/dev/osd_internals/last_epoch_started.rst b/doc/dev/osd_internals/last_epoch_started.rst
new file mode 100644
index 000000000..c31cc66b5
--- /dev/null
+++ b/doc/dev/osd_internals/last_epoch_started.rst
@@ -0,0 +1,60 @@
+======================
+last_epoch_started
+======================
+
+``info.last_epoch_started`` records an activation epoch ``e`` for interval ``i``
+such that all writes committed in ``i`` or earlier are reflected in the
+local info/log and no writes after ``i`` are reflected in the local
+info/log.  Since no committed write is ever divergent, even if we
+get an authoritative log/info with an older ``info.last_epoch_started``,
+we can leave our ``info.last_epoch_started`` alone since no writes could
+have committed in any intervening interval (See PG::proc_master_log).
+
+``info.history.last_epoch_started`` records a lower bound on the most
+recent interval in which the PG as a whole went active and accepted
+writes.  On a particular OSD it is also an upper bound on the
+activation epoch of intervals in which writes in the local PG log
+occurred:  we update it before accepting writes.  Because all
+committed writes are committed by all acting set OSDs, any
+non-divergent writes ensure that ``history.last_epoch_started`` was
+recorded by all acting set members in the interval.  Once peering has
+queried one OSD from each interval back to some seen
+``history.last_epoch_started``, it follows that no interval after the max
+``history.last_epoch_started`` can have reported writes as committed
+(since we record it before recording client writes in an interval).
+Thus, the minimum ``last_update`` across all infos with
+``info.last_epoch_started >= MAX(history.last_epoch_started)`` must be an
+upper bound on writes reported as committed to the client.
+
+We update ``info.last_epoch_started`` with the initial activation message,
+but we only update ``history.last_epoch_started`` after the new
+``info.last_epoch_started`` is persisted (possibly along with the first
+write).  This ensures that we do not require an OSD with the most
+recent ``info.last_epoch_started`` until all acting set OSDs have recorded
+it.
+
+In ``find_best_info``, we do include ``info.last_epoch_started`` values when
+calculating ``max_last_epoch_started_found`` because we want to avoid
+designating a log entry divergent which in a prior interval would have
+been non-divergent since it might have been used to serve a read.  In
+``activate()``, we use the peer's ``last_epoch_started`` value as a bound on
+how far back divergent log entries can be found.
+
+However, in a case like
+
+.. code::
+
+  calc_acting osd.0 1.4e( v 473'302 (292'200,473'302] local-les=473 n=4 ec=5 les/c 473/473 556/556/556
+  calc_acting osd.1 1.4e( v 473'302 (293'202,473'302] lb 0//0//-1 local-les=477 n=0 ec=5 les/c 473/473 556/556/556
+  calc_acting osd.4 1.4e( v 473'302 (120'121,473'302] local-les=473 n=4 ec=5 les/c 473/473 556/556/556
+  calc_acting osd.5 1.4e( empty local-les=0 n=0 ec=5 les/c 473/473 556/556/556
+
+since osd.1 is the only one which recorded info.les=477, while osd.4,osd.0
+(which were the acting set in that interval) did not (osd.4 restarted and osd.0
+did not get the message in time), the PG is marked incomplete when
+either osd.4 or osd.0 would have been valid choices. To avoid this, we do not
+consider ``info.les`` for incomplete peers when calculating
+``min_last_epoch_started_found``.  It would not have been in the acting
+set, so we must have another OSD from that interval anyway (if
+``maybe_went_rw``).  If that OSD does not remember that ``info.les``, then we
+cannot have served reads.
diff --git a/doc/dev/osd_internals/log_based_pg.rst b/doc/dev/osd_internals/log_based_pg.rst
new file mode 100644
index 000000000..5d1e560c0
--- /dev/null
+++ b/doc/dev/osd_internals/log_based_pg.rst
@@ -0,0 +1,208 @@
+.. _log-based-pg:
+
+============
+Log Based PG
+============
+
+Background
+==========
+
+Why PrimaryLogPG?
+-----------------
+
+Currently, consistency for all ceph pool types is ensured by primary
+log-based replication. This goes for both erasure-coded (EC) and
+replicated pools.
+
+Primary log-based replication
+-----------------------------
+
+Reads must return data written by any write which completed (where the
+client could possibly have received a commit message).  There are lots
+of ways to handle this, but Ceph's architecture makes it easy for
+everyone at any map epoch to know who the primary is.  Thus, the easy
+answer is to route all writes for a particular PG through a single
+ordering primary and then out to the replicas.  Though we only
+actually need to serialize writes on a single RADOS object (and even then,
+the partial ordering only really needs to provide an ordering between
+writes on overlapping regions), we might as well serialize writes on
+the whole PG since it lets us represent the current state of the PG
+using two numbers: the epoch of the map on the primary in which the
+most recent write started (this is a bit stranger than it might seem
+since map distribution itself is asynchronous -- see Peering and the
+concept of interval changes) and an increasing per-PG version number
+-- this is referred to in the code with type ``eversion_t`` and stored as
+``pg_info_t::last_update``.  Furthermore, we maintain a log of "recent"
+operations extending back at least far enough to include any
+*unstable* writes (writes which have been started but not committed)
+and objects which aren't uptodate locally (see recovery and
+backfill).  In practice, the log will extend much further
+(``osd_min_pg_log_entries`` when clean and ``osd_max_pg_log_entries`` when not
+clean) because it's handy for quickly performing recovery.
+
+Using this log, as long as we talk to a non-empty subset of the OSDs
+which must have accepted any completed writes from the most recent
+interval in which we accepted writes, we can determine a conservative
+log which must contain any write which has been reported to a client
+as committed.  There is some freedom here, we can choose any log entry
+between the oldest head remembered by an element of that set (any
+newer cannot have completed without that log containing it) and the
+newest head remembered (clearly, all writes in the log were started,
+so it's fine for us to remember them) as the new head.  This is the
+main point of divergence between replicated pools and EC pools in
+``PG/PrimaryLogPG``: replicated pools try to choose the newest valid
+option to avoid the client needing to replay those operations and
+instead recover the other copies.  EC pools instead try to choose
+the *oldest* option available to them.
+
+The reason for this gets to the heart of the rest of the differences
+in implementation: one copy will not generally be enough to
+reconstruct an EC object.  Indeed, there are encodings where some log
+combinations would leave unrecoverable objects (as with a ``k=4,m=2`` encoding
+where 3 of the replicas remember a write, but the other 3 do not -- we
+don't have 3 copies of either version).  For this reason, log entries
+representing *unstable* writes (writes not yet committed to the
+client) must be rollbackable using only local information on EC pools.
+Log entries in general may therefore be rollbackable (and in that case,
+via a delayed application or via a set of instructions for rolling
+back an inplace update) or not.  Replicated pool log entries are
+never able to be rolled back.
+
+For more details, see ``PGLog.h/cc``, ``osd_types.h:pg_log_t``,
+``osd_types.h:pg_log_entry_t``, and peering in general.
+
+ReplicatedBackend/ECBackend unification strategy
+================================================
+
+PGBackend
+---------
+
+The fundamental difference between replication and erasure coding
+is that replication can do destructive updates while erasure coding
+cannot.  It would be really annoying if we needed to have two entire
+implementations of ``PrimaryLogPG`` since there
+are really only a few fundamental differences:
+
+#. How reads work -- async only, requires remote reads for EC
+#. How writes work -- either restricted to append, or must write aside and do a
+   tpc
+#. Whether we choose the oldest or newest possible head entry during peering
+#. A bit of extra information in the log entry to enable rollback
+
+and so many similarities
+
+#. All of the stats and metadata for objects
+#. The high level locking rules for mixing client IO with recovery and scrub
+#. The high level locking rules for mixing reads and writes without exposing
+   uncommitted state (which might be rolled back or forgotten later)
+#. The process, metadata, and protocol needed to determine the set of osds
+   which participated in the most recent interval in which we accepted writes
+#. etc.
+
+Instead, we choose a few abstractions (and a few kludges) to paper over the differences:
+
+#. ``PGBackend``
+#. ``PGTransaction``
+#. ``PG::choose_acting`` chooses between ``calc_replicated_acting`` and ``calc_ec_acting``
+#. Various bits of the write pipeline disallow some operations based on pool
+   type -- like omap operations, class operation reads, and writes which are
+   not aligned appends (officially, so far) for EC
+#. Misc other kludges here and there
+
+``PGBackend`` and ``PGTransaction`` enable abstraction of differences 1 and 2 above
+and the addition of 4 as needed to the log entries.
+
+The replicated implementation is in ``ReplicatedBackend.h/cc`` and doesn't
+require much additional explanation.  More detail on the ``ECBackend`` can be
+found in ``doc/dev/osd_internals/erasure_coding/ecbackend.rst``.
+
+PGBackend Interface Explanation
+===============================
+
+Note: this is from a design document that predated the Firefly release
+and is probably out of date w.r.t. some of the method names.
+
+Readable vs Degraded
+--------------------
+
+For a replicated pool, an object is readable IFF it is present on
+the primary (at the right version).  For an EC pool, we need at least
+`m` shards present to perform a read, and we need it on the primary.  For
+this reason, ``PGBackend`` needs to include some interfaces for determining
+when recovery is required to serve a read vs a write.  This also
+changes the rules for when peering has enough logs to prove that it
+
+Core Changes:
+
+- | ``PGBackend`` needs to be able to return ``IsPG(Recoverable|Readable)Predicate``
+  | objects to allow the user to make these determinations.
+
+Client Reads
+------------
+
+Reads from a replicated pool can always be satisfied
+synchronously by the primary OSD.  Within an erasure coded pool,
+the primary will need to request data from some number of replicas in
+order to satisfy a read.  ``PGBackend`` will therefore need to provide
+separate ``objects_read_sync`` and ``objects_read_async`` interfaces where
+the former won't be implemented by the ``ECBackend``.
+
+``PGBackend`` interfaces:
+
+- ``objects_read_sync``
+- ``objects_read_async``
+
+Scrubs
+------
+
+We currently have two scrub modes with different default frequencies:
+
+#. [shallow] scrub: compares the set of objects and metadata, but not
+   the contents
+#. deep scrub: compares the set of objects, metadata, and a CRC32 of
+   the object contents (including omap)
+
+The primary requests a scrubmap from each replica for a particular
+range of objects.  The replica fills out this scrubmap for the range
+of objects including, if the scrub is deep, a CRC32 of the contents of
+each object.  The primary gathers these scrubmaps from each replica
+and performs a comparison identifying inconsistent objects.
+
+Most of this can work essentially unchanged with erasure coded PG with
+the caveat that the ``PGBackend`` implementation must be in charge of
+actually doing the scan.
+
+
+``PGBackend`` interfaces:
+
+- ``be_*``
+
+Recovery
+--------
+
+The logic for recovering an object depends on the backend.  With
+the current replicated strategy, we first pull the object replica
+to the primary and then concurrently push it out to the replicas.
+With the erasure coded strategy, we probably want to read the
+minimum number of replica chunks required to reconstruct the object
+and push out the replacement chunks concurrently.
+
+Another difference is that objects in erasure coded PG may be
+unrecoverable without being unfound.  The ``unfound`` state
+should probably be renamed to ``unrecoverable``.  Also, the
+``PGBackend`` implementation will have to be able to direct the search
+for PG replicas with unrecoverable object chunks and to be able
+to determine whether a particular object is recoverable.
+
+
+Core changes:
+
+- ``s/unfound/unrecoverable``
+
+PGBackend interfaces:
+
+- `on_local_recover_start <https://github.com/ceph/ceph/blob/firefly/src/osd/PGBackend.h#L60>`_
+- `on_local_recover <https://github.com/ceph/ceph/blob/firefly/src/osd/PGBackend.h#L66>`_
+- `on_global_recover <https://github.com/ceph/ceph/blob/firefly/src/osd/PGBackend.h#L78>`_
+- `on_peer_recover <https://github.com/ceph/ceph/blob/firefly/src/osd/PGBackend.h#L83>`_
+- `begin_peer_recover <https://github.com/ceph/ceph/blob/firefly/src/osd/PGBackend.h#L90>`_
diff --git a/doc/dev/osd_internals/manifest.rst b/doc/dev/osd_internals/manifest.rst
new file mode 100644
index 000000000..2e8f9ae44
--- /dev/null
+++ b/doc/dev/osd_internals/manifest.rst
@@ -0,0 +1,623 @@
+========
+Manifest
+========
+
+
+Introduction
+============
+
+As described in ``../deduplication.rst``, adding transparent redirect
+machinery to RADOS would enable a more capable tiering solution
+than RADOS currently has with "cache/tiering".
+
+See ``../deduplication.rst``
+
+At a high level, each object has a piece of metadata embedded in
+the ``object_info_t`` which can map subsets of the object data payload
+to (refcounted) objects in other pools.
+
+This document exists to detail:
+
+1. Manifest data structures
+2. Rados operations for manipulating manifests.
+3. Status and Plans
+
+
+Intended Usage Model
+====================
+
+RBD
+---
+
+For RBD, the primary goal is for either an OSD-internal agent or a
+cluster-external agent to be able to transparently shift portions
+of the consituent 4MB extents between a dedup pool and a hot base
+pool.
+
+As such, RBD operations (including class operations and snapshots)
+must have the same observable results regardless of the current
+status of the object.
+
+Moreover, tiering/dedup operations must interleave with RBD operations
+without changing the result.
+
+Thus, here is a sketch of how I'd expect a tiering agent to perform
+basic operations:
+
+* Demote cold RBD chunk to slow pool:
+
+  1. Read object, noting current user_version.
+  2. In memory, run CDC implementation to fingerprint object.
+  3. Write out each resulting extent to an object in the cold pool
+     using the CAS class.
+  4. Submit operation to base pool:
+
+     * ``ASSERT_VER`` with the user version from the read to fail if the
+       object has been mutated since the read.
+     * ``SET_CHUNK`` for each of the extents to the corresponding object
+       in the base pool.
+     * ``EVICT_CHUNK`` for each extent to free up space in the base pool.
+       Results in each chunk being marked ``MISSING``.
+
+  RBD users should then either see the state prior to the demotion or
+  subsequent to it.
+
+  Note that between 3 and 4, we potentially leak references, so a
+  periodic scrub would be needed to validate refcounts.
+
+* Promote cold RBD chunk to fast pool.
+
+  1. Submit ``TIER_PROMOTE``
+
+For clones, all of the above would be identical except that the
+initial read would need a ``LIST_SNAPS`` to determine which clones exist
+and the ``PROMOTE`` or ``SET_CHUNK``/``EVICT`` operations would need to include
+the ``cloneid``.
+
+RadosGW
+-------
+
+For reads, RADOS Gateway (RGW) could operate as RBD does above relying on the
+manifest machinery in the OSD to hide the distinction between the object
+being dedup'd or present in the base pool
+
+For writes, RGW could operate as RBD does above, but could
+optionally have the freedom to fingerprint prior to doing the write.
+In that case, it could immediately write out the target objects to the
+CAS pool and then atomically write an object with the corresponding
+chunks set.
+
+Status and Future Work
+======================
+
+At the moment, initial versions of a manifest data structure along
+with IO path support and rados control operations exist.  This section
+is meant to outline next steps.
+
+At a high level, our future work plan is:
+
+- Cleanups: Address immediate inconsistencies and shortcomings outlined
+  in the next section.
+- Testing: Rados relies heavily on teuthology failure testing to validate
+  features like cache/tiering.  We'll need corresponding tests for
+  manifest operations.
+- Snapshots: We want to be able to deduplicate portions of clones
+  below the level of the rados snapshot system.  As such, the
+  rados operations below need to be extended to work correctly on
+  clones (e.g.: we should be able to call ``SET_CHUNK`` on a clone, clear the
+  corresponding extent in the base pool, and correctly maintain OSD metadata).
+- Cache/tiering: Ultimately, we'd like to be able to deprecate the existing
+  cache/tiering implementation, but to do that we need to ensure that we
+  can address the same use cases.
+
+
+Cleanups
+--------
+
+The existing implementation has some things that need to be cleaned up:
+
+* ``SET_REDIRECT``: Should create the object if it doesn't exist, otherwise
+  one couldn't create an object atomically as a redirect.
+* ``SET_CHUNK``:
+
+  * Appears to trigger a new clone as user_modify gets set in
+    ``do_osd_ops``.  This probably isn't desirable, see Snapshots section
+    below for some options on how generally to mix these operations
+    with snapshots.  At a minimum, ``SET_CHUNK`` probably shouldn't set
+    user_modify.
+  * Appears to assume that the corresponding section of the object
+    does not exist (sets ``FLAG_MISSING``) but does not check whether the
+    corresponding extent exists already in the object.  Should always
+    leave the extent clean.
+  * Appears to clear the manifest unconditionally if not chunked,
+    that's probably wrong.  We should return an error if it's a
+    ``REDIRECT`` ::
+
+	case CEPH_OSD_OP_SET_CHUNK:
+	  if (oi.manifest.is_redirect()) {
+	    result = -EINVAL;
+	    goto fail;
+	  }
+
+
+* ``TIER_PROMOTE``:
+
+  * ``SET_REDIRECT`` clears the contents of the object.  ``PROMOTE`` appears
+    to copy them back in, but does not unset the redirect or clear the
+    reference. This violates the invariant that a redirect object
+    should be empty in the base pool.  In particular, as long as the
+    redirect is set, it appears that all operations will be proxied
+    even after the promote defeating the purpose.  We do want ``PROMOTE``
+    to be able to atomically replace a redirect with the actual
+    object, so the solution is to clear the redirect at the end of the
+    promote.
+  * For a chunked manifest, we appear to flush prior to promoting.
+    Promotion will often be used to prepare an object for low latency
+    reads and writes, accordingly, the only effect should be to read
+    any ``MISSING`` extents into the base pool.  No flushing should be done.
+
+* High Level:
+
+  * It appears that ``FLAG_DIRTY`` should never be used for an extent pointing
+    at a dedup extent.  Writing the mutated extent back to the dedup pool
+    requires writing a new object since the previous one cannot be mutated,
+    just as it would if it hadn't been dedup'd yet.  Thus, we should always
+    drop the reference and remove the manifest pointer.
+
+  * There isn't currently a way to "evict" an object region.  With the above
+    change to ``SET_CHUNK`` to always retain the existing object region, we
+    need an ``EVICT_CHUNK`` operation to then remove the extent.
+
+
+Testing
+-------
+
+We rely really heavily on randomized failure testing.  As such, we need
+to extend that testing to include dedup/manifest support as well.  Here's
+a short list of the touchpoints:
+
+* Thrasher tests like ``qa/suites/rados/thrash/workloads/cache-snaps.yaml``
+
+  That test, of course, tests the existing cache/tiering machinery.  Add
+  additional files to that directory that instead setup a dedup pool.  Add
+  support to ``ceph_test_rados`` (``src/test/osd/TestRados*``).
+
+* RBD tests
+
+  Add a test that runs an RBD workload concurrently with blind
+  promote/evict operations.
+
+* RGW
+
+  Add a test that runs a rgw workload concurrently with blind
+  promote/evict operations.
+
+
+Snapshots
+---------
+
+Fundamentally we need to be able to manipulate the manifest
+status of clones because we want to be able to dynamically promote,
+flush (if the state was dirty when the clone was created), and evict
+extents from clones.
+
+As such, the plan is to allow the ``object_manifest_t`` for each clone
+to be independent.  Here's an incomplete list of the high level
+tasks:
+
+* Modify the op processing pipeline to permit ``SET_CHUNK``, ``EVICT_CHUNK``
+  to operation directly on clones.
+* Ensure that recovery checks the object_manifest prior to trying to
+  use the overlaps in clone_range.  ``ReplicatedBackend::calc_*_subsets``
+  are the two methods that would likely need to be modified.
+
+See ``snaps.rst`` for a rundown of the ``librados`` snapshot system and OSD
+support details.  I'd like to call out one particular data structure
+we may want to exploit.
+
+The dedup-tool needs to be updated to use ``LIST_SNAPS`` to discover
+clones as part of leak detection.
+
+An important question is how we deal with the fact that many clones
+will frequently have references to the same backing chunks at the same
+offset.  In particular, ``make_writeable`` will generally create a clone
+that shares the same ``object_manifest_t`` references with the exception
+of any extents modified in that transaction.  The metadata that
+commits as part of that transaction must therefore map onto the same
+refcount as before because otherwise we'd have to first increment
+refcounts on backing objects (or risk a reference to a dead object)
+Thus, we introduce a simple convention: consecutive clones which
+share a reference at the same offset share the same refcount.  This
+means that a write that invokes ``make_writeable`` may decrease refcounts,
+but not increase them.  This has some conquences for removing clones.
+Consider the following sequence ::
+
+  write foo [0, 1024)
+  flush foo ->
+    head: [0, 512) aaa, [512, 1024) bbb
+    refcount(aaa)=1, refcount(bbb)=1
+  snapshot 10
+  write foo [0, 512) ->
+    head:               [512, 1024) bbb
+    10  : [0, 512) aaa, [512, 1024) bbb
+    refcount(aaa)=1, refcount(bbb)=1
+  flush foo ->
+    head: [0, 512) ccc, [512, 1024) bbb
+    10  : [0, 512) aaa, [512, 1024) bbb
+    refcount(aaa)=1, refcount(bbb)=1, refcount(ccc)=1
+  snapshot 20
+  write foo [0, 512) (same contents as the original write)
+    head:               [512, 1024) bbb
+    20  : [0, 512) ccc, [512, 1024) bbb
+    10  : [0, 512) aaa, [512, 1024) bbb
+    refcount(aaa)=?, refcount(bbb)=1
+  flush foo
+    head: [0, 512) aaa, [512, 1024) bbb
+    20  : [0, 512) ccc, [512, 1024) bbb
+    10  : [0, 512) aaa, [512, 1024) bbb
+    refcount(aaa)=?, refcount(bbb)=1, refcount(ccc)=1
+
+What should be the refcount for ``aaa`` be at the end?  By our
+above rule, it should be ``2`` since the two ```aaa``` refs are not
+contiguous.  However, consider removing clone ``20`` ::
+
+  initial:
+    head: [0, 512) aaa, [512, 1024) bbb
+    20  : [0, 512) ccc, [512, 1024) bbb
+    10  : [0, 512) aaa, [512, 1024) bbb
+    refcount(aaa)=2, refcount(bbb)=1, refcount(ccc)=1
+  trim 20
+    head: [0, 512) aaa, [512, 1024) bbb
+    10  : [0, 512) aaa, [512, 1024) bbb
+    refcount(aaa)=?, refcount(bbb)=1, refcount(ccc)=0
+
+At this point, our rule dictates that ``refcount(aaa)`` is `1`.
+This means that removing ``20`` needs to check for refs held by
+the clones on either side which will then match.
+
+See ``osd_types.h:object_manifest_t::calc_refs_to_drop_on_removal``
+for the logic implementing this rule.
+
+This seems complicated, but it gets us two valuable properties:
+
+1) The refcount change from make_writeable will not block on
+   incrementing a ref
+2) We don't need to load the ``object_manifest_t`` for every clone
+   to determine how to handle removing one -- just the ones
+   immediately preceding and succeeding it.
+
+All clone operations will need to consider adjacent ``chunk_maps``
+when adding or removing references.
+
+Cache/Tiering
+-------------
+
+There already exists a cache/tiering mechanism based on whiteouts.
+One goal here should ultimately be for this manifest machinery to
+provide a complete replacement.
+
+See ``cache-pool.rst``
+
+The manifest machinery already shares some code paths with the
+existing cache/tiering code, mainly ``stat_flush``.
+
+In no particular order, here's in incomplete list of things that need
+to be wired up to provide feature parity:
+
+* Online object access information: The osd already has pool configs
+  for maintaining bloom filters which provide estimates of access
+  recency for objects.  We probably need to modify this to permit
+  hitset maintenance for a normal pool -- there are already
+  ``CEPH_OSD_OP_PG_HITSET*`` interfaces for querying them.
+* Tiering agent: The osd already has a background tiering agent which
+  would need to be modified to instead flush and evict using
+  manifests.
+
+* Use exiting existing features regarding the cache flush policy such as
+  histset, age, ratio.
+  - hitset
+  - age, ratio, bytes
+
+* Add tiering-mode to ``manifest-tiering``
+  - Writeback
+  - Read-only
+
+
+Data Structures
+===============
+
+Each RADOS object contains an ``object_manifest_t`` embedded within the
+``object_info_t`` (see ``osd_types.h``):
+
+::
+  
+        struct object_manifest_t {
+                enum {
+                        TYPE_NONE = 0,
+                        TYPE_REDIRECT = 1,
+                        TYPE_CHUNKED = 2,
+                };
+                uint8_t type;  // redirect, chunked, ...
+                hobject_t redirect_target;
+                std::map<uint64_t, chunk_info_t> chunk_map;
+        }
+
+The ``type`` enum reflects three possible states an object can be in:
+
+1. ``TYPE_NONE``: normal RADOS object
+2. ``TYPE_REDIRECT``: object payload is backed by a single object
+   specified by ``redirect_target``
+3. ``TYPE_CHUNKED: object payload is distributed among objects with
+   size and offset specified by the ``chunk_map``. ``chunk_map`` maps
+   the offset of the chunk to a ``chunk_info_t`` as shown below, also
+   specifying the ``length``, target `OID`, and ``flags``.
+
+::
+
+        struct chunk_info_t {
+          typedef enum {
+            FLAG_DIRTY = 1, 
+            FLAG_MISSING = 2,
+            FLAG_HAS_REFERENCE = 4,
+            FLAG_HAS_FINGERPRINT = 8,
+          } cflag_t;
+          uint32_t offset;
+          uint32_t length;
+          hobject_t oid;
+          cflag_t flags;   // FLAG_*
+
+
+``FLAG_DIRTY`` at this time can happen if an extent with a fingerprint
+is written.  This should be changed to drop the fingerprint instead.
+
+
+Request Handling
+================
+
+Similarly to cache/tiering, the initial touchpoint is
+``maybe_handle_manifest_detail``.
+
+For manifest operations listed below, we return ``NOOP`` and continue onto
+dedicated handling within ``do_osd_ops``.
+
+For redirect objects which haven't been promoted (apparently ``oi.size >
+0`` indicates that it's present?) we proxy reads and writes.
+
+For reads on ``TYPE_CHUNKED``, if ``can_proxy_chunked_read`` (basically, all
+of the ops are reads of extents in the ``object_manifest_t chunk_map``),
+we proxy requests to those objects.
+
+
+RADOS Interface
+================
+
+To set up deduplication one must provision two pools. One will act as the 
+base pool and the other will act as the chunk pool. The base pool need to be
+configured with the ``fingerprint_algorithm`` option as follows.
+
+::
+
+  ceph osd pool set $BASE_POOL fingerprint_algorithm sha1|sha256|sha512 
+  --yes-i-really-mean-it
+
+Create objects ::
+
+  rados -p base_pool put foo ./foo
+  rados -p chunk_pool put foo-chunk ./foo-chunk
+
+Make a manifest object ::
+
+  rados -p base_pool set-chunk foo $START_OFFSET $END_OFFSET --target-pool chunk_pool foo-chunk $START_OFFSET --with-reference
+
+Operations:
+
+* ``set-redirect``
+
+  Set a redirection between a ``base_object`` in the ``base_pool`` and a ``target_object`` 
+  in the ``target_pool``.
+  A redirected object will forward all operations from the client to the 
+  ``target_object``. ::
+
+        void set_redirect(const std::string& tgt_obj, const IoCtx& tgt_ioctx,
+		      uint64_t tgt_version, int flag = 0);
+  
+        rados -p base_pool set-redirect <base_object> --target-pool <target_pool> 
+         <target_object>
+
+  Returns ``ENOENT`` if the object does not exist (TODO: why?)
+  Returns ``EINVAL`` if the object already is a redirect.
+
+  Takes a reference to target as part of operation, can possibly leak a ref
+  if the acting set resets and the client dies between taking the ref and
+  recording the redirect.
+
+  Truncates object, clears omap, and clears xattrs as a side effect.
+
+  At the top of ``do_osd_ops``, does not set user_modify.
+
+  This operation is not a user mutation and does not trigger a clone to be created.
+
+  There are two purposes of ``set_redirect``:
+
+  1. Redirect all operation to the target object (like proxy)
+  2. Cache when ``tier_promote`` is called (redirect will be cleared at this time).
+
+* ``set-chunk`` 
+
+  Set the ``chunk-offset`` in a ``source_object`` to make a link between it and a 
+  ``target_object``. ::
+
+        void set_chunk(uint64_t src_offset, uint64_t src_length, const IoCtx& tgt_ioctx,
+                   std::string tgt_oid, uint64_t tgt_offset, int flag = 0);
+  
+        rados -p base_pool set-chunk <source_object> <offset> <length> --target-pool 
+         <caspool> <target_object> <target-offset> 
+
+  Returns ``ENOENT`` if the object does not exist (TODO: why?)
+  Returns ``EINVAL`` if the object already is a redirect.
+  Returns ``EINVAL`` if on ill-formed parameter buffer.
+  Returns ``ENOTSUPP`` if existing mapped chunks overlap with new chunk mapping.
+
+  Takes references to targets as part of operation, can possibly leak refs
+  if the acting set resets and the client dies between taking the ref and
+  recording the redirect.
+
+  Truncates object, clears omap, and clears xattrs as a side effect.
+
+  This operation is not a user mutation and does not trigger a clone to be created.
+
+  TODO: ``SET_CHUNK`` appears to clear the manifest unconditionally if it's not chunked. ::
+
+       if (!oi.manifest.is_chunked()) {
+         oi.manifest.clear();
+       }
+
+* ``evict-chunk``
+
+  Clears an extent from an object leaving only the manifest link between
+  it and the ``target_object``. ::
+
+        void evict_chunk(
+	  uint64_t offset, uint64_t length, int flag = 0);
+  
+        rados -p base_pool evict-chunk <offset> <length> <object>
+
+  Returns ``EINVAL`` if the extent is not present in the manifest.
+
+  Note: this does not exist yet.
+
+
+* ``tier-promote`` 
+
+  Promotes the object ensuring that subsequent reads and writes will be local ::
+
+        void tier_promote();
+
+        rados -p base_pool tier-promote <obj-name> 
+
+  Returns ``ENOENT`` if the object does not exist
+
+  For a redirect manifest, copies data to head.
+
+  TODO: Promote on a redirect object needs to clear the redirect.
+
+  For a chunked manifest, reads all MISSING extents into the base pool,
+  subsequent reads and writes will be served from the base pool.
+
+  Implementation Note: For a chunked manifest, calls ``start_copy`` on itself.  The
+  resulting ``copy_get`` operation will issue reads which will then be redirected by
+  the normal manifest read machinery.
+
+  Does not set the ``user_modify`` flag.
+
+  Future work will involve adding support for specifying a ``clone_id``.
+
+* ``unset-manifest``
+
+  Unset the manifest info in the object that has manifest. ::
+
+        void unset_manifest();
+
+        rados -p base_pool unset-manifest <obj-name>
+
+  Clears manifest chunks or redirect.  Lazily releases references, may
+  leak.
+
+  ``do_osd_ops`` seems not to include it in the ``user_modify=false`` ``ignorelist``,
+  and so will trigger a snapshot.  Note, this will be true even for a
+  redirect though ``SET_REDIRECT`` does not flip ``user_modify``.  This should
+  be fixed -- ``unset-manifest`` should not be a ``user_modify``.
+
+* ``tier-flush``
+
+  Flush the object which has chunks to the chunk pool. ::
+
+        void tier_flush();
+
+        rados -p base_pool tier-flush <obj-name>
+
+  Included in the ``user_modify=false`` ``ignorelist``, does not trigger a clone.
+
+  Does not evict the extents.
+
+
+ceph-dedup-tool
+===============
+
+``ceph-dedup-tool`` has two features: finding an optimal chunk offset for dedup chunking 
+and fixing the reference count (see ``./refcount.rst``).
+
+* Find an optimal chunk offset
+
+  a. Fixed chunk  
+
+  To find out a fixed chunk length, you need to run the following command many 
+  times while changing the ``chunk_size``. ::
+
+            ceph-dedup-tool --op estimate --pool $POOL --chunk-size chunk_size  
+              --chunk-algorithm fixed --fingerprint-algorithm sha1|sha256|sha512
+
+  b. Rabin chunk(Rabin-Karp algorithm) 
+
+  Rabin-Karp is a string-searching algorithm based
+  on a rolling hash. But a rolling hash is not enough to do deduplication because 
+  we don't know the chunk boundary. So, we need content-based slicing using 
+  a rolling hash for content-defined chunking.
+  The current implementation uses the simplest approach: look for chunk boundaries 
+  by inspecting the rolling hash for pattern (like the
+  lower N bits are all zeroes). 
+      
+  Users who want to use deduplication need to find an ideal chunk offset.
+  To find out ideal chunk offset, users should discover
+  the optimal configuration for their data workload via ``ceph-dedup-tool``.
+  This information will then be used for object chunking through
+  the ``set-chunk`` API. ::
+
+              ceph-dedup-tool --op estimate --pool $POOL --min-chunk min_size  
+                --chunk-algorithm rabin --fingerprint-algorithm rabin
+
+  ``ceph-dedup-tool`` has many options to utilize ``rabin chunk``.
+  These are options for ``rabin chunk``. ::
+
+              --mod-prime <uint64_t>
+              --rabin-prime <uint64_t>
+              --pow <uint64_t>
+              --chunk-mask-bit <uint32_t>
+              --window-size <uint32_t>
+              --min-chunk <uint32_t>
+              --max-chunk <uint64_t>
+
+   Users need to refer following equation to use above options for ``rabin chunk``. ::
+
+              rabin_hash = 
+                (rabin_hash * rabin_prime + new_byte - old_byte * pow) % (mod_prime)
+
+  c. Fixed chunk vs content-defined chunk
+
+  Content-defined chunking may or not be optimal solution.
+  For example,
+
+  Data chunk ``A`` : ``abcdefgabcdefgabcdefg``
+
+  Let's think about Data chunk ``A``'s deduplication. The ideal chunk offset is
+  from ``1`` to ``7`` (``abcdefg``). So, if we use fixed chunk, ``7`` is optimal chunk length.
+  But, in the case of content-based slicing, the optimal chunk length
+  could not be found (dedup ratio will not be 100%).
+  Because we need to find optimal parameter such
+  as boundary bit, window size and prime value. This is as easy as fixed chunk.
+  But, content defined chunking is very effective in the following case.
+
+    Data chunk ``B`` : ``abcdefgabcdefgabcdefg``
+
+    Data chunk ``C`` : ``Tabcdefgabcdefgabcdefg``
+
+
+* Fix reference count
+  
+  The key idea behind of reference counting for dedup is false-positive, which means 
+  ``(manifest object (no ref),, chunk object(has ref))`` happen instead of 
+  ``(manifest object (has ref), chunk 1(no ref))``.
+  To fix such inconsistencies, ``ceph-dedup-tool`` supports ``chunk_scrub``. ::
+
+          ceph-dedup-tool --op chunk_scrub --chunk_pool $CHUNK_POOL
+
diff --git a/doc/dev/osd_internals/map_message_handling.rst b/doc/dev/osd_internals/map_message_handling.rst
new file mode 100644
index 000000000..f8104f3fd
--- /dev/null
+++ b/doc/dev/osd_internals/map_message_handling.rst
@@ -0,0 +1,131 @@
+===========================
+Map and PG Message handling
+===========================
+
+Overview
+--------
+The OSD handles routing incoming messages to PGs, creating the PG if necessary
+in some cases.
+
+PG messages generally come in two varieties:
+
+  1. Peering Messages
+  2. Ops/SubOps
+
+There are several ways in which a message might be dropped or delayed.  It is
+important that the message delaying does not result in a violation of certain
+message ordering requirements on the way to the relevant PG handling logic:
+
+  1. Ops referring to the same object must not be reordered.
+  2. Peering messages must not be reordered.
+  3. Subops must not be reordered.
+
+MOSDMap
+-------
+MOSDMap messages may come from either monitors or other OSDs.  Upon receipt, the
+OSD must perform several tasks:
+
+  1. Persist the new maps to the filestore.
+     Several PG operations rely on having access to maps dating back to the last
+     time the PG was clean.
+  2. Update and persist the superblock.
+  3. Update OSD state related to the current map.
+  4. Expose new maps to PG processes via *OSDService*.
+  5. Remove PGs due to pool removal.
+  6. Queue dummy events to trigger PG map catchup.
+
+Each PG asynchronously catches up to the currently published map during
+process_peering_events before processing the event.  As a result, different
+PGs may have different views as to the "current" map.
+
+One consequence of this design is that messages containing submessages from
+multiple PGs (MOSDPGInfo, MOSDPGQuery, MOSDPGNotify) must tag each submessage
+with the PG's epoch as well as tagging the message as a whole with the OSD's
+current published epoch.
+
+MOSDPGOp/MOSDPGSubOp
+--------------------
+See OSD::dispatch_op, OSD::handle_op, OSD::handle_sub_op
+
+MOSDPGOps are used by clients to initiate rados operations. MOSDSubOps are used
+between OSDs to coordinate most non peering activities including replicating
+MOSDPGOp operations.
+
+OSD::require_same_or_newer map checks that the current OSDMap is at least
+as new as the map epoch indicated on the message.  If not, the message is
+queued in OSD::waiting_for_osdmap via OSD::wait_for_new_map.  Note, this
+cannot violate the above conditions since any two messages will be queued
+in order of receipt and if a message is received with epoch e0, a later message
+from the same source must be at epoch at least e0.  Note that two PGs from
+the same OSD count for these purposes as different sources for single PG
+messages.  That is, messages from different PGs may be reordered.
+
+
+MOSDPGOps follow the following process:
+
+  1. OSD::handle_op: validates permissions and crush mapping.
+     discard the request if they are not connected and the client cannot get the reply ( See OSD::op_is_discardable )
+     See OSDService::handle_misdirected_op
+     See PG::op_has_sufficient_caps
+     See OSD::require_same_or_newer_map
+  2. OSD::enqueue_op
+
+MOSDSubOps follow the following process:
+
+  1. OSD::handle_sub_op checks that sender is an OSD
+  2. OSD::enqueue_op
+
+OSD::enqueue_op calls PG::queue_op which checks waiting_for_map before calling OpWQ::queue which adds the op to the queue of the PG responsible for handling it.
+
+OSD::dequeue_op is then eventually called, with a lock on the PG.  At
+this time, the op is passed to PG::do_request, which checks that:
+
+  1. the PG map is new enough (PG::must_delay_op)
+  2. the client requesting the op has enough permissions (PG::op_has_sufficient_caps)
+  3. the op is not to be discarded (PG::can_discard_{request,op,subop,scan,backfill})
+  4. the PG is active (PG::flushed boolean)
+  5. the op is a CEPH_MSG_OSD_OP and the PG is in PG_STATE_ACTIVE state and not in PG_STATE_REPLAY 
+
+If these conditions are not met, the op is either discarded or queued for later processing. If all conditions are met, the op is processed according to its type:
+
+  1. CEPH_MSG_OSD_OP is handled by PG::do_op
+  2. MSG_OSD_SUBOP is handled by PG::do_sub_op
+  3. MSG_OSD_SUBOPREPLY is handled by PG::do_sub_op_reply
+  4. MSG_OSD_PG_SCAN is handled by PG::do_scan
+  5. MSG_OSD_PG_BACKFILL is handled by PG::do_backfill
+
+CEPH_MSG_OSD_OP processing
+--------------------------
+
+PrimaryLogPG::do_op handles CEPH_MSG_OSD_OP op and will queue it
+
+  1. in wait_for_all_missing if it is a CEPH_OSD_OP_PGLS for a designated snapid and some object updates are still missing
+  2. in waiting_for_active if the op may write but the scrubber is working
+  3. in waiting_for_missing_object if the op requires an object or a snapdir or a specific snap that is still missing
+  4. in waiting_for_degraded_object if the op may write an object or a snapdir that is degraded, or if another object blocks it ("blocked_by")
+  5. in waiting_for_backfill_pos if the op requires an object that will be available after the backfill is complete
+  6. in waiting_for_ack if an ack from another OSD is expected
+  7. in waiting_for_ondisk if the op is waiting for a write to complete
+
+Peering Messages
+----------------
+See OSD::handle_pg_(notify|info|log|query)
+
+Peering messages are tagged with two epochs:
+
+  1. epoch_sent: map epoch at which the message was sent
+  2. query_epoch: map epoch at which the message triggering the message was sent
+
+These are the same in cases where there was no triggering message.  We discard
+a peering message if the message's query_epoch if the PG in question has entered
+a new epoch (See PG::old_peering_evt, PG::queue_peering_event).  Notifies,
+infos, notifies, and logs are all handled as PG::PeeringMachine events and
+are wrapped by PG::queue_* by PG::CephPeeringEvts, which include the created
+state machine event along with epoch_sent and query_epoch in order to
+generically check PG::old_peering_message upon insertion and removal from the
+queue.
+
+Note, notifies, logs, and infos can trigger the creation of a PG.  See
+OSD::get_or_create_pg.
+
+
diff --git a/doc/dev/osd_internals/osd_overview.rst b/doc/dev/osd_internals/osd_overview.rst
new file mode 100644
index 000000000..192ddf8ca
--- /dev/null
+++ b/doc/dev/osd_internals/osd_overview.rst
@@ -0,0 +1,106 @@
+===
+OSD
+===
+
+Concepts
+--------
+
+*Messenger*
+   See src/msg/Messenger.h
+
+   Handles sending and receipt of messages on behalf of the OSD.  The OSD uses
+   two messengers: 
+
+     1. cluster_messenger - handles traffic to other OSDs, monitors
+     2. client_messenger - handles client traffic
+
+	 This division allows the OSD to be configured with different interfaces for
+	 client and cluster traffic.
+
+*Dispatcher*
+   See src/msg/Dispatcher.h
+
+	 OSD implements the Dispatcher interface.  Of particular note is ms_dispatch,
+	 which serves as the entry point for messages received via either the client
+	 or cluster messenger.  Because there are two messengers, ms_dispatch may be
+	 called from at least two threads.  The osd_lock is always held during
+	 ms_dispatch.
+
+*WorkQueue*
+  See src/common/WorkQueue.h
+
+  The WorkQueue class abstracts the process of queueing independent tasks
+  for asynchronous execution.  Each OSD process contains workqueues for
+  distinct tasks:
+
+    1. OpWQ: handles ops (from clients) and subops (from other OSDs).
+       Runs in the op_tp threadpool.
+    2. PeeringWQ: handles peering tasks and pg map advancement
+       Runs in the op_tp threadpool.
+       See Peering
+    3. CommandWQ: handles commands (pg query, etc)
+       Runs in the command_tp threadpool.
+    4. RecoveryWQ: handles recovery tasks.
+       Runs in the recovery_tp threadpool.
+    5. SnapTrimWQ: handles snap trimming
+       Runs in the disk_tp threadpool.
+       See SnapTrimmer
+    6. ScrubWQ: handles primary scrub path
+       Runs in the disk_tp threadpool.
+       See Scrub
+    7. ScrubFinalizeWQ: handles primary scrub finalize
+       Runs in the disk_tp threadpool.
+       See Scrub
+    8. RepScrubWQ: handles replica scrub path
+       Runs in the disk_tp threadpool
+       See Scrub
+    9. RemoveWQ: Asynchronously removes old pg directories
+       Runs in the disk_tp threadpool
+       See PGRemoval
+
+*ThreadPool*
+  See src/common/WorkQueue.h
+  See also above.
+
+  There are 4 OSD threadpools:
+
+    1. op_tp: handles ops and subops
+    2. recovery_tp: handles recovery tasks
+    3. disk_tp: handles disk intensive tasks
+    4. command_tp: handles commands
+
+*OSDMap*
+  See src/osd/OSDMap.h
+
+  The crush algorithm takes two inputs: a picture of the cluster
+  with status information about which nodes are up/down and in/out, 
+  and the pgid to place.  The former is encapsulated by the OSDMap.
+  Maps are numbered by *epoch* (epoch_t).  These maps are passed around
+  within the OSD as std::tr1::shared_ptr<const OSDMap>.
+
+  See MapHandling
+
+*PG*
+  See src/osd/PG.* src/osd/PrimaryLogPG.*
+
+  Objects in rados are hashed into *PGs* and *PGs* are placed via crush onto
+  OSDs.  The PG structure is responsible for handling requests pertaining to
+  a particular *PG* as well as for maintaining relevant metadata and controlling
+  recovery.
+
+*OSDService*
+  See src/osd/OSD.cc OSDService
+
+  The OSDService acts as a broker between PG threads and OSD state which allows
+  PGs to perform actions using OSD services such as workqueues and messengers.
+  This is still a work in progress.  Future cleanups will focus on moving such
+  state entirely from the OSD into the OSDService.
+
+Overview
+--------
+  See src/ceph_osd.cc
+
+  The OSD process represents one leaf device in the crush hierarchy.  There
+  might be one OSD process per physical machine, or more than one if, for
+  example, the user configures one OSD instance per disk.
+  
diff --git a/doc/dev/osd_internals/osd_throttles.rst b/doc/dev/osd_internals/osd_throttles.rst
new file mode 100644
index 000000000..0703b797e
--- /dev/null
+++ b/doc/dev/osd_internals/osd_throttles.rst
@@ -0,0 +1,93 @@
+=============
+OSD Throttles
+=============
+
+There are three significant throttles in the FileStore OSD back end:
+wbthrottle, op_queue_throttle, and a throttle based on journal usage.
+
+WBThrottle
+----------
+The WBThrottle is defined in src/os/filestore/WBThrottle.[h,cc] and
+included in FileStore as FileStore::wbthrottle.  The intention is to
+bound the amount of outstanding IO we need to do to flush the journal.
+At the same time, we don't want to necessarily do it inline in case we
+might be able to combine several IOs on the same object close together
+in time.  Thus, in FileStore::_write, we queue the fd for asynchronous
+flushing and block in FileStore::_do_op if we have exceeded any hard
+limits until the background flusher catches up.
+
+The relevant config options are filestore_wbthrottle*.  There are
+different defaults for XFS and Btrfs.  Each set has hard and soft
+limits on bytes (total dirty bytes), ios (total dirty ios), and
+inodes (total dirty fds).  The WBThrottle will begin flushing
+when any of these hits the soft limit and will block in throttle()
+while any has exceeded the hard limit.
+
+Tighter soft limits will cause writeback to happen more quickly,
+but may cause the OSD to miss oportunities for write coalescing.
+Tighter hard limits may cause a reduction in latency variance by
+reducing time spent flushing the journal, but may reduce writeback
+parallelism.
+
+op_queue_throttle
+-----------------
+The op queue throttle is intended to bound the amount of queued but
+uncompleted work in the filestore by delaying threads calling
+queue_transactions more and more based on how many ops and bytes are
+currently queued.  The throttle is taken in queue_transactions and
+released when the op is applied to the file system.  This period
+includes time spent in the journal queue, time spent writing to the
+journal, time spent in the actual op queue, time spent waiting for the
+wbthrottle to open up (thus, the wbthrottle can push back indirectly
+on the queue_transactions caller), and time spent actually applying
+the op to the file system.  A BackoffThrottle is used to gradually
+delay the queueing thread after each throttle becomes more than
+filestore_queue_low_threshhold full (a ratio of
+filestore_queue_max_(bytes|ops)).  The throttles will block once the
+max value is reached (filestore_queue_max_(bytes|ops)).
+
+The significant config options are:
+filestore_queue_low_threshhold
+filestore_queue_high_threshhold
+filestore_expected_throughput_ops
+filestore_expected_throughput_bytes
+filestore_queue_high_delay_multiple
+filestore_queue_max_delay_multiple
+
+While each throttle is at less than low_threshold of the max,
+no delay happens.  Between low and high, the throttle will
+inject a per-op delay (per op or byte) ramping from 0 at low to
+high_delay_multiple/expected_throughput at high.  From high to
+1, the delay will ramp from high_delay_multiple/expected_throughput
+to max_delay_multiple/expected_throughput.
+
+filestore_queue_high_delay_multiple and
+filestore_queue_max_delay_multiple probably do not need to be
+changed.
+
+Setting these properly should help to smooth out op latencies by
+mostly avoiding the hard limit.
+
+See FileStore::throttle_ops and FileSTore::thottle_bytes.
+
+journal usage throttle
+----------------------
+See src/os/filestore/JournalThrottle.h/cc
+
+The intention of the journal usage throttle is to gradually slow
+down queue_transactions callers as the journal fills up in order
+to smooth out hiccup during filestore syncs.  JournalThrottle
+wraps a BackoffThrottle and tracks journaled but not flushed
+journal entries so that the throttle can be released when the
+journal is flushed.  The configs work very similarly to the
+op_queue_throttle.
+
+The significant config options are:
+journal_throttle_low_threshhold
+journal_throttle_high_threshhold
+filestore_expected_throughput_ops
+filestore_expected_throughput_bytes
+journal_throttle_high_multiple
+journal_throttle_max_multiple
+
+.. literalinclude:: osd_throttles.txt
diff --git a/doc/dev/osd_internals/osd_throttles.txt b/doc/dev/osd_internals/osd_throttles.txt
new file mode 100644
index 000000000..0332377ef
--- /dev/null
+++ b/doc/dev/osd_internals/osd_throttles.txt
@@ -0,0 +1,21 @@
+                                                                                                   Messenger throttle (number and size)
+         |-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+                                                                     FileStore op_queue throttle (number and size, includes a soft throttle based on filestore_expected_throughput_(ops|bytes))
+                                                                |--------------------------------------------------------|
+                                                                                                                                                                   WBThrottle
+                                                                                                                       |---------------------------------------------------------------------------------------------------------|
+                                                                                                                                                                 Journal (size, includes a soft throttle based on filestore_expected_throughput_bytes)
+                                                                 |-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+                                                                                                                       |----------------------------------------------------------------------------------------------------> flushed ----------------> synced
+                                                                                                                       |
+Op: Read Header --DispatchQ--> OSD::_dispatch --OpWQ--> PG::do_request --journalq--> Journal --FileStore::OpWQ--> Apply Thread --Finisher--> op_applied ------------------------------------------------------------->  Complete
+                                              |                                                                                                                                                                      |
+SubOp:                                        --Messenger--> ReadHeader --DispatchQ--> OSD::_dispatch --OpWQ--> PG::do_request --journalq--> Journal --FileStore::OpWQ--> Apply Thread --Finisher--> sub_op_applied -
+                                                                                                                                                                              |
+                                                                                                                                                                              |-----------------------------> flushed ----------------> synced
+                                                                                                                                                |------------------------------------------------------------------------------------------|
+                                                                                                                                                                                        Journal (size)
+                                                                                                                                                                              |---------------------------------|
+                                                                                                                                                                                        WBThrottle
+                                                                                                                         |-----------------------------------------------------|
+                                                                                                                            FileStore op_queue throttle (number and size)
diff --git a/doc/dev/osd_internals/osdmap_versions.txt b/doc/dev/osd_internals/osdmap_versions.txt
new file mode 100644
index 000000000..12fab5ae3
--- /dev/null
+++ b/doc/dev/osd_internals/osdmap_versions.txt
@@ -0,0 +1,259 @@
+releases:
+
+ <0.48  pre-argonaut, dev
+ 0.48   argonaut
+ 0.56   bobtail
+ 0.61   cuttlefish
+ 0.67   dumpling
+ 0.72   emperor
+ 0.80   firefly
+ 0.87   giant
+ 0.94   hammer
+ 9.1.0  infernalis rc
+ 9.2.0  infernalis
+ 10.2.0 jewel
+ 11.2.0 kraken
+ 12.2.0 luminous
+ 13.2.0 mimic
+ 14.2.0 nautilus (to-be)
+
+osdmap:
+
+type / v / cv / ev / commit / version / date
+
+map / 1 / - / - / 017788a6ecb570038632de31904dd2e1314dc7b7 / 0.11 / 2009
+inc / 1 / - / - /
+      * initial
+map / 2 / - / - / 020350e19a5dc03cd6cedd7494e434295580615f / 0.13 / 2009
+inc / 2 / - / - /
+      * pg_temp
+map / 3 / - / - / 1ebcebf6fff056a0c0bdf82dde69356e271be27e / 0.19 / 2009
+inc / 3 / - / - /
+      * heartbeat_addr
+map / 4 / - / - / 3ced5e7de243edeccfd20a90ec2034206c920795 / 0.19 / 2010
+inc / 4 / - / - /
+      * pools removed from map
+map / 5 / - / 5 / c4892bed6f49df396df3cbf9ed561c7315bd2442 / 0.20 / 2010
+inc / 5 / - / 5 /
+      * pool names moved to first part of encoding
+        * adds CEPH_OSDMAP_INC_VERSION_EXT (for extended part of map)
+        * adds CEPH_OSDMAP_VERSION_EXT (for extended part of map)
+        * adds 'ev' (extended version) during encode() and decode
+map / 5 / - / 5 / bc9cb9311f1b946898b5256eab500856fccf5c83 / 0.22 / 2010
+inc / 5 / - / 6 /
+      * separate up client/osd
+        * increments CEPH_OSDMAP_INC_VERSION_EXT to 6
+        * CEPH_OSDMAP_INC_VERSION stays at 5
+map / 5 / - / 6 / 7f70112052c7fc3ba46f9e475fa575d85e8b16b2 / 0.22 / 2010
+inc / 5 / - / 6 /
+      * add osd_cluster_addr to full map
+        * increments CEPH_OSDMAP_VERSION_EXT to 6
+        * CEPH_OSDMAP_VERSION stays at 5
+map / 5 / - / 7 / 2ced4e24aef64f2bc7d55b73abb888c124512eac / 0.28 / 2011
+inc / 5 / - / 7 /
+      * add cluster_snapshot field
+        * increments CEPH_OSDMAP_VERSION_EXT to 7
+        * increments CEPH_OSDMAP_INC_VERSION_EXT to 7
+        * CEPH_OSDMAP_INC_VERSION stays at 5
+        * CEPH_OSDMAP_VERSION stays at 5
+map / 6 / - / 7 / d1ce410842ca51fad3aa100a52815a39e5fe6af6 / 0.35 / 2011
+inc / 6 / - / 7 /
+      * encode/decode old + new versions
+        * adds encode_client_old() (basically transitioning from uint32 to
+          uint64)
+        * bumps osdmap version to 6, old clients stay at 5
+        * starts using in-function versions (i.e., _u16 v = 6)
+map / 6 / - / 7 / b297d1edecaf31a48cff6c37df2ee266e51cdec1 / 0.38 / 2011
+inc / 6 / - / 7 /
+      * make encoding conditional based on features
+        * essentially checks whether features & CEPH_FEATURE_PGID64 and opts
+          to either use encode_client_old() or encode()
+map / 6 / - / 7 / 0f0c59478894c9ca7fa04fc32e854648192a9fae / 0.38 / 2011
+inc / 6 / - / 7 /
+      * move stuff from osdmap.h to osdmap.cc
+map / 6 / 8 / ca4311e5e39cec8fad85fad3e67eea968707e9eb / 0.47 / 2012
+inc / 6 / 8 /
+      * store uuid per osd
+        * bumps osdmap::incremental extended version to 8; in function
+        * bumps osdmap's extended version to 8; in function
+map / 6 / - / 8 / 5125daa6d78e173a8dbc75723a8fdcd279a44bcd / 0.47 / 2012
+inc / 6 / - / 8 /
+      * drop defines
+        * drops defines for CEPH_OSDMAP_*_VERSION from rados.h
+map / 6 / 9 / e9f051ef3c49a080b24d7811a16aefb64beacbbd / 0.53 / 2012
+inc / 6 / 9 /
+      * add osd_xinfo_t
+        * osdmap::incremental ext version bumped to 9
+        * osdmap's ext version bumped to 9
+        * because we're adding osd_xinfo_t to the map
+map / 6 / - / 10 / 1fee4ccd5277b52292e255daf458330eef5f0255 / 0.64 / 2013
+inc / 6 / - / 10 /
+      * encode front hb addr
+        * osdmap::incremental ext version bumped to 10
+        * osdmap's ext versiont bumped to 10
+        * because we're adding osd_addrs->hb_front_addr to map
+
+// below we have the change to ENCODE_START() for osdmap and others
+// this means client-usable data and extended osd data get to have their
+// own ENCODE_START()'s, hence their versions start at 1 again.
+
+map / 7 / 1 / 1 / 3d7c69fb0986337dc72e466dc39d93e5ab406062 / 0.77 / 2014
+inc / 7 / 1 / 1 / b55c45e85dbd5d2513a4c56b3b74dcafd03f20b1 / 0.77 / 2014
+      * introduces ENCODE_START() approach to osdmap, and the 'features'
+        argument we currently see in ::encode() functions
+      * same, but for osdmap::incremental
+map / 7 / 1 / 1 / b9208b47745fdd53d36b682bebfc01e913347092 / 0.77 / 2014
+inc / 7 / 1 / 2 /
+      * include features argument in incremental.
+map / 7 / 2 / 1 / cee914290c5540eb1fb9d70faac70a581381c29b / 0.78 / 2014
+inc / 7 / 2 / 2 /
+      * add osd_primary_affinity
+map / 7 / 3 / 1 / c4f8f265955d54f33c79cde02c1ab2fe69ab1ab0 / 0.78 / 2014
+inc / 7 / 3 / 2 /
+      * add new/old erasure code profiles
+map / 8 / 3 / 1 / 3dcf5b9636bb9e0cd6484d18f151b457e1a0c328 / 0.91 / 2014
+inc / 8 / 3 / 2 /
+      * encode crc
+map / 8 / 3 / 1 / 04679c5451e353c966f6ed00b33fa97be8072a79 / 9.1.0 / 2015
+inc / 8 / 3 / 2 /
+      * simply ensures encode_features are filled to CEPH_FEATURE_PGID64 when
+        decoding an incremental if struct_v >= 6; else keeps it at zero.
+      * otherwise, if we get an incremental from hammer (which has
+        struct_v = 6) we would be decoding it as if it were a map from before
+        CEPH_FEATURES_PGID64 (which was introduced in 0.35, pre-argonaut)
+map / 8 / 3 / 2 / 5c6b9d9dcd0a225e3a2b154c20a623868c269346 / 12.0.1 / 2017
+inc / 8 / 3 / 3 /
+      * add (near)full_ratio
+        * used to live in pgmap, moving to osdmap for luminous
+      * conditional on SERVER_LUMINOUS feature being present
+        * osdmap::incremental::encode(): conditional on ev >= 3
+        * osdmap::incremental::decode(): conditional on ev >= 3, else -1
+        * osdmap::encode(): conditional on ev >= 2
+        * osdmap::decode(): conditional on ev >= 0, else 0
+map / 8 / 4 / 2 / 27d6f4373bafa24450f6dbb4e4252c2d9c2c1448 / 12.0.2 / 2017
+inc / 8 / 4 / 3 /
+      * add pg_remap and pg_remap_items
+        * first forces a pg to map to a particular value; second replaces
+          specific osds with specific other osds in crush mapping.
+      * inc conditional on SERVER_LUMINOUS feature being present
+        * osdmap::incremental::encode(): conditional on cv >= 4
+        * osdmap::incremental::decode(): conditional on cv >= 4
+      * map conditional on OSDMAP_REMAP feature being present
+        * osdmap::encode(): if not feature, cv = 3; encode on cv >= 4
+        * osdmap::decode(): conditional on cv >= 4
+map / 8 / 4 / 3 / 27d6f4373bafa24450f6dbb4e4252c2d9c2c1448 / 12.0.2 / 2017
+inc / 8 / 4 / 4 /
+      * handle backfillfull_ratio like nearfull and full
+      * inc:
+        * osdmap::incremental::encode(): conditional on ev >= 3
+        * osdmap::incremental::decode(): conditional on ev >= 4, else -1
+      * map:
+        * osdmap::encode(): conditional on ev >= 2
+        * osdmap::decode(): conditional on ev >= 3, else 0
+map / 8 / 4 / 3 / a1c66468232002c9f36033226f5db0a5751e8d18 / 12.0.3 / 2017
+inc / 8 / 4 / 4 /
+      * add require_min_compat_client field
+      * inc:
+        * osdmap::incremental::encode() conditional on ev >= 4
+        * osdmap::incremental::decode() conditional on ev >= 4
+      map:
+        * osdmap::encode() conditional on ev >= 3
+        * osdmap::decode() conditional on ev >= 3
+map / 8 / 4 / 4 / 4a09e9431de3084b1ca98af11b28f822fde4ffbe / 12.0.3 / 2017
+inc / 8 / 4 / 5 /
+      * bumps encoding version for require_min_compat_client
+          * otherwise osdmap::decode() would throw exception when decoding
+            old maps
+      * inc:
+        * osdmap::incremental::encode() no conditional on ev >= 3
+        * osdmap::incremental::decode() conditional on ev >= 5
+      * map:
+      * osdmap::encode() conditional on ev >= 2
+      * osdmap::decode() conditional on ev >= 4
+map / 8 / 4 / 5 / 3d4c4d9d9da07e1456331c43acc998d2008ca8ea / 12.1.0 / 2017
+inc / 8 / 4 / 6 /
+      * add require_osd_release numeric field
+      * new_require_min_compat_client:
+        * osdmap::incremental::encode() conditional on ev >= 5
+        * osdmap::encode() conditional on ev >= 4
+      * require_osd_release:
+        * osdmap::incremental::encode() conditional on ev >= 6
+        * osdmap::incremental::decode() conditional on ev >= 6 (else, -1)
+        * osdmap::encode() conditional on ev >= 5
+        * osdmap::decode() conditional on ev >= 5 (else, -1)
+map / 8 / 4 / 5 / f22997e24bda4e6476e15d5d4ad9737861f9741f / 12.1.0 / 2017
+inc / 8 / 4 / 6 /
+        * switch (require_)min_compat_client to integers instead of strings
+        * osdmap::incremental::encode() conditional on ev >= 6
+        * osdmap::incremental::decode():
+          * if ev == 5, decode string and translate to release number
+          * if ev >= 6, decode integer
+        * osdmap::encode() conditional on ev >= 4
+        * osdmap::decode():
+          * if ev == 4, decode string and translate to release number
+          * if ev >= 5, decode integer
+map / 8 / 4 / 6 / a8fb39b57884d96201fa502b17bc9395ec38c1b3 / 12.1.0 / 2017
+inc / 8 / 5 / 6 /
+        * make incremental's `new_state` 32 bits instead of 8 bits
+        * implies forcing 8 bits on
+          * osdmap::incremental::encode_client_old()
+          * osdmap::incremental::encode_classic()
+          * osdmap::incremental::decode_classic()
+        * osdmap::incremental::encode() conditional on cv >= 5, else force 8b.
+        * osdmap::incremental::decode() conditional on cv >= 5, else force 8b.
+map / 8 / 5 / 6 / 3c1e58215bbb98f71aae30904f9010a57a58da81 / 12.1.0 / 2017
+inc / 8 / 5 / 6 /
+        * same as above
+map / 8 / 6 / 6 / 48158ec579b708772fae82daaa6cb5dcaf5ac5dd / 12.1.0 / 2017
+inc / 8 / 5 / 6 /
+        * add crush_version
+        * osdmap::encode() conditional on cv >= 6
+        * osdmap::decode() conditional on cv >= 6
+map / 8 / 7 / 6 / 553048fbf97af999783deb7e992c8ecfa5e55500 / 13.0.2 / 2017
+inc / 8 / 6 / 6 /
+        * track newly removed and purged snaps in each epoch
+          * new_removed_snaps
+          * new_purged_snaps
+        * osdmap::encode() conditional on cv >= 7
+          * if SERVER_MIMIC not in features, cv = 6
+        * osdmap::decode() conditional cv >= 7
+map / 8 / 8 / 6 / f99c2a9fec65ad3ce275ef24bd167ee03275d3d7 / 14.0.1 / 2018
+inc / 8 / 7 / 6 /
+        * fix pre-addrvec compat
+        * osdmap::encode() conditional on cv >= 8, else encode client addrs
+          one by one in a loop.
+        * osdmap::decode() just bumps version (?)
+map / 8 / 8 / 7 / 9fb1e521c7c75c124b0dbf193e8b65ff1b5f461e / 14.0.1 / 2018
+inc / 8 / 7 / 7 /
+        * make cluster addrs into addrvecs too
+          * this will allow single-step upgrade from msgr1 to msgr2
+map / 8 / 9 / 7 / d414f0b43a69f3c2db8e454d795be881496237c6 / 14.0.1 / 2018
+inc / 8 / 8 / 7 /
+        * store last_up_change and last_in_change
+        * osdmap::encode() conditional on cv >= 9
+        * osdmap::decode() conditional on cv >= 9
+
+
+
+osd_info_t:
+v / commit / version / date / reason
+
+1 / e574c84a6a0c5a5070dc72d5f5d3d17914ef824a / 0.19 / 2010 / add struct_v
+
+osd_xinfo_t:
+v / commit / version / date
+
+1 / e9f051ef3c49a080b24d7811a16aefb64beacbbd / 0.53 / 2012
+      * add osd_xinfo_t
+2 / 31743d50a109a463d664ec9cf764d5405db507bd / 0.75 / 2013
+      * add features bit mask to osd_xinfo_t
+3 / 87722a42c286d4d12190b86b6d06d388e2953ba0 / 0.82 / 2014
+      * remember osd weight when auto-marking osds out
+
+rados.h:
+v / commit / version / date / reason
+
+- / 147c6f51e34a875ab65624df04baa8ef89296ddd / 0.19 / 2010 / move versions
+  3 / CEPH_OSDMAP_INC_VERSION
+  3 / CEPH_OSDMAP_VERSION
+  2 / CEPH_PG_POOL_VERSION
diff --git a/doc/dev/osd_internals/partial_object_recovery.rst b/doc/dev/osd_internals/partial_object_recovery.rst
new file mode 100644
index 000000000..a22f63348
--- /dev/null
+++ b/doc/dev/osd_internals/partial_object_recovery.rst
@@ -0,0 +1,148 @@
+=======================
+Partial Object Recovery
+=======================
+
+Partial Object Recovery improves the efficiency of log-based recovery (vs
+backfill). Original log-based recovery calculates missing_set based on pg_log
+differences.
+
+The whole object should be recovery from one OSD to another
+if the object is indicated modified by pg_log regardless of how much
+content in the object is really modified. That means a 4M object,
+which is just modified 4k inside, should recovery the whole 4M object
+rather than the modified 4k content. In addition, object map should be
+also recovered even if it is not modified at all.
+
+Partial Object Recovery is designed to solve the problem mentioned above.
+In order to achieve the goals, two things should be done: 
+
+1. logging where the object is modified is necessary
+2. logging whether the object_map of an object is modified is also necessary
+
+class ObjectCleanRegion is introduced to do what we want.
+clean_offsets is a variable of interval_set<uint64_t>
+and is used to indicate the unmodified content in an object.
+clean_omap is a variable of bool indicating whether object_map is modified.
+new_object means that osd does not exist for an object
+max_num_intervals is an upbound of the number of intervals in clean_offsets
+so that the memory cost of clean_offsets is always bounded.
+
+The shortest clean interval will be trimmed if the number of intervals
+in clean_offsets exceeds the boundary.
+
+    etc. max_num_intervals=2, clean_offsets:{[5~10], [20~5]}
+
+    then new interval [30~10] will evict out the shortest one [20~5]
+
+    finally, clean_offsets becomes {[5~10], [30~10]}
+
+Procedures for Partial Object Recovery
+======================================
+
+Firstly, OpContext and pg_log_entry_t should contain ObjectCleanRegion.
+In do_osd_ops(), finish_copyfrom(), finish_promote(), corresponding content
+in ObjectCleanRegion should mark dirty so that trace the modification of an object.
+Also update ObjectCleanRegion in OpContext to its pg_log_entry_t.
+
+Secondly, pg_missing_set can build and rebuild correctly.
+when calculating pg_missing_set during peering process,
+also merge ObjectCleanRegion in each pg_log_entry_t.
+
+    etc. object aa has pg_log:
+        26'101 {[0~4096, 8192~MAX], false}
+
+        26'104 {0~8192, 12288~MAX, false}
+
+        28'108 {[0~12288, 16384~MAX], true}
+
+    missing_set for object aa: merge pg_log above --> {[0~4096, 16384~MAX], true}.
+    which means 4096~16384 is modified and object_map is also modified on version 28'108
+
+Also, OSD may be crash after merge log.
+Therefore, we need to read_log and rebuild pg_missing_set. For example, pg_log is:
+
+    object aa: 26'101 {[0~4096, 8192~MAX], false}
+
+    object bb: 26'102 {[0~4096, 8192~MAX], false}
+
+    object cc: 26'103 {[0~4096, 8192~MAX], false}
+
+    object aa: 26'104 {0~8192, 12288~MAX, false}
+
+    object dd: 26'105 {[0~4096, 8192~MAX], false}
+
+    object aa: 28'108 {[0~12288, 16384~MAX], true}
+
+Originally, if bb,cc,dd is recovered, and aa is not.
+So we need to rebuild pg_missing_set for object aa,
+and find aa is modified on version 28'108.
+If version in object_info is 26'96 < 28'108,
+we don't need to consider 26'104 and 26'101 because the whole object will be recovered.
+However, Partial Object Recovery should also require us to rebuild ObjectCleanRegion.
+
+Knowing whether the object is modified is not enough.
+
+Therefore, we also need to traverse the pg_log before,
+that says 26'104 and 26'101 also > object_info(26'96)
+and rebuild pg_missing_set for object aa based on those three logs: 28'108, 26'104, 26'101.
+The way how to merge logs is the same as mentioned above
+
+Finally, finish the push and pull process based on pg_missing_set.
+Updating copy_subset in recovery_info based on ObjectCleanRegion in pg_missing_set.
+copy_subset indicates the intervals of content need to pull and push.
+
+The complicated part here is submit_push_data
+and serval cases should be considered separately.
+what we need to consider is how to deal with the object data,
+object data makes up of omap_header, xattrs, omap, data:
+    
+case 1: first && complete: since object recovering is finished in a single PushOp,
+we would like to preserve the original object and overwrite on the object directly.
+Object will not be removed and touch a new one.
+
+    issue 1: As object is not removed, old xattrs remain in the old object
+    but maybe updated in new object. Overwriting for the same key or adding new keys is correct,
+    but removing keys will be wrong. 
+    In order to solve this issue, We need to remove the all original xattrs in the object, and then update new xattrs.
+
+    issue 2: As object is not removed,
+    object_map may be recovered depending on the clean_omap.
+    Therefore, if recovering clean_omap, we need to remove old omap of the object for the same reason
+    since omap updating may also be a deletion.
+    Thus, in this case, we should do:
+
+        1) clear xattrs of the object
+        2) clear omap of the object if omap recovery is needed
+        3) truncate the object into recovery_info.size
+        4) recovery omap_header
+        5) recovery xattrs, and recover omap if needed
+        6) punch zeros for original object if fiemap tells nothing there
+        7) overwrite object content which is modified
+        8) finish recovery
+
+case 2:  first && !complete: object recovering should be done in multiple times.
+Here, target_oid will indicate a new temp_object in pgid_TEMP,
+so the issues are a bit difference.
+
+    issue 1: As object is newly created, there is no need to deal with xattrs
+
+    issue 2: As object is newly created,
+    and object_map may not be transmitted depending on clean_omap.
+    Therefore, if clean_omap is true, we need to clone object_map from original object.
+    issue 3: As object is newly created, and unmodified data will not be transmitted.
+    Therefore, we need to clone unmodified data from the original object.
+    Thus, in this case, we should do:
+
+        1) remove the temp object
+        2) create a new temp object
+        3) set alloc_hint for the new temp object
+        4) truncate new temp object to recovery_info.size
+        5) recovery omap_header
+        6) clone object_map from original object if omap is clean
+        7) clone unmodified object_data from original object
+        8) punch zeros for the new temp object
+        9) recovery xattrs, and recover omap if needed
+        10) overwrite object content which is modified
+        11) remove the original object
+        12) move and rename the new temp object to replace the original object
+        13) finish recovery
diff --git a/doc/dev/osd_internals/pg.rst b/doc/dev/osd_internals/pg.rst
new file mode 100644
index 000000000..397d4ab5d
--- /dev/null
+++ b/doc/dev/osd_internals/pg.rst
@@ -0,0 +1,31 @@
+====
+PG
+====
+
+Concepts
+--------
+
+*Peering Interval*
+  See PG::start_peering_interval.
+  See PG::acting_up_affected
+  See PG::PeeringState::Reset
+
+  A peering interval is a maximal set of contiguous map epochs in which the
+  up and acting sets did not change.  PG::PeeringMachine represents a
+  transition from one interval to another as passing through
+  PeeringState::Reset.  On PG::PeeringState::AdvMap PG::acting_up_affected can
+  cause the pg to transition to Reset.
+  
+
+Peering Details and Gotchas
+---------------------------
+For an overview of peering, see `Peering <../../peering>`_.
+
+  * PG::flushed defaults to false and is set to false in
+    PG::start_peering_interval.  Upon transitioning to PG::PeeringState::Started
+    we send a transaction through the pg op sequencer which, upon complete,
+    sends a FlushedEvt which sets flushed to true.  The primary cannot go
+    active until this happens (See PG::PeeringState::WaitFlushedPeering).
+    Replicas can go active but cannot serve ops (writes or reads).
+    This is necessary because we cannot read our ondisk state until unstable
+    transactions from the previous interval have cleared.
diff --git a/doc/dev/osd_internals/pg_removal.rst b/doc/dev/osd_internals/pg_removal.rst
new file mode 100644
index 000000000..c5fe0e1ab
--- /dev/null
+++ b/doc/dev/osd_internals/pg_removal.rst
@@ -0,0 +1,56 @@
+==========
+PG Removal
+==========
+
+See OSD::_remove_pg, OSD::RemoveWQ
+
+There are two ways for a pg to be removed from an OSD:
+
+  1. MOSDPGRemove from the primary
+  2. OSD::advance_map finds that the pool has been removed
+
+In either case, our general strategy for removing the pg is to
+atomically set the metadata objects (pg->log_oid, pg->biginfo_oid) to
+backfill and asynchronously remove the pg collections.  We do not do
+this inline because scanning the collections to remove the objects is
+an expensive operation.
+
+OSDService::deleting_pgs tracks all pgs in the process of being
+deleted.  Each DeletingState object in deleting_pgs lives while at
+least one reference to it remains.  Each item in RemoveWQ carries a
+reference to the DeletingState for the relevant pg such that
+deleting_pgs.lookup(pgid) will return a null ref only if there are no
+collections currently being deleted for that pg. 
+
+The DeletingState for a pg also carries information about the status
+of the current deletion and allows the deletion to be cancelled.
+The possible states are:
+
+  1. QUEUED: the PG is in the RemoveWQ
+  2. CLEARING_DIR: the PG's contents are being removed synchronously
+  3. DELETING_DIR: the PG's directories and metadata being queued for removal
+  4. DELETED_DIR: the final removal transaction has been queued
+  5. CANCELED: the deletion has been cancelled
+
+In 1 and 2, the deletion can be cancelled.  Each state transition
+method (and check_canceled) returns false if deletion has been
+cancelled and true if the state transition was successful.  Similarly,
+try_stop_deletion() returns true if it succeeds in cancelling the
+deletion.  Additionally, try_stop_deletion() in the event that it
+fails to stop the deletion will not return until the final removal
+transaction is queued.  This ensures that any operations queued after
+that point will be ordered after the pg deletion.
+
+OSD::_create_lock_pg must handle two cases:
+
+  1. Either there is no DeletingStateRef for the pg, or it failed to cancel
+  2. We succeeded in cancelling the deletion.
+
+In case 1., we proceed as if there were no deletion occurring, except that
+we avoid writing to the PG until the deletion finishes.  In case 2., we
+proceed as in case 1., except that we first mark the PG as backfilling.
+
+Similarly, OSD::osr_registry ensures that the OpSequencers for those
+pgs can be reused for a new pg if created before the old one is fully
+removed, ensuring that operations on the new pg are sequenced properly
+with respect to operations on the old one.
diff --git a/doc/dev/osd_internals/pgpool.rst b/doc/dev/osd_internals/pgpool.rst
new file mode 100644
index 000000000..45a252bd4
--- /dev/null
+++ b/doc/dev/osd_internals/pgpool.rst
@@ -0,0 +1,22 @@
+==================
+PGPool
+==================
+
+PGPool is a structure used to manage and update the status of removed
+snapshots.  It does this by maintaining two fields, cached_removed_snaps - the
+current removed snap set and newly_removed_snaps - newly removed snaps in the
+last epoch. In OSD::load_pgs the osd map is recovered from the pg's file store
+and passed down to OSD::_get_pool where a PGPool object is initialised with the
+map.
+
+With each new map we receive we call PGPool::update with the new map. In that
+function we build a list of newly removed snaps
+(pg_pool_t::build_removed_snaps) and merge that with our cached_removed_snaps.
+This function included checks to make sure we only do this update when things
+have changed or there has been a map gap.
+
+When we activate the pg we initialise the snap trim queue from
+cached_removed_snaps and subtract the purged_snaps we have already purged
+leaving us with the list of snaps that need to be trimmed. Trimming is later
+performed asynchronously by the snap_trim_wq.
+
diff --git a/doc/dev/osd_internals/recovery_reservation.rst b/doc/dev/osd_internals/recovery_reservation.rst
new file mode 100644
index 000000000..a24ac1b15
--- /dev/null
+++ b/doc/dev/osd_internals/recovery_reservation.rst
@@ -0,0 +1,83 @@
+====================
+Recovery Reservation
+====================
+
+Recovery reservation extends and subsumes backfill reservation. The
+reservation system from backfill recovery is used for local and remote
+reservations.
+
+When a PG goes active, first it determines what type of recovery is
+necessary, if any. It may need log-based recovery, backfill recovery,
+both, or neither.
+
+In log-based recovery, the primary first acquires a local reservation
+from the OSDService's local_reserver. Then a MRemoteReservationRequest
+message is sent to each replica in order of OSD number. These requests
+will always be granted (i.e., cannot be rejected), but they may take
+some time to be granted if the remotes have already granted all their
+remote reservation slots.
+
+After all reservations are acquired, log-based recovery proceeds as it
+would without the reservation system.
+
+After log-based recovery completes, the primary releases all remote
+reservations. The local reservation remains held. The primary then
+determines whether backfill is necessary. If it is not necessary, the
+primary releases its local reservation and waits in the Recovered state
+for all OSDs to indicate that they are clean.
+
+If backfill recovery occurs after log-based recovery, the local
+reservation does not need to be reacquired since it is still held from
+before. If it occurs immediately after activation (log-based recovery
+not possible/necessary), the local reservation is acquired according to
+the typical process.
+
+Once the primary has its local reservation, it requests a remote
+reservation from the backfill target. This reservation CAN be rejected,
+for instance if the OSD is too full (backfillfull_ratio osd setting).
+If the reservation is rejected, the primary drops its local
+reservation, waits (osd_backfill_retry_interval), and then retries. It
+will retry indefinitely.
+
+Once the primary has the local and remote reservations, backfill
+proceeds as usual. After backfill completes the remote reservation is
+dropped.
+
+Finally, after backfill (or log-based recovery if backfill was not
+necessary), the primary drops the local reservation and enters the
+Recovered state. Once all the PGs have reported they are clean, the
+primary enters the Clean state and marks itself active+clean.
+
+-----------------
+Dump Reservations
+-----------------
+
+An OSD daemon command dumps total local and remote reservations::
+
+  ceph daemon osd.<id> dump_recovery_reservations
+
+
+--------------
+Things to Note
+--------------
+
+We always grab the local reservation first, to prevent a circular
+dependency. We grab remote reservations in order of OSD number for the
+same reason.
+
+The recovery reservation state chart controls the PG state as reported
+to the monitor. The state chart can set:
+
+ - recovery_wait: waiting for local/remote reservations
+ - recovering: recovering
+ - recovery_toofull: recovery stopped, OSD(s) above full ratio
+ - backfill_wait: waiting for remote backfill reservations
+ - backfilling: backfilling
+ - backfill_toofull: backfill stopped, OSD(s) above backfillfull ratio
+
+
+--------
+See Also
+--------
+
+The Active substate of the automatically generated OSD state diagram.
diff --git a/doc/dev/osd_internals/refcount.rst b/doc/dev/osd_internals/refcount.rst
new file mode 100644
index 000000000..4d75ae019
--- /dev/null
+++ b/doc/dev/osd_internals/refcount.rst
@@ -0,0 +1,45 @@
+========
+Refcount
+========
+
+
+Introduction
+============
+
+Dedupliation, as described in ../deduplication.rst, needs a way to
+maintain a target pool of deduplicated chunks with atomic ref
+refcounting.  To that end, there exists an osd object class 
+refcount responsible for using the object class machinery to
+maintain refcounts on deduped chunks and ultimately remove them
+as the refcount hits 0.
+
+Class Interface
+===============
+
+See cls/refcount/cls_refcount_client*
+
+* cls_refcount_get
+
+  Atomically increments the refcount with specified tag ::
+
+    void cls_refcount_get(librados::ObjectWriteOperation& op, const string& tag, bool implicit_ref = false);
+
+* cls_refcount_put
+
+  Atomically decrements the refcount specified by passed tag ::
+
+    void cls_refcount_put(librados::ObjectWriteOperation& op, const string& tag, bool implicit_ref = false);
+
+* cls_refcount_Set
+
+  Atomically sets the set of refcounts with passed list of tags ::
+
+    void cls_refcount_set(librados::ObjectWriteOperation& op, list<string>& refs);
+
+* cls_refcount_read
+
+  Dumps the current set of ref tags for the object ::
+
+    int cls_refcount_read(librados::IoCtx& io_ctx, string& oid, list<string> *refs, bool implicit_ref = false);
+
+  
diff --git a/doc/dev/osd_internals/scrub.rst b/doc/dev/osd_internals/scrub.rst
new file mode 100644
index 000000000..149509799
--- /dev/null
+++ b/doc/dev/osd_internals/scrub.rst
@@ -0,0 +1,41 @@
+
+Scrub internals and diagnostics
+===============================
+
+Scrubbing Behavior Table
+------------------------
+
++-------------------------------------------------+----------+-----------+---------------+----------------------+
+|                                          Flags  | none     | noscrub   | nodeep_scrub  | noscrub/nodeep_scrub |
++=================================================+==========+===========+===============+======================+
+| Periodic tick                                   |   S      |    X      |     S         |         X            |
++-------------------------------------------------+----------+-----------+---------------+----------------------+
+| Periodic tick after osd_deep_scrub_interval     |   D      |    D      |     S         |         X            |
++-------------------------------------------------+----------+-----------+---------------+----------------------+
+| Initiated scrub                                 |   S      |    S      |     S         |         S            |
++-------------------------------------------------+----------+-----------+---------------+----------------------+
+| Initiated scrub after osd_deep_scrub_interval   |   D      |    D      |     S         |         S            |
++-------------------------------------------------+----------+-----------+---------------+----------------------+
+| Initiated deep scrub                            |   D      |    D      |     D         |         D            |
++-------------------------------------------------+----------+-----------+---------------+----------------------+
+
+- X = Do nothing
+- S = Do regular scrub
+- D = Do deep scrub
+
+State variables
+---------------
+
+- Periodic tick state is ``!must_scrub && !must_deep_scrub && !time_for_deep``
+- Periodic tick after ``osd_deep_scrub_interval state is !must_scrub && !must_deep_scrub && time_for_deep``
+- Initiated scrub state is ``must_scrub && !must_deep_scrub && !time_for_deep``
+- Initiated scrub after ``osd_deep_scrub_interval`` state is ``must_scrub && !must_deep_scrub && time_for_deep``
+- Initiated deep scrub state is ``must_scrub && must_deep_scrub``
+
+Scrub Reservations
+------------------
+
+An OSD daemon command dumps total local and remote reservations::
+
+  ceph daemon osd.<id> dump_scrub_reservations
+
diff --git a/doc/dev/osd_internals/snaps.rst b/doc/dev/osd_internals/snaps.rst
new file mode 100644
index 000000000..5ebd0884a
--- /dev/null
+++ b/doc/dev/osd_internals/snaps.rst
@@ -0,0 +1,128 @@
+======
+Snaps
+======
+
+Overview
+--------
+Rados supports two related snapshotting mechanisms:
+
+  1. *pool snaps*: snapshots are implicitly applied to all objects
+     in a pool
+  2. *self managed snaps*: the user must provide the current *SnapContext*
+     on each write.
+
+These two are mutually exclusive, only one or the other can be used on
+a particular pool.
+
+The *SnapContext* is the set of snapshots currently defined for an object
+as well as the most recent snapshot (the *seq*) requested from the mon for
+sequencing purposes (a *SnapContext* with a newer *seq* is considered to
+be more recent).
+
+The difference between *pool snaps* and *self managed snaps* from the
+OSD's point of view lies in whether the *SnapContext* comes to the OSD
+via the client's MOSDOp or via the most recent OSDMap.
+
+See OSD::make_writeable
+
+Ondisk Structures
+-----------------
+Each object has in the PG collection a *head* object (or *snapdir*, which we
+will come to shortly) and possibly a set of *clone* objects.
+Each hobject_t has a snap field.  For the *head* (the only writeable version
+of an object), the snap field is set to CEPH_NOSNAP.  For the *clones*, the
+snap field is set to the *seq* of the *SnapContext* at their creation.
+When the OSD services a write, it first checks whether the most recent
+*clone* is tagged with a snapid prior to the most recent snap represented
+in the *SnapContext*.  If so, at least one snapshot has occurred between
+the time of the write and the time of the last clone.  Therefore, prior
+to performing the mutation, the OSD creates a new clone for servicing
+reads on snaps between the snapid of the last clone and the most recent
+snapid.
+
+The *head* object contains a *SnapSet* encoded in an attribute, which tracks
+
+  1. The full set of snaps defined for the object
+  2. The full set of clones which currently exist
+  3. Overlapping intervals between clones for tracking space usage
+  4. Clone size
+
+If the *head* is deleted while there are still clones, a *snapdir* object
+is created instead to house the *SnapSet*.
+
+Additionally, the *object_info_t* on each clone includes a vector of snaps
+for which clone is defined.
+
+Snap Removal
+------------
+To remove a snapshot, a request is made to the *Monitor* cluster to
+add the snapshot id to the list of purged snaps (or to remove it from
+the set of pool snaps in the case of *pool snaps*).  In either case,
+the *PG* adds the snap to its *snap_trimq* for trimming.
+
+A clone can be removed when all of its snaps have been removed.  In
+order to determine which clones might need to be removed upon snap
+removal, we maintain a mapping from snap to *hobject_t* using the
+*SnapMapper*.
+
+See PrimaryLogPG::SnapTrimmer, SnapMapper
+
+This trimming is performed asynchronously by the snap_trim_wq while the
+PG is clean and not scrubbing.
+
+  #. The next snap in PG::snap_trimq is selected for trimming
+  #. We determine the next object for trimming out of PG::snap_mapper.
+     For each object, we create a log entry and repop updating the
+     object info and the snap set (including adjusting the overlaps).
+     If the object is a clone which no longer belongs to any live snapshots,
+     it is removed here. (See PrimaryLogPG::trim_object() when new_snaps
+     is empty.)
+  #. We also locally update our *SnapMapper* instance with the object's
+     new snaps.
+  #. The log entry containing the modification of the object also
+     contains the new set of snaps, which the replica uses to update
+     its own *SnapMapper* instance.
+  #. The primary shares the info with the replica, which persists
+     the new set of purged_snaps along with the rest of the info.
+
+
+
+Recovery
+--------
+Because the trim operations are implemented using repops and log entries,
+normal PG peering and recovery maintain the snap trimmer operations with
+the caveat that push and removal operations need to update the local
+*SnapMapper* instance.  If the purged_snaps update is lost, we merely
+retrim a now empty snap.
+
+SnapMapper
+----------
+*SnapMapper* is implemented on top of map_cacher<string, bufferlist>,
+which provides an interface over a backing store such as the file system
+with async transactions.  While transactions are incomplete, the map_cacher
+instance buffers unstable keys allowing consistent access without having
+to flush the filestore.  *SnapMapper* provides two mappings:
+
+  1. hobject_t -> set<snapid_t>: stores the set of snaps for each clone
+     object
+  2. snapid_t -> hobject_t: stores the set of hobjects with the snapshot
+     as one of its snaps
+
+Assumption: there are lots of hobjects and relatively few snaps.  The
+first encoding has a stringification of the object as the key and an
+encoding of the set of snaps as a value.  The second mapping, because there
+might be many hobjects for a single snap, is stored as a collection of keys
+of the form stringify(snap)_stringify(object) such that stringify(snap)
+is constant length.  These keys have a bufferlist encoding
+pair<snapid, hobject_t> as a value.  Thus, creating or trimming a single
+object does not involve reading all objects for any snap.  Additionally,
+upon construction, the *SnapMapper* is provided with a mask for filtering
+the objects in the single SnapMapper keyspace belonging to that PG.
+
+Split
+-----
+The snapid_t -> hobject_t key entries are arranged such that for any PG,
+up to 8 prefixes need to be checked to determine all hobjects in a particular
+snap for a particular PG.  Upon split, the prefixes to check on the parent
+are adjusted such that only the objects remaining in the PG will be visible.
+The children will immediately have the correct mapping.
diff --git a/doc/dev/osd_internals/stale_read.rst b/doc/dev/osd_internals/stale_read.rst
new file mode 100644
index 000000000..67eac9d7f
--- /dev/null
+++ b/doc/dev/osd_internals/stale_read.rst
@@ -0,0 +1,101 @@
+Preventing Stale Reads
+======================
+
+We write synchronously to all replicas before sending an ack to the
+client, which ensures that we do not introduce potential inconsistency
+in the write path.  However, we only read from one replica, and the
+client will use whatever OSDMap is has to identify which OSD to read
+from.  In most cases, this is fine: either the client map is correct,
+or the OSD that we think is the primary for the object knows that it
+is not the primary anymore, and can feed the client an updated map
+indicating a newer primary.
+
+They key is to ensure that this is *always* true.  In particular, we
+need to ensure that an OSD that is fenced off from its peers and has
+not learned about a map update does not continue to service read
+requests from similarly stale clients at any point after which a new
+primary may have been allowed to make a write.
+
+We accomplish this via a mechanism that works much like a read lease.
+Each pool may have a ``read_lease_interval`` property which defines
+how long this is, although by default we simply set it to
+``osd_pool_default_read_lease_ratio`` (default: .8) times the
+``osd_heartbeat_grace``.  (This way the lease will generally have
+expired by the time we mark a failed OSD down.)
+
+readable_until
+--------------
+
+Primary and replica both track a couple of values:
+
+* *readable_until* is how long we are allowed to service (read)
+  requests before *our* "lease" expires.
+* *readable_until_ub* is an upper bound on *readable_until* for any
+  OSD in the acting set.
+
+The primary manages these two values by sending *pg_lease_t* messages
+to replicas that increase the upper bound.  Once all acting OSDs have
+acknowledged they've seen the higher bound, the primary increases its
+own *readable_until* and shares that (in a subsequent *pg_lease_t*
+message).  The resulting invariant is that any acting OSDs'
+*readable_until* is always <= any acting OSDs' *readable_until_ub*.
+
+In order to avoid any problems with clock skew, we use monotonic
+clocks (which are only accurate locally and unaffected by time
+adjustments) throughout to manage these leases.  Peer OSDs calculate
+upper and lower bounds on the deltas between OSD-local clocks,
+allowing the primary to share timestamps based on its local clock
+while replicas translate that to an appropriate bound in for their own
+local clocks.
+
+Prior Intervals
+---------------
+
+Whenever there is an interval change, we need to have an upper bound
+on the *readable_until* values for any OSDs in the prior interval.
+All OSDs from that interval have this value (*readable_until_ub*), and
+share it as part of the pg_history_t during peering.
+
+Because peering may involve OSDs that were not already communicating
+before and may not have bounds on their clock deltas, the bound in
+*pg_history_t* is shared as a simple duration before the upper bound
+expires.  This means that the bound slips forward in time due to the
+transit time for the peering message, but that is generally quite
+short, and moving the bound later in time is safe since it is an
+*upper* bound.
+
+PG "laggy" state
+----------------
+
+While the PG is active, *pg_lease_t* and *pg_lease_ack_t* messages are
+regularly exchanged.  However, if a client request comes in and the
+lease has expired (*readable_until* has passed), the PG will go into a
+*LAGGY* state and request will be blocked.  Once the lease is renewed,
+the request(s) will be requeued.
+
+PG "wait" state
+---------------
+
+If peering completes but the prior interval's OSDs may still be
+readable, the PG will go into the *WAIT* state until sufficient time
+has passed.  Any OSD requests will block during that period.  Recovery
+may proceed while in this state, since the logical, user-visible
+content of objects does not change.
+
+Dead OSDs
+---------
+
+Generally speaking, we need to wait until prior intervals' OSDs *know*
+that they should no longer be readable.  If an OSD is known to have
+crashed (e.g., because the process is no longer running, which we may
+infer because we get a ECONNREFUSED error), then we can infer that it
+is not readable.
+
+Similarly, if an OSD is marked down, gets a map update telling it so,
+and then informs the monitor that it knows it was marked down, we can
+similarly infer that it is not still serving requests for a prior interval.
+
+When a PG is in the *WAIT* state, it will watch new maps for OSDs'
+*dead_epoch* value indicating they are aware of their dead-ness.  If
+all down OSDs from prior interval are so aware, we can exit the WAIT
+state early.
diff --git a/doc/dev/osd_internals/watch_notify.rst b/doc/dev/osd_internals/watch_notify.rst
new file mode 100644
index 000000000..8c2ce09ba
--- /dev/null
+++ b/doc/dev/osd_internals/watch_notify.rst
@@ -0,0 +1,81 @@
+============
+Watch Notify
+============
+
+See librados for the watch/notify interface.
+
+Overview
+--------
+The object_info (See osd/osd_types.h) tracks the set of watchers for
+a particular object persistently in the object_info_t::watchers map.
+In order to track notify progress, we also maintain some ephemeral
+structures associated with the ObjectContext.
+
+Each Watch has an associated Watch object (See osd/Watch.h).  The
+ObjectContext for a watched object will have a (strong) reference
+to one Watch object per watch, and each Watch object holds a
+reference to the corresponding ObjectContext.  This circular reference
+is deliberate and is broken when the Watch state is discarded on
+a new peering interval or removed upon timeout expiration or an
+unwatch operation.
+
+A watch tracks the associated connection via a strong
+ConnectionRef Watch::conn.  The associated connection has a
+WatchConState stashed in the OSD::Session for tracking associated
+Watches in order to be able to notify them upon ms_handle_reset()
+(via WatchConState::reset()).
+
+Each Watch object tracks the set of currently un-acked notifies.
+start_notify() on a Watch object adds a reference to a new in-progress
+Notify to the Watch and either:
+
+* if the Watch is *connected*, sends a Notify message to the client
+* if the Watch is *unconnected*, does nothing.
+
+When the Watch becomes connected (in PrimaryLogPG::do_osd_op_effects),
+Notifies are resent to all remaining tracked Notify objects.
+
+Each Notify object tracks the set of un-notified Watchers via
+calls to complete_watcher().  Once the remaining set is empty or the
+timeout expires (cb, registered in init()) a notify completion
+is sent to the client.
+
+Watch Lifecycle
+---------------
+A watch may be in one of 5 states:
+
+1. Non existent.
+2. On disk, but not registered with an object context.
+3. Connected
+4. Disconnected, callback registered with timer
+5. Disconnected, callback in queue for scrub or is_degraded
+
+Case 2 occurs between when an OSD goes active and the ObjectContext
+for an object with watchers is loaded into memory due to an access.
+During Case 2, no state is registered for the watch.  Case 2
+transitions to Case 4 in PrimaryLogPG::populate_obc_watchers() during
+PrimaryLogPG::find_object_context.  Case 1 becomes case 3 via
+OSD::do_osd_op_effects due to a watch operation.  Case 4,5 become case
+3 in the same way. Case 3 becomes case 4 when the connection resets
+on a watcher's session.
+
+Cases 4&5 can use some explanation.  Normally, when a Watch enters Case
+4, a callback is registered with the OSDService::watch_timer to be
+called at timeout expiration.  At the time that the callback is
+called, however, the pg might be in a state where it cannot write
+to the object in order to remove the watch (i.e., during a scrub
+or while the object is degraded).  In that case, we use
+Watch::get_delayed_cb() to generate another Context for use from
+the callbacks_for_degraded_object and Scrubber::callbacks lists.
+In either case, Watch::unregister_cb() does the right thing
+(SafeTimer::cancel_event() is harmless for contexts not registered
+with the timer).
+
+Notify Lifecycle
+----------------
+The notify timeout is simpler: a timeout callback is registered when
+the notify is init()'d.  If all watchers ack notifies before the
+timeout occurs, the timeout is canceled and the client is notified
+of the notify completion.  Otherwise, the timeout fires, the Notify
+object pings each Watch via cancel_notify to remove itself, and
+sends the notify completion to the client early.
diff --git a/doc/dev/osd_internals/wbthrottle.rst b/doc/dev/osd_internals/wbthrottle.rst
new file mode 100644
index 000000000..9b67efbb6
--- /dev/null
+++ b/doc/dev/osd_internals/wbthrottle.rst
@@ -0,0 +1,28 @@
+==================
+Writeback Throttle
+==================
+
+Previously, the filestore had a problem when handling large numbers of
+small ios.  We throttle dirty data implicitly via the journal, but
+a large number of inodes can be dirtied without filling the journal
+resulting in a very long sync time when the sync finally does happen.
+The flusher was not an adequate solution to this problem since it
+forced writeback of small writes too eagerly killing performance.
+
+WBThrottle tracks unflushed io per hobject_t and ::fsyncs in lru
+order once the start_flusher threshold is exceeded for any of
+dirty bytes, dirty ios, or dirty inodes.  While any of these exceed
+the hard_limit, we block on throttle() in _do_op.
+
+See src/os/WBThrottle.h, src/osd/WBThrottle.cc
+
+To track the open FDs through the writeback process, there is now an
+fdcache to cache open fds.  lfn_open now returns a cached FDRef which
+implicitly closes the fd once all references have expired.
+
+Filestore syncs have a sideeffect of flushing all outstanding objects
+in the wbthrottle.
+
+lfn_unlink clears the cached FDRef and wbthrottle entries for the
+unlinked object when the last link is removed and asserts that all
+outstanding FDRefs for that object are dead.
diff --git a/doc/dev/peering.rst b/doc/dev/peering.rst
new file mode 100644
index 000000000..7ee5debc9
--- /dev/null
+++ b/doc/dev/peering.rst
@@ -0,0 +1,259 @@
+======================
+Peering
+======================
+
+Concepts
+--------
+
+*Peering*
+   the process of bringing all of the OSDs that store
+   a Placement Group (PG) into agreement about the state
+   of all of the objects (and their metadata) in that PG.
+   Note that agreeing on the state does not mean that
+   they all have the latest contents.
+
+*Acting set*
+   the ordered list of OSDs who are (or were as of some epoch)
+   responsible for a particular PG.
+
+*Up set*
+   the ordered list of OSDs responsible for a particular PG for
+   a particular epoch according to CRUSH.  Normally this
+   is the same as the *acting set*, except when the *acting set* has been
+   explicitly overridden via *PG temp* in the OSDMap.
+
+*PG temp* 
+   a temporary placement group acting set used while backfilling the
+   primary osd. Let say acting is [0,1,2] and we are
+   active+clean. Something happens and acting is now [3,1,2]. osd 3 is
+   empty and can't serve reads although it is the primary. osd.3 will
+   see that and request a *PG temp* of [1,2,3] to the monitors using a
+   MOSDPGTemp message so that osd.1 temporarily becomes the
+   primary. It will select osd.3 as a backfill peer and continue to
+   serve reads and writes while osd.3 is backfilled. When backfilling
+   is complete, *PG temp* is discarded and the acting set changes back
+   to [3,1,2] and osd.3 becomes the primary.
+
+*current interval* or *past interval*
+   a sequence of OSD map epochs during which the *acting set* and *up
+   set* for particular PG do not change
+
+*primary*
+   the (by convention first) member of the *acting set*,
+   who is responsible for coordination peering, and is
+   the only OSD that will accept client initiated
+   writes to objects in a placement group.
+
+*replica*
+   a non-primary OSD in the *acting set* for a placement group
+   (and who has been recognized as such and *activated* by the primary).
+
+*stray*
+   an OSD who is not a member of the current *acting set*, but
+   has not yet been told that it can delete its copies of a
+   particular placement group.
+
+*recovery*
+   ensuring that copies of all of the objects in a PG
+   are on all of the OSDs in the *acting set*.  Once
+   *peering* has been performed, the primary can start
+   accepting write operations, and *recovery* can proceed
+   in the background.
+
+*PG info* basic metadata about the PG's creation epoch, the version
+   for the most recent write to the PG, *last epoch started*, *last
+   epoch clean*, and the beginning of the *current interval*.  Any
+   inter-OSD communication about PGs includes the *PG info*, such that
+   any OSD that knows a PG exists (or once existed) also has a lower
+   bound on *last epoch clean* or *last epoch started*.
+
+*PG log*
+   a list of recent updates made to objects in a PG.
+   Note that these logs can be truncated after all OSDs
+   in the *acting set* have acknowledged up to a certain
+   point.
+
+*missing set*
+   Each OSD notes update log entries and if they imply updates to
+   the contents of an object, adds that object to a list of needed
+   updates.  This list is called the *missing set* for that <OSD,PG>.
+
+*Authoritative History*
+   a complete, and fully ordered set of operations that, if
+   performed, would bring an OSD's copy of a Placement Group
+   up to date.
+
+*epoch*
+   a (monotonically increasing) OSD map version number
+
+*last epoch start*
+   the last epoch at which all nodes in the *acting set*
+   for a particular placement group agreed on an
+   *authoritative history*.  At this point, *peering* is
+   deemed to have been successful.
+
+*up_thru*
+   before a primary can successfully complete the *peering* process,
+   it must inform a monitor that is alive through the current
+   OSD map epoch by having the monitor set its *up_thru* in the osd
+   map.  This helps peering ignore previous *acting sets* for which
+   peering never completed after certain sequences of failures, such as
+   the second interval below:
+
+   - *acting set* = [A,B]
+   - *acting set* = [A]
+   - *acting set* = [] very shortly after (e.g., simultaneous failure, but staggered detection)
+   - *acting set* = [B] (B restarts, A does not)
+
+*last epoch clean*
+   the last epoch at which all nodes in the *acting set*
+   for a particular placement group were completely
+   up to date (both PG logs and object contents).
+   At this point, *recovery* is deemed to have been
+   completed.
+
+Description of the Peering Process
+----------------------------------
+
+The *Golden Rule* is that no write operation to any PG
+is acknowledged to a client until it has been persisted
+by all members of the *acting set* for that PG.  This means
+that if we can communicate with at least one member of
+each *acting set* since the last successful *peering*, someone
+will have a record of every (acknowledged) operation
+since the last successful *peering*.
+This means that it should be possible for the current
+primary to construct and disseminate a new *authoritative history*.
+
+It is also important to appreciate the role of the OSD map
+(list of all known OSDs and their states, as well as some
+information about the placement groups) in the *peering*
+process:
+
+   When OSDs go up or down (or get added or removed)
+   this has the potential to affect the *active sets*
+   of many placement groups.
+
+   Before a primary successfully completes the *peering*
+   process, the OSD map must reflect that the OSD was alive
+   and well as of the first epoch in the *current interval*.
+
+   Changes can only be made after successful *peering*.
+
+Thus, a new primary can use the latest OSD map along with a recent
+history of past maps to generate a set of *past intervals* to
+determine which OSDs must be consulted before we can successfully
+*peer*.  The set of past intervals is bounded by *last epoch started*,
+the most recent *past interval* for which we know *peering* completed.
+The process by which an OSD discovers a PG exists in the first place is
+by exchanging *PG info* messages, so the OSD always has some lower
+bound on *last epoch started*.
+
+The high level process is for the current PG primary to:
+
+  1. get a recent OSD map (to identify the members of the all
+     interesting *acting sets*, and confirm that we are still the
+     primary).
+
+  #. generate a list of *past intervals* since *last epoch started*.
+     Consider the subset of those for which *up_thru* was greater than
+     the first interval epoch by the last interval epoch's OSD map; that is,
+     the subset for which *peering* could have completed before the *acting
+     set* changed to another set of OSDs.
+
+     Successful *peering* will require that we be able to contact at
+     least one OSD from each of *past interval*'s *acting set*.
+
+  #. ask every node in that list for its *PG info*, which includes the most
+     recent write made to the PG, and a value for *last epoch started*.  If
+     we learn about a *last epoch started* that is newer than our own, we can
+     prune older *past intervals* and reduce the peer OSDs we need to contact.
+
+  #. if anyone else has (in its PG log) operations that I do not have,
+     instruct them to send me the missing log entries so that the primary's
+     *PG log* is up to date (includes the newest write)..
+
+  #. for each member of the current *acting set*:
+
+     a. ask it for copies of all PG log entries since *last epoch start*
+	so that I can verify that they agree with mine (or know what
+	objects I will be telling it to delete).
+
+	If the cluster failed before an operation was persisted by all
+	members of the *acting set*, and the subsequent *peering* did not
+	remember that operation, and a node that did remember that
+	operation later rejoined, its logs would record a different
+	(divergent) history than the *authoritative history* that was
+	reconstructed in the *peering* after the failure.
+
+	Since the *divergent* events were not recorded in other logs
+	from that *acting set*, they were not acknowledged to the client,
+	and there is no harm in discarding them (so that all OSDs agree
+	on the *authoritative history*).  But, we will have to instruct
+	any OSD that stores data from a divergent update to delete the
+	affected (and now deemed to be apocryphal) objects.
+
+     #. ask it for its *missing set* (object updates recorded
+	in its PG log, but for which it does not have the new data).
+	This is the list of objects that must be fully replicated
+	before we can accept writes.
+
+  #. at this point, the primary's PG log contains an *authoritative history* of
+     the placement group, and the OSD now has sufficient
+     information to bring any other OSD in the *acting set* up to date.
+
+  #. if the primary's *up_thru* value in the current OSD map is not greater than
+     or equal to the first epoch in the *current interval*, send a request to the
+     monitor to update it, and wait until receive an updated OSD map that reflects
+     the change.
+
+  #. for each member of the current *acting set*:
+
+     a. send them log updates to bring their PG logs into agreement with
+	my own (*authoritative history*) ... which may involve deciding
+	to delete divergent objects.
+
+     #. await acknowledgment that they have persisted the PG log entries.
+
+  #. at this point all OSDs in the *acting set* agree on all of the meta-data,
+     and would (in any future *peering*) return identical accounts of all
+     updates.
+
+     a. start accepting client write operations (because we have unanimous
+	agreement on the state of the objects into which those updates are
+	being accepted).  Note, however, that if a client tries to write to an
+        object it will be promoted to the front of the recovery queue, and the
+        write willy be applied after it is fully replicated to the current *acting set*.
+
+     #. update the *last epoch started* value in our local *PG info*, and instruct
+	other *active set* OSDs to do the same.
+
+     #. start pulling object data updates that other OSDs have, but I do not.  We may
+	need to query OSDs from additional *past intervals* prior to *last epoch started*
+	(the last time *peering* completed) and following *last epoch clean* (the last epoch that
+	recovery completed) in order to find copies of all objects.
+
+     #. start pushing object data updates to other OSDs that do not yet have them.
+
+	We push these updates from the primary (rather than having the replicas
+	pull them) because this allows the primary to ensure that a replica has
+	the current contents before sending it an update write.  It also makes
+	it possible for a single read (from the primary) to be used to write
+	the data to multiple replicas.  If each replica did its own pulls,
+	the data might have to be read multiple times.
+
+  #. once all replicas store the all copies of all objects (that
+     existed prior to the start of this epoch) we can update *last
+     epoch clean* in the *PG info*, and we can dismiss all of the
+     *stray* replicas, allowing them to delete their copies of objects
+     for which they are no longer in the *acting set*.
+
+     We could not dismiss the *strays* prior to this because it was possible
+     that one of those *strays* might hold the sole surviving copy of an
+     old object (all of whose copies disappeared before they could be
+     replicated on members of the current *acting set*).
+
+State Model
+-----------
+
+.. graphviz:: peering_graph.generated.dot
diff --git a/doc/dev/perf.rst b/doc/dev/perf.rst
new file mode 100644
index 000000000..57742eec4
--- /dev/null
+++ b/doc/dev/perf.rst
@@ -0,0 +1,55 @@
+Using perf
+==========
+
+Top::
+
+  sudo perf top -p `pidof ceph-osd`
+
+To capture some data with call graphs::
+
+  sudo perf record -p `pidof ceph-osd` -F 99 --call-graph dwarf -- sleep 60
+
+To view by caller (where you can see what each top function calls)::
+
+  sudo perf report --call-graph caller
+
+To view by callee (where you can see who calls each top function)::  
+
+  sudo perf report --call-graph callee
+
+:note: If the caller/callee views look the same you may be
+       suffering from a kernel bug; upgrade to 4.8 or later.
+
+Common Issues
+-------------
+
+Ceph use `RelWithDebInfo` as its default `CMAKE_BUILD_TYPE`. Hence `-O2 -g` is
+used to compile the tree in this case. And the `-O2` optimization level
+enables `-fomit-frame-pointer` by default. But this prevents stack profilers
+from accessing the complete stack information. So one can disable this option
+when launching `cmake` ::
+
+  cmake -DCMAKE_CXX_FLAGS="-fno-omit-frame-pointer"
+
+or when building the tree::
+
+  make CMAKE_CXX_FLAGS="-fno-omit-frame-pointer"
+
+
+Flamegraphs
+-----------
+
+First, get things set up::
+
+  cd ~/src
+  git clone https://github.com/brendangregg/FlameGraph
+
+Run ceph, then record some perf data::
+
+  sudo perf record -p `pidof ceph-osd` -F 99 --call-graph dwarf -- sleep 60
+
+Then generate the flamegraph::
+
+  sudo perf script | ~/src/FlameGraph/stackcollapse-perf.pl > /tmp/folded
+  ~/src/FlameGraph/flamegraph.pl /tmp/folded > /tmp/perf.svg
+  firefox /tmp/perf.svg
diff --git a/doc/dev/perf_counters.rst b/doc/dev/perf_counters.rst
new file mode 100644
index 000000000..2f49f772f
--- /dev/null
+++ b/doc/dev/perf_counters.rst
@@ -0,0 +1,198 @@
+===============
+ Perf counters
+===============
+
+The perf counters provide generic internal infrastructure for gauges and counters.  The counted values can be both integer and float.  There is also an "average" type (normally float) that combines a sum and num counter which can be divided to provide an average.
+
+The intention is that this data will be collected and aggregated by a tool like ``collectd`` or ``statsd`` and fed into a tool like ``graphite`` for graphing and analysis.  Also, note the :doc:`../mgr/prometheus`.
+
+Access
+------
+
+The perf counter data is accessed via the admin socket.  For example::
+
+   ceph daemon osd.0 perf schema
+   ceph daemon osd.0 perf dump
+
+
+Collections
+-----------
+
+The values are grouped into named collections, normally representing a subsystem or an instance of a subsystem.  For example, the internal ``throttle`` mechanism reports statistics on how it is throttling, and each instance is named something like::
+
+
+    throttle-msgr_dispatch_throttler-hbserver
+    throttle-msgr_dispatch_throttler-client
+    throttle-filestore_bytes
+    ...
+
+
+Schema
+------
+
+The ``perf schema`` command dumps a json description of which values are available, and what their type is.  Each named value as a ``type`` bitfield, with the following bits defined.
+
++------+-------------------------------------+
+| bit  | meaning                             |
++======+=====================================+
+| 1    | floating point value                |
++------+-------------------------------------+
+| 2    | unsigned 64-bit integer value       |
++------+-------------------------------------+
+| 4    | average (sum + count pair), where   |
++------+-------------------------------------+
+| 8    | counter (vs gauge)                  |
++------+-------------------------------------+
+
+Every value will have either bit 1 or 2 set to indicate the type
+(float or integer).
+
+If bit 8 is set (counter), the value is monotonically increasing and
+the reader may want to subtract off the previously read value to get
+the delta during the previous interval.
+
+If bit 4 is set (average), there will be two values to read, a sum and
+a count.  If it is a counter, the average for the previous interval
+would be sum delta (since the previous read) divided by the count
+delta.  Alternatively, dividing the values outright would provide the
+lifetime average value.  Normally these are used to measure latencies
+(number of requests and a sum of request latencies), and the average
+for the previous interval is what is interesting.
+
+Instead of interpreting the bit fields, the ``metric type`` has a
+value of either ``guage`` or ``counter``, and the ``value type``
+property will be one of ``real``, ``integer``, ``real-integer-pair``
+(for a sum + real count pair), or ``integer-integer-pair`` (for a
+sum + integer count pair).
+
+Here is an example of the schema output::
+
+  {
+    "throttle-bluestore_throttle_bytes": {
+        "val": {
+            "type": 2,
+            "metric_type": "gauge",
+            "value_type": "integer",
+            "description": "Currently available throttle",
+            "nick": ""
+        },
+        "max": {
+            "type": 2,
+            "metric_type": "gauge",
+            "value_type": "integer",
+            "description": "Max value for throttle",
+            "nick": ""
+        },
+        "get_started": {
+            "type": 10,
+            "metric_type": "counter",
+            "value_type": "integer",
+            "description": "Number of get calls, increased before wait",
+            "nick": ""
+        },
+        "get": {
+            "type": 10,
+            "metric_type": "counter",
+            "value_type": "integer",
+            "description": "Gets",
+            "nick": ""
+        },
+        "get_sum": {
+            "type": 10,
+            "metric_type": "counter",
+            "value_type": "integer",
+            "description": "Got data",
+            "nick": ""
+        },
+        "get_or_fail_fail": {
+            "type": 10,
+            "metric_type": "counter",
+            "value_type": "integer",
+            "description": "Get blocked during get_or_fail",
+            "nick": ""
+        },
+        "get_or_fail_success": {
+            "type": 10,
+            "metric_type": "counter",
+            "value_type": "integer",
+            "description": "Successful get during get_or_fail",
+            "nick": ""
+        },
+        "take": {
+            "type": 10,
+            "metric_type": "counter",
+            "value_type": "integer",
+            "description": "Takes",
+            "nick": ""
+        },
+        "take_sum": {
+            "type": 10,
+            "metric_type": "counter",
+            "value_type": "integer",
+            "description": "Taken data",
+            "nick": ""
+        },
+        "put": {
+            "type": 10,
+            "metric_type": "counter",
+            "value_type": "integer",
+            "description": "Puts",
+            "nick": ""
+        },
+        "put_sum": {
+            "type": 10,
+            "metric_type": "counter",
+            "value_type": "integer",
+            "description": "Put data",
+            "nick": ""
+        },
+        "wait": {
+            "type": 5,
+            "metric_type": "gauge",
+            "value_type": "real-integer-pair",
+            "description": "Waiting latency",
+            "nick": ""
+        }
+  }
+
+
+Dump
+----
+
+The actual dump is similar to the schema, except that average values are grouped.  For example::
+
+ {
+   "throttle-msgr_dispatch_throttler-hbserver" : {
+      "get_or_fail_fail" : 0,
+      "get_sum" : 0,
+      "max" : 104857600,
+      "put" : 0,
+      "val" : 0,
+      "take" : 0,
+      "get_or_fail_success" : 0,
+      "wait" : {
+         "avgcount" : 0,
+         "sum" : 0
+      },
+      "get" : 0,
+      "take_sum" : 0,
+      "put_sum" : 0
+   },
+   "throttle-msgr_dispatch_throttler-client" : {
+      "get_or_fail_fail" : 0,
+      "get_sum" : 82760,
+      "max" : 104857600,
+      "put" : 2637,
+      "val" : 0,
+      "take" : 0,
+      "get_or_fail_success" : 0,
+      "wait" : {
+         "avgcount" : 0,
+         "sum" : 0
+      },
+      "get" : 2637,
+      "take_sum" : 0,
+      "put_sum" : 82760
+   }
+ }
+
diff --git a/doc/dev/perf_histograms.rst b/doc/dev/perf_histograms.rst
new file mode 100644
index 000000000..c277ac209
--- /dev/null
+++ b/doc/dev/perf_histograms.rst
@@ -0,0 +1,677 @@
+=================
+ Perf histograms
+=================
+
+The perf histograms build on perf counters infrastructure. Histograms are built for a number of counters and simplify gathering data on which groups of counter values occur most often over time. 
+Perf histograms are currently unsigned 64-bit integer counters, so they're mostly useful for time and sizes. Data dumped by perf histogram can then be feed into other analysis tools/scripts.
+
+Access
+------
+
+The perf histogram data are accessed via the admin socket.  For example::
+
+   ceph daemon osd.0 perf histogram schema
+   ceph daemon osd.0 perf histogram dump
+
+
+Collections
+-----------
+
+The histograms are grouped into named collections, normally representing a subsystem or an instance of a subsystem.  For example, the internal ``throttle`` mechanism reports statistics on how it is throttling, and each instance is named something like::
+
+
+    op_r_latency_out_bytes_histogram
+    op_rw_latency_in_bytes_histogram
+    op_rw_latency_out_bytes_histogram
+    ...
+
+
+Schema
+------
+
+The ``perf histogram schema`` command dumps a json description of which values are available, and what their type is.  Each named value as a ``type`` bitfield, with the 5-th bit always set and following bits defined.
+
++------+-------------------------------------+
+| bit  | meaning                             |
++======+=====================================+
+| 1    | floating point value                |
++------+-------------------------------------+
+| 2    | unsigned 64-bit integer value       |
++------+-------------------------------------+
+| 4    | average (sum + count pair)          |
++------+-------------------------------------+
+| 8    | counter (vs gauge)                  |
++------+-------------------------------------+
+
+In other words, histogram of type "18" is a histogram of unsigned 64-bit integer values (16 + 2).
+
+Here is an example of the schema output::
+
+    {
+        "AsyncMessenger::Worker-0": {},
+        "AsyncMessenger::Worker-1": {},
+        "AsyncMessenger::Worker-2": {},
+        "mutex-WBThrottle::lock": {},
+        "objecter": {},
+        "osd": {
+            "op_r_latency_out_bytes_histogram": {
+                "type": 18,
+                "description": "Histogram of operation latency (including queue time) +   da  ta read",
+                "nick": ""
+            },
+            "op_w_latency_in_bytes_histogram": {
+                "type": 18,
+                "description": "Histogram of operation latency (including queue time) +   da  ta written",
+                "nick": ""
+            },
+            "op_rw_latency_in_bytes_histogram": {
+                "type": 18,
+                "description": "Histogram of rw operation latency (including queue time)   +   data written",
+                "nick": ""
+            },
+            "op_rw_latency_out_bytes_histogram": {
+                "type": 18,
+                "description": "Histogram of rw operation latency (including queue time)   +   data read",
+                "nick": ""
+            }
+        }
+    }
+
+
+Dump
+----
+
+The actual dump is similar to the schema, except that there are actual value groups.  For example::
+
+    "osd": {
+        "op_r_latency_out_bytes_histogram": {
+            "axes": [
+                {
+                    "name": "Latency (usec)",
+                    "min": 0,
+                    "quant_size": 100000,
+                    "buckets": 32,
+                    "scale_type": "log2",
+                    "ranges": [
+                        {
+                            "max": -1
+                        },
+                        {
+                            "min": 0,
+                            "max": 99999
+                        },
+                        {
+                            "min": 100000,
+                            "max": 199999
+                        },
+                        {
+                            "min": 200000,
+                            "max": 399999
+                        },
+                        {
+                            "min": 400000,
+                            "max": 799999
+                        },
+                        {
+                            "min": 800000,
+                            "max": 1599999
+                        },
+                        {
+                            "min": 1600000,
+                            "max": 3199999
+                        },
+                        {
+                            "min": 3200000,
+                            "max": 6399999
+                        },
+                        {
+                            "min": 6400000,
+                            "max": 12799999
+                        },
+                        {
+                            "min": 12800000,
+                            "max": 25599999
+                        },
+                        {
+                            "min": 25600000,
+                            "max": 51199999
+                        },
+                        {
+                            "min": 51200000,
+                            "max": 102399999
+                        },
+                        {
+                            "min": 102400000,
+                            "max": 204799999
+                        },
+                        {
+                            "min": 204800000,
+                            "max": 409599999
+                        },
+                        {
+                            "min": 409600000,
+                            "max": 819199999
+                        },
+                        {
+                            "min": 819200000,
+                            "max": 1638399999
+                        },
+                        {
+                            "min": 1638400000,
+                            "max": 3276799999
+                        },
+                        {
+                            "min": 3276800000,
+                            "max": 6553599999
+                        },
+                        {
+                            "min": 6553600000,
+                            "max": 13107199999
+                        },
+                        {
+                            "min": 13107200000,
+                            "max": 26214399999
+                        },
+                        {
+                            "min": 26214400000,
+                            "max": 52428799999
+                        },
+                        {
+                            "min": 52428800000,
+                            "max": 104857599999
+                        },
+                        {
+                            "min": 104857600000,
+                            "max": 209715199999
+                        },
+                        {
+                            "min": 209715200000,
+                            "max": 419430399999
+                        },
+                        {
+                            "min": 419430400000,
+                            "max": 838860799999
+                        },
+                        {
+                            "min": 838860800000,
+                            "max": 1677721599999
+                        },
+                        {
+                            "min": 1677721600000,
+                            "max": 3355443199999
+                        },
+                        {
+                            "min": 3355443200000,
+                            "max": 6710886399999
+                        },
+                        {
+                            "min": 6710886400000,
+                            "max": 13421772799999
+                        },
+                        {
+                            "min": 13421772800000,
+                            "max": 26843545599999
+                        },
+                        {
+                            "min": 26843545600000,
+                            "max": 53687091199999
+                        },
+                        },
+                        {
+                            "min": 53687091200000
+                        }
+                    ]
+                },
+                {
+                    "name": "Request size (bytes)",
+                    "min": 0,
+                    "quant_size": 512,
+                    "buckets": 32,
+                    "scale_type": "log2",
+                    "ranges": [
+                        {
+                            "max": -1
+                        },
+                        {
+                            "min": 0,
+                            "max": 511
+                        },
+                        {
+                            "min": 512,
+                            "max": 1023
+                        },
+                        {
+                            "min": 1024,
+                            "max": 2047
+                        },
+                        {
+                            "min": 2048,
+                            "max": 4095
+                        },
+                        {
+                            "min": 4096,
+                            "max": 8191
+                        },
+                        {
+                            "min": 8192,
+                            "max": 16383
+                        },
+                        {
+                            "min": 16384,
+                            "max": 32767
+                        },
+                        {
+                            "min": 32768,
+                            "max": 65535
+                        },
+                        {
+                            "min": 65536,
+                            "max": 131071
+                        },
+                        {
+                            "min": 131072,
+                            "max": 262143
+                        },
+                        {
+                            "min": 262144,
+                            "max": 524287
+                        },
+                        {
+                            "min": 524288,
+                            "max": 1048575
+                        },
+                        {
+                            "min": 1048576,
+                            "max": 2097151
+                        },
+                        {
+                            "min": 2097152,
+                            "max": 4194303
+                        },
+                        {
+                            "min": 4194304,
+                            "max": 8388607
+                        },
+                        {
+                            "min": 8388608,
+                            "max": 16777215
+                        },
+                        {
+                            "min": 16777216,
+                            "max": 33554431
+                        },
+                        {
+                            "min": 33554432,
+                            "max": 67108863
+                        },
+                        {
+                            "min": 67108864,
+                            "max": 134217727
+                        },
+                        {
+                            "min": 134217728,
+                            "max": 268435455
+                        },
+                        {
+                            "min": 268435456,
+                            "max": 536870911
+                        },
+                        {
+                            "min": 536870912,
+                            "max": 1073741823
+                        },
+                        {
+                            "min": 1073741824,
+                            "max": 2147483647
+                        },
+                        {
+                            "min": 2147483648,
+                            "max": 4294967295
+                        },
+                        {
+                            "min": 4294967296,
+                            "max": 8589934591
+                        },
+                        {
+                            "min": 8589934592,
+                            "max": 17179869183
+                        },
+                        {
+                            "min": 17179869184,
+                            "max": 34359738367
+                        },
+                        {
+                            "min": 34359738368,
+                            "max": 68719476735
+                        },
+                        {
+                            "min": 68719476736,
+                            "max": 137438953471
+                        },
+                        {
+                            "min": 137438953472,
+                            "max": 274877906943
+                        },
+                        {
+                            "min": 274877906944
+                        }
+                    ]
+                }
+            ],
+            "values": [
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0
+                ]
+            ]
+        }
+    },
+
+This represents the 2d histogram, consisting of 9 history entrires and 32 value groups per each history entry. 
+"Ranges" element denote value bounds for each of value groups. "Buckets" denote amount of value groups ("buckets"), 
+"Min" is a minimum accepted valaue, "quant_size" is quantization unit and "scale_type" is either "log2" (logarhitmic
+scale) or "linear" (linear scale).
+You can use histogram_dump.py tool (see src/tools/histogram_dump.py) for quick visualisation of existing histogram
+data.
diff --git a/doc/dev/placement-group.rst b/doc/dev/placement-group.rst
new file mode 100644
index 000000000..e29be2fa6
--- /dev/null
+++ b/doc/dev/placement-group.rst
@@ -0,0 +1,210 @@
+============================
+ PG (Placement Group) notes
+============================
+
+Miscellaneous copy-pastes from emails, when this gets cleaned up it
+should move out of /dev.
+
+Overview
+========
+
+PG = "placement group". When placing data in the cluster, objects are
+mapped into PGs, and those PGs are mapped onto OSDs. We use the
+indirection so that we can group objects, which reduces the amount of
+per-object metadata we need to keep track of and processes we need to
+run (it would be prohibitively expensive to track eg the placement
+history on a per-object basis). Increasing the number of PGs can
+reduce the variance in per-OSD load across your cluster, but each PG
+requires a bit more CPU and memory on the OSDs that are storing it. We
+try and ballpark it at 100 PGs/OSD, although it can vary widely
+without ill effects depending on your cluster. You hit a bug in how we
+calculate the initial PG number from a cluster description.
+
+There are a couple of different categories of PGs; the 6 that exist
+(in the original emailer's ``ceph -s`` output) are "local" PGs which
+are tied to a specific OSD. However, those aren't actually used in a
+standard Ceph configuration.
+
+
+Mapping algorithm (simplified)
+==============================
+
+| > How does the Object->PG mapping look like, do you map more than one object on
+| > one PG, or do you sometimes map an object to more than one PG? How about the
+| > mapping of PGs to OSDs, does one PG belong to exactly one OSD?
+| >
+| > Does one PG represent a fixed amount of storage space?
+
+Many objects map to one PG.
+
+Each object maps to exactly one PG.
+
+One PG maps to a single list of OSDs, where the first one in the list
+is the primary and the rest are replicas.
+
+Many PGs can map to one OSD.
+
+A PG represents nothing but a grouping of objects; you configure the
+number of PGs you want, number of OSDs * 100 is a good starting point
+, and all of your stored objects are pseudo-randomly evenly distributed
+to the PGs. So a PG explicitly does NOT represent a fixed amount of
+storage; it represents 1/pg_num'th of the storage you happen to have
+on your OSDs.
+
+Ignoring the finer points of CRUSH and custom placement, it goes
+something like this in pseudocode::
+
+	locator = object_name
+	obj_hash = hash(locator)
+	pg = obj_hash % num_pg
+	OSDs_for_pg = crush(pg)  # returns a list of OSDs
+	primary = osds_for_pg[0]
+	replicas = osds_for_pg[1:]
+
+If you want to understand the crush() part in the above, imagine a
+perfectly spherical datacenter in a vacuum ;) that is, if all OSDs
+have weight 1.0, and there is no topology to the data center (all OSDs
+are on the top level), and you use defaults, etc, it simplifies to
+consistent hashing; you can think of it as::
+
+	def crush(pg):
+	   all_osds = ['osd.0', 'osd.1', 'osd.2', ...]
+	   result = []
+	   # size is the number of copies; primary+replicas
+	   while len(result) < size:
+	       r = hash(pg)
+	       chosen = all_osds[ r % len(all_osds) ]
+	       if chosen in result:
+	           # OSD can be picked only once
+	           continue
+	       result.append(chosen)
+	   return result
+
+User-visible PG States
+======================
+
+.. todo:: diagram of states and how they can overlap
+
+*creating*
+  the PG is still being created
+
+*active*
+  requests to the PG will be processed
+
+*clean*
+  all objects in the PG are replicated the correct number of times
+
+*down*
+  a replica with necessary data is down, so the pg is offline
+
+*recovery_unfound*
+  recovery could not finish because object(s) are unfound.
+
+*backfill_unfound*
+  backfill could not finish because object(s) are unfound.
+
+*premerge*
+  the PG is in a quiesced-IO state due to an impending PG merge.  That
+  happens when pg_num_pending < pg_num, and applies to the PGs with
+  pg_num_pending <= ps < pg_num as well as the corresponding peer PG
+  that it is merging with.
+
+*scrubbing*
+  the PG is being checked for inconsistencies
+
+*degraded*
+  some objects in the PG are not replicated enough times yet
+
+*inconsistent*
+  replicas of the PG are not consistent (e.g. objects are
+  the wrong size, objects are missing from one replica *after* recovery
+  finished, etc.)
+
+*peering*
+  the PG is undergoing the :doc:`/dev/peering` process
+
+*repair*
+  the PG is being checked and any inconsistencies found will be repaired (if possible)
+
+*recovering*
+  objects are being migrated/synchronized with replicas
+
+*backfill_wait*
+  the PG is waiting in line to start backfill
+
+*incomplete*
+  a pg is missing a necessary period of history from its
+  log.  If you see this state, report a bug, and try to start any
+  failed OSDs that may contain the needed information.
+
+*stale*
+  the PG is in an unknown state - the monitors have not received
+  an update for it since the PG mapping changed.
+
+*remapped*
+  the PG is temporarily mapped to a different set of OSDs from what
+  CRUSH specified
+
+*deep*
+  In conjunction with *scrubbing* the scrub is a deep scrub
+
+*backfilling*
+  a special case of recovery, in which the entire contents of
+  the PG are scanned and synchronized, instead of inferring what
+  needs to be transferred from the PG logs of recent operations
+
+*backfill_toofull*
+  backfill reservation rejected, OSD too full
+
+*recovery_wait*
+  the PG is waiting for the local/remote recovery reservations
+
+*undersized*
+  the PG can't select enough OSDs given its size
+
+*activating*
+  the PG is peered but not yet active
+
+*peered*
+  the PG peered but can't go active
+
+*snaptrim*
+  the PG is trimming snaps
+
+*snaptrim_wait*
+  the PG is queued to trim snaps
+
+*recovery_toofull*
+  recovery reservation rejected, OSD too full
+
+*snaptrim_error*
+  the PG could not complete snap trimming due to errors
+
+*forced_recovery*
+  the PG has been marked for highest priority recovery
+
+*forced_backfill*
+  the PG has been marked for highest priority backfill
+
+*failed_repair*
+  an attempt to repair the PG has failed.  Manual intervention is required.
+
+
+OMAP STATISTICS
+===============
+
+Omap statistics are gathered during deep scrub and displayed in the output of
+the following commands::
+
+  ceph pg dump
+  ceph pg dump all
+  ceph pg dump summary
+  ceph pg dump pgs
+  ceph pg dump pools
+  ceph pg ls
+
+As these statistics are not updated continuously they may be quite inaccurate in
+an environment where deep scrubs are run infrequently and/or there is a lot of
+omap activity. As such they should not be relied on for exact accuracy but
+rather used as a guide. Running a deep scrub and checking these statistics
+immediately afterwards should give a good indication of current omap usage.
diff --git a/doc/dev/quick_guide.rst b/doc/dev/quick_guide.rst
new file mode 100644
index 000000000..8e27dfb4b
--- /dev/null
+++ b/doc/dev/quick_guide.rst
@@ -0,0 +1,140 @@
+=================================
+ Developer Guide (Quick)
+=================================
+
+This guide will describe how to build and test Ceph for development.
+
+Development
+-----------
+
+The ``run-make-check.sh`` script will install Ceph dependencies,
+compile everything in debug mode and run a number of tests to verify
+the result behaves as expected.
+
+.. prompt:: bash $
+
+   ./run-make-check.sh
+
+Optionally if you want to work on a specific component of Ceph,
+install the dependencies and build Ceph in debug mode with required cmake flags.
+
+Example:
+
+.. prompt:: bash $
+
+   ./install-deps.sh
+   ./do_cmake.sh -DWITH_MANPAGE=OFF -DWITH_BABELTRACE=OFF -DWITH_MGR_DASHBOARD_FRONTEND=OFF
+
+Running a development deployment
+--------------------------------
+Ceph contains a script called ``vstart.sh`` (see also :doc:`/dev/dev_cluster_deployement`) which allows developers to quickly test their code using
+a simple deployment on your development system. Once the build finishes successfully, start the ceph
+deployment using the following command:
+
+.. prompt:: bash $
+
+   cd ceph/build  # Assuming this is where you ran cmake
+   make vstart
+   ../src/vstart.sh -d -n -x
+
+You can also configure ``vstart.sh`` to use only one monitor and one metadata server by using the following:
+
+.. prompt:: bash $
+
+   MON=1 MDS=1 ../src/vstart.sh -d -n -x
+
+The system creates two pools on startup: `cephfs_data_a` and `cephfs_metadata_a`.  Let's get some stats on
+the current pools:
+
+.. code-block:: console
+
+  $ bin/ceph osd pool stats
+  *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
+  pool cephfs_data_a id 1
+    nothing is going on
+	
+  pool cephfs_metadata_a id 2
+    nothing is going on
+	
+  $ bin/ceph osd pool stats cephfs_data_a
+  *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
+  pool cephfs_data_a id 1
+    nothing is going on
+
+  $ bin/rados df
+  POOL_NAME         USED OBJECTS CLONES COPIES MISSING_ON_PRIMARY UNFOUND DEGRADED RD_OPS RD WR_OPS WR
+  cephfs_data_a        0       0      0      0                  0       0        0      0  0      0    0
+  cephfs_metadata_a 2246      21      0     63                  0       0        0      0  0     42 8192
+
+  total_objects    21
+  total_used       244G
+  total_space      1180G
+
+
+Make a pool and run some benchmarks against it:
+
+.. prompt:: bash $
+
+   bin/ceph osd pool create mypool
+   bin/rados -p mypool bench 10 write -b 123
+
+Place a file into the new pool:
+
+.. prompt:: bash $
+
+   bin/rados -p mypool put objectone <somefile>
+   bin/rados -p mypool put objecttwo <anotherfile>
+
+List the objects in the pool:
+
+.. prompt:: bash $
+
+   bin/rados -p mypool ls
+
+Once you are done, type the following to stop the development ceph deployment:
+
+.. prompt:: bash $
+
+   ../src/stop.sh
+
+Resetting your vstart environment
+---------------------------------
+
+The vstart script creates out/ and dev/ directories which contain
+the cluster's state.  If you want to quickly reset your environment,
+you might do something like this:
+
+.. prompt:: bash [build]$
+
+   ../src/stop.sh
+   rm -rf out dev
+   MDS=1 MON=1 OSD=3 ../src/vstart.sh -n -d
+
+Running a RadosGW development environment
+-----------------------------------------
+
+Set the ``RGW`` environment variable when running vstart.sh to enable the RadosGW.
+
+.. prompt:: bash $
+
+   cd build
+   RGW=1 ../src/vstart.sh -d -n -x
+
+You can now use the swift python client to communicate with the RadosGW.
+
+.. prompt:: bash $
+
+   swift -A http://localhost:8000/auth -U test:tester -K testing list
+   swift -A http://localhost:8000/auth -U test:tester -K testing upload mycontainer ceph
+   swift -A http://localhost:8000/auth -U test:tester -K testing list
+
+
+Run unit tests
+--------------
+
+The tests are located in `src/tests`.  To run them type:
+
+.. prompt:: bash $
+
+   make check
+
diff --git a/doc/dev/rados-client-protocol.rst b/doc/dev/rados-client-protocol.rst
new file mode 100644
index 000000000..920c65f39
--- /dev/null
+++ b/doc/dev/rados-client-protocol.rst
@@ -0,0 +1,117 @@
+RADOS client protocol
+=====================
+
+This is very incomplete, but one must start somewhere.
+
+Basics
+------
+
+Requests are MOSDOp messages.  Replies are MOSDOpReply messages.
+
+An object request is targeted at an hobject_t, which includes a pool,
+hash value, object name, placement key (usually empty), and snapid.
+
+The hash value is a 32-bit hash value, normally generated by hashing
+the object name.  The hobject_t can be arbitrarily constructed,
+though, with any hash value and name.  Note that in the MOSDOp these
+components are spread across several fields and not logically
+assembled in an actual hobject_t member (mainly historical reasons).
+
+A request can also target a PG.  In this case, the *ps* value matches
+a specific PG, the object name is empty, and (hopefully) the ops in
+the request are PG ops.
+
+Either way, the request ultimately targets a PG, either by using the
+explicit pgid or by folding the hash value onto the current number of
+pgs in the pool.  The client sends the request to the primary for the
+associated PG.
+
+Each request is assigned a unique tid.
+
+Resends
+-------
+
+If there is a connection drop, the client will resend any outstanding
+requests.
+
+Any time there is a PG mapping change such that the primary changes,
+the client is responsible for resending the request.  Note that
+although there may be an interval change from the OSD's perspective
+(triggering PG peering), if the primary doesn't change then the client
+need not resend.
+
+There are a few exceptions to this rule:
+
+ * There is a last_force_op_resend field in the pg_pool_t in the
+   OSDMap.  If this changes, then the clients are forced to resend any
+   outstanding requests. (This happens when tiering is adjusted, for
+   example.)
+ * Some requests are such that they are resent on *any* PG interval
+   change, as defined by pg_interval_t's is_new_interval() (the same
+   criteria used by peering in the OSD).
+ * If the PAUSE OSDMap flag is set and unset.
+
+Each time a request is sent to the OSD the *attempt* field is incremented. The
+first time it is 0, the next 1, etc.
+
+Backoff
+-------
+
+Ordinarily the OSD will simply queue any requests it can't immediately
+process in memory until such time as it can.  This can become
+problematic because the OSD limits the total amount of RAM consumed by
+incoming messages: if either of the thresholds for the number of
+messages or the number of bytes is reached, new messages will not be
+read off the network socket, causing backpressure through the network.
+
+In some cases, though, the OSD knows or expects that a PG or object
+will be unavailable for some time and does not want to consume memory
+by queuing requests.  In these cases it can send a MOSDBackoff message
+to the client.
+
+A backoff request has four properties:
+
+#. the op code (block, unblock, or ack-block)
+#. *id*, a unique id assigned within this session
+#. hobject_t begin
+#. hobject_t end
+
+There are two types of backoff: a *PG* backoff will plug all requests
+targeting an entire PG at the client, as described by a range of the
+hash/hobject_t space [begin,end), while an *object* backoff will plug
+all requests targeting a single object (begin == end).
+
+When the client receives a *block* backoff message, it is now
+responsible for *not* sending any requests for hobject_ts described by
+the backoff.  The backoff remains in effect until the backoff is
+cleared (via an 'unblock' message) or the OSD session is closed.  A
+*ack_block* message is sent back to the OSD immediately to acknowledge
+receipt of the backoff.
+
+When an unblock is
+received, it will reference a specific id that the client previous had
+blocked.  However, the range described by the unblock may be smaller
+than the original range, as the PG may have split on the OSD.  The unblock
+should *only* unblock the range specified in the unblock message.  Any requests
+that fall within the unblock request range are reexamined and, if no other
+installed backoff applies, resent.
+
+On the OSD, Backoffs are also tracked across ranges of the hash space, and
+exist in three states:
+
+#. new
+#. acked
+#. deleting
+
+A newly installed backoff is set to *new* and a message is sent to the
+client.  When the *ack-block* message is received it is changed to the
+*acked* state.  The OSD may process other messages from the client that
+are covered by the backoff in the *new* state, but once the backoff is
+*acked* it should never see a blocked request unless there is a bug.
+
+If the OSD wants to a remove a backoff in the *acked* state it can
+simply remove it and notify the client.  If the backoff is in the
+*new* state it must move it to the *deleting* state and continue to
+use it to discard client requests until the *ack-block* message is
+received, at which point it can finally be removed.  This is necessary to
+preserve the order of operations processed by the OSD.
diff --git a/doc/dev/radosgw/admin/adminops_nonimplemented.rst b/doc/dev/radosgw/admin/adminops_nonimplemented.rst
new file mode 100644
index 000000000..e579bd5aa
--- /dev/null
+++ b/doc/dev/radosgw/admin/adminops_nonimplemented.rst
@@ -0,0 +1,495 @@
+==================
+ Admin Operations
+==================
+
+An admin API request will be done on a URI that starts with the configurable 'admin'
+resource entry point. Authorization for the admin API duplicates the S3 authorization
+mechanism. Some operations require that the user holds special administrative capabilities.
+The response entity type (XML or JSON) may be specified as the 'format' option in the
+request and defaults to JSON if not specified.
+
+Get Object
+==========
+
+Get an existing object. NOTE: Does not require owner to be non-suspended.
+
+Syntax
+~~~~~~
+
+::
+
+	GET /{admin}/bucket?object&format=json HTTP/1.1
+	Host {fqdn}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``bucket``
+
+:Description: The bucket containing the object to be retrieved.
+:Type: String
+:Example: ``foo_bucket``
+:Required: Yes
+
+``object``
+
+:Description: The object to be retrieved.
+:Type: String
+:Example: ``foo.txt``
+:Required: Yes
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If successful, returns the desired object.
+
+``object``
+
+:Description: The desired object.
+:Type: Object
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``NoSuchObject``
+
+:Description: Specified object does not exist.
+:Code: 404 Not Found
+
+Head Object
+===========
+
+Verify the existence of an object. If the object exists,
+metadata headers for the object will be returned.
+
+Syntax
+~~~~~~
+
+::
+
+	HEAD /{admin}/bucket?object HTTP/1.1
+	Host {fqdn}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``bucket``
+
+:Description: The bucket containing the object to be retrieved.
+:Type: String
+:Example: ``foo_bucket``
+:Required: Yes
+
+``object``
+
+:Description: The object to be retrieved.
+:Type: String
+:Example: ``foo.txt``
+:Required: Yes
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+None.
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``NoSuchObject``
+
+:Description: Specified object does not exist.
+:Code: 404 Not Found
+
+Get Zone Info
+=============
+
+Get cluster information.
+
+Syntax
+~~~~~~
+
+::
+
+	GET /{admin}/zone&format=json HTTP/1.1
+	Host {fqdn}
+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If successful, returns cluster pool configuration.
+
+``zone``
+
+:Description: Contains current cluster pool configuration.
+:Type: Container
+
+``domain_root``
+
+:Description: root of all buckets.
+:Type: String
+:Parent: ``cluster``
+
+``control_pool``
+
+:Description: 
+:Type: String
+:Parent: ``cluster``
+
+``gc_pool``
+
+:Description: Garbage collection pool.
+:Type: String
+:Parent: ``cluster``
+
+``log_pool``
+
+:Description: Log pool.
+:Type: String
+:Parent: ``cluster``
+
+``intent_log_pool``
+
+:Description: Intent log pool.
+:Type: String
+:Parent: ``cluster``
+
+``usage_log_pool``
+
+:Description: Usage log pool.
+:Type: String
+:Parent: ``cluster``
+
+``user_keys_pool``
+
+:Description: User key pool.
+:Type: String
+:Parent: ``cluster``
+
+``user_email_pool``
+
+:Description: User email pool.
+:Type: String
+:Parent: ``cluster``
+
+``user_swift_pool``
+
+:Description: Pool of swift users.
+:Type: String
+:Parent: ``cluster``
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+None.
+
+Example Response
+~~~~~~~~~~~~~~~~
+
+::
+
+	HTTP/1.1 200
+	Content-Type: application/json
+
+	{
+	  "domain_root": ".rgw",
+	  "control_pool": ".rgw.control",
+	  "gc_pool": ".rgw.gc",
+	  "log_pool": ".log",
+	  "intent_log_pool": ".intent-log",
+	  "usage_log_pool": ".usage",
+	  "user_keys_pool": ".users",
+	  "user_email_pool": ".users.email",
+	  "user_swift_pool": ".users.swift",
+	  "user_uid_pool ": ".users.uid"
+	}
+
+
+
+Add Placement Pool
+==================
+
+Make a pool available for data placement.
+
+Syntax
+~~~~~~
+
+::
+
+	PUT /{admin}/pool?format=json HTTP/1.1
+	Host {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``pool``
+
+:Description: The pool to be made available for data placement.
+:Type: String
+:Example: ``foo_pool``
+:Required: Yes
+
+``create``
+
+:Description: Creates the data pool if it does not exist.
+:Type: Boolean
+:Example: False [False]
+:Required: No
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+TBD.
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+TBD.
+
+Remove Placement Pool
+=====================
+
+Make a pool unavailable for data placement.
+
+Syntax
+~~~~~~
+
+::
+
+	DELETE /{admin}/pool?format=json HTTP/1.1
+	Host {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``pool``
+
+:Description: The existing pool to be made available for data placement.
+:Type: String
+:Example: ``foo_pool``
+:Required: Yes
+
+``destroy``
+
+:Description: Destroys the pool after removing it from the active set.
+:Type: Boolean
+:Example: False [False]
+:Required: No
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+TBD.
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+TBD.
+
+List Available Data Placement Pools
+===================================
+
+List current pools available for data placement.
+
+Syntax
+~~~~~~
+
+::
+
+	GET /{admin}/pool?format=json HTTP/1.1
+	Host {fqdn}
+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If successful, returns a list of pools available for data placement.
+
+``pools``
+
+:Description: Contains currently available pools for data placement.
+:Type: Container
+
+
+
+List Expired Garbage Collection Items
+=====================================
+
+List objects scheduled for garbage collection.
+
+Syntax
+~~~~~~
+
+::
+
+	GET /{admin}/garbage?format=json HTTP/1.1
+	Host {fqdn}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+None.
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If expired garbage collection items exist, a list of such objects
+will be returned.
+
+``garbage``
+
+:Description: Expired garbage collection items.
+:Type: Container
+
+``object``
+
+:Description: A container garbage collection object information.
+:Type: Container
+:Parent: ``garbage``
+
+``name``
+
+:Description: The name of the object.
+:Type: String
+:Parent: ``object``
+
+``expired``
+
+:Description: The date at which the object expired.
+:Type: String
+:Parent: ``object``
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+TBD.
+
+Manually Processes Garbage Collection Items
+===========================================
+
+List objects scheduled for garbage collection.
+
+Syntax
+~~~~~~
+
+::
+
+	DELETE /{admin}/garbage?format=json HTTP/1.1
+	Host {fqdn}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+None.
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If expired garbage collection items exist, a list of removed objects
+will be returned.
+
+``garbage``
+
+:Description: Expired garbage collection items.
+:Type: Container
+
+``object``
+
+:Description: A container garbage collection object information.
+:Type: Container
+:Parent: ``garbage``
+
+``name``
+
+:Description: The name of the object.
+:Type: String
+:Parent: ``object``
+
+``expired``
+
+:Description: The date at which the object expired.
+:Type: String
+:Parent: ``object``
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+TBD.
+
+Show Log Objects
+================
+
+Show log objects
+
+Syntax
+~~~~~~
+
+::
+
+	GET /{admin}/log?format=json HTTP/1.1
+	Host {fqdn}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``object``
+
+:Description: The log object to return.
+:Type: String:
+:Example: ``2012-10-11-09-4165.2-foo_bucket``
+:Required: No
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If no object is specified, returns the full list of log objects.
+
+``log-objects``
+
+:Description: A list of log objects.
+:Type: Container
+
+``object``
+
+:Description: The name of the log object.
+:Type: String
+
+``log``
+
+:Description: The contents of the log object.
+:Type: Container
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+None.
+
+Standard Error Responses
+========================
+
+``AccessDenied``
+
+:Description: Access denied.
+:Code: 403 Forbidden
+
+``InternalError``
+
+:Description: Internal server error.
+:Code: 500 Internal Server Error
+
+``NoSuchUser``
+
+:Description: User does not exist.
+:Code: 404 Not Found
+
+``NoSuchBucket``
+
+:Description: Bucket does not exist.
+:Code: 404 Not Found
+
+``NoSuchKey``
+
+:Description: No such access key.
+:Code: 404 Not Found
diff --git a/doc/dev/radosgw/index.rst b/doc/dev/radosgw/index.rst
new file mode 100644
index 000000000..5f77609d2
--- /dev/null
+++ b/doc/dev/radosgw/index.rst
@@ -0,0 +1,13 @@
+=======================================
+ RADOS Gateway developer documentation
+=======================================
+
+.. rubric:: Contents
+
+.. toctree::
+   :maxdepth: 1   
+   
+   
+   usage
+   Admin Ops Nonimplemented <admin/adminops_nonimplemented>
+   s3_compliance
diff --git a/doc/dev/radosgw/s3_compliance.rst b/doc/dev/radosgw/s3_compliance.rst
new file mode 100644
index 000000000..50aeda36a
--- /dev/null
+++ b/doc/dev/radosgw/s3_compliance.rst
@@ -0,0 +1,304 @@
+===============================
+Rados Gateway S3 API Compliance
+===============================
+
+.. warning::
+	This document is a draft, it might not be accurate
+
+----------------------
+Naming code reference
+----------------------
+
+Here comes a BNF definition on how to name a feature in the code for referencing purpose : ::
+
+    name ::= request_type "_" ( header | operation ) ( "_" header_option )?
+    
+    request_type ::= "req" | "res"
+    
+    header ::= string
+    
+    operation ::= method resource
+    
+    method ::= "GET" | "PUT" | "POST" | "DELETE" | "OPTIONS" | "HEAD"
+    
+    resource ::= string
+    
+    header_option ::= string
+
+----------------------
+Common Request Headers
+----------------------
+
+S3 Documentation reference : http://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonRequestHeaders.html
+
++----------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| Header               | Supported? | Code Links                                                                                              | Tests links |
++======================+============+=========================================================================================================+=============+
+| Authorization        | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1962 |             |
+|                      |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L2051 |             |
++----------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| Content-Length       | Yes        |                                                                                                         |             |
++----------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| Content-Type         | Yes        |                                                                                                         |             |
++----------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| Content-MD5          | Yes        | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1249      |             |
+|                      |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1306      |             |
++----------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| Date                 | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_auth_s3.cc#L164  |             |
++----------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| Expect               | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest.cc#L1227    |             |
+|                      |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L802  |             |
+|                      |            | https://github.com/ceph/ceph/blob/76040d90f7eb9f9921a3b8dcd0f821ac2cd9c492/src/rgw/rgw_main.cc#L372     |             |
++----------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| Host                 | ?          |                                                                                                         |             |
++----------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| x-amz-date           | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_auth_s3.cc#L169  |             |
+|                      |            | should take precedence over DATE as mentioned here ->                                                   |             |
+|                      |            | http://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonRequestHeaders.html                            |             |
++----------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| x-amz-security-token | No         |                                                                                                         |             |
++----------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+
+-----------------------
+Common Response Headers
+-----------------------
+
+S3 Documentation reference : http://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html
+
++---------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| Header              | Supported? | Code Links                                                                                              | Tests links |
++=====================+============+=========================================================================================================+=============+
+| Content-Length      | Yes        |                                                                                                         |             |
++---------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| Connection          | ?          |                                                                                                         |             |
++---------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| Date                | ?          |                                                                                                         |             |
++---------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| ETag                | Yes        | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1312      |             |
+|                     |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1436      |             |
+|                     |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L2222      |             |
+|                     |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L118  |             |
+|                     |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L268  |             |
+|                     |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L516  |             |
+|                     |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1336 |             |
+|                     |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1486 |             |
+|                     |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1548 |             |
++---------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| Server              | No         |                                                                                                         |             |
++---------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| x-amz-delete-marker | No         |                                                                                                         |             |
++---------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| x-amz-id-2          | No         |                                                                                                         |             |
++---------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| x-amz-request-id    | Yes        | https://github.com/ceph/ceph/commit/b711e3124f8f73c17ebd19b38807a1b77f201e44                            |             |
++---------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| x-amz-version-id    | No         |                                                                                                         |             |
++---------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+
+-------------------------
+Operations on the Service
+-------------------------
+
+S3 Documentation reference : http://docs.aws.amazon.com/AmazonS3/latest/API/RESTServiceOps.html
+
++------+-----------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| Type | Operation | Supported? | Code links                                                                                              | Tests links |
++======+===========+============+=========================================================================================================+=============+
+| GET  | Service   | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L2094 |             |
+|      |           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1676 |             |
+|      |           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L185  |             |
++------+-----------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+
+---------------------
+Operations on Buckets
+---------------------
+
+S3 Documentation reference : http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketOps.html
+
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| Type   | Operation              | Supported? | Code links                                                                                                 | Tests links |
++========+========================+============+============================================================================================================+=============+
+| DELETE | Bucket                 | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1728    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/e91042171939b6bf82a56a1015c5cae792d228ad/src/rgw/rgw_rest_bucket.cc#L250 |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/e91042171939b6bf82a56a1015c5cae792d228ad/src/rgw/rgw_rest_bucket.cc#L212 |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/25948319c4d256c4aeb0137eb88947e54d14cc79/src/rgw/rgw_bucket.cc#L856      |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/25948319c4d256c4aeb0137eb88947e54d14cc79/src/rgw/rgw_bucket.cc#L513      |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/25948319c4d256c4aeb0137eb88947e54d14cc79/src/rgw/rgw_bucket.cc#L286      |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L461     |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| DELETE | Bucket cors            | ?          | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1731    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1916         |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| DELETE | Bucket lifecycle       | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| DELETE | Bucket policy          | ?          |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| DELETE | Bucket tagging         | ?          |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| DELETE | Bucket website         | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | Bucket                 | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1676    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L185     |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | Bucket acl             | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1697    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1728         |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1344    |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | Bucket cors            | ?          | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1698    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1845         |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/76040d90f7eb9f9921a3b8dcd0f821ac2cd9c492/src/rgw/rgw_main.cc#L345        |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | Bucket lifecycle       | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | Bucket location        | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | Bucket policy          | ?          | https://github.com/ceph/ceph/blob/e91042171939b6bf82a56a1015c5cae792d228ad/src/rgw/rgw_rest_bucket.cc#L232 |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/e91042171939b6bf82a56a1015c5cae792d228ad/src/rgw/rgw_rest_bucket.cc#L58  |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | Bucket logging         | ?          | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1695    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L287     |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | Bucket notification    | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | Bucket tagging         | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | Bucket Object versions | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | Bucket requestPayment  | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | Bucket versioning      | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | Bucket website         | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| GET    | List Multipart uploads | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1701    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest.cc#L877        |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L2355         |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L2363         |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| HEAD   | Bucket                 | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1713    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1689    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L826          |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L834          |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| PUT    | Bucket                 | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1725    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L382     |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L437     |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L901          |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L945          |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| PUT    | Bucket acl             | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1721    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1354    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1373    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1739         |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1753         |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| PUT    | Bucket cors            | ?          | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1723    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1398    |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1858         |             |
+|        |                        |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1866         |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| PUT    | Bucket lifecycle       | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| PUT    | Bucket policy          | ?          |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| PUT    | Bucket logging         | ?          |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| PUT    | Bucket notification    | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| PUT    | Bucket tagging         | ?          |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| PUT    | Bucket requestPayment  | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| PUT    | Bucket versioning      | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+| PUT    | Bucket website         | No         |                                                                                                            |             |
++--------+------------------------+------------+------------------------------------------------------------------------------------------------------------+-------------+
+
+---------------------
+Operations on Objects
+---------------------
+
+S3 Documentation reference : http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectOps.html
+
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| Type    | Operation                 | Supported? | Code links                                                                                              | Tests links |
++=========+===========================+============+=========================================================================================================+=============+
+| DELETE  | Object                    | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1796 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1516      |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1524      |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| DELETE  | Multiple objects          | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1739 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1616 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1626 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1641 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1667 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1516      |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1524      |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| GET     | Object                    | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1767 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L71   |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L397       |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L424       |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L497       |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L562       |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L626       |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L641       |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L706       |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| GET     | Object acl                | Yes        |                                                                                                         |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| GET     | Object torrent            | No         |                                                                                                         |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| HEAD    | Object                    | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1777 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L71   |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L397       |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L424       |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L497       |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L562       |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L626       |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L641       |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L706       |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| OPTIONS | Object                    | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1814 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1418 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1951      |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1968      |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1993      |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| POST    | Object                    | Yes        | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1742 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L631  |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L694  |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L700  |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L707  |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L759  |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L771  |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L781  |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L795  |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L929  |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1037 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1059 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/8a2eb18494005aa968b71f18121da8ebab48e950/src/rgw/rgw_rest_s3.cc#L1134 |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1344      |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1360      |             |
+|         |                           |            | https://github.com/ceph/ceph/blob/b139a7cd34b4e203ab164ada7a8fa590b50d8b13/src/rgw/rgw_op.cc#L1365      |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| POST    | Object restore            | ?          |                                                                                                         |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| PUT     | Object                    | Yes        |                                                                                                         |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| PUT     | Object acl                | Yes        |                                                                                                         |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| PUT     | Object copy               | Yes        |                                                                                                         |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| PUT     | Initate multipart upload  | Yes        |                                                                                                         |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| PUT     | Upload Part               | Yes        |                                                                                                         |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| PUT     | Upload Part copy          | ?          |                                                                                                         |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| PUT     | Complete multipart upload | Yes        |                                                                                                         |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| PUT     | Abort multipart upload    | Yes        |                                                                                                         |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
+| PUT     | List parts                | Yes        |                                                                                                         |             |
++---------+---------------------------+------------+---------------------------------------------------------------------------------------------------------+-------------+
diff --git a/doc/dev/radosgw/usage.rst b/doc/dev/radosgw/usage.rst
new file mode 100644
index 000000000..6c856fc7f
--- /dev/null
+++ b/doc/dev/radosgw/usage.rst
@@ -0,0 +1,84 @@
+============================
+Usage Design Overview
+============================
+
+
+
+
+Testing
+-------
+
+The current usage testing does the following:
+
+Following these operations:
+
+ - Create a few buckets
+ - Remove buckets
+ - Create a bucket
+ - Put object
+ - Remove object
+
+Test:
+
+1. Verify that 'usage show' with delete_obj category isn't empty after no more than 45 seconds (wait to flush)
+2. Check the following
+
+  - 'usage show'
+
+    - does not error out
+    - num of entries > 0
+    - num of summary entries > 0
+    - for every entry in categories check successful_ops > 0
+    - check that correct uid in the user summary
+
+
+  - 'usage show' with specified uid (--uid=<uid>')
+
+    - num of entries > 0
+    - num of summary entries > 0
+    - for every entry in categories check successful_ops > 0
+    - check that correct uid in the user summary
+
+  - 'usage show' with specified uid and specified categories (create_bucket,
+    put_obj, delete_obj, delete_bucket)
+
+    - for each category:
+      - does not error out
+      - num of entries > 0
+      - user in user summary is correct user
+      - length of categories entries under user summary is exactly 1
+      - name of category under user summary is correct name
+      - successful ops for the category > 0
+
+  - 'usage trim' with specified uid
+    - does not error
+    - check following 'usage show' shows complete usage info cleared for user
+
+
+Additional required testing:
+
+ - test multiple users
+
+   Do the same as in (2), with multiple users being set up.
+
+ - test with multiple buckets (> 1000 * factor, e.g., 2000)
+
+   Create multiple buckets, put objects in each. Account the number written data and verify
+   that usage reports show the expected number (up to a certain delta).
+
+ - verify usage show with a date/time range
+
+   Take timestamp of the beginning of the test, and the end of the test. Round timestamps to the
+   nearest hour (downward from start of test, upward from the end of test). List data starting
+   at end-time, make sure that no data is being shown. List data ending at start-time, make sure
+   that no data is shown. List data beginning at start-time, make sure that correct data is
+   displayed. List data ending end end-time, make sure that correct data is displayed. List
+   data beginning in begin-time, ending in end-time, make sure that correct data is displayed.
+
+ - verify usage trim with a date/time range
+
+   Take timestamp of the beginning of the test, and the end of the test. Round timestamps to the
+   nearest hour (downward from start of test, upward from the end of test). Trim data starting
+   at end-time, make sure that no data has been trimmed. Trim data ending at start-time, make sure
+   that no data has been trimmed.  Trim data beginning in begin-time, ending in end-time, make sure
+   that all data has been trimmed.
diff --git a/doc/dev/rbd-diff.rst b/doc/dev/rbd-diff.rst
new file mode 100644
index 000000000..083c13165
--- /dev/null
+++ b/doc/dev/rbd-diff.rst
@@ -0,0 +1,146 @@
+RBD Incremental Backup
+======================
+
+This is a simple streaming file format for representing a diff between
+two snapshots (or a snapshot and the head) of an RBD image.
+
+Header
+~~~~~~
+
+"rbd diff v1\\n"
+
+Metadata records
+~~~~~~~~~~~~~~~~
+
+Every record has a one byte "tag" that identifies the record type,
+followed by some other data.
+
+Metadata records come in the first part of the image.  Order is not
+important, as long as all the metadata records come before the data
+records.
+
+From snap
+---------
+
+- u8: 'f'
+- le32: snap name length
+- snap name
+
+To snap
+-------
+
+- u8: 't'
+- le32: snap name length
+- snap name
+
+Size
+----
+
+- u8: 's'
+- le64: (ending) image size
+
+Data Records
+~~~~~~~~~~~~
+
+These records come in the second part of the sequence.
+
+Updated data
+------------
+
+- u8: 'w'
+- le64: offset
+- le64: length
+- length bytes of actual data
+
+Zero data
+---------
+
+- u8: 'z'
+- le64: offset
+- le64: length
+
+
+Final Record
+~~~~~~~~~~~~
+
+End
+---
+
+- u8: 'e'
+
+
+Header
+~~~~~~
+
+"rbd diff v2\\n"
+
+Metadata records
+~~~~~~~~~~~~~~~~
+
+Every record has a one byte "tag" that identifies the record type,
+followed by length of data, and then some other data.
+
+Metadata records come in the first part of the image.  Order is not
+important, as long as all the metadata records come before the data
+records.
+
+In v2, we have the following metadata in each section:
+(1 Bytes) tag.
+(8 Bytes) length.
+(n Bytes) data.
+
+In this way, we can skip the unrecognized tag.
+
+From snap
+---------
+
+- u8: 'f'
+- le64: length of appending data (4 + length)
+- le32: snap name length
+- snap name
+
+To snap
+-------
+
+- u8: 't'
+- le64: length of appending data (4 + length)
+- le32: snap name length
+- snap name
+
+Size
+----
+
+- u8: 's'
+- le64: length of appending data (8)
+- le64: (ending) image size
+
+Data Records
+~~~~~~~~~~~~
+
+These records come in the second part of the sequence.
+
+Updated data
+------------
+
+- u8: 'w'
+- le64: length of appending data (8 + 8 + length)
+- le64: offset
+- le64: length
+- length bytes of actual data
+
+Zero data
+---------
+
+- u8: 'z'
+- le64: length of appending data (8 + 8)
+- le64: offset
+- le64: length
+
+
+Final Record
+~~~~~~~~~~~~
+
+End
+---
+
+- u8: 'e'
diff --git a/doc/dev/rbd-export.rst b/doc/dev/rbd-export.rst
new file mode 100644
index 000000000..2edb637f6
--- /dev/null
+++ b/doc/dev/rbd-export.rst
@@ -0,0 +1,104 @@
+RBD Export & Import
+===================
+
+This is a file format of an RBD image or snapshot. It's a sparse format
+for the full image. There are three recording sections in the file.
+
+(1) Header.
+(2) Metadata.
+(3) Diffs.
+
+Header
+~~~~~~
+
+"rbd image v2\\n"
+
+Metadata records
+~~~~~~~~~~~~~~~~
+
+Every record has a one byte "tag" that identifies the record type,
+followed by length of data, and then some other data.
+
+Metadata records come in the first part of the image.  Order is not
+important, as long as all the metadata records come before the data
+records.
+
+In v2, we have the following metadata in each section:
+(1 Bytes) tag.
+(8 Bytes) length.
+(n Bytes) data.
+
+In this way, we can skip the unrecognized tag.
+
+Image order
+-----------
+
+- u8: 'O'
+- le64: length of appending data (8)
+- le64: image order
+
+Image format
+------------
+
+- u8: 'F'
+- le64: length of appending data (8)
+- le64: image format
+
+Image Features
+--------------
+
+- u8: 'T'
+- le64: length of appending data (8)
+- le64: image features
+
+Image Stripe unit
+-----------------
+
+- u8: 'U'
+- le64: length of appending data (8)
+- le64: image striping unit
+
+Image Stripe count
+------------------
+
+- u8: 'C'
+- le64: length of appending data (8)
+- le64: image striping count
+
+ImageMeta Key and Value
+-----------------------
+
+- u8: 'M'
+- le64: length of appending data (length of key + length of value + 4 * 2)
+- string: image-meta key
+- string: image-meta value
+
+Final Record
+~~~~~~~~~~~~
+
+End
+---
+
+- u8: 'E'
+
+
+Diffs records
+~~~~~~~~~~~~~
+
+Record the all snapshots and the HEAD in this section. 
+
+Snap Protection status
+----------------------
+
+Record the snapshot's protection status if `--export-format=2`.
+- u8: 'p'
+- le64: length of appending data (8)
+- u8: snap protection status (0 for false, 1 for true)
+
+Others
+------
+
+- le64: number of diffs
+- Diffs ...
+
+Detail please refer to rbd-diff.rst
diff --git a/doc/dev/rbd-layering.rst b/doc/dev/rbd-layering.rst
new file mode 100644
index 000000000..e6e224ce4
--- /dev/null
+++ b/doc/dev/rbd-layering.rst
@@ -0,0 +1,281 @@
+============
+RBD Layering
+============
+
+RBD layering refers to the creation of copy-on-write clones of block
+devices. This allows for fast image creation, for example to clone a
+golden master image of a virtual machine into a new instance. To
+simplify the semantics, you can only create a clone of a snapshot -
+snapshots are always read-only, so the rest of the image is
+unaffected, and there's no possibility of writing to them
+accidentally.
+
+From a user's perspective, a clone is just like any other rbd image.
+You can take snapshots of them, read/write them, resize them, etc.
+There are no restrictions on clones from a user's viewpoint.
+
+Note: the terms `child` and `parent` below mean an rbd image created
+by cloning, and the rbd image snapshot a child was cloned from.
+
+Command line interface
+----------------------
+
+Before cloning a snapshot, you must mark it as protected, to prevent
+it from being deleted while child images refer to it:
+::
+
+    $ rbd snap protect pool/image@snap
+
+Then you can perform the clone:
+::
+
+    $ rbd clone [--parent] pool/parent@snap [--image] pool2/child1
+
+You can create a clone with different object sizes from the parent:
+::
+
+    $ rbd clone --order 25 pool/parent@snap pool2/child2
+
+To delete the parent, you must first mark it unprotected, which checks
+that there are no children left:
+::
+
+    $ rbd snap unprotect pool/image@snap
+    Cannot unprotect: Still in use by pool2/image2
+    $ rbd children pool/image@snap
+    pool2/child1
+    pool2/child2
+    $ rbd flatten pool2/child1
+    $ rbd rm pool2/child2
+    $ rbd snap rm pool/image@snap
+    Cannot remove a protected snapshot: pool/image@snap
+    $ rbd snap unprotect pool/image@snap
+
+Then the snapshot can be deleted like normal:
+::
+
+    $ rbd snap rm pool/image@snap
+
+Implementation
+--------------
+
+Data Flow
+^^^^^^^^^
+
+In the initial implementation, called 'trivial layering', there will
+be no tracking of which objects exist in a clone. A read that hits a
+non-existent object will attempt to read from the parent snapshot, and
+this will continue recursively until an object exists or an image with
+no parent is found. This is done through the normal read path from
+the parent, so differing object sizes between parents and children
+do not matter.
+
+Before a write to an object is performed, the object is checked for
+existence. If it doesn't exist, a copy-up operation is performed,
+which means reading the relevant range of data from the parent
+snapshot and writing it (plus the original write) to the child
+image. To prevent races with multiple writes trying to copy-up the
+same object, this copy-up operation will include an atomic create. If
+the atomic create fails, the original write is done instead. This
+copy-up operation is implemented as a class method so that extra
+metadata can be stored by it in the future. In trivial layering, the
+copy-up operation copies the entire range needed to the child object
+(that is, the full size of the child object). A future optimization
+could make this copy-up more fine-grained.
+
+Another future optimization could be storing a bitmap of which objects
+actually exist in a child. This would obviate the check for existence
+before each write, and let reads go directly to the parent if needed.
+
+These optimizations are discussed in:
+
+http://marc.info/?l=ceph-devel&m=129867273303846
+
+Parent/Child relationships
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Children store a reference to their parent in their header, as a tuple
+of (pool id, image id, snapshot id). This is enough information to
+open the parent and read from it.
+
+In addition to knowing which parent a given image has, we want to be
+able to tell if a protected snapshot still has children. This is
+accomplished with a new per-pool object, `rbd_children`, which maps
+(parent pool id, parent image id, parent snapshot id) to a list of
+child image ids. This is stored in the same pool as the child image
+because the client creating a clone already has read/write access to
+everything in this pool, but may not have write access to the parent's
+pool. This lets a client with read-only access to one pool clone a
+snapshot from that pool into a pool they have full access to. It
+increases the cost of unprotecting an image, since this needs to check
+for children in every pool, but this is a rare operation. It would
+likely only be done before removing old images, which is already much
+more expensive because it involves deleting every data object in the
+image.
+
+Protection
+^^^^^^^^^^
+
+Internally, protection_state is a field in the header object that
+can be in three states. "protected", "unprotected", and
+"unprotecting". The first two are set as the result of "rbd
+protect/unprotect". The "unprotecting" state is set while the "rbd
+unprotect" command checks for any child images. Only snapshots in the
+"protected" state may be cloned, so the "unprotected" state prevents
+a race like:
+
+1. A: walk through all pools, look for clones, find none
+2. B: create a clone
+3. A: unprotect parent
+4. A: rbd snap rm pool/parent@snap
+
+Resizing
+^^^^^^^^
+
+Resizing an rbd image is like truncating a sparse file. New space is
+treated as zeroes, and shrinking an rbd image deletes the contents
+beyond the old bounds. This means that if you have a 10G image full of
+data, and you resize it down to 5G and then up to 10G again, the last
+5G is treated as zeroes (and any objects that held that data were
+removed when the image was shrunk).
+
+Layering complicates this because the absence of an object no longer
+implies it should be treated as zeroes - if the object is part of a
+clone, it may mean that some data needs to be read from the parent.
+
+To preserve the resizing behavior for clones, we need to keep track of
+which objects could be stored in the parent. We can track this as the
+amount of overlap the child has with the parent, since resizing only
+changes the end of an image. When a child is created, its overlap
+is the size of the parent snapshot. On each subsequent resize, the
+overlap is `min(overlap, new_size)`. That is, shrinking the image
+may shrinks the overlap, but increasing the image's size does not
+change the overlap.
+
+Objects that do not exist past the overlap are treated as zeroes.
+Objects that do not exist before that point fall back to reading
+from the parent.
+
+Since this overlap changes over time, we store it as part of the
+metadata for a snapshot as well.
+
+Renaming
+^^^^^^^^
+
+Currently the rbd header object (that stores all the metadata about an
+image) is named after the name of the image. This makes renaming
+disrupt clients who have the image open (such as children reading from
+a parent). To avoid this, we can name the header object by the
+id of the image, which does not change. That is, the name of the
+header object could be `rbd_header.$id`, where $id is a unique id for
+the image in the pool.
+
+When a client opens an image, all it knows is the name. There is
+already a per-pool `rbd_directory` object that maps image names to
+ids, but if we relied on it to get the id, we could not open any
+images in that pool if that single object was unavailable. To avoid
+this dependency, we can store the id of an image in an object called
+`rbd_id.$image_name`, where $image_name is the name of the image. The
+per-pool `rbd_directory` object is still useful for listing all images
+in a pool, however.
+
+Header changes
+--------------
+
+The header needs a few new fields:
+
+* int64_t parent_pool_id
+* string parent_image_id
+* uint64_t parent_snap_id
+* uint64_t overlap (how much of the image may be referring to the parent)
+
+These are stored in a "parent" key, which is only present if the image
+has a parent.
+
+cls_rbd
+^^^^^^^
+
+Some new methods are needed:
+::
+
+    /***************** methods on the rbd header *********************/
+    /**
+     * Sets the parent and overlap keys.
+     * Fails if any of these keys exist, since the image already
+     * had a parent.
+     */
+    set_parent(uint64_t pool_id, string image_id, uint64_t snap_id)
+
+    /**
+     * returns the parent pool id, image id, snap id, and overlap, or -ENOENT
+     * if parent_pool_id does not exist or is -1
+     */
+    get_parent(uint64_t snapid)
+
+    /**
+     * Removes the parent key
+     */
+    remove_parent() // after all parent data is copied to the child
+
+    /*************** methods on the rbd_children object *****************/
+
+    add_child(uint64_t parent_pool_id, string parent_image_id,
+              uint64_t parent_snap_id, string image_id);
+    remove_child(uint64_t parent_pool_id, string parent_image_id,
+                 uint64_t parent_snap_id, string image_id);
+    /**
+     * List ids of a given parent
+     */
+    get_children(uint64_t parent_pool_id, string parent_image_id,
+                 uint64_t parent_snap_id, uint64_t max_return,
+                 string start);
+    /**
+     * list parent
+     */
+    get_parents(uint64_t max_return, uint64_t start_pool_id,
+                string start_image_id, string start_snap_id);
+
+
+    /************ methods on the rbd_id.$image_name object **************/
+
+    set_id(string id)
+    get_id()
+
+    /************** methods on the rbd_directory object *****************/
+
+    dir_get_id(string name);
+    dir_get_name(string id);
+    dir_list(string start_after, uint64_t max_return);
+    dir_add_image(string name, string id);
+    dir_remove_image(string name, string id);
+    dir_rename_image(string src, string dest, string id);
+
+Two existing methods will change if the image supports
+layering:
+::
+
+    snapshot_add - stores current overlap and has_parent with
+                   other snapshot metadata (images that don't have
+                   layering enabled aren't affected)
+
+    set_size     - will adjust the parent overlap down as needed.
+
+librbd
+^^^^^^
+
+Opening a child image opens its parent (and this will continue
+recursively as needed). This means that an ImageCtx will contain a
+pointer to the parent image context. Differing object sizes won't
+matter, since reading from the parent will go through the parent
+image context.
+
+Discard will need to change for layered images so that it only
+truncates objects, and does not remove them. If we removed objects, we
+could not tell if we needed to read them from the parent.
+
+A new clone method will be added, which takes the same arguments as
+create except size (size of the parent image is used).
+
+Instead of expanding the rbd_info struct, we will break the metadata
+retrieval into several API calls.  Right now, the only users of
+rbd_stat() other than 'rbd info' only use it to retrieve image size.
diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst
new file mode 100644
index 000000000..e9fe94d82
--- /dev/null
+++ b/doc/dev/release-checklists.rst
@@ -0,0 +1,107 @@
+==================
+Release checklists
+==================
+
+Dev Kickoff
+===========
+
+These steps should be taken when starting a new major release, just after
+the previous release has been tagged (vX.2.0) and that tag has been merged
+back into master.
+
+X is the release we are just starting development on.  X-1 is the one
+that was just released (X-1).2.0.
+
+Versions and tags
+-----------------
+
+- [x] Update CMakeLists.txt VERSION (right at the top to X.0.0)
+- [x] Update src/ceph_release with the new release name, number, and type ('dev')
+- [ ] Initial tag vX.0.0 (so that we can distinguish from (and sort
+  after) the backported (X-1).2.Z versions.
+
+
+Define release names and constants
+----------------------------------
+
+Make sure X (and, ideally, X+1) is defined:
+
+- [x] src/common/ceph_releases.h (`ceph_release_t`)
+- [x] src/common/ceph_strings.cc (`ceph_release_name()`)
+- [x] src/include/rados.h (`CEPH_RELEASE_*` and `MAX`)
+- [x] src/mon/mon_types.h (`ceph::features::mon::FEATURE_*` and related structs and helpers; note that monmaptool CLI test output will need adjustment)
+- [x] src/mds/cephfs_features.h (`CEPHFS_CURRENT_RELEASE`)
+
+Scripts
+~~~~~~~
+
+- [x] src/script/backport-resolve-issue (`releases()`, `ver_to_release()`... but for X-1)
+- [x] src/script/ceph-release-notes (X-1)
+- [x] ceph-build.git scripts/build_utils.sh `release_from_version()`
+
+Misc
+~~~~
+- [x] update src/ceph-volume/ceph_volume/__init__.py (`__release__`)
+
+Feature bits
+------------
+
+- [x] ensure that `SERVER_X` is defined
+- [x] change any features `DEPRECATED` in release X-3 are now marked `RETIRED`.
+- [ ] look for features that (1) were present in X-2 and (2) have no
+  client dependency and mark them `DEPRECATED` as of X.
+
+
+Compatsets
+----------
+
+- [x] mon/Monitor.h (`CEPH_MON_FEATURE_INCOMPAT_X`)
+- [x] mon/Monitor.cc (include in `get_supported_features()`)
+- [x] mon/Monitor.cc (`apply_monmap_to_compatset_features()`)
+- [x] mon/Monitor.cc (`calc_quorum_requirements()`)
+
+Mon
+---
+
+- [x] qa/standalone/mon/misc adjust `TEST_mon_features` (add X cases and adjust `--mon-debug-no-require-X`)
+- [x] mon/MgrMonitor.cc adjust `always_on_modules`
+- [x] common/options.cc define `mon_debug_no_require_X`
+- [x] common/options.cc remove `mon_debug_no_require_X-2`
+- [x] mon/OSDMonitor.cc `create_initial`: adjust new `require_osd_release`, and add associated `mon_debug_no_require_X`
+- [x] mon/OSDMonitor.cc `preprocess_boot`: adjust "disallow boot of " condition to disallow X if `require_osd_release` < X-2.
+- [x] mon/OSDMonitor.cc: adjust "osd require-osd-release" to (1) allow setting X, and (2) check that all mons *and* OSDs have X
+- [x] mon/MonCommands.h: adjust "osd require-osd-release" allows options to include X
+- [x] qa/workunits/cephtool/test.sh: adjust `require-osd-release` test
+
+
+Code cleanup
+------------
+
+- [ ] search code for "after X-1" or "X" for conditional checks
+- [ ] search code for X-2 and X-3 (`CEPH_FEATURE_SERVER_*` and
+  `ceph_release_t::*`)
+- [ ] search code for `require_osd_release`
+- [ ] search code for `min_mon_release`
+
+QA suite
+--------
+
+- [ ] create qa/suites/upgrade/(X-1)-x
+- [x] remove qa/suites/upgrade/(X-3)-x-*
+- [x] remove qa/suites/rados/upgrade/(X-3)-x-singleton symlink
+- [x] create qa/releases/X.yaml
+- [x] create qa/suites/rados/thrash-old-clients/1-install/(X-1).yaml
+
+
+
+First release candidate
+=======================
+
+- [ ] src/ceph_release: change type to `rc`
+
+
+First stable release
+====================
+
+- [ ] src/ceph_release: change type `stable`
+- [ ] generate new object corpus for encoding/decoding tests - see :doc:`corpus`
diff --git a/doc/dev/release-process.rst b/doc/dev/release-process.rst
new file mode 100644
index 000000000..3750759b8
--- /dev/null
+++ b/doc/dev/release-process.rst
@@ -0,0 +1,225 @@
+======================
+  Ceph Release Process
+======================
+
+Prerequisites
+=============
+
+Signing Machine
+---------------
+The signing machine is a virtual machine in the `Sepia lab
+<https://wiki.sepia.ceph.com/doku.php?id=start>`_. SSH access to the signing
+machine is limited to the usual Infrastructure Admins along with a few other
+component leads (e.g., nfs-ganesha, ceph-iscsi).
+
+The ``ubuntu`` user on the machine has some `build scripts <https://github.com/ceph/ceph-build/tree/main/scripts>`_ that help with pulling, pushing, and signing packages.
+
+The GPG signing key permanently lives on a `Nitrokey Pro <https://shop.nitrokey.com/shop/product/nkpr2-nitrokey-pro-2-3>`_ and is passed through to the VM via RHV. This helps to ensure that the key cannot be exported or leave the datacenter in any way.
+
+New Major Releases
+------------------
+For each new major (alphabetical) release, you must create one ``ceph-release`` RPM for each RPM repo (e.g., one for el8 and one for el9). `chacra <https://github.com/ceph/chacra>`_ is a python service we use to store DEB and RPM repos. The chacra repos are configured to include this ceph-release RPM, but it must be built separately. You must make sure that chacra is properly configured to include this RPM for each particular release.
+
+1. Update chacra so it is aware of the new Ceph release.  See `this PR <https://github.com/ceph/chacra/pull/219>`_ for an example.
+2. Redeploy chacra (e.g., ``ansible-playbook chacra.ceph.com.yml``)
+3. Run https://jenkins.ceph.com/view/all/job/ceph-release-rpm/
+
+Summarized build process
+========================
+
+1. QE finishes testing and finds a stopping point.  That commit is pushed to the ``$release-release`` branch in ceph.git (e.g., ``quincy-release``).  This allows work to continue in the working ``$release`` branch without having to freeze it during the release process.
+2. The Ceph Council approves and notifies the "Build Lead".
+3. The "Build Lead" starts the `Jenkins multijob <https://jenkins.ceph.com/view/all/job/ceph>`_, which triggers all builds.
+4. Packages are pushed to chacra.ceph.com.
+5. Packages are pulled from chacra.ceph.com to the Signer VM.
+6. Packages are signed.
+7. Packages are pushed to download.ceph.com.
+8. Release containers are built and pushed to quay.io.
+
+Hotfix Release Process Deviation
+--------------------------------
+
+A hotfix release has a couple differences.
+
+1. Check out the most recent tag. For example, if we're releasing a hotfix on top of 17.2.3, ``git checkout -f -B quincy-release origin/v17.2.3``
+2. ``git cherry-pick -x`` the necessary hotfix commits
+3. ``git push -f origin quincy-release``
+4. Notify the "Build Lead" to start the build.
+5. The "Build Lead" should set ``RELEASE_TYPE=HOTFIX`` instead of ``STABLE``.
+
+Security Release Process Deviation
+----------------------------------
+
+A security/CVE release is similar to a hotfix release with two differences:
+
+    1. The fix should be pushed to the `ceph-private <https://github.com/ceph/ceph-private>`_ repo instead of ceph.git (requires GitHub Admin Role).
+    2. The tags (e.g., v17.2.4) must be manually pushed to ceph.git by the "Build Lead."
+
+1. Check out the most recent tag. For example, if we're releasing a security fix on top of 17.2.3, ``git checkout -f -B quincy-release origin/v17.2.3``
+2. ``git cherry-pick -x`` the necessary security fix commits
+3. ``git remote add security git@github.com:ceph/ceph-private.git``
+4. ``git push -f security quincy-release``
+5. Notify the "Build Lead" to start the build.
+6. The "Build Lead" should set ``RELEASE_TYPE=SECURITY`` instead of ``STABLE``.
+7. Finally, the `ceph-tag <https://github.com/ceph/ceph-build/blob/main/ansible/roles/ceph-release/tasks/push.yml>`_ steps need to be manually run by the "Build Lead" as close to the Announcement time as possible::
+
+    # Example using quincy pretending 17.2.4 is the security release version
+    # Add the ceph-releases repo (also requires GitHub Admin Role). The `ceph-setup <https://jenkins.ceph.com/job/ceph-setup>`_ job will have already created and pushed the tag to ceph-releases.git.
+    git remote add releases git@github.com:ceph/ceph-releases.git
+    git fetch --all
+    # Check out the version commit
+    git checkout -f -B quincy-release releases/quincy-release
+    git push -f origin quincy-release
+    git push origin v17.2.4
+    # Now create a Pull Request of quincy-release targeting quincy to merge the version commit and security fixes back into the quincy branch
+
+1. Preparing the release branch
+===============================
+
+Once QE has determined a stopping point in the working (e.g., ``quincy``) branch, that commit should be pushed to the corresponding ``quincy-release`` branch.
+
+Notify the "Build Lead" that the release branch is ready.
+
+2. Starting the build
+=====================
+
+We'll use a stable/regular 15.2.17 release of Octopus as an example throughout this document.
+
+1. Browse to https://jenkins.ceph.com/view/all/job/ceph/build?delay=0sec
+2. Log in with GitHub OAuth
+3. Set the parameters as necessary::
+
+    BRANCH=octopus
+    TAG=checked
+    VERSION=15.2.17
+    RELEASE_TYPE=STABLE
+    ARCHS=x86_64 arm64
+
+4. Use https://docs.ceph.com/en/latest/start/os-recommendations/?highlight=debian#platforms to determine the ``DISTROS`` parameter.  For example,
+
+    +-------------------+-------------------------------------------+
+    | Release           | Distro Codemap                            |
+    +===================+===========================================+
+    | octopus (15.X.X)  | ``focal bionic centos7 centos8 buster``   |
+    +-------------------+-------------------------------------------+
+    | pacific (16.X.X)  | ``focal bionic centos8 buster bullseye``  |
+    +-------------------+-------------------------------------------+
+    | quincy (17.X.X)   | ``focal centos8 centos9 bullseye``        |
+    +-------------------+-------------------------------------------+
+
+5. Click ``Build``.
+
+3. Release Notes
+================
+
+Packages take hours to build. Use those hours to create the Release Notes and Announcements:
+
+1. ceph.git Release Notes (e.g., `v15.2.17's ceph.git (docs.ceph.com) PR <https://github.com/ceph/ceph/pull/47198>`_)
+2. ceph.io Release Notes (e.g., `v15.2.17's ceph.io.git (www.ceph.io) PR <https://github.com/ceph/ceph.io/pull/427>`_)
+3. E-mail announcement
+
+See `the Ceph Tracker wiki page that explains how to write the release notes <https://tracker.ceph.com/projects/ceph-releases/wiki/HOWTO_write_the_release_notes>`_. 
+
+4. Signing and Publishing the Build
+===================================
+
+#. Obtain the sha1 of the version commit from the `build job <https://jenkins.ceph.com/view/all/job/ceph>`_ or the ``sha1`` file created by the `ceph-setup <https://jenkins.ceph.com/job/ceph-setup/>`_ job.
+
+#. Download the packages from chacra.ceph.com to the signing virtual machine. These packages get downloaded to ``/opt/repos`` where the `Sepia Lab Long Running (Ceph) Cluster <https://wiki.sepia.ceph.com/doku.php?id=services:longrunningcluster>`_ is mounted.
+
+   .. prompt:: bash $
+
+      ssh ubuntu@signer.front.sepia.ceph.com
+      sync-pull ceph [pacific|quincy|etc] <sha1>
+ 
+   Example::
+    
+      $ sync-pull ceph octopus 8a82819d84cf884bd39c17e3236e0632ac146dc4
+      sync for: ceph octopus
+      ********************************************
+      Found the most packages (332) in ubuntu/bionic.
+      No JSON object could be decoded
+      No JSON object could be decoded
+      ubuntu@chacra.ceph.com:/opt/repos/ceph/octopus/8a82819d84cf884bd39c17e3236e0632ac146dc4/ubuntu/bionic/flavors/default/* /opt/repos/ceph/octopus-15.2.17/debian/jessie/
+      --------------------------------------------
+      receiving incremental file list
+      db/
+       db/checksums.db
+              180.22K 100%    2.23MB/s    0:00:00 (xfr#1, to-chk=463/467)
+      db/contents.cache.db
+              507.90K 100%    1.95MB/s    0:00:00 (xfr#2, to-chk=462/467)
+      db/packages.db
+      
+      etc...
+
+#. Sign the DEBs:
+
+   .. prompt:: bash
+
+      merfi gpg /opt/repos/ceph/octopus-15.2.17/debian
+
+   Example::
+
+      $ merfi gpg /opt/repos/ceph/octopus-15.2.17/debian
+      --> Starting path collection, looking for files to sign
+      --> 18 matching paths found
+      --> will sign with the following commands:
+      --> gpg --batch --yes --armor --detach-sig --output Release.gpg Release
+      --> gpg --batch --yes --clearsign --output InRelease Release
+      --> signing: /opt/repos/ceph/octopus-15.2.17/debian/jessie/dists/bionic/Release
+      --> Running command: gpg --batch --yes --armor --detach-sig --output Release.gpg Release
+      --> Running command: gpg --batch --yes --clearsign --output InRelease Release
+      --> signing: /opt/repos/ceph/octopus-15.2.17/debian/jessie/dists/focal/Release
+      --> Running command: gpg --batch --yes --armor --detach-sig --output Release.gpg Release
+      --> Running command: gpg --batch --yes --clearsign --output InRelease Release
+      
+      etc...
+
+#. Sign the RPMs:
+
+   .. prompt:: bash
+
+      sign-rpms octopus
+
+   Example::
+   
+      $ sign-rpms octopus
+      Checking packages in: /opt/repos/ceph/octopus-15.2.17/centos/7
+      signing:  /opt/repos/ceph/octopus-15.2.17/centos/7/SRPMS/ceph-release-1-1.el7.src.rpm
+      /opt/repos/ceph/octopus-15.2.17/centos/7/SRPMS/ceph-release-1-1.el7.src.rpm:
+      signing:  /opt/repos/ceph/octopus-15.2.17/centos/7/SRPMS/ceph-15.2.17-0.el7.src.rpm
+      /opt/repos/ceph/octopus-15.2.17/centos/7/SRPMS/ceph-15.2.17-0.el7.src.rpm:
+      signing:  /opt/repos/ceph/octopus-15.2.17/centos/7/noarch/ceph-mgr-modules-core-15.2.17-0.el7.noarch.rpm
+   
+      etc...
+
+5. Publish the packages to download.ceph.com:
+
+   .. prompt:: bash $
+
+      sync-push octopus
+
+5. Build Containers
+===================
+
+Start the following two jobs:
+
+#. https://2.jenkins.ceph.com/job/ceph-container-build-ceph-base-push-imgs/
+#. https://2.jenkins.ceph.com/job/ceph-container-build-ceph-base-push-imgs-arm64/
+
+6. Announce the Release
+=======================
+
+Version Commit PR
+-----------------
+
+The `ceph-tag Jenkins job <https://jenkins.ceph.com/job/ceph-tag>`_ creates a Pull Request in ceph.git that targets the release branch.
+
+If this was a regular release (not a hotfix release or a security release), the only commit in that Pull Request should be the version commit.  For example, see `v15.2.17's version commit PR <https://github.com/ceph/ceph/pull/47520>`_. 
+
+Request a review and then merge the Pull Request.
+
+Announcing
+----------
+
+Publish the Release Notes on ceph.io before announcing the release by email, because the e-mail announcement references the ceph.io blog post.
diff --git a/doc/dev/seastore.rst b/doc/dev/seastore.rst
new file mode 100644
index 000000000..eb89d8219
--- /dev/null
+++ b/doc/dev/seastore.rst
@@ -0,0 +1,323 @@
+==========
+ SeaStore
+==========
+
+Goals and Basics
+================
+
+* Target NVMe devices.  Not primarily concerned with pmem or HDD.
+* make use of SPDK for user-space driven IO
+* Use Seastar futures programming model to facilitate
+  run-to-completion and a sharded memory/processing model
+* Allow zero- (or minimal) data copying on read and write paths when
+  combined with a seastar-based messenger using DPDK
+
+Motivation and background
+-------------------------
+
+All flash devices are internally structured in terms of segments that
+can be written efficiently but must be erased in their entirety.  The
+NVMe device generally has limited knowledge about what data in a
+segment is still "live" (hasn't been logically discarded), making the
+inevitable garbage collection within the device inefficient.  We can
+design an on-disk layout that is friendly to GC at lower layers and
+drive garbage collection at higher layers.
+
+In principle a fine-grained discard could communicate our intent to
+the device, but in practice discard is poorly implemented in the
+device and intervening software layers.
+
+The basic idea is that all data will be stream out sequentially to
+large segments on the device.  In the SSD hardware, segments are
+likely to be on the order of 100's of MB to tens of GB.
+
+SeaStore's logical segments would ideally be perfectly aligned with
+the hardware segments.  In practice, it may be challenging to
+determine geometry and to sufficiently hint to the device that LBAs
+being written should be aligned to the underlying hardware.  In the
+worst case, we can structure our logical segments to correspond to
+e.g. 5x the physical segment size so that we have about ~20% of our
+data misaligned.
+
+When we reach some utilization threshold, we mix cleaning work in with
+the ongoing write workload in order to evacuate live data from
+previously written segments.  Once they are completely free we can
+discard the entire segment so that it can be erased and reclaimed by
+the device.
+
+The key is to mix a small bit of cleaning work with every write
+transaction to avoid spikes and variance in write latency.
+  
+Data layout basics
+------------------
+
+One or more cores/shards will be reading and writing to the device at
+once.  Each shard will have its own independent data it is operating
+on and stream to its own open segments.  Devices that support streams
+can be hinted accordingly so that data from different shards is not
+mixed on the underlying media.
+
+Persistent Memory
+-----------------
+
+As the initial sequential design above matures, we'll introduce
+persistent memory support for metadata and caching structures.
+
+Design
+======
+
+The design is based heavily on both f2fs and btrfs.  Each reactor
+manages its own root.  Prior to reusing a segment, we rewrite any live
+blocks to an open segment.
+
+Because we are only writing sequentially to open segments, we must
+“clean� one byte of an existing segment for every byte written at
+steady state.  Generally, we’ll need to reserve some portion of the
+usable capacity in order to ensure that write amplification remains
+acceptably low (20% for 2x? -- TODO: find prior work).  As a design
+choice, we want to avoid a background gc scheme as it tends to
+complicate estimating operation cost and tends to introduce
+non-deterministic latency behavior.  Thus, we want a set of structures
+that permits us to relocate blocks from existing segments inline with
+ongoing client IO.
+
+To that end, at a high level, we’ll maintain 2 basic metadata trees.
+First, we need a tree mapping ghobject_t->onode_t (onode_by_hobject).
+Second, we need a way to find live blocks within a segment and a way
+to decouple internal references from physical locations (lba_tree).
+
+Each onode contains xattrs directly as well as the top of the omap and
+extent trees (optimization: we ought to be able to fit small enough
+objects into the onode).
+
+Segment Layout
+--------------
+
+The backing storage is abstracted into a set of segments.  Each
+segment can be in one of 3 states: empty, open, closed.  The byte
+contents of a segment are a sequence of records.  A record is prefixed
+by a header (including length and checksums) and contains a sequence
+of deltas and/or blocks.  Each delta describes a logical mutation for
+some block.  Each included block is an aligned extent addressable by
+<segment_id_t, segment_offset_t>.  A transaction can be implemented by
+constructing a record combining deltas and updated blocks and writing
+it to an open segment.
+
+Note that segments will generally be large (something like >=256MB),
+so there will not typically be very many of them.
+
+record: [ header | delta | delta... | block | block ... ]
+segment: [ record ... ]
+
+See src/crimson/os/seastore/journal.h for Journal implementation
+See src/crimson/os/seastore/seastore_types.h for most seastore structures.
+
+Each shard will keep open N segments for writes
+
+- HDD: N is probably 1 on one shard
+- NVME/SSD: N is probably 2/shard, one for "journal" and one for
+  finished data records as their lifetimes are different.
+
+I think the exact number to keep open and how to partition writes
+among them will be a tuning question -- gc/layout should be flexible.
+Where practical, the goal is probably to partition blocks by expected
+lifetime so that a segment either has long lived or short lived
+blocks.
+
+The backing physical layer is exposed via a segment based interface.
+See src/crimson/os/seastore/segment_manager.h
+
+Journal and Atomicity
+---------------------
+
+One open segment is designated to be the journal.  A transaction is
+represented by an atomically written record.  A record will contain
+blocks written as part of the transaction as well as deltas which
+are logical mutations to existing physical extents.  Transaction deltas
+are always written to the journal.  If the transaction is associated
+with blocks written to other segments, final record with the deltas
+should be written only once the other blocks are persisted.  Crash
+recovery is done by finding the segment containing the beginning of
+the current journal, loading the root node, replaying the deltas, and
+loading blocks into the cache as needed.
+
+See src/crimson/os/seastore/journal.h
+
+Block Cache
+-----------
+
+Every block is in one of two states:
+
+- clean: may be in cache or not, reads may cause cache residence or
+  not
+- dirty: the current version of the record requires overlaying deltas
+  from the journal.  Must be fully present in the cache.
+
+Periodically, we need to trim the journal (else, we’d have to replay
+journal deltas from the beginning of time).  To do this, we need to
+create a checkpoint by rewriting the root blocks and all currently
+dirty blocks.  Note, we can do journal checkpoints relatively
+infrequently, and they needn’t block the write stream.
+
+Note, deltas may not be byte range modifications.  Consider a btree
+node structured with keys to the left and values to the right (common
+trick for improving point query/key scan performance).  Inserting a
+key/value into that node at the min would involve moving a bunch of
+bytes, which would be expensive (or verbose) to express purely as a
+sequence of byte operations.  As such, each delta indicates the type
+as well as the location of the corresponding extent.  Each block
+type can therefore implement CachedExtent::apply_delta as appopriate.
+
+See src/os/crimson/seastore/cached_extent.h.
+See src/os/crimson/seastore/cache.h.
+
+GC
+---
+
+Prior to reusing a segment, we must relocate all live blocks.  Because
+we only write sequentially to empty segments, for every byte we write
+to currently open segments, we need to clean a byte of an existing
+closed segment.  As a design choice, we’d like to avoid background
+work as it complicates estimating operation cost and has a tendency to
+create non-deterministic latency spikes.  Thus, under normal operation
+each seastore reactor will be inserting enough work to clean a segment
+at the same rate as incoming operations.
+
+In order to make this cheap for sparse segments, we need a way to
+positively identify dead blocks.  Thus, for every block written, an
+entry will be added to the lba tree with a pointer to the previous lba
+in the segment.  Any transaction that moves a block or modifies the
+reference set of an existing one will include deltas/blocks required
+to update the lba tree to update or remove the previous block
+allocation.  The gc state thus simply needs to maintain an iterator
+(of a sort) into the lba tree segment linked list for segment
+currently being cleaned and a pointer to the next record to be
+examined -- records not present in the allocation tree may still
+contain roots (like allocation tree blocks) and so the record metadata
+must be checked for a flag indicating root blocks.
+
+For each transaction, we evaluate a heuristic function of the
+currently available space and currently live space in order to
+determine whether we need to do cleaning work (could be simply a range
+of live/used space ratios).
+
+TODO: there is not yet a GC implementation
+
+Logical Layout
+==============
+
+Using the above block and delta semantics, we build two root level trees:
+- onode tree: maps hobject_t to onode_t
+- lba_tree: maps lba_t to lba_range_t
+
+Each of the above structures is comprised of blocks with mutations
+encoded in deltas.  Each node of the above trees maps onto a block.
+Each block is either physically addressed (root blocks and the
+lba_tree nodes) or is logically addressed (everything else).
+Physically addressed blocks are located by a paddr_t: <segment_id_t,
+segment_off_t> tuple and are marked as physically addressed in the
+record.  Logical blocks are addressed by laddr_t and require a lookup in
+the lba_tree to address.
+
+Because the cache/transaction machinery lives below the level of the
+lba tree, we can represent atomic mutations of the lba tree and other
+structures by simply including both in a transaction.
+
+LBAManager/BtreeLBAManager
+--------------------------
+
+Implementations of the LBAManager interface are responsible for managing
+the logical->physical mapping -- see crimson/os/seastore/lba_manager.h.
+
+The BtreeLBAManager implements this interface directly on top of
+Journal and SegmentManager using a wandering btree approach.
+
+Because SegmentManager does not let us predict the location of a
+committed record (a property of both SMR and Zone devices), references
+to blocks created within the same transaction will necessarily be
+*relative* addresses.  The BtreeLBAManager maintains an invariant by
+which the in-memory copy of any block will contain only absolute
+addresses when !is_pending() -- on_commit and complete_load fill in
+absolute addresses based on the actual block addr and on_delta_write
+does so based on the just committed record.  When is_pending(), if
+is_initial_pending references in memory are block_relative (because
+they will be written to the original block location) and
+record_relative otherwise (value will be written to delta).
+
+TransactionManager
+------------------
+
+The TransactionManager is responsible for presenting a unified
+interface on top of the Journal, SegmentManager, Cache, and
+LBAManager.  Users can allocate and mutate extents based on logical
+addresses with segment cleaning handled in the background.
+
+See crimson/os/seastore/transaction_manager.h
+
+Next Steps
+==========
+
+Journal
+-------
+
+- Support for scanning a segment to find physically addressed blocks
+- Add support for trimming the journal and releasing segments.
+
+Cache
+-----
+
+- Support for rewriting dirty blocks
+
+  - Need to add support to CachedExtent for finding/updating
+    dependent blocks
+  - Need to add support for adding dirty block writout to
+    try_construct_record
+
+LBAManager
+----------
+
+- Add support for pinning
+- Add segment -> laddr for use in GC
+- Support for locating remaining used blocks in segments
+
+GC
+---
+
+- Initial implementation
+- Support in BtreeLBAManager for tracking used blocks in segments
+- Heuristic for identifying segments to clean
+
+Other
+------
+
+- Add support for periodically generating a journal checkpoint.
+- Onode tree
+- Extent tree
+- Remaining ObjectStore integration
+
+ObjectStore considerations
+==========================
+
+Splits, merges, and sharding
+----------------------------
+
+One of the current ObjectStore requirements is to be able to split a
+collection (PG) in O(1) time.  Starting in mimic, we also need to be
+able to merge two collections into one (i.e., exactly the reverse of a
+split).
+
+However, the PGs that we split into would hash to different shards of
+the OSD in the current sharding scheme.  One can imagine replacing
+that sharding scheme with a temporary mapping directing the smaller
+child PG to the right shard since we generally then migrate that PG to
+another OSD anyway, but this wouldn't help us in the merge case where
+the constituent pieces may start out on different shards and
+ultimately need to be handled in the same collection (and be operated
+on via single transactions).
+
+This suggests that we likely need a way for data written via one shard
+to "switch ownership" and later be read and managed by a different
+shard.
+
+
+
diff --git a/doc/dev/sepia.rst b/doc/dev/sepia.rst
new file mode 100644
index 000000000..3064900c5
--- /dev/null
+++ b/doc/dev/sepia.rst
@@ -0,0 +1,8 @@
+Sepia community test lab
+========================
+
+The Ceph community maintains a test lab that is open to active contributors to
+the Ceph project. Please see the `Sepia wiki`_ for more information.
+
+.. _Sepia wiki: https://wiki.sepia.ceph.com/doku.php
+
diff --git a/doc/dev/session_authentication.rst b/doc/dev/session_authentication.rst
new file mode 100644
index 000000000..48fab623d
--- /dev/null
+++ b/doc/dev/session_authentication.rst
@@ -0,0 +1,160 @@
+==============================================
+Session Authentication for the Cephx Protocol
+==============================================
+Peter Reiher
+7/30/12
+
+The original Cephx protocol authenticated the client to the authenticator and set up a session 
+key used to authenticate the client to the server it needs to talk to.  It did not, however,
+authenticate the ongoing messages between the client and server.  Based on the fact that they
+share a secret key, these ongoing session messages can be easily authenticated by using the
+key to sign the messages.  
+
+This document describes changes to the code that allow such ongoing session authentication.
+The changes allow for future changes that permit other authentication protocols (and the 
+existing null NONE and UNKNOWN protocols) to handle signatures, but the only protocol that 
+actually does signatures, at the time of the writing, is the Cephx protocol.
+
+Introduction
+-------------
+
+This code comes into play after the Cephx protocol has completed.  At this point, the client and
+server share a secret key.  This key will be used for authentication.  For other protocols, there
+may or may not be such a key in place, and perhaps the actual procedures used to perform 
+signing will be different, so the code is written to be general.
+
+The "session" here is represented by an established pipe.  For such pipes, there should be a
+``session\_security`` structure attached to the pipe.  Whenever a message is to be sent on the
+pipe, code that handles the signature for this kind of session security will be called.  On the
+other end of the pipe, code that checks this kind of session security's message signatures will
+be called.  Messages that fail the signature check will not be processed further.  That implies
+that the sender had better be in agreement with the receiver on the session security being used,
+since otherwise messages will be uniformly dropped between them.
+
+The code is also prepared to handle encryption and decryption of session messages, which would
+add secrecy to the integrity provided by the signatures.  No protocol currently implemented
+encrypts the ongoing session messages, though.
+
+For this functionality to work, several steps are required.  First, the sender and receiver must have 
+a successful run of the cephx protocol to establish a shared key.  They must store that key somewhere
+that the pipe can get at later, to permit messages to be signed with it.  Sent messages must be
+signed, and received messages must have their signatures checked.
+
+The signature could be computed in a variety of ways, but currently its size is limited to 64 bits.
+A message's signature is placed in its footer, in a field called ``sig``.
+
+The signature code in Cephx can be turned on and off at runtime, using a Ceph boolean option called 
+``cephx\_sign\_messages``.  It is currently set to false, by default, so no messages will be signed.  It
+must be changed to true to cause signatures to be calculated and checked.
+
+Storing the Key
+---------------
+
+The key is needed to create signatures on the sending end and check signatures on the receiving end.
+In the future, if asymmetric crypto is an option, it's possible that two keys (a private one for
+this end of the pipe and a public one for the other end) would need to be stored. At this time,
+messages going in both directions will be signed with the same key, so only that key needs to be
+saved.
+
+The key is saved when the pipe is established.  On the client side, this happens in ``connect()``,
+which is located in ``msg/Pipe.cc``.  The key is obtained from a run of the Cephx protocol,
+which results in a successfully checked authorizer structure.  If there is such an authorizer
+available, the code calls ``get\_auth\_session\_handler()`` to create a new authentication session handler
+and stores it in the pipe data structure.  On the server side, a similar thing is done in 
+``accept()`` after the authorizer provided by the client has been verified.
+
+Once these things are done on either end of the connection, session authentication can start.
+
+These routines (``connect()`` and ``accept()``) are also used to handle situations where a new 
+session is being set up.  At this stage, no authorizer has been created yet, so there's no key.
+Special cases in the code that calls the signature code skip these calls when the 
+``CEPH\_AUTH\_UNKNOWN`` protocol is in use.  This protocol label is on the pre-authorizer
+messages in a session, indicating that negotiation on an authentication protocol is ongoing and
+thus signature is not possible.  There will be a reliable authentication operation later in this 
+session before anything sensitive should be passed, so this is not a security problem.
+
+Signing Messages
+----------------
+
+Messages are signed in the ``write\_message`` call located in ``msg/Pipe.cc``.  The actual
+signature process is to encrypt the CRCs for the message using the shared key.  Thus, we must
+defer signing until all CRCs have been computed.  The header CRC is computed last, so we
+call ``sign\_message()`` as soon as we've calculated that CRC.  
+
+``sign\_message()`` is a virtual function defined in ``auth/AuthSessionHandler.h``.  Thus,
+a specific version of it must be written for each authentication protocol supported.  Currently,
+only UNKNOWN, NONE and CEPHX are supported.  So there is a separate version of ``sign\_message()`` in
+``auth/unknown/AuthUnknownSessionHandler.h``, ``auth/none/AuthNoneSessionHandler.h`` and 
+``auth/cephx/CephxSessionHandler.cc``.  The UNKNOWN and NONE versions simply return 0, indicating 
+success.
+
+The CEPHX version is more extensive.  It is found in ``auth/cephx/CephxSessionHandler.cc``.  
+The first thing done is to determine if the run time option to handle signatures (see above) is on.  
+If not, the Cephx version of ``sign\_message()`` simply returns success without actually calculating 
+a signature or inserting it into the message.
+
+If the run time option is enabled, ``sign\_message()`` copies all of the message's CRCs (one from the
+header and three from the footer) into a buffer.  It calls ``encode\_encrypt()`` on the buffer,
+using the key obtained from the pipe's ``session\_security`` structure.  64 bits of the encrypted
+result are put into the message footer's signature field and a footer flag is set to indicate that
+the message was signed.  (This flag is a sanity check.  It is not regarded as definitive 
+evidence that the message was signed.  The presence of a ``session\_security`` structure at the
+receiving end requires a signature regardless of the value of this flag.)  If this all goes well,
+``sign\_message()`` returns 0.  If there is a problem anywhere along the line and no signature
+was computed, it returns ``SESSION\_SIGNATURE\_FAILURE``.
+
+Checking Signatures 
+-------------------
+
+The signature is checked by a routine called ``check\_message\_signature()``.  This is also a
+virtual function, defined in ``auth/AuthSessionHandler.h``.  So again there are specific versions
+for supported authentication protocols, such as UNKNOWN, NONE and CEPHX.  Again, the UNKNOWN and
+NONE versions are stored in ``auth/unknown/AuthUnknownSessionHandler.h`` and 
+``auth/none/AuthNoneSessionHandler.h``, respectively, and again they simply return 0, indicating
+success.
+
+The CEPHX version of ``check\_message\_signature()`` performs a real signature check.  This routine 
+(stored in ``auth/cephx/CephxSessionHandler.cc``) exits with success if the run time option has 
+disabled signatures.  Otherwise, it takes the CRCs from the header and footer, encrypts the result, 
+and compares it to the signature stored in the footer.  Since an earlier routine has checked that
+the CRCs actually match the contents of the message, it is unnecessary to recompute the CRCs
+on the raw data in the message.  The encryption is performed with the same ``encode\_encrypt()``
+routine used on the sending end, using the key stored in the local ``session\_security``
+data structure.  
+
+If everything checks out, the CEPHX routine returns 0, indicating success.  If there is a 
+problem, the routine returns ``SESSION\_SIGNATURE\_FAILURE``.
+
+Adding New Session Authentication Methods
+-----------------------------------------
+
+For the purpose of session authentication only (not the basic authentication of client and 
+server currently performed by the Cephx protocol), in addition to adding a new protocol, that
+protocol must have a ``sign\_message()`` routine and a ``check\_message\_signature`` routine.
+These routines will take a message pointer as a parameter and return 0 on success.  The procedure
+used to sign and check will be specific to the new method, but probably there will be a
+``session\_security`` structure attached to the pipe that contains a cryptographic key.  This
+structure will be either an ``AuthSessionHandler`` (found in ``auth/AuthSessionHandler.h``) 
+or a structure derived from that type.
+
+Adding Encryption to Sessions
+-----------------------------
+
+The existing code is partially, but not fully, set up to allow sessions to have their packets
+encrypted.  Part of adding encryption would be similar to adding a new authentication method.
+But one would also need to add calls to the encryption and decryption routines in ``write\_message()`` 
+and ``read\_message()``.  These calls would probably go near where the current calls for
+authentication are made.  You should consider whether you want to replace the existing calls
+with something more general that does whatever the chosen form of session security requires,
+rather than explicitly saying ``sign`` or ``encrypt``.
+
+Session Security Statistics
+---------------------------
+
+The existing Cephx authentication code keeps statistics on how many messages were signed, how
+many message signature were checked, and how many checks succeeded and failed.  It is prepared
+to keep similar statistics on encryption and decryption.  These statistics can be accessed through
+the call ``printAuthSessionHandlerStats`` in ``auth/AuthSessionHandler.cc``.  
+
+If new authentication or encryption methods are added, they should include code that keeps these
+statistics.
diff --git a/doc/dev/testing.rst b/doc/dev/testing.rst
new file mode 100644
index 000000000..1d99848a7
--- /dev/null
+++ b/doc/dev/testing.rst
@@ -0,0 +1,40 @@
+Testing notes
+=============
+
+
+build-integration-branch
+------------------------
+
+Setup
+^^^^^
+
+#. Create a github token at `<https://github.com/settings/tokens>`_
+   and put it in ``~/.github_token``.  Note that only the
+   ``public_repo`` under the ``repo`` section needs to be checked.
+
+#. Create a ceph repo label `wip-yourname-testing` if you don't
+   already have one at `<https://github.com/ceph/ceph/labels>`_.
+
+#. Create the ``ci`` remote::
+
+     git remote add ci git@github.com:ceph/ceph-ci
+
+Using
+^^^^^
+
+#. Tag some subset of `needs-qa` commits with your label (usually `wip-yourname-testing`).
+
+#. Create the integration branch::
+
+     git checkout master
+     git pull
+     ../src/script/build-integration-branch wip-yourname-testing
+
+#. Smoke test::
+
+     make && ctest -j12
+
+#. Push to ceph-ci::
+
+     git push ci $(git rev-parse --abbrev-ref HEAD)
+
diff --git a/doc/dev/versions.rst b/doc/dev/versions.rst
new file mode 100644
index 000000000..34ed74724
--- /dev/null
+++ b/doc/dev/versions.rst
@@ -0,0 +1,42 @@
+==================
+Public OSD Version
+==================
+
+We maintain two versions on disk: an eversion_t pg_log.head and a
+version_t info.user_version. Each object is tagged with both the pg
+version and user_version it was last modified with.  The PG version is
+modified by manipulating OpContext::at_version and then persisting it
+to the pg log as transactions, and is incremented in all the places it
+used to be. The user_version is modified by manipulating the new
+OpContext::user_at_version and is also persisted via the pg log
+transactions.
+user_at_version is modified only in PrimaryLogPG::prepare_transaction
+when the op was a "user modify" (a non-watch write), and the durable
+user_version is updated according to the following rules:
+1) set user_at_version to the maximum of ctx->new_obs.oi.user_version+1
+and info.last_user_version+1.
+2) set user_at_version to the maximum of itself and
+ctx->at_version.version.
+3) ctx->new_obs.oi.user_version = ctx->user_at_version (to change the
+object's user_version)
+
+This set of update semantics mean that for traditional pools the
+user_version will be equal to the past reassert_version, while for
+caching pools the object and PG user-version will be able to cross
+pools without making a total mess of things.
+In order to support old clients, we keep the old reassert_version but
+rename it to "bad_replay_version"; we fill it in as before: for writes
+it is set to the at_version (and is the proper replay version); for
+watches it is set to our user version; for ENOENT replies it is set to
+the replay version's epoch but the user_version's version. We also now
+fill in the version_t portion of the bad_replay_version on read ops as
+well as write ops, which should be fine for all old clients.
+
+For new clients, we prevent them from reading bad_replay_version and
+add two proper members: user_version and replay_version; user_version
+is filled in on every operation (reads included) while replay_version
+is filled in for writes.
+
+The objclass function get_current_version() now always returns the
+pg->info.last_user_version, which means it is guaranteed to contain
+the version of the last user update in the PG (including on reads!).
diff --git a/doc/dev/vstart-ganesha.rst b/doc/dev/vstart-ganesha.rst
new file mode 100644
index 000000000..4e77deb8b
--- /dev/null
+++ b/doc/dev/vstart-ganesha.rst
@@ -0,0 +1,45 @@
+==============================
+NFS CephFS-RGW Developer Guide
+==============================
+
+CephFS exports are supported since Octopus and RGW exports are supported since
+Quincy.
+
+Configuring NFS Ganesha to export CephFS with vstart
+====================================================
+
+1) Using ``cephadm``
+
+    .. code:: bash
+
+        $ MDS=1 MON=1 OSD=3 NFS=1 ../src/vstart.sh -n -d --cephadm
+
+    This will deploy a single NFS Ganesha daemon using ``vstart.sh``, where the
+    daemon will listen on the default NFS Ganesha port. Also cephfs export is
+    created.
+
+2) Using test orchestrator
+
+    .. code:: bash
+
+       $ MDS=1 MON=1 OSD=3 NFS=1 ../src/vstart.sh -n -d
+
+    Environment variable ``NFS`` is the number of NFS Ganesha daemons to be
+    deployed, each listening on a random port.
+
+    .. note:: NFS Ganesha packages must be pre-installed for this to work.
+
+Configuring NFS Ganesha to export RGW with vstart
+=================================================
+
+1) Using ``cephadm``
+
+    .. code:: bash
+
+        $ MON=1 OSD=3 RGW=1 NFS=1 ../src/vstart.sh -n -d --cephadm
+
+    This will deploy a single NFS Ganesha daemon using ``vstart.sh``, where the
+    daemon will listen on the default NFS Ganesha port. Also rgw export is
+    created.
+
+    .. note:: boto python module must be pre-installed for this to work.
diff --git a/doc/dev/wireshark.rst b/doc/dev/wireshark.rst
new file mode 100644
index 000000000..e03b3621c
--- /dev/null
+++ b/doc/dev/wireshark.rst
@@ -0,0 +1,41 @@
+=====================
+ Wireshark Dissector
+=====================
+
+Wireshark has support for the Ceph protocol and it will be shipped in the 1.12.1
+release.
+
+Using
+=====
+
+To use the Wireshark dissector you must build it from `git`__, the process is
+outlined in great detail in the `Building and Installing`__ section of the
+`Wireshark Users Guide`__.
+
+__ `Wireshark git`_
+__ WSUG_BI_
+__ WSUG_
+
+Developing
+==========
+
+The Ceph dissector lives in `Wireshark git`_ at
+``epan/dissectors/packet-ceph.c``.  At the top of that file there are some
+comments explaining how to insert new functionality or to update the encoding
+of existing types.
+
+Before you start hacking on Wireshark code you should look at the
+``doc/README.developer`` and ``doc/README.dissector`` documents as they explain
+the basics of writing dissectors.  After reading those two documents you should
+be prepared to work on the Ceph dissector.  `The Wireshark
+developers guide`__ also contains a lot of useful information but it is less
+directed and is more useful as a reference then an introduction.
+
+__ WSDG_
+
+.. _WSUG: https://www.wireshark.org/docs/wsug_html_chunked/
+.. _WSDG: https://www.wireshark.org/docs/wsdg_html_chunked/
+.. _WSUG_BI: https://www.wireshark.org/docs/wsug_html_chunked/ChapterBuildInstall.html
+.. _Wireshark git: https://www.wireshark.org/develop.html
+
+.. vi: textwidth=80 noexpandtab
diff --git a/doc/dev/zoned-storage.rst b/doc/dev/zoned-storage.rst
new file mode 100644
index 000000000..27f592b4c
--- /dev/null
+++ b/doc/dev/zoned-storage.rst
@@ -0,0 +1,134 @@
+=======================
+ Zoned Storage Support
+=======================
+
+http://zonedstorage.io
+
+Zoned Storage is a class of storage devices that enables host and storage
+devices to cooperate to achieve higher storage capacities, increased throughput,
+and lower latencies. The zoned storage interface is available through the SCSI
+Zoned Block Commands (ZBC) and Zoned Device ATA Command Set (ZAC) standards on
+Shingled Magnetic Recording (SMR) hard disks today and is also being adopted for
+NVMe Solid State Disks with the upcoming NVMe Zoned Namespaces (ZNS) standard.
+
+This project aims to enable Ceph to work on zoned storage drives and at the same
+time explore research problems related to adopting this new interface.  The
+first target is to enable non-ovewrite workloads (e.g. RGW) on host-managed SMR
+(HM-SMR) drives and explore cleaning (garbage collection) policies.  HM-SMR
+drives are high capacity hard drives with the ZBC/ZAC interface.  The longer
+term goal is to support ZNS SSDs, as they become available, as well as overwrite
+workloads.
+
+The first patch in these series enabled writing data to HM-SMR drives.  This
+patch introduces ZonedFreelistManger, a FreelistManager implementation that
+passes enough information to ZonedAllocator to correctly initialize state of
+zones by tracking the write pointer and the number of dead bytes per zone.  We
+have to introduce a new FreelistManager implementation because with zoned
+devices a region of disk can be in three states (empty, used, and dead), whereas
+current BitmapFreelistManager tracks only two states (empty and used).  It is
+not possible to accurately initialize the state of zones in ZonedAllocator by
+tracking only two states.  The third planned patch will introduce a rudimentary
+cleaner to form a baseline for further research.
+
+Currently we can perform basic RADOS benchmarks on an OSD running on an HM-SMR
+drives, restart the OSD, and read the written data, and write new data, as can
+be seen below.
+
+Please contact Abutalib Aghayev <agayev@psu.edu> for questions.
+
+::
+   
+  $ sudo zbd report -i -n /dev/sdc
+  Device /dev/sdc:
+      Vendor ID: ATA HGST HSH721414AL T240
+      Zone model: host-managed
+      Capacity: 14000.520 GB (27344764928 512-bytes sectors)
+      Logical blocks: 3418095616 blocks of 4096 B
+      Physical blocks: 3418095616 blocks of 4096 B
+      Zones: 52156 zones of 256.0 MB
+      Maximum number of open zones: no limit
+      Maximum number of active zones: no limit
+  52156 / 52156 zones
+  $ MON=1 OSD=1 MDS=0 sudo ../src/vstart.sh --new --localhost --bluestore --bluestore-devs /dev/sdc --bluestore-zoned
+  <snipped verbose output>
+  $ sudo ./bin/ceph osd pool create bench 32 32
+  pool 'bench' created
+  $ sudo ./bin/rados bench -p bench 10 write --no-cleanup
+  hints = 1
+  Maintaining 16 concurrent writes of 4194304 bytes to objects of size 4194304 for up to 10 seconds or 0 objects
+  Object prefix: benchmark_data_h0.cc.journaling712.narwhal.p_29846
+    sec Cur ops   started  finished  avg MB/s  cur MB/s last lat(s)  avg lat(s)
+      0       0         0         0         0         0           -           0
+      1      16        45        29   115.943       116    0.384175    0.407806
+      2      16        86        70   139.949       164    0.259845    0.391488
+      3      16       125       109   145.286       156     0.31727    0.404727
+      4      16       162       146   145.953       148    0.826671    0.409003
+      5      16       203       187   149.553       164     0.44815    0.404303
+      6      16       242       226   150.621       156    0.227488    0.409872
+      7      16       281       265   151.384       156    0.411896    0.408686
+      8      16       320       304   151.956       156    0.435135    0.411473
+      9      16       359       343   152.401       156    0.463699    0.408658
+     10      15       396       381   152.356       152    0.409554    0.410851
+  Total time run:         10.3305
+  Total writes made:      396
+  Write size:             4194304
+  Object size:            4194304
+  Bandwidth (MB/sec):     153.333
+  Stddev Bandwidth:       13.6561
+  Max bandwidth (MB/sec): 164
+  Min bandwidth (MB/sec): 116
+  Average IOPS:           38
+  Stddev IOPS:            3.41402
+  Max IOPS:               41
+  Min IOPS:               29
+  Average Latency(s):     0.411226
+  Stddev Latency(s):      0.180238
+  Max latency(s):         1.00844
+  Min latency(s):         0.108616
+  $ sudo ../src/stop.sh
+  $ # Notice the lack of "--new" parameter to vstart.sh
+  $ MON=1 OSD=1 MDS=0 sudo ../src/vstart.sh --localhost --bluestore --bluestore-devs /dev/sdc --bluestore-zoned  
+  <snipped verbose output>
+  $ sudo ./bin/rados bench -p bench 10 rand
+  hints = 1
+    sec Cur ops   started  finished  avg MB/s  cur MB/s last lat(s)  avg lat(s)
+      0       0         0         0         0         0           -           0
+      1      16        61        45   179.903       180    0.117329    0.244067
+      2      16       116       100   199.918       220    0.144162    0.292305
+      3      16       174       158   210.589       232    0.170941    0.285481
+      4      16       251       235   234.918       308    0.241175    0.256543
+      5      16       316       300   239.914       260    0.206044    0.255882
+      6      15       392       377   251.206       308    0.137972    0.247426
+      7      15       458       443   252.984       264   0.0800146    0.245138
+      8      16       529       513   256.346       280    0.103529    0.239888
+      9      16       587       571   253.634       232    0.145535      0.2453
+     10      15       646       631   252.254       240    0.837727    0.246019
+  Total time run:       10.272
+  Total reads made:     646
+  Read size:            4194304
+  Object size:          4194304
+  Bandwidth (MB/sec):   251.558
+  Average IOPS:         62
+  Stddev IOPS:          10.005
+  Max IOPS:             77
+  Min IOPS:             45
+  Average Latency(s):   0.249385
+  Max latency(s):       0.888654
+  Min latency(s):       0.0103208
+  $ sudo ./bin/rados bench -p bench 10 write --no-cleanup
+  hints = 1
+  Maintaining 16 concurrent writes of 4194304 bytes to objects of size 4194304 for up to 10 seconds or 0 objects
+  Object prefix: benchmark_data_h0.aa.journaling712.narwhal.p_64416
+    sec Cur ops   started  finished  avg MB/s  cur MB/s last lat(s)  avg lat(s)
+      0       0         0         0         0         0           -           0
+      1      16        46        30   119.949       120     0.52627    0.396166
+      2      16        82        66   131.955       144     0.48087    0.427311
+      3      16       123       107   142.627       164      0.3287    0.420614
+      4      16       158       142   141.964       140    0.405177    0.425993
+      5      16       192       176   140.766       136    0.514565    0.425175
+      6      16       224       208   138.635       128     0.69184    0.436672
+      7      16       261       245   139.967       148    0.459929    0.439502
+      8      16       301       285   142.468       160    0.250846    0.434799
+      9      16       336       320   142.189       140    0.621686    0.435457
+     10      16       374       358   143.166       152    0.460593    0.436384
+  
diff --git a/doc/favicon.ico b/doc/favicon.ico
new file mode 100644
index 000000000..90e538ba7
--- /dev/null
+++ b/doc/favicon.ico
diff --git a/doc/foundation.rst b/doc/foundation.rst
new file mode 100644
index 000000000..3d0e318d8
--- /dev/null
+++ b/doc/foundation.rst
@@ -0,0 +1,124 @@
+
+.. _foundation:
+
+=================
+ Ceph Foundation
+=================
+
+The Ceph Foundation exists to enable industry members to collaborate
+and pool resources to support the Ceph project community. The
+Foundation provides an open, collaborative, and neutral home for
+project stakeholders to coordinate their development and community
+investments in the Ceph ecosystem.
+
+The Ceph Foundation is organized as a directed fund under the Linux
+Foundation. Premier and General Member organizations contribute a
+yearly fee to become members. Associate members are educational
+institutions or government organizations and are invited to join at no
+cost.
+
+For more information, see `https://ceph.com/foundation
+<https://ceph.com/foundation>`_.
+
+
+Members
+=======
+
+Premier
+-------
+
+* `Amihan <https://amihan.net>`_
+* `Bloomberg <https://bloomberg.com>`_
+* `Canonical <https://www.canonical.com/>`_
+* `China Mobile <https://www.chinamobileltd.com/>`_
+* `DigitalOcean <https://www.digitalocean.com/>`_
+* `Intel <http://www.intel.com/>`_
+* `OVH <https://www.ovh.com/>`_
+* `Red Hat <https://www.redhat.com/>`_
+* `Samsung Electronics <https://samsung.com/>`_
+* `SoftIron <https://www.softiron.com/>`_
+* `SUSE <https://www.suse.com/>`_
+* `Western Digital <https://www.wdc.com/>`_
+* `XSKY <https://www.xsky.com/en/>`_
+* `ZTE <https://www.zte.com.cn/global/>`_
+
+General
+-------
+
+* `ARM <http://www.arm.com/>`_
+* `Catalyst Cloud <https://catalystcloud.nz/>`_
+* `Cloudbase Solutions <https://cloudbase.it/>`_
+* `Clyso <https://www.clyso.com/en/>`_
+* `croit <http://www.croit.io/>`_
+* `DiDi <https://www.didiglobal.com/>`_
+* `EasyStack <https://www.easystack.io/>`_
+* `ISS <http://iss-integration.com/>`_
+* `QCT <https://www.qct.io/>`_
+* `SinoRail <http://www.sinorail.com/>`_
+* `Vexxhost <https://vexxhost.com>`_
+
+Associate
+---------
+
+* `Boston University <http://www.bu.com/>`_
+* `Center for Research in Open Source Systems (CROSS) <http://cross.ucsc.edu/>`_
+* `CERN <https://home.cern/>`_
+* `FASRC <https://www.rc.fas.harvard.edu/>`_
+* `grnet <https://grnet.gr/>`_
+* `Monash University <http://www.monash.edu/>`_
+* `NRF SARAO <http://www.ska.ac.za/about/sarao/>`_
+* `Science & Technology Facilities Councel (STFC) <https://stfc.ukri.org/>`_
+* `University of Michigan <http://www.osris.org/>`_
+* `SWITCH <https://switch.ch/>`_
+
+Governing Board
+===============
+
+The Governing Board consists of all Premier members, a representative
+for the General members, a representative for the Associate members,
+and a representative from the Ceph Leadership Team (the technical
+governance body). The board is responsible for:
+
+* Building and approving an annual budget for spending in support of
+  the Ceph project
+* Establishing ad-hoc committees to address current needs of the
+  project
+* Coordinating outreach or marketing
+* Meeting regularly to discuss Foundation activities, the status of
+  the Ceph project, and overall project strategy
+* Voting on any decisions or matters before the board
+
+The Ceph Foundation board is not responsible for and does not have any
+direct control over the technical governance of Ceph. Development and
+engineering activities are managed through traditional open source
+processes and are overseen by the :ref:`clt`. For more
+information see :ref:`governance`.
+
+Members
+-------
+
+* Anjaneya "Reddy" Chagam (Intel)
+* Dan van der Ster (CERN) - Associate member representative
+* Haomai Wang (XSKY)
+* James Page (Canonical)
+* Lenz Grimmer (SUSE) - Ceph Leadership Team representative
+* Lars Marowsky-Bree (SUSE)
+* Matias Bjorling (Western Digital)
+* Matthew Leonard (Bloomberg)
+* Mike Perez (Red Hat) - Ceph community manager
+* Myoungwon Oh (Samsung Electronics)
+* Paul Emmerich (croit) - General member representative
+* Pawel Sadowski (OVH)
+* Phil Straw (SoftIron)
+* Robin Johnson (DigitalOcean)
+* Sage Weil (Red Hat) - Ceph project leader
+* Winston Damarillio (Amihan)
+* Xie Xingguo (ZTE)
+* Zhang Shaowen (China Mobile)
+
+Joining
+=======
+
+For information about joining the Ceph Foundation, please contact
+membership@linuxfoundation.org.
+
diff --git a/doc/glossary.rst b/doc/glossary.rst
new file mode 100644
index 000000000..16594e6c2
--- /dev/null
+++ b/doc/glossary.rst
@@ -0,0 +1,356 @@
+===============
+ Ceph Glossary
+===============
+
+.. glossary::
+
+	:ref:`BlueStore<rados_config_storage_devices_bluestore>`
+                OSD BlueStore is a storage back end used by OSD daemons, and
+                was designed specifically for use with Ceph. BlueStore was
+                introduced in the Ceph Kraken release. In the Ceph Luminous
+                release, BlueStore became Ceph's default storage back end,
+                supplanting FileStore. Unlike :term:`filestore`, BlueStore
+                stores objects directly on Ceph block devices without any file
+                system interface. Since Luminous (12.2), BlueStore has been
+                Ceph's default and recommended storage back end.
+
+	Ceph
+                Ceph is a distributed network storage and file system with
+                distributed metadata management and POSIX semantics.
+
+	Ceph Block Device
+                A software instrument that orchestrates the storage of
+                block-based data in Ceph. Ceph Block Device (also called "RBD",
+                or "RADOS block device") splits block-based application data
+                into "chunks". RADOS stores these chunks as objects. Ceph Block
+                Device orchestrates the storage of those objects across the
+                storage cluster. See also :term:`RBD`.
+
+	Ceph Block Storage
+                One of the three kinds of storage supported by Ceph (the other
+                two are object storage and file storage). Ceph Block Storage is
+                the block storage "product", which refers to block-storage
+                related services and capabilities when used in conjunction with
+                the collection of (1) ``librbd`` (a python module that provides
+                file-like access to :term:`RBD` images), (2) a hypervisor such
+                as QEMU or Xen, and (3) a hypervisor abstraction layer such as
+                ``libvirt``.
+
+	Ceph Client
+                Any of the Ceph components that can access a Ceph Storage
+                Cluster. This includes the Ceph Object Gateway, the Ceph Block
+                Device, the Ceph File System, and their corresponding
+                libraries. It also includes kernel modules, and FUSEs
+                (Filesystems in USERspace).
+
+	Ceph Client Libraries
+                The collection of libraries that can be used to interact with
+                components of the Ceph Cluster.
+
+	Ceph Cluster Map
+                See :term:`Cluster Map`
+
+	Ceph Dashboard
+                :ref:`The Ceph Dashboard<mgr-dashboard>` is a built-in
+                web-based Ceph management and monitoring application through
+                which you can inspect and administer various resources within
+                the cluster. It is implemented as a :ref:`ceph-manager-daemon`
+                module.
+
+	Ceph File System
+                See :term:`CephFS`
+
+	:ref:`CephFS<ceph-file-system>`
+                The **Ceph F**\ile **S**\ystem, or CephFS, is a
+                POSIX-compliant file system built on top of Ceph’s distributed
+                object store, RADOS.  See :ref:`CephFS Architecture
+                <arch-cephfs>` for more details.
+
+	Ceph Interim Release
+                See :term:`Releases`.
+
+	Ceph Kernel Modules
+                The collection of kernel modules that can be used to interact
+                with the Ceph Cluster (for example: ``ceph.ko``, ``rbd.ko``).
+
+	:ref:`Ceph Manager<ceph-manager-daemon>`
+                The Ceph manager daemon (ceph-mgr) is a daemon that runs
+                alongside monitor daemons to provide monitoring and interfacing
+                to external monitoring and management systems. Since the
+                Luminous release (12.x), no Ceph cluster functions properly
+                unless it contains a running ceph-mgr daemon. 
+
+	Ceph Manager Dashboard
+                See :term:`Ceph Dashboard`.
+
+	Ceph Metadata Server
+                See :term:`MDS`.
+
+	Ceph Monitor
+                A daemon that maintains a map of the state of the cluster. This
+                "cluster state" includes the monitor map, the manager map, the
+                OSD map, and the CRUSH map. A Ceph cluster must contain a
+                minimum of three running monitors in order to be both redundant
+                and highly-available. Ceph monitors and the nodes on which they
+                run are often referred to as "mon"s. See :ref:`Monitor Config
+                Reference <monitor-config-reference>`.
+
+	Ceph Node
+               A Ceph node is a unit of the Ceph Cluster that communicates with
+               other nodes in the Ceph Cluster in order to replicate and
+               redistribute data. All of the nodes together are called the
+               :term:`Ceph Storage Cluster`. Ceph nodes include :term:`OSD`\s,
+               :term:`Ceph Monitor`\s, :term:`Ceph Manager`\s, and
+               :term:`MDS`\es. The term "node" is usually equivalent to "host"
+               in the Ceph documentation. If you have a running Ceph Cluster,
+               you can list all of the nodes in it by running the command
+               ``ceph node ls all``.
+                
+	:ref:`Ceph Object Gateway<object-gateway>`
+                An object storage interface built on top of librados. Ceph
+                Object Gateway provides a RESTful gateway between applications
+                and Ceph storage clusters.
+
+	Ceph Object Storage
+                See :term:`Ceph Object Store`.
+
+	Ceph Object Store
+                A Ceph Object Store consists of a :term:`Ceph Storage Cluster`
+                and a :term:`Ceph Object Gateway` (RGW).
+
+	:ref:`Ceph OSD<rados_configuration_storage-devices_ceph_osd>`
+                Ceph **O**\bject **S**\torage **D**\aemon. The Ceph OSD
+                software, which interacts with logical disks (:term:`OSD`).
+                Around 2013, there was an attempt by "research and industry"
+                (Sage's own words) to insist on using the term "OSD" to mean
+                only "Object Storage Device", but the Ceph community has always
+                persisted in using the term to mean "Object Storage Daemon" and
+                no less an authority than Sage Weil himself confirms in
+                November of 2022 that "Daemon is more accurate for how Ceph is
+                built" (private correspondence between Zac Dover and Sage Weil,
+                07 Nov 2022). 
+
+	Ceph OSD Daemon
+                See :term:`Ceph OSD`.
+
+	Ceph OSD Daemons
+                See :term:`Ceph OSD`.
+
+	Ceph Platform
+                All Ceph software, which includes any piece of code hosted at
+                `https://github.com/ceph`_.
+
+	Ceph Point Release
+                See :term:`Releases`.
+
+	Ceph Project
+                The aggregate term for the people, software, mission and
+                infrastructure of Ceph.
+
+	Ceph Release
+                See :term:`Releases`.
+
+	Ceph Release Candidate
+                See :term:`Releases`.
+
+	Ceph Stable Release
+                See :term:`Releases`.
+
+	Ceph Stack
+		A collection of two or more components of Ceph.
+
+	:ref:`Ceph Storage Cluster<arch-ceph-storage-cluster>`
+                The collection of :term:`Ceph Monitor`\s, :term:`Ceph
+                Manager`\s, :term:`Ceph Metadata Server`\s, and :term:`OSD`\s
+                that work together to store and replicate data for use by
+                applications, Ceph Users, and :term:`Ceph Client`\s. Ceph
+                Storage Clusters receive data from :term:`Ceph Client`\s.
+
+	cephx
+                The Ceph authentication protocol. Cephx operates like Kerberos,
+                but it has no single point of failure.
+
+	Cloud Platforms
+	Cloud Stacks
+                Third party cloud provisioning platforms such as OpenStack,
+                CloudStack, OpenNebula, and Proxmox VE.
+
+	Cluster Map
+                The set of maps consisting of the monitor map, OSD map, PG map,
+                MDS map, and CRUSH map, which together report the state of the
+                Ceph cluster. See :ref:`the "Cluster Map" section of the
+                Architecture document<architecture_cluster_map>` for details.
+
+	CRUSH
+                Controlled Replication Under Scalable Hashing. It is the
+                algorithm Ceph uses to compute object storage locations.
+
+	CRUSH rule
+                The CRUSH data placement rule that applies to a particular
+                pool(s).
+
+        DAS
+                **D**\irect-\ **A**\ttached **S**\torage. Storage that is
+                attached directly to the computer accessing it, without passing
+                through a network.  Contrast with NAS and SAN.
+
+	:ref:`Dashboard<mgr-dashboard>`
+                A built-in web-based Ceph management and monitoring application
+                to administer various aspects and objects of the cluster. The
+                dashboard is implemented as a Ceph Manager module. See
+                :ref:`mgr-dashboard` for more details.
+
+	Dashboard Module
+                Another name for :term:`Dashboard`.
+
+	Dashboard Plugin
+	filestore
+                A back end for OSD daemons, where a Journal is needed and files
+                are written to the filesystem.
+
+        FQDN
+                **F**\ully **Q**\ualified **D**\omain **N**\ame. A domain name
+                that is applied to a node in a network and that specifies the
+                node's exact location in the tree hierarchy of the DNS.
+
+                In the context of Ceph cluster administration, FQDNs are often
+                applied to hosts. In this documentation, the term "FQDN" is
+                used mostly to distinguish between FQDNs and relatively simpler
+                hostnames, which do not specify the exact location of the host
+                in the tree hierarchy of the DNS but merely name the host.
+
+	Host
+                Any single machine or server in a Ceph Cluster. See :term:`Ceph
+                Node`.
+
+	LVM tags
+                Extensible metadata for LVM volumes and groups. It is used to
+                store Ceph-specific information about devices and its
+                relationship with OSDs.
+
+	:ref:`MDS<cephfs_add_remote_mds>`
+                The Ceph **M**\eta\ **D**\ata **S**\erver daemon. Also referred
+                to as "ceph-mds". The Ceph metadata server daemon must be
+                running in any Ceph cluster that runs the CephFS file system.
+                The MDS stores all filesystem metadata. 
+
+	MGR
+                The Ceph manager software, which collects all the state from
+                the whole cluster in one place.
+
+	MON
+		The Ceph monitor software.
+
+	Node
+                See :term:`Ceph Node`.
+
+	Object Storage Device
+                See :term:`OSD`.
+
+	OSD
+                Probably :term:`Ceph OSD`, but not necessarily. Sometimes
+                (especially in older correspondence, and especially in
+                documentation that is not written specifically for Ceph), "OSD"
+                means "**O**\bject **S**\torage **D**\evice", which refers to a
+                physical or logical storage unit (for example: LUN). The Ceph
+                community has always used the term "OSD" to refer to
+                :term:`Ceph OSD Daemon` despite an industry push in the
+                mid-2010s to insist that "OSD" should refer to "Object Storage
+                Device", so it is important to know which meaning is intended. 
+
+	OSD fsid
+                This is a unique identifier used to identify an OSD. It is
+                found in the OSD path in a file called ``osd_fsid``. The
+                term ``fsid`` is used interchangeably with ``uuid``
+
+	OSD id
+                The integer that defines an OSD. It is generated by the
+                monitors during the creation of each OSD.
+
+	OSD uuid
+                This is the unique identifier of an OSD. This term is used
+                interchangeably with ``fsid``
+
+	:ref:`Pool<rados_pools>`
+		A pool is a logical partition used to store objects.
+
+	Pools
+                See :term:`pool`.
+
+	RADOS
+                **R**\eliable **A**\utonomic **D**\istributed **O**\bject
+                **S**\tore. RADOS is the object store that provides a scalable
+                service for variably-sized objects. The RADOS object store is
+                the core component of a Ceph cluster.  `This blog post from
+                2009
+                <https://ceph.io/en/news/blog/2009/the-rados-distributed-object-store/>`_
+                provides a beginner's introduction to RADOS. Readers interested
+                in a deeper understanding of RADOS are directed to `RADOS: A
+                Scalable, Reliable Storage Service for Petabyte-scale Storage
+                Clusters <https://ceph.io/assets/pdfs/weil-rados-pdsw07.pdf>`_.
+
+	RADOS Cluster
+                A proper subset of the Ceph Cluster consisting of
+                :term:`OSD`\s, :term:`Ceph Monitor`\s, and :term:`Ceph
+                Manager`\s.
+                
+	RADOS Gateway
+                See :term:`RGW`.
+
+	RBD
+                The block storage component of Ceph. Also called "RADOS Block
+                Device" or :term:`Ceph Block Device`.
+
+        Releases
+
+	        Ceph Interim Release
+                        A version of Ceph that has not yet been put through
+                        quality assurance testing. May contain new features.
+
+                Ceph Point Release
+                        Any ad hoc release that includes only bug fixes and
+                        security fixes.
+
+                Ceph Release
+                        Any distinct numbered version of Ceph.
+
+                Ceph Release Candidate
+                        A major version of Ceph that has undergone initial
+                        quality assurance testing and is ready for beta
+                        testers.
+
+                Ceph Stable Release
+                        A major version of Ceph where all features from the
+                        preceding interim releases have been put through
+                        quality assurance testing successfully.
+
+	Reliable Autonomic Distributed Object Store
+                The core set of storage software which stores the user's data
+                (MON+OSD). See also :term:`RADOS`.
+
+	:ref:`RGW<object-gateway>`
+                **R**\ADOS **G**\ate **W**\ay.
+
+                The component of Ceph that provides a gateway to both the
+                Amazon S3 RESTful API and the OpenStack Swift API. Also called
+                "RADOS Gateway" and "Ceph Object Gateway".
+
+        secrets
+                Secrets are credentials used to perform digital authentication
+                whenever privileged users must access systems that require
+                authentication. Secrets can be passwords, API keys, tokens, SSH
+                keys, private certificates, or encryption keys.
+
+        SDS
+                Software-defined storage.
+
+	systemd oneshot
+                A systemd ``type`` where a command is defined in ``ExecStart``
+                which will exit upon completion (it is not intended to
+                daemonize)
+
+	Teuthology
+		The collection of software that performs scripted tests on Ceph.
+
+.. _https://github.com/ceph: https://github.com/ceph
+.. _Cluster Map: ../architecture#cluster-map   
diff --git a/doc/governance.rst b/doc/governance.rst
new file mode 100644
index 000000000..0796fad91
--- /dev/null
+++ b/doc/governance.rst
@@ -0,0 +1,103 @@
+.. _governance:
+
+============
+ Governance
+============
+
+The Ceph open source community is guided by a few different groups.
+
+Project Leader
+--------------
+
+The Ceph project is currently led by Sage Weil <sage@redhat.com>.  The
+project leader is responsible for guiding the overall direction of the
+project and ensuring that the developer and user communities are
+healthy.
+
+
+Committers
+----------
+
+Committers are project contributors who have write access to the central
+Ceph code repositories, currently hosted on GitHub.  This group of developers
+is collectively empowered to make changes to the Ceph source code.
+
+Generally speaking, no individual should make a change in isolation:
+all code contributions go through a collaborative review process (and
+undergo testing) before being merged.  The specifics of this process
+are dynamic and evolving over time.
+
+New committers are added to the project (or committers removed from
+the project) at the discretion of the Ceph Leadership Team (below).
+The criteria for becoming a contributor include a consistent level of
+quality and engagement in the project over time.
+
+
+.. _clt:
+
+Ceph Leadership Team
+--------------------
+
+The Ceph Leadership Team (CLT) is a collection of component leads and
+other core developers who collectively make technical decisions for
+the project.  These decisions are generally made by consensus,
+although voting may be used if necessary.
+
+The CLT meets weekly via video chat to discuss any pending issues or
+decisions.  Minutes for the CLT meetings are published at
+`https://pad.ceph.com/p/clt-weekly-minutes <https://pad.ceph.com/p/clt-weekly-minutes>`_.
+
+Committers are added to or removed from the CLT at the discretion of
+the CLT itself.
+
+Current CLT members are:
+
+ * Abhishek Lekshmanan <abhishek@suse.com>
+ * Casey Bodley <cbodley@redhat.com>
+ * Ernesto Puerta <epuerta@redhat.com>
+ * Gregory Farnum <gfarnum@redhat.com>
+ * Haomai Wang <haomai@xsky.com>
+ * Jason Dillaman <dillaman@redhat.com>
+ * Josh Durgin <jdurgin@redhat.com>
+ * João Eduardo Luis <joao@suse.de>
+ * Ken Dreyer <kdreyer@redhat.com>
+ * Matt Benjamin <mbenjami@redhat.com>
+ * Myoungwon Oh <omwmw@sk.com>
+ * Neha Ojha <nojha@redhat.com>
+ * Patrick Donnelly <pdonnell@redhat.com>
+ * Sage Weil <sage@redhat.com>
+ * Sebastian Wagner <swagner@suse.com>
+ * Xie Xingguo <xie.xingguo@zte.com.cn>
+ * Yehuda Sadeh <yehuda@redhat.com>
+
+Component Leads
+---------------
+
+Each major subcomponent of the Ceph project has a lead engineer who is
+responsible for guiding and coordinating development.  The leads are
+nominated or appointed at the discretion of the project leader or the
+CLT.  Leads responsibilities include:
+
+ * guiding the (usually) daily "stand-up" coordination calls over video chat
+ * building the development roadmap for each release cycle
+ * coordinating development activity between contributors
+ * ensuring that contributions are reviewed
+ * ensuring that different proposed changes do not conflict
+ * ensuring that testing remains robust (new features include tests, changes do not break tests, etc.)
+
+All component leads are included on the CLT.  They are expected to
+report progress and status updates to the rest of the leadership team
+and to help facilitate any cross-component coordination of
+development.
+
+The Ceph Foundation
+-------------------
+
+The Ceph Foundation is organized as a directed fund under the Linux
+Foundation and is tasked with supporting the Ceph project community
+and ecosystem.  It has no direct control over the technical direction
+of the Ceph open source project beyond offering feedback and input
+into the collaborative development process.
+
+For more information, see :ref:`foundation`.
+
diff --git a/doc/images/CRUSH.jpg b/doc/images/CRUSH.jpg
new file mode 100644
index 000000000..ac7975935
--- /dev/null
+++ b/doc/images/CRUSH.jpg
diff --git a/doc/images/HAProxy_for_RGW.svg b/doc/images/HAProxy_for_RGW.svg
new file mode 100644
index 000000000..c2fc95f1f
--- /dev/null
+++ b/doc/images/HAProxy_for_RGW.svg
@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:lucid="lucid" width="1182" height="877"><g transform="translate(-539 -40)" lucid:page-tab-id="0_0"><path d="M0 0h1870.4v1323.2H0z" fill="#fff"/><g filter="url(#a)"><path d="M1380 320c0-22.1 17.9-40 40-40h240c22.1 0 40 17.9 40 40v280c0 22.1-17.9 40-40 40h-240c-22.1 0-40-17.9-40-40z" stroke="#5e5e5e" stroke-width="3" fill="#fff"/><use xlink:href="#b" transform="matrix(1,0,0,1,1385,285) translate(14.01 17.77777777777778)"/><use xlink:href="#c" transform="matrix(1,0,0,1,1385,285) translate(65.80012345679013 17.77777777777778)"/></g><g filter="url(#a)"><path d="M920 320c0-22.1 17.9-40 40-40h240c22.1 0 40 17.9 40 40v280c0 22.1-17.9 40-40 40H960c-22.1 0-40-17.9-40-40z" stroke="#5e5e5e" stroke-width="3" fill="#fff"/><use xlink:href="#b" transform="matrix(1,0,0,1,925,285) translate(14.01 17.77777777777778)"/><use xlink:href="#d" transform="matrix(1,0,0,1,925,285) translate(65.80012345679013 17.77777777777778)"/></g><g filter="url(#a)"><path d="M560 320c0-22.1 17.9-40 40-40h240c22.1 0 40 17.9 40 40v280c0 22.1-17.9 40-40 40H600c-22.1 0-40-17.9-40-40z" stroke="#5e5e5e" stroke-width="3" fill="#fff"/><use xlink:href="#b" transform="matrix(1,0,0,1,565,285) translate(14.01 17.77777777777778)"/><use xlink:href="#e" transform="matrix(1,0,0,1,565,285) translate(65.80012345679013 17.77777777777778)"/></g><path d="M708 60c17.67 0 32 17.9 32 40s-14.33 40-32 40h-96c-17.67 0-32-17.9-32-40s14.33-40 32-40z" stroke="#5e5e5e" stroke-width="3" fill="#c1e4f7"/><use xlink:href="#f" transform="matrix(1,0,0,1,585,65) translate(18.95061728395062 39.65277777777778)"/><use xlink:href="#g" transform="matrix(1,0,0,1,585,65) translate(79.32098765432099 39.65277777777778)"/><path d="M1608 63c17.67 0 32 17.9 32 40s-14.33 40-32 40h-96c-17.67 0-32-17.9-32-40s14.33-40 32-40z" stroke="#5e5e5e" stroke-width="3" fill="#c1e4f7"/><use xlink:href="#f" transform="matrix(1,0,0,1,1485,68) translate(7.870370370370381 39.65277777777778)"/><use xlink:href="#h" transform="matrix(1,0,0,1,1485,68) translate(68.24074074074075 39.65277777777778)"/><use xlink:href="#c" transform="matrix(1,0,0,1,1485,68) translate(126.14197530864197 39.65277777777778)"/><path d="M700 360c0 22.1-17.9 40-40 40s-40-17.9-40-40 17.9-40 40-40 40 17.9 40 40z" stroke="#5e5e5e" stroke-width="3" fill="#b2b2b2"/><use xlink:href="#i" transform="matrix(1,0,0,1,625,325) translate(17.098765432098766 39.65277777777778)"/><path d="M1058.5 360c0 22.1-17.9 40-40 40s-40-17.9-40-40 17.9-40 40-40 40 17.9 40 40z" stroke="#5e5e5e" stroke-width="3" fill="#a3d977"/><use xlink:href="#i" transform="matrix(1,0,0,1,983.4990644204674,325) translate(17.098765432098766 39.65277777777778)"/><path d="M732.7 360H945.8" stroke="#5e5e5e" stroke-width="8" fill="none"/><path d="M714.45 360l14.26-4.64v9.28zM964.05 360l-14.26 4.64v-9.28z" stroke="#5e5e5e" stroke-width="8" fill="#5e5e5e"/><path d="M580 506c0-3.3 2.7-6 6-6h148c3.3 0 6 2.7 6 6v68c0 3.3-2.7 6-6 6H586c-3.3 0-6-2.7-6-6z" stroke="#5e5e5e" stroke-width="3" fill="#b2b2b2"/><use xlink:href="#j" transform="matrix(1,0,0,1,592,512) translate(14.97530864197531 31.77777777777778)"/><use xlink:href="#e" transform="matrix(1,0,0,1,592,512) translate(108.67901234567901 31.77777777777778)"/><path d="M1518.5 360c0 22.1-17.9 40-40 40s-40-17.9-40-40 17.9-40 40-40 40 17.9 40 40z" stroke="#5e5e5e" stroke-width="3" fill="#b2b2b2"/><use xlink:href="#i" transform="matrix(1,0,0,1,1443.4990644204672,325) translate(17.098765432098766 39.65277777777778)"/><path d="M1091.2 360h314.6" stroke="#5e5e5e" stroke-width="8" fill="none"/><path d="M1072.94 360l14.27-4.64v9.28zM1424.05 360l-14.26 4.64v-9.28z" stroke="#5e5e5e" stroke-width="8" fill="#5e5e5e"/><path d="M680 426.33c0-3.3 2.7-6 6-6h148c3.3 0 6 2.7 6 6V474c0 3.3-2.7 6-6 6H686c-3.3 0-6-2.7-6-6z" stroke="#5e5e5e" stroke-width="3" fill="#b2b2b2"/><use xlink:href="#k" transform="matrix(1,0,0,1,692,432.3333333333333) translate(3.864197530864189 17.77777777777778)"/><use xlink:href="#l" transform="matrix(1,0,0,1,692,432.3333333333333) translate(119.79012345679013 17.77777777777778)"/><use xlink:href="#m" transform="matrix(1,0,0,1,692,432.3333333333333) translate(32.19753086419753 44.44444444444444)"/><path d="M1188 60c17.67 0 32 17.9 32 40s-14.33 40-32 40h-96c-17.67 0-32-17.9-32-40s14.33-40 32-40z" stroke="#5e5e5e" stroke-width="3" fill="#c1e4f7"/><use xlink:href="#f" transform="matrix(1,0,0,1,1065,65) translate(18.95061728395062 39.65277777777778)"/><use xlink:href="#g" transform="matrix(1,0,0,1,1065,65) translate(79.32098765432099 39.65277777777778)"/><path d="M938.5 506c0-3.3 2.7-6 6-6h148c3.3 0 6 2.7 6 6v68c0 3.3-2.7 6-6 6h-148c-3.3 0-6-2.7-6-6z" stroke="#5e5e5e" stroke-width="3" fill="#a3d977"/><use xlink:href="#j" transform="matrix(1,0,0,1,950.4990644204674,512) translate(14.97530864197531 31.77777777777778)"/><use xlink:href="#d" transform="matrix(1,0,0,1,950.4990644204674,512) translate(108.67901234567901 31.77777777777778)"/><path d="M1398.5 506c0-3.3 2.7-6 6-6h148c3.3 0 6 2.7 6 6v68c0 3.3-2.7 6-6 6h-148c-3.3 0-6-2.7-6-6z" stroke="#5e5e5e" stroke-width="3" fill="#b2b2b2"/><use xlink:href="#j" transform="matrix(1,0,0,1,1410.4990644204672,512) translate(13.154320987654323 31.77777777777778)"/><use xlink:href="#c" transform="matrix(1,0,0,1,1410.4990644204672,512) translate(106.85802469135803 31.77777777777778)"/><path d="M660 402.5V480" stroke="#5e5e5e" stroke-width="2" fill="none"/><path d="M659.98 401.5l1.02-.03v1.06h-2v-1.1z" fill="#5e5e5e"/><path d="M660 495.26L655.36 481h9.28z" stroke="#5e5e5e" stroke-width="2" fill="#5e5e5e"/><path d="M1018.5 402.5V480" stroke="#5e5e5e" stroke-width="2" fill="none"/><path d="M1018.48 401.5l1.02-.03v1.06h-2v-1.1z" fill="#5e5e5e"/><path d="M1018.5 495.26l-4.64-14.26h9.27z" stroke="#5e5e5e" stroke-width="2" fill="#5e5e5e"/><path d="M1478.5 402.5V480" stroke="#5e5e5e" stroke-width="2" fill="none"/><path d="M1478.48 401.5l1.02-.03v1.06h-2v-1.1z" fill="#5e5e5e"/><path d="M1478.5 495.26l-4.64-14.26h9.27z" stroke="#5e5e5e" stroke-width="2" fill="#5e5e5e"/><path d="M560 823c0-3.3 2.7-6 6-6h148c3.3 0 6 2.7 6 6v68c0 3.3-2.7 6-6 6H566c-3.3 0-6-2.7-6-6z" stroke="#5e5e5e" stroke-width="3" fill="#ff8f80"/><use xlink:href="#n" transform="matrix(1,0,0,1,572,829) translate(40.90123456790124 17.77777777777778)"/><use xlink:href="#o" transform="matrix(1,0,0,1,572,829) translate(18.648148148148145 44.44444444444444)"/><use xlink:href="#e" transform="matrix(1,0,0,1,572,829) translate(105.00617283950618 44.44444444444444)"/><path d="M740 823c0-3.3 2.7-6 6-6h148c3.3 0 6 2.7 6 6v68c0 3.3-2.7 6-6 6H746c-3.3 0-6-2.7-6-6z" stroke="#5e5e5e" stroke-width="3" fill="#ff8f80"/><use xlink:href="#n" transform="matrix(1,0,0,1,752,829) translate(40.90123456790124 17.77777777777778)"/><use xlink:href="#o" transform="matrix(1,0,0,1,752,829) translate(18.648148148148145 44.44444444444444)"/><use xlink:href="#d" transform="matrix(1,0,0,1,752,829) translate(105.00617283950618 44.44444444444444)"/><path d="M920 823c0-3.3 2.7-6 6-6h148c3.3 0 6 2.7 6 6v68c0 3.3-2.7 6-6 6H926c-3.3 0-6-2.7-6-6z" stroke="#5e5e5e" stroke-width="3" fill="#ff8f80"/><use xlink:href="#n" transform="matrix(1,0,0,1,932,829) translate(40.90123456790124 17.77777777777778)"/><use xlink:href="#o" transform="matrix(1,0,0,1,932,829) translate(18.648148148148145 44.44444444444444)"/><use xlink:href="#p" transform="matrix(1,0,0,1,932,829) translate(105.00617283950618 44.44444444444444)"/><path d="M1480 823c0-3.3 2.7-6 6-6h148c3.3 0 6 2.7 6 6v68c0 3.3-2.7 6-6 6h-148c-3.3 0-6-2.7-6-6z" stroke="#5e5e5e" stroke-width="3" fill="#ff8f80"/><use xlink:href="#n" transform="matrix(1,0,0,1,1492,829) translate(40.90123456790124 17.77777777777778)"/><use xlink:href="#o" transform="matrix(1,0,0,1,1492,829) translate(18.648148148148145 44.44444444444444)"/><use xlink:href="#q" transform="matrix(1,0,0,1,1492,829) translate(105.00617283950618 44.44444444444444)"/><path d="M1058.5 426c0-3.3 2.7-6 6-6h148c3.3 0 6 2.7 6 6v47.67c0 3.3-2.7 6-6 6h-148c-3.3 0-6-2.7-6-6z" stroke="#5e5e5e" stroke-width="3" fill="#a3d977"/><use xlink:href="#k" transform="matrix(1,0,0,1,1070.4990644204674,432) translate(3.864197530864189 17.77777777777778)"/><use xlink:href="#r" transform="matrix(1,0,0,1,1070.4990644204674,432) translate(119.79012345679013 17.77777777777778)"/><use xlink:href="#s" transform="matrix(1,0,0,1,1070.4990644204674,432) translate(34.11111111111111 44.44444444444444)"/><path d="M1518.5 426c0-3.3 2.7-6 6-6h148c3.3 0 6 2.7 6 6v47.67c0 3.3-2.7 6-6 6h-148c-3.3 0-6-2.7-6-6z" stroke="#5e5e5e" stroke-width="3" fill="#b2b2b2"/><use xlink:href="#k" transform="matrix(1,0,0,1,1530.4990644204672,432) translate(2.0432098765432016 17.77777777777778)"/><use xlink:href="#t" transform="matrix(1,0,0,1,1530.4990644204672,432) translate(117.96913580246914 17.77777777777778)"/><use xlink:href="#m" transform="matrix(1,0,0,1,1530.4990644204672,432) translate(32.19753086419753 44.44444444444444)"/><path d="M1138.5 417.5v-5.74c0-3.3-2.7-6-6-6h-83.23" stroke="#5e5e5e" stroke-width="2" fill="none"/><path d="M1139.5 418.5h-2v-1.03h2z" fill="#5e5e5e"/><path d="M1044.84 395.98l10.1 11.08-8.44 3.83z" stroke="#5e5e5e" stroke-width="2" fill="#5e5e5e"/><path d="M1598.5 417.5v-6.98c0-3.3-2.7-6-6-6H1511" stroke="#5e5e5e" stroke-width="2" fill="none"/><path d="M1599.5 418.5h-2v-1.03h2z" fill="#5e5e5e"/><path d="M1507.48 393.63l8.8 12.14-8.82 2.86z" stroke="#5e5e5e" stroke-width="2" fill="#5e5e5e"/><path d="M760 417.83V406c0-3.3-2.7-6-6-6h-56.86" stroke="#5e5e5e" stroke-width="2" fill="none"/><path d="M761 418.83h-2v-1.02h2z" fill="#5e5e5e"/><path d="M691.67 391.43l11.58 9.54-7.82 4.98z" stroke="#5e5e5e" stroke-width="2" fill="#5e5e5e"/><path d="M659.9 582.5L641.6 797.07" stroke="#b2b2b2" stroke-width="2" fill="none"/><path d="M660.9 582.6l-1.98-.16.08-.94h2z" fill="#b2b2b2"/><path d="M640.28 812.28l-3.4-14.6 9.23.78z" stroke="#b2b2b2" stroke-width="2" fill="#b2b2b2"/><path d="M660.56 582.33l149 217.9" stroke="#b2b2b2" stroke-width="2" fill="none"/><path d="M661.4 581.78l-1.65 1.13-.96-1.4h2.4z" fill="#b2b2b2"/><path d="M818.17 812.83l-11.87-9.16 7.65-5.23z" stroke="#b2b2b2" stroke-width="2" fill="#b2b2b2"/><path d="M660.82 582.07L984.76 805" stroke="#b2b2b2" stroke-width="2" fill="none"/><path d="M660.28 582.9l-2.04-1.4h3z" fill="#b2b2b2"/><path d="M997.33 813.67l-14.38-4.27 5.26-7.64z" stroke="#b2b2b2" stroke-width="2" fill="#b2b2b2"/><path d="M660.97 581.75l881.12 229.1" stroke="#b2b2b2" stroke-width="2" fill="none"/><path d="M660.74 582.73l-3.9-1.02.06-.2h4.16z" fill="#b2b2b2"/><path d="M1556.87 814.7l-14.98.88 2.33-8.97z" stroke="#b2b2b2" stroke-width="2" fill="#b2b2b2"/><path d="M1017.65 582.03l-361.9 223.74" stroke="#a3d977" stroke-width="2" fill="none"/><path d="M1018.15 582.9l-.86-1.4h3.1z" fill="#a3d977"/><path d="M642.75 813.8l9.7-11.45 4.87 7.9z" stroke="#a3d977" stroke-width="2" fill="#a3d977"/><path d="M1017.85 582.26L831.97 801.4" stroke="#a3d977" stroke-width="2" fill="none"/><path d="M1018.6 582.93l-1.53-1.3.12-.13h2.6z" fill="#a3d977"/><path d="M822.1 813.03l5.7-13.88 7.06 6z" stroke="#a3d977" stroke-width="2" fill="#a3d977"/><path d="M1018.4 582.5l-18.24 213.6" stroke="#a3d977" stroke-width="2" fill="none"/><path d="M1019.4 582.6l-1.98-.16.08-.94h2z" fill="#a3d977"/><path d="M998.86 811.3l-3.4-14.6 9.23.78z" stroke="#a3d977" stroke-width="2" fill="#a3d977"/><path d="M1019.42 581.9l523.6 226.26" stroke="#a3d977" stroke-width="2" fill="none"/><path d="M1019.04 582.82l-3.06-1.32h3.64z" fill="#a3d977"/><path d="M1557.03 814.22l-14.93-1.4 3.67-8.52z" stroke="#a3d977" stroke-width="2" fill="#a3d977"/><path d="M1477.54 581.77L657.82 810.53" stroke="#b2b2b2" stroke-width="2" fill="none"/><path d="M1481.66 581.66l-3.88 1.08-.35-1.24h4.18z" fill="#b2b2b2"/><path d="M643.12 814.63l12.5-8.3 2.48 8.93z" stroke="#b2b2b2" stroke-width="2" fill="#b2b2b2"/><path d="M1477.56 581.83L837.43 809.3" stroke="#b2b2b2" stroke-width="2" fill="none"/><path d="M1477.87 582.8l-.46-1.3h4.1z" fill="#b2b2b2"/><path d="M823.05 814.42l11.9-9.15 3.1 8.74z" stroke="#b2b2b2" stroke-width="2" fill="#b2b2b2"/><path d="M1477.6 581.94l-460.98 225.43" stroke="#b2b2b2" stroke-width="2" fill="none"/><path d="M1478.02 582.85l-.66-1.35h3.42z" fill="#b2b2b2"/><path d="M1002.9 814.08l10.8-10.43 4.06 8.33z" stroke="#b2b2b2" stroke-width="2" fill="#b2b2b2"/><path d="M1478.83 582.44l75.08 215.6" stroke="#b2b2b2" stroke-width="2" fill="none"/><path d="M1479.78 582.14l-1.9.66-.44-1.3h2.12z" fill="#b2b2b2"/><path d="M1558.94 812.44l-9.07-11.94 8.75-3.05z" stroke="#b2b2b2" stroke-width="2" fill="#b2b2b2"/><path d="M860 450.14l178.5-.28" stroke="#a3d977" stroke-width="2" fill="none"/><path d="M844.74 450.16L859 445.5v9.27zM1053.76 449.84l-14.26 4.66v-9.27z" stroke="#a3d977" stroke-width="2" fill="#a3d977"/><path d="M1238.5 449.5h260" stroke="#a3d977" stroke-width="2" fill="none"/><path d="M1223.24 449.5l14.26-4.64v9.28zM1513.76 449.5l-14.26 4.64v-9.28z" stroke="#a3d977" stroke-width="2" fill="#a3d977"/><path d="M1140 142.5v81.7c0 3.3-2.7 6-6 6h-103.9c-3.3 0-6 2.68-6 6v64.2" stroke="#5e5e5e" stroke-width="2" fill="none"/><path d="M1141 142.53h-2v-1.03h2z" fill="#5e5e5e"/><path d="M1024.1 315.66l-4.63-14.27h9.27z" stroke="#5e5e5e" stroke-width="2" fill="#5e5e5e"/><path d="M1560 145.5v80c0 3.3-2.7 6-6 6h-529.5c-3.3 0-6 2.7-6 6V300" stroke="#5e5e5e" stroke-width="2" fill="none"/><path d="M1561 145.53h-2v-1.03h2z" fill="#5e5e5e"/><path d="M1018.5 315.26l-4.64-14.26h9.27z" stroke="#5e5e5e" stroke-width="2" fill="#5e5e5e"/><path d="M660 142.5V224c0 3.3 2.7 6 6 6h346.5c3.3 0 6 2.7 6 6v64" stroke="#5e5e5e" stroke-width="2" fill="none"/><path d="M661 142.53h-2v-1.03h2z" fill="#5e5e5e"/><path d="M1018.5 315.26l-4.64-14.26h9.27z" stroke="#5e5e5e" stroke-width="2" fill="#5e5e5e"/><path d="M1250.5 400c0 2.5-2 4.5-4.5 4.5s-4.5-2-4.5-4.5 2-4.5 4.5-4.5 4.5 2 4.5 4.5zm18.3 0c0 2.5-2.03 4.5-4.5 4.5-2.5 0-4.5-2-4.5-4.5s2-4.5 4.5-4.5c2.47 0 4.5 2 4.5 4.5zm18.27 0c0 2.5-2 4.5-4.5 4.5-2.48 0-4.5-2-4.5-4.5s2.02-4.5 4.5-4.5c2.5 0 4.5 2 4.5 4.5zm18.3 0c0 2.5-2.03 4.5-4.5 4.5-2.5 0-4.5-2-4.5-4.5s2-4.5 4.5-4.5c2.47 0 4.5 2 4.5 4.5zm18.27 0c0 2.5-2 4.5-4.5 4.5-2.48 0-4.5-2-4.5-4.5s2.02-4.5 4.5-4.5c2.5 0 4.5 2 4.5 4.5zm18.3 0c0 2.5-2.03 4.5-4.5 4.5-2.5 0-4.5-2-4.5-4.5s2-4.5 4.5-4.5c2.47 0 4.5 2 4.5 4.5zm18.27 0c0 2.5-2 4.5-4.5 4.5-2.47 0-4.5-2-4.5-4.5s2.03-4.5 4.5-4.5c2.5 0 4.5 2 4.5 4.5zm18.3 0c0 2.5-2 4.5-4.5 4.5s-4.5-2-4.5-4.5 2-4.5 4.5-4.5 4.5 2 4.5 4.5z" fill="#5e5e5e"/><path d="M1246.12 404.5h-4.62v-9h4.62zM1378.5 404.5h-4.62v-9h4.62zM1090.5 858.5c0 2.5-2 4.5-4.5 4.5s-4.5-2-4.5-4.5 2-4.5 4.5-4.5 4.5 2 4.5 4.5zm17.64 0c0 2.5-2.02 4.5-4.5 4.5-2.5 0-4.5-2-4.5-4.5s2-4.5 4.5-4.5c2.48 0 4.5 2 4.5 4.5zm17.63 0c0 2.5-2 4.5-4.5 4.5-2.48 0-4.5-2-4.5-4.5s2.02-4.5 4.5-4.5c2.5 0 4.5 2 4.5 4.5zm17.64 0c0 2.5-2 4.5-4.5 4.5-2.48 0-4.5-2-4.5-4.5s2.02-4.5 4.5-4.5c2.5 0 4.5 2 4.5 4.5zm17.65 0c0 2.5-2.02 4.5-4.5 4.5-2.5 0-4.5-2-4.5-4.5s2-4.5 4.5-4.5c2.48 0 4.5 2 4.5 4.5zm17.63 0c0 2.5-2 4.5-4.5 4.5-2.48 0-4.5-2-4.5-4.5s2.02-4.5 4.5-4.5c2.5 0 4.5 2 4.5 4.5zm17.64 0c0 2.5-2.02 4.5-4.5 4.5-2.5 0-4.5-2-4.5-4.5s2-4.5 4.5-4.5c2.48 0 4.5 2 4.5 4.5zm17.63 0c0 2.5-2 4.5-4.5 4.5-2.48 0-4.5-2-4.5-4.5s2.02-4.5 4.5-4.5c2.5 0 4.5 2 4.5 4.5zm17.64 0c0 2.5-2.02 4.5-4.5 4.5-2.5 0-4.5-2-4.5-4.5s2-4.5 4.5-4.5c2.48 0 4.5 2 4.5 4.5zm17.63 0c0 2.5-2.02 4.5-4.5 4.5-2.5 0-4.5-2-4.5-4.5s2-4.5 4.5-4.5c2.48 0 4.5 2 4.5 4.5zm17.63 0c0 2.5-2 4.5-4.5 4.5-2.48 0-4.5-2-4.5-4.5s2.02-4.5 4.5-4.5c2.5 0 4.5 2 4.5 4.5zm17.64 0c0 2.5-2 4.5-4.5 4.5s-4.5-2-4.5-4.5 2-4.5 4.5-4.5 4.5 2 4.5 4.5zm17.64 0c0 2.5-2.02 4.5-4.5 4.5-2.5 0-4.5-2-4.5-4.5s2-4.5 4.5-4.5c2.48 0 4.5 2 4.5 4.5zm17.63 0c0 2.5-2 4.5-4.5 4.5-2.48 0-4.5-2-4.5-4.5s2.02-4.5 4.5-4.5c2.5 0 4.5 2 4.5 4.5zm17.64 0c0 2.5-2 4.5-4.5 4.5-2.48 0-4.5-2-4.5-4.5s2.02-4.5 4.5-4.5c2.5 0 4.5 2 4.5 4.5zm17.65 0c0 2.5-2.02 4.5-4.5 4.5-2.5 0-4.5-2-4.5-4.5s2-4.5 4.5-4.5c2.48 0 4.5 2 4.5 4.5zm17.63 0c0 2.5-2 4.5-4.5 4.5-2.48 0-4.5-2-4.5-4.5s2.02-4.5 4.5-4.5c2.5 0 4.5 2 4.5 4.5zm17.64 0c0 2.5-2.02 4.5-4.5 4.5-2.5 0-4.5-2-4.5-4.5s2-4.5 4.5-4.5c2.48 0 4.5 2 4.5 4.5zm17.63 0c0 2.5-2 4.5-4.5 4.5-2.48 0-4.5-2-4.5-4.5s2.02-4.5 4.5-4.5c2.5 0 4.5 2 4.5 4.5zm17.64 0c0 2.5-2.02 4.5-4.5 4.5-2.5 0-4.5-2-4.5-4.5s2-4.5 4.5-4.5c2.48 0 4.5 2 4.5 4.5zm17.63 0c0 2.5-2.02 4.5-4.5 4.5-2.5 0-4.5-2-4.5-4.5s2-4.5 4.5-4.5c2.48 0 4.5 2 4.5 4.5zm17.63 0c0 2.5-2 4.5-4.5 4.5-2.48 0-4.5-2-4.5-4.5s2.02-4.5 4.5-4.5c2.5 0 4.5 2 4.5 4.5zm17.64 0c0 2.5-2 4.5-4.5 4.5s-4.5-2-4.5-4.5 2-4.5 4.5-4.5 4.5 2 4.5 4.5z" fill="#5e5e5e"/><path d="M1086.12 863h-4.62v-9h4.62zM1478.5 863h-4.62v-9h4.62z" fill="#5e5e5e"/><path d="M742.5 540H754c3.3 0 6-2.7 6-6v-34" stroke="#5e5e5e" stroke-width="2" fill="none"/><path d="M742.53 541h-1.03v-2h1.03z" fill="#5e5e5e"/><path d="M760 484.74l4.64 14.26h-9.28z" stroke="#5e5e5e" stroke-width="2" fill="#5e5e5e"/><path d="M1101 540h31.5c3.3 0 6-2.7 6-6v-34.33" stroke="#5e5e5e" stroke-width="2" fill="none"/><path d="M1101.02 541H1100v-2h1.02z" fill="#5e5e5e"/><path d="M1138.5 484.4l4.63 14.27h-9.27z" stroke="#5e5e5e" stroke-width="2" fill="#5e5e5e"/><path d="M1561 540h31.5c3.3 0 6-2.7 6-6v-34.33" stroke="#5e5e5e" stroke-width="2" fill="none"/><path d="M1561.02 541H1560v-2h1.02z" fill="#5e5e5e"/><path d="M1598.5 484.4l4.63 14.27h-9.27z" stroke="#5e5e5e" stroke-width="2" fill="#5e5e5e"/><defs><path fill="#333" d="M197 0v-115H63V0H30v-248h33v105h134v-105h34V0h-34" id="u"/><path fill="#333" d="M100-194c62-1 85 37 85 99 1 63-27 99-86 99S16-35 15-95c0-66 28-99 85-99zM99-20c44 1 53-31 53-75 0-43-8-75-51-75s-53 32-53 75 10 74 51 75" id="v"/><path fill="#333" d="M135-143c-3-34-86-38-87 0 15 53 115 12 119 90S17 21 10-45l28-5c4 36 97 45 98 0-10-56-113-15-118-90-4-57 82-63 122-42 12 7 21 19 24 35" id="w"/><path fill="#333" d="M59-47c-2 24 18 29 38 22v24C64 9 27 4 27-40v-127H5v-23h24l9-43h21v43h35v23H59v120" id="x"/><g id="b"><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#u"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,15.987654320987653,0)" xlink:href="#v"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,28.333333333333332,0)" xlink:href="#w"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,39.44444444444444,0)" xlink:href="#x"/></g><path fill="#333" d="M190 0L58-211 59 0H30v-248h39L202-35l-2-213h31V0h-41" id="y"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#y" id="c"/><filter id="a" filterUnits="objectBoundingBox" x="-.03" y="-.01" width="1.06" height="1.06"><feOffset result="offOut" in="SourceAlpha" dy="6"/><feGaussianBlur result="blurOut" in="offOut" stdDeviation="5"/><feColorMatrix result="colorOut" in="blurOut" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0.5019607843137255 0"/><feBlend in="SourceGraphic" in2="colorOut"/></filter><path fill="#333" d="M101-251c82-7 93 87 43 132L82-64C71-53 59-42 53-27h129V0H18c2-99 128-94 128-182 0-28-16-43-45-43s-46 15-49 41l-32-3c6-41 34-60 81-64" id="z"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#z" id="d"/><path fill="#333" d="M27 0v-27h64v-190l-56 39v-29l58-41h29v221h61V0H27" id="A"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#A" id="e"/><path fill="#333" d="M233-177c-1 41-23 64-60 70L243 0h-38l-65-103H63V0H30v-248c88 3 205-21 203 71zM63-129c60-2 137 13 137-47 0-61-80-42-137-45v92" id="B"/><path fill="#333" d="M143 4C61 4 22-44 18-125c-5-107 100-154 193-111 17 8 29 25 37 43l-32 9c-13-25-37-40-76-40-61 0-88 39-88 99 0 61 29 100 91 101 35 0 62-11 79-27v-45h-74v-28h105v86C228-13 192 4 143 4" id="C"/><path fill="#333" d="M266 0h-40l-56-210L115 0H75L2-248h35L96-30l15-64 43-154h32l59 218 59-218h35" id="D"/><g id="f"><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#B"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,15.987654320987653,0)" xlink:href="#C"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,33.2716049382716,0)" xlink:href="#D"/></g><path fill="#333" d="M96-169c-40 0-48 33-48 73s9 75 48 75c24 0 41-14 43-38l32 2c-6 37-31 61-74 61-59 0-76-41-82-99-10-93 101-131 147-64 4 7 5 14 7 22l-32 3c-4-21-16-35-41-35" id="E"/><path fill="#333" d="M24 0v-261h32V0H24" id="F"/><path fill="#333" d="M24-231v-30h32v30H24zM24 0v-190h32V0H24" id="G"/><path fill="#333" d="M100-194c63 0 86 42 84 106H49c0 40 14 67 53 68 26 1 43-12 49-29l28 8c-11 28-37 45-77 45C44 4 14-33 15-96c1-61 26-98 85-98zm52 81c6-60-76-77-97-28-3 7-6 17-6 28h103" id="H"/><path fill="#333" d="M117-194c89-4 53 116 60 194h-32v-121c0-31-8-49-39-48C34-167 62-67 57 0H25l-1-190h30c1 10-1 24 2 32 11-22 29-35 61-36" id="I"/><g id="g"><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#E"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,11.11111111111111,0)" xlink:href="#F"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,15.987654320987653,0)" xlink:href="#G"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,20.864197530864196,0)" xlink:href="#H"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,33.20987654320987,0)" xlink:href="#I"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,45.55555555555555,0)" xlink:href="#x"/></g><g id="h"><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#E"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,11.11111111111111,0)" xlink:href="#F"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,15.987654320987653,0)" xlink:href="#G"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,20.864197530864196,0)" xlink:href="#H"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,33.20987654320987,0)" xlink:href="#I"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,45.55555555555555,0)" xlink:href="#x"/></g><path fill="#333" d="M137 0h-34L2-248h35l83 218 83-218h36" id="J"/><path fill="#333" d="M33 0v-248h34V0H33" id="K"/><path fill="#333" d="M30-248c87 1 191-15 191 75 0 78-77 80-158 76V0H30v-248zm33 125c57 0 124 11 124-50 0-59-68-47-124-48v98" id="L"/><g id="i"><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#J"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,14.814814814814813,0)" xlink:href="#K"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,20.98765432098765,0)" xlink:href="#L"/></g><path fill="#333" d="M205 0l-28-72H64L36 0H1l101-248h38L239 0h-34zm-38-99l-47-123c-12 45-31 82-46 123h93" id="M"/><path fill="#333" d="M114-163C36-179 61-72 57 0H25l-1-190h30c1 12-1 29 2 39 6-27 23-49 58-41v29" id="N"/><path fill="#333" d="M141 0L90-78 38 0H4l68-98-65-92h35l48 74 47-74h35l-64 92 68 98h-35" id="O"/><path fill="#333" d="M179-190L93 31C79 59 56 82 12 73V49c39 6 53-20 64-50L1-190h34L92-34l54-156h33" id="P"/><g id="j"><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#u"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,15.987654320987653,0)" xlink:href="#M"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,30.80246913580247,0)" xlink:href="#L"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,45.61728395061728,0)" xlink:href="#N"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,52.962962962962955,0)" xlink:href="#v"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,65.30864197530863,0)" xlink:href="#O"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,76.41975308641975,0)" xlink:href="#P"/></g><path fill="#333" d="M194 0L95-120 63-95V0H30v-248h33v124l119-124h40L117-140 236 0h-42" id="Q"/><path fill="#333" d="M115-194c55 1 70 41 70 98S169 2 115 4C84 4 66-9 55-30l1 105H24l-1-265h31l2 30c10-21 28-34 59-34zm-8 174c40 0 45-34 45-75s-6-73-45-74c-42 0-51 32-51 76 0 43 10 73 51 73" id="R"/><path fill="#333" d="M141-36C126-15 110 5 73 4 37 3 15-17 15-53c-1-64 63-63 125-63 3-35-9-54-41-54-24 1-41 7-42 31l-33-3c5-37 33-52 76-52 45 0 72 20 72 64v82c-1 20 7 32 28 27v20c-31 9-61-2-59-35zM48-53c0 20 12 33 32 33 41-3 63-29 60-74-43 2-92-5-92 41" id="S"/><path fill="#333" d="M108 0H70L1-190h34L89-25l56-165h34" id="T"/><path fill="#333" d="M85-194c31 0 48 13 60 33l-1-100h32l1 261h-30c-2-10 0-23-3-31C134-8 116 4 85 4 32 4 16-35 15-94c0-66 23-100 70-100zm9 24c-40 0-46 34-46 75 0 40 6 74 45 74 42 0 51-32 51-76 0-42-9-74-50-73" id="U"/><g id="k"><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#Q"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,14.814814814814813,0)" xlink:href="#H"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,27.160493827160494,0)" xlink:href="#H"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,39.50617283950617,0)" xlink:href="#R"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,51.85185185185185,0)" xlink:href="#S"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,64.19753086419753,0)" xlink:href="#F"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,69.07407407407408,0)" xlink:href="#G"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,73.95061728395062,0)" xlink:href="#T"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,85.06172839506173,0)" xlink:href="#H"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,97.40740740740742,0)" xlink:href="#U"/></g><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#A" id="l"/><path fill="#333" d="M115-194c53 0 69 39 70 98 0 66-23 100-70 100C84 3 66-7 56-30L54 0H23l1-261h32v101c10-23 28-34 59-34zm-8 174c40 0 45-34 45-75 0-40-5-75-45-74-42 0-51 32-51 76 0 43 10 73 51 73" id="V"/><path fill="#333" d="M143 0L79-87 56-68V0H24v-261h32v163l83-92h37l-77 82L181 0h-38" id="W"/><path fill="#333" d="M84 4C-5 8 30-112 23-190h32v120c0 31 7 50 39 49 72-2 45-101 50-169h31l1 190h-30c-1-10 1-25-2-33-11 22-28 36-60 37" id="X"/><g id="m"><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#V"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,12.345679012345679,0)" xlink:href="#S"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,24.691358024691358,0)" xlink:href="#E"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,35.80246913580247,0)" xlink:href="#W"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,46.91358024691358,0)" xlink:href="#X"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,59.25925925925926,0)" xlink:href="#R"/></g><g id="n"><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#B"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,15.987654320987653,0)" xlink:href="#C"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,33.2716049382716,0)" xlink:href="#D"/></g><path fill="#333" d="M210-169c-67 3-38 105-44 169h-31v-121c0-29-5-50-35-48C34-165 62-65 56 0H25l-1-190h30c1 10-1 24 2 32 10-44 99-50 107 0 11-21 27-35 58-36 85-2 47 119 55 194h-31v-121c0-29-5-49-35-48" id="Y"/><g id="o"><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#U"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,12.345679012345679,0)" xlink:href="#S"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,24.691358024691358,0)" xlink:href="#H"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,37.03703703703704,0)" xlink:href="#Y"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,55.49382716049382,0)" xlink:href="#v"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,67.8395061728395,0)" xlink:href="#I"/></g><path fill="#333" d="M126-127c33 6 58 20 58 59 0 88-139 92-164 29-3-8-5-16-6-25l32-3c6 27 21 44 54 44 32 0 52-15 52-46 0-38-36-46-79-43v-28c39 1 72-4 72-42 0-27-17-43-46-43-28 0-47 15-49 41l-32-3c6-42 35-63 81-64 48-1 79 21 79 65 0 36-21 52-52 59" id="Z"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#Z" id="p"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#I" id="q"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#z" id="r"/><g id="s"><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#Y"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,18.456790123456788,0)" xlink:href="#S"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,30.80246913580247,0)" xlink:href="#w"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,41.91358024691358,0)" xlink:href="#x"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,48.086419753086425,0)" xlink:href="#H"/><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,60.4320987654321,0)" xlink:href="#N"/></g><use transform="matrix(0.06172839506172839,0,0,0.06172839506172839,0,0)" xlink:href="#y" id="t"/></defs></g></svg>
\ No newline at end of file
diff --git a/doc/images/RADOS.jpg b/doc/images/RADOS.jpg
new file mode 100644
index 000000000..0793a8ce4
--- /dev/null
+++ b/doc/images/RADOS.jpg
diff --git a/doc/images/RBD.jpg b/doc/images/RBD.jpg
new file mode 100644
index 000000000..c2a2b8f16
--- /dev/null
+++ b/doc/images/RBD.jpg
diff --git a/doc/images/RDBSnapshots.jpg b/doc/images/RDBSnapshots.jpg
new file mode 100644
index 000000000..493d4686a
--- /dev/null
+++ b/doc/images/RDBSnapshots.jpg
diff --git a/doc/images/dashboard/invalid-credentials.png b/doc/images/dashboard/invalid-credentials.png
new file mode 100644
index 000000000..abd78b024
--- /dev/null
+++ b/doc/images/dashboard/invalid-credentials.png
diff --git a/doc/images/docreviewprocess.jpg b/doc/images/docreviewprocess.jpg
new file mode 100644
index 000000000..33f7163e6
--- /dev/null
+++ b/doc/images/docreviewprocess.jpg
diff --git a/doc/images/esx_chap.png b/doc/images/esx_chap.png
new file mode 100644
index 000000000..caa70668a
--- /dev/null
+++ b/doc/images/esx_chap.png
diff --git a/doc/images/esx_config_iscsi_main.png b/doc/images/esx_config_iscsi_main.png
new file mode 100644
index 000000000..7443379da
--- /dev/null
+++ b/doc/images/esx_config_iscsi_main.png
diff --git a/doc/images/esx_iscsi_recov_timeout.png b/doc/images/esx_iscsi_recov_timeout.png
new file mode 100644
index 000000000..99840ae9c
--- /dev/null
+++ b/doc/images/esx_iscsi_recov_timeout.png
diff --git a/doc/images/esx_web_client_storage_main.png b/doc/images/esx_web_client_storage_main.png
new file mode 100644
index 000000000..56442a8b7
--- /dev/null
+++ b/doc/images/esx_web_client_storage_main.png
diff --git a/doc/images/keycloak-adduser.png b/doc/images/keycloak-adduser.png
new file mode 100644
index 000000000..03d9048f3
--- /dev/null
+++ b/doc/images/keycloak-adduser.png
diff --git a/doc/images/keycloak-userclientmapper.png b/doc/images/keycloak-userclientmapper.png
new file mode 100644
index 000000000..814b5e07e
--- /dev/null
+++ b/doc/images/keycloak-userclientmapper.png
diff --git a/doc/images/keycloak-usercredentials.png b/doc/images/keycloak-usercredentials.png
new file mode 100644
index 000000000..a01ab7922
--- /dev/null
+++ b/doc/images/keycloak-usercredentials.png
diff --git a/doc/images/keycloak-userdetails.png b/doc/images/keycloak-userdetails.png
new file mode 100644
index 000000000..3251a3907
--- /dev/null
+++ b/doc/images/keycloak-userdetails.png
diff --git a/doc/images/keycloak-usertags.png b/doc/images/keycloak-usertags.png
new file mode 100644
index 000000000..ee3fecf48
--- /dev/null
+++ b/doc/images/keycloak-usertags.png
diff --git a/doc/images/region-sync.png b/doc/images/region-sync.png
new file mode 100644
index 000000000..7f889a0e1
--- /dev/null
+++ b/doc/images/region-sync.png
diff --git a/doc/images/region-sync.svg b/doc/images/region-sync.svg
new file mode 100644
index 000000000..93b66f31d
--- /dev/null
+++ b/doc/images/region-sync.svg
@@ -0,0 +1,30426 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:osb="http://www.openswatchbook.org/uri/2009/osb"
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:xlink="http://www.w3.org/1999/xlink"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="744.09448819"
+   height="1052.3622047"
+   id="svg104536"
+   version="1.1"
+   inkscape:version="0.48.4 r9939"
+   sodipodi:docname="region-sync.svg"
+   inkscape:export-filename="/home/john/Pictures/region-sync.png"
+   inkscape:export-xdpi="90"
+   inkscape:export-ydpi="90">
+  <defs
+     id="defs104538">
+    <linearGradient
+       id="linearGradient126200"
+       osb:paint="solid">
+      <stop
+         style="stop-color:#ffffff;stop-opacity:1;"
+         offset="0"
+         id="stop126202" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4884_">
+      <rect
+         height="11.504"
+         width="174.63"
+         y="360.13901"
+         x="-679.57202"
+         id="use84863" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4886_">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use84873" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4888_">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use84881" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4890_">
+      <rect
+         height="67.402"
+         width="42.448002"
+         y="350.939"
+         x="-535.73901"
+         id="use84891" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4892_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-535.73901"
+         id="use84899" />
+    </clipPath>
+    <clipPath
+       id="clipPath99071">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99073" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4894_">
+      <rect
+         height="67.402"
+         width="42.448002"
+         y="350.939"
+         x="-536.966"
+         id="use84911" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4896_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-536.966"
+         id="use84919" />
+    </clipPath>
+    <clipPath
+       id="clipPath99079">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99081" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4898_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-538.19202"
+         id="use84931" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4900_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-538.19202"
+         id="use84939" />
+    </clipPath>
+    <clipPath
+       id="clipPath99087">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99089" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4902_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-539.41901"
+         id="use84951" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4904_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-539.41901"
+         id="use84959" />
+    </clipPath>
+    <clipPath
+       id="clipPath99095">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99097" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4906_">
+      <rect
+         height="67.402"
+         width="42.448002"
+         y="350.939"
+         x="-540.646"
+         id="use84971" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4908_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-540.646"
+         id="use84979" />
+    </clipPath>
+    <clipPath
+       id="clipPath99103">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99105" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4910_">
+      <rect
+         height="67.402"
+         width="42.448002"
+         y="350.939"
+         x="-541.87299"
+         id="use84991" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4912_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-541.87299"
+         id="use84999" />
+    </clipPath>
+    <clipPath
+       id="clipPath99111">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99113" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4914_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-543.099"
+         id="use85011" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4916_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-543.099"
+         id="use85019" />
+    </clipPath>
+    <clipPath
+       id="clipPath99119">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99121" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4918_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-544.32599"
+         id="use85031" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4920_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-544.32599"
+         id="use85039" />
+    </clipPath>
+    <clipPath
+       id="clipPath99127">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99129" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4922_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-545.55298"
+         id="use85051" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4924_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-545.55298"
+         id="use85059" />
+    </clipPath>
+    <clipPath
+       id="clipPath99135">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99137" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4926_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-546.77899"
+         id="use85071" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4928_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-546.77899"
+         id="use85079" />
+    </clipPath>
+    <clipPath
+       id="clipPath99143">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99145" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4930_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-548.00598"
+         id="use85091" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4932_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-548.00598"
+         id="use85099" />
+    </clipPath>
+    <clipPath
+       id="clipPath99151">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99153" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4934_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-549.23297"
+         id="use85111" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4936_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-549.23297"
+         id="use85119" />
+    </clipPath>
+    <clipPath
+       id="clipPath99159">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99161" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4938_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-550.46002"
+         id="use85131" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4940_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-550.46002"
+         id="use85139" />
+    </clipPath>
+    <clipPath
+       id="clipPath99167">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99169" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4942_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-551.68597"
+         id="use85151" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4944_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-551.68597"
+         id="use85159" />
+    </clipPath>
+    <clipPath
+       id="clipPath99175">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99177" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4946_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-552.91302"
+         id="use85171" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4948_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-552.91302"
+         id="use85179" />
+    </clipPath>
+    <clipPath
+       id="clipPath99183">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99185" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4950_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-554.14001"
+         id="use85191" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4952_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-554.14001"
+         id="use85199" />
+    </clipPath>
+    <clipPath
+       id="clipPath99191">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99193" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4954_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-555.367"
+         id="use85211" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4956_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-555.367"
+         id="use85219" />
+    </clipPath>
+    <clipPath
+       id="clipPath99199">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99201" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4958_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-556.59302"
+         id="use85231" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4960_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-556.59302"
+         id="use85239" />
+    </clipPath>
+    <clipPath
+       id="clipPath99207">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99209" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4962_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-557.82001"
+         id="use85251" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4964_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-557.82001"
+         id="use85259" />
+    </clipPath>
+    <clipPath
+       id="clipPath99215">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99217" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4966_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-559.047"
+         id="use85271" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4968_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-559.047"
+         id="use85279" />
+    </clipPath>
+    <clipPath
+       id="clipPath99223">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99225" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4970_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-560.27301"
+         id="use85291" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4972_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-560.27301"
+         id="use85299" />
+    </clipPath>
+    <clipPath
+       id="clipPath99231">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99233" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4974_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-561.5"
+         id="use85311" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4976_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-561.5"
+         id="use85319" />
+    </clipPath>
+    <clipPath
+       id="clipPath99239">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99241" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4978_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-562.72699"
+         id="use85331" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4980_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-562.72699"
+         id="use85339" />
+    </clipPath>
+    <clipPath
+       id="clipPath99247">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99249" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4982_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-563.95398"
+         id="use85351" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4984_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-563.95398"
+         id="use85359" />
+    </clipPath>
+    <clipPath
+       id="clipPath99255">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99257" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4986_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-565.17999"
+         id="use85371" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4988_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-565.17999"
+         id="use85379" />
+    </clipPath>
+    <clipPath
+       id="clipPath99263">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99265" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4990_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-566.40698"
+         id="use85391" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4992_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-566.40698"
+         id="use85399" />
+    </clipPath>
+    <clipPath
+       id="clipPath99271">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99273" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4994_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-567.63397"
+         id="use85411" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4996_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-567.63397"
+         id="use85419" />
+    </clipPath>
+    <clipPath
+       id="clipPath99279">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99281" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4998_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-568.86102"
+         id="use85431" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5000_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-568.86102"
+         id="use85439" />
+    </clipPath>
+    <clipPath
+       id="clipPath99287">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99289" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5002_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-570.08698"
+         id="use85451" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5004_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-570.08698"
+         id="use85459" />
+    </clipPath>
+    <clipPath
+       id="clipPath99295">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99297" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5006_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-571.315"
+         id="use85471" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5008_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-571.31403"
+         id="use85479" />
+    </clipPath>
+    <clipPath
+       id="clipPath99303">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99305" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5010_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-572.54102"
+         id="use85491" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5012_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-572.54102"
+         id="use85499" />
+    </clipPath>
+    <clipPath
+       id="clipPath99311">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99313" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5014_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-573.76801"
+         id="use85511" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5016_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-573.76801"
+         id="use85519" />
+    </clipPath>
+    <clipPath
+       id="clipPath99319">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99321" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5018_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-574.99402"
+         id="use85531" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5020_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-574.99402"
+         id="use85539" />
+    </clipPath>
+    <clipPath
+       id="clipPath99327">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99329" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5022_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-576.22101"
+         id="use85551" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5024_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-576.22101"
+         id="use85559" />
+    </clipPath>
+    <clipPath
+       id="clipPath99335">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99337" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5026_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-577.448"
+         id="use85571" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5028_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-577.448"
+         id="use85579" />
+    </clipPath>
+    <clipPath
+       id="clipPath99343">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99345" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5030_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-578.67499"
+         id="use85591" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5032_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-578.67499"
+         id="use85599" />
+    </clipPath>
+    <clipPath
+       id="clipPath99351">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99353" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5034_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-579.901"
+         id="use85611" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5036_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-579.901"
+         id="use85619" />
+    </clipPath>
+    <clipPath
+       id="clipPath99359">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99361" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5038_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-581.12799"
+         id="use85631" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5040_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-581.12799"
+         id="use85639" />
+    </clipPath>
+    <clipPath
+       id="clipPath99367">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99369" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5042_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-582.35498"
+         id="use85651" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5044_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-582.35498"
+         id="use85659" />
+    </clipPath>
+    <clipPath
+       id="clipPath99375">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99377" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5046_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-583.58197"
+         id="use85671" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5048_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-583.58197"
+         id="use85679" />
+    </clipPath>
+    <clipPath
+       id="clipPath99383">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99385" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5050_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-584.80798"
+         id="use85691" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5052_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-584.80798"
+         id="use85699" />
+    </clipPath>
+    <clipPath
+       id="clipPath99391">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99393" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5054_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-586.03497"
+         id="use85711" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5056_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-586.03497"
+         id="use85719" />
+    </clipPath>
+    <clipPath
+       id="clipPath99399">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99401" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5058_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-587.26202"
+         id="use85731" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5060_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-587.26202"
+         id="use85739" />
+    </clipPath>
+    <clipPath
+       id="clipPath99407">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99409" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5062_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-588.48901"
+         id="use85751" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5064_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-588.48901"
+         id="use85759" />
+    </clipPath>
+    <clipPath
+       id="clipPath99415">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99417" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5066_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-589.71503"
+         id="use85771" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5068_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-589.71503"
+         id="use85779" />
+    </clipPath>
+    <clipPath
+       id="clipPath99423">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99425" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5070_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-590.94202"
+         id="use85791" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5072_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-590.94202"
+         id="use85799" />
+    </clipPath>
+    <clipPath
+       id="clipPath99431">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99433" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5074_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-592.16901"
+         id="use85811" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5076_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-592.16901"
+         id="use85819" />
+    </clipPath>
+    <clipPath
+       id="clipPath99439">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99441" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5078_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-593.396"
+         id="use85831" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5080_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-593.396"
+         id="use85839" />
+    </clipPath>
+    <clipPath
+       id="clipPath99447">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99449" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5082_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-594.62299"
+         id="use85851" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5084_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-594.62299"
+         id="use85859" />
+    </clipPath>
+    <clipPath
+       id="clipPath99455">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99457" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5086_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-595.849"
+         id="use85871" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5088_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-595.849"
+         id="use85879" />
+    </clipPath>
+    <clipPath
+       id="clipPath99463">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99465" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5090_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-597.07599"
+         id="use85891" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5092_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-597.07599"
+         id="use85899" />
+    </clipPath>
+    <clipPath
+       id="clipPath99471">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99473" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5094_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-598.30298"
+         id="use85911" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5096_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-598.30298"
+         id="use85919" />
+    </clipPath>
+    <clipPath
+       id="clipPath99479">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99481" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5098_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-599.53003"
+         id="use85931" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5100_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-599.53003"
+         id="use85939" />
+    </clipPath>
+    <clipPath
+       id="clipPath99487">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99489" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5102_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-600.75598"
+         id="use85951" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5104_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-600.75598"
+         id="use85959" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4873_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_"
+         overflow="visible"
+         id="use84798" />
+    </clipPath>
+    <clipPath
+       id="clipPath99497">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_"
+         overflow="visible"
+         id="use99499" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4875_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4874_"
+         overflow="visible"
+         id="use84810" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4878_"
+       gradientUnits="userSpaceOnUse"
+       x1="40.563"
+       y1="147.0757"
+       x2="41.563"
+       y2="147.0757"
+       gradientTransform="matrix(0,-10.1526,-10.1526,0,900.78,782.1111)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84821" />
+      <stop
+         offset="0.486"
+         style="stop-color:#B0D2D9"
+         id="stop84823" />
+      <stop
+         offset="0.8287"
+         style="stop-color:#E6EFF1"
+         id="stop84825" />
+      <stop
+         offset="0.9939"
+         style="stop-color:#FFFFFF"
+         id="stop84827" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84829" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4877_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4876_"
+         overflow="visible"
+         id="use84818" />
+    </clipPath>
+    <clipPath
+       id="clipPath99511">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_"
+         overflow="visible"
+         id="use99513" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4880_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4879_"
+         overflow="visible"
+         id="use84841" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4882_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4881_"
+         overflow="visible"
+         id="use84849" />
+    </clipPath>
+    <clipPath
+       id="clipPath99519">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_"
+         overflow="visible"
+         id="use99521" />
+    </clipPath>
+    <clipPath
+       id="clipPath99523">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_"
+         overflow="visible"
+         id="use99525" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4871_"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94051"
+       x2="-33.205299"
+       y2="249.94051"
+       gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84774" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84776" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4870_">
+      <polygon
+         points="-640.956,278.632 -640.956,273.567 -654.922,273.567 -656.715,276.108 -654.922,278.632 "
+         id="use84771" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4868_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84759" />
+    </clipPath>
+    <clipPath
+       id="clipPath99534">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99536" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4866_"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.9404"
+       x2="-33.205299"
+       y2="249.9404"
+       gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84735" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84737" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4865_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4864_"
+         overflow="visible"
+         id="use84732" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4863_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4862_"
+         overflow="visible"
+         id="use84720" />
+    </clipPath>
+    <clipPath
+       id="clipPath99545">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4862_"
+         overflow="visible"
+         id="use99547" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4861_"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94031"
+       x2="-33.205299"
+       y2="249.94031"
+       gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84696" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84698" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4860_">
+      <polygon
+         points="-521.64,278.632 -507.673,278.632 -507.673,273.567 -521.64,273.567 -523.432,276.108 "
+         id="use84693" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4858_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84681" />
+    </clipPath>
+    <clipPath
+       id="clipPath99556">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99558" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4856_"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94031"
+       x2="-33.205101"
+       y2="249.94031"
+       gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84659" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84661" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4855_">
+      <polygon
+         points="-521.64,208.524 -507.673,208.524 -507.673,203.458 -521.64,203.458 -523.432,206 "
+         id="use84656" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4853_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84644" />
+    </clipPath>
+    <clipPath
+       id="clipPath99567">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99569" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4851_"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.9404"
+       x2="-33.205101"
+       y2="249.9404"
+       gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84622" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84624" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4850_">
+      <polygon
+         points="-588.288,203.458 -590.081,206 -588.288,208.524 -574.322,208.524 -574.322,203.458 "
+         id="use84619" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4848_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84607" />
+    </clipPath>
+    <clipPath
+       id="clipPath99578">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99580" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4846_"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94051"
+       x2="-33.205101"
+       y2="249.94051"
+       gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84585" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84587" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4845_">
+      <polygon
+         points="-654.922,203.458 -656.715,206 -654.922,208.524 -640.956,208.524 -640.956,203.458 "
+         id="use84582" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500" />
+    </clipPath>
+    <clipPath
+       id="clipPath99589">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99591" />
+    </clipPath>
+    <clipPath
+       id="clipPath99593">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99595" />
+    </clipPath>
+    <clipPath
+       id="clipPath99597">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99599" />
+    </clipPath>
+    <clipPath
+       id="clipPath99601">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99603" />
+    </clipPath>
+    <clipPath
+       id="clipPath99605">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99607" />
+    </clipPath>
+    <clipPath
+       id="clipPath99609">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99611" />
+    </clipPath>
+    <clipPath
+       id="clipPath99613">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99615" />
+    </clipPath>
+    <clipPath
+       id="clipPath99617">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99619" />
+    </clipPath>
+    <clipPath
+       id="clipPath99621">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99623" />
+    </clipPath>
+    <clipPath
+       id="clipPath99625">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99627" />
+    </clipPath>
+    <clipPath
+       id="clipPath99629">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99631" />
+    </clipPath>
+    <clipPath
+       id="clipPath99633">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99635" />
+    </clipPath>
+    <clipPath
+       id="clipPath99637">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99639" />
+    </clipPath>
+    <clipPath
+       id="clipPath99641">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99643" />
+    </clipPath>
+    <clipPath
+       id="clipPath99645">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99647" />
+    </clipPath>
+    <clipPath
+       id="clipPath99649">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99651" />
+    </clipPath>
+    <clipPath
+       id="clipPath99653">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99655" />
+    </clipPath>
+    <clipPath
+       id="clipPath99657">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99659" />
+    </clipPath>
+    <clipPath
+       id="clipPath99661">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99663" />
+    </clipPath>
+    <clipPath
+       id="clipPath99665">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99667" />
+    </clipPath>
+    <clipPath
+       id="clipPath99669">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99671" />
+    </clipPath>
+    <clipPath
+       id="clipPath99673">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99675" />
+    </clipPath>
+    <clipPath
+       id="clipPath99677">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99679" />
+    </clipPath>
+    <clipPath
+       id="clipPath99681">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99683" />
+    </clipPath>
+    <clipPath
+       id="clipPath99685">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99687" />
+    </clipPath>
+    <clipPath
+       id="clipPath99689">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99691" />
+    </clipPath>
+    <clipPath
+       id="clipPath99693">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99695" />
+    </clipPath>
+    <clipPath
+       id="clipPath99697">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99699" />
+    </clipPath>
+    <clipPath
+       id="clipPath99701">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99703" />
+    </clipPath>
+    <clipPath
+       id="clipPath99705">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99707" />
+    </clipPath>
+    <clipPath
+       id="clipPath99709">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99711" />
+    </clipPath>
+    <clipPath
+       id="clipPath99713">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99715" />
+    </clipPath>
+    <clipPath
+       id="clipPath99717">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99719" />
+    </clipPath>
+    <clipPath
+       id="clipPath99721">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99723" />
+    </clipPath>
+    <clipPath
+       id="clipPath99725">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99727" />
+    </clipPath>
+    <clipPath
+       id="clipPath99729">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99731" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4841_"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.813"
+       x2="37.392799"
+       y2="152.813"
+       gradientTransform="matrix(0,-9.614,-9.614,0,942.8785,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84478" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84480" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84482" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84484" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84486" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4840_">
+      <rect
+         height="9.6140003"
+         width="11.316"
+         y="214.106"
+         x="-531.92297"
+         id="use84475" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4838_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84445" />
+    </clipPath>
+    <clipPath
+       id="clipPath99743">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99745" />
+    </clipPath>
+    <clipPath
+       id="clipPath99747">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99749" />
+    </clipPath>
+    <clipPath
+       id="clipPath99751">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99753" />
+    </clipPath>
+    <clipPath
+       id="clipPath99755">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99757" />
+    </clipPath>
+    <clipPath
+       id="clipPath99759">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99761" />
+    </clipPath>
+    <clipPath
+       id="clipPath99763">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99765" />
+    </clipPath>
+    <clipPath
+       id="clipPath99767">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99769" />
+    </clipPath>
+    <clipPath
+       id="clipPath99771">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99773" />
+    </clipPath>
+    <clipPath
+       id="clipPath99775">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99777" />
+    </clipPath>
+    <clipPath
+       id="clipPath99779">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99781" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4836_"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.8129"
+       x2="37.392799"
+       y2="152.8129"
+       gradientTransform="matrix(0,-9.614,-9.614,0,876.3496,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84423" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84425" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84427" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84429" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84431" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4835_">
+      <rect
+         height="9.6140003"
+         width="11.315"
+         y="214.106"
+         x="-598.45099"
+         id="use84420" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4833_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84390" />
+    </clipPath>
+    <clipPath
+       id="clipPath99793">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99795" />
+    </clipPath>
+    <clipPath
+       id="clipPath99797">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99799" />
+    </clipPath>
+    <clipPath
+       id="clipPath99801">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99803" />
+    </clipPath>
+    <clipPath
+       id="clipPath99805">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99807" />
+    </clipPath>
+    <clipPath
+       id="clipPath99809">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99811" />
+    </clipPath>
+    <clipPath
+       id="clipPath99813">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99815" />
+    </clipPath>
+    <clipPath
+       id="clipPath99817">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99819" />
+    </clipPath>
+    <clipPath
+       id="clipPath99821">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99823" />
+    </clipPath>
+    <clipPath
+       id="clipPath99825">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99827" />
+    </clipPath>
+    <clipPath
+       id="clipPath99829">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99831" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4831_"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.813"
+       x2="37.392799"
+       y2="152.813"
+       gradientTransform="matrix(0,-9.614,-9.614,0,809.7396,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84368" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84370" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84372" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84374" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84376" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4830_">
+      <rect
+         height="9.6140003"
+         width="11.316"
+         y="214.106"
+         x="-665.06201"
+         id="use84365" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4828_"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-734.6116,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84329" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84331" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84333" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84335" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84337" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4827_">
+      <polygon
+         points="-636.358,231.934 -682.207,231.929 -682.208,251.221 -659.342,256.788 -636.356,251.283 "
+         id="use84326" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4825_"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-668.057,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84298" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84300" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84302" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84304" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84306" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4824_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4823_"
+         overflow="visible"
+         id="use84295" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4822_"
+       gradientUnits="userSpaceOnUse"
+       x1="145.317"
+       y1="2.8515"
+       x2="146.317"
+       y2="2.8515"
+       gradientTransform="matrix(0.0299,24.8863,24.8863,-0.0299,-601.3687,-3384.4197)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84267" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84269" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84271" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84273" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84275" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4821_">
+      <polygon
+         points="-503.122,251.283 -503.124,231.934 -548.973,231.929 -548.973,251.221 -526.108,256.788 "
+         id="use84264" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4819_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84232" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4817_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,284.9195,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84220" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84222" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4816_">
+      <path
+         d="m -516.985,346.449 c 0,1.487 1.205,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use84217"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4814_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84197" />
+    </clipPath>
+    <clipPath
+       id="clipPath99874">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99876" />
+    </clipPath>
+    <clipPath
+       id="clipPath99878">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99880" />
+    </clipPath>
+    <clipPath
+       id="clipPath99882">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99884" />
+    </clipPath>
+    <clipPath
+       id="clipPath99886">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99888" />
+    </clipPath>
+    <clipPath
+       id="clipPath99890">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99892" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4812_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,272.9481,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84185" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84187" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4811_">
+      <path
+         d="m -528.956,346.449 c 0,1.487 1.205,2.692 2.691,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.691,1.206 -2.691,2.692"
+         id="use84182"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4809_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84162" />
+    </clipPath>
+    <clipPath
+       id="clipPath99901">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99903" />
+    </clipPath>
+    <clipPath
+       id="clipPath99905">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99907" />
+    </clipPath>
+    <clipPath
+       id="clipPath99909">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99911" />
+    </clipPath>
+    <clipPath
+       id="clipPath99913">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99915" />
+    </clipPath>
+    <clipPath
+       id="clipPath99917">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99919" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4807_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,260.9767,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84150" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84152" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4806_">
+      <path
+         d="m -540.927,346.449 c 0,1.487 1.205,2.692 2.691,2.692 1.488,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.691,1.206 -2.691,2.692"
+         id="use84147"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4804_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84127" />
+    </clipPath>
+    <clipPath
+       id="clipPath99928">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99930" />
+    </clipPath>
+    <clipPath
+       id="clipPath99932">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99934" />
+    </clipPath>
+    <clipPath
+       id="clipPath99936">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99938" />
+    </clipPath>
+    <clipPath
+       id="clipPath99940">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99942" />
+    </clipPath>
+    <clipPath
+       id="clipPath99944">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99946" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4802_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,249.0053,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84115" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84117" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4801_">
+      <path
+         d="m -552.899,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use84112"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4799_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84092" />
+    </clipPath>
+    <clipPath
+       id="clipPath99955">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99957" />
+    </clipPath>
+    <clipPath
+       id="clipPath99959">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99961" />
+    </clipPath>
+    <clipPath
+       id="clipPath99963">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99965" />
+    </clipPath>
+    <clipPath
+       id="clipPath99967">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99969" />
+    </clipPath>
+    <clipPath
+       id="clipPath99971">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99973" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4797_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,237.034,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84080" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84082" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4796_">
+      <path
+         d="m -564.87,346.449 c 0,1.487 1.205,2.692 2.691,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.691,1.206 -2.691,2.692"
+         id="use84077"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4794_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84057" />
+    </clipPath>
+    <clipPath
+       id="clipPath99982">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99984" />
+    </clipPath>
+    <clipPath
+       id="clipPath99986">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99988" />
+    </clipPath>
+    <clipPath
+       id="clipPath99990">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99992" />
+    </clipPath>
+    <clipPath
+       id="clipPath99994">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99996" />
+    </clipPath>
+    <clipPath
+       id="clipPath99998">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100000" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4792_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,225.0626,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84045" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84047" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4791_">
+      <path
+         d="m -576.842,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use84042"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4789_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84022" />
+    </clipPath>
+    <clipPath
+       id="clipPath100009">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100011" />
+    </clipPath>
+    <clipPath
+       id="clipPath100013">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100015" />
+    </clipPath>
+    <clipPath
+       id="clipPath100017">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100019" />
+    </clipPath>
+    <clipPath
+       id="clipPath100021">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100023" />
+    </clipPath>
+    <clipPath
+       id="clipPath100025">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100027" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4787_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84010" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84012" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4786_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4785_"
+         overflow="visible"
+         id="use84007" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4784_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_"
+         overflow="visible"
+         id="use83987" />
+    </clipPath>
+    <clipPath
+       id="clipPath100036">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_"
+         overflow="visible"
+         id="use100038" />
+    </clipPath>
+    <clipPath
+       id="clipPath100040">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_"
+         overflow="visible"
+         id="use100042" />
+    </clipPath>
+    <clipPath
+       id="clipPath100044">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_"
+         overflow="visible"
+         id="use100046" />
+    </clipPath>
+    <clipPath
+       id="clipPath100048">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_"
+         overflow="visible"
+         id="use100050" />
+    </clipPath>
+    <clipPath
+       id="clipPath100052">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_"
+         overflow="visible"
+         id="use100054" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4782_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83975" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83977" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4781_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4780_"
+         overflow="visible"
+         id="use83972" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4779_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_"
+         overflow="visible"
+         id="use83952" />
+    </clipPath>
+    <clipPath
+       id="clipPath100063">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_"
+         overflow="visible"
+         id="use100065" />
+    </clipPath>
+    <clipPath
+       id="clipPath100067">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_"
+         overflow="visible"
+         id="use100069" />
+    </clipPath>
+    <clipPath
+       id="clipPath100071">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_"
+         overflow="visible"
+         id="use100073" />
+    </clipPath>
+    <clipPath
+       id="clipPath100075">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_"
+         overflow="visible"
+         id="use100077" />
+    </clipPath>
+    <clipPath
+       id="clipPath100079">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_"
+         overflow="visible"
+         id="use100081" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4777_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83940" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83942" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4776_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4775_"
+         overflow="visible"
+         id="use83937" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4774_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_"
+         overflow="visible"
+         id="use83917" />
+    </clipPath>
+    <clipPath
+       id="clipPath100090">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_"
+         overflow="visible"
+         id="use100092" />
+    </clipPath>
+    <clipPath
+       id="clipPath100094">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_"
+         overflow="visible"
+         id="use100096" />
+    </clipPath>
+    <clipPath
+       id="clipPath100098">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_"
+         overflow="visible"
+         id="use100100" />
+    </clipPath>
+    <clipPath
+       id="clipPath100102">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_"
+         overflow="visible"
+         id="use100104" />
+    </clipPath>
+    <clipPath
+       id="clipPath100106">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_"
+         overflow="visible"
+         id="use100108" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4772_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,177.1768,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83905" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83907" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4771_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4770_"
+         overflow="visible"
+         id="use83902" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4769_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_"
+         overflow="visible"
+         id="use83882" />
+    </clipPath>
+    <clipPath
+       id="clipPath100117">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_"
+         overflow="visible"
+         id="use100119" />
+    </clipPath>
+    <clipPath
+       id="clipPath100121">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_"
+         overflow="visible"
+         id="use100123" />
+    </clipPath>
+    <clipPath
+       id="clipPath100125">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_"
+         overflow="visible"
+         id="use100127" />
+    </clipPath>
+    <clipPath
+       id="clipPath100129">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_"
+         overflow="visible"
+         id="use100131" />
+    </clipPath>
+    <clipPath
+       id="clipPath100133">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_"
+         overflow="visible"
+         id="use100135" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4767_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83870" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83872" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4766_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4765_"
+         overflow="visible"
+         id="use83867" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4764_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_"
+         overflow="visible"
+         id="use83847" />
+    </clipPath>
+    <clipPath
+       id="clipPath100144">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_"
+         overflow="visible"
+         id="use100146" />
+    </clipPath>
+    <clipPath
+       id="clipPath100148">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_"
+         overflow="visible"
+         id="use100150" />
+    </clipPath>
+    <clipPath
+       id="clipPath100152">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_"
+         overflow="visible"
+         id="use100154" />
+    </clipPath>
+    <clipPath
+       id="clipPath100156">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_"
+         overflow="visible"
+         id="use100158" />
+    </clipPath>
+    <clipPath
+       id="clipPath100160">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_"
+         overflow="visible"
+         id="use100162" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4762_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83835" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83837" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4761_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4760_"
+         overflow="visible"
+         id="use83832" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4759_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_"
+         overflow="visible"
+         id="use83812" />
+    </clipPath>
+    <clipPath
+       id="clipPath100171">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_"
+         overflow="visible"
+         id="use100173" />
+    </clipPath>
+    <clipPath
+       id="clipPath100175">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_"
+         overflow="visible"
+         id="use100177" />
+    </clipPath>
+    <clipPath
+       id="clipPath100179">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_"
+         overflow="visible"
+         id="use100181" />
+    </clipPath>
+    <clipPath
+       id="clipPath100183">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_"
+         overflow="visible"
+         id="use100185" />
+    </clipPath>
+    <clipPath
+       id="clipPath100187">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_"
+         overflow="visible"
+         id="use100189" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4757_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83800" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83802" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4756_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4755_"
+         overflow="visible"
+         id="use83797" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4754_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_"
+         overflow="visible"
+         id="use83777" />
+    </clipPath>
+    <clipPath
+       id="clipPath100198">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_"
+         overflow="visible"
+         id="use100200" />
+    </clipPath>
+    <clipPath
+       id="clipPath100202">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_"
+         overflow="visible"
+         id="use100204" />
+    </clipPath>
+    <clipPath
+       id="clipPath100206">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_"
+         overflow="visible"
+         id="use100208" />
+    </clipPath>
+    <clipPath
+       id="clipPath100210">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_"
+         overflow="visible"
+         id="use100212" />
+    </clipPath>
+    <clipPath
+       id="clipPath100214">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_"
+         overflow="visible"
+         id="use100216" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4752_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83765" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83767" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4751_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4750_"
+         overflow="visible"
+         id="use83762" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4749_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4748_"
+         overflow="visible"
+         id="use83742" />
+    </clipPath>
+    <clipPath
+       id="clipPath100225">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4748_"
+         overflow="visible"
+         id="use100227" />
+    </clipPath>
+    <clipPath
+       id="clipPath100229">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4748_"
+         overflow="visible"
+         id="use100231" />
+    </clipPath>
+    <clipPath
+       id="clipPath100233">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4748_"
+         overflow="visible"
+         id="use100235" />
+    </clipPath>
+    <clipPath
+       id="clipPath100237">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4748_"
+         overflow="visible"
+         id="use100239" />
+    </clipPath>
+    <clipPath
+       id="clipPath100241">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4748_"
+         overflow="visible"
+         id="use100243" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4747_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,284.9195,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83730" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83732" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4746_">
+      <path
+         d="m -516.985,334.666 c 0,1.488 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use83727"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4744_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83707" />
+    </clipPath>
+    <clipPath
+       id="clipPath100252">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100254" />
+    </clipPath>
+    <clipPath
+       id="clipPath100256">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100258" />
+    </clipPath>
+    <clipPath
+       id="clipPath100260">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100262" />
+    </clipPath>
+    <clipPath
+       id="clipPath100264">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100266" />
+    </clipPath>
+    <clipPath
+       id="clipPath100268">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100270" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4742_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3544"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,272.9481,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83695" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83697" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4741_">
+      <path
+         d="m -528.956,334.666 c 0,1.488 1.205,2.694 2.691,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.691,1.206 -2.691,2.692"
+         id="use83692"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4739_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83672" />
+    </clipPath>
+    <clipPath
+       id="clipPath100279">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100281" />
+    </clipPath>
+    <clipPath
+       id="clipPath100283">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100285" />
+    </clipPath>
+    <clipPath
+       id="clipPath100287">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100289" />
+    </clipPath>
+    <clipPath
+       id="clipPath100291">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100293" />
+    </clipPath>
+    <clipPath
+       id="clipPath100295">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100297" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4737_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3544"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,260.9767,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83660" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83662" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4736_">
+      <path
+         d="m -540.927,334.666 c 0,1.488 1.205,2.694 2.691,2.694 1.488,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.691,1.206 -2.691,2.692"
+         id="use83657"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4734_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83637" />
+    </clipPath>
+    <clipPath
+       id="clipPath100306">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100308" />
+    </clipPath>
+    <clipPath
+       id="clipPath100310">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100312" />
+    </clipPath>
+    <clipPath
+       id="clipPath100314">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100316" />
+    </clipPath>
+    <clipPath
+       id="clipPath100318">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100320" />
+    </clipPath>
+    <clipPath
+       id="clipPath100322">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100324" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4732_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3544"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,249.0053,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83625" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83627" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4731_">
+      <path
+         d="m -552.899,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83622"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4729_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83602" />
+    </clipPath>
+    <clipPath
+       id="clipPath100333">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100335" />
+    </clipPath>
+    <clipPath
+       id="clipPath100337">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100339" />
+    </clipPath>
+    <clipPath
+       id="clipPath100341">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100343" />
+    </clipPath>
+    <clipPath
+       id="clipPath100345">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100347" />
+    </clipPath>
+    <clipPath
+       id="clipPath100349">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100351" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4727_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.35419"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,225.0626,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83590" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83592" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4726_">
+      <path
+         d="m -576.842,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83587"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4724_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83567" />
+    </clipPath>
+    <clipPath
+       id="clipPath100360">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100362" />
+    </clipPath>
+    <clipPath
+       id="clipPath100364">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100366" />
+    </clipPath>
+    <clipPath
+       id="clipPath100368">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100370" />
+    </clipPath>
+    <clipPath
+       id="clipPath100372">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100374" />
+    </clipPath>
+    <clipPath
+       id="clipPath100376">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100378" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4722_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83555" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83557" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4721_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4720_"
+         overflow="visible"
+         id="use83552" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4719_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_"
+         overflow="visible"
+         id="use83532" />
+    </clipPath>
+    <clipPath
+       id="clipPath100387">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_"
+         overflow="visible"
+         id="use100389" />
+    </clipPath>
+    <clipPath
+       id="clipPath100391">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_"
+         overflow="visible"
+         id="use100393" />
+    </clipPath>
+    <clipPath
+       id="clipPath100395">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_"
+         overflow="visible"
+         id="use100397" />
+    </clipPath>
+    <clipPath
+       id="clipPath100399">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_"
+         overflow="visible"
+         id="use100401" />
+    </clipPath>
+    <clipPath
+       id="clipPath100403">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_"
+         overflow="visible"
+         id="use100405" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4717_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83520" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83522" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4716_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4715_"
+         overflow="visible"
+         id="use83517" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4714_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_"
+         overflow="visible"
+         id="use83497" />
+    </clipPath>
+    <clipPath
+       id="clipPath100414">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_"
+         overflow="visible"
+         id="use100416" />
+    </clipPath>
+    <clipPath
+       id="clipPath100418">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_"
+         overflow="visible"
+         id="use100420" />
+    </clipPath>
+    <clipPath
+       id="clipPath100422">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_"
+         overflow="visible"
+         id="use100424" />
+    </clipPath>
+    <clipPath
+       id="clipPath100426">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_"
+         overflow="visible"
+         id="use100428" />
+    </clipPath>
+    <clipPath
+       id="clipPath100430">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_"
+         overflow="visible"
+         id="use100432" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4712_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83485" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83487" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4711_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4710_"
+         overflow="visible"
+         id="use83482" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4709_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_"
+         overflow="visible"
+         id="use83462" />
+    </clipPath>
+    <clipPath
+       id="clipPath100441">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_"
+         overflow="visible"
+         id="use100443" />
+    </clipPath>
+    <clipPath
+       id="clipPath100445">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_"
+         overflow="visible"
+         id="use100447" />
+    </clipPath>
+    <clipPath
+       id="clipPath100449">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_"
+         overflow="visible"
+         id="use100451" />
+    </clipPath>
+    <clipPath
+       id="clipPath100453">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_"
+         overflow="visible"
+         id="use100455" />
+    </clipPath>
+    <clipPath
+       id="clipPath100457">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_"
+         overflow="visible"
+         id="use100459" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4707_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3544"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83450" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83452" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4706_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4705_"
+         overflow="visible"
+         id="use83447" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4704_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_"
+         overflow="visible"
+         id="use83427" />
+    </clipPath>
+    <clipPath
+       id="clipPath100468">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_"
+         overflow="visible"
+         id="use100470" />
+    </clipPath>
+    <clipPath
+       id="clipPath100472">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_"
+         overflow="visible"
+         id="use100474" />
+    </clipPath>
+    <clipPath
+       id="clipPath100476">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_"
+         overflow="visible"
+         id="use100478" />
+    </clipPath>
+    <clipPath
+       id="clipPath100480">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_"
+         overflow="visible"
+         id="use100482" />
+    </clipPath>
+    <clipPath
+       id="clipPath100484">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_"
+         overflow="visible"
+         id="use100486" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4702_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83415" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83417" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4701_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4700_"
+         overflow="visible"
+         id="use83412" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4699_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_"
+         overflow="visible"
+         id="use83392" />
+    </clipPath>
+    <clipPath
+       id="clipPath100495">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_"
+         overflow="visible"
+         id="use100497" />
+    </clipPath>
+    <clipPath
+       id="clipPath100499">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_"
+         overflow="visible"
+         id="use100501" />
+    </clipPath>
+    <clipPath
+       id="clipPath100503">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_"
+         overflow="visible"
+         id="use100505" />
+    </clipPath>
+    <clipPath
+       id="clipPath100507">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_"
+         overflow="visible"
+         id="use100509" />
+    </clipPath>
+    <clipPath
+       id="clipPath100511">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_"
+         overflow="visible"
+         id="use100513" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4697_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.35419"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83380" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83382" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4696_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4695_"
+         overflow="visible"
+         id="use83377" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4694_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_"
+         overflow="visible"
+         id="use83357" />
+    </clipPath>
+    <clipPath
+       id="clipPath100522">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_"
+         overflow="visible"
+         id="use100524" />
+    </clipPath>
+    <clipPath
+       id="clipPath100526">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_"
+         overflow="visible"
+         id="use100528" />
+    </clipPath>
+    <clipPath
+       id="clipPath100530">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_"
+         overflow="visible"
+         id="use100532" />
+    </clipPath>
+    <clipPath
+       id="clipPath100534">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_"
+         overflow="visible"
+         id="use100536" />
+    </clipPath>
+    <clipPath
+       id="clipPath100538">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_"
+         overflow="visible"
+         id="use100540" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4692_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83345" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83347" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4691_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4690_"
+         overflow="visible"
+         id="use83342" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4689_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4688_"
+         overflow="visible"
+         id="use83322" />
+    </clipPath>
+    <clipPath
+       id="clipPath100549">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4688_"
+         overflow="visible"
+         id="use100551" />
+    </clipPath>
+    <clipPath
+       id="clipPath100553">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4688_"
+         overflow="visible"
+         id="use100555" />
+    </clipPath>
+    <clipPath
+       id="clipPath100557">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4688_"
+         overflow="visible"
+         id="use100559" />
+    </clipPath>
+    <clipPath
+       id="clipPath100561">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4688_"
+         overflow="visible"
+         id="use100563" />
+    </clipPath>
+    <clipPath
+       id="clipPath100565">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4688_"
+         overflow="visible"
+         id="use100567" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4687_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,284.9569,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83310" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83312" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4686_">
+      <path
+         d="m -516.985,323.119 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.204 -2.692,2.69"
+         id="use83307"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4684_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83287" />
+    </clipPath>
+    <clipPath
+       id="clipPath100576">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100578" />
+    </clipPath>
+    <clipPath
+       id="clipPath100580">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100582" />
+    </clipPath>
+    <clipPath
+       id="clipPath100584">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100586" />
+    </clipPath>
+    <clipPath
+       id="clipPath100588">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100590" />
+    </clipPath>
+    <clipPath
+       id="clipPath100592">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100594" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4682_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,272.9856,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83275" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83277" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4681_">
+      <path
+         d="m -528.956,323.119 c 0,1.487 1.205,2.694 2.691,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.691,1.204 -2.691,2.69"
+         id="use83272"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4679_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83252" />
+    </clipPath>
+    <clipPath
+       id="clipPath100603">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100605" />
+    </clipPath>
+    <clipPath
+       id="clipPath100607">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100609" />
+    </clipPath>
+    <clipPath
+       id="clipPath100611">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100613" />
+    </clipPath>
+    <clipPath
+       id="clipPath100615">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100617" />
+    </clipPath>
+    <clipPath
+       id="clipPath100619">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100621" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4677_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,261.0142,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83240" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83242" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4676_">
+      <path
+         d="m -540.927,323.119 c 0,1.487 1.205,2.694 2.691,2.694 1.488,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.203,-2.69 -2.691,-2.69 -1.486,0 -2.691,1.204 -2.691,2.69"
+         id="use83237"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4674_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83217" />
+    </clipPath>
+    <clipPath
+       id="clipPath100630">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100632" />
+    </clipPath>
+    <clipPath
+       id="clipPath100634">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100636" />
+    </clipPath>
+    <clipPath
+       id="clipPath100638">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100640" />
+    </clipPath>
+    <clipPath
+       id="clipPath100642">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100644" />
+    </clipPath>
+    <clipPath
+       id="clipPath100646">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100648" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4672_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,249.0428,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83205" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83207" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4671_">
+      <path
+         d="m -552.899,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use83202"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4669_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83182" />
+    </clipPath>
+    <clipPath
+       id="clipPath100657">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100659" />
+    </clipPath>
+    <clipPath
+       id="clipPath100661">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100663" />
+    </clipPath>
+    <clipPath
+       id="clipPath100665">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100667" />
+    </clipPath>
+    <clipPath
+       id="clipPath100669">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100671" />
+    </clipPath>
+    <clipPath
+       id="clipPath100673">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100675" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4667_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,237.0715,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83170" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83172" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4666_">
+      <path
+         d="m -564.87,323.119 c 0,1.487 1.205,2.694 2.691,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.691,1.204 -2.691,2.69"
+         id="use83167"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4664_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83147" />
+    </clipPath>
+    <clipPath
+       id="clipPath100684">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100686" />
+    </clipPath>
+    <clipPath
+       id="clipPath100688">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100690" />
+    </clipPath>
+    <clipPath
+       id="clipPath100692">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100694" />
+    </clipPath>
+    <clipPath
+       id="clipPath100696">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100698" />
+    </clipPath>
+    <clipPath
+       id="clipPath100700">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100702" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4662_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,225.1001,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83135" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83137" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4661_">
+      <path
+         d="m -576.842,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use83132"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4659_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83112" />
+    </clipPath>
+    <clipPath
+       id="clipPath100711">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100713" />
+    </clipPath>
+    <clipPath
+       id="clipPath100715">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100717" />
+    </clipPath>
+    <clipPath
+       id="clipPath100719">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100721" />
+    </clipPath>
+    <clipPath
+       id="clipPath100723">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100725" />
+    </clipPath>
+    <clipPath
+       id="clipPath100727">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100729" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4657_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83100" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83102" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4656_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4655_"
+         overflow="visible"
+         id="use83097" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4654_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_"
+         overflow="visible"
+         id="use83077" />
+    </clipPath>
+    <clipPath
+       id="clipPath100738">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_"
+         overflow="visible"
+         id="use100740" />
+    </clipPath>
+    <clipPath
+       id="clipPath100742">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_"
+         overflow="visible"
+         id="use100744" />
+    </clipPath>
+    <clipPath
+       id="clipPath100746">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_"
+         overflow="visible"
+         id="use100748" />
+    </clipPath>
+    <clipPath
+       id="clipPath100750">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_"
+         overflow="visible"
+         id="use100752" />
+    </clipPath>
+    <clipPath
+       id="clipPath100754">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_"
+         overflow="visible"
+         id="use100756" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4652_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83065" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83067" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4651_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4650_"
+         overflow="visible"
+         id="use83062" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4649_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_"
+         overflow="visible"
+         id="use83042" />
+    </clipPath>
+    <clipPath
+       id="clipPath100765">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_"
+         overflow="visible"
+         id="use100767" />
+    </clipPath>
+    <clipPath
+       id="clipPath100769">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_"
+         overflow="visible"
+         id="use100771" />
+    </clipPath>
+    <clipPath
+       id="clipPath100773">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_"
+         overflow="visible"
+         id="use100775" />
+    </clipPath>
+    <clipPath
+       id="clipPath100777">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_"
+         overflow="visible"
+         id="use100779" />
+    </clipPath>
+    <clipPath
+       id="clipPath100781">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_"
+         overflow="visible"
+         id="use100783" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4647_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83030" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83032" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4646_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4645_"
+         overflow="visible"
+         id="use83027" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4644_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_"
+         overflow="visible"
+         id="use83007" />
+    </clipPath>
+    <clipPath
+       id="clipPath100792">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_"
+         overflow="visible"
+         id="use100794" />
+    </clipPath>
+    <clipPath
+       id="clipPath100796">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_"
+         overflow="visible"
+         id="use100798" />
+    </clipPath>
+    <clipPath
+       id="clipPath100800">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_"
+         overflow="visible"
+         id="use100802" />
+    </clipPath>
+    <clipPath
+       id="clipPath100804">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_"
+         overflow="visible"
+         id="use100806" />
+    </clipPath>
+    <clipPath
+       id="clipPath100808">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_"
+         overflow="visible"
+         id="use100810" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4642_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82995" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82997" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4641_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4640_"
+         overflow="visible"
+         id="use82992" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4639_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_"
+         overflow="visible"
+         id="use82972" />
+    </clipPath>
+    <clipPath
+       id="clipPath100819">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_"
+         overflow="visible"
+         id="use100821" />
+    </clipPath>
+    <clipPath
+       id="clipPath100823">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_"
+         overflow="visible"
+         id="use100825" />
+    </clipPath>
+    <clipPath
+       id="clipPath100827">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_"
+         overflow="visible"
+         id="use100829" />
+    </clipPath>
+    <clipPath
+       id="clipPath100831">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_"
+         overflow="visible"
+         id="use100833" />
+    </clipPath>
+    <clipPath
+       id="clipPath100835">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_"
+         overflow="visible"
+         id="use100837" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4637_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82960" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82962" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4636_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4635_"
+         overflow="visible"
+         id="use82957" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4634_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_"
+         overflow="visible"
+         id="use82937" />
+    </clipPath>
+    <clipPath
+       id="clipPath100846">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_"
+         overflow="visible"
+         id="use100848" />
+    </clipPath>
+    <clipPath
+       id="clipPath100850">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_"
+         overflow="visible"
+         id="use100852" />
+    </clipPath>
+    <clipPath
+       id="clipPath100854">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_"
+         overflow="visible"
+         id="use100856" />
+    </clipPath>
+    <clipPath
+       id="clipPath100858">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_"
+         overflow="visible"
+         id="use100860" />
+    </clipPath>
+    <clipPath
+       id="clipPath100862">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_"
+         overflow="visible"
+         id="use100864" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4632_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82925" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82927" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4631_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4630_"
+         overflow="visible"
+         id="use82922" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4629_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_"
+         overflow="visible"
+         id="use82902" />
+    </clipPath>
+    <clipPath
+       id="clipPath100873">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_"
+         overflow="visible"
+         id="use100875" />
+    </clipPath>
+    <clipPath
+       id="clipPath100877">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_"
+         overflow="visible"
+         id="use100879" />
+    </clipPath>
+    <clipPath
+       id="clipPath100881">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_"
+         overflow="visible"
+         id="use100883" />
+    </clipPath>
+    <clipPath
+       id="clipPath100885">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_"
+         overflow="visible"
+         id="use100887" />
+    </clipPath>
+    <clipPath
+       id="clipPath100889">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_"
+         overflow="visible"
+         id="use100891" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4627_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46021"
+       x2="256.37631"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82890" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82892" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4626_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4625_"
+         overflow="visible"
+         id="use82887" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4624_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4623_"
+         overflow="visible"
+         id="use82867" />
+    </clipPath>
+    <clipPath
+       id="clipPath100900">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4623_"
+         overflow="visible"
+         id="use100902" />
+    </clipPath>
+    <clipPath
+       id="clipPath100904">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4623_"
+         overflow="visible"
+         id="use100906" />
+    </clipPath>
+    <clipPath
+       id="clipPath100908">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4623_"
+         overflow="visible"
+         id="use100910" />
+    </clipPath>
+    <clipPath
+       id="clipPath100912">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4623_"
+         overflow="visible"
+         id="use100914" />
+    </clipPath>
+    <clipPath
+       id="clipPath100916">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4623_"
+         overflow="visible"
+         id="use100918" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4622_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,284.9569,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82855" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82857" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4621_">
+      <path
+         d="m -516.985,311.531 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.205 -2.692,2.69"
+         id="use82852"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4619_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82832" />
+    </clipPath>
+    <clipPath
+       id="clipPath100927">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100929" />
+    </clipPath>
+    <clipPath
+       id="clipPath100931">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100933" />
+    </clipPath>
+    <clipPath
+       id="clipPath100935">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100937" />
+    </clipPath>
+    <clipPath
+       id="clipPath100939">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100941" />
+    </clipPath>
+    <clipPath
+       id="clipPath100943">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100945" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4617_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,272.9856,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82820" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82822" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4616_">
+      <path
+         d="m -528.956,311.531 c 0,1.487 1.205,2.694 2.691,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.691,1.205 -2.691,2.69"
+         id="use82817"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4614_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82797" />
+    </clipPath>
+    <clipPath
+       id="clipPath100954">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100956" />
+    </clipPath>
+    <clipPath
+       id="clipPath100958">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100960" />
+    </clipPath>
+    <clipPath
+       id="clipPath100962">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100964" />
+    </clipPath>
+    <clipPath
+       id="clipPath100966">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100968" />
+    </clipPath>
+    <clipPath
+       id="clipPath100970">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100972" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4612_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,261.0142,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82785" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82787" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4611_">
+      <path
+         d="m -540.927,311.531 c 0,1.487 1.205,2.694 2.691,2.694 1.488,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.203,-2.69 -2.691,-2.69 -1.486,0 -2.691,1.205 -2.691,2.69"
+         id="use82782"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4609_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82762" />
+    </clipPath>
+    <clipPath
+       id="clipPath100981">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100983" />
+    </clipPath>
+    <clipPath
+       id="clipPath100985">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100987" />
+    </clipPath>
+    <clipPath
+       id="clipPath100989">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100991" />
+    </clipPath>
+    <clipPath
+       id="clipPath100993">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100995" />
+    </clipPath>
+    <clipPath
+       id="clipPath100997">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100999" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4607_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,249.0428,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82750" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82752" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4606_">
+      <path
+         d="m -552.899,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82747"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4604_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82727" />
+    </clipPath>
+    <clipPath
+       id="clipPath101008">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101010" />
+    </clipPath>
+    <clipPath
+       id="clipPath101012">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101014" />
+    </clipPath>
+    <clipPath
+       id="clipPath101016">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101018" />
+    </clipPath>
+    <clipPath
+       id="clipPath101020">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101022" />
+    </clipPath>
+    <clipPath
+       id="clipPath101024">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101026" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4602_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,237.0715,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82715" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82717" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4601_">
+      <path
+         d="m -564.87,311.531 c 0,1.487 1.205,2.694 2.691,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.691,1.205 -2.691,2.69"
+         id="use82712"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4599_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82692" />
+    </clipPath>
+    <clipPath
+       id="clipPath101035">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101037" />
+    </clipPath>
+    <clipPath
+       id="clipPath101039">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101041" />
+    </clipPath>
+    <clipPath
+       id="clipPath101043">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101045" />
+    </clipPath>
+    <clipPath
+       id="clipPath101047">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101049" />
+    </clipPath>
+    <clipPath
+       id="clipPath101051">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101053" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4597_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,225.1001,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82680" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82682" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4596_">
+      <path
+         d="m -576.842,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82677"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4594_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82657" />
+    </clipPath>
+    <clipPath
+       id="clipPath101062">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101064" />
+    </clipPath>
+    <clipPath
+       id="clipPath101066">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101068" />
+    </clipPath>
+    <clipPath
+       id="clipPath101070">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101072" />
+    </clipPath>
+    <clipPath
+       id="clipPath101074">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101076" />
+    </clipPath>
+    <clipPath
+       id="clipPath101078">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101080" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4592_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82645" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82647" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4591_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4590_"
+         overflow="visible"
+         id="use82642" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4589_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_"
+         overflow="visible"
+         id="use82622" />
+    </clipPath>
+    <clipPath
+       id="clipPath101089">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_"
+         overflow="visible"
+         id="use101091" />
+    </clipPath>
+    <clipPath
+       id="clipPath101093">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_"
+         overflow="visible"
+         id="use101095" />
+    </clipPath>
+    <clipPath
+       id="clipPath101097">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_"
+         overflow="visible"
+         id="use101099" />
+    </clipPath>
+    <clipPath
+       id="clipPath101101">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_"
+         overflow="visible"
+         id="use101103" />
+    </clipPath>
+    <clipPath
+       id="clipPath101105">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_"
+         overflow="visible"
+         id="use101107" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4587_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,201.1574,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82610" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82612" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4586_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4585_"
+         overflow="visible"
+         id="use82607" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4584_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_"
+         overflow="visible"
+         id="use82587" />
+    </clipPath>
+    <clipPath
+       id="clipPath101116">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_"
+         overflow="visible"
+         id="use101118" />
+    </clipPath>
+    <clipPath
+       id="clipPath101120">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_"
+         overflow="visible"
+         id="use101122" />
+    </clipPath>
+    <clipPath
+       id="clipPath101124">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_"
+         overflow="visible"
+         id="use101126" />
+    </clipPath>
+    <clipPath
+       id="clipPath101128">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_"
+         overflow="visible"
+         id="use101130" />
+    </clipPath>
+    <clipPath
+       id="clipPath101132">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_"
+         overflow="visible"
+         id="use101134" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4582_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82575" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82577" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4581_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4580_"
+         overflow="visible"
+         id="use82572" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4579_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_"
+         overflow="visible"
+         id="use82552" />
+    </clipPath>
+    <clipPath
+       id="clipPath101143">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_"
+         overflow="visible"
+         id="use101145" />
+    </clipPath>
+    <clipPath
+       id="clipPath101147">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_"
+         overflow="visible"
+         id="use101149" />
+    </clipPath>
+    <clipPath
+       id="clipPath101151">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_"
+         overflow="visible"
+         id="use101153" />
+    </clipPath>
+    <clipPath
+       id="clipPath101155">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_"
+         overflow="visible"
+         id="use101157" />
+    </clipPath>
+    <clipPath
+       id="clipPath101159">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_"
+         overflow="visible"
+         id="use101161" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4577_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82540" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82542" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4576_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4575_"
+         overflow="visible"
+         id="use82537" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4574_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_"
+         overflow="visible"
+         id="use82517" />
+    </clipPath>
+    <clipPath
+       id="clipPath101170">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_"
+         overflow="visible"
+         id="use101172" />
+    </clipPath>
+    <clipPath
+       id="clipPath101174">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_"
+         overflow="visible"
+         id="use101176" />
+    </clipPath>
+    <clipPath
+       id="clipPath101178">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_"
+         overflow="visible"
+         id="use101180" />
+    </clipPath>
+    <clipPath
+       id="clipPath101182">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_"
+         overflow="visible"
+         id="use101184" />
+    </clipPath>
+    <clipPath
+       id="clipPath101186">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_"
+         overflow="visible"
+         id="use101188" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4572_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82505" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82507" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4571_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4570_"
+         overflow="visible"
+         id="use82502" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4569_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use82482" />
+    </clipPath>
+    <clipPath
+       id="clipPath101197">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use101199" />
+    </clipPath>
+    <clipPath
+       id="clipPath101201">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use101203" />
+    </clipPath>
+    <clipPath
+       id="clipPath101205">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use101207" />
+    </clipPath>
+    <clipPath
+       id="clipPath101209">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use101211" />
+    </clipPath>
+    <clipPath
+       id="clipPath101213">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use101215" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4567_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82470" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82472" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4566_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4565_"
+         overflow="visible"
+         id="use82467" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447" />
+    </clipPath>
+    <clipPath
+       id="clipPath101224">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use101226" />
+    </clipPath>
+    <clipPath
+       id="clipPath101228">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use101230" />
+    </clipPath>
+    <clipPath
+       id="clipPath101232">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use101234" />
+    </clipPath>
+    <clipPath
+       id="clipPath101236">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use101238" />
+    </clipPath>
+    <clipPath
+       id="clipPath101240">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use101242" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4562_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82435" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82437" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4561_">
+      <path
+         d="m -660.642,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82432"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4559_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82412" />
+    </clipPath>
+    <clipPath
+       id="clipPath101251">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101253" />
+    </clipPath>
+    <clipPath
+       id="clipPath101255">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101257" />
+    </clipPath>
+    <clipPath
+       id="clipPath101259">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101261" />
+    </clipPath>
+    <clipPath
+       id="clipPath101263">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101265" />
+    </clipPath>
+    <clipPath
+       id="clipPath101267">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101269" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4557_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82400" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82402" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4556_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4555_"
+         overflow="visible"
+         id="use82397" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4554_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82373" />
+    </clipPath>
+    <clipPath
+       id="clipPath101278">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101280" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4552_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4551_"
+         overflow="visible"
+         id="use82359" />
+    </clipPath>
+    <clipPath
+       id="clipPath101284">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4551_"
+         overflow="visible"
+         id="use101286" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4550_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4549_"
+         overflow="visible"
+         id="use82345" />
+    </clipPath>
+    <clipPath
+       id="clipPath101290">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4549_"
+         overflow="visible"
+         id="use101292" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4548_"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1255.1711,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82319" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82321" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82323" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82325" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82327" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82329" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82331" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82333" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4547_">
+      <polygon
+         points="-637.65,267.32 -637.642,253.133 -659.18,258.293 -679.997,253.224 -680.004,267.353 "
+         id="use82316" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4545_"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1321.2666,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82290" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82292" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82294" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82296" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82298" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82300" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82302" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82304" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4544_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4543_"
+         overflow="visible"
+         id="use82287" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4542_"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1387.9897,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82261" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82263" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82265" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82267" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82269" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82271" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82273" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82275" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4541_">
+      <polygon
+         points="-547.185,267.353 -504.831,267.32 -504.824,253.133 -526.362,258.293 -547.178,253.224 "
+         id="use82258" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4525_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4524_"
+         overflow="visible"
+         id="use82164" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4527_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4526_"
+         overflow="visible"
+         id="use82174" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4530_"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3308.0217,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82185" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82187" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4529_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4528_"
+         overflow="visible"
+         id="use82182" />
+    </clipPath>
+    <clipPath
+       id="clipPath101336">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4526_"
+         overflow="visible"
+         id="use101338" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4532_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4531_"
+         overflow="visible"
+         id="use82197" />
+    </clipPath>
+    <clipPath
+       id="clipPath101342">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4524_"
+         overflow="visible"
+         id="use101344" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4534_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4533_"
+         overflow="visible"
+         id="use82209" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4537_"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3481.4861,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82220" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82222" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4536_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4535_"
+         overflow="visible"
+         id="use82217" />
+    </clipPath>
+    <clipPath
+       id="clipPath101353">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4533_"
+         overflow="visible"
+         id="use101355" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4539_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4538_"
+         overflow="visible"
+         id="use82232" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4523_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82132" />
+    </clipPath>
+    <clipPath
+       id="clipPath101361">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101363" />
+    </clipPath>
+    <clipPath
+       id="clipPath101365">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101367" />
+    </clipPath>
+    <clipPath
+       id="clipPath101369">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101371" />
+    </clipPath>
+    <clipPath
+       id="clipPath101373">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101375" />
+    </clipPath>
+    <clipPath
+       id="clipPath101377">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101379" />
+    </clipPath>
+    <clipPath
+       id="clipPath101381">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101383" />
+    </clipPath>
+    <clipPath
+       id="clipPath101385">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101387" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4521_"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82116" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82118" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82120" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82122" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4520_">
+      <polygon
+         points="-547.302,196.684 -528.257,196.685 -526.139,198.478 -524.02,196.685 -504.975,196.685 -504.975,188.287 -547.302,188.287 "
+         id="use82113" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4518_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82083" />
+    </clipPath>
+    <clipPath
+       id="clipPath101398">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101400" />
+    </clipPath>
+    <clipPath
+       id="clipPath101402">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101404" />
+    </clipPath>
+    <clipPath
+       id="clipPath101406">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101408" />
+    </clipPath>
+    <clipPath
+       id="clipPath101410">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101412" />
+    </clipPath>
+    <clipPath
+       id="clipPath101414">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101416" />
+    </clipPath>
+    <clipPath
+       id="clipPath101418">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101420" />
+    </clipPath>
+    <clipPath
+       id="clipPath101422">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101424" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4516_"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,902.0769,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82067" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82069" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82071" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82073" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4515_">
+      <polygon
+         points="-590.704,196.685 -571.658,196.685 -571.658,188.287 -613.986,188.287 -613.986,196.684 -594.94,196.685 -592.822,198.478 "
+         id="use82064" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4511_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82010" />
+    </clipPath>
+    <clipPath
+       id="clipPath101435">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101437" />
+    </clipPath>
+    <clipPath
+       id="clipPath101439">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101441" />
+    </clipPath>
+    <clipPath
+       id="clipPath101443">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101445" />
+    </clipPath>
+    <clipPath
+       id="clipPath101447">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101449" />
+    </clipPath>
+    <clipPath
+       id="clipPath101451">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101453" />
+    </clipPath>
+    <clipPath
+       id="clipPath101455">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101457" />
+    </clipPath>
+    <clipPath
+       id="clipPath101459">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101461" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4509_"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop81994" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop81996" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop81998" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82000" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4508_">
+      <polygon
+         points="-657.048,196.685 -638.002,196.685 -638.002,188.287 -680.33,188.287 -680.33,196.684 -661.285,196.685 -659.167,198.478 "
+         id="use81991" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient104534"
+       xlink:href="#SVGID_4509_"
+       inkscape:collect="always" />
+    <defs
+       id="defs84496">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4842_" />
+    </defs>
+    <clipPath
+       id="clipPath103391">
+      <use
+         id="use103393"
+         overflow="visible"
+         xlink:href="#SVGID_4842_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4873_-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84798-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath113428">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113430" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4875_-9">
+      <rect
+         height="10.152"
+         width="174.95799"
+         y="360.13901"
+         x="-679.90002"
+         id="use84810-6" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4878_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="40.563"
+       y1="147.0757"
+       x2="41.563"
+       y2="147.0757"
+       gradientTransform="matrix(0,-10.1526,-10.1526,0,900.78,782.1111)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84821-2" />
+      <stop
+         offset="0.486"
+         style="stop-color:#B0D2D9"
+         id="stop84823-4" />
+      <stop
+         offset="0.8287"
+         style="stop-color:#E6EFF1"
+         id="stop84825-9" />
+      <stop
+         offset="0.9939"
+         style="stop-color:#FFFFFF"
+         id="stop84827-7" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84829-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4877_-0">
+      <rect
+         height="10.152"
+         width="174.95799"
+         y="360.13901"
+         x="-679.90002"
+         id="use84818-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath113442">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113444" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4880_-0">
+      <rect
+         height="10.152"
+         width="174.94701"
+         y="360.13901"
+         x="-679.88898"
+         id="use84841-6" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4882_-9">
+      <rect
+         height="10.153"
+         width="174.94701"
+         y="360.138"
+         x="-679.88898"
+         id="use84849-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath113450">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113452" />
+    </clipPath>
+    <clipPath
+       id="clipPath113454">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113456" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4871_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94051"
+       x2="-33.205299"
+       y2="249.94051"
+       gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84774-6" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84776-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4870_-6">
+      <polygon
+         points="-654.922,273.567 -656.715,276.108 -654.922,278.632 -640.956,278.632 -640.956,273.567 "
+         id="use84771-5" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4868_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84759-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath113465">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113467" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4866_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.9404"
+       x2="-33.205299"
+       y2="249.9404"
+       gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84735-4" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84737-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4865_-6">
+      <polygon
+         points="-588.288,273.567 -590.081,276.108 -588.288,278.632 -574.322,278.632 -574.322,273.567 "
+         id="use84732-8" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4863_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84720-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath113476">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113478" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4851_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.9404"
+       x2="-33.205101"
+       y2="249.9404"
+       gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84622-4" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84624-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4850_-1">
+      <polygon
+         points="-588.288,203.458 -590.081,206 -588.288,208.524 -574.322,208.524 -574.322,203.458 "
+         id="use84619-3" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4848_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84607-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath113487">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113489" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4846_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94051"
+       x2="-33.205101"
+       y2="249.94051"
+       gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84585-1" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84587-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4845_-3">
+      <polygon
+         points="-654.922,203.458 -656.715,206 -654.922,208.524 -640.956,208.524 -640.956,203.458 "
+         id="use84582-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath113498">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113500" />
+    </clipPath>
+    <clipPath
+       id="clipPath113502">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113504" />
+    </clipPath>
+    <clipPath
+       id="clipPath113506">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113508" />
+    </clipPath>
+    <clipPath
+       id="clipPath113510">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113512" />
+    </clipPath>
+    <clipPath
+       id="clipPath113514">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113516" />
+    </clipPath>
+    <clipPath
+       id="clipPath113518">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113520" />
+    </clipPath>
+    <clipPath
+       id="clipPath113522">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113524" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4838_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84445-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath113528">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113530" />
+    </clipPath>
+    <clipPath
+       id="clipPath113532">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113534" />
+    </clipPath>
+    <clipPath
+       id="clipPath113536">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113538" />
+    </clipPath>
+    <clipPath
+       id="clipPath113540">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113542" />
+    </clipPath>
+    <clipPath
+       id="clipPath113544">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113546" />
+    </clipPath>
+    <clipPath
+       id="clipPath113548">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113550" />
+    </clipPath>
+    <clipPath
+       id="clipPath113552">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113554" />
+    </clipPath>
+    <clipPath
+       id="clipPath113556">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113558" />
+    </clipPath>
+    <clipPath
+       id="clipPath113560">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113562" />
+    </clipPath>
+    <clipPath
+       id="clipPath113564">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113566" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4836_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.8129"
+       x2="37.392799"
+       y2="152.8129"
+       gradientTransform="matrix(0,-9.614,-9.614,0,876.3496,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84423-1" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84425-8" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84427-7" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84429-9" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84431-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4835_-5">
+      <rect
+         height="9.6140003"
+         width="11.315"
+         y="214.106"
+         x="-598.45099"
+         id="use84420-5" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4833_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84390-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath113578">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113580" />
+    </clipPath>
+    <clipPath
+       id="clipPath113582">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113584" />
+    </clipPath>
+    <clipPath
+       id="clipPath113586">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113588" />
+    </clipPath>
+    <clipPath
+       id="clipPath113590">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113592" />
+    </clipPath>
+    <clipPath
+       id="clipPath113594">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113596" />
+    </clipPath>
+    <clipPath
+       id="clipPath113598">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113600" />
+    </clipPath>
+    <clipPath
+       id="clipPath113602">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113604" />
+    </clipPath>
+    <clipPath
+       id="clipPath113606">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113608" />
+    </clipPath>
+    <clipPath
+       id="clipPath113610">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113612" />
+    </clipPath>
+    <clipPath
+       id="clipPath113614">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113616" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4831_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.813"
+       x2="37.392799"
+       y2="152.813"
+       gradientTransform="matrix(0,-9.614,-9.614,0,809.7396,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84368-9" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84370-7" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84372-0" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84374-4" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84376-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4830_-7">
+      <rect
+         height="9.6140003"
+         width="11.316"
+         y="214.106"
+         x="-665.06201"
+         id="use84365-8" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4828_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-734.6116,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84329-2" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84331-0" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84333-0" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84335-1" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84337-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4827_-3">
+      <polygon
+         points="-682.208,251.221 -659.342,256.788 -636.356,251.283 -636.358,231.934 -682.207,231.929 "
+         id="use84326-6" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4825_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-668.057,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84298-6" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84300-1" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84302-6" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84304-9" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84306-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4824_-9">
+      <polygon
+         points="-615.653,251.221 -592.788,256.788 -569.802,251.283 -569.804,231.934 -615.653,231.929 "
+         id="use84295-9" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4787_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84010-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84012-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4786_-8">
+      <path
+         d="m -588.813,346.449 c 0,1.487 1.205,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use84007-6"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4784_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83987-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath113649">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113651" />
+    </clipPath>
+    <clipPath
+       id="clipPath113653">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113655" />
+    </clipPath>
+    <clipPath
+       id="clipPath113657">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113659" />
+    </clipPath>
+    <clipPath
+       id="clipPath113661">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113663" />
+    </clipPath>
+    <clipPath
+       id="clipPath113665">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113667" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4782_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83975-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83977-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4781_-3">
+      <path
+         d="m -600.785,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83972-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4779_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83952-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath113676">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113678" />
+    </clipPath>
+    <clipPath
+       id="clipPath113680">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113682" />
+    </clipPath>
+    <clipPath
+       id="clipPath113684">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113686" />
+    </clipPath>
+    <clipPath
+       id="clipPath113688">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113690" />
+    </clipPath>
+    <clipPath
+       id="clipPath113692">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113694" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4777_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83940-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83942-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4776_-4">
+      <path
+         d="m -612.756,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83937-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4774_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83917-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath113703">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113705" />
+    </clipPath>
+    <clipPath
+       id="clipPath113707">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113709" />
+    </clipPath>
+    <clipPath
+       id="clipPath113711">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113713" />
+    </clipPath>
+    <clipPath
+       id="clipPath113715">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113717" />
+    </clipPath>
+    <clipPath
+       id="clipPath113719">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113721" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4772_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,177.1768,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83905-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83907-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4771_-5">
+      <path
+         d="m -624.728,346.449 c 0,1.487 1.206,2.692 2.693,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.693,1.206 -2.693,2.692"
+         id="use83902-7"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4769_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83882-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath113730">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113732" />
+    </clipPath>
+    <clipPath
+       id="clipPath113734">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113736" />
+    </clipPath>
+    <clipPath
+       id="clipPath113738">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113740" />
+    </clipPath>
+    <clipPath
+       id="clipPath113742">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113744" />
+    </clipPath>
+    <clipPath
+       id="clipPath113746">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113748" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4767_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83870-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83872-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4766_-5">
+      <path
+         d="m -636.699,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83867-0"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4764_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83847-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath113757">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113759" />
+    </clipPath>
+    <clipPath
+       id="clipPath113761">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113763" />
+    </clipPath>
+    <clipPath
+       id="clipPath113765">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113767" />
+    </clipPath>
+    <clipPath
+       id="clipPath113769">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113771" />
+    </clipPath>
+    <clipPath
+       id="clipPath113773">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113775" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4762_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83835-5" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83837-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4761_-9">
+      <path
+         d="m -648.671,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83832-7"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4759_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83812-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath113784">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113786" />
+    </clipPath>
+    <clipPath
+       id="clipPath113788">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113790" />
+    </clipPath>
+    <clipPath
+       id="clipPath113792">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113794" />
+    </clipPath>
+    <clipPath
+       id="clipPath113796">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113798" />
+    </clipPath>
+    <clipPath
+       id="clipPath113800">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113802" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4757_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83800-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83802-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4756_-5">
+      <path
+         d="m -660.642,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83797-6"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4754_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83777-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath113811">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113813" />
+    </clipPath>
+    <clipPath
+       id="clipPath113815">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113817" />
+    </clipPath>
+    <clipPath
+       id="clipPath113819">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113821" />
+    </clipPath>
+    <clipPath
+       id="clipPath113823">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113825" />
+    </clipPath>
+    <clipPath
+       id="clipPath113827">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113829" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4752_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83765-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83767-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4751_-0">
+      <path
+         d="m -672.614,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83762-6"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4722_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83555-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83557-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4721_-5">
+      <path
+         d="m -588.813,334.666 c 0,1.488 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use83552-6"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4719_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83532-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath113843">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113845" />
+    </clipPath>
+    <clipPath
+       id="clipPath113847">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113849" />
+    </clipPath>
+    <clipPath
+       id="clipPath113851">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113853" />
+    </clipPath>
+    <clipPath
+       id="clipPath113855">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113857" />
+    </clipPath>
+    <clipPath
+       id="clipPath113859">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113861" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4717_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83520-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83522-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4716_-1">
+      <path
+         d="m -600.785,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83517-1"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4714_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83497-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath113870">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113872" />
+    </clipPath>
+    <clipPath
+       id="clipPath113874">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113876" />
+    </clipPath>
+    <clipPath
+       id="clipPath113878">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113880" />
+    </clipPath>
+    <clipPath
+       id="clipPath113882">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113884" />
+    </clipPath>
+    <clipPath
+       id="clipPath113886">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113888" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4712_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83485-6" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83487-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4711_-6">
+      <path
+         d="m -612.756,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83482-0"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4709_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83462-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath113897">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113899" />
+    </clipPath>
+    <clipPath
+       id="clipPath113901">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113903" />
+    </clipPath>
+    <clipPath
+       id="clipPath113905">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113907" />
+    </clipPath>
+    <clipPath
+       id="clipPath113909">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113911" />
+    </clipPath>
+    <clipPath
+       id="clipPath113913">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113915" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4707_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3544"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83450-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83452-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4706_-5">
+      <path
+         d="m -636.699,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83447-9"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4704_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83427-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath113924">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113926" />
+    </clipPath>
+    <clipPath
+       id="clipPath113928">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113930" />
+    </clipPath>
+    <clipPath
+       id="clipPath113932">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113934" />
+    </clipPath>
+    <clipPath
+       id="clipPath113936">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113938" />
+    </clipPath>
+    <clipPath
+       id="clipPath113940">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113942" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4702_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83415-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83417-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4701_-3">
+      <path
+         d="m -648.671,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83412-9"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4699_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83392-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath113951">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113953" />
+    </clipPath>
+    <clipPath
+       id="clipPath113955">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113957" />
+    </clipPath>
+    <clipPath
+       id="clipPath113959">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113961" />
+    </clipPath>
+    <clipPath
+       id="clipPath113963">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113965" />
+    </clipPath>
+    <clipPath
+       id="clipPath113967">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113969" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4697_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.35419"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83380-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83382-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4696_-2">
+      <path
+         d="m -660.642,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83377-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4694_-4">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83357-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath113978">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113980" />
+    </clipPath>
+    <clipPath
+       id="clipPath113982">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113984" />
+    </clipPath>
+    <clipPath
+       id="clipPath113986">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113988" />
+    </clipPath>
+    <clipPath
+       id="clipPath113990">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113992" />
+    </clipPath>
+    <clipPath
+       id="clipPath113994">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113996" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4692_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83345-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83347-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4691_-2">
+      <path
+         d="m -672.614,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83342-7"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4657_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83100-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83102-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4656_-1">
+      <path
+         d="m -588.813,323.119 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.204 -2.692,2.69"
+         id="use83097-8"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4654_-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83077-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath114010">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114012" />
+    </clipPath>
+    <clipPath
+       id="clipPath114014">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114016" />
+    </clipPath>
+    <clipPath
+       id="clipPath114018">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114020" />
+    </clipPath>
+    <clipPath
+       id="clipPath114022">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114024" />
+    </clipPath>
+    <clipPath
+       id="clipPath114026">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114028" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4652_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83065-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83067-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4651_-4">
+      <path
+         d="m -612.756,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use83062-2"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4649_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83042-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath114037">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114039" />
+    </clipPath>
+    <clipPath
+       id="clipPath114041">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114043" />
+    </clipPath>
+    <clipPath
+       id="clipPath114045">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114047" />
+    </clipPath>
+    <clipPath
+       id="clipPath114049">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114051" />
+    </clipPath>
+    <clipPath
+       id="clipPath114053">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114055" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4647_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83030-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83032-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4646_-1">
+      <path
+         d="m -624.728,323.119 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.204 -2.693,2.69"
+         id="use83027-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4644_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83007-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath114064">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114066" />
+    </clipPath>
+    <clipPath
+       id="clipPath114068">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114070" />
+    </clipPath>
+    <clipPath
+       id="clipPath114072">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114074" />
+    </clipPath>
+    <clipPath
+       id="clipPath114076">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114078" />
+    </clipPath>
+    <clipPath
+       id="clipPath114080">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114082" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4642_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82995-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82997-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4641_-8">
+      <path
+         d="m -636.699,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82992-0"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4639_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82972-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath114091">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114093" />
+    </clipPath>
+    <clipPath
+       id="clipPath114095">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114097" />
+    </clipPath>
+    <clipPath
+       id="clipPath114099">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114101" />
+    </clipPath>
+    <clipPath
+       id="clipPath114103">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114105" />
+    </clipPath>
+    <clipPath
+       id="clipPath114107">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114109" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4637_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82960-5" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82962-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4636_-6">
+      <path
+         d="m -648.671,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82957-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4634_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82937-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath114118">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114120" />
+    </clipPath>
+    <clipPath
+       id="clipPath114122">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114124" />
+    </clipPath>
+    <clipPath
+       id="clipPath114126">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114128" />
+    </clipPath>
+    <clipPath
+       id="clipPath114130">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114132" />
+    </clipPath>
+    <clipPath
+       id="clipPath114134">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114136" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4632_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82925-6" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82927-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4631_-1">
+      <path
+         d="m -660.642,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82922-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4629_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82902-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath114145">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114147" />
+    </clipPath>
+    <clipPath
+       id="clipPath114149">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114151" />
+    </clipPath>
+    <clipPath
+       id="clipPath114153">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114155" />
+    </clipPath>
+    <clipPath
+       id="clipPath114157">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114159" />
+    </clipPath>
+    <clipPath
+       id="clipPath114161">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114163" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4627_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46021"
+       x2="256.37631"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82890-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82892-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4626_-6">
+      <path
+         d="m -672.614,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82887-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4592_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82645-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82647-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4591_-6">
+      <path
+         d="m -588.813,311.531 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.205 -2.692,2.69"
+         id="use82642-0"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4589_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82622-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114177">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114179" />
+    </clipPath>
+    <clipPath
+       id="clipPath114181">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114183" />
+    </clipPath>
+    <clipPath
+       id="clipPath114185">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114187" />
+    </clipPath>
+    <clipPath
+       id="clipPath114189">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114191" />
+    </clipPath>
+    <clipPath
+       id="clipPath114193">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114195" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4587_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,201.1574,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82610-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82612-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4586_-0">
+      <path
+         d="m -600.785,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.203,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82607-0"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4584_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82587-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath114204">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114206" />
+    </clipPath>
+    <clipPath
+       id="clipPath114208">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114210" />
+    </clipPath>
+    <clipPath
+       id="clipPath114212">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114214" />
+    </clipPath>
+    <clipPath
+       id="clipPath114216">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114218" />
+    </clipPath>
+    <clipPath
+       id="clipPath114220">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114222" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4582_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82575-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82577-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4581_-6">
+      <path
+         d="m -612.756,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82572-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4579_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82552-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath114231">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114233" />
+    </clipPath>
+    <clipPath
+       id="clipPath114235">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114237" />
+    </clipPath>
+    <clipPath
+       id="clipPath114239">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114241" />
+    </clipPath>
+    <clipPath
+       id="clipPath114243">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114245" />
+    </clipPath>
+    <clipPath
+       id="clipPath114247">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114249" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4577_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82540-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82542-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4576_-2">
+      <path
+         d="m -624.728,311.531 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.205 -2.693,2.69"
+         id="use82537-2"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4574_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82517-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114258">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114260" />
+    </clipPath>
+    <clipPath
+       id="clipPath114262">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114264" />
+    </clipPath>
+    <clipPath
+       id="clipPath114266">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114268" />
+    </clipPath>
+    <clipPath
+       id="clipPath114270">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114272" />
+    </clipPath>
+    <clipPath
+       id="clipPath114274">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114276" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4572_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82505-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82507-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4571_-1">
+      <path
+         d="m -636.699,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82502-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4569_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82482-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath114285">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114287" />
+    </clipPath>
+    <clipPath
+       id="clipPath114289">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114291" />
+    </clipPath>
+    <clipPath
+       id="clipPath114293">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114295" />
+    </clipPath>
+    <clipPath
+       id="clipPath114297">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114299" />
+    </clipPath>
+    <clipPath
+       id="clipPath114301">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114303" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4567_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82470-6" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82472-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4566_-1">
+      <path
+         d="m -648.671,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82467-9"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-4">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82447-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath114312">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114314" />
+    </clipPath>
+    <clipPath
+       id="clipPath114316">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114318" />
+    </clipPath>
+    <clipPath
+       id="clipPath114320">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114322" />
+    </clipPath>
+    <clipPath
+       id="clipPath114324">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114326" />
+    </clipPath>
+    <clipPath
+       id="clipPath114328">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114330" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4562_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82435-2" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82437-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4561_-1">
+      <path
+         d="m -660.642,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82432-6"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4559_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82412-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath114339">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114341" />
+    </clipPath>
+    <clipPath
+       id="clipPath114343">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114345" />
+    </clipPath>
+    <clipPath
+       id="clipPath114347">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114349" />
+    </clipPath>
+    <clipPath
+       id="clipPath114351">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114353" />
+    </clipPath>
+    <clipPath
+       id="clipPath114355">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114357" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4557_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82400-6" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82402-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4556_-5">
+      <path
+         d="m -672.614,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82397-5"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4552_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82359-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath114366">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114368" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4550_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82345-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114372">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114374" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4548_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1255.1711,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82319-4" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82321-6" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82323-0" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82325-7" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82327-0" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82329-5" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82331-6" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82333-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4547_-4">
+      <polygon
+         points="-659.18,258.293 -679.997,253.224 -680.004,267.353 -637.65,267.32 -637.642,253.133 "
+         id="use82316-8" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4545_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1321.2666,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82290-3" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82292-3" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82294-2" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82296-5" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82298-7" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82300-2" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82302-9" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82304-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4544_-2">
+      <polygon
+         points="-593.085,258.293 -613.901,253.224 -613.908,267.353 -571.554,267.32 -571.547,253.133 "
+         id="use82287-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4525_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82164-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4527_-8">
+      <rect
+         height="64.251999"
+         width="175.168"
+         y="293.259"
+         x="-679.67902"
+         id="use82174-4" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4530_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3308.0217,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82185-5" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82187-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4529_-6">
+      <rect
+         height="64.037003"
+         width="174.953"
+         y="293.367"
+         x="-679.57202"
+         id="use82182-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath114407">
+      <rect
+         height="64.251999"
+         width="175.168"
+         y="293.259"
+         x="-679.67902"
+         id="use114409" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4532_-7">
+      <rect
+         height="64.251999"
+         width="175.168"
+         y="293.259"
+         x="-679.67902"
+         id="use82197-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114413">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114415" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4534_-4">
+      <rect
+         height="64.251999"
+         width="45.873001"
+         y="293.259"
+         x="-441.56699"
+         id="use82209-7" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4537_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3481.4861,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82220-6" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82222-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4536_-8">
+      <rect
+         height="64.037003"
+         width="45.659"
+         y="293.367"
+         x="-441.45999"
+         id="use82217-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114424">
+      <rect
+         height="64.251999"
+         width="45.873001"
+         y="293.259"
+         x="-441.56699"
+         id="use114426" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4539_-9">
+      <rect
+         height="64.251999"
+         width="45.873001"
+         y="293.259"
+         x="-441.56699"
+         id="use82232-6" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4518_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82083-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114432">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114434" />
+    </clipPath>
+    <clipPath
+       id="clipPath114436">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114438" />
+    </clipPath>
+    <clipPath
+       id="clipPath114440">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114442" />
+    </clipPath>
+    <clipPath
+       id="clipPath114444">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114446" />
+    </clipPath>
+    <clipPath
+       id="clipPath114448">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114450" />
+    </clipPath>
+    <clipPath
+       id="clipPath114452">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114454" />
+    </clipPath>
+    <clipPath
+       id="clipPath114456">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114458" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4516_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,902.0769,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82067-5" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82069-0" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82071-8" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82073-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4515_-1">
+      <polygon
+         points="-613.986,188.287 -613.986,196.684 -594.94,196.685 -592.822,198.478 -590.704,196.685 -571.658,196.685 -571.658,188.287 "
+         id="use82064-3" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4511_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82010-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114469">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114471" />
+    </clipPath>
+    <clipPath
+       id="clipPath114473">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114475" />
+    </clipPath>
+    <clipPath
+       id="clipPath114477">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114479" />
+    </clipPath>
+    <clipPath
+       id="clipPath114481">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114483" />
+    </clipPath>
+    <clipPath
+       id="clipPath114485">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114487" />
+    </clipPath>
+    <clipPath
+       id="clipPath114489">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114491" />
+    </clipPath>
+    <clipPath
+       id="clipPath114493">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114495" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient104534-0"
+       xlink:href="#SVGID_4509_-4"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4509_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop81994-0" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop81996-4" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop81998-3" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82000-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4508_-0">
+      <polygon
+         points="-680.33,188.287 -680.33,196.684 -661.285,196.685 -659.167,198.478 -657.048,196.685 -638.002,196.685 -638.002,188.287 "
+         id="use81991-1" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient115446"
+       xlink:href="#SVGID_4509_-4"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4657_-1-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83100-8-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83102-8-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4656_-1-2">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4655_-2-7"
+         overflow="visible"
+         id="use83097-8-0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4550_-0-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82345-2-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath118798">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use118800" />
+    </clipPath>
+    <defs
+       id="defs83093-1-3">
+      <path
+         d="m -588.813,323.119 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.204 -2.692,2.69"
+         id="SVGID_4655_-2-7"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath102313-6-0">
+      <use
+         id="use102315-6-2"
+         overflow="visible"
+         xlink:href="#SVGID_4655_-2-7"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)"
+       y2="-148.4601"
+       x2="256.37631"
+       y1="-148.4601"
+       x1="255.37621"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient102317-9-1">
+      <stop
+         id="stop102319-7-7"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop102321-3-3"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4873_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84798-79" />
+    </clipPath>
+    <clipPath
+       id="clipPath119610">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119612" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4875_-5">
+      <rect
+         height="10.152"
+         width="174.95799"
+         y="360.13901"
+         x="-679.90002"
+         id="use84810-0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4878_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="40.563"
+       y1="147.0757"
+       x2="41.563"
+       y2="147.0757"
+       gradientTransform="matrix(0,-10.1526,-10.1526,0,900.78,782.1111)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84821-4" />
+      <stop
+         offset="0.486"
+         style="stop-color:#B0D2D9"
+         id="stop84823-2" />
+      <stop
+         offset="0.8287"
+         style="stop-color:#E6EFF1"
+         id="stop84825-99" />
+      <stop
+         offset="0.9939"
+         style="stop-color:#FFFFFF"
+         id="stop84827-79" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84829-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4877_-3">
+      <rect
+         height="10.152"
+         width="174.95799"
+         y="360.13901"
+         x="-679.90002"
+         id="use84818-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath119624">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119626" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4880_-1">
+      <rect
+         height="10.152"
+         width="174.94701"
+         y="360.13901"
+         x="-679.88898"
+         id="use84841-0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4882_-6">
+      <rect
+         height="10.153"
+         width="174.94701"
+         y="360.138"
+         x="-679.88898"
+         id="use84849-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath119632">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119634" />
+    </clipPath>
+    <clipPath
+       id="clipPath119636">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119638" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4871_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94051"
+       x2="-33.205299"
+       y2="249.94051"
+       gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84774-61" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84776-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4870_-2">
+      <polygon
+         points="-654.922,278.632 -640.956,278.632 -640.956,273.567 -654.922,273.567 -656.715,276.108 "
+         id="use84771-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4868_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84759-44" />
+    </clipPath>
+    <clipPath
+       id="clipPath119647">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119649" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4866_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.9404"
+       x2="-33.205299"
+       y2="249.9404"
+       gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84735-9" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84737-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4865_-0">
+      <polygon
+         points="-588.288,278.632 -574.322,278.632 -574.322,273.567 -588.288,273.567 -590.081,276.108 "
+         id="use84732-6" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4863_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84720-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath119658">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119660" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4851_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.9404"
+       x2="-33.205101"
+       y2="249.9404"
+       gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84622-0" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84624-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4850_-19">
+      <polygon
+         points="-588.288,208.524 -574.322,208.524 -574.322,203.458 -588.288,203.458 -590.081,206 "
+         id="use84619-38" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4848_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84607-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath119669">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119671" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4846_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94051"
+       x2="-33.205101"
+       y2="249.94051"
+       gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84585-6" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84587-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4845_-4">
+      <polygon
+         points="-654.922,208.524 -640.956,208.524 -640.956,203.458 -654.922,203.458 -656.715,206 "
+         id="use84582-3" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath119680">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119682" />
+    </clipPath>
+    <clipPath
+       id="clipPath119684">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119686" />
+    </clipPath>
+    <clipPath
+       id="clipPath119688">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119690" />
+    </clipPath>
+    <clipPath
+       id="clipPath119692">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119694" />
+    </clipPath>
+    <clipPath
+       id="clipPath119696">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119698" />
+    </clipPath>
+    <clipPath
+       id="clipPath119700">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119702" />
+    </clipPath>
+    <clipPath
+       id="clipPath119704">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119706" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4838_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84445-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath119710">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119712" />
+    </clipPath>
+    <clipPath
+       id="clipPath119714">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119716" />
+    </clipPath>
+    <clipPath
+       id="clipPath119718">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119720" />
+    </clipPath>
+    <clipPath
+       id="clipPath119722">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119724" />
+    </clipPath>
+    <clipPath
+       id="clipPath119726">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119728" />
+    </clipPath>
+    <clipPath
+       id="clipPath119730">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119732" />
+    </clipPath>
+    <clipPath
+       id="clipPath119734">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119736" />
+    </clipPath>
+    <clipPath
+       id="clipPath119738">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119740" />
+    </clipPath>
+    <clipPath
+       id="clipPath119742">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119744" />
+    </clipPath>
+    <clipPath
+       id="clipPath119746">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119748" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4836_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.8129"
+       x2="37.392799"
+       y2="152.8129"
+       gradientTransform="matrix(0,-9.614,-9.614,0,876.3496,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84423-5" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84425-7" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84427-8" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84429-1" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84431-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4835_-3">
+      <rect
+         height="9.6140003"
+         width="11.315"
+         y="214.106"
+         x="-598.45099"
+         id="use84420-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4833_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84390-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath119760">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119762" />
+    </clipPath>
+    <clipPath
+       id="clipPath119764">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119766" />
+    </clipPath>
+    <clipPath
+       id="clipPath119768">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119770" />
+    </clipPath>
+    <clipPath
+       id="clipPath119772">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119774" />
+    </clipPath>
+    <clipPath
+       id="clipPath119776">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119778" />
+    </clipPath>
+    <clipPath
+       id="clipPath119780">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119782" />
+    </clipPath>
+    <clipPath
+       id="clipPath119784">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119786" />
+    </clipPath>
+    <clipPath
+       id="clipPath119788">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119790" />
+    </clipPath>
+    <clipPath
+       id="clipPath119792">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119794" />
+    </clipPath>
+    <clipPath
+       id="clipPath119796">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119798" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4831_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.813"
+       x2="37.392799"
+       y2="152.813"
+       gradientTransform="matrix(0,-9.614,-9.614,0,809.7396,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84368-1" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84370-0" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84372-8" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84374-42" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84376-15" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4830_-5">
+      <rect
+         height="9.6140003"
+         width="11.316"
+         y="214.106"
+         x="-665.06201"
+         id="use84365-9" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4828_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-734.6116,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84329-5" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84331-2" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84333-3" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84335-9" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84337-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4827_-8">
+      <polygon
+         points="-636.356,251.283 -636.358,231.934 -682.207,231.929 -682.208,251.221 -659.342,256.788 "
+         id="use84326-1" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4825_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-668.057,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84298-1" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84300-7" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84302-5" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84304-7" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84306-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4824_-6">
+      <polygon
+         points="-569.802,251.283 -569.804,231.934 -615.653,231.929 -615.653,251.221 -592.788,256.788 "
+         id="use84295-8" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4787_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84010-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84012-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4786_-2">
+      <path
+         d="m -588.813,346.449 c 0,1.487 1.205,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use84007-5"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4784_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83987-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath119831">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119833" />
+    </clipPath>
+    <clipPath
+       id="clipPath119835">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119837" />
+    </clipPath>
+    <clipPath
+       id="clipPath119839">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119841" />
+    </clipPath>
+    <clipPath
+       id="clipPath119843">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119845" />
+    </clipPath>
+    <clipPath
+       id="clipPath119847">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119849" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4782_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83975-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83977-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4781_-1">
+      <path
+         d="m -600.785,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83972-45"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4779_-85">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83952-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath119858">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119860" />
+    </clipPath>
+    <clipPath
+       id="clipPath119862">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119864" />
+    </clipPath>
+    <clipPath
+       id="clipPath119866">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119868" />
+    </clipPath>
+    <clipPath
+       id="clipPath119870">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119872" />
+    </clipPath>
+    <clipPath
+       id="clipPath119874">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119876" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4777_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83940-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83942-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4776_-7">
+      <path
+         d="m -612.756,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83937-8"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4774_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83917-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath119885">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119887" />
+    </clipPath>
+    <clipPath
+       id="clipPath119889">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119891" />
+    </clipPath>
+    <clipPath
+       id="clipPath119893">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119895" />
+    </clipPath>
+    <clipPath
+       id="clipPath119897">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119899" />
+    </clipPath>
+    <clipPath
+       id="clipPath119901">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119903" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4772_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,177.1768,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83905-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83907-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4771_-2">
+      <path
+         d="m -624.728,346.449 c 0,1.487 1.206,2.692 2.693,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.693,1.206 -2.693,2.692"
+         id="use83902-70"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4769_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83882-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath119912">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119914" />
+    </clipPath>
+    <clipPath
+       id="clipPath119916">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119918" />
+    </clipPath>
+    <clipPath
+       id="clipPath119920">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119922" />
+    </clipPath>
+    <clipPath
+       id="clipPath119924">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119926" />
+    </clipPath>
+    <clipPath
+       id="clipPath119928">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119930" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4767_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83870-91" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83872-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4766_-6">
+      <path
+         d="m -636.699,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83867-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4764_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83847-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath119939">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119941" />
+    </clipPath>
+    <clipPath
+       id="clipPath119943">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119945" />
+    </clipPath>
+    <clipPath
+       id="clipPath119947">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119949" />
+    </clipPath>
+    <clipPath
+       id="clipPath119951">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119953" />
+    </clipPath>
+    <clipPath
+       id="clipPath119955">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119957" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4762_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83835-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83837-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4761_-3">
+      <path
+         d="m -648.671,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83832-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4759_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83812-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath119966">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119968" />
+    </clipPath>
+    <clipPath
+       id="clipPath119970">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119972" />
+    </clipPath>
+    <clipPath
+       id="clipPath119974">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119976" />
+    </clipPath>
+    <clipPath
+       id="clipPath119978">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119980" />
+    </clipPath>
+    <clipPath
+       id="clipPath119982">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119984" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4757_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83800-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83802-76" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4756_-1">
+      <path
+         d="m -660.642,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83797-7"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4754_-4">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83777-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath119993">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119995" />
+    </clipPath>
+    <clipPath
+       id="clipPath119997">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119999" />
+    </clipPath>
+    <clipPath
+       id="clipPath120001">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120003" />
+    </clipPath>
+    <clipPath
+       id="clipPath120005">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120007" />
+    </clipPath>
+    <clipPath
+       id="clipPath120009">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120011" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4752_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83765-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83767-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4751_-6">
+      <path
+         d="m -672.614,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83762-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4749_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83742-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath120020">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120022" />
+    </clipPath>
+    <clipPath
+       id="clipPath120024">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120026" />
+    </clipPath>
+    <clipPath
+       id="clipPath120028">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120030" />
+    </clipPath>
+    <clipPath
+       id="clipPath120032">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120034" />
+    </clipPath>
+    <clipPath
+       id="clipPath120036">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120038" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4722_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83555-85" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83557-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4721_-8">
+      <path
+         d="m -588.813,334.666 c 0,1.488 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use83552-2"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4719_-85">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83532-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath120047">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120049" />
+    </clipPath>
+    <clipPath
+       id="clipPath120051">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120053" />
+    </clipPath>
+    <clipPath
+       id="clipPath120055">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120057" />
+    </clipPath>
+    <clipPath
+       id="clipPath120059">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120061" />
+    </clipPath>
+    <clipPath
+       id="clipPath120063">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120065" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4717_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83520-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83522-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4716_-0">
+      <path
+         d="m -600.785,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83517-0"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4714_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83497-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath120074">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120076" />
+    </clipPath>
+    <clipPath
+       id="clipPath120078">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120080" />
+    </clipPath>
+    <clipPath
+       id="clipPath120082">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120084" />
+    </clipPath>
+    <clipPath
+       id="clipPath120086">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120088" />
+    </clipPath>
+    <clipPath
+       id="clipPath120090">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120092" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4712_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83485-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83487-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4711_-68">
+      <path
+         d="m -612.756,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83482-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4709_-2">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83462-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath120101">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120103" />
+    </clipPath>
+    <clipPath
+       id="clipPath120105">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120107" />
+    </clipPath>
+    <clipPath
+       id="clipPath120109">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120111" />
+    </clipPath>
+    <clipPath
+       id="clipPath120113">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120115" />
+    </clipPath>
+    <clipPath
+       id="clipPath120117">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120119" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4707_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3544"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83450-77" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83452-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4706_-2">
+      <path
+         d="m -636.699,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83447-7"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4704_-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83427-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath120128">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120130" />
+    </clipPath>
+    <clipPath
+       id="clipPath120132">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120134" />
+    </clipPath>
+    <clipPath
+       id="clipPath120136">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120138" />
+    </clipPath>
+    <clipPath
+       id="clipPath120140">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120142" />
+    </clipPath>
+    <clipPath
+       id="clipPath120144">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120146" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4702_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83415-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83417-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4701_-5">
+      <path
+         d="m -648.671,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83412-8"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4699_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83392-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath120155">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120157" />
+    </clipPath>
+    <clipPath
+       id="clipPath120159">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120161" />
+    </clipPath>
+    <clipPath
+       id="clipPath120163">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120165" />
+    </clipPath>
+    <clipPath
+       id="clipPath120167">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120169" />
+    </clipPath>
+    <clipPath
+       id="clipPath120171">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120173" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4697_-62"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.35419"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83380-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83382-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4696_-20">
+      <path
+         d="m -660.642,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83377-7"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4694_-2">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83357-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath120182">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120184" />
+    </clipPath>
+    <clipPath
+       id="clipPath120186">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120188" />
+    </clipPath>
+    <clipPath
+       id="clipPath120190">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120192" />
+    </clipPath>
+    <clipPath
+       id="clipPath120194">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120196" />
+    </clipPath>
+    <clipPath
+       id="clipPath120198">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120200" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4692_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83345-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83347-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4691_-7">
+      <path
+         d="m -672.614,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83342-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4689_-2">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83322-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath120209">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120211" />
+    </clipPath>
+    <clipPath
+       id="clipPath120213">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120215" />
+    </clipPath>
+    <clipPath
+       id="clipPath120217">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120219" />
+    </clipPath>
+    <clipPath
+       id="clipPath120221">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120223" />
+    </clipPath>
+    <clipPath
+       id="clipPath120225">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120227" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4657_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83100-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83102-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4656_-7">
+      <path
+         d="m -588.813,323.119 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.204 -2.692,2.69"
+         id="use83097-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4654_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83077-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath120236">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120238" />
+    </clipPath>
+    <clipPath
+       id="clipPath120240">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120242" />
+    </clipPath>
+    <clipPath
+       id="clipPath120244">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120246" />
+    </clipPath>
+    <clipPath
+       id="clipPath120248">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120250" />
+    </clipPath>
+    <clipPath
+       id="clipPath120252">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120254" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4652_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83065-42" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83067-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4651_-1">
+      <path
+         d="m -612.756,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use83062-1"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4649_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83042-49" />
+    </clipPath>
+    <clipPath
+       id="clipPath120263">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120265" />
+    </clipPath>
+    <clipPath
+       id="clipPath120267">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120269" />
+    </clipPath>
+    <clipPath
+       id="clipPath120271">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120273" />
+    </clipPath>
+    <clipPath
+       id="clipPath120275">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120277" />
+    </clipPath>
+    <clipPath
+       id="clipPath120279">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120281" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4647_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83030-6" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83032-35" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4646_-8">
+      <path
+         d="m -624.728,323.119 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.204 -2.693,2.69"
+         id="use83027-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4644_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83007-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath120290">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120292" />
+    </clipPath>
+    <clipPath
+       id="clipPath120294">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120296" />
+    </clipPath>
+    <clipPath
+       id="clipPath120298">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120300" />
+    </clipPath>
+    <clipPath
+       id="clipPath120302">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120304" />
+    </clipPath>
+    <clipPath
+       id="clipPath120306">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120308" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4642_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82995-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82997-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4641_-9">
+      <path
+         d="m -636.699,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82992-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4639_-4">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82972-97" />
+    </clipPath>
+    <clipPath
+       id="clipPath120317">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120319" />
+    </clipPath>
+    <clipPath
+       id="clipPath120321">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120323" />
+    </clipPath>
+    <clipPath
+       id="clipPath120325">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120327" />
+    </clipPath>
+    <clipPath
+       id="clipPath120329">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120331" />
+    </clipPath>
+    <clipPath
+       id="clipPath120333">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120335" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4637_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82960-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82962-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4636_-2">
+      <path
+         d="m -648.671,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82957-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4634_-2">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82937-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath120344">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120346" />
+    </clipPath>
+    <clipPath
+       id="clipPath120348">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120350" />
+    </clipPath>
+    <clipPath
+       id="clipPath120352">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120354" />
+    </clipPath>
+    <clipPath
+       id="clipPath120356">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120358" />
+    </clipPath>
+    <clipPath
+       id="clipPath120360">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120362" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4632_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82925-64" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82927-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4631_-9">
+      <path
+         d="m -660.642,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82922-36"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4629_-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82902-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath120371">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120373" />
+    </clipPath>
+    <clipPath
+       id="clipPath120375">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120377" />
+    </clipPath>
+    <clipPath
+       id="clipPath120379">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120381" />
+    </clipPath>
+    <clipPath
+       id="clipPath120383">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120385" />
+    </clipPath>
+    <clipPath
+       id="clipPath120387">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120389" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4627_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46021"
+       x2="256.37631"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82890-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82892-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4626_-2">
+      <path
+         d="m -672.614,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82887-9"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4624_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82867-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath120398">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120400" />
+    </clipPath>
+    <clipPath
+       id="clipPath120402">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120404" />
+    </clipPath>
+    <clipPath
+       id="clipPath120406">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120408" />
+    </clipPath>
+    <clipPath
+       id="clipPath120410">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120412" />
+    </clipPath>
+    <clipPath
+       id="clipPath120414">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120416" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4592_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82645-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82647-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4591_-1">
+      <path
+         d="m -588.813,311.531 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.205 -2.692,2.69"
+         id="use82642-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4589_-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82622-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath120425">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120427" />
+    </clipPath>
+    <clipPath
+       id="clipPath120429">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120431" />
+    </clipPath>
+    <clipPath
+       id="clipPath120433">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120435" />
+    </clipPath>
+    <clipPath
+       id="clipPath120437">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120439" />
+    </clipPath>
+    <clipPath
+       id="clipPath120441">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120443" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4587_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,201.1574,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82610-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82612-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4586_-7">
+      <path
+         d="m -600.785,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.203,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82607-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4584_-4">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82587-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath120452">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120454" />
+    </clipPath>
+    <clipPath
+       id="clipPath120456">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120458" />
+    </clipPath>
+    <clipPath
+       id="clipPath120460">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120462" />
+    </clipPath>
+    <clipPath
+       id="clipPath120464">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120466" />
+    </clipPath>
+    <clipPath
+       id="clipPath120468">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120470" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4582_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82575-5" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82577-03" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4581_-7">
+      <path
+         d="m -612.756,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82572-9"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4579_-65">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82552-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath120479">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120481" />
+    </clipPath>
+    <clipPath
+       id="clipPath120483">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120485" />
+    </clipPath>
+    <clipPath
+       id="clipPath120487">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120489" />
+    </clipPath>
+    <clipPath
+       id="clipPath120491">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120493" />
+    </clipPath>
+    <clipPath
+       id="clipPath120495">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120497" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4577_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82540-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82542-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4576_-5">
+      <path
+         d="m -624.728,311.531 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.205 -2.693,2.69"
+         id="use82537-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4574_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82517-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath120506">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120508" />
+    </clipPath>
+    <clipPath
+       id="clipPath120510">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120512" />
+    </clipPath>
+    <clipPath
+       id="clipPath120514">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120516" />
+    </clipPath>
+    <clipPath
+       id="clipPath120518">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120520" />
+    </clipPath>
+    <clipPath
+       id="clipPath120522">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120524" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4572_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82505-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82507-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4571_-3">
+      <path
+         d="m -636.699,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82502-9"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4569_-54">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82482-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath120533">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120535" />
+    </clipPath>
+    <clipPath
+       id="clipPath120537">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120539" />
+    </clipPath>
+    <clipPath
+       id="clipPath120541">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120543" />
+    </clipPath>
+    <clipPath
+       id="clipPath120545">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120547" />
+    </clipPath>
+    <clipPath
+       id="clipPath120549">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120551" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4567_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82470-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82472-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4566_-8">
+      <path
+         d="m -648.671,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82467-6"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82447-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath120560">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120562" />
+    </clipPath>
+    <clipPath
+       id="clipPath120564">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120566" />
+    </clipPath>
+    <clipPath
+       id="clipPath120568">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120570" />
+    </clipPath>
+    <clipPath
+       id="clipPath120572">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120574" />
+    </clipPath>
+    <clipPath
+       id="clipPath120576">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120578" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4562_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82435-29" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82437-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4561_-11">
+      <path
+         d="m -660.642,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82432-8"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4559_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82412-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath120587">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120589" />
+    </clipPath>
+    <clipPath
+       id="clipPath120591">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120593" />
+    </clipPath>
+    <clipPath
+       id="clipPath120595">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120597" />
+    </clipPath>
+    <clipPath
+       id="clipPath120599">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120601" />
+    </clipPath>
+    <clipPath
+       id="clipPath120603">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120605" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4557_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82400-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82402-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4556_-7">
+      <path
+         d="m -672.614,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82397-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4552_-54">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82359-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath120614">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120616" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4550_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82345-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath120620">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120622" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4548_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1255.1711,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82319-6" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82321-0" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82323-4" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82325-70" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82327-00" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82329-8" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82331-67" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82333-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4547_-3">
+      <polygon
+         points="-680.004,267.353 -637.65,267.32 -637.642,253.133 -659.18,258.293 -679.997,253.224 "
+         id="use82316-0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4545_-93"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1321.2666,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82290-1" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82292-0" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82294-1" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82296-0" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82298-2" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82300-3" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82302-1" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82304-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4544_-1">
+      <polygon
+         points="-613.908,267.353 -571.554,267.32 -571.547,253.133 -593.085,258.293 -613.901,253.224 "
+         id="use82287-5" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4525_-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82164-7" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4527_-9">
+      <rect
+         height="64.251999"
+         width="175.168"
+         y="293.259"
+         x="-679.67902"
+         id="use82174-42" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4530_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3308.0217,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82185-2" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82187-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4529_-4">
+      <rect
+         height="64.037003"
+         width="174.953"
+         y="293.367"
+         x="-679.57202"
+         id="use82182-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath120655">
+      <rect
+         height="64.251999"
+         width="175.168"
+         y="293.259"
+         x="-679.67902"
+         id="use120657" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4532_-5">
+      <rect
+         height="64.251999"
+         width="175.168"
+         y="293.259"
+         x="-679.67902"
+         id="use82197-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath120661">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120663" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4534_-1">
+      <rect
+         height="64.251999"
+         width="45.873001"
+         y="293.259"
+         x="-441.56699"
+         id="use82209-3" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4537_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3481.4861,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82220-9" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82222-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4536_-89">
+      <rect
+         height="64.037003"
+         width="45.659"
+         y="293.367"
+         x="-441.45999"
+         id="use82217-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath120672">
+      <rect
+         height="64.251999"
+         width="45.873001"
+         y="293.259"
+         x="-441.56699"
+         id="use120674" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4539_-1">
+      <rect
+         height="64.251999"
+         width="45.873001"
+         y="293.259"
+         x="-441.56699"
+         id="use82232-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4518_-35">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82083-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath120680">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120682" />
+    </clipPath>
+    <clipPath
+       id="clipPath120684">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120686" />
+    </clipPath>
+    <clipPath
+       id="clipPath120688">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120690" />
+    </clipPath>
+    <clipPath
+       id="clipPath120692">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120694" />
+    </clipPath>
+    <clipPath
+       id="clipPath120696">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120698" />
+    </clipPath>
+    <clipPath
+       id="clipPath120700">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120702" />
+    </clipPath>
+    <clipPath
+       id="clipPath120704">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120706" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4516_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,902.0769,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82067-0" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82069-6" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82071-2" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82073-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4515_-11">
+      <polygon
+         points="-590.704,196.685 -571.658,196.685 -571.658,188.287 -613.986,188.287 -613.986,196.684 -594.94,196.685 -592.822,198.478 "
+         id="use82064-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4511_-38">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82010-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath120717">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120719" />
+    </clipPath>
+    <clipPath
+       id="clipPath120721">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120723" />
+    </clipPath>
+    <clipPath
+       id="clipPath120725">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120727" />
+    </clipPath>
+    <clipPath
+       id="clipPath120729">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120731" />
+    </clipPath>
+    <clipPath
+       id="clipPath120733">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120735" />
+    </clipPath>
+    <clipPath
+       id="clipPath120737">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120739" />
+    </clipPath>
+    <clipPath
+       id="clipPath120741">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120743" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient104534-4"
+       xlink:href="#SVGID_4509_-0"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4509_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop81994-08" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop81996-2" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop81998-7" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82000-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4508_-3">
+      <polygon
+         points="-657.048,196.685 -638.002,196.685 -638.002,188.287 -680.33,188.287 -680.33,196.684 -661.285,196.685 -659.167,198.478 "
+         id="use81991-0" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient121727"
+       xlink:href="#SVGID_4509_-0"
+       inkscape:collect="always" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200"
+       id="linearGradient126204"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218"
+       gradientUnits="userSpaceOnUse" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200"
+       id="linearGradient126210"
+       gradientUnits="userSpaceOnUse"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-7"
+       id="linearGradient126210-7"
+       gradientUnits="userSpaceOnUse"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <linearGradient
+       id="linearGradient126200-7"
+       osb:paint="solid">
+      <stop
+         style="stop-color:#ffffff;stop-opacity:1;"
+         offset="0"
+         id="stop126202-2" />
+    </linearGradient>
+    <linearGradient
+       y2="442.86218"
+       x2="219"
+       y1="442.86218"
+       x1="111"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126228"
+       xlink:href="#linearGradient126200-7"
+       inkscape:collect="always" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-2"
+       id="linearGradient126210-9"
+       gradientUnits="userSpaceOnUse"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <linearGradient
+       id="linearGradient126200-2"
+       osb:paint="solid">
+      <stop
+         style="stop-color:#ffffff;stop-opacity:1;"
+         offset="0"
+         id="stop126202-7" />
+    </linearGradient>
+    <linearGradient
+       y2="442.86218"
+       x2="219"
+       y1="442.86218"
+       x1="111"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126435"
+       xlink:href="#linearGradient126200-2"
+       inkscape:collect="always" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-2"
+       id="linearGradient126487"
+       gradientUnits="userSpaceOnUse"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218"
+       gradientTransform="translate(190.5,52.1205)" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-2-0"
+       id="linearGradient126520-9"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="translate(190.5,52.1205)"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <linearGradient
+       id="linearGradient126200-2-0"
+       osb:paint="solid">
+      <stop
+         style="stop-color:#ffffff;stop-opacity:1;"
+         offset="0"
+         id="stop126202-7-1" />
+    </linearGradient>
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-2-2"
+       id="linearGradient126520-4"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="translate(190.5,52.1205)"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <linearGradient
+       id="linearGradient126200-2-2"
+       osb:paint="solid">
+      <stop
+         style="stop-color:#ffffff;stop-opacity:1;"
+         offset="0"
+         id="stop126202-7-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4843_-9">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-9-6">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-9-6-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60-1-9" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-9-6-8">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60-1-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-9-1">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60-9" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-9-1-4">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60-9-6" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-9-6-8-6">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60-1-2-2" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4856_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94031"
+       x2="-33.205101"
+       y2="249.94031"
+       gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84659-2" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84661-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4855_-2">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4854_"
+         overflow="visible"
+         id="use84656-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4853_-8">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4852_"
+         overflow="visible"
+         id="use84644-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath126768">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4852_"
+         overflow="visible"
+         id="use126770" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4841_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.813"
+       x2="37.392799"
+       y2="152.813"
+       gradientTransform="matrix(0,-9.614,-9.614,0,942.8785,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84478-8" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84480-2" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84482-3" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84484-9" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84486-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4840_-7">
+      <rect
+         height="9.6140003"
+         width="11.316"
+         y="214.106"
+         x="-531.92297"
+         id="use84475-9" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4523_-1">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use82132-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath126782">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126784" />
+    </clipPath>
+    <clipPath
+       id="clipPath126786">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126788" />
+    </clipPath>
+    <clipPath
+       id="clipPath126790">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126792" />
+    </clipPath>
+    <clipPath
+       id="clipPath126794">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126796" />
+    </clipPath>
+    <clipPath
+       id="clipPath126798">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126800" />
+    </clipPath>
+    <clipPath
+       id="clipPath126802">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126804" />
+    </clipPath>
+    <clipPath
+       id="clipPath126806">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126808" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4521_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82116-6" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82118-6" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82120-5" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82122-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4520_-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4519_"
+         overflow="visible"
+         id="use82113-2" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126904"
+       xlink:href="#SVGID_4521_-3"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4856_-4-6"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94031"
+       x2="-33.205101"
+       y2="249.94031"
+       gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84659-2-0" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84661-1-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4855_-2-2">
+      <polygon
+         points="-507.673,203.458 -521.64,203.458 -523.432,206 -521.64,208.524 -507.673,208.524 "
+         id="use84656-1-4" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4853_-8-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84644-1-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath127140">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127142" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4523_-1-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82132-5-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath127146">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127148" />
+    </clipPath>
+    <clipPath
+       id="clipPath127150">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127152" />
+    </clipPath>
+    <clipPath
+       id="clipPath127154">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127156" />
+    </clipPath>
+    <clipPath
+       id="clipPath127158">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127160" />
+    </clipPath>
+    <clipPath
+       id="clipPath127162">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127164" />
+    </clipPath>
+    <clipPath
+       id="clipPath127166">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127168" />
+    </clipPath>
+    <clipPath
+       id="clipPath127170">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127172" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126904-6"
+       xlink:href="#SVGID_4521_-3-3"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4521_-3-3"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82116-6-3" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82118-6-5" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82120-5-4" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82122-9-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4520_-5-3">
+      <polygon
+         points="-524.02,196.685 -504.975,196.685 -504.975,188.287 -547.302,188.287 -547.302,196.684 -528.257,196.685 -526.139,198.478 "
+         id="use82113-2-8" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4856_-4-1"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94031"
+       x2="-33.205101"
+       y2="249.94031"
+       gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84659-2-9" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84661-1-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4855_-2-24">
+      <polygon
+         id="use84656-1-9"
+         points="-521.64,208.524 -507.673,208.524 -507.673,203.458 -521.64,203.458 -523.432,206 " />
+    </clipPath>
+    <clipPath
+       id="SVGID_4853_-8-0">
+      <rect
+         id="use84644-1-6"
+         x="-682.31403"
+         y="181.315"
+         width="200.271"
+         height="205.44099" />
+    </clipPath>
+    <clipPath
+       id="clipPath9068">
+      <rect
+         id="use9070"
+         x="-682.31403"
+         y="181.315"
+         width="200.271"
+         height="205.44099" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4523_-1-5">
+      <rect
+         id="use82132-5-25"
+         x="-682.31403"
+         y="181.315"
+         width="200.271"
+         height="205.44099" />
+    </clipPath>
+    <clipPath
+       id="clipPath9074">
+      <rect
+         id="use9076"
+         x="-682.31403"
+         y="181.315"
+         width="200.271"
+         height="205.44099" />
+    </clipPath>
+    <clipPath
+       id="clipPath9078">
+      <rect
+         id="use9080"
+         x="-682.31403"
+         y="181.315"
+         width="200.271"
+         height="205.44099" />
+    </clipPath>
+    <clipPath
+       id="clipPath9082">
+      <rect
+         id="use9084"
+         x="-682.31403"
+         y="181.315"
+         width="200.271"
+         height="205.44099" />
+    </clipPath>
+    <clipPath
+       id="clipPath9086">
+      <rect
+         id="use9088"
+         x="-682.31403"
+         y="181.315"
+         width="200.271"
+         height="205.44099" />
+    </clipPath>
+    <clipPath
+       id="clipPath9090">
+      <rect
+         id="use9092"
+         x="-682.31403"
+         y="181.315"
+         width="200.271"
+         height="205.44099" />
+    </clipPath>
+    <clipPath
+       id="clipPath9094">
+      <rect
+         id="use9096"
+         x="-682.31403"
+         y="181.315"
+         width="200.271"
+         height="205.44099" />
+    </clipPath>
+    <clipPath
+       id="clipPath9098">
+      <rect
+         id="use9100"
+         x="-682.31403"
+         y="181.315"
+         width="200.271"
+         height="205.44099" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126904-61"
+       xlink:href="#SVGID_4521_-3-0"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4521_-3-0"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82116-6-7" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82118-6-2" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82120-5-1" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82122-9-34" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4520_-5-6">
+      <polygon
+         id="use82113-2-5"
+         points="-528.257,196.685 -526.139,198.478 -524.02,196.685 -504.975,196.685 -504.975,188.287 -547.302,188.287 -547.302,196.684 " />
+    </clipPath>
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-2-0"
+       id="linearGradient9382"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="translate(28.50002,64.927434)"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <clipPath
+       id="SVGID_4550_-6">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4549_-8"
+         overflow="visible"
+         id="use82345-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath9448">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4549_-8"
+         overflow="visible"
+         id="use9450" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4559_-4">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82412-03" />
+    </clipPath>
+    <clipPath
+       id="clipPath9526">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9528" />
+    </clipPath>
+    <clipPath
+       id="clipPath9530">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9532" />
+    </clipPath>
+    <clipPath
+       id="clipPath9534">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9536" />
+    </clipPath>
+    <clipPath
+       id="clipPath9538">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9540" />
+    </clipPath>
+    <clipPath
+       id="clipPath9542">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9544" />
+    </clipPath>
+    <defs
+       id="defs82478">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4568_" />
+    </defs>
+    <clipPath
+       id="clipPath101841">
+      <use
+         id="use101843"
+         overflow="visible"
+         xlink:href="#SVGID_4568_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs82443">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4563_" />
+    </defs>
+    <clipPath
+       id="clipPath101814">
+      <use
+         id="use101816"
+         overflow="visible"
+         xlink:href="#SVGID_4563_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-2">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-4" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-9">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-9-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-1-6" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-9-3-4">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-1-6-0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-9-3-4-4">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-1-6-0-9" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-9-3-43">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-1-6-01" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-9-8">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-1-9" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-3-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-2-3" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-39">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-27" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4550_-6-4">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4549_-8-0"
+         overflow="visible"
+         id="use82345-0-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath9882">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4549_-8-0"
+         overflow="visible"
+         id="use9884" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4856_-4-65"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94031"
+       x2="-33.205101"
+       y2="249.94031"
+       gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84659-2-8" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84661-1-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4855_-2-4">
+      <polygon
+         points="-523.432,206 -521.64,208.524 -507.673,208.524 -507.673,203.458 -521.64,203.458 "
+         id="use84656-1-0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4853_-8-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84644-1-67" />
+    </clipPath>
+    <clipPath
+       id="clipPath9893">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9895" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4523_-1-2">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82132-5-21" />
+    </clipPath>
+    <clipPath
+       id="clipPath9899">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9901" />
+    </clipPath>
+    <clipPath
+       id="clipPath9903">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9905" />
+    </clipPath>
+    <clipPath
+       id="clipPath9907">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9909" />
+    </clipPath>
+    <clipPath
+       id="clipPath9911">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9913" />
+    </clipPath>
+    <clipPath
+       id="clipPath9915">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9917" />
+    </clipPath>
+    <clipPath
+       id="clipPath9919">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9921" />
+    </clipPath>
+    <clipPath
+       id="clipPath9923">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9925" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126904-5"
+       xlink:href="#SVGID_4521_-3-09"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4521_-3-09"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82116-6-0" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82118-6-1" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82120-5-5" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82122-9-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4520_-5-2">
+      <polygon
+         points="-504.975,196.685 -504.975,188.287 -547.302,188.287 -547.302,196.684 -528.257,196.685 -526.139,198.478 -524.02,196.685 "
+         id="use82113-2-6" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-9-6-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60-1-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4873_-98">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_-8"
+         overflow="visible"
+         id="use84798-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath9939">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_-8"
+         overflow="visible"
+         id="use9941" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4875_-1">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4874_-4"
+         overflow="visible"
+         id="use84810-3" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4878_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="40.563"
+       y1="147.0757"
+       x2="41.563"
+       y2="147.0757"
+       gradientTransform="matrix(0,-10.1526,-10.1526,0,900.78,782.1111)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84821-5" />
+      <stop
+         offset="0.486"
+         style="stop-color:#B0D2D9"
+         id="stop84823-1" />
+      <stop
+         offset="0.8287"
+         style="stop-color:#E6EFF1"
+         id="stop84825-0" />
+      <stop
+         offset="0.9939"
+         style="stop-color:#FFFFFF"
+         id="stop84827-2" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84829-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4877_-4">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4876_-1"
+         overflow="visible"
+         id="use84818-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath9953">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_-8"
+         overflow="visible"
+         id="use9955" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4880_-2">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4879_-3"
+         overflow="visible"
+         id="use84841-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4882_-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4881_-4"
+         overflow="visible"
+         id="use84849-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath9961">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_-8"
+         overflow="visible"
+         id="use9963" />
+    </clipPath>
+    <clipPath
+       id="clipPath9965">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_-8"
+         overflow="visible"
+         id="use9967" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4871_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94051"
+       x2="-33.205299"
+       y2="249.94051"
+       gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84774-5" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84776-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4870_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4869_-7"
+         overflow="visible"
+         id="use84771-7" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4868_-1">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4867_-0"
+         overflow="visible"
+         id="use84759-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath9976">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4867_-0"
+         overflow="visible"
+         id="use9978" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4866_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.9404"
+       x2="-33.205299"
+       y2="249.9404"
+       gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84735-7" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84737-90" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4865_-5">
+      <polygon
+         points="-574.322,278.632 -574.322,273.567 -588.288,273.567 -590.081,276.108 -588.288,278.632 "
+         id="use84732-4" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4863_-4">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84720-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath9987">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9989" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath9993">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use9995" />
+    </clipPath>
+    <clipPath
+       id="clipPath9997">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use9999" />
+    </clipPath>
+    <clipPath
+       id="clipPath10001">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use10003" />
+    </clipPath>
+    <clipPath
+       id="clipPath10005">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use10007" />
+    </clipPath>
+    <clipPath
+       id="clipPath10009">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use10011" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4828_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-734.6116,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84329-0" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84331-5" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84333-31" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84335-7" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84337-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4827_-1">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4826_-2"
+         overflow="visible"
+         id="use84326-14" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126904-61-3"
+       xlink:href="#SVGID_4521_-3-0-3"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4521_-3-0-3"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82116-6-7-7" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82118-6-2-5" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82120-5-1-8" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82122-9-34-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4520_-5-6-7">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4519_-5-8"
+         overflow="visible"
+         id="use82113-2-5-0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4523_-1-5-1">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8-1"
+         overflow="visible"
+         id="use82132-5-25-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath10031">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8-1"
+         overflow="visible"
+         id="use10033" />
+    </clipPath>
+    <clipPath
+       id="clipPath10035">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8-1"
+         overflow="visible"
+         id="use10037" />
+    </clipPath>
+    <clipPath
+       id="clipPath10039">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8-1"
+         overflow="visible"
+         id="use10041" />
+    </clipPath>
+    <clipPath
+       id="clipPath10043">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8-1"
+         overflow="visible"
+         id="use10045" />
+    </clipPath>
+    <clipPath
+       id="clipPath10047">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8-1"
+         overflow="visible"
+         id="use10049" />
+    </clipPath>
+    <clipPath
+       id="clipPath10051">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8-1"
+         overflow="visible"
+         id="use10053" />
+    </clipPath>
+    <clipPath
+       id="clipPath10055">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8-1"
+         overflow="visible"
+         id="use10057" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4853_-8-0-4">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4852_-4-9"
+         overflow="visible"
+         id="use84644-1-6-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath10061">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4852_-4-9"
+         overflow="visible"
+         id="use10063" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4856_-4-1-0"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94031"
+       x2="-33.205101"
+       y2="249.94031"
+       gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84659-2-9-3" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84661-1-0-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4855_-2-24-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4854_-1-2"
+         overflow="visible"
+         id="use84656-1-9-9" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4825_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-668.057,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84298-2" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84300-4" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84302-9" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84304-0" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84306-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4824_-4">
+      <polygon
+         points="-569.804,231.934 -615.653,231.929 -615.653,251.221 -592.788,256.788 -569.802,251.283 "
+         id="use84295-5" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4787_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84010-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84012-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4786_-1">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4785_-4"
+         overflow="visible"
+         id="use84007-57" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4784_-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_-9"
+         overflow="visible"
+         id="use83987-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath10085">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_-9"
+         overflow="visible"
+         id="use10087" />
+    </clipPath>
+    <clipPath
+       id="clipPath10089">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_-9"
+         overflow="visible"
+         id="use10091" />
+    </clipPath>
+    <clipPath
+       id="clipPath10093">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_-9"
+         overflow="visible"
+         id="use10095" />
+    </clipPath>
+    <clipPath
+       id="clipPath10097">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_-9"
+         overflow="visible"
+         id="use10099" />
+    </clipPath>
+    <clipPath
+       id="clipPath10101">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_-9"
+         overflow="visible"
+         id="use10103" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4782_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83975-2" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83977-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4781_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4780_-6"
+         overflow="visible"
+         id="use83972-9" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4779_-83">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_-7"
+         overflow="visible"
+         id="use83952-22" />
+    </clipPath>
+    <clipPath
+       id="clipPath10112">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_-7"
+         overflow="visible"
+         id="use10114" />
+    </clipPath>
+    <clipPath
+       id="clipPath10116">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_-7"
+         overflow="visible"
+         id="use10118" />
+    </clipPath>
+    <clipPath
+       id="clipPath10120">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_-7"
+         overflow="visible"
+         id="use10122" />
+    </clipPath>
+    <clipPath
+       id="clipPath10124">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_-7"
+         overflow="visible"
+         id="use10126" />
+    </clipPath>
+    <clipPath
+       id="clipPath10128">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_-7"
+         overflow="visible"
+         id="use10130" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4777_-42"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83940-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83942-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4776_-8">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4775_-6"
+         overflow="visible"
+         id="use83937-7" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4774_-60">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_-2"
+         overflow="visible"
+         id="use83917-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath10139">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_-2"
+         overflow="visible"
+         id="use10141" />
+    </clipPath>
+    <clipPath
+       id="clipPath10143">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_-2"
+         overflow="visible"
+         id="use10145" />
+    </clipPath>
+    <clipPath
+       id="clipPath10147">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_-2"
+         overflow="visible"
+         id="use10149" />
+    </clipPath>
+    <clipPath
+       id="clipPath10151">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_-2"
+         overflow="visible"
+         id="use10153" />
+    </clipPath>
+    <clipPath
+       id="clipPath10155">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_-2"
+         overflow="visible"
+         id="use10157" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4772_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,177.1768,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83905-30" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83907-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4771_-8">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4770_-0"
+         overflow="visible"
+         id="use83902-8" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4769_-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_-0"
+         overflow="visible"
+         id="use83882-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath10166">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_-0"
+         overflow="visible"
+         id="use10168" />
+    </clipPath>
+    <clipPath
+       id="clipPath10170">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_-0"
+         overflow="visible"
+         id="use10172" />
+    </clipPath>
+    <clipPath
+       id="clipPath10174">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_-0"
+         overflow="visible"
+         id="use10176" />
+    </clipPath>
+    <clipPath
+       id="clipPath10178">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_-0"
+         overflow="visible"
+         id="use10180" />
+    </clipPath>
+    <clipPath
+       id="clipPath10182">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_-0"
+         overflow="visible"
+         id="use10184" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4767_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83870-2" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83872-06" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4766_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4765_-7"
+         overflow="visible"
+         id="use83867-8" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4764_-8">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_-6"
+         overflow="visible"
+         id="use83847-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath10193">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_-6"
+         overflow="visible"
+         id="use10195" />
+    </clipPath>
+    <clipPath
+       id="clipPath10197">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_-6"
+         overflow="visible"
+         id="use10199" />
+    </clipPath>
+    <clipPath
+       id="clipPath10201">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_-6"
+         overflow="visible"
+         id="use10203" />
+    </clipPath>
+    <clipPath
+       id="clipPath10205">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_-6"
+         overflow="visible"
+         id="use10207" />
+    </clipPath>
+    <clipPath
+       id="clipPath10209">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_-6"
+         overflow="visible"
+         id="use10211" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4762_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83835-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83837-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4761_-32">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4760_-8"
+         overflow="visible"
+         id="use83832-48" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4759_-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_-6"
+         overflow="visible"
+         id="use83812-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath10220">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_-6"
+         overflow="visible"
+         id="use10222" />
+    </clipPath>
+    <clipPath
+       id="clipPath10224">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_-6"
+         overflow="visible"
+         id="use10226" />
+    </clipPath>
+    <clipPath
+       id="clipPath10228">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_-6"
+         overflow="visible"
+         id="use10230" />
+    </clipPath>
+    <clipPath
+       id="clipPath10232">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_-6"
+         overflow="visible"
+         id="use10234" />
+    </clipPath>
+    <clipPath
+       id="clipPath10236">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_-6"
+         overflow="visible"
+         id="use10238" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4757_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83800-5" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83802-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4756_-57">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4755_-2"
+         overflow="visible"
+         id="use83797-4" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4754_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_-7"
+         overflow="visible"
+         id="use83777-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath10247">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_-7"
+         overflow="visible"
+         id="use10249" />
+    </clipPath>
+    <clipPath
+       id="clipPath10251">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_-7"
+         overflow="visible"
+         id="use10253" />
+    </clipPath>
+    <clipPath
+       id="clipPath10255">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_-7"
+         overflow="visible"
+         id="use10257" />
+    </clipPath>
+    <clipPath
+       id="clipPath10259">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_-7"
+         overflow="visible"
+         id="use10261" />
+    </clipPath>
+    <clipPath
+       id="clipPath10263">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_-7"
+         overflow="visible"
+         id="use10265" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4752_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83765-41" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83767-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4751_-1">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4750_-6"
+         overflow="visible"
+         id="use83762-4" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4722_-33"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83555-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83557-25" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4721_-50">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4720_-7"
+         overflow="visible"
+         id="use83552-66" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4719_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_-5"
+         overflow="visible"
+         id="use83532-81" />
+    </clipPath>
+    <clipPath
+       id="clipPath10279">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_-5"
+         overflow="visible"
+         id="use10281" />
+    </clipPath>
+    <clipPath
+       id="clipPath10283">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_-5"
+         overflow="visible"
+         id="use10285" />
+    </clipPath>
+    <clipPath
+       id="clipPath10287">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_-5"
+         overflow="visible"
+         id="use10289" />
+    </clipPath>
+    <clipPath
+       id="clipPath10291">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_-5"
+         overflow="visible"
+         id="use10293" />
+    </clipPath>
+    <clipPath
+       id="clipPath10295">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_-5"
+         overflow="visible"
+         id="use10297" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4717_-52"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83520-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83522-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4716_-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4715_-7"
+         overflow="visible"
+         id="use83517-8" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4714_-7">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_-2"
+         overflow="visible"
+         id="use83497-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath10306">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_-2"
+         overflow="visible"
+         id="use10308" />
+    </clipPath>
+    <clipPath
+       id="clipPath10310">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_-2"
+         overflow="visible"
+         id="use10312" />
+    </clipPath>
+    <clipPath
+       id="clipPath10314">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_-2"
+         overflow="visible"
+         id="use10316" />
+    </clipPath>
+    <clipPath
+       id="clipPath10318">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_-2"
+         overflow="visible"
+         id="use10320" />
+    </clipPath>
+    <clipPath
+       id="clipPath10322">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_-2"
+         overflow="visible"
+         id="use10324" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4712_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83485-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83487-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4711_-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4710_-7"
+         overflow="visible"
+         id="use83482-7" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4709_-6">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_-8"
+         overflow="visible"
+         id="use83462-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath10333">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_-8"
+         overflow="visible"
+         id="use10335" />
+    </clipPath>
+    <clipPath
+       id="clipPath10337">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_-8"
+         overflow="visible"
+         id="use10339" />
+    </clipPath>
+    <clipPath
+       id="clipPath10341">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_-8"
+         overflow="visible"
+         id="use10343" />
+    </clipPath>
+    <clipPath
+       id="clipPath10345">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_-8"
+         overflow="visible"
+         id="use10347" />
+    </clipPath>
+    <clipPath
+       id="clipPath10349">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_-8"
+         overflow="visible"
+         id="use10351" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4707_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3544"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83450-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83452-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4706_-25">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4705_-7"
+         overflow="visible"
+         id="use83447-5" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4704_-1">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_-5"
+         overflow="visible"
+         id="use83427-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath10360">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_-5"
+         overflow="visible"
+         id="use10362" />
+    </clipPath>
+    <clipPath
+       id="clipPath10364">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_-5"
+         overflow="visible"
+         id="use10366" />
+    </clipPath>
+    <clipPath
+       id="clipPath10368">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_-5"
+         overflow="visible"
+         id="use10370" />
+    </clipPath>
+    <clipPath
+       id="clipPath10372">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_-5"
+         overflow="visible"
+         id="use10374" />
+    </clipPath>
+    <clipPath
+       id="clipPath10376">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_-5"
+         overflow="visible"
+         id="use10378" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4702_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83415-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83417-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4701_-9">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4700_-3"
+         overflow="visible"
+         id="use83412-5" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4699_-07">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_-1"
+         overflow="visible"
+         id="use83392-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath10387">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_-1"
+         overflow="visible"
+         id="use10389" />
+    </clipPath>
+    <clipPath
+       id="clipPath10391">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_-1"
+         overflow="visible"
+         id="use10393" />
+    </clipPath>
+    <clipPath
+       id="clipPath10395">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_-1"
+         overflow="visible"
+         id="use10397" />
+    </clipPath>
+    <clipPath
+       id="clipPath10399">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_-1"
+         overflow="visible"
+         id="use10401" />
+    </clipPath>
+    <clipPath
+       id="clipPath10403">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_-1"
+         overflow="visible"
+         id="use10405" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4697_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.35419"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83380-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83382-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4696_-9">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4695_-3"
+         overflow="visible"
+         id="use83377-9" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4694_-6">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_-2"
+         overflow="visible"
+         id="use83357-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath10414">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_-2"
+         overflow="visible"
+         id="use10416" />
+    </clipPath>
+    <clipPath
+       id="clipPath10418">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_-2"
+         overflow="visible"
+         id="use10420" />
+    </clipPath>
+    <clipPath
+       id="clipPath10422">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_-2"
+         overflow="visible"
+         id="use10424" />
+    </clipPath>
+    <clipPath
+       id="clipPath10426">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_-2"
+         overflow="visible"
+         id="use10428" />
+    </clipPath>
+    <clipPath
+       id="clipPath10430">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_-2"
+         overflow="visible"
+         id="use10432" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4692_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83345-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83347-37" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4691_-8">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4690_-8"
+         overflow="visible"
+         id="use83342-0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4657_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83100-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83102-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4656_-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4655_-6"
+         overflow="visible"
+         id="use83097-44" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4654_-2">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_-9"
+         overflow="visible"
+         id="use83077-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath10446">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_-9"
+         overflow="visible"
+         id="use10448" />
+    </clipPath>
+    <clipPath
+       id="clipPath10450">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_-9"
+         overflow="visible"
+         id="use10452" />
+    </clipPath>
+    <clipPath
+       id="clipPath10454">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_-9"
+         overflow="visible"
+         id="use10456" />
+    </clipPath>
+    <clipPath
+       id="clipPath10458">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_-9"
+         overflow="visible"
+         id="use10460" />
+    </clipPath>
+    <clipPath
+       id="clipPath10462">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_-9"
+         overflow="visible"
+         id="use10464" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4652_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83065-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83067-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4651_-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4650_-3"
+         overflow="visible"
+         id="use83062-0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4649_-4">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_-2"
+         overflow="visible"
+         id="use83042-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath10473">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_-2"
+         overflow="visible"
+         id="use10475" />
+    </clipPath>
+    <clipPath
+       id="clipPath10477">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_-2"
+         overflow="visible"
+         id="use10479" />
+    </clipPath>
+    <clipPath
+       id="clipPath10481">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_-2"
+         overflow="visible"
+         id="use10483" />
+    </clipPath>
+    <clipPath
+       id="clipPath10485">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_-2"
+         overflow="visible"
+         id="use10487" />
+    </clipPath>
+    <clipPath
+       id="clipPath10489">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_-2"
+         overflow="visible"
+         id="use10491" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4647_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83030-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83032-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4646_-87">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4645_-7"
+         overflow="visible"
+         id="use83027-49" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4644_-6">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_-8"
+         overflow="visible"
+         id="use83007-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath10500">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_-8"
+         overflow="visible"
+         id="use10502" />
+    </clipPath>
+    <clipPath
+       id="clipPath10504">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_-8"
+         overflow="visible"
+         id="use10506" />
+    </clipPath>
+    <clipPath
+       id="clipPath10508">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_-8"
+         overflow="visible"
+         id="use10510" />
+    </clipPath>
+    <clipPath
+       id="clipPath10512">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_-8"
+         overflow="visible"
+         id="use10514" />
+    </clipPath>
+    <clipPath
+       id="clipPath10516">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_-8"
+         overflow="visible"
+         id="use10518" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4642_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82995-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82997-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4641_-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4640_-2"
+         overflow="visible"
+         id="use82992-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4639_-45">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_-3"
+         overflow="visible"
+         id="use82972-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath10527">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_-3"
+         overflow="visible"
+         id="use10529" />
+    </clipPath>
+    <clipPath
+       id="clipPath10531">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_-3"
+         overflow="visible"
+         id="use10533" />
+    </clipPath>
+    <clipPath
+       id="clipPath10535">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_-3"
+         overflow="visible"
+         id="use10537" />
+    </clipPath>
+    <clipPath
+       id="clipPath10539">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_-3"
+         overflow="visible"
+         id="use10541" />
+    </clipPath>
+    <clipPath
+       id="clipPath10543">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_-3"
+         overflow="visible"
+         id="use10545" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4637_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82960-54" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82962-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4636_-9">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4635_-5"
+         overflow="visible"
+         id="use82957-7" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4634_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_-6"
+         overflow="visible"
+         id="use82937-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath10554">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_-6"
+         overflow="visible"
+         id="use10556" />
+    </clipPath>
+    <clipPath
+       id="clipPath10558">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_-6"
+         overflow="visible"
+         id="use10560" />
+    </clipPath>
+    <clipPath
+       id="clipPath10562">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_-6"
+         overflow="visible"
+         id="use10564" />
+    </clipPath>
+    <clipPath
+       id="clipPath10566">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_-6"
+         overflow="visible"
+         id="use10568" />
+    </clipPath>
+    <clipPath
+       id="clipPath10570">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_-6"
+         overflow="visible"
+         id="use10572" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4632_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82925-5" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82927-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4631_-7">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4630_-9"
+         overflow="visible"
+         id="use82922-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4629_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_-3"
+         overflow="visible"
+         id="use82902-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath10581">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_-3"
+         overflow="visible"
+         id="use10583" />
+    </clipPath>
+    <clipPath
+       id="clipPath10585">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_-3"
+         overflow="visible"
+         id="use10587" />
+    </clipPath>
+    <clipPath
+       id="clipPath10589">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_-3"
+         overflow="visible"
+         id="use10591" />
+    </clipPath>
+    <clipPath
+       id="clipPath10593">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_-3"
+         overflow="visible"
+         id="use10595" />
+    </clipPath>
+    <clipPath
+       id="clipPath10597">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_-3"
+         overflow="visible"
+         id="use10599" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4627_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46021"
+       x2="256.37631"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82890-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82892-95" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4626_-3">
+      <path
+         d="m -672.614,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82887-94"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4592_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82645-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82647-71" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4591_-2">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4590_-7"
+         overflow="visible"
+         id="use82642-05" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4589_-7">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_-2"
+         overflow="visible"
+         id="use82622-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath10613">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_-2"
+         overflow="visible"
+         id="use10615" />
+    </clipPath>
+    <clipPath
+       id="clipPath10617">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_-2"
+         overflow="visible"
+         id="use10619" />
+    </clipPath>
+    <clipPath
+       id="clipPath10621">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_-2"
+         overflow="visible"
+         id="use10623" />
+    </clipPath>
+    <clipPath
+       id="clipPath10625">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_-2"
+         overflow="visible"
+         id="use10627" />
+    </clipPath>
+    <clipPath
+       id="clipPath10629">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_-2"
+         overflow="visible"
+         id="use10631" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4587_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,201.1574,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82610-2" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82612-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4586_-03">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4585_-8"
+         overflow="visible"
+         id="use82607-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4584_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_-4"
+         overflow="visible"
+         id="use82587-10" />
+    </clipPath>
+    <clipPath
+       id="clipPath10640">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_-4"
+         overflow="visible"
+         id="use10642" />
+    </clipPath>
+    <clipPath
+       id="clipPath10644">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_-4"
+         overflow="visible"
+         id="use10646" />
+    </clipPath>
+    <clipPath
+       id="clipPath10648">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_-4"
+         overflow="visible"
+         id="use10650" />
+    </clipPath>
+    <clipPath
+       id="clipPath10652">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_-4"
+         overflow="visible"
+         id="use10654" />
+    </clipPath>
+    <clipPath
+       id="clipPath10656">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_-4"
+         overflow="visible"
+         id="use10658" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4582_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82575-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82577-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4581_-2">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4580_-5"
+         overflow="visible"
+         id="use82572-40" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4579_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_-9"
+         overflow="visible"
+         id="use82552-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath10667">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_-9"
+         overflow="visible"
+         id="use10669" />
+    </clipPath>
+    <clipPath
+       id="clipPath10671">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_-9"
+         overflow="visible"
+         id="use10673" />
+    </clipPath>
+    <clipPath
+       id="clipPath10675">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_-9"
+         overflow="visible"
+         id="use10677" />
+    </clipPath>
+    <clipPath
+       id="clipPath10679">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_-9"
+         overflow="visible"
+         id="use10681" />
+    </clipPath>
+    <clipPath
+       id="clipPath10683">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_-9"
+         overflow="visible"
+         id="use10685" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4577_-27"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82540-6" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82542-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4576_-1">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4575_-2"
+         overflow="visible"
+         id="use82537-22" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4574_-9">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_-4"
+         overflow="visible"
+         id="use82517-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath10694">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_-4"
+         overflow="visible"
+         id="use10696" />
+    </clipPath>
+    <clipPath
+       id="clipPath10698">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_-4"
+         overflow="visible"
+         id="use10700" />
+    </clipPath>
+    <clipPath
+       id="clipPath10702">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_-4"
+         overflow="visible"
+         id="use10704" />
+    </clipPath>
+    <clipPath
+       id="clipPath10706">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_-4"
+         overflow="visible"
+         id="use10708" />
+    </clipPath>
+    <clipPath
+       id="clipPath10710">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_-4"
+         overflow="visible"
+         id="use10712" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4572_-41"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82505-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82507-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4571_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4570_-9"
+         overflow="visible"
+         id="use82502-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4569_-7">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use82482-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath10721">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use10723" />
+    </clipPath>
+    <clipPath
+       id="clipPath10725">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use10727" />
+    </clipPath>
+    <clipPath
+       id="clipPath10729">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use10731" />
+    </clipPath>
+    <clipPath
+       id="clipPath10733">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use10735" />
+    </clipPath>
+    <clipPath
+       id="clipPath10737">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use10739" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4567_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82470-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82472-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4566_-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4565_-3"
+         overflow="visible"
+         id="use82467-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-73">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath10748">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use10750" />
+    </clipPath>
+    <clipPath
+       id="clipPath10752">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use10754" />
+    </clipPath>
+    <clipPath
+       id="clipPath10756">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use10758" />
+    </clipPath>
+    <clipPath
+       id="clipPath10760">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use10762" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4557_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82400-5" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82402-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4556_-9">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4555_-7"
+         overflow="visible"
+         id="use82397-7" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4552_-4">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4551_-7"
+         overflow="visible"
+         id="use82359-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath10771">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4551_-7"
+         overflow="visible"
+         id="use10773" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4550_-50">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4549_-7"
+         overflow="visible"
+         id="use82345-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath10777">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4549_-7"
+         overflow="visible"
+         id="use10779" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4548_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1255.1711,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82319-3" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82321-8" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82323-7" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82325-9" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82327-2" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82329-0" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82331-0" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82333-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4547_-1">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4546_-1"
+         overflow="visible"
+         id="use82316-1" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4545_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1321.2666,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82290-7" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82292-4" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82294-4" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82296-50" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82298-8" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82300-4" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82302-5" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82304-53" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4544_-16">
+      <polygon
+         points="-571.554,267.32 -571.547,253.133 -593.085,258.293 -613.901,253.224 -613.908,267.353 "
+         id="use82287-4" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4525_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4524_-5"
+         overflow="visible"
+         id="use82164-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4527_-7">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4526_-7"
+         overflow="visible"
+         id="use82174-5" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4530_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3308.0217,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82185-1" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82187-47" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4529_-9">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4528_-0"
+         overflow="visible"
+         id="use82182-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath10812">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4526_-7"
+         overflow="visible"
+         id="use10814" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4532_-8">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4531_-4"
+         overflow="visible"
+         id="use82197-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath10818">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4524_-5"
+         overflow="visible"
+         id="use10820" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4534_-49">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4533_-1"
+         overflow="visible"
+         id="use82209-0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4537_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3481.4861,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82220-2" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82222-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4536_-4">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4535_-5"
+         overflow="visible"
+         id="use82217-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath10829">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4533_-1"
+         overflow="visible"
+         id="use10831" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4539_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4538_-7"
+         overflow="visible"
+         id="use82232-1" />
+    </clipPath>
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-2-0-0"
+       id="linearGradient9382-1"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="translate(197.19124,64.89703)"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <linearGradient
+       id="linearGradient126200-2-0-0"
+       osb:paint="solid">
+      <stop
+         style="stop-color:#ffffff;stop-opacity:1;"
+         offset="0"
+         id="stop126202-7-1-6" />
+    </linearGradient>
+    <linearGradient
+       id="SVGID_4757_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83800-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83802-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4756_-3">
+      <path
+         d="m -660.642,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83797-44"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4754_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83777-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath14916">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14918" />
+    </clipPath>
+    <clipPath
+       id="clipPath14920">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14922" />
+    </clipPath>
+    <clipPath
+       id="clipPath14924">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14926" />
+    </clipPath>
+    <clipPath
+       id="clipPath14928">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14930" />
+    </clipPath>
+    <clipPath
+       id="clipPath14932">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14934" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4752_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83765-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83767-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4751_-5">
+      <path
+         d="m -672.614,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83762-8"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4697_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.35419"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83380-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83382-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4696_-0">
+      <path
+         d="m -660.642,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83377-5"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4694_-29">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83357-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath14948">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14950" />
+    </clipPath>
+    <clipPath
+       id="clipPath14952">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14954" />
+    </clipPath>
+    <clipPath
+       id="clipPath14956">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14958" />
+    </clipPath>
+    <clipPath
+       id="clipPath14960">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14962" />
+    </clipPath>
+    <clipPath
+       id="clipPath14964">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14966" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4692_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83345-5" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83347-07" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4691_-6">
+      <path
+         d="m -672.614,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83342-2"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4632_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82925-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82927-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4631_-16">
+      <path
+         d="m -660.642,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82922-1"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4629_-99">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82902-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath14980">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14982" />
+    </clipPath>
+    <clipPath
+       id="clipPath14984">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14986" />
+    </clipPath>
+    <clipPath
+       id="clipPath14988">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14990" />
+    </clipPath>
+    <clipPath
+       id="clipPath14992">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14994" />
+    </clipPath>
+    <clipPath
+       id="clipPath14996">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use14998" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4627_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46021"
+       x2="256.37631"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82890-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82892-90" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4626_-69">
+      <path
+         d="m -672.614,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82887-7"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <linearGradient
+       y2="-148.46021"
+       x2="256.37631"
+       y1="-148.46021"
+       x1="255.37621"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1054.4158)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient15092"
+       xlink:href="#SVGID_4627_-3"
+       inkscape:collect="always" />
+    <clipPath
+       id="SVGID_4564_-7-9-3-4-4-7">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-1-6-0-9-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-9-3-43-6">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-1-6-01-4" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-9-8-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-1-9-3" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4787_-8-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84010-1-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84012-3-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4786_-1-6">
+      <path
+         d="m -588.813,346.449 c 0,1.487 1.205,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use84007-57-5"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4722_-33-2"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83555-7-5" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83557-25-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4721_-50-0">
+      <path
+         d="m -588.813,334.666 c 0,1.488 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use83552-66-1"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4657_-5-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83100-0-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83102-0-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4656_-5-8">
+      <path
+         d="m -588.813,323.119 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.204 -2.692,2.69"
+         id="use83097-44-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-3-3-7">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-2-3-9" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4557_-8-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82400-5-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82402-1-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4556_-9-2">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4555_-7-9"
+         overflow="visible"
+         id="use82397-7-5" />
+    </clipPath>
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#SVGID_4557_-8-1"
+       id="linearGradient15887"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#SVGID_4557_-8-1-0"
+       id="linearGradient15887-0"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021" />
+    <linearGradient
+       id="SVGID_4557_-8-1-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82400-5-0-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82402-1-3-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4556_-9-2-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4555_-7-9-1"
+         overflow="visible"
+         id="use82397-7-5-6" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-3-3-7-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-2-3-9-1" />
+    </clipPath>
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#SVGID_4557_-8-1-0-3"
+       id="linearGradient15887-0-3"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021" />
+    <linearGradient
+       id="SVGID_4557_-8-1-0-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82400-5-0-3-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82402-1-3-5-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4556_-9-2-5-8">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4555_-7-9-1-0"
+         overflow="visible"
+         id="use82397-7-5-6-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-3-3-7-5-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-2-3-9-1-0" />
+    </clipPath>
+  </defs>
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="0.93303299"
+     inkscape:cx="321.56814"
+     inkscape:cy="515.95091"
+     inkscape:document-units="px"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     inkscape:window-width="1535"
+     inkscape:window-height="876"
+     inkscape:window-x="65"
+     inkscape:window-y="24"
+     inkscape:window-maximized="1" />
+  <metadata
+     id="metadata104541">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1">
+    <rect
+       style="fill:none;stroke:#5e6a71;stroke-width:0.48500001;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:10;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0"
+       id="rect125410-1-2-7"
+       width="94.101288"
+       height="87.10128"
+       x="316.64056"
+       y="418.58807" />
+    <rect
+       style="fill:url(#linearGradient9382-1);fill-opacity:1;stroke:none"
+       id="rect126198-3-8-1"
+       width="108"
+       height="27"
+       x="308.19122"
+       y="494.25922" />
+    <line
+       style="fill:none;stroke:#5e6a71;stroke-width:1.0268178;stroke-miterlimit:10;stroke-dasharray:3.07565499, 3.07565499"
+       stroke-miterlimit="10"
+       x1="277.38089"
+       y1="315.69797"
+       x2="277.38089"
+       y2="533.44318"
+       id="line84244" />
+    <rect
+       style="fill:url(#linearGradient9382);fill-opacity:1;stroke:none"
+       id="rect126198-3-8"
+       width="108"
+       height="27"
+       x="139.50002"
+       y="494.28964" />
+    <g
+       transform="matrix(1.3542118,0,0,2.25,996.72494,-25.560183)"
+       id="g82158"
+       inkscape:transform-center-x="262.3321"
+       inkscape:transform-center-y="50.765304">
+      <defs
+         id="defs82160">
+        <rect
+           id="SVGID_4524_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101582">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4524_"
+           overflow="visible"
+           id="use101584" />
+      </clipPath>
+      <g
+         clip-path="url(#SVGID_4525_)"
+         id="g82166">
+        <g
+           id="g82168">
+          <defs
+             id="defs82170">
+            <rect
+               id="SVGID_4526_"
+               x="-679.67902"
+               y="293.259"
+               width="175.168"
+               height="64.251999" />
+          </defs>
+          <clipPath
+             id="clipPath101590">
+            <use
+               height="1052.3622"
+               width="744.09448"
+               y="0"
+               x="0"
+               style="overflow:visible"
+               xlink:href="#SVGID_4526_"
+               overflow="visible"
+               id="use101592" />
+          </clipPath>
+          <g
+             clip-path="url(#SVGID_4527_)"
+             id="g82176">
+            <defs
+               id="defs82178">
+              <rect
+                 id="SVGID_4528_"
+                 x="-679.57202"
+                 y="293.367"
+                 width="174.953"
+                 height="64.037003" />
+            </defs>
+            <clipPath
+               id="clipPath101597">
+              <use
+                 height="1052.3622"
+                 width="744.09448"
+                 y="0"
+                 x="0"
+                 style="overflow:visible"
+                 xlink:href="#SVGID_4528_"
+                 overflow="visible"
+                 id="use101599" />
+            </clipPath>
+            <linearGradient
+               id="linearGradient101601"
+               gradientUnits="userSpaceOnUse"
+               x1="126.8011"
+               y1="60.902401"
+               x2="127.8011"
+               y2="60.902401"
+               gradientTransform="matrix(0,64.0388,-64.0388,0,3308.0217,-7826.8223)">
+              <stop
+                 offset="0"
+                 style="stop-color:#E6E8E7"
+                 id="stop101603" />
+              <stop
+                 offset="1"
+                 style="stop-color:#FFFFFF"
+                 id="stop101605" />
+            </linearGradient>
+            <rect
+               style="fill:url(#SVGID_4530_)"
+               x="-679.57202"
+               y="293.366"
+               clip-path="url(#SVGID_4529_)"
+               width="174.953"
+               height="64.038002"
+               id="rect82189" />
+          </g>
+          <g
+             clip-path="url(#SVGID_4527_)"
+             id="g82191">
+            <defs
+               id="defs82193">
+              <rect
+                 id="SVGID_4531_"
+                 x="-679.67902"
+                 y="293.259"
+                 width="175.168"
+                 height="64.251999" />
+            </defs>
+            <clipPath
+               id="clipPath101611">
+              <use
+                 height="1052.3622"
+                 width="744.09448"
+                 y="0"
+                 x="0"
+                 style="overflow:visible"
+                 xlink:href="#SVGID_4531_"
+                 overflow="visible"
+                 id="use101613" />
+            </clipPath>
+            <rect
+               style="fill:none;stroke:#f15d57;stroke-width:0.214;stroke-miterlimit:10"
+               x="-679.57202"
+               y="293.366"
+               clip-path="url(#SVGID_4532_)"
+               stroke-miterlimit="10"
+               width="174.953"
+               height="64.038002"
+               id="rect82199" />
+          </g>
+        </g>
+      </g>
+      <g
+         clip-path="url(#SVGID_4525_)"
+         id="g82201">
+        <g
+           id="g82203">
+          <defs
+             id="defs82205">
+            <rect
+               id="SVGID_4533_"
+               x="-441.56699"
+               y="293.259"
+               width="45.873001"
+               height="64.251999" />
+          </defs>
+          <clipPath
+             id="clipPath101620">
+            <use
+               height="1052.3622"
+               width="744.09448"
+               y="0"
+               x="0"
+               style="overflow:visible"
+               xlink:href="#SVGID_4533_"
+               overflow="visible"
+               id="use101622" />
+          </clipPath>
+          <g
+             clip-path="url(#SVGID_4534_)"
+             id="g82211">
+            <defs
+               id="defs82213">
+              <rect
+                 id="SVGID_4535_"
+                 x="-441.45999"
+                 y="293.367"
+                 width="45.659"
+                 height="64.037003" />
+            </defs>
+            <clipPath
+               id="clipPath101627">
+              <use
+                 height="1052.3622"
+                 width="744.09448"
+                 y="0"
+                 x="0"
+                 style="overflow:visible"
+                 xlink:href="#SVGID_4535_"
+                 overflow="visible"
+                 id="use101629" />
+            </clipPath>
+            <linearGradient
+               id="linearGradient101631"
+               gradientUnits="userSpaceOnUse"
+               x1="126.8011"
+               y1="60.902401"
+               x2="127.8011"
+               y2="60.902401"
+               gradientTransform="matrix(0,64.0388,-64.0388,0,3481.4861,-7826.8223)">
+              <stop
+                 offset="0"
+                 style="stop-color:#E6E8E7"
+                 id="stop101633" />
+              <stop
+                 offset="1"
+                 style="stop-color:#FFFFFF"
+                 id="stop101635" />
+            </linearGradient>
+            <rect
+               style="fill:url(#SVGID_4537_)"
+               x="-441.45999"
+               y="293.366"
+               clip-path="url(#SVGID_4536_)"
+               width="45.659"
+               height="64.038002"
+               id="rect82224" />
+          </g>
+          <g
+             clip-path="url(#SVGID_4534_)"
+             id="g82226">
+            <defs
+               id="defs82228">
+              <rect
+                 id="SVGID_4538_"
+                 x="-441.56699"
+                 y="293.259"
+                 width="45.873001"
+                 height="64.251999" />
+            </defs>
+            <clipPath
+               id="clipPath101641">
+              <use
+                 height="1052.3622"
+                 width="744.09448"
+                 y="0"
+                 x="0"
+                 style="overflow:visible"
+                 xlink:href="#SVGID_4538_"
+                 overflow="visible"
+                 id="use101643" />
+            </clipPath>
+            <rect
+               style="fill:none;stroke:#f15d57;stroke-width:0.214;stroke-miterlimit:10"
+               x="-441.45999"
+               y="293.366"
+               clip-path="url(#SVGID_4539_)"
+               stroke-miterlimit="10"
+               width="45.659"
+               height="64.038002"
+               id="rect82234" />
+          </g>
+        </g>
+      </g>
+    </g>
+    <rect
+       style="fill:#f15d57"
+       x="224.23932"
+       y="574.69031"
+       width="95.793755"
+       height="3.0712502"
+       id="rect82238"
+       inkscape:transform-center-x="437.28349"
+       inkscape:transform-center-y="-79.564828" />
+    <line
+       style="fill:none;stroke:#5e6a71;stroke-width:0.48150003;stroke-miterlimit:10;stroke-dasharray:1.44225, 1.44225"
+       stroke-miterlimit="10"
+       x1="272.20041"
+       y1="577.34521"
+       x2="272.20041"
+       y2="631.72772"
+       id="line82244"
+       inkscape:transform-center-x="437.21928"
+       inkscape:transform-center-y="-51.254289" />
+    <polyline
+       style="fill:none;stroke:#5e6a71;stroke-width:0.214;stroke-miterlimit:10;stroke-dasharray:0.641, 0.641"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       stroke-miterlimit="10"
+       points="  -603.212,292.128 -603.212,283.832 -582.187,283.832 -582.187,292.128 "
+       id="polyline82246"
+       inkscape:transform-center-x="437.22048"
+       inkscape:transform-center-y="-33.395941" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82281"
+       inkscape:transform-center-x="437.2835"
+       inkscape:transform-center-y="-95.804196">
+      <defs
+         id="defs82283">
+        <polygon
+           id="SVGID_4543_"
+           points="-613.901,253.224 -613.908,267.353 -571.554,267.32 -571.547,253.133 -593.085,258.293 " />
+      </defs>
+      <clipPath
+         id="clipPath101684">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4543_"
+           overflow="visible"
+           id="use101686" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101688"
+         gradientUnits="userSpaceOnUse"
+         x1="76.3899"
+         y1="97.785698"
+         x2="77.3899"
+         y2="97.785698"
+         gradientTransform="matrix(0,-19.5733,-19.5733,0,1321.2666,1762.5587)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101690" />
+        <stop
+           offset="0.0507"
+           style="stop-color:#68B4BF"
+           id="stop101692" />
+        <stop
+           offset="0.2116"
+           style="stop-color:#96C6CF"
+           id="stop101694" />
+        <stop
+           offset="0.3736"
+           style="stop-color:#BAD8DD"
+           id="stop101696" />
+        <stop
+           offset="0.5342"
+           style="stop-color:#D7E6EA"
+           id="stop101698" />
+        <stop
+           offset="0.693"
+           style="stop-color:#EBF2F3"
+           id="stop101700" />
+        <stop
+           offset="0.8492"
+           style="stop-color:#F8FBFB"
+           id="stop101702" />
+        <stop
+           offset="1"
+           style="stop-color:#FFFFFF"
+           id="stop101704" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4545_)"
+         x="-613.90802"
+         y="253.133"
+         clip-path="url(#SVGID_4544_)"
+         width="42.361"
+         height="14.22"
+         id="rect82306" />
+    </g>
+    <text
+       y="567.99066"
+       x="255.24185"
+       id="text84246"
+       inkscape:transform-center-x="438.70839"
+       inkscape:transform-center-y="-90.601429"
+       style="font-size:27px">
+      <tspan
+         style="font-size:8px;fill:#37424b;font-family:ApexSans-Medium"
+         x="255.24185"
+         y="567.99066"
+         font-size="3.4203"
+         id="tspan84248">US-EAST</tspan>
+    </text>
+    <path
+       style="fill:#e6e8e8"
+       inkscape:connector-curvature="0"
+       d="m 319.54934,544.29505 -0.0157,31.37625 -94.815,0.072 0.0157,-31.2435 46.48275,11.31975 0.1125,0.027 0.11475,-0.027 48.105,-11.5245 z m 0.4815,-0.60975 -48.699,11.66625 -47.079,-11.46375 -0.0135,32.337 95.77575,-0.0743 0.0158,-32.46525 z"
+       id="path82308"
+       inkscape:transform-center-x="437.28456"
+       inkscape:transform-center-y="-95.835733" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82339"
+       inkscape:transform-center-x="449.81598"
+       inkscape:transform-center-y="45.666809">
+      <defs
+         id="defs82341">
+        <rect
+           id="SVGID_4549_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101738">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4549_"
+           overflow="visible"
+           id="use101740" />
+      </clipPath>
+      <path
+         style="fill:#f15d57"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4550_)"
+         d="m -602.031,326.467 c 0,0.212 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.173 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+         id="path82347" />
+      <path
+         style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4550_)"
+         stroke-miterlimit="10"
+         d="m -602.031,326.467 c 0,0.212 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.173 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+         id="path82349" />
+    </g>
+    <text
+       y="704.88873"
+       x="275.14355"
+       style="font-size:9.75217533px;fill:#ffffff;font-family:ApexSans-Medium"
+       font-size="4.3343"
+       id="text82351"
+       inkscape:transform-center-x="450.06672"
+       inkscape:transform-center-y="45.543285">M</text>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82353"
+       inkscape:transform-center-x="503.22873"
+       inkscape:transform-center-y="71.647559">
+      <defs
+         id="defs82355">
+        <rect
+           id="SVGID_4551_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101748">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4551_"
+           overflow="visible"
+           id="use101750" />
+      </clipPath>
+      <path
+         style="fill:#f15d57"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4552_)"
+         d="m -625.77,338.014 c 0,0.213 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.172 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+         id="path82361" />
+      <path
+         style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4552_)"
+         stroke-miterlimit="10"
+         d="m -625.77,338.014 c 0,0.213 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.172 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+         id="path82363" />
+    </g>
+    <text
+       y="730.87036"
+       x="201.73129"
+       style="font-size:9.75217533px;fill:#ffffff;font-family:ApexSans-Medium"
+       font-size="4.3343"
+       id="text82365"
+       inkscape:transform-center-x="503.47899"
+       inkscape:transform-center-y="71.524915">M</text>
+    <rect
+       style="fill:#ffffff"
+       x="87.659889"
+       y="664.59583"
+       width="21.581999"
+       height="21.584251"
+       id="rect82381"
+       inkscape:transform-center-x="610.96881"
+       inkscape:transform-center-y="19.597191" />
+    <rect
+       style="fill:none;stroke:#81d0db;stroke-width:0.96300006;stroke-miterlimit:10"
+       x="87.659889"
+       y="664.59583"
+       stroke-miterlimit="10"
+       width="21.581999"
+       height="21.584251"
+       id="rect82383"
+       inkscape:transform-center-x="610.96881"
+       inkscape:transform-center-y="19.597191" />
+    <rect
+       style="fill:#81d0db"
+       x="89.169548"
+       y="666.1123"
+       width="18.557999"
+       height="18.553501"
+       id="rect82385"
+       inkscape:transform-center-x="610.97115"
+       inkscape:transform-center-y="19.598295" />
+    <rect
+       style="fill:#ffffff"
+       x="90.654625"
+       y="667.59052"
+       width="15.592501"
+       height="15.594751"
+       id="rect82387"
+       inkscape:transform-center-x="610.96883"
+       inkscape:transform-center-y="19.597131" />
+    <rect
+       style="fill:#ffffff"
+       x="93.307411"
+       y="670.24774"
+       width="10.282501"
+       height="10.278"
+       id="rect82389"
+       inkscape:transform-center-x="610.97103"
+       inkscape:transform-center-y="19.595982" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82391"
+       inkscape:transform-center-x="610.97111"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82393">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4555_"
+           d="m -672.614,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101773">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4555_"
+           overflow="visible"
+           id="use101775" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101777"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.46021"
+         x2="256.3761"
+         y2="-148.46021"
+         gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101779" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101781" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4557_)"
+         x="-672.61401"
+         y="308.841"
+         clip-path="url(#SVGID_4556_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82404" />
+    </g>
+    <rect
+       id="rect82451"
+       height="9.5930004"
+       width="9.592"
+       clip-path="url(#SVGID_4564_)"
+       y="306.73599"
+       x="-650.77399"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <rect
+       id="rect82453"
+       height="9.5930004"
+       width="9.592"
+       stroke-miterlimit="10"
+       clip-path="url(#SVGID_4564_)"
+       y="306.73599"
+       x="-650.77399"
+       style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <rect
+       id="rect82455"
+       height="8.2460003"
+       width="8.2480001"
+       clip-path="url(#SVGID_4564_)"
+       y="307.41"
+       x="-650.10303"
+       style="fill:#81d0db"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <rect
+       id="rect82457"
+       height="6.9310002"
+       width="6.9299998"
+       clip-path="url(#SVGID_4564_)"
+       y="308.06699"
+       x="-649.44299"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <rect
+       id="rect82459"
+       height="4.5679998"
+       width="4.5700002"
+       clip-path="url(#SVGID_4564_)"
+       y="309.24799"
+       x="-648.26398"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82461"
+       inkscape:transform-center-x="557.09937"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82463">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4565_"
+           d="m -648.671,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101827">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4565_"
+           overflow="visible"
+           id="use101829" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101831"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.4601"
+         x2="256.3761"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101833" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101835" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4567_)"
+         x="-648.67102"
+         y="308.841"
+         clip-path="url(#SVGID_4566_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82474" />
+    </g>
+    <path
+       id="path82484"
+       d="m -646.611,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+       clip-path="url(#SVGID_4569_)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <rect
+       id="rect82486"
+       height="9.5930004"
+       width="9.592"
+       clip-path="url(#SVGID_4569_)"
+       y="306.73599"
+       x="-638.80298"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <rect
+       id="rect82488"
+       height="9.5930004"
+       width="9.592"
+       stroke-miterlimit="10"
+       clip-path="url(#SVGID_4569_)"
+       y="306.73599"
+       x="-638.80298"
+       style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <rect
+       id="rect82490"
+       height="8.2460003"
+       width="8.2480001"
+       clip-path="url(#SVGID_4569_)"
+       y="307.41"
+       x="-638.13202"
+       style="fill:#81d0db"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <rect
+       id="rect82492"
+       height="6.9310002"
+       width="6.9299998"
+       clip-path="url(#SVGID_4569_)"
+       y="308.06699"
+       x="-637.47198"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <rect
+       id="rect82494"
+       height="4.5679998"
+       width="4.5700002"
+       clip-path="url(#SVGID_4569_)"
+       y="309.24799"
+       x="-636.29199"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82496"
+       inkscape:transform-center-x="530.16231"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82498">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4570_"
+           d="m -636.699,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101854">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4570_"
+           overflow="visible"
+           id="use101856" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101858"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.46001"
+         x2="256.3761"
+         y2="-148.46001"
+         gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101860" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101862" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4572_)"
+         x="-636.69897"
+         y="308.841"
+         clip-path="url(#SVGID_4571_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82509" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82511"
+       inkscape:transform-center-x="511.76858"
+       inkscape:transform-center-y="19.597166">
+      <defs
+         id="defs82513">
+        <rect
+           id="SVGID_4573_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101868">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4573_"
+           overflow="visible"
+           id="use101870" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4574_)"
+         d="m -634.639,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82519" />
+      <rect
+         style="fill:#ffffff"
+         x="-626.83099"
+         y="306.73599"
+         clip-path="url(#SVGID_4574_)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82521" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-626.83099"
+         y="306.73599"
+         clip-path="url(#SVGID_4574_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82523" />
+      <rect
+         style="fill:#81d0db"
+         x="-626.15997"
+         y="307.41"
+         clip-path="url(#SVGID_4574_)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82525" />
+      <rect
+         style="fill:#ffffff"
+         x="-625.5"
+         y="308.06699"
+         clip-path="url(#SVGID_4574_)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82527" />
+      <rect
+         style="fill:#ffffff"
+         x="-624.32098"
+         y="309.24799"
+         clip-path="url(#SVGID_4574_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82529" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82531"
+       inkscape:transform-center-x="503.22762"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82533">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4575_"
+           d="m -624.728,311.531 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.205 -2.693,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101881">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4575_"
+           overflow="visible"
+           id="use101883" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101885"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.4601"
+         x2="256.3761"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101887" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101889" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4577_)"
+         x="-624.72803"
+         y="308.841"
+         clip-path="url(#SVGID_4576_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82544" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82546"
+       inkscape:transform-center-x="484.83387"
+       inkscape:transform-center-y="19.597166">
+      <defs
+         id="defs82548">
+        <rect
+           id="SVGID_4578_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101895">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4578_"
+           overflow="visible"
+           id="use101897" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4579_)"
+         d="m -622.668,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82554" />
+      <rect
+         style="fill:#ffffff"
+         x="-614.85999"
+         y="306.73599"
+         clip-path="url(#SVGID_4579_)"
+         width="9.5930004"
+         height="9.5930004"
+         id="rect82556" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-614.86102"
+         y="306.73599"
+         clip-path="url(#SVGID_4579_)"
+         stroke-miterlimit="10"
+         width="9.5930004"
+         height="9.5930004"
+         id="rect82558" />
+      <rect
+         style="fill:#81d0db"
+         x="-614.18799"
+         y="307.41"
+         clip-path="url(#SVGID_4579_)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82560" />
+      <rect
+         style="fill:#ffffff"
+         x="-613.52899"
+         y="308.06699"
+         clip-path="url(#SVGID_4579_)"
+         width="6.9310002"
+         height="6.9310002"
+         id="rect82562" />
+      <rect
+         style="fill:#ffffff"
+         x="-612.349"
+         y="309.24799"
+         clip-path="url(#SVGID_4579_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82564" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82566"
+       inkscape:transform-center-x="476.2917"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82568">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4580_"
+           d="m -612.756,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101908">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4580_"
+           overflow="visible"
+           id="use101910" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101912"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.4601"
+         x2="256.3761"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101914" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101916" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4582_)"
+         x="-612.75598"
+         y="308.841"
+         clip-path="url(#SVGID_4581_)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect82579" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82581"
+       inkscape:transform-center-x="457.89686"
+       inkscape:transform-center-y="19.597166">
+      <defs
+         id="defs82583">
+        <rect
+           id="SVGID_4583_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101922">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4583_"
+           overflow="visible"
+           id="use101924" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4584_)"
+         d="m -610.696,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82589" />
+      <rect
+         style="fill:#ffffff"
+         x="-602.888"
+         y="306.73599"
+         clip-path="url(#SVGID_4584_)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82591" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-602.888"
+         y="306.73599"
+         clip-path="url(#SVGID_4584_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82593" />
+      <rect
+         style="fill:#81d0db"
+         x="-602.21698"
+         y="307.41"
+         clip-path="url(#SVGID_4584_)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82595" />
+      <rect
+         style="fill:#ffffff"
+         x="-601.55701"
+         y="308.06699"
+         clip-path="url(#SVGID_4584_)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82597" />
+      <rect
+         style="fill:#ffffff"
+         x="-600.37799"
+         y="309.24799"
+         clip-path="url(#SVGID_4584_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82599" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82601"
+       inkscape:transform-center-x="449.35694"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82603">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4585_"
+           d="m -600.785,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.203,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101935">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4585_"
+           overflow="visible"
+           id="use101937" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101939"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.46021"
+         x2="256.3761"
+         y2="-148.46021"
+         gradientTransform="matrix(0,5.3836,5.3836,0,201.1574,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101941" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101943" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4587_)"
+         x="-600.78497"
+         y="308.841"
+         clip-path="url(#SVGID_4586_)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect82614" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82616"
+       inkscape:transform-center-x="430.96208"
+       inkscape:transform-center-y="19.597166">
+      <defs
+         id="defs82618">
+        <rect
+           id="SVGID_4588_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101949">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4588_"
+           overflow="visible"
+           id="use101951" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4589_)"
+         d="m -598.725,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82624" />
+      <rect
+         style="fill:#ffffff"
+         x="-590.91699"
+         y="306.73599"
+         clip-path="url(#SVGID_4589_)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82626" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-590.91699"
+         y="306.73599"
+         clip-path="url(#SVGID_4589_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82628" />
+      <rect
+         style="fill:#81d0db"
+         x="-590.24597"
+         y="307.41"
+         clip-path="url(#SVGID_4589_)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82630" />
+      <rect
+         style="fill:#ffffff"
+         x="-589.586"
+         y="308.06699"
+         clip-path="url(#SVGID_4589_)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82632" />
+      <rect
+         style="fill:#ffffff"
+         x="-588.40698"
+         y="309.24799"
+         clip-path="url(#SVGID_4589_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82634" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82636"
+       inkscape:transform-center-x="422.41995"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82638">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4590_"
+           d="m -588.813,311.531 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.205 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101962">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4590_"
+           overflow="visible"
+           id="use101964" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101966"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.4601"
+         x2="256.3761"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101968" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101970" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4592_)"
+         x="-588.81299"
+         y="308.841"
+         clip-path="url(#SVGID_4591_)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect82649" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82861"
+       inkscape:transform-center-x="440.81033"
+       inkscape:transform-center-y="37.558931">
+      <defs
+         id="defs82863">
+        <rect
+           id="SVGID_4623_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102138">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4623_"
+           overflow="visible"
+           id="use102140" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4624_)"
+         d="m -514.925,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82869" />
+      <rect
+         style="fill:#ffffff"
+         x="-674.71698"
+         y="318.323"
+         clip-path="url(#SVGID_4624_)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82871" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-674.71698"
+         y="318.323"
+         clip-path="url(#SVGID_4624_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82873" />
+      <rect
+         style="fill:#81d0db"
+         x="-674.04602"
+         y="318.99701"
+         clip-path="url(#SVGID_4624_)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82875" />
+      <rect
+         style="fill:#ffffff"
+         x="-673.38599"
+         y="319.65399"
+         clip-path="url(#SVGID_4624_)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82877" />
+      <rect
+         style="fill:#ffffff"
+         x="-672.20697"
+         y="320.83499"
+         clip-path="url(#SVGID_4624_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82879" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82881"
+       inkscape:transform-center-x="610.97111"
+       inkscape:transform-center-y="45.671291">
+      <defs
+         id="defs82883">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4625_"
+           d="m -672.614,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath102151">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4625_"
+           overflow="visible"
+           id="use102153" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102155"
+         gradientUnits="userSpaceOnUse"
+         x1="255.37621"
+         y1="-148.46021"
+         x2="256.37631"
+         y2="-148.46021"
+         gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1054.4158)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102157" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102159" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4627_)"
+         x="-672.61401"
+         y="320.42899"
+         clip-path="url(#SVGID_4626_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82894" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82896"
+       inkscape:transform-center-x="592.57619"
+       inkscape:transform-center-y="45.667929">
+      <defs
+         id="defs82898">
+        <rect
+           id="SVGID_4628_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102165">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4628_"
+           overflow="visible"
+           id="use102167" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4629_)"
+         d="m -670.553,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82904" />
+      <rect
+         style="fill:#ffffff"
+         x="-662.745"
+         y="318.323"
+         clip-path="url(#SVGID_4629_)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82906" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-662.74597"
+         y="318.323"
+         clip-path="url(#SVGID_4629_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82908" />
+      <rect
+         style="fill:#81d0db"
+         x="-662.07397"
+         y="318.99701"
+         clip-path="url(#SVGID_4629_)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82910" />
+      <rect
+         style="fill:#ffffff"
+         x="-661.414"
+         y="319.65399"
+         clip-path="url(#SVGID_4629_)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82912" />
+      <rect
+         style="fill:#ffffff"
+         x="-660.23499"
+         y="320.83499"
+         clip-path="url(#SVGID_4629_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82914" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82916"
+       inkscape:transform-center-x="584.03412"
+       inkscape:transform-center-y="45.671291">
+      <defs
+         id="defs82918">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4630_"
+           d="m -660.642,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath102178">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4630_"
+           overflow="visible"
+           id="use102180" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102182"
+         gradientUnits="userSpaceOnUse"
+         x1="255.37621"
+         y1="-148.4601"
+         x2="256.37631"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1054.4158)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102184" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102186" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4632_)"
+         x="-660.64203"
+         y="320.42899"
+         clip-path="url(#SVGID_4631_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82929" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82931"
+       inkscape:transform-center-x="565.64033"
+       inkscape:transform-center-y="45.667929">
+      <defs
+         id="defs82933">
+        <rect
+           id="SVGID_4633_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102192">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4633_"
+           overflow="visible"
+           id="use102194" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4634_)"
+         d="m -658.582,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82939" />
+      <rect
+         style="fill:#ffffff"
+         x="-650.77399"
+         y="318.323"
+         clip-path="url(#SVGID_4634_)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82941" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-650.77399"
+         y="318.323"
+         clip-path="url(#SVGID_4634_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82943" />
+      <rect
+         style="fill:#81d0db"
+         x="-650.10303"
+         y="318.99701"
+         clip-path="url(#SVGID_4634_)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82945" />
+      <rect
+         style="fill:#ffffff"
+         x="-649.44299"
+         y="319.65399"
+         clip-path="url(#SVGID_4634_)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82947" />
+      <rect
+         style="fill:#ffffff"
+         x="-648.26398"
+         y="320.83499"
+         clip-path="url(#SVGID_4634_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82949" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82951"
+       inkscape:transform-center-x="557.09937"
+       inkscape:transform-center-y="45.671291">
+      <defs
+         id="defs82953">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4635_"
+           d="m -648.671,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath102205">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4635_"
+           overflow="visible"
+           id="use102207" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102209"
+         gradientUnits="userSpaceOnUse"
+         x1="255.37621"
+         y1="-148.4601"
+         x2="256.37631"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1054.4158)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102211" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102213" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4637_)"
+         x="-648.67102"
+         y="320.42899"
+         clip-path="url(#SVGID_4636_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82964" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82966"
+       inkscape:transform-center-x="538.70558"
+       inkscape:transform-center-y="45.667929">
+      <defs
+         id="defs82968">
+        <rect
+           id="SVGID_4638_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102219">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4638_"
+           overflow="visible"
+           id="use102221" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4639_)"
+         d="m -646.611,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82974" />
+      <rect
+         style="fill:#ffffff"
+         x="-638.80298"
+         y="318.323"
+         clip-path="url(#SVGID_4639_)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82976" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-638.80298"
+         y="318.323"
+         clip-path="url(#SVGID_4639_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82978" />
+      <rect
+         style="fill:#81d0db"
+         x="-638.13202"
+         y="318.99701"
+         clip-path="url(#SVGID_4639_)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82980" />
+      <rect
+         style="fill:#ffffff"
+         x="-637.47198"
+         y="319.65399"
+         clip-path="url(#SVGID_4639_)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82982" />
+      <rect
+         style="fill:#ffffff"
+         x="-636.29199"
+         y="320.83499"
+         clip-path="url(#SVGID_4639_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82984" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82986"
+       inkscape:transform-center-x="530.16231"
+       inkscape:transform-center-y="45.671291">
+      <defs
+         id="defs82988">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4640_"
+           d="m -636.699,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath102232">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4640_"
+           overflow="visible"
+           id="use102234" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102236"
+         gradientUnits="userSpaceOnUse"
+         x1="255.37621"
+         y1="-148.46001"
+         x2="256.37631"
+         y2="-148.46001"
+         gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1054.4158)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102238" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102240" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4642_)"
+         x="-636.69897"
+         y="320.42899"
+         clip-path="url(#SVGID_4641_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82999" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83001"
+       inkscape:transform-center-x="511.76858"
+       inkscape:transform-center-y="45.667929">
+      <defs
+         id="defs83003">
+        <rect
+           id="SVGID_4643_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102246">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4643_"
+           overflow="visible"
+           id="use102248" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4644_)"
+         d="m -634.639,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83009" />
+      <rect
+         style="fill:#ffffff"
+         x="-626.83099"
+         y="318.323"
+         clip-path="url(#SVGID_4644_)"
+         width="9.592"
+         height="9.5930004"
+         id="rect83011" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-626.83099"
+         y="318.323"
+         clip-path="url(#SVGID_4644_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect83013" />
+      <rect
+         style="fill:#81d0db"
+         x="-626.15997"
+         y="318.99701"
+         clip-path="url(#SVGID_4644_)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect83015" />
+      <rect
+         style="fill:#ffffff"
+         x="-625.5"
+         y="319.65399"
+         clip-path="url(#SVGID_4644_)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect83017" />
+      <rect
+         style="fill:#ffffff"
+         x="-624.32098"
+         y="320.83499"
+         clip-path="url(#SVGID_4644_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83019" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83021"
+       inkscape:transform-center-x="503.22762"
+       inkscape:transform-center-y="45.671291">
+      <defs
+         id="defs83023">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4645_"
+           d="m -624.728,323.119 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.204 -2.693,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath102259">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4645_"
+           overflow="visible"
+           id="use102261" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102263"
+         gradientUnits="userSpaceOnUse"
+         x1="255.37621"
+         y1="-148.4601"
+         x2="256.37631"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1054.4158)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102265" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102267" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4647_)"
+         x="-624.72803"
+         y="320.42899"
+         clip-path="url(#SVGID_4646_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect83034" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83036"
+       inkscape:transform-center-x="484.83387"
+       inkscape:transform-center-y="45.667929">
+      <defs
+         id="defs83038">
+        <rect
+           id="SVGID_4648_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102273">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4648_"
+           overflow="visible"
+           id="use102275" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4649_)"
+         d="m -622.668,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83044" />
+      <rect
+         style="fill:#ffffff"
+         x="-614.85999"
+         y="318.323"
+         clip-path="url(#SVGID_4649_)"
+         width="9.5930004"
+         height="9.5930004"
+         id="rect83046" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-614.86102"
+         y="318.323"
+         clip-path="url(#SVGID_4649_)"
+         stroke-miterlimit="10"
+         width="9.5930004"
+         height="9.5930004"
+         id="rect83048" />
+      <rect
+         style="fill:#81d0db"
+         x="-614.18799"
+         y="318.99701"
+         clip-path="url(#SVGID_4649_)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect83050" />
+      <rect
+         style="fill:#ffffff"
+         x="-613.52899"
+         y="319.65399"
+         clip-path="url(#SVGID_4649_)"
+         width="6.9310002"
+         height="6.9310002"
+         id="rect83052" />
+      <rect
+         style="fill:#ffffff"
+         x="-612.349"
+         y="320.83499"
+         clip-path="url(#SVGID_4649_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83054" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83056"
+       inkscape:transform-center-x="476.2917"
+       inkscape:transform-center-y="45.671291">
+      <defs
+         id="defs83058">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4650_"
+           d="m -612.756,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath102286">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4650_"
+           overflow="visible"
+           id="use102288" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102290"
+         gradientUnits="userSpaceOnUse"
+         x1="255.37621"
+         y1="-148.4601"
+         x2="256.37631"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1054.4158)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102292" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102294" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4652_)"
+         x="-612.75598"
+         y="320.42899"
+         clip-path="url(#SVGID_4651_)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect83069" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83071"
+       inkscape:transform-center-x="444.42945"
+       inkscape:transform-center-y="45.667929">
+      <defs
+         id="defs83073">
+        <rect
+           id="SVGID_4653_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102300">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4653_"
+           overflow="visible"
+           id="use102302" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4654_)"
+         d="m -610.696,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83079" />
+      <rect
+         style="fill:#ffffff"
+         x="-590.91699"
+         y="318.323"
+         clip-path="url(#SVGID_4654_)"
+         width="9.592"
+         height="9.5930004"
+         id="rect83081" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-590.91699"
+         y="318.323"
+         clip-path="url(#SVGID_4654_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect83083" />
+      <rect
+         style="fill:#81d0db"
+         x="-590.24597"
+         y="318.99701"
+         clip-path="url(#SVGID_4654_)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect83085" />
+      <rect
+         style="fill:#ffffff"
+         x="-589.586"
+         y="319.65399"
+         clip-path="url(#SVGID_4654_)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect83087" />
+      <rect
+         style="fill:#ffffff"
+         x="-588.40698"
+         y="320.83499"
+         clip-path="url(#SVGID_4654_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83089" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83091"
+       inkscape:transform-center-x="422.41995"
+       inkscape:transform-center-y="45.671291">
+      <defs
+         id="defs83093">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4655_"
+           d="m -588.813,323.119 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.204 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath102313">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4655_"
+           overflow="visible"
+           id="use102315" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102317"
+         gradientUnits="userSpaceOnUse"
+         x1="255.37621"
+         y1="-148.4601"
+         x2="256.37631"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102319" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102321" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4657_)"
+         x="-588.81299"
+         y="320.42899"
+         clip-path="url(#SVGID_4656_)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect83104" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83316"
+       inkscape:transform-center-x="440.81033"
+       inkscape:transform-center-y="63.583551">
+      <defs
+         id="defs83318">
+        <rect
+           id="SVGID_4688_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102489">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4688_"
+           overflow="visible"
+           id="use102491" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4689_)"
+         d="m -514.925,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83324" />
+      <rect
+         style="fill:#ffffff"
+         x="-674.71698"
+         y="329.87"
+         clip-path="url(#SVGID_4689_)"
+         width="9.592"
+         height="9.592"
+         id="rect83326" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-674.71698"
+         y="329.87"
+         clip-path="url(#SVGID_4689_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83328" />
+      <rect
+         style="fill:#81d0db"
+         x="-674.04602"
+         y="330.543"
+         clip-path="url(#SVGID_4689_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83330" />
+      <rect
+         style="fill:#ffffff"
+         x="-673.38599"
+         y="331.20099"
+         clip-path="url(#SVGID_4689_)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83332" />
+      <rect
+         style="fill:#ffffff"
+         x="-672.20697"
+         y="332.38199"
+         clip-path="url(#SVGID_4689_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83334" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83336"
+       inkscape:transform-center-x="610.97111"
+       inkscape:transform-center-y="71.649806">
+      <defs
+         id="defs83338">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4690_"
+           d="m -672.614,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102502">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4690_"
+           overflow="visible"
+           id="use102504" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102506"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4301"
+         x2="256.3544"
+         y2="-148.4301"
+         gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102508" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102510" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4692_)"
+         x="-672.61401"
+         y="331.974"
+         clip-path="url(#SVGID_4691_)"
+         width="5.3839998"
+         height="5.3860002"
+         id="rect83349" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83351"
+       inkscape:transform-center-x="592.57619"
+       inkscape:transform-center-y="71.647547">
+      <defs
+         id="defs83353">
+        <rect
+           id="SVGID_4693_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102516">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4693_"
+           overflow="visible"
+           id="use102518" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4694_)"
+         d="m -670.553,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83359" />
+      <rect
+         style="fill:#ffffff"
+         x="-662.745"
+         y="329.87"
+         clip-path="url(#SVGID_4694_)"
+         width="9.592"
+         height="9.592"
+         id="rect83361" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-662.74597"
+         y="329.87"
+         clip-path="url(#SVGID_4694_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83363" />
+      <rect
+         style="fill:#81d0db"
+         x="-662.07397"
+         y="330.543"
+         clip-path="url(#SVGID_4694_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83365" />
+      <rect
+         style="fill:#ffffff"
+         x="-661.414"
+         y="331.20099"
+         clip-path="url(#SVGID_4694_)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83367" />
+      <rect
+         style="fill:#ffffff"
+         x="-660.23499"
+         y="332.38199"
+         clip-path="url(#SVGID_4694_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83369" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83371"
+       inkscape:transform-center-x="584.03412"
+       inkscape:transform-center-y="71.648681">
+      <defs
+         id="defs83373">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4695_"
+           d="m -660.642,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102529">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4695_"
+           overflow="visible"
+           id="use102531" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102533"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4299"
+         x2="256.35419"
+         y2="-148.4299"
+         gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102535" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102537" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4697_)"
+         x="-660.64203"
+         y="331.974"
+         clip-path="url(#SVGID_4696_)"
+         width="5.3839998"
+         height="5.3850002"
+         id="rect83384" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83386"
+       inkscape:transform-center-x="565.64033"
+       inkscape:transform-center-y="71.647547">
+      <defs
+         id="defs83388">
+        <rect
+           id="SVGID_4698_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102543">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4698_"
+           overflow="visible"
+           id="use102545" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4699_)"
+         d="m -658.582,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83394" />
+      <rect
+         style="fill:#ffffff"
+         x="-650.77399"
+         y="329.87"
+         clip-path="url(#SVGID_4699_)"
+         width="9.592"
+         height="9.592"
+         id="rect83396" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-650.77399"
+         y="329.87"
+         clip-path="url(#SVGID_4699_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83398" />
+      <rect
+         style="fill:#81d0db"
+         x="-650.10303"
+         y="330.543"
+         clip-path="url(#SVGID_4699_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83400" />
+      <rect
+         style="fill:#ffffff"
+         x="-649.44299"
+         y="331.20099"
+         clip-path="url(#SVGID_4699_)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83402" />
+      <rect
+         style="fill:#ffffff"
+         x="-648.26398"
+         y="332.38199"
+         clip-path="url(#SVGID_4699_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83404" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83406"
+       inkscape:transform-center-x="557.09937"
+       inkscape:transform-center-y="71.649806">
+      <defs
+         id="defs83408">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4700_"
+           d="m -648.671,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102556">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4700_"
+           overflow="visible"
+           id="use102558" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102560"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3544"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102562" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102564" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4702_)"
+         x="-648.67102"
+         y="331.974"
+         clip-path="url(#SVGID_4701_)"
+         width="5.3839998"
+         height="5.3860002"
+         id="rect83419" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83421"
+       inkscape:transform-center-x="538.70558"
+       inkscape:transform-center-y="71.647547">
+      <defs
+         id="defs83423">
+        <rect
+           id="SVGID_4703_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102570">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4703_"
+           overflow="visible"
+           id="use102572" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4704_)"
+         d="m -646.611,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83429" />
+      <rect
+         style="fill:#ffffff"
+         x="-638.80298"
+         y="329.87"
+         clip-path="url(#SVGID_4704_)"
+         width="9.592"
+         height="9.592"
+         id="rect83431" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-638.80298"
+         y="329.87"
+         clip-path="url(#SVGID_4704_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83433" />
+      <rect
+         style="fill:#81d0db"
+         x="-638.13202"
+         y="330.543"
+         clip-path="url(#SVGID_4704_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83435" />
+      <rect
+         style="fill:#ffffff"
+         x="-637.47198"
+         y="331.20099"
+         clip-path="url(#SVGID_4704_)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83437" />
+      <rect
+         style="fill:#ffffff"
+         x="-636.29199"
+         y="332.38199"
+         clip-path="url(#SVGID_4704_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83439" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83441"
+       inkscape:transform-center-x="530.16231"
+       inkscape:transform-center-y="71.649806">
+      <defs
+         id="defs83443">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4705_"
+           d="m -636.699,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102583">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4705_"
+           overflow="visible"
+           id="use102585" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102587"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4299"
+         x2="256.3544"
+         y2="-148.4299"
+         gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102589" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102591" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4707_)"
+         x="-636.69897"
+         y="331.974"
+         clip-path="url(#SVGID_4706_)"
+         width="5.3839998"
+         height="5.3860002"
+         id="rect83454" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83456"
+       inkscape:transform-center-x="498.30125"
+       inkscape:transform-center-y="71.647547">
+      <defs
+         id="defs83458">
+        <rect
+           id="SVGID_4708_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102597">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4708_"
+           overflow="visible"
+           id="use102599" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4709_)"
+         d="m -634.639,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83464" />
+      <rect
+         style="fill:#ffffff"
+         x="-614.85999"
+         y="329.87"
+         clip-path="url(#SVGID_4709_)"
+         width="9.5930004"
+         height="9.592"
+         id="rect83466" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-614.86102"
+         y="329.87"
+         clip-path="url(#SVGID_4709_)"
+         stroke-miterlimit="10"
+         width="9.5930004"
+         height="9.592"
+         id="rect83468" />
+      <rect
+         style="fill:#81d0db"
+         x="-614.18799"
+         y="330.543"
+         clip-path="url(#SVGID_4709_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83470" />
+      <rect
+         style="fill:#ffffff"
+         x="-613.52899"
+         y="331.20099"
+         clip-path="url(#SVGID_4709_)"
+         width="6.9310002"
+         height="6.9299998"
+         id="rect83472" />
+      <rect
+         style="fill:#ffffff"
+         x="-612.349"
+         y="332.38199"
+         clip-path="url(#SVGID_4709_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83474" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83476"
+       inkscape:transform-center-x="476.2917"
+       inkscape:transform-center-y="71.649806">
+      <defs
+         id="defs83478">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4710_"
+           d="m -612.756,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102610">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4710_"
+           overflow="visible"
+           id="use102612" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102614"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3544"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102616" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102618" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4712_)"
+         x="-612.75598"
+         y="331.974"
+         clip-path="url(#SVGID_4711_)"
+         width="5.3829999"
+         height="5.3860002"
+         id="rect83489" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83491"
+       inkscape:transform-center-x="457.89686"
+       inkscape:transform-center-y="71.647547">
+      <defs
+         id="defs83493">
+        <rect
+           id="SVGID_4713_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102624">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4713_"
+           overflow="visible"
+           id="use102626" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4714_)"
+         d="m -610.696,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83499" />
+      <rect
+         style="fill:#ffffff"
+         x="-602.888"
+         y="329.87"
+         clip-path="url(#SVGID_4714_)"
+         width="9.592"
+         height="9.592"
+         id="rect83501" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-602.888"
+         y="329.87"
+         clip-path="url(#SVGID_4714_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83503" />
+      <rect
+         style="fill:#81d0db"
+         x="-602.21698"
+         y="330.543"
+         clip-path="url(#SVGID_4714_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83505" />
+      <rect
+         style="fill:#ffffff"
+         x="-601.55701"
+         y="331.20099"
+         clip-path="url(#SVGID_4714_)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83507" />
+      <rect
+         style="fill:#ffffff"
+         x="-600.37799"
+         y="332.38199"
+         clip-path="url(#SVGID_4714_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83509" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83511"
+       inkscape:transform-center-x="449.35694"
+       inkscape:transform-center-y="71.649806">
+      <defs
+         id="defs83513">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4715_"
+           d="m -600.785,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102637">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4715_"
+           overflow="visible"
+           id="use102639" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102641"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4301"
+         x2="256.3544"
+         y2="-148.4301"
+         gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102643" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102645" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4717_)"
+         x="-600.78497"
+         y="331.974"
+         clip-path="url(#SVGID_4716_)"
+         width="5.3829999"
+         height="5.3860002"
+         id="rect83524" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83526"
+       inkscape:transform-center-x="430.96208"
+       inkscape:transform-center-y="71.647547">
+      <defs
+         id="defs83528">
+        <rect
+           id="SVGID_4718_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102651">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4718_"
+           overflow="visible"
+           id="use102653" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4719_)"
+         d="m -598.725,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83534" />
+      <rect
+         style="fill:#ffffff"
+         x="-590.91699"
+         y="329.87"
+         clip-path="url(#SVGID_4719_)"
+         width="9.592"
+         height="9.592"
+         id="rect83536" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-590.91699"
+         y="329.87"
+         clip-path="url(#SVGID_4719_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83538" />
+      <rect
+         style="fill:#81d0db"
+         x="-590.24597"
+         y="330.543"
+         clip-path="url(#SVGID_4719_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83540" />
+      <rect
+         style="fill:#ffffff"
+         x="-589.586"
+         y="331.20099"
+         clip-path="url(#SVGID_4719_)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83542" />
+      <rect
+         style="fill:#ffffff"
+         x="-588.40698"
+         y="332.38199"
+         clip-path="url(#SVGID_4719_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83544" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83546"
+       inkscape:transform-center-x="422.41995"
+       inkscape:transform-center-y="71.649806">
+      <defs
+         id="defs83548">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4720_"
+           d="m -588.813,334.666 c 0,1.488 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102664">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4720_"
+           overflow="visible"
+           id="use102666" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102668"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3544"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102670" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102672" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4722_)"
+         x="-588.81299"
+         y="331.974"
+         clip-path="url(#SVGID_4721_)"
+         width="5.3829999"
+         height="5.3860002"
+         id="rect83559" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83736"
+       inkscape:transform-center-x="440.81033"
+       inkscape:transform-center-y="89.828699">
+      <defs
+         id="defs83738">
+        <rect
+           id="SVGID_4748_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102813">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4748_"
+           overflow="visible"
+           id="use102815" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4749_)"
+         d="m -514.925,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83744" />
+      <rect
+         style="fill:#ffffff"
+         x="-674.71698"
+         y="341.65302"
+         clip-path="url(#SVGID_4749_)"
+         width="9.592"
+         height="9.592"
+         id="rect83746" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-674.71698"
+         y="341.65302"
+         clip-path="url(#SVGID_4749_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83748" />
+      <rect
+         style="fill:#81d0db"
+         x="-674.04602"
+         y="342.32599"
+         clip-path="url(#SVGID_4749_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83750" />
+      <rect
+         style="fill:#ffffff"
+         x="-673.38599"
+         y="342.98401"
+         clip-path="url(#SVGID_4749_)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83752" />
+      <rect
+         style="fill:#ffffff"
+         x="-672.20697"
+         y="344.164"
+         clip-path="url(#SVGID_4749_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83754" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83756"
+       inkscape:transform-center-x="610.97111"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83758">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4750_"
+           d="m -672.614,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102826">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4750_"
+           overflow="visible"
+           id="use102828" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102830"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4301"
+         x2="256.3541"
+         y2="-148.4301"
+         gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102832" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102834" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4752_)"
+         x="-672.61401"
+         y="343.75699"
+         clip-path="url(#SVGID_4751_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect83769" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83771"
+       inkscape:transform-center-x="592.57619"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83773">
+        <rect
+           id="SVGID_4753_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102840">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4753_"
+           overflow="visible"
+           id="use102842" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4754_)"
+         d="m -670.553,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83779" />
+      <rect
+         style="fill:#ffffff"
+         x="-662.745"
+         y="341.65302"
+         clip-path="url(#SVGID_4754_)"
+         width="9.592"
+         height="9.592"
+         id="rect83781" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-662.74597"
+         y="341.65302"
+         clip-path="url(#SVGID_4754_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83783" />
+      <rect
+         style="fill:#81d0db"
+         x="-662.07397"
+         y="342.32599"
+         clip-path="url(#SVGID_4754_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83785" />
+      <rect
+         style="fill:#ffffff"
+         x="-661.414"
+         y="342.98401"
+         clip-path="url(#SVGID_4754_)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83787" />
+      <rect
+         style="fill:#ffffff"
+         x="-660.23499"
+         y="344.164"
+         clip-path="url(#SVGID_4754_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83789" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83791"
+       inkscape:transform-center-x="584.03412"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83793">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4755_"
+           d="m -660.642,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102853">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4755_"
+           overflow="visible"
+           id="use102855" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102857"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4299"
+         x2="256.3541"
+         y2="-148.4299"
+         gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102859" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102861" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4757_)"
+         x="-660.64203"
+         y="343.75699"
+         clip-path="url(#SVGID_4756_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect83804" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83806"
+       inkscape:transform-center-x="565.64033"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83808">
+        <rect
+           id="SVGID_4758_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102867">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4758_"
+           overflow="visible"
+           id="use102869" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4759_)"
+         d="m -658.582,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83814" />
+      <rect
+         style="fill:#ffffff"
+         x="-650.77399"
+         y="341.65302"
+         clip-path="url(#SVGID_4759_)"
+         width="9.592"
+         height="9.592"
+         id="rect83816" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-650.77399"
+         y="341.65302"
+         clip-path="url(#SVGID_4759_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83818" />
+      <rect
+         style="fill:#81d0db"
+         x="-650.10303"
+         y="342.32599"
+         clip-path="url(#SVGID_4759_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83820" />
+      <rect
+         style="fill:#ffffff"
+         x="-649.44299"
+         y="342.98401"
+         clip-path="url(#SVGID_4759_)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83822" />
+      <rect
+         style="fill:#ffffff"
+         x="-648.26398"
+         y="344.164"
+         clip-path="url(#SVGID_4759_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83824" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83826"
+       inkscape:transform-center-x="557.09937"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83828">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4760_"
+           d="m -648.671,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102880">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4760_"
+           overflow="visible"
+           id="use102882" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102884"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3541"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102886" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102888" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4762_)"
+         x="-648.67102"
+         y="343.75699"
+         clip-path="url(#SVGID_4761_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect83839" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83841"
+       inkscape:transform-center-x="538.70558"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83843">
+        <rect
+           id="SVGID_4763_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102894">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4763_"
+           overflow="visible"
+           id="use102896" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4764_)"
+         d="m -646.611,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83849" />
+      <rect
+         style="fill:#ffffff"
+         x="-638.80298"
+         y="341.65302"
+         clip-path="url(#SVGID_4764_)"
+         width="9.592"
+         height="9.592"
+         id="rect83851" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-638.80298"
+         y="341.65302"
+         clip-path="url(#SVGID_4764_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83853" />
+      <rect
+         style="fill:#81d0db"
+         x="-638.13202"
+         y="342.32599"
+         clip-path="url(#SVGID_4764_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83855" />
+      <rect
+         style="fill:#ffffff"
+         x="-637.47198"
+         y="342.98401"
+         clip-path="url(#SVGID_4764_)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83857" />
+      <rect
+         style="fill:#ffffff"
+         x="-636.29199"
+         y="344.164"
+         clip-path="url(#SVGID_4764_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83859" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83861"
+       inkscape:transform-center-x="530.16231"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83863">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4765_"
+           d="m -636.699,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102907">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4765_"
+           overflow="visible"
+           id="use102909" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102911"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4299"
+         x2="256.3541"
+         y2="-148.4299"
+         gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102913" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102915" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4767_)"
+         x="-636.69897"
+         y="343.75699"
+         clip-path="url(#SVGID_4766_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect83874" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83876"
+       inkscape:transform-center-x="511.76858"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83878">
+        <rect
+           id="SVGID_4768_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102921">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4768_"
+           overflow="visible"
+           id="use102923" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4769_)"
+         d="m -634.639,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83884" />
+      <rect
+         style="fill:#ffffff"
+         x="-626.83099"
+         y="341.65302"
+         clip-path="url(#SVGID_4769_)"
+         width="9.592"
+         height="9.592"
+         id="rect83886" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-626.83099"
+         y="341.65302"
+         clip-path="url(#SVGID_4769_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83888" />
+      <rect
+         style="fill:#81d0db"
+         x="-626.15997"
+         y="342.32599"
+         clip-path="url(#SVGID_4769_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83890" />
+      <rect
+         style="fill:#ffffff"
+         x="-625.5"
+         y="342.98401"
+         clip-path="url(#SVGID_4769_)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83892" />
+      <rect
+         style="fill:#ffffff"
+         x="-624.32098"
+         y="344.164"
+         clip-path="url(#SVGID_4769_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83894" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83896"
+       inkscape:transform-center-x="503.22762"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83898">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4770_"
+           d="m -624.728,346.449 c 0,1.487 1.206,2.692 2.693,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.693,1.206 -2.693,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102934">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4770_"
+           overflow="visible"
+           id="use102936" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102938"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3541"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,177.1768,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102940" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102942" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4772_)"
+         x="-624.72803"
+         y="343.75699"
+         clip-path="url(#SVGID_4771_)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect83909" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83911"
+       inkscape:transform-center-x="484.83387"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83913">
+        <rect
+           id="SVGID_4773_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102948">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4773_"
+           overflow="visible"
+           id="use102950" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4774_)"
+         d="m -622.668,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83919" />
+      <rect
+         style="fill:#ffffff"
+         x="-614.85999"
+         y="341.65302"
+         clip-path="url(#SVGID_4774_)"
+         width="9.5930004"
+         height="9.592"
+         id="rect83921" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-614.86102"
+         y="341.65302"
+         clip-path="url(#SVGID_4774_)"
+         stroke-miterlimit="10"
+         width="9.5930004"
+         height="9.592"
+         id="rect83923" />
+      <rect
+         style="fill:#81d0db"
+         x="-614.18799"
+         y="342.32599"
+         clip-path="url(#SVGID_4774_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83925" />
+      <rect
+         style="fill:#ffffff"
+         x="-613.52899"
+         y="342.98401"
+         clip-path="url(#SVGID_4774_)"
+         width="6.9310002"
+         height="6.9299998"
+         id="rect83927" />
+      <rect
+         style="fill:#ffffff"
+         x="-612.349"
+         y="344.164"
+         clip-path="url(#SVGID_4774_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83929" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83931"
+       inkscape:transform-center-x="476.2917"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83933">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4775_"
+           d="m -612.756,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102961">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4775_"
+           overflow="visible"
+           id="use102963" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102965"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3541"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102967" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102969" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4777_)"
+         x="-612.75598"
+         y="343.75699"
+         clip-path="url(#SVGID_4776_)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect83944" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83946"
+       inkscape:transform-center-x="457.89686"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83948">
+        <rect
+           id="SVGID_4778_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102975">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4778_"
+           overflow="visible"
+           id="use102977" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4779_)"
+         d="m -610.696,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83954" />
+      <rect
+         style="fill:#ffffff"
+         x="-602.888"
+         y="341.65302"
+         clip-path="url(#SVGID_4779_)"
+         width="9.592"
+         height="9.592"
+         id="rect83956" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-602.888"
+         y="341.65302"
+         clip-path="url(#SVGID_4779_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83958" />
+      <rect
+         style="fill:#81d0db"
+         x="-602.21698"
+         y="342.32599"
+         clip-path="url(#SVGID_4779_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83960" />
+      <rect
+         style="fill:#ffffff"
+         x="-601.55701"
+         y="342.98401"
+         clip-path="url(#SVGID_4779_)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83962" />
+      <rect
+         style="fill:#ffffff"
+         x="-600.37799"
+         y="344.164"
+         clip-path="url(#SVGID_4779_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83964" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83966"
+       inkscape:transform-center-x="449.35694"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83968">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4780_"
+           d="m -600.785,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102988">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4780_"
+           overflow="visible"
+           id="use102990" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102992"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4301"
+         x2="256.3541"
+         y2="-148.4301"
+         gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102994" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102996" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4782_)"
+         x="-600.78497"
+         y="343.75699"
+         clip-path="url(#SVGID_4781_)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect83979" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g83981"
+       inkscape:transform-center-x="430.96208"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83983">
+        <rect
+           id="SVGID_4783_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath103002">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4783_"
+           overflow="visible"
+           id="use103004" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4784_)"
+         d="m -598.725,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83989" />
+      <rect
+         style="fill:#ffffff"
+         x="-590.91699"
+         y="341.65302"
+         clip-path="url(#SVGID_4784_)"
+         width="9.592"
+         height="9.592"
+         id="rect83991" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-590.91699"
+         y="341.65302"
+         clip-path="url(#SVGID_4784_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83993" />
+      <rect
+         style="fill:#81d0db"
+         x="-590.24597"
+         y="342.32599"
+         clip-path="url(#SVGID_4784_)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83995" />
+      <rect
+         style="fill:#ffffff"
+         x="-589.586"
+         y="342.98401"
+         clip-path="url(#SVGID_4784_)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83997" />
+      <rect
+         style="fill:#ffffff"
+         x="-588.40698"
+         y="344.164"
+         clip-path="url(#SVGID_4784_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83999" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g84001"
+       inkscape:transform-center-x="422.41995"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs84003">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4785_"
+           d="m -588.813,346.449 c 0,1.487 1.205,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath103015">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4785_"
+           overflow="visible"
+           id="use103017" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient103019"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3541"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop103021" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop103023" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4787_)"
+         x="-588.81299"
+         y="343.75699"
+         clip-path="url(#SVGID_4786_)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect84014" />
+    </g>
+    <rect
+       style="fill:#f15d57"
+       x="220.3154"
+       y="492.99048"
+       width="103.6395"
+       height="3.8114998"
+       id="rect84287"
+       inkscape:transform-center-x="437.28453"
+       inkscape:transform-center-y="-160.89453" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g84289"
+       inkscape:transform-center-x="437.28348"
+       inkscape:transform-center-y="-131.54432">
+      <defs
+         id="defs84291">
+        <polygon
+           id="SVGID_4823_"
+           points="-592.788,256.788 -569.802,251.283 -569.804,231.934 -615.653,231.929 -615.653,251.221 " />
+      </defs>
+      <clipPath
+         id="clipPath103235">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4823_"
+           overflow="visible"
+           id="use103237" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient103239"
+         gradientUnits="userSpaceOnUse"
+         x1="145.3168"
+         y1="2.8518"
+         x2="146.3168"
+         y2="2.8518"
+         gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-668.057,-3384.4426)">
+        <stop
+           offset="0"
+           style="stop-color:#FFFFFF"
+           id="stop103241" />
+        <stop
+           offset="0.23"
+           style="stop-color:#FFFFFF"
+           id="stop103243" />
+        <stop
+           offset="0.5671"
+           style="stop-color:#F9F9F9"
+           id="stop103245" />
+        <stop
+           offset="0.8674"
+           style="stop-color:#EDEEEE"
+           id="stop103247" />
+        <stop
+           offset="1"
+           style="stop-color:#E6E8E7"
+           id="stop103249" />
+      </linearGradient>
+      <polygon
+         style="fill:url(#SVGID_4825_)"
+         clip-path="url(#SVGID_4824_)"
+         points="-615.683,231.929 -569.802,231.874 -569.772,256.788 -615.653,256.843 "
+         id="polygon84308" />
+    </g>
+    <path
+       style="fill:#e6e8e8"
+       inkscape:connector-curvature="0"
+       d="m 220.79459,496.5208 102.67875,0.009 10e-4,43.1055 -51.4755,12.33 -51.20775,-12.46725 0.003,-42.97725 z m -0.4815,-0.4815 v 43.83675 l 51.687,12.58425 51.957,-12.44475 -0.005,-43.96725 -103.6395,-0.009 z"
+       id="path84310"
+       inkscape:transform-center-x="437.28484"
+       inkscape:transform-center-y="-131.54096" />
+    <text
+       y="513.2843"
+       x="243.95297"
+       id="text84312"
+       inkscape:transform-center-x="437.86922"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px">
+      <tspan
+         style="font-size:11px;fill:#37424b;font-family:ApexSans-Book"
+         x="243.95297"
+         y="513.2843"
+         font-size="5.1244"
+         id="tspan84314">RADOSGW</tspan>
+    </text>
+    <polygon
+       style="fill:#81d0db"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       points="-592.696,269.217 -590.628,267.424 -594.747,267.424 "
+       id="polygon84355"
+       inkscape:transform-center-x="437.19348"
+       inkscape:transform-center-y="-77.629816" />
+    <path
+       id="path84524"
+       d="m -365.856,302.434 c 0,-2.331 1.892,-4.222 4.225,-4.222 2.334,0 4.223,1.891 4.223,4.222 0,2.334 -1.889,4.226 -4.223,4.226 -2.333,0 -4.225,-1.892 -4.225,-4.226"
+       clip-path="url(#SVGID_4843_)"
+       inkscape:connector-curvature="0"
+       style="fill:#f15d57"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <path
+       id="path84526"
+       d="m -361.631,301.547 c 0.488,0 0.888,0.398 0.888,0.887 0,0.493 -0.4,0.892 -0.888,0.89 -0.491,0 -0.888,-0.397 -0.888,-0.89 0,-0.489 0.397,-0.887 0.888,-0.887"
+       clip-path="url(#SVGID_4843_)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <path
+       id="path84528"
+       d="m -347.575,299.54 c 0.487,0 0.886,0.395 0.886,0.886 0,0.494 -0.399,0.891 -0.886,0.891 -0.493,0 -0.888,-0.397 -0.888,-0.891 0,-0.491 0.395,-0.886 0.888,-0.886"
+       clip-path="url(#SVGID_4843_)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <path
+       id="path84530"
+       d="m -350.839,302.809 c 0.49,0 0.886,0.397 0.886,0.886 0,0.494 -0.396,0.893 -0.886,0.893 -0.491,0 -0.889,-0.399 -0.889,-0.893 0,-0.489 0.398,-0.886 0.889,-0.886"
+       clip-path="url(#SVGID_4843_)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <path
+       id="path84532"
+       d="m -350.838,299.54 c 0.489,0 0.885,0.395 0.885,0.886 0,0.494 -0.396,0.891 -0.885,0.891 -0.492,0 -0.89,-0.397 -0.89,-0.891 0,-0.491 0.398,-0.886 0.89,-0.886"
+       clip-path="url(#SVGID_4843_)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <path
+       id="path84534"
+       d="m -347.576,302.809 c 0.488,0 0.888,0.397 0.888,0.886 0,0.494 -0.4,0.893 -0.888,0.893 -0.491,0 -0.888,-0.399 -0.888,-0.893 0,-0.489 0.397,-0.886 0.888,-0.886"
+       clip-path="url(#SVGID_4843_)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <path
+       id="path84536"
+       d="m -344.311,302.809 c 0.488,0 0.886,0.397 0.886,0.886 0,0.494 -0.398,0.893 -0.886,0.893 -0.491,0 -0.888,-0.399 -0.888,-0.893 0,-0.489 0.397,-0.886 0.888,-0.886"
+       clip-path="url(#SVGID_4843_)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <path
+       id="path84538"
+       d="m -344.31,299.54 c 0.487,0 0.885,0.395 0.885,0.886 0,0.494 -0.398,0.891 -0.885,0.891 -0.492,0 -0.888,-0.397 -0.888,-0.891 0,-0.491 0.396,-0.886 0.888,-0.886"
+       clip-path="url(#SVGID_4843_)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <line
+       id="line84540"
+       y2="308.98499"
+       x2="-343.42499"
+       y1="308.98499"
+       x1="-365.664"
+       stroke-miterlimit="10"
+       clip-path="url(#SVGID_4843_)"
+       style="fill:none;stroke:#ffffff;stroke-width:0.428;stroke-miterlimit:10"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <line
+       id="line84542"
+       y2="311.16699"
+       x2="-343.42499"
+       y1="311.16699"
+       x1="-365.664"
+       stroke-miterlimit="10"
+       clip-path="url(#SVGID_4843_)"
+       style="fill:none;stroke:#ffffff;stroke-width:0.428;stroke-miterlimit:10"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <line
+       id="line84544"
+       y2="313.35001"
+       x2="-343.42499"
+       y1="313.35001"
+       x1="-365.664"
+       stroke-miterlimit="10"
+       clip-path="url(#SVGID_4843_)"
+       style="fill:none;stroke:#ffffff;stroke-width:0.428;stroke-miterlimit:10"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <polygon
+       id="polygon84552"
+       points="-601.13,292.47 -605.249,292.47 -603.198,294.263 "
+       clip-path="url(#SVGID_4843_)"
+       style="fill:#f15d57"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <polygon
+       id="polygon84554"
+       points="-590.618,292.47 -594.737,292.47 -592.686,294.263 "
+       clip-path="url(#SVGID_4843_)"
+       style="fill:#f15d57"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <polygon
+       id="polygon84556"
+       points="-580.105,292.47 -584.224,292.47 -582.173,294.263 "
+       clip-path="url(#SVGID_4843_)"
+       style="fill:#f15d57"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)" />
+    <polygon
+       id="polygon84564"
+       points="-436.6,208.359 -434.532,206.566 -438.651,206.566 "
+       clip-path="url(#SVGID_4843_)"
+       style="fill:#f15d57"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <polygon
+       id="polygon84566"
+       points="-426.089,208.359 -424.021,206.566 -428.14,206.566 "
+       clip-path="url(#SVGID_4843_)"
+       style="fill:#f15d57"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <polygon
+       id="polygon84568"
+       points="-415.575,208.359 -413.507,206.566 -417.626,206.566 "
+       clip-path="url(#SVGID_4843_)"
+       style="fill:#f15d57"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <polygon
+       id="polygon84570"
+       points="-296.627,290.885 -294.559,289.091 -298.678,289.091 "
+       clip-path="url(#SVGID_4843_)"
+       style="fill:#f15d57"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g84714"
+       inkscape:transform-center-x="434.5295"
+       inkscape:transform-center-y="-60.127066">
+      <defs
+         id="defs84716">
+        <rect
+           id="SVGID_4862_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath103539">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4862_"
+           overflow="visible"
+           id="use103541" />
+      </clipPath>
+      <path
+         style="fill:#5e6a71"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4863_)"
+         d="m -593.321,276.099 c 0,-0.349 0.282,-0.632 0.632,-0.632 0.349,0 0.632,0.283 0.632,0.632 0,0.35 -0.283,0.633 -0.632,0.633 -0.35,0 -0.632,-0.283 -0.632,-0.633"
+         id="path84722" />
+      <line
+         style="fill:none;stroke:#5e6a71;stroke-width:0.214;stroke-miterlimit:10"
+         clip-path="url(#SVGID_4863_)"
+         stroke-miterlimit="10"
+         x1="-592.68903"
+         y1="276.099"
+         x2="-589.79303"
+         y2="276.099"
+         id="line84724" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g84726"
+       inkscape:transform-center-x="413.59995"
+       inkscape:transform-center-y="-60.127083">
+      <defs
+         id="defs84728">
+        <polygon
+           id="SVGID_4864_"
+           points="-590.081,276.108 -588.288,278.632 -574.322,278.632 -574.322,273.567 -588.288,273.567 " />
+      </defs>
+      <clipPath
+         id="clipPath103548">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4864_"
+           overflow="visible"
+           id="use103550" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient103552"
+         gradientUnits="userSpaceOnUse"
+         x1="-34.205299"
+         y1="249.9404"
+         x2="-33.205299"
+         y2="249.9404"
+         gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,105.3813)">
+        <stop
+           offset="0"
+           style="stop-color:#E6E8E7"
+           id="stop103554" />
+        <stop
+           offset="1"
+           style="stop-color:#FFFFFF"
+           id="stop103556" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4866_)"
+         x="-590.08099"
+         y="273.56699"
+         clip-path="url(#SVGID_4865_)"
+         width="15.759"
+         height="5.0650001"
+         id="rect84739" />
+    </g>
+    <polygon
+       style="fill:none;stroke:#e6e8e8;stroke-width:0.214;stroke-miterlimit:10"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       stroke-miterlimit="10"
+       points="-590.082,276.108 -588.289,278.632 -574.323,278.632 -574.323,273.567 -588.289,273.567 "
+       id="polygon84741"
+       inkscape:transform-center-x="413.60223"
+       inkscape:transform-center-y="-60.127066" />
+    <text
+       y="597.62451"
+       x="287.82977"
+       id="text84743"
+       inkscape:transform-center-x="412.33742"
+       inkscape:transform-center-y="-60.270044"
+       style="font-size:27px">
+      <tspan
+         style="font-size:5.77170038px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="287.82977"
+         y="597.62451"
+         font-size="2.5652"
+         id="tspan84745">N</tspan>
+      <tspan
+         style="font-size:5.77170038px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="291.70874"
+         y="597.62451"
+         font-size="2.5652"
+         id="tspan84747">A</tspan>
+      <tspan
+         style="font-size:5.77170038px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="294.77774"
+         y="597.62451"
+         font-size="2.5652"
+         id="tspan84749">TIV</tspan>
+      <tspan
+         style="font-size:5.77170038px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="302.49078"
+         y="597.62451"
+         font-size="2.5652"
+         id="tspan84751">E</tspan>
+    </text>
+    <g
+       transform="matrix(1.3542118,0,0,2.25,996.72494,-31.560183)"
+       id="g84792"
+       inkscape:transform-center-x="262.77489"
+       inkscape:transform-center-y="141.90493">
+      <defs
+         id="defs84794">
+        <rect
+           id="SVGID_4872_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath103597">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4872_"
+           overflow="visible"
+           id="use103599" />
+      </clipPath>
+      <path
+         style="fill:#5e6a71"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4873_)"
+         d="m -438.652,188.287 c 0,0.349 -0.282,0.632 -0.632,0.632 -0.349,0 -0.632,-0.283 -0.632,-0.632 0,-0.35 0.283,-0.633 0.632,-0.633 0.35,0 0.632,0.284 0.632,0.633"
+         id="path84800" />
+      <g
+         clip-path="url(#SVGID_4873_)"
+         id="g84802">
+        <g
+           id="g84804">
+          <defs
+             id="defs84806">
+            <rect
+               id="SVGID_4874_"
+               x="-679.90002"
+               y="360.13901"
+               width="174.95799"
+               height="10.152" />
+          </defs>
+          <clipPath
+             id="clipPath103606">
+            <use
+               height="1052.3622"
+               width="744.09448"
+               y="0"
+               x="0"
+               style="overflow:visible"
+               xlink:href="#SVGID_4874_"
+               overflow="visible"
+               id="use103608" />
+          </clipPath>
+          <g
+             clip-path="url(#SVGID_4875_)"
+             id="g84812">
+            <defs
+               id="defs84814">
+              <rect
+                 id="SVGID_4876_"
+                 x="-679.90002"
+                 y="360.13901"
+                 width="174.95799"
+                 height="10.152" />
+            </defs>
+            <clipPath
+               id="clipPath103613">
+              <use
+                 height="1052.3622"
+                 width="744.09448"
+                 y="0"
+                 x="0"
+                 style="overflow:visible"
+                 xlink:href="#SVGID_4876_"
+                 overflow="visible"
+                 id="use103615" />
+            </clipPath>
+            <linearGradient
+               id="linearGradient103617"
+               gradientUnits="userSpaceOnUse"
+               x1="40.563"
+               y1="147.0757"
+               x2="41.563"
+               y2="147.0757"
+               gradientTransform="matrix(0,-10.1526,-10.1526,0,900.78,782.1111)">
+              <stop
+                 offset="0"
+                 style="stop-color:#55AEB9"
+                 id="stop103619" />
+              <stop
+                 offset="0.486"
+                 style="stop-color:#B0D2D9"
+                 id="stop103621" />
+              <stop
+                 offset="0.8287"
+                 style="stop-color:#E6EFF1"
+                 id="stop103623" />
+              <stop
+                 offset="0.9939"
+                 style="stop-color:#FFFFFF"
+                 id="stop103625" />
+              <stop
+                 offset="1"
+                 style="stop-color:#FFFFFF"
+                 id="stop103627" />
+            </linearGradient>
+            <rect
+               style="fill:url(#SVGID_4878_)"
+               x="-679.90002"
+               y="360.138"
+               clip-path="url(#SVGID_4877_)"
+               width="174.95799"
+               height="10.153"
+               id="rect84831" />
+          </g>
+        </g>
+      </g>
+      <g
+         clip-path="url(#SVGID_4873_)"
+         id="g84833">
+        <g
+           id="g84835">
+          <defs
+             id="defs84837">
+            <rect
+               id="SVGID_4879_"
+               x="-679.88898"
+               y="360.13901"
+               width="174.94701"
+               height="10.152" />
+          </defs>
+          <clipPath
+             id="clipPath103634">
+            <use
+               height="1052.3622"
+               width="744.09448"
+               y="0"
+               x="0"
+               style="overflow:visible"
+               xlink:href="#SVGID_4879_"
+               overflow="visible"
+               id="use103636" />
+          </clipPath>
+          <g
+             clip-path="url(#SVGID_4880_)"
+             id="g84843">
+            <defs
+               id="defs84845">
+              <rect
+                 id="SVGID_4881_"
+                 x="-679.88898"
+                 y="360.138"
+                 width="174.94701"
+                 height="10.153" />
+            </defs>
+            <clipPath
+               id="clipPath103641">
+              <use
+                 height="1052.3622"
+                 width="744.09448"
+                 y="0"
+                 x="0"
+                 style="overflow:visible"
+                 xlink:href="#SVGID_4881_"
+                 overflow="visible"
+                 id="use103643" />
+            </clipPath>
+            <polygon
+               style="fill:#f15d57"
+               clip-path="url(#SVGID_4882_)"
+               points="-679.89,364.81 -679.89,367.139 -679.89,370.291 -504.943,370.291 -504.943,360.138 -679.89,360.139 "
+               id="polygon84851" />
+          </g>
+        </g>
+      </g>
+      <rect
+         style="fill:#f15d57"
+         x="-679.90198"
+         y="360.267"
+         clip-path="url(#SVGID_4873_)"
+         width="174.96001"
+         height="2.641"
+         id="rect84853" />
+      <rect
+         style="fill:#55aeb9"
+         x="-679.88898"
+         y="370.29199"
+         clip-path="url(#SVGID_4873_)"
+         width="174.94701"
+         height="1.352"
+         id="rect84855" />
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-size:8px;font-style:normal;font-weight:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans"
+       x="285.86542"
+       y="577.40869"
+       id="text113406"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan113408"
+         x="285.86542"
+         y="577.40869" /></text>
+    <text
+       y="832.10559"
+       x="242.41013"
+       id="text84343-2"
+       inkscape:transform-center-x="587.62742"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px;text-align:center;text-anchor:middle">
+      <tspan
+         style="font-size:11px;text-align:center;text-anchor:middle;fill:#37424b;font-family:ApexSans-Book"
+         x="197.43796"
+         y="832.10559"
+         font-size="5.1244"
+         id="tspan84345-2">MASTER REGION</tspan>
+    </text>
+    <text
+       y="926.36633"
+       x="288.45654"
+       id="text84343-2-6"
+       inkscape:transform-center-x="587.62742"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px;text-align:center;text-anchor:middle">
+      <tspan
+         style="font-size:11px;text-align:center;text-anchor:middle;fill:#37424b;font-family:ApexSans-Book"
+         x="249.24219"
+         y="926.36633"
+         font-size="5.1244"
+         id="tspan84345-2-5" />
+    </text>
+    <flowRoot
+       xml:space="preserve"
+       id="flowRoot126352"
+       style="fill:black;stroke:none;stroke-opacity:1;stroke-width:1px;stroke-linejoin:miter;stroke-linecap:butt;fill-opacity:1;font-family:Sans;font-style:normal;font-weight:normal;font-size:9px;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;text-align:center"><flowRegion
+         id="flowRegion126354"><rect
+           id="rect126356"
+           width="0"
+           height="51"
+           x="247"
+           y="928.36218" /></flowRegion><flowPara
+         id="flowPara126358" /></flowRoot>    <flowRoot
+       xml:space="preserve"
+       id="flowRoot126403"
+       style="fill:black;stroke:none;stroke-opacity:1;stroke-width:1px;stroke-linejoin:miter;stroke-linecap:butt;fill-opacity:1;font-family:Sans;font-style:normal;font-weight:normal;font-size:9px;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;text-align:center"><flowRegion
+         id="flowRegion126405"><rect
+           id="rect126407"
+           width="185"
+           height="81"
+           x="286"
+           y="941.36218" /></flowRegion><flowPara
+         id="flowPara126409" /></flowRoot>    <flowRoot
+       xml:space="preserve"
+       id="flowRoot126411"
+       style="fill:black;stroke:none;stroke-opacity:1;stroke-width:1px;stroke-linejoin:miter;stroke-linecap:butt;fill-opacity:1;font-family:Sans;font-style:normal;font-weight:normal;font-size:9px;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;text-align:center"><flowRegion
+         id="flowRegion126413"><rect
+           id="rect126415"
+           width="89"
+           height="64"
+           x="285"
+           y="943.36218" /></flowRegion><flowPara
+         id="flowPara126417" /></flowRoot>    <text
+       xml:space="preserve"
+       style="font-size:9px;font-style:normal;font-weight:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans"
+       x="272"
+       y="529.64587"
+       id="text126456"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan126458"
+         x="272"
+         y="529.64587">MASTER</tspan><tspan
+         sodipodi:role="line"
+         x="272"
+         y="540.89587"
+         id="tspan126460">ZONE</tspan></text>
+    <g
+       transform="matrix(3.375,0,0,3.375,2054.0831,-334.61629)"
+       id="g82107">
+      <defs
+         id="defs82109">
+        <polygon
+           id="SVGID_4519_"
+           points="-504.975,196.685 -504.975,188.287 -547.302,188.287 -547.302,196.684 -528.257,196.685 -526.139,198.478 -524.02,196.685 " />
+      </defs>
+      <clipPath
+         id="clipPath126820">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4519_"
+           overflow="visible"
+           id="use126822" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient126824"
+         gradientUnits="userSpaceOnUse"
+         x1="40.840599"
+         y1="146.6937"
+         x2="41.840599"
+         y2="146.6937"
+         gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)">
+        <stop
+           offset="0"
+           style="stop-color:#37424B"
+           id="stop126826" />
+        <stop
+           offset="0.12"
+           style="stop-color:#37424B"
+           id="stop126828" />
+        <stop
+           offset="0.88"
+           style="stop-color:#5E6A71"
+           id="stop126830" />
+        <stop
+           offset="1"
+           style="stop-color:#5E6A71"
+           id="stop126832" />
+      </linearGradient>
+      <rect
+         style="fill:url(#linearGradient126904)"
+         x="-547.302"
+         y="188.287"
+         clip-path="url(#SVGID_4520_-5)"
+         width="42.327"
+         height="10.191"
+         id="rect82124" />
+    </g>
+    <g
+       transform="matrix(3.375,0,0,3.375,2054.0831,-334.61629)"
+       id="g82126">
+      <defs
+         id="defs82128">
+        <rect
+           id="SVGID_4522_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath126838">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4522_"
+           overflow="visible"
+           id="use126840" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4523_-1)"
+         d="m -539.397,191.908 c -0.353,0 -0.64,-0.286 -0.64,-0.639 0,-0.354 0.287,-0.642 0.64,-0.641 0.354,0 0.639,0.287 0.639,0.641 0,0.353 -0.285,0.639 -0.639,0.639"
+         id="path82134" />
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4523_-1)"
+         d="m -539.397,194.263 c -0.353,0 -0.64,-0.286 -0.64,-0.639 0,-0.355 0.287,-0.642 0.64,-0.642 0.354,0 0.639,0.287 0.639,0.642 0,0.353 -0.285,0.639 -0.639,0.639"
+         id="path82136" />
+      <circle
+         style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10"
+         sodipodi:ry="0.63999999"
+         sodipodi:rx="0.63999999"
+         sodipodi:cy="191.26801"
+         sodipodi:cx="-541.625"
+         d="m -540.985,191.26801 c 0,0.35346 -0.28654,0.64 -0.64,0.64 -0.35346,0 -0.64,-0.28654 -0.64,-0.64 0,-0.35347 0.28654,-0.64 0.64,-0.64 0.35346,0 0.64,0.28653 0.64,0.64 z"
+         clip-path="url(#SVGID_4523_-1)"
+         stroke-miterlimit="10"
+         cx="-541.625"
+         cy="191.26801"
+         r="0.63999999"
+         id="circle82138" />
+      <circle
+         style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10"
+         sodipodi:ry="0.63999999"
+         sodipodi:rx="0.63999999"
+         sodipodi:cy="193.623"
+         sodipodi:cx="-541.625"
+         d="m -540.985,193.623 c 0,0.35346 -0.28654,0.64 -0.64,0.64 -0.35346,0 -0.64,-0.28654 -0.64,-0.64 0,-0.35346 0.28654,-0.64 0.64,-0.64 0.35346,0 0.64,0.28654 0.64,0.64 z"
+         clip-path="url(#SVGID_4523_-1)"
+         stroke-miterlimit="10"
+         cx="-541.625"
+         cy="193.623"
+         r="0.63999999"
+         id="circle82140" />
+      <circle
+         style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10"
+         sodipodi:ry="0.63999999"
+         sodipodi:rx="0.63999999"
+         sodipodi:cy="191.26801"
+         sodipodi:cx="-543.85303"
+         d="m -543.21303,191.26801 c 0,0.35346 -0.28654,0.64 -0.64,0.64 -0.35346,0 -0.64,-0.28654 -0.64,-0.64 0,-0.35347 0.28654,-0.64 0.64,-0.64 0.35346,0 0.64,0.28653 0.64,0.64 z"
+         clip-path="url(#SVGID_4523_-1)"
+         stroke-miterlimit="10"
+         cx="-543.85303"
+         cy="191.26801"
+         r="0.63999999"
+         id="circle82142" />
+      <circle
+         style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10"
+         sodipodi:ry="0.63999999"
+         sodipodi:rx="0.63999999"
+         sodipodi:cy="193.623"
+         sodipodi:cx="-543.854"
+         d="m -543.214,193.623 c 0,0.35346 -0.28654,0.64 -0.64,0.64 -0.35347,0 -0.64,-0.28654 -0.64,-0.64 0,-0.35346 0.28653,-0.64 0.64,-0.64 0.35346,0 0.64,0.28654 0.64,0.64 z"
+         clip-path="url(#SVGID_4523_-1)"
+         stroke-miterlimit="10"
+         cx="-543.854"
+         cy="193.623"
+         r="0.63999999"
+         id="circle82144" />
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4523_-1)"
+         d="m -537.169,191.908 c -0.354,0 -0.64,-0.286 -0.64,-0.639 0,-0.354 0.286,-0.642 0.64,-0.641 0.354,0 0.64,0.287 0.64,0.641 0,0.353 -0.286,0.639 -0.64,0.639"
+         id="path82146" />
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4523_-1)"
+         d="m -537.169,194.263 c -0.354,0 -0.64,-0.286 -0.64,-0.639 0,-0.355 0.286,-0.642 0.64,-0.642 0.354,0 0.64,0.287 0.64,0.642 0,0.353 -0.286,0.639 -0.64,0.639"
+         id="path82148" />
+    </g>
+    <text
+       y="319.3797"
+       x="267.68884"
+       id="text82150"
+       style="font-size:40.5px">
+      <tspan
+         style="font-size:10.80911255px;fill:#ffffff;font-family:ApexSans-Medium"
+         x="267.68884"
+         y="319.3797"
+         font-size="3.2027"
+         id="tspan82152">AP</tspan>
+      <tspan
+         style="font-size:10.80911255px;fill:#ffffff;font-family:ApexSans-Medium"
+         x="281.37094"
+         y="319.3797"
+         font-size="3.2027"
+         id="tspan82154">P</tspan>
+    </text>
+    <g
+       transform="matrix(3.375,0,0,3.375,2054.0831,-334.61629)"
+       id="g84638">
+      <defs
+         id="defs84640">
+        <rect
+           id="SVGID_4852_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath126879">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4852_"
+           overflow="visible"
+           id="use126881" />
+      </clipPath>
+      <path
+         style="fill:#5e6a71"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4853_-8)"
+         d="m -526.673,205.991 c 0,-0.349 0.282,-0.632 0.632,-0.632 0.349,0 0.632,0.283 0.632,0.632 0,0.35 -0.283,0.633 -0.632,0.633 -0.35,0 -0.632,-0.283 -0.632,-0.633"
+         id="path84646" />
+      <line
+         style="fill:none;stroke:#5e6a71;stroke-width:0.214;stroke-miterlimit:10"
+         clip-path="url(#SVGID_4853_-8)"
+         stroke-miterlimit="10"
+         x1="-526.03998"
+         y1="205.991"
+         x2="-523.14398"
+         y2="205.991"
+         id="line84648" />
+    </g>
+    <g
+       transform="matrix(3.375,0,0,3.375,2054.0831,-334.61629)"
+       id="g84650">
+      <defs
+         id="defs84652">
+        <polygon
+           id="SVGID_4854_"
+           points="-507.673,203.458 -521.64,203.458 -523.432,206 -521.64,208.524 -507.673,208.524 " />
+      </defs>
+      <clipPath
+         id="clipPath126888">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4854_"
+           overflow="visible"
+           id="use126890" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient126892"
+         gradientUnits="userSpaceOnUse"
+         x1="-34.205299"
+         y1="249.94031"
+         x2="-33.205101"
+         y2="249.94031"
+         gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)">
+        <stop
+           offset="0"
+           style="stop-color:#E6E8E7"
+           id="stop126894" />
+        <stop
+           offset="1"
+           style="stop-color:#FFFFFF"
+           id="stop126896" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4856_-4)"
+         x="-523.43201"
+         y="203.45799"
+         clip-path="url(#SVGID_4855_-2)"
+         width="15.759"
+         height="5.066"
+         id="rect84663" />
+    </g>
+    <polygon
+       style="fill:none;stroke:#e6e8e8;stroke-width:0.214;stroke-miterlimit:10"
+       transform="matrix(3.375,0,0,3.375,2054.0831,-334.61629)"
+       stroke-miterlimit="10"
+       points="-507.674,203.459 -521.64,203.459 -523.433,206 -521.64,208.524 -507.674,208.524 "
+       id="polygon84665" />
+    <text
+       y="363.54565"
+       x="306.66861"
+       id="text84667"
+       style="font-size:40.5px">
+      <tspan
+         style="font-size:8.65755081px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="306.66861"
+         y="363.54565"
+         font-size="2.5652"
+         id="tspan84669">RE</tspan>
+      <tspan
+         style="font-size:8.65755081px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="315.84521"
+         y="363.54565"
+         font-size="2.5652"
+         id="tspan84671">S</tspan>
+      <tspan
+         style="font-size:8.65755081px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="319.98297"
+         y="363.54565"
+         font-size="2.5652"
+         id="tspan84673">T</tspan>
+    </text>
+    <text
+       xml:space="preserve"
+       style="font-size:9px;font-style:normal;font-weight:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans"
+       x="232.25806"
+       y="178.16862"
+       id="text127482"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan127484"
+         x="232.25806"
+         y="178.16862" /></text>
+    <text
+       y="247.60512"
+       x="322.41013"
+       id="text84343-2-0"
+       inkscape:transform-center-x="587.62742"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px;text-align:center;text-anchor:middle">
+      <tspan
+         style="font-size:11px;text-align:center;text-anchor:middle;fill:#37424b;font-family:ApexSans-Book"
+         x="277.43796"
+         y="247.60512"
+         font-size="5.1244"
+         id="tspan84345-2-6">MASTER REGION</tspan>
+    </text>
+    <text
+       y="705.37921"
+       x="255.23141"
+       style="font-size:9.75217533px;fill:#ffffff;font-family:ApexSans-Medium"
+       font-size="4.3343"
+       id="text82351-9"
+       inkscape:transform-center-x="450.06672"
+       inkscape:transform-center-y="45.543285">M</text>
+    <g
+       id="g9504"
+       transform="translate(-230,34)">
+      <g
+         inkscape:transform-center-y="45.666809"
+         inkscape:transform-center-x="449.81598"
+         id="g82339-6"
+         transform="matrix(2.25,0,0,2.25,1701.0081,-85.193202)">
+        <defs
+           id="defs82341-9">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4549_-8" />
+        </defs>
+        <clipPath
+           id="clipPath101738-3">
+          <use
+             id="use101740-3"
+             overflow="visible"
+             xlink:href="#SVGID_4549_-8"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path82347-9"
+           d="m -602.031,326.467 c 0,0.212 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.173 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+           clip-path="url(#SVGID_4550_-6)"
+           inkscape:connector-curvature="0"
+           style="fill:#f15d57" />
+        <path
+           id="path82349-9"
+           d="m -602.031,326.467 c 0,0.212 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.173 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4550_-6)"
+           inkscape:connector-curvature="0"
+           style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10" />
+      </g>
+      <text
+         inkscape:transform-center-y="45.543285"
+         inkscape:transform-center-x="450.06672"
+         id="text82351-9-4"
+         font-size="4.3343"
+         style="font-size:9.75217533px;fill:#ffffff;font-family:ApexSans-Medium"
+         x="350.6293"
+         y="645.37921">M</text>
+    </g>
+    <path
+       id="path82449-2"
+       d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+       clip-path="url(#SVGID_4564_-7)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1767.5425,-25.466067)" />
+    <path
+       id="path82449-2-2"
+       d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+       clip-path="url(#SVGID_4564_-7-3)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1578.7139,-25.075587)" />
+    <path
+       id="path82449-2-0"
+       d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+       clip-path="url(#SVGID_4564_-7-9)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1767.5425,0.59793262)" />
+    <path
+       id="path82449-2-0-1"
+       d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+       clip-path="url(#SVGID_4564_-7-9-3)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1767.6425,26.597933)" />
+    <path
+       id="path82449-2-0-1-5"
+       d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+       clip-path="url(#SVGID_4564_-7-9-3-4)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1767.7422,53.197933)" />
+    <text
+       y="292.96655"
+       x="321.78174"
+       id="text84343-2-0-4-3"
+       inkscape:transform-center-x="587.62742"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px;text-align:center;text-anchor:middle">
+      <tspan
+         style="font-size:11px;text-align:center;text-anchor:middle;fill:#37424b;font-family:ApexSans-Book"
+         x="276.80957"
+         y="292.96655"
+         font-size="5.1244"
+         id="tspan84345-2-6-5-3">WRITE / READ</tspan>
+    </text>
+    <text
+       y="262.38412"
+       x="322.19797"
+       id="text84343-2-0-4-6"
+       inkscape:transform-center-x="587.62742"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px;text-align:center;text-anchor:middle">
+      <tspan
+         style="font-size:11px;text-align:center;text-anchor:middle;fill:#37424b;font-family:ApexSans-Book"
+         x="277.2258"
+         y="262.38412"
+         font-size="5.1244"
+         id="tspan84345-2-6-5-6">(United States)</tspan>
+    </text>
+    <g
+       transform="matrix(1.3542118,0,0,2.25,1325.4162,-25.59059)"
+       id="g82158-9"
+       inkscape:transform-center-x="262.3321"
+       inkscape:transform-center-y="50.765304">
+      <defs
+         id="defs82160-5">
+        <rect
+           id="SVGID_4524_-5"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101582-0">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4524_-5"
+           overflow="visible"
+           id="use101584-7" />
+      </clipPath>
+      <g
+         clip-path="url(#SVGID_4525_-0)"
+         id="g82166-4">
+        <g
+           id="g82168-7">
+          <defs
+             id="defs82170-2">
+            <rect
+               id="SVGID_4526_-7"
+               x="-679.67902"
+               y="293.259"
+               width="175.168"
+               height="64.251999" />
+          </defs>
+          <clipPath
+             id="clipPath101590-3">
+            <use
+               height="1052.3622"
+               width="744.09448"
+               y="0"
+               x="0"
+               style="overflow:visible"
+               xlink:href="#SVGID_4526_-7"
+               overflow="visible"
+               id="use101592-1" />
+          </clipPath>
+          <g
+             clip-path="url(#SVGID_4527_-7)"
+             id="g82176-6">
+            <defs
+               id="defs82178-7">
+              <rect
+                 id="SVGID_4528_-0"
+                 x="-679.57202"
+                 y="293.367"
+                 width="174.953"
+                 height="64.037003" />
+            </defs>
+            <clipPath
+               id="clipPath101597-1">
+              <use
+                 height="1052.3622"
+                 width="744.09448"
+                 y="0"
+                 x="0"
+                 style="overflow:visible"
+                 xlink:href="#SVGID_4528_-0"
+                 overflow="visible"
+                 id="use101599-3" />
+            </clipPath>
+            <linearGradient
+               id="linearGradient101601-1"
+               gradientUnits="userSpaceOnUse"
+               x1="126.8011"
+               y1="60.902401"
+               x2="127.8011"
+               y2="60.902401"
+               gradientTransform="matrix(0,64.0388,-64.0388,0,3308.0217,-7826.8223)">
+              <stop
+                 offset="0"
+                 style="stop-color:#E6E8E7"
+                 id="stop101603-3" />
+              <stop
+                 offset="1"
+                 style="stop-color:#FFFFFF"
+                 id="stop101605-0" />
+            </linearGradient>
+            <rect
+               style="fill:url(#SVGID_4530_-2)"
+               x="-679.57202"
+               y="293.366"
+               clip-path="url(#SVGID_4529_-9)"
+               width="174.953"
+               height="64.038002"
+               id="rect82189-5" />
+          </g>
+          <g
+             clip-path="url(#SVGID_4527_-7)"
+             id="g82191-9">
+            <defs
+               id="defs82193-6">
+              <rect
+                 id="SVGID_4531_-4"
+                 x="-679.67902"
+                 y="293.259"
+                 width="175.168"
+                 height="64.251999" />
+            </defs>
+            <clipPath
+               id="clipPath101611-1">
+              <use
+                 height="1052.3622"
+                 width="744.09448"
+                 y="0"
+                 x="0"
+                 style="overflow:visible"
+                 xlink:href="#SVGID_4531_-4"
+                 overflow="visible"
+                 id="use101613-8" />
+            </clipPath>
+            <rect
+               style="fill:none;stroke:#f15d57;stroke-width:0.214;stroke-miterlimit:10"
+               x="-679.57202"
+               y="293.366"
+               clip-path="url(#SVGID_4532_-8)"
+               stroke-miterlimit="10"
+               width="174.953"
+               height="64.038002"
+               id="rect82199-7" />
+          </g>
+        </g>
+      </g>
+      <g
+         clip-path="url(#SVGID_4525_-0)"
+         id="g82201-2">
+        <g
+           id="g82203-6">
+          <defs
+             id="defs82205-4">
+            <rect
+               id="SVGID_4533_-1"
+               x="-441.56699"
+               y="293.259"
+               width="45.873001"
+               height="64.251999" />
+          </defs>
+          <clipPath
+             id="clipPath101620-7">
+            <use
+               height="1052.3622"
+               width="744.09448"
+               y="0"
+               x="0"
+               style="overflow:visible"
+               xlink:href="#SVGID_4533_-1"
+               overflow="visible"
+               id="use101622-3" />
+          </clipPath>
+          <g
+             clip-path="url(#SVGID_4534_-49)"
+             id="g82211-7">
+            <defs
+               id="defs82213-4">
+              <rect
+                 id="SVGID_4535_-5"
+                 x="-441.45999"
+                 y="293.367"
+                 width="45.659"
+                 height="64.037003" />
+            </defs>
+            <clipPath
+               id="clipPath101627-6">
+              <use
+                 height="1052.3622"
+                 width="744.09448"
+                 y="0"
+                 x="0"
+                 style="overflow:visible"
+                 xlink:href="#SVGID_4535_-5"
+                 overflow="visible"
+                 id="use101629-8" />
+            </clipPath>
+            <linearGradient
+               id="linearGradient101631-5"
+               gradientUnits="userSpaceOnUse"
+               x1="126.8011"
+               y1="60.902401"
+               x2="127.8011"
+               y2="60.902401"
+               gradientTransform="matrix(0,64.0388,-64.0388,0,3481.4861,-7826.8223)">
+              <stop
+                 offset="0"
+                 style="stop-color:#E6E8E7"
+                 id="stop101633-8" />
+              <stop
+                 offset="1"
+                 style="stop-color:#FFFFFF"
+                 id="stop101635-5" />
+            </linearGradient>
+            <rect
+               style="fill:url(#SVGID_4537_-7)"
+               x="-441.45999"
+               y="293.366"
+               clip-path="url(#SVGID_4536_-4)"
+               width="45.659"
+               height="64.038002"
+               id="rect82224-8" />
+          </g>
+          <g
+             clip-path="url(#SVGID_4534_-49)"
+             id="g82226-0">
+            <defs
+               id="defs82228-3">
+              <rect
+                 id="SVGID_4538_-7"
+                 x="-441.56699"
+                 y="293.259"
+                 width="45.873001"
+                 height="64.251999" />
+            </defs>
+            <clipPath
+               id="clipPath101641-2">
+              <use
+                 height="1052.3622"
+                 width="744.09448"
+                 y="0"
+                 x="0"
+                 style="overflow:visible"
+                 xlink:href="#SVGID_4538_-7"
+                 overflow="visible"
+                 id="use101643-6" />
+            </clipPath>
+            <rect
+               style="fill:none;stroke:#f15d57;stroke-width:0.214;stroke-miterlimit:10"
+               x="-441.45999"
+               y="293.366"
+               clip-path="url(#SVGID_4539_-0)"
+               stroke-miterlimit="10"
+               width="45.659"
+               height="64.038002"
+               id="rect82234-0" />
+          </g>
+        </g>
+      </g>
+    </g>
+    <rect
+       style="fill:#f15d57"
+       x="404.21451"
+       y="574.65991"
+       width="95.778"
+       height="3.0712502"
+       id="rect82236-5"
+       inkscape:transform-center-x="586.0074"
+       inkscape:transform-center-y="-79.564828" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82310-8"
+       inkscape:transform-center-x="585.9972"
+       inkscape:transform-center-y="-95.804196">
+      <defs
+         id="defs82312-6">
+        <polygon
+           id="SVGID_4546_-1"
+           points="-679.997,253.224 -680.004,267.353 -637.65,267.32 -637.642,253.133 -659.18,258.293 " />
+      </defs>
+      <clipPath
+         id="clipPath101711-5">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4546_-1"
+           overflow="visible"
+           id="use101713-5" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101715-3"
+         gradientUnits="userSpaceOnUse"
+         x1="76.3899"
+         y1="97.785698"
+         x2="77.3899"
+         y2="97.785698"
+         gradientTransform="matrix(0,-19.5733,-19.5733,0,1255.1711,1762.5587)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101717-8" />
+        <stop
+           offset="0.0507"
+           style="stop-color:#68B4BF"
+           id="stop101719-2" />
+        <stop
+           offset="0.2116"
+           style="stop-color:#96C6CF"
+           id="stop101721-7" />
+        <stop
+           offset="0.3736"
+           style="stop-color:#BAD8DD"
+           id="stop101723-6" />
+        <stop
+           offset="0.5342"
+           style="stop-color:#D7E6EA"
+           id="stop101725-5" />
+        <stop
+           offset="0.693"
+           style="stop-color:#EBF2F3"
+           id="stop101727-2" />
+        <stop
+           offset="0.8492"
+           style="stop-color:#F8FBFB"
+           id="stop101729-5" />
+        <stop
+           offset="1"
+           style="stop-color:#FFFFFF"
+           id="stop101731-7" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4548_-6)"
+         x="-680.00299"
+         y="253.133"
+         clip-path="url(#SVGID_4547_-1)"
+         width="42.361"
+         height="14.22"
+         id="rect82335-2" />
+    </g>
+    <path
+       style="fill:#e6e8e8"
+       inkscape:connector-curvature="0"
+       d="m 499.52681,544.26464 -0.0157,31.37625 -94.815,0.072 0.0158,-31.2435 46.48274,11.31975 0.1125,0.027 0.11475,-0.027 48.10501,-11.5245 z m 0.4815,-0.60975 -48.699,11.66625 -47.079,-11.46375 -0.0135,32.337 95.77575,-0.0743 0.0157,-32.46525 z"
+       id="path82337-5"
+       inkscape:transform-center-x="585.99834"
+       inkscape:transform-center-y="-95.835733" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82339-8"
+       inkscape:transform-center-x="449.81598"
+       inkscape:transform-center-y="45.666809">
+      <defs
+         id="defs82341-98">
+        <rect
+           id="SVGID_4549_-7"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101738-4">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4549_-7"
+           overflow="visible"
+           id="use101740-4" />
+      </clipPath>
+      <path
+         style="fill:#f15d57"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4550_-50)"
+         d="m -602.031,326.467 c 0,0.212 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.173 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+         id="path82347-4" />
+      <path
+         style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4550_-50)"
+         stroke-miterlimit="10"
+         d="m -602.031,326.467 c 0,0.212 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.173 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+         id="path82349-4" />
+    </g>
+    <text
+       y="704.8584"
+       x="603.83478"
+       style="font-size:9.75217533px;fill:#ffffff;font-family:ApexSans-Medium"
+       font-size="4.3343"
+       id="text82351-1"
+       inkscape:transform-center-x="450.06672"
+       inkscape:transform-center-y="45.543285">M</text>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82353-8"
+       inkscape:transform-center-x="503.22873"
+       inkscape:transform-center-y="71.647559">
+      <defs
+         id="defs82355-0">
+        <rect
+           id="SVGID_4551_-7"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101748-6">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4551_-7"
+           overflow="visible"
+           id="use101750-2" />
+      </clipPath>
+      <path
+         style="fill:#f15d57"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4552_-4)"
+         d="m -625.77,338.014 c 0,0.213 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.172 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+         id="path82361-9" />
+      <path
+         style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4552_-4)"
+         stroke-miterlimit="10"
+         d="m -625.77,338.014 c 0,0.213 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.172 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+         id="path82363-6" />
+    </g>
+    <text
+       y="730.84003"
+       x="530.42249"
+       style="font-size:9.75217533px;fill:#ffffff;font-family:ApexSans-Medium"
+       font-size="4.3343"
+       id="text82365-8"
+       inkscape:transform-center-x="503.47899"
+       inkscape:transform-center-y="71.524915">M</text>
+    <rect
+       style="fill:#ffffff"
+       x="416.3511"
+       y="664.56543"
+       width="21.581999"
+       height="21.584251"
+       id="rect82381-2"
+       inkscape:transform-center-x="610.96881"
+       inkscape:transform-center-y="19.597191" />
+    <rect
+       style="fill:none;stroke:#81d0db;stroke-width:0.96300006;stroke-miterlimit:10"
+       x="416.3511"
+       y="664.56543"
+       stroke-miterlimit="10"
+       width="21.581999"
+       height="21.584251"
+       id="rect82383-1"
+       inkscape:transform-center-x="610.96881"
+       inkscape:transform-center-y="19.597191" />
+    <rect
+       style="fill:#81d0db"
+       x="417.8631"
+       y="666.08191"
+       width="18.557999"
+       height="18.553501"
+       id="rect82385-3"
+       inkscape:transform-center-x="610.97115"
+       inkscape:transform-center-y="19.598295" />
+    <rect
+       style="fill:#ffffff"
+       x="419.34586"
+       y="667.56012"
+       width="15.592501"
+       height="15.594751"
+       id="rect82387-5"
+       inkscape:transform-center-x="610.96883"
+       inkscape:transform-center-y="19.597131" />
+    <rect
+       style="fill:#ffffff"
+       x="422.00085"
+       y="670.21729"
+       width="10.282501"
+       height="10.278"
+       id="rect82389-9"
+       inkscape:transform-center-x="610.97103"
+       inkscape:transform-center-y="19.595982" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4666,-25.591067)"
+       id="g82391-8"
+       inkscape:transform-center-x="610.97111"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82393-4">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4555_-7"
+           d="m -672.614,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101773-5">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4555_-7"
+           overflow="visible"
+           id="use101775-6" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101777-2"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.46021"
+         x2="256.3761"
+         y2="-148.46021"
+         gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101779-4" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101781-1" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4557_-8)"
+         x="-672.61401"
+         y="308.841"
+         clip-path="url(#SVGID_4556_-9)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82404-9" />
+    </g>
+    <rect
+       id="rect82451-3"
+       height="9.5930004"
+       width="9.592"
+       clip-path="url(#SVGID_4564_-73)"
+       y="306.73599"
+       x="-650.77399"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <rect
+       id="rect82453-2"
+       height="9.5930004"
+       width="9.592"
+       stroke-miterlimit="10"
+       clip-path="url(#SVGID_4564_-73)"
+       y="306.73599"
+       x="-650.77399"
+       style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <rect
+       id="rect82455-8"
+       height="8.2460003"
+       width="8.2480001"
+       clip-path="url(#SVGID_4564_-73)"
+       y="307.41"
+       x="-650.10303"
+       style="fill:#81d0db"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <rect
+       id="rect82457-2"
+       height="6.9310002"
+       width="6.9299998"
+       clip-path="url(#SVGID_4564_-73)"
+       y="308.06699"
+       x="-649.44299"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <rect
+       id="rect82459-8"
+       height="4.5679998"
+       width="4.5700002"
+       clip-path="url(#SVGID_4564_-73)"
+       y="309.24799"
+       x="-648.26398"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82461-4"
+       inkscape:transform-center-x="557.09937"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82463-7">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4565_-3"
+           d="m -648.671,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101827-5">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4565_-3"
+           overflow="visible"
+           id="use101829-7" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101831-3"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.4601"
+         x2="256.3761"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101833-4" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101835-4" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4567_-1)"
+         x="-648.67102"
+         y="308.841"
+         clip-path="url(#SVGID_4566_-3)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82474-5" />
+    </g>
+    <path
+       id="path82484-3"
+       d="m -646.611,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+       clip-path="url(#SVGID_4569_-7)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <rect
+       id="rect82486-0"
+       height="9.5930004"
+       width="9.592"
+       clip-path="url(#SVGID_4569_-7)"
+       y="306.73599"
+       x="-638.80298"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <rect
+       id="rect82488-6"
+       height="9.5930004"
+       width="9.592"
+       stroke-miterlimit="10"
+       clip-path="url(#SVGID_4569_-7)"
+       y="306.73599"
+       x="-638.80298"
+       style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <rect
+       id="rect82490-6"
+       height="8.2460003"
+       width="8.2480001"
+       clip-path="url(#SVGID_4569_-7)"
+       y="307.41"
+       x="-638.13202"
+       style="fill:#81d0db"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <rect
+       id="rect82492-4"
+       height="6.9310002"
+       width="6.9299998"
+       clip-path="url(#SVGID_4569_-7)"
+       y="308.06699"
+       x="-637.47198"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <rect
+       id="rect82494-9"
+       height="4.5679998"
+       width="4.5700002"
+       clip-path="url(#SVGID_4569_-7)"
+       y="309.24799"
+       x="-636.29199"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82496-3"
+       inkscape:transform-center-x="530.16231"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82498-5">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4570_-9"
+           d="m -636.699,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101854-8">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4570_-9"
+           overflow="visible"
+           id="use101856-3" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101858-4"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.46001"
+         x2="256.3761"
+         y2="-148.46001"
+         gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101860-6" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101862-5" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4572_-41)"
+         x="-636.69897"
+         y="308.841"
+         clip-path="url(#SVGID_4571_-0)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82509-9" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82511-7"
+       inkscape:transform-center-x="511.76858"
+       inkscape:transform-center-y="19.597166">
+      <defs
+         id="defs82513-7">
+        <rect
+           id="SVGID_4573_-4"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101868-0">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4573_-4"
+           overflow="visible"
+           id="use101870-7" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4574_-9)"
+         d="m -634.639,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82519-6" />
+      <rect
+         style="fill:#ffffff"
+         x="-626.83099"
+         y="306.73599"
+         clip-path="url(#SVGID_4574_-9)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82521-0" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-626.83099"
+         y="306.73599"
+         clip-path="url(#SVGID_4574_-9)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82523-1" />
+      <rect
+         style="fill:#81d0db"
+         x="-626.15997"
+         y="307.41"
+         clip-path="url(#SVGID_4574_-9)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82525-5" />
+      <rect
+         style="fill:#ffffff"
+         x="-625.5"
+         y="308.06699"
+         clip-path="url(#SVGID_4574_-9)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82527-3" />
+      <rect
+         style="fill:#ffffff"
+         x="-624.32098"
+         y="309.24799"
+         clip-path="url(#SVGID_4574_-9)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82529-8" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82531-3"
+       inkscape:transform-center-x="503.22762"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82533-9">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4575_-2"
+           d="m -624.728,311.531 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.205 -2.693,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101881-9">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4575_-2"
+           overflow="visible"
+           id="use101883-6" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101885-7"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.4601"
+         x2="256.3761"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101887-1" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101889-2" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4577_-27)"
+         x="-624.72803"
+         y="308.841"
+         clip-path="url(#SVGID_4576_-1)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82544-5" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82546-5"
+       inkscape:transform-center-x="484.83387"
+       inkscape:transform-center-y="19.597166">
+      <defs
+         id="defs82548-4">
+        <rect
+           id="SVGID_4578_-9"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101895-1">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4578_-9"
+           overflow="visible"
+           id="use101897-5" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4579_-0)"
+         d="m -622.668,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82554-9" />
+      <rect
+         style="fill:#ffffff"
+         x="-614.85999"
+         y="306.73599"
+         clip-path="url(#SVGID_4579_-0)"
+         width="9.5930004"
+         height="9.5930004"
+         id="rect82556-4" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-614.86102"
+         y="306.73599"
+         clip-path="url(#SVGID_4579_-0)"
+         stroke-miterlimit="10"
+         width="9.5930004"
+         height="9.5930004"
+         id="rect82558-0" />
+      <rect
+         style="fill:#81d0db"
+         x="-614.18799"
+         y="307.41"
+         clip-path="url(#SVGID_4579_-0)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82560-5" />
+      <rect
+         style="fill:#ffffff"
+         x="-613.52899"
+         y="308.06699"
+         clip-path="url(#SVGID_4579_-0)"
+         width="6.9310002"
+         height="6.9310002"
+         id="rect82562-1" />
+      <rect
+         style="fill:#ffffff"
+         x="-612.349"
+         y="309.24799"
+         clip-path="url(#SVGID_4579_-0)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82564-1" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82566-5"
+       inkscape:transform-center-x="476.2917"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82568-0">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4580_-5"
+           d="m -612.756,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101908-7">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4580_-5"
+           overflow="visible"
+           id="use101910-9" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101912-3"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.4601"
+         x2="256.3761"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101914-7" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101916-2" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4582_-8)"
+         x="-612.75598"
+         y="308.841"
+         clip-path="url(#SVGID_4581_-2)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect82579-9" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82581-3"
+       inkscape:transform-center-x="457.89686"
+       inkscape:transform-center-y="19.597166">
+      <defs
+         id="defs82583-0">
+        <rect
+           id="SVGID_4583_-4"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101922-4">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4583_-4"
+           overflow="visible"
+           id="use101924-4" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4584_-0)"
+         d="m -610.696,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82589-5" />
+      <rect
+         style="fill:#ffffff"
+         x="-602.888"
+         y="306.73599"
+         clip-path="url(#SVGID_4584_-0)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82591-2" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-602.888"
+         y="306.73599"
+         clip-path="url(#SVGID_4584_-0)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82593-2" />
+      <rect
+         style="fill:#81d0db"
+         x="-602.21698"
+         y="307.41"
+         clip-path="url(#SVGID_4584_-0)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82595-8" />
+      <rect
+         style="fill:#ffffff"
+         x="-601.55701"
+         y="308.06699"
+         clip-path="url(#SVGID_4584_-0)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82597-7" />
+      <rect
+         style="fill:#ffffff"
+         x="-600.37799"
+         y="309.24799"
+         clip-path="url(#SVGID_4584_-0)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82599-7" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82601-6"
+       inkscape:transform-center-x="449.35694"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82603-1">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4585_-8"
+           d="m -600.785,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.203,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101935-9">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4585_-8"
+           overflow="visible"
+           id="use101937-8" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101939-7"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.46021"
+         x2="256.3761"
+         y2="-148.46021"
+         gradientTransform="matrix(0,5.3836,5.3836,0,201.1574,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101941-5" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101943-8" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4587_-9)"
+         x="-600.78497"
+         y="308.841"
+         clip-path="url(#SVGID_4586_-03)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect82614-5" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82616-8"
+       inkscape:transform-center-x="430.96208"
+       inkscape:transform-center-y="19.597166">
+      <defs
+         id="defs82618-9">
+        <rect
+           id="SVGID_4588_-2"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath101949-9">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4588_-2"
+           overflow="visible"
+           id="use101951-6" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4589_-7)"
+         d="m -598.725,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82624-1" />
+      <rect
+         style="fill:#ffffff"
+         x="-590.91699"
+         y="306.73599"
+         clip-path="url(#SVGID_4589_-7)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82626-0" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-590.91699"
+         y="306.73599"
+         clip-path="url(#SVGID_4589_-7)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82628-0" />
+      <rect
+         style="fill:#81d0db"
+         x="-590.24597"
+         y="307.41"
+         clip-path="url(#SVGID_4589_-7)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82630-0" />
+      <rect
+         style="fill:#ffffff"
+         x="-589.586"
+         y="308.06699"
+         clip-path="url(#SVGID_4589_-7)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82632-3" />
+      <rect
+         style="fill:#ffffff"
+         x="-588.40698"
+         y="309.24799"
+         clip-path="url(#SVGID_4589_-7)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82634-1" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82636-3"
+       inkscape:transform-center-x="422.41995"
+       inkscape:transform-center-y="19.598312">
+      <defs
+         id="defs82638-5">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4590_-7"
+           d="m -588.813,311.531 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.205 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath101962-9">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4590_-7"
+           overflow="visible"
+           id="use101964-0" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101966-2"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3761"
+         y1="-148.4601"
+         x2="256.3761"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1066.0031)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101968-4" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop101970-4" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4592_-6)"
+         x="-588.81299"
+         y="308.841"
+         clip-path="url(#SVGID_4591_-2)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect82649-2" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82896-3"
+       inkscape:transform-center-x="592.57619"
+       inkscape:transform-center-y="45.667929">
+      <defs
+         id="defs82898-8">
+        <rect
+           id="SVGID_4628_-3"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102165-4">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4628_-3"
+           overflow="visible"
+           id="use102167-0" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4629_-0)"
+         d="m -670.553,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82904-4" />
+      <rect
+         style="fill:#ffffff"
+         x="-662.745"
+         y="318.323"
+         clip-path="url(#SVGID_4629_-0)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82906-1" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-662.74597"
+         y="318.323"
+         clip-path="url(#SVGID_4629_-0)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82908-3" />
+      <rect
+         style="fill:#81d0db"
+         x="-662.07397"
+         y="318.99701"
+         clip-path="url(#SVGID_4629_-0)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82910-4" />
+      <rect
+         style="fill:#ffffff"
+         x="-661.414"
+         y="319.65399"
+         clip-path="url(#SVGID_4629_-0)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82912-1" />
+      <rect
+         style="fill:#ffffff"
+         x="-660.23499"
+         y="320.83499"
+         clip-path="url(#SVGID_4629_-0)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82914-4" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82916-7"
+       inkscape:transform-center-x="584.03412"
+       inkscape:transform-center-y="45.671291">
+      <defs
+         id="defs82918-4">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4630_-9"
+           d="m -660.642,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath102178-5">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4630_-9"
+           overflow="visible"
+           id="use102180-1" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102182-9"
+         gradientUnits="userSpaceOnUse"
+         x1="255.37621"
+         y1="-148.4601"
+         x2="256.37631"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1054.4158)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102184-5" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102186-5" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4632_-7)"
+         x="-660.64203"
+         y="320.42899"
+         clip-path="url(#SVGID_4631_-7)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82929-5" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82931-1"
+       inkscape:transform-center-x="565.64033"
+       inkscape:transform-center-y="45.667929">
+      <defs
+         id="defs82933-7">
+        <rect
+           id="SVGID_4633_-6"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102192-4">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4633_-6"
+           overflow="visible"
+           id="use102194-8" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4634_-0)"
+         d="m -658.582,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82939-0" />
+      <rect
+         style="fill:#ffffff"
+         x="-650.77399"
+         y="318.323"
+         clip-path="url(#SVGID_4634_-0)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82941-6" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-650.77399"
+         y="318.323"
+         clip-path="url(#SVGID_4634_-0)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82943-7" />
+      <rect
+         style="fill:#81d0db"
+         x="-650.10303"
+         y="318.99701"
+         clip-path="url(#SVGID_4634_-0)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82945-4" />
+      <rect
+         style="fill:#ffffff"
+         x="-649.44299"
+         y="319.65399"
+         clip-path="url(#SVGID_4634_-0)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82947-0" />
+      <rect
+         style="fill:#ffffff"
+         x="-648.26398"
+         y="320.83499"
+         clip-path="url(#SVGID_4634_-0)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82949-2" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82951-7"
+       inkscape:transform-center-x="557.09937"
+       inkscape:transform-center-y="45.671291">
+      <defs
+         id="defs82953-8">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4635_-5"
+           d="m -648.671,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath102205-2">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4635_-5"
+           overflow="visible"
+           id="use102207-1" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102209-9"
+         gradientUnits="userSpaceOnUse"
+         x1="255.37621"
+         y1="-148.4601"
+         x2="256.37631"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1054.4158)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102211-5" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102213-4" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4637_-4)"
+         x="-648.67102"
+         y="320.42899"
+         clip-path="url(#SVGID_4636_-9)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82964-5" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82966-8"
+       inkscape:transform-center-x="538.70558"
+       inkscape:transform-center-y="45.667929">
+      <defs
+         id="defs82968-8">
+        <rect
+           id="SVGID_4638_-3"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102219-2">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4638_-3"
+           overflow="visible"
+           id="use102221-0" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4639_-45)"
+         d="m -646.611,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82974-0" />
+      <rect
+         style="fill:#ffffff"
+         x="-638.80298"
+         y="318.323"
+         clip-path="url(#SVGID_4639_-45)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82976-3" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-638.80298"
+         y="318.323"
+         clip-path="url(#SVGID_4639_-45)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82978-1" />
+      <rect
+         style="fill:#81d0db"
+         x="-638.13202"
+         y="318.99701"
+         clip-path="url(#SVGID_4639_-45)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82980-7" />
+      <rect
+         style="fill:#ffffff"
+         x="-637.47198"
+         y="319.65399"
+         clip-path="url(#SVGID_4639_-45)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82982-8" />
+      <rect
+         style="fill:#ffffff"
+         x="-636.29199"
+         y="320.83499"
+         clip-path="url(#SVGID_4639_-45)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82984-6" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g82986-8"
+       inkscape:transform-center-x="530.16231"
+       inkscape:transform-center-y="45.671291">
+      <defs
+         id="defs82988-7">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4640_-2"
+           d="m -636.699,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath102232-2">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4640_-2"
+           overflow="visible"
+           id="use102234-7" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102236-4"
+         gradientUnits="userSpaceOnUse"
+         x1="255.37621"
+         y1="-148.46001"
+         x2="256.37631"
+         y2="-148.46001"
+         gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1054.4158)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102238-1" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102240-5" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4642_-8)"
+         x="-636.69897"
+         y="320.42899"
+         clip-path="url(#SVGID_4641_-3)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect82999-8" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83001-3"
+       inkscape:transform-center-x="511.76858"
+       inkscape:transform-center-y="45.667929">
+      <defs
+         id="defs83003-9">
+        <rect
+           id="SVGID_4643_-8"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102246-1">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4643_-8"
+           overflow="visible"
+           id="use102248-6" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4644_-6)"
+         d="m -634.639,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83009-0" />
+      <rect
+         style="fill:#ffffff"
+         x="-626.83099"
+         y="318.323"
+         clip-path="url(#SVGID_4644_-6)"
+         width="9.592"
+         height="9.5930004"
+         id="rect83011-4" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-626.83099"
+         y="318.323"
+         clip-path="url(#SVGID_4644_-6)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect83013-7" />
+      <rect
+         style="fill:#81d0db"
+         x="-626.15997"
+         y="318.99701"
+         clip-path="url(#SVGID_4644_-6)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect83015-7" />
+      <rect
+         style="fill:#ffffff"
+         x="-625.5"
+         y="319.65399"
+         clip-path="url(#SVGID_4644_-6)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect83017-1" />
+      <rect
+         style="fill:#ffffff"
+         x="-624.32098"
+         y="320.83499"
+         clip-path="url(#SVGID_4644_-6)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83019-2" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83021-5"
+       inkscape:transform-center-x="503.22762"
+       inkscape:transform-center-y="45.671291">
+      <defs
+         id="defs83023-1">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4645_-7"
+           d="m -624.728,323.119 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.204 -2.693,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath102259-7">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4645_-7"
+           overflow="visible"
+           id="use102261-1" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102263-7"
+         gradientUnits="userSpaceOnUse"
+         x1="255.37621"
+         y1="-148.4601"
+         x2="256.37631"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1054.4158)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102265-2" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102267-2" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4647_-0)"
+         x="-624.72803"
+         y="320.42899"
+         clip-path="url(#SVGID_4646_-87)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect83034-4" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83036-0"
+       inkscape:transform-center-x="484.83387"
+       inkscape:transform-center-y="45.667929">
+      <defs
+         id="defs83038-8">
+        <rect
+           id="SVGID_4648_-2"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102273-9">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4648_-2"
+           overflow="visible"
+           id="use102275-0" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4649_-4)"
+         d="m -622.668,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83044-7" />
+      <rect
+         style="fill:#ffffff"
+         x="-614.85999"
+         y="318.323"
+         clip-path="url(#SVGID_4649_-4)"
+         width="9.5930004"
+         height="9.5930004"
+         id="rect83046-7" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-614.86102"
+         y="318.323"
+         clip-path="url(#SVGID_4649_-4)"
+         stroke-miterlimit="10"
+         width="9.5930004"
+         height="9.5930004"
+         id="rect83048-4" />
+      <rect
+         style="fill:#81d0db"
+         x="-614.18799"
+         y="318.99701"
+         clip-path="url(#SVGID_4649_-4)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect83050-0" />
+      <rect
+         style="fill:#ffffff"
+         x="-613.52899"
+         y="319.65399"
+         clip-path="url(#SVGID_4649_-4)"
+         width="6.9310002"
+         height="6.9310002"
+         id="rect83052-4" />
+      <rect
+         style="fill:#ffffff"
+         x="-612.349"
+         y="320.83499"
+         clip-path="url(#SVGID_4649_-4)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83054-5" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83056-3"
+       inkscape:transform-center-x="476.2917"
+       inkscape:transform-center-y="45.671291">
+      <defs
+         id="defs83058-3">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4650_-3"
+           d="m -612.756,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath102286-6">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4650_-3"
+           overflow="visible"
+           id="use102288-9" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102290-5"
+         gradientUnits="userSpaceOnUse"
+         x1="255.37621"
+         y1="-148.4601"
+         x2="256.37631"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1054.4158)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102292-1" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102294-8" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4652_-6)"
+         x="-612.75598"
+         y="320.42899"
+         clip-path="url(#SVGID_4651_-3)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect83069-2" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83071-4"
+       inkscape:transform-center-x="444.42945"
+       inkscape:transform-center-y="45.667929">
+      <defs
+         id="defs83073-0">
+        <rect
+           id="SVGID_4653_-9"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102300-5">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4653_-9"
+           overflow="visible"
+           id="use102302-8" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4654_-2)"
+         d="m -610.696,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83079-6" />
+      <rect
+         style="fill:#ffffff"
+         x="-590.91699"
+         y="318.323"
+         clip-path="url(#SVGID_4654_-2)"
+         width="9.592"
+         height="9.5930004"
+         id="rect83081-7" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-590.91699"
+         y="318.323"
+         clip-path="url(#SVGID_4654_-2)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect83083-5" />
+      <rect
+         style="fill:#81d0db"
+         x="-590.24597"
+         y="318.99701"
+         clip-path="url(#SVGID_4654_-2)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect83085-8" />
+      <rect
+         style="fill:#ffffff"
+         x="-589.586"
+         y="319.65399"
+         clip-path="url(#SVGID_4654_-2)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect83087-9" />
+      <rect
+         style="fill:#ffffff"
+         x="-588.40698"
+         y="320.83499"
+         clip-path="url(#SVGID_4654_-2)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83089-2" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83091-0"
+       inkscape:transform-center-x="422.41995"
+       inkscape:transform-center-y="45.671291">
+      <defs
+         id="defs83093-8">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4655_-6"
+           d="m -588.813,323.119 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.204 -2.692,2.69" />
+      </defs>
+      <clipPath
+         id="clipPath102313-1">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4655_-6"
+           overflow="visible"
+           id="use102315-0" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102317-3"
+         gradientUnits="userSpaceOnUse"
+         x1="255.37621"
+         y1="-148.4601"
+         x2="256.37631"
+         y2="-148.4601"
+         gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102319-0" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102321-7" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4657_-5)"
+         x="-588.81299"
+         y="320.42899"
+         clip-path="url(#SVGID_4656_-5)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect83104-3" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4666,-25.59059)"
+       id="g83336-4"
+       inkscape:transform-center-x="610.97111"
+       inkscape:transform-center-y="71.649806">
+      <defs
+         id="defs83338-2">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4690_-8"
+           d="m -672.614,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102502-9">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4690_-8"
+           overflow="visible"
+           id="use102504-7" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102506-5"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4301"
+         x2="256.3544"
+         y2="-148.4301"
+         gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102508-0" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102510-4" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4692_-5)"
+         x="-672.61401"
+         y="331.974"
+         clip-path="url(#SVGID_4691_-8)"
+         width="5.3839998"
+         height="5.3860002"
+         id="rect83349-8" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83351-8"
+       inkscape:transform-center-x="592.57619"
+       inkscape:transform-center-y="71.647547">
+      <defs
+         id="defs83353-8">
+        <rect
+           id="SVGID_4693_-2"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102516-9">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4693_-2"
+           overflow="visible"
+           id="use102518-7" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4694_-6)"
+         d="m -670.553,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83359-7" />
+      <rect
+         style="fill:#ffffff"
+         x="-662.745"
+         y="329.87"
+         clip-path="url(#SVGID_4694_-6)"
+         width="9.592"
+         height="9.592"
+         id="rect83361-9" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-662.74597"
+         y="329.87"
+         clip-path="url(#SVGID_4694_-6)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83363-3" />
+      <rect
+         style="fill:#81d0db"
+         x="-662.07397"
+         y="330.543"
+         clip-path="url(#SVGID_4694_-6)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83365-6" />
+      <rect
+         style="fill:#ffffff"
+         x="-661.414"
+         y="331.20099"
+         clip-path="url(#SVGID_4694_-6)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83367-6" />
+      <rect
+         style="fill:#ffffff"
+         x="-660.23499"
+         y="332.38199"
+         clip-path="url(#SVGID_4694_-6)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83369-3" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83371-6"
+       inkscape:transform-center-x="584.03412"
+       inkscape:transform-center-y="71.648681">
+      <defs
+         id="defs83373-8">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4695_-3"
+           d="m -660.642,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102529-6">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4695_-3"
+           overflow="visible"
+           id="use102531-5" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102533-6"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4299"
+         x2="256.35419"
+         y2="-148.4299"
+         gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102535-6" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102537-0" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4697_-7)"
+         x="-660.64203"
+         y="331.974"
+         clip-path="url(#SVGID_4696_-9)"
+         width="5.3839998"
+         height="5.3850002"
+         id="rect83384-7" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83386-5"
+       inkscape:transform-center-x="565.64033"
+       inkscape:transform-center-y="71.647547">
+      <defs
+         id="defs83388-6">
+        <rect
+           id="SVGID_4698_-1"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102543-9">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4698_-1"
+           overflow="visible"
+           id="use102545-4" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4699_-07)"
+         d="m -658.582,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83394-3" />
+      <rect
+         style="fill:#ffffff"
+         x="-650.77399"
+         y="329.87"
+         clip-path="url(#SVGID_4699_-07)"
+         width="9.592"
+         height="9.592"
+         id="rect83396-6" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-650.77399"
+         y="329.87"
+         clip-path="url(#SVGID_4699_-07)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83398-1" />
+      <rect
+         style="fill:#81d0db"
+         x="-650.10303"
+         y="330.543"
+         clip-path="url(#SVGID_4699_-07)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83400-3" />
+      <rect
+         style="fill:#ffffff"
+         x="-649.44299"
+         y="331.20099"
+         clip-path="url(#SVGID_4699_-07)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83402-2" />
+      <rect
+         style="fill:#ffffff"
+         x="-648.26398"
+         y="332.38199"
+         clip-path="url(#SVGID_4699_-07)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83404-1" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83406-2"
+       inkscape:transform-center-x="557.09937"
+       inkscape:transform-center-y="71.649806">
+      <defs
+         id="defs83408-0">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4700_-3"
+           d="m -648.671,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102556-3">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4700_-3"
+           overflow="visible"
+           id="use102558-7" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102560-3"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3544"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102562-2" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102564-2" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4702_-3)"
+         x="-648.67102"
+         y="331.974"
+         clip-path="url(#SVGID_4701_-9)"
+         width="5.3839998"
+         height="5.3860002"
+         id="rect83419-9" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83421-8"
+       inkscape:transform-center-x="538.70558"
+       inkscape:transform-center-y="71.647547">
+      <defs
+         id="defs83423-5">
+        <rect
+           id="SVGID_4703_-5"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102570-9">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4703_-5"
+           overflow="visible"
+           id="use102572-0" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4704_-1)"
+         d="m -646.611,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83429-3" />
+      <rect
+         style="fill:#ffffff"
+         x="-638.80298"
+         y="329.87"
+         clip-path="url(#SVGID_4704_-1)"
+         width="9.592"
+         height="9.592"
+         id="rect83431-4" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-638.80298"
+         y="329.87"
+         clip-path="url(#SVGID_4704_-1)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83433-7" />
+      <rect
+         style="fill:#81d0db"
+         x="-638.13202"
+         y="330.543"
+         clip-path="url(#SVGID_4704_-1)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83435-2" />
+      <rect
+         style="fill:#ffffff"
+         x="-637.47198"
+         y="331.20099"
+         clip-path="url(#SVGID_4704_-1)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83437-6" />
+      <rect
+         style="fill:#ffffff"
+         x="-636.29199"
+         y="332.38199"
+         clip-path="url(#SVGID_4704_-1)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83439-4" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83441-7"
+       inkscape:transform-center-x="530.16231"
+       inkscape:transform-center-y="71.649806">
+      <defs
+         id="defs83443-2">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4705_-7"
+           d="m -636.699,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102583-7">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4705_-7"
+           overflow="visible"
+           id="use102585-7" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102587-0"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4299"
+         x2="256.3544"
+         y2="-148.4299"
+         gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102589-5" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102591-0" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4707_-8)"
+         x="-636.69897"
+         y="331.974"
+         clip-path="url(#SVGID_4706_-25)"
+         width="5.3839998"
+         height="5.3860002"
+         id="rect83454-4" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83456-8"
+       inkscape:transform-center-x="498.30125"
+       inkscape:transform-center-y="71.647547">
+      <defs
+         id="defs83458-2">
+        <rect
+           id="SVGID_4708_-8"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102597-0">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4708_-8"
+           overflow="visible"
+           id="use102599-7" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4709_-6)"
+         d="m -634.639,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83464-3" />
+      <rect
+         style="fill:#ffffff"
+         x="-614.85999"
+         y="329.87"
+         clip-path="url(#SVGID_4709_-6)"
+         width="9.5930004"
+         height="9.592"
+         id="rect83466-8" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-614.86102"
+         y="329.87"
+         clip-path="url(#SVGID_4709_-6)"
+         stroke-miterlimit="10"
+         width="9.5930004"
+         height="9.592"
+         id="rect83468-0" />
+      <rect
+         style="fill:#81d0db"
+         x="-614.18799"
+         y="330.543"
+         clip-path="url(#SVGID_4709_-6)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83470-5" />
+      <rect
+         style="fill:#ffffff"
+         x="-613.52899"
+         y="331.20099"
+         clip-path="url(#SVGID_4709_-6)"
+         width="6.9310002"
+         height="6.9299998"
+         id="rect83472-2" />
+      <rect
+         style="fill:#ffffff"
+         x="-612.349"
+         y="332.38199"
+         clip-path="url(#SVGID_4709_-6)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83474-0" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83476-5"
+       inkscape:transform-center-x="476.2917"
+       inkscape:transform-center-y="71.649806">
+      <defs
+         id="defs83478-8">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4710_-7"
+           d="m -612.756,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102610-6">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4710_-7"
+           overflow="visible"
+           id="use102612-0" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102614-3"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3544"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102616-0" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102618-7" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4712_-3)"
+         x="-612.75598"
+         y="331.974"
+         clip-path="url(#SVGID_4711_-5)"
+         width="5.3829999"
+         height="5.3860002"
+         id="rect83489-5" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83491-9"
+       inkscape:transform-center-x="457.89686"
+       inkscape:transform-center-y="71.647547">
+      <defs
+         id="defs83493-3">
+        <rect
+           id="SVGID_4713_-2"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102624-1">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4713_-2"
+           overflow="visible"
+           id="use102626-1" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4714_-7)"
+         d="m -610.696,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83499-1" />
+      <rect
+         style="fill:#ffffff"
+         x="-602.888"
+         y="329.87"
+         clip-path="url(#SVGID_4714_-7)"
+         width="9.592"
+         height="9.592"
+         id="rect83501-0" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-602.888"
+         y="329.87"
+         clip-path="url(#SVGID_4714_-7)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83503-1" />
+      <rect
+         style="fill:#81d0db"
+         x="-602.21698"
+         y="330.543"
+         clip-path="url(#SVGID_4714_-7)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83505-7" />
+      <rect
+         style="fill:#ffffff"
+         x="-601.55701"
+         y="331.20099"
+         clip-path="url(#SVGID_4714_-7)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83507-1" />
+      <rect
+         style="fill:#ffffff"
+         x="-600.37799"
+         y="332.38199"
+         clip-path="url(#SVGID_4714_-7)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83509-7" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83511-7"
+       inkscape:transform-center-x="449.35694"
+       inkscape:transform-center-y="71.649806">
+      <defs
+         id="defs83513-5">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4715_-7"
+           d="m -600.785,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102637-7">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4715_-7"
+           overflow="visible"
+           id="use102639-2" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102641-0"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4301"
+         x2="256.3544"
+         y2="-148.4301"
+         gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102643-7" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102645-3" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4717_-52)"
+         x="-600.78497"
+         y="331.974"
+         clip-path="url(#SVGID_4716_-5)"
+         width="5.3829999"
+         height="5.3860002"
+         id="rect83524-7" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83526-2"
+       inkscape:transform-center-x="430.96208"
+       inkscape:transform-center-y="71.647547">
+      <defs
+         id="defs83528-5">
+        <rect
+           id="SVGID_4718_-5"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102651-0">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4718_-5"
+           overflow="visible"
+           id="use102653-4" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4719_-0)"
+         d="m -598.725,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83534-1" />
+      <rect
+         style="fill:#ffffff"
+         x="-590.91699"
+         y="329.87"
+         clip-path="url(#SVGID_4719_-0)"
+         width="9.592"
+         height="9.592"
+         id="rect83536-2" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-590.91699"
+         y="329.87"
+         clip-path="url(#SVGID_4719_-0)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83538-7" />
+      <rect
+         style="fill:#81d0db"
+         x="-590.24597"
+         y="330.543"
+         clip-path="url(#SVGID_4719_-0)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83540-4" />
+      <rect
+         style="fill:#ffffff"
+         x="-589.586"
+         y="331.20099"
+         clip-path="url(#SVGID_4719_-0)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83542-2" />
+      <rect
+         style="fill:#ffffff"
+         x="-588.40698"
+         y="332.38199"
+         clip-path="url(#SVGID_4719_-0)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83544-4" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83546-3"
+       inkscape:transform-center-x="422.41995"
+       inkscape:transform-center-y="71.649806">
+      <defs
+         id="defs83548-5">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4720_-7"
+           d="m -588.813,334.666 c 0,1.488 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102664-4">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4720_-7"
+           overflow="visible"
+           id="use102666-8" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102668-0"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3544"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1042.9659)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102670-7" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102672-2" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4722_-33)"
+         x="-588.81299"
+         y="331.974"
+         clip-path="url(#SVGID_4721_-50)"
+         width="5.3829999"
+         height="5.3860002"
+         id="rect83559-7" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4666,-25.59059)"
+       id="g83756-0"
+       inkscape:transform-center-x="610.97111"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83758-9">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4750_-6"
+           d="m -672.614,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102826-5">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4750_-6"
+           overflow="visible"
+           id="use102828-9" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102830-6"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4301"
+         x2="256.3541"
+         y2="-148.4301"
+         gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102832-7" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102834-9" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4752_-4)"
+         x="-672.61401"
+         y="343.75699"
+         clip-path="url(#SVGID_4751_-1)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect83769-5" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83771-2"
+       inkscape:transform-center-x="592.57619"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83773-9">
+        <rect
+           id="SVGID_4753_-7"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102840-9">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4753_-7"
+           overflow="visible"
+           id="use102842-4" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4754_-0)"
+         d="m -670.553,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83779-9" />
+      <rect
+         style="fill:#ffffff"
+         x="-662.745"
+         y="341.65302"
+         clip-path="url(#SVGID_4754_-0)"
+         width="9.592"
+         height="9.592"
+         id="rect83781-6" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-662.74597"
+         y="341.65302"
+         clip-path="url(#SVGID_4754_-0)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83783-5" />
+      <rect
+         style="fill:#81d0db"
+         x="-662.07397"
+         y="342.32599"
+         clip-path="url(#SVGID_4754_-0)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83785-2" />
+      <rect
+         style="fill:#ffffff"
+         x="-661.414"
+         y="342.98401"
+         clip-path="url(#SVGID_4754_-0)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83787-5" />
+      <rect
+         style="fill:#ffffff"
+         x="-660.23499"
+         y="344.164"
+         clip-path="url(#SVGID_4754_-0)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83789-9" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83791-4"
+       inkscape:transform-center-x="584.03412"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83793-0">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4755_-2"
+           d="m -660.642,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102853-1">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4755_-2"
+           overflow="visible"
+           id="use102855-9" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102857-9"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4299"
+         x2="256.3541"
+         y2="-148.4299"
+         gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102859-0" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102861-9" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4757_-6)"
+         x="-660.64203"
+         y="343.75699"
+         clip-path="url(#SVGID_4756_-57)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect83804-6" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83806-2"
+       inkscape:transform-center-x="565.64033"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83808-9">
+        <rect
+           id="SVGID_4758_-6"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102867-3">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4758_-6"
+           overflow="visible"
+           id="use102869-5" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4759_-5)"
+         d="m -658.582,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83814-1" />
+      <rect
+         style="fill:#ffffff"
+         x="-650.77399"
+         y="341.65302"
+         clip-path="url(#SVGID_4759_-5)"
+         width="9.592"
+         height="9.592"
+         id="rect83816-2" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-650.77399"
+         y="341.65302"
+         clip-path="url(#SVGID_4759_-5)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83818-3" />
+      <rect
+         style="fill:#81d0db"
+         x="-650.10303"
+         y="342.32599"
+         clip-path="url(#SVGID_4759_-5)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83820-0" />
+      <rect
+         style="fill:#ffffff"
+         x="-649.44299"
+         y="342.98401"
+         clip-path="url(#SVGID_4759_-5)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83822-4" />
+      <rect
+         style="fill:#ffffff"
+         x="-648.26398"
+         y="344.164"
+         clip-path="url(#SVGID_4759_-5)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83824-9" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83826-5"
+       inkscape:transform-center-x="557.09937"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83828-3">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4760_-8"
+           d="m -648.671,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102880-6">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4760_-8"
+           overflow="visible"
+           id="use102882-7" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102884-8"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3541"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102886-2" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102888-2" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4762_-0)"
+         x="-648.67102"
+         y="343.75699"
+         clip-path="url(#SVGID_4761_-32)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect83839-2" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83841-0"
+       inkscape:transform-center-x="538.70558"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83843-2">
+        <rect
+           id="SVGID_4763_-6"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102894-0">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4763_-6"
+           overflow="visible"
+           id="use102896-6" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4764_-8)"
+         d="m -646.611,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83849-9" />
+      <rect
+         style="fill:#ffffff"
+         x="-638.80298"
+         y="341.65302"
+         clip-path="url(#SVGID_4764_-8)"
+         width="9.592"
+         height="9.592"
+         id="rect83851-9" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-638.80298"
+         y="341.65302"
+         clip-path="url(#SVGID_4764_-8)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83853-5" />
+      <rect
+         style="fill:#81d0db"
+         x="-638.13202"
+         y="342.32599"
+         clip-path="url(#SVGID_4764_-8)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83855-9" />
+      <rect
+         style="fill:#ffffff"
+         x="-637.47198"
+         y="342.98401"
+         clip-path="url(#SVGID_4764_-8)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83857-0" />
+      <rect
+         style="fill:#ffffff"
+         x="-636.29199"
+         y="344.164"
+         clip-path="url(#SVGID_4764_-8)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83859-1" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83861-1"
+       inkscape:transform-center-x="530.16231"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83863-9">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4765_-7"
+           d="m -636.699,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102907-7">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4765_-7"
+           overflow="visible"
+           id="use102909-7" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102911-0"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4299"
+         x2="256.3541"
+         y2="-148.4299"
+         gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102913-9" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102915-0" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4767_-9)"
+         x="-636.69897"
+         y="343.75699"
+         clip-path="url(#SVGID_4766_-0)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect83874-3" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83876-5"
+       inkscape:transform-center-x="511.76858"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83878-1">
+        <rect
+           id="SVGID_4768_-0"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102921-8">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4768_-0"
+           overflow="visible"
+           id="use102923-0" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4769_-5)"
+         d="m -634.639,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83884-6" />
+      <rect
+         style="fill:#ffffff"
+         x="-626.83099"
+         y="341.65302"
+         clip-path="url(#SVGID_4769_-5)"
+         width="9.592"
+         height="9.592"
+         id="rect83886-7" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-626.83099"
+         y="341.65302"
+         clip-path="url(#SVGID_4769_-5)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83888-0" />
+      <rect
+         style="fill:#81d0db"
+         x="-626.15997"
+         y="342.32599"
+         clip-path="url(#SVGID_4769_-5)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83890-1" />
+      <rect
+         style="fill:#ffffff"
+         x="-625.5"
+         y="342.98401"
+         clip-path="url(#SVGID_4769_-5)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83892-0" />
+      <rect
+         style="fill:#ffffff"
+         x="-624.32098"
+         y="344.164"
+         clip-path="url(#SVGID_4769_-5)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83894-2" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83896-1"
+       inkscape:transform-center-x="503.22762"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83898-4">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4770_-0"
+           d="m -624.728,346.449 c 0,1.487 1.206,2.692 2.693,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.693,1.206 -2.693,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102934-3">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4770_-0"
+           overflow="visible"
+           id="use102936-2" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102938-9"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3541"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,177.1768,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102940-4" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102942-8" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4772_-5)"
+         x="-624.72803"
+         y="343.75699"
+         clip-path="url(#SVGID_4771_-8)"
+         width="5.3839998"
+         height="5.3839998"
+         id="rect83909-9" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83911-4"
+       inkscape:transform-center-x="484.83387"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83913-1">
+        <rect
+           id="SVGID_4773_-2"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102948-6">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4773_-2"
+           overflow="visible"
+           id="use102950-1" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4774_-60)"
+         d="m -622.668,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83919-9" />
+      <rect
+         style="fill:#ffffff"
+         x="-614.85999"
+         y="341.65302"
+         clip-path="url(#SVGID_4774_-60)"
+         width="9.5930004"
+         height="9.592"
+         id="rect83921-3" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-614.86102"
+         y="341.65302"
+         clip-path="url(#SVGID_4774_-60)"
+         stroke-miterlimit="10"
+         width="9.5930004"
+         height="9.592"
+         id="rect83923-1" />
+      <rect
+         style="fill:#81d0db"
+         x="-614.18799"
+         y="342.32599"
+         clip-path="url(#SVGID_4774_-60)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83925-1" />
+      <rect
+         style="fill:#ffffff"
+         x="-613.52899"
+         y="342.98401"
+         clip-path="url(#SVGID_4774_-60)"
+         width="6.9310002"
+         height="6.9299998"
+         id="rect83927-5" />
+      <rect
+         style="fill:#ffffff"
+         x="-612.349"
+         y="344.164"
+         clip-path="url(#SVGID_4774_-60)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83929-6" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83931-6"
+       inkscape:transform-center-x="476.2917"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83933-7">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4775_-6"
+           d="m -612.756,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102961-7">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4775_-6"
+           overflow="visible"
+           id="use102963-9" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102965-3"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3541"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102967-4" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102969-9" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4777_-42)"
+         x="-612.75598"
+         y="343.75699"
+         clip-path="url(#SVGID_4776_-8)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect83944-6" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83946-6"
+       inkscape:transform-center-x="457.89686"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83948-3">
+        <rect
+           id="SVGID_4778_-7"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath102975-2">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4778_-7"
+           overflow="visible"
+           id="use102977-3" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4779_-83)"
+         d="m -610.696,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83954-0" />
+      <rect
+         style="fill:#ffffff"
+         x="-602.888"
+         y="341.65302"
+         clip-path="url(#SVGID_4779_-83)"
+         width="9.592"
+         height="9.592"
+         id="rect83956-5" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-602.888"
+         y="341.65302"
+         clip-path="url(#SVGID_4779_-83)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83958-2" />
+      <rect
+         style="fill:#81d0db"
+         x="-602.21698"
+         y="342.32599"
+         clip-path="url(#SVGID_4779_-83)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83960-4" />
+      <rect
+         style="fill:#ffffff"
+         x="-601.55701"
+         y="342.98401"
+         clip-path="url(#SVGID_4779_-83)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83962-5" />
+      <rect
+         style="fill:#ffffff"
+         x="-600.37799"
+         y="344.164"
+         clip-path="url(#SVGID_4779_-83)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83964-3" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83966-0"
+       inkscape:transform-center-x="449.35694"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs83968-6">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4780_-6"
+           d="m -600.785,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath102988-6">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4780_-6"
+           overflow="visible"
+           id="use102990-7" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient102992-7"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.4301"
+         x2="256.3541"
+         y2="-148.4301"
+         gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop102994-1" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop102996-1" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4782_-6)"
+         x="-600.78497"
+         y="343.75699"
+         clip-path="url(#SVGID_4781_-0)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect83979-0" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g83981-7"
+       inkscape:transform-center-x="430.96208"
+       inkscape:transform-center-y="98.159342">
+      <defs
+         id="defs83983-7">
+        <rect
+           id="SVGID_4783_-9"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath103002-6">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4783_-9"
+           overflow="visible"
+           id="use103004-4" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4784_-5)"
+         d="m -598.725,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path83989-6" />
+      <rect
+         style="fill:#ffffff"
+         x="-590.91699"
+         y="341.65302"
+         clip-path="url(#SVGID_4784_-5)"
+         width="9.592"
+         height="9.592"
+         id="rect83991-7" />
+      <rect
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-590.91699"
+         y="341.65302"
+         clip-path="url(#SVGID_4784_-5)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.592"
+         id="rect83993-9" />
+      <rect
+         style="fill:#81d0db"
+         x="-590.24597"
+         y="342.32599"
+         clip-path="url(#SVGID_4784_-5)"
+         width="8.2480001"
+         height="8.2469997"
+         id="rect83995-0" />
+      <rect
+         style="fill:#ffffff"
+         x="-589.586"
+         y="342.98401"
+         clip-path="url(#SVGID_4784_-5)"
+         width="6.9299998"
+         height="6.9299998"
+         id="rect83997-8" />
+      <rect
+         style="fill:#ffffff"
+         x="-588.40698"
+         y="344.164"
+         clip-path="url(#SVGID_4784_-5)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect83999-5" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g84001-7"
+       inkscape:transform-center-x="422.41995"
+       inkscape:transform-center-y="98.159295">
+      <defs
+         id="defs84003-1">
+        <path
+           inkscape:connector-curvature="0"
+           id="SVGID_4785_-4"
+           d="m -588.813,346.449 c 0,1.487 1.205,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692" />
+      </defs>
+      <clipPath
+         id="clipPath103015-1">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4785_-4"
+           overflow="visible"
+           id="use103017-4" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient103019-6"
+         gradientUnits="userSpaceOnUse"
+         x1="255.3541"
+         y1="-148.42999"
+         x2="256.3541"
+         y2="-148.42999"
+         gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1031.1833)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop103021-6" />
+        <stop
+           offset="1"
+           style="stop-color:#81D1DB"
+           id="stop103023-8" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4787_-8)"
+         x="-588.81299"
+         y="343.75699"
+         clip-path="url(#SVGID_4786_-1)"
+         width="5.3829999"
+         height="5.3839998"
+         id="rect84014-2" />
+    </g>
+    <text
+       y="567.96027"
+       x="435.21774"
+       id="text84236-1"
+       inkscape:transform-center-x="587.42372"
+       inkscape:transform-center-y="-90.601429"
+       style="font-size:27px">
+      <tspan
+         style="font-size:8px;fill:#37424b;font-family:ApexSans-Medium"
+         x="435.21774"
+         y="567.96027"
+         font-size="3.4203"
+         id="tspan84238-2">EU-EAST</tspan>
+    </text>
+    <rect
+       style="fill:#f15d57"
+       x="399.26007"
+       y="492.96008"
+       width="103.6395"
+       height="3.8114998"
+       id="rect84318-4"
+       inkscape:transform-center-x="587.03108"
+       inkscape:transform-center-y="-160.89453" />
+    <g
+       transform="translate(328.69122,-0.03041)"
+       id="g9328-2">
+      <line
+         id="line84244-3-6"
+         y2="533.36865"
+         x2="121.81088"
+         y1="315.62341"
+         x1="121.81088"
+         stroke-miterlimit="10"
+         style="fill:none;stroke:#5e6a71;stroke-width:1.0268178;stroke-miterlimit:10;stroke-dasharray:3.07565499, 3.07565499" />
+      <g
+         id="g82107-4-5"
+         transform="matrix(3.375,0,0,3.375,1898.5131,-334.69083)">
+        <defs
+           id="defs82109-6-6">
+          <polygon
+             points="-504.975,196.685 -504.975,188.287 -547.302,188.287 -547.302,196.684 -528.257,196.685 -526.139,198.478 -524.02,196.685 "
+             id="SVGID_4519_-5-8" />
+        </defs>
+        <clipPath
+           id="clipPath126820-6-9">
+          <use
+             id="use126822-8-2"
+             overflow="visible"
+             xlink:href="#SVGID_4519_-5-8"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)"
+           y2="146.6937"
+           x2="41.840599"
+           y1="146.6937"
+           x1="40.840599"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient126824-4-4">
+          <stop
+             id="stop126826-2-0"
+             style="stop-color:#37424B"
+             offset="0" />
+          <stop
+             id="stop126828-3-6"
+             style="stop-color:#37424B"
+             offset="0.12" />
+          <stop
+             id="stop126830-7-8"
+             style="stop-color:#5E6A71"
+             offset="0.88" />
+          <stop
+             id="stop126832-6-2"
+             style="stop-color:#5E6A71"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect82124-4-8"
+           height="10.191"
+           width="42.327"
+           clip-path="url(#SVGID_4520_-5-6-7)"
+           y="188.287"
+           x="-547.302"
+           style="fill:url(#linearGradient126904-61-3)" />
+      </g>
+      <g
+         id="g82126-0-8"
+         transform="matrix(3.375,0,0,3.375,1898.5131,-334.69083)">
+        <defs
+           id="defs82128-8-2">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4522_-8-1" />
+        </defs>
+        <clipPath
+           id="clipPath126838-9-7">
+          <use
+             id="use126840-87-3"
+             overflow="visible"
+             xlink:href="#SVGID_4522_-8-1"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path82134-5-4"
+           d="m -539.397,191.908 c -0.353,0 -0.64,-0.286 -0.64,-0.639 0,-0.354 0.287,-0.642 0.64,-0.641 0.354,0 0.639,0.287 0.639,0.641 0,0.353 -0.285,0.639 -0.639,0.639"
+           clip-path="url(#SVGID_4523_-1-5-1)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <path
+           id="path82136-0-8"
+           d="m -539.397,194.263 c -0.353,0 -0.64,-0.286 -0.64,-0.639 0,-0.355 0.287,-0.642 0.64,-0.642 0.354,0 0.639,0.287 0.639,0.642 0,0.353 -0.285,0.639 -0.639,0.639"
+           clip-path="url(#SVGID_4523_-1-5-1)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <circle
+           id="circle82138-2-3"
+           r="0.63999999"
+           cy="191.26801"
+           cx="-541.625"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4523_-1-5-1)"
+           d="m -540.985,191.26801 c 0,0.35346 -0.28654,0.64 -0.64,0.64 -0.35346,0 -0.64,-0.28654 -0.64,-0.64 0,-0.35347 0.28654,-0.64 0.64,-0.64 0.35346,0 0.64,0.28653 0.64,0.64 z"
+           sodipodi:cx="-541.625"
+           sodipodi:cy="191.26801"
+           sodipodi:rx="0.63999999"
+           sodipodi:ry="0.63999999"
+           style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10" />
+        <circle
+           id="circle82140-3-7"
+           r="0.63999999"
+           cy="193.623"
+           cx="-541.625"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4523_-1-5-1)"
+           d="m -540.985,193.623 c 0,0.35346 -0.28654,0.64 -0.64,0.64 -0.35346,0 -0.64,-0.28654 -0.64,-0.64 0,-0.35346 0.28654,-0.64 0.64,-0.64 0.35346,0 0.64,0.28654 0.64,0.64 z"
+           sodipodi:cx="-541.625"
+           sodipodi:cy="193.623"
+           sodipodi:rx="0.63999999"
+           sodipodi:ry="0.63999999"
+           style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10" />
+        <circle
+           id="circle82142-3-6"
+           r="0.63999999"
+           cy="191.26801"
+           cx="-543.85303"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4523_-1-5-1)"
+           d="m -543.21303,191.26801 c 0,0.35346 -0.28654,0.64 -0.64,0.64 -0.35346,0 -0.64,-0.28654 -0.64,-0.64 0,-0.35347 0.28654,-0.64 0.64,-0.64 0.35346,0 0.64,0.28653 0.64,0.64 z"
+           sodipodi:cx="-543.85303"
+           sodipodi:cy="191.26801"
+           sodipodi:rx="0.63999999"
+           sodipodi:ry="0.63999999"
+           style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10" />
+        <circle
+           id="circle82144-4-4"
+           r="0.63999999"
+           cy="193.623"
+           cx="-543.854"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4523_-1-5-1)"
+           d="m -543.214,193.623 c 0,0.35346 -0.28654,0.64 -0.64,0.64 -0.35347,0 -0.64,-0.28654 -0.64,-0.64 0,-0.35346 0.28653,-0.64 0.64,-0.64 0.35346,0 0.64,0.28654 0.64,0.64 z"
+           sodipodi:cx="-543.854"
+           sodipodi:cy="193.623"
+           sodipodi:rx="0.63999999"
+           sodipodi:ry="0.63999999"
+           style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10" />
+        <path
+           id="path82146-1-5"
+           d="m -537.169,191.908 c -0.354,0 -0.64,-0.286 -0.64,-0.639 0,-0.354 0.286,-0.642 0.64,-0.641 0.354,0 0.64,0.287 0.64,0.641 0,0.353 -0.286,0.639 -0.64,0.639"
+           clip-path="url(#SVGID_4523_-1-5-1)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <path
+           id="path82148-5-2"
+           d="m -537.169,194.263 c -0.354,0 -0.64,-0.286 -0.64,-0.639 0,-0.355 0.286,-0.642 0.64,-0.642 0.354,0 0.64,0.287 0.64,0.642 0,0.353 -0.286,0.639 -0.64,0.639"
+           clip-path="url(#SVGID_4523_-1-5-1)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+      </g>
+      <text
+         style="font-size:40.5px"
+         id="text82150-7-1"
+         x="112.11884"
+         y="319.30515">
+        <tspan
+           id="tspan82152-4-3"
+           font-size="3.2027"
+           y="319.30515"
+           x="112.11884"
+           style="font-size:10.80911255px;fill:#ffffff;font-family:ApexSans-Medium">AP</tspan>
+        <tspan
+           id="tspan82154-9-7"
+           font-size="3.2027"
+           y="319.30515"
+           x="125.80093"
+           style="font-size:10.80911255px;fill:#ffffff;font-family:ApexSans-Medium">P</tspan>
+      </text>
+      <g
+         id="g84638-4-9"
+         transform="matrix(3.375,0,0,3.375,1898.5131,-334.69083)">
+        <defs
+           id="defs84640-9-8">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4852_-4-9" />
+        </defs>
+        <clipPath
+           id="clipPath126879-0-5">
+          <use
+             id="use126881-7-3"
+             overflow="visible"
+             xlink:href="#SVGID_4852_-4-9"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path84646-0-7"
+           d="m -526.673,205.991 c 0,-0.349 0.282,-0.632 0.632,-0.632 0.349,0 0.632,0.283 0.632,0.632 0,0.35 -0.283,0.633 -0.632,0.633 -0.35,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4853_-8-0-4)"
+           inkscape:connector-curvature="0"
+           style="fill:#5e6a71" />
+        <line
+           id="line84648-6-5"
+           y2="205.991"
+           x2="-523.14398"
+           y1="205.991"
+           x1="-526.03998"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4853_-8-0-4)"
+           style="fill:none;stroke:#5e6a71;stroke-width:0.214;stroke-miterlimit:10" />
+      </g>
+      <g
+         id="g84650-5-5"
+         transform="matrix(3.375,0,0,3.375,1898.5131,-334.69083)">
+        <defs
+           id="defs84652-4-9">
+          <polygon
+             points="-521.64,203.458 -523.432,206 -521.64,208.524 -507.673,208.524 -507.673,203.458 "
+             id="SVGID_4854_-1-2" />
+        </defs>
+        <clipPath
+           id="clipPath126888-1-5">
+          <use
+             id="use126890-4-7"
+             overflow="visible"
+             xlink:href="#SVGID_4854_-1-2"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)"
+           y2="249.94031"
+           x2="-33.205101"
+           y1="249.94031"
+           x1="-34.205299"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient126892-7-0">
+          <stop
+             id="stop126894-5-7"
+             style="stop-color:#E6E8E7"
+             offset="0" />
+          <stop
+             id="stop126896-6-7"
+             style="stop-color:#FFFFFF"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect84663-7-0"
+           height="5.066"
+           width="15.759"
+           clip-path="url(#SVGID_4855_-2-24-5)"
+           y="203.45799"
+           x="-523.43201"
+           style="fill:url(#SVGID_4856_-4-1-0)" />
+      </g>
+      <polygon
+         id="polygon84665-6-1"
+         points="-521.64,203.459 -523.433,206 -521.64,208.524 -507.674,208.524 -507.674,203.459 "
+         stroke-miterlimit="10"
+         transform="matrix(3.375,0,0,3.375,1898.5131,-334.69083)"
+         style="fill:none;stroke:#e6e8e8;stroke-width:0.214;stroke-miterlimit:10" />
+      <text
+         style="font-size:40.5px"
+         id="text84667-5-9"
+         x="151.0986"
+         y="363.4711">
+        <tspan
+           id="tspan84669-5-9"
+           font-size="2.5652"
+           y="363.4711"
+           x="151.0986"
+           style="font-size:8.65755081px;fill:#5e6a71;font-family:ApexSans-Book">RE</tspan>
+        <tspan
+           id="tspan84671-5-4"
+           font-size="2.5652"
+           y="363.4711"
+           x="160.27521"
+           style="font-size:8.65755081px;fill:#5e6a71;font-family:ApexSans-Book">S</tspan>
+        <tspan
+           id="tspan84673-2-5"
+           font-size="2.5652"
+           y="363.4711"
+           x="164.41296"
+           style="font-size:8.65755081px;fill:#5e6a71;font-family:ApexSans-Book">T</tspan>
+      </text>
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g84320-7"
+       inkscape:transform-center-x="587.03111"
+       inkscape:transform-center-y="-131.54432">
+      <defs
+         id="defs84322-0">
+        <polygon
+           id="SVGID_4826_-2"
+           points="-659.342,256.788 -636.356,251.283 -636.358,231.934 -682.207,231.929 -682.208,251.221 " />
+      </defs>
+      <clipPath
+         id="clipPath103260-5">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4826_-2"
+           overflow="visible"
+           id="use103262-4" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient103264-9"
+         gradientUnits="userSpaceOnUse"
+         x1="145.3168"
+         y1="2.8518"
+         x2="146.3168"
+         y2="2.8518"
+         gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-734.6116,-3384.4426)">
+        <stop
+           offset="0"
+           style="stop-color:#FFFFFF"
+           id="stop103266-7" />
+        <stop
+           offset="0.23"
+           style="stop-color:#FFFFFF"
+           id="stop103268-8" />
+        <stop
+           offset="0.5671"
+           style="stop-color:#F9F9F9"
+           id="stop103270-3" />
+        <stop
+           offset="0.8674"
+           style="stop-color:#EDEEEE"
+           id="stop103272-6" />
+        <stop
+           offset="1"
+           style="stop-color:#E6E8E7"
+           id="stop103274-7" />
+      </linearGradient>
+      <polygon
+         style="fill:url(#SVGID_4828_-8)"
+         clip-path="url(#SVGID_4827_-1)"
+         points="-682.238,231.929 -636.356,231.874 -636.326,256.788 -682.208,256.843 "
+         id="polygon84339-1" />
+    </g>
+    <path
+       style="fill:#e6e8e8"
+       inkscape:connector-curvature="0"
+       d="m 399.73931,496.49039 102.67875,0.009 10e-4,43.1055 -51.4755,12.33 -51.20775,-12.46725 0.002,-42.97725 z m -0.4815,-0.4815 v 43.83675 l 51.687,12.58425 51.957,-12.44475 -0.005,-43.96725 -103.6395,-0.009 z"
+       id="path84341-5"
+       inkscape:transform-center-x="587.03109"
+       inkscape:transform-center-y="-131.54096" />
+    <text
+       y="513.25391"
+       x="422.89722"
+       id="text84343-4"
+       inkscape:transform-center-x="587.62742"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px">
+      <tspan
+         style="font-size:11px;fill:#37424b;font-family:ApexSans-Book"
+         x="422.89722"
+         y="513.25391"
+         font-size="5.1244"
+         id="tspan84345-6">RADOSGW</tspan>
+    </text>
+    <line
+       style="fill:none;stroke:#5e6a71;stroke-width:0.48150003;stroke-miterlimit:10;stroke-dasharray:1.44225, 1.44225"
+       stroke-miterlimit="10"
+       x1="450.98538"
+       y1="577.31482"
+       x2="450.98538"
+       y2="631.69733"
+       id="line84349-2"
+       inkscape:transform-center-x="587.12555"
+       inkscape:transform-center-y="-51.254289" />
+    <polyline
+       style="fill:none;stroke:#5e6a71;stroke-width:0.214;stroke-miterlimit:10;stroke-dasharray:0.641, 0.641"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       stroke-miterlimit="10"
+       points="  -669.837,292.128 -669.837,283.832 -648.812,283.832 -648.812,292.128 "
+       id="polyline84351-0"
+       inkscape:transform-center-x="587.12673"
+       inkscape:transform-center-y="-33.395941" />
+    <polygon
+       style="fill:#81d0db"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       points="-657.251,267.424 -661.37,267.424 -659.319,269.217 "
+       id="polygon84353-1"
+       inkscape:transform-center-x="587.09523"
+       inkscape:transform-center-y="-77.629816" />
+    <polygon
+       id="polygon84546-2"
+       points="-671.896,292.47 -669.845,294.263 -667.777,292.47 "
+       clip-path="url(#SVGID_4843_-5)"
+       style="fill:#f15d57"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <polygon
+       id="polygon84548-8"
+       points="-661.384,292.47 -659.333,294.263 -657.265,292.47 "
+       clip-path="url(#SVGID_4843_-5)"
+       style="fill:#f15d57"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <polygon
+       id="polygon84550-0"
+       points="-650.871,292.47 -648.82,294.263 -646.752,292.47 "
+       clip-path="url(#SVGID_4843_-5)"
+       style="fill:#f15d57"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g84753-5"
+       inkscape:transform-center-x="584.45598"
+       inkscape:transform-center-y="-60.127066">
+      <defs
+         id="defs84755-4">
+        <rect
+           id="SVGID_4867_-0"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath103568-9">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4867_-0"
+           overflow="visible"
+           id="use103570-2" />
+      </clipPath>
+      <path
+         style="fill:#5e6a71"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4868_-1)"
+         d="m -659.955,276.099 c 0,-0.349 0.282,-0.632 0.632,-0.632 0.349,0 0.632,0.283 0.632,0.632 0,0.35 -0.283,0.633 -0.632,0.633 -0.35,0 -0.632,-0.283 -0.632,-0.633"
+         id="path84761-2" />
+      <line
+         style="fill:none;stroke:#5e6a71;stroke-width:0.214;stroke-miterlimit:10"
+         clip-path="url(#SVGID_4868_-1)"
+         stroke-miterlimit="10"
+         x1="-659.323"
+         y1="276.099"
+         x2="-656.427"
+         y2="276.099"
+         id="line84763-3" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       id="g84765-0"
+       inkscape:transform-center-x="563.5265"
+       inkscape:transform-center-y="-60.127083">
+      <defs
+         id="defs84767-2">
+        <polygon
+           id="SVGID_4869_-7"
+           points="-656.715,276.108 -654.922,278.632 -640.956,278.632 -640.956,273.567 -654.922,273.567 " />
+      </defs>
+      <clipPath
+         id="clipPath103577-0">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4869_-7"
+           overflow="visible"
+           id="use103579-1" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient103581-2"
+         gradientUnits="userSpaceOnUse"
+         x1="-34.205299"
+         y1="249.94051"
+         x2="-33.205299"
+         y2="249.94051"
+         gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,105.3813)">
+        <stop
+           offset="0"
+           style="stop-color:#E6E8E7"
+           id="stop103583-7" />
+        <stop
+           offset="1"
+           style="stop-color:#FFFFFF"
+           id="stop103585-7" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4871_-4)"
+         x="-656.71503"
+         y="273.56699"
+         clip-path="url(#SVGID_4870_-0)"
+         width="15.759"
+         height="5.0650001"
+         id="rect84778-5" />
+    </g>
+    <polygon
+       style="fill:none;stroke:#e6e8e8;stroke-width:0.214;stroke-miterlimit:10"
+       transform="matrix(2.25,0,0,2.25,1934.4644,-25.59059)"
+       stroke-miterlimit="10"
+       points="-656.715,276.108 -654.922,278.632 -640.956,278.632 -640.956,273.567 -654.922,273.567 "
+       id="polygon84780-4"
+       inkscape:transform-center-x="563.52648"
+       inkscape:transform-center-y="-60.127066" />
+    <text
+       y="597.59412"
+       x="466.59583"
+       id="text84782-6"
+       inkscape:transform-center-x="562.26258"
+       inkscape:transform-center-y="-60.270044"
+       style="font-size:27px">
+      <tspan
+         style="font-size:5.77170038px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="466.59583"
+         y="597.59412"
+         font-size="2.5652"
+         id="tspan84784-6">N</tspan>
+      <tspan
+         style="font-size:5.77170038px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="470.47482"
+         y="597.59412"
+         font-size="2.5652"
+         id="tspan84786-0">A</tspan>
+      <tspan
+         style="font-size:5.77170038px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="473.54385"
+         y="597.59412"
+         font-size="2.5652"
+         id="tspan84788-6">TIV</tspan>
+      <tspan
+         style="font-size:5.77170038px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="481.25684"
+         y="597.59412"
+         font-size="2.5652"
+         id="tspan84790-4">E</tspan>
+    </text>
+    <g
+       transform="matrix(1.3542118,0,0,2.25,1325.4162,-31.59059)"
+       id="g84792-1"
+       inkscape:transform-center-x="262.77489"
+       inkscape:transform-center-y="141.90493">
+      <defs
+         id="defs84794-1">
+        <rect
+           id="SVGID_4872_-8"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath103597-2">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4872_-8"
+           overflow="visible"
+           id="use103599-5" />
+      </clipPath>
+      <path
+         style="fill:#5e6a71"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4873_-98)"
+         d="m -438.652,188.287 c 0,0.349 -0.282,0.632 -0.632,0.632 -0.349,0 -0.632,-0.283 -0.632,-0.632 0,-0.35 0.283,-0.633 0.632,-0.633 0.35,0 0.632,0.284 0.632,0.633"
+         id="path84800-2" />
+      <g
+         clip-path="url(#SVGID_4873_-98)"
+         id="g84802-7">
+        <g
+           id="g84804-9">
+          <defs
+             id="defs84806-7">
+            <rect
+               id="SVGID_4874_-4"
+               x="-679.90002"
+               y="360.13901"
+               width="174.95799"
+               height="10.152" />
+          </defs>
+          <clipPath
+             id="clipPath103606-3">
+            <use
+               height="1052.3622"
+               width="744.09448"
+               y="0"
+               x="0"
+               style="overflow:visible"
+               xlink:href="#SVGID_4874_-4"
+               overflow="visible"
+               id="use103608-9" />
+          </clipPath>
+          <g
+             clip-path="url(#SVGID_4875_-1)"
+             id="g84812-3">
+            <defs
+               id="defs84814-8">
+              <rect
+                 id="SVGID_4876_-1"
+                 x="-679.90002"
+                 y="360.13901"
+                 width="174.95799"
+                 height="10.152" />
+            </defs>
+            <clipPath
+               id="clipPath103613-6">
+              <use
+                 height="1052.3622"
+                 width="744.09448"
+                 y="0"
+                 x="0"
+                 style="overflow:visible"
+                 xlink:href="#SVGID_4876_-1"
+                 overflow="visible"
+                 id="use103615-0" />
+            </clipPath>
+            <linearGradient
+               id="linearGradient103617-6"
+               gradientUnits="userSpaceOnUse"
+               x1="40.563"
+               y1="147.0757"
+               x2="41.563"
+               y2="147.0757"
+               gradientTransform="matrix(0,-10.1526,-10.1526,0,900.78,782.1111)">
+              <stop
+                 offset="0"
+                 style="stop-color:#55AEB9"
+                 id="stop103619-5" />
+              <stop
+                 offset="0.486"
+                 style="stop-color:#B0D2D9"
+                 id="stop103621-2" />
+              <stop
+                 offset="0.8287"
+                 style="stop-color:#E6EFF1"
+                 id="stop103623-9" />
+              <stop
+                 offset="0.9939"
+                 style="stop-color:#FFFFFF"
+                 id="stop103625-9" />
+              <stop
+                 offset="1"
+                 style="stop-color:#FFFFFF"
+                 id="stop103627-0" />
+            </linearGradient>
+            <rect
+               style="fill:url(#SVGID_4878_-2)"
+               x="-679.90002"
+               y="360.138"
+               clip-path="url(#SVGID_4877_-4)"
+               width="174.95799"
+               height="10.153"
+               id="rect84831-7" />
+          </g>
+        </g>
+      </g>
+      <g
+         clip-path="url(#SVGID_4873_-98)"
+         id="g84833-5">
+        <g
+           id="g84835-6">
+          <defs
+             id="defs84837-5">
+            <rect
+               id="SVGID_4879_-3"
+               x="-679.88898"
+               y="360.13901"
+               width="174.94701"
+               height="10.152" />
+          </defs>
+          <clipPath
+             id="clipPath103634-6">
+            <use
+               height="1052.3622"
+               width="744.09448"
+               y="0"
+               x="0"
+               style="overflow:visible"
+               xlink:href="#SVGID_4879_-3"
+               overflow="visible"
+               id="use103636-3" />
+          </clipPath>
+          <g
+             clip-path="url(#SVGID_4880_-2)"
+             id="g84843-9">
+            <defs
+               id="defs84845-0">
+              <rect
+                 id="SVGID_4881_-4"
+                 x="-679.88898"
+                 y="360.138"
+                 width="174.94701"
+                 height="10.153" />
+            </defs>
+            <clipPath
+               id="clipPath103641-9">
+              <use
+                 height="1052.3622"
+                 width="744.09448"
+                 y="0"
+                 x="0"
+                 style="overflow:visible"
+                 xlink:href="#SVGID_4881_-4"
+                 overflow="visible"
+                 id="use103643-2" />
+            </clipPath>
+            <polygon
+               style="fill:#f15d57"
+               clip-path="url(#SVGID_4882_-3)"
+               points="-504.943,360.138 -679.89,360.139 -679.89,364.81 -679.89,367.139 -679.89,370.291 -504.943,370.291 "
+               id="polygon84851-1" />
+          </g>
+        </g>
+      </g>
+      <rect
+         style="fill:#f15d57"
+         x="-679.90198"
+         y="360.267"
+         clip-path="url(#SVGID_4873_-98)"
+         width="174.96001"
+         height="2.641"
+         id="rect84853-2" />
+      <rect
+         style="fill:#55aeb9"
+         x="-679.88898"
+         y="370.29199"
+         clip-path="url(#SVGID_4873_-98)"
+         width="174.94701"
+         height="1.352"
+         id="rect84855-1" />
+    </g>
+    <text
+       y="832.0752"
+       x="571.10132"
+       id="text84343-2-09"
+       inkscape:transform-center-x="587.62742"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px;text-align:center;text-anchor:middle">
+      <tspan
+         style="font-size:11px;text-align:center;text-anchor:middle;fill:#37424b;font-family:ApexSans-Book"
+         x="526.12915"
+         y="832.0752"
+         font-size="5.1244"
+         id="tspan84345-2-7">SECONDARY REGION</tspan>
+    </text>
+    <polygon
+       id="polygon84546-9-3-6"
+       points="-667.777,292.47 -671.896,292.47 -669.845,294.263 "
+       clip-path="url(#SVGID_4843_-9-6-5)"
+       style="fill:#f15d57"
+       transform="matrix(2.25,0,0,2.25,1917.6233,-169.74298)" />
+    <text
+       xml:space="preserve"
+       style="font-size:9px;font-style:normal;font-weight:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans"
+       x="450.92264"
+       y="529.61548"
+       id="text126655-6-3"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan126657-2-8"
+         x="450.92264"
+         y="529.61548">MASTER</tspan><tspan
+         sodipodi:role="line"
+         x="450.92264"
+         y="540.86548"
+         id="tspan126659-4-8">ZONE</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-size:9px;font-style:normal;font-weight:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans"
+       x="363.76648"
+       y="400.13824"
+       id="text127418-7"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan127420-8"
+         x="363.76648"
+         y="400.13824">METADATA</tspan><tspan
+         sodipodi:role="line"
+         x="363.76648"
+         y="411.38824"
+         id="tspan127422-7">SYNC</tspan></text>
+    <text
+       y="247.57471"
+       x="491.10132"
+       id="text84343-2-0-2"
+       inkscape:transform-center-x="587.62742"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px;text-align:center;text-anchor:middle">
+      <tspan
+         style="font-size:11px;text-align:center;text-anchor:middle;fill:#37424b;font-family:ApexSans-Book"
+         x="446.12915"
+         y="247.57471"
+         font-size="5.1244"
+         id="tspan84345-2-6-0">SECONDARY REGION</tspan>
+    </text>
+    <text
+       y="705.34888"
+       x="583.92261"
+       style="font-size:9.75217533px;fill:#ffffff;font-family:ApexSans-Medium"
+       font-size="4.3343"
+       id="text82351-9-7"
+       inkscape:transform-center-x="450.06672"
+       inkscape:transform-center-y="45.543285">M</text>
+    <g
+       id="g9504-6"
+       transform="translate(98.691216,33.96959)">
+      <g
+         inkscape:transform-center-y="45.666809"
+         inkscape:transform-center-x="449.81598"
+         id="g82339-6-5"
+         transform="matrix(2.25,0,0,2.25,1701.0081,-85.193202)">
+        <defs
+           id="defs82341-9-3">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4549_-8-0" />
+        </defs>
+        <clipPath
+           id="clipPath101738-3-9">
+          <use
+             id="use101740-3-4"
+             overflow="visible"
+             xlink:href="#SVGID_4549_-8-0"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path82347-9-8"
+           d="m -602.031,326.467 c 0,0.212 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.173 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+           clip-path="url(#SVGID_4550_-6-4)"
+           inkscape:connector-curvature="0"
+           style="fill:#f15d57" />
+        <path
+           id="path82349-9-3"
+           d="m -602.031,326.467 c 0,0.212 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.173 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4550_-6-4)"
+           inkscape:connector-curvature="0"
+           style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10" />
+      </g>
+      <text
+         inkscape:transform-center-y="45.543285"
+         inkscape:transform-center-x="450.06672"
+         id="text82351-9-4-7"
+         font-size="4.3343"
+         style="font-size:9.75217533px;fill:#ffffff;font-family:ApexSans-Medium"
+         x="350.6293"
+         y="645.37921">M</text>
+    </g>
+    <path
+       id="path82449-2-4"
+       d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+       clip-path="url(#SVGID_4564_-7-39)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,2096.2338,-25.49647)" />
+    <path
+       id="path82449-2-2-4"
+       d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+       clip-path="url(#SVGID_4564_-7-3-3)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,1907.5274,-25.10599)" />
+    <path
+       id="path82449-2-0-6"
+       d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+       clip-path="url(#SVGID_4564_-7-9-8)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,2096.2338,0.56753)" />
+    <path
+       id="path82449-2-0-1-9"
+       d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+       clip-path="url(#SVGID_4564_-7-9-3-43)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,2096.3338,26.56753)" />
+    <path
+       id="path82449-2-0-1-5-7"
+       d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+       clip-path="url(#SVGID_4564_-7-9-3-4-4)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(2.25,0,0,2.25,2096.4335,53.16753)" />
+    <text
+       y="293.44104"
+       x="496.78619"
+       id="text84343-2-0-4-8"
+       inkscape:transform-center-x="587.62742"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px;text-align:center;text-anchor:middle">
+      <tspan
+         style="font-size:11px;text-align:center;text-anchor:middle;fill:#37424b;font-family:ApexSans-Book"
+         x="451.81403"
+         y="293.44104"
+         font-size="5.1244"
+         id="tspan84345-2-6-5-9">WRITE / READ</tspan>
+    </text>
+    <text
+       y="262.3537"
+       x="490.88916"
+       id="text84343-2-0-4-6-1"
+       inkscape:transform-center-x="587.62742"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px;text-align:center;text-anchor:middle">
+      <tspan
+         style="font-size:11px;text-align:center;text-anchor:middle;fill:#37424b;font-family:ApexSans-Book"
+         x="445.91699"
+         y="262.3537"
+         font-size="5.1244"
+         id="tspan84345-2-6-5-6-6">(Europe)</tspan>
+    </text>
+    <g
+       id="g15866"
+       transform="translate(66.384186,10.199929)">
+      <rect
+         inkscape:transform-center-y="19.597191"
+         inkscape:transform-center-x="610.96881"
+         id="rect82381-2-3"
+         height="21.584251"
+         width="21.581999"
+         y="680.3775"
+         x="349.96692"
+         style="fill:#ffffff" />
+      <rect
+         inkscape:transform-center-y="19.597191"
+         inkscape:transform-center-x="610.96881"
+         id="rect82383-1-0"
+         height="21.584251"
+         width="21.581999"
+         stroke-miterlimit="10"
+         y="680.3775"
+         x="349.96692"
+         style="fill:none;stroke:#81d0db;stroke-width:0.96300006;stroke-miterlimit:10" />
+      <rect
+         inkscape:transform-center-y="19.598295"
+         inkscape:transform-center-x="610.97115"
+         id="rect82385-3-3"
+         height="18.553501"
+         width="18.557999"
+         y="681.89398"
+         x="351.47656"
+         style="fill:#81d0db" />
+      <rect
+         inkscape:transform-center-y="19.597131"
+         inkscape:transform-center-x="610.96883"
+         id="rect82387-5-5"
+         height="15.594751"
+         width="15.592501"
+         y="683.37219"
+         x="352.96164"
+         style="fill:#ffffff" />
+      <rect
+         inkscape:transform-center-y="19.595982"
+         inkscape:transform-center-x="610.97103"
+         id="rect82389-9-0"
+         height="10.278"
+         width="10.282501"
+         y="686.02936"
+         x="355.61444"
+         style="fill:#ffffff" />
+      <g
+         inkscape:transform-center-y="19.598312"
+         inkscape:transform-center-x="610.97111"
+         id="g82391-8-0"
+         transform="matrix(2.25,0,0,2.25,1868.0802,-9.7784893)">
+        <defs
+           id="defs82393-4-0">
+          <path
+             d="m -672.614,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+             id="SVGID_4555_-7-9"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath101773-5-5">
+          <use
+             id="use101775-6-5"
+             overflow="visible"
+             xlink:href="#SVGID_4555_-7-9"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)"
+           y2="-148.46021"
+           x2="256.3761"
+           y1="-148.46021"
+           x1="255.3761"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient101777-2-8">
+          <stop
+             id="stop101779-4-4"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop101781-1-3"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect82404-9-9"
+           height="5.3839998"
+           width="5.3839998"
+           clip-path="url(#SVGID_4556_-9-2)"
+           y="308.841"
+           x="-672.61401"
+           style="fill:url(#linearGradient15887)" />
+      </g>
+      <path
+         transform="matrix(2.25,0,0,2.25,1841.021,-9.2938893)"
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4564_-7-3-3-7)"
+         d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82449-2-2-4-4" />
+    </g>
+    <g
+       id="g15866-2"
+       transform="translate(66.384186,36.234929)">
+      <rect
+         inkscape:transform-center-y="19.597191"
+         inkscape:transform-center-x="610.96881"
+         id="rect82381-2-3-5"
+         height="21.584251"
+         width="21.581999"
+         y="680.3775"
+         x="349.96692"
+         style="fill:#ffffff" />
+      <rect
+         inkscape:transform-center-y="19.597191"
+         inkscape:transform-center-x="610.96881"
+         id="rect82383-1-0-2"
+         height="21.584251"
+         width="21.581999"
+         stroke-miterlimit="10"
+         y="680.3775"
+         x="349.96692"
+         style="fill:none;stroke:#81d0db;stroke-width:0.96300006;stroke-miterlimit:10" />
+      <rect
+         inkscape:transform-center-y="19.598295"
+         inkscape:transform-center-x="610.97115"
+         id="rect82385-3-3-6"
+         height="18.553501"
+         width="18.557999"
+         y="681.89398"
+         x="351.47656"
+         style="fill:#81d0db" />
+      <rect
+         inkscape:transform-center-y="19.597131"
+         inkscape:transform-center-x="610.96883"
+         id="rect82387-5-5-5"
+         height="15.594751"
+         width="15.592501"
+         y="683.37219"
+         x="352.96164"
+         style="fill:#ffffff" />
+      <rect
+         inkscape:transform-center-y="19.595982"
+         inkscape:transform-center-x="610.97103"
+         id="rect82389-9-0-6"
+         height="10.278"
+         width="10.282501"
+         y="686.02936"
+         x="355.61444"
+         style="fill:#ffffff" />
+      <g
+         inkscape:transform-center-y="19.598312"
+         inkscape:transform-center-x="610.97111"
+         id="g82391-8-0-8"
+         transform="matrix(2.25,0,0,2.25,1868.0802,-9.7784893)">
+        <defs
+           id="defs82393-4-0-3">
+          <path
+             d="m -672.614,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+             id="SVGID_4555_-7-9-1"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath101773-5-5-0">
+          <use
+             id="use101775-6-5-6"
+             overflow="visible"
+             xlink:href="#SVGID_4555_-7-9-1"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)"
+           y2="-148.46021"
+           x2="256.3761"
+           y1="-148.46021"
+           x1="255.3761"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient101777-2-8-6">
+          <stop
+             id="stop101779-4-4-3"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop101781-1-3-8"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect82404-9-9-8"
+           height="5.3839998"
+           width="5.3839998"
+           clip-path="url(#SVGID_4556_-9-2-5)"
+           y="308.841"
+           x="-672.61401"
+           style="fill:url(#linearGradient15887-0)" />
+      </g>
+      <path
+         transform="matrix(2.25,0,0,2.25,1841.021,-9.2938893)"
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4564_-7-3-3-7-5)"
+         d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82449-2-2-4-4-2" />
+    </g>
+    <g
+       id="g15866-2-3"
+       transform="translate(66.384186,62.334929)">
+      <rect
+         inkscape:transform-center-y="19.597191"
+         inkscape:transform-center-x="610.96881"
+         id="rect82381-2-3-5-9"
+         height="21.584251"
+         width="21.581999"
+         y="680.3775"
+         x="349.96692"
+         style="fill:#ffffff" />
+      <rect
+         inkscape:transform-center-y="19.597191"
+         inkscape:transform-center-x="610.96881"
+         id="rect82383-1-0-2-8"
+         height="21.584251"
+         width="21.581999"
+         stroke-miterlimit="10"
+         y="680.3775"
+         x="349.96692"
+         style="fill:none;stroke:#81d0db;stroke-width:0.96300006;stroke-miterlimit:10" />
+      <rect
+         inkscape:transform-center-y="19.598295"
+         inkscape:transform-center-x="610.97115"
+         id="rect82385-3-3-6-8"
+         height="18.553501"
+         width="18.557999"
+         y="681.89398"
+         x="351.47656"
+         style="fill:#81d0db" />
+      <rect
+         inkscape:transform-center-y="19.597131"
+         inkscape:transform-center-x="610.96883"
+         id="rect82387-5-5-5-5"
+         height="15.594751"
+         width="15.592501"
+         y="683.37219"
+         x="352.96164"
+         style="fill:#ffffff" />
+      <rect
+         inkscape:transform-center-y="19.595982"
+         inkscape:transform-center-x="610.97103"
+         id="rect82389-9-0-6-5"
+         height="10.278"
+         width="10.282501"
+         y="686.02936"
+         x="355.61444"
+         style="fill:#ffffff" />
+      <g
+         inkscape:transform-center-y="19.598312"
+         inkscape:transform-center-x="610.97111"
+         id="g82391-8-0-8-1"
+         transform="matrix(2.25,0,0,2.25,1868.0802,-9.7784893)">
+        <defs
+           id="defs82393-4-0-3-9">
+          <path
+             d="m -672.614,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+             id="SVGID_4555_-7-9-1-0"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath101773-5-5-0-4">
+          <use
+             id="use101775-6-5-6-5"
+             overflow="visible"
+             xlink:href="#SVGID_4555_-7-9-1-0"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)"
+           y2="-148.46021"
+           x2="256.3761"
+           y1="-148.46021"
+           x1="255.3761"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient101777-2-8-6-8">
+          <stop
+             id="stop101779-4-4-3-2"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop101781-1-3-8-5"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect82404-9-9-8-1"
+           height="5.3839998"
+           width="5.3839998"
+           clip-path="url(#SVGID_4556_-9-2-5-8)"
+           y="308.841"
+           x="-672.61401"
+           style="fill:url(#linearGradient15887-0-3)" />
+      </g>
+      <path
+         transform="matrix(2.25,0,0,2.25,1841.021,-9.2938893)"
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4564_-7-3-3-7-5-3)"
+         d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82449-2-2-4-4-2-3" />
+    </g>
+  </g>
+</svg>
diff --git a/doc/images/rgw-encryption-barbican.png b/doc/images/rgw-encryption-barbican.png
new file mode 100644
index 000000000..e3c10c4a7
--- /dev/null
+++ b/doc/images/rgw-encryption-barbican.png
diff --git a/doc/images/stack.png b/doc/images/stack.png
new file mode 100644
index 000000000..38eac41b0
--- /dev/null
+++ b/doc/images/stack.png
diff --git a/doc/images/win2016_iscsi_advanced_window.png b/doc/images/win2016_iscsi_advanced_window.png
new file mode 100755
index 000000000..cba70ea89
--- /dev/null
+++ b/doc/images/win2016_iscsi_advanced_window.png
diff --git a/doc/images/win2016_iscsi_connect_to_target.png b/doc/images/win2016_iscsi_connect_to_target.png
new file mode 100755
index 000000000..29d9e1267
--- /dev/null
+++ b/doc/images/win2016_iscsi_connect_to_target.png
diff --git a/doc/images/win2016_iscsi_devices_mpio.png b/doc/images/win2016_iscsi_devices_mpio.png
new file mode 100755
index 000000000..135d41bf5
--- /dev/null
+++ b/doc/images/win2016_iscsi_devices_mpio.png
diff --git a/doc/images/win2016_iscsi_discovery_tab.png b/doc/images/win2016_iscsi_discovery_tab.png
new file mode 100755
index 000000000..4ac5f71d9
--- /dev/null
+++ b/doc/images/win2016_iscsi_discovery_tab.png
diff --git a/doc/images/win2016_iscsi_target_tab.png b/doc/images/win2016_iscsi_target_tab.png
new file mode 100755
index 000000000..75437692b
--- /dev/null
+++ b/doc/images/win2016_iscsi_target_tab.png
diff --git a/doc/images/win2016_iscsi_target_tab2.png b/doc/images/win2016_iscsi_target_tab2.png
new file mode 100755
index 000000000..302fef918
--- /dev/null
+++ b/doc/images/win2016_iscsi_target_tab2.png
diff --git a/doc/images/win2016_mpclaim_output.png b/doc/images/win2016_mpclaim_output.png
new file mode 100644
index 000000000..73e1e5ed2
--- /dev/null
+++ b/doc/images/win2016_mpclaim_output.png
diff --git a/doc/images/win2016_mpio_set_failover_only.png b/doc/images/win2016_mpio_set_failover_only.png
new file mode 100755
index 000000000..a988e8a07
--- /dev/null
+++ b/doc/images/win2016_mpio_set_failover_only.png
diff --git a/doc/images/zone-sync.png b/doc/images/zone-sync.png
new file mode 100644
index 000000000..c7bf9efba
--- /dev/null
+++ b/doc/images/zone-sync.png
diff --git a/doc/images/zone-sync.svg b/doc/images/zone-sync.svg
new file mode 100644
index 000000000..e0bcb0c5b
--- /dev/null
+++ b/doc/images/zone-sync.svg
@@ -0,0 +1,21935 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:osb="http://www.openswatchbook.org/uri/2009/osb"
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:xlink="http://www.w3.org/1999/xlink"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="744.09448819"
+   height="1052.3622047"
+   id="svg104536"
+   version="1.1"
+   inkscape:version="0.48.4 r9939"
+   sodipodi:docname="zone-sync.svg"
+   inkscape:export-filename="/home/john/Pictures/zone-sync.png"
+   inkscape:export-xdpi="90"
+   inkscape:export-ydpi="90">
+  <defs
+     id="defs104538">
+    <linearGradient
+       id="linearGradient126200"
+       osb:paint="solid">
+      <stop
+         style="stop-color:#ffffff;stop-opacity:1;"
+         offset="0"
+         id="stop126202" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4884_">
+      <rect
+         height="11.504"
+         width="174.63"
+         y="360.13901"
+         x="-679.57202"
+         id="use84863" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4886_">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use84873" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4888_">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use84881" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4890_">
+      <rect
+         height="67.402"
+         width="42.448002"
+         y="350.939"
+         x="-535.73901"
+         id="use84891" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4892_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-535.73901"
+         id="use84899" />
+    </clipPath>
+    <clipPath
+       id="clipPath99071">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99073" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4894_">
+      <rect
+         height="67.402"
+         width="42.448002"
+         y="350.939"
+         x="-536.966"
+         id="use84911" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4896_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-536.966"
+         id="use84919" />
+    </clipPath>
+    <clipPath
+       id="clipPath99079">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99081" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4898_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-538.19202"
+         id="use84931" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4900_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-538.19202"
+         id="use84939" />
+    </clipPath>
+    <clipPath
+       id="clipPath99087">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99089" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4902_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-539.41901"
+         id="use84951" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4904_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-539.41901"
+         id="use84959" />
+    </clipPath>
+    <clipPath
+       id="clipPath99095">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99097" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4906_">
+      <rect
+         height="67.402"
+         width="42.448002"
+         y="350.939"
+         x="-540.646"
+         id="use84971" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4908_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-540.646"
+         id="use84979" />
+    </clipPath>
+    <clipPath
+       id="clipPath99103">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99105" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4910_">
+      <rect
+         height="67.402"
+         width="42.448002"
+         y="350.939"
+         x="-541.87299"
+         id="use84991" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4912_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-541.87299"
+         id="use84999" />
+    </clipPath>
+    <clipPath
+       id="clipPath99111">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99113" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4914_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-543.099"
+         id="use85011" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4916_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-543.099"
+         id="use85019" />
+    </clipPath>
+    <clipPath
+       id="clipPath99119">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99121" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4918_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-544.32599"
+         id="use85031" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4920_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-544.32599"
+         id="use85039" />
+    </clipPath>
+    <clipPath
+       id="clipPath99127">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99129" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4922_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-545.55298"
+         id="use85051" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4924_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-545.55298"
+         id="use85059" />
+    </clipPath>
+    <clipPath
+       id="clipPath99135">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99137" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4926_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-546.77899"
+         id="use85071" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4928_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-546.77899"
+         id="use85079" />
+    </clipPath>
+    <clipPath
+       id="clipPath99143">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99145" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4930_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-548.00598"
+         id="use85091" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4932_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-548.00598"
+         id="use85099" />
+    </clipPath>
+    <clipPath
+       id="clipPath99151">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99153" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4934_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-549.23297"
+         id="use85111" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4936_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-549.23297"
+         id="use85119" />
+    </clipPath>
+    <clipPath
+       id="clipPath99159">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99161" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4938_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-550.46002"
+         id="use85131" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4940_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-550.46002"
+         id="use85139" />
+    </clipPath>
+    <clipPath
+       id="clipPath99167">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99169" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4942_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-551.68597"
+         id="use85151" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4944_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-551.68597"
+         id="use85159" />
+    </clipPath>
+    <clipPath
+       id="clipPath99175">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99177" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4946_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-552.91302"
+         id="use85171" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4948_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-552.91302"
+         id="use85179" />
+    </clipPath>
+    <clipPath
+       id="clipPath99183">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99185" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4950_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-554.14001"
+         id="use85191" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4952_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-554.14001"
+         id="use85199" />
+    </clipPath>
+    <clipPath
+       id="clipPath99191">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99193" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4954_">
+      <rect
+         height="67.402"
+         width="42.446999"
+         y="350.939"
+         x="-555.367"
+         id="use85211" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4956_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-555.367"
+         id="use85219" />
+    </clipPath>
+    <clipPath
+       id="clipPath99199">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99201" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4958_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-556.59302"
+         id="use85231" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4960_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-556.59302"
+         id="use85239" />
+    </clipPath>
+    <clipPath
+       id="clipPath99207">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99209" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4962_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-557.82001"
+         id="use85251" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4964_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-557.82001"
+         id="use85259" />
+    </clipPath>
+    <clipPath
+       id="clipPath99215">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99217" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4966_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-559.047"
+         id="use85271" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4968_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-559.047"
+         id="use85279" />
+    </clipPath>
+    <clipPath
+       id="clipPath99223">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99225" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4970_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-560.27301"
+         id="use85291" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4972_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-560.27301"
+         id="use85299" />
+    </clipPath>
+    <clipPath
+       id="clipPath99231">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99233" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4974_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-561.5"
+         id="use85311" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4976_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-561.5"
+         id="use85319" />
+    </clipPath>
+    <clipPath
+       id="clipPath99239">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99241" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4978_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-562.72699"
+         id="use85331" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4980_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-562.72699"
+         id="use85339" />
+    </clipPath>
+    <clipPath
+       id="clipPath99247">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99249" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4982_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-563.95398"
+         id="use85351" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4984_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-563.95398"
+         id="use85359" />
+    </clipPath>
+    <clipPath
+       id="clipPath99255">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99257" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4986_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-565.17999"
+         id="use85371" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4988_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-565.17999"
+         id="use85379" />
+    </clipPath>
+    <clipPath
+       id="clipPath99263">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99265" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4990_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-566.40698"
+         id="use85391" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4992_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-566.40698"
+         id="use85399" />
+    </clipPath>
+    <clipPath
+       id="clipPath99271">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99273" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4994_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-567.63397"
+         id="use85411" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4996_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-567.63397"
+         id="use85419" />
+    </clipPath>
+    <clipPath
+       id="clipPath99279">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99281" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4998_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-568.86102"
+         id="use85431" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5000_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-568.86102"
+         id="use85439" />
+    </clipPath>
+    <clipPath
+       id="clipPath99287">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99289" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5002_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-570.08698"
+         id="use85451" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5004_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-570.08698"
+         id="use85459" />
+    </clipPath>
+    <clipPath
+       id="clipPath99295">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99297" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5006_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-571.315"
+         id="use85471" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5008_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-571.31403"
+         id="use85479" />
+    </clipPath>
+    <clipPath
+       id="clipPath99303">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99305" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5010_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-572.54102"
+         id="use85491" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5012_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-572.54102"
+         id="use85499" />
+    </clipPath>
+    <clipPath
+       id="clipPath99311">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99313" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5014_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-573.76801"
+         id="use85511" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5016_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-573.76801"
+         id="use85519" />
+    </clipPath>
+    <clipPath
+       id="clipPath99319">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99321" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5018_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-574.99402"
+         id="use85531" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5020_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-574.99402"
+         id="use85539" />
+    </clipPath>
+    <clipPath
+       id="clipPath99327">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99329" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5022_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-576.22101"
+         id="use85551" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5024_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-576.22101"
+         id="use85559" />
+    </clipPath>
+    <clipPath
+       id="clipPath99335">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99337" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5026_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-577.448"
+         id="use85571" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5028_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-577.448"
+         id="use85579" />
+    </clipPath>
+    <clipPath
+       id="clipPath99343">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99345" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5030_">
+      <rect
+         height="67.402"
+         width="42.445999"
+         y="350.939"
+         x="-578.67499"
+         id="use85591" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5032_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-578.67499"
+         id="use85599" />
+    </clipPath>
+    <clipPath
+       id="clipPath99351">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99353" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5034_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-579.901"
+         id="use85611" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5036_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-579.901"
+         id="use85619" />
+    </clipPath>
+    <clipPath
+       id="clipPath99359">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99361" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5038_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-581.12799"
+         id="use85631" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5040_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-581.12799"
+         id="use85639" />
+    </clipPath>
+    <clipPath
+       id="clipPath99367">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99369" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5042_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-582.35498"
+         id="use85651" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5044_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-582.35498"
+         id="use85659" />
+    </clipPath>
+    <clipPath
+       id="clipPath99375">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99377" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5046_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-583.58197"
+         id="use85671" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5048_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-583.58197"
+         id="use85679" />
+    </clipPath>
+    <clipPath
+       id="clipPath99383">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99385" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5050_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-584.80798"
+         id="use85691" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5052_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-584.80798"
+         id="use85699" />
+    </clipPath>
+    <clipPath
+       id="clipPath99391">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99393" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5054_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-586.03497"
+         id="use85711" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5056_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-586.03497"
+         id="use85719" />
+    </clipPath>
+    <clipPath
+       id="clipPath99399">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99401" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5058_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-587.26202"
+         id="use85731" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5060_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-587.26202"
+         id="use85739" />
+    </clipPath>
+    <clipPath
+       id="clipPath99407">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99409" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5062_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-588.48901"
+         id="use85751" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5064_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-588.48901"
+         id="use85759" />
+    </clipPath>
+    <clipPath
+       id="clipPath99415">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99417" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5066_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-589.71503"
+         id="use85771" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5068_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-589.71503"
+         id="use85779" />
+    </clipPath>
+    <clipPath
+       id="clipPath99423">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99425" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5070_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-590.94202"
+         id="use85791" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5072_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-590.94202"
+         id="use85799" />
+    </clipPath>
+    <clipPath
+       id="clipPath99431">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99433" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5074_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-592.16901"
+         id="use85811" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5076_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-592.16901"
+         id="use85819" />
+    </clipPath>
+    <clipPath
+       id="clipPath99439">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99441" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5078_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-593.396"
+         id="use85831" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5080_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-593.396"
+         id="use85839" />
+    </clipPath>
+    <clipPath
+       id="clipPath99447">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99449" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5082_">
+      <rect
+         height="67.402"
+         width="42.445"
+         y="350.939"
+         x="-594.62299"
+         id="use85851" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5084_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-594.62299"
+         id="use85859" />
+    </clipPath>
+    <clipPath
+       id="clipPath99455">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99457" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5086_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-595.849"
+         id="use85871" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5088_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-595.849"
+         id="use85879" />
+    </clipPath>
+    <clipPath
+       id="clipPath99463">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99465" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5090_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-597.07599"
+         id="use85891" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5092_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-597.07599"
+         id="use85899" />
+    </clipPath>
+    <clipPath
+       id="clipPath99471">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99473" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5094_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-598.30298"
+         id="use85911" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5096_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-598.30298"
+         id="use85919" />
+    </clipPath>
+    <clipPath
+       id="clipPath99479">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99481" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5098_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-599.53003"
+         id="use85931" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5100_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-599.53003"
+         id="use85939" />
+    </clipPath>
+    <clipPath
+       id="clipPath99487">
+      <rect
+         height="67.402"
+         width="107.465"
+         y="350.939"
+         x="-600.75598"
+         id="use99489" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5102_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-600.75598"
+         id="use85951" />
+    </clipPath>
+    <clipPath
+       id="SVGID_5104_">
+      <rect
+         height="67.402"
+         width="42.444"
+         y="350.939"
+         x="-600.75598"
+         id="use85959" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4873_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_"
+         overflow="visible"
+         id="use84798" />
+    </clipPath>
+    <clipPath
+       id="clipPath99497">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_"
+         overflow="visible"
+         id="use99499" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4875_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4874_"
+         overflow="visible"
+         id="use84810" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4878_"
+       gradientUnits="userSpaceOnUse"
+       x1="40.563"
+       y1="147.0757"
+       x2="41.563"
+       y2="147.0757"
+       gradientTransform="matrix(0,-10.1526,-10.1526,0,900.78,782.1111)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84821" />
+      <stop
+         offset="0.486"
+         style="stop-color:#B0D2D9"
+         id="stop84823" />
+      <stop
+         offset="0.8287"
+         style="stop-color:#E6EFF1"
+         id="stop84825" />
+      <stop
+         offset="0.9939"
+         style="stop-color:#FFFFFF"
+         id="stop84827" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84829" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4877_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4876_"
+         overflow="visible"
+         id="use84818" />
+    </clipPath>
+    <clipPath
+       id="clipPath99511">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_"
+         overflow="visible"
+         id="use99513" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4880_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4879_"
+         overflow="visible"
+         id="use84841" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4882_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4881_"
+         overflow="visible"
+         id="use84849" />
+    </clipPath>
+    <clipPath
+       id="clipPath99519">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_"
+         overflow="visible"
+         id="use99521" />
+    </clipPath>
+    <clipPath
+       id="clipPath99523">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4872_"
+         overflow="visible"
+         id="use99525" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4871_"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94051"
+       x2="-33.205299"
+       y2="249.94051"
+       gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84774" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84776" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4870_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4869_"
+         overflow="visible"
+         id="use84771" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4868_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4867_"
+         overflow="visible"
+         id="use84759" />
+    </clipPath>
+    <clipPath
+       id="clipPath99534">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4867_"
+         overflow="visible"
+         id="use99536" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4866_"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.9404"
+       x2="-33.205299"
+       y2="249.9404"
+       gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84735" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84737" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4865_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4864_"
+         overflow="visible"
+         id="use84732" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4863_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4862_"
+         overflow="visible"
+         id="use84720" />
+    </clipPath>
+    <clipPath
+       id="clipPath99545">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4862_"
+         overflow="visible"
+         id="use99547" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4861_"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94031"
+       x2="-33.205299"
+       y2="249.94031"
+       gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84696" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84698" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4860_">
+      <polygon
+         points="-521.64,278.632 -507.673,278.632 -507.673,273.567 -521.64,273.567 -523.432,276.108 "
+         id="use84693" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4858_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84681" />
+    </clipPath>
+    <clipPath
+       id="clipPath99556">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99558" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4856_"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94031"
+       x2="-33.205101"
+       y2="249.94031"
+       gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84659" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84661" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4855_">
+      <polygon
+         points="-521.64,208.524 -507.673,208.524 -507.673,203.458 -521.64,203.458 -523.432,206 "
+         id="use84656" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4853_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84644" />
+    </clipPath>
+    <clipPath
+       id="clipPath99567">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99569" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4851_"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.9404"
+       x2="-33.205101"
+       y2="249.9404"
+       gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84622" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84624" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4850_">
+      <polygon
+         points="-588.288,203.458 -590.081,206 -588.288,208.524 -574.322,208.524 -574.322,203.458 "
+         id="use84619" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4848_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84607" />
+    </clipPath>
+    <clipPath
+       id="clipPath99578">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99580" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4846_"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94051"
+       x2="-33.205101"
+       y2="249.94051"
+       gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84585" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84587" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4845_">
+      <polygon
+         points="-654.922,203.458 -656.715,206 -654.922,208.524 -640.956,208.524 -640.956,203.458 "
+         id="use84582" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500" />
+    </clipPath>
+    <clipPath
+       id="clipPath99589">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99591" />
+    </clipPath>
+    <clipPath
+       id="clipPath99593">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99595" />
+    </clipPath>
+    <clipPath
+       id="clipPath99597">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99599" />
+    </clipPath>
+    <clipPath
+       id="clipPath99601">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99603" />
+    </clipPath>
+    <clipPath
+       id="clipPath99605">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99607" />
+    </clipPath>
+    <clipPath
+       id="clipPath99609">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99611" />
+    </clipPath>
+    <clipPath
+       id="clipPath99613">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99615" />
+    </clipPath>
+    <clipPath
+       id="clipPath99617">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99619" />
+    </clipPath>
+    <clipPath
+       id="clipPath99621">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99623" />
+    </clipPath>
+    <clipPath
+       id="clipPath99625">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99627" />
+    </clipPath>
+    <clipPath
+       id="clipPath99629">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99631" />
+    </clipPath>
+    <clipPath
+       id="clipPath99633">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99635" />
+    </clipPath>
+    <clipPath
+       id="clipPath99637">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99639" />
+    </clipPath>
+    <clipPath
+       id="clipPath99641">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99643" />
+    </clipPath>
+    <clipPath
+       id="clipPath99645">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99647" />
+    </clipPath>
+    <clipPath
+       id="clipPath99649">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99651" />
+    </clipPath>
+    <clipPath
+       id="clipPath99653">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99655" />
+    </clipPath>
+    <clipPath
+       id="clipPath99657">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99659" />
+    </clipPath>
+    <clipPath
+       id="clipPath99661">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99663" />
+    </clipPath>
+    <clipPath
+       id="clipPath99665">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99667" />
+    </clipPath>
+    <clipPath
+       id="clipPath99669">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99671" />
+    </clipPath>
+    <clipPath
+       id="clipPath99673">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99675" />
+    </clipPath>
+    <clipPath
+       id="clipPath99677">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99679" />
+    </clipPath>
+    <clipPath
+       id="clipPath99681">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99683" />
+    </clipPath>
+    <clipPath
+       id="clipPath99685">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99687" />
+    </clipPath>
+    <clipPath
+       id="clipPath99689">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99691" />
+    </clipPath>
+    <clipPath
+       id="clipPath99693">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99695" />
+    </clipPath>
+    <clipPath
+       id="clipPath99697">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99699" />
+    </clipPath>
+    <clipPath
+       id="clipPath99701">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99703" />
+    </clipPath>
+    <clipPath
+       id="clipPath99705">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99707" />
+    </clipPath>
+    <clipPath
+       id="clipPath99709">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99711" />
+    </clipPath>
+    <clipPath
+       id="clipPath99713">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99715" />
+    </clipPath>
+    <clipPath
+       id="clipPath99717">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99719" />
+    </clipPath>
+    <clipPath
+       id="clipPath99721">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99723" />
+    </clipPath>
+    <clipPath
+       id="clipPath99725">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99727" />
+    </clipPath>
+    <clipPath
+       id="clipPath99729">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use99731" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4841_"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.813"
+       x2="37.392799"
+       y2="152.813"
+       gradientTransform="matrix(0,-9.614,-9.614,0,942.8785,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84478" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84480" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84482" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84484" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84486" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4840_">
+      <rect
+         height="9.6140003"
+         width="11.316"
+         y="214.106"
+         x="-531.92297"
+         id="use84475" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4838_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84445" />
+    </clipPath>
+    <clipPath
+       id="clipPath99743">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99745" />
+    </clipPath>
+    <clipPath
+       id="clipPath99747">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99749" />
+    </clipPath>
+    <clipPath
+       id="clipPath99751">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99753" />
+    </clipPath>
+    <clipPath
+       id="clipPath99755">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99757" />
+    </clipPath>
+    <clipPath
+       id="clipPath99759">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99761" />
+    </clipPath>
+    <clipPath
+       id="clipPath99763">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99765" />
+    </clipPath>
+    <clipPath
+       id="clipPath99767">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99769" />
+    </clipPath>
+    <clipPath
+       id="clipPath99771">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99773" />
+    </clipPath>
+    <clipPath
+       id="clipPath99775">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99777" />
+    </clipPath>
+    <clipPath
+       id="clipPath99779">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99781" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4836_"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.8129"
+       x2="37.392799"
+       y2="152.8129"
+       gradientTransform="matrix(0,-9.614,-9.614,0,876.3496,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84423" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84425" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84427" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84429" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84431" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4835_">
+      <rect
+         height="9.6140003"
+         width="11.315"
+         y="214.106"
+         x="-598.45099"
+         id="use84420" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4833_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84390" />
+    </clipPath>
+    <clipPath
+       id="clipPath99793">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99795" />
+    </clipPath>
+    <clipPath
+       id="clipPath99797">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99799" />
+    </clipPath>
+    <clipPath
+       id="clipPath99801">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99803" />
+    </clipPath>
+    <clipPath
+       id="clipPath99805">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99807" />
+    </clipPath>
+    <clipPath
+       id="clipPath99809">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99811" />
+    </clipPath>
+    <clipPath
+       id="clipPath99813">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99815" />
+    </clipPath>
+    <clipPath
+       id="clipPath99817">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99819" />
+    </clipPath>
+    <clipPath
+       id="clipPath99821">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99823" />
+    </clipPath>
+    <clipPath
+       id="clipPath99825">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99827" />
+    </clipPath>
+    <clipPath
+       id="clipPath99829">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99831" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4831_"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.813"
+       x2="37.392799"
+       y2="152.813"
+       gradientTransform="matrix(0,-9.614,-9.614,0,809.7396,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84368" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84370" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84372" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84374" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84376" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4830_">
+      <rect
+         height="9.6140003"
+         width="11.316"
+         y="214.106"
+         x="-665.06201"
+         id="use84365" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4828_"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-734.6116,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84329" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84331" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84333" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84335" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84337" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4827_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4826_"
+         overflow="visible"
+         id="use84326" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4825_"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-668.057,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84298" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84300" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84302" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84304" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84306" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4824_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4823_"
+         overflow="visible"
+         id="use84295" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4822_"
+       gradientUnits="userSpaceOnUse"
+       x1="145.317"
+       y1="2.8515"
+       x2="146.317"
+       y2="2.8515"
+       gradientTransform="matrix(0.0299,24.8863,24.8863,-0.0299,-601.3687,-3384.4197)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84267" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84269" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84271" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84273" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84275" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4821_">
+      <polygon
+         points="-503.122,251.283 -503.124,231.934 -548.973,231.929 -548.973,251.221 -526.108,256.788 "
+         id="use84264" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4819_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84232" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4817_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,284.9195,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84220" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84222" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4816_">
+      <path
+         d="m -516.985,346.449 c 0,1.487 1.205,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use84217"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4814_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84197" />
+    </clipPath>
+    <clipPath
+       id="clipPath99874">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99876" />
+    </clipPath>
+    <clipPath
+       id="clipPath99878">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99880" />
+    </clipPath>
+    <clipPath
+       id="clipPath99882">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99884" />
+    </clipPath>
+    <clipPath
+       id="clipPath99886">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99888" />
+    </clipPath>
+    <clipPath
+       id="clipPath99890">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99892" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4812_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,272.9481,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84185" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84187" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4811_">
+      <path
+         d="m -528.956,346.449 c 0,1.487 1.205,2.692 2.691,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.691,1.206 -2.691,2.692"
+         id="use84182"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4809_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84162" />
+    </clipPath>
+    <clipPath
+       id="clipPath99901">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99903" />
+    </clipPath>
+    <clipPath
+       id="clipPath99905">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99907" />
+    </clipPath>
+    <clipPath
+       id="clipPath99909">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99911" />
+    </clipPath>
+    <clipPath
+       id="clipPath99913">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99915" />
+    </clipPath>
+    <clipPath
+       id="clipPath99917">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99919" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4807_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,260.9767,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84150" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84152" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4806_">
+      <path
+         d="m -540.927,346.449 c 0,1.487 1.205,2.692 2.691,2.692 1.488,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.691,1.206 -2.691,2.692"
+         id="use84147"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4804_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84127" />
+    </clipPath>
+    <clipPath
+       id="clipPath99928">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99930" />
+    </clipPath>
+    <clipPath
+       id="clipPath99932">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99934" />
+    </clipPath>
+    <clipPath
+       id="clipPath99936">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99938" />
+    </clipPath>
+    <clipPath
+       id="clipPath99940">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99942" />
+    </clipPath>
+    <clipPath
+       id="clipPath99944">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99946" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4802_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,249.0053,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84115" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84117" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4801_">
+      <path
+         d="m -552.899,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use84112"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4799_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84092" />
+    </clipPath>
+    <clipPath
+       id="clipPath99955">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99957" />
+    </clipPath>
+    <clipPath
+       id="clipPath99959">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99961" />
+    </clipPath>
+    <clipPath
+       id="clipPath99963">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99965" />
+    </clipPath>
+    <clipPath
+       id="clipPath99967">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99969" />
+    </clipPath>
+    <clipPath
+       id="clipPath99971">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99973" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4797_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,237.034,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84080" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84082" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4796_">
+      <path
+         d="m -564.87,346.449 c 0,1.487 1.205,2.692 2.691,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.691,1.206 -2.691,2.692"
+         id="use84077"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4794_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84057" />
+    </clipPath>
+    <clipPath
+       id="clipPath99982">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99984" />
+    </clipPath>
+    <clipPath
+       id="clipPath99986">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99988" />
+    </clipPath>
+    <clipPath
+       id="clipPath99990">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99992" />
+    </clipPath>
+    <clipPath
+       id="clipPath99994">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use99996" />
+    </clipPath>
+    <clipPath
+       id="clipPath99998">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100000" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4792_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,225.0626,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84045" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84047" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4791_">
+      <path
+         d="m -576.842,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use84042"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4789_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84022" />
+    </clipPath>
+    <clipPath
+       id="clipPath100009">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100011" />
+    </clipPath>
+    <clipPath
+       id="clipPath100013">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100015" />
+    </clipPath>
+    <clipPath
+       id="clipPath100017">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100019" />
+    </clipPath>
+    <clipPath
+       id="clipPath100021">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100023" />
+    </clipPath>
+    <clipPath
+       id="clipPath100025">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100027" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4787_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84010" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84012" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4786_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4785_"
+         overflow="visible"
+         id="use84007" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4784_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_"
+         overflow="visible"
+         id="use83987" />
+    </clipPath>
+    <clipPath
+       id="clipPath100036">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_"
+         overflow="visible"
+         id="use100038" />
+    </clipPath>
+    <clipPath
+       id="clipPath100040">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_"
+         overflow="visible"
+         id="use100042" />
+    </clipPath>
+    <clipPath
+       id="clipPath100044">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_"
+         overflow="visible"
+         id="use100046" />
+    </clipPath>
+    <clipPath
+       id="clipPath100048">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_"
+         overflow="visible"
+         id="use100050" />
+    </clipPath>
+    <clipPath
+       id="clipPath100052">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4783_"
+         overflow="visible"
+         id="use100054" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4782_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83975" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83977" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4781_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4780_"
+         overflow="visible"
+         id="use83972" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4779_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_"
+         overflow="visible"
+         id="use83952" />
+    </clipPath>
+    <clipPath
+       id="clipPath100063">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_"
+         overflow="visible"
+         id="use100065" />
+    </clipPath>
+    <clipPath
+       id="clipPath100067">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_"
+         overflow="visible"
+         id="use100069" />
+    </clipPath>
+    <clipPath
+       id="clipPath100071">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_"
+         overflow="visible"
+         id="use100073" />
+    </clipPath>
+    <clipPath
+       id="clipPath100075">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_"
+         overflow="visible"
+         id="use100077" />
+    </clipPath>
+    <clipPath
+       id="clipPath100079">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4778_"
+         overflow="visible"
+         id="use100081" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4777_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83940" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83942" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4776_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4775_"
+         overflow="visible"
+         id="use83937" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4774_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_"
+         overflow="visible"
+         id="use83917" />
+    </clipPath>
+    <clipPath
+       id="clipPath100090">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_"
+         overflow="visible"
+         id="use100092" />
+    </clipPath>
+    <clipPath
+       id="clipPath100094">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_"
+         overflow="visible"
+         id="use100096" />
+    </clipPath>
+    <clipPath
+       id="clipPath100098">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_"
+         overflow="visible"
+         id="use100100" />
+    </clipPath>
+    <clipPath
+       id="clipPath100102">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_"
+         overflow="visible"
+         id="use100104" />
+    </clipPath>
+    <clipPath
+       id="clipPath100106">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4773_"
+         overflow="visible"
+         id="use100108" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4772_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,177.1768,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83905" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83907" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4771_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4770_"
+         overflow="visible"
+         id="use83902" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4769_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_"
+         overflow="visible"
+         id="use83882" />
+    </clipPath>
+    <clipPath
+       id="clipPath100117">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_"
+         overflow="visible"
+         id="use100119" />
+    </clipPath>
+    <clipPath
+       id="clipPath100121">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_"
+         overflow="visible"
+         id="use100123" />
+    </clipPath>
+    <clipPath
+       id="clipPath100125">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_"
+         overflow="visible"
+         id="use100127" />
+    </clipPath>
+    <clipPath
+       id="clipPath100129">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_"
+         overflow="visible"
+         id="use100131" />
+    </clipPath>
+    <clipPath
+       id="clipPath100133">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4768_"
+         overflow="visible"
+         id="use100135" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4767_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83870" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83872" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4766_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4765_"
+         overflow="visible"
+         id="use83867" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4764_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_"
+         overflow="visible"
+         id="use83847" />
+    </clipPath>
+    <clipPath
+       id="clipPath100144">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_"
+         overflow="visible"
+         id="use100146" />
+    </clipPath>
+    <clipPath
+       id="clipPath100148">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_"
+         overflow="visible"
+         id="use100150" />
+    </clipPath>
+    <clipPath
+       id="clipPath100152">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_"
+         overflow="visible"
+         id="use100154" />
+    </clipPath>
+    <clipPath
+       id="clipPath100156">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_"
+         overflow="visible"
+         id="use100158" />
+    </clipPath>
+    <clipPath
+       id="clipPath100160">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4763_"
+         overflow="visible"
+         id="use100162" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4762_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83835" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83837" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4761_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4760_"
+         overflow="visible"
+         id="use83832" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4759_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_"
+         overflow="visible"
+         id="use83812" />
+    </clipPath>
+    <clipPath
+       id="clipPath100171">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_"
+         overflow="visible"
+         id="use100173" />
+    </clipPath>
+    <clipPath
+       id="clipPath100175">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_"
+         overflow="visible"
+         id="use100177" />
+    </clipPath>
+    <clipPath
+       id="clipPath100179">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_"
+         overflow="visible"
+         id="use100181" />
+    </clipPath>
+    <clipPath
+       id="clipPath100183">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_"
+         overflow="visible"
+         id="use100185" />
+    </clipPath>
+    <clipPath
+       id="clipPath100187">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4758_"
+         overflow="visible"
+         id="use100189" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4757_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83800" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83802" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4756_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4755_"
+         overflow="visible"
+         id="use83797" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4754_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_"
+         overflow="visible"
+         id="use83777" />
+    </clipPath>
+    <clipPath
+       id="clipPath100198">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_"
+         overflow="visible"
+         id="use100200" />
+    </clipPath>
+    <clipPath
+       id="clipPath100202">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_"
+         overflow="visible"
+         id="use100204" />
+    </clipPath>
+    <clipPath
+       id="clipPath100206">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_"
+         overflow="visible"
+         id="use100208" />
+    </clipPath>
+    <clipPath
+       id="clipPath100210">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_"
+         overflow="visible"
+         id="use100212" />
+    </clipPath>
+    <clipPath
+       id="clipPath100214">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4753_"
+         overflow="visible"
+         id="use100216" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4752_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83765" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83767" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4751_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4750_"
+         overflow="visible"
+         id="use83762" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4749_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4748_"
+         overflow="visible"
+         id="use83742" />
+    </clipPath>
+    <clipPath
+       id="clipPath100225">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4748_"
+         overflow="visible"
+         id="use100227" />
+    </clipPath>
+    <clipPath
+       id="clipPath100229">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4748_"
+         overflow="visible"
+         id="use100231" />
+    </clipPath>
+    <clipPath
+       id="clipPath100233">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4748_"
+         overflow="visible"
+         id="use100235" />
+    </clipPath>
+    <clipPath
+       id="clipPath100237">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4748_"
+         overflow="visible"
+         id="use100239" />
+    </clipPath>
+    <clipPath
+       id="clipPath100241">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4748_"
+         overflow="visible"
+         id="use100243" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4747_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,284.9195,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83730" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83732" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4746_">
+      <path
+         d="m -516.985,334.666 c 0,1.488 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use83727"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4744_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83707" />
+    </clipPath>
+    <clipPath
+       id="clipPath100252">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100254" />
+    </clipPath>
+    <clipPath
+       id="clipPath100256">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100258" />
+    </clipPath>
+    <clipPath
+       id="clipPath100260">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100262" />
+    </clipPath>
+    <clipPath
+       id="clipPath100264">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100266" />
+    </clipPath>
+    <clipPath
+       id="clipPath100268">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100270" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4742_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3544"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,272.9481,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83695" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83697" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4741_">
+      <path
+         d="m -528.956,334.666 c 0,1.488 1.205,2.694 2.691,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.691,1.206 -2.691,2.692"
+         id="use83692"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4739_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83672" />
+    </clipPath>
+    <clipPath
+       id="clipPath100279">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100281" />
+    </clipPath>
+    <clipPath
+       id="clipPath100283">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100285" />
+    </clipPath>
+    <clipPath
+       id="clipPath100287">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100289" />
+    </clipPath>
+    <clipPath
+       id="clipPath100291">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100293" />
+    </clipPath>
+    <clipPath
+       id="clipPath100295">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100297" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4737_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3544"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,260.9767,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83660" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83662" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4736_">
+      <path
+         d="m -540.927,334.666 c 0,1.488 1.205,2.694 2.691,2.694 1.488,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.691,1.206 -2.691,2.692"
+         id="use83657"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4734_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83637" />
+    </clipPath>
+    <clipPath
+       id="clipPath100306">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100308" />
+    </clipPath>
+    <clipPath
+       id="clipPath100310">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100312" />
+    </clipPath>
+    <clipPath
+       id="clipPath100314">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100316" />
+    </clipPath>
+    <clipPath
+       id="clipPath100318">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100320" />
+    </clipPath>
+    <clipPath
+       id="clipPath100322">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100324" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4732_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3544"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,249.0053,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83625" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83627" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4731_">
+      <path
+         d="m -552.899,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83622"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4729_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83602" />
+    </clipPath>
+    <clipPath
+       id="clipPath100333">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100335" />
+    </clipPath>
+    <clipPath
+       id="clipPath100337">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100339" />
+    </clipPath>
+    <clipPath
+       id="clipPath100341">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100343" />
+    </clipPath>
+    <clipPath
+       id="clipPath100345">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100347" />
+    </clipPath>
+    <clipPath
+       id="clipPath100349">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100351" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4727_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.35419"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,225.0626,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83590" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83592" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4726_">
+      <path
+         d="m -576.842,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83587"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4724_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83567" />
+    </clipPath>
+    <clipPath
+       id="clipPath100360">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100362" />
+    </clipPath>
+    <clipPath
+       id="clipPath100364">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100366" />
+    </clipPath>
+    <clipPath
+       id="clipPath100368">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100370" />
+    </clipPath>
+    <clipPath
+       id="clipPath100372">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100374" />
+    </clipPath>
+    <clipPath
+       id="clipPath100376">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100378" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4722_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83555" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83557" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4721_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4720_"
+         overflow="visible"
+         id="use83552" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4719_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_"
+         overflow="visible"
+         id="use83532" />
+    </clipPath>
+    <clipPath
+       id="clipPath100387">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_"
+         overflow="visible"
+         id="use100389" />
+    </clipPath>
+    <clipPath
+       id="clipPath100391">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_"
+         overflow="visible"
+         id="use100393" />
+    </clipPath>
+    <clipPath
+       id="clipPath100395">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_"
+         overflow="visible"
+         id="use100397" />
+    </clipPath>
+    <clipPath
+       id="clipPath100399">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_"
+         overflow="visible"
+         id="use100401" />
+    </clipPath>
+    <clipPath
+       id="clipPath100403">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4718_"
+         overflow="visible"
+         id="use100405" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4717_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83520" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83522" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4716_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4715_"
+         overflow="visible"
+         id="use83517" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4714_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_"
+         overflow="visible"
+         id="use83497" />
+    </clipPath>
+    <clipPath
+       id="clipPath100414">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_"
+         overflow="visible"
+         id="use100416" />
+    </clipPath>
+    <clipPath
+       id="clipPath100418">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_"
+         overflow="visible"
+         id="use100420" />
+    </clipPath>
+    <clipPath
+       id="clipPath100422">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_"
+         overflow="visible"
+         id="use100424" />
+    </clipPath>
+    <clipPath
+       id="clipPath100426">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_"
+         overflow="visible"
+         id="use100428" />
+    </clipPath>
+    <clipPath
+       id="clipPath100430">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4713_"
+         overflow="visible"
+         id="use100432" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4712_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83485" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83487" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4711_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4710_"
+         overflow="visible"
+         id="use83482" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4709_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_"
+         overflow="visible"
+         id="use83462" />
+    </clipPath>
+    <clipPath
+       id="clipPath100441">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_"
+         overflow="visible"
+         id="use100443" />
+    </clipPath>
+    <clipPath
+       id="clipPath100445">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_"
+         overflow="visible"
+         id="use100447" />
+    </clipPath>
+    <clipPath
+       id="clipPath100449">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_"
+         overflow="visible"
+         id="use100451" />
+    </clipPath>
+    <clipPath
+       id="clipPath100453">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_"
+         overflow="visible"
+         id="use100455" />
+    </clipPath>
+    <clipPath
+       id="clipPath100457">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4708_"
+         overflow="visible"
+         id="use100459" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4707_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3544"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83450" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83452" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4706_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4705_"
+         overflow="visible"
+         id="use83447" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4704_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_"
+         overflow="visible"
+         id="use83427" />
+    </clipPath>
+    <clipPath
+       id="clipPath100468">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_"
+         overflow="visible"
+         id="use100470" />
+    </clipPath>
+    <clipPath
+       id="clipPath100472">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_"
+         overflow="visible"
+         id="use100474" />
+    </clipPath>
+    <clipPath
+       id="clipPath100476">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_"
+         overflow="visible"
+         id="use100478" />
+    </clipPath>
+    <clipPath
+       id="clipPath100480">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_"
+         overflow="visible"
+         id="use100482" />
+    </clipPath>
+    <clipPath
+       id="clipPath100484">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4703_"
+         overflow="visible"
+         id="use100486" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4702_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83415" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83417" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4701_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4700_"
+         overflow="visible"
+         id="use83412" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4699_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_"
+         overflow="visible"
+         id="use83392" />
+    </clipPath>
+    <clipPath
+       id="clipPath100495">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_"
+         overflow="visible"
+         id="use100497" />
+    </clipPath>
+    <clipPath
+       id="clipPath100499">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_"
+         overflow="visible"
+         id="use100501" />
+    </clipPath>
+    <clipPath
+       id="clipPath100503">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_"
+         overflow="visible"
+         id="use100505" />
+    </clipPath>
+    <clipPath
+       id="clipPath100507">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_"
+         overflow="visible"
+         id="use100509" />
+    </clipPath>
+    <clipPath
+       id="clipPath100511">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4698_"
+         overflow="visible"
+         id="use100513" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4697_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.35419"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83380" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83382" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4696_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4695_"
+         overflow="visible"
+         id="use83377" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4694_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_"
+         overflow="visible"
+         id="use83357" />
+    </clipPath>
+    <clipPath
+       id="clipPath100522">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_"
+         overflow="visible"
+         id="use100524" />
+    </clipPath>
+    <clipPath
+       id="clipPath100526">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_"
+         overflow="visible"
+         id="use100528" />
+    </clipPath>
+    <clipPath
+       id="clipPath100530">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_"
+         overflow="visible"
+         id="use100532" />
+    </clipPath>
+    <clipPath
+       id="clipPath100534">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_"
+         overflow="visible"
+         id="use100536" />
+    </clipPath>
+    <clipPath
+       id="clipPath100538">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4693_"
+         overflow="visible"
+         id="use100540" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4692_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83345" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83347" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4691_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4690_"
+         overflow="visible"
+         id="use83342" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4689_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4688_"
+         overflow="visible"
+         id="use83322" />
+    </clipPath>
+    <clipPath
+       id="clipPath100549">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4688_"
+         overflow="visible"
+         id="use100551" />
+    </clipPath>
+    <clipPath
+       id="clipPath100553">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4688_"
+         overflow="visible"
+         id="use100555" />
+    </clipPath>
+    <clipPath
+       id="clipPath100557">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4688_"
+         overflow="visible"
+         id="use100559" />
+    </clipPath>
+    <clipPath
+       id="clipPath100561">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4688_"
+         overflow="visible"
+         id="use100563" />
+    </clipPath>
+    <clipPath
+       id="clipPath100565">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4688_"
+         overflow="visible"
+         id="use100567" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4687_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,284.9569,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83310" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83312" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4686_">
+      <path
+         d="m -516.985,323.119 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.204 -2.692,2.69"
+         id="use83307"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4684_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83287" />
+    </clipPath>
+    <clipPath
+       id="clipPath100576">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100578" />
+    </clipPath>
+    <clipPath
+       id="clipPath100580">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100582" />
+    </clipPath>
+    <clipPath
+       id="clipPath100584">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100586" />
+    </clipPath>
+    <clipPath
+       id="clipPath100588">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100590" />
+    </clipPath>
+    <clipPath
+       id="clipPath100592">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100594" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4682_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,272.9856,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83275" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83277" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4681_">
+      <path
+         d="m -528.956,323.119 c 0,1.487 1.205,2.694 2.691,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.691,1.204 -2.691,2.69"
+         id="use83272"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4679_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83252" />
+    </clipPath>
+    <clipPath
+       id="clipPath100603">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100605" />
+    </clipPath>
+    <clipPath
+       id="clipPath100607">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100609" />
+    </clipPath>
+    <clipPath
+       id="clipPath100611">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100613" />
+    </clipPath>
+    <clipPath
+       id="clipPath100615">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100617" />
+    </clipPath>
+    <clipPath
+       id="clipPath100619">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100621" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4677_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,261.0142,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83240" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83242" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4676_">
+      <path
+         d="m -540.927,323.119 c 0,1.487 1.205,2.694 2.691,2.694 1.488,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.203,-2.69 -2.691,-2.69 -1.486,0 -2.691,1.204 -2.691,2.69"
+         id="use83237"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4674_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83217" />
+    </clipPath>
+    <clipPath
+       id="clipPath100630">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100632" />
+    </clipPath>
+    <clipPath
+       id="clipPath100634">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100636" />
+    </clipPath>
+    <clipPath
+       id="clipPath100638">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100640" />
+    </clipPath>
+    <clipPath
+       id="clipPath100642">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100644" />
+    </clipPath>
+    <clipPath
+       id="clipPath100646">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100648" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4672_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,249.0428,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83205" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83207" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4671_">
+      <path
+         d="m -552.899,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use83202"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4669_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83182" />
+    </clipPath>
+    <clipPath
+       id="clipPath100657">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100659" />
+    </clipPath>
+    <clipPath
+       id="clipPath100661">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100663" />
+    </clipPath>
+    <clipPath
+       id="clipPath100665">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100667" />
+    </clipPath>
+    <clipPath
+       id="clipPath100669">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100671" />
+    </clipPath>
+    <clipPath
+       id="clipPath100673">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100675" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4667_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,237.0715,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83170" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83172" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4666_">
+      <path
+         d="m -564.87,323.119 c 0,1.487 1.205,2.694 2.691,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.691,1.204 -2.691,2.69"
+         id="use83167"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4664_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83147" />
+    </clipPath>
+    <clipPath
+       id="clipPath100684">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100686" />
+    </clipPath>
+    <clipPath
+       id="clipPath100688">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100690" />
+    </clipPath>
+    <clipPath
+       id="clipPath100692">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100694" />
+    </clipPath>
+    <clipPath
+       id="clipPath100696">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100698" />
+    </clipPath>
+    <clipPath
+       id="clipPath100700">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100702" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4662_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,225.1001,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83135" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83137" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4661_">
+      <path
+         d="m -576.842,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use83132"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4659_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83112" />
+    </clipPath>
+    <clipPath
+       id="clipPath100711">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100713" />
+    </clipPath>
+    <clipPath
+       id="clipPath100715">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100717" />
+    </clipPath>
+    <clipPath
+       id="clipPath100719">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100721" />
+    </clipPath>
+    <clipPath
+       id="clipPath100723">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100725" />
+    </clipPath>
+    <clipPath
+       id="clipPath100727">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100729" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4657_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83100" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83102" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4656_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4655_"
+         overflow="visible"
+         id="use83097" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4654_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_"
+         overflow="visible"
+         id="use83077" />
+    </clipPath>
+    <clipPath
+       id="clipPath100738">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_"
+         overflow="visible"
+         id="use100740" />
+    </clipPath>
+    <clipPath
+       id="clipPath100742">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_"
+         overflow="visible"
+         id="use100744" />
+    </clipPath>
+    <clipPath
+       id="clipPath100746">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_"
+         overflow="visible"
+         id="use100748" />
+    </clipPath>
+    <clipPath
+       id="clipPath100750">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_"
+         overflow="visible"
+         id="use100752" />
+    </clipPath>
+    <clipPath
+       id="clipPath100754">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4653_"
+         overflow="visible"
+         id="use100756" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4652_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83065" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83067" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4651_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4650_"
+         overflow="visible"
+         id="use83062" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4649_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_"
+         overflow="visible"
+         id="use83042" />
+    </clipPath>
+    <clipPath
+       id="clipPath100765">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_"
+         overflow="visible"
+         id="use100767" />
+    </clipPath>
+    <clipPath
+       id="clipPath100769">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_"
+         overflow="visible"
+         id="use100771" />
+    </clipPath>
+    <clipPath
+       id="clipPath100773">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_"
+         overflow="visible"
+         id="use100775" />
+    </clipPath>
+    <clipPath
+       id="clipPath100777">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_"
+         overflow="visible"
+         id="use100779" />
+    </clipPath>
+    <clipPath
+       id="clipPath100781">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4648_"
+         overflow="visible"
+         id="use100783" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4647_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83030" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83032" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4646_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4645_"
+         overflow="visible"
+         id="use83027" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4644_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_"
+         overflow="visible"
+         id="use83007" />
+    </clipPath>
+    <clipPath
+       id="clipPath100792">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_"
+         overflow="visible"
+         id="use100794" />
+    </clipPath>
+    <clipPath
+       id="clipPath100796">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_"
+         overflow="visible"
+         id="use100798" />
+    </clipPath>
+    <clipPath
+       id="clipPath100800">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_"
+         overflow="visible"
+         id="use100802" />
+    </clipPath>
+    <clipPath
+       id="clipPath100804">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_"
+         overflow="visible"
+         id="use100806" />
+    </clipPath>
+    <clipPath
+       id="clipPath100808">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4643_"
+         overflow="visible"
+         id="use100810" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4642_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82995" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82997" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4641_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4640_"
+         overflow="visible"
+         id="use82992" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4639_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_"
+         overflow="visible"
+         id="use82972" />
+    </clipPath>
+    <clipPath
+       id="clipPath100819">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_"
+         overflow="visible"
+         id="use100821" />
+    </clipPath>
+    <clipPath
+       id="clipPath100823">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_"
+         overflow="visible"
+         id="use100825" />
+    </clipPath>
+    <clipPath
+       id="clipPath100827">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_"
+         overflow="visible"
+         id="use100829" />
+    </clipPath>
+    <clipPath
+       id="clipPath100831">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_"
+         overflow="visible"
+         id="use100833" />
+    </clipPath>
+    <clipPath
+       id="clipPath100835">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4638_"
+         overflow="visible"
+         id="use100837" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4637_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82960" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82962" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4636_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4635_"
+         overflow="visible"
+         id="use82957" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4634_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_"
+         overflow="visible"
+         id="use82937" />
+    </clipPath>
+    <clipPath
+       id="clipPath100846">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_"
+         overflow="visible"
+         id="use100848" />
+    </clipPath>
+    <clipPath
+       id="clipPath100850">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_"
+         overflow="visible"
+         id="use100852" />
+    </clipPath>
+    <clipPath
+       id="clipPath100854">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_"
+         overflow="visible"
+         id="use100856" />
+    </clipPath>
+    <clipPath
+       id="clipPath100858">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_"
+         overflow="visible"
+         id="use100860" />
+    </clipPath>
+    <clipPath
+       id="clipPath100862">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4633_"
+         overflow="visible"
+         id="use100864" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4632_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82925" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82927" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4631_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4630_"
+         overflow="visible"
+         id="use82922" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4629_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_"
+         overflow="visible"
+         id="use82902" />
+    </clipPath>
+    <clipPath
+       id="clipPath100873">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_"
+         overflow="visible"
+         id="use100875" />
+    </clipPath>
+    <clipPath
+       id="clipPath100877">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_"
+         overflow="visible"
+         id="use100879" />
+    </clipPath>
+    <clipPath
+       id="clipPath100881">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_"
+         overflow="visible"
+         id="use100883" />
+    </clipPath>
+    <clipPath
+       id="clipPath100885">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_"
+         overflow="visible"
+         id="use100887" />
+    </clipPath>
+    <clipPath
+       id="clipPath100889">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4628_"
+         overflow="visible"
+         id="use100891" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4627_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46021"
+       x2="256.37631"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82890" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82892" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4626_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4625_"
+         overflow="visible"
+         id="use82887" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4624_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4623_"
+         overflow="visible"
+         id="use82867" />
+    </clipPath>
+    <clipPath
+       id="clipPath100900">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4623_"
+         overflow="visible"
+         id="use100902" />
+    </clipPath>
+    <clipPath
+       id="clipPath100904">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4623_"
+         overflow="visible"
+         id="use100906" />
+    </clipPath>
+    <clipPath
+       id="clipPath100908">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4623_"
+         overflow="visible"
+         id="use100910" />
+    </clipPath>
+    <clipPath
+       id="clipPath100912">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4623_"
+         overflow="visible"
+         id="use100914" />
+    </clipPath>
+    <clipPath
+       id="clipPath100916">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4623_"
+         overflow="visible"
+         id="use100918" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4622_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,284.9569,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82855" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82857" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4621_">
+      <path
+         d="m -516.985,311.531 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.205 -2.692,2.69"
+         id="use82852"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4619_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82832" />
+    </clipPath>
+    <clipPath
+       id="clipPath100927">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100929" />
+    </clipPath>
+    <clipPath
+       id="clipPath100931">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100933" />
+    </clipPath>
+    <clipPath
+       id="clipPath100935">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100937" />
+    </clipPath>
+    <clipPath
+       id="clipPath100939">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100941" />
+    </clipPath>
+    <clipPath
+       id="clipPath100943">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100945" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4617_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,272.9856,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82820" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82822" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4616_">
+      <path
+         d="m -528.956,311.531 c 0,1.487 1.205,2.694 2.691,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.691,1.205 -2.691,2.69"
+         id="use82817"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4614_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82797" />
+    </clipPath>
+    <clipPath
+       id="clipPath100954">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100956" />
+    </clipPath>
+    <clipPath
+       id="clipPath100958">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100960" />
+    </clipPath>
+    <clipPath
+       id="clipPath100962">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100964" />
+    </clipPath>
+    <clipPath
+       id="clipPath100966">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100968" />
+    </clipPath>
+    <clipPath
+       id="clipPath100970">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100972" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4612_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,261.0142,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82785" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82787" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4611_">
+      <path
+         d="m -540.927,311.531 c 0,1.487 1.205,2.694 2.691,2.694 1.488,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.203,-2.69 -2.691,-2.69 -1.486,0 -2.691,1.205 -2.691,2.69"
+         id="use82782"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4609_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82762" />
+    </clipPath>
+    <clipPath
+       id="clipPath100981">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100983" />
+    </clipPath>
+    <clipPath
+       id="clipPath100985">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100987" />
+    </clipPath>
+    <clipPath
+       id="clipPath100989">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100991" />
+    </clipPath>
+    <clipPath
+       id="clipPath100993">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100995" />
+    </clipPath>
+    <clipPath
+       id="clipPath100997">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use100999" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4607_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,249.0428,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82750" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82752" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4606_">
+      <path
+         d="m -552.899,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82747"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4604_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82727" />
+    </clipPath>
+    <clipPath
+       id="clipPath101008">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101010" />
+    </clipPath>
+    <clipPath
+       id="clipPath101012">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101014" />
+    </clipPath>
+    <clipPath
+       id="clipPath101016">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101018" />
+    </clipPath>
+    <clipPath
+       id="clipPath101020">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101022" />
+    </clipPath>
+    <clipPath
+       id="clipPath101024">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101026" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4602_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,237.0715,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82715" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82717" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4601_">
+      <path
+         d="m -564.87,311.531 c 0,1.487 1.205,2.694 2.691,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.691,1.205 -2.691,2.69"
+         id="use82712"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4599_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82692" />
+    </clipPath>
+    <clipPath
+       id="clipPath101035">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101037" />
+    </clipPath>
+    <clipPath
+       id="clipPath101039">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101041" />
+    </clipPath>
+    <clipPath
+       id="clipPath101043">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101045" />
+    </clipPath>
+    <clipPath
+       id="clipPath101047">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101049" />
+    </clipPath>
+    <clipPath
+       id="clipPath101051">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101053" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4597_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,225.1001,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82680" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82682" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4596_">
+      <path
+         d="m -576.842,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82677"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4594_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82657" />
+    </clipPath>
+    <clipPath
+       id="clipPath101062">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101064" />
+    </clipPath>
+    <clipPath
+       id="clipPath101066">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101068" />
+    </clipPath>
+    <clipPath
+       id="clipPath101070">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101072" />
+    </clipPath>
+    <clipPath
+       id="clipPath101074">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101076" />
+    </clipPath>
+    <clipPath
+       id="clipPath101078">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101080" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4592_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82645" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82647" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4591_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4590_"
+         overflow="visible"
+         id="use82642" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4589_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_"
+         overflow="visible"
+         id="use82622" />
+    </clipPath>
+    <clipPath
+       id="clipPath101089">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_"
+         overflow="visible"
+         id="use101091" />
+    </clipPath>
+    <clipPath
+       id="clipPath101093">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_"
+         overflow="visible"
+         id="use101095" />
+    </clipPath>
+    <clipPath
+       id="clipPath101097">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_"
+         overflow="visible"
+         id="use101099" />
+    </clipPath>
+    <clipPath
+       id="clipPath101101">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_"
+         overflow="visible"
+         id="use101103" />
+    </clipPath>
+    <clipPath
+       id="clipPath101105">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4588_"
+         overflow="visible"
+         id="use101107" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4587_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,201.1574,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82610" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82612" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4586_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4585_"
+         overflow="visible"
+         id="use82607" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4584_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_"
+         overflow="visible"
+         id="use82587" />
+    </clipPath>
+    <clipPath
+       id="clipPath101116">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_"
+         overflow="visible"
+         id="use101118" />
+    </clipPath>
+    <clipPath
+       id="clipPath101120">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_"
+         overflow="visible"
+         id="use101122" />
+    </clipPath>
+    <clipPath
+       id="clipPath101124">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_"
+         overflow="visible"
+         id="use101126" />
+    </clipPath>
+    <clipPath
+       id="clipPath101128">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_"
+         overflow="visible"
+         id="use101130" />
+    </clipPath>
+    <clipPath
+       id="clipPath101132">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4583_"
+         overflow="visible"
+         id="use101134" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4582_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82575" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82577" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4581_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4580_"
+         overflow="visible"
+         id="use82572" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4579_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_"
+         overflow="visible"
+         id="use82552" />
+    </clipPath>
+    <clipPath
+       id="clipPath101143">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_"
+         overflow="visible"
+         id="use101145" />
+    </clipPath>
+    <clipPath
+       id="clipPath101147">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_"
+         overflow="visible"
+         id="use101149" />
+    </clipPath>
+    <clipPath
+       id="clipPath101151">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_"
+         overflow="visible"
+         id="use101153" />
+    </clipPath>
+    <clipPath
+       id="clipPath101155">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_"
+         overflow="visible"
+         id="use101157" />
+    </clipPath>
+    <clipPath
+       id="clipPath101159">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4578_"
+         overflow="visible"
+         id="use101161" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4577_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82540" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82542" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4576_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4575_"
+         overflow="visible"
+         id="use82537" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4574_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_"
+         overflow="visible"
+         id="use82517" />
+    </clipPath>
+    <clipPath
+       id="clipPath101170">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_"
+         overflow="visible"
+         id="use101172" />
+    </clipPath>
+    <clipPath
+       id="clipPath101174">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_"
+         overflow="visible"
+         id="use101176" />
+    </clipPath>
+    <clipPath
+       id="clipPath101178">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_"
+         overflow="visible"
+         id="use101180" />
+    </clipPath>
+    <clipPath
+       id="clipPath101182">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_"
+         overflow="visible"
+         id="use101184" />
+    </clipPath>
+    <clipPath
+       id="clipPath101186">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4573_"
+         overflow="visible"
+         id="use101188" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4572_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82505" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82507" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4571_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4570_"
+         overflow="visible"
+         id="use82502" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4569_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use82482" />
+    </clipPath>
+    <clipPath
+       id="clipPath101197">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use101199" />
+    </clipPath>
+    <clipPath
+       id="clipPath101201">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use101203" />
+    </clipPath>
+    <clipPath
+       id="clipPath101205">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use101207" />
+    </clipPath>
+    <clipPath
+       id="clipPath101209">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use101211" />
+    </clipPath>
+    <clipPath
+       id="clipPath101213">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4568_"
+         overflow="visible"
+         id="use101215" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4567_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82470" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82472" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4566_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4565_"
+         overflow="visible"
+         id="use82467" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447" />
+    </clipPath>
+    <clipPath
+       id="clipPath101224">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use101226" />
+    </clipPath>
+    <clipPath
+       id="clipPath101228">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use101230" />
+    </clipPath>
+    <clipPath
+       id="clipPath101232">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use101234" />
+    </clipPath>
+    <clipPath
+       id="clipPath101236">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use101238" />
+    </clipPath>
+    <clipPath
+       id="clipPath101240">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use101242" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4562_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82435" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82437" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4561_">
+      <path
+         d="m -660.642,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82432"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4559_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82412" />
+    </clipPath>
+    <clipPath
+       id="clipPath101251">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101253" />
+    </clipPath>
+    <clipPath
+       id="clipPath101255">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101257" />
+    </clipPath>
+    <clipPath
+       id="clipPath101259">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101261" />
+    </clipPath>
+    <clipPath
+       id="clipPath101263">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101265" />
+    </clipPath>
+    <clipPath
+       id="clipPath101267">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101269" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4557_"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82400" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82402" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4556_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4555_"
+         overflow="visible"
+         id="use82397" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4554_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82373" />
+    </clipPath>
+    <clipPath
+       id="clipPath101278">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101280" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4552_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4551_"
+         overflow="visible"
+         id="use82359" />
+    </clipPath>
+    <clipPath
+       id="clipPath101284">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4551_"
+         overflow="visible"
+         id="use101286" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4550_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4549_"
+         overflow="visible"
+         id="use82345" />
+    </clipPath>
+    <clipPath
+       id="clipPath101290">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4549_"
+         overflow="visible"
+         id="use101292" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4548_"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1255.1711,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82319" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82321" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82323" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82325" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82327" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82329" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82331" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82333" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4547_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4546_"
+         overflow="visible"
+         id="use82316" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4545_"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1321.2666,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82290" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82292" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82294" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82296" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82298" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82300" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82302" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82304" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4544_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4543_"
+         overflow="visible"
+         id="use82287" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4542_"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1387.9897,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82261" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82263" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82265" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82267" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82269" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82271" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82273" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82275" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4541_">
+      <polygon
+         points="-547.185,267.353 -504.831,267.32 -504.824,253.133 -526.362,258.293 -547.178,253.224 "
+         id="use82258" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4525_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4524_"
+         overflow="visible"
+         id="use82164" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4527_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4526_"
+         overflow="visible"
+         id="use82174" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4530_"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3308.0217,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82185" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82187" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4529_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4528_"
+         overflow="visible"
+         id="use82182" />
+    </clipPath>
+    <clipPath
+       id="clipPath101336">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4526_"
+         overflow="visible"
+         id="use101338" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4532_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4531_"
+         overflow="visible"
+         id="use82197" />
+    </clipPath>
+    <clipPath
+       id="clipPath101342">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4524_"
+         overflow="visible"
+         id="use101344" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4534_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4533_"
+         overflow="visible"
+         id="use82209" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4537_"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3481.4861,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82220" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82222" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4536_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4535_"
+         overflow="visible"
+         id="use82217" />
+    </clipPath>
+    <clipPath
+       id="clipPath101353">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4533_"
+         overflow="visible"
+         id="use101355" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4539_">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4538_"
+         overflow="visible"
+         id="use82232" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4523_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82132" />
+    </clipPath>
+    <clipPath
+       id="clipPath101361">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101363" />
+    </clipPath>
+    <clipPath
+       id="clipPath101365">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101367" />
+    </clipPath>
+    <clipPath
+       id="clipPath101369">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101371" />
+    </clipPath>
+    <clipPath
+       id="clipPath101373">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101375" />
+    </clipPath>
+    <clipPath
+       id="clipPath101377">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101379" />
+    </clipPath>
+    <clipPath
+       id="clipPath101381">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101383" />
+    </clipPath>
+    <clipPath
+       id="clipPath101385">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101387" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4521_"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82116" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82118" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82120" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82122" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4520_">
+      <polygon
+         points="-547.302,196.684 -528.257,196.685 -526.139,198.478 -524.02,196.685 -504.975,196.685 -504.975,188.287 -547.302,188.287 "
+         id="use82113" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4518_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82083" />
+    </clipPath>
+    <clipPath
+       id="clipPath101398">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101400" />
+    </clipPath>
+    <clipPath
+       id="clipPath101402">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101404" />
+    </clipPath>
+    <clipPath
+       id="clipPath101406">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101408" />
+    </clipPath>
+    <clipPath
+       id="clipPath101410">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101412" />
+    </clipPath>
+    <clipPath
+       id="clipPath101414">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101416" />
+    </clipPath>
+    <clipPath
+       id="clipPath101418">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101420" />
+    </clipPath>
+    <clipPath
+       id="clipPath101422">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101424" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4516_"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,902.0769,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82067" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82069" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82071" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82073" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4515_">
+      <polygon
+         points="-590.704,196.685 -571.658,196.685 -571.658,188.287 -613.986,188.287 -613.986,196.684 -594.94,196.685 -592.822,198.478 "
+         id="use82064" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4511_">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82010" />
+    </clipPath>
+    <clipPath
+       id="clipPath101435">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101437" />
+    </clipPath>
+    <clipPath
+       id="clipPath101439">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101441" />
+    </clipPath>
+    <clipPath
+       id="clipPath101443">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101445" />
+    </clipPath>
+    <clipPath
+       id="clipPath101447">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101449" />
+    </clipPath>
+    <clipPath
+       id="clipPath101451">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101453" />
+    </clipPath>
+    <clipPath
+       id="clipPath101455">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101457" />
+    </clipPath>
+    <clipPath
+       id="clipPath101459">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use101461" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4509_"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop81994" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop81996" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop81998" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82000" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4508_">
+      <polygon
+         points="-657.048,196.685 -638.002,196.685 -638.002,188.287 -680.33,188.287 -680.33,196.684 -661.285,196.685 -659.167,198.478 "
+         id="use81991" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient104534"
+       xlink:href="#SVGID_4509_"
+       inkscape:collect="always" />
+    <defs
+       id="defs84496">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4842_" />
+    </defs>
+    <clipPath
+       id="clipPath103391">
+      <use
+         id="use103393"
+         overflow="visible"
+         xlink:href="#SVGID_4842_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4873_-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84798-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath113428">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113430" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4875_-9">
+      <rect
+         height="10.152"
+         width="174.95799"
+         y="360.13901"
+         x="-679.90002"
+         id="use84810-6" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4878_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="40.563"
+       y1="147.0757"
+       x2="41.563"
+       y2="147.0757"
+       gradientTransform="matrix(0,-10.1526,-10.1526,0,900.78,782.1111)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84821-2" />
+      <stop
+         offset="0.486"
+         style="stop-color:#B0D2D9"
+         id="stop84823-4" />
+      <stop
+         offset="0.8287"
+         style="stop-color:#E6EFF1"
+         id="stop84825-9" />
+      <stop
+         offset="0.9939"
+         style="stop-color:#FFFFFF"
+         id="stop84827-7" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84829-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4877_-0">
+      <rect
+         height="10.152"
+         width="174.95799"
+         y="360.13901"
+         x="-679.90002"
+         id="use84818-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath113442">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113444" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4880_-0">
+      <rect
+         height="10.152"
+         width="174.94701"
+         y="360.13901"
+         x="-679.88898"
+         id="use84841-6" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4882_-9">
+      <rect
+         height="10.153"
+         width="174.94701"
+         y="360.138"
+         x="-679.88898"
+         id="use84849-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath113450">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113452" />
+    </clipPath>
+    <clipPath
+       id="clipPath113454">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113456" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4871_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94051"
+       x2="-33.205299"
+       y2="249.94051"
+       gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84774-6" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84776-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4870_-6">
+      <polygon
+         points="-654.922,273.567 -656.715,276.108 -654.922,278.632 -640.956,278.632 -640.956,273.567 "
+         id="use84771-5" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4868_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84759-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath113465">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113467" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4866_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.9404"
+       x2="-33.205299"
+       y2="249.9404"
+       gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84735-4" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84737-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4865_-6">
+      <polygon
+         points="-588.288,273.567 -590.081,276.108 -588.288,278.632 -574.322,278.632 -574.322,273.567 "
+         id="use84732-8" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4863_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84720-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath113476">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113478" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4851_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.9404"
+       x2="-33.205101"
+       y2="249.9404"
+       gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84622-4" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84624-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4850_-1">
+      <polygon
+         points="-588.288,203.458 -590.081,206 -588.288,208.524 -574.322,208.524 -574.322,203.458 "
+         id="use84619-3" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4848_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84607-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath113487">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113489" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4846_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94051"
+       x2="-33.205101"
+       y2="249.94051"
+       gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84585-1" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84587-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4845_-3">
+      <polygon
+         points="-654.922,203.458 -656.715,206 -654.922,208.524 -640.956,208.524 -640.956,203.458 "
+         id="use84582-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath113498">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113500" />
+    </clipPath>
+    <clipPath
+       id="clipPath113502">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113504" />
+    </clipPath>
+    <clipPath
+       id="clipPath113506">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113508" />
+    </clipPath>
+    <clipPath
+       id="clipPath113510">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113512" />
+    </clipPath>
+    <clipPath
+       id="clipPath113514">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113516" />
+    </clipPath>
+    <clipPath
+       id="clipPath113518">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113520" />
+    </clipPath>
+    <clipPath
+       id="clipPath113522">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use113524" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4838_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84445-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath113528">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113530" />
+    </clipPath>
+    <clipPath
+       id="clipPath113532">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113534" />
+    </clipPath>
+    <clipPath
+       id="clipPath113536">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113538" />
+    </clipPath>
+    <clipPath
+       id="clipPath113540">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113542" />
+    </clipPath>
+    <clipPath
+       id="clipPath113544">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113546" />
+    </clipPath>
+    <clipPath
+       id="clipPath113548">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113550" />
+    </clipPath>
+    <clipPath
+       id="clipPath113552">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113554" />
+    </clipPath>
+    <clipPath
+       id="clipPath113556">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113558" />
+    </clipPath>
+    <clipPath
+       id="clipPath113560">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113562" />
+    </clipPath>
+    <clipPath
+       id="clipPath113564">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113566" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4836_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.8129"
+       x2="37.392799"
+       y2="152.8129"
+       gradientTransform="matrix(0,-9.614,-9.614,0,876.3496,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84423-1" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84425-8" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84427-7" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84429-9" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84431-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4835_-5">
+      <rect
+         height="9.6140003"
+         width="11.315"
+         y="214.106"
+         x="-598.45099"
+         id="use84420-5" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4833_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84390-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath113578">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113580" />
+    </clipPath>
+    <clipPath
+       id="clipPath113582">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113584" />
+    </clipPath>
+    <clipPath
+       id="clipPath113586">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113588" />
+    </clipPath>
+    <clipPath
+       id="clipPath113590">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113592" />
+    </clipPath>
+    <clipPath
+       id="clipPath113594">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113596" />
+    </clipPath>
+    <clipPath
+       id="clipPath113598">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113600" />
+    </clipPath>
+    <clipPath
+       id="clipPath113602">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113604" />
+    </clipPath>
+    <clipPath
+       id="clipPath113606">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113608" />
+    </clipPath>
+    <clipPath
+       id="clipPath113610">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113612" />
+    </clipPath>
+    <clipPath
+       id="clipPath113614">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113616" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4831_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.813"
+       x2="37.392799"
+       y2="152.813"
+       gradientTransform="matrix(0,-9.614,-9.614,0,809.7396,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84368-9" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84370-7" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84372-0" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84374-4" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84376-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4830_-7">
+      <rect
+         height="9.6140003"
+         width="11.316"
+         y="214.106"
+         x="-665.06201"
+         id="use84365-8" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4828_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-734.6116,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84329-2" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84331-0" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84333-0" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84335-1" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84337-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4827_-3">
+      <polygon
+         points="-682.208,251.221 -659.342,256.788 -636.356,251.283 -636.358,231.934 -682.207,231.929 "
+         id="use84326-6" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4825_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-668.057,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84298-6" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84300-1" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84302-6" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84304-9" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84306-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4824_-9">
+      <polygon
+         points="-615.653,251.221 -592.788,256.788 -569.802,251.283 -569.804,231.934 -615.653,231.929 "
+         id="use84295-9" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4787_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84010-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84012-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4786_-8">
+      <path
+         d="m -588.813,346.449 c 0,1.487 1.205,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use84007-6"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4784_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83987-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath113649">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113651" />
+    </clipPath>
+    <clipPath
+       id="clipPath113653">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113655" />
+    </clipPath>
+    <clipPath
+       id="clipPath113657">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113659" />
+    </clipPath>
+    <clipPath
+       id="clipPath113661">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113663" />
+    </clipPath>
+    <clipPath
+       id="clipPath113665">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113667" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4782_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83975-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83977-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4781_-3">
+      <path
+         d="m -600.785,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83972-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4779_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83952-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath113676">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113678" />
+    </clipPath>
+    <clipPath
+       id="clipPath113680">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113682" />
+    </clipPath>
+    <clipPath
+       id="clipPath113684">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113686" />
+    </clipPath>
+    <clipPath
+       id="clipPath113688">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113690" />
+    </clipPath>
+    <clipPath
+       id="clipPath113692">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113694" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4777_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83940-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83942-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4776_-4">
+      <path
+         d="m -612.756,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83937-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4774_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83917-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath113703">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113705" />
+    </clipPath>
+    <clipPath
+       id="clipPath113707">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113709" />
+    </clipPath>
+    <clipPath
+       id="clipPath113711">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113713" />
+    </clipPath>
+    <clipPath
+       id="clipPath113715">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113717" />
+    </clipPath>
+    <clipPath
+       id="clipPath113719">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113721" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4772_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,177.1768,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83905-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83907-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4771_-5">
+      <path
+         d="m -624.728,346.449 c 0,1.487 1.206,2.692 2.693,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.693,1.206 -2.693,2.692"
+         id="use83902-7"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4769_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83882-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath113730">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113732" />
+    </clipPath>
+    <clipPath
+       id="clipPath113734">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113736" />
+    </clipPath>
+    <clipPath
+       id="clipPath113738">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113740" />
+    </clipPath>
+    <clipPath
+       id="clipPath113742">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113744" />
+    </clipPath>
+    <clipPath
+       id="clipPath113746">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113748" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4767_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83870-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83872-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4766_-5">
+      <path
+         d="m -636.699,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83867-0"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4764_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83847-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath113757">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113759" />
+    </clipPath>
+    <clipPath
+       id="clipPath113761">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113763" />
+    </clipPath>
+    <clipPath
+       id="clipPath113765">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113767" />
+    </clipPath>
+    <clipPath
+       id="clipPath113769">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113771" />
+    </clipPath>
+    <clipPath
+       id="clipPath113773">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113775" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4762_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83835-5" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83837-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4761_-9">
+      <path
+         d="m -648.671,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83832-7"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4759_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83812-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath113784">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113786" />
+    </clipPath>
+    <clipPath
+       id="clipPath113788">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113790" />
+    </clipPath>
+    <clipPath
+       id="clipPath113792">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113794" />
+    </clipPath>
+    <clipPath
+       id="clipPath113796">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113798" />
+    </clipPath>
+    <clipPath
+       id="clipPath113800">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113802" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4757_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83800-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83802-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4756_-5">
+      <path
+         d="m -660.642,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83797-6"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4754_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83777-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath113811">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113813" />
+    </clipPath>
+    <clipPath
+       id="clipPath113815">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113817" />
+    </clipPath>
+    <clipPath
+       id="clipPath113819">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113821" />
+    </clipPath>
+    <clipPath
+       id="clipPath113823">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113825" />
+    </clipPath>
+    <clipPath
+       id="clipPath113827">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113829" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4752_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83765-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83767-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4751_-0">
+      <path
+         d="m -672.614,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83762-6"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4722_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83555-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83557-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4721_-5">
+      <path
+         d="m -588.813,334.666 c 0,1.488 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use83552-6"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4719_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83532-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath113843">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113845" />
+    </clipPath>
+    <clipPath
+       id="clipPath113847">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113849" />
+    </clipPath>
+    <clipPath
+       id="clipPath113851">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113853" />
+    </clipPath>
+    <clipPath
+       id="clipPath113855">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113857" />
+    </clipPath>
+    <clipPath
+       id="clipPath113859">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113861" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4717_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83520-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83522-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4716_-1">
+      <path
+         d="m -600.785,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83517-1"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4714_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83497-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath113870">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113872" />
+    </clipPath>
+    <clipPath
+       id="clipPath113874">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113876" />
+    </clipPath>
+    <clipPath
+       id="clipPath113878">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113880" />
+    </clipPath>
+    <clipPath
+       id="clipPath113882">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113884" />
+    </clipPath>
+    <clipPath
+       id="clipPath113886">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113888" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4712_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83485-6" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83487-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4711_-6">
+      <path
+         d="m -612.756,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83482-0"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4709_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83462-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath113897">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113899" />
+    </clipPath>
+    <clipPath
+       id="clipPath113901">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113903" />
+    </clipPath>
+    <clipPath
+       id="clipPath113905">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113907" />
+    </clipPath>
+    <clipPath
+       id="clipPath113909">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113911" />
+    </clipPath>
+    <clipPath
+       id="clipPath113913">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113915" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4707_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3544"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83450-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83452-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4706_-5">
+      <path
+         d="m -636.699,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83447-9"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4704_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83427-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath113924">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113926" />
+    </clipPath>
+    <clipPath
+       id="clipPath113928">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113930" />
+    </clipPath>
+    <clipPath
+       id="clipPath113932">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113934" />
+    </clipPath>
+    <clipPath
+       id="clipPath113936">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113938" />
+    </clipPath>
+    <clipPath
+       id="clipPath113940">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113942" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4702_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83415-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83417-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4701_-3">
+      <path
+         d="m -648.671,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83412-9"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4699_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83392-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath113951">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113953" />
+    </clipPath>
+    <clipPath
+       id="clipPath113955">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113957" />
+    </clipPath>
+    <clipPath
+       id="clipPath113959">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113961" />
+    </clipPath>
+    <clipPath
+       id="clipPath113963">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113965" />
+    </clipPath>
+    <clipPath
+       id="clipPath113967">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113969" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4697_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.35419"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83380-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83382-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4696_-2">
+      <path
+         d="m -660.642,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83377-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4694_-4">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83357-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath113978">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113980" />
+    </clipPath>
+    <clipPath
+       id="clipPath113982">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113984" />
+    </clipPath>
+    <clipPath
+       id="clipPath113986">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113988" />
+    </clipPath>
+    <clipPath
+       id="clipPath113990">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113992" />
+    </clipPath>
+    <clipPath
+       id="clipPath113994">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use113996" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4692_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83345-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83347-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4691_-2">
+      <path
+         d="m -672.614,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83342-7"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4657_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83100-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83102-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4656_-1">
+      <path
+         d="m -588.813,323.119 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.204 -2.692,2.69"
+         id="use83097-8"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4654_-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83077-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath114010">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114012" />
+    </clipPath>
+    <clipPath
+       id="clipPath114014">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114016" />
+    </clipPath>
+    <clipPath
+       id="clipPath114018">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114020" />
+    </clipPath>
+    <clipPath
+       id="clipPath114022">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114024" />
+    </clipPath>
+    <clipPath
+       id="clipPath114026">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114028" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4652_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83065-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83067-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4651_-4">
+      <path
+         d="m -612.756,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use83062-2"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4649_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83042-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath114037">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114039" />
+    </clipPath>
+    <clipPath
+       id="clipPath114041">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114043" />
+    </clipPath>
+    <clipPath
+       id="clipPath114045">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114047" />
+    </clipPath>
+    <clipPath
+       id="clipPath114049">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114051" />
+    </clipPath>
+    <clipPath
+       id="clipPath114053">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114055" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4647_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83030-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83032-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4646_-1">
+      <path
+         d="m -624.728,323.119 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.204 -2.693,2.69"
+         id="use83027-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4644_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83007-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath114064">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114066" />
+    </clipPath>
+    <clipPath
+       id="clipPath114068">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114070" />
+    </clipPath>
+    <clipPath
+       id="clipPath114072">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114074" />
+    </clipPath>
+    <clipPath
+       id="clipPath114076">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114078" />
+    </clipPath>
+    <clipPath
+       id="clipPath114080">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114082" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4642_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82995-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82997-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4641_-8">
+      <path
+         d="m -636.699,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82992-0"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4639_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82972-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath114091">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114093" />
+    </clipPath>
+    <clipPath
+       id="clipPath114095">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114097" />
+    </clipPath>
+    <clipPath
+       id="clipPath114099">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114101" />
+    </clipPath>
+    <clipPath
+       id="clipPath114103">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114105" />
+    </clipPath>
+    <clipPath
+       id="clipPath114107">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114109" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4637_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82960-5" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82962-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4636_-6">
+      <path
+         d="m -648.671,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82957-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4634_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82937-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath114118">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114120" />
+    </clipPath>
+    <clipPath
+       id="clipPath114122">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114124" />
+    </clipPath>
+    <clipPath
+       id="clipPath114126">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114128" />
+    </clipPath>
+    <clipPath
+       id="clipPath114130">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114132" />
+    </clipPath>
+    <clipPath
+       id="clipPath114134">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114136" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4632_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82925-6" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82927-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4631_-1">
+      <path
+         d="m -660.642,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82922-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4629_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82902-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath114145">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114147" />
+    </clipPath>
+    <clipPath
+       id="clipPath114149">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114151" />
+    </clipPath>
+    <clipPath
+       id="clipPath114153">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114155" />
+    </clipPath>
+    <clipPath
+       id="clipPath114157">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114159" />
+    </clipPath>
+    <clipPath
+       id="clipPath114161">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114163" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4627_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46021"
+       x2="256.37631"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82890-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82892-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4626_-6">
+      <path
+         d="m -672.614,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82887-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4592_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82645-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82647-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4591_-6">
+      <path
+         d="m -588.813,311.531 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.205 -2.692,2.69"
+         id="use82642-0"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4589_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82622-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114177">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114179" />
+    </clipPath>
+    <clipPath
+       id="clipPath114181">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114183" />
+    </clipPath>
+    <clipPath
+       id="clipPath114185">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114187" />
+    </clipPath>
+    <clipPath
+       id="clipPath114189">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114191" />
+    </clipPath>
+    <clipPath
+       id="clipPath114193">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114195" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4587_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,201.1574,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82610-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82612-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4586_-0">
+      <path
+         d="m -600.785,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.203,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82607-0"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4584_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82587-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath114204">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114206" />
+    </clipPath>
+    <clipPath
+       id="clipPath114208">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114210" />
+    </clipPath>
+    <clipPath
+       id="clipPath114212">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114214" />
+    </clipPath>
+    <clipPath
+       id="clipPath114216">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114218" />
+    </clipPath>
+    <clipPath
+       id="clipPath114220">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114222" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4582_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82575-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82577-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4581_-6">
+      <path
+         d="m -612.756,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82572-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4579_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82552-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath114231">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114233" />
+    </clipPath>
+    <clipPath
+       id="clipPath114235">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114237" />
+    </clipPath>
+    <clipPath
+       id="clipPath114239">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114241" />
+    </clipPath>
+    <clipPath
+       id="clipPath114243">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114245" />
+    </clipPath>
+    <clipPath
+       id="clipPath114247">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114249" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4577_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82540-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82542-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4576_-2">
+      <path
+         d="m -624.728,311.531 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.205 -2.693,2.69"
+         id="use82537-2"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4574_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82517-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114258">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114260" />
+    </clipPath>
+    <clipPath
+       id="clipPath114262">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114264" />
+    </clipPath>
+    <clipPath
+       id="clipPath114266">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114268" />
+    </clipPath>
+    <clipPath
+       id="clipPath114270">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114272" />
+    </clipPath>
+    <clipPath
+       id="clipPath114274">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114276" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4572_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82505-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82507-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4571_-1">
+      <path
+         d="m -636.699,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82502-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4569_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82482-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath114285">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114287" />
+    </clipPath>
+    <clipPath
+       id="clipPath114289">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114291" />
+    </clipPath>
+    <clipPath
+       id="clipPath114293">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114295" />
+    </clipPath>
+    <clipPath
+       id="clipPath114297">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114299" />
+    </clipPath>
+    <clipPath
+       id="clipPath114301">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114303" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4567_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82470-6" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82472-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4566_-1">
+      <path
+         d="m -648.671,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82467-9"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-4">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82447-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath114312">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114314" />
+    </clipPath>
+    <clipPath
+       id="clipPath114316">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114318" />
+    </clipPath>
+    <clipPath
+       id="clipPath114320">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114322" />
+    </clipPath>
+    <clipPath
+       id="clipPath114324">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114326" />
+    </clipPath>
+    <clipPath
+       id="clipPath114328">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114330" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4562_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82435-2" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82437-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4561_-1">
+      <path
+         d="m -660.642,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82432-6"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4559_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82412-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath114339">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114341" />
+    </clipPath>
+    <clipPath
+       id="clipPath114343">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114345" />
+    </clipPath>
+    <clipPath
+       id="clipPath114347">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114349" />
+    </clipPath>
+    <clipPath
+       id="clipPath114351">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114353" />
+    </clipPath>
+    <clipPath
+       id="clipPath114355">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114357" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4557_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82400-6" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82402-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4556_-5">
+      <path
+         d="m -672.614,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82397-5"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4552_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82359-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath114366">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114368" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4550_-0">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82345-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114372">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114374" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4548_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1255.1711,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82319-4" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82321-6" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82323-0" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82325-7" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82327-0" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82329-5" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82331-6" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82333-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4547_-4">
+      <polygon
+         points="-659.18,258.293 -679.997,253.224 -680.004,267.353 -637.65,267.32 -637.642,253.133 "
+         id="use82316-8" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4545_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1321.2666,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82290-3" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82292-3" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82294-2" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82296-5" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82298-7" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82300-2" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82302-9" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82304-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4544_-2">
+      <polygon
+         points="-593.085,258.293 -613.901,253.224 -613.908,267.353 -571.554,267.32 -571.547,253.133 "
+         id="use82287-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4525_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82164-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4527_-8">
+      <rect
+         height="64.251999"
+         width="175.168"
+         y="293.259"
+         x="-679.67902"
+         id="use82174-4" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4530_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3308.0217,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82185-5" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82187-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4529_-6">
+      <rect
+         height="64.037003"
+         width="174.953"
+         y="293.367"
+         x="-679.57202"
+         id="use82182-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath114407">
+      <rect
+         height="64.251999"
+         width="175.168"
+         y="293.259"
+         x="-679.67902"
+         id="use114409" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4532_-7">
+      <rect
+         height="64.251999"
+         width="175.168"
+         y="293.259"
+         x="-679.67902"
+         id="use82197-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114413">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114415" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4534_-4">
+      <rect
+         height="64.251999"
+         width="45.873001"
+         y="293.259"
+         x="-441.56699"
+         id="use82209-7" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4537_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3481.4861,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82220-6" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82222-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4536_-8">
+      <rect
+         height="64.037003"
+         width="45.659"
+         y="293.367"
+         x="-441.45999"
+         id="use82217-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114424">
+      <rect
+         height="64.251999"
+         width="45.873001"
+         y="293.259"
+         x="-441.56699"
+         id="use114426" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4539_-9">
+      <rect
+         height="64.251999"
+         width="45.873001"
+         y="293.259"
+         x="-441.56699"
+         id="use82232-6" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4518_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82083-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114432">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114434" />
+    </clipPath>
+    <clipPath
+       id="clipPath114436">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114438" />
+    </clipPath>
+    <clipPath
+       id="clipPath114440">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114442" />
+    </clipPath>
+    <clipPath
+       id="clipPath114444">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114446" />
+    </clipPath>
+    <clipPath
+       id="clipPath114448">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114450" />
+    </clipPath>
+    <clipPath
+       id="clipPath114452">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114454" />
+    </clipPath>
+    <clipPath
+       id="clipPath114456">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114458" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4516_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,902.0769,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82067-5" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82069-0" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82071-8" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82073-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4515_-1">
+      <polygon
+         points="-613.986,188.287 -613.986,196.684 -594.94,196.685 -592.822,198.478 -590.704,196.685 -571.658,196.685 -571.658,188.287 "
+         id="use82064-3" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4511_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82010-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath114469">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114471" />
+    </clipPath>
+    <clipPath
+       id="clipPath114473">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114475" />
+    </clipPath>
+    <clipPath
+       id="clipPath114477">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114479" />
+    </clipPath>
+    <clipPath
+       id="clipPath114481">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114483" />
+    </clipPath>
+    <clipPath
+       id="clipPath114485">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114487" />
+    </clipPath>
+    <clipPath
+       id="clipPath114489">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114491" />
+    </clipPath>
+    <clipPath
+       id="clipPath114493">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use114495" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient104534-0"
+       xlink:href="#SVGID_4509_-4"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4509_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop81994-0" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop81996-4" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop81998-3" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82000-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4508_-0">
+      <polygon
+         points="-680.33,188.287 -680.33,196.684 -661.285,196.685 -659.167,198.478 -657.048,196.685 -638.002,196.685 -638.002,188.287 "
+         id="use81991-1" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient115446"
+       xlink:href="#SVGID_4509_-4"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4657_-1-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83100-8-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83102-8-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4656_-1-2">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4655_-2-7"
+         overflow="visible"
+         id="use83097-8-0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4550_-0-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82345-2-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath118798">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use118800" />
+    </clipPath>
+    <defs
+       id="defs83093-1-3">
+      <path
+         d="m -588.813,323.119 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.204 -2.692,2.69"
+         id="SVGID_4655_-2-7"
+         inkscape:connector-curvature="0" />
+    </defs>
+    <clipPath
+       id="clipPath102313-6-0">
+      <use
+         id="use102315-6-2"
+         overflow="visible"
+         xlink:href="#SVGID_4655_-2-7"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <linearGradient
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)"
+       y2="-148.4601"
+       x2="256.37631"
+       y1="-148.4601"
+       x1="255.37621"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient102317-9-1">
+      <stop
+         id="stop102319-7-7"
+         style="stop-color:#55AEB9"
+         offset="0" />
+      <stop
+         id="stop102321-3-3"
+         style="stop-color:#81D1DB"
+         offset="1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4873_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84798-79" />
+    </clipPath>
+    <clipPath
+       id="clipPath119610">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119612" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4875_-5">
+      <rect
+         height="10.152"
+         width="174.95799"
+         y="360.13901"
+         x="-679.90002"
+         id="use84810-0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4878_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="40.563"
+       y1="147.0757"
+       x2="41.563"
+       y2="147.0757"
+       gradientTransform="matrix(0,-10.1526,-10.1526,0,900.78,782.1111)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84821-4" />
+      <stop
+         offset="0.486"
+         style="stop-color:#B0D2D9"
+         id="stop84823-2" />
+      <stop
+         offset="0.8287"
+         style="stop-color:#E6EFF1"
+         id="stop84825-99" />
+      <stop
+         offset="0.9939"
+         style="stop-color:#FFFFFF"
+         id="stop84827-79" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84829-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4877_-3">
+      <rect
+         height="10.152"
+         width="174.95799"
+         y="360.13901"
+         x="-679.90002"
+         id="use84818-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath119624">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119626" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4880_-1">
+      <rect
+         height="10.152"
+         width="174.94701"
+         y="360.13901"
+         x="-679.88898"
+         id="use84841-0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4882_-6">
+      <rect
+         height="10.153"
+         width="174.94701"
+         y="360.138"
+         x="-679.88898"
+         id="use84849-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath119632">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119634" />
+    </clipPath>
+    <clipPath
+       id="clipPath119636">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119638" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4871_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94051"
+       x2="-33.205299"
+       y2="249.94051"
+       gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84774-61" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84776-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4870_-2">
+      <polygon
+         points="-654.922,278.632 -640.956,278.632 -640.956,273.567 -654.922,273.567 -656.715,276.108 "
+         id="use84771-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4868_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84759-44" />
+    </clipPath>
+    <clipPath
+       id="clipPath119647">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119649" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4866_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.9404"
+       x2="-33.205299"
+       y2="249.9404"
+       gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,105.3813)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84735-9" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84737-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4865_-0">
+      <polygon
+         points="-588.288,278.632 -574.322,278.632 -574.322,273.567 -588.288,273.567 -590.081,276.108 "
+         id="use84732-6" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4863_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84720-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath119658">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119660" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4851_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.9404"
+       x2="-33.205101"
+       y2="249.9404"
+       gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84622-0" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84624-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4850_-19">
+      <polygon
+         points="-588.288,208.524 -574.322,208.524 -574.322,203.458 -588.288,203.458 -590.081,206 "
+         id="use84619-38" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4848_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84607-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath119669">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119671" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4846_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94051"
+       x2="-33.205101"
+       y2="249.94051"
+       gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84585-6" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84587-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4845_-4">
+      <polygon
+         points="-654.922,208.524 -640.956,208.524 -640.956,203.458 -654.922,203.458 -656.715,206 "
+         id="use84582-3" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath119680">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119682" />
+    </clipPath>
+    <clipPath
+       id="clipPath119684">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119686" />
+    </clipPath>
+    <clipPath
+       id="clipPath119688">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119690" />
+    </clipPath>
+    <clipPath
+       id="clipPath119692">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119694" />
+    </clipPath>
+    <clipPath
+       id="clipPath119696">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119698" />
+    </clipPath>
+    <clipPath
+       id="clipPath119700">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119702" />
+    </clipPath>
+    <clipPath
+       id="clipPath119704">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use119706" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4838_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84445-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath119710">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119712" />
+    </clipPath>
+    <clipPath
+       id="clipPath119714">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119716" />
+    </clipPath>
+    <clipPath
+       id="clipPath119718">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119720" />
+    </clipPath>
+    <clipPath
+       id="clipPath119722">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119724" />
+    </clipPath>
+    <clipPath
+       id="clipPath119726">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119728" />
+    </clipPath>
+    <clipPath
+       id="clipPath119730">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119732" />
+    </clipPath>
+    <clipPath
+       id="clipPath119734">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119736" />
+    </clipPath>
+    <clipPath
+       id="clipPath119738">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119740" />
+    </clipPath>
+    <clipPath
+       id="clipPath119742">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119744" />
+    </clipPath>
+    <clipPath
+       id="clipPath119746">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119748" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4836_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.8129"
+       x2="37.392799"
+       y2="152.8129"
+       gradientTransform="matrix(0,-9.614,-9.614,0,876.3496,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84423-5" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84425-7" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84427-8" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84429-1" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84431-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4835_-3">
+      <rect
+         height="9.6140003"
+         width="11.315"
+         y="214.106"
+         x="-598.45099"
+         id="use84420-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4833_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84390-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath119760">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119762" />
+    </clipPath>
+    <clipPath
+       id="clipPath119764">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119766" />
+    </clipPath>
+    <clipPath
+       id="clipPath119768">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119770" />
+    </clipPath>
+    <clipPath
+       id="clipPath119772">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119774" />
+    </clipPath>
+    <clipPath
+       id="clipPath119776">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119778" />
+    </clipPath>
+    <clipPath
+       id="clipPath119780">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119782" />
+    </clipPath>
+    <clipPath
+       id="clipPath119784">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119786" />
+    </clipPath>
+    <clipPath
+       id="clipPath119788">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119790" />
+    </clipPath>
+    <clipPath
+       id="clipPath119792">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119794" />
+    </clipPath>
+    <clipPath
+       id="clipPath119796">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119798" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4831_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.813"
+       x2="37.392799"
+       y2="152.813"
+       gradientTransform="matrix(0,-9.614,-9.614,0,809.7396,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84368-1" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84370-0" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84372-8" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84374-42" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84376-15" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4830_-5">
+      <rect
+         height="9.6140003"
+         width="11.316"
+         y="214.106"
+         x="-665.06201"
+         id="use84365-9" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4828_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-734.6116,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84329-5" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84331-2" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84333-3" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84335-9" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84337-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4827_-8">
+      <polygon
+         points="-636.356,251.283 -636.358,231.934 -682.207,231.929 -682.208,251.221 -659.342,256.788 "
+         id="use84326-1" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4825_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="145.3168"
+       y1="2.8518"
+       x2="146.3168"
+       y2="2.8518"
+       gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-668.057,-3384.4426)">
+      <stop
+         offset="0"
+         style="stop-color:#FFFFFF"
+         id="stop84298-1" />
+      <stop
+         offset="0.23"
+         style="stop-color:#FFFFFF"
+         id="stop84300-7" />
+      <stop
+         offset="0.5671"
+         style="stop-color:#F9F9F9"
+         id="stop84302-5" />
+      <stop
+         offset="0.8674"
+         style="stop-color:#EDEEEE"
+         id="stop84304-7" />
+      <stop
+         offset="1"
+         style="stop-color:#E6E8E7"
+         id="stop84306-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4824_-6">
+      <polygon
+         points="-569.802,251.283 -569.804,231.934 -615.653,231.929 -615.653,251.221 -592.788,256.788 "
+         id="use84295-8" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4787_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop84010-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop84012-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4786_-2">
+      <path
+         d="m -588.813,346.449 c 0,1.487 1.205,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use84007-5"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4784_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83987-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath119831">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119833" />
+    </clipPath>
+    <clipPath
+       id="clipPath119835">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119837" />
+    </clipPath>
+    <clipPath
+       id="clipPath119839">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119841" />
+    </clipPath>
+    <clipPath
+       id="clipPath119843">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119845" />
+    </clipPath>
+    <clipPath
+       id="clipPath119847">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119849" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4782_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83975-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83977-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4781_-1">
+      <path
+         d="m -600.785,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83972-45"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4779_-85">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83952-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath119858">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119860" />
+    </clipPath>
+    <clipPath
+       id="clipPath119862">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119864" />
+    </clipPath>
+    <clipPath
+       id="clipPath119866">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119868" />
+    </clipPath>
+    <clipPath
+       id="clipPath119870">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119872" />
+    </clipPath>
+    <clipPath
+       id="clipPath119874">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119876" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4777_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83940-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83942-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4776_-7">
+      <path
+         d="m -612.756,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83937-8"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4774_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83917-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath119885">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119887" />
+    </clipPath>
+    <clipPath
+       id="clipPath119889">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119891" />
+    </clipPath>
+    <clipPath
+       id="clipPath119893">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119895" />
+    </clipPath>
+    <clipPath
+       id="clipPath119897">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119899" />
+    </clipPath>
+    <clipPath
+       id="clipPath119901">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119903" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4772_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,177.1768,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83905-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83907-6" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4771_-2">
+      <path
+         d="m -624.728,346.449 c 0,1.487 1.206,2.692 2.693,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.693,1.206 -2.693,2.692"
+         id="use83902-70"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4769_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83882-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath119912">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119914" />
+    </clipPath>
+    <clipPath
+       id="clipPath119916">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119918" />
+    </clipPath>
+    <clipPath
+       id="clipPath119920">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119922" />
+    </clipPath>
+    <clipPath
+       id="clipPath119924">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119926" />
+    </clipPath>
+    <clipPath
+       id="clipPath119928">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119930" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4767_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83870-91" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83872-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4766_-6">
+      <path
+         d="m -636.699,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83867-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4764_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83847-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath119939">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119941" />
+    </clipPath>
+    <clipPath
+       id="clipPath119943">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119945" />
+    </clipPath>
+    <clipPath
+       id="clipPath119947">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119949" />
+    </clipPath>
+    <clipPath
+       id="clipPath119951">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119953" />
+    </clipPath>
+    <clipPath
+       id="clipPath119955">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119957" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4762_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3541"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83835-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83837-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4761_-3">
+      <path
+         d="m -648.671,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83832-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4759_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83812-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath119966">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119968" />
+    </clipPath>
+    <clipPath
+       id="clipPath119970">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119972" />
+    </clipPath>
+    <clipPath
+       id="clipPath119974">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119976" />
+    </clipPath>
+    <clipPath
+       id="clipPath119978">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119980" />
+    </clipPath>
+    <clipPath
+       id="clipPath119982">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119984" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4757_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3541"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83800-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83802-76" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4756_-1">
+      <path
+         d="m -660.642,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83797-7"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4754_-4">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83777-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath119993">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119995" />
+    </clipPath>
+    <clipPath
+       id="clipPath119997">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use119999" />
+    </clipPath>
+    <clipPath
+       id="clipPath120001">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120003" />
+    </clipPath>
+    <clipPath
+       id="clipPath120005">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120007" />
+    </clipPath>
+    <clipPath
+       id="clipPath120009">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120011" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4752_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3541"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1031.1833)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83765-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83767-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4751_-6">
+      <path
+         d="m -672.614,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83762-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4749_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83742-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath120020">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120022" />
+    </clipPath>
+    <clipPath
+       id="clipPath120024">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120026" />
+    </clipPath>
+    <clipPath
+       id="clipPath120028">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120030" />
+    </clipPath>
+    <clipPath
+       id="clipPath120032">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120034" />
+    </clipPath>
+    <clipPath
+       id="clipPath120036">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120038" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4722_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83555-85" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83557-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4721_-8">
+      <path
+         d="m -588.813,334.666 c 0,1.488 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+         id="use83552-2"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4719_-85">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83532-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath120047">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120049" />
+    </clipPath>
+    <clipPath
+       id="clipPath120051">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120053" />
+    </clipPath>
+    <clipPath
+       id="clipPath120055">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120057" />
+    </clipPath>
+    <clipPath
+       id="clipPath120059">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120061" />
+    </clipPath>
+    <clipPath
+       id="clipPath120063">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120065" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4717_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83520-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83522-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4716_-0">
+      <path
+         d="m -600.785,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83517-0"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4714_-3">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83497-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath120074">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120076" />
+    </clipPath>
+    <clipPath
+       id="clipPath120078">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120080" />
+    </clipPath>
+    <clipPath
+       id="clipPath120082">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120084" />
+    </clipPath>
+    <clipPath
+       id="clipPath120086">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120088" />
+    </clipPath>
+    <clipPath
+       id="clipPath120090">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120092" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4712_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83485-7" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83487-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4711_-68">
+      <path
+         d="m -612.756,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83482-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4709_-2">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83462-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath120101">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120103" />
+    </clipPath>
+    <clipPath
+       id="clipPath120105">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120107" />
+    </clipPath>
+    <clipPath
+       id="clipPath120109">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120111" />
+    </clipPath>
+    <clipPath
+       id="clipPath120113">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120115" />
+    </clipPath>
+    <clipPath
+       id="clipPath120117">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120119" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4707_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.3544"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83450-77" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83452-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4706_-2">
+      <path
+         d="m -636.699,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83447-7"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4704_-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83427-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath120128">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120130" />
+    </clipPath>
+    <clipPath
+       id="clipPath120132">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120134" />
+    </clipPath>
+    <clipPath
+       id="clipPath120136">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120138" />
+    </clipPath>
+    <clipPath
+       id="clipPath120140">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120142" />
+    </clipPath>
+    <clipPath
+       id="clipPath120144">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120146" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4702_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.42999"
+       x2="256.3544"
+       y2="-148.42999"
+       gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83415-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83417-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4701_-5">
+      <path
+         d="m -648.671,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83412-8"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4699_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83392-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath120155">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120157" />
+    </clipPath>
+    <clipPath
+       id="clipPath120159">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120161" />
+    </clipPath>
+    <clipPath
+       id="clipPath120163">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120165" />
+    </clipPath>
+    <clipPath
+       id="clipPath120167">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120169" />
+    </clipPath>
+    <clipPath
+       id="clipPath120171">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120173" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4697_-62"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4299"
+       x2="256.35419"
+       y2="-148.4299"
+       gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83380-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83382-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4696_-20">
+      <path
+         d="m -660.642,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83377-7"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4694_-2">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83357-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath120182">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120184" />
+    </clipPath>
+    <clipPath
+       id="clipPath120186">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120188" />
+    </clipPath>
+    <clipPath
+       id="clipPath120190">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120192" />
+    </clipPath>
+    <clipPath
+       id="clipPath120194">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120196" />
+    </clipPath>
+    <clipPath
+       id="clipPath120198">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120200" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4692_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3541"
+       y1="-148.4301"
+       x2="256.3544"
+       y2="-148.4301"
+       gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1042.9659)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83345-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83347-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4691_-7">
+      <path
+         d="m -672.614,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+         id="use83342-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4689_-2">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83322-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath120209">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120211" />
+    </clipPath>
+    <clipPath
+       id="clipPath120213">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120215" />
+    </clipPath>
+    <clipPath
+       id="clipPath120217">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120219" />
+    </clipPath>
+    <clipPath
+       id="clipPath120221">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120223" />
+    </clipPath>
+    <clipPath
+       id="clipPath120225">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120227" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4657_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83100-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83102-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4656_-7">
+      <path
+         d="m -588.813,323.119 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.204 -2.692,2.69"
+         id="use83097-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4654_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83077-8" />
+    </clipPath>
+    <clipPath
+       id="clipPath120236">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120238" />
+    </clipPath>
+    <clipPath
+       id="clipPath120240">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120242" />
+    </clipPath>
+    <clipPath
+       id="clipPath120244">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120246" />
+    </clipPath>
+    <clipPath
+       id="clipPath120248">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120250" />
+    </clipPath>
+    <clipPath
+       id="clipPath120252">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120254" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4652_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83065-42" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83067-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4651_-1">
+      <path
+         d="m -612.756,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use83062-1"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4649_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83042-49" />
+    </clipPath>
+    <clipPath
+       id="clipPath120263">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120265" />
+    </clipPath>
+    <clipPath
+       id="clipPath120267">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120269" />
+    </clipPath>
+    <clipPath
+       id="clipPath120271">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120273" />
+    </clipPath>
+    <clipPath
+       id="clipPath120275">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120277" />
+    </clipPath>
+    <clipPath
+       id="clipPath120279">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120281" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4647_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop83030-6" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop83032-35" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4646_-8">
+      <path
+         d="m -624.728,323.119 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.204 -2.693,2.69"
+         id="use83027-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4644_-8">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use83007-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath120290">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120292" />
+    </clipPath>
+    <clipPath
+       id="clipPath120294">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120296" />
+    </clipPath>
+    <clipPath
+       id="clipPath120298">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120300" />
+    </clipPath>
+    <clipPath
+       id="clipPath120302">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120304" />
+    </clipPath>
+    <clipPath
+       id="clipPath120306">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120308" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4642_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46001"
+       x2="256.37631"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82995-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82997-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4641_-9">
+      <path
+         d="m -636.699,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82992-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4639_-4">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82972-97" />
+    </clipPath>
+    <clipPath
+       id="clipPath120317">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120319" />
+    </clipPath>
+    <clipPath
+       id="clipPath120321">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120323" />
+    </clipPath>
+    <clipPath
+       id="clipPath120325">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120327" />
+    </clipPath>
+    <clipPath
+       id="clipPath120329">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120331" />
+    </clipPath>
+    <clipPath
+       id="clipPath120333">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120335" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4637_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82960-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82962-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4636_-2">
+      <path
+         d="m -648.671,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82957-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4634_-2">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82937-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath120344">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120346" />
+    </clipPath>
+    <clipPath
+       id="clipPath120348">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120350" />
+    </clipPath>
+    <clipPath
+       id="clipPath120352">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120354" />
+    </clipPath>
+    <clipPath
+       id="clipPath120356">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120358" />
+    </clipPath>
+    <clipPath
+       id="clipPath120360">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120362" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4632_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.4601"
+       x2="256.37631"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82925-64" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82927-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4631_-9">
+      <path
+         d="m -660.642,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82922-36"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4629_-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82902-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath120371">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120373" />
+    </clipPath>
+    <clipPath
+       id="clipPath120375">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120377" />
+    </clipPath>
+    <clipPath
+       id="clipPath120379">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120381" />
+    </clipPath>
+    <clipPath
+       id="clipPath120383">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120385" />
+    </clipPath>
+    <clipPath
+       id="clipPath120387">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120389" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4627_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="255.37621"
+       y1="-148.46021"
+       x2="256.37631"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1054.4158)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82890-4" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82892-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4626_-2">
+      <path
+         d="m -672.614,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+         id="use82887-9"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4624_-1">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82867-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath120398">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120400" />
+    </clipPath>
+    <clipPath
+       id="clipPath120402">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120404" />
+    </clipPath>
+    <clipPath
+       id="clipPath120406">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120408" />
+    </clipPath>
+    <clipPath
+       id="clipPath120410">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120412" />
+    </clipPath>
+    <clipPath
+       id="clipPath120414">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120416" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4592_-1"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82645-1" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82647-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4591_-1">
+      <path
+         d="m -588.813,311.531 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.205 -2.692,2.69"
+         id="use82642-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4589_-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82622-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath120425">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120427" />
+    </clipPath>
+    <clipPath
+       id="clipPath120429">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120431" />
+    </clipPath>
+    <clipPath
+       id="clipPath120433">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120435" />
+    </clipPath>
+    <clipPath
+       id="clipPath120437">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120439" />
+    </clipPath>
+    <clipPath
+       id="clipPath120441">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120443" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4587_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,201.1574,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82610-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82612-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4586_-7">
+      <path
+         d="m -600.785,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.203,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82607-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4584_-4">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82587-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath120452">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120454" />
+    </clipPath>
+    <clipPath
+       id="clipPath120456">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120458" />
+    </clipPath>
+    <clipPath
+       id="clipPath120460">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120462" />
+    </clipPath>
+    <clipPath
+       id="clipPath120464">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120466" />
+    </clipPath>
+    <clipPath
+       id="clipPath120468">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120470" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4582_-6"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82575-5" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82577-03" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4581_-7">
+      <path
+         d="m -612.756,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82572-9"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4579_-65">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82552-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath120479">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120481" />
+    </clipPath>
+    <clipPath
+       id="clipPath120483">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120485" />
+    </clipPath>
+    <clipPath
+       id="clipPath120487">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120489" />
+    </clipPath>
+    <clipPath
+       id="clipPath120491">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120493" />
+    </clipPath>
+    <clipPath
+       id="clipPath120495">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120497" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4577_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82540-0" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82542-7" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4576_-5">
+      <path
+         d="m -624.728,311.531 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.205 -2.693,2.69"
+         id="use82537-4"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4574_-6">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82517-7" />
+    </clipPath>
+    <clipPath
+       id="clipPath120506">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120508" />
+    </clipPath>
+    <clipPath
+       id="clipPath120510">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120512" />
+    </clipPath>
+    <clipPath
+       id="clipPath120514">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120516" />
+    </clipPath>
+    <clipPath
+       id="clipPath120518">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120520" />
+    </clipPath>
+    <clipPath
+       id="clipPath120522">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120524" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4572_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46001"
+       x2="256.3761"
+       y2="-148.46001"
+       gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82505-8" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82507-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4571_-3">
+      <path
+         d="m -636.699,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82502-9"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4569_-54">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82482-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath120533">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120535" />
+    </clipPath>
+    <clipPath
+       id="clipPath120537">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120539" />
+    </clipPath>
+    <clipPath
+       id="clipPath120541">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120543" />
+    </clipPath>
+    <clipPath
+       id="clipPath120545">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120547" />
+    </clipPath>
+    <clipPath
+       id="clipPath120549">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120551" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4567_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82470-3" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82472-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4566_-8">
+      <path
+         d="m -648.671,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82467-6"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82447-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath120560">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120562" />
+    </clipPath>
+    <clipPath
+       id="clipPath120564">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120566" />
+    </clipPath>
+    <clipPath
+       id="clipPath120568">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120570" />
+    </clipPath>
+    <clipPath
+       id="clipPath120572">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120574" />
+    </clipPath>
+    <clipPath
+       id="clipPath120576">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120578" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4562_-5"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.4601"
+       x2="256.3761"
+       y2="-148.4601"
+       gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82435-29" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82437-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4561_-11">
+      <path
+         d="m -660.642,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82432-8"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4559_-7">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82412-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath120587">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120589" />
+    </clipPath>
+    <clipPath
+       id="clipPath120591">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120593" />
+    </clipPath>
+    <clipPath
+       id="clipPath120595">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120597" />
+    </clipPath>
+    <clipPath
+       id="clipPath120599">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120601" />
+    </clipPath>
+    <clipPath
+       id="clipPath120603">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120605" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4557_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="255.3761"
+       y1="-148.46021"
+       x2="256.3761"
+       y2="-148.46021"
+       gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82400-9" />
+      <stop
+         offset="1"
+         style="stop-color:#81D1DB"
+         id="stop82402-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4556_-7">
+      <path
+         d="m -672.614,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+         id="use82397-3"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4552_-54">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82359-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath120614">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120616" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4550_-5">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82345-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath120620">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120622" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4548_-8"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1255.1711,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82319-6" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82321-0" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82323-4" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82325-70" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82327-00" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82329-8" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82331-67" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82333-8" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4547_-3">
+      <polygon
+         points="-680.004,267.353 -637.65,267.32 -637.642,253.133 -659.18,258.293 -679.997,253.224 "
+         id="use82316-0" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4545_-93"
+       gradientUnits="userSpaceOnUse"
+       x1="76.3899"
+       y1="97.785698"
+       x2="77.3899"
+       y2="97.785698"
+       gradientTransform="matrix(0,-19.5733,-19.5733,0,1321.2666,1762.5587)">
+      <stop
+         offset="0"
+         style="stop-color:#55AEB9"
+         id="stop82290-1" />
+      <stop
+         offset="0.0507"
+         style="stop-color:#68B4BF"
+         id="stop82292-0" />
+      <stop
+         offset="0.2116"
+         style="stop-color:#96C6CF"
+         id="stop82294-1" />
+      <stop
+         offset="0.3736"
+         style="stop-color:#BAD8DD"
+         id="stop82296-0" />
+      <stop
+         offset="0.5342"
+         style="stop-color:#D7E6EA"
+         id="stop82298-2" />
+      <stop
+         offset="0.693"
+         style="stop-color:#EBF2F3"
+         id="stop82300-3" />
+      <stop
+         offset="0.8492"
+         style="stop-color:#F8FBFB"
+         id="stop82302-1" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82304-2" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4544_-1">
+      <polygon
+         points="-613.908,267.353 -571.554,267.32 -571.547,253.133 -593.085,258.293 -613.901,253.224 "
+         id="use82287-5" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4525_-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82164-7" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4527_-9">
+      <rect
+         height="64.251999"
+         width="175.168"
+         y="293.259"
+         x="-679.67902"
+         id="use82174-42" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4530_-9"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3308.0217,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82185-2" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82187-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4529_-4">
+      <rect
+         height="64.037003"
+         width="174.953"
+         y="293.367"
+         x="-679.57202"
+         id="use82182-9" />
+    </clipPath>
+    <clipPath
+       id="clipPath120655">
+      <rect
+         height="64.251999"
+         width="175.168"
+         y="293.259"
+         x="-679.67902"
+         id="use120657" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4532_-5">
+      <rect
+         height="64.251999"
+         width="175.168"
+         y="293.259"
+         x="-679.67902"
+         id="use82197-4" />
+    </clipPath>
+    <clipPath
+       id="clipPath120661">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120663" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4534_-1">
+      <rect
+         height="64.251999"
+         width="45.873001"
+         y="293.259"
+         x="-441.56699"
+         id="use82209-3" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4537_-2"
+       gradientUnits="userSpaceOnUse"
+       x1="126.8011"
+       y1="60.902401"
+       x2="127.8011"
+       y2="60.902401"
+       gradientTransform="matrix(0,64.0388,-64.0388,0,3481.4861,-7826.8223)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop82220-9" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop82222-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4536_-89">
+      <rect
+         height="64.037003"
+         width="45.659"
+         y="293.367"
+         x="-441.45999"
+         id="use82217-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath120672">
+      <rect
+         height="64.251999"
+         width="45.873001"
+         y="293.259"
+         x="-441.56699"
+         id="use120674" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4539_-1">
+      <rect
+         height="64.251999"
+         width="45.873001"
+         y="293.259"
+         x="-441.56699"
+         id="use82232-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4518_-35">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82083-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath120680">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120682" />
+    </clipPath>
+    <clipPath
+       id="clipPath120684">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120686" />
+    </clipPath>
+    <clipPath
+       id="clipPath120688">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120690" />
+    </clipPath>
+    <clipPath
+       id="clipPath120692">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120694" />
+    </clipPath>
+    <clipPath
+       id="clipPath120696">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120698" />
+    </clipPath>
+    <clipPath
+       id="clipPath120700">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120702" />
+    </clipPath>
+    <clipPath
+       id="clipPath120704">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120706" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4516_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,902.0769,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82067-0" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82069-6" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82071-2" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82073-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4515_-11">
+      <polygon
+         points="-590.704,196.685 -571.658,196.685 -571.658,188.287 -613.986,188.287 -613.986,196.684 -594.94,196.685 -592.822,198.478 "
+         id="use82064-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4511_-38">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82010-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath120717">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120719" />
+    </clipPath>
+    <clipPath
+       id="clipPath120721">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120723" />
+    </clipPath>
+    <clipPath
+       id="clipPath120725">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120727" />
+    </clipPath>
+    <clipPath
+       id="clipPath120729">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120731" />
+    </clipPath>
+    <clipPath
+       id="clipPath120733">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120735" />
+    </clipPath>
+    <clipPath
+       id="clipPath120737">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120739" />
+    </clipPath>
+    <clipPath
+       id="clipPath120741">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use120743" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient104534-4"
+       xlink:href="#SVGID_4509_-0"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4509_-0"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop81994-08" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop81996-2" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop81998-7" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82000-4" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4508_-3">
+      <polygon
+         points="-657.048,196.685 -638.002,196.685 -638.002,188.287 -680.33,188.287 -680.33,196.684 -661.285,196.685 -659.167,198.478 "
+         id="use81991-0" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,835.7324,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient121727"
+       xlink:href="#SVGID_4509_-0"
+       inkscape:collect="always" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200"
+       id="linearGradient126204"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218"
+       gradientUnits="userSpaceOnUse" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200"
+       id="linearGradient126210"
+       gradientUnits="userSpaceOnUse"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-7"
+       id="linearGradient126210-7"
+       gradientUnits="userSpaceOnUse"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <linearGradient
+       id="linearGradient126200-7"
+       osb:paint="solid">
+      <stop
+         style="stop-color:#ffffff;stop-opacity:1;"
+         offset="0"
+         id="stop126202-2" />
+    </linearGradient>
+    <linearGradient
+       y2="442.86218"
+       x2="219"
+       y1="442.86218"
+       x1="111"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126228"
+       xlink:href="#linearGradient126200-7"
+       inkscape:collect="always" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-2"
+       id="linearGradient126210-9"
+       gradientUnits="userSpaceOnUse"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <linearGradient
+       id="linearGradient126200-2"
+       osb:paint="solid">
+      <stop
+         style="stop-color:#ffffff;stop-opacity:1;"
+         offset="0"
+         id="stop126202-7" />
+    </linearGradient>
+    <linearGradient
+       y2="442.86218"
+       x2="219"
+       y1="442.86218"
+       x1="111"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126435"
+       xlink:href="#linearGradient126200-2"
+       inkscape:collect="always" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-2"
+       id="linearGradient126487"
+       gradientUnits="userSpaceOnUse"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218"
+       gradientTransform="translate(190.5,52.1205)" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-2"
+       id="linearGradient126520"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="translate(190.5,52.1205)"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-2-0"
+       id="linearGradient126520-9"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="translate(190.5,52.1205)"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <linearGradient
+       id="linearGradient126200-2-0"
+       osb:paint="solid">
+      <stop
+         style="stop-color:#ffffff;stop-opacity:1;"
+         offset="0"
+         id="stop126202-7-1" />
+    </linearGradient>
+    <linearGradient
+       y2="442.86218"
+       x2="219"
+       y1="442.86218"
+       x1="111"
+       gradientTransform="translate(190.5,52.1205)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126539"
+       xlink:href="#linearGradient126200-2-0"
+       inkscape:collect="always" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-2-2"
+       id="linearGradient126520-4"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="translate(190.5,52.1205)"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <linearGradient
+       id="linearGradient126200-2-2"
+       osb:paint="solid">
+      <stop
+         style="stop-color:#ffffff;stop-opacity:1;"
+         offset="0"
+         id="stop126202-7-2" />
+    </linearGradient>
+    <linearGradient
+       y2="442.86218"
+       x2="219"
+       y1="442.86218"
+       x1="111"
+       gradientTransform="translate(190.5,52.1205)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126539-0"
+       xlink:href="#linearGradient126200-2-2"
+       inkscape:collect="always" />
+    <clipPath
+       id="SVGID_4843_-9">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-9-6">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-9-6-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60-1-9" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-9-6-8">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60-1-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-9-1">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60-9" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-9-1-4">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60-9-6" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4843_-9-6-8-6">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4842_"
+         overflow="visible"
+         id="use84500-60-1-2-2" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4856_-4"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94031"
+       x2="-33.205101"
+       y2="249.94031"
+       gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84659-2" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84661-1" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4855_-2">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4854_"
+         overflow="visible"
+         id="use84656-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4853_-8">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4852_"
+         overflow="visible"
+         id="use84644-1" />
+    </clipPath>
+    <clipPath
+       id="clipPath126768">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4852_"
+         overflow="visible"
+         id="use126770" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4841_-7"
+       gradientUnits="userSpaceOnUse"
+       x1="36.392799"
+       y1="152.813"
+       x2="37.392799"
+       y2="152.813"
+       gradientTransform="matrix(0,-9.614,-9.614,0,942.8785,573.6009)">
+      <stop
+         offset="0"
+         style="stop-color:#5E6A71"
+         id="stop84478-8" />
+      <stop
+         offset="0.1421"
+         style="stop-color:#667077"
+         id="stop84480-2" />
+      <stop
+         offset="0.3823"
+         style="stop-color:#7A8187"
+         id="stop84482-3" />
+      <stop
+         offset="0.6911"
+         style="stop-color:#9EA1A6"
+         id="stop84484-9" />
+      <stop
+         offset="1"
+         style="stop-color:#CECED1"
+         id="stop84486-5" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4840_-7">
+      <rect
+         height="9.6140003"
+         width="11.316"
+         y="214.106"
+         x="-531.92297"
+         id="use84475-9" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4523_-1">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use82132-5" />
+    </clipPath>
+    <clipPath
+       id="clipPath126782">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126784" />
+    </clipPath>
+    <clipPath
+       id="clipPath126786">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126788" />
+    </clipPath>
+    <clipPath
+       id="clipPath126790">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126792" />
+    </clipPath>
+    <clipPath
+       id="clipPath126794">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126796" />
+    </clipPath>
+    <clipPath
+       id="clipPath126798">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126800" />
+    </clipPath>
+    <clipPath
+       id="clipPath126802">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126804" />
+    </clipPath>
+    <clipPath
+       id="clipPath126806">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_"
+         overflow="visible"
+         id="use126808" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4521_-3"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82116-6" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82118-6" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82120-5" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82122-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4520_-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4519_"
+         overflow="visible"
+         id="use82113-2" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126904"
+       xlink:href="#SVGID_4521_-3"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4856_-4-6"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94031"
+       x2="-33.205101"
+       y2="249.94031"
+       gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84659-2-0" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84661-1-9" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4855_-2-2">
+      <polygon
+         points="-507.673,203.458 -521.64,203.458 -523.432,206 -521.64,208.524 -507.673,208.524 "
+         id="use84656-1-4" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4853_-8-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use84644-1-3" />
+    </clipPath>
+    <clipPath
+       id="clipPath127140">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127142" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4523_-1-9">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82132-5-2" />
+    </clipPath>
+    <clipPath
+       id="clipPath127146">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127148" />
+    </clipPath>
+    <clipPath
+       id="clipPath127150">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127152" />
+    </clipPath>
+    <clipPath
+       id="clipPath127154">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127156" />
+    </clipPath>
+    <clipPath
+       id="clipPath127158">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127160" />
+    </clipPath>
+    <clipPath
+       id="clipPath127162">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127164" />
+    </clipPath>
+    <clipPath
+       id="clipPath127166">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127168" />
+    </clipPath>
+    <clipPath
+       id="clipPath127170">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use127172" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126904-6"
+       xlink:href="#SVGID_4521_-3-3"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4521_-3-3"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82116-6-3" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82118-6-5" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82120-5-4" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82122-9-3" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4520_-5-3">
+      <polygon
+         points="-524.02,196.685 -504.975,196.685 -504.975,188.287 -547.302,188.287 -547.302,196.684 -528.257,196.685 -526.139,198.478 "
+         id="use82113-2-8" />
+    </clipPath>
+    <linearGradient
+       id="SVGID_4856_-4-1"
+       gradientUnits="userSpaceOnUse"
+       x1="-34.205299"
+       y1="249.94031"
+       x2="-33.205101"
+       y2="249.94031"
+       gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)">
+      <stop
+         offset="0"
+         style="stop-color:#E6E8E7"
+         id="stop84659-2-9" />
+      <stop
+         offset="1"
+         style="stop-color:#FFFFFF"
+         id="stop84661-1-0" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4855_-2-24">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4854_-1"
+         overflow="visible"
+         id="use84656-1-9" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4853_-8-0">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4852_-4"
+         overflow="visible"
+         id="use84644-1-6" />
+    </clipPath>
+    <clipPath
+       id="clipPath9068">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4852_-4"
+         overflow="visible"
+         id="use9070" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4523_-1-5">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8"
+         overflow="visible"
+         id="use82132-5-25" />
+    </clipPath>
+    <clipPath
+       id="clipPath9074">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8"
+         overflow="visible"
+         id="use9076" />
+    </clipPath>
+    <clipPath
+       id="clipPath9078">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8"
+         overflow="visible"
+         id="use9080" />
+    </clipPath>
+    <clipPath
+       id="clipPath9082">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8"
+         overflow="visible"
+         id="use9084" />
+    </clipPath>
+    <clipPath
+       id="clipPath9086">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8"
+         overflow="visible"
+         id="use9088" />
+    </clipPath>
+    <clipPath
+       id="clipPath9090">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8"
+         overflow="visible"
+         id="use9092" />
+    </clipPath>
+    <clipPath
+       id="clipPath9094">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8"
+         overflow="visible"
+         id="use9096" />
+    </clipPath>
+    <clipPath
+       id="clipPath9098">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4522_-8"
+         overflow="visible"
+         id="use9100" />
+    </clipPath>
+    <linearGradient
+       y2="146.6937"
+       x2="41.840599"
+       y1="146.6937"
+       x1="40.840599"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient126904-61"
+       xlink:href="#SVGID_4521_-3-0"
+       inkscape:collect="always" />
+    <linearGradient
+       id="SVGID_4521_-3-0"
+       gradientUnits="userSpaceOnUse"
+       x1="40.840599"
+       y1="146.6937"
+       x2="41.840599"
+       y2="146.6937"
+       gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)">
+      <stop
+         offset="0"
+         style="stop-color:#37424B"
+         id="stop82116-6-7" />
+      <stop
+         offset="0.12"
+         style="stop-color:#37424B"
+         id="stop82118-6-2" />
+      <stop
+         offset="0.88"
+         style="stop-color:#5E6A71"
+         id="stop82120-5-1" />
+      <stop
+         offset="1"
+         style="stop-color:#5E6A71"
+         id="stop82122-9-34" />
+    </linearGradient>
+    <clipPath
+       id="SVGID_4520_-5-6">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4519_-5"
+         overflow="visible"
+         id="use82113-2-5" />
+    </clipPath>
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-2-0"
+       id="linearGradient9379"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="translate(190.5,52.1205)"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient126200-2-0"
+       id="linearGradient9382"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="translate(28.50002,64.927434)"
+       x1="111"
+       y1="442.86218"
+       x2="219"
+       y2="442.86218" />
+    <clipPath
+       id="SVGID_4550_-6">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4549_-8"
+         overflow="visible"
+         id="use82345-0" />
+    </clipPath>
+    <clipPath
+       id="clipPath9448">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4549_-8"
+         overflow="visible"
+         id="use9450" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4559_-4">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use82412-03" />
+    </clipPath>
+    <clipPath
+       id="clipPath9526">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9528" />
+    </clipPath>
+    <clipPath
+       id="clipPath9530">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9532" />
+    </clipPath>
+    <clipPath
+       id="clipPath9534">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9536" />
+    </clipPath>
+    <clipPath
+       id="clipPath9538">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9540" />
+    </clipPath>
+    <clipPath
+       id="clipPath9542">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="use9544" />
+    </clipPath>
+    <defs
+       id="defs82478">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4568_" />
+    </defs>
+    <clipPath
+       id="clipPath101841">
+      <use
+         id="use101843"
+         overflow="visible"
+         xlink:href="#SVGID_4568_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <defs
+       id="defs82443">
+      <rect
+         height="205.44099"
+         width="200.271"
+         y="181.315"
+         x="-682.31403"
+         id="SVGID_4563_" />
+    </defs>
+    <clipPath
+       id="clipPath101814">
+      <use
+         id="use101816"
+         overflow="visible"
+         xlink:href="#SVGID_4563_"
+         style="overflow:visible"
+         x="0"
+         y="0"
+         width="744.09448"
+         height="1052.3622" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-2">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-4" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-2" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-9">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-1" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-9-3">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-1-6" />
+    </clipPath>
+    <clipPath
+       id="SVGID_4564_-7-9-3-4">
+      <use
+         height="1052.3622"
+         width="744.09448"
+         y="0"
+         x="0"
+         style="overflow:visible"
+         xlink:href="#SVGID_4563_"
+         overflow="visible"
+         id="use82447-2-1-6-0" />
+    </clipPath>
+  </defs>
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="1.0729879"
+     inkscape:cx="202.21003"
+     inkscape:cy="536.61458"
+     inkscape:document-units="px"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     inkscape:window-width="1535"
+     inkscape:window-height="876"
+     inkscape:window-x="65"
+     inkscape:window-y="24"
+     inkscape:window-maximized="1" />
+  <metadata
+     id="metadata104541">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1">
+    <line
+       style="fill:none;stroke:#5e6a71;stroke-width:1.0268178;stroke-miterlimit:10;stroke-dasharray:3.07565499, 3.07565499"
+       stroke-miterlimit="10"
+       x1="277.38089"
+       y1="315.69797"
+       x2="277.38089"
+       y2="533.44318"
+       id="line84244" />
+    <rect
+       style="fill:none;stroke:#5e6a71;stroke-width:0.48500001;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:10;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0"
+       id="rect125410-1-2"
+       width="94.101288"
+       height="87.10128"
+       x="151.94936"
+       y="418.6185" />
+    <rect
+       style="fill:url(#linearGradient9382);fill-opacity:1;stroke:none"
+       id="rect126198-3-8"
+       width="108"
+       height="27"
+       x="139.50002"
+       y="494.28964" />
+    <rect
+       style="fill:#f15d57"
+       x="75.523293"
+       y="574.69031"
+       width="95.778"
+       height="3.0712502"
+       id="rect82236"
+       inkscape:transform-center-x="586.0074"
+       inkscape:transform-center-y="-79.564828" />
+    <rect
+       style="fill:#f15d57"
+       x="224.23932"
+       y="574.69031"
+       width="95.793755"
+       height="3.0712502"
+       id="rect82238"
+       inkscape:transform-center-x="437.28349"
+       inkscape:transform-center-y="-79.564828" />
+    <line
+       style="fill:none;stroke:#5e6a71;stroke-width:0.48150003;stroke-miterlimit:10;stroke-dasharray:1.44225, 1.44225"
+       stroke-miterlimit="10"
+       x1="272.20041"
+       y1="577.34521"
+       x2="272.20041"
+       y2="631.72772"
+       id="line82244"
+       inkscape:transform-center-x="437.21928"
+       inkscape:transform-center-y="-51.254289" />
+    <polyline
+       style="fill:none;stroke:#5e6a71;stroke-width:0.214;stroke-miterlimit:10;stroke-dasharray:0.641, 0.641"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       stroke-miterlimit="10"
+       points="  -603.212,292.128 -603.212,283.832 -582.187,283.832 -582.187,292.128 "
+       id="polyline82246"
+       inkscape:transform-center-x="437.22048"
+       inkscape:transform-center-y="-33.395941" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82281"
+       inkscape:transform-center-x="437.2835"
+       inkscape:transform-center-y="-95.804196">
+      <defs
+         id="defs82283">
+        <polygon
+           id="SVGID_4543_"
+           points="-613.901,253.224 -613.908,267.353 -571.554,267.32 -571.547,253.133 -593.085,258.293 " />
+      </defs>
+      <clipPath
+         id="clipPath101684">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4543_"
+           overflow="visible"
+           id="use101686" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101688"
+         gradientUnits="userSpaceOnUse"
+         x1="76.3899"
+         y1="97.785698"
+         x2="77.3899"
+         y2="97.785698"
+         gradientTransform="matrix(0,-19.5733,-19.5733,0,1321.2666,1762.5587)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101690" />
+        <stop
+           offset="0.0507"
+           style="stop-color:#68B4BF"
+           id="stop101692" />
+        <stop
+           offset="0.2116"
+           style="stop-color:#96C6CF"
+           id="stop101694" />
+        <stop
+           offset="0.3736"
+           style="stop-color:#BAD8DD"
+           id="stop101696" />
+        <stop
+           offset="0.5342"
+           style="stop-color:#D7E6EA"
+           id="stop101698" />
+        <stop
+           offset="0.693"
+           style="stop-color:#EBF2F3"
+           id="stop101700" />
+        <stop
+           offset="0.8492"
+           style="stop-color:#F8FBFB"
+           id="stop101702" />
+        <stop
+           offset="1"
+           style="stop-color:#FFFFFF"
+           id="stop101704" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4545_)"
+         x="-613.90802"
+         y="253.133"
+         clip-path="url(#SVGID_4544_)"
+         width="42.361"
+         height="14.22"
+         id="rect82306" />
+    </g>
+    <path
+       style="fill:#e6e8e8"
+       inkscape:connector-curvature="0"
+       d="m 319.54934,544.29505 -0.0157,31.37625 -94.815,0.072 0.0157,-31.2435 46.48275,11.31975 0.1125,0.027 0.11475,-0.027 48.105,-11.5245 z m 0.4815,-0.60975 -48.699,11.66625 -47.079,-11.46375 -0.0135,32.337 95.77575,-0.0743 0.0158,-32.46525 z"
+       id="path82308"
+       inkscape:transform-center-x="437.28456"
+       inkscape:transform-center-y="-95.835733" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g82310"
+       inkscape:transform-center-x="585.9972"
+       inkscape:transform-center-y="-95.804196">
+      <defs
+         id="defs82312">
+        <polygon
+           id="SVGID_4546_"
+           points="-679.997,253.224 -680.004,267.353 -637.65,267.32 -637.642,253.133 -659.18,258.293 " />
+      </defs>
+      <clipPath
+         id="clipPath101711">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4546_"
+           overflow="visible"
+           id="use101713" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient101715"
+         gradientUnits="userSpaceOnUse"
+         x1="76.3899"
+         y1="97.785698"
+         x2="77.3899"
+         y2="97.785698"
+         gradientTransform="matrix(0,-19.5733,-19.5733,0,1255.1711,1762.5587)">
+        <stop
+           offset="0"
+           style="stop-color:#55AEB9"
+           id="stop101717" />
+        <stop
+           offset="0.0507"
+           style="stop-color:#68B4BF"
+           id="stop101719" />
+        <stop
+           offset="0.2116"
+           style="stop-color:#96C6CF"
+           id="stop101721" />
+        <stop
+           offset="0.3736"
+           style="stop-color:#BAD8DD"
+           id="stop101723" />
+        <stop
+           offset="0.5342"
+           style="stop-color:#D7E6EA"
+           id="stop101725" />
+        <stop
+           offset="0.693"
+           style="stop-color:#EBF2F3"
+           id="stop101727" />
+        <stop
+           offset="0.8492"
+           style="stop-color:#F8FBFB"
+           id="stop101729" />
+        <stop
+           offset="1"
+           style="stop-color:#FFFFFF"
+           id="stop101731" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4548_)"
+         x="-680.00299"
+         y="253.133"
+         clip-path="url(#SVGID_4547_)"
+         width="42.361"
+         height="14.22"
+         id="rect82335" />
+    </g>
+    <path
+       style="fill:#e6e8e8"
+       inkscape:connector-curvature="0"
+       d="m 170.83559,544.29505 -0.0157,31.37625 -94.814996,0.072 0.01575,-31.2435 46.482736,11.31975 0.1125,0.027 0.11475,-0.027 48.10501,-11.5245 z m 0.4815,-0.60975 -48.699,11.66625 -47.078996,-11.46375 -0.0135,32.337 95.775746,-0.0743 0.0157,-32.46525 z"
+       id="path82337"
+       inkscape:transform-center-x="585.99834"
+       inkscape:transform-center-y="-95.835733" />
+    <text
+       y="567.99066"
+       x="104.52653"
+       id="text84236"
+       inkscape:transform-center-x="587.42372"
+       inkscape:transform-center-y="-90.601429"
+       style="font-size:27px">
+      <tspan
+         style="font-size:8px;fill:#37424b;font-family:ApexSans-Medium"
+         x="104.52653"
+         y="567.99066"
+         font-size="3.4203"
+         id="tspan84238">US-WEST</tspan>
+    </text>
+    <text
+       y="567.99066"
+       x="255.24185"
+       id="text84246"
+       inkscape:transform-center-x="438.70839"
+       inkscape:transform-center-y="-90.601429"
+       style="font-size:27px"
+       inkscape:export-xdpi="90"
+       inkscape:export-ydpi="90"
+       inkscape:export-filename="/home/john/Pictures/zone-sync.png">
+      <tspan
+         style="font-size:8px;fill:#37424b;font-family:ApexSans-Medium"
+         x="255.24185"
+         y="567.99066"
+         font-size="3.4203"
+         id="tspan84248">US-EAST</tspan>
+    </text>
+    <rect
+       style="fill:#f15d57"
+       x="220.3154"
+       y="492.99048"
+       width="103.6395"
+       height="3.8114998"
+       id="rect84287"
+       inkscape:transform-center-x="437.28453"
+       inkscape:transform-center-y="-160.89453" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g84289"
+       inkscape:transform-center-x="437.28348"
+       inkscape:transform-center-y="-131.54432">
+      <defs
+         id="defs84291">
+        <polygon
+           id="SVGID_4823_"
+           points="-592.788,256.788 -569.802,251.283 -569.804,231.934 -615.653,231.929 -615.653,251.221 " />
+      </defs>
+      <clipPath
+         id="clipPath103235">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4823_"
+           overflow="visible"
+           id="use103237" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient103239"
+         gradientUnits="userSpaceOnUse"
+         x1="145.3168"
+         y1="2.8518"
+         x2="146.3168"
+         y2="2.8518"
+         gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-668.057,-3384.4426)">
+        <stop
+           offset="0"
+           style="stop-color:#FFFFFF"
+           id="stop103241" />
+        <stop
+           offset="0.23"
+           style="stop-color:#FFFFFF"
+           id="stop103243" />
+        <stop
+           offset="0.5671"
+           style="stop-color:#F9F9F9"
+           id="stop103245" />
+        <stop
+           offset="0.8674"
+           style="stop-color:#EDEEEE"
+           id="stop103247" />
+        <stop
+           offset="1"
+           style="stop-color:#E6E8E7"
+           id="stop103249" />
+      </linearGradient>
+      <polygon
+         style="fill:url(#SVGID_4825_)"
+         clip-path="url(#SVGID_4824_)"
+         points="-615.683,231.929 -569.802,231.874 -569.772,256.788 -615.653,256.843 "
+         id="polygon84308" />
+    </g>
+    <path
+       style="fill:#e6e8e8"
+       inkscape:connector-curvature="0"
+       d="m 220.79459,496.5208 102.67875,0.009 10e-4,43.1055 -51.4755,12.33 -51.20775,-12.46725 0.003,-42.97725 z m -0.4815,-0.4815 v 43.83675 l 51.687,12.58425 51.957,-12.44475 -0.005,-43.96725 -103.6395,-0.009 z"
+       id="path84310"
+       inkscape:transform-center-x="437.28484"
+       inkscape:transform-center-y="-131.54096" />
+    <text
+       y="513.2843"
+       x="243.95297"
+       id="text84312"
+       inkscape:transform-center-x="437.86922"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px">
+      <tspan
+         style="font-size:11px;fill:#37424b;font-family:ApexSans-Book"
+         x="243.95297"
+         y="513.2843"
+         font-size="5.1244"
+         id="tspan84314">RADOSGW</tspan>
+    </text>
+    <rect
+       style="fill:#f15d57"
+       x="70.568871"
+       y="492.99048"
+       width="103.6395"
+       height="3.8114998"
+       id="rect84318"
+       inkscape:transform-center-x="587.03108"
+       inkscape:transform-center-y="-160.89453" />
+    <g
+       id="g9328">
+      <line
+         id="line84244-3"
+         y2="533.36865"
+         x2="121.81088"
+         y1="315.62341"
+         x1="121.81088"
+         stroke-miterlimit="10"
+         style="fill:none;stroke:#5e6a71;stroke-width:1.0268178;stroke-miterlimit:10;stroke-dasharray:3.07565499, 3.07565499" />
+      <g
+         id="g82107-4"
+         transform="matrix(3.375,0,0,3.375,1898.5131,-334.69083)">
+        <defs
+           id="defs82109-6">
+          <polygon
+             points="-547.302,188.287 -547.302,196.684 -528.257,196.685 -526.139,198.478 -524.02,196.685 -504.975,196.685 -504.975,188.287 "
+             id="SVGID_4519_-5" />
+        </defs>
+        <clipPath
+           id="clipPath126820-6">
+          <use
+             id="use126822-8"
+             overflow="visible"
+             xlink:href="#SVGID_4519_-5"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)"
+           y2="146.6937"
+           x2="41.840599"
+           y1="146.6937"
+           x1="40.840599"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient126824-4">
+          <stop
+             id="stop126826-2"
+             style="stop-color:#37424B"
+             offset="0" />
+          <stop
+             id="stop126828-3"
+             style="stop-color:#37424B"
+             offset="0.12" />
+          <stop
+             id="stop126830-7"
+             style="stop-color:#5E6A71"
+             offset="0.88" />
+          <stop
+             id="stop126832-6"
+             style="stop-color:#5E6A71"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect82124-4"
+           height="10.191"
+           width="42.327"
+           clip-path="url(#SVGID_4520_-5-6)"
+           y="188.287"
+           x="-547.302"
+           style="fill:url(#linearGradient126904-61)" />
+      </g>
+      <g
+         id="g82126-0"
+         transform="matrix(3.375,0,0,3.375,1898.5131,-334.69083)">
+        <defs
+           id="defs82128-8">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4522_-8" />
+        </defs>
+        <clipPath
+           id="clipPath126838-9">
+          <use
+             id="use126840-87"
+             overflow="visible"
+             xlink:href="#SVGID_4522_-8"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path82134-5"
+           d="m -539.397,191.908 c -0.353,0 -0.64,-0.286 -0.64,-0.639 0,-0.354 0.287,-0.642 0.64,-0.641 0.354,0 0.639,0.287 0.639,0.641 0,0.353 -0.285,0.639 -0.639,0.639"
+           clip-path="url(#SVGID_4523_-1-5)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <path
+           id="path82136-0"
+           d="m -539.397,194.263 c -0.353,0 -0.64,-0.286 -0.64,-0.639 0,-0.355 0.287,-0.642 0.64,-0.642 0.354,0 0.639,0.287 0.639,0.642 0,0.353 -0.285,0.639 -0.639,0.639"
+           clip-path="url(#SVGID_4523_-1-5)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <circle
+           id="circle82138-2"
+           r="0.63999999"
+           cy="191.26801"
+           cx="-541.625"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4523_-1-5)"
+           d="m -540.985,191.26801 c 0,0.35346 -0.28654,0.64 -0.64,0.64 -0.35346,0 -0.64,-0.28654 -0.64,-0.64 0,-0.35347 0.28654,-0.64 0.64,-0.64 0.35346,0 0.64,0.28653 0.64,0.64 z"
+           sodipodi:cx="-541.625"
+           sodipodi:cy="191.26801"
+           sodipodi:rx="0.63999999"
+           sodipodi:ry="0.63999999"
+           style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10" />
+        <circle
+           id="circle82140-3"
+           r="0.63999999"
+           cy="193.623"
+           cx="-541.625"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4523_-1-5)"
+           d="m -540.985,193.623 c 0,0.35346 -0.28654,0.64 -0.64,0.64 -0.35346,0 -0.64,-0.28654 -0.64,-0.64 0,-0.35346 0.28654,-0.64 0.64,-0.64 0.35346,0 0.64,0.28654 0.64,0.64 z"
+           sodipodi:cx="-541.625"
+           sodipodi:cy="193.623"
+           sodipodi:rx="0.63999999"
+           sodipodi:ry="0.63999999"
+           style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10" />
+        <circle
+           id="circle82142-3"
+           r="0.63999999"
+           cy="191.26801"
+           cx="-543.85303"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4523_-1-5)"
+           d="m -543.21303,191.26801 c 0,0.35346 -0.28654,0.64 -0.64,0.64 -0.35346,0 -0.64,-0.28654 -0.64,-0.64 0,-0.35347 0.28654,-0.64 0.64,-0.64 0.35346,0 0.64,0.28653 0.64,0.64 z"
+           sodipodi:cx="-543.85303"
+           sodipodi:cy="191.26801"
+           sodipodi:rx="0.63999999"
+           sodipodi:ry="0.63999999"
+           style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10" />
+        <circle
+           id="circle82144-4"
+           r="0.63999999"
+           cy="193.623"
+           cx="-543.854"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4523_-1-5)"
+           d="m -543.214,193.623 c 0,0.35346 -0.28654,0.64 -0.64,0.64 -0.35347,0 -0.64,-0.28654 -0.64,-0.64 0,-0.35346 0.28653,-0.64 0.64,-0.64 0.35346,0 0.64,0.28654 0.64,0.64 z"
+           sodipodi:cx="-543.854"
+           sodipodi:cy="193.623"
+           sodipodi:rx="0.63999999"
+           sodipodi:ry="0.63999999"
+           style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10" />
+        <path
+           id="path82146-1"
+           d="m -537.169,191.908 c -0.354,0 -0.64,-0.286 -0.64,-0.639 0,-0.354 0.286,-0.642 0.64,-0.641 0.354,0 0.64,0.287 0.64,0.641 0,0.353 -0.286,0.639 -0.64,0.639"
+           clip-path="url(#SVGID_4523_-1-5)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <path
+           id="path82148-5"
+           d="m -537.169,194.263 c -0.354,0 -0.64,-0.286 -0.64,-0.639 0,-0.355 0.286,-0.642 0.64,-0.642 0.354,0 0.64,0.287 0.64,0.642 0,0.353 -0.286,0.639 -0.64,0.639"
+           clip-path="url(#SVGID_4523_-1-5)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+      </g>
+      <text
+         style="font-size:40.5px"
+         id="text82150-7"
+         x="112.11884"
+         y="319.30515">
+        <tspan
+           id="tspan82152-4"
+           font-size="3.2027"
+           y="319.30515"
+           x="112.11884"
+           style="font-size:10.80911255px;fill:#ffffff;font-family:ApexSans-Medium">AP</tspan>
+        <tspan
+           id="tspan82154-9"
+           font-size="3.2027"
+           y="319.30515"
+           x="125.80093"
+           style="font-size:10.80911255px;fill:#ffffff;font-family:ApexSans-Medium">P</tspan>
+      </text>
+      <g
+         id="g84638-4"
+         transform="matrix(3.375,0,0,3.375,1898.5131,-334.69083)">
+        <defs
+           id="defs84640-9">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4852_-4" />
+        </defs>
+        <clipPath
+           id="clipPath126879-0">
+          <use
+             id="use126881-7"
+             overflow="visible"
+             xlink:href="#SVGID_4852_-4"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path84646-0"
+           d="m -526.673,205.991 c 0,-0.349 0.282,-0.632 0.632,-0.632 0.349,0 0.632,0.283 0.632,0.632 0,0.35 -0.283,0.633 -0.632,0.633 -0.35,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4853_-8-0)"
+           inkscape:connector-curvature="0"
+           style="fill:#5e6a71" />
+        <line
+           id="line84648-6"
+           y2="205.991"
+           x2="-523.14398"
+           y1="205.991"
+           x1="-526.03998"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4853_-8-0)"
+           style="fill:none;stroke:#5e6a71;stroke-width:0.214;stroke-miterlimit:10" />
+      </g>
+      <g
+         id="g84650-5"
+         transform="matrix(3.375,0,0,3.375,1898.5131,-334.69083)">
+        <defs
+           id="defs84652-4">
+          <polygon
+             points="-521.64,203.458 -523.432,206 -521.64,208.524 -507.673,208.524 -507.673,203.458 "
+             id="SVGID_4854_-1" />
+        </defs>
+        <clipPath
+           id="clipPath126888-1">
+          <use
+             id="use126890-4"
+             overflow="visible"
+             xlink:href="#SVGID_4854_-1"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)"
+           y2="249.94031"
+           x2="-33.205101"
+           y1="249.94031"
+           x1="-34.205299"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient126892-7">
+          <stop
+             id="stop126894-5"
+             style="stop-color:#E6E8E7"
+             offset="0" />
+          <stop
+             id="stop126896-6"
+             style="stop-color:#FFFFFF"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect84663-7"
+           height="5.066"
+           width="15.759"
+           clip-path="url(#SVGID_4855_-2-24)"
+           y="203.45799"
+           x="-523.43201"
+           style="fill:url(#SVGID_4856_-4-1)" />
+      </g>
+      <polygon
+         id="polygon84665-6"
+         points="-521.64,203.459 -523.433,206 -521.64,208.524 -507.674,208.524 -507.674,203.459 "
+         stroke-miterlimit="10"
+         transform="matrix(3.375,0,0,3.375,1898.5131,-334.69083)"
+         style="fill:none;stroke:#e6e8e8;stroke-width:0.214;stroke-miterlimit:10" />
+      <text
+         style="font-size:40.5px"
+         id="text84667-5"
+         x="151.0986"
+         y="363.4711">
+        <tspan
+           id="tspan84669-5"
+           font-size="2.5652"
+           y="363.4711"
+           x="151.0986"
+           style="font-size:8.65755081px;fill:#5e6a71;font-family:ApexSans-Book">RE</tspan>
+        <tspan
+           id="tspan84671-5"
+           font-size="2.5652"
+           y="363.4711"
+           x="160.27521"
+           style="font-size:8.65755081px;fill:#5e6a71;font-family:ApexSans-Book">S</tspan>
+        <tspan
+           id="tspan84673-2"
+           font-size="2.5652"
+           y="363.4711"
+           x="164.41296"
+           style="font-size:8.65755081px;fill:#5e6a71;font-family:ApexSans-Book">T</tspan>
+      </text>
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g84320"
+       inkscape:transform-center-x="587.03111"
+       inkscape:transform-center-y="-131.54432">
+      <defs
+         id="defs84322">
+        <polygon
+           id="SVGID_4826_"
+           points="-659.342,256.788 -636.356,251.283 -636.358,231.934 -682.207,231.929 -682.208,251.221 " />
+      </defs>
+      <clipPath
+         id="clipPath103260">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4826_"
+           overflow="visible"
+           id="use103262" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient103264"
+         gradientUnits="userSpaceOnUse"
+         x1="145.3168"
+         y1="2.8518"
+         x2="146.3168"
+         y2="2.8518"
+         gradientTransform="matrix(0.0299,24.8865,24.8865,-0.0299,-734.6116,-3384.4426)">
+        <stop
+           offset="0"
+           style="stop-color:#FFFFFF"
+           id="stop103266" />
+        <stop
+           offset="0.23"
+           style="stop-color:#FFFFFF"
+           id="stop103268" />
+        <stop
+           offset="0.5671"
+           style="stop-color:#F9F9F9"
+           id="stop103270" />
+        <stop
+           offset="0.8674"
+           style="stop-color:#EDEEEE"
+           id="stop103272" />
+        <stop
+           offset="1"
+           style="stop-color:#E6E8E7"
+           id="stop103274" />
+      </linearGradient>
+      <polygon
+         style="fill:url(#SVGID_4828_)"
+         clip-path="url(#SVGID_4827_)"
+         points="-682.238,231.929 -636.356,231.874 -636.326,256.788 -682.208,256.843 "
+         id="polygon84339" />
+    </g>
+    <path
+       style="fill:#e6e8e8"
+       inkscape:connector-curvature="0"
+       d="m 71.048094,496.5208 102.678746,0.009 10e-4,43.1055 -51.4755,12.33 -51.207746,-12.46725 0.0023,-42.97725 z m -0.4815,-0.4815 v 43.83675 l 51.686996,12.58425 51.957,-12.44475 -0.005,-43.96725 -103.639496,-0.009 z"
+       id="path84341"
+       inkscape:transform-center-x="587.03109"
+       inkscape:transform-center-y="-131.54096" />
+    <text
+       y="513.2843"
+       x="94.206009"
+       id="text84343"
+       inkscape:transform-center-x="587.62742"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px">
+      <tspan
+         style="font-size:11px;fill:#37424b;font-family:ApexSans-Book"
+         x="94.206009"
+         y="513.2843"
+         font-size="5.1244"
+         id="tspan84345">RADOSGW</tspan>
+    </text>
+    <line
+       style="fill:none;stroke:#5e6a71;stroke-width:0.48150003;stroke-miterlimit:10;stroke-dasharray:1.44225, 1.44225"
+       stroke-miterlimit="10"
+       x1="122.29416"
+       y1="577.34521"
+       x2="122.29416"
+       y2="631.72772"
+       id="line84349"
+       inkscape:transform-center-x="587.12555"
+       inkscape:transform-center-y="-51.254289" />
+    <polyline
+       style="fill:none;stroke:#5e6a71;stroke-width:0.214;stroke-miterlimit:10;stroke-dasharray:0.641, 0.641"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       stroke-miterlimit="10"
+       points="  -669.837,292.128 -669.837,283.832 -648.812,283.832 -648.812,292.128 "
+       id="polyline84351"
+       inkscape:transform-center-x="587.12673"
+       inkscape:transform-center-y="-33.395941" />
+    <polygon
+       style="fill:#81d0db"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       points="-659.319,269.217 -657.251,267.424 -661.37,267.424 "
+       id="polygon84353"
+       inkscape:transform-center-x="587.09523"
+       inkscape:transform-center-y="-77.629816" />
+    <polygon
+       style="fill:#81d0db"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       points="-592.696,269.217 -590.628,267.424 -594.747,267.424 "
+       id="polygon84355"
+       inkscape:transform-center-x="437.19348"
+       inkscape:transform-center-y="-77.629816" />
+    <path
+       id="path84524"
+       d="m -365.856,302.434 c 0,-2.331 1.892,-4.222 4.225,-4.222 2.334,0 4.223,1.891 4.223,4.222 0,2.334 -1.889,4.226 -4.223,4.226 -2.333,0 -4.225,-1.892 -4.225,-4.226"
+       clip-path="url(#SVGID_4843_)"
+       inkscape:connector-curvature="0"
+       style="fill:#f15d57"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <path
+       id="path84526"
+       d="m -361.631,301.547 c 0.488,0 0.888,0.398 0.888,0.887 0,0.493 -0.4,0.892 -0.888,0.89 -0.491,0 -0.888,-0.397 -0.888,-0.89 0,-0.489 0.397,-0.887 0.888,-0.887"
+       clip-path="url(#SVGID_4843_)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <path
+       id="path84528"
+       d="m -347.575,299.54 c 0.487,0 0.886,0.395 0.886,0.886 0,0.494 -0.399,0.891 -0.886,0.891 -0.493,0 -0.888,-0.397 -0.888,-0.891 0,-0.491 0.395,-0.886 0.888,-0.886"
+       clip-path="url(#SVGID_4843_)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <path
+       id="path84530"
+       d="m -350.839,302.809 c 0.49,0 0.886,0.397 0.886,0.886 0,0.494 -0.396,0.893 -0.886,0.893 -0.491,0 -0.889,-0.399 -0.889,-0.893 0,-0.489 0.398,-0.886 0.889,-0.886"
+       clip-path="url(#SVGID_4843_)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <path
+       id="path84532"
+       d="m -350.838,299.54 c 0.489,0 0.885,0.395 0.885,0.886 0,0.494 -0.396,0.891 -0.885,0.891 -0.492,0 -0.89,-0.397 -0.89,-0.891 0,-0.491 0.398,-0.886 0.89,-0.886"
+       clip-path="url(#SVGID_4843_)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <path
+       id="path84534"
+       d="m -347.576,302.809 c 0.488,0 0.888,0.397 0.888,0.886 0,0.494 -0.4,0.893 -0.888,0.893 -0.491,0 -0.888,-0.399 -0.888,-0.893 0,-0.489 0.397,-0.886 0.888,-0.886"
+       clip-path="url(#SVGID_4843_)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <path
+       id="path84536"
+       d="m -344.311,302.809 c 0.488,0 0.886,0.397 0.886,0.886 0,0.494 -0.398,0.893 -0.886,0.893 -0.491,0 -0.888,-0.399 -0.888,-0.893 0,-0.489 0.397,-0.886 0.888,-0.886"
+       clip-path="url(#SVGID_4843_)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <path
+       id="path84538"
+       d="m -344.31,299.54 c 0.487,0 0.885,0.395 0.885,0.886 0,0.494 -0.398,0.891 -0.885,0.891 -0.492,0 -0.888,-0.397 -0.888,-0.891 0,-0.491 0.396,-0.886 0.888,-0.886"
+       clip-path="url(#SVGID_4843_)"
+       inkscape:connector-curvature="0"
+       style="fill:#ffffff"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <line
+       id="line84540"
+       y2="308.98499"
+       x2="-343.42499"
+       y1="308.98499"
+       x1="-365.664"
+       stroke-miterlimit="10"
+       clip-path="url(#SVGID_4843_)"
+       style="fill:none;stroke:#ffffff;stroke-width:0.428;stroke-miterlimit:10"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <line
+       id="line84542"
+       y2="311.16699"
+       x2="-343.42499"
+       y1="311.16699"
+       x1="-365.664"
+       stroke-miterlimit="10"
+       clip-path="url(#SVGID_4843_)"
+       style="fill:none;stroke:#ffffff;stroke-width:0.428;stroke-miterlimit:10"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <line
+       id="line84544"
+       y2="313.35001"
+       x2="-343.42499"
+       y1="313.35001"
+       x1="-365.664"
+       stroke-miterlimit="10"
+       clip-path="url(#SVGID_4843_)"
+       style="fill:none;stroke:#ffffff;stroke-width:0.428;stroke-miterlimit:10"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <polygon
+       id="polygon84564"
+       points="-436.6,208.359 -434.532,206.566 -438.651,206.566 "
+       clip-path="url(#SVGID_4843_)"
+       style="fill:#f15d57"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <polygon
+       id="polygon84566"
+       points="-426.089,208.359 -424.021,206.566 -428.14,206.566 "
+       clip-path="url(#SVGID_4843_)"
+       style="fill:#f15d57"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <polygon
+       id="polygon84568"
+       points="-415.575,208.359 -413.507,206.566 -417.626,206.566 "
+       clip-path="url(#SVGID_4843_)"
+       style="fill:#f15d57"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <polygon
+       id="polygon84570"
+       points="-296.627,290.885 -294.559,289.091 -298.678,289.091 "
+       clip-path="url(#SVGID_4843_)"
+       style="fill:#f15d57"
+       transform="matrix(1.5,0,0,1.5,1063.2832,192.41394)" />
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g84714"
+       inkscape:transform-center-x="434.5295"
+       inkscape:transform-center-y="-60.127066">
+      <defs
+         id="defs84716">
+        <rect
+           id="SVGID_4862_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath103539">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4862_"
+           overflow="visible"
+           id="use103541" />
+      </clipPath>
+      <path
+         style="fill:#5e6a71"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4863_)"
+         d="m -593.321,276.099 c 0,-0.349 0.282,-0.632 0.632,-0.632 0.349,0 0.632,0.283 0.632,0.632 0,0.35 -0.283,0.633 -0.632,0.633 -0.35,0 -0.632,-0.283 -0.632,-0.633"
+         id="path84722" />
+      <line
+         style="fill:none;stroke:#5e6a71;stroke-width:0.214;stroke-miterlimit:10"
+         clip-path="url(#SVGID_4863_)"
+         stroke-miterlimit="10"
+         x1="-592.68903"
+         y1="276.099"
+         x2="-589.79303"
+         y2="276.099"
+         id="line84724" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g84726"
+       inkscape:transform-center-x="413.59995"
+       inkscape:transform-center-y="-60.127083">
+      <defs
+         id="defs84728">
+        <polygon
+           id="SVGID_4864_"
+           points="-590.081,276.108 -588.288,278.632 -574.322,278.632 -574.322,273.567 -588.288,273.567 " />
+      </defs>
+      <clipPath
+         id="clipPath103548">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4864_"
+           overflow="visible"
+           id="use103550" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient103552"
+         gradientUnits="userSpaceOnUse"
+         x1="-34.205299"
+         y1="249.9404"
+         x2="-33.205299"
+         y2="249.9404"
+         gradientTransform="matrix(0,-5.065,-5.065,0,683.7546,105.3813)">
+        <stop
+           offset="0"
+           style="stop-color:#E6E8E7"
+           id="stop103554" />
+        <stop
+           offset="1"
+           style="stop-color:#FFFFFF"
+           id="stop103556" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4866_)"
+         x="-590.08099"
+         y="273.56699"
+         clip-path="url(#SVGID_4865_)"
+         width="15.759"
+         height="5.0650001"
+         id="rect84739" />
+    </g>
+    <polygon
+       style="fill:none;stroke:#e6e8e8;stroke-width:0.214;stroke-miterlimit:10"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       stroke-miterlimit="10"
+       points="-590.082,276.108 -588.289,278.632 -574.323,278.632 -574.323,273.567 -588.289,273.567 "
+       id="polygon84741"
+       inkscape:transform-center-x="413.60223"
+       inkscape:transform-center-y="-60.127066" />
+    <text
+       y="597.62451"
+       x="287.82977"
+       id="text84743"
+       inkscape:transform-center-x="412.33742"
+       inkscape:transform-center-y="-60.270044"
+       style="font-size:27px">
+      <tspan
+         style="font-size:5.77170038px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="287.82977"
+         y="597.62451"
+         font-size="2.5652"
+         id="tspan84745">N</tspan>
+      <tspan
+         style="font-size:5.77170038px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="291.70874"
+         y="597.62451"
+         font-size="2.5652"
+         id="tspan84747">A</tspan>
+      <tspan
+         style="font-size:5.77170038px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="294.77774"
+         y="597.62451"
+         font-size="2.5652"
+         id="tspan84749">TIV</tspan>
+      <tspan
+         style="font-size:5.77170038px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="302.49078"
+         y="597.62451"
+         font-size="2.5652"
+         id="tspan84751">E</tspan>
+    </text>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g84753"
+       inkscape:transform-center-x="584.45598"
+       inkscape:transform-center-y="-60.127066">
+      <defs
+         id="defs84755">
+        <rect
+           id="SVGID_4867_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath103568">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4867_"
+           overflow="visible"
+           id="use103570" />
+      </clipPath>
+      <path
+         style="fill:#5e6a71"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4868_)"
+         d="m -659.955,276.099 c 0,-0.349 0.282,-0.632 0.632,-0.632 0.349,0 0.632,0.283 0.632,0.632 0,0.35 -0.283,0.633 -0.632,0.633 -0.35,0 -0.632,-0.283 -0.632,-0.633"
+         id="path84761" />
+      <line
+         style="fill:none;stroke:#5e6a71;stroke-width:0.214;stroke-miterlimit:10"
+         clip-path="url(#SVGID_4868_)"
+         stroke-miterlimit="10"
+         x1="-659.323"
+         y1="276.099"
+         x2="-656.427"
+         y2="276.099"
+         id="line84763" />
+    </g>
+    <g
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       id="g84765"
+       inkscape:transform-center-x="563.5265"
+       inkscape:transform-center-y="-60.127083">
+      <defs
+         id="defs84767">
+        <polygon
+           id="SVGID_4869_"
+           points="-656.715,276.108 -654.922,278.632 -640.956,278.632 -640.956,273.567 -654.922,273.567 " />
+      </defs>
+      <clipPath
+         id="clipPath103577">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4869_"
+           overflow="visible"
+           id="use103579" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient103581"
+         gradientUnits="userSpaceOnUse"
+         x1="-34.205299"
+         y1="249.94051"
+         x2="-33.205299"
+         y2="249.94051"
+         gradientTransform="matrix(0,-5.065,-5.065,0,617.1209,105.3813)">
+        <stop
+           offset="0"
+           style="stop-color:#E6E8E7"
+           id="stop103583" />
+        <stop
+           offset="1"
+           style="stop-color:#FFFFFF"
+           id="stop103585" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4871_)"
+         x="-656.71503"
+         y="273.56699"
+         clip-path="url(#SVGID_4870_)"
+         width="15.759"
+         height="5.0650001"
+         id="rect84778" />
+    </g>
+    <polygon
+       style="fill:none;stroke:#e6e8e8;stroke-width:0.214;stroke-miterlimit:10"
+       transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+       stroke-miterlimit="10"
+       points="-656.715,276.108 -654.922,278.632 -640.956,278.632 -640.956,273.567 -654.922,273.567 "
+       id="polygon84780"
+       inkscape:transform-center-x="563.52648"
+       inkscape:transform-center-y="-60.127066" />
+    <text
+       y="597.62451"
+       x="137.9046"
+       id="text84782"
+       inkscape:transform-center-x="562.26258"
+       inkscape:transform-center-y="-60.270044"
+       style="font-size:27px">
+      <tspan
+         style="font-size:5.77170038px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="137.9046"
+         y="597.62451"
+         font-size="2.5652"
+         id="tspan84784">N</tspan>
+      <tspan
+         style="font-size:5.77170038px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="141.7836"
+         y="597.62451"
+         font-size="2.5652"
+         id="tspan84786">A</tspan>
+      <tspan
+         style="font-size:5.77170038px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="144.85263"
+         y="597.62451"
+         font-size="2.5652"
+         id="tspan84788">TIV</tspan>
+      <tspan
+         style="font-size:5.77170038px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="152.56561"
+         y="597.62451"
+         font-size="2.5652"
+         id="tspan84790">E</tspan>
+    </text>
+    <text
+       xml:space="preserve"
+       style="font-size:8px;font-style:normal;font-weight:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans"
+       x="285.86542"
+       y="577.40869"
+       id="text113406"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan113408"
+         x="285.86542"
+         y="577.40869" /></text>
+    <text
+       y="832.10559"
+       x="242.41013"
+       id="text84343-2"
+       inkscape:transform-center-x="587.62742"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px;text-align:center;text-anchor:middle">
+      <tspan
+         style="font-size:11px;text-align:center;text-anchor:middle;fill:#37424b;font-family:ApexSans-Book"
+         x="197.43796"
+         y="832.10559"
+         font-size="5.1244"
+         id="tspan84345-2">MASTER REGION</tspan>
+    </text>
+    <text
+       y="926.36633"
+       x="288.45654"
+       id="text84343-2-6"
+       inkscape:transform-center-x="587.62742"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px;text-align:center;text-anchor:middle">
+      <tspan
+         style="font-size:11px;text-align:center;text-anchor:middle;fill:#37424b;font-family:ApexSans-Book"
+         x="249.24219"
+         y="926.36633"
+         font-size="5.1244"
+         id="tspan84345-2-5" />
+    </text>
+    <flowRoot
+       xml:space="preserve"
+       id="flowRoot126352"
+       style="fill:black;stroke:none;stroke-opacity:1;stroke-width:1px;stroke-linejoin:miter;stroke-linecap:butt;fill-opacity:1;font-family:Sans;font-style:normal;font-weight:normal;font-size:9px;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;text-align:center"><flowRegion
+         id="flowRegion126354"><rect
+           id="rect126356"
+           width="0"
+           height="51"
+           x="247"
+           y="928.36218" /></flowRegion><flowPara
+         id="flowPara126358" /></flowRoot>    <flowRoot
+       xml:space="preserve"
+       id="flowRoot126403"
+       style="fill:black;stroke:none;stroke-opacity:1;stroke-width:1px;stroke-linejoin:miter;stroke-linecap:butt;fill-opacity:1;font-family:Sans;font-style:normal;font-weight:normal;font-size:9px;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;text-align:center"><flowRegion
+         id="flowRegion126405"><rect
+           id="rect126407"
+           width="185"
+           height="81"
+           x="286"
+           y="941.36218" /></flowRegion><flowPara
+         id="flowPara126409" /></flowRoot>    <flowRoot
+       xml:space="preserve"
+       id="flowRoot126411"
+       style="fill:black;stroke:none;stroke-opacity:1;stroke-width:1px;stroke-linejoin:miter;stroke-linecap:butt;fill-opacity:1;font-family:Sans;font-style:normal;font-weight:normal;font-size:9px;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;text-align:center"><flowRegion
+         id="flowRegion126413"><rect
+           id="rect126415"
+           width="89"
+           height="64"
+           x="285"
+           y="943.36218" /></flowRegion><flowPara
+         id="flowPara126417" /></flowRoot>    <text
+       xml:space="preserve"
+       style="font-size:9px;font-style:normal;font-weight:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans"
+       x="272"
+       y="529.64587"
+       id="text126456"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan126458"
+         x="272"
+         y="529.64587">MASTER</tspan><tspan
+         sodipodi:role="line"
+         x="272"
+         y="540.89587"
+         id="tspan126460">ZONE</tspan></text>
+    <polygon
+       id="polygon84546-9-3"
+       points="-667.777,292.47 -671.896,292.47 -669.845,294.263 "
+       clip-path="url(#SVGID_4843_-9-6)"
+       style="fill:#f15d57"
+       transform="matrix(2.25,0,0,2.25,1658.932,-169.71257)" />
+    <text
+       xml:space="preserve"
+       style="font-size:9px;font-style:normal;font-weight:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans"
+       x="122.23143"
+       y="529.64587"
+       id="text126655-6"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan126657-2"
+         x="122.23143"
+         y="529.64587">SECONDARY</tspan><tspan
+         sodipodi:role="line"
+         x="122.23143"
+         y="540.89587"
+         id="tspan126659-4">ZONE</tspan></text>
+    <g
+       transform="matrix(3.375,0,0,3.375,2054.0831,-334.61629)"
+       id="g82107">
+      <defs
+         id="defs82109">
+        <polygon
+           id="SVGID_4519_"
+           points="-504.975,196.685 -504.975,188.287 -547.302,188.287 -547.302,196.684 -528.257,196.685 -526.139,198.478 -524.02,196.685 " />
+      </defs>
+      <clipPath
+         id="clipPath126820">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4519_"
+           overflow="visible"
+           id="use126822" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient126824"
+         gradientUnits="userSpaceOnUse"
+         x1="40.840599"
+         y1="146.6937"
+         x2="41.840599"
+         y2="146.6937"
+         gradientTransform="matrix(0,-10.1906,-10.1906,0,968.7597,614.6687)">
+        <stop
+           offset="0"
+           style="stop-color:#37424B"
+           id="stop126826" />
+        <stop
+           offset="0.12"
+           style="stop-color:#37424B"
+           id="stop126828" />
+        <stop
+           offset="0.88"
+           style="stop-color:#5E6A71"
+           id="stop126830" />
+        <stop
+           offset="1"
+           style="stop-color:#5E6A71"
+           id="stop126832" />
+      </linearGradient>
+      <rect
+         style="fill:url(#linearGradient126904)"
+         x="-547.302"
+         y="188.287"
+         clip-path="url(#SVGID_4520_-5)"
+         width="42.327"
+         height="10.191"
+         id="rect82124" />
+    </g>
+    <g
+       transform="matrix(3.375,0,0,3.375,2054.0831,-334.61629)"
+       id="g82126">
+      <defs
+         id="defs82128">
+        <rect
+           id="SVGID_4522_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath126838">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4522_"
+           overflow="visible"
+           id="use126840" />
+      </clipPath>
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4523_-1)"
+         d="m -539.397,191.908 c -0.353,0 -0.64,-0.286 -0.64,-0.639 0,-0.354 0.287,-0.642 0.64,-0.641 0.354,0 0.639,0.287 0.639,0.641 0,0.353 -0.285,0.639 -0.639,0.639"
+         id="path82134" />
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4523_-1)"
+         d="m -539.397,194.263 c -0.353,0 -0.64,-0.286 -0.64,-0.639 0,-0.355 0.287,-0.642 0.64,-0.642 0.354,0 0.639,0.287 0.639,0.642 0,0.353 -0.285,0.639 -0.639,0.639"
+         id="path82136" />
+      <circle
+         style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10"
+         sodipodi:ry="0.63999999"
+         sodipodi:rx="0.63999999"
+         sodipodi:cy="191.26801"
+         sodipodi:cx="-541.625"
+         d="m -540.985,191.26801 c 0,0.35346 -0.28654,0.64 -0.64,0.64 -0.35346,0 -0.64,-0.28654 -0.64,-0.64 0,-0.35347 0.28654,-0.64 0.64,-0.64 0.35346,0 0.64,0.28653 0.64,0.64 z"
+         clip-path="url(#SVGID_4523_-1)"
+         stroke-miterlimit="10"
+         cx="-541.625"
+         cy="191.26801"
+         r="0.63999999"
+         id="circle82138" />
+      <circle
+         style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10"
+         sodipodi:ry="0.63999999"
+         sodipodi:rx="0.63999999"
+         sodipodi:cy="193.623"
+         sodipodi:cx="-541.625"
+         d="m -540.985,193.623 c 0,0.35346 -0.28654,0.64 -0.64,0.64 -0.35346,0 -0.64,-0.28654 -0.64,-0.64 0,-0.35346 0.28654,-0.64 0.64,-0.64 0.35346,0 0.64,0.28654 0.64,0.64 z"
+         clip-path="url(#SVGID_4523_-1)"
+         stroke-miterlimit="10"
+         cx="-541.625"
+         cy="193.623"
+         r="0.63999999"
+         id="circle82140" />
+      <circle
+         style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10"
+         sodipodi:ry="0.63999999"
+         sodipodi:rx="0.63999999"
+         sodipodi:cy="191.26801"
+         sodipodi:cx="-543.85303"
+         d="m -543.21303,191.26801 c 0,0.35346 -0.28654,0.64 -0.64,0.64 -0.35346,0 -0.64,-0.28654 -0.64,-0.64 0,-0.35347 0.28654,-0.64 0.64,-0.64 0.35346,0 0.64,0.28653 0.64,0.64 z"
+         clip-path="url(#SVGID_4523_-1)"
+         stroke-miterlimit="10"
+         cx="-543.85303"
+         cy="191.26801"
+         r="0.63999999"
+         id="circle82142" />
+      <circle
+         style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10"
+         sodipodi:ry="0.63999999"
+         sodipodi:rx="0.63999999"
+         sodipodi:cy="193.623"
+         sodipodi:cx="-543.854"
+         d="m -543.214,193.623 c 0,0.35346 -0.28654,0.64 -0.64,0.64 -0.35347,0 -0.64,-0.28654 -0.64,-0.64 0,-0.35346 0.28653,-0.64 0.64,-0.64 0.35346,0 0.64,0.28654 0.64,0.64 z"
+         clip-path="url(#SVGID_4523_-1)"
+         stroke-miterlimit="10"
+         cx="-543.854"
+         cy="193.623"
+         r="0.63999999"
+         id="circle82144" />
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4523_-1)"
+         d="m -537.169,191.908 c -0.354,0 -0.64,-0.286 -0.64,-0.639 0,-0.354 0.286,-0.642 0.64,-0.641 0.354,0 0.64,0.287 0.64,0.641 0,0.353 -0.286,0.639 -0.64,0.639"
+         id="path82146" />
+      <path
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4523_-1)"
+         d="m -537.169,194.263 c -0.354,0 -0.64,-0.286 -0.64,-0.639 0,-0.355 0.286,-0.642 0.64,-0.642 0.354,0 0.64,0.287 0.64,0.642 0,0.353 -0.286,0.639 -0.64,0.639"
+         id="path82148" />
+    </g>
+    <text
+       y="319.3797"
+       x="267.68884"
+       id="text82150"
+       style="font-size:40.5px">
+      <tspan
+         style="font-size:10.80911255px;fill:#ffffff;font-family:ApexSans-Medium"
+         x="267.68884"
+         y="319.3797"
+         font-size="3.2027"
+         id="tspan82152">AP</tspan>
+      <tspan
+         style="font-size:10.80911255px;fill:#ffffff;font-family:ApexSans-Medium"
+         x="281.37094"
+         y="319.3797"
+         font-size="3.2027"
+         id="tspan82154">P</tspan>
+    </text>
+    <g
+       transform="matrix(3.375,0,0,3.375,2054.0831,-334.61629)"
+       id="g84638">
+      <defs
+         id="defs84640">
+        <rect
+           id="SVGID_4852_"
+           x="-682.31403"
+           y="181.315"
+           width="200.271"
+           height="205.44099" />
+      </defs>
+      <clipPath
+         id="clipPath126879">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4852_"
+           overflow="visible"
+           id="use126881" />
+      </clipPath>
+      <path
+         style="fill:#5e6a71"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4853_-8)"
+         d="m -526.673,205.991 c 0,-0.349 0.282,-0.632 0.632,-0.632 0.349,0 0.632,0.283 0.632,0.632 0,0.35 -0.283,0.633 -0.632,0.633 -0.35,0 -0.632,-0.283 -0.632,-0.633"
+         id="path84646" />
+      <line
+         style="fill:none;stroke:#5e6a71;stroke-width:0.214;stroke-miterlimit:10"
+         clip-path="url(#SVGID_4853_-8)"
+         stroke-miterlimit="10"
+         x1="-526.03998"
+         y1="205.991"
+         x2="-523.14398"
+         y2="205.991"
+         id="line84648" />
+    </g>
+    <g
+       transform="matrix(3.375,0,0,3.375,2054.0831,-334.61629)"
+       id="g84650">
+      <defs
+         id="defs84652">
+        <polygon
+           id="SVGID_4854_"
+           points="-507.673,203.458 -521.64,203.458 -523.432,206 -521.64,208.524 -507.673,208.524 " />
+      </defs>
+      <clipPath
+         id="clipPath126888">
+        <use
+           height="1052.3622"
+           width="744.09448"
+           y="0"
+           x="0"
+           style="overflow:visible"
+           xlink:href="#SVGID_4854_"
+           overflow="visible"
+           id="use126890" />
+      </clipPath>
+      <linearGradient
+         id="linearGradient126892"
+         gradientUnits="userSpaceOnUse"
+         x1="-34.205299"
+         y1="249.94031"
+         x2="-33.205101"
+         y2="249.94031"
+         gradientTransform="matrix(0,-5.065,-5.065,0,750.4031,35.2731)">
+        <stop
+           offset="0"
+           style="stop-color:#E6E8E7"
+           id="stop126894" />
+        <stop
+           offset="1"
+           style="stop-color:#FFFFFF"
+           id="stop126896" />
+      </linearGradient>
+      <rect
+         style="fill:url(#SVGID_4856_-4)"
+         x="-523.43201"
+         y="203.45799"
+         clip-path="url(#SVGID_4855_-2)"
+         width="15.759"
+         height="5.066"
+         id="rect84663" />
+    </g>
+    <polygon
+       style="fill:none;stroke:#e6e8e8;stroke-width:0.214;stroke-miterlimit:10"
+       transform="matrix(3.375,0,0,3.375,2054.0831,-334.61629)"
+       stroke-miterlimit="10"
+       points="-507.674,203.459 -521.64,203.459 -523.433,206 -521.64,208.524 -507.674,208.524 "
+       id="polygon84665" />
+    <text
+       y="363.54565"
+       x="306.66861"
+       id="text84667"
+       style="font-size:40.5px">
+      <tspan
+         style="font-size:8.65755081px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="306.66861"
+         y="363.54565"
+         font-size="2.5652"
+         id="tspan84669">RE</tspan>
+      <tspan
+         style="font-size:8.65755081px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="315.84521"
+         y="363.54565"
+         font-size="2.5652"
+         id="tspan84671">S</tspan>
+      <tspan
+         style="font-size:8.65755081px;fill:#5e6a71;font-family:ApexSans-Book"
+         x="319.98297"
+         y="363.54565"
+         font-size="2.5652"
+         id="tspan84673">T</tspan>
+    </text>
+    <text
+       xml:space="preserve"
+       style="font-size:9px;font-style:normal;font-weight:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans"
+       x="197.07527"
+       y="438.16864"
+       id="text127418"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan127420"
+         x="197.07527"
+         y="438.16864">DATA</tspan><tspan
+         sodipodi:role="line"
+         x="197.07527"
+         y="449.41864"
+         id="tspan127422">SYNC</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-size:9px;font-style:normal;font-weight:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans"
+       x="232.25806"
+       y="178.16862"
+       id="text127482"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan127484"
+         x="232.25806"
+         y="178.16862" /></text>
+    <text
+       y="247.60512"
+       x="242.41013"
+       id="text84343-2-0"
+       inkscape:transform-center-x="587.62742"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px;text-align:center;text-anchor:middle">
+      <tspan
+         style="font-size:11px;text-align:center;text-anchor:middle;fill:#37424b;font-family:ApexSans-Book"
+         x="197.43796"
+         y="247.60512"
+         font-size="5.1244"
+         id="tspan84345-2-6">MASTER REGION</tspan>
+    </text>
+    <g
+       id="g17978">
+      <g
+         inkscape:transform-center-y="50.765304"
+         inkscape:transform-center-x="262.3321"
+         id="g82158"
+         transform="matrix(1.3542118,0,0,2.25,996.72494,-25.560183)">
+        <defs
+           id="defs82160">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4524_" />
+        </defs>
+        <clipPath
+           id="clipPath101582">
+          <use
+             id="use101584"
+             overflow="visible"
+             xlink:href="#SVGID_4524_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <g
+           id="g82166"
+           clip-path="url(#SVGID_4525_)">
+          <g
+             id="g82168">
+            <defs
+               id="defs82170">
+              <rect
+                 height="64.251999"
+                 width="175.168"
+                 y="293.259"
+                 x="-679.67902"
+                 id="SVGID_4526_" />
+            </defs>
+            <clipPath
+               id="clipPath101590">
+              <use
+                 id="use101592"
+                 overflow="visible"
+                 xlink:href="#SVGID_4526_"
+                 style="overflow:visible"
+                 x="0"
+                 y="0"
+                 width="744.09448"
+                 height="1052.3622" />
+            </clipPath>
+            <g
+               id="g82176"
+               clip-path="url(#SVGID_4527_)">
+              <defs
+                 id="defs82178">
+                <rect
+                   height="64.037003"
+                   width="174.953"
+                   y="293.367"
+                   x="-679.57202"
+                   id="SVGID_4528_" />
+              </defs>
+              <clipPath
+                 id="clipPath101597">
+                <use
+                   id="use101599"
+                   overflow="visible"
+                   xlink:href="#SVGID_4528_"
+                   style="overflow:visible"
+                   x="0"
+                   y="0"
+                   width="744.09448"
+                   height="1052.3622" />
+              </clipPath>
+              <linearGradient
+                 gradientTransform="matrix(0,64.0388,-64.0388,0,3308.0217,-7826.8223)"
+                 y2="60.902401"
+                 x2="127.8011"
+                 y1="60.902401"
+                 x1="126.8011"
+                 gradientUnits="userSpaceOnUse"
+                 id="linearGradient101601">
+                <stop
+                   id="stop101603"
+                   style="stop-color:#E6E8E7"
+                   offset="0" />
+                <stop
+                   id="stop101605"
+                   style="stop-color:#FFFFFF"
+                   offset="1" />
+              </linearGradient>
+              <rect
+                 id="rect82189"
+                 height="64.038002"
+                 width="174.953"
+                 clip-path="url(#SVGID_4529_)"
+                 y="293.366"
+                 x="-679.57202"
+                 style="fill:url(#SVGID_4530_)" />
+            </g>
+            <g
+               id="g82191"
+               clip-path="url(#SVGID_4527_)">
+              <defs
+                 id="defs82193">
+                <rect
+                   height="64.251999"
+                   width="175.168"
+                   y="293.259"
+                   x="-679.67902"
+                   id="SVGID_4531_" />
+              </defs>
+              <clipPath
+                 id="clipPath101611">
+                <use
+                   id="use101613"
+                   overflow="visible"
+                   xlink:href="#SVGID_4531_"
+                   style="overflow:visible"
+                   x="0"
+                   y="0"
+                   width="744.09448"
+                   height="1052.3622" />
+              </clipPath>
+              <rect
+                 id="rect82199"
+                 height="64.038002"
+                 width="174.953"
+                 stroke-miterlimit="10"
+                 clip-path="url(#SVGID_4532_)"
+                 y="293.366"
+                 x="-679.57202"
+                 style="fill:none;stroke:#f15d57;stroke-width:0.214;stroke-miterlimit:10" />
+            </g>
+          </g>
+        </g>
+        <g
+           id="g82201"
+           clip-path="url(#SVGID_4525_)">
+          <g
+             id="g82203">
+            <defs
+               id="defs82205">
+              <rect
+                 height="64.251999"
+                 width="45.873001"
+                 y="293.259"
+                 x="-441.56699"
+                 id="SVGID_4533_" />
+            </defs>
+            <clipPath
+               id="clipPath101620">
+              <use
+                 id="use101622"
+                 overflow="visible"
+                 xlink:href="#SVGID_4533_"
+                 style="overflow:visible"
+                 x="0"
+                 y="0"
+                 width="744.09448"
+                 height="1052.3622" />
+            </clipPath>
+            <g
+               id="g82211"
+               clip-path="url(#SVGID_4534_)">
+              <defs
+                 id="defs82213">
+                <rect
+                   height="64.037003"
+                   width="45.659"
+                   y="293.367"
+                   x="-441.45999"
+                   id="SVGID_4535_" />
+              </defs>
+              <clipPath
+                 id="clipPath101627">
+                <use
+                   id="use101629"
+                   overflow="visible"
+                   xlink:href="#SVGID_4535_"
+                   style="overflow:visible"
+                   x="0"
+                   y="0"
+                   width="744.09448"
+                   height="1052.3622" />
+              </clipPath>
+              <linearGradient
+                 gradientTransform="matrix(0,64.0388,-64.0388,0,3481.4861,-7826.8223)"
+                 y2="60.902401"
+                 x2="127.8011"
+                 y1="60.902401"
+                 x1="126.8011"
+                 gradientUnits="userSpaceOnUse"
+                 id="linearGradient101631">
+                <stop
+                   id="stop101633"
+                   style="stop-color:#E6E8E7"
+                   offset="0" />
+                <stop
+                   id="stop101635"
+                   style="stop-color:#FFFFFF"
+                   offset="1" />
+              </linearGradient>
+              <rect
+                 id="rect82224"
+                 height="64.038002"
+                 width="45.659"
+                 clip-path="url(#SVGID_4536_)"
+                 y="293.366"
+                 x="-441.45999"
+                 style="fill:url(#SVGID_4537_)" />
+            </g>
+            <g
+               id="g82226"
+               clip-path="url(#SVGID_4534_)">
+              <defs
+                 id="defs82228">
+                <rect
+                   height="64.251999"
+                   width="45.873001"
+                   y="293.259"
+                   x="-441.56699"
+                   id="SVGID_4538_" />
+              </defs>
+              <clipPath
+                 id="clipPath101641">
+                <use
+                   id="use101643"
+                   overflow="visible"
+                   xlink:href="#SVGID_4538_"
+                   style="overflow:visible"
+                   x="0"
+                   y="0"
+                   width="744.09448"
+                   height="1052.3622" />
+              </clipPath>
+              <rect
+                 id="rect82234"
+                 height="64.038002"
+                 width="45.659"
+                 stroke-miterlimit="10"
+                 clip-path="url(#SVGID_4539_)"
+                 y="293.366"
+                 x="-441.45999"
+                 style="fill:none;stroke:#f15d57;stroke-width:0.214;stroke-miterlimit:10" />
+            </g>
+          </g>
+        </g>
+      </g>
+      <g
+         inkscape:transform-center-y="45.666809"
+         inkscape:transform-center-x="449.81598"
+         id="g82339"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs82341">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4549_" />
+        </defs>
+        <clipPath
+           id="clipPath101738">
+          <use
+             id="use101740"
+             overflow="visible"
+             xlink:href="#SVGID_4549_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path82347"
+           d="m -602.031,326.467 c 0,0.212 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.173 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+           clip-path="url(#SVGID_4550_)"
+           inkscape:connector-curvature="0"
+           style="fill:#f15d57" />
+        <path
+           id="path82349"
+           d="m -602.031,326.467 c 0,0.212 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.173 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4550_)"
+           inkscape:connector-curvature="0"
+           style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10" />
+      </g>
+      <text
+         inkscape:transform-center-y="45.543285"
+         inkscape:transform-center-x="450.06672"
+         id="text82351"
+         font-size="4.3343"
+         style="font-size:9.75217533px;fill:#ffffff;font-family:ApexSans-Medium"
+         x="275.14355"
+         y="704.88873">M</text>
+      <g
+         inkscape:transform-center-y="71.647559"
+         inkscape:transform-center-x="503.22873"
+         id="g82353"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs82355">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4551_" />
+        </defs>
+        <clipPath
+           id="clipPath101748">
+          <use
+             id="use101750"
+             overflow="visible"
+             xlink:href="#SVGID_4551_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path82361"
+           d="m -625.77,338.014 c 0,0.213 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.172 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+           clip-path="url(#SVGID_4552_)"
+           inkscape:connector-curvature="0"
+           style="fill:#f15d57" />
+        <path
+           id="path82363"
+           d="m -625.77,338.014 c 0,0.213 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.172 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4552_)"
+           inkscape:connector-curvature="0"
+           style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10" />
+      </g>
+      <text
+         inkscape:transform-center-y="71.524915"
+         inkscape:transform-center-x="503.47899"
+         id="text82365"
+         font-size="4.3343"
+         style="font-size:9.75217533px;fill:#ffffff;font-family:ApexSans-Medium"
+         x="201.73129"
+         y="730.87036">M</text>
+      <rect
+         inkscape:transform-center-y="19.597191"
+         inkscape:transform-center-x="610.96881"
+         id="rect82381"
+         height="21.584251"
+         width="21.581999"
+         y="664.59583"
+         x="87.659889"
+         style="fill:#ffffff" />
+      <rect
+         inkscape:transform-center-y="19.597191"
+         inkscape:transform-center-x="610.96881"
+         id="rect82383"
+         height="21.584251"
+         width="21.581999"
+         stroke-miterlimit="10"
+         y="664.59583"
+         x="87.659889"
+         style="fill:none;stroke:#81d0db;stroke-width:0.96300006;stroke-miterlimit:10" />
+      <rect
+         inkscape:transform-center-y="19.598295"
+         inkscape:transform-center-x="610.97115"
+         id="rect82385"
+         height="18.553501"
+         width="18.557999"
+         y="666.1123"
+         x="89.169548"
+         style="fill:#81d0db" />
+      <rect
+         inkscape:transform-center-y="19.597131"
+         inkscape:transform-center-x="610.96883"
+         id="rect82387"
+         height="15.594751"
+         width="15.592501"
+         y="667.59052"
+         x="90.654625"
+         style="fill:#ffffff" />
+      <rect
+         inkscape:transform-center-y="19.595982"
+         inkscape:transform-center-x="610.97103"
+         id="rect82389"
+         height="10.278"
+         width="10.282501"
+         y="670.24774"
+         x="93.307411"
+         style="fill:#ffffff" />
+      <g
+         inkscape:transform-center-y="19.598312"
+         inkscape:transform-center-x="610.97111"
+         id="g82391"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs82393">
+          <path
+             d="m -672.614,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+             id="SVGID_4555_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath101773">
+          <use
+             id="use101775"
+             overflow="visible"
+             xlink:href="#SVGID_4555_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1066.0031)"
+           y2="-148.46021"
+           x2="256.3761"
+           y1="-148.46021"
+           x1="255.3761"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient101777">
+          <stop
+             id="stop101779"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop101781"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect82404"
+           height="5.3839998"
+           width="5.3839998"
+           clip-path="url(#SVGID_4556_)"
+           y="308.841"
+           x="-672.61401"
+           style="fill:url(#SVGID_4557_)" />
+      </g>
+      <rect
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+         style="fill:#ffffff"
+         x="-650.77399"
+         y="306.73599"
+         clip-path="url(#SVGID_4564_)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82451" />
+      <rect
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-650.77399"
+         y="306.73599"
+         clip-path="url(#SVGID_4564_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82453" />
+      <rect
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+         style="fill:#81d0db"
+         x="-650.10303"
+         y="307.41"
+         clip-path="url(#SVGID_4564_)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82455" />
+      <rect
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+         style="fill:#ffffff"
+         x="-649.44299"
+         y="308.06699"
+         clip-path="url(#SVGID_4564_)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82457" />
+      <rect
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+         style="fill:#ffffff"
+         x="-648.26398"
+         y="309.24799"
+         clip-path="url(#SVGID_4564_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82459" />
+      <g
+         inkscape:transform-center-y="19.598312"
+         inkscape:transform-center-x="557.09937"
+         id="g82461"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs82463">
+          <path
+             d="m -648.671,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+             id="SVGID_4565_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath101827">
+          <use
+             id="use101829"
+             overflow="visible"
+             xlink:href="#SVGID_4565_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1066.0031)"
+           y2="-148.4601"
+           x2="256.3761"
+           y1="-148.4601"
+           x1="255.3761"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient101831">
+          <stop
+             id="stop101833"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop101835"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect82474"
+           height="5.3839998"
+           width="5.3839998"
+           clip-path="url(#SVGID_4566_)"
+           y="308.841"
+           x="-648.67102"
+           style="fill:url(#SVGID_4567_)" />
+      </g>
+      <path
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4569_)"
+         d="m -646.611,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82484" />
+      <rect
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+         style="fill:#ffffff"
+         x="-638.80298"
+         y="306.73599"
+         clip-path="url(#SVGID_4569_)"
+         width="9.592"
+         height="9.5930004"
+         id="rect82486" />
+      <rect
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+         style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10"
+         x="-638.80298"
+         y="306.73599"
+         clip-path="url(#SVGID_4569_)"
+         stroke-miterlimit="10"
+         width="9.592"
+         height="9.5930004"
+         id="rect82488" />
+      <rect
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+         style="fill:#81d0db"
+         x="-638.13202"
+         y="307.41"
+         clip-path="url(#SVGID_4569_)"
+         width="8.2480001"
+         height="8.2460003"
+         id="rect82490" />
+      <rect
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+         style="fill:#ffffff"
+         x="-637.47198"
+         y="308.06699"
+         clip-path="url(#SVGID_4569_)"
+         width="6.9299998"
+         height="6.9310002"
+         id="rect82492" />
+      <rect
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+         style="fill:#ffffff"
+         x="-636.29199"
+         y="309.24799"
+         clip-path="url(#SVGID_4569_)"
+         width="4.5700002"
+         height="4.5679998"
+         id="rect82494" />
+      <g
+         inkscape:transform-center-y="19.598312"
+         inkscape:transform-center-x="530.16231"
+         id="g82496"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs82498">
+          <path
+             d="m -636.699,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.485 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+             id="SVGID_4570_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath101854">
+          <use
+             id="use101856"
+             overflow="visible"
+             xlink:href="#SVGID_4570_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1066.0031)"
+           y2="-148.46001"
+           x2="256.3761"
+           y1="-148.46001"
+           x1="255.3761"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient101858">
+          <stop
+             id="stop101860"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop101862"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect82509"
+           height="5.3839998"
+           width="5.3839998"
+           clip-path="url(#SVGID_4571_)"
+           y="308.841"
+           x="-636.69897"
+           style="fill:url(#SVGID_4572_)" />
+      </g>
+      <g
+         inkscape:transform-center-y="19.597166"
+         inkscape:transform-center-x="511.76858"
+         id="g82511"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs82513">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4573_" />
+        </defs>
+        <clipPath
+           id="clipPath101868">
+          <use
+             id="use101870"
+             overflow="visible"
+             xlink:href="#SVGID_4573_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path82519"
+           d="m -634.639,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4574_)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <rect
+           id="rect82521"
+           height="9.5930004"
+           width="9.592"
+           clip-path="url(#SVGID_4574_)"
+           y="306.73599"
+           x="-626.83099"
+           style="fill:#ffffff" />
+        <rect
+           id="rect82523"
+           height="9.5930004"
+           width="9.592"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4574_)"
+           y="306.73599"
+           x="-626.83099"
+           style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10" />
+        <rect
+           id="rect82525"
+           height="8.2460003"
+           width="8.2480001"
+           clip-path="url(#SVGID_4574_)"
+           y="307.41"
+           x="-626.15997"
+           style="fill:#81d0db" />
+        <rect
+           id="rect82527"
+           height="6.9310002"
+           width="6.9299998"
+           clip-path="url(#SVGID_4574_)"
+           y="308.06699"
+           x="-625.5"
+           style="fill:#ffffff" />
+        <rect
+           id="rect82529"
+           height="4.5679998"
+           width="4.5700002"
+           clip-path="url(#SVGID_4574_)"
+           y="309.24799"
+           x="-624.32098"
+           style="fill:#ffffff" />
+      </g>
+      <g
+         inkscape:transform-center-y="19.598312"
+         inkscape:transform-center-x="503.22762"
+         id="g82531"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs82533">
+          <path
+             d="m -624.728,311.531 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.205 -2.693,2.69"
+             id="SVGID_4575_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath101881">
+          <use
+             id="use101883"
+             overflow="visible"
+             xlink:href="#SVGID_4575_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1066.0031)"
+           y2="-148.4601"
+           x2="256.3761"
+           y1="-148.4601"
+           x1="255.3761"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient101885">
+          <stop
+             id="stop101887"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop101889"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect82544"
+           height="5.3839998"
+           width="5.3839998"
+           clip-path="url(#SVGID_4576_)"
+           y="308.841"
+           x="-624.72803"
+           style="fill:url(#SVGID_4577_)" />
+      </g>
+      <g
+         inkscape:transform-center-y="19.597166"
+         inkscape:transform-center-x="484.83387"
+         id="g82546"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs82548">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4578_" />
+        </defs>
+        <clipPath
+           id="clipPath101895">
+          <use
+             id="use101897"
+             overflow="visible"
+             xlink:href="#SVGID_4578_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path82554"
+           d="m -622.668,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4579_)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <rect
+           id="rect82556"
+           height="9.5930004"
+           width="9.5930004"
+           clip-path="url(#SVGID_4579_)"
+           y="306.73599"
+           x="-614.85999"
+           style="fill:#ffffff" />
+        <rect
+           id="rect82558"
+           height="9.5930004"
+           width="9.5930004"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4579_)"
+           y="306.73599"
+           x="-614.86102"
+           style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10" />
+        <rect
+           id="rect82560"
+           height="8.2460003"
+           width="8.2480001"
+           clip-path="url(#SVGID_4579_)"
+           y="307.41"
+           x="-614.18799"
+           style="fill:#81d0db" />
+        <rect
+           id="rect82562"
+           height="6.9310002"
+           width="6.9310002"
+           clip-path="url(#SVGID_4579_)"
+           y="308.06699"
+           x="-613.52899"
+           style="fill:#ffffff" />
+        <rect
+           id="rect82564"
+           height="4.5679998"
+           width="4.5700002"
+           clip-path="url(#SVGID_4579_)"
+           y="309.24799"
+           x="-612.349"
+           style="fill:#ffffff" />
+      </g>
+      <g
+         inkscape:transform-center-y="19.598312"
+         inkscape:transform-center-x="476.2917"
+         id="g82566"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs82568">
+          <path
+             d="m -612.756,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+             id="SVGID_4580_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath101908">
+          <use
+             id="use101910"
+             overflow="visible"
+             xlink:href="#SVGID_4580_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1066.0031)"
+           y2="-148.4601"
+           x2="256.3761"
+           y1="-148.4601"
+           x1="255.3761"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient101912">
+          <stop
+             id="stop101914"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop101916"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect82579"
+           height="5.3839998"
+           width="5.3829999"
+           clip-path="url(#SVGID_4581_)"
+           y="308.841"
+           x="-612.75598"
+           style="fill:url(#SVGID_4582_)" />
+      </g>
+      <g
+         inkscape:transform-center-y="19.597166"
+         inkscape:transform-center-x="457.89686"
+         id="g82581"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs82583">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4583_" />
+        </defs>
+        <clipPath
+           id="clipPath101922">
+          <use
+             id="use101924"
+             overflow="visible"
+             xlink:href="#SVGID_4583_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path82589"
+           d="m -610.696,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4584_)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <rect
+           id="rect82591"
+           height="9.5930004"
+           width="9.592"
+           clip-path="url(#SVGID_4584_)"
+           y="306.73599"
+           x="-602.888"
+           style="fill:#ffffff" />
+        <rect
+           id="rect82593"
+           height="9.5930004"
+           width="9.592"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4584_)"
+           y="306.73599"
+           x="-602.888"
+           style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10" />
+        <rect
+           id="rect82595"
+           height="8.2460003"
+           width="8.2480001"
+           clip-path="url(#SVGID_4584_)"
+           y="307.41"
+           x="-602.21698"
+           style="fill:#81d0db" />
+        <rect
+           id="rect82597"
+           height="6.9310002"
+           width="6.9299998"
+           clip-path="url(#SVGID_4584_)"
+           y="308.06699"
+           x="-601.55701"
+           style="fill:#ffffff" />
+        <rect
+           id="rect82599"
+           height="4.5679998"
+           width="4.5700002"
+           clip-path="url(#SVGID_4584_)"
+           y="309.24799"
+           x="-600.37799"
+           style="fill:#ffffff" />
+      </g>
+      <g
+         inkscape:transform-center-y="19.598312"
+         inkscape:transform-center-x="449.35694"
+         id="g82601"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs82603">
+          <path
+             d="m -600.785,311.531 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.203,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.205 -2.692,2.69"
+             id="SVGID_4585_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath101935">
+          <use
+             id="use101937"
+             overflow="visible"
+             xlink:href="#SVGID_4585_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3836,5.3836,0,201.1574,-1066.0031)"
+           y2="-148.46021"
+           x2="256.3761"
+           y1="-148.46021"
+           x1="255.3761"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient101939">
+          <stop
+             id="stop101941"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop101943"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect82614"
+           height="5.3839998"
+           width="5.3829999"
+           clip-path="url(#SVGID_4586_)"
+           y="308.841"
+           x="-600.78497"
+           style="fill:url(#SVGID_4587_)" />
+      </g>
+      <g
+         inkscape:transform-center-y="19.597166"
+         inkscape:transform-center-x="430.96208"
+         id="g82616"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs82618">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4588_" />
+        </defs>
+        <clipPath
+           id="clipPath101949">
+          <use
+             id="use101951"
+             overflow="visible"
+             xlink:href="#SVGID_4588_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path82624"
+           d="m -598.725,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4589_)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <rect
+           id="rect82626"
+           height="9.5930004"
+           width="9.592"
+           clip-path="url(#SVGID_4589_)"
+           y="306.73599"
+           x="-590.91699"
+           style="fill:#ffffff" />
+        <rect
+           id="rect82628"
+           height="9.5930004"
+           width="9.592"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4589_)"
+           y="306.73599"
+           x="-590.91699"
+           style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10" />
+        <rect
+           id="rect82630"
+           height="8.2460003"
+           width="8.2480001"
+           clip-path="url(#SVGID_4589_)"
+           y="307.41"
+           x="-590.24597"
+           style="fill:#81d0db" />
+        <rect
+           id="rect82632"
+           height="6.9310002"
+           width="6.9299998"
+           clip-path="url(#SVGID_4589_)"
+           y="308.06699"
+           x="-589.586"
+           style="fill:#ffffff" />
+        <rect
+           id="rect82634"
+           height="4.5679998"
+           width="4.5700002"
+           clip-path="url(#SVGID_4589_)"
+           y="309.24799"
+           x="-588.40698"
+           style="fill:#ffffff" />
+      </g>
+      <g
+         inkscape:transform-center-y="19.598312"
+         inkscape:transform-center-x="422.41995"
+         id="g82636"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs82638">
+          <path
+             d="m -588.813,311.531 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.485 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.205 -2.692,2.69"
+             id="SVGID_4590_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath101962">
+          <use
+             id="use101964"
+             overflow="visible"
+             xlink:href="#SVGID_4590_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1066.0031)"
+           y2="-148.4601"
+           x2="256.3761"
+           y1="-148.4601"
+           x1="255.3761"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient101966">
+          <stop
+             id="stop101968"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop101970"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect82649"
+           height="5.3839998"
+           width="5.3829999"
+           clip-path="url(#SVGID_4591_)"
+           y="308.841"
+           x="-588.81299"
+           style="fill:url(#SVGID_4592_)" />
+      </g>
+      <g
+         inkscape:transform-center-y="37.558931"
+         inkscape:transform-center-x="440.81033"
+         id="g82861"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs82863">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4623_" />
+        </defs>
+        <clipPath
+           id="clipPath102138">
+          <use
+             id="use102140"
+             overflow="visible"
+             xlink:href="#SVGID_4623_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path82869"
+           d="m -514.925,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4624_)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <rect
+           id="rect82871"
+           height="9.5930004"
+           width="9.592"
+           clip-path="url(#SVGID_4624_)"
+           y="318.323"
+           x="-674.71698"
+           style="fill:#ffffff" />
+        <rect
+           id="rect82873"
+           height="9.5930004"
+           width="9.592"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4624_)"
+           y="318.323"
+           x="-674.71698"
+           style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10" />
+        <rect
+           id="rect82875"
+           height="8.2460003"
+           width="8.2480001"
+           clip-path="url(#SVGID_4624_)"
+           y="318.99701"
+           x="-674.04602"
+           style="fill:#81d0db" />
+        <rect
+           id="rect82877"
+           height="6.9310002"
+           width="6.9299998"
+           clip-path="url(#SVGID_4624_)"
+           y="319.65399"
+           x="-673.38599"
+           style="fill:#ffffff" />
+        <rect
+           id="rect82879"
+           height="4.5679998"
+           width="4.5700002"
+           clip-path="url(#SVGID_4624_)"
+           y="320.83499"
+           x="-672.20697"
+           style="fill:#ffffff" />
+      </g>
+      <g
+         inkscape:transform-center-y="45.671291"
+         inkscape:transform-center-x="610.97111"
+         id="g82881"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs82883">
+          <path
+             d="m -672.614,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+             id="SVGID_4625_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath102151">
+          <use
+             id="use102153"
+             overflow="visible"
+             xlink:href="#SVGID_4625_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3836,5.3836,0,129.3288,-1054.4158)"
+           y2="-148.46021"
+           x2="256.37631"
+           y1="-148.46021"
+           x1="255.37621"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient102155">
+          <stop
+             id="stop102157"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop102159"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect82894"
+           height="5.3839998"
+           width="5.3839998"
+           clip-path="url(#SVGID_4626_)"
+           y="320.42899"
+           x="-672.61401"
+           style="fill:url(#SVGID_4627_)" />
+      </g>
+      <g
+         inkscape:transform-center-y="45.667929"
+         inkscape:transform-center-x="592.57619"
+         id="g82896"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs82898">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4628_" />
+        </defs>
+        <clipPath
+           id="clipPath102165">
+          <use
+             id="use102167"
+             overflow="visible"
+             xlink:href="#SVGID_4628_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path82904"
+           d="m -670.553,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4629_)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <rect
+           id="rect82906"
+           height="9.5930004"
+           width="9.592"
+           clip-path="url(#SVGID_4629_)"
+           y="318.323"
+           x="-662.745"
+           style="fill:#ffffff" />
+        <rect
+           id="rect82908"
+           height="9.5930004"
+           width="9.592"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4629_)"
+           y="318.323"
+           x="-662.74597"
+           style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10" />
+        <rect
+           id="rect82910"
+           height="8.2460003"
+           width="8.2480001"
+           clip-path="url(#SVGID_4629_)"
+           y="318.99701"
+           x="-662.07397"
+           style="fill:#81d0db" />
+        <rect
+           id="rect82912"
+           height="6.9310002"
+           width="6.9299998"
+           clip-path="url(#SVGID_4629_)"
+           y="319.65399"
+           x="-661.414"
+           style="fill:#ffffff" />
+        <rect
+           id="rect82914"
+           height="4.5679998"
+           width="4.5700002"
+           clip-path="url(#SVGID_4629_)"
+           y="320.83499"
+           x="-660.23499"
+           style="fill:#ffffff" />
+      </g>
+      <g
+         inkscape:transform-center-y="45.671291"
+         inkscape:transform-center-x="584.03412"
+         id="g82916"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs82918">
+          <path
+             d="m -660.642,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+             id="SVGID_4630_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath102178">
+          <use
+             id="use102180"
+             overflow="visible"
+             xlink:href="#SVGID_4630_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3836,5.3836,0,141.3002,-1054.4158)"
+           y2="-148.4601"
+           x2="256.37631"
+           y1="-148.4601"
+           x1="255.37621"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient102182">
+          <stop
+             id="stop102184"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop102186"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect82929"
+           height="5.3839998"
+           width="5.3839998"
+           clip-path="url(#SVGID_4631_)"
+           y="320.42899"
+           x="-660.64203"
+           style="fill:url(#SVGID_4632_)" />
+      </g>
+      <g
+         inkscape:transform-center-y="45.667929"
+         inkscape:transform-center-x="565.64033"
+         id="g82931"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs82933">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4633_" />
+        </defs>
+        <clipPath
+           id="clipPath102192">
+          <use
+             id="use102194"
+             overflow="visible"
+             xlink:href="#SVGID_4633_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path82939"
+           d="m -658.582,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4634_)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <rect
+           id="rect82941"
+           height="9.5930004"
+           width="9.592"
+           clip-path="url(#SVGID_4634_)"
+           y="318.323"
+           x="-650.77399"
+           style="fill:#ffffff" />
+        <rect
+           id="rect82943"
+           height="9.5930004"
+           width="9.592"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4634_)"
+           y="318.323"
+           x="-650.77399"
+           style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10" />
+        <rect
+           id="rect82945"
+           height="8.2460003"
+           width="8.2480001"
+           clip-path="url(#SVGID_4634_)"
+           y="318.99701"
+           x="-650.10303"
+           style="fill:#81d0db" />
+        <rect
+           id="rect82947"
+           height="6.9310002"
+           width="6.9299998"
+           clip-path="url(#SVGID_4634_)"
+           y="319.65399"
+           x="-649.44299"
+           style="fill:#ffffff" />
+        <rect
+           id="rect82949"
+           height="4.5679998"
+           width="4.5700002"
+           clip-path="url(#SVGID_4634_)"
+           y="320.83499"
+           x="-648.26398"
+           style="fill:#ffffff" />
+      </g>
+      <g
+         inkscape:transform-center-y="45.671291"
+         inkscape:transform-center-x="557.09937"
+         id="g82951"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs82953">
+          <path
+             d="m -648.671,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.204,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+             id="SVGID_4635_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath102205">
+          <use
+             id="use102207"
+             overflow="visible"
+             xlink:href="#SVGID_4635_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3836,5.3836,0,153.2715,-1054.4158)"
+           y2="-148.4601"
+           x2="256.37631"
+           y1="-148.4601"
+           x1="255.37621"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient102209">
+          <stop
+             id="stop102211"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop102213"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect82964"
+           height="5.3839998"
+           width="5.3839998"
+           clip-path="url(#SVGID_4636_)"
+           y="320.42899"
+           x="-648.67102"
+           style="fill:url(#SVGID_4637_)" />
+      </g>
+      <g
+         inkscape:transform-center-y="45.667929"
+         inkscape:transform-center-x="538.70558"
+         id="g82966"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs82968">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4638_" />
+        </defs>
+        <clipPath
+           id="clipPath102219">
+          <use
+             id="use102221"
+             overflow="visible"
+             xlink:href="#SVGID_4638_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path82974"
+           d="m -646.611,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4639_)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <rect
+           id="rect82976"
+           height="9.5930004"
+           width="9.592"
+           clip-path="url(#SVGID_4639_)"
+           y="318.323"
+           x="-638.80298"
+           style="fill:#ffffff" />
+        <rect
+           id="rect82978"
+           height="9.5930004"
+           width="9.592"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4639_)"
+           y="318.323"
+           x="-638.80298"
+           style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10" />
+        <rect
+           id="rect82980"
+           height="8.2460003"
+           width="8.2480001"
+           clip-path="url(#SVGID_4639_)"
+           y="318.99701"
+           x="-638.13202"
+           style="fill:#81d0db" />
+        <rect
+           id="rect82982"
+           height="6.9310002"
+           width="6.9299998"
+           clip-path="url(#SVGID_4639_)"
+           y="319.65399"
+           x="-637.47198"
+           style="fill:#ffffff" />
+        <rect
+           id="rect82984"
+           height="4.5679998"
+           width="4.5700002"
+           clip-path="url(#SVGID_4639_)"
+           y="320.83499"
+           x="-636.29199"
+           style="fill:#ffffff" />
+      </g>
+      <g
+         inkscape:transform-center-y="45.671291"
+         inkscape:transform-center-x="530.16231"
+         id="g82986"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs82988">
+          <path
+             d="m -636.699,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.207 2.692,-2.694 0,-1.486 -1.205,-2.69 -2.692,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+             id="SVGID_4640_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath102232">
+          <use
+             id="use102234"
+             overflow="visible"
+             xlink:href="#SVGID_4640_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3836,5.3836,0,165.2429,-1054.4158)"
+           y2="-148.46001"
+           x2="256.37631"
+           y1="-148.46001"
+           x1="255.37621"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient102236">
+          <stop
+             id="stop102238"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop102240"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect82999"
+           height="5.3839998"
+           width="5.3839998"
+           clip-path="url(#SVGID_4641_)"
+           y="320.42899"
+           x="-636.69897"
+           style="fill:url(#SVGID_4642_)" />
+      </g>
+      <g
+         inkscape:transform-center-y="45.667929"
+         inkscape:transform-center-x="511.76858"
+         id="g83001"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83003">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4643_" />
+        </defs>
+        <clipPath
+           id="clipPath102246">
+          <use
+             id="use102248"
+             overflow="visible"
+             xlink:href="#SVGID_4643_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path83009"
+           d="m -634.639,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4644_)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83011"
+           height="9.5930004"
+           width="9.592"
+           clip-path="url(#SVGID_4644_)"
+           y="318.323"
+           x="-626.83099"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83013"
+           height="9.5930004"
+           width="9.592"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4644_)"
+           y="318.323"
+           x="-626.83099"
+           style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10" />
+        <rect
+           id="rect83015"
+           height="8.2460003"
+           width="8.2480001"
+           clip-path="url(#SVGID_4644_)"
+           y="318.99701"
+           x="-626.15997"
+           style="fill:#81d0db" />
+        <rect
+           id="rect83017"
+           height="6.9310002"
+           width="6.9299998"
+           clip-path="url(#SVGID_4644_)"
+           y="319.65399"
+           x="-625.5"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83019"
+           height="4.5679998"
+           width="4.5700002"
+           clip-path="url(#SVGID_4644_)"
+           y="320.83499"
+           x="-624.32098"
+           style="fill:#ffffff" />
+      </g>
+      <g
+         inkscape:transform-center-y="45.671291"
+         inkscape:transform-center-x="503.22762"
+         id="g83021"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83023">
+          <path
+             d="m -624.728,323.119 c 0,1.487 1.206,2.694 2.693,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.693,1.204 -2.693,2.69"
+             id="SVGID_4645_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath102259">
+          <use
+             id="use102261"
+             overflow="visible"
+             xlink:href="#SVGID_4645_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3836,5.3836,0,177.2143,-1054.4158)"
+           y2="-148.4601"
+           x2="256.37631"
+           y1="-148.4601"
+           x1="255.37621"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient102263">
+          <stop
+             id="stop102265"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop102267"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect83034"
+           height="5.3839998"
+           width="5.3839998"
+           clip-path="url(#SVGID_4646_)"
+           y="320.42899"
+           x="-624.72803"
+           style="fill:url(#SVGID_4647_)" />
+      </g>
+      <g
+         inkscape:transform-center-y="45.667929"
+         inkscape:transform-center-x="484.83387"
+         id="g83036"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83038">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4648_" />
+        </defs>
+        <clipPath
+           id="clipPath102273">
+          <use
+             id="use102275"
+             overflow="visible"
+             xlink:href="#SVGID_4648_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path83044"
+           d="m -622.668,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4649_)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83046"
+           height="9.5930004"
+           width="9.5930004"
+           clip-path="url(#SVGID_4649_)"
+           y="318.323"
+           x="-614.85999"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83048"
+           height="9.5930004"
+           width="9.5930004"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4649_)"
+           y="318.323"
+           x="-614.86102"
+           style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10" />
+        <rect
+           id="rect83050"
+           height="8.2460003"
+           width="8.2480001"
+           clip-path="url(#SVGID_4649_)"
+           y="318.99701"
+           x="-614.18799"
+           style="fill:#81d0db" />
+        <rect
+           id="rect83052"
+           height="6.9310002"
+           width="6.9310002"
+           clip-path="url(#SVGID_4649_)"
+           y="319.65399"
+           x="-613.52899"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83054"
+           height="4.5679998"
+           width="4.5700002"
+           clip-path="url(#SVGID_4649_)"
+           y="320.83499"
+           x="-612.349"
+           style="fill:#ffffff" />
+      </g>
+      <g
+         inkscape:transform-center-y="45.671291"
+         inkscape:transform-center-x="476.2917"
+         id="g83056"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83058">
+          <path
+             d="m -612.756,323.119 c 0,1.487 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.486,0 -2.692,1.204 -2.692,2.69"
+             id="SVGID_4650_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath102286">
+          <use
+             id="use102288"
+             overflow="visible"
+             xlink:href="#SVGID_4650_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3836,5.3836,0,189.1858,-1054.4158)"
+           y2="-148.4601"
+           x2="256.37631"
+           y1="-148.4601"
+           x1="255.37621"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient102290">
+          <stop
+             id="stop102292"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop102294"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect83069"
+           height="5.3839998"
+           width="5.3829999"
+           clip-path="url(#SVGID_4651_)"
+           y="320.42899"
+           x="-612.75598"
+           style="fill:url(#SVGID_4652_)" />
+      </g>
+      <g
+         inkscape:transform-center-y="45.667929"
+         inkscape:transform-center-x="444.42945"
+         id="g83071"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83073">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4653_" />
+        </defs>
+        <clipPath
+           id="clipPath102300">
+          <use
+             id="use102302"
+             overflow="visible"
+             xlink:href="#SVGID_4653_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path83079"
+           d="m -610.696,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4654_)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83081"
+           height="9.5930004"
+           width="9.592"
+           clip-path="url(#SVGID_4654_)"
+           y="318.323"
+           x="-590.91699"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83083"
+           height="9.5930004"
+           width="9.592"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4654_)"
+           y="318.323"
+           x="-590.91699"
+           style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10" />
+        <rect
+           id="rect83085"
+           height="8.2460003"
+           width="8.2480001"
+           clip-path="url(#SVGID_4654_)"
+           y="318.99701"
+           x="-590.24597"
+           style="fill:#81d0db" />
+        <rect
+           id="rect83087"
+           height="6.9310002"
+           width="6.9299998"
+           clip-path="url(#SVGID_4654_)"
+           y="319.65399"
+           x="-589.586"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83089"
+           height="4.5679998"
+           width="4.5700002"
+           clip-path="url(#SVGID_4654_)"
+           y="320.83499"
+           x="-588.40698"
+           style="fill:#ffffff" />
+      </g>
+      <g
+         inkscape:transform-center-y="45.671291"
+         inkscape:transform-center-x="422.41995"
+         id="g83091"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83093">
+          <path
+             d="m -588.813,323.119 c 0,1.487 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.207 2.691,-2.694 0,-1.486 -1.204,-2.69 -2.691,-2.69 -1.487,0 -2.692,1.204 -2.692,2.69"
+             id="SVGID_4655_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath102313">
+          <use
+             id="use102315"
+             overflow="visible"
+             xlink:href="#SVGID_4655_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3836,5.3836,0,213.1288,-1054.4158)"
+           y2="-148.4601"
+           x2="256.37631"
+           y1="-148.4601"
+           x1="255.37621"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient102317">
+          <stop
+             id="stop102319"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop102321"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect83104"
+           height="5.3839998"
+           width="5.3829999"
+           clip-path="url(#SVGID_4656_)"
+           y="320.42899"
+           x="-588.81299"
+           style="fill:url(#SVGID_4657_)" />
+      </g>
+      <g
+         inkscape:transform-center-y="63.583551"
+         inkscape:transform-center-x="440.81033"
+         id="g83316"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83318">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4688_" />
+        </defs>
+        <clipPath
+           id="clipPath102489">
+          <use
+             id="use102491"
+             overflow="visible"
+             xlink:href="#SVGID_4688_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path83324"
+           d="m -514.925,323.119 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4689_)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83326"
+           height="9.592"
+           width="9.592"
+           clip-path="url(#SVGID_4689_)"
+           y="329.87"
+           x="-674.71698"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83328"
+           height="9.592"
+           width="9.592"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4689_)"
+           y="329.87"
+           x="-674.71698"
+           style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10" />
+        <rect
+           id="rect83330"
+           height="8.2469997"
+           width="8.2480001"
+           clip-path="url(#SVGID_4689_)"
+           y="330.543"
+           x="-674.04602"
+           style="fill:#81d0db" />
+        <rect
+           id="rect83332"
+           height="6.9299998"
+           width="6.9299998"
+           clip-path="url(#SVGID_4689_)"
+           y="331.20099"
+           x="-673.38599"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83334"
+           height="4.5679998"
+           width="4.5700002"
+           clip-path="url(#SVGID_4689_)"
+           y="332.38199"
+           x="-672.20697"
+           style="fill:#ffffff" />
+      </g>
+      <g
+         inkscape:transform-center-y="71.649806"
+         inkscape:transform-center-x="610.97111"
+         id="g83336"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83338">
+          <path
+             d="m -672.614,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+             id="SVGID_4690_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath102502">
+          <use
+             id="use102504"
+             overflow="visible"
+             xlink:href="#SVGID_4690_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1042.9659)"
+           y2="-148.4301"
+           x2="256.3544"
+           y1="-148.4301"
+           x1="255.3541"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient102506">
+          <stop
+             id="stop102508"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop102510"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect83349"
+           height="5.3860002"
+           width="5.3839998"
+           clip-path="url(#SVGID_4691_)"
+           y="331.974"
+           x="-672.61401"
+           style="fill:url(#SVGID_4692_)" />
+      </g>
+      <g
+         inkscape:transform-center-y="71.647547"
+         inkscape:transform-center-x="592.57619"
+         id="g83351"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83353">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4693_" />
+        </defs>
+        <clipPath
+           id="clipPath102516">
+          <use
+             id="use102518"
+             overflow="visible"
+             xlink:href="#SVGID_4693_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path83359"
+           d="m -670.553,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4694_)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83361"
+           height="9.592"
+           width="9.592"
+           clip-path="url(#SVGID_4694_)"
+           y="329.87"
+           x="-662.745"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83363"
+           height="9.592"
+           width="9.592"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4694_)"
+           y="329.87"
+           x="-662.74597"
+           style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10" />
+        <rect
+           id="rect83365"
+           height="8.2469997"
+           width="8.2480001"
+           clip-path="url(#SVGID_4694_)"
+           y="330.543"
+           x="-662.07397"
+           style="fill:#81d0db" />
+        <rect
+           id="rect83367"
+           height="6.9299998"
+           width="6.9299998"
+           clip-path="url(#SVGID_4694_)"
+           y="331.20099"
+           x="-661.414"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83369"
+           height="4.5679998"
+           width="4.5700002"
+           clip-path="url(#SVGID_4694_)"
+           y="332.38199"
+           x="-660.23499"
+           style="fill:#ffffff" />
+      </g>
+      <g
+         inkscape:transform-center-y="71.648681"
+         inkscape:transform-center-x="584.03412"
+         id="g83371"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83373">
+          <path
+             d="m -660.642,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+             id="SVGID_4695_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath102529">
+          <use
+             id="use102531"
+             overflow="visible"
+             xlink:href="#SVGID_4695_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1042.9659)"
+           y2="-148.4299"
+           x2="256.35419"
+           y1="-148.4299"
+           x1="255.3541"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient102533">
+          <stop
+             id="stop102535"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop102537"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect83384"
+           height="5.3850002"
+           width="5.3839998"
+           clip-path="url(#SVGID_4696_)"
+           y="331.974"
+           x="-660.64203"
+           style="fill:url(#SVGID_4697_)" />
+      </g>
+      <g
+         inkscape:transform-center-y="71.647547"
+         inkscape:transform-center-x="565.64033"
+         id="g83386"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83388">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4698_" />
+        </defs>
+        <clipPath
+           id="clipPath102543">
+          <use
+             id="use102545"
+             overflow="visible"
+             xlink:href="#SVGID_4698_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path83394"
+           d="m -658.582,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4699_)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83396"
+           height="9.592"
+           width="9.592"
+           clip-path="url(#SVGID_4699_)"
+           y="329.87"
+           x="-650.77399"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83398"
+           height="9.592"
+           width="9.592"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4699_)"
+           y="329.87"
+           x="-650.77399"
+           style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10" />
+        <rect
+           id="rect83400"
+           height="8.2469997"
+           width="8.2480001"
+           clip-path="url(#SVGID_4699_)"
+           y="330.543"
+           x="-650.10303"
+           style="fill:#81d0db" />
+        <rect
+           id="rect83402"
+           height="6.9299998"
+           width="6.9299998"
+           clip-path="url(#SVGID_4699_)"
+           y="331.20099"
+           x="-649.44299"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83404"
+           height="4.5679998"
+           width="4.5700002"
+           clip-path="url(#SVGID_4699_)"
+           y="332.38199"
+           x="-648.26398"
+           style="fill:#ffffff" />
+      </g>
+      <g
+         inkscape:transform-center-y="71.649806"
+         inkscape:transform-center-x="557.09937"
+         id="g83406"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83408">
+          <path
+             d="m -648.671,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+             id="SVGID_4700_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath102556">
+          <use
+             id="use102558"
+             overflow="visible"
+             xlink:href="#SVGID_4700_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1042.9659)"
+           y2="-148.42999"
+           x2="256.3544"
+           y1="-148.42999"
+           x1="255.3541"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient102560">
+          <stop
+             id="stop102562"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop102564"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect83419"
+           height="5.3860002"
+           width="5.3839998"
+           clip-path="url(#SVGID_4701_)"
+           y="331.974"
+           x="-648.67102"
+           style="fill:url(#SVGID_4702_)" />
+      </g>
+      <g
+         inkscape:transform-center-y="71.647547"
+         inkscape:transform-center-x="538.70558"
+         id="g83421"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83423">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4703_" />
+        </defs>
+        <clipPath
+           id="clipPath102570">
+          <use
+             id="use102572"
+             overflow="visible"
+             xlink:href="#SVGID_4703_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path83429"
+           d="m -646.611,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4704_)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83431"
+           height="9.592"
+           width="9.592"
+           clip-path="url(#SVGID_4704_)"
+           y="329.87"
+           x="-638.80298"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83433"
+           height="9.592"
+           width="9.592"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4704_)"
+           y="329.87"
+           x="-638.80298"
+           style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10" />
+        <rect
+           id="rect83435"
+           height="8.2469997"
+           width="8.2480001"
+           clip-path="url(#SVGID_4704_)"
+           y="330.543"
+           x="-638.13202"
+           style="fill:#81d0db" />
+        <rect
+           id="rect83437"
+           height="6.9299998"
+           width="6.9299998"
+           clip-path="url(#SVGID_4704_)"
+           y="331.20099"
+           x="-637.47198"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83439"
+           height="4.5679998"
+           width="4.5700002"
+           clip-path="url(#SVGID_4704_)"
+           y="332.38199"
+           x="-636.29199"
+           style="fill:#ffffff" />
+      </g>
+      <g
+         inkscape:transform-center-y="71.649806"
+         inkscape:transform-center-x="530.16231"
+         id="g83441"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83443">
+          <path
+             d="m -636.699,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.692,-1.206 2.692,-2.694 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+             id="SVGID_4705_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath102583">
+          <use
+             id="use102585"
+             overflow="visible"
+             xlink:href="#SVGID_4705_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1042.9659)"
+           y2="-148.4299"
+           x2="256.3544"
+           y1="-148.4299"
+           x1="255.3541"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient102587">
+          <stop
+             id="stop102589"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop102591"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect83454"
+           height="5.3860002"
+           width="5.3839998"
+           clip-path="url(#SVGID_4706_)"
+           y="331.974"
+           x="-636.69897"
+           style="fill:url(#SVGID_4707_)" />
+      </g>
+      <g
+         inkscape:transform-center-y="71.647547"
+         inkscape:transform-center-x="498.30125"
+         id="g83456"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83458">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4708_" />
+        </defs>
+        <clipPath
+           id="clipPath102597">
+          <use
+             id="use102599"
+             overflow="visible"
+             xlink:href="#SVGID_4708_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path83464"
+           d="m -634.639,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4709_)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83466"
+           height="9.592"
+           width="9.5930004"
+           clip-path="url(#SVGID_4709_)"
+           y="329.87"
+           x="-614.85999"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83468"
+           height="9.592"
+           width="9.5930004"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4709_)"
+           y="329.87"
+           x="-614.86102"
+           style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10" />
+        <rect
+           id="rect83470"
+           height="8.2469997"
+           width="8.2480001"
+           clip-path="url(#SVGID_4709_)"
+           y="330.543"
+           x="-614.18799"
+           style="fill:#81d0db" />
+        <rect
+           id="rect83472"
+           height="6.9299998"
+           width="6.9310002"
+           clip-path="url(#SVGID_4709_)"
+           y="331.20099"
+           x="-613.52899"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83474"
+           height="4.5679998"
+           width="4.5700002"
+           clip-path="url(#SVGID_4709_)"
+           y="332.38199"
+           x="-612.349"
+           style="fill:#ffffff" />
+      </g>
+      <g
+         inkscape:transform-center-y="71.649806"
+         inkscape:transform-center-x="476.2917"
+         id="g83476"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83478">
+          <path
+             d="m -612.756,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+             id="SVGID_4710_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath102610">
+          <use
+             id="use102612"
+             overflow="visible"
+             xlink:href="#SVGID_4710_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1042.9659)"
+           y2="-148.42999"
+           x2="256.3544"
+           y1="-148.42999"
+           x1="255.3541"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient102614">
+          <stop
+             id="stop102616"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop102618"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect83489"
+           height="5.3860002"
+           width="5.3829999"
+           clip-path="url(#SVGID_4711_)"
+           y="331.974"
+           x="-612.75598"
+           style="fill:url(#SVGID_4712_)" />
+      </g>
+      <g
+         inkscape:transform-center-y="71.647547"
+         inkscape:transform-center-x="457.89686"
+         id="g83491"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83493">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4713_" />
+        </defs>
+        <clipPath
+           id="clipPath102624">
+          <use
+             id="use102626"
+             overflow="visible"
+             xlink:href="#SVGID_4713_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path83499"
+           d="m -610.696,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4714_)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83501"
+           height="9.592"
+           width="9.592"
+           clip-path="url(#SVGID_4714_)"
+           y="329.87"
+           x="-602.888"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83503"
+           height="9.592"
+           width="9.592"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4714_)"
+           y="329.87"
+           x="-602.888"
+           style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10" />
+        <rect
+           id="rect83505"
+           height="8.2469997"
+           width="8.2480001"
+           clip-path="url(#SVGID_4714_)"
+           y="330.543"
+           x="-602.21698"
+           style="fill:#81d0db" />
+        <rect
+           id="rect83507"
+           height="6.9299998"
+           width="6.9299998"
+           clip-path="url(#SVGID_4714_)"
+           y="331.20099"
+           x="-601.55701"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83509"
+           height="4.5679998"
+           width="4.5700002"
+           clip-path="url(#SVGID_4714_)"
+           y="332.38199"
+           x="-600.37799"
+           style="fill:#ffffff" />
+      </g>
+      <g
+         inkscape:transform-center-y="71.649806"
+         inkscape:transform-center-x="449.35694"
+         id="g83511"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83513">
+          <path
+             d="m -600.785,334.666 c 0,1.488 1.206,2.694 2.692,2.694 1.488,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+             id="SVGID_4715_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath102637">
+          <use
+             id="use102639"
+             overflow="visible"
+             xlink:href="#SVGID_4715_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1042.9659)"
+           y2="-148.4301"
+           x2="256.3544"
+           y1="-148.4301"
+           x1="255.3541"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient102641">
+          <stop
+             id="stop102643"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop102645"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect83524"
+           height="5.3860002"
+           width="5.3829999"
+           clip-path="url(#SVGID_4716_)"
+           y="331.974"
+           x="-600.78497"
+           style="fill:url(#SVGID_4717_)" />
+      </g>
+      <g
+         inkscape:transform-center-y="71.647547"
+         inkscape:transform-center-x="430.96208"
+         id="g83526"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83528">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4718_" />
+        </defs>
+        <clipPath
+           id="clipPath102651">
+          <use
+             id="use102653"
+             overflow="visible"
+             xlink:href="#SVGID_4718_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path83534"
+           d="m -598.725,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4719_)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83536"
+           height="9.592"
+           width="9.592"
+           clip-path="url(#SVGID_4719_)"
+           y="329.87"
+           x="-590.91699"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83538"
+           height="9.592"
+           width="9.592"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4719_)"
+           y="329.87"
+           x="-590.91699"
+           style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10" />
+        <rect
+           id="rect83540"
+           height="8.2469997"
+           width="8.2480001"
+           clip-path="url(#SVGID_4719_)"
+           y="330.543"
+           x="-590.24597"
+           style="fill:#81d0db" />
+        <rect
+           id="rect83542"
+           height="6.9299998"
+           width="6.9299998"
+           clip-path="url(#SVGID_4719_)"
+           y="331.20099"
+           x="-589.586"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83544"
+           height="4.5679998"
+           width="4.5700002"
+           clip-path="url(#SVGID_4719_)"
+           y="332.38199"
+           x="-588.40698"
+           style="fill:#ffffff" />
+      </g>
+      <g
+         inkscape:transform-center-y="71.649806"
+         inkscape:transform-center-x="422.41995"
+         id="g83546"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83548">
+          <path
+             d="m -588.813,334.666 c 0,1.488 1.205,2.694 2.692,2.694 1.487,0 2.691,-1.206 2.691,-2.694 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+             id="SVGID_4720_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath102664">
+          <use
+             id="use102666"
+             overflow="visible"
+             xlink:href="#SVGID_4720_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1042.9659)"
+           y2="-148.42999"
+           x2="256.3544"
+           y1="-148.42999"
+           x1="255.3541"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient102668">
+          <stop
+             id="stop102670"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop102672"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect83559"
+           height="5.3860002"
+           width="5.3829999"
+           clip-path="url(#SVGID_4721_)"
+           y="331.974"
+           x="-588.81299"
+           style="fill:url(#SVGID_4722_)" />
+      </g>
+      <g
+         inkscape:transform-center-y="89.828699"
+         inkscape:transform-center-x="440.81033"
+         id="g83736"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83738">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4748_" />
+        </defs>
+        <clipPath
+           id="clipPath102813">
+          <use
+             id="use102815"
+             overflow="visible"
+             xlink:href="#SVGID_4748_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path83744"
+           d="m -514.925,334.666 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4749_)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83746"
+           height="9.592"
+           width="9.592"
+           clip-path="url(#SVGID_4749_)"
+           y="341.65302"
+           x="-674.71698"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83748"
+           height="9.592"
+           width="9.592"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4749_)"
+           y="341.65302"
+           x="-674.71698"
+           style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10" />
+        <rect
+           id="rect83750"
+           height="8.2469997"
+           width="8.2480001"
+           clip-path="url(#SVGID_4749_)"
+           y="342.32599"
+           x="-674.04602"
+           style="fill:#81d0db" />
+        <rect
+           id="rect83752"
+           height="6.9299998"
+           width="6.9299998"
+           clip-path="url(#SVGID_4749_)"
+           y="342.98401"
+           x="-673.38599"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83754"
+           height="4.5679998"
+           width="4.5700002"
+           clip-path="url(#SVGID_4749_)"
+           y="344.164"
+           x="-672.20697"
+           style="fill:#ffffff" />
+      </g>
+      <g
+         inkscape:transform-center-y="98.159295"
+         inkscape:transform-center-x="610.97111"
+         id="g83756"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83758">
+          <path
+             d="m -672.614,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+             id="SVGID_4750_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath102826">
+          <use
+             id="use102828"
+             overflow="visible"
+             xlink:href="#SVGID_4750_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3844,5.3844,0,129.2913,-1031.1833)"
+           y2="-148.4301"
+           x2="256.3541"
+           y1="-148.4301"
+           x1="255.3541"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient102830">
+          <stop
+             id="stop102832"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop102834"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect83769"
+           height="5.3839998"
+           width="5.3839998"
+           clip-path="url(#SVGID_4751_)"
+           y="343.75699"
+           x="-672.61401"
+           style="fill:url(#SVGID_4752_)" />
+      </g>
+      <g
+         inkscape:transform-center-y="98.159342"
+         inkscape:transform-center-x="592.57619"
+         id="g83771"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83773">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4753_" />
+        </defs>
+        <clipPath
+           id="clipPath102840">
+          <use
+             id="use102842"
+             overflow="visible"
+             xlink:href="#SVGID_4753_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path83779"
+           d="m -670.553,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4754_)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83781"
+           height="9.592"
+           width="9.592"
+           clip-path="url(#SVGID_4754_)"
+           y="341.65302"
+           x="-662.745"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83783"
+           height="9.592"
+           width="9.592"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4754_)"
+           y="341.65302"
+           x="-662.74597"
+           style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10" />
+        <rect
+           id="rect83785"
+           height="8.2469997"
+           width="8.2480001"
+           clip-path="url(#SVGID_4754_)"
+           y="342.32599"
+           x="-662.07397"
+           style="fill:#81d0db" />
+        <rect
+           id="rect83787"
+           height="6.9299998"
+           width="6.9299998"
+           clip-path="url(#SVGID_4754_)"
+           y="342.98401"
+           x="-661.414"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83789"
+           height="4.5679998"
+           width="4.5700002"
+           clip-path="url(#SVGID_4754_)"
+           y="344.164"
+           x="-660.23499"
+           style="fill:#ffffff" />
+      </g>
+      <g
+         inkscape:transform-center-y="98.159295"
+         inkscape:transform-center-x="584.03412"
+         id="g83791"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83793">
+          <path
+             d="m -660.642,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+             id="SVGID_4755_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath102853">
+          <use
+             id="use102855"
+             overflow="visible"
+             xlink:href="#SVGID_4755_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3844,5.3844,0,141.2627,-1031.1833)"
+           y2="-148.4299"
+           x2="256.3541"
+           y1="-148.4299"
+           x1="255.3541"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient102857">
+          <stop
+             id="stop102859"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop102861"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect83804"
+           height="5.3839998"
+           width="5.3839998"
+           clip-path="url(#SVGID_4756_)"
+           y="343.75699"
+           x="-660.64203"
+           style="fill:url(#SVGID_4757_)" />
+      </g>
+      <g
+         inkscape:transform-center-y="98.159342"
+         inkscape:transform-center-x="565.64033"
+         id="g83806"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83808">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4758_" />
+        </defs>
+        <clipPath
+           id="clipPath102867">
+          <use
+             id="use102869"
+             overflow="visible"
+             xlink:href="#SVGID_4758_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path83814"
+           d="m -658.582,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4759_)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83816"
+           height="9.592"
+           width="9.592"
+           clip-path="url(#SVGID_4759_)"
+           y="341.65302"
+           x="-650.77399"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83818"
+           height="9.592"
+           width="9.592"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4759_)"
+           y="341.65302"
+           x="-650.77399"
+           style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10" />
+        <rect
+           id="rect83820"
+           height="8.2469997"
+           width="8.2480001"
+           clip-path="url(#SVGID_4759_)"
+           y="342.32599"
+           x="-650.10303"
+           style="fill:#81d0db" />
+        <rect
+           id="rect83822"
+           height="6.9299998"
+           width="6.9299998"
+           clip-path="url(#SVGID_4759_)"
+           y="342.98401"
+           x="-649.44299"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83824"
+           height="4.5679998"
+           width="4.5700002"
+           clip-path="url(#SVGID_4759_)"
+           y="344.164"
+           x="-648.26398"
+           style="fill:#ffffff" />
+      </g>
+      <g
+         inkscape:transform-center-y="98.159295"
+         inkscape:transform-center-x="557.09937"
+         id="g83826"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83828">
+          <path
+             d="m -648.671,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.204,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+             id="SVGID_4760_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath102880">
+          <use
+             id="use102882"
+             overflow="visible"
+             xlink:href="#SVGID_4760_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3844,5.3844,0,153.234,-1031.1833)"
+           y2="-148.42999"
+           x2="256.3541"
+           y1="-148.42999"
+           x1="255.3541"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient102884">
+          <stop
+             id="stop102886"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop102888"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect83839"
+           height="5.3839998"
+           width="5.3839998"
+           clip-path="url(#SVGID_4761_)"
+           y="343.75699"
+           x="-648.67102"
+           style="fill:url(#SVGID_4762_)" />
+      </g>
+      <g
+         inkscape:transform-center-y="98.159342"
+         inkscape:transform-center-x="538.70558"
+         id="g83841"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83843">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4763_" />
+        </defs>
+        <clipPath
+           id="clipPath102894">
+          <use
+             id="use102896"
+             overflow="visible"
+             xlink:href="#SVGID_4763_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path83849"
+           d="m -646.611,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4764_)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83851"
+           height="9.592"
+           width="9.592"
+           clip-path="url(#SVGID_4764_)"
+           y="341.65302"
+           x="-638.80298"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83853"
+           height="9.592"
+           width="9.592"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4764_)"
+           y="341.65302"
+           x="-638.80298"
+           style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10" />
+        <rect
+           id="rect83855"
+           height="8.2469997"
+           width="8.2480001"
+           clip-path="url(#SVGID_4764_)"
+           y="342.32599"
+           x="-638.13202"
+           style="fill:#81d0db" />
+        <rect
+           id="rect83857"
+           height="6.9299998"
+           width="6.9299998"
+           clip-path="url(#SVGID_4764_)"
+           y="342.98401"
+           x="-637.47198"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83859"
+           height="4.5679998"
+           width="4.5700002"
+           clip-path="url(#SVGID_4764_)"
+           y="344.164"
+           x="-636.29199"
+           style="fill:#ffffff" />
+      </g>
+      <g
+         inkscape:transform-center-y="98.159295"
+         inkscape:transform-center-x="530.16231"
+         id="g83861"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83863">
+          <path
+             d="m -636.699,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.692,-1.205 2.692,-2.692 0,-1.486 -1.205,-2.692 -2.692,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+             id="SVGID_4765_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath102907">
+          <use
+             id="use102909"
+             overflow="visible"
+             xlink:href="#SVGID_4765_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3844,5.3844,0,165.2054,-1031.1833)"
+           y2="-148.4299"
+           x2="256.3541"
+           y1="-148.4299"
+           x1="255.3541"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient102911">
+          <stop
+             id="stop102913"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop102915"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect83874"
+           height="5.3839998"
+           width="5.3839998"
+           clip-path="url(#SVGID_4766_)"
+           y="343.75699"
+           x="-636.69897"
+           style="fill:url(#SVGID_4767_)" />
+      </g>
+      <g
+         inkscape:transform-center-y="98.159342"
+         inkscape:transform-center-x="511.76858"
+         id="g83876"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83878">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4768_" />
+        </defs>
+        <clipPath
+           id="clipPath102921">
+          <use
+             id="use102923"
+             overflow="visible"
+             xlink:href="#SVGID_4768_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path83884"
+           d="m -634.639,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4769_)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83886"
+           height="9.592"
+           width="9.592"
+           clip-path="url(#SVGID_4769_)"
+           y="341.65302"
+           x="-626.83099"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83888"
+           height="9.592"
+           width="9.592"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4769_)"
+           y="341.65302"
+           x="-626.83099"
+           style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10" />
+        <rect
+           id="rect83890"
+           height="8.2469997"
+           width="8.2480001"
+           clip-path="url(#SVGID_4769_)"
+           y="342.32599"
+           x="-626.15997"
+           style="fill:#81d0db" />
+        <rect
+           id="rect83892"
+           height="6.9299998"
+           width="6.9299998"
+           clip-path="url(#SVGID_4769_)"
+           y="342.98401"
+           x="-625.5"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83894"
+           height="4.5679998"
+           width="4.5700002"
+           clip-path="url(#SVGID_4769_)"
+           y="344.164"
+           x="-624.32098"
+           style="fill:#ffffff" />
+      </g>
+      <g
+         inkscape:transform-center-y="98.159295"
+         inkscape:transform-center-x="503.22762"
+         id="g83896"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83898">
+          <path
+             d="m -624.728,346.449 c 0,1.487 1.206,2.692 2.693,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.693,1.206 -2.693,2.692"
+             id="SVGID_4770_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath102934">
+          <use
+             id="use102936"
+             overflow="visible"
+             xlink:href="#SVGID_4770_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3844,5.3844,0,177.1768,-1031.1833)"
+           y2="-148.42999"
+           x2="256.3541"
+           y1="-148.42999"
+           x1="255.3541"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient102938">
+          <stop
+             id="stop102940"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop102942"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect83909"
+           height="5.3839998"
+           width="5.3839998"
+           clip-path="url(#SVGID_4771_)"
+           y="343.75699"
+           x="-624.72803"
+           style="fill:url(#SVGID_4772_)" />
+      </g>
+      <g
+         inkscape:transform-center-y="98.159342"
+         inkscape:transform-center-x="484.83387"
+         id="g83911"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83913">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4773_" />
+        </defs>
+        <clipPath
+           id="clipPath102948">
+          <use
+             id="use102950"
+             overflow="visible"
+             xlink:href="#SVGID_4773_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path83919"
+           d="m -622.668,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4774_)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83921"
+           height="9.592"
+           width="9.5930004"
+           clip-path="url(#SVGID_4774_)"
+           y="341.65302"
+           x="-614.85999"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83923"
+           height="9.592"
+           width="9.5930004"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4774_)"
+           y="341.65302"
+           x="-614.86102"
+           style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10" />
+        <rect
+           id="rect83925"
+           height="8.2469997"
+           width="8.2480001"
+           clip-path="url(#SVGID_4774_)"
+           y="342.32599"
+           x="-614.18799"
+           style="fill:#81d0db" />
+        <rect
+           id="rect83927"
+           height="6.9299998"
+           width="6.9310002"
+           clip-path="url(#SVGID_4774_)"
+           y="342.98401"
+           x="-613.52899"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83929"
+           height="4.5679998"
+           width="4.5700002"
+           clip-path="url(#SVGID_4774_)"
+           y="344.164"
+           x="-612.349"
+           style="fill:#ffffff" />
+      </g>
+      <g
+         inkscape:transform-center-y="98.159295"
+         inkscape:transform-center-x="476.2917"
+         id="g83931"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83933">
+          <path
+             d="m -612.756,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+             id="SVGID_4775_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath102961">
+          <use
+             id="use102963"
+             overflow="visible"
+             xlink:href="#SVGID_4775_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3844,5.3844,0,189.1483,-1031.1833)"
+           y2="-148.42999"
+           x2="256.3541"
+           y1="-148.42999"
+           x1="255.3541"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient102965">
+          <stop
+             id="stop102967"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop102969"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect83944"
+           height="5.3839998"
+           width="5.3829999"
+           clip-path="url(#SVGID_4776_)"
+           y="343.75699"
+           x="-612.75598"
+           style="fill:url(#SVGID_4777_)" />
+      </g>
+      <g
+         inkscape:transform-center-y="98.159342"
+         inkscape:transform-center-x="457.89686"
+         id="g83946"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83948">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4778_" />
+        </defs>
+        <clipPath
+           id="clipPath102975">
+          <use
+             id="use102977"
+             overflow="visible"
+             xlink:href="#SVGID_4778_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path83954"
+           d="m -610.696,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4779_)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83956"
+           height="9.592"
+           width="9.592"
+           clip-path="url(#SVGID_4779_)"
+           y="341.65302"
+           x="-602.888"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83958"
+           height="9.592"
+           width="9.592"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4779_)"
+           y="341.65302"
+           x="-602.888"
+           style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10" />
+        <rect
+           id="rect83960"
+           height="8.2469997"
+           width="8.2480001"
+           clip-path="url(#SVGID_4779_)"
+           y="342.32599"
+           x="-602.21698"
+           style="fill:#81d0db" />
+        <rect
+           id="rect83962"
+           height="6.9299998"
+           width="6.9299998"
+           clip-path="url(#SVGID_4779_)"
+           y="342.98401"
+           x="-601.55701"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83964"
+           height="4.5679998"
+           width="4.5700002"
+           clip-path="url(#SVGID_4779_)"
+           y="344.164"
+           x="-600.37799"
+           style="fill:#ffffff" />
+      </g>
+      <g
+         inkscape:transform-center-y="98.159295"
+         inkscape:transform-center-x="449.35694"
+         id="g83966"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83968">
+          <path
+             d="m -600.785,346.449 c 0,1.487 1.206,2.692 2.692,2.692 1.488,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.203,-2.692 -2.691,-2.692 -1.486,0 -2.692,1.206 -2.692,2.692"
+             id="SVGID_4780_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath102988">
+          <use
+             id="use102990"
+             overflow="visible"
+             xlink:href="#SVGID_4780_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3844,5.3844,0,201.1199,-1031.1833)"
+           y2="-148.4301"
+           x2="256.3541"
+           y1="-148.4301"
+           x1="255.3541"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient102992">
+          <stop
+             id="stop102994"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop102996"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect83979"
+           height="5.3839998"
+           width="5.3829999"
+           clip-path="url(#SVGID_4781_)"
+           y="343.75699"
+           x="-600.78497"
+           style="fill:url(#SVGID_4782_)" />
+      </g>
+      <g
+         inkscape:transform-center-y="98.159342"
+         inkscape:transform-center-x="430.96208"
+         id="g83981"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs83983">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4783_" />
+        </defs>
+        <clipPath
+           id="clipPath103002">
+          <use
+             id="use103004"
+             overflow="visible"
+             xlink:href="#SVGID_4783_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path83989"
+           d="m -598.725,346.449 c 0,-0.349 0.283,-0.632 0.632,-0.632 0.351,0 0.634,0.283 0.634,0.632 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+           clip-path="url(#SVGID_4784_)"
+           inkscape:connector-curvature="0"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83991"
+           height="9.592"
+           width="9.592"
+           clip-path="url(#SVGID_4784_)"
+           y="341.65302"
+           x="-590.91699"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83993"
+           height="9.592"
+           width="9.592"
+           stroke-miterlimit="10"
+           clip-path="url(#SVGID_4784_)"
+           y="341.65302"
+           x="-590.91699"
+           style="fill:none;stroke:#81d0db;stroke-width:0.428;stroke-miterlimit:10" />
+        <rect
+           id="rect83995"
+           height="8.2469997"
+           width="8.2480001"
+           clip-path="url(#SVGID_4784_)"
+           y="342.32599"
+           x="-590.24597"
+           style="fill:#81d0db" />
+        <rect
+           id="rect83997"
+           height="6.9299998"
+           width="6.9299998"
+           clip-path="url(#SVGID_4784_)"
+           y="342.98401"
+           x="-589.586"
+           style="fill:#ffffff" />
+        <rect
+           id="rect83999"
+           height="4.5679998"
+           width="4.5700002"
+           clip-path="url(#SVGID_4784_)"
+           y="344.164"
+           x="-588.40698"
+           style="fill:#ffffff" />
+      </g>
+      <g
+         inkscape:transform-center-y="98.159295"
+         inkscape:transform-center-x="422.41995"
+         id="g84001"
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)">
+        <defs
+           id="defs84003">
+          <path
+             d="m -588.813,346.449 c 0,1.487 1.205,2.692 2.692,2.692 1.487,0 2.691,-1.205 2.691,-2.692 0,-1.486 -1.204,-2.692 -2.691,-2.692 -1.487,0 -2.692,1.206 -2.692,2.692"
+             id="SVGID_4785_"
+             inkscape:connector-curvature="0" />
+        </defs>
+        <clipPath
+           id="clipPath103015">
+          <use
+             id="use103017"
+             overflow="visible"
+             xlink:href="#SVGID_4785_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <linearGradient
+           gradientTransform="matrix(0,5.3844,5.3844,0,213.0913,-1031.1833)"
+           y2="-148.42999"
+           x2="256.3541"
+           y1="-148.42999"
+           x1="255.3541"
+           gradientUnits="userSpaceOnUse"
+           id="linearGradient103019">
+          <stop
+             id="stop103021"
+             style="stop-color:#55AEB9"
+             offset="0" />
+          <stop
+             id="stop103023"
+             style="stop-color:#81D1DB"
+             offset="1" />
+        </linearGradient>
+        <rect
+           id="rect84014"
+           height="5.3839998"
+           width="5.3829999"
+           clip-path="url(#SVGID_4786_)"
+           y="343.75699"
+           x="-588.81299"
+           style="fill:url(#SVGID_4787_)" />
+      </g>
+      <polygon
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+         style="fill:#f15d57"
+         clip-path="url(#SVGID_4843_)"
+         points="-667.777,292.47 -671.896,292.47 -669.845,294.263 "
+         id="polygon84546" />
+      <polygon
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+         style="fill:#f15d57"
+         clip-path="url(#SVGID_4843_)"
+         points="-657.265,292.47 -661.384,292.47 -659.333,294.263 "
+         id="polygon84548" />
+      <polygon
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+         style="fill:#f15d57"
+         clip-path="url(#SVGID_4843_)"
+         points="-646.752,292.47 -650.871,292.47 -648.82,294.263 "
+         id="polygon84550" />
+      <polygon
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+         style="fill:#f15d57"
+         clip-path="url(#SVGID_4843_)"
+         points="-601.13,292.47 -605.249,292.47 -603.198,294.263 "
+         id="polygon84552" />
+      <polygon
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+         style="fill:#f15d57"
+         clip-path="url(#SVGID_4843_)"
+         points="-590.618,292.47 -594.737,292.47 -592.686,294.263 "
+         id="polygon84554" />
+      <polygon
+         transform="matrix(2.25,0,0,2.25,1605.7731,-25.560183)"
+         style="fill:#f15d57"
+         clip-path="url(#SVGID_4843_)"
+         points="-580.105,292.47 -584.224,292.47 -582.173,294.263 "
+         id="polygon84556" />
+      <g
+         inkscape:transform-center-y="141.90493"
+         inkscape:transform-center-x="262.77489"
+         id="g84792"
+         transform="matrix(1.3542118,0,0,2.25,996.72494,-31.560183)">
+        <defs
+           id="defs84794">
+          <rect
+             height="205.44099"
+             width="200.271"
+             y="181.315"
+             x="-682.31403"
+             id="SVGID_4872_" />
+        </defs>
+        <clipPath
+           id="clipPath103597">
+          <use
+             id="use103599"
+             overflow="visible"
+             xlink:href="#SVGID_4872_"
+             style="overflow:visible"
+             x="0"
+             y="0"
+             width="744.09448"
+             height="1052.3622" />
+        </clipPath>
+        <path
+           id="path84800"
+           d="m -438.652,188.287 c 0,0.349 -0.282,0.632 -0.632,0.632 -0.349,0 -0.632,-0.283 -0.632,-0.632 0,-0.35 0.283,-0.633 0.632,-0.633 0.35,0 0.632,0.284 0.632,0.633"
+           clip-path="url(#SVGID_4873_)"
+           inkscape:connector-curvature="0"
+           style="fill:#5e6a71" />
+        <g
+           id="g84802"
+           clip-path="url(#SVGID_4873_)">
+          <g
+             id="g84804">
+            <defs
+               id="defs84806">
+              <rect
+                 height="10.152"
+                 width="174.95799"
+                 y="360.13901"
+                 x="-679.90002"
+                 id="SVGID_4874_" />
+            </defs>
+            <clipPath
+               id="clipPath103606">
+              <use
+                 id="use103608"
+                 overflow="visible"
+                 xlink:href="#SVGID_4874_"
+                 style="overflow:visible"
+                 x="0"
+                 y="0"
+                 width="744.09448"
+                 height="1052.3622" />
+            </clipPath>
+            <g
+               id="g84812"
+               clip-path="url(#SVGID_4875_)">
+              <defs
+                 id="defs84814">
+                <rect
+                   height="10.152"
+                   width="174.95799"
+                   y="360.13901"
+                   x="-679.90002"
+                   id="SVGID_4876_" />
+              </defs>
+              <clipPath
+                 id="clipPath103613">
+                <use
+                   id="use103615"
+                   overflow="visible"
+                   xlink:href="#SVGID_4876_"
+                   style="overflow:visible"
+                   x="0"
+                   y="0"
+                   width="744.09448"
+                   height="1052.3622" />
+              </clipPath>
+              <linearGradient
+                 gradientTransform="matrix(0,-10.1526,-10.1526,0,900.78,782.1111)"
+                 y2="147.0757"
+                 x2="41.563"
+                 y1="147.0757"
+                 x1="40.563"
+                 gradientUnits="userSpaceOnUse"
+                 id="linearGradient103617">
+                <stop
+                   id="stop103619"
+                   style="stop-color:#55AEB9"
+                   offset="0" />
+                <stop
+                   id="stop103621"
+                   style="stop-color:#B0D2D9"
+                   offset="0.486" />
+                <stop
+                   id="stop103623"
+                   style="stop-color:#E6EFF1"
+                   offset="0.8287" />
+                <stop
+                   id="stop103625"
+                   style="stop-color:#FFFFFF"
+                   offset="0.9939" />
+                <stop
+                   id="stop103627"
+                   style="stop-color:#FFFFFF"
+                   offset="1" />
+              </linearGradient>
+              <rect
+                 id="rect84831"
+                 height="10.153"
+                 width="174.95799"
+                 clip-path="url(#SVGID_4877_)"
+                 y="360.138"
+                 x="-679.90002"
+                 style="fill:url(#SVGID_4878_)" />
+            </g>
+          </g>
+        </g>
+        <g
+           id="g84833"
+           clip-path="url(#SVGID_4873_)">
+          <g
+             id="g84835">
+            <defs
+               id="defs84837">
+              <rect
+                 height="10.152"
+                 width="174.94701"
+                 y="360.13901"
+                 x="-679.88898"
+                 id="SVGID_4879_" />
+            </defs>
+            <clipPath
+               id="clipPath103634">
+              <use
+                 id="use103636"
+                 overflow="visible"
+                 xlink:href="#SVGID_4879_"
+                 style="overflow:visible"
+                 x="0"
+                 y="0"
+                 width="744.09448"
+                 height="1052.3622" />
+            </clipPath>
+            <g
+               id="g84843"
+               clip-path="url(#SVGID_4880_)">
+              <defs
+                 id="defs84845">
+                <rect
+                   height="10.153"
+                   width="174.94701"
+                   y="360.138"
+                   x="-679.88898"
+                   id="SVGID_4881_" />
+              </defs>
+              <clipPath
+                 id="clipPath103641">
+                <use
+                   id="use103643"
+                   overflow="visible"
+                   xlink:href="#SVGID_4881_"
+                   style="overflow:visible"
+                   x="0"
+                   y="0"
+                   width="744.09448"
+                   height="1052.3622" />
+              </clipPath>
+              <polygon
+                 id="polygon84851"
+                 points="-679.89,364.81 -679.89,367.139 -679.89,370.291 -504.943,370.291 -504.943,360.138 -679.89,360.139 "
+                 clip-path="url(#SVGID_4882_)"
+                 style="fill:#f15d57" />
+            </g>
+          </g>
+        </g>
+        <rect
+           id="rect84853"
+           height="2.641"
+           width="174.96001"
+           clip-path="url(#SVGID_4873_)"
+           y="360.267"
+           x="-679.90198"
+           style="fill:#f15d57" />
+        <rect
+           id="rect84855"
+           height="1.352"
+           width="174.94701"
+           clip-path="url(#SVGID_4873_)"
+           y="370.29199"
+           x="-679.88898"
+           style="fill:#55aeb9" />
+      </g>
+      <text
+         inkscape:transform-center-y="45.543285"
+         inkscape:transform-center-x="450.06672"
+         id="text82351-9"
+         font-size="4.3343"
+         style="font-size:9.75217533px;fill:#ffffff;font-family:ApexSans-Medium"
+         x="255.23141"
+         y="705.37921">M</text>
+      <g
+         transform="translate(-230,34)"
+         id="g9504">
+        <g
+           transform="matrix(2.25,0,0,2.25,1701.0081,-85.193202)"
+           id="g82339-6"
+           inkscape:transform-center-x="449.81598"
+           inkscape:transform-center-y="45.666809">
+          <defs
+             id="defs82341-9">
+            <rect
+               id="SVGID_4549_-8"
+               x="-682.31403"
+               y="181.315"
+               width="200.271"
+               height="205.44099" />
+          </defs>
+          <clipPath
+             id="clipPath101738-3">
+            <use
+               height="1052.3622"
+               width="744.09448"
+               y="0"
+               x="0"
+               style="overflow:visible"
+               xlink:href="#SVGID_4549_-8"
+               overflow="visible"
+               id="use101740-3" />
+          </clipPath>
+          <path
+             style="fill:#f15d57"
+             inkscape:connector-curvature="0"
+             clip-path="url(#SVGID_4550_-6)"
+             d="m -602.031,326.467 c 0,0.212 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.173 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+             id="path82347-9" />
+          <path
+             style="fill:none;stroke:#ffffff;stroke-width:0.32100001;stroke-miterlimit:10"
+             inkscape:connector-curvature="0"
+             clip-path="url(#SVGID_4550_-6)"
+             stroke-miterlimit="10"
+             d="m -602.031,326.467 c 0,0.212 0.172,0.385 0.385,0.385 h 6.697 c 0.212,0 0.385,-0.173 0.385,-0.385 v -6.696 c 0,-0.213 -0.173,-0.385 -0.385,-0.385 h -6.697 c -0.213,0 -0.385,0.172 -0.385,0.385 v 6.696 z"
+             id="path82349-9" />
+        </g>
+        <text
+           y="645.37921"
+           x="350.6293"
+           style="font-size:9.75217533px;fill:#ffffff;font-family:ApexSans-Medium"
+           font-size="4.3343"
+           id="text82351-9-4"
+           inkscape:transform-center-x="450.06672"
+           inkscape:transform-center-y="45.543285">M</text>
+      </g>
+      <path
+         transform="matrix(2.25,0,0,2.25,1767.5425,-25.466067)"
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4564_-7)"
+         d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82449-2" />
+      <path
+         transform="matrix(2.25,0,0,2.25,1578.7139,-25.075587)"
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4564_-7-3)"
+         d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82449-2-2" />
+      <path
+         transform="matrix(2.25,0,0,2.25,1767.5425,0.59793262)"
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4564_-7-9)"
+         d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82449-2-0" />
+      <path
+         transform="matrix(2.25,0,0,2.25,1767.6425,26.597933)"
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4564_-7-9-3)"
+         d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82449-2-0-1" />
+      <path
+         transform="matrix(2.25,0,0,2.25,1767.7422,53.197933)"
+         style="fill:#ffffff"
+         inkscape:connector-curvature="0"
+         clip-path="url(#SVGID_4564_-7-9-3-4)"
+         d="m -658.582,311.532 c 0,-0.349 0.283,-0.631 0.632,-0.631 0.351,0 0.634,0.282 0.634,0.631 0,0.35 -0.283,0.633 -0.634,0.633 -0.349,0 -0.632,-0.283 -0.632,-0.633"
+         id="path82449-2-0-1-5" />
+    </g>
+    <text
+       y="293.47144"
+       x="168.09497"
+       id="text84343-2-0-4"
+       inkscape:transform-center-x="587.62742"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px;text-align:center;text-anchor:middle">
+      <tspan
+         style="font-size:11px;text-align:center;text-anchor:middle;fill:#37424b;font-family:ApexSans-Book"
+         x="123.1228"
+         y="293.47144"
+         font-size="5.1244"
+         id="tspan84345-2-6-5">READ ONLY</tspan>
+    </text>
+    <text
+       y="292.96655"
+       x="321.78174"
+       id="text84343-2-0-4-3"
+       inkscape:transform-center-x="587.62742"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px;text-align:center;text-anchor:middle">
+      <tspan
+         style="font-size:11px;text-align:center;text-anchor:middle;fill:#37424b;font-family:ApexSans-Book"
+         x="276.80957"
+         y="292.96655"
+         font-size="5.1244"
+         id="tspan84345-2-6-5-3">WRITE / READ</tspan>
+    </text>
+    <text
+       y="262.38412"
+       x="242.19797"
+       id="text84343-2-0-4-6"
+       inkscape:transform-center-x="587.62742"
+       inkscape:transform-center-y="-134.7035"
+       style="font-size:27px;text-align:center;text-anchor:middle">
+      <tspan
+         style="font-size:11px;text-align:center;text-anchor:middle;fill:#37424b;font-family:ApexSans-Book"
+         x="197.2258"
+         y="262.38412"
+         font-size="5.1244"
+         id="tspan84345-2-6-5-6">(United States)</tspan>
+    </text>
+  </g>
+</svg>
diff --git a/doc/images/zone-sync2.png b/doc/images/zone-sync2.png
new file mode 100644
index 000000000..67963dcd1
--- /dev/null
+++ b/doc/images/zone-sync2.png
diff --git a/doc/index.rst b/doc/index.rst
new file mode 100644
index 000000000..b339e4e87
--- /dev/null
+++ b/doc/index.rst
@@ -0,0 +1,121 @@
+=================
+ Welcome to Ceph
+=================
+
+Ceph uniquely delivers **object, block, and file storage in one unified
+system**.
+
+.. warning:: 
+
+   :ref:`If this is your first time using Ceph, read the "Basic Workflow" 
+   page in the Ceph Developer Guide to learn how to contribute to the 
+   Ceph project. (Click anywhere in this paragraph to read the "Basic 
+   Workflow" page of the Ceph Developer Guide.) <basic workflow dev guide>`.
+
+.. raw:: html
+
+      <style type="text/css">div.body h3{margin:5px 0px 0px 0px;}</style>
+      <table cellpadding="10"><colgroup><col width="33%"><col width="33%">
+      <col width="33%"></colgroup><tbody valign="top"><tr><td><h3>Ceph Object Store</h3>
+
+- RESTful Interface
+- S3- and Swift-compliant APIs
+- S3-style subdomains
+- Unified S3/Swift namespace
+- User management
+- Usage tracking
+- Striped objects
+- Cloud solution integration
+- Multi-site deployment
+- Multi-site replication
+
+.. raw:: html
+
+	</td><td><h3>Ceph Block Device</h3>
+
+
+- Thin-provisioned
+- Images up to 16 exabytes
+- Configurable striping
+- In-memory caching
+- Snapshots
+- Copy-on-write cloning
+- Kernel driver support
+- KVM/libvirt support
+- Back-end for cloud solutions
+- Incremental backup
+- Disaster recovery (multisite asynchronous replication)
+
+.. raw:: html
+
+	</td><td><h3>Ceph File System</h3>
+
+- POSIX-compliant semantics
+- Separates metadata from data
+- Dynamic rebalancing
+- Subdirectory snapshots
+- Configurable striping
+- Kernel driver support
+- FUSE support
+- NFS/CIFS deployable
+- Use with Hadoop (replace HDFS)
+
+.. raw:: html
+
+	</td></tr><tr><td>
+
+See `Ceph Object Store`_ for additional details.
+
+.. raw:: html
+
+	</td><td>
+
+See `Ceph Block Device`_ for additional details.
+
+.. raw:: html
+
+	</td><td>
+
+See `Ceph File System`_ for additional details.
+
+.. raw::	html
+
+	</td></tr></tbody></table>
+
+Ceph is highly reliable, easy to manage, and free. The power of Ceph
+can transform your company's IT infrastructure and your ability to manage vast
+amounts of data. To try Ceph, see our `Getting Started`_ guides. To learn more
+about Ceph, see our `Architecture`_ section.
+
+
+
+.. _Ceph Object Store: radosgw
+.. _Ceph Block Device: rbd
+.. _Ceph File System: cephfs
+.. _Getting Started: install
+.. _Architecture: architecture
+
+.. toctree::
+   :maxdepth: 3
+   :hidden:
+
+   start/intro
+   install/index
+   cephadm/index
+   rados/index
+   cephfs/index
+   rbd/index
+   radosgw/index
+   mgr/index
+   mgr/dashboard
+   api/index
+   architecture
+   Developer Guide <dev/developer_guide/index>
+   dev/internals
+   governance
+   foundation
+   ceph-volume/index
+   Ceph Releases (general) <https://docs.ceph.com/en/latest/releases/general/>
+   Ceph Releases (index) <https://docs.ceph.com/en/latest/releases/>
+   security/index
+   Glossary <glossary>
diff --git a/doc/install/build-ceph.rst b/doc/install/build-ceph.rst
new file mode 100644
index 000000000..5c93ca52d
--- /dev/null
+++ b/doc/install/build-ceph.rst
@@ -0,0 +1,115 @@
+============
+ Build Ceph
+============
+
+You can get Ceph software by retrieving Ceph source code and building it yourself.
+To build Ceph, you need to set up a development environment, compile Ceph,
+and then either install in user space or build packages and install the packages.
+
+Build Prerequisites
+===================
+
+
+.. tip:: Check this section to see if there are specific prerequisites for your
+   Linux/Unix distribution.
+
+A debug build of Ceph may take around 40 gigabytes. If you want to build Ceph in
+a virtual machine (VM) please make sure total disk space on the VM is at least
+60 gigabytes.
+
+Please also be aware that some distributions of Linux, like CentOS, use Linux
+Volume Manager (LVM) for the default installation. LVM may reserve a large
+portion of disk space of a typical sized virtual disk for the operating system.
+
+Before you can build Ceph source code, you need to install several libraries
+and tools::
+
+	./install-deps.sh
+
+.. note:: Some distributions that support Google's memory profiler tool may use
+   a different package name (e.g., ``libgoogle-perftools4``).
+
+Build Ceph
+==========
+
+Ceph is built using cmake. To build Ceph, navigate to your cloned Ceph
+repository and execute the following::
+
+    cd ceph
+    ./do_cmake.sh
+    cd build
+    make
+
+.. note:: By default do_cmake.sh will build a debug version of ceph that may
+   perform up to 5 times slower with certain workloads. Pass 
+   '-DCMAKE_BUILD_TYPE=RelWithDebInfo' to do_cmake.sh if you would like to
+   build a release version of the ceph executables instead.
+
+.. topic:: Hyperthreading
+
+	You can use ``make -j`` to execute multiple jobs depending upon your system. For 
+	example, ``make -j4`` for a dual core processor may build faster.
+
+See `Installing a Build`_ to install a build in user space.
+
+Build Ceph Packages
+===================
+
+To build packages, you must clone the `Ceph`_ repository. You can create 
+installation packages from the latest code using ``dpkg-buildpackage`` for 
+Debian/Ubuntu or ``rpmbuild`` for the RPM Package Manager.
+
+.. tip:: When building on a multi-core CPU, use the ``-j`` and the number of 
+   cores * 2. For example, use ``-j4`` for a dual-core processor to accelerate 
+   the build.
+
+
+Advanced Package Tool (APT)
+---------------------------
+
+To create ``.deb`` packages for Debian/Ubuntu, ensure that you have cloned the 
+`Ceph`_ repository, installed the `Build Prerequisites`_ and installed 
+``debhelper``::
+
+	sudo apt-get install debhelper
+
+Once you have installed debhelper, you can build the packages::
+
+	sudo dpkg-buildpackage
+
+For multi-processor CPUs use the ``-j`` option to accelerate the build.
+
+
+RPM Package Manager
+-------------------
+
+To create ``.rpm`` packages, ensure that you have cloned the `Ceph`_ repository,
+installed the `Build Prerequisites`_ and installed ``rpm-build`` and 
+``rpmdevtools``::
+
+	yum install rpm-build rpmdevtools
+
+Once you have installed the tools, setup an RPM compilation environment::
+
+	rpmdev-setuptree
+
+Fetch the source tarball for the RPM compilation environment::
+
+	wget -P ~/rpmbuild/SOURCES/ https://download.ceph.com/tarballs/ceph-<version>.tar.bz2
+
+Or from the EU mirror::
+
+	wget -P ~/rpmbuild/SOURCES/ http://eu.ceph.com/tarballs/ceph-<version>.tar.bz2
+
+Extract the specfile::
+
+    tar --strip-components=1 -C ~/rpmbuild/SPECS/ --no-anchored -xvjf ~/rpmbuild/SOURCES/ceph-<version>.tar.bz2 "ceph.spec"
+
+Build the RPM packages::
+
+	rpmbuild -ba ~/rpmbuild/SPECS/ceph.spec
+
+For multi-processor CPUs use the ``-j`` option to accelerate the build.
+
+.. _Ceph: ../clone-source
+.. _Installing a Build: ../install-storage-cluster#installing-a-build
diff --git a/doc/install/clone-source.rst b/doc/install/clone-source.rst
new file mode 100644
index 000000000..2d09ef9eb
--- /dev/null
+++ b/doc/install/clone-source.rst
@@ -0,0 +1,195 @@
+=========================================
+ Cloning the Ceph Source Code Repository
+=========================================
+
+To clone a Ceph branch of the Ceph source code, go to `github Ceph
+Repository`_, select a branch (``main`` by default), and click the **Download
+ZIP** button.
+
+.. _github Ceph Repository: https://github.com/ceph/ceph
+
+To clone the entire git repository, :ref:`install <install-git>` and configure
+``git``.
+
+.. _install-git:
+
+Install Git
+===========
+
+To install ``git`` on Debian/Ubuntu, run the following command:
+
+.. prompt:: bash $
+
+   sudo apt-get install git
+
+
+To install ``git`` on CentOS/RHEL, run the following command:
+
+.. prompt:: bash $
+
+   sudo yum install git
+
+
+You must have a ``github`` account. If you do not have a ``github``
+account, go to `github.com`_ and register.  Follow the directions for setting
+up git at `Set Up Git`_.
+
+.. _github.com: https://github.com
+.. _Set Up Git: https://help.github.com/linux-set-up-git
+
+
+Add SSH Keys (Optional)
+=======================
+
+To commit code to Ceph or to clone the respository by using SSH
+(``git@github.com:ceph/ceph.git``), you must generate SSH keys for github. 
+
+.. tip:: If you want only to clone the repository, you can 
+   use ``git clone --recursive https://github.com/ceph/ceph.git`` 
+   without generating SSH keys.
+
+To generate SSH keys for ``github``, run the following command:
+
+.. prompt:: bash $
+
+   ssh-keygen
+
+To print the SSH key that you just generated and that you will add to your
+``github`` account, use the ``cat`` command. (The following example assumes you
+used the default file path.):
+
+.. prompt:: bash $
+
+   cat .ssh/id_rsa.pub
+
+Copy the public key.
+
+Go to your ``github`` account, click "Account Settings" (represented by the
+'tools' icon), and click "SSH Keys" on the left side navbar.
+
+Click "Add SSH key" in the "SSH Keys" list, enter a name for the key, paste the
+key you generated, and press the "Add key" button.
+
+
+Clone the Source
+================
+
+To clone the Ceph source code repository, run the following command:
+
+.. prompt:: bash $
+
+   git clone --recursive https://github.com/ceph/ceph.git
+
+After ``git clone`` has run, you should have a full copy of the Ceph 
+repository.
+
+.. tip:: Make sure you maintain the latest copies of the submodules included in
+   the repository. Running ``git status`` will tell you whether the submodules
+   are out of date. See :ref:`update-submodules` for more information.
+
+
+.. prompt:: bash $
+
+   cd ceph
+   git status
+
+.. _update-submodules:
+
+Updating Submodules
+-------------------
+
+#. Determine whether your submodules are out of date:
+
+   .. prompt:: bash $
+
+      git status
+
+   A. If your submodules are up to date 
+         If your submodules are up to date, the following console output will
+         appear: 
+
+         ::
+   
+           On branch main
+           Your branch is up to date with 'origin/main'.
+           
+           nothing to commit, working tree clean
+   
+         If you see this console output, then your submodules are up to date.
+         You do not need this procedure.
+
+
+   B. If your submodules are not up to date 
+         If your submodules are not up to date, you will see a message that
+         includes a list of "untracked files". The example here shows such a
+         list, which was generated from a real situation in which the
+         submodules were no longer current. Your list of files will not be the
+         same as this list of files, but this list is provided as an example.
+         If in your case any untracked files are listed, then you should
+         continue to the next step of this procedure.
+
+         ::
+
+            On branch main
+            Your branch is up to date with 'origin/main'.
+            
+            Untracked files:
+              (use "git add <file>..." to include in what will be committed)
+            src/pybind/cephfs/build/
+            src/pybind/cephfs/cephfs.c
+            src/pybind/cephfs/cephfs.egg-info/
+            src/pybind/rados/build/
+            src/pybind/rados/rados.c
+            src/pybind/rados/rados.egg-info/
+            src/pybind/rbd/build/
+            src/pybind/rbd/rbd.c
+            src/pybind/rbd/rbd.egg-info/
+            src/pybind/rgw/build/
+            src/pybind/rgw/rgw.c
+            src/pybind/rgw/rgw.egg-info/
+            
+            nothing added to commit but untracked files present (use "git add" to track)
+
+#. If your submodules are out of date, run the following commands:
+
+   .. prompt:: bash $
+
+      git submodule update --force --init --recursive
+      git clean -fdx
+      git submodule foreach git clean -fdx
+
+   If you still have problems with a submodule directory, use ``rm -rf
+   [directory name]`` to remove the directory. Then run ``git submodule update
+   --init --recursive`` again.
+
+#. Run ``git status`` again:
+
+   .. prompt:: bash $
+
+      git status
+   
+   Your submodules are up to date if you see the following message:
+
+   ::
+
+     On branch main
+     Your branch is up to date with 'origin/main'.
+     
+     nothing to commit, working tree clean
+
+Choose a Branch
+===============
+
+Once you clone the source code and submodules, your Ceph repository 
+will be on the ``main`` branch by default, which is the unstable 
+development branch. You may choose other branches too.
+
+- ``main``: The unstable development branch.
+- ``stable-release-name``: The name of the stable, `Active Releases`_. e.g. ``Pacific``
+- ``next``: The release candidate branch.
+
+::
+
+	git checkout main
+
+.. _Active Releases: https://docs.ceph.com/en/latest/releases/#active-releases
diff --git a/doc/install/containers.rst b/doc/install/containers.rst
new file mode 100644
index 000000000..49c976199
--- /dev/null
+++ b/doc/install/containers.rst
@@ -0,0 +1,113 @@
+.. _containers:
+
+Ceph Container Images
+=====================
+
+.. important::
+
+   Using the ``:latest`` tag is discouraged. If you use the ``:latest`` 
+   tag, there is no guarantee that the same image will be on each of 
+   your hosts.  Under these conditions, upgrades might not work 
+   properly. Remember that ``:latest`` is a relative tag, and a moving
+   target.
+
+   Instead of the ``:latest`` tag, use explicit tags or image IDs. For
+   example:
+
+   ``podman pull ceph/ceph:v15.2.0``
+
+Official Releases
+-----------------
+
+Ceph Container images are available from Quay:
+
+  https://quay.io/repository/ceph/ceph
+  https://hub.docker.com/r/ceph
+
+ceph/ceph
+^^^^^^^^^
+
+- General purpose Ceph container with all necessary daemons and
+  dependencies installed.
+
++----------------------+--------------------------------------------------------------+
+| Tag                  | Meaning                                                      |
++----------------------+--------------------------------------------------------------+
+| vRELNUM              | Latest release in this series (e.g., *v14* = Nautilus)       |
++----------------------+--------------------------------------------------------------+
+| vRELNUM.2            | Latest *stable* release in this stable series (e.g., *v14.2*)|
++----------------------+--------------------------------------------------------------+
+| vRELNUM.Y.Z          | A specific release (e.g., *v14.2.4*)                         |
++----------------------+--------------------------------------------------------------+
+| vRELNUM.Y.Z-YYYYMMDD | A specific build (e.g., *v14.2.4-20191203*)                  |
++----------------------+--------------------------------------------------------------+
+
+Legacy container images
+-----------------------
+
+Legacy container images are available from Docker Hub at::
+  
+  https://hub.docker.com/r/ceph
+
+ceph/daemon-base
+^^^^^^^^^^^^^^^^
+
+- General purpose Ceph container with all necessary daemons and
+  dependencies installed.
+- Basically the same as *ceph/ceph*, but with different tags.
+- Note that all of the *-devel* tags (and the *latest-master* tag) are based on
+  unreleased and generally untested packages from https://shaman.ceph.com.
+
+:note: This image will soon become an alias to *ceph/ceph*.
+
++------------------------+---------------------------------------------------------+
+| Tag                    | Meaning                                                 |
++------------------------+---------------------------------------------------------+
+| latest-master          | Build of master branch a last ceph-container.git update |
++------------------------+---------------------------------------------------------+
+| latest-master-devel    | Daily build of the master branch                        |
++------------------------+---------------------------------------------------------+
+| latest-RELEASE-devel   | Daily build of the *RELEASE* (e.g., nautilus) branch    |
++------------------------+---------------------------------------------------------+
+
+
+ceph/daemon
+^^^^^^^^^^^
+
+- *ceph/daemon-base* plus a collection of BASH scripts that are used
+  by ceph-nano and ceph-ansible to manage a Ceph cluster.
+
++------------------------+---------------------------------------------------------+
+| Tag                    | Meaning                                                 |
++------------------------+---------------------------------------------------------+
+| latest-master          | Build of master branch a last ceph-container.git update |
++------------------------+---------------------------------------------------------+
+| latest-master-devel    | Daily build of the master branch                        |
++------------------------+---------------------------------------------------------+
+| latest-RELEASE-devel   | Daily build of the *RELEASE* (e.g., nautilus) branch    |
++------------------------+---------------------------------------------------------+
+
+
+Development builds
+------------------
+
+We automatically build container images for development ``wip-*``
+branches in the ceph-ci.git repositories and push them to Quay at::
+
+  https://quay.io/organization/ceph-ci
+
+ceph-ci/ceph
+^^^^^^^^^^^^
+
+- This is analogous to the ceph/ceph image above
+- TODO: remove the ``wip-*`` limitation and also build ceph.git branches.
+
++------------------------------------+------------------------------------------------------+
+| Tag                                | Meaning                                              |
++------------------------------------+------------------------------------------------------+
+| BRANCH                             | Latest build of a given GIT branch (e.g., *wip-foo*) |
++------------------------------------+------------------------------------------------------+
+| BRANCH-SHORTSHA1-BASEOS-ARCH-devel | A specific build of a branch                         |
++------------------------------------+------------------------------------------------------+
+| SHA1                               | A specific build                                     |
++------------------------------------+------------------------------------------------------+
diff --git a/doc/install/get-packages.rst b/doc/install/get-packages.rst
new file mode 100644
index 000000000..261815187
--- /dev/null
+++ b/doc/install/get-packages.rst
@@ -0,0 +1,402 @@
+.. _packages:
+
+==============
+ Get Packages
+==============
+
+To install Ceph and other enabling software, you need to retrieve packages from
+the Ceph repository. 
+
+There are three ways to get packages:
+
+- **Cephadm:** Cephadm can configure your Ceph repositories for you
+  based on a release name or a specific Ceph version.  Each
+  :term:`Ceph Node` in your cluster must have internet access.
+
+- **Configure Repositories Manually:** You can manually configure your
+  package management tool to retrieve Ceph packages and all enabling
+  software.  Each :term:`Ceph Node` in your cluster must have internet
+  access.
+
+- **Download Packages Manually:** Downloading packages manually is a convenient
+  way to install Ceph if your environment does not allow a :term:`Ceph Node` to
+  access the internet.
+
+Install packages with cephadm
+=============================
+
+#. Download the cephadm script
+
+.. prompt:: bash $
+   :substitutions:
+
+   curl --silent --remote-name --location https://github.com/ceph/ceph/raw/|stable-release|/src/cephadm/cephadm
+   chmod +x cephadm
+
+#. Configure the Ceph repository based on the release name::
+
+     ./cephadm add-repo --release nautilus
+
+   For Octopus (15.2.0) and later releases, you can also specify a specific
+   version::
+
+     ./cephadm add-repo --version 15.2.1
+
+   For development packages, you can specify a specific branch name::
+
+     ./cephadm add-repo --dev my-branch
+
+#. Install the appropriate packages.  You can install them using your
+   package management tool (e.g., APT, Yum) directly, or you can also
+   use the cephadm wrapper.  For example::
+
+     ./cephadm install ceph-common
+   
+     
+Configure Repositories Manually
+===============================
+
+All Ceph deployments require Ceph packages (except for development). You should
+also add keys and recommended packages.
+
+- **Keys: (Recommended)** Whether you add repositories or download packages
+  manually, you should download keys to verify the packages. If you do not get
+  the keys, you may encounter security warnings.
+
+- **Ceph: (Required)** All Ceph deployments require Ceph release packages,
+  except for deployments that use development packages (development, QA, and
+  bleeding edge deployments only).
+
+- **Ceph Development: (Optional)** If you are developing for Ceph, testing Ceph
+  development builds, or if you want features from the bleeding edge of Ceph
+  development, you may get Ceph development packages.
+
+
+
+Add Keys
+--------
+
+Add a key to your system's list of trusted keys to avoid a security warning. For
+major releases (e.g., ``luminous``, ``mimic``, ``nautilus``) and development releases
+(``release-name-rc1``, ``release-name-rc2``), use the ``release.asc`` key.
+
+
+APT
+~~~
+
+To install the ``release.asc`` key, execute the following::
+
+	wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add -
+
+
+RPM
+~~~
+
+To install the ``release.asc`` key, execute the following::
+
+	sudo rpm --import 'https://download.ceph.com/keys/release.asc'
+
+Ceph Release Packages
+---------------------
+
+Release repositories use the ``release.asc`` key to verify packages.
+To install Ceph packages with the Advanced Package Tool (APT) or
+Yellowdog Updater, Modified (YUM), you must add Ceph repositories.
+
+You may find releases for Debian/Ubuntu (installed with APT) at::
+
+	https://download.ceph.com/debian-{release-name}
+
+You may find releases for CentOS/RHEL and others (installed with YUM) at::
+
+	https://download.ceph.com/rpm-{release-name}
+
+For Octopus and later releases, you can also configure a repository for a
+specific version ``x.y.z``.  For Debian/Ubuntu packages::
+
+  https://download.ceph.com/debian-{version}
+
+For RPMs::
+
+  https://download.ceph.com/rpm-{version}
+
+The major releases of Ceph are summarized at: `Releases`_
+
+.. tip:: For non-US users: There might be a mirror close to you where
+         to download Ceph from. For more information see: `Ceph Mirrors`_.
+
+Debian Packages
+~~~~~~~~~~~~~~~
+
+Add a Ceph package repository to your system's list of APT sources. For newer
+versions of Debian/Ubuntu, call ``lsb_release -sc`` on the command line to
+get the short codename, and replace ``{codename}`` in the following command.
+
+.. prompt:: bash $
+   :substitutions:
+
+   sudo apt-add-repository 'deb https://download.ceph.com/debian-|stable-release|/ {codename} main'
+
+For early Linux distributions, you may execute the following command
+
+.. prompt:: bash $
+   :substitutions:
+
+   echo deb https://download.ceph.com/debian-|stable-release|/ $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list
+
+For earlier Ceph releases, replace ``{release-name}`` with the name  with the
+name of the Ceph release. You may call ``lsb_release -sc`` on the command  line
+to get the short codename, and replace ``{codename}`` in the following command.
+
+.. prompt:: bash $
+
+   sudo apt-add-repository 'deb https://download.ceph.com/debian-{release-name}/ {codename} main'
+
+For older Linux distributions, replace ``{release-name}`` with the name of the
+release
+
+.. prompt:: bash $
+
+	echo deb https://download.ceph.com/debian-{release-name}/ $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list
+
+For development release packages, add our package repository to your system's
+list of APT sources.  See `the testing Debian repository`_ for a complete list
+of Debian and Ubuntu releases supported.
+
+.. prompt:: bash $
+
+   echo deb https://download.ceph.com/debian-testing/ $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list
+
+.. tip:: For non-US users: There might be a mirror close to you where
+         to download Ceph from. For more information see: `Ceph Mirrors`_.
+
+
+RPM Packages
+~~~~~~~~~~~~
+
+RHEL
+^^^^
+
+For major releases, you may add a Ceph entry to the ``/etc/yum.repos.d``
+directory. Create a ``ceph.repo`` file. In the example below, replace
+``{ceph-release}`` with  a major release of Ceph (e.g., ``|stable-release|``)
+and ``{distro}`` with your Linux distribution (e.g., ``el8``, etc.).  You
+may view https://download.ceph.com/rpm-{ceph-release}/ directory to see which
+distributions Ceph supports. Some Ceph packages (e.g., EPEL) must take priority
+over standard packages, so you must ensure that you set
+``priority=2``.
+
+.. code-block:: ini
+
+	[ceph]
+	name=Ceph packages for $basearch
+	baseurl=https://download.ceph.com/rpm-{ceph-release}/{distro}/$basearch
+	enabled=1
+	priority=2
+	gpgcheck=1
+	gpgkey=https://download.ceph.com/keys/release.asc
+
+	[ceph-noarch]
+	name=Ceph noarch packages
+	baseurl=https://download.ceph.com/rpm-{ceph-release}/{distro}/noarch
+	enabled=1
+	priority=2
+	gpgcheck=1
+	gpgkey=https://download.ceph.com/keys/release.asc
+
+	[ceph-source]
+	name=Ceph source packages
+	baseurl=https://download.ceph.com/rpm-{ceph-release}/{distro}/SRPMS
+	enabled=0
+	priority=2
+	gpgcheck=1
+	gpgkey=https://download.ceph.com/keys/release.asc
+
+
+For specific packages, you may retrieve them by downloading the release package
+by name. Our development process generates a new release of Ceph every 3-4
+weeks. These packages are faster-moving than the major releases.  Development
+packages have new features integrated quickly, while still undergoing several
+weeks of QA prior to release.
+
+The repository package installs the repository details on your local system for
+use with ``yum``. Replace ``{distro}`` with your Linux distribution, and
+``{release}`` with the specific release of Ceph
+
+.. prompt:: bash $
+
+    su -c 'rpm -Uvh https://download.ceph.com/rpms/{distro}/x86_64/ceph-{release}.el7.noarch.rpm'
+
+You can download the RPMs directly from
+
+.. code-block:: none
+
+   https://download.ceph.com/rpm-testing
+
+.. tip:: For non-US users: There might be a mirror close to you where
+         to download Ceph from. For more information see: `Ceph Mirrors`_.
+
+openSUSE Leap 15.1
+^^^^^^^^^^^^^^^^^^
+
+You need to add the Ceph package repository to your list of zypper sources. This can be done with the following command
+
+.. code-block:: bash
+
+    zypper ar https://download.opensuse.org/repositories/filesystems:/ceph/openSUSE_Leap_15.1/filesystems:ceph.repo
+
+openSUSE Tumbleweed
+^^^^^^^^^^^^^^^^^^^
+
+The newest major release of Ceph is already available through the normal Tumbleweed repositories.
+There's no need to add another package repository manually.
+
+
+Ceph Development Packages
+-------------------------
+
+If you are developing Ceph and need to deploy and test specific Ceph branches,
+ensure that you remove repository entries for major releases first.
+
+
+DEB Packages
+~~~~~~~~~~~~
+
+We automatically build Ubuntu packages for current development branches in the
+Ceph source code repository.  These packages are intended for developers and QA
+only.
+
+Add the package repository to your system's list of APT sources, but
+replace ``{BRANCH}`` with the branch you'd like to use (e.g.,
+wip-hack, master).  See `the shaman page`_ for a complete
+list of distributions we build.
+
+.. prompt:: bash $
+
+    curl -L https://shaman.ceph.com/api/repos/ceph/{BRANCH}/latest/ubuntu/$(lsb_release -sc)/repo/ | sudo tee /etc/apt/sources.list.d/shaman.list
+
+.. note:: If the repository is not ready an HTTP 504 will be returned
+
+The use of ``latest`` in the url, means it will figure out which is the last
+commit that has been built. Alternatively, a specific sha1 can be specified.
+For Ubuntu Xenial and the master branch of Ceph, it would look like
+
+.. prompt:: bash $
+
+    curl -L https://shaman.ceph.com/api/repos/ceph/master/53e772a45fdf2d211c0c383106a66e1feedec8fd/ubuntu/xenial/repo/ | sudo tee /etc/apt/sources.list.d/shaman.list
+
+
+.. warning:: Development repositories are no longer available after two weeks.
+
+RPM Packages
+~~~~~~~~~~~~
+
+For current development branches, you may add a Ceph entry to the
+``/etc/yum.repos.d`` directory. The `the shaman page`_ can be used to retrieve the full details
+of a repo file. It can be retrieved via an HTTP request, for example
+
+.. prompt:: bash $
+
+    curl -L https://shaman.ceph.com/api/repos/ceph/{BRANCH}/latest/centos/7/repo/ | sudo tee /etc/yum.repos.d/shaman.repo
+
+The use of ``latest`` in the url, means it will figure out which is the last
+commit that has been built. Alternatively, a specific sha1 can be specified.
+For CentOS 7 and the master branch of Ceph, it would look like
+
+.. prompt:: bash $
+
+    curl -L https://shaman.ceph.com/api/repos/ceph/master/53e772a45fdf2d211c0c383106a66e1feedec8fd/centos/7/repo/ | sudo tee /etc/apt/sources.list.d/shaman.list
+
+
+.. warning:: Development repositories are no longer available after two weeks.
+
+.. note:: If the repository is not ready an HTTP 504 will be returned
+
+Download Packages Manually
+--------------------------
+
+If you are attempting to install behind a firewall in an environment without internet
+access, you must retrieve the packages (mirrored with all the necessary dependencies)
+before attempting an install.
+
+Debian Packages
+~~~~~~~~~~~~~~~
+
+Ceph requires additional third party libraries.
+
+- libaio1
+- libsnappy1
+- libcurl3
+- curl
+- libgoogle-perftools4
+- google-perftools
+- libleveldb1
+
+
+The repository package installs the repository details on your local system for
+use with ``apt``. Replace ``{release}`` with the latest Ceph release. Replace
+``{version}`` with the latest Ceph version number. Replace ``{distro}`` with
+your Linux distribution codename. Replace ``{arch}`` with the CPU architecture.
+
+.. prompt:: bash $
+
+	wget -q https://download.ceph.com/debian-{release}/pool/main/c/ceph/ceph_{version}{distro}_{arch}.deb
+
+
+RPM Packages
+~~~~~~~~~~~~
+
+Ceph requires additional third party libraries.
+To add the EPEL repository, execute the following
+
+.. prompt:: bash $
+
+   sudo yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+
+Ceph requires the following packages:
+
+- snappy
+- leveldb
+- gdisk
+- python-argparse
+- gperftools-libs
+
+
+Packages are currently built for the RHEL/CentOS7 (``el7``) platforms.  The
+repository package installs the repository details on your local system for use
+with ``yum``. Replace ``{distro}`` with your distribution.
+
+.. prompt:: bash $
+   :substitutions:
+
+   su -c 'rpm -Uvh https://download.ceph.com/rpm-|stable-release|/{distro}/noarch/ceph-{version}.{distro}.noarch.rpm'
+
+For example, for CentOS 8  (``el8``)
+
+.. prompt:: bash $
+   :substitutions:
+
+   su -c 'rpm -Uvh https://download.ceph.com/rpm-|stable-release|/el8/noarch/ceph-release-1-0.el8.noarch.rpm'
+
+You can download the RPMs directly from
+
+.. code-block:: none
+   :substitutions:
+
+   https://download.ceph.com/rpm-|stable-release|
+
+
+For earlier Ceph releases, replace ``{release-name}`` with the name
+with the name of the Ceph release. You may call ``lsb_release -sc`` on the command
+line to get the short codename.
+
+.. prompt:: bash $
+
+	su -c 'rpm -Uvh https://download.ceph.com/rpm-{release-name}/{distro}/noarch/ceph-{version}.{distro}.noarch.rpm'
+
+
+
+.. _Releases: https://docs.ceph.com/en/latest/releases/
+.. _the testing Debian repository: https://download.ceph.com/debian-testing/dists
+.. _the shaman page: https://shaman.ceph.com
+.. _Ceph Mirrors: ../mirrors
diff --git a/doc/install/get-tarballs.rst b/doc/install/get-tarballs.rst
new file mode 100644
index 000000000..175d0399b
--- /dev/null
+++ b/doc/install/get-tarballs.rst
@@ -0,0 +1,14 @@
+====================================
+ Downloading a Ceph Release Tarball
+====================================
+
+As Ceph development progresses, the Ceph team releases new versions of the
+source code. You may download source code tarballs for Ceph releases here:
+
+`Ceph Release Tarballs`_
+
+.. tip:: For international users: There might be a mirror close to you where download Ceph from. For more information see: `Ceph Mirrors`_.
+
+
+.. _Ceph Release Tarballs: https://download.ceph.com/tarballs/
+.. _Ceph Mirrors: ../mirrors
diff --git a/doc/install/index.rst b/doc/install/index.rst
new file mode 100644
index 000000000..841febbe3
--- /dev/null
+++ b/doc/install/index.rst
@@ -0,0 +1,73 @@
+.. _install-overview:
+
+===============
+Installing Ceph
+===============
+
+There are several different ways to install Ceph.  Choose the
+method that best suits your needs.
+
+Recommended methods
+~~~~~~~~~~~~~~~~~~~
+
+:ref:`Cephadm <cephadm>` installs and manages a Ceph cluster using containers and
+systemd, with tight integration with the CLI and dashboard GUI.
+
+* cephadm only supports Octopus and newer releases.
+* cephadm is fully integrated with the new orchestration API and
+  fully supports the new CLI and dashboard features to manage
+  cluster deployment.
+* cephadm requires container support (podman or docker) and
+  Python 3.
+
+`Rook <https://rook.io/>`_ deploys and manages Ceph clusters running
+in Kubernetes, while also enabling management of storage resources and
+provisioning via Kubernetes APIs.  We recommend Rook as the way to run Ceph in
+Kubernetes or to connect an existing Ceph storage cluster to Kubernetes.
+
+* Rook only supports Nautilus and newer releases of Ceph.
+* Rook is the preferred method for running Ceph on Kubernetes, or for
+  connecting a Kubernetes cluster to an existing (external) Ceph
+  cluster.
+* Rook supports the new orchestrator API. New management features
+  in the CLI and dashboard are fully supported.
+
+Other methods
+~~~~~~~~~~~~~
+
+`ceph-ansible <https://docs.ceph.com/ceph-ansible/>`_ deploys and manages
+Ceph clusters using Ansible.
+
+* ceph-ansible is widely deployed.
+* ceph-ansible is not integrated with the new orchestrator APIs,
+  introduced in Nautlius and Octopus, which means that newer
+  management features and dashboard integration are not available.
+
+
+`ceph-deploy <https://docs.ceph.com/projects/ceph-deploy/en/latest/>`_ is a tool for quickly deploying clusters.
+
+  .. IMPORTANT::
+
+   ceph-deploy is no longer actively maintained. It is not tested on versions of Ceph newer than Nautilus. It does not support RHEL8, CentOS 8, or newer operating systems.
+
+`DeepSea <https://github.com/SUSE/DeepSea>`_ installs Ceph using Salt.
+
+`jaas.ai/ceph-mon <https://jaas.ai/ceph-mon>`_ installs Ceph using Juju.
+
+`github.com/openstack/puppet-ceph <https://github.com/openstack/puppet-ceph>`_  installs Ceph via Puppet.
+
+Ceph can also be :ref:`installed manually <install-manual>`.
+
+
+.. toctree::
+   :hidden:
+
+   index_manual
+
+Windows
+~~~~~~~
+
+For Windows installations, please consult this document:
+`Windows installation guide`_.
+
+.. _Windows installation guide: ./windows-install
diff --git a/doc/install/index_manual.rst b/doc/install/index_manual.rst
new file mode 100644
index 000000000..60dccfde2
--- /dev/null
+++ b/doc/install/index_manual.rst
@@ -0,0 +1,71 @@
+.. _install-manual:
+
+=======================
+ Installation (Manual)
+=======================
+
+
+Get Software
+============
+
+There are several methods for getting Ceph software. The easiest and most common
+method is to `get packages`_ by adding repositories for use with package
+management tools such as the Advanced Package Tool (APT) or Yellowdog Updater,
+Modified (YUM). You may also retrieve pre-compiled packages from the Ceph
+repository. Finally, you can retrieve tarballs or clone the Ceph source code
+repository and build Ceph yourself.
+
+
+.. toctree::
+   :maxdepth: 1
+
+	Get Packages <get-packages>
+	Get Tarballs <get-tarballs>
+	Clone Source <clone-source>
+	Build Ceph <build-ceph>
+    	Ceph Mirrors <mirrors>
+	Ceph Containers <containers>
+
+
+Install Software
+================
+
+Once you have the Ceph software (or added repositories), installing the software
+is easy. To install packages on each :term:`Ceph Node` in your cluster. You may
+use  ``cephadm`` to install Ceph for your storage cluster, or use package
+management tools. You should install Yum Priorities for RHEL/CentOS and other
+distributions that use Yum if you intend to install the Ceph Object Gateway or
+QEMU.
+
+.. toctree::
+   :maxdepth: 1
+
+	Install cephadm <../cephadm/install>
+    	Install Ceph Storage Cluster <install-storage-cluster>
+	Install Virtualization for Block <install-vm-cloud>
+
+
+Deploy a Cluster Manually
+=========================
+
+Once you have Ceph installed on your nodes, you can deploy a cluster manually.
+The manual procedure is primarily for exemplary purposes for those developing
+deployment scripts with Chef, Juju, Puppet, etc.
+
+.. toctree::
+
+	Manual Deployment <manual-deployment>
+	Manual Deployment on FreeBSD <manual-freebsd-deployment>
+
+Upgrade Software
+================
+
+As new versions of Ceph become available, you may upgrade your cluster to take
+advantage of new functionality. Read the upgrade documentation before you
+upgrade your cluster. Sometimes upgrading Ceph requires you to follow an upgrade
+sequence.
+
+.. toctree::
+   :maxdepth: 2
+
+.. _get packages: ../get-packages
diff --git a/doc/install/install-storage-cluster.rst b/doc/install/install-storage-cluster.rst
new file mode 100644
index 000000000..dcb9f5040
--- /dev/null
+++ b/doc/install/install-storage-cluster.rst
@@ -0,0 +1,87 @@
+==============================
+ Install Ceph Storage Cluster
+==============================
+
+This guide describes installing Ceph packages manually. This procedure
+is only for users who are not installing with a deployment tool such as
+``cephadm``, ``chef``, ``juju``, etc. 
+
+
+Installing with APT
+===================
+
+Once you have added either release or development packages to APT, you should
+update APT's database and install Ceph::
+
+	sudo apt-get update && sudo apt-get install ceph ceph-mds
+
+
+Installing with RPM
+===================
+
+To install Ceph with RPMs, execute the following steps:
+
+
+#. Install ``yum-plugin-priorities``. ::
+
+	sudo yum install yum-plugin-priorities
+
+#. Ensure ``/etc/yum/pluginconf.d/priorities.conf`` exists.
+
+#. Ensure ``priorities.conf`` enables the plugin. :: 
+
+	[main]
+	enabled = 1
+
+#. Ensure your YUM ``ceph.repo`` entry includes ``priority=2``. See
+   `Get Packages`_ for details::
+
+	[ceph]
+	name=Ceph packages for $basearch
+	baseurl=https://download.ceph.com/rpm-{ceph-release}/{distro}/$basearch
+	enabled=1
+	priority=2
+	gpgcheck=1
+	gpgkey=https://download.ceph.com/keys/release.asc
+
+	[ceph-noarch]
+	name=Ceph noarch packages
+	baseurl=https://download.ceph.com/rpm-{ceph-release}/{distro}/noarch
+	enabled=1
+	priority=2
+	gpgcheck=1
+	gpgkey=https://download.ceph.com/keys/release.asc
+
+	[ceph-source]
+	name=Ceph source packages
+	baseurl=https://download.ceph.com/rpm-{ceph-release}/{distro}/SRPMS
+	enabled=0
+	priority=2
+	gpgcheck=1
+	gpgkey=https://download.ceph.com/keys/release.asc
+
+
+#. Install pre-requisite packages::  
+
+	sudo yum install snappy leveldb gdisk python-argparse gperftools-libs
+
+
+Once you have added either release or development packages, or added a
+``ceph.repo`` file to ``/etc/yum.repos.d``, you can install Ceph packages. :: 
+
+	sudo yum install ceph
+
+
+Installing a Build
+==================
+
+If you build Ceph from source code, you may install Ceph in user space
+by executing the following:: 
+
+	sudo make install
+
+If you install Ceph locally, ``make`` will place the executables in
+``usr/local/bin``. You may add the Ceph configuration file to the
+``usr/local/bin`` directory to run Ceph from a single directory.
+
+.. _Get Packages: ../get-packages
diff --git a/doc/install/install-vm-cloud.rst b/doc/install/install-vm-cloud.rst
new file mode 100644
index 000000000..876422865
--- /dev/null
+++ b/doc/install/install-vm-cloud.rst
@@ -0,0 +1,132 @@
+=========================================
+ Install Virtualization for Block Device
+=========================================
+
+If you intend to use Ceph Block Devices and the Ceph Storage Cluster as a
+backend for Virtual Machines (VMs) or  :term:`Cloud Platforms` the QEMU/KVM and
+``libvirt`` packages are important for enabling VMs and cloud platforms.
+Examples of VMs include: QEMU/KVM, XEN, VMWare, LXC, VirtualBox, etc. Examples
+of Cloud Platforms include OpenStack, CloudStack, OpenNebula, etc.
+
+
+.. ditaa::
+
+            +---------------------------------------------------+
+            |                     libvirt                       |
+            +------------------------+--------------------------+
+                                     |
+                                     | configures
+                                     v
+            +---------------------------------------------------+
+            |                       QEMU                        |
+            +---------------------------------------------------+
+            |                      librbd                       |
+            +---------------------------------------------------+
+            |                     librados                      |
+            +------------------------+-+------------------------+
+            |          OSDs          | |        Monitors        |
+            +------------------------+ +------------------------+
+
+
+Install QEMU
+============
+
+QEMU KVM can interact with Ceph Block Devices via ``librbd``, which is an
+important feature for using Ceph with cloud platforms. Once you install QEMU,
+see `QEMU and Block Devices`_ for usage.
+
+
+Debian Packages
+---------------
+
+QEMU packages are incorporated into Ubuntu 12.04 Precise Pangolin and later
+versions. To  install QEMU, execute the following::
+
+	sudo apt-get install qemu
+
+
+RPM Packages
+------------
+
+To install QEMU, execute the following:
+
+
+#. Update your repositories. ::
+
+	sudo yum update
+
+#. Install QEMU for Ceph. ::
+
+	sudo yum install qemu-kvm qemu-kvm-tools qemu-img
+
+#. Install additional QEMU packages (optional)::
+
+	sudo yum install qemu-guest-agent qemu-guest-agent-win32
+
+
+Building QEMU
+-------------
+
+To build QEMU from source, use the following procedure::
+
+	cd {your-development-directory}
+	git clone git://git.qemu.org/qemu.git
+	cd qemu
+	./configure --enable-rbd
+	make; make install
+
+
+
+Install libvirt
+===============
+
+To use ``libvirt`` with Ceph, you must have a running Ceph Storage Cluster, and
+you must have installed and configured QEMU. See `Using libvirt with Ceph Block
+Device`_ for usage.
+
+
+Debian Packages
+---------------
+
+``libvirt`` packages are incorporated into Ubuntu 12.04 Precise Pangolin and
+later versions of Ubuntu. To install ``libvirt`` on these distributions,
+execute the following::
+
+	sudo apt-get update && sudo apt-get install libvirt-bin
+
+
+RPM Packages
+------------
+
+To use ``libvirt`` with a Ceph Storage Cluster, you must  have a running Ceph
+Storage Cluster and you must also install a version of QEMU with ``rbd`` format
+support.  See `Install QEMU`_ for details.
+
+
+``libvirt`` packages are incorporated into the recent CentOS/RHEL distributions.
+To install ``libvirt``, execute the following::
+
+	sudo yum install libvirt
+
+
+Building ``libvirt``
+--------------------
+
+To build ``libvirt`` from source, clone the ``libvirt`` repository and use
+`AutoGen`_ to generate the build. Then, execute ``make`` and ``make install`` to
+complete the installation. For example::
+
+	git clone git://libvirt.org/libvirt.git
+	cd libvirt
+	./autogen.sh
+	make
+	sudo make install
+
+See `libvirt Installation`_ for details.
+
+
+
+.. _libvirt Installation: http://www.libvirt.org/compiling.html
+.. _AutoGen: http://www.gnu.org/software/autogen/
+.. _QEMU and Block Devices: ../../rbd/qemu-rbd
+.. _Using libvirt with Ceph Block Device: ../../rbd/libvirt
diff --git a/doc/install/manual-deployment.rst b/doc/install/manual-deployment.rst
new file mode 100644
index 000000000..9ad652634
--- /dev/null
+++ b/doc/install/manual-deployment.rst
@@ -0,0 +1,529 @@
+===================
+ Manual Deployment
+===================
+
+All Ceph clusters require at least one monitor, and at least as many OSDs as
+copies of an object stored on the cluster.  Bootstrapping the initial monitor(s)
+is the first step in deploying a Ceph Storage Cluster. Monitor deployment also
+sets important criteria for the entire cluster, such as the number of replicas
+for pools, the number of placement groups per OSD, the heartbeat intervals,
+whether authentication is required, etc. Most of these values are set by
+default, so it's useful to know about them when setting up your cluster for
+production.
+
+We will set up a cluster with ``node1`` as  the monitor node, and ``node2`` and
+``node3`` for OSD nodes.
+
+
+
+.. ditaa::
+
+           /------------------\         /----------------\
+           |    Admin Node    |         |     node1      |
+           |                  +-------->+                |
+           |                  |         | cCCC           |
+           \---------+--------/         \----------------/
+                     |
+                     |                  /----------------\
+                     |                  |     node2      |
+                     +----------------->+                |
+                     |                  | cCCC           |
+                     |                  \----------------/
+                     |
+                     |                  /----------------\
+                     |                  |     node3      |
+                     +----------------->|                |
+                                        | cCCC           |
+                                        \----------------/
+
+
+Monitor Bootstrapping
+=====================
+
+Bootstrapping a monitor (a Ceph Storage Cluster, in theory) requires
+a number of things:
+
+- **Unique Identifier:** The ``fsid`` is a unique identifier for the cluster,
+  and stands for File System ID from the days when the Ceph Storage Cluster was
+  principally for the Ceph File System. Ceph now supports native interfaces,
+  block devices, and object storage gateway interfaces too, so ``fsid`` is a
+  bit of a misnomer.
+
+- **Cluster Name:** Ceph clusters have a cluster name, which is a simple string
+  without spaces. The default cluster name is ``ceph``, but you may specify
+  a different cluster name. Overriding the default cluster name is
+  especially useful when you are working with multiple clusters and you need to
+  clearly understand which cluster your are working with.
+
+  For example, when you run multiple clusters in a :ref:`multisite configuration <multisite>`,
+  the cluster name (e.g., ``us-west``, ``us-east``) identifies the cluster for
+  the current CLI session. **Note:** To identify the cluster name on the
+  command line interface, specify the Ceph configuration file with the
+  cluster name (e.g., ``ceph.conf``, ``us-west.conf``, ``us-east.conf``, etc.).
+  Also see CLI usage (``ceph --cluster {cluster-name}``).
+
+- **Monitor Name:** Each monitor instance within a cluster has a unique name.
+  In common practice, the Ceph Monitor name is the host name (we recommend one
+  Ceph Monitor per host, and no commingling of Ceph OSD Daemons with
+  Ceph Monitors). You may retrieve the short hostname with ``hostname -s``.
+
+- **Monitor Map:** Bootstrapping the initial monitor(s) requires you to
+  generate a monitor map. The monitor map requires the ``fsid``, the cluster
+  name (or uses the default), and at least one host name and its IP address.
+
+- **Monitor Keyring**: Monitors communicate with each other via a
+  secret key. You must generate a keyring with a monitor secret and provide
+  it when bootstrapping the initial monitor(s).
+
+- **Administrator Keyring**: To use the ``ceph`` CLI tools, you must have
+  a ``client.admin`` user. So you must generate the admin user and keyring,
+  and you must also add the ``client.admin`` user to the monitor keyring.
+
+The foregoing requirements do not imply the creation of a Ceph Configuration
+file. However, as a best practice, we recommend creating a Ceph configuration
+file and populating it with the ``fsid``, the ``mon initial members`` and the
+``mon host`` settings.
+
+You can get and set all of the monitor settings at runtime as well. However,
+a Ceph Configuration file may contain only those settings that override the
+default values. When you add settings to a Ceph configuration file, these
+settings override the default settings. Maintaining those settings in a
+Ceph configuration file makes it easier to maintain your cluster.
+
+The procedure is as follows:
+
+
+#. Log in to the initial monitor node(s)::
+
+	ssh {hostname}
+
+   For example::
+
+	ssh node1
+
+
+#. Ensure you have a directory for the Ceph configuration file. By default,
+   Ceph uses ``/etc/ceph``. When you install ``ceph``, the installer will
+   create the ``/etc/ceph`` directory automatically. ::
+
+	ls /etc/ceph
+
+
+#. Create a Ceph configuration file. By default, Ceph uses
+   ``ceph.conf``, where ``ceph`` reflects the cluster name. ::
+
+	sudo vim /etc/ceph/ceph.conf
+
+
+#. Generate a unique ID (i.e., ``fsid``) for your cluster. ::
+
+	uuidgen
+
+
+#. Add the unique ID to your Ceph configuration file. ::
+
+	fsid = {UUID}
+
+   For example::
+
+	fsid = a7f64266-0894-4f1e-a635-d0aeaca0e993
+
+
+#. Add the initial monitor(s) to your Ceph configuration file. ::
+
+	mon initial members = {hostname}[,{hostname}]
+
+   For example::
+
+	mon initial members = node1
+
+
+#. Add the IP address(es) of the initial monitor(s) to your Ceph configuration
+   file and save the file. ::
+
+	mon host = {ip-address}[,{ip-address}]
+
+   For example::
+
+	mon host = 192.168.0.1
+
+   **Note:** You may use IPv6 addresses instead of IPv4 addresses, but
+   you must set ``ms bind ipv6`` to ``true``. See `Network Configuration
+   Reference`_ for details about network configuration.
+
+#. Create a keyring for your cluster and generate a monitor secret key. ::
+
+	sudo ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'
+
+
+#. Generate an administrator keyring, generate a ``client.admin`` user and add
+   the user to the keyring. ::
+
+	sudo ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'
+
+#. Generate a bootstrap-osd keyring, generate a ``client.bootstrap-osd`` user and add
+   the user to the keyring. ::
+
+	sudo ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'
+
+#. Add the generated keys to the ``ceph.mon.keyring``. ::
+
+	sudo ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring
+	sudo ceph-authtool /tmp/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring
+
+#. Change the owner for ``ceph.mon.keyring``. ::
+
+	sudo chown ceph:ceph /tmp/ceph.mon.keyring
+
+#. Generate a monitor map using the hostname(s), host IP address(es) and the FSID.
+   Save it as ``/tmp/monmap``::
+
+	monmaptool --create --add {hostname} {ip-address} --fsid {uuid} /tmp/monmap
+
+   For example::
+
+	monmaptool --create --add node1 192.168.0.1 --fsid a7f64266-0894-4f1e-a635-d0aeaca0e993 /tmp/monmap
+
+
+#. Create a default data directory (or directories) on the monitor host(s). ::
+
+	sudo mkdir /var/lib/ceph/mon/{cluster-name}-{hostname}
+
+   For example::
+
+	sudo -u ceph mkdir /var/lib/ceph/mon/ceph-node1
+
+   See `Monitor Config Reference - Data`_ for details.
+
+#. Populate the monitor daemon(s) with the monitor map and keyring. ::
+
+	sudo -u ceph ceph-mon [--cluster {cluster-name}] --mkfs -i {hostname} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring
+
+   For example::
+
+	sudo -u ceph ceph-mon --mkfs -i node1 --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring
+
+
+#. Consider settings for a Ceph configuration file. Common settings include
+   the following::
+
+	[global]
+	fsid = {cluster-id}
+	mon initial members = {hostname}[, {hostname}]
+	mon host = {ip-address}[, {ip-address}]
+	public network = {network}[, {network}]
+	cluster network = {network}[, {network}]
+	auth cluster required = cephx
+	auth service required = cephx
+	auth client required = cephx
+	osd journal size = {n}
+	osd pool default size = {n}  # Write an object n times.
+	osd pool default min size = {n} # Allow writing n copies in a degraded state.
+	osd pool default pg num = {n}
+	osd pool default pgp num = {n}
+	osd crush chooseleaf type = {n}
+
+   In the foregoing example, the ``[global]`` section of the configuration might
+   look like this::
+
+	[global]
+	fsid = a7f64266-0894-4f1e-a635-d0aeaca0e993
+	mon initial members = node1
+	mon host = 192.168.0.1
+	public network = 192.168.0.0/24
+	auth cluster required = cephx
+	auth service required = cephx
+	auth client required = cephx
+	osd journal size = 1024
+	osd pool default size = 3
+	osd pool default min size = 2
+	osd pool default pg num = 333
+	osd pool default pgp num = 333
+	osd crush chooseleaf type = 1
+
+
+#. Start the monitor(s).
+
+   Start the service with systemd::
+
+	sudo systemctl start ceph-mon@node1
+
+#. Verify that the monitor is running. ::
+
+	sudo ceph -s
+
+   You should see output that the monitor you started is up and running, and
+   you should see a health error indicating that placement groups are stuck
+   inactive. It should look something like this::
+
+      cluster:
+        id:     a7f64266-0894-4f1e-a635-d0aeaca0e993
+        health: HEALTH_OK
+
+      services:
+        mon: 1 daemons, quorum node1
+        mgr: node1(active)
+        osd: 0 osds: 0 up, 0 in
+
+      data:
+        pools:   0 pools, 0 pgs
+        objects: 0 objects, 0 bytes
+        usage:   0 kB used, 0 kB / 0 kB avail
+        pgs:
+
+
+   **Note:** Once you add OSDs and start them, the placement group health errors
+   should disappear. See `Adding OSDs`_ for details.
+
+Manager daemon configuration
+============================
+
+On each node where you run a ceph-mon daemon, you should also set up a ceph-mgr daemon.
+
+See :ref:`mgr-administrator-guide`
+
+Adding OSDs
+===========
+
+Once you have your initial monitor(s) running, you should add OSDs. Your cluster
+cannot reach an ``active + clean`` state until you have enough OSDs to handle the
+number of copies of an object (e.g., ``osd pool default size = 2`` requires at
+least two OSDs). After bootstrapping your monitor, your cluster has a default
+CRUSH map; however, the CRUSH map doesn't have any Ceph OSD Daemons mapped to
+a Ceph Node.
+
+
+Short Form
+----------
+
+Ceph provides the ``ceph-volume`` utility, which can prepare a logical volume, disk, or partition
+for use with Ceph. The ``ceph-volume`` utility creates the OSD ID by
+incrementing the index. Additionally, ``ceph-volume`` will add the new OSD to the
+CRUSH map under the host for you. Execute ``ceph-volume -h`` for CLI details.
+The ``ceph-volume`` utility automates the steps of the `Long Form`_ below. To
+create the first two OSDs with the short form procedure, execute the following
+on  ``node2`` and ``node3``:
+
+bluestore
+^^^^^^^^^
+#. Create the OSD. ::
+
+	ssh {node-name}
+	sudo ceph-volume lvm create --data {data-path}
+
+   For example::
+
+	ssh node1
+	sudo ceph-volume lvm create --data /dev/hdd1
+
+Alternatively, the creation process can be split in two phases (prepare, and
+activate):
+
+#. Prepare the OSD. ::
+
+	ssh {node-name}
+	sudo ceph-volume lvm prepare --data {data-path} {data-path}
+
+   For example::
+
+	ssh node1
+	sudo ceph-volume lvm prepare --data /dev/hdd1
+
+   Once prepared, the ``ID`` and ``FSID`` of the prepared OSD are required for
+   activation. These can be obtained by listing OSDs in the current server::
+
+    sudo ceph-volume lvm list
+
+#. Activate the OSD::
+
+	sudo ceph-volume lvm activate {ID} {FSID}
+
+   For example::
+
+	sudo ceph-volume lvm activate 0 a7f64266-0894-4f1e-a635-d0aeaca0e993
+
+
+filestore
+^^^^^^^^^
+#. Create the OSD. ::
+
+	ssh {node-name}
+	sudo ceph-volume lvm create --filestore --data {data-path} --journal {journal-path}
+
+   For example::
+
+	ssh node1
+	sudo ceph-volume lvm create --filestore --data /dev/hdd1 --journal /dev/hdd2
+
+Alternatively, the creation process can be split in two phases (prepare, and
+activate):
+
+#. Prepare the OSD. ::
+
+	ssh {node-name}
+	sudo ceph-volume lvm prepare --filestore --data {data-path} --journal {journal-path}
+
+   For example::
+
+	ssh node1
+	sudo ceph-volume lvm prepare --filestore --data /dev/hdd1 --journal /dev/hdd2
+
+   Once prepared, the ``ID`` and ``FSID`` of the prepared OSD are required for
+   activation. These can be obtained by listing OSDs in the current server::
+
+    sudo ceph-volume lvm list
+
+#. Activate the OSD::
+
+	sudo ceph-volume lvm activate --filestore {ID} {FSID}
+
+   For example::
+
+	sudo ceph-volume lvm activate --filestore 0 a7f64266-0894-4f1e-a635-d0aeaca0e993
+
+
+Long Form
+---------
+
+Without the benefit of any helper utilities, create an OSD and add it to the
+cluster and CRUSH map with the following procedure. To create the first two
+OSDs with the long form procedure, execute the following steps for each OSD.
+
+.. note:: This procedure does not describe deployment on top of dm-crypt
+          making use of the dm-crypt 'lockbox'.
+
+#. Connect to the OSD host and become root. ::
+
+     ssh {node-name}
+     sudo bash
+
+#. Generate a UUID for the OSD. ::
+
+     UUID=$(uuidgen)
+
+#. Generate a cephx key for the OSD. ::
+
+     OSD_SECRET=$(ceph-authtool --gen-print-key)
+
+#. Create the OSD. Note that an OSD ID can be provided as an
+   additional argument to ``ceph osd new`` if you need to reuse a
+   previously-destroyed OSD id. We assume that the
+   ``client.bootstrap-osd`` key is present on the machine.  You may
+   alternatively execute this command as ``client.admin`` on a
+   different host where that key is present.::
+
+     ID=$(echo "{\"cephx_secret\": \"$OSD_SECRET\"}" | \
+	ceph osd new $UUID -i - \
+	-n client.bootstrap-osd -k /var/lib/ceph/bootstrap-osd/ceph.keyring)
+
+   It is also possible to include a ``crush_device_class`` property in the JSON
+   to set an initial class other than the default (``ssd`` or ``hdd`` based on
+   the auto-detected device type).
+
+#. Create the default directory on your new OSD. ::
+
+     mkdir /var/lib/ceph/osd/ceph-$ID
+
+#. If the OSD is for a drive other than the OS drive, prepare it
+   for use with Ceph, and mount it to the directory you just created. ::
+
+     mkfs.xfs /dev/{DEV}
+     mount /dev/{DEV} /var/lib/ceph/osd/ceph-$ID
+
+#. Write the secret to the OSD keyring file. ::
+
+     ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-$ID/keyring \
+          --name osd.$ID --add-key $OSD_SECRET
+
+#. Initialize the OSD data directory. ::
+
+     ceph-osd -i $ID --mkfs --osd-uuid $UUID
+
+#. Fix ownership. ::
+
+     chown -R ceph:ceph /var/lib/ceph/osd/ceph-$ID
+
+#. After you add an OSD to Ceph, the OSD is in your configuration. However,
+   it is not yet running. You must start
+   your new OSD before it can begin receiving data.
+
+   For modern systemd distributions::
+
+     systemctl enable ceph-osd@$ID
+     systemctl start ceph-osd@$ID
+
+   For example::
+
+     systemctl enable ceph-osd@12
+     systemctl start ceph-osd@12
+
+
+Adding MDS
+==========
+
+In the below instructions, ``{id}`` is an arbitrary name, such as the hostname of the machine.
+
+#. Create the mds data directory.::
+
+	mkdir -p /var/lib/ceph/mds/{cluster-name}-{id}
+
+#. Create a keyring.::
+
+	ceph-authtool --create-keyring /var/lib/ceph/mds/{cluster-name}-{id}/keyring --gen-key -n mds.{id}
+
+#. Import the keyring and set caps.::
+
+	ceph auth add mds.{id} osd "allow rwx" mds "allow *" mon "allow profile mds" -i /var/lib/ceph/mds/{cluster}-{id}/keyring
+
+#. Add to ceph.conf.::
+
+	[mds.{id}]
+	host = {id}
+
+#. Start the daemon the manual way.::
+
+	ceph-mds --cluster {cluster-name} -i {id} -m {mon-hostname}:{mon-port} [-f]
+
+#. Start the daemon the right way (using ceph.conf entry).::
+
+	service ceph start
+
+#. If starting the daemon fails with this error::
+
+	mds.-1.0 ERROR: failed to authenticate: (22) Invalid argument
+
+   Then make sure you do not have a keyring set in ceph.conf in the global section; move it to the client section; or add a keyring setting specific to this mds daemon. And verify that you see the same key in the mds data directory and ``ceph auth get mds.{id}`` output.
+
+#. Now you are ready to `create a Ceph file system`_.
+
+
+Summary
+=======
+
+Once you have your monitor and two OSDs up and running, you can watch the
+placement groups peer by executing the following::
+
+	ceph -w
+
+To view the tree, execute the following::
+
+	ceph osd tree
+
+You should see output that looks something like this::
+
+	# id	weight	type name	up/down	reweight
+	-1	2	root default
+	-2	2		host node1
+	0	1			osd.0	up	1
+	-3	1		host node2
+	1	1			osd.1	up	1
+
+To add (or remove) additional monitors, see `Add/Remove Monitors`_.
+To add (or remove) additional Ceph OSD Daemons, see `Add/Remove OSDs`_.
+
+
+.. _Add/Remove Monitors: ../../rados/operations/add-or-rm-mons
+.. _Add/Remove OSDs: ../../rados/operations/add-or-rm-osds
+.. _Network Configuration Reference: ../../rados/configuration/network-config-ref
+.. _Monitor Config Reference - Data: ../../rados/configuration/mon-config-ref#data
+.. _create a Ceph file system: ../../cephfs/createfs
diff --git a/doc/install/manual-freebsd-deployment.rst b/doc/install/manual-freebsd-deployment.rst
new file mode 100644
index 000000000..f597574f4
--- /dev/null
+++ b/doc/install/manual-freebsd-deployment.rst
@@ -0,0 +1,575 @@
+==============================
+ Manual Deployment on FreeBSD
+==============================
+
+This a largely a copy of the regular Manual Deployment with FreeBSD specifics.
+The difference lies in two parts: The underlying diskformat, and the way to use
+the tools.
+
+All Ceph clusters require at least one monitor, and at least as many OSDs as
+copies of an object stored on the cluster.  Bootstrapping the initial monitor(s)
+is the first step in deploying a Ceph Storage Cluster. Monitor deployment also
+sets important criteria for the entire cluster, such as the number of replicas
+for pools, the number of placement groups per OSD, the heartbeat intervals,
+whether authentication is required, etc. Most of these values are set by
+default, so it's useful to know about them when setting up your cluster for
+production.
+
+We will set up a cluster with ``node1`` as  the monitor node, and ``node2`` and
+``node3`` for OSD nodes.
+
+
+
+.. ditaa::
+
+           /------------------\         /----------------\
+           |    Admin Node    |         |     node1      |
+           |                  +-------->+                |
+           |                  |         | cCCC           |
+           \---------+--------/         \----------------/
+                     |
+                     |                  /----------------\
+                     |                  |     node2      |
+                     +----------------->+                |
+                     |                  | cCCC           |
+                     |                  \----------------/
+                     |
+                     |                  /----------------\
+                     |                  |     node3      |
+                     +----------------->|                |
+                                        | cCCC           |
+                                        \----------------/
+
+
+
+Disklayout on FreeBSD
+=====================
+
+Current implementation works on ZFS pools
+
+* All Ceph data is created in /var/lib/ceph
+* Log files go into /var/log/ceph
+* PID files go into /var/log/run
+* One ZFS pool is allocated per OSD, like::
+
+    gpart create -s GPT ada1
+    gpart add -t freebsd-zfs -l osd.1 ada1
+    zpool create -m /var/lib/ceph/osd/osd.1 osd.1 gpt/osd.1
+
+* Some cache and log (ZIL) can be attached.
+  Please note that this is different from the Ceph journals. Cache and log are
+  totally transparent for Ceph, and help the file system to keep the system
+  consistent and help performance.
+  Assuming that ada2 is an SSD::
+
+    gpart create -s GPT ada2
+    gpart add -t freebsd-zfs -l osd.1-log -s 1G ada2
+    zpool add osd.1 log gpt/osd.1-log
+    gpart add -t freebsd-zfs -l osd.1-cache -s 10G ada2
+    zpool add osd.1 log gpt/osd.1-cache
+
+* Note: *UFS2 does not allow large xattribs*
+
+
+Configuration
+-------------
+
+As per FreeBSD default parts of extra software go into ``/usr/local/``. Which
+means that for ``/etc/ceph.conf`` the default location is
+``/usr/local/etc/ceph/ceph.conf``. Smartest thing to do is to create a softlink
+from ``/etc/ceph`` to ``/usr/local/etc/ceph``::
+
+  ln -s /usr/local/etc/ceph /etc/ceph
+
+A sample file is provided in ``/usr/local/share/doc/ceph/sample.ceph.conf``
+Note that ``/usr/local/etc/ceph/ceph.conf`` will be found by most tools,
+linking it to ``/etc/ceph/ceph.conf`` will help with any scripts that are found
+in extra tools, scripts, and/or discussionlists.
+
+Monitor Bootstrapping
+=====================
+
+Bootstrapping a monitor (a Ceph Storage Cluster, in theory) requires
+a number of things:
+
+- **Unique Identifier:** The ``fsid`` is a unique identifier for the cluster,
+  and stands for File System ID from the days when the Ceph Storage Cluster was
+  principally for the Ceph File System. Ceph now supports native interfaces,
+  block devices, and object storage gateway interfaces too, so ``fsid`` is a
+  bit of a misnomer.
+
+- **Cluster Name:** Ceph clusters have a cluster name, which is a simple string
+  without spaces. The default cluster name is ``ceph``, but you may specify
+  a different cluster name. Overriding the default cluster name is
+  especially useful when you are working with multiple clusters and you need to
+  clearly understand which cluster your are working with.
+
+  For example, when you run multiple clusters in a :ref:`multisite configuration <multisite>`,
+  the cluster name (e.g., ``us-west``, ``us-east``) identifies the cluster for
+  the current CLI session. **Note:** To identify the cluster name on the
+  command line interface, specify the a Ceph configuration file with the
+  cluster name (e.g., ``ceph.conf``, ``us-west.conf``, ``us-east.conf``, etc.).
+  Also see CLI usage (``ceph --cluster {cluster-name}``).
+
+- **Monitor Name:** Each monitor instance within a cluster has a unique name.
+  In common practice, the Ceph Monitor name is the host name (we recommend one
+  Ceph Monitor per host, and no commingling of Ceph OSD Daemons with
+  Ceph Monitors). You may retrieve the short hostname with ``hostname -s``.
+
+- **Monitor Map:** Bootstrapping the initial monitor(s) requires you to
+  generate a monitor map. The monitor map requires the ``fsid``, the cluster
+  name (or uses the default), and at least one host name and its IP address.
+
+- **Monitor Keyring**: Monitors communicate with each other via a
+  secret key. You must generate a keyring with a monitor secret and provide
+  it when bootstrapping the initial monitor(s).
+
+- **Administrator Keyring**: To use the ``ceph`` CLI tools, you must have
+  a ``client.admin`` user. So you must generate the admin user and keyring,
+  and you must also add the ``client.admin`` user to the monitor keyring.
+
+The foregoing requirements do not imply the creation of a Ceph Configuration
+file. However, as a best practice, we recommend creating a Ceph configuration
+file and populating it with the ``fsid``, the ``mon initial members`` and the
+``mon host`` settings.
+
+You can get and set all of the monitor settings at runtime as well. However,
+a Ceph Configuration file may contain only those settings that override the
+default values. When you add settings to a Ceph configuration file, these
+settings override the default settings. Maintaining those settings in a
+Ceph configuration file makes it easier to maintain your cluster.
+
+The procedure is as follows:
+
+
+#. Log in to the initial monitor node(s)::
+
+	ssh {hostname}
+
+   For example::
+
+	ssh node1
+
+
+#. Ensure you have a directory for the Ceph configuration file. By default,
+   Ceph uses ``/etc/ceph``. When you install ``ceph``, the installer will
+   create the ``/etc/ceph`` directory automatically. ::
+
+	ls /etc/ceph
+
+#. Create a Ceph configuration file. By default, Ceph uses
+   ``ceph.conf``, where ``ceph`` reflects the cluster name. ::
+
+	sudo vim /etc/ceph/ceph.conf
+
+
+#. Generate a unique ID (i.e., ``fsid``) for your cluster. ::
+
+	uuidgen
+
+
+#. Add the unique ID to your Ceph configuration file. ::
+
+	fsid = {UUID}
+
+   For example::
+
+	fsid = a7f64266-0894-4f1e-a635-d0aeaca0e993
+
+
+#. Add the initial monitor(s) to your Ceph configuration file. ::
+
+	mon initial members = {hostname}[,{hostname}]
+
+   For example::
+
+	mon initial members = node1
+
+
+#. Add the IP address(es) of the initial monitor(s) to your Ceph configuration
+   file and save the file. ::
+
+	mon host = {ip-address}[,{ip-address}]
+
+   For example::
+
+	mon host = 192.168.0.1
+
+   **Note:** You may use IPv6 addresses instead of IPv4 addresses, but
+   you must set ``ms bind ipv6`` to ``true``. See `Network Configuration
+   Reference`_ for details about network configuration.
+
+#. Create a keyring for your cluster and generate a monitor secret key. ::
+
+	ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'
+
+
+#. Generate an administrator keyring, generate a ``client.admin`` user and add
+   the user to the keyring. ::
+
+	sudo ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'
+
+
+#. Add the ``client.admin`` key to the ``ceph.mon.keyring``. ::
+
+	ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring
+
+
+#. Generate a monitor map using the hostname(s), host IP address(es) and the FSID.
+   Save it as ``/tmp/monmap``::
+
+	monmaptool --create --add {hostname} {ip-address} --fsid {uuid} /tmp/monmap
+
+   For example::
+
+	monmaptool --create --add node1 192.168.0.1 --fsid a7f64266-0894-4f1e-a635-d0aeaca0e993 /tmp/monmap
+
+
+#. Create a default data directory (or directories) on the monitor host(s). ::
+
+	sudo mkdir /var/lib/ceph/mon/{cluster-name}-{hostname}
+
+   For example::
+
+	sudo mkdir /var/lib/ceph/mon/ceph-node1
+
+   See `Monitor Config Reference - Data`_ for details.
+
+#. Populate the monitor daemon(s) with the monitor map and keyring. ::
+
+	sudo -u ceph ceph-mon [--cluster {cluster-name}] --mkfs -i {hostname} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring
+
+   For example::
+
+	sudo -u ceph ceph-mon --mkfs -i node1 --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring
+
+
+#. Consider settings for a Ceph configuration file. Common settings include
+   the following::
+
+	[global]
+	fsid = {cluster-id}
+	mon initial members = {hostname}[, {hostname}]
+	mon host = {ip-address}[, {ip-address}]
+	public network = {network}[, {network}]
+	cluster network = {network}[, {network}]
+	auth cluster required = cephx
+	auth service required = cephx
+	auth client required = cephx
+	osd journal size = {n}
+	osd pool default size = {n}  # Write an object n times.
+	osd pool default min size = {n} # Allow writing n copy in a degraded state.
+	osd pool default pg num = {n}
+	osd pool default pgp num = {n}
+	osd crush chooseleaf type = {n}
+
+   In the foregoing example, the ``[global]`` section of the configuration might
+   look like this::
+
+	[global]
+	fsid = a7f64266-0894-4f1e-a635-d0aeaca0e993
+	mon initial members = node1
+	mon host = 192.168.0.1
+	public network = 192.168.0.0/24
+	auth cluster required = cephx
+	auth service required = cephx
+	auth client required = cephx
+	osd journal size = 1024
+	osd pool default size = 3
+	osd pool default min size = 2
+	osd pool default pg num = 333
+	osd pool default pgp num = 333
+	osd crush chooseleaf type = 1
+
+#. Touch the ``done`` file.
+
+   Mark that the monitor is created and ready to be started::
+
+	sudo touch /var/lib/ceph/mon/ceph-node1/done
+
+#. And for FreeBSD an entry for every monitor needs to be added to the config
+   file. (The requirement will be removed in future releases).
+
+   The entry should look like::
+
+     [mon]
+         [mon.node1]
+             host = node1    # this name can be resolve
+
+
+#. Start the monitor(s).
+
+   For Ubuntu, use Upstart::
+
+	sudo start ceph-mon id=node1 [cluster={cluster-name}]
+
+   In this case, to allow the start of the daemon at each reboot you
+   must create two empty files like this::
+
+	sudo touch /var/lib/ceph/mon/{cluster-name}-{hostname}/upstart
+
+   For example::
+
+	sudo touch /var/lib/ceph/mon/ceph-node1/upstart
+
+   For Debian/CentOS/RHEL, use sysvinit::
+
+	sudo /etc/init.d/ceph start mon.node1
+
+   For FreeBSD we use the rc.d init scripts (called bsdrc in Ceph)::
+
+	sudo service ceph start start mon.node1
+
+   For this to work /etc/rc.conf also needs the entry to enable ceph::
+     cat 'ceph_enable="YES"' >> /etc/rc.conf
+
+
+#. Verify that Ceph created the default pools. ::
+
+	ceph osd lspools
+
+   You should see output like this::
+
+	0 data
+	1 metadata
+	2 rbd
+
+#. Verify that the monitor is running. ::
+
+	ceph -s
+
+   You should see output that the monitor you started is up and running, and
+   you should see a health error indicating that placement groups are stuck
+   inactive. It should look something like this::
+
+	cluster a7f64266-0894-4f1e-a635-d0aeaca0e993
+	  health HEALTH_ERR 192 pgs stuck inactive; 192 pgs stuck unclean; no osds
+	  monmap e1: 1 mons at {node1=192.168.0.1:6789/0}, election epoch 1, quorum 0 node1
+	  osdmap e1: 0 osds: 0 up, 0 in
+	  pgmap v2: 192 pgs, 3 pools, 0 bytes data, 0 objects
+	     0 kB used, 0 kB / 0 kB avail
+	     192 creating
+
+   **Note:** Once you add OSDs and start them, the placement group health errors
+   should disappear. See the next section for details.
+
+.. _freebsd_adding_osds:
+
+Adding OSDs
+===========
+
+Once you have your initial monitor(s) running, you should add OSDs. Your cluster
+cannot reach an ``active + clean`` state until you have enough OSDs to handle the
+number of copies of an object (e.g., ``osd pool default size = 2`` requires at
+least two OSDs). After bootstrapping your monitor, your cluster has a default
+CRUSH map; however, the CRUSH map doesn't have any Ceph OSD Daemons mapped to
+a Ceph Node.
+
+
+Long Form
+---------
+
+Without the benefit of any helper utilities, create an OSD and add it to the
+cluster and CRUSH map with the following procedure. To create the first two
+OSDs with the long form procedure, execute the following on ``node2`` and
+``node3``:
+
+#. Connect to the OSD host. ::
+
+	ssh {node-name}
+
+#. Generate a UUID for the OSD. ::
+
+	uuidgen
+
+
+#. Create the OSD. If no UUID is given, it will be set automatically when the
+   OSD starts up. The following command will output the OSD number, which you
+   will need for subsequent steps. ::
+
+	ceph osd create [{uuid} [{id}]]
+
+
+#. Create the default directory on your new OSD. ::
+
+	ssh {new-osd-host}
+	sudo mkdir /var/lib/ceph/osd/{cluster-name}-{osd-number}
+
+   Above are the ZFS instructions to do this for FreeBSD.
+
+
+#. If the OSD is for a drive other than the OS drive, prepare it
+   for use with Ceph, and mount it to the directory you just created.
+
+
+#. Initialize the OSD data directory. ::
+
+	ssh {new-osd-host}
+	sudo ceph-osd -i {osd-num} --mkfs --mkkey --osd-uuid [{uuid}]
+
+   The directory must be empty before you can run ``ceph-osd`` with the
+   ``--mkkey`` option. In addition, the ceph-osd tool requires specification
+   of custom cluster names with the ``--cluster`` option.
+
+
+#. Register the OSD authentication key. The value of ``ceph`` for
+   ``ceph-{osd-num}`` in the path is the ``$cluster-$id``.  If your
+   cluster name differs from ``ceph``, use your cluster name instead.::
+
+	sudo ceph auth add osd.{osd-num} osd 'allow *' mon 'allow profile osd' -i /var/lib/ceph/osd/{cluster-name}-{osd-num}/keyring
+
+
+#. Add your Ceph Node to the CRUSH map. ::
+
+	ceph [--cluster {cluster-name}] osd crush add-bucket {hostname} host
+
+   For example::
+
+	ceph osd crush add-bucket node1 host
+
+
+#. Place the Ceph Node under the root ``default``. ::
+
+	ceph osd crush move node1 root=default
+
+
+#. Add the OSD to the CRUSH map so that it can begin receiving data. You may
+   also decompile the CRUSH map, add the OSD to the device list, add the host as a
+   bucket (if it's not already in the CRUSH map), add the device as an item in the
+   host, assign it a weight, recompile it and set it. ::
+
+	ceph [--cluster {cluster-name}] osd crush add {id-or-name} {weight} [{bucket-type}={bucket-name} ...]
+
+   For example::
+
+	ceph osd crush add osd.0 1.0 host=node1
+
+
+#. After you add an OSD to Ceph, the OSD is in your configuration. However,
+   it is not yet running. The OSD is ``down`` and ``in``. You must start
+   your new OSD before it can begin receiving data.
+
+   For Ubuntu, use Upstart::
+
+	sudo start ceph-osd id={osd-num} [cluster={cluster-name}]
+
+   For example::
+
+	sudo start ceph-osd id=0
+	sudo start ceph-osd id=1
+
+   For Debian/CentOS/RHEL, use sysvinit::
+
+	sudo /etc/init.d/ceph start osd.{osd-num} [--cluster {cluster-name}]
+
+   For example::
+
+	sudo /etc/init.d/ceph start osd.0
+	sudo /etc/init.d/ceph start osd.1
+
+   In this case, to allow the start of the daemon at each reboot you
+   must create an empty file like this::
+
+	sudo touch /var/lib/ceph/osd/{cluster-name}-{osd-num}/sysvinit
+
+   For example::
+
+	sudo touch /var/lib/ceph/osd/ceph-0/sysvinit
+	sudo touch /var/lib/ceph/osd/ceph-1/sysvinit
+
+   Once you start your OSD, it is ``up`` and ``in``.
+
+   For FreeBSD using rc.d init.
+
+   After adding the OSD to ``ceph.conf``::
+
+	sudo service ceph start osd.{osd-num}
+
+   For example::
+
+	sudo service ceph start osd.0
+	sudo service ceph start osd.1
+
+   In this case, to allow the start of the daemon at each reboot you
+   must create an empty file like this::
+
+	sudo touch /var/lib/ceph/osd/{cluster-name}-{osd-num}/bsdrc
+
+   For example::
+
+	sudo touch /var/lib/ceph/osd/ceph-0/bsdrc
+	sudo touch /var/lib/ceph/osd/ceph-1/bsdrc
+
+   Once you start your OSD, it is ``up`` and ``in``.
+
+
+
+Adding MDS
+==========
+
+In the below instructions, ``{id}`` is an arbitrary name, such as the hostname of the machine.
+
+#. Create the mds data directory.::
+
+	mkdir -p /var/lib/ceph/mds/{cluster-name}-{id}
+
+#. Create a keyring.::
+
+	ceph-authtool --create-keyring /var/lib/ceph/mds/{cluster-name}-{id}/keyring --gen-key -n mds.{id}
+
+#. Import the keyring and set caps.::
+
+	ceph auth add mds.{id} osd "allow rwx" mds "allow *" mon "allow profile mds" -i /var/lib/ceph/mds/{cluster}-{id}/keyring
+
+#. Add to ceph.conf.::
+
+	[mds.{id}]
+	host = {id}
+
+#. Start the daemon the manual way.::
+
+	ceph-mds --cluster {cluster-name} -i {id} -m {mon-hostname}:{mon-port} [-f]
+
+#. Start the daemon the right way (using ceph.conf entry).::
+
+	service ceph start
+
+#. If starting the daemon fails with this error::
+
+	mds.-1.0 ERROR: failed to authenticate: (22) Invalid argument
+
+   Then make sure you do not have a keyring set in ceph.conf in the global section; move it to the client section; or add a keyring setting specific to this mds daemon. And verify that you see the same key in the mds data directory and ``ceph auth get mds.{id}`` output.
+
+#. Now you are ready to `create a Ceph file system`_.
+
+
+Summary
+=======
+
+Once you have your monitor and two OSDs up and running, you can watch the
+placement groups peer by executing the following::
+
+	ceph -w
+
+To view the tree, execute the following::
+
+	ceph osd tree
+
+You should see output that looks something like this::
+
+	# id	weight	type name	up/down	reweight
+	-1	2	root default
+	-2	2		host node1
+	0	1			osd.0	up	1
+	-3	1		host node2
+	1	1			osd.1	up	1
+
+To add (or remove) additional monitors, see `Add/Remove Monitors`_.
+To add (or remove) additional Ceph OSD Daemons, see `Add/Remove OSDs`_.
+
+
+.. _Add/Remove Monitors: ../../rados/operations/add-or-rm-mons
+.. _Add/Remove OSDs: ../../rados/operations/add-or-rm-osds
+.. _Network Configuration Reference: ../../rados/configuration/network-config-ref
+.. _Monitor Config Reference - Data: ../../rados/configuration/mon-config-ref#data
+.. _create a Ceph file system: ../../cephfs/createfs
diff --git a/doc/install/mirrors.rst b/doc/install/mirrors.rst
new file mode 100644
index 000000000..35df93f62
--- /dev/null
+++ b/doc/install/mirrors.rst
@@ -0,0 +1,67 @@
+=============
+ Ceph Mirrors
+=============
+
+For improved user experience multiple mirrors for Ceph are available around the
+world.
+
+These mirrors are kindly sponsored by various companies who want to support the
+Ceph project.
+
+
+Locations
+=========
+
+These mirrors are available on the following locations:
+
+- **EU: Netherlands**: http://eu.ceph.com/
+- **AU: Australia**: http://au.ceph.com/
+- **SE: Sweden**: http://se.ceph.com/
+- **DE: Germany**: http://de.ceph.com/
+- **HK: Hong Kong**: http://hk.ceph.com/
+- **FR: France**: http://fr.ceph.com/
+- **UK: UK**: http://uk.ceph.com
+- **US-East: US East Coast**: http://us-east.ceph.com/
+- **US-Mid-West: Chicago**: http://mirrors.gigenet.com/ceph/
+- **US-West: US West Coast**: http://us-west.ceph.com/
+- **CN: China**: http://mirrors.ustc.edu.cn/ceph/
+
+You can replace all download.ceph.com URLs with any of the mirrors, for example:
+
+- http://download.ceph.com/tarballs/
+- http://download.ceph.com/debian-hammer/
+- http://download.ceph.com/rpm-hammer/
+
+Change this to:
+
+- http://eu.ceph.com/tarballs/
+- http://eu.ceph.com/debian-hammer/
+- http://eu.ceph.com/rpm-hammer/
+
+
+Mirroring
+=========
+
+You can easily mirror Ceph yourself using a Bash script and rsync. A easy to use
+script can be found at `Github`_.
+
+When mirroring Ceph, please keep the following guidelines in mind:
+
+- Choose a mirror close to you
+- Do not sync in a interval shorter than 3 hours
+- Avoid syncing at minute 0 of the hour, use something between 0 and 59
+
+
+Becoming a mirror
+=================
+
+If you want to provide a public mirror for other users of Ceph you can opt to
+become a official mirror.
+
+To make sure all mirrors meet the same standards some requirements have been
+set for all mirrors. These can be found on `Github`_.
+
+If you want to apply for an official mirror, please contact the ceph-users mailinglist.
+
+
+.. _Github: https://github.com/ceph/ceph/tree/master/mirroring
diff --git a/doc/install/windows-basic-config.rst b/doc/install/windows-basic-config.rst
new file mode 100644
index 000000000..0fe8a1b1b
--- /dev/null
+++ b/doc/install/windows-basic-config.rst
@@ -0,0 +1,48 @@
+:orphan:
+
+===========================
+Windows basic configuration
+===========================
+
+This page describes the minimum Ceph configuration required for using the
+client components on Windows.
+
+ceph.conf
+=========
+
+The default location for the ``ceph.conf`` file on Windows is
+``%ProgramData%\ceph\ceph.conf``, which usually expands to
+``C:\ProgramData\ceph\ceph.conf``.
+
+Below you may find a sample. Please fill in the monitor addresses
+accordingly.
+
+.. code:: ini
+
+    [global]
+        log to stderr = true
+        ; Uncomment the following in order to use the Windows Event Log
+        ; log to syslog = true
+
+        run dir = C:/ProgramData/ceph/out
+        crash dir = C:/ProgramData/ceph/out
+
+        ; Use the following to change the cephfs client log level
+        ; debug client = 2
+    [client]
+        keyring = C:/ProgramData/ceph/keyring
+        ; log file = C:/ProgramData/ceph/out/$name.$pid.log
+        admin socket = C:/ProgramData/ceph/out/$name.$pid.asok
+
+        ; client_permissions = true
+        ; client_mount_uid = 1000
+        ; client_mount_gid = 1000
+    [global]
+        mon host = <ceph_monitor_addresses>
+
+Don't forget to also copy your keyring file to the specified location and make
+sure that the configured directories exist (e.g. ``C:\ProgramData\ceph\out``).
+
+Please use slashes ``/`` instead of backslashes ``\`` as path separators
+within ``ceph.conf``.
+
diff --git a/doc/install/windows-install.rst b/doc/install/windows-install.rst
new file mode 100644
index 000000000..fb5b5b6f5
--- /dev/null
+++ b/doc/install/windows-install.rst
@@ -0,0 +1,88 @@
+:orphan:
+
+==========================
+Installing Ceph on Windows
+==========================
+
+The Ceph client tools and libraries can be natively used on Windows. This avoids
+the need of having additional layers such as iSCSI gateways or SMB shares,
+drastically improving the performance.
+
+Prerequisites
+=============
+
+Supported platforms
+-------------------
+
+Windows Server 2019 and Windows Server 2016 are supported. Previous Windows
+Server versions, including Windows client versions such as Windows 10, might
+work but haven't been tested.
+
+Windows Server 2016 does not provide Unix sockets, in which case some commands
+might be unavailable.
+
+Secure boot
+-----------
+
+The ``WNBD`` driver hasn't been signed by Microsoft, which means that Secure Boot
+must be disabled.
+
+Dokany
+------
+
+In order to mount Ceph filesystems, ``ceph-dokan`` requires Dokany to be
+installed. You may fetch the installer as well as the source code from the
+Dokany Github repository: https://github.com/dokan-dev/dokany/releases
+
+The minimum supported Dokany version is 1.3.1. At the time of the writing,
+Dokany 2.0 is in Beta stage and is unsupported.
+
+Unlike ``WNBD``, Dokany isn't included in the Ceph MSI installer.
+
+MSI installer
+=============
+
+Using the MSI installer is the recommended way of installing Ceph on Windows.
+It can be downloaded from here: https://cloudbase.it/ceph-for-windows/
+
+As mentioned earlier, the Ceph installer does not include Dokany, which has
+to be installed separately.
+
+A server reboot is required after uninstalling the driver, otherwise subsequent
+install attempts may fail.
+
+The following project allows building the MSI installer:
+https://github.com/cloudbase/ceph-windows-installer. It can either use prebuilt
+Ceph and WNBD binaries or compile them from scratch.
+
+Manual installation
+===================
+
+The following document describes the build process and manual installation:
+https://github.com/ceph/ceph/blob/master/README.windows.rst
+
+Configuration
+=============
+
+Please check the `Windows configuration sample`_ to get started.
+
+You'll also need a keyring file. The `General CephFS Prerequisites`_ page provides a
+simple example, showing how a new CephX user can be created and how its secret
+key can be retrieved.
+
+For more details on CephX user management, see the `Client Authentication`_
+and :ref:`User Management <user-management>`.
+
+Further reading
+===============
+
+* `RBD Windows documentation`_
+* `CephFS Windows documentation`_
+* `Windows troubleshooting`_
+
+.. _CephFS Windows documentation: ../../cephfs/ceph-dokan
+.. _Windows configuration sample: ../windows-basic-config
+.. _RBD Windows documentation: ../../rbd/rbd-windows/
+.. _Windows troubleshooting: ../windows-troubleshooting
+.. _General CephFS Prerequisites: ../../cephfs/mount-prerequisites
+.. _Client Authentication: ../../cephfs/client-auth
diff --git a/doc/install/windows-troubleshooting.rst b/doc/install/windows-troubleshooting.rst
new file mode 100644
index 000000000..355fd8803
--- /dev/null
+++ b/doc/install/windows-troubleshooting.rst
@@ -0,0 +1,96 @@
+:orphan:
+
+===============================
+Troubleshooting Ceph on Windows
+===============================
+
+MSI installer
+~~~~~~~~~~~~~
+
+The MSI source code can be consulted here:
+https://github.com/cloudbase/ceph-windows-installer
+
+The following command can be used to generate MSI logs::
+
+    msiexec.exe /i $msi_full_path /l*v! $log_file
+
+WNBD driver installation failures will be logged here: ``C:\Windows\inf\setupapi.dev.log``.
+A server reboot is required after uninstalling the driver, otherwise subsequent
+install attempts may fail.
+
+Wnbd
+~~~~
+
+For ``WNBD`` troubleshooting, please check this page: https://github.com/cloudbase/wnbd#troubleshooting
+
+Privileges
+~~~~~~~~~~
+
+Most ``rbd-wnbd`` and ``rbd device`` commands require privileged rights. Make
+sure to use an elevated PowerShell or CMD command prompt.
+
+Crash dumps
+~~~~~~~~~~~
+
+Userspace crash dumps can be placed at a configurable location and enabled for all
+applications or just predefined ones, as outlined here:
+https://docs.microsoft.com/en-us/windows/win32/wer/collecting-user-mode-dumps.
+
+Whenever a Windows application crashes, an event will be submitted to the ``Application``
+Windows Event Log, having Event ID 1000. The entry will also include the process id,
+the faulting module name and path as well as the exception code.
+
+Please note that in order to analyze crash dumps, the debug symbols are required.
+We're currently buidling Ceph using ``MinGW``, so by default ``DWARF`` symbols will
+be embedded in the binaries. ``windbg`` does not support such symbols but ``gdb``
+can be used.
+
+``gdb`` can debug running Windows processes but it cannot open Windows minidumps.
+The following ``gdb`` fork may be used until this functionality is merged upstream:
+https://github.com/ssbssa/gdb/releases. As an alternative, ``DWARF`` symbols
+can be converted using ``cv2pdb`` but be aware that this tool has limitted C++
+support.
+
+ceph tool
+~~~~~~~~~
+
+The ``ceph`` Python tool can't be used on Windows natively yet. With minor
+changes it may run, but the main issue is that Python doesn't currently allow
+using ``AF_UNIX`` on Windows: https://bugs.python.org/issue33408
+
+As an alternative, the ``ceph`` tool can be used through Windows Subsystem
+for Linux (WSL). For example, running Windows RBD daemons may be contacted by
+using:
+
+.. code:: bash
+
+    ceph daemon /mnt/c/ProgramData/ceph/out/ceph-client.admin.61436.1209215304.asok help
+
+IO counters
+~~~~~~~~~~~
+
+Along with the standard RBD perf counters, the ``libwnbd`` IO counters may be
+retrieved using:
+
+.. code:: PowerShell
+
+    rbd-wnbd stats $imageName
+
+At the same time, WNBD driver counters can be fetched using:
+
+.. code:: PowerShell
+
+    wnbd-client stats $mappingId
+
+Note that the ``wnbd-client`` mapping identifier will be the full RBD image spec
+(the ``device`` column of the ``rbd device list`` output).
+
+Missing libraries
+~~~~~~~~~~~~~~~~~
+
+The Ceph tools can silently exit with a -1073741515 return code if one of the
+required DLLs is missing or unsupported.
+
+The `Dependency Walker`_ tool can be used to determine the missing library.
+
+.. _Dependency Walker: https://www.dependencywalker.com/
diff --git a/doc/logo.png b/doc/logo.png
new file mode 100644
index 000000000..4cd8c7cc9
--- /dev/null
+++ b/doc/logo.png
diff --git a/doc/man/8/CMakeLists.txt b/doc/man/8/CMakeLists.txt
new file mode 100644
index 000000000..8e1b9373f
--- /dev/null
+++ b/doc/man/8/CMakeLists.txt
@@ -0,0 +1,89 @@
+set(client_srcs
+  ceph-syn.rst
+  ceph-conf.rst
+  ceph.rst
+  ceph-authtool.rst
+  ceph-kvstore-tool.rst
+  rados.rst
+  ceph-post-file.rst
+  ceph-dencoder.rst)
+
+set(server_srcs
+  ceph-deploy.rst
+  crushtool.rst
+  ceph-run.rst
+  mount.ceph.rst
+  mount.fuse.ceph.rst
+  ceph-create-keys.rst)
+if(WITH_TESTS)
+list(APPEND server_srcs
+  ceph-debugpack.rst)
+endif(WITH_TESTS)
+
+set(osd_srcs
+  ceph-clsinfo.rst
+  ceph-volume.rst
+  ceph-volume-systemd.rst
+  ceph-osd.rst
+  osdmaptool.rst
+  ceph-bluestore-tool.rst)
+
+set(mon_srcs
+  ceph-mon.rst
+  monmaptool.rst)
+
+list(APPEND man_srcs
+  ${client_srcs}
+  ${server_srcs}
+  ${osd_srcs}
+  ${mon_srcs}
+  ceph-mds.rst
+  cephfs-top.rst
+  librados-config.rst
+  cephadm.rst
+  cephfs-mirror.rst)
+
+if(HAVE_LIBFUSE)
+  list(APPEND man_srcs
+    ceph-fuse.rst
+    rbd-fuse.rst)
+endif()
+
+if(WITH_RADOSGW)
+  list(APPEND man_srcs
+	radosgw.rst
+	radosgw-admin.rst
+	rgw-orphan-list.rst
+	ceph-diff-sorted.rst)
+endif()
+
+if(WITH_RBD)
+  list(APPEND man_srcs
+	ceph-rbdnamer.rst
+	rbd-mirror.rst
+	rbd-replay-many.rst
+	rbd-replay-prep.rst
+	rbd-replay.rst
+	rbdmap.rst
+	rbd.rst)
+  if(LINUX)
+    list(APPEND man_srcs rbd-nbd.rst)
+  endif()
+  if(FREEBSD)
+    list(APPEND man_srcs rbd-ggate.rst)
+  endif()
+endif()
+
+list(APPEND man_srcs ceph-immutable-object-cache.rst)
+foreach(man ${man_srcs})
+  list(APPEND sphinx_input ${CMAKE_CURRENT_SOURCE_DIR}/${man})
+  # mount.ceph.rst => mount if we use
+  # get_filename_component(cmd ${man} NAME_WE)
+  string(REGEX REPLACE ".rst$" "" cmd ${man})
+  list(APPEND sphinx_output ${sphinx_output_dir}/${cmd}.8)
+  install(FILES ${sphinx_output_dir}/${cmd}.8
+    DESTINATION ${CEPH_MAN_DIR}/man8)
+endforeach()
+
+set(sphinx_input ${sphinx_input} PARENT_SCOPE)
+set(sphinx_output ${sphinx_output} PARENT_SCOPE)
diff --git a/doc/man/8/ceph-authtool.rst b/doc/man/8/ceph-authtool.rst
new file mode 100644
index 000000000..af9ee7123
--- /dev/null
+++ b/doc/man/8/ceph-authtool.rst
@@ -0,0 +1,206 @@
+:orphan:
+
+=================================================
+ ceph-authtool -- ceph keyring manipulation tool
+=================================================
+
+.. program:: ceph-authtool
+
+Synopsis
+========
+
+| **ceph-authtool** *keyringfile*
+  [ -l | --list ]
+  [ -p | --print-key ]
+  [ -C | --create-keyring ]
+  [ -g | --gen-key ]
+  [ --gen-print-key ]
+  [ --import-keyring *otherkeyringfile* ]
+  [ -n | --name *entityname* ]
+  [ -a | --add-key *base64_key* ]
+  [ --cap *subsystem* *capability* ]
+  [ --caps *capfile* ]
+  [ --mode *mode* ]
+
+
+Description
+===========
+
+**ceph-authtool** is a utility to create, view, and modify a Ceph keyring
+file. A keyring file stores one or more Ceph authentication keys and
+possibly an associated capability specification. Each key is
+associated with an entity name, of the form
+``{client,mon,mds,osd}.name``.
+
+**WARNING** Ceph provides authentication and protection against
+man-in-the-middle attacks once secret keys are in place.  However,
+data over the wire is not encrypted, which may include the messages
+used to configure said keys. The system is primarily intended to be
+used in trusted environments.
+
+Options
+=======
+
+.. option:: -l, --list
+
+   will list all keys and capabilities present in the keyring
+
+.. option:: -p, --print-key
+
+   will print an encoded key for the specified entityname. This is
+   suitable for the ``mount -o secret=`` argument
+
+.. option:: -C, --create-keyring
+
+   will create a new keyring, overwriting any existing keyringfile
+
+.. option:: -g, --gen-key
+
+   will generate a new secret key for the specified entityname
+
+.. option:: --gen-print-key
+
+   will generate a new secret key for the specified entityname,
+   without altering the keyringfile, printing the secret to stdout
+
+.. option:: --import-keyring *secondkeyringfile*
+
+   will import the content of a given keyring to the keyringfile
+
+.. option:: -n, --name *name*
+
+   specify entityname to operate on
+
+.. option:: -a, --add-key *base64_key*
+
+   will add an encoded key to the keyring
+
+.. option:: --cap *subsystem* *capability*
+
+   will set the capability for given subsystem
+
+.. option:: --caps *capsfile*
+
+   will set all of capabilities associated with a given key, for all subsystems
+
+ .. option:: --mode *mode*
+
+    will set the desired file mode to the keyring e.g: 0644, defaults to 0600
+
+
+Capabilities
+============
+
+The subsystem is the name of a Ceph subsystem: ``mon``, ``mds``, or
+``osd``.
+
+The capability is a string describing what the given user is allowed
+to do. This takes the form of a comma separated list of allow
+clauses with a permission specifier containing one or more of rwx for
+read, write, and execute permission. The ``allow *`` grants full
+superuser permissions for the given subsystem.
+
+For example::
+
+	# can read, write, and execute objects
+        osd = "allow rwx"
+
+	# can access mds server
+        mds = "allow"
+
+	# can modify cluster state (i.e., is a server daemon)
+        mon = "allow rwx"
+
+A librados user restricted to a single pool might look like::
+
+        mon = "allow r"
+
+        osd = "allow rw pool foo"
+
+A client using rbd with read access to one pool and read/write access to another::
+
+        mon = "allow r"
+
+        osd = "allow class-read object_prefix rbd_children, allow pool templates r class-read, allow pool vms rwx"
+
+A client mounting the file system with minimal permissions would need caps like::
+
+        mds = "allow"
+
+        osd = "allow rw pool data"
+
+        mon = "allow r"
+
+
+OSD Capabilities
+================
+
+In general, an osd capability follows the grammar::
+
+        osdcap  := grant[,grant...]
+        grant   := allow (match capspec | capspec match)
+        match   := [ pool[=]<poolname> | object_prefix <prefix>
+                    | namespace[=]<rados-namespace>
+                    | tag <application-name> <key>=<value> ]
+        capspec := * | [r][w][x] [class-read] [class-write]
+
+The capspec determines what kind of operations the entity can perform::
+
+    r           = read access to objects
+    w           = write access to objects
+    x           = can call any class method (same as class-read class-write)
+    class-read  = can call class methods that are reads
+    class-write = can call class methods that are writes
+    * or "all"  = equivalent to rwx, plus the ability to run osd admin commands,
+                  i.e. ceph osd tell ...
+
+The match criteria restrict a grant based on the pool being accessed.
+Grants are additive if the client fulfills the match condition. For
+example, if a client has the osd capabilities: "allow r object_prefix
+prefix, allow w pool foo, allow x pool bar", then it has rw access to
+pool foo, rx access to pool bar, and r access to objects whose
+names begin with 'prefix' in any pool.
+
+Caps file format
+================
+
+The caps file format consists of zero or more key/value pairs, one per
+line. The key and value are separated by an ``=``, and the value must
+be quoted (with ``'`` or ``"``) if it contains any whitespace. The key
+is the name of the Ceph subsystem (``osd``, ``mds``, ``mon``), and the
+value is the capability string (see above).
+
+
+Example
+=======
+
+To create a new keyring containing a key for client.foo with a 0644 file mode::
+
+        ceph-authtool -C -n client.foo --gen-key keyring --mode 0644
+
+To associate some capabilities with the key (namely, the ability to
+mount a Ceph file system)::
+
+        ceph-authtool -n client.foo --cap mds 'allow' --cap osd 'allow rw pool=data' --cap mon 'allow r' keyring
+
+To display the contents of the keyring::
+
+        ceph-authtool -l keyring
+
+When mounting a Ceph file system, you can grab the appropriately encoded secret key with::
+
+        mount -t ceph serverhost:/ mountpoint -o name=foo,secret=`ceph-authtool -p -n client.foo keyring`
+
+
+Availability
+============
+
+**ceph-authtool** is part of Ceph, a massively scalable, open-source, distributed storage system. Please
+refer to the Ceph documentation at http://ceph.com/docs for more
+information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8)
diff --git a/doc/man/8/ceph-bluestore-tool.rst b/doc/man/8/ceph-bluestore-tool.rst
new file mode 100644
index 000000000..bb67ccc71
--- /dev/null
+++ b/doc/man/8/ceph-bluestore-tool.rst
@@ -0,0 +1,212 @@
+:orphan:
+
+======================================================
+ ceph-bluestore-tool -- bluestore administrative tool
+======================================================
+
+.. program:: ceph-bluestore-tool
+
+Synopsis
+========
+
+| **ceph-bluestore-tool** *command*
+  [ --dev *device* ... ]
+  [ --path *osd path* ]
+  [ --out-dir *dir* ]
+  [ --log-file | -l *filename* ]
+  [ --deep ]
+| **ceph-bluestore-tool** fsck|repair --path *osd path* [ --deep ]
+| **ceph-bluestore-tool** show-label --dev *device* ...
+| **ceph-bluestore-tool** prime-osd-dir --dev *device* --path *osd path*
+| **ceph-bluestore-tool** bluefs-export --path *osd path* --out-dir *dir*
+| **ceph-bluestore-tool** bluefs-bdev-new-wal --path *osd path* --dev-target *new-device*
+| **ceph-bluestore-tool** bluefs-bdev-new-db --path *osd path* --dev-target *new-device*
+| **ceph-bluestore-tool** bluefs-bdev-migrate --path *osd path* --dev-target *new-device* --devs-source *device1* [--devs-source *device2*]
+| **ceph-bluestore-tool** free-dump|free-score --path *osd path* [ --allocator block/bluefs-wal/bluefs-db/bluefs-slow ]
+| **ceph-bluestore-tool** reshard --path *osd path* --sharding *new sharding* [ --sharding-ctrl *control string* ]
+| **ceph-bluestore-tool** show-sharding --path *osd path*
+
+
+Description
+===========
+
+**ceph-bluestore-tool** is a utility to perform low-level administrative
+operations on a BlueStore instance.
+
+Commands
+========
+
+:command:`help`
+
+   show help
+
+:command:`fsck` [ --deep ]
+
+   run consistency check on BlueStore metadata.  If *--deep* is specified, also read all object data and verify checksums.
+
+:command:`repair`
+
+   Run a consistency check *and* repair any errors we can.
+
+:command:`bluefs-export`
+
+   Export the contents of BlueFS (i.e., RocksDB files) to an output directory.
+
+:command:`bluefs-bdev-sizes` --path *osd path*
+
+   Print the device sizes, as understood by BlueFS, to stdout.
+
+:command:`bluefs-bdev-expand` --path *osd path*
+
+   Instruct BlueFS to check the size of its block devices and, if they have
+   expanded, make use of the additional space. Please note that only the new
+   files created by BlueFS will be allocated on the preferred block device if
+   it has enough free space, and the existing files that have spilled over to
+   the slow device will be gradually removed when RocksDB performs compaction.
+   In other words, if there is any data spilled over to the slow device, it
+   will be moved to the fast device over time.
+
+:command:`bluefs-bdev-new-wal` --path *osd path* --dev-target *new-device*
+
+   Adds WAL device to BlueFS, fails if WAL device already exists.
+
+:command:`bluefs-bdev-new-db` --path *osd path* --dev-target *new-device*
+
+   Adds DB device to BlueFS, fails if DB device already exists.
+   
+:command:`bluefs-bdev-migrate` --dev-target *new-device* --devs-source *device1* [--devs-source *device2*]
+
+   Moves BlueFS data from source device(s) to the target one, source devices
+   (except the main one) are removed on success. Target device can be both
+   already attached or new device. In the latter case it's added to OSD
+   replacing one of the source devices. Following replacement rules apply
+   (in the order of precedence, stop on the first match):
+
+      - if source list has DB volume - target device replaces it.
+      - if source list has WAL volume - target device replace it.
+      - if source list has slow volume only - operation isn't permitted, requires explicit allocation via new-db/new-wal command.
+
+:command:`show-label` --dev *device* [...]
+
+   Show device label(s).	   
+
+:command:`free-dump` --path *osd path* [ --allocator block/bluefs-wal/bluefs-db/bluefs-slow ]
+
+   Dump all free regions in allocator.
+
+:command:`free-score` --path *osd path* [ --allocator block/bluefs-wal/bluefs-db/bluefs-slow ]
+
+   Give a [0-1] number that represents quality of fragmentation in allocator.
+   0 represents case when all free space is in one chunk. 1 represents worst possible fragmentation.
+
+:command:`reshard` --path *osd path* --sharding *new sharding* [ --resharding-ctrl *control string* ]
+
+   Changes sharding of BlueStore's RocksDB. Sharding is build on top of RocksDB column families.
+   This option allows to test performance of *new sharding* without need to redeploy OSD.
+   Resharding is usually a long process, which involves walking through entire RocksDB key space
+   and moving some of them to different column families.
+   Option --resharding-ctrl provides performance control over resharding process.
+   Interrupted resharding will prevent OSD from running.
+   Interrupted resharding does not corrupt data. It is always possible to continue previous resharding,
+   or select any other sharding scheme, including reverting to original one.
+
+:command:`show-sharding` --path *osd path*
+
+   Show sharding that is currently applied to BlueStore's RocksDB.
+
+Options
+=======
+
+.. option:: --dev *device*
+
+   Add *device* to the list of devices to consider
+
+.. option:: --devs-source *device*
+
+   Add *device* to the list of devices to consider as sources for migrate operation
+
+.. option:: --dev-target *device*
+
+   Specify target *device* migrate operation or device to add for adding new DB/WAL.
+
+.. option:: --path *osd path*
+
+   Specify an osd path.  In most cases, the device list is inferred from the symlinks present in *osd path*.  This is usually simpler than explicitly specifying the device(s) with --dev.
+
+.. option:: --out-dir *dir*
+
+   Output directory for bluefs-export
+
+.. option:: -l, --log-file *log file*
+
+   file to log to
+
+.. option:: --log-level *num*
+
+   debug log level.  Default is 30 (extremely verbose), 20 is very
+   verbose, 10 is verbose, and 1 is not very verbose.
+
+.. option:: --deep
+
+   deep scrub/repair (read and validate object data, not just metadata)
+
+.. option:: --allocator *name*
+
+   Useful for *free-dump* and *free-score* actions. Selects allocator(s).
+
+.. option:: --resharding-ctrl *control string*
+
+   Provides control over resharding process. Specifies how often refresh RocksDB iterator,
+   and how large should commit batch be before committing to RocksDB. Option format is:
+   <iterator_refresh_bytes>/<iterator_refresh_keys>/<batch_commit_bytes>/<batch_commit_keys>
+   Default: 10000000/10000/1000000/1000
+
+Device labels
+=============
+
+Every BlueStore block device has a single block label at the beginning of the
+device.  You can dump the contents of the label with::
+
+  ceph-bluestore-tool show-label --dev *device*
+
+The main device will have a lot of metadata, including information
+that used to be stored in small files in the OSD data directory.  The
+auxiliary devices (db and wal) will only have the minimum required
+fields (OSD UUID, size, device type, birth time).
+
+OSD directory priming
+=====================
+
+You can generate the content for an OSD data directory that can start up a
+BlueStore OSD with the *prime-osd-dir* command::
+
+  ceph-bluestore-tool prime-osd-dir --dev *main device* --path /var/lib/ceph/osd/ceph-*id*
+
+BlueFS log rescue
+=====================
+
+Some versions of BlueStore were susceptible to BlueFS log growing extremaly large -
+beyond the point of making booting OSD impossible. This state is indicated by
+booting that takes very long and fails in _replay function.
+
+This can be fixed by::
+  ceph-bluestore-tool fsck --path *osd path* --bluefs_replay_recovery=true
+
+It is advised to first check if rescue process would be successfull::
+  ceph-bluestore-tool fsck --path *osd path* \
+  --bluefs_replay_recovery=true --bluefs_replay_recovery_disable_compact=true
+
+If above fsck is successful fix procedure can be applied.
+
+Availability
+============
+
+**ceph-bluestore-tool** is part of Ceph, a massively scalable,
+open-source, distributed storage system. Please refer to the Ceph
+documentation at http://ceph.com/docs for more information.
+
+
+See also
+========
+
+:doc:`ceph-osd <ceph-osd>`\(8)
diff --git a/doc/man/8/ceph-clsinfo.rst b/doc/man/8/ceph-clsinfo.rst
new file mode 100644
index 000000000..0188ce131
--- /dev/null
+++ b/doc/man/8/ceph-clsinfo.rst
@@ -0,0 +1,49 @@
+:orphan:
+
+===============================================
+ ceph-clsinfo -- show class object information
+===============================================
+
+.. program:: ceph-clsinfo
+
+Synopsis
+========
+
+| **ceph-clsinfo** [ *options* ] ... *filename*
+
+
+Description
+===========
+
+**ceph-clsinfo** can show name, version, and architecture information
+about a specific class object.
+
+
+Options
+=======
+
+.. option:: -n, --name
+
+   Shows the class name
+
+.. option:: -v, --version
+
+   Shows the class version
+
+.. option:: -a, --arch
+
+   Shows the class architecture
+
+
+Availability
+============
+
+**ceph-clsinfo** is part of Ceph, a massively scalable, open-source, distributed storage system. Please
+refer to the Ceph documentation at http://ceph.com/docs for more
+information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8)
diff --git a/doc/man/8/ceph-conf.rst b/doc/man/8/ceph-conf.rst
new file mode 100644
index 000000000..b8b99c2aa
--- /dev/null
+++ b/doc/man/8/ceph-conf.rst
@@ -0,0 +1,149 @@
+:orphan:
+
+==================================
+ ceph-conf -- ceph conf file tool
+==================================
+
+.. program:: ceph-conf
+
+Synopsis
+========
+
+| **ceph-conf** -c *conffile* --list-all-sections
+| **ceph-conf** -c *conffile* -L
+| **ceph-conf** -c *conffile* -l *prefix*
+| **ceph-conf** *key* -s *section1* ...
+| **ceph-conf** [-s *section* ] [-r] --lookup *key*
+| **ceph-conf** [-s *section* ] *key*
+
+
+Description
+===========
+
+**ceph-conf** is a utility for getting information from a ceph
+configuration file. As with most Ceph programs, you can specify which
+Ceph configuration file to use with the ``-c`` flag.
+
+Note that unlike other ceph tools, **ceph-conf** will *only* read from
+config files (or return compiled-in default values)--it will *not*
+fetch config values from the monitor cluster.  For this reason it is
+recommended that **ceph-conf** only be used in legacy environments
+that are strictly config-file based.  New deployments and tools should
+instead rely on either querying the monitor explicitly for
+configuration (e.g., ``ceph config get <daemon> <option>``) or use
+daemons themselves to fetch effective config options (e.g.,
+``ceph-osd -i 123 --show-config-value osd_data``).  The latter option
+has the advantages of drawing from compiled-in defaults (which
+occasionally vary between daemons), config files, and the monitor's
+config database, providing the exact value that that daemon would be
+using if it were started.
+
+Actions
+=======
+
+**ceph-conf** performs one of the following actions:
+
+.. option:: -L, --list-all-sections
+
+   list all sections in the configuration file.
+
+.. option:: -l, --list-sections *prefix*
+
+   list the sections with the given *prefix*. For example, ``--list-sections mon``
+   would list all sections beginning with ``mon``.
+
+.. option:: --lookup *key*
+
+   search and print the specified configuration setting. Note:  ``--lookup`` is
+   the default action. If no other actions are given on the command line, we will
+   default to doing a lookup.
+
+.. option:: -h, --help
+
+   print a summary of usage.
+
+
+Options
+=======
+
+.. option:: -c *conffile*
+
+   the Ceph configuration file.
+
+.. option:: --filter-key *key*
+
+   filter section list to only include sections with given *key* defined.
+
+.. option:: --filter-key-value *key* ``=`` *value*
+
+   filter section list to only include sections with given *key*/*value* pair.
+
+.. option:: --name *type.id*
+
+   the Ceph name in which the sections are searched (default 'client.admin').
+   For example, if we specify ``--name osd.0``, the following sections will be
+   searched: [osd.0], [osd], [global]
+
+.. option:: --pid *pid*
+
+   override the ``$pid`` when expanding options. For example, if an option is
+   configured like ``/var/log/$name.$pid.log``, the ``$pid`` portion in its
+   value will be substituded using the PID of **ceph-conf** instead of the
+   PID of the process specfied using the ``--name`` option.
+
+.. option:: -r, --resolve-search
+
+   search for the first file that exists and can be opened in the resulted
+   comma delimited search list.
+
+.. option:: -s, --section
+
+   additional sections to search.  These additional sections will be searched
+   before the sections that would normally be searched. As always, the first
+   matching entry we find will be returned.
+
+
+Examples
+========
+
+To find out what value osd 0 will use for the "osd data" option::
+
+        ceph-conf -c foo.conf  --name osd.0 --lookup "osd data"
+
+To find out what value will mds a use for the "log file" option::
+
+        ceph-conf -c foo.conf  --name mds.a "log file"
+
+To list all sections that begin with "osd"::
+
+        ceph-conf -c foo.conf -l osd
+
+To list all sections::
+
+        ceph-conf -c foo.conf -L
+
+To print the path of the "keyring" used by "client.0"::
+
+       ceph-conf --name client.0 -r -l keyring
+
+
+Files
+=====
+
+``/etc/ceph/$cluster.conf``, ``~/.ceph/$cluster.conf``, ``$cluster.conf``
+
+the Ceph configuration files to use if not specified.
+
+
+Availability
+============
+
+**ceph-conf** is part of Ceph, a massively scalable, open-source, distributed storage system.  Please refer
+to the Ceph documentation at http://ceph.com/docs for more
+information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
diff --git a/doc/man/8/ceph-create-keys.rst b/doc/man/8/ceph-create-keys.rst
new file mode 100644
index 000000000..20b6560b2
--- /dev/null
+++ b/doc/man/8/ceph-create-keys.rst
@@ -0,0 +1,67 @@
+:orphan:
+
+===============================================
+ceph-create-keys -- ceph keyring generate tool
+===============================================
+
+.. program:: ceph-create-keys
+
+Synopsis
+========
+
+| **ceph-create-keys** [-h] [-v] [-t seconds] [--cluster *name*] --id *id*
+
+
+Description
+===========
+
+:program:`ceph-create-keys` is a utility to generate bootstrap keyrings using
+the given monitor when it is ready.
+
+It creates following auth entities (or users)
+
+``client.admin``
+
+    and its key for your client host.
+
+``client.bootstrap-{osd, rgw, mds}``
+
+    and their keys for bootstrapping corresponding services
+
+To list all users in the cluster::
+
+    ceph auth ls
+
+
+Options
+=======
+
+.. option:: --cluster
+
+   name of the cluster (default 'ceph').
+
+.. option:: -t
+
+   time out after **seconds** (default: 600) waiting for a response from the monitor
+
+.. option:: -i, --id
+
+   id of a ceph-mon that is coming up. **ceph-create-keys** will wait until it joins quorum.
+
+.. option:: -v, --verbose
+
+   be more verbose.
+
+
+Availability
+============
+
+**ceph-create-keys** is part of Ceph, a massively scalable, open-source, distributed storage system.  Please refer
+to the Ceph documentation at http://ceph.com/docs for more
+information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8)
diff --git a/doc/man/8/ceph-debugpack.rst b/doc/man/8/ceph-debugpack.rst
new file mode 100644
index 000000000..4f2c4f2f6
--- /dev/null
+++ b/doc/man/8/ceph-debugpack.rst
@@ -0,0 +1,50 @@
+:orphan:
+
+=============================================
+ ceph-debugpack -- ceph debug packer utility
+=============================================
+
+.. program:: ceph-debugpack
+
+Synopsis
+========
+
+| **ceph-debugpack** [ *options* ] *filename.tar.gz*
+
+
+Description
+===========
+
+**ceph-debugpack** will build a tarball containing various items that are
+useful for debugging crashes. The resulting tarball can be shared with
+Ceph developers when debugging a problem.
+
+The tarball will include the binaries for ceph-mds, ceph-osd, and ceph-mon, radosgw, any
+log files, the ceph.conf configuration file, any core files we can
+find, and (if the system is running) dumps of the current cluster state
+as reported by 'ceph report'.
+
+
+Options
+=======
+
+.. option:: -c ceph.conf, --conf=ceph.conf
+
+   Use *ceph.conf* configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` to determine monitor addresses during
+   startup.
+
+
+Availability
+============
+
+**ceph-debugpack** is part of Ceph, a massively scalable, open-source, distributed storage system. Please
+refer to the Ceph documentation at http://ceph.com/docs for more
+information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8)
+:doc:`ceph-post-file <ceph-post-file>`\(8)
diff --git a/doc/man/8/ceph-dencoder.rst b/doc/man/8/ceph-dencoder.rst
new file mode 100644
index 000000000..09032aa75
--- /dev/null
+++ b/doc/man/8/ceph-dencoder.rst
@@ -0,0 +1,151 @@
+:orphan:
+
+==============================================
+ ceph-dencoder -- ceph encoder/decoder utility
+==============================================
+
+.. program:: ceph-dencoder
+
+Synopsis
+========
+
+| **ceph-dencoder** [commands...]
+
+
+Description
+===========
+
+**ceph-dencoder** is a utility to encode, decode, and dump ceph data
+structures.  It is used for debugging and for testing inter-version
+compatibility.
+
+**ceph-dencoder** takes a simple list of commands and performs them
+in order.
+
+Commands
+========
+
+.. option:: version
+
+   Print the version string for the **ceph-dencoder** binary.
+
+.. option:: import <file>
+
+   Read a binary blob of encoded data from the given file.  It will be
+   placed in an in-memory buffer.
+
+.. option:: export <file>
+
+   Write the contents of the current in-memory buffer to the given
+   file.
+
+.. option:: list_types
+
+   List the data types known to this build of **ceph-dencoder**.
+
+.. option:: type <name>
+
+   Select the given type for future ``encode`` or ``decode`` operations.
+
+.. option:: skip <bytes>
+
+   Seek <bytes> into the imported file before reading data structure, use
+   this with objects that have a preamble/header before the object of interest.
+
+.. option:: decode
+
+   Decode the contents of the in-memory buffer into an instance of the
+   previously selected type.  If there is an error, report it.
+
+.. option:: encode
+
+   Encode the contents of the in-memory instance of the previously
+   selected type to the in-memory buffer.
+
+.. option:: dump_json
+
+   Print a JSON-formatted description of the in-memory object.
+
+.. option:: count_tests
+
+   Print the number of built-in test instances of the previously
+   selected type that **ceph-dencoder** is able to generate.
+
+.. option:: select_test <n>
+
+   Select the given build-in test instance as a the in-memory instance
+   of the type.
+
+.. option:: get_features
+
+   Print the decimal value of the feature set supported by this version
+   of **ceph-dencoder**.  Each bit represents a feature.  These correspond to
+   CEPH_FEATURE_* defines in src/include/ceph_features.h.
+
+.. option:: set_features <f>
+
+   Set the feature bits provided to ``encode`` to *f*.  This allows
+   you to encode objects such that they can be understood by old
+   versions of the software (for those types that support it).
+
+Example
+=======
+
+Say you want to examine an attribute on an object stored by ``ceph-osd``.  You can do this:
+
+::
+
+    $ cd /mnt/osd.12/current/2.b_head
+    $ attr -l foo_bar_head_EFE6384B
+    Attribute "ceph.snapset" has a 31 byte value for foo_bar_head_EFE6384B
+    Attribute "ceph._" has a 195 byte value for foo_bar_head_EFE6384B
+    $ attr foo_bar_head_EFE6384B -g ceph._ -q > /tmp/a
+    $ ceph-dencoder type object_info_t import /tmp/a decode dump_json
+    { "oid": { "oid": "foo",
+          "key": "bar",
+          "snapid": -2,
+          "hash": 4024842315,
+          "max": 0},
+      "locator": { "pool": 2,
+          "preferred": -1,
+          "key": "bar"},
+      "category": "",
+      "version": "9'1",
+      "prior_version": "0'0",
+      "last_reqid": "client.4116.0:1",
+      "size": 1681,
+      "mtime": "2012-02-21 08:58:23.666639",
+      "lost": 0,
+      "wrlock_by": "unknown.0.0:0",
+      "snaps": [],
+      "truncate_seq": 0,
+      "truncate_size": 0,
+      "watchers": {}}
+
+Alternatively, perhaps you wish to dump an internal CephFS metadata object, you might
+do that like this:
+
+::
+
+   $ rados -p metadata get mds_snaptable mds_snaptable.bin
+   $ ceph-dencoder type SnapServer skip 8 import mds_snaptable.bin decode dump_json
+   { "snapserver": { "last_snap": 1,
+      "pending_noop": [],
+      "snaps": [],
+      "need_to_purge": {},
+      "pending_create": [],
+      "pending_destroy": []}} 
+
+
+Availability
+============
+
+**ceph-dencoder** is part of Ceph, a massively scalable, open-source, distributed storage system. Please
+refer to the Ceph documentation at http://ceph.com/docs for more
+information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8)
diff --git a/doc/man/8/ceph-deploy.rst b/doc/man/8/ceph-deploy.rst
new file mode 100644
index 000000000..3d099252a
--- /dev/null
+++ b/doc/man/8/ceph-deploy.rst
@@ -0,0 +1,529 @@
+:orphan:
+
+.. _ceph-deploy:
+
+=====================================
+ ceph-deploy -- Ceph deployment tool
+=====================================
+
+.. program:: ceph-deploy
+
+Synopsis
+========
+
+| **ceph-deploy** **new** [*initial-monitor-node(s)*]
+
+| **ceph-deploy** **install** [*ceph-node*] [*ceph-node*...]
+
+| **ceph-deploy** **mon** *create-initial*
+
+| **ceph-deploy** **osd** *create* *--data* *device* *ceph-node*
+
+| **ceph-deploy** **admin** [*admin-node*][*ceph-node*...]
+
+| **ceph-deploy** **purgedata** [*ceph-node*][*ceph-node*...]
+
+| **ceph-deploy** **forgetkeys**
+
+Description
+===========
+
+:program:`ceph-deploy` is a tool which allows easy and quick deployment of a
+Ceph cluster without involving complex and detailed manual configuration. It
+uses ssh to gain access to other Ceph nodes from the admin node, sudo for
+administrator privileges on them and the underlying Python scripts automates
+the manual process of Ceph installation on each node from the admin node itself.
+It can be easily run on an workstation and doesn't require servers, databases or
+any other automated tools. With :program:`ceph-deploy`, it is really easy to set
+up and take down a cluster. However, it is not a generic deployment tool. It is
+a specific tool which is designed for those who want to get Ceph up and running
+quickly with only the unavoidable initial configuration settings and without the
+overhead of installing other tools like ``Chef``, ``Puppet`` or ``Juju``. Those
+who want to customize security settings, partitions or directory locations and
+want to set up a cluster following detailed manual steps, should use other tools
+i.e, ``Chef``, ``Puppet``, ``Juju`` or ``Crowbar``.
+
+With :program:`ceph-deploy`, you can install Ceph packages on remote nodes,
+create a cluster, add monitors, gather/forget keys, add OSDs and metadata
+servers, configure admin hosts or take down the cluster.
+
+Commands
+========
+
+new
+---
+
+Start deploying a new cluster and write a configuration file and keyring for it.
+It tries to copy ssh keys from admin node to gain passwordless ssh to monitor
+node(s), validates host IP, creates a cluster with a new initial monitor node or
+nodes for monitor quorum, a ceph configuration file, a monitor secret keyring and
+a log file for the new cluster. It populates the newly created Ceph configuration
+file with ``fsid`` of cluster, hostnames and IP addresses of initial monitor
+members under ``[global]`` section.
+
+Usage::
+
+	ceph-deploy new [MON][MON...]
+
+Here, [MON] is the initial monitor hostname (short hostname i.e, ``hostname -s``).
+
+Other options like :option:`--no-ssh-copykey`, :option:`--fsid`,
+:option:`--cluster-network` and :option:`--public-network` can also be used with
+this command.
+
+If more than one network interface is used, ``public network`` setting has to be
+added under ``[global]`` section of Ceph configuration file. If the public subnet
+is given, ``new`` command will choose the one IP from the remote host that exists
+within the subnet range. Public network can also be added at runtime using
+:option:`--public-network` option with the command as mentioned above.
+
+
+install
+-------
+
+Install Ceph packages on remote hosts. As a first step it installs
+``yum-plugin-priorities`` in admin and other nodes using passwordless ssh and sudo
+so that Ceph packages from upstream repository get more priority. It then detects
+the platform and distribution for the hosts and installs Ceph normally by
+downloading distro compatible packages if adequate repo for Ceph is already added.
+``--release`` flag is used to get the latest release for installation. During
+detection of platform and distribution before installation, if it finds the
+``distro.init`` to be ``sysvinit`` (Fedora, CentOS/RHEL etc), it doesn't allow
+installation with custom cluster name and uses the default name ``ceph`` for the
+cluster.
+
+If the user explicitly specifies a custom repo url with :option:`--repo-url` for
+installation, anything detected from the configuration will be overridden and
+the custom repository location will be used for installation of Ceph packages.
+If required, valid custom repositories are also detected and installed. In case
+of installation from a custom repo a boolean is used to determine the logic
+needed to proceed with a custom repo installation. A custom repo install helper
+is used that goes through config checks to retrieve repos (and any extra repos
+defined) and installs them. ``cd_conf`` is the object built from ``argparse``
+that holds the flags and information needed to determine what metadata from the
+configuration is to be used.
+
+A user can also opt to install only the repository without installing Ceph and
+its dependencies by using :option:`--repo` option.
+
+Usage::
+
+	ceph-deploy install [HOST][HOST...]
+
+Here, [HOST] is/are the host node(s) where Ceph is to be installed.
+
+An option ``--release`` is used to install a release known as CODENAME
+(default: firefly).
+
+Other options like :option:`--testing`, :option:`--dev`, :option:`--adjust-repos`,
+:option:`--no-adjust-repos`, :option:`--repo`, :option:`--local-mirror`,
+:option:`--repo-url` and :option:`--gpg-url` can also be used with this command.
+
+
+mds
+---
+
+Deploy Ceph mds on remote hosts. A metadata server is needed to use CephFS and
+the ``mds`` command is used to create one on the desired host node. It uses the
+subcommand ``create`` to do so. ``create`` first gets the hostname and distro
+information of the desired mds host. It then tries to read the ``bootstrap-mds``
+key for the cluster and deploy it in the desired host. The key generally has a
+format of ``{cluster}.bootstrap-mds.keyring``. If it doesn't finds a keyring,
+it runs ``gatherkeys`` to get the keyring. It then creates a mds on the desired
+host under the path ``/var/lib/ceph/mds/`` in ``/var/lib/ceph/mds/{cluster}-{name}``
+format and a bootstrap keyring under ``/var/lib/ceph/bootstrap-mds/`` in
+``/var/lib/ceph/bootstrap-mds/{cluster}.keyring`` format. It then runs appropriate
+commands based on ``distro.init`` to start the ``mds``.
+
+Usage::
+
+	ceph-deploy mds create [HOST[:DAEMON-NAME]] [HOST[:DAEMON-NAME]...]
+
+The [DAEMON-NAME] is optional.
+
+
+mon
+---
+
+Deploy Ceph monitor on remote hosts. ``mon`` makes use of certain subcommands
+to deploy Ceph monitors on other nodes.
+
+Subcommand ``create-initial`` deploys for monitors defined in
+``mon initial members`` under ``[global]`` section in Ceph configuration file,
+wait until they form quorum and then gatherkeys, reporting the monitor status
+along the process. If monitors don't form quorum the command will eventually
+time out.
+
+Usage::
+
+	ceph-deploy mon create-initial
+
+Subcommand ``create`` is used to deploy Ceph monitors by explicitly specifying
+the hosts which are desired to be made monitors. If no hosts are specified it
+will default to use the ``mon initial members`` defined under ``[global]``
+section of Ceph configuration file. ``create`` first detects platform and distro
+for desired hosts and checks if hostname is compatible for deployment. It then
+uses the monitor keyring initially created using ``new`` command and deploys the
+monitor in desired host. If multiple hosts were specified during ``new`` command
+i.e, if there are multiple hosts in ``mon initial members`` and multiple keyrings
+were created then a concatenated keyring is used for deployment of monitors. In
+this process a keyring parser is used which looks for ``[entity]`` sections in
+monitor keyrings and returns a list of those sections. A helper is then used to
+collect all keyrings into a single blob that will be used to inject it to monitors
+with :option:`--mkfs` on remote nodes. All keyring files are concatenated to be
+in a directory ending with ``.keyring``. During this process the helper uses list
+of sections returned by keyring parser to check if an entity is already present
+in a keyring and if not, adds it. The concatenated keyring is used for deployment
+of monitors to desired multiple hosts.
+
+Usage::
+
+	ceph-deploy mon create [HOST] [HOST...]
+
+Here, [HOST] is hostname of desired monitor host(s).
+
+Subcommand ``add`` is used to add a monitor to an existing cluster. It first
+detects platform and distro for desired host and checks if hostname is compatible
+for deployment. It then uses the monitor keyring, ensures configuration for new
+monitor host and adds the monitor to the cluster. If the section for the monitor
+exists and defines a monitor address that will be used, otherwise it will fallback by
+resolving the hostname to an IP. If :option:`--address` is used it will override
+all other options. After adding the monitor to the cluster, it gives it some time
+to start. It then looks for any monitor errors and checks monitor status. Monitor
+errors arise if the monitor is not added in ``mon initial members``, if it doesn't
+exist in ``monmap`` and if neither ``public_addr`` nor ``public_network`` keys
+were defined for monitors. Under such conditions, monitors may not be able to
+form quorum. Monitor status tells if the monitor is up and running normally. The
+status is checked by running ``ceph daemon mon.hostname mon_status`` on remote
+end which provides the output and returns a boolean status of what is going on.
+``False`` means a monitor that is not fine even if it is up and running, while
+``True`` means the monitor is up and running correctly.
+
+Usage::
+
+	ceph-deploy mon add [HOST]
+
+	ceph-deploy mon add [HOST] --address [IP]
+
+Here, [HOST] is the hostname and [IP] is the IP address of the desired monitor
+node. Please note, unlike other ``mon`` subcommands, only one node can be
+specified at a time.
+
+Subcommand ``destroy`` is used to completely remove monitors on remote hosts.
+It takes hostnames as arguments. It stops the monitor, verifies if ``ceph-mon``
+daemon really stopped, creates an archive directory ``mon-remove`` under
+``/var/lib/ceph/``, archives old monitor directory in
+``{cluster}-{hostname}-{stamp}`` format in it and removes the monitor from
+cluster by running ``ceph remove...`` command.
+
+Usage::
+
+	ceph-deploy mon destroy [HOST] [HOST...]
+
+Here, [HOST] is hostname of monitor that is to be removed.
+
+
+gatherkeys
+----------
+
+Gather authentication keys for provisioning new nodes. It takes hostnames as
+arguments. It checks for and fetches ``client.admin`` keyring, monitor keyring
+and ``bootstrap-mds/bootstrap-osd`` keyring from monitor host. These
+authentication keys are used when new ``monitors/OSDs/MDS`` are added to the
+cluster.
+
+Usage::
+
+	ceph-deploy gatherkeys [HOST] [HOST...]
+
+Here, [HOST] is hostname of the monitor from where keys are to be pulled.
+
+
+disk
+----
+
+Manage disks on a remote host. It actually triggers the ``ceph-volume`` utility
+and its subcommands to manage disks.
+
+Subcommand ``list`` lists disk partitions and Ceph OSDs.
+
+Usage::
+
+	ceph-deploy disk list HOST
+
+
+Subcommand ``zap`` zaps/erases/destroys a device's partition table and
+contents.  It actually uses ``ceph-volume lvm zap`` remotely, alternatively
+allowing someone to remove the Ceph metadata from the logical volume.
+
+osd
+---
+
+Manage OSDs by preparing data disk on remote host. ``osd`` makes use of certain
+subcommands for managing OSDs.
+
+Subcommand ``create`` prepares a device for Ceph OSD. It first checks against
+multiple OSDs getting created and warns about the possibility of more than the
+recommended which would cause issues with max allowed PIDs in a system. It then
+reads the bootstrap-osd key for the cluster or writes the bootstrap key if not
+found.
+It then uses :program:`ceph-volume` utility's ``lvm create`` subcommand to
+prepare the disk, (and journal if using filestore) and deploy the OSD on the desired host.
+Once prepared, it gives some time to the OSD to start and checks for any
+possible errors and if found, reports to the user.
+
+Bluestore Usage::
+
+	ceph-deploy osd create --data DISK HOST
+
+Filestore Usage::
+
+	ceph-deploy osd create --data DISK --journal JOURNAL HOST
+
+
+.. note:: For other flags available, please see the man page or the --help menu
+          on ceph-deploy osd create
+
+Subcommand ``list`` lists devices associated to Ceph as part of an OSD.
+It uses the ``ceph-volume lvm list`` output that has a rich output, mapping
+OSDs to devices and other interesting information about the OSD setup.
+
+Usage::
+
+	ceph-deploy osd list HOST
+
+
+admin
+-----
+
+Push configuration and ``client.admin`` key to a remote host. It takes
+the ``{cluster}.client.admin.keyring`` from admin node and writes it under
+``/etc/ceph`` directory of desired node.
+
+Usage::
+
+	ceph-deploy admin [HOST] [HOST...]
+
+Here, [HOST] is desired host to be configured for Ceph administration.
+
+
+config
+------
+
+Push/pull configuration file to/from a remote host. It uses ``push`` subcommand
+to takes the configuration file from admin host and write it to remote host under
+``/etc/ceph`` directory. It uses ``pull`` subcommand to do the opposite i.e, pull
+the configuration file under ``/etc/ceph`` directory of remote host to admin node.
+
+Usage::
+
+	ceph-deploy config push [HOST] [HOST...]
+
+	ceph-deploy config pull [HOST] [HOST...]
+
+Here, [HOST] is the hostname of the node where config file will be pushed to or
+pulled from.
+
+
+uninstall
+---------
+
+Remove Ceph packages from remote hosts. It detects the platform and distro of
+selected host and uninstalls Ceph packages from it. However, some dependencies
+like ``librbd1`` and ``librados2`` will not be removed because they can cause
+issues with ``qemu-kvm``.
+
+Usage::
+
+	ceph-deploy uninstall [HOST] [HOST...]
+
+Here, [HOST] is hostname of the node from where Ceph will be uninstalled.
+
+
+purge
+-----
+
+Remove Ceph packages from remote hosts and purge all data. It detects the
+platform and distro of selected host, uninstalls Ceph packages and purges all
+data. However, some dependencies like ``librbd1`` and ``librados2`` will not be
+removed because they can cause issues with ``qemu-kvm``.
+
+Usage::
+
+	ceph-deploy purge [HOST] [HOST...]
+
+Here, [HOST] is hostname of the node from where Ceph will be purged.
+
+
+purgedata
+---------
+
+Purge (delete, destroy, discard, shred) any Ceph data from ``/var/lib/ceph``.
+Once it detects the platform and distro of desired host, it first checks if Ceph
+is still installed on the selected host and if installed, it won't purge data
+from it. If Ceph is already uninstalled from the host, it tries to remove the
+contents of ``/var/lib/ceph``. If it fails then probably OSDs are still mounted
+and needs to be unmounted to continue. It unmount the OSDs and tries to remove
+the contents of ``/var/lib/ceph`` again and checks for errors. It also removes
+contents of ``/etc/ceph``. Once all steps are successfully completed, all the
+Ceph data from the selected host are removed.
+
+Usage::
+
+	ceph-deploy purgedata [HOST] [HOST...]
+
+Here, [HOST] is hostname of the node from where Ceph data will be purged.
+
+
+forgetkeys
+----------
+
+Remove authentication keys from the local directory. It removes all the
+authentication keys i.e, monitor keyring, client.admin keyring, bootstrap-osd
+and bootstrap-mds keyring from the node.
+
+Usage::
+
+	ceph-deploy forgetkeys
+
+
+pkg
+---
+
+Manage packages on remote hosts. It is used for installing or removing packages
+from remote hosts. The package names for installation or removal are to be
+specified after the command. Two options :option:`--install` and
+:option:`--remove` are used for this purpose.
+
+Usage::
+
+	ceph-deploy pkg --install [PKGs] [HOST] [HOST...]
+
+	ceph-deploy pkg --remove [PKGs] [HOST] [HOST...]
+
+Here, [PKGs] is comma-separated package names and [HOST] is hostname of the
+remote node where packages are to be installed or removed from.
+
+
+Options
+=======
+
+.. option:: --address
+
+	IP address of the host node to be added to the cluster.
+
+.. option:: --adjust-repos
+
+	Install packages modifying source repos.
+
+.. option:: --ceph-conf
+
+	Use (or reuse) a given ``ceph.conf`` file.
+
+.. option:: --cluster
+
+	Name of the cluster.
+
+.. option:: --dev
+
+	Install a bleeding edge built from Git branch or tag (default: master).
+
+.. option:: --cluster-network
+
+	Specify the (internal) cluster network.
+
+.. option:: --dmcrypt
+
+	Encrypt [data-path] and/or journal devices with ``dm-crypt``.
+
+.. option:: --dmcrypt-key-dir
+
+	Directory where ``dm-crypt`` keys are stored.
+
+.. option:: --install
+
+	Comma-separated package(s) to install on remote hosts.
+
+.. option:: --fs-type
+
+	Filesystem to use to format disk ``(xfs, btrfs or ext4)``.  Note that support for btrfs and ext4 is no longer tested or recommended; please use xfs.
+
+.. option:: --fsid
+
+	Provide an alternate FSID for ``ceph.conf`` generation.
+
+.. option:: --gpg-url
+
+	Specify a GPG key url to be used with custom repos (defaults to ceph.com).
+
+.. option:: --keyrings
+
+	Concatenate multiple keyrings to be seeded on new monitors.
+
+.. option:: --local-mirror
+
+	Fetch packages and push them to hosts for a local repo mirror.
+
+.. option:: --mkfs
+
+	Inject keys to MONs on remote nodes.
+
+.. option:: --no-adjust-repos
+
+	Install packages without modifying source repos.
+
+.. option:: --no-ssh-copykey
+
+	Do not attempt to copy ssh keys.
+
+.. option:: --overwrite-conf
+
+	Overwrite an existing conf file on remote host (if present).
+
+.. option:: --public-network
+
+	Specify the public network for a cluster.
+
+.. option:: --remove
+
+	Comma-separated package(s) to remove from remote hosts.
+
+.. option:: --repo
+
+	Install repo files only (skips package installation).
+
+.. option:: --repo-url
+
+	Specify a repo url that mirrors/contains Ceph packages.
+
+.. option:: --testing
+
+	Install the latest development release.
+
+.. option:: --username
+
+	The username to connect to the remote host.
+
+.. option:: --version
+
+	The current installed version of :program:`ceph-deploy`.
+
+.. option:: --zap-disk
+
+	Destroy the partition table and content of a disk.
+
+
+Availability
+============
+
+:program:`ceph-deploy` is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the documentation at https://ceph.com/ceph-deploy/docs for more information.
+
+
+See also
+========
+
+:doc:`ceph-mon <ceph-mon>`\(8),
+:doc:`ceph-osd <ceph-osd>`\(8),
+:doc:`ceph-volume <ceph-volume>`\(8),
+:doc:`ceph-mds <ceph-mds>`\(8)
diff --git a/doc/man/8/ceph-diff-sorted.rst b/doc/man/8/ceph-diff-sorted.rst
new file mode 100644
index 000000000..99e958336
--- /dev/null
+++ b/doc/man/8/ceph-diff-sorted.rst
@@ -0,0 +1,71 @@
+:orphan:
+
+==========================================================
+ ceph-diff-sorted -- compare two sorted files line by line
+==========================================================
+
+.. program:: ceph-diff-sorted
+
+Synopsis
+========
+
+| **ceph-diff-sorted** *file1* *file2*
+
+Description
+===========
+
+:program:`ceph-diff-sorted` is a simplifed *diff* utility optimized
+for comparing two files with lines that are lexically sorted.
+
+The output is simplified in comparison to that of the standard `diff`
+tool available in POSIX systems. Angle brackets ('<' and '>') are used
+to show lines that appear in one file but not the other. The output is
+not compatible with the `patch` tool.
+
+This tool was created in order to perform diffs of large files (e.g.,
+containing billions of lines) that the standard `diff` tool cannot
+handle efficiently. Knowing that the lines are sorted allows this to
+be done efficiently with minimal memory overhead.
+
+The sorting of each file needs to be done lexcially. Most POSIX
+systems use the *LANG* environment variable to determine the `sort`
+tool's sorting order. To sort lexically we would need something such
+as:
+
+        $ LANG=C sort some-file.txt >some-file-sorted.txt
+
+Examples
+========
+
+Compare two files::
+
+        $ ceph-diff-sorted fileA.txt fileB.txt
+
+Exit Status
+===========
+
+When complete, the exit status will be set to one of the following:
+
+0
+  files same
+1
+  files different
+2
+  usage problem (e.g., wrong number of command-line arguments)
+3
+  problem opening input file
+4
+  bad file content (e.g., unsorted order or empty lines)
+
+
+Availability
+============
+
+:program:`ceph-diff-sorted` is part of Ceph, a massively scalable,
+open-source, distributed storage system.  Please refer to the Ceph
+documentation at http://ceph.com/docs for more information.
+
+See also
+========
+
+:doc:`rgw-orphan-list <rgw-orphan-list>`\(8)
diff --git a/doc/man/8/ceph-fuse.rst b/doc/man/8/ceph-fuse.rst
new file mode 100644
index 000000000..f230626b8
--- /dev/null
+++ b/doc/man/8/ceph-fuse.rst
@@ -0,0 +1,85 @@
+:orphan:
+
+=========================================
+ ceph-fuse -- FUSE-based client for ceph
+=========================================
+
+.. program:: ceph-fuse
+
+Synopsis
+========
+
+| **ceph-fuse** [-n *client.username*] [ -m *monaddr*:*port* ] *mountpoint* [ *fuse options* ]
+
+
+Description
+===========
+
+**ceph-fuse** is a FUSE ("Filesystem in USErspace") client for Ceph
+distributed file system. It will mount a ceph file system specified via the -m
+option or described by ceph.conf (see below) at the specific mount point. See
+`Mount CephFS using FUSE`_ for detailed information.
+
+The file system can be unmounted with::
+
+        fusermount -u mountpoint
+
+or by sending ``SIGINT`` to the ``ceph-fuse`` process.
+
+
+Options
+=======
+
+Any options not recognized by ceph-fuse will be passed on to libfuse.
+
+.. option:: -o opt,[opt...]
+
+   Mount options.
+
+.. option:: -d
+
+   Run in foreground, send all log output to stderr and enable FUSE debugging (-o debug).
+
+.. option:: -c ceph.conf, --conf=ceph.conf
+
+   Use *ceph.conf* configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` to determine monitor addresses during startup.
+
+.. option:: -m monaddress[:port]
+
+   Connect to specified monitor (instead of looking through ceph.conf).
+
+.. option:: -n client.{cephx-username}
+
+   Pass the name of CephX user whose secret key is be to used for mounting.
+
+.. option:: -k <path-to-keyring>
+
+   Provide path to keyring; useful when it's absent in standard locations.
+
+.. option:: --client_mountpoint/-r root_directory
+
+   Use root_directory as the mounted root, rather than the full Ceph tree.
+
+.. option:: -f
+
+   Foreground: do not daemonize after startup (run in foreground). Do not generate a pid file.
+
+.. option:: -s
+
+   Disable multi-threaded operation.
+
+Availability
+============
+
+**ceph-fuse** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at http://ceph.com/docs for more information.
+
+
+See also
+========
+
+fusermount(8),
+:doc:`ceph <ceph>`\(8)
+
+.. _Mount CephFS using FUSE: ../../../cephfs/mount-using-fuse/
diff --git a/doc/man/8/ceph-immutable-object-cache.rst b/doc/man/8/ceph-immutable-object-cache.rst
new file mode 100644
index 000000000..2e971abdc
--- /dev/null
+++ b/doc/man/8/ceph-immutable-object-cache.rst
@@ -0,0 +1,76 @@
+:orphan:
+
+======================================================================
+ ceph-immutable-object-cache -- Ceph daemon for immutable object cache
+======================================================================
+
+.. program:: ceph-immutable-object-cache
+
+Synopsis
+========
+
+| **ceph-immutable-object-cache**
+
+
+Description
+===========
+
+:program:`ceph-immutable-object-cache` is a daemon for object cache of RADOS
+objects among Ceph clusters. It will promote the objects to a local directory
+upon promote requests and future reads will be serviced from these cached
+objects.
+
+It connects to local clusters via the RADOS protocol, relying on
+default search paths to find ceph.conf files, monitor addresses and
+authentication information for them, i.e. ``/etc/ceph/$cluster.conf``,
+``/etc/ceph/$cluster.keyring``, and
+``/etc/ceph/$cluster.$name.keyring``, where ``$cluster`` is the
+human-friendly name of the cluster, and ``$name`` is the rados user to
+connect as, e.g. ``client.ceph-immutable-object-cache``.
+
+
+Options
+=======
+
+.. option:: -c ceph.conf, --conf=ceph.conf
+
+   Use ``ceph.conf`` configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` to determine monitor addresses during startup.
+
+.. option:: -m monaddress[:port]
+
+   Connect to specified monitor (instead of looking through ``ceph.conf``).
+
+.. option:: -i ID, --id ID
+
+   Set the ID portion of name for ceph-immutable-object-cache
+
+.. option:: -n TYPE.ID, --name TYPE.ID
+
+   Set the rados user name for the gateway (eg. client.ceph-immutable-object-cache)
+
+.. option:: --cluster NAME
+
+   Set the cluster name (default: ceph)
+
+.. option:: -d
+
+   Run in foreground, log to stderr
+
+.. option:: -f
+
+   Run in foreground, log to usual location
+
+
+Availability
+============
+
+:program:`ceph-immutable-object-cache` is part of Ceph, a massively scalable, open-source, distributed
+storage system. Please refer to the Ceph documentation at http://ceph.com/docs for
+more information.
+
+
+See also
+========
+
+:doc:`rbd <rbd>`\(8)
diff --git a/doc/man/8/ceph-kvstore-tool.rst b/doc/man/8/ceph-kvstore-tool.rst
new file mode 100644
index 000000000..1eb99c030
--- /dev/null
+++ b/doc/man/8/ceph-kvstore-tool.rst
@@ -0,0 +1,98 @@
+:orphan:
+
+=====================================================
+ ceph-kvstore-tool -- ceph kvstore manipulation tool
+=====================================================
+
+.. program:: ceph-kvstore-tool
+
+Synopsis
+========
+
+| **ceph-kvstore-tool** <leveldb|rocksdb|bluestore-kv> <store path> *command* [args...]
+
+
+Description
+===========
+
+:program:`ceph-kvstore-tool` is a kvstore manipulation tool. It allows users to manipulate
+leveldb/rocksdb's data (like OSD's omap) offline.
+
+Commands
+========
+
+:program:`ceph-kvstore-tool` utility uses many commands for debugging purpose
+which are as follows:
+
+:command:`list [prefix]`
+    Print key of all KV pairs stored with the URL encoded prefix.
+
+:command:`list-crc [prefix]`
+    Print CRC of all KV pairs stored with the URL encoded prefix.
+
+:command:`dump [prefix]`
+    Print key and value of all KV pairs stored with the URL encoded prefix.
+
+:command:`exists <prefix> [key]`
+    Check if there is any KV pair stored with the URL encoded prefix. If key
+    is also specified, check for the key with the prefix instead.
+
+:command:`get <prefix> <key> [out <file>]`
+    Get the value of the KV pair stored with the URL encoded prefix and key.
+    If file is also specified, write the value to the file.
+
+:command:`crc <prefix> <key>`
+    Get the CRC of the KV pair stored with the URL encoded prefix and key. 
+
+:command:`get-size [<prefix> <key>]`
+    Get estimated store size or size of value specified by prefix and key.
+
+:command:`set <prefix> <key> [ver <N>|in <file>]`
+    Set the value of the KV pair stored with the URL encoded prefix and key. 
+    The value could be *version_t* or text.
+
+:command:`rm <prefix> <key>`
+    Remove the KV pair stored with the URL encoded prefix and key.
+
+:command:`rm-prefix <prefix>`
+    Remove all KV pairs stored with the URL encoded prefix.
+
+:command:`store-copy <path> [num-keys-per-tx]`
+    Copy all KV pairs to another directory specified by ``path``. 
+    [num-keys-per-tx] is the number of KV pairs copied for a transaction.
+
+:command:`store-crc <path>`
+    Store CRC of all KV pairs to a file specified by ``path``.
+
+:command:`compact`
+    Subcommand ``compact`` is used to compact all data of kvstore. It will open
+    the database, and trigger a database's compaction. After compaction, some 
+    disk space may be released.
+
+:command:`compact-prefix <prefix>`
+    Compact all entries specified by the URL encoded prefix. 
+   
+:command:`compact-range <prefix> <start> <end>`
+    Compact some entries specified by the URL encoded prefix and range.
+
+:command:`destructive-repair`
+    Make a (potentially destructive) effort to recover a corrupted database.
+    Note that in the case of rocksdb this may corrupt an otherwise uncorrupted
+    database--use this only as a last resort!
+
+:command:`stats`
+    Prints statistics from underlying key-value database. This is only for informative purposes.
+    Format and information content may vary between releases. For RocksDB information includes
+    compactions stats, performance counters, memory usage and internal RocksDB stats. 
+
+Availability
+============
+
+**ceph-kvstore-tool** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at http://ceph.com/docs for more information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8)
diff --git a/doc/man/8/ceph-mds.rst b/doc/man/8/ceph-mds.rst
new file mode 100644
index 000000000..a07b9105f
--- /dev/null
+++ b/doc/man/8/ceph-mds.rst
@@ -0,0 +1,82 @@
+:orphan:
+
+=========================================
+ ceph-mds -- ceph metadata server daemon
+=========================================
+
+.. program:: ceph-mds
+
+Synopsis
+========
+
+| **ceph-mds** -i <*ID*> [flags]
+
+
+Description
+===========
+
+**ceph-mds** is the metadata server daemon for the Ceph distributed file
+system. One or more instances of ceph-mds collectively manage the file
+system namespace, coordinating access to the shared OSD cluster.
+
+Each ceph-mds daemon instance should have a unique name. The name is used
+to identify daemon instances in the ceph.conf.
+
+Once the daemon has started, the monitor cluster will normally assign
+it a logical rank, or put it in a standby pool to take over for
+another daemon that crashes. Some of the specified options can cause
+other behaviors.
+
+
+Options
+=======
+
+.. option:: -f, --foreground
+
+   Foreground: do not daemonize after startup (run in foreground). Do
+   not generate a pid file. Useful when run via :doc:`ceph-run
+   <ceph-run>`\(8).
+
+.. option:: -d
+
+   Debug mode: like ``-f``, but also send all log output to stderr.
+
+.. option:: --setuser userorgid
+
+   Set uid after starting.  If a username is specified, the user
+   record is looked up to get a uid and a gid, and the gid is also set
+   as well, unless --setgroup is also specified.
+
+.. option:: --setgroup grouporgid
+
+   Set gid after starting.  If a group name is specified the group
+   record is looked up to get a gid.
+
+.. option:: -c ceph.conf, --conf=ceph.conf
+
+   Use *ceph.conf* configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` to determine monitor addresses during
+   startup.
+
+.. option:: -m monaddress[:port]
+
+   Connect to specified monitor (instead of looking through
+   ``ceph.conf``).
+
+.. option:: --id/-i ID
+
+   Set ID portion of the MDS name.
+
+Availability
+============
+
+**ceph-mds** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to the Ceph documentation at
+http://ceph.com/docs for more information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
+:doc:`ceph-mon <ceph-mon>`\(8),
+:doc:`ceph-osd <ceph-osd>`\(8)
diff --git a/doc/man/8/ceph-mon.rst b/doc/man/8/ceph-mon.rst
new file mode 100644
index 000000000..aeb6a38fc
--- /dev/null
+++ b/doc/man/8/ceph-mon.rst
@@ -0,0 +1,99 @@
+:orphan:
+
+=================================
+ ceph-mon -- ceph monitor daemon
+=================================
+
+.. program:: ceph-mon
+
+Synopsis
+========
+
+| **ceph-mon** -i *monid* [ --mon-data *mondatapath* ]
+
+
+Description
+===========
+
+**ceph-mon** is the cluster monitor daemon for the Ceph distributed
+file system. One or more instances of **ceph-mon** form a Paxos
+part-time parliament cluster that provides extremely reliable and
+durable storage of cluster membership, configuration, and state.
+
+The *mondatapath* refers to a directory on a local file system storing
+monitor data. It is normally specified via the ``mon data`` option in
+the configuration file.
+
+Options
+=======
+
+.. option:: -f, --foreground
+
+   Foreground: do not daemonize after startup (run in foreground). Do
+   not generate a pid file. Useful when run via :doc:`ceph-run <ceph-run>`\(8).
+
+.. option:: -d
+
+   Debug mode: like ``-f``, but also send all log output to stderr.
+
+.. option:: --setuser userorgid
+
+   Set uid after starting.  If a username is specified, the user
+   record is looked up to get a uid and a gid, and the gid is also set
+   as well, unless --setgroup is also specified.
+
+.. option:: --setgroup grouporgid
+
+   Set gid after starting.  If a group name is specified the group
+   record is looked up to get a gid.
+
+.. option:: -c ceph.conf, --conf=ceph.conf
+
+   Use *ceph.conf* configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` to determine monitor addresses during
+   startup.
+
+.. option:: --mkfs
+
+   Initialize the ``mon data`` directory with seed information to form
+   and initial ceph file system or to join an existing monitor
+   cluster.  Three pieces of information must be provided:
+
+   - The cluster fsid.  This can come from a monmap (``--monmap <path>``) or
+     explicitly via ``--fsid <uuid>``.
+   - A list of monitors and their addresses.  This list of monitors
+     can come from a monmap (``--monmap <path>``), the ``mon host``
+     configuration value (in *ceph.conf* or via ``-m
+     host1,host2,...``), or (for backward compatibility) the deprecated ``mon addr`` lines in *ceph.conf*.  If this
+     monitor is to be part of the initial monitor quorum for a new
+     Ceph cluster, then it must be included in the initial list,
+     matching either the name or address of a monitor in the list.
+     When matching by address, either the ``public addr`` or ``public
+     subnet`` options may be used.
+   - The monitor secret key ``mon.``.  This must be included in the
+     keyring provided via ``--keyring <path>``.
+
+.. option:: --keyring
+
+   Specify a keyring for use with ``--mkfs``.
+
+.. option:: --no-config-file
+
+    Signal that we don't want to rely on a *ceph.conf*, either user provided
+    or the default, to run the daemon.  This will entail providing all
+    necessary options to the daemon as arguments.
+
+Availability
+============
+
+**ceph-mon** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer
+to the Ceph documentation at http://ceph.com/docs for more
+information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
+:doc:`ceph-mds <ceph-mds>`\(8),
+:doc:`ceph-osd <ceph-osd>`\(8)
diff --git a/doc/man/8/ceph-objectstore-tool.rst b/doc/man/8/ceph-objectstore-tool.rst
new file mode 100644
index 000000000..19acc5913
--- /dev/null
+++ b/doc/man/8/ceph-objectstore-tool.rst
@@ -0,0 +1,488 @@
+:orphan:
+
+==============================================================
+ceph-objectstore-tool -- modify or examine the state of an OSD
+==============================================================
+
+Synopsis
+========
+
+
+| **ceph-objectstore-tool** --data-path *path to osd* [--op *list* ]
+
+
+
+Possible object operations:
+
+* (get|set)-bytes [file]
+* set-(attr|omap) [file]
+* (get|rm)-attr|omap)
+* get-omaphdr
+* set-omaphdr [file]
+* list-attrs
+* list-omap
+* remove|removeall
+* dump
+* set-size
+* clear-data-digest
+* remove-clone-metadata 
+
+
+Description
+===========
+
+**ceph-objectstore-tool** is a tool for modifying the state of an OSD. It facilitates manipulating an object's content, removing an object, listing the omap, manipulating the omap header, manipulating the omap key, listing object attributes, and manipulating object attribute keys.
+
+**ceph-objectstore-tool** provides two main modes: (1) a mode that specifies the "--op" argument (for example, **ceph-objectstore-tool** --data-path $PATH_TO_OSD --op $SELECT_OPERATION [--pgid $PGID] [--dry-run]), and (2) a mode for positional object operations. If the second mode is used, the object can be specified by ID or by the JSON output of the --op list. 
+
+| **ceph-objectstore-tool** --data-path *path to osd* [--pgid *$PG_ID* ][--op *command*]
+| **ceph-objectstore-tool** --data-path *path to osd* [ --op *list $OBJECT_ID*]
+
+Possible -op commands::
+
+* info
+* log
+* remove
+* mkfs
+* fsck
+* repair
+* fuse
+* dup
+* export
+* export-remove
+* import
+* list
+* list-slow-omap
+* fix-lost
+* list-pgs
+* dump-journal
+* dump-super
+* meta-list
+* get-osdmap
+* set-osdmap
+* get-inc-osdmap
+* set-inc-osdmap
+* mark-complete
+* reset-last-complete
+* apply-layour-settings
+* update-mon-db
+* dump-export
+* trim-pg-log
+
+Installation
+============
+
+The `ceph-osd` package provides **ceph-objectstore-tool**.
+
+
+Examples
+========
+
+Modifying Objects
+-----------------
+These commands modify state of an OSD. The OSD must not be running when ceph-objectstore-tool is used.
+
+Listing Objects and Placement Groups
+------------------------------------
+
+Make sure that the target OSD is down::
+
+   systemctl status ceph-osd@$OSD_NUMBER
+
+Identify all objects within an OSD::
+
+   ceph-objectstore-tool --data-path $PATH_TO_OSD --op list
+
+Identify all objects within a placement group::
+
+   ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID --op list
+
+Identify the placement group (PG) that an object belongs to::
+
+   ceph-objectstore-tool --data-path $PATH_TO_OSD --op list $OBJECT_ID
+
+
+Fixing Lost Objects   
+-------------------
+
+Make sure the OSD is down::
+
+   systemctl status ceph-osd@OSD_NUMBER
+
+Fix all lost objects::
+
+   ceph-objectstore-tool --data-path $PATH_TO_OSD --op fix-lost
+
+Fix all the lost objects within a specified placement group::
+
+   ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID --op fix-lost
+
+Fix a lost object by its identifier::
+
+   ceph-objectstore-tool --data-path $PATH_TO_OSD --op fix-lost $OBJECT_ID
+
+Fix legacy lost objects::
+
+   ceph-objectstore-tool --data-path $PATH_TO_OSD --op fix-lost
+
+
+Manipulating an object's content
+--------------------------------
+
+1. Make sure that the target OSD is down::
+   
+    systemctl status ceph-osd@$OSD_NUMBER
+
+2. Find the object by listing the objects of the OSD or placement group.
+
+3. Before setting the bytes on the object, make a backup and a working copy of the object. Here is the syntactic form of that command::
+   
+    ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT get-bytes > $OBJECT_FILE_NAME
+
+For example::
+
+   [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' get-bytes > zone_info.default.backup
+
+   [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' get-bytes > zone_info.default.working-copy
+
+The first command creates the back-up copy, and the second command creates the working copy.
+
+4. Edit the working copy object file.
+
+5. Set the bytes of the object::
+     
+     ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT set-bytes < $OBJECT_FILE_NAME
+
+For example::
+
+   [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' set-bytes < zone_info.default.working-copy
+ 
+
+Removing an Object
+------------------
+
+Use **ceph-objectstore-tool** to remove objects. When an object is removed, its contents and references are removed from the placement group (PG).
+
+Remove an object (syntax)::
+
+   ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT remove
+
+Remove an object (example)::
+
+[root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' remove
+
+
+Listing the Object Map
+----------------------
+
+Use the ceph-objectstore-tool to list the contents of the object map (OMAP). The output is a list of keys.
+
+
+1. Verify the appropriate OSD is down:
+
+   Syntax::
+
+    systemctl status ceph-osd@$OSD_NUMBER
+
+   Example::
+
+    [root@osd ~]# systemctl status ceph-osd@1
+
+2. List the object map:
+
+   Syntax::
+
+    ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT list-omap
+
+   Example::
+
+    [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' list-omap
+
+
+Manipulating the Object Map Header
+----------------------------------
+The **ceph-objectstore-tool** utility will output the object map (OMAP) header with the values associated with the object's keys.
+
+Note: If using FileStore as the OSD backend object store, then add the `--journal-path $PATH_TO_JOURNAL` argument when getting or setting the object map header, where the `$PATH_TO_JOURNAL` variable is the absolute path to the OSD journal; for example `/var/lib/ceph/osd/ceph-0/journal`.
+
+Prerequisites
+^^^^^^^^^^^^^
+
+    * Having root access to the Ceph OSD node.
+    * Stopping the ceph-osd daemon. 
+
+Procedure
+^^^^^^^^^
+
+  Verify that the target OSD is down:
+
+  Syntax::
+
+    systemctl status ceph-osd@$OSD_NUMBER
+
+  Example::
+
+    [root@osd ~]# systemctl status ceph-osd@1
+
+  Get the object map header:
+
+  Syntax::
+
+        ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT get-omaphdr > $OBJECT_MAP_FILE_NAME
+
+  Example::
+
+        [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}'  get-omaphdr > zone_info.default.omaphdr.txt
+
+  Set the object map header:
+
+  Syntax::
+
+        ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT get-omaphdr < $OBJECT_MAP_FILE_NAME
+
+  Example::
+
+   [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}'  set-omaphdr < zone_info.default.omaphdr.txt
+
+
+Manipulating the Object Map Key
+-------------------------------
+
+Use the **ceph-objectstore-tool** utility to change the object map (OMAP) key. You need to provide the data path, the placement group identifier (PG ID), the object, and the key in the OMAP.
+Note
+
+If using FileStore as the OSD backend object store, then add the `--journal-path $PATH_TO_JOURNAL` argument when getting, setting or removing the object map key, where the `$PATH_TO_JOURNAL` variable is the absolute path to the OSD journal; for example `/var/lib/ceph/osd/ceph-0/journal`.
+
+Prerequisites
+
+    * Having root access to the Ceph OSD node.
+    * Stopping the ceph-osd daemon. 
+
+Procedure
+
+    Get the object map key:
+
+    Syntax::
+     
+       ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT get-omap $KEY > $OBJECT_MAP_FILE_NAME
+
+   Example::
+
+    [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}'  get-omap "" > zone_info.default.omap.txt
+
+   Set the object map key:
+
+   Syntax::
+
+    ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT set-omap $KEY < $OBJECT_MAP_FILE_NAME
+
+   Example::
+
+    [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' set-omap "" < zone_info.default.omap.txt
+
+   Remove the object map key:
+
+   Syntax::
+
+    ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT rm-omap $KEY
+
+   Example::
+
+    [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' rm-omap ""
+
+
+Listing an Object's Attributes
+-------------------------------
+
+Use the **ceph-objectstore-tool** utility to list an object's attributes. The output provides you with the object's keys and values.
+Note
+
+If you are using FileStore as the OSD backend object store and the journal is on a different disk, you must add the `--journal-path $PATH_TO_JOURNAL` argument when listing an object's attributes, where the `$PATH_TO_JOURNAL` variable is the absolute path to the OSD journal; for example `/var/lib/ceph/osd/ceph-0/journal`.
+
+Prerequisites
+^^^^^^^^^^^^^
+
+    * Having root access to the Ceph OSD node.
+    * Stopping the ceph-osd daemon. 
+
+Procedure
+^^^^^^^^^
+
+   Verify that the target OSD is down:
+
+   Syntax::
+
+    systemctl status ceph-osd@$OSD_NUMBER
+
+   Example::
+
+    [root@osd ~]# systemctl status ceph-osd@1
+
+   List the object's attributes:
+
+   Syntax::
+
+    ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT list-attrs
+
+   Example::
+
+    [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' list-attrs
+
+
+MANIPULATING THE OBJECT ATTRIBUTE KEY
+-------------------------------------
+
+Use the ceph-objectstore-tool utility to change an object's attributes. To manipulate the object's attributes you need the data and journal paths, the placement group identifier (PG ID), the object, and the key in the object's attribute.
+Note
+
+If you are using FileStore as the OSD backend object store and the journal is on a different disk, you must add the `--journal-path $PATH_TO_JOURNAL` argument when getting, setting or removing the object's attributes. Where the `$PATH_TO_JOURNAL` variable is the absolute path to the OSD journal, for example `/var/lib/ceph/osd/ceph-0/journal`.
+
+Prerequisites
+
+    * Having root access to the Ceph OSD node.  
+    * Stopping the ceph-osd daemon. 
+
+Procedure
+
+    Verify that the target OSD is down.
+
+ Syntax::
+
+    systemctl status ceph-osd@$OSD_NUMBER
+
+ Example::
+
+    [root@osd ~]# systemctl status ceph-osd@1
+
+ Get the object's attributes:
+
+ Syntax::
+
+   ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT get-attrs $KEY > $OBJECT_ATTRS_FILE_NAME
+
+ Example::
+
+   [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0  --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' get-attrs "oid" > zone_info.default.attr.txt
+
+ Set an object's attributes:
+
+ Syntax::
+
+   ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT  set-attrs $KEY < $OBJECT_ATTRS_FILE_NAME
+
+ Example::
+
+   [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' set-attrs "oid" < zone_info.default.attr.txt
+
+ Remove an object's attributes:
+
+ Syntax::
+
+   ceph-objectstore-tool --data-path $PATH_TO_OSD --pgid $PG_ID $OBJECT rm-attrs $KEY
+
+ Example::
+
+   [root@osd ~]# ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --pgid 0.1c '{"oid":"zone_info.default","key":"","snapid":-2,"hash":235010478,"max":0,"pool":11,"namespace":""}' rm-attrs "oid"
+
+
+Options
+=======
+
+.. option:: --help          
+
+   produce help message
+
+.. option:: --type arg        
+
+   Arg is one of [bluestore (default), filestore, memstore]. This option is needed only if the tool can't tell the type from --data-path.
+ 
+.. option:: --data-path arg
+
+   path to object store, mandatory
+   
+.. option:: --journal-path arg
+
+   path to journal, use if tool can't find it
+   
+.. option:: --pgid arg
+
+   PG id, mandatory for info, log, remove, export, export-remove, mark-complete, trim-pg-log, and mandatory for apply-layout-settings if --pool is not specified
+                             
+.. option:: --pool arg
+
+   Pool name, mandatory for apply-layout-settings if --pgid is not specified
+
+.. option:: --op arg
+
+   Arg is one of [info, log, remove, mkfs, fsck, repair, fuse, dup, export, export-remove, import, list, fix-lost, list-pgs, dump-journal, dump-super, meta-list, get-osdmap, set-osdmap, get-inc-osdmap, set-inc-osdmap, mark-complete, reset-last-complete, apply-layout-settings, update-mon-db, dump-export, trim-pg-log]
+
+.. option:: --epoch arg
+
+   epoch# for get-osdmap and get-inc-osdmap, the current epoch in use if not specified
+
+.. option:: --file arg             
+   
+   path of file to export, export-remove, import, get-osdmap, set-osdmap, get-inc-osdmap or set-inc-osdmap
+
+.. option:: --mon-store-path arg
+
+   path of monstore to update-mon-db
+
+.. option:: --fsid arg
+
+   fsid for new store created by mkfs
+
+.. option:: --target-data-path arg
+
+   path of target object store (for --op dup)
+   
+.. option:: --mountpoint arg
+
+   fuse mountpoint
+
+.. option:: --format arg (=json-pretty) 
+
+   Output format which may be json, json-pretty, xml, xml-pretty
+
+.. option:: --debug
+
+   Enable diagnostic output to stderr
+
+.. option:: --force
+
+   Ignore some types of errors and proceed with operation - USE WITH CAUTION: CORRUPTION POSSIBLE NOW OR IN THE FUTURE
+
+.. option:: --skip-journal-replay
+
+   Disable journal replay
+
+.. option:: --skip-mount-omap
+
+   Disable mounting of omap
+
+.. option:: --head
+
+   Find head/snapdir when searching for objects by name
+
+.. option:: --dry-run
+
+   Don't modify the objectstore
+
+.. option:: --namespace arg
+
+   Specify namespace when searching for objects
+
+.. option:: --rmtype arg      
+
+   Specify corrupting object removal 'snapmap' or 'nosnapmap' - TESTING USE ONLY
+
+
+
+Error Codes
+===========
+"Mount failed with '(11) Resource temporarily unavailable" - This might mean that you have attempted to run **ceph-objectstore-tool** on a running OSD.
+
+Availability
+============
+
+**ceph-objectstore-tool** is part of Ceph, a massively scalable, open-source, distributed storage system. **ceph-objectstore-tool** is provided by the package `ceph-osd`. Refer to the Ceph documentation at htpp://ceph.com/docs for more information.
diff --git a/doc/man/8/ceph-osd.rst b/doc/man/8/ceph-osd.rst
new file mode 100644
index 000000000..2e9424b11
--- /dev/null
+++ b/doc/man/8/ceph-osd.rst
@@ -0,0 +1,140 @@
+:orphan:
+
+.. _ceph_osd-daemon:
+
+========================================
+ ceph-osd -- ceph object storage daemon
+========================================
+
+.. program:: ceph-osd
+
+Synopsis
+========
+
+| **ceph-osd** -i *osdnum* [ --osd-data *datapath* ] [ --osd-journal
+  *journal* ] [ --mkfs ] [ --mkjournal ] [--flush-journal] [--check-allows-journal] [--check-wants-journal] [--check-needs-journal] [ --mkkey ] [ --osdspec-affinity ]
+
+
+Description
+===========
+
+**ceph-osd** is the object storage daemon for the Ceph distributed file
+system. It is responsible for storing objects on a local file system
+and providing access to them over the network.
+
+The datapath argument should be a directory on a xfs file system
+where the object data resides. The journal is optional, and is only
+useful performance-wise when it resides on a different disk than
+datapath with low latency (ideally, an NVRAM device).
+
+
+Options
+=======
+
+.. option:: -f, --foreground
+
+   Foreground: do not daemonize after startup (run in foreground). Do
+   not generate a pid file. Useful when run via :doc:`ceph-run <ceph-run>`\(8).
+
+.. option:: -d
+
+   Debug mode: like ``-f``, but also send all log output to stderr.
+
+.. option:: --setuser userorgid
+
+   Set uid after starting.  If a username is specified, the user
+   record is looked up to get a uid and a gid, and the gid is also set
+   as well, unless --setgroup is also specified.
+
+.. option:: --setgroup grouporgid
+
+   Set gid after starting.  If a group name is specified the group
+   record is looked up to get a gid.
+
+.. option:: --osd-data osddata
+
+   Use object store at *osddata*.
+
+.. option:: --osd-journal journal
+
+   Journal updates to *journal*.
+
+.. option:: --check-wants-journal
+
+   Check whether a journal is desired.
+
+.. option:: --check-allows-journal
+
+   Check whether a journal is allowed.
+
+.. option:: --check-needs-journal
+
+   Check whether a journal is required.
+
+.. option:: --mkfs
+
+   Create an empty object repository. This also initializes the journal
+   (if one is defined).
+
+.. option:: --mkkey
+
+   Generate a new secret key. This is normally used in combination
+   with ``--mkfs`` as it is more convenient than generating a key by
+   hand with :doc:`ceph-authtool <ceph-authtool>`\(8).
+
+.. option:: --mkjournal
+
+   Create a new journal file to match an existing object repository.
+   This is useful if the journal device or file is wiped out due to a
+   disk or file system failure.
+
+.. option:: --flush-journal
+
+   Flush the journal to permanent store. This runs in the foreground
+   so you know when it's completed. This can be useful if you want to
+   resize the journal or need to otherwise destroy it: this guarantees
+   you won't lose data.
+
+.. option:: --get-cluster-fsid
+
+   Print the cluster fsid (uuid) and exit.
+
+.. option:: --get-osd-fsid
+
+   Print the OSD's fsid and exit.  The OSD's uuid is generated at
+   --mkfs time and is thus unique to a particular instantiation of
+   this OSD.
+
+.. option:: --get-journal-fsid
+
+   Print the journal's uuid.  The journal fsid is set to match the OSD
+   fsid at --mkfs time.
+
+.. option:: -c ceph.conf, --conf=ceph.conf
+
+   Use *ceph.conf* configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` for runtime configuration options.
+
+.. option:: -m monaddress[:port]
+
+   Connect to specified monitor (instead of looking through
+   ``ceph.conf``).
+
+.. option:: --osdspec-affinity
+
+   Set an affinity to a certain OSDSpec.
+   This option can only be used in conjunction with --mkfs.
+
+Availability
+============
+
+**ceph-osd** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at http://ceph.com/docs for more information.
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
+:doc:`ceph-mds <ceph-mds>`\(8),
+:doc:`ceph-mon <ceph-mon>`\(8),
+:doc:`ceph-authtool <ceph-authtool>`\(8)
diff --git a/doc/man/8/ceph-post-file.rst b/doc/man/8/ceph-post-file.rst
new file mode 100644
index 000000000..7e4899f5a
--- /dev/null
+++ b/doc/man/8/ceph-post-file.rst
@@ -0,0 +1,71 @@
+:orphan:
+
+==================================================
+ ceph-post-file -- post files for ceph developers
+==================================================
+
+.. program:: ceph-post-file
+
+Synopsis
+========
+
+| **ceph-post-file** [-d *description] [-u *user*] *file or dir* ...
+
+
+Description
+===========
+
+**ceph-post-file** will upload files or directories to ceph.com for
+later analysis by Ceph developers.
+
+Each invocation uploads files or directories to a separate directory
+with a unique tag.  That tag can be passed to a developer or
+referenced in a bug report (http://tracker.ceph.com/).  Once the
+upload completes, the directory is marked non-readable and
+non-writeable to prevent access or modification by other users.
+
+Warning
+=======
+
+Basic measures are taken to make posted data be visible only to
+developers with access to ceph.com infrastructure. However, users
+should think twice and/or take appropriate precautions before
+posting potentially sensitive data (for example, logs or data
+directories that contain Ceph secrets).
+
+
+Options
+=======
+
+.. option:: -d *description*, --description *description*
+
+   Add a short description for the upload.  This is a good opportunity
+   to reference a bug number.  There is no default value.
+
+.. option:: -u *user*
+
+   Set the user metadata for the upload.  This defaults to `whoami`@`hostname -f`.
+
+Examples
+========
+
+To upload a single log::
+
+   ceph-post-file /var/log/ceph/ceph-mon.`hostname`.log
+
+To upload several directories::
+
+   ceph-post-file -d 'mon data directories' /var/log/ceph/mon/*
+
+
+Availability
+============
+
+**ceph-post-file** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at http://ceph.com/docs for more information.
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
+:doc:`ceph-debugpack <ceph-debugpack>`\(8),
diff --git a/doc/man/8/ceph-rbdnamer.rst b/doc/man/8/ceph-rbdnamer.rst
new file mode 100644
index 000000000..d31f61658
--- /dev/null
+++ b/doc/man/8/ceph-rbdnamer.rst
@@ -0,0 +1,36 @@
+:orphan:
+
+==================================================
+ ceph-rbdnamer -- udev helper to name RBD devices
+==================================================
+
+.. program:: ceph-rbdnamer
+
+
+Synopsis
+========
+
+| **ceph-rbdnamer** *num*
+
+
+Description
+===========
+
+**ceph-rbdnamer** prints the pool, namespace, image and snapshot names
+for a given RBD device to stdout. It is used by `udev` device manager
+to set up RBD device symlinks. The appropriate `udev` rules are
+provided in a file named `50-rbd.rules`.
+
+Availability
+============
+
+**ceph-rbdnamer** is part of Ceph, a massively scalable, open-source, distributed storage system.  Please
+refer to the Ceph documentation at http://ceph.com/docs for more
+information.
+
+
+See also
+========
+
+:doc:`rbd <rbd>`\(8),
+:doc:`ceph <ceph>`\(8)
diff --git a/doc/man/8/ceph-run.rst b/doc/man/8/ceph-run.rst
new file mode 100644
index 000000000..ed76c2848
--- /dev/null
+++ b/doc/man/8/ceph-run.rst
@@ -0,0 +1,45 @@
+:orphan:
+
+=========================================
+ ceph-run -- restart daemon on core dump
+=========================================
+
+.. program:: ceph-run
+
+Synopsis
+========
+
+| **ceph-run** *command* ...
+
+
+Description
+===========
+
+**ceph-run** is a simple wrapper that will restart a daemon if it exits
+with a signal indicating it crashed and possibly core dumped (that is,
+signals 3, 4, 5, 6, 8, or 11).
+
+The command should run the daemon in the foreground. For Ceph daemons,
+that means the ``-f`` option.
+
+
+Options
+=======
+
+None
+
+
+Availability
+============
+
+**ceph-run** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at http://ceph.com/docs for more information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
+:doc:`ceph-mon <ceph-mon>`\(8),
+:doc:`ceph-mds <ceph-mds>`\(8),
+:doc:`ceph-osd <ceph-osd>`\(8)
diff --git a/doc/man/8/ceph-syn.rst b/doc/man/8/ceph-syn.rst
new file mode 100644
index 000000000..a30c460cb
--- /dev/null
+++ b/doc/man/8/ceph-syn.rst
@@ -0,0 +1,99 @@
+:orphan:
+
+===============================================
+ ceph-syn -- ceph synthetic workload generator
+===============================================
+
+.. program:: ceph-syn
+
+Synopsis
+========
+
+| **ceph-syn** [ -m *monaddr*:*port* ] --syn *command* *...*
+
+
+Description
+===========
+
+**ceph-syn** is a simple synthetic workload generator for the Ceph
+distributed file system. It uses the userspace client library to
+generate simple workloads against a currently running file system. The
+file system need not be mounted via ceph-fuse(8) or the kernel client.
+
+One or more ``--syn`` command arguments specify the particular
+workload, as documented below.
+
+
+Options
+=======
+
+.. option:: -d
+
+   Detach from console and daemonize after startup.
+
+.. option:: -c ceph.conf, --conf=ceph.conf
+
+   Use *ceph.conf* configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` to determine monitor addresses during
+   startup.
+
+.. option:: -m monaddress[:port]
+
+   Connect to specified monitor (instead of looking through
+   ``ceph.conf``).
+
+.. option:: --num_client num
+
+   Run num different clients, each in a separate thread.
+
+.. option:: --syn workloadspec
+
+   Run the given workload. May be specified as many times as
+   needed. Workloads will normally run sequentially.
+
+
+Workloads
+=========
+
+Each workload should be preceded by ``--syn`` on the command
+line. This is not a complete list.
+
+:command:`mknap` *path* *snapname*
+  Create a snapshot called *snapname* on *path*.
+
+:command:`rmsnap` *path* *snapname*
+  Delete snapshot called *snapname* on *path*.
+
+:command:`rmfile` *path*
+  Delete/unlink *path*.
+
+:command:`writefile` *sizeinmb* *blocksize*
+  Create a file, named after our client id, that is *sizeinmb* MB by
+  writing *blocksize* chunks.
+
+:command:`readfile` *sizeinmb* *blocksize*
+  Read file, named after our client id, that is *sizeinmb* MB by
+  writing *blocksize* chunks.
+
+:command:`rw` *sizeinmb* *blocksize*
+  Write file, then read it back, as above.
+
+:command:`makedirs` *numsubdirs* *numfiles* *depth*
+  Create a hierarchy of directories that is *depth* levels deep. Give
+  each directory *numsubdirs* subdirectories and *numfiles* files.
+
+:command:`walk`
+  Recursively walk the file system (like find).
+
+
+Availability
+============
+
+**ceph-syn** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at http://ceph.com/docs for more information.
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
+:doc:`ceph-fuse <ceph-fuse>`\(8)
diff --git a/doc/man/8/ceph-volume-systemd.rst b/doc/man/8/ceph-volume-systemd.rst
new file mode 100644
index 000000000..b8578796c
--- /dev/null
+++ b/doc/man/8/ceph-volume-systemd.rst
@@ -0,0 +1,55 @@
+:orphan:
+
+=======================================================
+ ceph-volume-systemd -- systemd ceph-volume helper tool
+=======================================================
+
+.. program:: ceph-volume-systemd
+
+Synopsis
+========
+
+| **ceph-volume-systemd** *systemd instance name*
+
+
+Description
+===========
+:program:`ceph-volume-systemd` is a systemd helper tool that receives input
+from (dynamically created) systemd units so that activation of OSDs can
+proceed.
+
+It translates the input into a system call to ceph-volume for activation
+purposes only.
+
+
+Examples
+========
+Its input is the ``systemd instance name`` (represented by ``%i`` in a systemd
+unit), and it should be in the following format::
+
+    <ceph-volume subcommand>-<extra metadata>
+
+In the case of ``lvm`` a call could look like::
+
+    /usr/bin/ceph-volume-systemd lvm-0-8715BEB4-15C5-49DE-BA6F-401086EC7B41
+
+Which in turn will call ``ceph-volume`` in the following way::
+
+    ceph-volume lvm trigger  0-8715BEB4-15C5-49DE-BA6F-401086EC7B41
+
+Any other subcommand will need to have implemented a ``trigger`` command that
+can consume the extra metadata in this format.
+
+
+Availability
+============
+
+:program:`ceph-volume-systemd` is part of Ceph, a massively scalable,
+open-source, distributed storage system. Please refer to the documentation at
+http://docs.ceph.com/ for more information.
+
+
+See also
+========
+
+:doc:`ceph-osd <ceph-osd>`\(8),
diff --git a/doc/man/8/ceph-volume.rst b/doc/man/8/ceph-volume.rst
new file mode 100644
index 000000000..5a28695ae
--- /dev/null
+++ b/doc/man/8/ceph-volume.rst
@@ -0,0 +1,425 @@
+:orphan:
+
+=======================================================
+ ceph-volume -- Ceph OSD deployment and inspection tool
+=======================================================
+
+.. program:: ceph-volume
+
+Synopsis
+========
+
+| **ceph-volume** [-h] [--cluster CLUSTER] [--log-level LOG_LEVEL]
+|                 [--log-path LOG_PATH]
+
+| **ceph-volume** **inventory**
+
+| **ceph-volume** **lvm** [ *trigger* | *create* | *activate* | *prepare*
+| *zap* | *list* | *batch* | *new-wal* | *new-db* | *migrate* ]
+
+| **ceph-volume** **simple** [ *trigger* | *scan* | *activate* ]
+
+
+Description
+===========
+
+:program:`ceph-volume` is a single purpose command line tool to deploy logical
+volumes as OSDs, trying to maintain a similar API to ``ceph-disk`` when
+preparing, activating, and creating OSDs.
+
+It deviates from ``ceph-disk`` by not interacting or relying on the udev rules
+that come installed for Ceph. These rules allow automatic detection of
+previously setup devices that are in turn fed into ``ceph-disk`` to activate
+them.
+
+
+Commands
+========
+
+inventory
+---------
+
+This subcommand provides information about a host's physical disc inventory and
+reports metadata about these discs. Among this metadata one can find disc
+specific data items (like model, size, rotational or solid state) as well as
+data items specific to ceph using a device, such as if it is available for
+use with ceph or if logical volumes are present.
+
+Examples::
+
+    ceph-volume inventory
+    ceph-volume inventory /dev/sda
+    ceph-volume inventory --format json-pretty
+
+Optional arguments:
+
+* [-h, --help]          show the help message and exit
+* [--format]            report format, valid values are ``plain`` (default),
+                        ``json`` and ``json-pretty``
+
+lvm
+---
+
+By making use of LVM tags, the ``lvm`` sub-command is able to store and later
+re-discover and query devices associated with OSDs so that they can later
+activated.
+
+Subcommands:
+
+**batch**
+Creates OSDs from a list of devices using a ``filestore``
+or ``bluestore`` (default) setup. It will create all necessary volume groups
+and logical volumes required to have a working OSD.
+
+Example usage with three devices::
+
+    ceph-volume lvm batch --bluestore /dev/sda /dev/sdb /dev/sdc
+
+Optional arguments:
+
+* [-h, --help]          show the help message and exit
+* [--bluestore]         Use the bluestore objectstore (default)
+* [--filestore]         Use the filestore objectstore
+* [--yes]               Skip the report and prompt to continue provisioning
+* [--prepare]           Only prepare OSDs, do not activate
+* [--dmcrypt]           Enable encryption for the underlying OSD devices
+* [--crush-device-class] Define a CRUSH device class to assign the OSD to
+* [--no-systemd]         Do not enable or create any systemd units
+* [--osds-per-device]   Provision more than 1 (the default) OSD per device
+* [--report]         Report what the potential outcome would be for the current input (requires devices to be passed in)
+* [--format]         Output format when reporting (used along with --report), can be one of 'pretty' (default) or 'json'
+* [--block-db-size]     Set (or override) the "bluestore_block_db_size" value, in bytes
+* [--journal-size]      Override the "osd_journal_size" value, in megabytes
+
+Required positional arguments:
+
+* <DEVICE>    Full path to a raw device, like ``/dev/sda``. Multiple
+              ``<DEVICE>`` paths can be passed in.
+
+
+**activate**
+Enables a systemd unit that persists the OSD ID and its UUID (also called
+``fsid`` in Ceph CLI tools), so that at boot time it can understand what OSD is
+enabled and needs to be mounted.
+
+Usage::
+
+    ceph-volume lvm activate --bluestore <osd id> <osd fsid>
+
+Optional Arguments:
+
+* [-h, --help]  show the help message and exit
+* [--auto-detect-objectstore] Automatically detect the objectstore by inspecting
+  the OSD
+* [--bluestore] bluestore objectstore (default)
+* [--filestore] filestore objectstore
+* [--all] Activate all OSDs found in the system
+* [--no-systemd] Skip creating and enabling systemd units and starting of OSD
+  services
+
+Multiple OSDs can be activated at once by using the (idempotent) ``--all`` flag::
+
+    ceph-volume lvm activate --all
+
+
+**prepare**
+Prepares a logical volume to be used as an OSD and journal using a ``filestore``
+or ``bluestore`` (default) setup. It will not create or modify the logical volumes
+except for adding extra metadata.
+
+Usage::
+
+    ceph-volume lvm prepare --filestore --data <data lv> --journal <journal device>
+
+Optional arguments:
+
+* [-h, --help]          show the help message and exit
+* [--journal JOURNAL]   A logical group name, path to a logical volume, or path to a device
+* [--bluestore]         Use the bluestore objectstore (default)
+* [--block.wal]         Path to a bluestore block.wal logical volume or partition
+* [--block.db]          Path to a bluestore block.db logical volume or partition
+* [--filestore]         Use the filestore objectstore
+* [--dmcrypt]           Enable encryption for the underlying OSD devices
+* [--osd-id OSD_ID]     Reuse an existing OSD id
+* [--osd-fsid OSD_FSID] Reuse an existing OSD fsid
+* [--crush-device-class] Define a CRUSH device class to assign the OSD to
+
+Required arguments:
+
+* --data                A logical group name or a path to a logical volume
+
+For encrypting an OSD, the ``--dmcrypt`` flag must be added when preparing
+(also supported in the ``create`` sub-command).
+
+
+**create**
+Wraps the two-step process to provision a new osd (calling ``prepare`` first
+and then ``activate``) into a single one. The reason to prefer ``prepare`` and
+then ``activate`` is to gradually introduce new OSDs into a cluster, and
+avoiding large amounts of data being rebalanced.
+
+The single-call process unifies exactly what ``prepare`` and ``activate`` do,
+with the convenience of doing it all at once. Flags and general usage are
+equivalent to those of the ``prepare`` and ``activate`` subcommand.
+
+**trigger**
+This subcommand is not meant to be used directly, and it is used by systemd so
+that it proxies input to ``ceph-volume lvm activate`` by parsing the
+input from systemd, detecting the UUID and ID associated with an OSD.
+
+Usage::
+
+    ceph-volume lvm trigger <SYSTEMD-DATA>
+
+The systemd "data" is expected to be in the format of::
+
+    <OSD ID>-<OSD UUID>
+
+The lvs associated with the OSD need to have been prepared previously,
+so that all needed tags and metadata exist.
+
+Positional arguments:
+
+* <SYSTEMD_DATA>  Data from a systemd unit containing ID and UUID of the OSD.
+
+**list**
+List devices or logical volumes associated with Ceph. An association is
+determined if a device has information relating to an OSD. This is
+verified by querying LVM's metadata and correlating it with devices.
+
+The lvs associated with the OSD need to have been prepared previously by
+ceph-volume so that all needed tags and metadata exist.
+
+Usage::
+
+    ceph-volume lvm list
+
+List a particular device, reporting all metadata about it::
+
+    ceph-volume lvm list /dev/sda1
+
+List a logical volume, along with all its metadata (vg is a volume
+group, and lv the logical volume name)::
+
+    ceph-volume lvm list {vg/lv}
+
+Positional arguments:
+
+* <DEVICE>  Either in the form of ``vg/lv`` for logical volumes,
+  ``/path/to/sda1`` or ``/path/to/sda`` for regular devices.
+
+
+**zap**
+Zaps the given logical volume or partition. If given a path to a logical
+volume it must be in the format of vg/lv. Any file systems present
+on the given lv or partition will be removed and all data will be purged.
+
+However, the lv or partition will be kept intact.
+
+Usage, for logical volumes::
+
+      ceph-volume lvm zap {vg/lv}
+
+Usage, for logical partitions::
+
+      ceph-volume lvm zap /dev/sdc1
+
+For full removal of the device use the ``--destroy`` flag (allowed for all
+device types)::
+
+      ceph-volume lvm zap --destroy /dev/sdc1
+
+Multiple devices can be removed by specifying the OSD ID and/or the OSD FSID::
+
+      ceph-volume lvm zap --destroy --osd-id 1
+      ceph-volume lvm zap --destroy --osd-id 1 --osd-fsid C9605912-8395-4D76-AFC0-7DFDAC315D59
+
+
+Positional arguments:
+
+* <DEVICE>  Either in the form of ``vg/lv`` for logical volumes,
+  ``/path/to/sda1`` or ``/path/to/sda`` for regular devices.
+
+
+new-wal
+^^^^^^^
+
+Attaches the given logical volume to OSD as a WAL. Logical volume
+name format is vg/lv. Fails if OSD has already got attached WAL.
+
+Usage::
+
+    ceph-volume lvm new-wal --osd-id OSD_ID --osd-fsid OSD_FSID --target <target lv>
+
+Optional arguments:
+
+.. option:: -h, --help
+
+   show the help message and exit
+
+.. option:: --no-systemd
+
+   Skip checking OSD systemd unit
+
+Required arguments:
+
+.. option:: --target
+
+   logical volume name to attach as WAL
+
+new-db
+^^^^^^
+
+Attaches the given logical volume to OSD as a DB. Logical volume
+name format is vg/lv. Fails if OSD has already got attached DB.
+
+Usage::
+
+    ceph-volume lvm new-db --osd-id OSD_ID --osd-fsid OSD_FSID --target <target lv>
+
+Optional arguments:
+
+.. option:: -h, --help
+
+   show the help message and exit
+
+.. option:: --no-systemd
+
+   Skip checking OSD systemd unit
+
+Required arguments:
+
+.. option:: --target
+
+   logical volume name to attach as DB
+
+migrate
+^^^^^^^
+
+Moves BlueFS data from source volume(s) to the target one, source volumes
+(except the main, i.e. data or block one) are removed on success. LVM volumes
+are permitted for Target only, both already attached or new one. In the latter
+case it is attached to the OSD replacing one of the source devices. Following
+replacement rules apply (in the order of precedence, stop on the first match):
+
+    - if source list has DB volume - target device replaces it.
+    - if source list has WAL volume - target device replace it.
+    - if source list has slow volume only - operation is not permitted,
+      requires explicit allocation via new-db/new-wal command.
+
+Usage::
+
+    ceph-volume lvm migrate --osd-id OSD_ID --osd-fsid OSD_FSID --target <target lv> --from {data|db|wal} [{data|db|wal} ...]
+
+Optional arguments:
+
+.. option:: -h, --help
+
+   show the help message and exit
+
+.. option:: --no-systemd
+
+   Skip checking OSD systemd unit
+
+Required arguments:
+
+.. option:: --from
+
+   list of source device type names
+
+.. option:: --target
+
+   logical volume to move data to
+
+simple
+------
+
+Scan legacy OSD directories or data devices that may have been created by
+ceph-disk, or manually.
+
+Subcommands:
+
+**activate**
+Enables a systemd unit that persists the OSD ID and its UUID (also called
+``fsid`` in Ceph CLI tools), so that at boot time it can understand what OSD is
+enabled and needs to be mounted, while reading information that was previously
+created and persisted at ``/etc/ceph/osd/`` in JSON format.
+
+Usage::
+
+    ceph-volume simple activate --bluestore <osd id> <osd fsid>
+
+Optional Arguments:
+
+* [-h, --help]  show the help message and exit
+* [--bluestore] bluestore objectstore (default)
+* [--filestore] filestore objectstore
+
+Note: It requires a matching JSON file with the following format::
+
+    /etc/ceph/osd/<osd id>-<osd fsid>.json
+
+
+**scan**
+Scan a running OSD or data device for an OSD for metadata that can later be
+used to activate and manage the OSD with ceph-volume. The scan method will
+create a JSON file with the required information plus anything found in the OSD
+directory as well.
+
+Optionally, the JSON blob can be sent to stdout for further inspection.
+
+Usage on all running OSDs::
+
+    ceph-volume simple scan
+
+Usage on data devices::
+
+    ceph-volume simple scan <data device>
+
+Running OSD directories::
+
+    ceph-volume simple scan <path to osd dir>
+
+
+Optional arguments:
+
+* [-h, --help]          show the help message and exit
+* [--stdout]            Send the JSON blob to stdout
+* [--force]             If the JSON file exists at destination, overwrite it
+
+Optional Positional arguments:
+
+* <DATA DEVICE or OSD DIR>  Actual data partition or a path to the running OSD
+
+**trigger**
+This subcommand is not meant to be used directly, and it is used by systemd so
+that it proxies input to ``ceph-volume simple activate`` by parsing the
+input from systemd, detecting the UUID and ID associated with an OSD.
+
+Usage::
+
+    ceph-volume simple trigger <SYSTEMD-DATA>
+
+The systemd "data" is expected to be in the format of::
+
+    <OSD ID>-<OSD UUID>
+
+The JSON file associated with the OSD need to have been persisted previously by
+a scan (or manually), so that all needed metadata can be used.
+
+Positional arguments:
+
+* <SYSTEMD_DATA>  Data from a systemd unit containing ID and UUID of the OSD.
+
+
+Availability
+============
+
+:program:`ceph-volume` is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the documentation at http://docs.ceph.com/ for more information.
+
+
+See also
+========
+
+:doc:`ceph-osd <ceph-osd>`\(8),
diff --git a/doc/man/8/ceph.rst b/doc/man/8/ceph.rst
new file mode 100644
index 000000000..7fefe58d6
--- /dev/null
+++ b/doc/man/8/ceph.rst
@@ -0,0 +1,1649 @@
+:orphan:
+
+==================================
+ ceph -- ceph administration tool
+==================================
+
+.. program:: ceph
+
+Synopsis
+========
+
+| **ceph** **auth** [ *add* \| *caps* \| *del* \| *export* \| *get* \| *get-key* \| *get-or-create* \| *get-or-create-key* \| *import* \| *list* \| *print-key* \| *print_key* ] ...
+
+| **ceph** **compact**
+
+| **ceph** **config** [ *dump* | *ls* | *help* | *get* | *show* | *show-with-defaults* | *set* | *rm* | *log* | *reset* | *assimilate-conf* | *generate-minimal-conf* ] ...
+
+| **ceph** **config-key** [ *rm* | *exists* | *get* | *ls* | *dump* | *set* ] ...
+
+| **ceph** **daemon** *<name>* \| *<path>* *<command>* ...
+
+| **ceph** **daemonperf** *<name>* \| *<path>* [ *interval* [ *count* ] ]
+
+| **ceph** **df** *{detail}*
+
+| **ceph** **fs** [ *ls* \| *new* \| *reset* \| *rm* \| *authorize* ] ...
+
+| **ceph** **fsid**
+
+| **ceph** **health** *{detail}*
+
+| **ceph** **injectargs** *<injectedargs>* [ *<injectedargs>*... ]
+
+| **ceph** **log** *<logtext>* [ *<logtext>*... ]
+
+| **ceph** **mds** [ *compat* \| *fail* \| *rm* \| *rmfailed* \| *set_state* \| *stat* \| *repaired* ] ...
+
+| **ceph** **mon** [ *add* \| *dump* \| *getmap* \| *remove* \| *stat* ] ...
+
+| **ceph** **osd** [ *blocklist* \| *blocked-by* \| *create* \| *new* \| *deep-scrub* \| *df* \| *down* \| *dump* \| *erasure-code-profile* \| *find* \| *getcrushmap* \| *getmap* \| *getmaxosd* \| *in* \| *ls* \| *lspools* \| *map* \| *metadata* \| *ok-to-stop* \| *out* \| *pause* \| *perf* \| *pg-temp* \| *force-create-pg* \| *primary-affinity* \| *primary-temp* \| *repair* \| *reweight* \| *reweight-by-pg* \| *rm* \| *destroy* \| *purge* \| *safe-to-destroy* \| *scrub* \| *set* \| *setcrushmap* \| *setmaxosd*  \| *stat* \| *tree* \| *unpause* \| *unset* ] ...
+
+| **ceph** **osd** **crush** [ *add* \| *add-bucket* \| *create-or-move* \| *dump* \| *get-tunable* \| *link* \| *move* \| *remove* \| *rename-bucket* \| *reweight* \| *reweight-all* \| *reweight-subtree* \| *rm* \| *rule* \| *set* \| *set-tunable* \| *show-tunables* \| *tunables* \| *unlink* ] ...
+
+| **ceph** **osd** **pool** [ *create* \| *delete* \| *get* \| *get-quota* \| *ls* \| *mksnap* \| *rename* \| *rmsnap* \| *set* \| *set-quota* \| *stats* ] ...
+
+| **ceph** **osd** **pool** **application** [ *disable* \| *enable* \| *get* \| *rm* \| *set* ] ...
+
+| **ceph** **osd** **tier** [ *add* \| *add-cache* \| *cache-mode* \| *remove* \| *remove-overlay* \| *set-overlay* ] ...
+
+| **ceph** **pg** [ *debug* \| *deep-scrub* \| *dump* \| *dump_json* \| *dump_pools_json* \| *dump_stuck* \| *getmap* \| *ls* \| *ls-by-osd* \| *ls-by-pool* \| *ls-by-primary* \| *map* \| *repair* \| *scrub* \| *stat* ] ...
+
+| **ceph** **quorum_status**
+
+| **ceph** **report** { *<tags>* [ *<tags>...* ] }
+
+| **ceph** **status**
+
+| **ceph** **sync** **force** {--yes-i-really-mean-it} {--i-know-what-i-am-doing}
+
+| **ceph** **tell** *<name (type.id)> <command> [options...]*
+
+| **ceph** **version**
+
+Description
+===========
+
+:program:`ceph` is a control utility which is used for manual deployment and maintenance
+of a Ceph cluster. It provides a diverse set of commands that allows deployment of
+monitors, OSDs, placement groups, MDS and overall maintenance, administration
+of the cluster.
+
+Commands
+========
+
+auth
+----
+
+Manage authentication keys. It is used for adding, removing, exporting
+or updating of authentication keys for a particular  entity such as a monitor or
+OSD. It uses some additional subcommands.
+
+Subcommand ``add`` adds authentication info for a particular entity from input
+file, or random key if no input is given and/or any caps specified in the command.
+
+Usage::
+
+	ceph auth add <entity> {<caps> [<caps>...]}
+
+Subcommand ``caps`` updates caps for **name** from caps specified in the command.
+
+Usage::
+
+	ceph auth caps <entity> <caps> [<caps>...]
+
+Subcommand ``del`` deletes all caps for ``name``.
+
+Usage::
+
+	ceph auth del <entity>
+
+Subcommand ``export`` writes keyring for requested entity, or master keyring if
+none given.
+
+Usage::
+
+	ceph auth export {<entity>}
+
+Subcommand ``get`` writes keyring file with requested key.
+
+Usage::
+
+	ceph auth get <entity>
+
+Subcommand ``get-key`` displays requested key.
+
+Usage::
+
+	ceph auth get-key <entity>
+
+Subcommand ``get-or-create`` adds authentication info for a particular entity
+from input file, or random key if no input given and/or any caps specified in the
+command.
+
+Usage::
+
+	ceph auth get-or-create <entity> {<caps> [<caps>...]}
+
+Subcommand ``get-or-create-key`` gets or adds key for ``name`` from system/caps
+pairs specified in the command.  If key already exists, any given caps must match
+the existing caps for that key.
+
+Usage::
+
+	ceph auth get-or-create-key <entity> {<caps> [<caps>...]}
+
+Subcommand ``import`` reads keyring from input file.
+
+Usage::
+
+	ceph auth import
+
+Subcommand ``ls`` lists authentication state.
+
+Usage::
+
+	ceph auth ls
+
+Subcommand ``print-key`` displays requested key.
+
+Usage::
+
+	ceph auth print-key <entity>
+
+Subcommand ``print_key`` displays requested key.
+
+Usage::
+
+	ceph auth print_key <entity>
+
+
+compact
+-------
+
+Causes compaction of monitor's leveldb storage.
+
+Usage::
+
+	ceph compact
+
+
+config
+------
+
+Configure the cluster. By default, Ceph daemons and clients retrieve their
+configuration options from monitor when they start, and are updated if any of
+the tracked options is changed at run time. It uses following additional
+subcommand.
+
+Subcommand ``dump`` to dump all options for the cluster
+
+Usage::
+
+	ceph config dump
+
+Subcommand ``ls`` to list all option names for the cluster
+
+Usage::
+
+	ceph config ls
+
+Subcommand ``help`` to describe the specified configuration option
+
+Usage::
+
+    ceph config help <option>
+
+Subcommand ``get`` to dump the option(s) for the specified entity.
+
+Usage::
+
+    ceph config get <who> {<option>}
+
+Subcommand ``show`` to display the running configuration of the specified
+entity. Please note, unlike ``get``, which only shows the options managed
+by monitor, ``show`` displays all the configurations being actively used.
+These options are pulled from several sources, for instance, the compiled-in
+default value, the monitor's configuration database, ``ceph.conf`` file on
+the host. The options can even be overridden at runtime. So, there is chance
+that the configuration options in the output of ``show`` could be different
+from those in the output of ``get``.
+
+Usage::
+
+	ceph config show {<who>}
+
+Subcommand ``show-with-defaults`` to display the running configuration along with the compiled-in defaults of the specified entity
+
+Usage::
+
+	ceph config show {<who>}
+
+Subcommand ``set`` to set an option for one or more specified entities
+
+Usage::
+
+    ceph config set <who> <option> <value> {--force}
+
+Subcommand ``rm`` to clear an option for one or more entities
+
+Usage::
+
+    ceph config rm <who> <option>
+
+Subcommand ``log`` to show recent history of config changes. If `count` option
+is omitted it defeaults to 10.
+
+Usage::
+
+    ceph config log {<count>}
+
+Subcommand ``reset`` to revert configuration to the specified historical version
+
+Usage::
+
+    ceph config reset <version>
+
+
+Subcommand ``assimilate-conf`` to assimilate options from stdin, and return a
+new, minimal conf file
+
+Usage::
+
+    ceph config assimilate-conf -i <input-config-path> > <output-config-path>
+    ceph config assimilate-conf < <input-config-path>
+
+Subcommand ``generate-minimal-conf`` to generate a minimal ``ceph.conf`` file,
+which can be used for bootstrapping a daemon or a client.
+
+Usage::
+
+    ceph config generate-minimal-conf > <minimal-config-path>
+
+
+config-key
+----------
+
+Manage configuration key. Config-key is a general purpose key/value service
+offered by the monitors. This service is mainly used by Ceph tools and daemons
+for persisting various settings. Among which, ceph-mgr modules uses it for
+storing their options. It uses some additional subcommands.
+
+Subcommand ``rm`` deletes configuration key.
+
+Usage::
+
+	ceph config-key rm <key>
+
+Subcommand ``exists`` checks for configuration keys existence.
+
+Usage::
+
+	ceph config-key exists <key>
+
+Subcommand ``get`` gets the configuration key.
+
+Usage::
+
+	ceph config-key get <key>
+
+Subcommand ``ls`` lists configuration keys.
+
+Usage::
+
+	ceph config-key ls
+
+Subcommand ``dump`` dumps configuration keys and values.
+
+Usage::
+
+	ceph config-key dump
+
+Subcommand ``set`` puts configuration key and value.
+
+Usage::
+
+	ceph config-key set <key> {<val>}
+
+
+daemon
+------
+
+Submit admin-socket commands.
+
+Usage::
+
+	ceph daemon {daemon_name|socket_path} {command} ...
+
+Example::
+
+	ceph daemon osd.0 help
+
+
+daemonperf
+----------
+
+Watch performance counters from a Ceph daemon.
+
+Usage::
+
+	ceph daemonperf {daemon_name|socket_path} [{interval} [{count}]]
+
+
+df
+--
+
+Show cluster's free space status.
+
+Usage::
+
+	ceph df {detail}
+
+.. _ceph features:
+
+features
+--------
+
+Show the releases and features of all connected daemons and clients connected
+to the cluster, along with the numbers of them in each bucket grouped by the
+corresponding features/releases. Each release of Ceph supports a different set
+of features, expressed by the features bitmask. New cluster features require
+that clients support the feature, or else they are not allowed to connect to
+these new features. As new features or capabilities are enabled after an
+upgrade, older clients are prevented from connecting.
+
+Usage::
+
+    ceph features
+
+fs
+--
+
+Manage cephfs file systems. It uses some additional subcommands.
+
+Subcommand ``ls`` to list file systems
+
+Usage::
+
+	ceph fs ls
+
+Subcommand ``new`` to make a new file system using named pools <metadata> and <data>
+
+Usage::
+
+	ceph fs new <fs_name> <metadata> <data>
+
+Subcommand ``reset`` is used for disaster recovery only: reset to a single-MDS map
+
+Usage::
+
+	ceph fs reset <fs_name> {--yes-i-really-mean-it}
+
+Subcommand ``rm`` to disable the named file system
+
+Usage::
+
+	ceph fs rm <fs_name> {--yes-i-really-mean-it}
+
+Subcommand ``authorize`` creates a new client that will be authorized for the
+given path in ``<fs_name>``. Pass ``/`` to authorize for the entire FS.
+``<perms>`` below can be ``r``, ``rw`` or ``rwp``.
+
+Usage::
+
+    ceph fs authorize <fs_name> client.<client_id> <path> <perms> [<path> <perms>...]
+
+fsid
+----
+
+Show cluster's FSID/UUID.
+
+Usage::
+
+	ceph fsid
+
+
+health
+------
+
+Show cluster's health.
+
+Usage::
+
+	ceph health {detail}
+
+
+heap
+----
+
+Show heap usage info (available only if compiled with tcmalloc)
+
+Usage::
+
+	ceph tell <name (type.id)> heap dump|start_profiler|stop_profiler|stats
+
+Subcommand ``release`` to make TCMalloc to releases no-longer-used memory back to the kernel at once. 
+
+Usage::
+
+	ceph tell <name (type.id)> heap release
+
+Subcommand ``(get|set)_release_rate`` get or set the TCMalloc memory release rate. TCMalloc releases 
+no-longer-used memory back to the kernel gradually. the rate controls how quickly this happens. 
+Increase this setting to make TCMalloc to return unused memory more frequently. 0 means never return
+memory to system, 1 means wait for 1000 pages after releasing a page to system. It is ``1.0`` by default..
+
+Usage::
+
+	ceph tell <name (type.id)> heap get_release_rate|set_release_rate {<val>}
+
+injectargs
+----------
+
+Inject configuration arguments into monitor.
+
+Usage::
+
+	ceph injectargs <injected_args> [<injected_args>...]
+
+
+log
+---
+
+Log supplied text to the monitor log.
+
+Usage::
+
+	ceph log <logtext> [<logtext>...]
+
+
+mds
+---
+
+Manage metadata server configuration and administration. It uses some
+additional subcommands.
+
+Subcommand ``compat`` manages compatible features. It uses some additional
+subcommands.
+
+Subcommand ``rm_compat`` removes compatible feature.
+
+Usage::
+
+	ceph mds compat rm_compat <int[0-]>
+
+Subcommand ``rm_incompat`` removes incompatible feature.
+
+Usage::
+
+	ceph mds compat rm_incompat <int[0-]>
+
+Subcommand ``show`` shows mds compatibility settings.
+
+Usage::
+
+	ceph mds compat show
+
+Subcommand ``fail`` forces mds to status fail.
+
+Usage::
+
+	ceph mds fail <role|gid>
+
+Subcommand ``rm`` removes inactive mds.
+
+Usage::
+
+	ceph mds rm <int[0-]> <name> (type.id)>
+
+Subcommand ``rmfailed`` removes failed mds.
+
+Usage::
+
+	ceph mds rmfailed <int[0-]>
+
+Subcommand ``set_state`` sets mds state of <gid> to <numeric-state>.
+
+Usage::
+
+	ceph mds set_state <int[0-]> <int[0-20]>
+
+Subcommand ``stat`` shows MDS status.
+
+Usage::
+
+	ceph mds stat
+
+Subcommand ``repaired`` mark a damaged MDS rank as no longer damaged.
+
+Usage::
+
+	ceph mds repaired <role>
+
+mon
+---
+
+Manage monitor configuration and administration. It uses some additional
+subcommands.
+
+Subcommand ``add`` adds new monitor named <name> at <addr>.
+
+Usage::
+
+	ceph mon add <name> <IPaddr[:port]>
+
+Subcommand ``dump`` dumps formatted monmap (optionally from epoch)
+
+Usage::
+
+	ceph mon dump {<int[0-]>}
+
+Subcommand ``getmap`` gets monmap.
+
+Usage::
+
+	ceph mon getmap {<int[0-]>}
+
+Subcommand ``remove`` removes monitor named <name>.
+
+Usage::
+
+	ceph mon remove <name>
+
+Subcommand ``stat`` summarizes monitor status.
+
+Usage::
+
+	ceph mon stat
+
+mgr
+---
+
+Ceph manager daemon configuration and management.
+
+Subcommand ``dump`` dumps the latest MgrMap, which describes the active
+and standby manager daemons.
+
+Usage::
+
+  ceph mgr dump
+
+Subcommand ``fail`` will mark a manager daemon as failed, removing it
+from the manager map.  If it is the active manager daemon a standby
+will take its place.
+
+Usage::
+
+  ceph mgr fail <name>
+
+Subcommand ``module ls`` will list currently enabled manager modules (plugins).
+
+Usage::
+
+  ceph mgr module ls
+
+Subcommand ``module enable`` will enable a manager module.  Available modules are included in MgrMap and visible via ``mgr dump``.
+
+Usage::
+
+  ceph mgr module enable <module>
+
+Subcommand ``module disable`` will disable an active manager module.
+
+Usage::
+
+  ceph mgr module disable <module>
+
+Subcommand ``metadata`` will report metadata about all manager daemons or, if the name is specified, a single manager daemon.
+
+Usage::
+
+  ceph mgr metadata [name]
+
+Subcommand ``versions`` will report a count of running daemon versions.
+
+Usage::
+
+  ceph mgr versions
+
+Subcommand ``count-metadata`` will report a count of any daemon metadata field.
+
+Usage::
+
+  ceph mgr count-metadata <field>
+
+.. _ceph-admin-osd:
+
+osd
+---
+
+Manage OSD configuration and administration. It uses some additional
+subcommands.
+
+Subcommand ``blocklist`` manage blocklisted clients. It uses some additional
+subcommands.
+
+Subcommand ``add`` add <addr> to blocklist (optionally until <expire> seconds
+from now)
+
+Usage::
+
+	ceph osd blocklist add <EntityAddr> {<float[0.0-]>}
+
+Subcommand ``ls`` show blocklisted clients
+
+Usage::
+
+	ceph osd blocklist ls
+
+Subcommand ``rm`` remove <addr> from blocklist
+
+Usage::
+
+	ceph osd blocklist rm <EntityAddr>
+
+Subcommand ``blocked-by`` prints a histogram of which OSDs are blocking their peers
+
+Usage::
+
+	ceph osd blocked-by
+
+Subcommand ``create`` creates new osd (with optional UUID and ID).
+
+This command is DEPRECATED as of the Luminous release, and will be removed in
+a future release.
+
+Subcommand ``new`` should instead be used.
+
+Usage::
+
+	ceph osd create {<uuid>} {<id>}
+
+Subcommand ``new`` can be used to create a new OSD or to recreate a previously
+destroyed OSD with a specific *id*. The new OSD will have the specified *uuid*,
+and the command expects a JSON file containing the base64 cephx key for auth
+entity *client.osd.<id>*, as well as optional base64 cepx key for dm-crypt
+lockbox access and a dm-crypt key. Specifying a dm-crypt requires specifying
+the accompanying lockbox cephx key.
+
+Usage::
+
+    ceph osd new {<uuid>} {<id>} -i {<params.json>}
+
+The parameters JSON file is optional but if provided, is expected to maintain
+a form of the following format::
+
+    {
+        "cephx_secret": "AQBWtwhZdBO5ExAAIDyjK2Bh16ZXylmzgYYEjg==",
+	"crush_device_class": "myclass"
+    }
+
+Or::
+
+    {
+        "cephx_secret": "AQBWtwhZdBO5ExAAIDyjK2Bh16ZXylmzgYYEjg==",
+        "cephx_lockbox_secret": "AQDNCglZuaeVCRAAYr76PzR1Anh7A0jswkODIQ==",
+        "dmcrypt_key": "<dm-crypt key>",
+	"crush_device_class": "myclass"
+    }
+
+Or::
+
+    {
+	"crush_device_class": "myclass"
+    }
+
+The "crush_device_class" property is optional. If specified, it will set the
+initial CRUSH device class for the new OSD.
+
+
+Subcommand ``crush`` is used for CRUSH management. It uses some additional
+subcommands.
+
+Subcommand ``add`` adds or updates crushmap position and weight for <name> with
+<weight> and location <args>.
+
+Usage::
+
+	ceph osd crush add <osdname (id|osd.id)> <float[0.0-]> <args> [<args>...]
+
+Subcommand ``add-bucket`` adds no-parent (probably root) crush bucket <name> of
+type <type>.
+
+Usage::
+
+	ceph osd crush add-bucket <name> <type>
+
+Subcommand ``create-or-move`` creates entry or moves existing entry for <name>
+<weight> at/to location <args>.
+
+Usage::
+
+	ceph osd crush create-or-move <osdname (id|osd.id)> <float[0.0-]> <args>
+	[<args>...]
+
+Subcommand ``dump`` dumps crush map.
+
+Usage::
+
+	ceph osd crush dump
+
+Subcommand ``get-tunable`` get crush tunable straw_calc_version
+
+Usage::
+
+	ceph osd crush get-tunable straw_calc_version
+
+Subcommand ``link`` links existing entry for <name> under location <args>.
+
+Usage::
+
+	ceph osd crush link <name> <args> [<args>...]
+
+Subcommand ``move`` moves existing entry for <name> to location <args>.
+
+Usage::
+
+	ceph osd crush move <name> <args> [<args>...]
+
+Subcommand ``remove`` removes <name> from crush map (everywhere, or just at
+<ancestor>).
+
+Usage::
+
+	ceph osd crush remove <name> {<ancestor>}
+
+Subcommand ``rename-bucket`` renames bucket <srcname> to <dstname>
+
+Usage::
+
+	ceph osd crush rename-bucket <srcname> <dstname>
+
+Subcommand ``reweight`` change <name>'s weight to <weight> in crush map.
+
+Usage::
+
+	ceph osd crush reweight <name> <float[0.0-]>
+
+Subcommand ``reweight-all`` recalculate the weights for the tree to
+ensure they sum correctly
+
+Usage::
+
+	ceph osd crush reweight-all
+
+Subcommand ``reweight-subtree`` changes all leaf items beneath <name>
+to <weight> in crush map
+
+Usage::
+
+	ceph osd crush reweight-subtree <name> <weight>
+
+Subcommand ``rm`` removes <name> from crush map (everywhere, or just at
+<ancestor>).
+
+Usage::
+
+	ceph osd crush rm <name> {<ancestor>}
+
+Subcommand ``rule`` is used for creating crush rules. It uses some additional
+subcommands.
+
+Subcommand ``create-erasure`` creates crush rule <name> for erasure coded pool
+created with <profile> (default default).
+
+Usage::
+
+	ceph osd crush rule create-erasure <name> {<profile>}
+
+Subcommand ``create-simple`` creates crush rule <name> to start from <root>,
+replicate across buckets of type <type>, using a choose mode of <firstn|indep>
+(default firstn; indep best for erasure pools).
+
+Usage::
+
+	ceph osd crush rule create-simple <name> <root> <type> {firstn|indep}
+
+Subcommand ``dump`` dumps crush rule <name> (default all).
+
+Usage::
+
+	ceph osd crush rule dump {<name>}
+
+Subcommand ``ls`` lists crush rules.
+
+Usage::
+
+	ceph osd crush rule ls
+
+Subcommand ``rm`` removes crush rule <name>.
+
+Usage::
+
+	ceph osd crush rule rm <name>
+
+Subcommand ``set`` used alone, sets crush map from input file.
+
+Usage::
+
+	ceph osd crush set
+
+Subcommand ``set`` with osdname/osd.id update crushmap position and weight
+for <name> to <weight> with location <args>.
+
+Usage::
+
+	ceph osd crush set <osdname (id|osd.id)> <float[0.0-]> <args> [<args>...]
+
+Subcommand ``set-tunable`` set crush tunable <tunable> to <value>.  The only
+tunable that can be set is straw_calc_version.
+
+Usage::
+
+	ceph osd crush set-tunable straw_calc_version <value>
+
+Subcommand ``show-tunables`` shows current crush tunables.
+
+Usage::
+
+	ceph osd crush show-tunables
+
+Subcommand ``tree`` shows the crush buckets and items in a tree view.
+
+Usage::
+
+	ceph osd crush tree
+
+Subcommand ``tunables`` sets crush tunables values to <profile>.
+
+Usage::
+
+	ceph osd crush tunables legacy|argonaut|bobtail|firefly|hammer|optimal|default
+
+Subcommand ``unlink`` unlinks <name> from crush map (everywhere, or just at
+<ancestor>).
+
+Usage::
+
+	ceph osd crush unlink <name> {<ancestor>}
+
+Subcommand ``df`` shows OSD utilization
+
+Usage::
+
+	ceph osd df {plain|tree}
+
+Subcommand ``deep-scrub`` initiates deep scrub on specified osd.
+
+Usage::
+
+	ceph osd deep-scrub <who>
+
+Subcommand ``down`` sets osd(s) <id> [<id>...] down.
+
+Usage::
+
+	ceph osd down <ids> [<ids>...]
+
+Subcommand ``dump`` prints summary of OSD map.
+
+Usage::
+
+	ceph osd dump {<int[0-]>}
+
+Subcommand ``erasure-code-profile`` is used for managing the erasure code
+profiles. It uses some additional subcommands.
+
+Subcommand ``get`` gets erasure code profile <name>.
+
+Usage::
+
+	ceph osd erasure-code-profile get <name>
+
+Subcommand ``ls`` lists all erasure code profiles.
+
+Usage::
+
+	ceph osd erasure-code-profile ls
+
+Subcommand ``rm`` removes erasure code profile <name>.
+
+Usage::
+
+	ceph osd erasure-code-profile rm <name>
+
+Subcommand ``set`` creates erasure code profile <name> with [<key[=value]> ...]
+pairs. Add a --force at the end to override an existing profile (IT IS RISKY).
+
+Usage::
+
+	ceph osd erasure-code-profile set <name> {<profile> [<profile>...]}
+
+Subcommand ``find`` find osd <id> in the CRUSH map and shows its location.
+
+Usage::
+
+	ceph osd find <int[0-]>
+
+Subcommand ``getcrushmap`` gets CRUSH map.
+
+Usage::
+
+	ceph osd getcrushmap {<int[0-]>}
+
+Subcommand ``getmap`` gets OSD map.
+
+Usage::
+
+	ceph osd getmap {<int[0-]>}
+
+Subcommand ``getmaxosd`` shows largest OSD id.
+
+Usage::
+
+	ceph osd getmaxosd
+
+Subcommand ``in`` sets osd(s) <id> [<id>...] in.
+
+Usage::
+
+	ceph osd in <ids> [<ids>...]
+
+Subcommand ``lost`` marks osd as permanently lost. THIS DESTROYS DATA IF NO
+MORE REPLICAS EXIST, BE CAREFUL.
+
+Usage::
+
+	ceph osd lost <int[0-]> {--yes-i-really-mean-it}
+
+Subcommand ``ls`` shows all OSD ids.
+
+Usage::
+
+	ceph osd ls {<int[0-]>}
+
+Subcommand ``lspools`` lists pools.
+
+Usage::
+
+	ceph osd lspools {<int>}
+
+Subcommand ``map`` finds pg for <object> in <pool>.
+
+Usage::
+
+	ceph osd map <poolname> <objectname>
+
+Subcommand ``metadata`` fetches metadata for osd <id>.
+
+Usage::
+
+	ceph osd metadata {int[0-]} (default all)
+
+Subcommand ``out`` sets osd(s) <id> [<id>...] out.
+
+Usage::
+
+	ceph osd out <ids> [<ids>...]
+
+Subcommand ``ok-to-stop`` checks whether the list of OSD(s) can be
+stopped without immediately making data unavailable.  That is, all
+data should remain readable and writeable, although data redundancy
+may be reduced as some PGs may end up in a degraded (but active)
+state.  It will return a success code if it is okay to stop the
+OSD(s), or an error code and informative message if it is not or if no
+conclusion can be drawn at the current time.  When ``--max <num>`` is
+provided, up to <num> OSDs IDs will return (including the provided
+OSDs) that can all be stopped simultaneously.  This allows larger sets
+of stoppable OSDs to be generated easily by providing a single
+starting OSD and a max.  Additional OSDs are drawn from adjacent locations
+in the CRUSH hierarchy.
+
+Usage::
+
+  ceph osd ok-to-stop <id> [<ids>...] [--max <num>]
+
+Subcommand ``pause`` pauses osd.
+
+Usage::
+
+	ceph osd pause
+
+Subcommand ``perf`` prints dump of OSD perf summary stats.
+
+Usage::
+
+	ceph osd perf
+
+Subcommand ``pg-temp`` set pg_temp mapping pgid:[<id> [<id>...]] (developers
+only).
+
+Usage::
+
+	ceph osd pg-temp <pgid> {<id> [<id>...]}
+
+Subcommand ``force-create-pg`` forces creation of pg <pgid>.
+
+Usage::
+
+	ceph osd force-create-pg <pgid>
+
+
+Subcommand ``pool`` is used for managing data pools. It uses some additional
+subcommands.
+
+Subcommand ``create`` creates pool.
+
+Usage::
+
+	ceph osd pool create <poolname> {<int[0-]>} {<int[0-]>} {replicated|erasure}
+	{<erasure_code_profile>} {<rule>} {<int>} {--autoscale-mode=<on,off,warn>}
+
+Subcommand ``delete`` deletes pool.
+
+Usage::
+
+	ceph osd pool delete <poolname> {<poolname>} {--yes-i-really-really-mean-it}
+
+Subcommand ``get`` gets pool parameter <var>.
+
+Usage::
+
+	ceph osd pool get <poolname> size|min_size|pg_num|pgp_num|crush_rule|write_fadvise_dontneed
+
+Only for tiered pools::
+
+	ceph osd pool get <poolname> hit_set_type|hit_set_period|hit_set_count|hit_set_fpp|
+	target_max_objects|target_max_bytes|cache_target_dirty_ratio|cache_target_dirty_high_ratio|
+	cache_target_full_ratio|cache_min_flush_age|cache_min_evict_age|
+	min_read_recency_for_promote|hit_set_grade_decay_rate|hit_set_search_last_n
+
+Only for erasure coded pools::
+
+	ceph osd pool get <poolname> erasure_code_profile
+
+Use ``all`` to get all pool parameters that apply to the pool's type::
+
+	ceph osd pool get <poolname> all
+
+Subcommand ``get-quota`` obtains object or byte limits for pool.
+
+Usage::
+
+	ceph osd pool get-quota <poolname>
+
+Subcommand ``ls`` list pools
+
+Usage::
+
+	ceph osd pool ls {detail}
+
+Subcommand ``mksnap`` makes snapshot <snap> in <pool>.
+
+Usage::
+
+	ceph osd pool mksnap <poolname> <snap>
+
+Subcommand ``rename`` renames <srcpool> to <destpool>.
+
+Usage::
+
+	ceph osd pool rename <poolname> <poolname>
+
+Subcommand ``rmsnap`` removes snapshot <snap> from <pool>.
+
+Usage::
+
+	ceph osd pool rmsnap <poolname> <snap>
+
+Subcommand ``set`` sets pool parameter <var> to <val>.
+
+Usage::
+
+	ceph osd pool set <poolname> size|min_size|pg_num|
+	pgp_num|crush_rule|hashpspool|nodelete|nopgchange|nosizechange|
+	hit_set_type|hit_set_period|hit_set_count|hit_set_fpp|debug_fake_ec_pool|
+	target_max_bytes|target_max_objects|cache_target_dirty_ratio|
+	cache_target_dirty_high_ratio|
+	cache_target_full_ratio|cache_min_flush_age|cache_min_evict_age|
+	min_read_recency_for_promote|write_fadvise_dontneed|hit_set_grade_decay_rate|
+	hit_set_search_last_n
+	<val> {--yes-i-really-mean-it}
+
+Subcommand ``set-quota`` sets object or byte limit on pool.
+
+Usage::
+
+	ceph osd pool set-quota <poolname> max_objects|max_bytes <val>
+
+Subcommand ``stats`` obtain stats from all pools, or from specified pool.
+
+Usage::
+
+	ceph osd pool stats {<name>}
+
+Subcommand ``application`` is used for adding an annotation to the given
+pool. By default, the possible applications are object, block, and file
+storage (corresponding app-names are "rgw", "rbd", and "cephfs"). However,
+there might be other applications as well. Based on the application, there
+may or may not be some processing conducted.
+
+Subcommand ``disable`` disables the given application on the given pool.
+
+Usage::
+
+        ceph osd pool application disable <pool-name> <app> {--yes-i-really-mean-it}
+
+Subcommand ``enable`` adds an annotation to the given pool for the mentioned
+application.
+
+Usage::
+
+        ceph osd pool application enable <pool-name> <app> {--yes-i-really-mean-it}
+
+Subcommand ``get`` displays the value for the given key that is associated
+with the given application of the given pool. Not passing the optional
+arguments would display all key-value pairs for all applications for all
+pools.
+
+Usage::
+
+        ceph osd pool application get {<pool-name>} {<app>} {<key>}
+
+Subcommand ``rm`` removes the key-value pair for the given key in the given
+application of the given pool.
+
+Usage::
+
+        ceph osd pool application rm <pool-name> <app> <key>
+
+Subcommand ``set`` associates or updates, if it already exists, a key-value
+pair with the given application for the given pool.
+
+Usage::
+
+        ceph osd pool application set <pool-name> <app> <key> <value>
+
+Subcommand ``primary-affinity`` adjust osd primary-affinity from 0.0 <=<weight>
+<= 1.0
+
+Usage::
+
+	ceph osd primary-affinity <osdname (id|osd.id)> <float[0.0-1.0]>
+
+Subcommand ``primary-temp`` sets primary_temp mapping pgid:<id>|-1 (developers
+only).
+
+Usage::
+
+	ceph osd primary-temp <pgid> <id>
+
+Subcommand ``repair`` initiates repair on a specified osd.
+
+Usage::
+
+	ceph osd repair <who>
+
+Subcommand ``reweight`` reweights osd to 0.0 < <weight> < 1.0.
+
+Usage::
+
+	osd reweight <int[0-]> <float[0.0-1.0]>
+
+Subcommand ``reweight-by-pg`` reweight OSDs by PG distribution
+[overload-percentage-for-consideration, default 120].
+
+Usage::
+
+	ceph osd reweight-by-pg {<int[100-]>} {<poolname> [<poolname...]}
+	{--no-increasing}
+
+Subcommand ``reweight-by-utilization`` reweights OSDs by utilization.  It only reweights
+outlier OSDs whose utilization exceeds the average, eg. the default 120%
+limits reweight to those OSDs that are more than 20% over the average.
+[overload-threshold, default 120 [max_weight_change, default 0.05 [max_osds_to_adjust, default 4]]] 
+
+Usage::
+
+	ceph osd reweight-by-utilization {<int[100-]> {<float[0.0-]> {<int[0-]>}}}
+	{--no-increasing}
+
+Subcommand ``rm`` removes osd(s) <id> [<id>...] from the OSD map.
+
+
+Usage::
+
+	ceph osd rm <ids> [<ids>...]
+
+Subcommand ``destroy`` marks OSD *id* as *destroyed*, removing its cephx
+entity's keys and all of its dm-crypt and daemon-private config key
+entries.
+
+This command will not remove the OSD from crush, nor will it remove the
+OSD from the OSD map. Instead, once the command successfully completes,
+the OSD will show marked as *destroyed*.
+
+In order to mark an OSD as destroyed, the OSD must first be marked as
+**lost**.
+
+Usage::
+
+    ceph osd destroy <id> {--yes-i-really-mean-it}
+
+
+Subcommand ``purge`` performs a combination of ``osd destroy``,
+``osd rm`` and ``osd crush remove``.
+
+Usage::
+
+    ceph osd purge <id> {--yes-i-really-mean-it}
+
+Subcommand ``safe-to-destroy`` checks whether it is safe to remove or
+destroy an OSD without reducing overall data redundancy or durability.
+It will return a success code if it is definitely safe, or an error
+code and informative message if it is not or if no conclusion can be
+drawn at the current time.
+
+Usage::
+
+  ceph osd safe-to-destroy <id> [<ids>...]
+
+Subcommand ``scrub`` initiates scrub on specified osd.
+
+Usage::
+
+	ceph osd scrub <who>
+
+Subcommand ``set`` sets cluster-wide <flag> by updating OSD map.
+The ``full`` flag is not honored anymore since the Mimic release, and
+``ceph osd set full`` is not supported in the Octopus release.
+
+Usage::
+
+	ceph osd set pause|noup|nodown|noout|noin|nobackfill|
+	norebalance|norecover|noscrub|nodeep-scrub|notieragent
+
+Subcommand ``setcrushmap`` sets crush map from input file.
+
+Usage::
+
+	ceph osd setcrushmap
+
+Subcommand ``setmaxosd`` sets new maximum osd value.
+
+Usage::
+
+	ceph osd setmaxosd <int[0-]>
+
+Subcommand ``set-require-min-compat-client`` enforces the cluster to be backward
+compatible with the specified client version. This subcommand prevents you from
+making any changes (e.g., crush tunables, or using new features) that
+would violate the current setting. Please note, This subcommand will fail if
+any connected daemon or client is not compatible with the features offered by
+the given <version>. To see the features and releases of all clients connected
+to cluster, please see `ceph features`_.
+
+Usage::
+
+    ceph osd set-require-min-compat-client <version>
+
+Subcommand ``stat`` prints summary of OSD map.
+
+Usage::
+
+	ceph osd stat
+
+Subcommand ``tier`` is used for managing tiers. It uses some additional
+subcommands.
+
+Subcommand ``add`` adds the tier <tierpool> (the second one) to base pool <pool>
+(the first one).
+
+Usage::
+
+	ceph osd tier add <poolname> <poolname> {--force-nonempty}
+
+Subcommand ``add-cache`` adds a cache <tierpool> (the second one) of size <size>
+to existing pool <pool> (the first one).
+
+Usage::
+
+	ceph osd tier add-cache <poolname> <poolname> <int[0-]>
+
+Subcommand ``cache-mode`` specifies the caching mode for cache tier <pool>.
+
+Usage::
+
+	ceph osd tier cache-mode <poolname> writeback|readproxy|readonly|none
+
+Subcommand ``remove`` removes the tier <tierpool> (the second one) from base pool
+<pool> (the first one).
+
+Usage::
+
+	ceph osd tier remove <poolname> <poolname>
+
+Subcommand ``remove-overlay`` removes the overlay pool for base pool <pool>.
+
+Usage::
+
+	ceph osd tier remove-overlay <poolname>
+
+Subcommand ``set-overlay`` set the overlay pool for base pool <pool> to be
+<overlaypool>.
+
+Usage::
+
+	ceph osd tier set-overlay <poolname> <poolname>
+
+Subcommand ``tree`` prints OSD tree.
+
+Usage::
+
+	ceph osd tree {<int[0-]>}
+
+Subcommand ``unpause`` unpauses osd.
+
+Usage::
+
+	ceph osd unpause
+
+Subcommand ``unset`` unsets cluster-wide <flag> by updating OSD map.
+
+Usage::
+
+	ceph osd unset pause|noup|nodown|noout|noin|nobackfill|
+	norebalance|norecover|noscrub|nodeep-scrub|notieragent
+
+
+pg
+--
+
+It is used for managing the placement groups in OSDs. It uses some
+additional subcommands.
+
+Subcommand ``debug`` shows debug info about pgs.
+
+Usage::
+
+	ceph pg debug unfound_objects_exist|degraded_pgs_exist
+
+Subcommand ``deep-scrub`` starts deep-scrub on <pgid>.
+
+Usage::
+
+	ceph pg deep-scrub <pgid>
+
+Subcommand ``dump`` shows human-readable versions of pg map (only 'all' valid
+with plain).
+
+Usage::
+
+	ceph pg dump {all|summary|sum|delta|pools|osds|pgs|pgs_brief} [{all|summary|sum|delta|pools|osds|pgs|pgs_brief...]}
+
+Subcommand ``dump_json`` shows human-readable version of pg map in json only.
+
+Usage::
+
+	ceph pg dump_json {all|summary|sum|delta|pools|osds|pgs|pgs_brief} [{all|summary|sum|delta|pools|osds|pgs|pgs_brief...]}
+
+Subcommand ``dump_pools_json`` shows pg pools info in json only.
+
+Usage::
+
+	ceph pg dump_pools_json
+
+Subcommand ``dump_stuck`` shows information about stuck pgs.
+
+Usage::
+
+	ceph pg dump_stuck {inactive|unclean|stale|undersized|degraded [inactive|unclean|stale|undersized|degraded...]}
+	{<int>}
+
+Subcommand ``getmap`` gets binary pg map to -o/stdout.
+
+Usage::
+
+	ceph pg getmap
+
+Subcommand ``ls`` lists pg with specific pool, osd, state
+
+Usage::
+
+	ceph pg ls {<int>} {<pg-state> [<pg-state>...]}
+
+Subcommand ``ls-by-osd`` lists pg on osd [osd]
+
+Usage::
+
+	ceph pg ls-by-osd <osdname (id|osd.id)> {<int>}
+	{<pg-state> [<pg-state>...]}
+
+Subcommand ``ls-by-pool`` lists pg with pool = [poolname]
+
+Usage::
+
+	ceph pg ls-by-pool <poolstr> {<int>} {<pg-state> [<pg-state>...]}
+
+Subcommand ``ls-by-primary`` lists pg with primary = [osd]
+
+Usage::
+
+	ceph pg ls-by-primary <osdname (id|osd.id)> {<int>}
+	{<pg-state> [<pg-state>...]}
+
+Subcommand ``map`` shows mapping of pg to osds.
+
+Usage::
+
+	ceph pg map <pgid>
+
+Subcommand ``repair`` starts repair on <pgid>.
+
+Usage::
+
+	ceph pg repair <pgid>
+
+Subcommand ``scrub`` starts scrub on <pgid>.
+
+Usage::
+
+	ceph pg scrub <pgid>
+
+Subcommand ``stat`` shows placement group status.
+
+Usage::
+
+	ceph pg stat
+
+
+quorum
+------
+
+Cause a specific MON to enter or exit quorum.
+
+Usage::
+
+	ceph tell mon.<id> quorum enter|exit
+
+quorum_status
+-------------
+
+Reports status of monitor quorum.
+
+Usage::
+
+	ceph quorum_status
+
+
+report
+------
+
+Reports full status of cluster, optional title tag strings.
+
+Usage::
+
+	ceph report {<tags> [<tags>...]}
+
+
+status
+------
+
+Shows cluster status.
+
+Usage::
+
+	ceph status
+
+
+tell
+----
+
+Sends a command to a specific daemon.
+
+Usage::
+
+	ceph tell <name (type.id)> <command> [options...]
+
+
+List all available commands.
+
+Usage::
+
+ 	ceph tell <name (type.id)> help
+
+version
+-------
+
+Show mon daemon version
+
+Usage::
+
+	ceph version
+
+Options
+=======
+
+.. option:: -i infile
+
+   will specify an input file to be passed along as a payload with the
+   command to the monitor cluster. This is only used for specific
+   monitor commands.
+
+.. option:: -o outfile
+
+   will write any payload returned by the monitor cluster with its
+   reply to outfile.  Only specific monitor commands (e.g. osd getmap)
+   return a payload.
+
+.. option:: --setuser user
+
+   will apply the appropriate user ownership to the file specified by
+   the option '-o'.
+
+.. option:: --setgroup group
+
+   will apply the appropriate group ownership to the file specified by
+   the option '-o'.
+
+.. option:: -c ceph.conf, --conf=ceph.conf
+
+   Use ceph.conf configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` to determine monitor addresses during startup.
+
+.. option:: --id CLIENT_ID, --user CLIENT_ID
+
+   Client id for authentication.
+
+.. option:: --name CLIENT_NAME, -n CLIENT_NAME
+
+	Client name for authentication.
+
+.. option:: --cluster CLUSTER
+
+	Name of the Ceph cluster.
+
+.. option:: --admin-daemon ADMIN_SOCKET, daemon DAEMON_NAME
+
+	Submit admin-socket commands via admin sockets in /var/run/ceph.
+
+.. option:: --admin-socket ADMIN_SOCKET_NOPE
+
+	You probably mean --admin-daemon
+
+.. option:: -s, --status
+
+	Show cluster status.
+
+.. option:: -w, --watch
+
+	Watch live cluster changes on the default 'cluster' channel
+
+.. option:: -W, --watch-channel
+
+	Watch live cluster changes on any channel (cluster, audit, cephadm, or * for all)
+
+.. option:: --watch-debug
+
+	Watch debug events.
+
+.. option:: --watch-info
+
+	Watch info events.
+
+.. option:: --watch-sec
+
+	Watch security events.
+
+.. option:: --watch-warn
+
+	Watch warning events.
+
+.. option:: --watch-error
+
+	Watch error events.
+
+.. option:: --version, -v
+
+	Display version.
+
+.. option:: --verbose
+
+	Make verbose.
+
+.. option:: --concise
+
+	Make less verbose.
+
+.. option:: -f {json,json-pretty,xml,xml-pretty,plain}, --format
+
+	Format of output.
+
+.. option:: --connect-timeout CLUSTER_TIMEOUT
+
+	Set a timeout for connecting to the cluster.
+
+.. option:: --no-increasing
+
+	 ``--no-increasing`` is off by default. So increasing the osd weight is allowed
+         using the ``reweight-by-utilization`` or ``test-reweight-by-utilization`` commands.
+         If this option is used with these commands, it will help not to increase osd weight
+         even the osd is under utilized.
+
+.. option:: --block
+
+	 block until completion (scrub and deep-scrub only)
+
+Availability
+============
+
+:program:`ceph` is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at http://ceph.com/docs for more information.
+
+
+See also
+========
+
+:doc:`ceph-mon <ceph-mon>`\(8),
+:doc:`ceph-osd <ceph-osd>`\(8),
+:doc:`ceph-mds <ceph-mds>`\(8)
diff --git a/doc/man/8/cephadm.rst b/doc/man/8/cephadm.rst
new file mode 100644
index 000000000..ebcc3a6a2
--- /dev/null
+++ b/doc/man/8/cephadm.rst
@@ -0,0 +1,526 @@
+:orphan:
+
+=========================================
+ cephadm -- manage the local cephadm host
+=========================================
+
+.. program:: cephadm
+
+Synopsis
+========
+
+| **cephadm**** [-h] [--image IMAGE] [--docker] [--data-dir DATA_DIR]
+|               [--log-dir LOG_DIR] [--logrotate-dir LOGROTATE_DIR]
+|               [--unit-dir UNIT_DIR] [--verbose] [--timeout TIMEOUT]
+|               [--retry RETRY] [--no-container-init]
+|               {version,pull,inspect-image,ls,list-networks,adopt,rm-daemon,rm-cluster,run,shell,enter,ceph-volume,unit,logs,bootstrap,deploy,check-host,prepare-host,add-repo,rm-repo,install}
+|               ...
+
+
+| **cephadm** **pull**
+
+| **cephadm** **inspect-image**
+
+| **cephadm** **ls** [-h] [--no-detail] [--legacy-dir LEGACY_DIR]
+
+| **cephadm** **list-networks**
+
+| **cephadm** **adopt** [-h] --name NAME --style STYLE [--cluster CLUSTER]
+|                       [--legacy-dir LEGACY_DIR] [--config-json CONFIG_JSON]
+|                       [--skip-firewalld] [--skip-pull]
+
+| **cephadm** **rm-daemon** [-h] --name NAME --fsid FSID [--force]
+|                           [--force-delete-data]
+
+| **cephadm** **rm-cluster** [-h] --fsid FSID [--force]
+
+| **cephadm** **run** [-h] --name NAME --fsid FSID
+
+| **cephadm** **shell** [-h] [--fsid FSID] [--name NAME] [--config CONFIG]
+                        [--keyring KEYRING] --mount [MOUNT [MOUNT ...]] [--env ENV]
+                        [--] [command [command ...]]
+
+| **cephadm** **enter** [-h] [--fsid FSID] --name NAME [command [command ...]]
+
+| **cephadm** **ceph-volume** [-h] [--fsid FSID] [--config-json CONFIG_JSON]
+                              [--config CONFIG] [--keyring KEYRING]
+                              command [command ...]
+
+| **cephadm** **unit**  [-h] [--fsid FSID] --name NAME command
+
+| **cephadm** **logs** [-h] [--fsid FSID] --name NAME [command [command ...]]
+
+| **cephadm** **bootstrap** [-h] [--config CONFIG] [--mon-id MON_ID]
+|                           [--mon-addrv MON_ADDRV] [--mon-ip MON_IP]
+|                           [--mgr-id MGR_ID] [--fsid FSID]
+|                           [--log-to-file] [--single-host-defaults]
+|                           [--output-dir OUTPUT_DIR]
+|                           [--output-keyring OUTPUT_KEYRING]
+|                           [--output-config OUTPUT_CONFIG]
+|                           [--output-pub-ssh-key OUTPUT_PUB_SSH_KEY]
+|                           [--skip-ssh]
+|                           [--initial-dashboard-user INITIAL_DASHBOARD_USER]
+|                           [--initial-dashboard-password INITIAL_DASHBOARD_PASSWORD]
+|                           [--ssl-dashboard-port SSL_DASHBOARD_PORT]
+|                           [--dashboard-key DASHBOARD_KEY]
+|                           [--dashboard-crt DASHBOARD_CRT]
+|                           [--ssh-config SSH_CONFIG]
+|                           [--ssh-private-key SSH_PRIVATE_KEY]
+|                           [--ssh-public-key SSH_PUBLIC_KEY]
+|                           [--ssh-user SSH_USER] [--skip-mon-network]
+|                           [--skip-dashboard] [--dashboard-password-noupdate]
+|                           [--no-minimize-config] [--skip-ping-check]
+|                           [--skip-pull] [--skip-firewalld] [--allow-overwrite]
+|                           [--allow-fqdn-hostname] [--skip-prepare-host]
+|                           [--orphan-initial-daemons] [--skip-monitoring-stack]
+|                           [--apply-spec APPLY_SPEC]
+|                           [--registry-url REGISTRY_URL]
+|                           [--registry-username REGISTRY_USERNAME]
+|                           [--registry-password REGISTRY_PASSWORD]
+|                           [--registry-json REGISTRY_JSON]
+
+
+
+| **cephadm** **deploy** [-h] --name NAME --fsid FSID [--config CONFIG]
+|                        [--config-json CONFIG_JSON] [--keyring KEYRING]
+|                        [--key KEY] [--osd-fsid OSD_FSID] [--skip-firewalld]
+|                        [--tcp-ports TCP_PORTS] [--reconfig] [--allow-ptrace]
+
+| **cephadm** **check-host** [-h] [--expect-hostname EXPECT_HOSTNAME]
+
+| **cephadm** **prepare-host**
+
+| **cephadm** **add-repo** [-h] [--release RELEASE] [--version VERSION]
+|                          [--dev DEV] [--dev-commit DEV_COMMIT]
+|                          [--gpg-url GPG_URL] [--repo-url REPO_URL]
+
+
+| **cephadm** **rm-repo**
+
+| **cephadm** **install** [-h] [packages [packages ...]]
+
+| **cephadm** **registry-login** [-h] [--registry-url REGISTRY_URL]
+|                                [--registry-username REGISTRY_USERNAME]
+|                                [--registry-password REGISTRY_PASSWORD]
+|                                [--registry-json REGISTRY_JSON] [--fsid FSID]
+
+
+
+Description
+===========
+
+:program:`cephadm` is a command line tool to manage the local host for the cephadm orchestrator.
+
+It provides commands to investigate and modify the state of the current host.
+
+:program:`cephadm` is not required on all hosts, but useful when investigating a particular
+daemon.
+
+Options
+=======
+
+.. option:: --image IMAGE
+
+   container image. Can also be set via the
+   "CEPHADM_IMAGE" env var (default: None)
+
+.. option:: --docker
+
+   use docker instead of podman (default: False)
+
+.. option:: --data-dir DATA_DIR
+
+   base directory for daemon data (default: /var/lib/ceph)
+
+.. option:: --log-dir LOG_DIR
+
+   base directory for daemon logs (default: /var/log/ceph)
+
+.. option:: --logrotate-dir LOGROTATE_DIR
+
+   location of logrotate configuration files (default: /etc/logrotate.d)
+
+.. option:: --unit-dir UNIT_DIR
+
+   base directory for systemd units (default: /etc/systemd/system)
+
+.. option:: --verbose, -v
+
+   Show debug-level log messages (default: False)
+
+.. option:: --timeout TIMEOUT
+
+   timeout in seconds (default: None)
+
+.. option:: --retry RETRY
+
+   max number of retries (default: 10)
+
+.. option:: --no-container-init
+
+   do not run podman/docker with `--init` (default: False)
+
+
+Commands
+========
+
+add-repo
+--------
+
+configure local package repository to also include the ceph repository.
+
+Arguments:
+
+* [--release RELEASE]       use latest version of a named release (e.g., octopus)
+* [--version VERSION]       use specific upstream version (x.y.z)
+* [--dev DEV]               use specified bleeding edge build from git branch or tag
+* [--dev-commit DEV_COMMIT] use specified bleeding edge build from git commit
+* [--gpg-url GPG_URL]       specify alternative GPG key location
+* [--repo-url REPO_URL]     specify alternative repo location
+
+
+adopt
+-----
+
+Adopt a daemon deployed with a different deployment tool.
+
+Arguments:
+
+* [--name NAME, -n NAME]       daemon name (type.id)
+* [--style STYLE]              deployment style (legacy, ...)
+* [--cluster CLUSTER]          cluster name
+* [--legacy-dir LEGACY_DIR]    base directory for legacy daemon data
+* [--config-json CONFIG_JSON]  Additional configuration information in JSON format
+* [--skip-firewalld]           Do not configure firewalld
+* [--skip-pull]                do not pull the latest image before adopting
+
+
+bootstrap
+---------
+
+Bootstrap a cluster on the local host. It deploys a MON and a MGR and then also automatically
+deploys the monitoring stack on this host (see --skip-monitoring-stack) and calls
+``ceph orch host add $(hostname)`` (see --skip-ssh).
+
+Arguments:
+
+* [--config CONFIG, -c CONFIG]    ceph conf file to incorporate
+* [--mon-id MON_ID]               mon id (default: local hostname)
+* [--mon-addrv MON_ADDRV]         mon IPs (e.g., [v2:localipaddr:3300,v1:localipaddr:6789])
+* [--mon-ip MON_IP]               mon IP
+* [--mgr-id MGR_ID]               mgr id (default: randomly generated)
+* [--fsid FSID]                   cluster FSID
+* [--log-to-file]                 configure cluster to log to traditional log files
+* [--single-host-defaults]        configure cluster to run on a single host
+* [--output-dir OUTPUT_DIR]       directory to write config, keyring, and pub key files
+* [--output-keyring OUTPUT_KEYRING] location to write keyring file with new cluster admin and mon keys
+* [--output-config OUTPUT_CONFIG] location to write conf file to connect to new cluster
+* [--output-pub-ssh-key OUTPUT_PUB_SSH_KEY] location to write the cluster's public SSH key
+* [--skip-ssh                     skip setup of ssh key on local host
+* [--initial-dashboard-user INITIAL_DASHBOARD_USER] Initial user for the dashboard
+* [--initial-dashboard-password INITIAL_DASHBOARD_PASSWORD] Initial password for the initial dashboard user
+* [--ssl-dashboard-port SSL_DASHBOARD_PORT] Port number used to connect with dashboard using SSL
+* [--dashboard-key DASHBOARD_KEY] Dashboard key
+* [--dashboard-crt DASHBOARD_CRT] Dashboard certificate
+* [--ssh-config SSH_CONFIG] SSH config
+* [--ssh-private-key SSH_PRIVATE_KEY] SSH private key
+* [--ssh-public-key SSH_PUBLIC_KEY] SSH public key
+* [--ssh-user SSH_USER]           set user for SSHing to cluster hosts, passwordless sudo will be needed for non-root users'
+* [--skip-mon-network]            set mon public_network based on bootstrap mon ip
+* [--skip-dashboard]              do not enable the Ceph Dashboard
+* [--dashboard-password-noupdate] stop forced dashboard password change
+* [--no-minimize-config]          do not assimilate and minimize the config file
+* [--skip-ping-check]             do not verify that mon IP is pingable
+* [--skip-pull]                   do not pull the latest image before bootstrapping
+* [--skip-firewalld]              Do not configure firewalld
+* [--allow-overwrite]             allow overwrite of existing --output-* config/keyring/ssh files
+* [--allow-fqdn-hostname]         allow hostname that is fully-qualified (contains ".")
+* [--skip-prepare-host]           Do not prepare host
+* [--orphan-initial-daemons]      Do not create initial mon, mgr, and crash service specs
+* [--skip-monitoring-stack]       Do not automatically provision monitoring stack] (prometheus, grafana, alertmanager, node-exporter)
+* [--apply-spec APPLY_SPEC]       Apply cluster spec after bootstrap (copy ssh key, add hosts and apply services)
+* [--registry-url REGISTRY_URL]   url of custom registry to login to. e.g. docker.io, quay.io
+* [--registry-username REGISTRY_USERNAME] username of account to login to on custom registry
+* [--registry-password REGISTRY_PASSWORD] password of account to login to on custom registry
+* [--registry-json REGISTRY_JSON] JSON file containing registry login info (see registry-login command documentation)
+
+
+ceph-volume
+-----------
+
+Run ceph-volume inside a container::
+
+    cephadm ceph-volume inventory
+
+Positional arguments:
+* [command]               command
+
+Arguments:
+
+* [--fsid FSID]                    cluster FSID
+* [--config-json CONFIG_JSON]      JSON file with config and (client.bootrap-osd) key
+* [--config CONFIG, -c CONFIG]     ceph conf file
+* [--keyring KEYRING, -k KEYRING]  ceph.keyring to pass through to the container
+
+
+check-host
+----------
+
+check host configuration to be suitable for a Ceph cluster.
+
+Arguments:
+
+* [--expect-hostname EXPECT_HOSTNAME] Check that hostname matches an expected value
+
+
+deploy
+------
+
+deploy a daemon on the local host. Used by the orchestrator CLI::
+
+    cephadm shell -- ceph orch apply <type> ...
+
+Arguments:
+
+* [--name NAME]               daemon name (type.id)
+* [--fsid FSID]               cluster FSID
+* [--config CONFIG, -c CONFIG] config file for new daemon
+* [--config-json CONFIG_JSON] Additional configuration information in JSON format
+* [--keyring KEYRING]         keyring for new daemon
+* [--key KEY]                 key for new daemon
+* [--osd-fsid OSD_FSID]       OSD uuid, if creating an OSD container
+* [--skip-firewalld]          Do not configure firewalld
+* [--tcp-ports                List of tcp ports to open in the host firewall
+* [--reconfig]                Reconfigure a previously deployed daemon
+* [--allow-ptrace]            Allow SYS_PTRACE on daemon container
+
+
+enter
+-----
+
+Run an interactive shell inside a running daemon container::
+
+    cephadm enter --name mgr.myhost.ysubfo
+
+Positional arguments:
+* [command]               command
+
+Arguments:
+
+* [--fsid FSID]           cluster FSID
+* [--name NAME, -n NAME]  daemon name (type.id)
+
+install
+-------
+
+install ceph package(s)
+
+Positional arguments:
+
+* [packages]    packages
+
+
+inspect-image
+-------------
+
+inspect local ceph container image.
+
+list-networks
+-------------
+
+list IP networks
+
+
+ls
+--
+
+list daemon instances known to cephadm on **this** host::
+
+    $ cephadm ls
+    [
+        {
+            "style": "cephadm:v1",
+            "name": "mgr.storage-14b-1.ysubfo",
+            "fsid": "5110cb22-8332-11ea-9148-0894ef7e8bdc",
+            "enabled": true,
+            "state": "running",
+            "container_id": "8562de72370a3836473ecfff8a22c9ccdd99815386b4692a2b30924fb5493c44",
+            "container_image_name": "docker.io/ceph/ceph:v15",
+            "container_image_id": "bc83a388465f0568dab4501fb7684398dca8b50ca12a342a57f21815721723c2",
+            "version": "15.2.1",
+            "started": "2020-04-21T01:16:41.831456",
+            "created": "2020-04-21T01:16:41.775024",
+            "deployed": "2020-04-21T01:16:41.415021",
+            "configured": "2020-04-21T01:16:41.775024"
+        },
+    ...
+
+Arguments:
+
+* [--no-detail]             Do not include daemon status
+* [--legacy-dir LEGACY_DIR] Base directory for legacy daemon data
+
+logs
+----
+
+print journald logs for a daemon container::
+
+    cephadm logs --name mgr.myhost.ysubfo
+
+This is similar to::
+
+    journalctl -u mgr.myhost.ysubfo
+
+Can also specify additional journal arguments::
+
+    cephadm logs --name mgr.myhost.ysubfo -- -n 20 # last 20 lines
+    cephadm logs --name mgr.myhost.ysubfo -- -f # follow the log
+
+
+Positional arguments:
+
+* [command]               command (optional)
+
+Arguments:
+
+* [--fsid FSID]           cluster FSID
+* [--name NAME, -n NAME]  daemon name (type.id)
+
+
+prepare-host
+------------
+
+prepare a host for cephadm use
+
+Arguments:
+
+* [--expect-hostname EXPECT_HOSTNAME] Set hostname
+
+
+pull
+----
+
+Pull the ceph image::
+
+    cephadm pull
+
+registry-login
+--------------
+
+Give cephadm login information for an authenticated registry (url, username and password).
+Cephadm will attempt to log the calling host into that registry::
+
+      cephadm registry-login --registry-url [REGISTRY_URL] --registry-username [USERNAME]
+                             --registry-password [PASSWORD]
+
+Can also use a JSON file containing the login info formatted as::
+
+      {
+       "url":"REGISTRY_URL",
+       "username":"REGISTRY_USERNAME",
+       "password":"REGISTRY_PASSWORD"
+      }
+
+and turn it in with command::
+
+      cephadm registry-login --registry-json [JSON FILE]
+
+Arguments:
+
+* [--registry-url REGISTRY_URL]   url of registry to login to. e.g. docker.io, quay.io
+* [--registry-username REGISTRY_USERNAME] username of account to login to on registry
+* [--registry-password REGISTRY_PASSWORD] password of account to login to on registry
+* [--registry-json REGISTRY_JSON] JSON file containing login info for custom registry
+* [--fsid FSID]                   cluster FSID
+
+rm-daemon
+---------
+
+Remove a specific daemon instance
+
+Arguments:
+
+* [--name NAME, -n NAME]  daemon name (type.id)
+* [--fsid FSID]           cluster FSID
+* [--force]               proceed, even though this may destroy valuable data
+* [--force-delete-data]   delete valuable daemon data instead of making a backup
+
+
+rm-cluster
+----------
+
+remove all daemons for a cluster
+
+Arguments:
+
+* [--fsid FSID]  cluster FSID
+* [--force]      proceed, even though this may destroy valuable data
+
+rm-repo
+-------
+
+remove package repository configuration
+
+run
+---
+
+run a ceph daemon, in a container, in the foreground
+
+Arguments:
+
+* [--name NAME, -n NAME]  daemon name (type.id)
+* [--fsid FSID]           cluster FSID
+
+
+shell
+-----
+
+Run an interactive shell::
+
+    cephadm shell
+
+Or one specific command inside a container::
+
+    cephadm shell -- ceph orch ls
+
+
+Positional arguments:
+
+* [command]               command (optional)
+
+Arguments:
+
+* [--fsid FSID]                   cluster FSID
+* [--name NAME, -n NAME]          daemon name (type.id)
+* [--config CONFIG, -c CONFIG]    ceph.conf to pass through to the container
+* [--keyring KEYRING, -k KEYRING] ceph.keyring to pass through to the container
+* [--mount MOUNT, -m MOUNT]       mount a file or directory under /mnt in the container
+* [--env ENV, -e ENV]             set environment variable
+
+
+unit
+----
+
+Operate on the daemon's systemd unit.
+
+Positional arguments:
+
+* [command]               systemd command (start, stop, restart, enable, disable, ...)
+
+Arguments:
+
+* [--fsid FSID]           cluster FSID
+* [--name NAME, -n NAME]  daemon name (type.id)
+
+
+Availability
+============
+
+:program:`cephadm` is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the documentation at http://docs.ceph.com/ for more information.
+
+
+See also
+========
+
+:doc:`ceph-volume <ceph-volume>`\(8),
diff --git a/doc/man/8/cephfs-mirror.rst b/doc/man/8/cephfs-mirror.rst
new file mode 100644
index 000000000..3448dd047
--- /dev/null
+++ b/doc/man/8/cephfs-mirror.rst
@@ -0,0 +1,66 @@
+:orphan:
+
+============================================================
+ cephfs-mirror -- Ceph daemon for mirroring CephFS snapshots
+============================================================
+
+.. program:: cephfs-mirror
+
+Synopsis
+========
+
+| **cephfs-mirror**
+
+
+Description
+===========
+
+:program:`cephfs-mirror` is a daemon for asynchronous mirroring of Ceph
+Filesystem snapshots among Ceph clusters.
+
+It connects to remote clusters via libcephfs, relying on default search
+paths to find ceph.conf files, i.e. ``/etc/ceph/$cluster.conf`` where
+``$cluster`` is the human-friendly name of the cluster.
+
+
+Options
+=======
+
+.. option:: -c ceph.conf, --conf=ceph.conf
+
+   Use ``ceph.conf`` configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` to determine monitor addresses during startup.
+
+.. option:: -i ID, --id ID
+
+   Set the ID portion of name for cephfs-mirror
+
+.. option:: -n TYPE.ID, --name TYPE.ID
+
+   Set the rados user name (eg. client.mirror)
+
+.. option:: --cluster NAME
+
+   Set the cluster name (default: ceph)
+
+.. option:: -d
+
+   Run in foreground, log to stderr
+
+.. option:: -f
+
+   Run in foreground, log to usual location
+
+
+Availability
+============
+
+:program:`cephfs-mirror` is part of Ceph, a massively scalable, open-source, distributed
+storage system. Please refer to the Ceph documentation at http://ceph.com/docs for
+more information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8)
diff --git a/doc/man/8/cephfs-top.rst b/doc/man/8/cephfs-top.rst
new file mode 100644
index 000000000..c3719cd36
--- /dev/null
+++ b/doc/man/8/cephfs-top.rst
@@ -0,0 +1,121 @@
+:orphan:
+
+==========================================
+ cephfs-top -- Ceph Filesystem Top Utility
+==========================================
+
+.. program:: cephfs-top
+
+Synopsis
+========
+
+| **cephfs-top** [flags]
+
+
+Description
+===========
+
+**cephfs-top** provides top(1) like functionality for Ceph Filesystem.
+Various client metrics are displayed and updated in realtime.
+
+Ceph Metadata Servers periodically send client metrics to Ceph Manager.
+``Stats`` plugin in Ceph Manager provides an interface to fetch these metrics.
+
+Options
+=======
+
+.. option:: --cluster
+
+   Cluster: Ceph cluster to connect. Defaults to ``ceph``.
+
+.. option:: --id
+
+   Id: Client used to connect to Ceph cluster. Defaults to ``fstop``.
+
+.. option:: --selftest
+
+   Perform a selftest. This mode performs a sanity check of ``stats`` module.
+
+Descriptions of fields
+======================
+
+.. describe:: chit
+
+   cap hit rate
+
+.. describe:: dlease
+
+   dentry lease rate
+
+.. describe:: ofiles
+
+   number of opened files
+
+.. describe:: oicaps
+
+   number of pinned caps
+
+.. describe:: oinodes
+
+   number of opened inodes
+
+.. describe:: rtio
+
+   total size of read IOs
+
+.. describe:: wtio
+
+   total size of write IOs
+
+.. describe:: raio
+
+   average size of read IOs
+
+.. describe:: waio
+
+   average size of write IOs
+
+.. describe:: rsp
+
+   speed of read IOs compared with the last refresh
+
+.. describe:: wsp
+
+   speed of write IOs compared with the last refresh
+
+.. describe:: rlatavg
+
+   average read latency
+
+.. describe:: rlatsd
+
+   standard deviation (variance) for read latency
+
+.. describe:: wlatavg
+
+   average write latency
+
+.. describe:: wlatsd
+
+   standard deviation (variance) for write latency
+
+.. describe:: mlatavg
+
+   average metadata latency
+
+.. describe:: mlatsd
+
+   standard deviation (variance) for metadata latency
+
+Availability
+============
+
+**cephfs-top** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to the Ceph documentation at
+http://ceph.com/ for more information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
+:doc:`ceph-mds <ceph-mds>`\(8)
diff --git a/doc/man/8/crushtool.rst b/doc/man/8/crushtool.rst
new file mode 100644
index 000000000..31bb8272e
--- /dev/null
+++ b/doc/man/8/crushtool.rst
@@ -0,0 +1,302 @@
+:orphan:
+
+==========================================
+ crushtool -- CRUSH map manipulation tool
+==========================================
+
+.. program:: crushtool
+
+Synopsis
+========
+
+| **crushtool** ( -d *map* | -c *map.txt* | --build --num_osds *numosds*
+  *layer1* *...* | --test ) [ -o *outfile* ]
+
+
+Description
+===========
+
+**crushtool** is a utility that lets you create, compile, decompile
+and test CRUSH map files.
+
+CRUSH is a pseudo-random data distribution algorithm that efficiently
+maps input values (which, in the context of Ceph, correspond to Placement
+Groups) across a heterogeneous, hierarchically structured device map.
+The algorithm was originally described in detail in the following paper
+(although it has evolved some since then)::
+
+   http://www.ssrc.ucsc.edu/Papers/weil-sc06.pdf
+
+The tool has four modes of operation.
+
+.. option:: --compile|-c map.txt
+
+   will compile a plaintext map.txt into a binary map file.
+
+.. option:: --decompile|-d map
+
+   will take the compiled map and decompile it into a plaintext source
+   file, suitable for editing.
+
+.. option:: --build --num_osds {num-osds} layer1 ...
+
+   will create map with the given layer structure. See below for a
+   detailed explanation.
+
+.. option:: --test
+
+   will perform a dry run of a CRUSH mapping for a range of input
+   values ``[--min-x,--max-x]`` (default ``[0,1023]``) which can be
+   thought of as simulated Placement Groups. See below for a more
+   detailed explanation.
+
+Unlike other Ceph tools, **crushtool** does not accept generic options
+such as **--debug-crush** from the command line. They can, however, be
+provided via the CEPH_ARGS environment variable. For instance, to
+silence all output from the CRUSH subsystem::
+
+    CEPH_ARGS="--debug-crush 0" crushtool ...
+
+
+Running tests with --test
+=========================
+
+The test mode will use the input crush map ( as specified with **-i
+map** ) and perform a dry run of CRUSH mapping or random placement
+(if **--simulate** is set ). On completion, two kinds of reports can be
+created. 
+1) The **--show-...** option outputs human readable information
+on stderr. 
+2) The **--output-csv** option creates CSV files that are
+documented by the **--help-output** option.
+
+Note: Each Placement Group (PG) has an integer ID which can be obtained
+from ``ceph pg dump`` (for example PG 2.2f means pool id 2, PG id 32).
+The pool and PG IDs are combined by a function to get a value which is
+given to CRUSH to map it to OSDs. crushtool does not know about PGs or
+pools; it only runs simulations by mapping values in the range
+``[--min-x,--max-x]``.
+
+
+.. option:: --show-statistics
+
+   Displays a summary of the distribution. For instance::
+
+       rule 1 (metadata) num_rep 5 result size == 5:	1024/1024
+
+   shows that rule **1** which is named **metadata** successfully
+   mapped **1024** values to **result size == 5** devices when trying
+   to map them to **num_rep 5** replicas. When it fails to provide the
+   required mapping, presumably because the number of **tries** must
+   be increased, a breakdown of the failures is displayed. For instance::
+
+       rule 1 (metadata) num_rep 10 result size == 8:	4/1024
+       rule 1 (metadata) num_rep 10 result size == 9:	93/1024
+       rule 1 (metadata) num_rep 10 result size == 10:	927/1024
+
+   shows that although **num_rep 10** replicas were required, **4**
+   out of **1024** values ( **4/1024** ) were mapped to **result size
+   == 8** devices only.
+
+.. option:: --show-mappings
+
+   Displays the mapping of each value in the range ``[--min-x,--max-x]``.
+   For instance::
+
+       CRUSH rule 1 x 24 [11,6]
+
+   shows that value **24** is mapped to devices **[11,6]** by rule
+   **1**.
+
+   One of the following is required when using the ``--show-mappings`` option:
+   
+        (a) ``--num-rep`` 
+        (b) both ``--min-rep`` and ``--max-rep``
+
+   ``--num-rep`` stands for "number of replicas, indicates the number of
+   replicas in a pool, and is used to specify an exact number of replicas (for
+   example ``--num-rep 5``). ``--min-rep`` and ``--max-rep`` are used together
+   to specify a range of replicas (for example, ``--min-rep 1 --max-rep 10``).
+
+.. option:: --show-bad-mappings
+
+   Displays which value failed to be mapped to the required number of
+   devices. For instance::
+
+     bad mapping rule 1 x 781 num_rep 7 result [8,10,2,11,6,9]
+
+   shows that when rule **1** was required to map **7** devices, it
+   could map only six : **[8,10,2,11,6,9]**.
+
+.. option:: --show-utilization
+
+   Displays the expected and actual utilization for each device, for
+   each number of replicas. For instance::
+
+     device 0: stored : 951      expected : 853.333
+     device 1: stored : 963      expected : 853.333
+     ...
+
+   shows that device **0** stored **951** values and was expected to store **853**.
+   Implies **--show-statistics**.
+
+.. option:: --show-utilization-all
+
+   Displays the same as **--show-utilization** but does not suppress
+   output when the weight of a device is zero.
+   Implies **--show-statistics**.
+
+.. option:: --show-choose-tries
+
+   Displays how many attempts were needed to find a device mapping.
+   For instance::
+
+      0:     95224
+      1:      3745
+      2:      2225
+      ..
+
+   shows that **95224** mappings succeeded without retries, **3745**
+   mappings succeeded with one attempts, etc. There are as many rows
+   as the value of the **--set-choose-total-tries** option.
+
+.. option:: --output-csv
+
+   Creates CSV files (in the current directory) containing information
+   documented by **--help-output**. The files are named after the rule
+   used when collecting the statistics. For instance, if the rule
+   : 'metadata' is used, the CSV files will be::
+
+      metadata-absolute_weights.csv
+      metadata-device_utilization.csv
+      ...
+
+   The first line of the file shortly explains the column layout. For
+   instance::
+
+      metadata-absolute_weights.csv
+      Device ID, Absolute Weight
+      0,1
+      ...
+
+.. option:: --output-name NAME
+
+   Prepend **NAME** to the file names generated when **--output-csv**
+   is specified. For instance **--output-name FOO** will create
+   files::
+
+      FOO-metadata-absolute_weights.csv
+      FOO-metadata-device_utilization.csv
+      ...
+
+The **--set-...** options can be used to modify the tunables of the
+input crush map. The input crush map is modified in
+memory. For example::
+
+      $ crushtool -i mymap --test --show-bad-mappings
+      bad mapping rule 1 x 781 num_rep 7 result [8,10,2,11,6,9]
+
+could be fixed by increasing the **choose-total-tries** as follows:
+
+      $ crushtool -i mymap --test \
+          --show-bad-mappings \
+          --set-choose-total-tries 500
+
+Building a map with --build
+===========================
+
+The build mode will generate hierarchical maps. The first argument
+specifies the number of devices (leaves) in the CRUSH hierarchy. Each
+layer describes how the layer (or devices) preceding it should be
+grouped.
+
+Each layer consists of::
+
+       bucket ( uniform | list | tree | straw | straw2 ) size
+
+The **bucket** is the type of the buckets in the layer
+(e.g. "rack"). Each bucket name will be built by appending a unique
+number to the **bucket** string (e.g. "rack0", "rack1"...).
+
+The second component is the type of bucket: **straw** should be used
+most of the time.
+
+The third component is the maximum size of the bucket. A size of zero
+means a bucket of infinite capacity.
+
+
+Example
+=======
+
+Suppose we have two rows with two racks each and 20 nodes per rack. Suppose
+each node contains 4 storage devices for Ceph OSD Daemons. This configuration
+allows us to deploy 320 Ceph OSD Daemons. Lets assume a 42U rack with 2U nodes,
+leaving an extra 2U for a rack switch.
+
+To reflect our hierarchy of devices, nodes, racks and rows, we would execute
+the following::
+
+    $ crushtool -o crushmap --build --num_osds 320 \
+           node straw 4 \
+           rack straw 20 \
+           row straw 2 \
+           root straw 0
+    # id	weight	type name	reweight
+    -87	320	root root
+    -85	160		row row0
+    -81	80			rack rack0
+    -1	4				node node0
+    0	1					osd.0	1
+    1	1					osd.1	1
+    2	1					osd.2	1
+    3	1					osd.3	1
+    -2	4				node node1
+    4	1					osd.4	1
+    5	1					osd.5	1
+    ...
+
+CRUSH rules are created so the generated crushmap can be
+tested. They are the same rules as the ones created by default when
+creating a new Ceph cluster. They can be further edited with::
+
+       # decompile
+       crushtool -d crushmap -o map.txt
+
+       # edit
+       emacs map.txt
+
+       # recompile
+       crushtool -c map.txt -o crushmap
+
+Reclassify
+==========
+
+The *reclassify* function allows users to transition from older maps that
+maintain parallel hierarchies for OSDs of different types to a modern CRUSH
+map that makes use of the *device class* feature.  For more information,
+see https://docs.ceph.com/en/latest/rados/operations/crush-map-edits/#migrating-from-a-legacy-ssd-rule-to-device-classes.
+
+Example output from --test
+==========================
+
+See https://github.com/ceph/ceph/blob/master/src/test/cli/crushtool/set-choose.t
+for sample ``crushtool --test`` commands and output produced thereby.
+
+Availability
+============
+
+**crushtool** is part of Ceph, a massively scalable, open-source, distributed storage system. Please
+refer to the Ceph documentation at http://ceph.com/docs for more
+information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
+:doc:`osdmaptool <osdmaptool>`\(8),
+
+Authors
+=======
+
+John Wilkins, Sage Weil, Loic Dachary
diff --git a/doc/man/8/librados-config.rst b/doc/man/8/librados-config.rst
new file mode 100644
index 000000000..940e8c2e4
--- /dev/null
+++ b/doc/man/8/librados-config.rst
@@ -0,0 +1,46 @@
+:orphan:
+
+=======================================================
+ librados-config -- display information about librados
+=======================================================
+
+.. program:: librados-config
+
+Synopsis
+========
+
+| **librados-config** [ --version ] [ --vernum ]
+
+
+Description
+===========
+
+**librados-config** is a utility that displays information about the
+  installed ``librados``.
+
+
+Options
+=======
+
+.. option:: --version
+
+   Display ``librados`` version
+
+.. option:: --vernum
+
+   Display the ``librados`` version code
+
+
+Availability
+============
+
+**librados-config** is part of Ceph, a massively scalable, open-source, distributed storage system.
+Please refer to the Ceph documentation at http://ceph.com/docs for
+more information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
+:doc:`rados <rados>`\(8)
diff --git a/doc/man/8/monmaptool.rst b/doc/man/8/monmaptool.rst
new file mode 100644
index 000000000..f564f8f2e
--- /dev/null
+++ b/doc/man/8/monmaptool.rst
@@ -0,0 +1,140 @@
+:orphan:
+
+==========================================================
+ monmaptool -- ceph monitor cluster map manipulation tool
+==========================================================
+
+.. program:: monmaptool
+
+Synopsis
+========
+
+| **monmaptool** <action> [options] *mapfilename*
+
+
+Description
+===========
+
+**monmaptool** is a utility to create, view, and modify a monitor
+cluster map for the Ceph distributed storage system. The monitor map
+specifies the only fixed addresses in the Ceph distributed system.
+All other daemons bind to arbitrary addresses and register themselves
+with the monitors.
+
+When creating a map with --create, a new monitor map with a new,
+random UUID will be created. It should be followed by one or more
+monitor addresses.
+
+The default Ceph monitor port for messenger protocol v1 is 6789, and
+3300 for protocol v2.
+
+Multiple actions can be performed per invocation.
+
+
+Options
+=======
+
+.. option:: --print
+
+   print a plaintext dump of the map, after any modifications are
+   made.
+
+.. option:: --feature-list [plain|parseable]
+
+   list the enabled features as well as the available ones.
+
+   By default, a human readable output is produced.
+
+.. option:: --create
+
+   create a new monitor map with a new UUID (and with it, a new,
+   empty Ceph cluster).
+
+.. option:: --clobber
+
+   allow monmaptool to create a new mapfilename in place of an existing map.
+
+   Only useful when *--create* is used.
+
+.. option:: --generate
+
+   generate a new monmap based on the values on the command line or specified
+   in the ceph configuration.  This is, in order of preference,
+
+      #. ``--monmap filename`` to specify a monmap to load
+      #. ``--mon-host 'host1,ip2'`` to specify a list of hosts or ip addresses
+      #. ``[mon.foo]`` sections containing ``mon addr`` settings in the config. Note that this method is not recommended and support will be removed in a future release.
+
+.. option:: --filter-initial-members
+
+   filter the initial monmap by applying the ``mon initial members``
+   setting.  Monitors not present in that list will be removed, and
+   initial members not present in the map will be added with dummy
+   addresses.
+
+.. option:: --add name ip[:port]
+
+   add a monitor with the specified ip:port to the map.
+
+   If the *nautilus* feature is set, and the port is not, the monitor
+   will be added for both messenger protocols.
+
+.. option:: --addv name [protocol:ip:port[,...]]
+
+   add a monitor with the specified version:ip:port to the map.
+
+.. option:: --rm name
+
+   remove the monitor with the specified name from the map.
+
+.. option:: --fsid uuid
+
+   set the fsid to the given uuid.  If not specified with *--create*, a random fsid will be generated.
+
+.. option:: --feature-set value [--optional|--persistent]
+
+   enable a feature.
+
+.. option:: --feature-unset value [--optional|--persistent]
+
+   disable a feature.
+
+.. option:: --enable-all-features
+
+   enable all supported features.
+
+.. option:: --set-min-mon-release release
+
+   set the min_mon_release.
+
+Example
+=======
+
+To create a new map with three monitors (for a fresh Ceph cluster)::
+
+        monmaptool --create --add nodeA 192.168.0.10 --add nodeB 192.168.0.11 \
+          --add nodeC 192.168.0.12 --enable-all-features --clobber monmap
+
+To display the contents of the map::
+
+        monmaptool --print monmap
+
+To replace one monitor::
+
+        monmaptool --rm nodeA monmap
+        monmaptool --add nodeA 192.168.0.9 monmap
+
+
+Availability
+============
+
+**monmaptool** is part of Ceph, a massively scalable, open-source, distributed 
+storage system. Please refer to the Ceph documentation at http://ceph.com/docs 
+for more information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
+:doc:`crushtool <crushtool>`\(8),
diff --git a/doc/man/8/mount.ceph.rst b/doc/man/8/mount.ceph.rst
new file mode 100644
index 000000000..1c67a12f8
--- /dev/null
+++ b/doc/man/8/mount.ceph.rst
@@ -0,0 +1,256 @@
+:orphan:
+
+========================================
+ mount.ceph -- mount a Ceph file system
+========================================
+
+.. program:: mount.ceph
+
+Synopsis
+========
+
+| **mount.ceph** [*mon1_socket*\ ,\ *mon2_socket*\ ,...]:/[*subdir*] *dir* [
+  -o *options* ]
+
+
+Description
+===========
+
+**mount.ceph** is a helper for mounting the Ceph file system on a Linux host.
+It serves to resolve monitor hostname(s) into IP addresses and read
+authentication keys from disk; the Linux kernel client component does most of
+the real work. In fact, it is possible to mount a non-authenticated Ceph file
+system without mount.ceph by specifying monitor address(es) by IP::
+
+        mount -t ceph 1.2.3.4:/ /mnt/mycephfs
+
+The first argument is the device part of the mount command. It includes host's
+socket and path within CephFS that will be mounted at the mount point. The
+socket, obviously, takes the form ip_address[:port]. If the port is not
+specified, the Ceph default of 6789 is assumed. Multiple monitor addresses can
+be passed by separating them by commas. Only one monitor is needed to mount
+successfully; the client will learn about all monitors from any responsive
+monitor. However, it is a good idea to specify more than one in case the one
+happens to be down at the time of mount.
+
+If the host portion of the device is left blank, then **mount.ceph** will
+attempt to determine monitor addresses using local configuration files
+and/or DNS SRV records. In similar way, if authentication is enabled on Ceph
+cluster (which is done using CephX) and options ``secret`` and ``secretfile``
+are not specified in the command, the mount helper will spawn a child process
+that will use the standard Ceph library routines to find a keyring and fetch
+the secret from it.
+
+A sub-directory of the file system can be mounted by specifying the (absolute)
+path to the sub-directory right after ":" after the socket in the device part
+of the mount command.
+
+Mount helper application conventions dictate that the first two options are
+device to be mounted and the mountpoint for that device. Options must be
+passed only after these fixed arguments.
+
+
+Options
+=======
+
+Basic
+-----
+
+:command:`conf`
+    Path to a ceph.conf file. This is used to initialize the Ceph context
+    for autodiscovery of monitor addresses and auth secrets. The default is
+    to use the standard search path for ceph.conf files.
+
+:command: `fs=<fs-name>`
+    Specify the non-default file system to be mounted. Not passing this
+    option mounts the default file system.
+
+:command: `mds_namespace=<fs-name>`
+    A synonym of "fs=" and its use is deprecated.
+
+:command:`mount_timeout`
+    int (seconds), Default: 60
+
+:command:`ms_mode=<legacy|crc|secure|prefer-crc|prefer-secure>`
+    Set the connection mode that the client uses for transport. The available
+    modes are:
+
+    - ``legacy``: use messenger v1 protocol to talk to the cluster
+
+    - ``crc``: use messenger v2, without on-the-wire encryption
+
+    - ``secure``: use messenger v2, with on-the-wire encryption
+
+    - ``prefer-crc``: crc mode, if denied agree to secure mode
+
+    - ``prefer-secure``: secure mode, if denied agree to crc mode
+
+:command:`name`
+    RADOS user to authenticate as when using CephX. Default: guest
+
+:command:`secret`
+    secret key for use with CephX. This option is insecure because it exposes
+    the secret on the command line. To avoid this, use the secretfile option.
+
+:command:`secretfile`
+    path to file containing the secret key to use with CephX
+
+:command:`recover_session=<no|clean>`
+    Set auto reconnect mode in the case where the client is blocklisted. The
+    available modes are ``no`` and ``clean``. The default is ``no``.
+
+    - ``no``: never attempt to reconnect when client detects that it has been
+       blocklisted. Blocklisted clients will not attempt to reconnect and
+       their operations will fail too.
+
+    - ``clean``: client reconnects to the Ceph cluster automatically when it
+      detects that it has been blocklisted. During reconnect, client drops
+      dirty data/metadata, invalidates page caches and writable file handles.
+      After reconnect, file locks become stale because the MDS loses track of
+      them. If an inode contains any stale file locks, read/write on the inode
+      is not allowed until applications release all stale file locks.
+
+Advanced
+--------
+:command:`cap_release_safety`
+    int, Default: calculated
+
+:command:`caps_wanted_delay_max`
+    int, cap release delay, Default: 60
+
+:command:`caps_wanted_delay_min`
+    int, cap release delay, Default: 5
+
+:command:`dirstat`
+    funky `cat dirname` for stats, Default: off
+
+:command:`nodirstat`
+    no funky `cat dirname` for stats
+
+:command:`ip`
+    my ip
+
+:command:`noasyncreaddir`
+    no dcache readdir
+
+:command:`nocrc`
+    no data crc on writes
+
+:command:`noshare`
+    create a new client instance, instead of sharing an existing instance of
+    a client mounting the same cluster
+
+:command:`osdkeepalive`
+    int, Default: 5
+
+:command:`osd_idle_ttl`
+    int (seconds), Default: 60
+
+:command:`rasize`
+    int (bytes), max readahead. Default: 8388608 (8192*1024)
+
+:command:`rbytes`
+    Report the recursive size of the directory contents for st_size on
+    directories.  Default: off
+
+:command:`norbytes`
+    Do not report the recursive size of the directory contents for
+    st_size on directories.
+
+:command:`readdir_max_bytes`
+    int, Default: 524288 (512*1024)
+
+:command:`readdir_max_entries`
+    int, Default: 1024
+
+:command:`rsize`
+    int (bytes), max read size. Default: 16777216 (16*1024*1024)
+
+:command:`snapdirname`
+    string, set the name of the hidden snapdir. Default: .snap
+
+:command:`write_congestion_kb`
+    int (kb), max writeback in flight. scale with available
+    memory. Default: calculated from available memory
+
+:command:`wsize`
+    int (bytes), max write size. Default: 16777216 (16*1024*1024) (writeback
+    uses smaller of wsize and stripe unit)
+
+:command:`wsync`
+    Execute all namespace operations synchronously. This ensures that the
+    namespace operation will only complete after receiving a reply from
+    the MDS. This is the default.
+
+:command:`nowsync`
+    Allow the client to do namespace operations asynchronously. When this
+    option is enabled, a namespace operation may complete before the MDS
+    replies, if it has sufficient capabilities to do so.
+
+Examples
+========
+
+Mount the full file system::
+
+    mount.ceph :/ /mnt/mycephfs
+
+Assuming mount.ceph is installed properly, it should be automatically invoked
+by mount(8)::
+
+    mount -t ceph :/ /mnt/mycephfs
+
+Mount only part of the namespace/file system::
+
+    mount.ceph :/some/directory/in/cephfs /mnt/mycephfs
+
+Mount non-default FS, in case cluster has multiple FSs::
+    mount -t ceph :/ /mnt/mycephfs2 -o fs=mycephfs2
+    
+    or
+    
+    mount -t ceph :/ /mnt/mycephfs2 -o mds_namespace=mycephfs2 # This option name is deprecated.
+
+Pass the monitor host's IP address, optionally::
+
+    mount.ceph 192.168.0.1:/ /mnt/mycephfs
+
+Pass the port along with IP address if it's running on a non-standard port::
+
+    mount.ceph 192.168.0.1:7000:/ /mnt/mycephfs
+
+If there are multiple monitors, passes addresses separated by a comma::
+
+   mount.ceph 192.168.0.1,192.168.0.2,192.168.0.3:/ /mnt/mycephfs
+
+If authentication is enabled on Ceph cluster::
+
+    mount.ceph :/ /mnt/mycephfs -o name=fs_username
+
+Pass secret key for CephX user optionally::
+
+    mount.ceph :/ /mnt/mycephfs -o name=fs_username,secret=AQATSKdNGBnwLhAAnNDKnH65FmVKpXZJVasUeQ==
+
+Pass file containing secret key to avoid leaving secret key in shell's command
+history::
+
+    mount.ceph :/ /mnt/mycephfs -o name=fs_username,secretfile=/etc/ceph/fs_username.secret
+
+
+Availability
+============
+
+**mount.ceph** is part of Ceph, a massively scalable, open-source, distributed
+storage system. Please refer to the Ceph documentation at http://ceph.com/docs
+for more information.
+
+Feature Availability
+====================
+
+The ``recover_session=`` option was added to mainline Linux kernels in v5.4.
+``wsync`` and ``nowsync`` were added in v5.7.
+
+See also
+========
+
+:doc:`ceph-fuse <ceph-fuse>`\(8),
+:doc:`ceph <ceph>`\(8)
diff --git a/doc/man/8/mount.fuse.ceph.rst b/doc/man/8/mount.fuse.ceph.rst
new file mode 100644
index 000000000..0b3b2d67d
--- /dev/null
+++ b/doc/man/8/mount.fuse.ceph.rst
@@ -0,0 +1,71 @@
+:orphan:
+
+====================================================
+ mount.fuse.ceph -- mount ceph-fuse from /etc/fstab.
+====================================================
+
+.. program:: mount.fuse.ceph
+
+Synopsis
+========
+
+| **mount.fuse.ceph** [-h] [-o OPTIONS [*OPTIONS* ...]]
+                      device [*device* ...]
+                      mountpoint [*mountpoint* ...]
+
+Description
+===========
+
+**mount.fuse.ceph** is a helper for mounting ceph-fuse from
+``/etc/fstab``.
+
+To use mount.fuse.ceph, add an entry in ``/etc/fstab`` like::
+
+  DEVICE    PATH        TYPE        OPTIONS
+  none      /mnt/ceph   fuse.ceph   ceph.id=admin,_netdev,defaults  0 0
+  none      /mnt/ceph   fuse.ceph   ceph.name=client.admin,_netdev,defaults  0 0
+  none      /mnt/ceph   fuse.ceph   ceph.id=myuser,ceph.conf=/etc/ceph/foo.conf,_netdev,defaults  0 0
+
+ceph-fuse options are specified in the ``OPTIONS`` column and must begin
+with '``ceph.``' prefix. This way ceph related fs options will be passed to
+ceph-fuse and others will be ignored by ceph-fuse.
+
+Options
+=======
+
+.. option:: ceph.id=<username>
+
+   Specify that the ceph-fuse will authenticate as the given user.
+
+.. option:: ceph.name=client.admin
+
+   Specify that the ceph-fuse will authenticate as client.admin
+
+.. option:: ceph.conf=/etc/ceph/foo.conf
+
+   Sets 'conf' option to /etc/ceph/foo.conf via ceph-fuse command line.
+
+
+Any valid ceph-fuse options can be passed this way.   
+
+Additional Info
+===============
+
+The old format /etc/fstab entries are also supported::
+
+  DEVICE                              PATH        TYPE        OPTIONS
+  id=admin                            /mnt/ceph   fuse.ceph   defaults   0 0
+  id=myuser,conf=/etc/ceph/foo.conf   /mnt/ceph   fuse.ceph   defaults   0 0
+
+Availability
+============
+
+**mount.fuse.ceph** is part of Ceph, a massively scalable, open-source, distributed storage system. Please
+refer to the Ceph documentation at http://ceph.com/docs for more
+information.
+
+See also
+========
+
+:doc:`ceph-fuse <ceph-fuse>`\(8),
+:doc:`ceph <ceph>`\(8)
diff --git a/doc/man/8/osdmaptool.rst b/doc/man/8/osdmaptool.rst
new file mode 100644
index 000000000..92ad8039c
--- /dev/null
+++ b/doc/man/8/osdmaptool.rst
@@ -0,0 +1,331 @@
+:orphan:
+
+.. _osdmaptool:
+
+======================================================
+ osdmaptool -- ceph osd cluster map manipulation tool
+======================================================
+
+.. program:: osdmaptool
+
+Synopsis
+========
+
+| **osdmaptool** *mapfilename* [--print] [--createsimple *numosd*
+  [--pgbits *bitsperosd* ] ] [--clobber]
+| **osdmaptool** *mapfilename* [--import-crush *crushmap*]
+| **osdmaptool** *mapfilename* [--export-crush *crushmap*]
+| **osdmaptool** *mapfilename* [--upmap *file*] [--upmap-max *max-optimizations*]
+  [--upmap-deviation *max-deviation*] [--upmap-pool *poolname*]
+  [--save] [--upmap-active]
+| **osdmaptool** *mapfilename* [--upmap-cleanup] [--upmap *file*]
+
+
+Description
+===========
+
+**osdmaptool** is a utility that lets you create, view, and manipulate
+OSD cluster maps from the Ceph distributed storage system. Notably, it
+lets you extract the embedded CRUSH map or import a new CRUSH map.
+It can also simulate the upmap balancer mode so you can get a sense of
+what is needed to balance your PGs.
+
+
+Options
+=======
+
+.. option:: --print
+
+   will simply make the tool print a plaintext dump of the map, after
+   any modifications are made.
+
+.. option:: --dump <format>
+
+   displays the map in plain text when <format> is 'plain', 'json' if specified
+   format is not supported. This is an alternative to the print option.
+
+.. option:: --clobber
+
+   will allow osdmaptool to overwrite mapfilename if changes are made.
+
+.. option:: --import-crush mapfile
+
+   will load the CRUSH map from mapfile and embed it in the OSD map.
+
+.. option:: --export-crush mapfile
+
+   will extract the CRUSH map from the OSD map and write it to
+   mapfile.
+
+.. option:: --createsimple numosd [--pg-bits bitsperosd] [--pgp-bits bits]
+
+   will create a relatively generic OSD map with the numosd devices.
+   If --pg-bits is specified, the initial placement group counts will
+   be set with bitsperosd bits per OSD. That is, the pg_num map
+   attribute will be set to numosd shifted by bitsperosd.
+   If --pgp-bits is specified, then the pgp_num map attribute will
+   be set to numosd shifted by bits. 
+
+.. option:: --create-from-conf
+
+   creates an osd map with default configurations.
+
+.. option:: --test-map-pgs [--pool poolid] [--range-first <first> --range-last <last>]
+
+   will print out the mappings from placement groups to OSDs.
+   If range is specified, then it iterates from first to last in the directory 
+   specified by argument to osdmaptool.
+   Eg: **osdmaptool --test-map-pgs --range-first 0 --range-last 2 osdmap_dir**.
+   This will iterate through the files named 0,1,2 in osdmap_dir.
+
+.. option:: --test-map-pgs-dump [--pool poolid] [--range-first <first> --range-last <last>]
+
+   will print out the summary of all placement groups and the mappings from them to the mapped OSDs.
+   If range is specified, then it iterates from first to last in the directory
+   specified by argument to osdmaptool.
+   Eg: **osdmaptool --test-map-pgs-dump --range-first 0 --range-last 2 osdmap_dir**.
+   This will iterate through the files named 0,1,2 in osdmap_dir.
+
+.. option:: --test-map-pgs-dump-all [--pool poolid] [--range-first <first> --range-last <last>]
+
+   will print out the summary of all placement groups and the mappings
+   from them to all the OSDs.
+   If range is specified, then it iterates from first to last in the directory
+   specified by argument to osdmaptool.
+   Eg: **osdmaptool --test-map-pgs-dump-all --range-first 0 --range-last 2 osdmap_dir**.
+   This will iterate through the files named 0,1,2 in osdmap_dir.
+
+.. option:: --test-random
+
+   does a random mapping of placement groups to the OSDs.
+
+.. option:: --test-map-pg <pgid>
+
+   map a particular placement group(specified by pgid) to the OSDs.
+
+.. option:: --test-map-object <objectname> [--pool <poolid>]
+
+   map a particular placement group(specified by objectname) to the OSDs.
+
+.. option:: --test-crush [--range-first <first> --range-last <last>]
+
+   map placement groups to acting OSDs.
+   If range is specified, then it iterates from first to last in the directory
+   specified by argument to osdmaptool.
+   Eg: **osdmaptool --test-crush --range-first 0 --range-last 2 osdmap_dir**.
+   This will iterate through the files named 0,1,2 in osdmap_dir.
+
+.. option:: --mark-up-in
+
+   mark osds up and in (but do not persist).
+
+.. option:: --mark-out
+
+   mark an osd as out (but do not persist)
+
+.. option:: --mark-up <osdid>
+
+   mark an osd as up (but do not persist)
+
+.. option:: --mark-in <osdid>
+
+   mark an osd as in (but do not persist)
+
+.. option:: --tree
+
+   Displays a hierarchical tree of the map.
+
+.. option:: --clear-temp
+
+   clears pg_temp and primary_temp variables.
+
+.. option:: --clean-temps
+
+   clean pg_temps.
+
+.. option:: --health
+
+   dump health checks
+
+.. option:: --with-default-pool
+
+   include default pool when creating map
+
+.. option:: --upmap-cleanup <file>
+
+   clean up pg_upmap[_items] entries, writing commands to <file> [default: - for stdout]
+
+.. option:: --upmap <file>
+
+   calculate pg upmap entries to balance pg layout writing commands to <file> [default: - for stdout]
+
+.. option:: --upmap-max <max-optimizations>
+
+   set max upmap entries to calculate [default: 10]
+
+.. option:: --upmap-deviation <max-deviation>
+
+   max deviation from target [default: 5]
+
+.. option:: --upmap-pool <poolname>
+
+   restrict upmap balancing to 1 pool or the option can be repeated for multiple pools
+
+.. option:: --upmap-active
+
+   Act like an active balancer, keep applying changes until balanced
+
+.. option:: --adjust-crush-weight <osdid:weight>[,<osdid:weight>,<...>]
+
+   Change CRUSH weight of <osdid>
+
+.. option:: --save
+
+   write modified osdmap with upmap or crush-adjust changes
+
+Example
+=======
+
+To create a simple map with 16 devices::
+
+        osdmaptool --createsimple 16 osdmap --clobber
+
+To view the result::
+
+        osdmaptool --print osdmap
+
+To view the mappings of placement groups for pool 1::
+
+        osdmaptool osdmap --test-map-pgs-dump --pool 1
+
+        pool 1 pg_num 8
+        1.0     [0,2,1] 0
+        1.1     [2,0,1] 2
+        1.2     [0,1,2] 0
+        1.3     [2,0,1] 2
+        1.4     [0,2,1] 0
+        1.5     [0,2,1] 0
+        1.6     [0,1,2] 0
+        1.7     [1,0,2] 1
+        #osd    count   first   primary c wt    wt
+        osd.0   8       5       5       1       1
+        osd.1   8       1       1       1       1
+        osd.2   8       2       2       1       1
+         in 3
+         avg 8 stddev 0 (0x) (expected 2.3094 0.288675x))
+         min osd.0 8
+         max osd.0 8
+        size 0  0
+        size 1  0
+        size 2  0
+        size 3  8
+
+In which,
+ #. pool 1 has 8 placement groups. And two tables follow:
+ #. A table for placement groups. Each row presents a placement group. With columns of:
+
+    * placement group id,
+    * acting set, and
+    * primary OSD.
+ #. A table for all OSDs. Each row presents an OSD. With columns of:
+
+    * count of placement groups being mapped to this OSD,
+    * count of placement groups where this OSD is the first one in their acting sets,
+    * count of placement groups where this OSD is the primary of them,
+    * the CRUSH weight of this OSD, and
+    * the weight of this OSD.
+ #. Looking at the number of placement groups held by 3 OSDs. We have
+
+    * avarge, stddev, stddev/average, expected stddev, expected stddev / average
+    * min and max
+ #. The number of placement groups mapping to n OSDs. In this case, all 8 placement
+    groups are mapping to 3 different OSDs.
+
+In a less-balanced cluster, we could have following output for the statistics of
+placement group distribution, whose standard deviation is 1.41421::
+
+        #osd    count   first   primary c wt    wt
+        osd.0   8       5       5       1       1
+        osd.1   8       1       1       1       1
+        osd.2   8       2       2       1       1
+
+        #osd    count   first    primary c wt    wt
+        osd.0   33      9        9       0.0145874     1
+        osd.1   34      14       14      0.0145874     1
+        osd.2   31      7        7       0.0145874     1
+        osd.3   31      13       13      0.0145874     1
+        osd.4   30      14       14      0.0145874     1
+        osd.5   33      7        7       0.0145874     1
+         in 6
+         avg 32 stddev 1.41421 (0.0441942x) (expected 5.16398 0.161374x))
+         min osd.4 30
+         max osd.1 34
+        size 00
+        size 10
+        size 20
+        size 364
+
+To simulate the active balancer in upmap mode::
+
+        osdmaptool --upmap upmaps.out --upmap-active --upmap-deviation 6 --upmap-max 11 osdmap
+
+   osdmaptool: osdmap file 'osdmap'
+   writing upmap command output to: upmaps.out
+   checking for upmap cleanups
+   upmap, max-count 11, max deviation 6
+   pools movies photos metadata data
+   prepared 11/11 changes
+   Time elapsed 0.00310404 secs
+   pools movies photos metadata data
+   prepared 11/11 changes
+   Time elapsed 0.00283402 secs
+   pools data metadata movies photos
+   prepared 11/11 changes
+   Time elapsed 0.003122 secs
+   pools photos metadata data movies
+   prepared 11/11 changes
+   Time elapsed 0.00324372 secs
+   pools movies metadata data photos
+   prepared 1/11 changes
+   Time elapsed 0.00222609 secs
+   pools data movies photos metadata
+   prepared 0/11 changes
+   Time elapsed 0.00209916 secs
+   Unable to find further optimization, or distribution is already perfect
+   osd.0 pgs 41
+   osd.1 pgs 42
+   osd.2 pgs 42
+   osd.3 pgs 41
+   osd.4 pgs 46
+   osd.5 pgs 39
+   osd.6 pgs 39
+   osd.7 pgs 43
+   osd.8 pgs 41
+   osd.9 pgs 46
+   osd.10 pgs 46
+   osd.11 pgs 46
+   osd.12 pgs 46
+   osd.13 pgs 41
+   osd.14 pgs 40
+   osd.15 pgs 40
+   osd.16 pgs 39
+   osd.17 pgs 46
+   osd.18 pgs 46
+   osd.19 pgs 39
+   osd.20 pgs 42
+   Total time elapsed 0.0167765 secs, 5 rounds
+
+
+Availability
+============
+
+**osdmaptool** is part of Ceph, a massively scalable, open-source, distributed storage system.  Please
+refer to the Ceph documentation at http://ceph.com/docs for more
+information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
+:doc:`crushtool <crushtool>`\(8),
diff --git a/doc/man/8/rados.rst b/doc/man/8/rados.rst
new file mode 100644
index 000000000..147313f14
--- /dev/null
+++ b/doc/man/8/rados.rst
@@ -0,0 +1,404 @@
+:orphan:
+
+=======================================
+ rados -- rados object storage utility
+=======================================
+
+.. program:: rados
+
+Synopsis
+========
+
+| **rados** [ *options* ] [ *command* ]
+
+
+Description
+===========
+
+**rados** is a utility for interacting with a Ceph object storage
+cluster (RADOS), part of the Ceph distributed storage system.
+
+
+Global Options
+==============
+
+.. option:: --object-locator object_locator
+
+   Set object_locator for operation.
+
+.. option:: -p pool, --pool pool
+
+   Interact with the given pool. Required by most commands.
+
+.. option:: --target-pool pool
+
+   Select target pool by name.
+
+.. option:: --pgid
+
+   As an alternative to ``--pool``, ``--pgid`` also allow users to specify the
+   PG id to which the command will be directed. With this option, certain
+   commands like ``ls`` allow users to limit the scope of the command to the given PG.
+
+.. option:: -N namespace, --namespace namespace
+
+   Specify the rados namespace to use for the object.
+
+.. option:: --all
+
+   Use with ls to list objects in all namespaces.
+   Put in CEPH_ARGS environment variable to make this the default.
+
+.. option:: --default
+
+   Use with ls to list objects in default namespace.
+   Takes precedence over --all in case --all is in environment.
+
+.. option:: -s snap, --snap snap
+
+   Read from the given pool snapshot. Valid for all pool-specific read operations.
+
+.. option:: --create
+
+   Create the pool or directory that was specified.
+
+.. option:: -i infile
+
+   will specify an input file to be passed along as a payload with the
+   command to the monitor cluster. This is only used for specific
+   monitor commands.
+
+.. option:: -m monaddress[:port]
+
+   Connect to specified monitor (instead of looking through ceph.conf).
+
+.. option:: -b block_size
+
+  Set the block size for put/get/append ops and for write benchmarking.
+
+.. option:: --striper
+
+   Uses the striping API of rados rather than the default one.
+   Available for stat, stat2, get, put, append, truncate, rm, ls
+   and all xattr related operation.
+
+.. option:: -O object_size, --object-size object_size
+
+   Set the object size for put/get ops and for write benchmarking.
+
+.. option:: --max-objects
+
+   Set the max number of objects for write benchmarking.
+
+.. option:: --lock-cookie locker-cookie
+
+   Will set the lock cookie for acquiring advisory lock (lock get command).
+   If the cookie is not empty, this option must be passed to lock break command
+   to find the correct lock when releasing lock.
+
+.. option:: --target-locator
+
+   Use with cp to specify the locator of the new object.
+
+.. option:: --target-nspace
+
+   Use with cp to specify the namespace of the new object.
+
+
+Bench options
+=============
+
+.. option:: -t N, --concurrent-ios=N
+
+   Set number of concurrent I/O operations.
+
+.. option:: --show-time
+
+   Prefix output with date/time.
+
+.. option:: --no-verify
+
+   Do not verify contents of read objects.
+
+.. option:: --write-object
+
+   Write contents to the objects.
+
+.. option:: --write-omap
+
+   Write contents to the omap.
+
+.. option:: --write-xattr
+
+   Write contents to the extended attributes.
+
+
+Load gen options
+================
+
+.. option:: --num-objects
+
+   Total number of objects.
+
+.. option:: --min-object-size
+
+  Min object size.
+
+.. option:: --max-object-size
+
+   Max object size.
+
+.. option:: --min-op-len
+
+   Min io size of operations.
+
+.. option:: --max-op-len
+
+   Max io size of operations.
+
+.. option:: --max-ops
+
+   Max number of operations.
+
+.. option:: --max-backlog
+
+   Max backlog size.
+
+.. option:: --read-percent
+
+   Percent of operations that are read.
+
+.. option:: --target-throughput
+
+   Target throughput (in bytes).
+
+.. option:: --run-length
+
+   Total time (in seconds).
+
+.. option:: --offset-align
+
+   At what boundary to align random op offsets.
+
+
+Cache pools options
+===================
+
+.. option:: --with-clones
+
+   Include clones when doing flush or evict.
+
+
+OMAP options
+============
+
+.. option:: --omap-key-file file
+
+   Read the omap key from a file.
+
+
+Generic options
+===============
+
+.. option:: -c FILE, --conf FILE
+
+   Read configuration from the given configuration file.
+
+.. option:: --id ID
+
+   Set ID portion of my name.
+
+.. option:: -n TYPE.ID, --name TYPE.ID
+
+   Set cephx user name.
+
+.. option:: --cluster NAME
+
+   Set cluster name (default: ceph).
+
+.. option:: --setuser USER
+
+   Set uid to user or uid (and gid to user's gid).
+
+.. option:: --setgroup GROUP
+
+   Set gid to group or gid.
+
+.. option:: --version
+
+   Show version and quit.
+
+
+Global commands
+===============
+
+:command:`lspools`
+  List object pools
+
+:command:`df`
+  Show utilization statistics, including disk usage (bytes) and object
+  counts, over the entire system and broken down by pool.
+
+:command:`list-inconsistent-pg` *pool*
+  List inconsistent PGs in given pool.
+
+:command:`list-inconsistent-obj` *pgid*
+  List inconsistent objects in given PG.
+
+:command:`list-inconsistent-snapset` *pgid*
+  List inconsistent snapsets in given PG.
+
+
+Pool specific commands
+======================
+
+:command:`get` *name* *outfile*
+  Read object name from the cluster and write it to outfile.
+
+:command:`put` *name* *infile* [--offset offset]
+  Write object name with start offset (default:0) to the cluster with contents from infile.
+  **Warning:** The put command creates a single RADOS object, sized just as
+  large as your input file. Unless your objects are of reasonable and consistent sizes, that
+  is probably not what you want -- consider using RGW/S3, CephFS, or RBD instead.
+
+:command:`append` *name* *infile*
+  Append object name to the cluster with contents from infile.
+
+:command:`rm` *name*
+  Remove object name.
+
+:command:`listwatchers` *name*
+  List the watchers of object name.
+
+:command:`ls` *outfile*
+  List objects in the given pool and write to outfile. Instead of ``--pool`` if ``--pgid`` will be specified, ``ls`` will only list the objects in the given PG.
+
+:command:`lssnap`
+  List snapshots for given pool.
+
+:command:`clonedata` *srcname* *dstname* --object-locator *key*
+  Clone object byte data from *srcname* to *dstname*.  Both objects must be stored with the locator key *key* (usually either *srcname* or *dstname*).  Object attributes and omap keys are not copied or cloned.
+
+:command:`mksnap` *foo*
+  Create pool snapshot named *foo*.
+
+:command:`rmsnap` *foo*
+  Remove pool snapshot named *foo*.
+
+:command:`bench` *seconds* *mode* [ -b *objsize* ] [ -t *threads* ]
+  Benchmark for *seconds*. The mode can be *write*, *seq*, or
+  *rand*. *seq* and *rand* are read benchmarks, either
+  sequential or random. Before running one of the reading benchmarks,
+  run a write benchmark with the *--no-cleanup* option. The default
+  object size is 4 MB, and the default number of simulated threads
+  (parallel writes) is 16. The *--run-name <label>* option is useful
+  for benchmarking a workload test from multiple clients. The *<label>*
+  is an arbitrary object name. It is "benchmark_last_metadata" by
+  default, and is used as the underlying object name for "read" and
+  "write" ops.
+  Note: -b *objsize* option is valid only in *write* mode.
+  Note: *write* and *seq* must be run on the same host otherwise the
+  objects created by *write* will have names that will fail *seq*.
+
+:command:`cleanup` [ --run-name *run_name* ] [ --prefix *prefix* ]
+  Clean up a previous benchmark operation.
+  Note: the default run-name is "benchmark_last_metadata"
+
+:command:`listxattr` *name*
+  List all extended attributes of an object.
+
+:command:`getxattr` *name* *attr*
+  Dump the extended attribute value of *attr* of an object.
+
+:command:`setxattr` *name* *attr* *value*
+  Set the value of *attr* in the extended attributes of an object.
+
+:command:`rmxattr` *name* *attr*
+  Remove *attr* from the extended attributes of an object.
+
+:command:`stat` *name*
+   Get stat (ie. mtime, size) of given object
+
+:command:`stat2` *name*
+   Get stat (similar to stat, but with high precision time) of given object
+
+:command:`listomapkeys` *name*
+  List all the keys stored in the object map of object name.
+
+:command:`listomapvals` *name*
+  List all key/value pairs stored in the object map of object name.
+  The values are dumped in hexadecimal.
+
+:command:`getomapval` [ --omap-key-file *file* ] *name* *key* [ *out-file* ]
+  Dump the hexadecimal value of key in the object map of object name.
+  If the optional *out-file* argument is not provided, the value will be
+  written to standard output.
+
+:command:`setomapval` [ --omap-key-file *file* ] *name* *key* [ *value* ]
+  Set the value of key in the object map of object name. If the optional
+  *value* argument is not provided, the value will be read from standard
+  input.
+
+:command:`rmomapkey` [ --omap-key-file *file* ] *name* *key*
+  Remove key from the object map of object name.
+
+:command:`getomapheader` *name*
+  Dump the hexadecimal value of the object map header of object name.
+
+:command:`setomapheader` *name* *value*
+  Set the value of the object map header of object name.
+
+:command:`export` *filename*
+  Serialize pool contents to a file or standard output.\n"
+
+:command:`import` [--dry-run] [--no-overwrite] < filename | - >
+  Load pool contents from a file or standard input
+
+
+Examples
+========
+
+To view cluster utilization::
+
+       rados df
+
+To get a list object in pool foo sent to stdout::
+
+       rados -p foo ls -
+
+To get a list of objects in PG 0.6::
+
+       rados --pgid 0.6 ls
+
+To write an object::
+
+       rados -p foo put myobject blah.txt
+
+To create a snapshot::
+
+       rados -p foo mksnap mysnap
+
+To delete the object::
+
+       rados -p foo rm myobject
+
+To read a previously snapshotted version of an object::
+
+       rados -p foo -s mysnap get myobject blah.txt.old
+
+To list inconsistent objects in PG 0.6::
+
+       rados list-inconsistent-obj 0.6 --format=json-pretty
+
+
+Availability
+============
+
+**rados** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at http://ceph.com/docs for more information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8)
diff --git a/doc/man/8/radosgw-admin.rst b/doc/man/8/radosgw-admin.rst
new file mode 100644
index 000000000..6fa2aba3e
--- /dev/null
+++ b/doc/man/8/radosgw-admin.rst
@@ -0,0 +1,1023 @@
+:orphan:
+
+=================================================================
+ radosgw-admin -- rados REST gateway user administration utility
+=================================================================
+
+.. program:: radosgw-admin
+
+Synopsis
+========
+
+| **radosgw-admin** *command* [ *options* *...* ]
+
+
+Description
+===========
+
+:program:`radosgw-admin` is a RADOS gateway user administration utility. It
+allows creating and modifying users.
+
+
+Commands
+========
+
+:program:`radosgw-admin` utility uses many commands for administration purpose
+which are as follows:
+
+:command:`user create`
+  Create a new user.
+
+:command:`user modify`
+  Modify a user.
+
+:command:`user info`
+  Display information of a user, and any potentially available
+  subusers and keys.
+
+:command:`user rename`
+  Renames a user.
+
+:command:`user rm`
+  Remove a user.
+
+:command:`user suspend`
+  Suspend a user.
+
+:command:`user enable`
+  Re-enable user after suspension.
+
+:command:`user check`
+  Check user info.
+
+:command:`user stats`
+  Show user stats as accounted by quota subsystem.
+
+:command:`user list`
+  List all users.
+
+:command:`caps add`
+  Add user capabilities.
+
+:command:`caps rm`
+  Remove user capabilities.
+
+:command:`subuser create`
+  Create a new subuser (primarily useful for clients using the Swift API).
+
+:command:`subuser modify`
+  Modify a subuser.
+
+:command:`subuser rm`
+  Remove a subuser.
+
+:command:`key create`
+  Create access key.
+
+:command:`key rm`
+  Remove access key.
+
+:command:`bucket list`
+  List buckets, or, if bucket specified with --bucket=<bucket>,
+  list its objects. If bucket specified adding --allow-unordered
+  removes ordering requirement, possibly generating results more
+  quickly in buckets with large number of objects.
+
+:command:`bucket limit check`
+  Show bucket sharding stats.
+
+:command:`bucket link`
+  Link bucket to specified user.
+
+:command:`bucket unlink`
+  Unlink bucket from specified user.
+
+:command:`bucket chown`
+  Link bucket to specified user and update object ACLs. 
+  Use --marker to resume if command gets interrupted.
+
+:command:`bucket stats`
+  Returns bucket statistics.
+
+:command:`bucket rm`
+  Remove a bucket.
+
+:command:`bucket check`
+  Check bucket index.
+
+:command:`bucket rewrite`
+  Rewrite all objects in the specified bucket.
+
+:command:`bucket radoslist`
+  List the rados objects that contain the data for all objects is
+  the designated bucket, if --bucket=<bucket> is specified, or
+  otherwise all buckets.
+
+:command:`bucket reshard`
+  Reshard a bucket.
+
+:command:`bucket sync disable`
+  Disable bucket sync.
+
+:command:`bucket sync enable`
+  Enable bucket sync.
+
+:command:`bi get`
+  Retrieve bucket index object entries.
+
+:command:`bi put`
+  Store bucket index object entries.
+
+:command:`bi list`
+  List raw bucket index entries.
+
+:command:`bi purge`
+  Purge bucket index entries.
+
+:command:`object rm`
+  Remove an object.
+
+:command:`object stat`
+  Stat an object for its metadata.
+
+:command:`object unlink`
+  Unlink object from bucket index.
+
+:command:`object rewrite`
+  Rewrite the specified object.
+
+:command:`objects expire`
+  Run expired objects cleanup.
+
+:command:`period rm`
+  Remove a period.
+
+:command:`period get`
+  Get the period info.
+
+:command:`period get-current`
+  Get the current period info.
+
+:command:`period pull`
+  Pull a period.
+
+:command:`period push`
+  Push a period.
+
+:command:`period list`
+  List all periods.
+
+:command:`period update`
+  Update the staging period.
+
+:command:`period commit`
+  Commit the staging period.
+
+:command:`quota set`
+  Set quota params.
+
+:command:`quota enable`
+  Enable quota.
+
+:command:`quota disable`
+  Disable quota.
+
+:command:`global quota get`
+  View global quota parameters.
+
+:command:`global quota set`
+  Set global quota parameters.
+
+:command:`global quota enable`
+  Enable a global quota.
+
+:command:`global quota disable`
+  Disable a global quota.
+
+:command:`realm create`
+  Create a new realm.
+
+:command:`realm rm`
+  Remove a realm.
+
+:command:`realm get`
+  Show the realm info.
+
+:command:`realm get-default`
+  Get the default realm name.
+
+:command:`realm list`
+  List all realms.
+
+:command:`realm list-periods`
+  List all realm periods.
+
+:command:`realm rename`
+  Rename a realm.
+
+:command:`realm set`
+  Set the realm info (requires infile).
+
+:command:`realm default`
+  Set the realm as default.
+
+:command:`realm pull`
+  Pull a realm and its current period.
+
+:command:`zonegroup add`
+  Add a zone to a zonegroup.
+
+:command:`zonegroup create`
+  Create a new zone group info.
+
+:command:`zonegroup default`
+  Set the default zone group.
+
+:command:`zonegroup rm`
+  Remove a zone group info.
+
+:command:`zonegroup get`
+  Show the zone group info.
+
+:command:`zonegroup modify`
+  Modify an existing zonegroup.
+
+:command:`zonegroup set`
+  Set the zone group info (requires infile).
+
+:command:`zonegroup remove`
+  Remove a zone from a zonegroup.
+
+:command:`zonegroup rename`
+  Rename a zone group.
+
+:command:`zonegroup list`
+  List all zone groups set on this cluster.
+
+:command:`zonegroup placement list`
+  List zonegroup's placement targets.
+
+:command:`zonegroup placement add`
+  Add a placement target id to a zonegroup.
+
+:command:`zonegroup placement modify`
+  Modify a placement target of a specific zonegroup.
+
+:command:`zonegroup placement rm`
+  Remove a placement target from a zonegroup.
+
+:command:`zonegroup placement default`
+  Set a zonegroup's default placement target.
+
+:command:`zone create`
+  Create a new zone.
+
+:command:`zone rm`
+  Remove a zone.
+
+:command:`zone get`
+  Show zone cluster params.
+
+:command:`zone set`
+  Set zone cluster params (requires infile).
+
+:command:`zone modify`
+  Modify an existing zone.
+
+:command:`zone list`
+  List all zones set on this cluster.
+
+:command:`metadata sync status`
+  Get metadata sync status.
+
+:command:`metadata sync init`
+  Init metadata sync.
+
+:command:`metadata sync run`
+  Run metadata sync.
+
+:command:`data sync status`
+  Get data sync status of the specified source zone.
+  
+:command:`data sync init`
+  Init data sync for the specified source zone.
+
+:command:`data sync run`
+  Run data sync for the specified source zone.
+
+:command:`sync error list`
+  list sync error.
+
+:command:`sync error trim`
+  trim sync error.
+
+:command:`zone rename`
+  Rename a zone.
+
+:command:`zone placement list`
+  List zone's placement targets.
+
+:command:`zone placement add`
+  Add a zone placement target.
+
+:command:`zone placement modify`
+  Modify a zone placement target.
+
+:command:`zone placement rm`
+  Remove a zone placement target.
+
+:command:`pool add`
+  Add an existing pool for data placement.
+
+:command:`pool rm`
+  Remove an existing pool from data placement set.
+
+:command:`pools list`
+  List placement active set.
+
+:command:`policy`
+  Display bucket/object policy.
+
+:command:`log list`
+  List log objects.
+
+:command:`log show`
+  Dump a log from specific object or (bucket + date + bucket-id).
+  (NOTE: required to specify formatting of date to "YYYY-MM-DD-hh")
+
+:command:`log rm`
+  Remove log object.
+
+:command:`usage show`
+  Show the usage information (with optional user and date range).
+
+:command:`usage trim`
+  Trim usage information (with optional user and date range).
+
+:command:`gc list`
+  Dump expired garbage collection objects (specify --include-all to list all
+  entries, including unexpired).
+
+:command:`gc process`
+  Manually process garbage.
+
+:command:`lc list`
+  List all bucket lifecycle progress.
+
+:command:`lc process`
+  Manually process lifecycle.
+
+:command:`metadata get`
+  Get metadata info.
+
+:command:`metadata put`
+  Put metadata info.
+
+:command:`metadata rm`
+  Remove metadata info.
+
+:command:`metadata list`
+  List metadata info.
+
+:command:`mdlog list`
+  List metadata log.
+
+:command:`mdlog trim`
+  Trim metadata log.
+
+:command:`mdlog status`
+  Read metadata log status.
+
+:command:`bilog list`
+  List bucket index log.
+
+:command:`bilog trim`
+  Trim bucket index log (use start-marker, end-marker).
+
+:command:`datalog list`
+  List data log.
+
+:command:`datalog trim`
+  Trim data log.
+
+:command:`datalog status`
+  Read data log status.
+
+:command:`orphans find`
+  Init and run search for leaked rados objects.
+  DEPRECATED. See the "rgw-orphan-list" tool.
+
+:command:`orphans finish`
+  Clean up search for leaked rados objects.
+  DEPRECATED. See the "rgw-orphan-list" tool.
+
+:command:`orphans list-jobs`
+  List the current job-ids for the orphans search.
+  DEPRECATED. See the "rgw-orphan-list" tool.
+
+:command:`role create`
+  create a new AWS role for use with STS.
+
+:command:`role rm`
+  Remove a role.
+
+:command:`role get`
+  Get a role.
+
+:command:`role list`
+  List the roles with specified path prefix.
+
+:command:`role modify`
+  Modify the assume role policy of an existing role.
+
+:command:`role-policy put`
+  Add/update permission policy to role.
+
+:command:`role-policy list`
+  List the policies attached to a role.
+
+:command:`role-policy get`
+  Get the specified inline policy document embedded with the given role.
+
+:command:`role-policy rm`
+  Remove the policy attached to a role
+
+:command:`reshard add`
+  Schedule a resharding of a bucket
+
+:command:`reshard list`
+  List all bucket resharding or scheduled to be resharded
+
+:command:`reshard process`
+  Process of scheduled reshard jobs
+
+:command:`reshard status`
+  Resharding status of a bucket
+
+:command:`reshard cancel`
+  Cancel resharding a bucket
+
+:command:`topic list`
+  List bucket notifications/pubsub topics                                                   
+
+:command:`topic get`
+  Get a bucket notifications/pubsub topic                                                   
+  
+:command:`topic rm`
+  Remove a bucket notifications/pubsub topic                                                
+
+:command:`subscription get`
+  Get a pubsub subscription definition
+
+:command:`subscription rm`
+  Remove a pubsub subscription
+
+:command:`subscription pull`
+  Show events in a pubsub subscription
+             
+:command:`subscription ack`
+  Ack (remove) an events in a pubsub subscription
+
+
+Options
+=======
+
+.. option:: -c ceph.conf, --conf=ceph.conf
+
+   Use ``ceph.conf`` configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` to determine monitor addresses during
+   startup.
+
+.. option:: -m monaddress[:port]
+
+   Connect to specified monitor (instead of looking through ceph.conf).
+
+.. option:: --tenant=<tenant>
+
+   Name of the tenant.
+
+.. option:: --uid=uid
+
+   The radosgw user ID.
+
+.. option:: --new-uid=uid
+
+   ID of the new user. Used with 'user rename' command.
+
+.. option:: --subuser=<name>
+
+	Name of the subuser.
+
+.. option:: --access-key=<key>
+
+        S3 access key.
+
+.. option:: --email=email
+
+   The e-mail address of the user.
+
+.. option:: --secret/--secret-key=<key>
+
+   The secret key.
+
+.. option:: --gen-access-key
+
+	Generate random access key (for S3).
+
+.. option:: --gen-secret
+
+	Generate random secret key.
+
+.. option:: --key-type=<type>
+
+	key type, options are: swift, s3.
+
+.. option:: --temp-url-key[-2]=<key>
+
+	Temporary url key.
+
+.. option:: --max-buckets
+
+	max number of buckets for a user (0 for no limit, negative value to disable bucket creation).
+	Default is 1000.
+
+.. option:: --access=<access>
+
+   Set the access permissions for the sub-user.
+   Available access permissions are read, write, readwrite and full.
+
+.. option:: --display-name=<name>
+
+   The display name of the user.
+
+.. option:: --admin
+
+   Set the admin flag on the user.
+
+.. option:: --system
+
+   Set the system flag on the user.
+
+.. option:: --bucket=[tenant-id/]bucket
+
+   Specify the bucket name.  If tenant-id is not specified, the tenant-id
+   of the user (--uid) is used.
+
+.. option:: --pool=<pool>
+
+   Specify the pool name.
+   Also used with `orphans find` as data pool to scan for leaked rados objects.
+
+.. option:: --object=object
+
+   Specify the object name.
+
+.. option:: --date=yyyy-mm-dd
+
+   The date in the format yyyy-mm-dd.
+
+.. option:: --start-date=yyyy-mm-dd
+
+   The start date in the format yyyy-mm-dd.
+
+.. option:: --end-date=yyyy-mm-dd
+
+   The end date in the format yyyy-mm-dd.
+
+.. option:: --bucket-id=<bucket-id>
+
+   Specify the bucket id.
+
+.. option:: --bucket-new-name=[tenant-id/]<bucket>
+
+   Optional for `bucket link`; use to rename a bucket.
+        While tenant-id/ can be specified, this is never
+        necessary for normal operation.
+
+.. option:: --shard-id=<shard-id>
+
+	Optional for mdlog list, bi list, data sync status. Required for ``mdlog trim``.
+
+.. option:: --max-entries=<entries>
+
+	Optional for listing operations to specify the max entires
+
+.. option:: --purge-data
+
+   When specified, user removal will also purge all the user data.
+
+.. option:: --purge-keys
+
+	When specified, subuser removal will also purge all the subuser keys.
+   
+.. option:: --purge-objects
+
+   When specified, the bucket removal will also purge all objects in it.
+
+.. option:: --metadata-key=<key>
+
+	Key to retrieve metadata from with ``metadata get``.
+
+.. option:: --remote=<remote>
+
+   Zone or zonegroup id of remote gateway.
+
+.. option:: --period=<id>
+
+   Period id.
+
+.. option:: --url=<url>
+
+   url for pushing/pulling period or realm.
+
+.. option:: --epoch=<number>
+
+   Period epoch.
+
+.. option:: --commit
+
+   Commit the period during 'period update'.
+
+.. option:: --staging
+
+   Get the staging period info.
+
+.. option:: --master
+
+   Set as master.
+
+.. option:: --master-zone=<id>
+
+   Master zone id.
+
+.. option:: --rgw-realm=<name>
+
+   The realm name.
+
+.. option:: --realm-id=<id>
+
+   The realm id.
+
+.. option:: --realm-new-name=<name>
+
+   New name of realm.
+
+.. option:: --rgw-zonegroup=<name>
+
+   The zonegroup name.
+
+.. option:: --zonegroup-id=<id>
+
+   The zonegroup id.
+
+.. option:: --zonegroup-new-name=<name>
+
+   The new name of the zonegroup.
+
+.. option:: --rgw-zone=<zone>
+
+	Zone in which radosgw is running.
+
+.. option:: --zone-id=<id>
+
+   The zone id.
+
+.. option:: --zone-new-name=<name>
+
+   The new name of the zone.
+
+.. option:: --source-zone
+
+   The source zone for data sync.
+
+.. option:: --default
+
+   Set the entity (realm, zonegroup, zone) as default.
+
+.. option:: --read-only
+
+   Set the zone as read-only when adding to the zonegroup.
+
+.. option:: --placement-id
+
+   Placement id for the zonegroup placement commands.
+
+.. option:: --tags=<list>
+
+   The list of tags for zonegroup placement add and modify commands.
+
+.. option:: --tags-add=<list>
+
+   The list of tags to add for zonegroup placement modify command.
+
+.. option:: --tags-rm=<list>
+
+   The list of tags to remove for zonegroup placement modify command.
+
+.. option:: --endpoints=<list>
+
+   The zone endpoints.
+
+.. option:: --index-pool=<pool>
+
+   The placement target index pool.
+
+.. option:: --data-pool=<pool>
+
+   The placement target data pool.
+
+.. option:: --data-extra-pool=<pool>
+
+   The placement target data extra (non-ec) pool.
+
+.. option:: --placement-index-type=<type>
+
+   The placement target index type (normal, indexless, or #id).
+
+.. option:: --tier-type=<type>
+
+   The zone tier type.
+
+.. option:: --tier-config=<k>=<v>[,...]
+
+   Set zone tier config keys, values.
+
+.. option:: --tier-config-rm=<k>[,...]
+
+   Unset zone tier config keys.
+
+.. option:: --sync-from-all[=false]
+
+   Set/reset whether zone syncs from all zonegroup peers.
+
+.. option:: --sync-from=[zone-name][,...]
+
+   Set the list of zones to sync from.
+
+.. option:: --sync-from-rm=[zone-name][,...]
+
+   Remove the zones from list of zones to sync from.
+
+.. option:: --bucket-index-max-shards
+
+   Override a zone's or zonegroup's default number of bucket index shards. This
+   option is accepted by the 'zone create', 'zone modify', 'zonegroup add',
+   and 'zonegroup modify' commands, and applies to buckets that are created
+   after the zone/zonegroup changes take effect.
+
+.. option:: --fix
+
+	Besides checking bucket index, will also fix it.
+
+.. option:: --check-objects
+
+	bucket check: Rebuilds bucket index according to actual objects state.
+
+.. option:: --format=<format>
+
+	Specify output format for certain operations. Supported formats: xml, json.
+
+.. option:: --sync-stats
+
+	Option for 'user stats' command. When specified, it will update user stats with
+	the current stats reported by user's buckets indexes.
+
+.. option:: --show-log-entries=<flag>
+
+	Enable/disable dump of log entries on log show.
+
+.. option:: --show-log-sum=<flag>
+
+	Enable/disable dump of log summation on log show.
+
+.. option:: --skip-zero-entries
+
+	Log show only dumps entries that don't have zero value in one of the numeric
+	field.
+
+.. option:: --infile
+
+	Specify a file to read in when setting data.
+
+.. option:: --categories=<list>
+
+	Comma separated list of categories, used in usage show.
+
+.. option:: --caps=<caps>
+
+	List of caps (e.g., "usage=read, write; user=read".
+
+.. option:: --compression=<compression-algorithm>
+
+    Placement target compression algorithm (lz4|snappy|zlib|zstd)
+
+.. option:: --yes-i-really-mean-it
+
+	Required for certain operations.
+
+.. option:: --min-rewrite-size
+
+    Specify the min object size for bucket rewrite (default 4M).
+
+.. option:: --max-rewrite-size
+
+    Specify the max object size for bucket rewrite (default ULLONG_MAX).
+
+.. option:: --min-rewrite-stripe-size
+
+    Specify the min stripe size for object rewrite (default 0). If the value
+    is set to 0, then the specified object will always be
+    rewritten for restriping.
+
+.. option:: --warnings-only
+
+   When specified with bucket limit check,
+   list only buckets nearing or over the current max objects per shard value.
+
+.. option:: --bypass-gc
+
+   When specified with bucket deletion,
+   triggers object deletions by not involving GC.
+
+.. option:: --inconsistent-index
+
+   When specified with bucket deletion and bypass-gc set to true,
+   ignores bucket index consistency.
+
+.. option:: --max-concurrent-ios
+
+        Maximum concurrent ios for bucket operations. Affects operations that
+        scan the bucket index, e.g., listing, deletion, and all scan/search
+        operations such as finding orphans or checking the bucket index.
+        Default is 32.
+
+Quota Options
+=============
+
+.. option:: --max-objects
+
+	Specify max objects (negative value to disable).
+
+.. option:: --max-size
+
+	Specify max size (in B/K/M/G/T, negative value to disable).
+
+.. option:: --quota-scope
+
+	The scope of quota (bucket, user).
+
+
+Orphans Search Options
+======================
+
+.. option:: --num-shards
+
+	Number of shards to use for keeping the temporary scan info
+
+.. option:: --orphan-stale-secs
+
+        Number of seconds to wait before declaring an object to be an orphan.
+        Default is 86400 (24 hours).
+
+.. option:: --job-id
+
+        Set the job id (for orphans find)
+
+
+Orphans list-jobs options
+=========================
+
+.. option:: --extra-info
+
+   Provide extra info in the job list.
+
+
+Role Options
+============
+
+.. option:: --role-name
+
+   The name of the role to create.
+
+.. option:: --path
+
+   The path to the role.
+
+.. option:: --assume-role-policy-doc
+
+   The trust relationship policy document that grants an entity permission to
+   assume the role.
+
+.. option:: --policy-name
+
+   The name of the policy document.
+
+.. option:: --policy-doc
+
+   The permission policy document.
+
+.. option:: --path-prefix
+
+   The path prefix for filtering the roles.
+
+
+Bucket Notifications/PubSub Options
+===================================
+.. option:: --topic                   
+
+   The bucket notifications/pubsub topic name.
+
+.. option:: --subscription
+
+   The pubsub subscription name.
+
+.. option:: --event-id
+
+   The event id in a pubsub subscription.
+
+
+Examples
+========
+
+Generate a new user::
+
+        $ radosgw-admin user create --display-name="johnny rotten" --uid=johnny
+        { "user_id": "johnny",
+          "rados_uid": 0,
+          "display_name": "johnny rotten",
+          "email": "",
+          "suspended": 0,
+          "subusers": [],
+          "keys": [
+                { "user": "johnny",
+                  "access_key": "TCICW53D9BQ2VGC46I44",
+                  "secret_key": "tfm9aHMI8X76L3UdgE+ZQaJag1vJQmE6HDb5Lbrz"}],
+          "swift_keys": []}
+
+Remove a user::
+
+        $ radosgw-admin user rm --uid=johnny
+
+Rename a user::
+
+        $ radosgw-admin user rename --uid=johny --new-uid=joe
+        
+Remove a user and all associated buckets with their contents::
+
+        $ radosgw-admin user rm --uid=johnny --purge-data
+
+Remove a bucket::
+
+	$ radosgw-admin bucket rm --bucket=foo
+
+Link bucket to specified user::
+	
+	$ radosgw-admin bucket link --bucket=foo --bucket_id=<bucket id> --uid=johnny
+
+Unlink bucket from specified user::
+
+        $ radosgw-admin bucket unlink --bucket=foo --uid=johnny
+
+Rename a bucket::
+
+        $ radosgw-admin bucket link --bucket=foo --bucket-new-name=bar --uid=johnny
+
+Move a bucket from the old global tenant space to a specified tenant::
+
+        $ radosgw-admin bucket link --bucket=/foo --uid=12345678$12345678'
+
+Link bucket to specified user and change object ACLs::
+
+        $ radosgw-admin bucket chown --bucket=/foo --uid=12345678$12345678'
+
+Show the logs of a bucket from April 1st, 2012::
+
+        $ radosgw-admin log show --bucket=foo --date=2012-04-01-01 --bucket-id=default.14193.1
+
+Show usage information for user from March 1st to (but not including) April 1st, 2012::
+
+        $ radosgw-admin usage show --uid=johnny \
+                        --start-date=2012-03-01 --end-date=2012-04-01
+
+Show only summary of usage information for all users::
+
+        $ radosgw-admin usage show --show-log-entries=false
+
+Trim usage information for user until March 1st, 2012::
+
+        $ radosgw-admin usage trim --uid=johnny --end-date=2012-04-01
+
+
+Availability
+============
+
+:program:`radosgw-admin` is part of Ceph, a massively scalable, open-source,
+distributed storage system.  Please refer to the Ceph documentation at
+http://ceph.com/docs for more information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8)
+:doc:`radosgw <radosgw>`\(8)
diff --git a/doc/man/8/radosgw.rst b/doc/man/8/radosgw.rst
new file mode 100644
index 000000000..84a1aea27
--- /dev/null
+++ b/doc/man/8/radosgw.rst
@@ -0,0 +1,253 @@
+:orphan:
+
+===============================
+ radosgw -- rados REST gateway
+===============================
+
+.. program:: radosgw
+
+Synopsis
+========
+
+| **radosgw**
+
+
+Description
+===========
+
+:program:`radosgw` is an HTTP REST gateway for the RADOS object store, a part
+of the Ceph distributed storage system. It is implemented as a FastCGI
+module using libfcgi, and can be used in conjunction with any FastCGI
+capable web server.
+
+
+Options
+=======
+
+.. option:: -c ceph.conf, --conf=ceph.conf
+
+   Use ``ceph.conf`` configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` to determine monitor addresses during startup.
+
+.. option:: -m monaddress[:port]
+
+   Connect to specified monitor (instead of looking through ``ceph.conf``).
+
+.. option:: -i ID, --id ID
+
+   Set the ID portion of name for radosgw
+
+.. option:: -n TYPE.ID, --name TYPE.ID
+
+   Set the rados user name for the gateway (eg. client.radosgw.gateway)
+
+.. option:: --cluster NAME
+
+   Set the cluster name (default: ceph)
+
+.. option:: -d
+
+   Run in foreground, log to stderr
+
+.. option:: -f
+
+   Run in foreground, log to usual location
+
+.. option:: --rgw-socket-path=path
+
+   Specify a unix domain socket path.
+
+.. option:: --rgw-region=region
+
+   The region where radosgw runs
+
+.. option:: --rgw-zone=zone
+
+   The zone where radosgw runs
+
+
+Configuration
+=============
+
+Earlier RADOS Gateway had to be configured with ``Apache`` and ``mod_fastcgi``.
+Now, ``mod_proxy_fcgi`` module is used instead of ``mod_fastcgi``.
+``mod_proxy_fcgi`` works differently than a traditional FastCGI module. This
+module requires the service of ``mod_proxy`` which provides support for the
+FastCGI protocol. So, to be able to handle FastCGI protocol, both ``mod_proxy``
+and ``mod_proxy_fcgi`` have to be present in the server. Unlike ``mod_fastcgi``,
+``mod_proxy_fcgi`` cannot start the application process. Some platforms have
+``fcgistarter`` for that purpose. However, external launching of application
+or process management may be available in the FastCGI application framework
+in use.
+
+``Apache`` can be configured in a way that enables ``mod_proxy_fcgi`` to be used
+with localhost tcp or through unix domain socket. ``mod_proxy_fcgi`` that doesn't
+support unix domain socket such as the ones in Apache 2.2 and earlier versions of
+Apache 2.4, needs to be configured for use with localhost tcp. Later versions of
+Apache like Apache 2.4.9 or later support unix domain socket and as such they
+allow for the configuration with unix domain socket instead of localhost tcp.
+
+The following steps show the configuration in Ceph's configuration file i.e,
+``/etc/ceph/ceph.conf`` and the gateway configuration file i.e,
+``/etc/httpd/conf.d/rgw.conf`` (RPM-based distros) or
+``/etc/apache2/conf-available/rgw.conf`` (Debian-based distros) with localhost
+tcp and through unix domain socket:
+
+#. For distros with Apache 2.2 and early versions of Apache 2.4 that use
+   localhost TCP and do not support Unix Domain Socket, append the following
+   contents to ``/etc/ceph/ceph.conf``::
+
+	[client.radosgw.gateway]
+	host = {hostname}
+	keyring = /etc/ceph/ceph.client.radosgw.keyring
+	rgw socket path = ""
+	log file = /var/log/ceph/client.radosgw.gateway.log
+	rgw frontends = fastcgi socket_port=9000 socket_host=0.0.0.0
+	rgw print continue = false
+
+#. Add the following content in the gateway configuration file:
+
+   For Debian/Ubuntu add in ``/etc/apache2/conf-available/rgw.conf``::
+
+		<VirtualHost *:80>
+		ServerName localhost
+		DocumentRoot /var/www/html
+
+		ErrorLog /var/log/apache2/rgw_error.log
+		CustomLog /var/log/apache2/rgw_access.log combined
+
+		# LogLevel debug
+
+		RewriteEngine On
+
+		RewriteRule .* - [E=HTTP_AUTHORIZATION:%{HTTP:Authorization},L]
+
+		SetEnv proxy-nokeepalive 1
+
+		ProxyPass / fcgi://localhost:9000/
+
+		</VirtualHost>
+
+   For CentOS/RHEL add in ``/etc/httpd/conf.d/rgw.conf``::
+
+		<VirtualHost *:80>
+		ServerName localhost
+		DocumentRoot /var/www/html
+
+		ErrorLog /var/log/httpd/rgw_error.log
+		CustomLog /var/log/httpd/rgw_access.log combined
+
+		# LogLevel debug
+
+		RewriteEngine On
+
+		RewriteRule .* - [E=HTTP_AUTHORIZATION:%{HTTP:Authorization},L]
+
+		SetEnv proxy-nokeepalive 1
+
+		ProxyPass / fcgi://localhost:9000/
+
+		</VirtualHost>
+
+#. For distros with Apache 2.4.9 or later that support Unix Domain Socket,
+   append the following configuration to ``/etc/ceph/ceph.conf``::
+
+	[client.radosgw.gateway]
+	host = {hostname}
+	keyring = /etc/ceph/ceph.client.radosgw.keyring
+	rgw socket path = /var/run/ceph/ceph.radosgw.gateway.fastcgi.sock
+	log file = /var/log/ceph/client.radosgw.gateway.log
+	rgw print continue = false
+
+#. Add the following content in the gateway configuration file:
+
+   For CentOS/RHEL add in ``/etc/httpd/conf.d/rgw.conf``::
+
+		<VirtualHost *:80>
+		ServerName localhost
+		DocumentRoot /var/www/html
+
+		ErrorLog /var/log/httpd/rgw_error.log
+		CustomLog /var/log/httpd/rgw_access.log combined
+
+		# LogLevel debug
+
+		RewriteEngine On
+
+		RewriteRule .* - [E=HTTP_AUTHORIZATION:%{HTTP:Authorization},L]
+
+		SetEnv proxy-nokeepalive 1
+
+		ProxyPass / unix:///var/run/ceph/ceph.radosgw.gateway.fastcgi.sock|fcgi://localhost:9000/
+
+		</VirtualHost>
+
+   Please note, ``Apache 2.4.7`` does not have Unix Domain Socket support in
+   it and as such it has to be configured with localhost tcp. The Unix Domain
+   Socket support is available in ``Apache 2.4.9`` and later versions.
+
+#. Generate a key for radosgw to use for authentication with the cluster. ::
+
+	ceph-authtool -C -n client.radosgw.gateway --gen-key /etc/ceph/keyring.radosgw.gateway
+	ceph-authtool -n client.radosgw.gateway --cap mon 'allow rw' --cap osd 'allow rwx' /etc/ceph/keyring.radosgw.gateway
+
+#. Add the key to the auth entries. ::
+
+	ceph auth add client.radosgw.gateway --in-file=keyring.radosgw.gateway
+
+#. Start Apache and radosgw.
+
+   Debian/Ubuntu::
+
+		sudo /etc/init.d/apache2 start
+		sudo /etc/init.d/radosgw start
+
+   CentOS/RHEL::
+
+		sudo apachectl start
+		sudo /etc/init.d/ceph-radosgw start
+
+Usage Logging
+=============
+
+:program:`radosgw` maintains an asynchronous usage log. It accumulates
+statistics about user operations and flushes it periodically. The
+logs can be accessed and managed through :program:`radosgw-admin`.
+
+The information that is being logged contains total data transfer,
+total operations, and total successful operations. The data is being
+accounted in an hourly resolution under the bucket owner, unless the
+operation was done on the service (e.g., when listing a bucket) in
+which case it is accounted under the operating user.
+
+Following is an example configuration::
+
+        [client.radosgw.gateway]
+            rgw enable usage log = true
+            rgw usage log tick interval = 30
+            rgw usage log flush threshold = 1024
+            rgw usage max shards = 32
+            rgw usage max user shards = 1
+
+
+The total number of shards determines how many total objects hold the
+usage log information. The per-user number of shards specify how many
+objects hold usage information for a single user. The tick interval
+configures the number of seconds between log flushes, and the flush
+threshold specify how many entries can be kept before resorting to
+synchronous flush.
+
+
+Availability
+============
+
+:program:`radosgw` is part of Ceph, a massively scalable, open-source, distributed
+storage system. Please refer to the Ceph documentation at http://ceph.com/docs for
+more information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8)
+:doc:`radosgw-admin <radosgw-admin>`\(8)
diff --git a/doc/man/8/rbd-fuse.rst b/doc/man/8/rbd-fuse.rst
new file mode 100644
index 000000000..5cdb1f735
--- /dev/null
+++ b/doc/man/8/rbd-fuse.rst
@@ -0,0 +1,61 @@
+:orphan:
+
+=======================================
+ rbd-fuse -- expose rbd images as files
+=======================================
+
+.. program:: rbd-fuse
+
+Synopsis
+========
+
+| **rbd-fuse** [ -p pool ] [-c conffile] *mountpoint* [ *fuse options* ]
+
+
+Note
+====
+
+**rbd-fuse** is not recommended for any production or high performance workloads.
+
+Description
+===========
+
+**rbd-fuse** is a FUSE ("Filesystem in USErspace") client for RADOS
+block device (rbd) images.  Given a pool containing rbd images,
+it will mount a userspace file system allowing access to those images
+as regular files at **mountpoint**.
+
+The file system can be unmounted with::
+
+        fusermount -u mountpoint
+
+or by sending ``SIGINT`` to the ``rbd-fuse`` process.
+
+
+Options
+=======
+
+Any options not recognized by rbd-fuse will be passed on to libfuse.
+
+.. option:: -c ceph.conf
+
+   Use *ceph.conf* configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` to determine monitor addresses during startup.
+
+.. option:: -p pool
+
+   Use *pool* as the pool to search for rbd images.  Default is ``rbd``.
+
+
+Availability
+============
+
+**rbd-fuse** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at http://ceph.com/docs for more information.
+
+
+See also
+========
+
+fusermount(8),
+:doc:`rbd <rbd>`\(8)
diff --git a/doc/man/8/rbd-ggate.rst b/doc/man/8/rbd-ggate.rst
new file mode 100644
index 000000000..67d0c81e8
--- /dev/null
+++ b/doc/man/8/rbd-ggate.rst
@@ -0,0 +1,79 @@
+:orphan:
+
+==================================================
+ rbd-ggate -- map rbd images via FreeBSD GEOM Gate
+==================================================
+
+.. program:: rbd-ggate
+
+Synopsis
+========
+
+| **rbd-ggate** [--read-only] [--exclusive] [--device *ggate device*] map *image-spec* | *snap-spec*
+| **rbd-ggate** unmap *ggate device*
+| **rbd-ggate** list
+
+Description
+===========
+
+**rbd-ggate** is a client for RADOS block device (rbd) images. It will
+map a rbd image to a ggate (FreeBSD GEOM Gate class) device, allowing
+access it as regular local block device.
+
+Commands
+========
+
+map
+---
+
+Spawn a process responsible for the creation of ggate device and
+forwarding I/O requests between the GEOM Gate kernel subsystem and
+RADOS.
+
+unmap
+-----
+
+Destroy ggate device and terminate the process responsible for it.
+
+list
+----
+
+List mapped ggate devices.
+
+Options
+=======
+
+.. option:: --device *ggate device*
+
+   Specify ggate device path.
+
+.. option:: --read-only
+
+   Map read-only.
+
+.. option:: --exclusive
+
+   Forbid writes by other clients.
+
+Image and snap specs
+====================
+
+| *image-spec* is [*pool-name*]/*image-name*
+| *snap-spec*  is [*pool-name*]/*image-name*\ @\ *snap-name*
+
+The default for *pool-name* is "rbd".  If an image name contains a slash
+character ('/'), *pool-name* is required.
+
+Availability
+============
+
+**rbd-ggate** is part of Ceph, a massively scalable, open-source,
+distributed storage system. Please refer to the Ceph documentation at
+http://ceph.com/docs for more information.
+
+
+See also
+========
+
+:doc:`rbd <rbd>`\(8)
+:doc:`ceph <ceph>`\(8)
diff --git a/doc/man/8/rbd-mirror.rst b/doc/man/8/rbd-mirror.rst
new file mode 100644
index 000000000..b35787fbf
--- /dev/null
+++ b/doc/man/8/rbd-mirror.rst
@@ -0,0 +1,75 @@
+:orphan:
+
+===================================================
+ rbd-mirror -- Ceph daemon for mirroring RBD images
+===================================================
+
+.. program:: rbd-mirror
+
+Synopsis
+========
+
+| **rbd-mirror**
+
+
+Description
+===========
+
+:program:`rbd-mirror` is a daemon for asynchronous mirroring of RADOS
+block device (rbd) images among Ceph clusters. It replays changes to
+images in remote clusters in a local cluster, for disaster recovery.
+
+It connects to remote clusters via the RADOS protocol, relying on
+default search paths to find ceph.conf files, monitor addresses and
+authentication information for them, i.e. ``/etc/ceph/$cluster.conf``,
+``/etc/ceph/$cluster.keyring``, and
+``/etc/ceph/$cluster.$name.keyring``, where ``$cluster`` is the
+human-friendly name of the cluster, and ``$name`` is the rados user to
+connect as, e.g. ``client.rbd-mirror``.
+
+
+Options
+=======
+
+.. option:: -c ceph.conf, --conf=ceph.conf
+
+   Use ``ceph.conf`` configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` to determine monitor addresses during startup.
+
+.. option:: -m monaddress[:port]
+
+   Connect to specified monitor (instead of looking through ``ceph.conf``).
+
+.. option:: -i ID, --id ID
+
+   Set the ID portion of name for rbd-mirror
+
+.. option:: -n TYPE.ID, --name TYPE.ID
+
+   Set the rados user name for the gateway (eg. client.rbd-mirror)
+
+.. option:: --cluster NAME
+
+   Set the cluster name (default: ceph)
+
+.. option:: -d
+
+   Run in foreground, log to stderr
+
+.. option:: -f
+
+   Run in foreground, log to usual location
+
+
+Availability
+============
+
+:program:`rbd-mirror` is part of Ceph, a massively scalable, open-source, distributed
+storage system. Please refer to the Ceph documentation at http://ceph.com/docs for
+more information.
+
+
+See also
+========
+
+:doc:`rbd <rbd>`\(8)
diff --git a/doc/man/8/rbd-nbd.rst b/doc/man/8/rbd-nbd.rst
new file mode 100644
index 000000000..8fcc5799d
--- /dev/null
+++ b/doc/man/8/rbd-nbd.rst
@@ -0,0 +1,89 @@
+:orphan:
+
+=========================================
+ rbd-nbd -- map rbd images to nbd device
+=========================================
+
+.. program:: rbd-nbd
+
+Synopsis
+========
+
+| **rbd-nbd** [-c conf] [--read-only] [--device *nbd device*] [--nbds_max *limit*] [--max_part *limit*] [--exclusive] [--encryption-format *format*] [--encryption-passphrase-file *passphrase-file*] [--io-timeout *seconds*] [--reattach-timeout *seconds*] map *image-spec* | *snap-spec*
+| **rbd-nbd** unmap *nbd device* | *image-spec* | *snap-spec*
+| **rbd-nbd** list-mapped
+| **rbd-nbd** attach --device *nbd device* *image-spec* | *snap-spec*
+| **rbd-nbd** detach *nbd device* | *image-spec* | *snap-spec*
+
+Description
+===========
+
+**rbd-nbd** is a client for RADOS block device (rbd) images like rbd kernel module.
+It will map a rbd image to a nbd (Network Block Device) device, allowing access it
+as regular local block device.
+
+Options
+=======
+
+.. option:: -c ceph.conf
+
+   Use *ceph.conf* configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` to determine monitor addresses during startup.
+
+.. option:: --read-only
+
+   Map read-only.
+
+.. option:: --nbds_max *limit*
+
+   Override the parameter nbds_max of NBD kernel module when modprobe, used to
+   limit the count of nbd device.
+
+.. option:: --max_part *limit*
+
+    Override for module param max_part.
+
+.. option:: --exclusive
+
+   Forbid writes by other clients.
+
+.. option:: --encryption-format
+
+   Image encryption format.
+   Possible values: *luks1*, *luks2*
+
+.. option:: --encryption-passphrase-file
+
+   Path of file containing a passphrase for unlocking image encryption.
+
+.. option:: --io-timeout *seconds*
+
+   Override device timeout. Linux kernel will default to a 30 second request timeout.
+   Allow the user to optionally specify an alternate timeout.
+
+.. option:: --reattach-timeout *seconds*
+
+   Specify timeout for the kernel to wait for a new rbd-nbd process is
+   attached after the old process is detached. The default is 30
+   second.
+
+Image and snap specs
+====================
+
+| *image-spec* is [*pool-name*]/*image-name*
+| *snap-spec*  is [*pool-name*]/*image-name*\ @\ *snap-name*
+
+The default for *pool-name* is "rbd".  If an image name contains a slash
+character ('/'), *pool-name* is required.
+
+Availability
+============
+
+**rbd-nbd** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at http://ceph.com/docs for more information.
+
+
+See also
+========
+
+:doc:`rbd <rbd>`\(8)
diff --git a/doc/man/8/rbd-replay-many.rst b/doc/man/8/rbd-replay-many.rst
new file mode 100644
index 000000000..5fb93498a
--- /dev/null
+++ b/doc/man/8/rbd-replay-many.rst
@@ -0,0 +1,73 @@
+:orphan:
+
+==================================================================================
+ rbd-replay-many -- replay a rados block device (RBD) workload on several clients
+==================================================================================
+
+.. program:: rbd-replay-many
+
+Synopsis
+========
+
+| **rbd-replay-many** [ *options* ] --original-image *name* *host1* [ *host2* [ ... ] ] -- *rbd_replay_args*
+
+
+Description
+===========
+
+**rbd-replay-many** is a utility for replaying a rados block device (RBD) workload on several clients.
+Although all clients use the same workload, they replay against separate images.
+This matches normal use of librbd, where each original client is a VM with its own image.
+
+Configuration and replay files are not automatically copied to clients.
+Replay images must already exist.
+
+
+Options
+=======
+
+.. option:: --original-image name
+
+   Specifies the name (and snap) of the originally traced image.
+   Necessary for correct name mapping.
+
+.. option:: --image-prefix prefix
+
+   Prefix of image names to replay against.
+   Specifying --image-prefix=foo results in clients replaying against foo-0, foo-1, etc.
+   Defaults to the original image name.
+
+.. option:: --exec program
+
+   Path to the rbd-replay executable.
+
+.. option:: --delay seconds
+
+   Delay between starting each client.  Defaults to 0.
+
+
+Examples
+========
+
+Typical usage::
+
+       rbd-replay-many host-0 host-1 --original-image=image -- -c ceph.conf replay.bin
+
+This results in the following commands being executed::
+
+       ssh host-0 'rbd-replay' --map-image 'image=image-0' -c ceph.conf replay.bin
+       ssh host-1 'rbd-replay' --map-image 'image=image-1' -c ceph.conf replay.bin
+
+
+Availability
+============
+
+**rbd-replay-many** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at http://ceph.com/docs for more information.
+
+
+See also
+========
+
+:doc:`rbd-replay <rbd-replay>`\(8),
+:doc:`rbd <rbd>`\(8)
diff --git a/doc/man/8/rbd-replay-prep.rst b/doc/man/8/rbd-replay-prep.rst
new file mode 100644
index 000000000..abb08decf
--- /dev/null
+++ b/doc/man/8/rbd-replay-prep.rst
@@ -0,0 +1,55 @@
+:orphan:
+
+====================================================================================
+ rbd-replay-prep -- prepare captured rados block device (RBD) workloads for replay
+====================================================================================
+
+.. program:: rbd-replay-prep
+
+Synopsis
+========
+
+| **rbd-replay-prep** [ --window *seconds* ] [ --anonymize ] *trace_dir* *replay_file*
+
+
+Description
+===========
+
+**rbd-replay-prep** processes raw rados block device (RBD) traces to prepare them for **rbd-replay**.
+
+
+Options
+=======
+
+.. option:: --window seconds
+
+   Requests further apart than 'seconds' seconds are assumed to be independent.
+
+.. option:: --anonymize
+
+   Anonymizes image and snap names.
+
+.. option:: --verbose
+
+   Print all processed events to console
+
+Examples
+========
+
+To prepare workload1-trace for replay::
+
+       rbd-replay-prep workload1-trace/ust/uid/1000/64-bit workload1
+
+
+Availability
+============
+
+**rbd-replay-prep** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at http://ceph.com/docs for more information.
+
+
+See also
+========
+
+:doc:`rbd-replay <rbd-replay>`\(8),
+:doc:`rbd <rbd>`\(8)
diff --git a/doc/man/8/rbd-replay.rst b/doc/man/8/rbd-replay.rst
new file mode 100644
index 000000000..74b8018f5
--- /dev/null
+++ b/doc/man/8/rbd-replay.rst
@@ -0,0 +1,78 @@
+:orphan:
+
+=========================================================
+ rbd-replay -- replay rados block device (RBD) workloads
+=========================================================
+
+.. program:: rbd-replay
+
+Synopsis
+========
+
+| **rbd-replay** [ *options* ] *replay_file*
+
+
+Description
+===========
+
+**rbd-replay** is a utility for replaying rados block device (RBD) workloads.
+
+
+Options
+=======
+
+.. option:: -c ceph.conf, --conf ceph.conf
+
+   Use ceph.conf configuration file instead of the default /etc/ceph/ceph.conf to
+   determine monitor addresses during startup.
+
+.. option:: -p pool, --pool pool
+
+   Interact with the given pool.  Defaults to 'rbd'.
+
+.. option:: --latency-multiplier
+
+   Multiplies inter-request latencies.  Default: 1.
+
+.. option:: --read-only
+
+   Only replay non-destructive requests.
+
+.. option:: --map-image rule
+
+   Add a rule to map image names in the trace to image names in the replay cluster.
+   A rule of image1@snap1=image2@snap2 would map snap1 of image1 to snap2 of image2.
+
+.. option:: --dump-perf-counters
+
+   **Experimental**
+   Dump performance counters to standard out before an image is closed.
+   Performance counters may be dumped multiple times if multiple images are closed,
+   or if the same image is opened and closed multiple times.
+   Performance counters and their meaning may change between versions.
+
+
+Examples
+========
+
+To replay workload1 as fast as possible::
+
+       rbd-replay --latency-multiplier=0 workload1
+
+To replay workload1 but use test_image instead of prod_image::
+
+       rbd-replay --map-image=prod_image=test_image workload1
+
+
+Availability
+============
+
+**rbd-replay** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at http://ceph.com/docs for more information.
+
+
+See also
+========
+
+:doc:`rbd-replay-prep <rbd-replay-prep>`\(8),
+:doc:`rbd <rbd>`\(8)
diff --git a/doc/man/8/rbd.rst b/doc/man/8/rbd.rst
new file mode 100644
index 000000000..57423348d
--- /dev/null
+++ b/doc/man/8/rbd.rst
@@ -0,0 +1,1036 @@
+:orphan:
+
+===============================================
+ rbd -- manage rados block device (RBD) images
+===============================================
+
+.. program:: rbd
+
+Synopsis
+========
+
+| **rbd** [ -c *ceph.conf* ] [ -m *monaddr* ] [--cluster *cluster-name*]
+  [ -p | --pool *pool* ] [ *command* ... ]
+
+
+Description
+===========
+
+**rbd** is a utility for manipulating rados block device (RBD) images,
+used by the Linux rbd driver and the rbd storage driver for QEMU/KVM.
+RBD images are simple block devices that are striped over objects and
+stored in a RADOS object store. The size of the objects the image is
+striped over must be a power of two.
+
+
+Options
+=======
+
+.. option:: -c ceph.conf, --conf ceph.conf
+
+   Use ceph.conf configuration file instead of the default /etc/ceph/ceph.conf to
+   determine monitor addresses during startup.
+
+.. option:: -m monaddress[:port]
+
+   Connect to specified monitor (instead of looking through ceph.conf).
+
+.. option:: --cluster cluster-name
+
+   Use different cluster name as compared to default cluster name *ceph*.
+
+.. option:: -p pool-name, --pool pool-name
+
+   Interact with the given pool. Required by most commands.
+
+.. option:: --namespace namespace-name
+
+   Use a pre-defined image namespace within a pool
+
+.. option:: --no-progress
+
+   Do not output progress information (goes to standard error by
+   default for some commands).
+
+
+Parameters
+==========
+
+.. option:: --image-format format-id
+
+   Specifies which object layout to use. The default is 2.
+
+   * format 1 - (deprecated) Use the original format for a new rbd image. This
+     format is understood by all versions of librbd and the kernel rbd module,
+     but does not support newer features like cloning.
+
+   * format 2 - Use the second rbd format, which is supported by librbd since
+     the Bobtail release and the kernel rbd module since kernel 3.10 (except
+     for "fancy" striping, which is supported since kernel 4.17). This adds
+     support for cloning and is more easily extensible to allow more
+     features in the future.
+
+.. option:: -s size-in-M/G/T, --size size-in-M/G/T
+
+   Specifies the size of the new rbd image or the new size of the existing rbd
+   image in M/G/T.  If no suffix is given, unit M is assumed.
+
+.. option:: --object-size size-in-B/K/M
+
+   Specifies the object size in B/K/M.  Object size will be rounded up the
+   nearest power of two; if no suffix is given, unit B is assumed.  The default
+   object size is 4M, smallest is 4K and maximum is 32M.
+
+.. option:: --stripe-unit size-in-B/K/M
+
+   Specifies the stripe unit size in B/K/M.  If no suffix is given, unit B is
+   assumed.  See striping section (below) for more details.
+
+.. option:: --stripe-count num
+
+   Specifies the number of objects to stripe over before looping back
+   to the first object.  See striping section (below) for more details.
+
+.. option:: --snap snap
+
+   Specifies the snapshot name for the specific operation.
+
+.. option:: --id username
+
+   Specifies the username (without the ``client.`` prefix) to use with the map command.
+
+.. option:: --keyring filename
+
+   Specifies a keyring file containing a secret for the specified user
+   to use with the map command.  If not specified, the default keyring
+   locations will be searched.
+
+.. option:: --keyfile filename
+
+   Specifies a file containing the secret key of ``--id user`` to use with the map command.
+   This option is overridden by ``--keyring`` if the latter is also specified.
+
+.. option:: --shared lock-tag
+
+   Option for `lock add` that allows multiple clients to lock the
+   same image if they use the same tag. The tag is an arbitrary
+   string. This is useful for situations where an image must
+   be open from more than one client at once, like during
+   live migration of a virtual machine, or for use underneath
+   a clustered file system.
+
+.. option:: --format format
+
+   Specifies output formatting (default: plain, json, xml)
+
+.. option:: --pretty-format
+
+   Make json or xml formatted output more human-readable.
+
+.. option:: -o krbd-options, --options krbd-options
+
+   Specifies which options to use when mapping or unmapping an image via the
+   rbd kernel driver.  krbd-options is a comma-separated list of options
+   (similar to mount(8) mount options).  See kernel rbd (krbd) options section
+   below for more details.
+
+.. option:: --read-only
+
+   Map the image read-only.  Equivalent to -o ro.
+
+.. option:: --image-feature feature-name
+
+   Specifies which RBD format 2 feature should be enabled when creating
+   an image. Multiple features can be enabled by repeating this option
+   multiple times. The following features are supported:
+
+   * layering: layering support
+   * striping: striping v2 support
+   * exclusive-lock: exclusive locking support
+   * object-map: object map support (requires exclusive-lock)
+   * fast-diff: fast diff calculations (requires object-map)
+   * deep-flatten: snapshot flatten support
+   * journaling: journaled IO support (requires exclusive-lock)
+   * data-pool: erasure coded pool support
+
+.. option:: --image-shared
+
+   Specifies that the image will be used concurrently by multiple clients.
+   This will disable features that are dependent upon exclusive ownership
+   of the image.
+
+.. option:: --whole-object
+
+   Specifies that the diff should be limited to the extents of a full object
+   instead of showing intra-object deltas. When the object map feature is
+   enabled on an image, limiting the diff to the object extents will
+   dramatically improve performance since the differences can be computed
+   by examining the in-memory object map instead of querying RADOS for each
+   object within the image.
+
+.. option:: --limit
+
+   Specifies the limit for the number of snapshots permitted.
+
+Commands
+========
+
+.. TODO rst "option" directive seems to require --foo style options, parsing breaks on subcommands.. the args show up as bold too
+
+:command:`bench` --io-type <read | write | readwrite | rw> [--io-size *size-in-B/K/M/G/T*] [--io-threads *num-ios-in-flight*] [--io-total *size-in-B/K/M/G/T*] [--io-pattern seq | rand] [--rw-mix-read *read proportion in readwrite*] *image-spec*
+  Generate a series of IOs to the image and measure the IO throughput and
+  latency.  If no suffix is given, unit B is assumed for both --io-size and
+  --io-total.  Defaults are: --io-size 4096, --io-threads 16, --io-total 1G,
+  --io-pattern seq, --rw-mix-read 50.
+
+:command:`children` *snap-spec*
+  List the clones of the image at the given snapshot. This checks
+  every pool, and outputs the resulting poolname/imagename.
+
+  This requires image format 2.
+
+:command:`clone` [--object-size *size-in-B/K/M*] [--stripe-unit *size-in-B/K/M* --stripe-count *num*] [--image-feature *feature-name*] [--image-shared] *parent-snap-spec* *child-image-spec*
+  Will create a clone (copy-on-write child) of the parent snapshot.
+  Object size will be identical to that of the parent image unless
+  specified. Size will be the same as the parent snapshot. The --stripe-unit
+  and --stripe-count arguments are optional, but must be used together.
+
+  The parent snapshot must be protected (see `rbd snap protect`).
+  This requires image format 2.
+
+:command:`config global get` *config-entity* *key*
+  Get a global-level configuration override.
+
+:command:`config global list` [--format plain | json | xml] [--pretty-format] *config-entity*
+  List global-level configuration overrides.
+
+:command:`config global set` *config-entity* *key* *value*
+  Set a global-level configuration override.
+
+:command:`config global remove` *config-entity* *key*
+  Remove a global-level configuration override.
+
+:command:`config image get` *image-spec* *key*
+  Get an image-level configuration override.
+
+:command:`config image list` [--format plain | json | xml] [--pretty-format] *image-spec*
+  List image-level configuration overrides.
+
+:command:`config image set` *image-spec* *key* *value*
+  Set an image-level configuration override.
+
+:command:`config image remove` *image-spec* *key*
+  Remove an image-level configuration override.
+
+:command:`config pool get` *pool-name* *key*
+  Get a pool-level configuration override.
+
+:command:`config pool list` [--format plain | json | xml] [--pretty-format] *pool-name*
+  List pool-level configuration overrides.
+
+:command:`config pool set` *pool-name* *key* *value*
+  Set a pool-level configuration override.
+
+:command:`config pool remove` *pool-name* *key*
+  Remove a pool-level configuration override.
+
+:command:`cp` (*src-image-spec* | *src-snap-spec*) *dest-image-spec*
+  Copy the content of a src-image into the newly created dest-image.
+  dest-image will have the same size, object size, and image format as src-image.
+
+:command:`create` (-s | --size *size-in-M/G/T*) [--image-format *format-id*] [--object-size *size-in-B/K/M*] [--stripe-unit *size-in-B/K/M* --stripe-count *num*] [--thick-provision] [--no-progress] [--image-feature *feature-name*]... [--image-shared] *image-spec*
+  Will create a new rbd image. You must also specify the size via --size.  The
+  --stripe-unit and --stripe-count arguments are optional, but must be used together.
+  If the --thick-provision is enabled, it will fully allocate storage for
+  the image at creation time. It will take a long time to do.
+  Note: thick provisioning requires zeroing the contents of the entire image.
+
+:command:`deep cp` (*src-image-spec* | *src-snap-spec*) *dest-image-spec*
+  Deep copy the content of a src-image into the newly created dest-image.
+  Dest-image will have the same size, object size, image format, and snapshots as src-image.
+
+:command:`device list` [-t | --device-type *device-type*] [--format plain | json | xml] --pretty-format
+  Show the rbd images that are mapped via the rbd kernel module
+  (default) or other supported device.
+
+:command:`device map` [-t | --device-type *device-type*] [--cookie *device-cookie*] [--show-cookie] [--read-only] [--exclusive] [-o | --options *device-options*] *image-spec* | *snap-spec*
+  Map the specified image to a block device via the rbd kernel module
+  (default) or other supported device (*nbd* on Linux or *ggate* on
+  FreeBSD).
+
+  The --options argument is a comma separated list of device type
+  specific options (opt1,opt2=val,...).
+
+:command:`device unmap` [-t | --device-type *device-type*] [-o | --options *device-options*] *image-spec* | *snap-spec* | *device-path*
+  Unmap the block device that was mapped via the rbd kernel module
+  (default) or other supported device.
+
+  The --options argument is a comma separated list of device type
+  specific options (opt1,opt2=val,...).
+
+:command:`device attach` [-t | --device-type *device-type*] --device *device-path* [--cookie *device-cookie*] [--show-cookie] [--read-only] [--exclusive] [--force] [-o | --options *device-options*] *image-spec* | *snap-spec*
+  Attach the specified image to the specified block device (currently only
+  `nbd` on Linux). This operation is unsafe and should not be normally used.
+  In particular, specifying the wrong image or the wrong block device may
+  lead to data corruption as no validation is performed by `nbd` kernel driver.
+
+  The --options argument is a comma separated list of device type
+  specific options (opt1,opt2=val,...).
+
+:command:`device detach` [-t | --device-type *device-type*] [-o | --options *device-options*] *image-spec* | *snap-spec* | *device-path*
+  Detach the block device that was mapped or attached (currently only `nbd`
+  on Linux). This operation is unsafe and should not be normally used.
+
+  The --options argument is a comma separated list of device type
+  specific options (opt1,opt2=val,...).
+
+:command:`diff` [--from-snap *snap-name*] [--whole-object] *image-spec* | *snap-spec*
+  Dump a list of byte extents in the image that have changed since the specified start
+  snapshot, or since the image was created.  Each output line includes the starting offset
+  (in bytes), the length of the region (in bytes), and either 'zero' or 'data' to indicate
+  whether the region is known to be zeros or may contain other data.
+
+:command:`du` [-p | --pool *pool-name*] [*image-spec* | *snap-spec*] [--merge-snapshots]
+  Will calculate the provisioned and actual disk usage of all images and
+  associated snapshots within the specified pool.  It can also be used against
+  individual images and snapshots.
+
+  If the RBD fast-diff feature is not enabled on images, this operation will
+  require querying the OSDs for every potential object within the image.
+
+  The --merge-snapshots will merge snapshots used space into their parent images.
+
+:command:`encryption format` *image-spec* *format* *passphrase-file* [--cipher-alg *alg*]
+  Formats image to an encrypted format.
+  All data previously written to the image will become unreadable.
+  A cloned image cannot be formatted, although encrypted images can be cloned.
+  Supported formats: *luks1*, *luks2*.
+  Supported cipher algorithms: *aes-128*, *aes-256* (default).
+
+:command:`export` [--export-format *format (1 or 2)*] (*image-spec* | *snap-spec*) [*dest-path*]
+  Export image to dest path (use - for stdout).
+  The --export-format accepts '1' or '2' currently. Format 2 allow us to export not only the content
+  of image, but also the snapshots and other properties, such as image_order, features.
+
+:command:`export-diff` [--from-snap *snap-name*] [--whole-object] (*image-spec* | *snap-spec*) *dest-path*
+  Export an incremental diff for an image to dest path (use - for stdout).  If
+  an initial snapshot is specified, only changes since that snapshot are included; otherwise,
+  any regions of the image that contain data are included.  The end snapshot is specified
+  using the standard --snap option or @snap syntax (see below).  The image diff format includes
+  metadata about image size changes, and the start and end snapshots.  It efficiently represents
+  discarded or 'zero' regions of the image.
+
+:command:`feature disable` *image-spec* *feature-name*...
+  Disable the specified feature on the specified image. Multiple features can
+  be specified.
+
+:command:`feature enable` *image-spec* *feature-name*...
+  Enable the specified feature on the specified image. Multiple features can
+  be specified.
+
+:command:`flatten` *image-spec*
+  If image is a clone, copy all shared blocks from the parent snapshot and
+  make the child independent of the parent, severing the link between
+  parent snap and child.  The parent snapshot can be unprotected and
+  deleted if it has no further dependent clones.
+
+  This requires image format 2.
+
+:command:`group create` *group-spec*
+  Create a group.
+
+:command:`group image add` *group-spec* *image-spec*
+  Add an image to a group.
+
+:command:`group image list` *group-spec*
+  List images in a group.
+
+:command:`group image remove` *group-spec* *image-spec*
+  Remove an image from a group.
+
+:command:`group ls` [-p | --pool *pool-name*]
+  List rbd groups.
+
+:command:`group rename` *src-group-spec* *dest-group-spec*
+  Rename a group.  Note: rename across pools is not supported.
+
+:command:`group rm` *group-spec*
+  Delete a group.
+
+:command:`group snap create` *group-snap-spec*
+  Make a snapshot of a group.
+
+:command:`group snap list` *group-spec*
+  List snapshots of a group.
+
+:command:`group snap rm` *group-snap-spec*
+  Remove a snapshot from a group.
+
+:command:`group snap rename` *group-snap-spec* *snap-name*
+  Rename group's snapshot.
+
+:command:`group snap rollback` *group-snap-spec*
+  Rollback group to snapshot.
+
+:command:`image-meta get` *image-spec* *key*
+  Get metadata value with the key.
+
+:command:`image-meta list` *image-spec*
+  Show metadata held on the image. The first column is the key
+  and the second column is the value.
+
+:command:`image-meta remove` *image-spec* *key*
+  Remove metadata key with the value.
+
+:command:`image-meta set` *image-spec* *key* *value*
+  Set metadata key with the value. They will displayed in `image-meta list`.
+
+:command:`import` [--export-format *format (1 or 2)*] [--image-format *format-id*] [--object-size *size-in-B/K/M*] [--stripe-unit *size-in-B/K/M* --stripe-count *num*] [--image-feature *feature-name*]... [--image-shared] *src-path* [*image-spec*]
+  Create a new image and imports its data from path (use - for
+  stdin).  The import operation will try to create sparse rbd images 
+  if possible.  For import from stdin, the sparsification unit is
+  the data block size of the destination image (object size).
+
+  The --stripe-unit and --stripe-count arguments are optional, but must be
+  used together.
+
+  The --export-format accepts '1' or '2' currently. Format 2 allow us to import not only the content
+  of image, but also the snapshots and other properties, such as image_order, features.
+
+:command:`import-diff` *src-path* *image-spec*
+  Import an incremental diff of an image and applies it to the current image.  If the diff
+  was generated relative to a start snapshot, we verify that snapshot already exists before
+  continuing.  If there was an end snapshot we verify it does not already exist before
+  applying the changes, and create the snapshot when we are done.
+  
+:command:`info` *image-spec* | *snap-spec*
+  Will dump information (such as size and object size) about a specific rbd image.
+  If image is a clone, information about its parent is also displayed.
+  If a snapshot is specified, whether it is protected is shown as well.
+
+:command:`journal client disconnect` *journal-spec*
+  Flag image journal client as disconnected.
+
+:command:`journal export` [--verbose] [--no-error] *src-journal-spec* *path-name*
+  Export image journal to path (use - for stdout). It can be make a backup
+  of the image journal especially before attempting dangerous operations.
+
+  Note that this command may not always work if the journal is badly corrupted.
+
+:command:`journal import` [--verbose] [--no-error] *path-name* *dest-journal-spec*
+  Import image journal from path (use - for stdin).
+
+:command:`journal info` *journal-spec*
+  Show information about image journal.
+
+:command:`journal inspect` [--verbose] *journal-spec*
+  Inspect and report image journal for structural errors.
+
+:command:`journal reset` *journal-spec*
+  Reset image journal.
+
+:command:`journal status` *journal-spec*
+  Show status of image journal.
+
+:command:`lock add` [--shared *lock-tag*] *image-spec* *lock-id*
+  Lock an image. The lock-id is an arbitrary name for the user's
+  convenience. By default, this is an exclusive lock, meaning it
+  will fail if the image is already locked. The --shared option
+  changes this behavior. Note that locking does not affect
+  any operation other than adding a lock. It does not
+  protect an image from being deleted.
+
+:command:`lock ls` *image-spec*
+  Show locks held on the image. The first column is the locker
+  to use with the `lock remove` command.
+
+:command:`lock rm` *image-spec* *lock-id* *locker*
+  Release a lock on an image. The lock id and locker are
+  as output by lock ls.
+
+:command:`ls` [-l | --long] [*pool-name*]
+  Will list all rbd images listed in the rbd_directory object.  With
+  -l, also show snapshots, and use longer-format output including
+  size, parent (if clone), format, etc.
+
+:command:`merge-diff` *first-diff-path* *second-diff-path* *merged-diff-path*
+  Merge two continuous incremental diffs of an image into one single diff. The
+  first diff's end snapshot must be equal with the second diff's start snapshot.
+  The first diff could be - for stdin, and merged diff could be - for stdout, which
+  enables multiple diff files to be merged using something like
+  'rbd merge-diff first second - | rbd merge-diff - third result'. Note this command
+  currently only support the source incremental diff with stripe_count == 1
+
+:command:`migration abort` *image-spec*
+  Cancel image migration. This step may be run after successful or
+  failed migration prepare or migration execute steps and returns the
+  image to its initial (before migration) state. All modifications to
+  the destination image are lost.
+
+:command:`migration commit` *image-spec*
+  Commit image migration. This step is run after a successful migration
+  prepare and migration execute steps and removes the source image data.
+
+:command:`migration execute` *image-spec*
+  Execute image migration. This step is run after a successful migration
+  prepare step and copies image data to the destination.
+
+:command:`migration prepare` [--order *order*] [--object-size *object-size*] [--image-feature *image-feature*] [--image-shared] [--stripe-unit *stripe-unit*] [--stripe-count *stripe-count*] [--data-pool *data-pool*] [--import-only] [--source-spec *json*] [--source-spec-path *path*] *src-image-spec* [*dest-image-spec*]
+  Prepare image migration. This is the first step when migrating an
+  image, i.e. changing the image location, format or other
+  parameters that can't be changed dynamically. The destination can
+  match the source, and in this case *dest-image-spec* can be omitted.
+  After this step the source image is set as a parent of the
+  destination image, and the image is accessible in copy-on-write mode
+  by its destination spec.
+
+  An image can also be migrated from a read-only import source by adding the
+  *--import-only* optional and providing a JSON-encoded *--source-spec* or a
+  path to a JSON-encoded source-spec file using the *--source-spec-path*
+  optionals.
+
+:command:`mirror image demote` *image-spec*
+  Demote a primary image to non-primary for RBD mirroring.
+
+:command:`mirror image disable` [--force] *image-spec*
+  Disable RBD mirroring for an image. If the mirroring is
+  configured in ``image`` mode for the image's pool, then it
+  can be explicitly disabled mirroring for each image within
+  the pool.
+
+:command:`mirror image enable` *image-spec* *mode*
+  Enable RBD mirroring for an image. If the mirroring is
+  configured in ``image`` mode for the image's pool, then it
+  can be explicitly enabled mirroring for each image within
+  the pool.
+
+  The mirror image mode can either be ``journal`` (default) or
+  ``snapshot``. The ``journal`` mode requires the RBD journaling
+  feature.
+
+:command:`mirror image promote` [--force] *image-spec*
+  Promote a non-primary image to primary for RBD mirroring.
+
+:command:`mirror image resync` *image-spec*
+  Force resync to primary image for RBD mirroring.
+
+:command:`mirror image status` *image-spec*
+  Show RBD mirroring status for an image.
+
+:command:`mirror pool demote` [*pool-name*]
+  Demote all primary images within a pool to non-primary.
+  Every mirroring enabled image will demoted in the pool.
+
+:command:`mirror pool disable` [*pool-name*]
+  Disable RBD mirroring by default within a pool. When mirroring
+  is disabled on a pool in this way, mirroring will also be
+  disabled on any images (within the pool) for which mirroring
+  was enabled explicitly.
+
+:command:`mirror pool enable` [*pool-name*] *mode*
+  Enable RBD mirroring by default within a pool.
+  The mirroring mode can either be ``pool`` or ``image``.
+  If configured in ``pool`` mode, all images in the pool
+  with the journaling feature enabled are mirrored.
+  If configured in ``image`` mode, mirroring needs to be
+  explicitly enabled (by ``mirror image enable`` command)
+  on each image.
+
+:command:`mirror pool info` [*pool-name*]
+  Show information about the pool mirroring configuration.
+  It includes mirroring mode, peer UUID, remote cluster name,
+  and remote client name.
+
+:command:`mirror pool peer add` [*pool-name*] *remote-cluster-spec*
+  Add a mirroring peer to a pool.
+  *remote-cluster-spec* is [*remote client name*\ @\ ]\ *remote cluster name*.
+
+  The default for *remote client name* is "client.admin".
+
+  This requires mirroring mode is enabled.
+
+:command:`mirror pool peer remove` [*pool-name*] *uuid*
+  Remove a mirroring peer from a pool. The peer uuid is available
+  from ``mirror pool info`` command.
+
+:command:`mirror pool peer set` [*pool-name*] *uuid* *key* *value*
+  Update mirroring peer settings.
+  The key can be either ``client`` or ``cluster``, and the value
+  is corresponding to remote client name or remote cluster name.
+
+:command:`mirror pool promote` [--force] [*pool-name*]
+  Promote all non-primary images within a pool to primary.
+  Every mirroring enabled image will promoted in the pool.
+
+:command:`mirror pool status` [--verbose] [*pool-name*]
+  Show status for all mirrored images in the pool.
+  With --verbose, also show additionally output status
+  details for every mirroring image in the pool.
+
+:command:`mirror snapshot schedule add` [-p | --pool *pool*] [--namespace *namespace*] [--image *image*] *interval* [*start-time*]
+  Add mirror snapshot schedule.
+
+:command:`mirror snapshot schedule list` [-R | --recursive] [--format *format*] [--pretty-format] [-p | --pool *pool*] [--namespace *namespace*] [--image *image*]
+  List mirror snapshot schedule.
+
+:command:`mirror snapshot schedule remove` [-p | --pool *pool*] [--namespace *namespace*] [--image *image*] *interval* [*start-time*]
+  Remove mirror snapshot schedule.
+
+:command:`mirror snapshot schedule status` [-p | --pool *pool*] [--format *format*] [--pretty-format] [--namespace *namespace*] [--image *image*]
+  Show mirror snapshot schedule status.
+
+:command:`mv` *src-image-spec* *dest-image-spec*
+  Rename an image.  Note: rename across pools is not supported.
+
+:command:`namespace create` *pool-name*/*namespace-name*
+  Create a new image namespace within the pool.
+
+:command:`namespace list` *pool-name*
+  List image namespaces defined within the pool.
+
+:command:`namespace remove` *pool-name*/*namespace-name*
+  Remove an empty image namespace from the pool.
+
+:command:`object-map check` *image-spec* | *snap-spec*
+  Verify the object map is correct.
+
+:command:`object-map rebuild` *image-spec* | *snap-spec*
+  Rebuild an invalid object map for the specified image. An image snapshot can be
+  specified to rebuild an invalid object map for a snapshot.
+
+:command:`pool init` [*pool-name*] [--force]
+  Initialize pool for use by RBD. Newly created pools must initialized
+  prior to use.
+
+:command:`resize` (-s | --size *size-in-M/G/T*) [--allow-shrink] *image-spec*
+  Resize rbd image. The size parameter also needs to be specified.
+  The --allow-shrink option lets the size be reduced.
+  
+:command:`rm` *image-spec*
+  Delete an rbd image (including all data blocks). If the image has
+  snapshots, this fails and nothing is deleted.
+
+:command:`snap create` *snap-spec*
+  Create a new snapshot. Requires the snapshot name parameter specified.
+
+:command:`snap limit clear` *image-spec*
+  Remove any previously set limit on the number of snapshots allowed on
+  an image.
+
+:command:`snap limit set` [--limit] *limit* *image-spec*
+  Set a limit for the number of snapshots allowed on an image.
+
+:command:`snap ls` *image-spec*
+  Dump the list of snapshots inside a specific image.
+
+:command:`snap protect` *snap-spec*
+  Protect a snapshot from deletion, so that clones can be made of it
+  (see `rbd clone`).  Snapshots must be protected before clones are made;
+  protection implies that there exist dependent cloned children that
+  refer to this snapshot.  `rbd clone` will fail on a nonprotected
+  snapshot.
+
+  This requires image format 2.
+
+:command:`snap purge` *image-spec*
+  Remove all unprotected snapshots from an image.
+
+:command:`snap rename` *src-snap-spec* *dest-snap-spec*
+  Rename a snapshot. Note: rename across pools and images is not supported.
+
+:command:`snap rm` [--force] *snap-spec*
+  Remove the specified snapshot.
+
+:command:`snap rollback` *snap-spec*
+  Rollback image content to snapshot. This will iterate through the entire blocks
+  array and update the data head content to the snapshotted version.
+
+:command:`snap unprotect` *snap-spec*
+  Unprotect a snapshot from deletion (undo `snap protect`).  If cloned
+  children remain, `snap unprotect` fails.  (Note that clones may exist
+  in different pools than the parent snapshot.)
+
+  This requires image format 2.
+
+:command:`sparsify` [--sparse-size *sparse-size*] *image-spec*
+  Reclaim space for zeroed image extents. The default sparse size is
+  4096 bytes and can be changed via --sparse-size option with the
+  following restrictions: it should be power of two, not less than
+  4096, and not larger than image object size.
+
+:command:`status` *image-spec*
+  Show the status of the image, including which clients have it open.
+
+:command:`trash ls` [*pool-name*]
+  List all entries from trash.
+
+:command:`trash mv` *image-spec*
+  Move an image to the trash. Images, even ones actively in-use by 
+  clones, can be moved to the trash and deleted at a later time.
+
+:command:`trash purge` [*pool-name*]
+  Remove all expired images from trash.
+
+:command:`trash restore` *image-id*  
+  Restore an image from trash.
+
+:command:`trash rm` *image-id* 
+  Delete an image from trash. If image deferment time has not expired
+  you can not removed it unless use force. But an actively in-use by clones 
+  or has snapshots can not be removed.
+
+:command:`trash purge schedule add` [-p | --pool *pool*] [--namespace *namespace*] *interval* [*start-time*]
+  Add trash purge schedule.
+
+:command:`trash purge schedule list` [-R | --recursive] [--format *format*] [--pretty-format] [-p | --pool *pool*] [--namespace *namespace*]
+  List trash purge schedule.
+
+:command:`trash purge schedule remove` [-p | --pool *pool*] [--namespace *namespace*] *interval* [*start-time*]
+  Remove trash purge schedule.
+
+:command:`trash purge schedule status` [-p | --pool *pool*] [--format *format*] [--pretty-format] [--namespace *namespace*]
+  Show trash purge schedule status.
+
+:command:`watch` *image-spec*
+  Watch events on image.
+
+Image, snap, group and journal specs
+====================================
+
+| *image-spec*      is [*pool-name*/[*namespace-name*/]]\ *image-name*
+| *snap-spec*       is [*pool-name*/[*namespace-name*/]]\ *image-name*\ @\ *snap-name*
+| *group-spec*      is [*pool-name*/[*namespace-name*/]]\ *group-name*
+| *group-snap-spec* is [*pool-name*/[*namespace-name*/]]\ *group-name*\ @\ *snap-name*
+| *journal-spec*    is [*pool-name*/[*namespace-name*/]]\ *journal-name*
+
+The default for *pool-name* is "rbd" and *namespace-name* is "". If an image
+name contains a slash character ('/'), *pool-name* is required.
+
+The *journal-name* is *image-id*.
+
+You may specify each name individually, using --pool, --namespace, --image, and
+--snap options, but this is discouraged in favor of the above spec syntax.
+
+Striping
+========
+
+RBD images are striped over many objects, which are then stored by the
+Ceph distributed object store (RADOS).  As a result, read and write
+requests for the image are distributed across many nodes in the
+cluster, generally preventing any single node from becoming a
+bottleneck when individual images get large or busy.
+
+The striping is controlled by three parameters:
+
+.. option:: object-size
+
+  The size of objects we stripe over is a power of two. It will be rounded up the nearest power of two.
+  The default object size is 4 MB, smallest is 4K and maximum is 32M.
+
+.. option:: stripe_unit
+
+  Each [*stripe_unit*] contiguous bytes are stored adjacently in the same object, before we move on
+  to the next object.
+
+.. option:: stripe_count
+
+  After we write [*stripe_unit*] bytes to [*stripe_count*] objects, we loop back to the initial object
+  and write another stripe, until the object reaches its maximum size.  At that point,
+  we move on to the next [*stripe_count*] objects.
+
+By default, [*stripe_unit*] is the same as the object size and [*stripe_count*] is 1.  Specifying a different
+[*stripe_unit*] and/or [*stripe_count*] is often referred to as using "fancy" striping and requires format 2.
+
+
+Kernel rbd (krbd) options
+=========================
+
+Most of these options are useful mainly for debugging and benchmarking.  The
+default values are set in the kernel and may therefore depend on the version of
+the running kernel.
+
+Per client instance `rbd device map` options:
+
+* fsid=aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee - FSID that should be assumed by
+  the client.
+
+* ip=a.b.c.d[:p] - IP and, optionally, port the client should use.
+
+* share - Enable sharing of client instances with other mappings (default).
+
+* noshare - Disable sharing of client instances with other mappings.
+
+* crc - Enable CRC32C checksumming for msgr1 on-the-wire protocol (default).
+  For msgr2.1 protocol this option is ignored: full checksumming is always on
+  in 'crc' mode and always off in 'secure' mode.
+
+* nocrc - Disable CRC32C checksumming for msgr1 on-the-wire protocol.  Note
+  that only payload checksumming is disabled, header checksumming is always on.
+  For msgr2.1 protocol this option is ignored.
+
+* cephx_require_signatures - Require msgr1 message signing feature (since 3.19,
+  default).  This option is deprecated and will be removed in the future as the
+  feature has been supported since the Bobtail release.
+
+* nocephx_require_signatures - Don't require msgr1 message signing feature
+  (since 3.19).  This option is deprecated and will be removed in the future.
+
+* tcp_nodelay - Disable Nagle's algorithm on client sockets (since 4.0,
+  default).
+
+* notcp_nodelay - Enable Nagle's algorithm on client sockets (since 4.0).
+
+* cephx_sign_messages - Enable message signing for msgr1 on-the-wire protocol
+  (since 4.4, default).  For msgr2.1 protocol this option is ignored: message
+  signing is built into 'secure' mode and not offered in 'crc' mode.
+
+* nocephx_sign_messages - Disable message signing for msgr1 on-the-wire protocol
+  (since 4.4).  For msgr2.1 protocol this option is ignored.
+
+* mount_timeout=x - A timeout on various steps in `rbd device map` and
+  `rbd device unmap` sequences (default is 60 seconds).  In particular,
+  since 4.2 this can be used to ensure that `rbd device unmap` eventually
+  times out when there is no network connection to a cluster.
+
+* osdkeepalive=x - OSD keepalive timeout (default is 5 seconds).
+
+* osd_idle_ttl=x - OSD idle TTL (default is 60 seconds).
+
+Per mapping (block device) `rbd device map` options:
+
+* rw - Map the image read-write (default).  Overridden by --read-only.
+
+* ro - Map the image read-only.  Equivalent to --read-only.
+
+* queue_depth=x - queue depth (since 4.2, default is 128 requests).
+
+* lock_on_read - Acquire exclusive lock on reads, in addition to writes and
+  discards (since 4.9).
+
+* exclusive - Disable automatic exclusive lock transitions (since 4.12).
+  Equivalent to --exclusive.
+
+* lock_timeout=x - A timeout on waiting for the acquisition of exclusive lock
+  (since 4.17, default is 0 seconds, meaning no timeout).
+
+* notrim - Turn off discard and write zeroes offload support to avoid
+  deprovisioning a fully provisioned image (since 4.17). When enabled, discard
+  requests will fail with -EOPNOTSUPP, write zeroes requests will fall back to
+  manually zeroing.
+
+* abort_on_full - Fail write requests with -ENOSPC when the cluster is full or
+  the data pool reaches its quota (since 5.0).  The default behaviour is to
+  block until the full condition is cleared.
+
+* alloc_size - Minimum allocation unit of the underlying OSD object store
+  backend (since 5.1, default is 64K bytes).  This is used to round off and
+  drop discards that are too small.  For bluestore, the recommended setting is
+  bluestore_min_alloc_size (currently set to 4K for all types of drives,
+  previously used to be set to 64K for hard disk drives and 16K for
+  solid-state drives).  For filestore with filestore_punch_hole = false, the
+  recommended setting is image object size (typically 4M).
+
+* crush_location=x - Specify the location of the client in terms of CRUSH
+  hierarchy (since 5.8).  This is a set of key-value pairs separated from
+  each other by '|', with keys separated from values by ':'.  Note that '|'
+  may need to be quoted or escaped to avoid it being interpreted as a pipe
+  by the shell.  The key is the bucket type name (e.g. rack, datacenter or
+  region with default bucket types) and the value is the bucket name.  For
+  example, to indicate that the client is local to rack "myrack", data center
+  "mydc" and region "myregion"::
+
+    crush_location=rack:myrack|datacenter:mydc|region:myregion
+
+  Each key-value pair stands on its own: "myrack" doesn't need to reside in
+  "mydc", which in turn doesn't need to reside in "myregion".  The location
+  is not a path to the root of the hierarchy but rather a set of nodes that
+  are matched independently, owning to the fact that bucket names are unique
+  within a CRUSH map.  "Multipath" locations are supported, so it is possible
+  to indicate locality for multiple parallel hierarchies::
+
+    crush_location=rack:myrack1|rack:myrack2|datacenter:mydc
+
+* read_from_replica=no - Disable replica reads, always pick the primary OSD
+  (since 5.8, default).
+
+* read_from_replica=balance - When issued a read on a replicated pool, pick
+  a random OSD for serving it (since 5.8).
+
+  This mode is safe for general use only since Octopus (i.e. after "ceph osd
+  require-osd-release octopus").  Otherwise it should be limited to read-only
+  workloads such as images mapped read-only everywhere or snapshots.
+
+* read_from_replica=localize - When issued a read on a replicated pool, pick
+  the most local OSD for serving it (since 5.8).  The locality metric is
+  calculated against the location of the client given with crush_location;
+  a match with the lowest-valued bucket type wins.  For example, with default
+  bucket types, an OSD in a matching rack is closer than an OSD in a matching
+  data center, which in turn is closer than an OSD in a matching region.
+
+  This mode is safe for general use only since Octopus (i.e. after "ceph osd
+  require-osd-release octopus").  Otherwise it should be limited to read-only
+  workloads such as images mapped read-only everywhere or snapshots.
+
+* compression_hint=none - Don't set compression hints (since 5.8, default).
+
+* compression_hint=compressible - Hint to the underlying OSD object store
+  backend that the data is compressible, enabling compression in passive mode
+  (since 5.8).
+
+* compression_hint=incompressible - Hint to the underlying OSD object store
+  backend that the data is incompressible, disabling compression in aggressive
+  mode (since 5.8).
+
+* ms_mode=legacy - Use msgr1 on-the-wire protocol (since 5.11, default).
+
+* ms_mode=crc - Use msgr2.1 on-the-wire protocol, select 'crc' mode, also
+  referred to as plain mode (since 5.11).  If the daemon denies 'crc' mode,
+  fail the connection.
+
+* ms_mode=secure - Use msgr2.1 on-the-wire protocol, select 'secure' mode
+  (since 5.11).  'secure' mode provides full in-transit encryption ensuring
+  both confidentiality and authenticity.  If the daemon denies 'secure' mode,
+  fail the connection.
+
+* ms_mode=prefer-crc - Use msgr2.1 on-the-wire protocol, select 'crc'
+  mode (since 5.11).  If the daemon denies 'crc' mode in favor of 'secure'
+  mode, agree to 'secure' mode.
+
+* ms_mode=prefer-secure - Use msgr2.1 on-the-wire protocol, select 'secure'
+  mode (since 5.11).  If the daemon denies 'secure' mode in favor of 'crc'
+  mode, agree to 'crc' mode.
+
+* rxbounce - Use a bounce buffer when receiving data (since 5.17).  The default
+  behaviour is to read directly into the destination buffer.  A bounce buffer
+  is needed if the destination buffer isn't guaranteed to be stable (i.e. remain
+  unchanged while it is being read to).  In particular this is the case for
+  Windows where a system-wide "dummy" (throwaway) page may be mapped into the
+  destination buffer in order to generate a single large I/O.  Otherwise,
+  "libceph: ... bad crc/signature" or "libceph: ... integrity error, bad crc"
+  errors and associated performance degradation are expected.
+
+* udev - Wait for udev device manager to finish executing all matching
+  "add" rules and release the device before exiting (default).  This option
+  is not passed to the kernel.
+
+* noudev - Don't wait for udev device manager.  When enabled, the device may
+  not be fully usable immediately on exit.
+
+`rbd device unmap` options:
+
+* force - Force the unmapping of a block device that is open (since 4.9).  The
+  driver will wait for running requests to complete and then unmap; requests
+  sent to the driver after initiating the unmap will be failed.
+
+* udev - Wait for udev device manager to finish executing all matching
+  "remove" rules and clean up after the device before exiting (default).
+  This option is not passed to the kernel.
+
+* noudev - Don't wait for udev device manager.
+
+
+Examples
+========
+
+To create a new rbd image that is 100 GB::
+
+       rbd create mypool/myimage --size 102400
+
+To use a non-default object size (8 MB)::
+
+       rbd create mypool/myimage --size 102400 --object-size 8M
+
+To delete an rbd image (be careful!)::
+
+       rbd rm mypool/myimage
+
+To create a new snapshot::
+
+       rbd snap create mypool/myimage@mysnap
+
+To create a copy-on-write clone of a protected snapshot::
+
+       rbd clone mypool/myimage@mysnap otherpool/cloneimage
+
+To see which clones of a snapshot exist::
+
+       rbd children mypool/myimage@mysnap
+
+To delete a snapshot::
+
+       rbd snap rm mypool/myimage@mysnap
+
+To map an image via the kernel with cephx enabled::
+
+       rbd device map mypool/myimage --id admin --keyfile secretfile
+
+To map an image via the kernel with different cluster name other than default *ceph*::
+
+       rbd device map mypool/myimage --cluster cluster-name
+
+To unmap an image::
+
+       rbd device unmap /dev/rbd0
+
+To create an image and a clone from it::
+
+       rbd import --image-format 2 image mypool/parent
+       rbd snap create mypool/parent@snap
+       rbd snap protect mypool/parent@snap
+       rbd clone mypool/parent@snap otherpool/child
+
+To create an image with a smaller stripe_unit (to better distribute small writes in some workloads)::
+
+       rbd create mypool/myimage --size 102400 --stripe-unit 65536B --stripe-count 16
+
+To change an image from one image format to another, export it and then
+import it as the desired image format::
+
+       rbd export mypool/myimage@snap /tmp/img
+       rbd import --image-format 2 /tmp/img mypool/myimage2
+
+To lock an image for exclusive use::
+
+       rbd lock add mypool/myimage mylockid
+
+To release a lock::
+
+       rbd lock remove mypool/myimage mylockid client.2485
+
+To list images from trash::
+
+       rbd trash ls mypool
+
+To defer delete an image (use *--expires-at* to set expiration time, default is now)::
+
+       rbd trash mv mypool/myimage --expires-at "tomorrow"
+
+To delete an image from trash (be careful!)::
+
+       rbd trash rm mypool/myimage-id
+
+To force delete an image from trash (be careful!)::
+
+       rbd trash rm mypool/myimage-id  --force
+
+To restore an image from trash::
+
+       rbd trash restore mypool/myimage-id
+
+To restore an image from trash and rename it::
+
+       rbd trash restore mypool/myimage-id --image mynewimage
+
+
+Availability
+============
+
+**rbd** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at http://ceph.com/docs for more information.
+
+
+See also
+========
+
+:doc:`ceph <ceph>`\(8),
+:doc:`rados <rados>`\(8)
diff --git a/doc/man/8/rbdmap.rst b/doc/man/8/rbdmap.rst
new file mode 100644
index 000000000..137920f4c
--- /dev/null
+++ b/doc/man/8/rbdmap.rst
@@ -0,0 +1,128 @@
+:orphan:
+
+=========================================
+ rbdmap -- map RBD devices at boot time
+=========================================
+
+.. program:: rbdmap
+
+Synopsis
+========
+
+| **rbdmap map**
+| **rbdmap unmap**
+
+
+Description
+===========
+
+**rbdmap** is a shell script that automates ``rbd map`` and ``rbd unmap``
+operations on one or more RBD (RADOS Block Device) images. While the script can be
+run manually by the system administrator at any time, the principal use case is
+automatic mapping/mounting of RBD images at boot time (and unmounting/unmapping
+at shutdown), as triggered by the init system (a systemd unit file,
+``rbdmap.service`` is included with the ceph-common package for this purpose).
+
+The script takes a single argument, which can be either "map" or "unmap".
+In either case, the script parses a configuration file (defaults to ``/etc/ceph/rbdmap``,
+but can be overridden via an environment variable ``RBDMAPFILE``). Each line
+of the configuration file corresponds to an RBD image which is to be mapped, or
+unmapped.
+
+The configuration file format is::
+
+    IMAGESPEC RBDOPTS
+
+where ``IMAGESPEC`` should be specified as ``POOLNAME/IMAGENAME`` (the pool
+name, a forward slash, and the image name), or merely ``IMAGENAME``, in which
+case the ``POOLNAME`` defaults to "rbd". ``RBDOPTS`` is an optional list of
+parameters to be passed to the underlying ``rbd map`` command. These parameters
+and their values should be specified as a comma-separated string::
+
+    PARAM1=VAL1,PARAM2=VAL2,...,PARAMN=VALN 
+
+This will cause the script to issue an ``rbd map`` command like the following::
+
+    rbd map POOLNAME/IMAGENAME --PARAM1 VAL1 --PARAM2 VAL2 
+
+(See the ``rbd`` manpage for a full list of possible options.)
+For parameters and values which contain commas or equality signs, a simple
+apostrophe can be used to prevent replacing them.
+
+When run as ``rbdmap map``, the script parses the configuration file, and for
+each RBD image specified attempts to first map the image (using the ``rbd map``
+command) and, second, to mount the image.
+
+When run as ``rbdmap unmap``, images listed in the configuration file will
+be unmounted and unmapped.
+
+``rbdmap unmap-all`` attempts to unmount and subsequently unmap all currently
+mapped RBD images, regardless of whether or not they are listed in the
+configuration file.
+
+If successful, the ``rbd map`` operation maps the image to a ``/dev/rbdX``
+device, at which point a udev rule is triggered to create a friendly device
+name symlink, ``/dev/rbd/POOLNAME/IMAGENAME``, pointing to the real mapped
+device.
+
+In order for mounting/unmounting to succeed, the friendly device name must
+have a corresponding entry in ``/etc/fstab``.
+
+When writing ``/etc/fstab`` entries for RBD images, it's a good idea to specify
+the "noauto" (or "nofail") mount option. This prevents the init system from
+trying to mount the device too early - before the device in question even
+exists. (Since ``rbdmap.service``
+executes a shell script, it is typically triggered quite late in the boot
+sequence.)
+
+
+Examples
+========
+
+Example ``/etc/ceph/rbdmap`` for three RBD images called "bar1", "bar2" and "bar3", 
+which are in pool "foopool"::
+
+    foopool/bar1    id=admin,keyring=/etc/ceph/ceph.client.admin.keyring
+    foopool/bar2    id=admin,keyring=/etc/ceph/ceph.client.admin.keyring
+    foopool/bar3    id=admin,keyring=/etc/ceph/ceph.client.admin.keyring,options='lock_on_read,queue_depth=1024'
+
+Each line in the file contains two strings: the image spec and the options to
+be passed to ``rbd map``. These two lines get transformed into the following
+commands::
+
+    rbd map foopool/bar1 --id admin --keyring /etc/ceph/ceph.client.admin.keyring
+    rbd map foopool/bar2 --id admin --keyring /etc/ceph/ceph.client.admin.keyring
+    rbd map foopool/bar2 --id admin --keyring /etc/ceph/ceph.client.admin.keyring --options lock_on_read,queue_depth=1024
+
+If the images had XFS file systems on them, the corresponding ``/etc/fstab``
+entries might look like this::
+
+    /dev/rbd/foopool/bar1 /mnt/bar1 xfs noauto 0 0
+    /dev/rbd/foopool/bar2 /mnt/bar2 xfs noauto 0 0
+    /dev/rbd/foopool/bar3 /mnt/bar3 xfs noauto 0 0
+
+After creating the images and populating the ``/etc/ceph/rbdmap`` file, making
+the images get automatically mapped and mounted at boot is just a matter of
+enabling that unit::
+
+    systemctl enable rbdmap.service
+
+
+Options
+=======
+
+None
+
+
+Availability
+============
+
+**rbdmap** is part of Ceph, a massively scalable, open-source, distributed
+storage system. Please refer to the Ceph documentation at
+http://ceph.com/docs for more information.
+
+
+See also
+========
+
+:doc:`rbd <rbd>`\(8),
diff --git a/doc/man/8/rgw-orphan-list.rst b/doc/man/8/rgw-orphan-list.rst
new file mode 100644
index 000000000..408242da2
--- /dev/null
+++ b/doc/man/8/rgw-orphan-list.rst
@@ -0,0 +1,69 @@
+:orphan:
+
+==================================================================
+ rgw-orphan-list -- list rados objects that are not indexed by rgw
+==================================================================
+
+.. program:: rgw-orphan-list
+
+Synopsis
+========
+
+| **rgw-orphan-list**
+
+Description
+===========
+
+:program:`rgw-orphan-list` is an *EXPERIMENTAL* RADOS gateway user
+administration utility. It produces a listing of rados objects that
+are not directly or indirectly referenced through the bucket indexes
+on a pool. It places the results and intermediate files on the local
+filesystem rather than on the ceph cluster itself, and therefore will
+not itself consume additional cluster storage.
+
+In theory orphans should not exist. However because ceph evolves
+rapidly, bugs do crop up, and they may result in orphans that are left
+behind.
+
+In its current form this utility does not take any command-line
+arguments or options. It will list the available pools and prompt the
+user to enter the pool they would like to list orphans for.
+
+Behind the scenes it runs `rados ls` and `radosgw-admin bucket
+radoslist ...` and produces a list of those entries that appear in the
+former but not the latter. Those entries are presumed to be the
+orphans.
+
+Warnings
+========
+
+This utility is currently considered *EXPERIMENTAL*.
+
+This utility will produce false orphan entries for unindexed buckets
+since such buckets have no bucket indices that can provide the
+starting point for tracing.
+
+Options
+=======
+
+At present there are no options.
+
+Examples
+========
+
+Launch the tool::
+
+        $ rgw-orphan-list
+
+Availability
+============
+
+:program:`radosgw-admin` is part of Ceph, a massively scalable, open-source,
+distributed storage system.  Please refer to the Ceph documentation at
+http://ceph.com/docs for more information.
+
+See also
+========
+
+:doc:`radosgw-admin <radosgw-admin>`\(8)
+:doc:`ceph-diff-sorted <ceph-diff-sorted>`\(8)
diff --git a/doc/man/CMakeLists.txt b/doc/man/CMakeLists.txt
new file mode 100644
index 000000000..e81631bab
--- /dev/null
+++ b/doc/man/CMakeLists.txt
@@ -0,0 +1,15 @@
+set(sphinx_input)
+set(sphinx_output)
+set(sphinx_output_dir ${CMAKE_BINARY_DIR}/doc/man)
+
+add_subdirectory(8)
+
+add_custom_command(
+  OUTPUT ${sphinx_output}
+  COMMAND ${SPHINX_BUILD} -b man -t man -d ${CMAKE_BINARY_DIR}/doc/doctrees -c ${CMAKE_SOURCE_DIR}/man ${CMAKE_CURRENT_SOURCE_DIR} ${sphinx_output_dir}
+  DEPENDS ${sphinx_input})
+
+add_custom_target(
+  manpages ALL
+  DEPENDS ${sphinx_output}
+  COMMENT "manpages building")
diff --git a/doc/man_index.rst b/doc/man_index.rst
new file mode 100644
index 000000000..30ab1c3eb
--- /dev/null
+++ b/doc/man_index.rst
@@ -0,0 +1,47 @@
+.. this is the man index/toctree reference. It is separate from the "regular"
+.. index so that it doesn't include things that are not tagged for man pages
+
+.. toctree::
+   :maxdepth: 3
+   :hidden:
+
+   man/8/ceph-authtool
+   man/8/ceph-clsinfo
+   man/8/ceph-conf
+   man/8/ceph-create-keys
+   man/8/ceph-debugpack
+   man/8/ceph-dencoder
+   man/8/ceph-volume
+   man/8/ceph-volume-systemd
+   man/8/ceph-fuse
+   man/8/ceph-mds
+   man/8/ceph-mon
+   man/8/ceph-osd
+   man/8/ceph-post-file
+   man/8/ceph-rbdnamer
+   man/8/ceph-run
+   man/8/ceph-syn
+   man/8/ceph
+   man/8/cephadm
+   man/8/cephfs-top
+   man/8/cephfs-mirror
+   man/8/crushtool
+   man/8/librados-config
+   man/8/monmaptool
+   man/8/mount.ceph
+   man/8/osdmaptool
+   man/8/rados
+   man/8/radosgw-admin
+   man/8/radosgw
+   man/8/rbd-fuse
+   man/8/rbd-ggate
+   man/8/rbd-mirror
+   man/8/rbd-nbd
+   man/8/rbd-replay-many
+   man/8/rbd-replay-prep
+   man/8/rbd-replay
+   man/8/rbd
+   man/8/rbdmap
+   man/8/rgw-orphan-list
+   man/8/ceph-immutable-object-cache
+   man/8/ceph-diff-sorted
diff --git a/doc/mgr/administrator.rst b/doc/mgr/administrator.rst
new file mode 100644
index 000000000..62294580a
--- /dev/null
+++ b/doc/mgr/administrator.rst
@@ -0,0 +1,179 @@
+.. _mgr-administrator-guide:
+
+ceph-mgr administrator's guide
+==============================
+
+Manual setup
+------------
+
+Usually, you would set up a ceph-mgr daemon using a tool such
+as ceph-ansible.  These instructions describe how to set up
+a ceph-mgr daemon manually.
+
+First, create an authentication key for your daemon::
+
+    ceph auth get-or-create mgr.$name mon 'allow profile mgr' osd 'allow *' mds 'allow *'
+
+Place that key into ``mgr data`` path, which for a cluster "ceph"
+and mgr $name "foo" would be ``/var/lib/ceph/mgr/ceph-foo``.
+
+Start the ceph-mgr daemon::
+
+    ceph-mgr -i $name
+
+Check that the mgr has come up by looking at the output
+of ``ceph status``, which should now include a mgr status line::
+
+    mgr active: $name
+
+Client authentication
+---------------------
+
+The manager is a new daemon which requires new CephX capabilities. If you upgrade
+a cluster from an old version of Ceph, or use the default install/deploy tools,
+your admin client should get this capability automatically. If you use tooling from
+elsewhere, you may get EACCES errors when invoking certain ceph cluster commands.
+To fix that, add a "mgr allow \*" stanza to your client's cephx capabilities by
+`Modifying User Capabilities`_.
+
+High availability
+-----------------
+
+In general, you should set up a ceph-mgr on each of the hosts
+running a ceph-mon daemon to achieve the same level of availability.
+
+By default, whichever ceph-mgr instance comes up first will be made
+active by the monitors, and the others will be standbys.  There is
+no requirement for quorum among the ceph-mgr daemons.
+
+If the active daemon fails to send a beacon to the monitors for
+more than ``mon mgr beacon grace`` (default 30s), then it will be replaced
+by a standby.
+
+If you want to pre-empt failover, you can explicitly mark a ceph-mgr
+daemon as failed using ``ceph mgr fail <mgr name>``.
+
+Performance and Scalability
+---------------------------
+
+All the mgr modules share a cache that can be enabled with
+``ceph config set mgr mgr_ttl_cache_expire_seconds <seconds>``, where seconds
+is the time to live of the cached python objects.
+
+It is recommended to enable the cache with a 10 seconds TTL when there are 500+
+osds or 10k+ pgs as internal structures might increase in size, and cause latency
+issues when requesting large structures. As an example, an OSDMap with 1000 osds
+has a aproximate size of 4MiB. With heavy load, on a 3000 osd cluster there has
+been a 1.5x improvement enabling the cache.
+
+Furthermore, you can run ``ceph daemon mgr.${MGRNAME} perf dump`` to retrieve perf
+counters of a mgr module. In ``mgr.cache_hit`` and ``mgr.cache_miss`` you'll find the
+hit/miss ratio of the mgr cache.
+
+Using modules
+-------------
+
+Use the command ``ceph mgr module ls`` to see which modules are
+available, and which are currently enabled.  Enable or disable modules
+using the commands ``ceph mgr module enable <module>`` and
+``ceph mgr module disable <module>`` respectively.
+
+If a module is *enabled* then the active ceph-mgr daemon will load
+and execute it.  In the case of modules that provide a service,
+such as an HTTP server, the module may publish its address when it
+is loaded.  To see the addresses of such modules, use the command
+``ceph mgr services``.
+
+Some modules may also implement a special standby mode which runs on
+standby ceph-mgr daemons as well as the active daemon.  This enables
+modules that provide services to redirect their clients to the active
+daemon, if the client tries to connect to a standby.
+
+Consult the documentation pages for individual manager modules for more
+information about what functionality each module provides.
+
+Here is an example of enabling the :term:`Dashboard` module:
+
+::
+
+	$ ceph mgr module ls
+	{
+		"enabled_modules": [
+			"restful",
+			"status"
+		],
+		"disabled_modules": [
+			"dashboard"
+		]
+	}
+
+	$ ceph mgr module enable dashboard
+	$ ceph mgr module ls
+	{
+		"enabled_modules": [
+			"restful",
+			"status",
+			"dashboard"
+		],
+		"disabled_modules": [
+		]
+	}
+
+	$ ceph mgr services
+	{
+		"dashboard": "http://myserver.com:7789/",
+		"restful": "https://myserver.com:8789/"
+	}
+
+
+The first time the cluster starts, it uses the ``mgr_initial_modules``
+setting to override which modules to enable.  However, this setting
+is ignored through the rest of the lifetime of the cluster: only
+use it for bootstrapping.  For example, before starting your
+monitor daemons for the first time, you might add a section like
+this to your ``ceph.conf``:
+
+::
+
+    [mon]
+        mgr_initial_modules = dashboard balancer
+
+Calling module commands
+-----------------------
+
+Where a module implements command line hooks, the commands will
+be accessible as ordinary Ceph commands.  Ceph will automatically incorporate
+module commands into the standard CLI interface and route them appropriately to
+the module.::
+
+    ceph <command | help>
+
+Configuration
+-------------
+
+``mgr_module_path``
+
+:Description: Path to load modules from
+:Type: String
+:Default: ``"<library dir>/mgr"``
+
+``mgr_data``
+
+:Description: Path to load daemon data (such as keyring)
+:Type: String
+:Default: ``"/var/lib/ceph/mgr/$cluster-$id"``
+
+``mgr_tick_period``
+
+:Description: How many seconds between mgr beacons to monitors, and other
+              periodic checks.
+:Type: Integer
+:Default: ``5``
+
+``mon_mgr_beacon_grace``
+
+:Description: How long after last beacon should a mgr be considered failed
+:Type: Integer
+:Default: ``30``
+
+.. _Modifying User Capabilities: ../../rados/operations/user-management/#modify-user-capabilities
diff --git a/doc/mgr/alerts.rst b/doc/mgr/alerts.rst
new file mode 100644
index 000000000..319d9d927
--- /dev/null
+++ b/doc/mgr/alerts.rst
@@ -0,0 +1,58 @@
+Alerts module
+=============
+
+The alerts module can send simple alert messages about cluster health
+via e-mail.  In the future, it will support other notification methods
+as well.
+
+:note: This module is *not* intended to be a robust monitoring
+       solution.  The fact that it is run as part of the Ceph cluster
+       itself is fundamentally limiting in that a failure of the
+       ceph-mgr daemon prevents alerts from being sent.  This module
+       can, however, be useful for standalone clusters that exist in
+       environments where existing monitoring infrastructure does not
+       exist.
+
+Enabling
+--------
+
+The *alerts* module is enabled with::
+
+  ceph mgr module enable alerts
+
+Configuration
+-------------
+
+To configure SMTP, all of the following config options must be set::
+
+  ceph config set mgr mgr/alerts/smtp_host *<smtp-server>*
+  ceph config set mgr mgr/alerts/smtp_destination *<email-address-to-send-to>*
+  ceph config set mgr mgr/alerts/smtp_sender *<from-email-address>*
+
+By default, the module will use SSL and port 465.  To change that,::
+
+  ceph config set mgr mgr/alerts/smtp_ssl false   # if not SSL
+  ceph config set mgr mgr/alerts/smtp_port *<port-number>*  # if not 465
+
+To authenticate to the SMTP server, you must set the user and password::
+
+  ceph config set mgr mgr/alerts/smtp_user *<username>*
+  ceph config set mgr mgr/alerts/smtp_password *<password>*
+
+By default, the name in the ``From:`` line is simply ``Ceph``.  To
+change that (e.g., to identify which cluster this is),::
+
+  ceph config set mgr mgr/alerts/smtp_from_name 'Ceph Cluster Foo'
+
+By default, the module will check the cluster health once per minute
+and, if there is a change, send a message.  To change that
+frequency,::
+
+  ceph config set mgr mgr/alerts/interval *<interval>*   # e.g., "5m" for 5 minutes
+
+Commands
+--------
+
+To force an alert to be send immediately,::
+
+  ceph alerts send
diff --git a/doc/mgr/ceph_api/index.rst b/doc/mgr/ceph_api/index.rst
new file mode 100644
index 000000000..f4e6836b0
--- /dev/null
+++ b/doc/mgr/ceph_api/index.rst
@@ -0,0 +1,90 @@
+.. _mgr-ceph-api:
+
+================
+Ceph RESTful API
+================
+
+Introduction
+============
+The **Ceph RESTful API** (henceforth **Ceph API**) is provided by the
+:ref:`mgr-dashboard` module. The Ceph API
+service is available at the same URL as the regular Ceph Dashboard, under the
+``/api`` base path (please refer to :ref:`dashboard-host-name-and-port`)::
+
+  http://<server_addr>:<server_port>/api
+
+or, if HTTPS is enabled (please refer to :ref:`dashboard-ssl-tls-support`)::
+
+  https://<server_addr>:<ssl_server_port>/api
+
+The Ceph API leverages the following standards:
+
+* `HTTP 1.1 <https://tools.ietf.org/html/rfc7231>`_ for API syntax and semantics,
+* `JSON <https://tools.ietf.org/html/rfc8259>`_ for content encoding,
+* `HTTP Content Negotiation <https://tools.ietf.org/html/rfc2295>`_ and `MIME <https://tools.ietf.org/html/rfc2045>`_ for versioning,
+* `OAuth 2.0 <https://tools.ietf.org/html/rfc6750>`_ and `JWT <https://tools.ietf.org/html/rfc7519>`_ for authentication and authorization.
+
+.. warning::
+  Some endpoints are still under active development, and should be carefully
+  used since new Ceph releases could bring backward incompatible changes.
+
+
+Authentication and Authorization
+================================
+
+Requests to the Ceph API pass through two access control checkpoints:
+
+* **Authentication**: ensures that the request is performed on behalf of an existing and valid user account.
+* **Authorization**: ensures that the previously authenticated user can in fact perform a specific action (create, read, update or delete) on the target endpoint.
+
+So, prior to start consuming the Ceph API, a valid JSON Web Token (JWT) has to
+be obtained, and it may then be reused for subsequent requests. The
+``/api/auth`` endpoint will provide the valid token:
+
+.. code-block:: sh
+
+  $ curl -X POST "https://example.com:8443/api/auth" \
+    -H  "Accept: application/vnd.ceph.api.v1.0+json" \
+    -H  "Content-Type: application/json" \
+    -d '{"username": <username>, "password": <password>}'
+
+  { "token": "<redacted_token>", ...}
+
+The token obtained must be passed together with every API request in the
+``Authorization`` HTTP header::
+
+  curl -H "Authorization: Bearer <token>" ...
+
+Authentication and authorization can be further configured from the
+Ceph CLI, the Ceph-Dashboard UI and the Ceph API itself (please refer to
+:ref:`dashboard-user-role-management`).
+
+Versioning
+==========
+
+One of the main goals of the Ceph API is to keep a stable interface. For this
+purpose, Ceph API is built upon the following principles:
+
+* **Mandatory**: in order to avoid implicit defaults, all endpoints require an explicit default version (starting with ``1.0``).
+* **Per-endpoint**: as this API wraps many different Ceph components, this allows for a finer-grained change control.
+   * **Content/MIME Type**: the version expected from a specific endpoint is stated by the ``Accept: application/vnd.ceph.api.v<major>.<minor>+json`` HTTP header. If the current Ceph API server is not able to address that specific major version, a `415 - Unsupported Media Type <https://tools.ietf.org/html/rfc7231#section-6.5.13>`_ response will be returned.
+* **Semantic Versioning**: with a ``major.minor`` version:
+   * Major changes are backward incompatible: they might result in non-additive changes to the request and/or response formats of a specific endpoint.
+   * Minor changes are backward/forward compatible: they basically consists of additive changes to the request or response formats of a specific endpoint.
+
+An example:
+
+.. code-block:: bash
+
+  $ curl -X GET "https://example.com:8443/api/osd" \
+    -H  "Accept: application/vnd.ceph.api.v1.0+json" \
+    -H  "Authorization: Bearer <token>"
+
+
+Specification
+=============
+
+.. openapi:: ../../../src/pybind/mgr/dashboard/openapi.yaml
+    :group:
+    :examples:
+    :encoding: utf-8
diff --git a/doc/mgr/cli_api.rst b/doc/mgr/cli_api.rst
new file mode 100644
index 000000000..81a99ae44
--- /dev/null
+++ b/doc/mgr/cli_api.rst
@@ -0,0 +1,39 @@
+CLI API Commands Module
+=======================
+
+The CLI API module exposes most ceph-mgr python API via CLI. Furthermore, this API can be
+benchmarked for further testing.
+
+Enabling
+--------
+
+The *cli api commands* module is enabled with::
+
+  ceph mgr module enable cli_api
+
+To check that it is enabled, run::
+
+  ceph mgr module ls | grep cli_api
+
+Usage
+--------
+
+To run a mgr module command, run::
+
+  ceph mgr cli <command> <param>
+
+For example, use the following command to print the list of servers::
+
+  ceph mgr cli list_servers
+
+List all available mgr module commands with::
+
+  ceph mgr cli --help
+
+To benchmark a command, run::
+
+  ceph mgr cli_benchmark <number of calls> <number of threads> <command> <param>
+
+For example, use the following command to benchmark the command to get osd_map::
+
+  ceph mgr cli_benchmark 100 10 get osd_map
diff --git a/doc/mgr/crash.rst b/doc/mgr/crash.rst
new file mode 100644
index 000000000..e12a8c6f8
--- /dev/null
+++ b/doc/mgr/crash.rst
@@ -0,0 +1,83 @@
+Crash Module
+============
+The crash module collects information about daemon crashdumps and stores
+it in the Ceph cluster for later analysis.
+
+Daemon crashdumps are dumped in /var/lib/ceph/crash by default; this can
+be configured with the option 'crash dir'.  Crash directories are named by
+time and date and a randomly-generated UUID, and contain a metadata file
+'meta' and a recent log file, with a "crash_id" that is the same.
+This module allows the metadata about those dumps to be persisted in
+the monitors' storage.
+
+Enabling
+--------
+
+The *crash* module is enabled with::
+
+  ceph mgr module enable crash
+
+Commands
+--------
+::
+
+  ceph crash post -i <metafile>
+
+Save a crash dump.  The metadata file is a JSON blob stored in the crash
+dir as ``meta``.  As usual, the ceph command can be invoked with ``-i -``,
+and will read from stdin.
+
+::
+
+  ceph crash rm <crashid>
+
+Remove a specific crash dump.
+
+::
+
+  ceph crash ls
+
+List the timestamp/uuid crashids for all new and archived crash info.
+
+::
+
+  ceph crash ls-new
+
+List the timestamp/uuid crashids for all newcrash info.
+
+::
+
+  ceph crash stat
+
+Show a summary of saved crash info grouped by age.
+
+::
+
+  ceph crash info <crashid>
+
+Show all details of a saved crash.
+
+::
+
+   ceph crash prune <keep>
+
+Remove saved crashes older than 'keep' days.  <keep> must be an integer.
+
+::
+
+   ceph crash archive <crashid>
+
+Archive a crash report so that it is no longer considered for the ``RECENT_CRASH`` health check and does not appear in the ``crash ls-new`` output (it will still appear in the ``crash ls`` output).
+
+::
+
+   ceph crash archive-all
+
+Archive all new crash reports.
+
+
+Options
+-------
+
+* ``mgr/crash/warn_recent_interval`` [default: 2 weeks] controls what constitutes "recent" for the purposes of raising the ``RECENT_CRASH`` health warning.
+* ``mgr/crash/retain_interval`` [default: 1 year] controls how long crash reports are retained by the cluster before they are automatically purged.
diff --git a/doc/mgr/dashboard.rst b/doc/mgr/dashboard.rst
new file mode 100644
index 000000000..ca5dd7fb8
--- /dev/null
+++ b/doc/mgr/dashboard.rst
@@ -0,0 +1,1619 @@
+.. _mgr-dashboard:
+
+Ceph Dashboard
+==============
+
+Overview
+--------
+
+The Ceph Dashboard is a built-in web-based Ceph management and monitoring
+application through which you can inspect and administer various aspects
+and resources within the cluster. It is implemented as a :ref:`ceph-manager-daemon` module.
+
+The original Ceph Dashboard that was shipped with Ceph Luminous started
+out as a simple read-only view into run-time information and performance
+data of Ceph clusters. It used a very simple architecture to achieve the
+original goal. However, there was growing demand for richer web-based
+management capabilities, to make it easier to administer Ceph for users that
+prefer a WebUI over the CLI.
+
+The new :term:`Ceph Dashboard` module adds web-based monitoring and
+administration to the Ceph Manager. The architecture and functionality of this new
+module are derived from
+and inspired by the `openATTIC Ceph management and monitoring tool
+<https://openattic.org/>`_. Development is actively driven by the
+openATTIC team at `SUSE <https://www.suse.com/>`_, with support from
+companies including `Red Hat <https://redhat.com/>`_ and members of the Ceph
+community.
+
+The dashboard module's backend code uses the CherryPy framework and implements
+a custom REST API. The WebUI implementation is based on
+Angular/TypeScript and includes both functionality from the original dashboard
+and new features originally developed for the standalone version
+of openATTIC. The Ceph Dashboard module is implemented as an
+application that provides a graphical representation of information and statistics
+through a web server hosted by ``ceph-mgr``.
+
+Feature Overview
+^^^^^^^^^^^^^^^^
+
+The dashboard provides the following features:
+
+* **Multi-User and Role Management**: The dashboard supports multiple user
+  accounts with different permissions (roles). User accounts and roles
+  can be managed via both the command line and the WebUI. The dashboard
+  supports various methods to enhance password security. Password
+  complexity rules may be configured, requiring users to change their password
+  after the first login or after a configurable time period. See
+  :ref:`dashboard-user-role-management` for details.
+* **Single Sign-On (SSO)**: The dashboard supports authentication
+  via an external identity provider using the SAML 2.0 protocol. See
+  :ref:`dashboard-sso-support` for details.
+* **SSL/TLS support**: All HTTP communication between the web browser and the
+  dashboard is secured via SSL. A self-signed certificate can be created with
+  a built-in command, but it's also possible to import custom certificates
+  signed and issued by a CA. See :ref:`dashboard-ssl-tls-support` for details.
+* **Auditing**: The dashboard backend can be configured to log all ``PUT``, ``POST``
+  and ``DELETE`` API requests in the Ceph audit log. See :ref:`dashboard-auditing`
+  for instructions on how to enable this feature.
+* **Internationalization (I18N)**: The language used for dashboard text can be
+  selected at run-time.
+
+The Ceph Dashboard offers the following monitoring and management capabilities:
+
+* **Overall cluster health**: Display performance and capacity metrics as well
+  as cluster status.
+* **Embedded Grafana Dashboards**: Ceph Dashboard
+  `Grafana`_ dashboards may be embedded in external applications and web pages
+  to surface information and performance metrics gathered by
+  the :ref:`mgr-prometheus` module. See
+  :ref:`dashboard-grafana` for details on how to configure this functionality.
+* **Cluster logs**: Display the latest updates to the cluster's event and
+  audit log files. Log entries can be filtered by priority, date or keyword.
+* **Hosts**: Display a list of all cluster hosts along with their
+  storage drives, which services are running, and which version of Ceph is
+  installed.
+* **Performance counters**: Display detailed service-specific statistics for
+  each running service.
+* **Monitors**: List all Mons, their quorum status, and open sessions.
+* **Monitoring**: Enable creation, re-creation, editing, and expiration of
+  Prometheus' silences, list the alerting configuration and all
+  configured and firing alerts. Show notifications for firing alerts.
+* **Configuration Editor**: Display all available configuration options,
+  their descriptions, types, default and currently set values.  These may be edited as well.
+* **Pools**: List Ceph pools and their details (e.g. applications,
+  pg-autoscaling, placement groups, replication size, EC profile, CRUSH
+  rulesets, quotas etc.)
+* **OSDs**: List OSDs, their status and usage statistics as well as
+  detailed information like attributes (OSD map), metadata, performance
+  counters and usage histograms for read/write operations. Mark OSDs
+  up/down/out, purge and reweight OSDs, perform scrub operations, modify
+  various scrub-related configuration options, select profiles to
+  adjust the level of backfilling activity. List all drives associated with an
+  OSD. Set and change the device class of an OSD, display and sort OSDs by
+  device class. Deploy OSDs on new drives and hosts.
+* **Device management**: List all hosts known by the orchestrator. List all
+  drives attached to a host and their properties. Display drive
+  health predictions and SMART data. Blink enclosure LEDs.
+* **iSCSI**: List all hosts that run the TCMU runner service, display all
+  images and their performance characteristics (read/write ops, traffic).
+  Create, modify, and delete iSCSI targets (via ``ceph-iscsi``). Display the
+  iSCSI gateway status and info about active initiators.
+  See :ref:`dashboard-iscsi-management` for instructions on how to configure
+  this feature.
+* **RBD**: List all RBD images and their properties (size, objects, features).
+  Create, copy, modify and delete RBD images (incl. snapshots) and manage RBD
+  namespaces. Define various I/O or bandwidth limitation settings on a global,
+  per-pool or per-image level. Create, delete and rollback snapshots of selected
+  images, protect/unprotect these snapshots against modification. Copy or clone
+  snapshots, flatten cloned images.
+* **RBD mirroring**: Enable and configure RBD mirroring to a remote Ceph server.
+  List active daemons and their status, pools and RBD images including
+  sync progress.
+* **CephFS**: List active file system clients and associated pools,
+  including usage statistics. Evict active CephFS clients. Manage CephFS
+  quotas and snapshots. Browse a CephFS directory structure.
+* **Object Gateway**: List all active object gateways and their performance
+  counters. Display and manage (add/edit/delete) object gateway users and their
+  details (e.g. quotas) as well as the users' buckets and their details (e.g.
+  placement targets, owner, quotas, versioning, multi-factor authentication).
+  See :ref:`dashboard-enabling-object-gateway` for configuration instructions.
+* **NFS**: Manage NFS exports of CephFS file systems and RGW S3 buckets via NFS
+  Ganesha. See :ref:`dashboard-nfs-ganesha-management` for details on how to
+  enable this functionality.
+* **Ceph Manager Modules**: Enable and disable Ceph Manager modules, manage
+  module-specific configuration settings.
+
+Overview of the Dashboard Landing Page
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Displays overall cluster status, performance, and capacity metrics. Shows instant
+feedback for changes in the cluster and provides easy access to subpages of the
+dashboard.
+
+.. _dashboard-landing-page-status:
+
+Status
+""""""
+
+* **Cluster Status**: Displays overall cluster health. In case of any error it
+  displays a short description of the error and provides a link to the logs.
+* **Hosts**: Displays the total number of hosts associated to the cluster and
+  links to a subpage that lists and describes each.
+* **Monitors**: Displays mons and their quorum status and
+  open sessions.  Links to a subpage that lists and describes each.
+* **OSDs**: Displays object storage daemons (ceph-osds) and
+  the numbers of OSDs running (up), in service
+  (in), and out of the cluster (out). Provides links to
+  subpages providing a list of all OSDs and related management actions.
+* **Managers**: Displays active and standby Ceph Manager
+  daemons (ceph-mgr).
+* **Object Gateway**: Displays active object gateways (RGWs) and
+  provides links to subpages that list all object gateway daemons.
+* **Metadata Servers**: Displays active and standby CephFS metadata
+  service daemons (ceph-mds).
+* **iSCSI Gateways**: Display iSCSI gateways available,
+  active (up), and inactive (down). Provides a link to a subpage
+  showing a list of all iSCSI Gateways.
+
+.. _dashboard-landing-page-capacity:
+
+Capacity
+""""""""
+
+* **Raw Capacity**: Displays the capacity used out of the total
+  physical capacity provided by storage nodes (OSDs).
+* **Objects**: Displays the number and status of RADOS objects
+  including the percentages of healthy, misplaced, degraded, and unfound
+  objects.
+* **PG Status**: Displays the total number of placement groups and
+  their status, including the percentage clean, working,
+  warning, and unknown.
+* **Pools**: Displays pools and links to a subpage listing details.
+* **PGs per OSD**: Displays the number of placement groups assigned to
+  object storage daemons.
+
+.. _dashboard-landing-page-performance:
+
+Performance
+"""""""""""
+
+* **Client READ/Write**: Displays an overview of
+  client input and output operations.
+* **Client Throughput**: Displays the data transfer rates to and from Ceph clients.
+* **Recovery throughput**: Displays rate of cluster healing and balancing operations.
+* **Scrubbing**: Displays light and deep scrub status.
+
+Supported Browsers
+^^^^^^^^^^^^^^^^^^
+
+Ceph Dashboard is primarily tested and developed using the following web
+browsers:
+
++---------------------------------------------------------------+---------------------------------------+
+|                            Browser                            |               Versions                |
++===============================================================+=======================================+
+| `Chrome <https://www.google.com/chrome/>`_ and                | latest 2 major versions               |
+| `Chromium <https://www.chromium.org/>`_ based browsers        |                                       |
++---------------------------------------------------------------+---------------------------------------+
+| `Firefox <https://www.mozilla.org/firefox/>`_                 | latest 2 major versions               |
++---------------------------------------------------------------+---------------------------------------+
+| `Firefox ESR <https://www.mozilla.org/firefox/enterprise/>`_  | latest major version                  |
++---------------------------------------------------------------+---------------------------------------+
+
+While Ceph Dashboard might work in older browsers, we cannot guarantee compatibility and
+recommend keeping your browser up to date.
+
+Enabling
+--------
+
+If you have installed ``ceph-mgr-dashboard`` from distribution packages, the
+package management system should take care of installing all required
+dependencies.
+
+If you're building Ceph from source and want to start the dashboard from your
+development environment, please see the files ``README.rst`` and ``HACKING.rst``
+in the source directory ``src/pybind/mgr/dashboard``.
+
+Within a running Ceph cluster, the Ceph Dashboard is enabled with:
+
+.. prompt:: bash $
+
+   ceph mgr module enable dashboard
+
+Configuration
+-------------
+
+.. _dashboard-ssl-tls-support:
+
+SSL/TLS Support
+^^^^^^^^^^^^^^^
+
+All HTTP connections to the dashboard are secured with SSL/TLS by default.
+
+To get the dashboard up and running quickly, you can generate and install a
+self-signed certificate:
+
+.. prompt:: bash $
+
+   ceph dashboard create-self-signed-cert
+
+Note that most web browsers will complain about self-signed certificates
+and require explicit confirmation before establishing a secure connection to the
+dashboard.
+
+To properly secure a deployment and to remove the warning, a
+certificate that is issued by a certificate authority (CA) should be used.
+
+For example, a key pair can be generated with a command similar to:
+
+.. prompt:: bash $
+
+   openssl req -new -nodes -x509 \
+   -subj "/O=IT/CN=ceph-mgr-dashboard" -days 3650 \
+   -keyout dashboard.key -out dashboard.crt -extensions v3_ca
+
+The ``dashboard.crt`` file should then be signed by a CA. Once that is done, you
+can enable it for Ceph manager instances by running the following commands:
+
+.. prompt:: bash $
+
+   ceph dashboard set-ssl-certificate -i dashboard.crt
+   ceph dashboard set-ssl-certificate-key -i dashboard.key
+
+If unique certificates are desired for each manager instance,
+the name of the instance can be included as follows (where ``$name`` is the name
+of the ``ceph-mgr`` instance, usually the hostname):
+
+.. prompt:: bash $
+
+   ceph dashboard set-ssl-certificate $name -i dashboard.crt
+   ceph dashboard set-ssl-certificate-key $name -i dashboard.key
+
+SSL can also be disabled by setting this configuration value:
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/dashboard/ssl false
+
+This might be useful if the dashboard will be running behind a proxy which does
+not support SSL for its upstream servers or other situations where SSL is not
+wanted or required. See :ref:`dashboard-proxy-configuration` for more details.
+
+.. warning::
+
+  Use caution when disabling SSL as usernames and passwords will be sent to the
+  dashboard unencrypted.
+
+
+.. note::
+
+  You must restart Ceph manager processes after changing the SSL
+  certificate and key. This can be accomplished by either running ``ceph mgr
+  fail mgr`` or by disabling and re-enabling the dashboard module (which also
+  triggers the manager to respawn itself):
+  
+  .. prompt:: bash $
+
+     ceph mgr module disable dashboard
+     ceph mgr module enable dashboard
+
+.. _dashboard-host-name-and-port:
+
+Host Name and Port
+^^^^^^^^^^^^^^^^^^
+
+Like most web applications, the dashboard binds to a TCP/IP address and TCP port.
+
+By default, the ``ceph-mgr`` daemon hosting the dashboard (i.e., the currently
+active manager) will bind to TCP port 8443 or 8080 when SSL is disabled.
+
+If no specific address has been configured, the web app will bind to ``::``,
+which corresponds to all available IPv4 and IPv6 addresses.
+
+These defaults can be changed via the configuration key facility on a
+cluster-wide level (so they apply to all manager instances) as follows:
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/dashboard/server_addr $IP
+   ceph config set mgr mgr/dashboard/server_port $PORT
+   ceph config set mgr mgr/dashboard/ssl_server_port $PORT
+
+Since each ``ceph-mgr`` hosts its own instance of the dashboard, it may be
+necessary to configure them separately. The IP address and port for a specific
+manager instance can be changed with the following commands:
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/dashboard/$name/server_addr $IP
+   ceph config set mgr mgr/dashboard/$name/server_port $PORT
+   ceph config set mgr mgr/dashboard/$name/ssl_server_port $PORT
+
+Replace ``$name`` with the ID of the ceph-mgr instance hosting the dashboard.
+
+.. note::
+
+  The command ``ceph mgr services`` will show you all endpoints that are
+  currently configured. Look for the ``dashboard`` key to obtain the URL for
+  accessing the dashboard.
+
+Username and Password
+^^^^^^^^^^^^^^^^^^^^^
+
+In order to be able to log in, you need to create a user account and associate
+it with at least one role. We provide a set of predefined *system roles* that
+you can use. For more details please refer to the `User and Role Management`_
+section.
+
+To create a user with the administrator role you can use the following
+commands:
+
+.. prompt:: bash $
+
+   ceph dashboard ac-user-create <username> -i <file-containing-password> administrator
+
+Account Lock-out
+^^^^^^^^^^^^^^^^
+
+It disables a user account if a user repeatedly enters the wrong credentials
+for multiple times. It is enabled by default to prevent brute-force or dictionary
+attacks. The user can get or set the default number of lock-out attempts using
+these commands respectively:
+
+.. prompt:: bash $
+
+   ceph dashboard get-account-lockout-attempts
+   ceph dashboard set-account-lockout-attempts <value:int>
+
+.. warning::
+
+  This feature can be disabled by setting the default number of lock-out attempts to 0.
+  However, by disabling this feature, the account is more vulnerable to brute-force or
+  dictionary based attacks. This can be disabled by:
+
+  .. prompt:: bash $
+
+     ceph dashboard set-account-lockout-attempts 0
+
+Enable a Locked User
+^^^^^^^^^^^^^^^^^^^^
+
+If a user account is disabled as a result of multiple invalid login attempts, then
+it needs to be manually enabled by the administrator. This can be done by the following
+command:
+
+.. prompt:: bash $
+
+   ceph dashboard ac-user-enable <username>
+
+Accessing the Dashboard
+^^^^^^^^^^^^^^^^^^^^^^^
+
+You can now access the dashboard using your (JavaScript-enabled) web browser, by
+pointing it to any of the host names or IP addresses and the selected TCP port
+where a manager instance is running: e.g., ``http(s)://<$IP>:<$PORT>/``.
+
+The dashboard page displays and requests a previously defined username and
+password.
+
+.. _dashboard-enabling-object-gateway:
+
+Enabling the Object Gateway Management Frontend
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When RGW is deployed with cephadm, the RGW credentials used by the
+dashboard will be automatically configured. You can also manually force the
+credentials to be set up with:
+
+.. prompt:: bash $
+
+   ceph dashboard set-rgw-credentials
+
+This will create an RGW user with uid ``dashboard`` for each realm in
+the system.
+
+If you've configured a custom 'admin' resource in your RGW admin API, you should set it here also:
+
+.. prompt:: bash $
+
+   ceph dashboard set-rgw-api-admin-resource <admin_resource>
+
+If you are using a self-signed certificate in your Object Gateway setup,
+you should disable certificate verification in the dashboard to avoid refused
+connections, e.g. caused by certificates signed by unknown CA or not matching
+the host name:
+
+.. prompt:: bash $
+
+   ceph dashboard set-rgw-api-ssl-verify False
+
+If the Object Gateway takes too long to process requests and the dashboard runs
+into timeouts, you can set the timeout value to your needs:
+
+.. prompt:: bash $
+
+   ceph dashboard set-rest-requests-timeout <seconds>
+
+The default value is 45 seconds.
+
+.. _dashboard-iscsi-management:
+
+Enabling iSCSI Management
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The Ceph Dashboard can manage iSCSI targets using the REST API provided by the
+``rbd-target-api`` service of the :ref:`ceph-iscsi`. Please make sure that it is
+installed and enabled on the iSCSI gateways.
+
+.. note::
+
+  The iSCSI management functionality of Ceph Dashboard depends on the latest
+  version 3 of the `ceph-iscsi <https://github.com/ceph/ceph-iscsi>`_ project.
+  Make sure that your operating system provides the correct version, otherwise
+  the dashboard will not enable the management features.
+
+If the ``ceph-iscsi`` REST API is configured in HTTPS mode and its using a self-signed
+certificate, you need to configure the dashboard to avoid SSL certificate
+verification when accessing ceph-iscsi API.
+
+To disable API SSL verification run the following command:
+
+.. prompt:: bash $
+
+   ceph dashboard set-iscsi-api-ssl-verification false
+
+The available iSCSI gateways must be defined using the following commands:
+
+.. prompt:: bash $
+
+   ceph dashboard iscsi-gateway-list
+   # Gateway URL format for a new gateway: <scheme>://<username>:<password>@<host>[:port]
+   ceph dashboard iscsi-gateway-add -i <file-containing-gateway-url> [<gateway_name>]
+   ceph dashboard iscsi-gateway-rm <gateway_name>
+
+
+.. _dashboard-grafana:
+
+Enabling the Embedding of Grafana Dashboards
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+`Grafana`_ pulls data from `Prometheus <https://prometheus.io/>`_. Although
+Grafana can use other data sources, the Grafana dashboards we provide contain
+queries that are specific to Prometheus. Our Grafana dashboards therefore
+require Prometheus as the data source. The Ceph :ref:`mgr-prometheus`
+module exports its data in the Prometheus exposition format. These Grafana
+dashboards rely on metric names from the Prometheus module and `Node exporter
+<https://prometheus.io/docs/guides/node-exporter/>`_. The Node exporter is a
+separate application that provides machine metrics.
+
+.. note::
+
+  Prometheus' security model presumes that untrusted users have access to the
+  Prometheus HTTP endpoint and logs. Untrusted users have access to all the
+  (meta)data Prometheus collects that is contained in the database, plus a
+  variety of operational and debugging information.
+
+  However, Prometheus' HTTP API is limited to read-only operations.
+  Configurations can *not* be changed using the API and secrets are not
+  exposed. Moreover, Prometheus has some built-in measures to mitigate the
+  impact of denial of service attacks.
+
+  Please see `Prometheus' Security model
+  <https://prometheus.io/docs/operating/security/>` for more detailed
+  information.
+
+Installation and Configuration using cephadm
+""""""""""""""""""""""""""""""""""""""""""""
+
+Grafana and Prometheus can be installed using :ref:`cephadm`. They will
+automatically be configured by ``cephadm``. Please see
+:ref:`mgr-cephadm-monitoring` documentation for more details on how to use
+``cephadm`` for installing and configuring Prometheus and Grafana.
+
+Manual Installation and Configuration
+"""""""""""""""""""""""""""""""""""""
+
+The following process describes how to configure Grafana and Prometheus
+manually. After you have installed Prometheus, Grafana, and the Node exporter
+on appropriate hosts, proceed with the following steps.
+
+#.  Enable the Ceph Exporter which comes as Ceph Manager module by running:
+
+    .. prompt:: bash $
+
+       ceph mgr module enable prometheus
+
+    More details can be found in the documentation of the :ref:`mgr-prometheus`.
+
+#.  Add the corresponding scrape configuration to Prometheus. This may look
+    like::
+
+      global:
+        scrape_interval: 5s
+
+      scrape_configs:
+        - job_name: 'prometheus'
+          static_configs:
+            - targets: ['localhost:9090']
+        - job_name: 'ceph'
+          static_configs:
+            - targets: ['localhost:9283']
+        - job_name: 'node-exporter'
+          static_configs:
+            - targets: ['localhost:9100']
+
+    .. note::
+
+      Please note that in the above example, Prometheus is configured
+      to scrape data from itself (port 9090), the Ceph manager module
+      `prometheus` (port 9283), which exports Ceph internal data, and the Node
+      Exporter (port 9100), which provides OS and hardware metrics for each host.
+
+      Depending on your configuration, you may need to change the hostname in
+      or add additional configuration entries for the Node
+      Exporter. It is unlikely that you will need to change the default TCP ports.
+
+      Moreover, you don't *need* to have more than one target for Ceph specific
+      data, provided by the `prometheus` mgr module. But it is recommended to
+      configure Prometheus to scrape Ceph specific data from all existing Ceph
+      managers. This enables a built-in high availability mechanism, so that
+      services run on a manager host will be restarted automatically on a different
+      manager host if one Ceph Manager goes down.
+
+#. Add Prometheus as data source to Grafana `using the Grafana Web UI <https://grafana.com/docs/grafana/latest/features/datasources/add-a-data-source/>`_.
+
+   .. IMPORTANT:: 
+      The data source must be named "Dashboard1".
+
+#.  Install the `vonage-status-panel and grafana-piechart-panel` plugins using:
+
+    .. prompt:: bash $
+
+      grafana-cli plugins install vonage-status-panel
+      grafana-cli plugins install grafana-piechart-panel
+
+#.  Add Dashboards to Grafana:
+
+    Dashboards can be added to Grafana by importing dashboard JSON files.
+    Use the following command to download the JSON files:
+
+    .. prompt:: bash $
+
+       wget https://raw.githubusercontent.com/ceph/ceph/main/monitoring/ceph-mixin/dashboards_out/<Dashboard-name>.json
+
+    You can find various dashboard JSON files `here <https://github.com/ceph/ceph/tree/
+    main/monitoring/ceph-mixin/dashboards_out>`_.
+
+    For Example, for ceph-cluster overview you can use:
+
+    .. prompt:: bash $
+
+       wget https://raw.githubusercontent.com/ceph/ceph/main/monitoring/ceph-mixin/dashboards_out/ceph-cluster.json
+
+    You may also author your own dashboards.
+
+#.  Configure anonymous mode in ``/etc/grafana/grafana.ini``::
+
+      [auth.anonymous]
+      enabled = true
+      org_name = Main Org.
+      org_role = Viewer
+
+    In newer versions of Grafana (starting with 6.2.0-beta1) a new setting named
+    ``allow_embedding`` has been introduced. This setting must be explicitly
+    set to ``true`` for the Grafana integration in Ceph Dashboard to work, as the
+    default is ``false``.
+
+    ::
+
+      [security]
+      allow_embedding = true
+
+Enabling RBD-Image monitoring
+"""""""""""""""""""""""""""""
+
+Monitoring of RBD images is disabled by default, as it can significantly impact
+performance. For more information please see :ref:`prometheus-rbd-io-statistics`.
+When disabled, the overview and details dashboards will be empty in Grafana and
+metrics will not be visible in Prometheus.
+
+Configuring Dashboard
+"""""""""""""""""""""
+
+After you have set up Grafana and Prometheus, you will need to configure the
+connection information that the Ceph Dashboard will use to access Grafana.
+
+You need to tell the dashboard on which URL the Grafana instance is
+running/deployed:
+
+.. prompt:: bash $
+
+   ceph dashboard set-grafana-api-url <grafana-server-url>  # default: ''
+
+The format of url is : `<protocol>:<IP-address>:<port>`
+
+.. note::
+
+  The Ceph Dashboard embeds Grafana dashboards via ``iframe`` HTML elements.
+  If Grafana is configured without SSL/TLS support, most browsers will block the
+  embedding of insecure content if SSL support is
+  enabled for the dashboard (which is the default). If you
+  can't see the embedded Grafana dashboards after enabling them as outlined
+  above, check your browser's documentation on how to unblock mixed content.
+  Alternatively, consider enabling SSL/TLS support in Grafana.
+
+If you are using a self-signed certificate for Grafana,
+disable certificate verification in the dashboard to avoid refused connections,
+which can be a result of certificates signed by an unknown CA or that do not
+match the host name:
+
+.. prompt:: bash $
+
+   ceph dashboard set-grafana-api-ssl-verify False
+
+You can also access Grafana directly to monitor your cluster.
+
+.. note::
+
+  Ceph Dashboard configuration information can also be unset. For example, to
+  clear the Grafana API URL we configured above:
+
+  .. prompt:: bash $
+
+     ceph dashboard reset-grafana-api-url
+
+Alternative URL for Browsers
+""""""""""""""""""""""""""""
+
+The Ceph Dashboard backend requires the Grafana URL to be able to verify the
+existence of Grafana Dashboards before the frontend even loads them. Due to the
+nature of how Grafana is implemented in Ceph Dashboard, this means that two
+working connections are required in order to be able to see Grafana graphs in
+Ceph Dashboard:
+
+- The backend (Ceph Mgr module) needs to verify the existence of the requested
+  graph. If this request succeeds, it lets the frontend know that it can safely
+  access Grafana.
+- The frontend then requests the Grafana graphs directly from the user's
+  browser using an iframe. The Grafana instance is accessed directly without any
+  detour through Ceph Dashboard.
+
+Now, it might be the case that your environment makes it difficult for the
+user's browser to directly access the URL configured in Ceph Dashboard. To solve
+this issue, a separate URL can be configured which will solely be used to tell
+the frontend (the user's browser) which URL it should use to access Grafana.
+This setting won't ever be changed automatically, unlike the GRAFANA_API_URL
+which is set by :ref:`cephadm` (only if cephadm is used to deploy monitoring
+services).
+
+To change the URL that is returned to the frontend issue the following command:
+
+.. prompt:: bash $
+
+   ceph dashboard set-grafana-frontend-api-url <grafana-server-url>
+
+If no value is set for that option, it will simply fall back to the value of the
+GRAFANA_API_URL option. If set, it will instruct the browser to use this URL to
+access Grafana.
+
+.. _dashboard-sso-support:
+
+Enabling Single Sign-On (SSO)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The Ceph Dashboard supports external authentication of users via the
+`SAML 2.0 <https://en.wikipedia.org/wiki/SAML_2.0>`_ protocol. You need to
+first create user accounts and associate them with desired roles, as
+authorization is performed by the Dashboard. However, the authentication
+process can be performed by an existing Identity Provider (IdP).
+
+.. note::
+
+  Ceph Dashboard SSO support relies on onelogin's
+  `python-saml <https://pypi.org/project/python-saml/>`_ library.
+  Please ensure that this library is installed on your system, either by using
+  your distribution's package management or via Python's `pip` installer.
+
+To configure SSO on Ceph Dashboard, you should use the following command:
+
+.. prompt:: bash $
+
+   ceph dashboard sso setup saml2 <ceph_dashboard_base_url> <idp_metadata> {<idp_username_attribute>} {<idp_entity_id>} {<sp_x_509_cert>} {<sp_private_key>}
+
+Parameters:
+
+* **<ceph_dashboard_base_url>**: Base URL where Ceph Dashboard is accessible (e.g., `https://cephdashboard.local`)
+* **<idp_metadata>**: URL to remote (`http://`, `https://`) or local (`file://`) path or content of the IdP metadata XML (e.g., `https://myidp/metadata`, `file:///home/myuser/metadata.xml`).
+* **<idp_username_attribute>** *(optional)*: Attribute that should be used to get the username from the authentication response. Defaults to `uid`.
+* **<idp_entity_id>** *(optional)*: Use this when more than one entity id exists on the IdP metadata.
+* **<sp_x_509_cert> / <sp_private_key>** *(optional)*: File path of the certificate that should be used by Ceph Dashboard (Service Provider) for signing and encryption.
+
+.. note::
+
+  The issuer value of SAML requests will follow this pattern:  **<ceph_dashboard_base_url>**/auth/saml2/metadata
+
+To display the current SAML 2.0 configuration, use the following command:
+
+.. prompt:: bash $
+
+   ceph dashboard sso show saml2
+
+.. note::
+
+  For more information about `onelogin_settings`, please check the `onelogin documentation <https://github.com/onelogin/python-saml>`_.
+
+To disable SSO:
+
+.. prompt:: bash $
+
+   ceph dashboard sso disable
+
+To check if SSO is enabled:
+
+.. prompt:: bash $
+
+   ceph dashboard sso status
+
+To enable SSO:
+
+.. prompt:: bash $
+
+   ceph dashboard sso enable saml2
+
+.. _dashboard-alerting:
+
+Enabling Prometheus Alerting
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To use Prometheus for alerting you must define `alerting rules
+<https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules>`_.
+These are managed by the `Alertmanager
+<https://prometheus.io/docs/alerting/alertmanager>`_.
+If you are not yet using the Alertmanager, `install it
+<https://github.com/prometheus/alertmanager#install>`_ as it receives
+and manages alerts from Prometheus.
+
+Alertmanager capabilities can be consumed by the dashboard in three different
+ways:
+
+#. Use the notification receiver of the dashboard.
+
+#. Use the Prometheus Alertmanager API.
+
+#. Use both sources simultaneously.
+
+All three methods notify you about alerts. You won't be notified
+twice if you use both sources, but you need to consume at least the Alertmanager API
+in order to manage silences.
+
+1. Use the notification receiver of the dashboard
+
+  This allows you to get notifications as `configured
+  <https://prometheus.io/docs/alerting/configuration/>`_ from the Alertmanager.
+  You will get notified inside the dashboard once a notification is send out,
+  but you are not able to manage alerts.
+
+  Add the dashboard receiver and the new route to your Alertmanager
+  configuration. This should look like::
+
+    route:
+      receiver: 'ceph-dashboard'
+    ...
+    receivers:
+      - name: 'ceph-dashboard'
+        webhook_configs:
+        - url: '<url-to-dashboard>/api/prometheus_receiver'
+
+
+  Ensure that the Alertmanager considers your SSL certificate in terms
+  of the dashboard as valid. For more information about the correct
+  configuration checkout the `<http_config> documentation
+  <https://prometheus.io/docs/alerting/configuration/#%3Chttp_config%3E>`_.
+
+2. Use the API of Prometheus and the Alertmanager
+
+  This allows you to manage alerts and silences and will enable the "Active
+  Alerts", "All Alerts" as well as the "Silences" tabs in the "Monitoring"
+  section of the "Cluster" menu entry.
+
+  Alerts can be sorted by name, job, severity, state and start time.
+  Unfortunately it's not possible to know when an alert was sent out through a
+  notification by the Alertmanager based on your configuration, that's why the
+  dashboard will notify the user on any visible change to an alert and will
+  notify the changed alert.
+
+  Silences can be sorted by id, creator, status, start, updated and end time.
+  Silences can be created in various ways, it's also possible to expire them.
+
+  #. Create from scratch
+
+  #. Based on a selected alert
+
+  #. Recreate from expired silence
+
+  #. Update a silence (which will recreate and expire it (default Alertmanager behaviour))
+
+  To use it, specify the host and port of the Alertmanager server:
+  
+  .. prompt:: bash $
+
+     ceph dashboard set-alertmanager-api-host <alertmanager-host:port>  # default: ''
+
+  For example:
+  
+  .. prompt:: bash $
+
+     ceph dashboard set-alertmanager-api-host 'http://localhost:9093'
+
+  To be able to see all configured alerts, you will need to configure the URL to
+  the Prometheus API. Using this API, the UI will also help you in verifying
+  that a new silence will match a corresponding alert.
+
+  
+  .. prompt:: bash $
+
+     ceph dashboard set-prometheus-api-host <prometheus-host:port>  # default: ''
+
+  For example:
+
+  .. prompt:: bash $
+
+     ceph dashboard set-prometheus-api-host 'http://localhost:9090'
+
+  After setting up the hosts, refresh your browser's dashboard window or tab.
+
+3. Use both methods
+
+  The behaviors of both methods are configured in a way that they
+  should not disturb each other, through annoying duplicated notifications
+  may pop up.
+
+If you are using a self-signed certificate in your Prometheus or your
+Alertmanager setup, you should disable certificate verification in the
+dashboard to avoid refused connections caused by certificates signed by
+an unknown CA or that do not match the host name.
+
+- For Prometheus:
+
+.. prompt:: bash $
+
+   ceph dashboard set-prometheus-api-ssl-verify False
+
+- For Alertmanager:
+
+.. prompt:: bash $
+
+   ceph dashboard set-alertmanager-api-ssl-verify False
+
+.. _dashboard-user-role-management:
+
+User and Role Management
+------------------------
+
+Password Policy
+^^^^^^^^^^^^^^^
+
+By default the password policy feature is enabled, which includes the
+following checks:
+
+- Is the password longer than N characters?
+- Are the old and new password the same?
+
+The password policy feature can be switched on or off completely:
+
+.. prompt:: bash $
+
+    ceph dashboard set-pwd-policy-enabled <true|false>
+
+The following individual checks can also be switched on or off:
+
+.. prompt:: bash $
+
+  ceph dashboard set-pwd-policy-check-length-enabled <true|false>
+  ceph dashboard set-pwd-policy-check-oldpwd-enabled <true|false>
+  ceph dashboard set-pwd-policy-check-username-enabled <true|false>
+  ceph dashboard set-pwd-policy-check-exclusion-list-enabled <true|false>
+  ceph dashboard set-pwd-policy-check-complexity-enabled <true|false>
+  ceph dashboard set-pwd-policy-check-sequential-chars-enabled <true|false>
+  ceph dashboard set-pwd-policy-check-repetitive-chars-enabled <true|false>
+
+Additionally the following options are available to configure password
+policy.
+
+- Minimum password length (defaults to 8):
+
+.. prompt:: bash $
+
+   ceph dashboard set-pwd-policy-min-length <N>
+
+- Minimum password complexity (defaults to 10):
+
+  .. prompt:: bash $
+
+     ceph dashboard set-pwd-policy-min-complexity <N>
+
+  Password complexity is calculated by classifying each character in
+  the password. The complexity count starts by 0. A character is rated by
+  the following rules in the given order.
+
+  - Increase by 1 if the character is a digit.
+  - Increase by 1 if the character is a lower case ASCII character.
+  - Increase by 2 if the character is an upper case ASCII character.
+  - Increase by 3 if the character is a special character like ``!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~``.
+  - Increase by 5 if the character has not been classified by one of the previous rules.
+
+- A list of comma separated words that are not allowed to be used in a
+  password:
+
+  .. prompt:: bash $
+
+     ceph dashboard set-pwd-policy-exclusion-list <word>[,...]
+
+
+User Accounts
+^^^^^^^^^^^^^
+
+The Ceph Dashboard supports multiple user accounts. Each user account
+consists of a username, a password (stored in encrypted form using ``bcrypt``),
+an optional name, and an optional email address.
+
+If a new user is created via the Web UI, it is possible to set an option that the
+user must assign a new password when they log in for the first time.
+
+User accounts are stored in the monitors' configuration database, and are
+available to all ``ceph-mgr`` instances.
+
+We provide a set of CLI commands to manage user accounts:
+
+- *Show User(s)*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-user-show [<username>]
+
+- *Create User*:
+  
+  .. prompt:: bash $
+
+     ceph dashboard ac-user-create [--enabled] [--force-password] [--pwd_update_required] <username> -i <file-containing-password> [<rolename>] [<name>] [<email>] [<pwd_expiration_date>]
+
+  To bypass password policy checks use the `force-password` option.
+  Add the option `pwd_update_required` so that a newly created user has
+  to change their password after the first login.
+
+- *Delete User*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-user-delete <username>
+
+- *Change Password*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-user-set-password [--force-password] <username> -i <file-containing-password>
+
+- *Change Password Hash*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-user-set-password-hash <username> -i <file-containing-password-hash>
+
+  The hash must be a bcrypt hash and salt, e.g. ``$2b$12$Pt3Vq/rDt2y9glTPSV.VFegiLkQeIpddtkhoFetNApYmIJOY8gau2``.
+  This can be used to import users from an external database.
+
+- *Modify User (name, and email)*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-user-set-info <username> <name> <email>
+
+- *Disable User*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-user-disable <username>
+
+- *Enable User*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-user-enable <username>
+
+User Roles and Permissions
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+User accounts are associated with a set of roles that define which
+dashboard functionality can be accessed.
+
+The Dashboard functionality/modules are grouped within a *security scope*.
+Security scopes are predefined and static. The current available security
+scopes are:
+
+- **hosts**: includes all features related to the ``Hosts`` menu
+  entry.
+- **config-opt**: includes all features related to management of Ceph
+  configuration options.
+- **pool**: includes all features related to pool management.
+- **osd**: includes all features related to OSD management.
+- **monitor**: includes all features related to monitor management.
+- **rbd-image**: includes all features related to RBD image
+  management.
+- **rbd-mirroring**: includes all features related to RBD mirroring
+  management.
+- **iscsi**: includes all features related to iSCSI management.
+- **rgw**: includes all features related to RADOS Gateway (RGW) management.
+- **cephfs**: includes all features related to CephFS management.
+- **nfs-ganesha**: includes all features related to NFS Ganesha management.
+- **manager**: include all features related to Ceph Manager
+  management.
+- **log**: include all features related to Ceph logs management.
+- **grafana**: include all features related to Grafana proxy.
+- **prometheus**: include all features related to Prometheus alert management.
+- **dashboard-settings**: allows to change dashboard settings.
+
+A *role* specifies a set of mappings between a *security scope* and a set of
+*permissions*. There are four types of permissions:
+
+- **read**
+- **create**
+- **update**
+- **delete**
+
+See below for an example of a role specification, in the form of a Python dictionary::
+
+  # example of a role
+  {
+    'role': 'my_new_role',
+    'description': 'My new role',
+    'scopes_permissions': {
+      'pool': ['read', 'create'],
+      'rbd-image': ['read', 'create', 'update', 'delete']
+    }
+  }
+
+The above role dictates that a user has *read* and *create* permissions for
+features related to pool management, and has full permissions for
+features related to RBD image management.
+
+The Dashboard provides a set of predefined roles that we call
+*system roles*, which can be used right away by a fresh Ceph Dashboard
+installation.
+
+The list of system roles are:
+
+- **administrator**: allows full permissions for all security scopes.
+- **read-only**: allows *read* permission for all security scopes except
+  dashboard settings.
+- **block-manager**: allows full permissions for *rbd-image*,
+  *rbd-mirroring*, and *iscsi* scopes.
+- **rgw-manager**: allows full permissions for the *rgw* scope
+- **cluster-manager**: allows full permissions for the *hosts*, *osd*,
+  *monitor*, *manager*, and *config-opt* scopes.
+- **pool-manager**: allows full permissions for the *pool* scope.
+- **cephfs-manager**: allows full permissions for the *cephfs* scope.
+
+The list of available roles can be retrieved with the following command:
+
+.. prompt:: bash $
+
+   ceph dashboard ac-role-show [<rolename>]
+
+You can also use the CLI to create new roles. The available commands are the
+following:
+
+- *Create Role*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-role-create <rolename> [<description>]
+
+- *Delete Role*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-role-delete <rolename>
+
+- *Add Scope Permissions to Role*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-role-add-scope-perms <rolename> <scopename> <permission> [<permission>...]
+
+- *Delete Scope Permission from Role*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-role-del-scope-perms <rolename> <scopename>
+
+To assign roles to users, the following commands are available:
+
+- *Set User Roles*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-user-set-roles <username> <rolename> [<rolename>...]
+
+- *Add Roles To User*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-user-add-roles <username> <rolename> [<rolename>...]
+
+- *Delete Roles from User*:
+
+  .. prompt:: bash $
+
+     ceph dashboard ac-user-del-roles <username> <rolename> [<rolename>...]
+
+
+Example of User and Custom Role Creation
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In this section we show a complete example of the commands that
+create a user account that can manage RBD images, view and create Ceph pools,
+and has read-only access to other scopes.
+
+1. *Create the user*:
+
+   .. prompt:: bash $
+
+      ceph dashboard ac-user-create bob -i <file-containing-password>
+
+2. *Create role and specify scope permissions*:
+
+   .. prompt:: bash $
+
+      ceph dashboard ac-role-create rbd/pool-manager
+      ceph dashboard ac-role-add-scope-perms rbd/pool-manager rbd-image read create update delete
+      ceph dashboard ac-role-add-scope-perms rbd/pool-manager pool read create
+
+3. *Associate roles to user*:
+
+   .. prompt:: bash $
+
+      ceph dashboard ac-user-set-roles bob rbd/pool-manager read-only
+
+.. _dashboard-proxy-configuration:
+
+Proxy Configuration
+-------------------
+
+In a Ceph cluster with multiple ``ceph-mgr`` instances, only the dashboard
+running on the currently active ``ceph-mgr`` daemon will serve incoming requests.
+Connections to the dashboard's TCP port on standby ``ceph-mgr`` instances
+will receive an HTTP redirect (303) to the active manager's dashboard URL.
+This enables you to point your browser to any ``ceph-mgr`` instance in
+order to access the dashboard.
+
+If you want to establish a fixed URL to reach the dashboard or if you don't want
+to allow direct connections to the manager nodes, you could set up a proxy that
+automatically forwards incoming requests to the active ``ceph-mgr``
+instance.
+
+Configuring a URL Prefix
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you are accessing the dashboard via a reverse proxy,
+you may wish to service it under a URL prefix. To get the dashboard
+to use hyperlinks that include your prefix, you can set the
+``url_prefix`` setting:
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/dashboard/url_prefix $PREFIX
+
+so you can access the dashboard at ``http://$IP:$PORT/$PREFIX/``.
+
+Disable the redirection
+^^^^^^^^^^^^^^^^^^^^^^^
+
+If the dashboard is behind a load-balancing proxy like `HAProxy <https://www.haproxy.org/>`_
+you might want to disable redirection to prevent situations in which
+internal (unresolvable) URLs are published to the frontend client. Use the
+following command to get the dashboard to respond with an HTTP error (500 by default)
+instead of redirecting to the active dashboard:
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/dashboard/standby_behaviour "error"
+
+To reset the setting to default redirection, use the following command:
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/dashboard/standby_behaviour "redirect"
+
+Configure the error status code
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When redirection is disabled, you may want to customize the HTTP status
+code of standby dashboards. To do so you need to run the command:
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/dashboard/standby_error_status_code 503
+
+Resolve IP address to hostname before redirect
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The redirect from a standby to the active dashboard is done via the IP
+address. This is done because resolving IP addresses to hostnames can be error
+prone in containerized environments. It is also the reason why the option is
+disabled by default.
+However, in some situations it might be helpful to redirect via the hostname.
+For example if the configured TLS certificate matches only the hostnames. To
+activate the redirection via the hostname run the following command::
+
+  $ ceph config set mgr mgr/dashboard/redirect_resolve_ip_addr True
+
+You can disable it again by::
+
+  $ ceph config set mgr mgr/dashboard/redirect_resolve_ip_addr False
+
+HAProxy example configuration
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Below you will find an example configuration for SSL/TLS passthrough using
+`HAProxy <https://www.haproxy.org/>`_.
+
+Please note that this configuration works under the following conditions.
+If the dashboard fails over, the front-end client might receive a HTTP redirect
+(303) response and will be redirected to an unresolvable host. This happens when
+failover occurs between two HAProxy health checks. In this situation the
+previously active dashboard node will now respond with a 303 which points to
+the new active node. To prevent that situation you should consider disabling
+redirection on standby nodes.
+
+::
+
+  defaults
+    log global
+    option log-health-checks
+    timeout connect 5s
+    timeout client 50s
+    timeout server 450s
+
+  frontend dashboard_front
+    mode http
+    bind *:80
+    option httplog
+    redirect scheme https code 301 if !{ ssl_fc }
+
+  frontend dashboard_front_ssl
+    mode tcp
+    bind *:443
+    option tcplog
+    default_backend dashboard_back_ssl
+
+  backend dashboard_back_ssl
+    mode tcp
+    option httpchk GET /
+    http-check expect status 200
+    server x <HOST>:<PORT> ssl check verify none
+    server y <HOST>:<PORT> ssl check verify none
+    server z <HOST>:<PORT> ssl check verify none
+
+.. _dashboard-auditing:
+
+Auditing API Requests
+---------------------
+
+The REST API can log PUT, POST and DELETE requests to the Ceph
+audit log. This feature is disabled by default, but can be enabled with the
+following command:
+
+.. prompt:: bash $
+
+   ceph dashboard set-audit-api-enabled <true|false>
+
+If enabled, the following parameters are logged per each request:
+
+* from - The origin of the request, e.g. https://[::1]:44410
+* path - The REST API path, e.g. /api/auth
+* method - e.g. PUT, POST or DELETE
+* user - The name of the user, otherwise 'None'
+
+The logging of the request payload (the arguments and their values) is enabled
+by default. Execute the following command to disable this behaviour:
+
+.. prompt:: bash $
+
+   ceph dashboard set-audit-api-log-payload <true|false>
+
+A log entry may look like this::
+
+  2018-10-22 15:27:01.302514 mgr.x [INF] [DASHBOARD] from='https://[::ffff:127.0.0.1]:37022' path='/api/rgw/user/klaus' method='PUT' user='admin' params='{"max_buckets": "1000", "display_name": "Klaus Mustermann", "uid": "klaus", "suspended": "0", "email": "klaus.mustermann@ceph.com"}'
+
+.. _dashboard-nfs-ganesha-management:
+
+NFS-Ganesha Management
+----------------------
+
+The dashboard requires enabling the NFS module which will be used to manage
+NFS clusters and NFS exports. For more information check :ref:`mgr-nfs`.
+
+Plug-ins
+--------
+
+Plug-ins extend the functionality of the Ceph Dashboard in a modular
+and loosely coupled fashion.
+
+.. _Grafana: https://grafana.com/
+
+.. include:: dashboard_plugins/feature_toggles.inc.rst
+.. include:: dashboard_plugins/debug.inc.rst
+.. include:: dashboard_plugins/motd.inc.rst
+
+
+Troubleshooting the Dashboard
+-----------------------------
+
+Locating the Dashboard
+^^^^^^^^^^^^^^^^^^^^^^
+
+If you are unsure of the location of the Ceph Dashboard, run the following command:
+
+.. prompt:: bash $
+
+   ceph mgr services | jq .dashboard
+
+::
+
+    "https://host:port"
+
+The command returns the URL where the Ceph Dashboard is located: ``https://<host>:<port>/``
+
+.. note::
+
+    Many Ceph tools return results in JSON format. We suggest that
+    you install the `jq <https://stedolan.github.io/jq>`_ command-line
+    utility to faciliate working with JSON data.
+
+
+Accessing the Dashboard
+^^^^^^^^^^^^^^^^^^^^^^^
+
+If you are unable to access the Ceph Dashboard, run the following
+commands:
+
+#. Verify the Ceph Dashboard module is enabled:
+
+   .. prompt:: bash $
+
+      ceph mgr module ls | jq .enabled_modules
+
+   Ensure the Ceph Dashboard module is listed in the return value of the
+   command. Example snipped output from the command above::
+
+    [
+      "dashboard",
+      "iostat",
+      "restful"
+    ]
+
+#. If it is not listed, activate the module with the following command:
+
+   .. prompt:: bash $
+
+      ceph mgr module enable dashboard
+
+#. Check the Ceph Dashboard and/or ``ceph-mgr`` log files for any errors.
+
+   * Check if ``ceph-mgr`` log messages are written to a file by:
+     
+     .. prompt:: bash $
+
+        ceph config get mgr log_to_file
+
+     ::
+
+        true
+
+   * Get the location of the log file (it's ``/var/log/ceph/<cluster-name>-<daemon-name>.log``
+     by default):
+
+     .. prompt:: bash $
+
+        ceph config get mgr log_file
+
+     ::
+
+        /var/log/ceph/$cluster-$name.log
+
+#. Ensure the SSL/TSL support is configured properly:
+
+   * Check if the SSL/TSL support is enabled:
+
+     .. prompt:: bash $
+
+        ceph config get mgr mgr/dashboard/ssl
+
+   * If the command returns ``true``, verify a certificate exists by:
+
+     .. prompt:: bash $
+
+        ceph config-key get mgr/dashboard/crt
+
+     and:
+
+     .. prompt:: bash $
+
+        ceph config-key get mgr/dashboard/key
+
+   * If it doesn't return ``true``, run the following command to generate a self-signed
+     certificate or follow the instructions outlined in
+     :ref:`dashboard-ssl-tls-support`:
+
+     .. prompt:: bash $
+
+        ceph dashboard create-self-signed-cert
+
+
+Trouble Logging into the Dashboard
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you are unable to log into the Ceph Dashboard and you receive the following
+error, run through the procedural checks below:
+
+.. image:: ../images/dashboard/invalid-credentials.png
+   :align: center
+
+#. Check that your user credentials are correct. If you are seeing the
+   notification message above when trying to log into the Ceph Dashboard, it
+   is likely you are using the wrong credentials. Double check your username
+   and password, and ensure that your keyboard's caps lock is not enabled by accident.
+
+#. If your user credentials are correct, but you are experiencing the same
+   error, check that the user account exists:
+
+   .. prompt:: bash $
+
+      ceph dashboard ac-user-show <username>
+
+   This command returns your user data. If the user does not exist, it will
+   print::
+
+      Error ENOENT: User <username> does not exist
+
+#. Check if the user is enabled:
+
+   .. prompt:: bash $
+
+      ceph dashboard ac-user-show <username> | jq .enabled
+
+   ::
+
+    true
+
+   Check if ``enabled`` is set to ``true`` for your user. If not the user is
+   not enabled, run:
+
+   .. prompt:: bash $
+
+      ceph dashboard ac-user-enable <username>
+
+Please see :ref:`dashboard-user-role-management` for more information.
+
+
+A Dashboard Feature is Not Working
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When an error occurs on the backend, you will usually receive an error
+notification on the frontend. Run through the following scenarios to debug.
+
+#. Check the Ceph Dashboard and ``ceph-mgr`` logfile(s) for any errors. These can
+   found by searching for keywords, such as *500 Internal Server Error*,
+   followed by ``traceback``. The end of a traceback contains more details about
+   what exact error occurred.
+#. Check your web browser's Javascript Console for any errors.
+
+
+Ceph Dashboard Logs
+^^^^^^^^^^^^^^^^^^^
+
+Dashboard Debug Flag
+""""""""""""""""""""
+
+With this flag enabled, error traceback is included in backend responses.
+
+To enable this flag via the Ceph Dashboard, navigate from *Cluster* to *Manager
+modules*. Select *Dashboard module* and click the edit button. Click the
+*debug* checkbox and update.
+
+To enable it via the CLI, run the following command:
+
+.. prompt:: bash $
+
+   ceph dashboard debug enable
+
+
+Setting Logging Level of Dashboard Module
+"""""""""""""""""""""""""""""""""""""""""
+
+Setting the logging level to debug makes the log more verbose and helpful for
+debugging.
+
+#. Increase the logging level of manager daemons:
+
+   .. prompt:: bash $
+
+      ceph tell mgr config set debug_mgr 20
+
+#. Adjust the logging level of the Ceph Dashboard module via the Dashboard or
+   CLI:
+
+   * Navigate from *Cluster* to *Manager modules*. Select *Dashboard module*
+     and click the edit button. Modify the ``log_level`` configuration.
+   * To adjust it via the CLI, run the following command:
+
+     .. prompt:: bash $
+
+        bin/ceph config set mgr mgr/dashboard/log_level debug
+
+3. High log levels can result in considerable log volume, which can
+easily fill up your filesystem. Set a calendar reminder for an hour, a day,
+or a week in the future to revert this temporary logging increase.  This looks
+something like this:
+
+   .. prompt:: bash $
+
+      ceph config log
+
+   ::
+
+      ...
+      --- 11 --- 2020-11-07 11:11:11.960659 --- mgr.x/dashboard/log_level = debug ---
+      ...
+    
+   .. prompt:: bash $
+
+      ceph config reset 11
+
+.. _centralized-logging:
+
+
+Reporting issues from Dashboard
+"""""""""""""""""""""""""""""""
+
+Ceph-Dashboard provides two ways to create an issue in the Ceph Issue Tracker,
+either using the Ceph command line interface or by using the Ceph Dashboard
+user interface.
+
+To create an issue in the Ceph Issue Tracker, a user needs to have an account
+on the issue tracker. Under the ``my account`` tab in the Ceph Issue Tracker,
+the user can see their API access key. This key is used for authentication
+when creating a new issue. To store the Ceph API access key, in the CLI run:
+
+.. prompt:: bash $
+
+   ``ceph dashboard set-issue-tracker-api-key -i <file-containing-key>``
+
+Then on successful update, you can create an issue using:
+
+.. prompt:: bash $
+
+   ``ceph dashboard create issue <project> <tracker_type> <subject> <description>``
+
+The available projects to create an issue on are:
+#. dashboard
+#. block
+#. object
+#. file_system
+#. ceph_manager
+#. orchestrator
+#. ceph_volume
+#. core_ceph
+
+The available tracker types are:
+#. bug
+#. feature
+
+The subject and description are then set by the user.
+
+The user can also create an issue using the Dashboard user interface. The settings
+icon drop down menu on the top right of the navigation bar has the option to
+``Raise an issue``. On clicking it, a modal dialog opens that has the option to
+select the project and tracker from their respective drop down menus. The subject
+and multiline description are added by the user. The user can then submit the issue.
diff --git a/doc/mgr/dashboard_plugins/debug.inc.rst b/doc/mgr/dashboard_plugins/debug.inc.rst
new file mode 100644
index 000000000..883419cbf
--- /dev/null
+++ b/doc/mgr/dashboard_plugins/debug.inc.rst
@@ -0,0 +1,43 @@
+.. _dashboard-debug:
+
+Debug
+^^^^^
+
+This plugin allows to customize the behaviour of the dashboard according to the
+debug mode. It can be enabled, disabled or checked with the following command:
+
+.. prompt:: bash $
+
+   ceph dashboard debug status
+
+::
+
+  Debug: 'disabled'
+
+.. prompt:: bash $
+
+   ceph dashboard debug enable
+
+::
+
+  Debug: 'enabled'
+
+.. prompt:: bash $
+
+   ceph dashboard debug disable
+
+::
+
+  Debug: 'disabled'
+
+By default, it's disabled. This is the recommended setting for production
+deployments. If required, debug mode can be enabled without need of restarting.
+Currently, disabled debug mode equals to CherryPy ``production`` environment,
+while when enabled, it uses ``test_suite`` defaults (please refer to
+`CherryPy Environments
+<https://docs.cherrypy.org/en/latest/config.html#environments>`_ for more
+details).
+
+It also adds request uuid (``unique_id``) to Cherrypy on versions that don't
+support this. It additionally prints the ``unique_id`` to error responses and
+log messages.
diff --git a/doc/mgr/dashboard_plugins/feature_toggles.inc.rst b/doc/mgr/dashboard_plugins/feature_toggles.inc.rst
new file mode 100644
index 000000000..7c96b0faa
--- /dev/null
+++ b/doc/mgr/dashboard_plugins/feature_toggles.inc.rst
@@ -0,0 +1,56 @@
+.. _dashboard-feature-toggles:
+
+Feature Toggles
+^^^^^^^^^^^^^^^
+
+This plug-in allows to enable or disable some features from the Ceph Dashboard
+on-demand. When a feature becomes disabled:
+
+- Its front-end elements (web pages, menu entries, charts, etc.) will become hidden.
+- Its associated REST API endpoints will reject any further requests (404, Not Found Error).
+
+The main purpose of this plug-in is to allow ad-hoc customizations of the workflows exposed
+by the dashboard. Additionally, it could allow for dynamically enabling experimental
+features with minimal configuration burden and no service impact.
+
+The list of features that can be enabled/disabled is:
+
+- **Block (RBD)**:
+   - Image Management: ``rbd``
+   - Mirroring: ``mirroring``
+   - iSCSI: ``iscsi``
+- **Filesystem (Cephfs)**: ``cephfs``
+- **Objects (RGW)**: ``rgw`` (including daemon, user and bucket management).
+- **NFS**: ``nfs-ganesha`` exports.
+
+By default all features come enabled.
+
+To retrieve a list of features and their current statuses:
+
+.. prompt:: bash $
+
+   ceph dashboard feature status
+
+::
+
+  Feature 'cephfs': 'enabled'
+  Feature 'iscsi': 'enabled'
+  Feature 'mirroring': 'enabled'
+  Feature 'rbd': 'enabled'
+  Feature 'rgw': 'enabled'
+  Feature 'nfs': 'enabled'
+
+To enable or disable the status of a single or multiple features:
+
+.. prompt:: bash $
+
+   ceph dashboard feature disable iscsi mirroring
+
+:: 
+
+  Feature 'iscsi': disabled
+  Feature 'mirroring': disabled
+
+After a feature status has changed, the API REST endpoints immediately respond to
+that change, while for the front-end UI elements, it may take up to 20 seconds to
+reflect it.
diff --git a/doc/mgr/dashboard_plugins/motd.inc.rst b/doc/mgr/dashboard_plugins/motd.inc.rst
new file mode 100644
index 000000000..0f9cc199a
--- /dev/null
+++ b/doc/mgr/dashboard_plugins/motd.inc.rst
@@ -0,0 +1,36 @@
+.. _dashboard-motd:
+
+Message of the day (MOTD)
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Displays a configured `message of the day` at the top of the Ceph Dashboard.
+
+The importance of a MOTD can be configured by its severity, which is
+`info`, `warning` or `danger`. The MOTD can expire after a given time,
+this means it will not be displayed in the UI anymore. Use the following
+syntax to specify the expiration time: `Ns|m|h|d|w` for seconds, minutes,
+hours, days and weeks. If the MOTD should expire after 2 hours, use `2h`
+or `5w` for 5 weeks. Use `0` to configure a MOTD that does not expire.
+
+To configure a MOTD, run the following command:
+
+.. prompt:: bash $
+
+   ceph dashboard motd set <severity:info|warning|danger> <expires> <message>
+
+To show the configured MOTD:
+
+.. prompt:: bash $
+
+   ceph dashboard motd get
+
+To clear the configured MOTD run:
+
+.. prompt:: bash $
+
+   ceph dashboard motd clear
+
+A MOTD with a `info` or `warning` severity can be closed by the user. The
+`info` MOTD is not displayed anymore until the local storage cookies are
+cleared or a new MOTD with a different severity is displayed. A MOTD with
+a 'warning' severity will be displayed again in a new session.
diff --git a/doc/mgr/diskprediction.rst b/doc/mgr/diskprediction.rst
new file mode 100644
index 000000000..af34a037c
--- /dev/null
+++ b/doc/mgr/diskprediction.rst
@@ -0,0 +1,59 @@
+.. _diskprediction:
+
+=====================
+Diskprediction Module
+=====================
+
+The *diskprediction* module leverages Ceph device health check to collect disk health metrics and uses internal predictor module to produce the disk failure prediction and returns back to Ceph. It doesn't require any external server for data analysis and output results. Its internal predictor's accuracy is around 70%.
+
+Enabling
+========
+
+Run the following command to enable the *diskprediction_local* module in the Ceph
+environment::
+
+    ceph mgr module enable diskprediction_local
+
+
+To enable the local predictor::
+
+    ceph config set global device_failure_prediction_mode local
+
+To disable prediction,::
+
+    ceph config set global device_failure_prediction_mode none
+
+
+*diskprediction_local* requires at least six datasets of device health metrics to
+make prediction of the devices' life expentancy. And these health metrics are
+collected only if health monitoring is :ref:`enabled <enabling-monitoring>`.
+
+Run the following command to retrieve the life expectancy of given device.
+
+::
+
+    ceph device predict-life-expectancy <device id>
+
+Configuration
+=============
+
+The module performs the prediction on a daily basis by default. You can adjust
+this interval with::
+
+  ceph config set mgr mgr/diskprediction_local/predict_interval <interval-in-seconds>
+
+Debugging
+=========
+
+If you want to debug the DiskPrediction module mapping to Ceph logging level,
+use the following command.
+
+::
+
+    [mgr]
+
+        debug mgr = 20
+
+With logging set to debug for the manager the module will print out logging
+message with prefix *mgr[diskprediction]* for easy filtering.
+
diff --git a/doc/mgr/hello.rst b/doc/mgr/hello.rst
new file mode 100644
index 000000000..725355fc9
--- /dev/null
+++ b/doc/mgr/hello.rst
@@ -0,0 +1,39 @@
+Hello World Module
+==================
+
+This is a simple module skeleton for documentation purposes.
+
+Enabling
+--------
+
+The *hello* module is enabled with::
+
+  ceph mgr module enable hello
+
+To check that it is enabled, run::
+
+  ceph mgr module ls
+
+After editing the module file (found in ``src/pybind/mgr/hello/module.py``), you can see changes by running::
+
+  ceph mgr module disable hello
+  ceph mgr module enable hello
+
+or::
+
+  init-ceph restart mgr
+
+To execute the module, run::
+
+  ceph hello
+
+The log is found at::
+
+  build/out/mgr.x.log
+
+
+Documenting
+-----------
+
+After adding a new mgr module, be sure to add its documentation to ``doc/mgr/module_name.rst``.
+Also, add a link to your new module into ``doc/mgr/index.rst``.
diff --git a/doc/mgr/index.rst b/doc/mgr/index.rst
new file mode 100644
index 000000000..66fa19f86
--- /dev/null
+++ b/doc/mgr/index.rst
@@ -0,0 +1,50 @@
+.. _ceph-manager-daemon:
+
+===================
+Ceph Manager Daemon
+===================
+
+The :term:`Ceph Manager` daemon (ceph-mgr) runs alongside monitor daemons,
+to provide additional monitoring and interfaces to external monitoring
+and management systems.
+
+Since the 12.x (*luminous*) Ceph release, the ceph-mgr daemon is required for
+normal operations.  The ceph-mgr daemon is an optional component in
+the 11.x (*kraken*) Ceph release.
+
+By default, the manager daemon requires no additional configuration, beyond
+ensuring it is running.  If there is no mgr daemon running, you will
+see a health warning to that effect, and some of the other information
+in the output of `ceph status` will be missing or stale until a mgr is started.
+
+Use your normal deployment tools, such as ceph-ansible or cephadm, to
+set up ceph-mgr daemons on each of your mon nodes.  It is not mandatory
+to place mgr daemons on the same nodes as mons, but it is almost always
+sensible.
+
+.. toctree::
+    :maxdepth: 1
+
+    Installation and Configuration <administrator>
+    Writing modules <modules>
+    Writing orchestrator plugins <orchestrator_modules>
+    Dashboard module <dashboard>
+    Ceph RESTful API <ceph_api/index>
+    Alerts module <alerts>
+    DiskPrediction module <diskprediction>
+    Local pool module <localpool>
+    RESTful module <restful>
+    Zabbix module <zabbix>
+    Prometheus module <prometheus>
+    Influx module <influx>
+    Hello module <hello>
+    Telegraf module <telegraf>
+    Telemetry module <telemetry>
+    Iostat module <iostat>
+    Crash module <crash>
+    Insights module <insights>
+    Orchestrator module <orchestrator>
+    Rook module <rook>
+    MDS Autoscaler module <mds_autoscaler>
+    NFS module <nfs>
+    CLI API Commands module <cli_api>
diff --git a/doc/mgr/influx.rst b/doc/mgr/influx.rst
new file mode 100644
index 000000000..9a770530a
--- /dev/null
+++ b/doc/mgr/influx.rst
@@ -0,0 +1,165 @@
+=============
+Influx Module 
+=============
+
+The influx module continuously collects and sends time series data to an
+influxdb database.
+
+The influx module was introduced in the 13.x *Mimic* release.
+
+--------
+Enabling 
+--------
+
+To enable the module, use the following command:
+
+::
+
+    ceph mgr module enable influx
+
+If you wish to subsequently disable the module, you can use the equivalent
+*disable* command:
+
+::
+
+    ceph mgr module disable influx
+
+-------------
+Configuration 
+-------------
+
+For the influx module to send statistics to an InfluxDB server, it
+is necessary to configure the servers address and some authentication
+credentials.
+
+Set configuration values using the following command:
+
+::
+
+    ceph config set mgr mgr/influx/<key> <value>
+
+
+The most important settings are ``hostname``, ``username`` and ``password``.  
+For example, a typical configuration might look like this:
+
+::
+
+    ceph config set mgr mgr/influx/hostname influx.mydomain.com
+    ceph config set mgr mgr/influx/username admin123
+    ceph config set mgr mgr/influx/password p4ssw0rd
+    
+Additional optional configuration settings are:
+
+:interval: Time between reports to InfluxDB.  Default 30 seconds.
+:database: InfluxDB database name.  Default "ceph".  You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.  
+:port: InfluxDB server port.  Default 8086
+:ssl: Use https connection for InfluxDB server. Use "true" or "false". Default false
+:verify_ssl: Verify https cert for InfluxDB server. Use "true" or "false". Default true
+:threads: How many worker threads should be spawned for sending data to InfluxDB. Default is 5
+:batch_size: How big batches of data points should be when sending to InfluxDB. Default is 5000
+
+---------
+Debugging 
+---------
+
+By default, a few debugging statements as well as error statements have been set to print in the log files. Users can add more if necessary.
+To make use of the debugging option in the module:
+
+- Add this to the ceph.conf file.::
+
+    [mgr]
+        debug_mgr = 20  
+
+- Use this command ``ceph influx self-test``.
+- Check the log files. Users may find it easier to filter the log files using *mgr[influx]*.
+
+--------------------
+Interesting counters
+--------------------
+
+The following tables describe a subset of the values output by
+this module.
+
+^^^^^
+Pools
+^^^^^
+
++---------------+-----------------------------------------------------+
+|Counter        | Description                                         |
++===============+=====================================================+
+|stored         | Bytes stored in the pool not including copies       |
++---------------+-----------------------------------------------------+
+|max_avail      | Max available number of bytes in the pool           |
++---------------+-----------------------------------------------------+
+|objects        | Number of objects in the pool                       |
++---------------+-----------------------------------------------------+
+|wr_bytes       | Number of bytes written in the pool                 |
++---------------+-----------------------------------------------------+
+|dirty          | Number of bytes dirty in the pool                   |
++---------------+-----------------------------------------------------+
+|rd_bytes       | Number of bytes read in the pool                    |
++---------------+-----------------------------------------------------+
+|stored_raw     | Bytes used in pool including copies made            |
++---------------+-----------------------------------------------------+
+
+^^^^
+OSDs
+^^^^
+
++------------+------------------------------------+
+|Counter     | Description                        |
++============+====================================+
+|op_w        | Client write operations            |
++------------+------------------------------------+
+|op_in_bytes | Client operations total write size |
++------------+------------------------------------+
+|op_r        | Client read operations             |
++------------+------------------------------------+
+|op_out_bytes| Client operations total read size  |
++------------+------------------------------------+
+
+
++------------------------+--------------------------------------------------------------------------+
+|Counter                 | Description                                                              |
++========================+==========================================================================+
+|op_wip                  | Replication operations currently being processed (primary)               |
++------------------------+--------------------------------------------------------------------------+
+|op_latency              | Latency of client operations (including queue time)                      |
++------------------------+--------------------------------------------------------------------------+
+|op_process_latency      | Latency of client operations (excluding queue time)                      |           
++------------------------+--------------------------------------------------------------------------+
+|op_prepare_latency      | Latency of client operations (excluding queue time and wait for finished)|
++------------------------+--------------------------------------------------------------------------+
+|op_r_latency            | Latency of read operation (including queue time)                         |
++------------------------+--------------------------------------------------------------------------+
+|op_r_process_latency    | Latency of read operation (excluding queue time)                         |
++------------------------+--------------------------------------------------------------------------+
+|op_w_in_bytes           | Client data written                                                      |
++------------------------+--------------------------------------------------------------------------+
+|op_w_latency            | Latency of write operation (including queue time)                        |
++------------------------+--------------------------------------------------------------------------+
+|op_w_process_latency    | Latency of write operation (excluding queue time)                        |
++------------------------+--------------------------------------------------------------------------+
+|op_w_prepare_latency    | Latency of write operations (excluding queue time and wait for finished) |
++------------------------+--------------------------------------------------------------------------+
+|op_rw                   | Client read-modify-write operations                                      |
++------------------------+--------------------------------------------------------------------------+
+|op_rw_in_bytes          | Client read-modify-write operations write in                             |
++------------------------+--------------------------------------------------------------------------+
+|op_rw_out_bytes         | Client read-modify-write operations read out                             |
++------------------------+--------------------------------------------------------------------------+
+|op_rw_latency           | Latency of read-modify-write operation (including queue time)            |
++------------------------+--------------------------------------------------------------------------+
+|op_rw_process_latency   | Latency of read-modify-write operation (excluding queue time)            |
++------------------------+--------------------------------------------------------------------------+
+|op_rw_prepare_latency   | Latency of read-modify-write operations (excluding queue time            |
+|                        | and wait for finished)                                                   |
++------------------------+--------------------------------------------------------------------------+
+|op_before_queue_op_lat  | Latency of IO before calling queue (before really queue into ShardedOpWq)|
+|                        | op_before_dequeue_op_lat                                                 |
++------------------------+--------------------------------------------------------------------------+
+|op_before_dequeue_op_lat| Latency of IO before calling dequeue_op(already dequeued and get PG lock)|
++------------------------+--------------------------------------------------------------------------+
+
+Latency counters are measured in microseconds unless otherwise specified in the description.
+
diff --git a/doc/mgr/insights.rst b/doc/mgr/insights.rst
new file mode 100644
index 000000000..37b8903f1
--- /dev/null
+++ b/doc/mgr/insights.rst
@@ -0,0 +1,52 @@
+Insights Module
+===============
+
+The insights module collects and exposes system information to the Insights Core
+data analysis framework. It is intended to replace explicit interrogation of
+Ceph CLIs and daemon admin sockets, reducing the API surface that Insights
+depends on. The insights reports contains the following:
+
+* **Health reports**. In addition to reporting the current health of the
+  cluster, the insights module reports a summary of the last 24 hours of health
+  checks. This feature is important for catching cluster health issues that are
+  transient and may not be present at the moment the report is generated. Health
+  checks are deduplicated to avoid unbounded data growth.
+
+* **Crash reports**. A summary of any daemon crashes in the past 24 hours is
+  included in the insights report. Crashes are reported as the number of crashes
+  per daemon type (e.g. `ceph-osd`) within the time window. Full details of a
+  crash may be obtained using the `crash module`_.
+
+* Software version, storage utilization, cluster maps, placement group summary,
+  monitor status, cluster configuration, and OSD metadata.
+
+Enabling
+--------
+
+The *insights* module is enabled with::
+
+  ceph mgr module enable insights
+
+Commands
+--------
+::
+
+  ceph insights
+
+Generate the full report.
+
+::
+
+  ceph insights prune-health <hours>
+
+Remove historical health data older than <hours>. Passing `0` for <hours> will
+clear all health data.
+
+This command is useful for cleaning the health history before automated nightly
+reports are generated, which may contain spurious health checks accumulated
+while performing system maintenance, or other health checks that have been
+resolved. There is no need to prune health data to reclaim storage space;
+garbage collection is performed regularly to remove old health data from
+persistent storage.
+
+.. _crash module: ../crash
diff --git a/doc/mgr/iostat.rst b/doc/mgr/iostat.rst
new file mode 100644
index 000000000..f9f849383
--- /dev/null
+++ b/doc/mgr/iostat.rst
@@ -0,0 +1,32 @@
+.. _mgr-iostat-overview:
+
+iostat
+======
+
+This module shows the current throughput and IOPS done on the Ceph cluster.
+
+Enabling
+--------
+
+To check if the *iostat* module is enabled, run::
+
+  ceph mgr module ls
+
+The module can be enabled with::
+
+  ceph mgr module enable iostat
+
+To execute the module, run::
+
+  ceph iostat
+
+To change the frequency at which the statistics are printed, use the ``-p``
+option::
+
+  ceph iostat -p <period in seconds>
+
+For example, use the following command to print the statistics every 5 seconds::
+
+  ceph iostat -p 5
+
+To stop the module, press Ctrl-C.
diff --git a/doc/mgr/localpool.rst b/doc/mgr/localpool.rst
new file mode 100644
index 000000000..fe8bd3942
--- /dev/null
+++ b/doc/mgr/localpool.rst
@@ -0,0 +1,37 @@
+Local Pool Module
+=================
+
+The *localpool* module can automatically create RADOS pools that are
+localized to a subset of the overall cluster.  For example, by default, it will
+create a pool for each distinct ``rack`` in the cluster.  This can be useful for
+deployments where it is desirable to distribute some data locally and other data
+globally across the cluster.  One use-case is measuring performance and testing
+behavior of specific drive, NIC, or chassis models in isolation.
+
+Enabling
+--------
+
+The *localpool* module is enabled with::
+
+  ceph mgr module enable localpool
+
+Configuring
+-----------
+
+The *localpool* module understands the following options:
+
+* **subtree** (default: `rack`): which CRUSH subtree type the module
+  should create a pool for.
+* **failure_domain** (default: `host`): what failure domain we should
+  separate data replicas across.
+* **pg_num** (default: `128`): number of PGs to create for each pool
+* **num_rep** (default: `3`): number of replicas for each pool.
+  (Currently, pools are always replicated.)
+* **min_size** (default: none): value to set min_size to (unchanged from Ceph's default if this option is not set)
+* **prefix** (default: `by-$subtreetype-`): prefix for the pool name.
+
+These options are set via the config-key interface.  For example, to
+change the replication level to 2x with only 64 PGs, ::
+
+  ceph config set mgr mgr/localpool/num_rep 2
+  ceph config set mgr mgr/localpool/pg_num 64
diff --git a/doc/mgr/mds_autoscaler.rst b/doc/mgr/mds_autoscaler.rst
new file mode 100644
index 000000000..46fc44155
--- /dev/null
+++ b/doc/mgr/mds_autoscaler.rst
@@ -0,0 +1,23 @@
+MDS Autoscaler Module
+=====================
+
+The MDS Autoscaler Module monitors file systems to ensure sufficient MDS
+daemons are available. It works by adjusting the placement specification for
+the orchestrator backend of the MDS service. To enable, use:
+
+.. sh:
+
+   ceph mgr module enable mds_autoscaler
+
+The module will monitor the following file system settings to inform
+placement count adjustments:
+
+- ``max_mds`` file system setting
+- ``standby_count_wanted`` file system setting
+
+The Ceph monitor daemons are still responsible for promoting or stopping MDS
+according to these settings. The ``mds_autoscaler`` simply adjusts the
+number of MDS which are spawned by the orchestrator.
+
+.. note: There is no CLI or module configurations as of now. Enable or disable
+   the module to turn on or off.
diff --git a/doc/mgr/modules.rst b/doc/mgr/modules.rst
new file mode 100644
index 000000000..8979b4e6a
--- /dev/null
+++ b/doc/mgr/modules.rst
@@ -0,0 +1,476 @@
+
+
+.. _mgr-module-dev:
+
+ceph-mgr module developer's guide
+=================================
+
+.. warning::
+
+    This is developer documentation, describing Ceph internals that
+    are only relevant to people writing ceph-mgr modules.
+
+Creating a module
+-----------------
+
+In pybind/mgr/, create a python module.  Within your module, create a class
+that inherits from ``MgrModule``.  For ceph-mgr to detect your module, your
+directory must contain a file called `module.py`.
+
+The most important methods to override are:
+
+* a ``serve`` member function for server-type modules.  This
+  function should block forever.
+* a ``notify`` member function if your module needs to
+  take action when new cluster data is available.
+* a ``handle_command`` member function if your module
+  exposes CLI commands.
+
+Some modules interface with external orchestrators to deploy
+Ceph services.  These also inherit from ``Orchestrator``, which adds
+additional methods to the base ``MgrModule`` class.  See
+:ref:`Orchestrator modules <orchestrator-modules>` for more on
+creating these modules.
+
+Installing a module
+-------------------
+
+Once your module is present in the location set by the
+``mgr module path`` configuration setting, you can enable it
+via the ``ceph mgr module enable`` command::
+
+  ceph mgr module enable mymodule
+
+Note that the MgrModule interface is not stable, so any modules maintained
+outside of the Ceph tree are liable to break when run against any newer
+or older versions of Ceph.
+
+Logging
+-------
+
+Logging in Ceph manager modules is done as in any other Python program. Just
+import the ``logging`` package and get a logger instance with the
+``logging.getLogger`` function.
+
+Each module has a ``log_level`` option that specifies the current Python
+logging level of the module.
+To change or query the logging level of the module use the following Ceph
+commands::
+
+  ceph config get mgr mgr/<module_name>/log_level
+  ceph config set mgr mgr/<module_name>/log_level <info|debug|critical|error|warning|>
+
+The logging level used upon the module's start is determined by the current
+logging level of the mgr daemon, unless if the ``log_level`` option was
+previously set with the ``config set ...`` command. The mgr daemon logging
+level is mapped to the module python logging level as follows:
+
+* <= 0 is CRITICAL
+* <= 1 is WARNING
+* <= 4 is INFO
+* <= +inf is DEBUG
+
+We can unset the module log level and fallback to the mgr daemon logging level
+by running the following command::
+
+  ceph config set mgr mgr/<module_name>/log_level ''
+
+By default, modules' logging messages are processed by the Ceph logging layer
+where they will be recorded in the mgr daemon's log file.
+But it's also possible to send a module's logging message to it's own file.
+
+The module's log file will be located in the same directory as the mgr daemon's
+log file with the following name pattern::
+
+   <mgr_daemon_log_file_name>.<module_name>.log
+
+To enable the file logging on a module use the following command::
+
+   ceph config set mgr mgr/<module_name>/log_to_file true
+
+When the module's file logging is enabled, module's logging messages stop
+being written to the mgr daemon's log file and are only written to the
+module's log file.
+
+It's also possible to check the status and disable the file logging with the
+following commands::
+
+   ceph config get mgr mgr/<module_name>/log_to_file
+   ceph config set mgr mgr/<module_name>/log_to_file false
+
+
+
+
+Exposing commands
+-----------------
+
+There are two approaches for exposing a command. The first one is to
+use the ``@CLICommand`` decorator to decorate the method which handles
+the command. like this
+
+.. code:: python
+
+   @CLICommand('antigravity send to blackhole',
+               perm='rw')
+   def send_to_blackhole(self, oid: str, blackhole: Optional[str] = None, inbuf: Optional[str] = None):
+       '''
+       Send the specified object to black hole
+       '''
+       obj = self.find_object(oid)
+       if obj is None:
+           return HandleCommandResult(-errno.ENOENT, stderr=f"object '{oid}' not found")
+       if blackhole is not None and inbuf is not None:
+           try:
+               location = self.decrypt(blackhole, passphrase=inbuf)
+           except ValueError:
+               return HandleCommandResult(-errno.EINVAL, stderr='unable to decrypt location')
+       else:
+           location = blackhole
+       self.send_object_to(obj, location)
+       return HandleCommandResult(stdout=f'the black hole swallowed '{oid}'")
+
+The first parameter passed to ``CLICommand`` is the "name" of the command.
+Since there are lots of commands in Ceph, we tend to group related commands
+with a common prefix. In this case, "antigravity" is used for this purpose.
+As the author is probably designing a module which is also able to launch
+rockets into the deep space.
+
+The `type annotations <https://www.python.org/dev/peps/pep-0484/>`_ for the
+method parameters are mandatory here, so the usage of the command can be
+properly reported to the ``ceph`` CLI, and the manager daemon can convert
+the serialized command parameters sent by the clients to the expected type
+before passing them to the handler method. With properly implemented types,
+one can also perform some sanity checks against the parameters!
+
+The names of the parameters are part of the command interface, so please
+try to take the backward compatibility into consideration when changing
+them. But you **cannot** change name of ``inbuf`` parameter, it is used
+to pass the content of the file specified by ``ceph --in-file`` option.
+
+The docstring of the method is used for the description of the command.
+
+The manager daemon cooks the usage of the command from these ingredients,
+like::
+
+  antigravity send to blackhole <oid> [<blackhole>]  Send the specified object to black hole
+
+as part of the output of ``ceph --help``.
+
+In addition to ``@CLICommand``, you could also use ``@CLIReadCommand`` or
+``@CLIWriteCommand`` if your command only requires read permissions or
+write permissions respectively.
+
+The second one is to set the ``COMMANDS`` class attribute of your module to
+a list of dicts like this::
+
+    COMMANDS = [
+        {
+            "cmd": "foobar name=myarg,type=CephString",
+            "desc": "Do something awesome",
+            "perm": "rw",
+            # optional:
+            "poll": "true"
+        }
+    ]
+
+The ``cmd`` part of each entry is parsed in the same way as internal
+Ceph mon and admin socket commands (see mon/MonCommands.h in
+the Ceph source for examples). Note that the "poll" field is optional,
+and is set to False by default; this indicates to the ``ceph`` CLI
+that it should call this command repeatedly and output results (see
+``ceph -h`` and its ``--period`` option).
+
+Each command is expected to return a tuple ``(retval, stdout, stderr)``.
+``retval`` is an integer representing a libc error code (e.g. EINVAL,
+EPERM, or 0 for no error), ``stdout`` is a string containing any
+non-error output, and ``stderr`` is a string containing any progress or
+error explanation output.  Either or both of the two strings may be empty.
+
+Implement the ``handle_command`` function to respond to the commands
+when they are sent:
+
+
+.. py:currentmodule:: mgr_module
+.. automethod:: MgrModule.handle_command
+
+Configuration options
+---------------------
+
+Modules can load and store configuration options using the
+``set_module_option`` and ``get_module_option`` methods.
+
+.. note:: Use ``set_module_option`` and ``get_module_option`` to
+   manage user-visible configuration options that are not blobs (like
+   certificates). If you want to persist module-internal data or
+   binary configuration data consider using the `KV store`_.
+
+You must declare your available configuration options in the
+``MODULE_OPTIONS`` class attribute, like this:
+
+::
+
+    MODULE_OPTIONS = [
+        {
+            "name": "my_option"
+        }
+    ]
+
+If you try to use set_module_option or get_module_option on options not declared
+in ``MODULE_OPTIONS``, an exception will be raised.
+
+You may choose to provide setter commands in your module to perform
+high level validation.  Users can also modify configuration using
+the normal `ceph config set` command, where the configuration options
+for a mgr module are named like `mgr/<module name>/<option>`.
+
+If a configuration option is different depending on which node the mgr
+is running on, then use *localized* configuration (
+``get_localized_module_option``, ``set_localized_module_option``).
+This may be necessary for options such as what address to listen on.
+Localized options may also be set externally with ``ceph config set``,
+where they key name is like ``mgr/<module name>/<mgr id>/<option>``
+
+If you need to load and store data (e.g. something larger, binary, or multiline),
+use the KV store instead of configuration options (see next section).
+
+Hints for using config options:
+
+* Reads are fast: ceph-mgr keeps a local in-memory copy, so in many cases
+  you can just do a get_module_option every time you use a option, rather than
+  copying it out into a variable.
+* Writes block until the value is persisted (i.e. round trip to the monitor),
+  but reads from another thread will see the new value immediately.
+* If a user has used `config set` from the command line, then the new
+  value will become visible to `get_module_option` immediately, although the
+  mon->mgr update is asynchronous, so `config set` will return a fraction
+  of a second before the new value is visible on the mgr.
+* To delete a config value (i.e. revert to default), just pass ``None`` to
+  set_module_option.
+
+.. automethod:: MgrModule.get_module_option
+.. automethod:: MgrModule.set_module_option
+.. automethod:: MgrModule.get_localized_module_option
+.. automethod:: MgrModule.set_localized_module_option
+
+KV store
+--------
+
+Modules have access to a private (per-module) key value store, which
+is implemented using the monitor's "config-key" commands.  Use
+the ``set_store`` and ``get_store`` methods to access the KV store from
+your module.
+
+The KV store commands work in a similar way to the configuration
+commands.  Reads are fast, operating from a local cache.  Writes block
+on persistence and do a round trip to the monitor.
+
+This data can be access from outside of ceph-mgr using the
+``ceph config-key [get|set]`` commands.  Key names follow the same
+conventions as configuration options.  Note that any values updated
+from outside of ceph-mgr will not be seen by running modules until
+the next restart.  Users should be discouraged from accessing module KV
+data externally -- if it is necessary for users to populate data, modules
+should provide special commands to set the data via the module.
+
+Use the ``get_store_prefix`` function to enumerate keys within
+a particular prefix (i.e. all keys starting with a particular substring).
+
+
+.. automethod:: MgrModule.get_store
+.. automethod:: MgrModule.set_store
+.. automethod:: MgrModule.get_localized_store
+.. automethod:: MgrModule.set_localized_store
+.. automethod:: MgrModule.get_store_prefix
+
+
+Accessing cluster data
+----------------------
+
+Modules have access to the in-memory copies of the Ceph cluster's
+state that the mgr maintains.  Accessor functions as exposed
+as members of MgrModule.
+
+Calls that access the cluster or daemon state are generally going
+from Python into native C++ routines.  There is some overhead to this,
+but much less than for example calling into a REST API or calling into
+an SQL database.
+
+There are no consistency rules about access to cluster structures or
+daemon metadata.  For example, an OSD might exist in OSDMap but
+have no metadata, or vice versa.  On a healthy cluster these
+will be very rare transient states, but modules should be written
+to cope with the possibility.
+
+Note that these accessors must not be called in the modules ``__init__``
+function. This will result in a circular locking exception.
+
+.. automethod:: MgrModule.get
+.. automethod:: MgrModule.get_server
+.. automethod:: MgrModule.list_servers
+.. automethod:: MgrModule.get_metadata
+.. automethod:: MgrModule.get_daemon_status
+.. automethod:: MgrModule.get_perf_schema
+.. automethod:: MgrModule.get_counter
+.. automethod:: MgrModule.get_mgr_id
+
+Exposing health checks
+----------------------
+
+Modules can raise first class Ceph health checks, which will be reported
+in the output of ``ceph status`` and in other places that report on the
+cluster's health.
+
+If you use ``set_health_checks`` to report a problem, be sure to call
+it again with an empty dict to clear your health check when the problem
+goes away.
+
+.. automethod:: MgrModule.set_health_checks
+
+What if the mons are down?
+--------------------------
+
+The manager daemon gets much of its state (such as the cluster maps)
+from the monitor.  If the monitor cluster is inaccessible, whichever
+manager was active will continue to run, with the latest state it saw
+still in memory.
+
+However, if you are creating a module that shows the cluster state
+to the user then you may well not want to mislead them by showing
+them that out of date state.
+
+To check if the manager daemon currently has a connection to
+the monitor cluster, use this function:
+
+.. automethod:: MgrModule.have_mon_connection
+
+Reporting if your module cannot run
+-----------------------------------
+
+If your module cannot be run for any reason (such as a missing dependency),
+then you can report that by implementing the ``can_run`` function.
+
+.. automethod:: MgrModule.can_run
+
+Note that this will only work properly if your module can always be imported:
+if you are importing a dependency that may be absent, then do it in a
+try/except block so that your module can be loaded far enough to use
+``can_run`` even if the dependency is absent.
+
+Sending commands
+----------------
+
+A non-blocking facility is provided for sending monitor commands
+to the cluster.
+
+.. automethod:: MgrModule.send_command
+
+Receiving notifications
+-----------------------
+
+The manager daemon calls the ``notify`` function on all active modules
+when certain important pieces of cluster state are updated, such as the
+cluster maps.
+
+The actual data is not passed into this function, rather it is a cue for
+the module to go and read the relevant structure if it is interested.  Most
+modules ignore most types of notification: to ignore a notification
+simply return from this function without doing anything.
+
+.. automethod:: MgrModule.notify
+
+Accessing RADOS or CephFS
+-------------------------
+
+If you want to use the librados python API to access data stored in
+the Ceph cluster, you can access the ``rados`` attribute of your
+``MgrModule`` instance.  This is an instance of ``rados.Rados`` which
+has been constructed for you using the existing Ceph context (an internal
+detail of the C++ Ceph code) of the mgr daemon.
+
+Always use this specially constructed librados instance instead of
+constructing one by hand.
+
+Similarly, if you are using libcephfs to access the file system, then
+use the libcephfs ``create_with_rados`` to construct it from the
+``MgrModule.rados`` librados instance, and thereby inherit the correct context.
+
+Remember that your module may be running while other parts of the cluster
+are down: do not assume that librados or libcephfs calls will return
+promptly -- consider whether to use timeouts or to block if the rest of
+the cluster is not fully available.
+
+Implementing standby mode
+-------------------------
+
+For some modules, it is useful to run on standby manager daemons as well
+as on the active daemon.  For example, an HTTP server can usefully
+serve HTTP redirect responses from the standby managers so that
+the user can point his browser at any of the manager daemons without
+having to worry about which one is active.
+
+Standby manager daemons look for a subclass of ``StandbyModule``
+in each module.  If the class is not found then the module is not
+used at all on standby daemons.  If the class is found, then
+its ``serve`` method is called.  Implementations of ``StandbyModule``
+must inherit from ``mgr_module.MgrStandbyModule``.
+
+The interface of ``MgrStandbyModule`` is much restricted compared to
+``MgrModule`` -- none of the Ceph cluster state is available to
+the module.  ``serve`` and ``shutdown`` methods are used in the same
+way as a normal module class.  The ``get_active_uri`` method enables
+the standby module to discover the address of its active peer in
+order to make redirects.  See the ``MgrStandbyModule`` definition
+in the Ceph source code for the full list of methods.
+
+For an example of how to use this interface, look at the source code
+of the ``dashboard`` module.
+
+Communicating between modules
+-----------------------------
+
+Modules can invoke member functions of other modules.
+
+.. automethod:: MgrModule.remote
+
+Be sure to handle ``ImportError`` to deal with the case that the desired
+module is not enabled.
+
+If the remote method raises a python exception, this will be converted
+to a RuntimeError on the calling side, where the message string describes
+the exception that was originally thrown.  If your logic intends
+to handle certain errors cleanly, it is better to modify the remote method
+to return an error value instead of raising an exception.
+
+At time of writing, inter-module calls are implemented without
+copies or serialization, so when you return a python object, you're 
+returning a reference to that object to the calling module.  It
+is recommend *not* to rely on this reference passing, as in future the
+implementation may change to serialize arguments and return
+values.
+
+
+Shutting down cleanly
+---------------------
+
+If a module implements the ``serve()`` method, it should also implement
+the ``shutdown()`` method to shutdown cleanly: misbehaving modules
+may otherwise prevent clean shutdown of ceph-mgr.
+
+Limitations
+-----------
+
+It is not possible to call back into C++ code from a module's
+``__init__()`` method.  For example calling ``self.get_module_option()`` at
+this point will result in an assertion failure in ceph-mgr.  For modules
+that implement the ``serve()`` method, it usually makes sense to do most
+initialization inside that method instead.
+
+Is something missing?
+---------------------
+
+The ceph-mgr python interface is not set in stone.  If you have a need
+that is not satisfied by the current interface, please bring it up
+on the ceph-devel mailing list.  While it is desired to avoid bloating
+the interface, it is not generally very hard to expose existing data
+to the Python code when there is a good reason.
+
diff --git a/doc/mgr/nfs.rst b/doc/mgr/nfs.rst
new file mode 100644
index 000000000..777e2ee81
--- /dev/null
+++ b/doc/mgr/nfs.rst
@@ -0,0 +1,612 @@
+.. _mgr-nfs:
+
+=============================
+CephFS & RGW Exports over NFS
+=============================
+
+CephFS namespaces and RGW buckets can be exported over NFS protocol
+using the `NFS-Ganesha NFS server`_.
+
+The ``nfs`` manager module provides a general interface for managing
+NFS exports of either CephFS directories or RGW buckets.  Exports can
+be managed either via the CLI ``ceph nfs export ...`` commands
+or via the dashboard.
+
+The deployment of the nfs-ganesha daemons can also be managed
+automatically if either the :ref:`cephadm` or :ref:`mgr-rook`
+orchestrators are enabled.  If neither are in use (e.g., Ceph is
+deployed via an external orchestrator like Ansible or Puppet), the
+nfs-ganesha daemons must be manually deployed; for more information,
+see :ref:`nfs-ganesha-config`.
+
+.. note:: Starting with Ceph Pacific, the ``nfs`` mgr module must be enabled.
+
+NFS Cluster management
+======================
+
+Create NFS Ganesha Cluster
+--------------------------
+
+.. code:: bash
+
+    $ ceph nfs cluster create <cluster_id> [<placement>] [--port <port>] [--ingress --virtual-ip <ip>]
+
+This creates a common recovery pool for all NFS Ganesha daemons, new user based on
+``cluster_id``, and a common NFS Ganesha config RADOS object.
+
+.. note:: Since this command also brings up NFS Ganesha daemons using a ceph-mgr
+   orchestrator module (see :doc:`/mgr/orchestrator`) such as cephadm or rook, at
+   least one such module must be enabled for it to work.
+
+   Currently, NFS Ganesha daemon deployed by cephadm listens on the standard
+   port. So only one daemon will be deployed on a host.
+
+``<cluster_id>`` is an arbitrary string by which this NFS Ganesha cluster will be
+known (e.g., ``mynfs``).
+
+``<placement>`` is an optional string signifying which hosts should have NFS Ganesha
+daemon containers running on them and, optionally, the total number of NFS
+Ganesha daemons on the cluster (should you want to have more than one NFS Ganesha
+daemon running per node). For example, the following placement string means
+"deploy NFS Ganesha daemons on nodes host1 and host2 (one daemon per host)::
+
+    "host1,host2"
+
+and this placement specification says to deploy single NFS Ganesha daemon each
+on nodes host1 and host2 (for a total of two NFS Ganesha daemons in the
+cluster)::
+
+    "2 host1,host2"
+
+NFS can be deployed on a port other than 2049 (the default) with ``--port <port>``.
+
+To deploy NFS with a high-availability front-end (virtual IP and load balancer), add the
+``--ingress`` flag and specify a virtual IP address. This will deploy a combination
+of keepalived and haproxy to provide an high-availability NFS frontend for the NFS
+service.
+
+.. note:: The ingress implementation is not yet complete.  Enabling
+	  ingress will deploy multiple ganesha instances and balance
+	  load across them, but a host failure will not immediately
+	  cause cephadm to deploy a replacement daemon before the NFS
+	  grace period expires.  This high-availability functionality
+	  is expected to be completed by the Quincy release (March
+	  2022).
+
+For more details, refer :ref:`orchestrator-cli-placement-spec` but keep
+in mind that specifying the placement via a YAML file is not supported.
+
+Ingress
+-------
+
+The core *nfs* service will deploy one or more nfs-ganesha daemons,
+each of which will provide a working NFS endpoint.  The IP for each
+NFS endpoint will depend on which host the nfs-ganesha daemons are
+deployed.  By default, daemons are placed semi-randomly, but users can
+also explicitly control where daemons are placed; see
+:ref:`orchestrator-cli-placement-spec`.
+
+When a cluster is created with ``--ingress``, an *ingress* service is
+additionally deployed to provide load balancing and high-availability
+for the NFS servers.  A virtual IP is used to provide a known, stable
+NFS endpoint that all clients can use to mount.  Ceph will take care
+of the details of NFS redirecting traffic on the virtual IP to the
+appropriate backend NFS servers, and redeploying NFS servers when they
+fail.
+
+Enabling ingress via the ``ceph nfs cluster create`` command deploys a
+simple ingress configuration with the most common configuration
+options.  Ingress can also be added to an existing NFS service (e.g.,
+one created without the ``--ingress`` flag), and the basic NFS service can
+also be modified after the fact to include non-default options, by modifying
+the services directly.  For more information, see :ref:`cephadm-ha-nfs`.
+
+Show NFS Cluster IP(s)
+----------------------
+
+To examine an NFS cluster's IP endpoints, including the IPs for the individual NFS
+daemons, and the virtual IP (if any) for the ingress service,
+
+.. code:: bash
+
+    $ ceph nfs cluster info [<cluster_id>]
+
+.. note:: This will not work with the rook backend. Instead, expose the port with
+   the kubectl patch command and fetch the port details with kubectl get services
+   command::
+
+    $ kubectl patch service -n rook-ceph -p '{"spec":{"type": "NodePort"}}' rook-ceph-nfs-<cluster-name>-<node-id>
+    $ kubectl get services -n rook-ceph rook-ceph-nfs-<cluster-name>-<node-id>
+
+
+Delete NFS Ganesha Cluster
+--------------------------
+
+.. code:: bash
+
+    $ ceph nfs cluster rm <cluster_id>
+
+This deletes the deployed cluster.
+
+Updating an NFS Cluster
+-----------------------
+
+In order to modify cluster parameters (like the port or placement), you need to
+use the orchestrator interface to update the NFS service spec.  The safest way to do
+that is to export the current spec, modify it, and then re-apply it.  For example,
+to modify the ``nfs.foo`` service,
+
+.. code:: bash
+
+    $ ceph orch ls --service-name nfs.foo --export > nfs.foo.yaml
+    $ vi nfs.foo.yaml
+    $ ceph orch apply -i nfs.foo.yaml
+
+For more information about the NFS service spec, see :ref:`deploy-cephadm-nfs-ganesha`.
+
+List NFS Ganesha Clusters
+-------------------------
+
+.. code:: bash
+
+    $ ceph nfs cluster ls
+
+This lists deployed clusters.
+
+.. _nfs-cluster-set:
+
+Set Customized NFS Ganesha Configuration
+----------------------------------------
+
+.. code:: bash
+
+    $ ceph nfs cluster config set <cluster_id> -i <config_file>
+
+With this the nfs cluster will use the specified config and it will have
+precedence over default config blocks.
+
+Example use cases include:
+
+#. Changing log level.  The logging level can be adjusted with the following config
+   fragment::
+
+     LOG {
+         COMPONENTS {
+             ALL = FULL_DEBUG;
+         }
+     }
+
+#. Adding custom export block.
+
+   The following sample block creates a single export. This export will not be
+   managed by `ceph nfs export` interface::
+
+    EXPORT {
+      Export_Id = 100;
+      Transports = TCP;
+      Path = /;
+      Pseudo = /ceph/;
+      Protocols = 4;
+      Access_Type = RW;
+      Attr_Expiration_Time = 0;
+      Squash = None;
+      FSAL {
+        Name = CEPH;
+        Filesystem = "filesystem name";
+        User_Id = "user id";
+        Secret_Access_Key = "secret key";
+      }
+    }
+
+.. note:: User specified in FSAL block should have proper caps for NFS-Ganesha
+   daemons to access ceph cluster. User can be created in following way using
+   `auth get-or-create`::
+
+         # ceph auth get-or-create client.<user_id> mon 'allow r' osd 'allow rw pool=.nfs namespace=<nfs_cluster_name>, allow rw tag cephfs data=<fs_name>' mds 'allow rw path=<export_path>'
+
+View Customized NFS Ganesha Configuration
+-----------------------------------------
+
+.. code:: bash
+
+    $ ceph nfs cluster config get <cluster_id>
+
+This will output the user defined configuration (if any).
+
+Reset NFS Ganesha Configuration
+-------------------------------
+
+.. code:: bash
+
+    $ ceph nfs cluster config reset <cluster_id>
+
+This removes the user defined configuration.
+
+.. note:: With a rook deployment, ganesha pods must be explicitly restarted
+   for the new config blocks to be effective.
+
+
+Export Management
+=================
+
+.. warning:: Currently, the nfs interface is not integrated with dashboard. Both
+   dashboard and nfs interface have different export requirements and
+   create exports differently. Management of dashboard created exports is not
+   supported.
+
+Create CephFS Export
+--------------------
+
+.. code:: bash
+
+    $ ceph nfs export create cephfs --cluster-id <cluster_id> --pseudo-path <pseudo_path> --fsname <fsname> [--readonly] [--path=/path/in/cephfs] [--client_addr <value>...] [--squash <value>]
+
+This creates export RADOS objects containing the export block, where
+
+``<cluster_id>`` is the NFS Ganesha cluster ID.
+
+``<pseudo_path>`` is the export position within the NFS v4 Pseudo Filesystem where the export will be available on the server. It must be an absolute path and unique.
+
+``<fsname>`` is the name of the FS volume used by the NFS Ganesha cluster
+that will serve this export.
+
+``<path>`` is the path within cephfs. Valid path should be given and default
+path is '/'. It need not be unique. Subvolume path can be fetched using:
+
+.. code::
+
+   $ ceph fs subvolume getpath <vol_name> <subvol_name> [--group_name <subvol_group_name>]
+
+``<client_addr>`` is the list of client address for which these export
+permissions will be applicable. By default all clients can access the export
+according to specified export permissions. See the `NFS-Ganesha Export Sample`_
+for permissible values.
+
+``<squash>`` defines the kind of user id squashing to be performed. The default
+value is `no_root_squash`. See the `NFS-Ganesha Export Sample`_ for
+permissible values.
+
+.. note:: Export creation is supported only for NFS Ganesha clusters deployed using nfs interface.
+
+Create RGW Export
+-----------------
+
+There are two kinds of RGW exports:
+
+- a *user* export will export all buckets owned by an
+  RGW user, where the top-level directory of the export is a list of buckets.
+- a *bucket* export will export a single bucket, where the top-level directory contains
+  the objects in the bucket.
+
+RGW bucket export
+^^^^^^^^^^^^^^^^^
+  
+To export a *bucket*:
+
+.. code::
+
+   $ ceph nfs export create rgw --cluster-id <cluster_id> --pseudo-path <pseudo_path> --bucket <bucket_name> [--user-id <user-id>] [--readonly] [--client_addr <value>...] [--squash <value>]
+
+For example, to export *mybucket* via NFS cluster *mynfs* at the pseudo-path */bucketdata* to any host in the ``192.168.10.0/24`` network
+
+.. code::
+
+   $ ceph nfs export create rgw --cluster-id mynfs --pseudo-path /bucketdata --bucket mybucket --client_addr 192.168.10.0/24
+
+.. note:: Export creation is supported only for NFS Ganesha clusters deployed using nfs interface.
+
+``<cluster_id>`` is the NFS Ganesha cluster ID.
+
+``<pseudo_path>`` is the export position within the NFS v4 Pseudo Filesystem where the export will be available on the server. It must be an absolute path and unique.
+
+``<bucket_name>`` is the name of the bucket that will be exported.
+
+``<user_id>`` is optional, and specifies which RGW user will be used for read and write
+operations to the bucket.  If it is not specified, the user who owns the bucket will be
+used.
+
+.. note:: Currently, if multi-site RGW is enabled, Ceph can only export RGW buckets in the default realm.
+
+``<client_addr>`` is the list of client address for which these export
+permissions will be applicable. By default all clients can access the export
+according to specified export permissions. See the `NFS-Ganesha Export Sample`_
+for permissible values.
+
+``<squash>`` defines the kind of user id squashing to be performed. The default
+value is `no_root_squash`. See the `NFS-Ganesha Export Sample`_ for
+permissible values.
+
+RGW user export
+^^^^^^^^^^^^^^^
+
+To export an RGW *user*:
+
+.. code::
+
+   $ ceph nfs export create rgw --cluster-id <cluster_id> --pseudo-path <pseudo_path> --user-id <user-id> [--readonly] [--client_addr <value>...] [--squash <value>]
+
+For example, to export *myuser* via NFS cluster *mynfs* at the pseudo-path */myuser* to any host in the ``192.168.10.0/24`` network
+
+.. code::
+
+   $ ceph nfs export create rgw --cluster-id mynfs --pseudo-path /bucketdata --user-id myuser --client_addr 192.168.10.0/24
+
+
+Delete Export
+-------------
+
+.. code:: bash
+
+    $ ceph nfs export rm <cluster_id> <pseudo_path>
+
+This deletes an export in an NFS Ganesha cluster, where:
+
+``<cluster_id>`` is the NFS Ganesha cluster ID.
+
+``<pseudo_path>`` is the pseudo root path (must be an absolute path).
+
+List Exports
+------------
+
+.. code:: bash
+
+    $ ceph nfs export ls <cluster_id> [--detailed]
+
+It lists exports for a cluster, where:
+
+``<cluster_id>`` is the NFS Ganesha cluster ID.
+
+With the ``--detailed`` option enabled it shows entire export block.
+
+Get Export
+----------
+
+.. code:: bash
+
+    $ ceph nfs export info <cluster_id> <pseudo_path>
+
+This displays export block for a cluster based on pseudo root name,
+where:
+
+``<cluster_id>`` is the NFS Ganesha cluster ID.
+
+``<pseudo_path>`` is the pseudo root path (must be an absolute path).
+
+
+Create or update export via JSON specification
+----------------------------------------------
+
+An existing export can be dumped in JSON format with:
+
+.. prompt:: bash #
+
+    ceph nfs export info *<cluster_id>* *<pseudo_path>*
+
+An export can be created or modified by importing a JSON description in the
+same format:
+
+.. prompt:: bash #
+
+    ceph nfs export apply *<cluster_id>* -i <json_file>
+
+For example,::
+
+   $ ceph nfs export info mynfs /cephfs > update_cephfs_export.json
+   $ cat update_cephfs_export.json
+   {
+     "export_id": 1,
+     "path": "/",
+     "cluster_id": "mynfs",
+     "pseudo": "/cephfs",
+     "access_type": "RW",
+     "squash": "no_root_squash",
+     "security_label": true,
+     "protocols": [
+       4
+     ],
+     "transports": [
+       "TCP"
+     ],
+     "fsal": {
+       "name": "CEPH",
+       "user_id": "nfs.mynfs.1",
+       "fs_name": "a",
+       "sec_label_xattr": ""
+     },
+     "clients": []
+   }
+
+The imported JSON can be a single dict describing a single export, or a JSON list
+containing multiple export dicts.
+
+The exported JSON can be modified and then reapplied.  Below, *pseudo*
+and *access_type* are modified.  When modifying an export, the
+provided JSON should fully describe the new state of the export (just
+as when creating a new export), with the exception of the
+authentication credentials, which will be carried over from the
+previous state of the export where possible.
+
+::
+
+   $ ceph nfs export apply mynfs -i update_cephfs_export.json
+   $ cat update_cephfs_export.json
+   {
+     "export_id": 1,
+     "path": "/",
+     "cluster_id": "mynfs",
+     "pseudo": "/cephfs_testing",
+     "access_type": "RO",
+     "squash": "no_root_squash",
+     "security_label": true,
+     "protocols": [
+       4
+     ],
+     "transports": [
+       "TCP"
+     ],
+     "fsal": {
+       "name": "CEPH",
+       "user_id": "nfs.mynfs.1",
+       "fs_name": "a",
+       "sec_label_xattr": ""
+     },
+     "clients": []
+   }
+
+An export can also be created or updated by injecting a Ganesha NFS EXPORT config
+fragment.  For example,::
+
+   $ ceph nfs export apply mynfs -i update_cephfs_export.conf
+   $ cat update_cephfs_export.conf
+   EXPORT {
+       FSAL {
+           name = "CEPH";
+           filesystem = "a";
+       }
+       export_id = 1;
+       path = "/";
+       pseudo = "/a";
+       access_type = "RW";
+       squash = "none";
+       attr_expiration_time = 0;
+       security_label = true;
+       protocols = 4;
+       transports = "TCP";
+   }
+
+
+Mounting
+========
+
+After the exports are successfully created and NFS Ganesha daemons are
+deployed, exports can be mounted with:
+
+.. code:: bash
+
+    $ mount -t nfs <ganesha-host-name>:<pseudo_path> <mount-point>
+
+For example, if the NFS cluster was created with ``--ingress --virtual-ip 192.168.10.10``
+and the export's pseudo-path was ``/foo``, the export can be mounted at ``/mnt`` with:
+
+.. code:: bash
+
+    $ mount -t nfs 192.168.10.10:/foo /mnt
+
+If the NFS service is running on a non-standard port number:
+
+.. code:: bash
+
+    $ mount -t nfs -o port=<ganesha-port> <ganesha-host-name>:<ganesha-pseudo_path> <mount-point>
+
+.. note:: Only NFS v4.0+ is supported.
+
+Troubleshooting
+===============
+
+Checking NFS-Ganesha logs with
+
+1) ``cephadm``: The NFS daemons can be listed with:
+
+   .. code:: bash
+
+	    $ ceph orch ps --daemon-type nfs
+
+   You can via the logs for a specific daemon (e.g., ``nfs.mynfs.0.0.myhost.xkfzal``) on
+   the relevant host with:
+
+   .. code:: bash
+
+      # cephadm logs --fsid <fsid> --name nfs.mynfs.0.0.myhost.xkfzal
+
+2) ``rook``:
+
+   .. code:: bash
+
+      $ kubectl logs -n rook-ceph rook-ceph-nfs-<cluster_id>-<node_id> nfs-ganesha
+
+The NFS log level can be adjusted using `nfs cluster config set` command (see :ref:`nfs-cluster-set`).
+
+
+.. _nfs-ganesha-config:
+
+
+Manual Ganesha deployment
+=========================
+
+It may be possible to deploy and manage the NFS ganesha daemons without
+orchestration frameworks such as cephadm or rook.
+
+.. note:: Manual configuration is not tested or fully documented; your
+          mileage may vary. If you make this work, please help us by
+          updating this documentation.
+
+Limitations
+------------
+
+If no orchestrator module is enabled for the Ceph Manager the NFS cluster
+management commands, such as those starting with ``ceph nfs cluster``, will not
+function. However, commands that manage NFS exports, like those prefixed with
+``ceph nfs export`` are expected to work as long as the necessary RADOS objects
+have already been created. The exact RADOS objects required are not documented
+at this time as support for this feature is incomplete. A curious reader can
+find some details about the object by reading the source code for the
+``mgr/nfs`` module (found in the ceph source tree under
+``src/pybind/mgr/nfs``).
+
+
+Requirements
+------------
+
+The following packages are required to enable CephFS and RGW exports with nfs-ganesha:
+
+-  ``nfs-ganesha``, ``nfs-ganesha-ceph``, ``nfs-ganesha-rados-grace`` and
+   ``nfs-ganesha-rados-urls`` packages (version 3.3 and above)
+
+Ganesha Configuration Hierarchy
+-------------------------------
+
+Cephadm and rook start each nfs-ganesha daemon with a minimal
+`bootstrap` configuration file that pulls from a shared `common`
+configuration stored in the ``.nfs`` RADOS pool and watches the common
+config for changes.  Each export is written to a separate RADOS object
+that is referenced by URL from the common config.
+
+.. ditaa::
+
+                             rados://$pool/$namespace/export-$i        rados://$pool/$namespace/userconf-nfs.$cluster_id
+                                      (export config)                          (user config)
+
+                        +----------+    +----------+    +----------+      +---------------------------+
+                        |          |    |          |    |          |      |                           |
+                        | export-1 |    | export-2 |    | export-3 |      | userconf-nfs.$cluster_id  |
+                        |          |    |          |    |          |      |                           |
+                        +----+-----+    +----+-----+    +-----+----+      +-------------+-------------+
+                             ^               ^                ^                         ^
+                             |               |                |                         |
+                             +--------------------------------+-------------------------+
+                                        %url |
+                                             |
+                                    +--------+--------+
+                                    |                 |  rados://$pool/$namespace/conf-nfs.$svc
+                                    |  conf+nfs.$svc  |  (common config)
+                                    |                 |
+                                    +--------+--------+
+                                             ^
+                                             |
+                                   watch_url |
+                     +----------------------------------------------+
+                     |                       |                      |
+                     |                       |                      |            RADOS
+             +----------------------------------------------------------------------------------+
+                     |                       |                      |            CONTAINER
+           watch_url |             watch_url |            watch_url |
+                     |                       |                      |
+            +--------+-------+      +--------+-------+      +-------+--------+
+            |                |      |                |      |                |  /etc/ganesha/ganesha.conf
+            |   nfs.$svc.a   |      |   nfs.$svc.b   |      |   nfs.$svc.c   |  (bootstrap config)
+            |                |      |                |      |                |
+            +----------------+      +----------------+      +----------------+
+
+
+.. _NFS-Ganesha NFS Server: https://github.com/nfs-ganesha/nfs-ganesha/wiki
+.. _NFS-Ganesha Export Sample: https://github.com/nfs-ganesha/nfs-ganesha/blob/next/src/config_samples/export.txt
diff --git a/doc/mgr/orchestrator.rst b/doc/mgr/orchestrator.rst
new file mode 100644
index 000000000..24704e5e1
--- /dev/null
+++ b/doc/mgr/orchestrator.rst
@@ -0,0 +1,240 @@
+
+.. _orchestrator-cli-module:
+
+================
+Orchestrator CLI
+================
+
+This module provides a command line interface (CLI) for orchestrator modules.
+Orchestrator modules are ``ceph-mgr`` plugins that interface with external
+orchestration services.
+
+Definition of Terms
+===================
+
+The orchestrator CLI unifies multiple external orchestrators, so we need a
+common nomenclature for the orchestrator module: 
+
++--------------------------------------+---------------------------------------+
+| *host*                               | hostname (not the DNS name) of the    |
+|                                      | physical host. Not the podname,       |
+|                                      | container name, or hostname inside    |
+|                                      | the container.                        |
++--------------------------------------+---------------------------------------+
+| *service type*                       | The type of the service. e.g., nfs,   |
+|                                      | mds, osd, mon, rgw, mgr, iscsi        |
++--------------------------------------+---------------------------------------+
+| *service*                            | A logical service. Typically          |
+|                                      | comprised of multiple service         |
+|                                      | instances on multiple hosts for HA    |
+|                                      |                                       |
+|                                      | * ``fs_name`` for mds type            |
+|                                      | * ``rgw_zone`` for rgw type           |
+|                                      | * ``ganesha_cluster_id`` for nfs type |
++--------------------------------------+---------------------------------------+
+| *daemon*                             | A single instance of a service.       |
+|                                      | Usually a daemon, but maybe not       |
+|                                      | (e.g., might be a kernel service      |
+|                                      | like LIO or knfsd or whatever)        |
+|                                      |                                       |
+|                                      | This identifier should                |
+|                                      | uniquely identify the instance.       |
++--------------------------------------+---------------------------------------+
+
+Here is how the names relate: 
+
+* A *service* has a specific *service type*.
+* A *daemon* is a physical instance of a *service type*.
+
+.. note::
+
+    Orchestrator modules might implement only a subset of the commands listed
+    below. The implementation of the commands may differ between modules.
+
+Status
+======
+
+.. prompt:: bash $
+
+   ceph orch status [--detail]
+
+This command shows the current orchestrator mode and its high-level status
+(whether the orchestrator plugin is available and operational).
+
+
+..
+    Turn On Device Lights
+    ^^^^^^^^^^^^^^^^^^^^^
+    ::
+
+        ceph orch device ident-on <dev_id>
+        ceph orch device ident-on <dev_name> <host>
+        ceph orch device fault-on <dev_id>
+        ceph orch device fault-on <dev_name> <host>
+
+        ceph orch device ident-off <dev_id> [--force=true]
+        ceph orch device ident-off <dev_id> <host> [--force=true]
+        ceph orch device fault-off <dev_id> [--force=true]
+        ceph orch device fault-off <dev_id> <host> [--force=true]
+
+    where ``dev_id`` is the device id as listed in ``osd metadata``,
+    ``dev_name`` is the name of the device on the system and ``host`` is the host as
+    returned by ``orchestrator host ls``
+
+        ceph orch osd ident-on {primary,journal,db,wal,all} <osd-id>
+        ceph orch osd ident-off {primary,journal,db,wal,all} <osd-id>
+        ceph orch osd fault-on {primary,journal,db,wal,all} <osd-id>
+        ceph orch osd fault-off {primary,journal,db,wal,all} <osd-id>
+
+    where ``journal`` is the filestore journal device, ``wal`` is the bluestore
+    write ahead log device, and ``all`` stands for all devices associated with the OSD
+
+
+.. _orchestrator-cli-stateless-services:
+
+Stateless services (MDS/RGW/NFS/rbd-mirror/iSCSI)
+=================================================
+
+.. note::
+
+   The orchestrator will not configure the services. See the relevant
+   documentation for details about how to configure particular services. 
+
+The ``name`` parameter identifies the kind of the group of instances. The
+following short list explains the meaning of the ``name`` parameter:
+
+* A CephFS file system identifies a group of MDS daemons.
+* A zone name identifies a group of RGWs.
+
+Creating/growing/shrinking/removing services:
+
+.. prompt:: bash $
+
+    ceph orch apply mds <fs_name> [--placement=<placement>] [--dry-run]
+    ceph orch apply rgw <name> [--realm=<realm>] [--zone=<zone>] [--port=<port>] [--ssl] [--placement=<placement>] [--dry-run]
+    ceph orch apply nfs <name> <pool> [--namespace=<namespace>] [--placement=<placement>] [--dry-run]
+    ceph orch rm <service_name> [--force]
+
+where ``placement`` is a :ref:`orchestrator-cli-placement-spec`.
+
+e.g., ``ceph orch apply mds myfs --placement="3 host1 host2 host3"``
+
+Service Commands:
+
+.. prompt:: bash $
+
+    ceph orch <start|stop|restart|redeploy|reconfig> <service_name>
+
+.. note:: These commands apply only to cephadm containerized daemons.
+
+Options
+=======
+
+.. option:: start
+
+   Start the daemon on the corresponding host.
+
+.. option:: stop
+
+   Stop the daemon on the corresponding host.
+
+.. option:: restart
+
+   Restart the daemon on the corresponding host.
+
+.. option:: redeploy
+
+   Redeploy the ceph daemon on the corresponding host. This will recreate the daemon directory
+   structure under ``/var/lib/ceph/<fsid>/<daemon-name>`` (if it doesn't exist), refresh its
+   configuration files, regenerate its unit-files and restarts the systemd daemon.
+
+.. option:: reconfig
+
+   Reconfigure the daemon on the corresponding host. This will refresh configuration files then restart the daemon.
+
+   .. note:: this command assumes the daemon directory ``/var/lib/ceph/<fsid>/<daemon-name>`` already exists.
+
+
+Configuring the Orchestrator CLI
+================================
+
+Enable the orchestrator by using the ``set backend`` command to select the orchestrator module that will be used:
+
+.. prompt:: bash $
+
+    ceph orch set backend <module>
+
+Example - Configuring the Orchestrator CLI
+------------------------------------------
+
+For example, to enable the Rook orchestrator module and use it with the CLI:
+
+.. prompt:: bash $
+
+    ceph mgr module enable rook
+    ceph orch set backend rook
+
+Confirm that the backend is properly configured:
+
+.. prompt:: bash $
+
+    ceph orch status
+
+Disable the Orchestrator
+------------------------
+
+To disable the orchestrator, use the empty string ``""``:
+
+.. prompt:: bash $
+
+    ceph orch set backend ""
+    ceph mgr module disable rook
+
+Current Implementation Status
+=============================
+
+This is an overview of the current implementation status of the orchestrators.
+
+=================================== ====== =========
+ Command                             Rook   Cephadm
+=================================== ====== =========
+ apply iscsi                         ⚪     ✔
+ apply mds                           ✔      ✔
+ apply mgr                           ⚪      ✔
+ apply mon                           ✔      ✔
+ apply nfs                           ✔      ✔
+ apply osd                           ✔      ✔
+ apply rbd-mirror                    ✔      ✔
+ apply cephfs-mirror                 ⚪      ✔
+ apply grafana                       ⚪      ✔
+ apply prometheus                    �      ✔
+ apply alertmanager                  �      ✔
+ apply node-exporter                 �      ✔
+ apply rgw                           ✔       ✔
+ apply container                     ⚪      ✔
+ apply snmp-gateway                  �      ✔
+ host add                            ⚪      ✔
+ host ls                             ✔      ✔
+ host rm                             ⚪      ✔
+ host maintenance enter              �      ✔
+ host maintenance exit               �      ✔
+ daemon status                       ⚪      ✔
+ daemon {stop,start,...}             ⚪      ✔
+ device {ident,fault}-(on,off}       ⚪      ✔
+ device ls                           ✔      ✔
+ iscsi add                           ⚪     ✔
+ mds add                             ⚪      ✔
+ nfs add                             ⚪      ✔
+ rbd-mirror add                      ⚪      ✔
+ rgw add                             ⚪     ✔
+ ls                                  ✔      ✔
+ ps                                  ✔      ✔
+ status                              ✔      ✔
+ upgrade                             �      ✔
+=================================== ====== =========
+
+where
+
+* ⚪ = not yet implemented
+* � = not applicable
+* ✔ = implemented
diff --git a/doc/mgr/orchestrator_modules.rst b/doc/mgr/orchestrator_modules.rst
new file mode 100644
index 000000000..a28b43059
--- /dev/null
+++ b/doc/mgr/orchestrator_modules.rst
@@ -0,0 +1,332 @@
+
+
+.. _orchestrator-modules:
+
+.. py:currentmodule:: orchestrator
+
+ceph-mgr orchestrator modules
+=============================
+
+.. warning::
+
+    This is developer documentation, describing Ceph internals that
+    are only relevant to people writing ceph-mgr orchestrator modules.
+
+In this context, *orchestrator* refers to some external service that
+provides the ability to discover devices and create Ceph services.  This
+includes external projects such as Rook.
+
+An *orchestrator module* is a ceph-mgr module (:ref:`mgr-module-dev`)
+which implements common management operations using a particular
+orchestrator.
+
+Orchestrator modules subclass the ``Orchestrator`` class: this class is
+an interface, it only provides method definitions to be implemented
+by subclasses.  The purpose of defining this common interface
+for different orchestrators is to enable common UI code, such as
+the dashboard, to work with various different backends.
+
+
+.. graphviz::
+
+    digraph G {
+        subgraph cluster_1 {
+            volumes [label="mgr/volumes"]
+            rook [label="mgr/rook"]
+            dashboard [label="mgr/dashboard"]
+            orchestrator_cli [label="mgr/orchestrator"]
+            orchestrator [label="Orchestrator Interface"]
+            cephadm [label="mgr/cephadm"]
+
+            label = "ceph-mgr";
+        }
+
+        volumes -> orchestrator
+        dashboard -> orchestrator
+        orchestrator_cli -> orchestrator
+        orchestrator -> rook -> rook_io
+        orchestrator -> cephadm
+
+
+        rook_io [label="Rook"]
+
+        rankdir="TB";
+    }
+
+Behind all the abstraction, the purpose of orchestrator modules is simple:
+enable Ceph to do things like discover available hardware, create and
+destroy OSDs, and run MDS and RGW services.
+
+A tutorial is not included here: for full and concrete examples, see
+the existing implemented orchestrator modules in the Ceph source tree.
+
+Glossary
+--------
+
+Stateful service
+  a daemon that uses local storage, such as OSD or mon.
+
+Stateless service
+  a daemon that doesn't use any local storage, such
+  as an MDS, RGW, nfs-ganesha, iSCSI gateway.
+
+Label
+  arbitrary string tags that may be applied by administrators
+  to hosts.  Typically administrators use labels to indicate
+  which hosts should run which kinds of service.  Labels are
+  advisory (from human input) and do not guarantee that hosts
+  have particular physical capabilities.
+
+Drive group
+  collection of block devices with common/shared OSD
+  formatting (typically one or more SSDs acting as
+  journals/dbs for a group of HDDs).
+
+Placement
+  choice of which host is used to run a service.
+
+Key Concepts
+------------
+
+The underlying orchestrator remains the source of truth for information
+about whether a service is running, what is running where, which
+hosts are available, etc.  Orchestrator modules should avoid taking
+any internal copies of this information, and read it directly from
+the orchestrator backend as much as possible.
+
+Bootstrapping hosts and adding them to the underlying orchestration
+system is outside the scope of Ceph's orchestrator interface.  Ceph
+can only work on hosts when the orchestrator is already aware of them.
+
+Where possible, placement of stateless services should be left up to the
+orchestrator.
+
+Completions and batching
+------------------------
+
+All methods that read or modify the state of the system can potentially
+be long running. Therefore the module needs to schedule those operations.
+
+Each orchestrator module implements its own underlying mechanisms
+for completions.  This might involve running the underlying operations
+in threads, or batching the operations up before later executing
+in one go in the background.  If implementing such a batching pattern, the
+module would do no work on any operation until it appeared in a list
+of completions passed into *process*.
+
+Error Handling
+--------------
+
+The main goal of error handling within orchestrator modules is to provide debug information to
+assist users when dealing with deployment errors.
+
+.. autoclass:: OrchestratorError
+.. autoclass:: NoOrchestrator
+.. autoclass:: OrchestratorValidationError
+
+
+In detail, orchestrators need to explicitly deal with different kinds of errors:
+
+1. No orchestrator configured
+
+   See :class:`NoOrchestrator`.
+
+2. An orchestrator doesn't implement a specific method.
+
+   For example, an Orchestrator doesn't support ``add_host``.
+
+   In this case, a ``NotImplementedError`` is raised.
+
+3. Missing features within implemented methods.
+
+   E.g. optional parameters to a command that are not supported by the
+   backend (e.g. the hosts field in :func:`Orchestrator.apply_mons` command with the rook backend).
+
+   See :class:`OrchestratorValidationError`.
+
+4. Input validation errors
+
+   The ``orchestrator`` module and other calling modules are supposed to
+   provide meaningful error messages.
+
+   See :class:`OrchestratorValidationError`.
+
+5. Errors when actually executing commands
+
+   The resulting Completion should contain an error string that assists in understanding the
+   problem. In addition, :func:`Completion.is_errored` is set to ``True``
+
+6. Invalid configuration in the orchestrator modules
+
+   This can be tackled similar to 5.
+
+
+All other errors are unexpected orchestrator issues and thus should raise an exception that are then
+logged into the mgr log file. If there is a completion object at that point,
+:func:`Completion.result` may contain an error message.
+
+
+Excluded functionality
+----------------------
+
+- Ceph's orchestrator interface is not a general purpose framework for
+  managing linux servers -- it is deliberately constrained to manage
+  the Ceph cluster's services only.
+- Multipathed storage is not handled (multipathing is unnecessary for
+  Ceph clusters).  Each drive is assumed to be visible only on
+  a single host.
+
+Host management
+---------------
+
+.. automethod:: Orchestrator.add_host
+.. automethod:: Orchestrator.remove_host
+.. automethod:: Orchestrator.get_hosts
+.. automethod:: Orchestrator.update_host_addr
+.. automethod:: Orchestrator.add_host_label
+.. automethod:: Orchestrator.remove_host_label
+
+.. autoclass:: HostSpec
+
+Devices
+-------
+
+.. automethod:: Orchestrator.get_inventory
+.. autoclass:: InventoryFilter
+
+.. py:currentmodule:: ceph.deployment.inventory
+
+.. autoclass:: Devices
+   :members:
+
+.. autoclass:: Device
+   :members:
+
+.. py:currentmodule:: orchestrator
+
+Placement
+---------
+
+A :ref:`orchestrator-cli-placement-spec` defines the placement of
+daemons of a specific service.
+
+In general, stateless services do not require any specific placement
+rules as they can run anywhere that sufficient system resources
+are available. However, some orchestrators may not include the
+functionality to choose a location in this way. Optionally, you can
+specify a location when creating a stateless service.
+
+
+.. py:currentmodule:: ceph.deployment.service_spec
+
+.. autoclass:: PlacementSpec
+   :members:
+
+.. py:currentmodule:: orchestrator
+
+
+Services
+--------
+
+.. autoclass:: ServiceDescription
+
+.. py:currentmodule:: ceph.deployment.service_spec
+
+.. autoclass:: ServiceSpec
+  :members:
+  :private-members:
+  :noindex:
+
+.. py:currentmodule:: orchestrator
+
+.. automethod:: Orchestrator.describe_service
+
+.. automethod:: Orchestrator.service_action
+.. automethod:: Orchestrator.remove_service
+
+
+Daemons
+-------
+
+.. automethod:: Orchestrator.list_daemons
+.. automethod:: Orchestrator.remove_daemons
+.. automethod:: Orchestrator.daemon_action
+
+.. autoclass:: DaemonDescription
+.. autoclass:: DaemonDescriptionStatus
+
+OSD management
+--------------
+
+.. automethod:: Orchestrator.create_osds
+
+.. automethod:: Orchestrator.blink_device_light
+.. autoclass:: DeviceLightLoc
+
+.. _orchestrator-osd-replace:
+
+OSD Replacement
+^^^^^^^^^^^^^^^
+
+See :ref:`rados-replacing-an-osd` for the underlying process.
+
+Replacing OSDs is fundamentally a two-staged process, as users need to
+physically replace drives. The orchestrator therefore exposes this two-staged process.
+
+Phase one is a call to :meth:`Orchestrator.remove_daemons` with ``destroy=True`` in order to mark
+the OSD as destroyed.
+
+
+Phase two is a call to  :meth:`Orchestrator.create_osds` with a Drive Group with
+
+.. py:currentmodule:: ceph.deployment.drive_group
+
+:attr:`DriveGroupSpec.osd_id_claims` set to the destroyed OSD ids.
+
+.. py:currentmodule:: orchestrator
+
+Services
+--------
+
+.. automethod:: Orchestrator.add_daemon
+.. automethod:: Orchestrator.apply_mon
+.. automethod:: Orchestrator.apply_mgr
+.. automethod:: Orchestrator.apply_mds
+.. automethod:: Orchestrator.apply_rbd_mirror
+
+.. py:currentmodule:: ceph.deployment.service_spec
+
+.. autoclass:: RGWSpec
+  :noindex:
+
+.. py:currentmodule:: orchestrator
+
+.. automethod:: Orchestrator.apply_rgw
+
+.. py:currentmodule:: ceph.deployment.service_spec
+
+.. autoclass:: NFSServiceSpec
+
+.. py:currentmodule:: orchestrator
+
+.. automethod:: Orchestrator.apply_nfs
+
+Upgrades
+--------
+
+.. automethod:: Orchestrator.upgrade_available
+.. automethod:: Orchestrator.upgrade_start
+.. automethod:: Orchestrator.upgrade_status
+.. autoclass:: UpgradeStatusSpec
+
+Utility
+-------
+
+.. automethod:: Orchestrator.available
+.. automethod:: Orchestrator.get_feature_set
+
+Client Modules
+--------------
+
+.. autoclass:: OrchestratorClientMixin
+   :members:
diff --git a/doc/mgr/prometheus.rst b/doc/mgr/prometheus.rst
new file mode 100644
index 000000000..d4a9fd911
--- /dev/null
+++ b/doc/mgr/prometheus.rst
@@ -0,0 +1,388 @@
+.. _mgr-prometheus:
+
+=================
+Prometheus Module
+=================
+
+Provides a Prometheus exporter to pass on Ceph performance counters
+from the collection point in ceph-mgr.  Ceph-mgr receives MMgrReport
+messages from all MgrClient processes (mons and OSDs, for instance)
+with performance counter schema data and actual counter data, and keeps
+a circular buffer of the last N samples.  This module creates an HTTP
+endpoint (like all Prometheus exporters) and retrieves the latest sample
+of every counter when polled (or "scraped" in Prometheus terminology).
+The HTTP path and query parameters are ignored; all extant counters
+for all reporting entities are returned in text exposition format.
+(See the Prometheus `documentation <https://prometheus.io/docs/instrumenting/exposition_formats/#text-format-details>`_.)
+
+Enabling prometheus output
+==========================
+
+The *prometheus* module is enabled with::
+
+  ceph mgr module enable prometheus
+
+Configuration
+-------------
+
+.. note::
+
+    The Prometheus manager module needs to be restarted for configuration changes to be applied.
+
+By default the module will accept HTTP requests on port ``9283`` on all IPv4
+and IPv6 addresses on the host.  The port and listen address are both
+configurable with ``ceph config set``, with keys
+``mgr/prometheus/server_addr`` and ``mgr/prometheus/server_port``.  This port
+is registered with Prometheus's `registry
+<https://github.com/prometheus/prometheus/wiki/Default-port-allocations>`_.
+
+::
+
+    ceph config set mgr mgr/prometheus/server_addr 0.0.0.0
+    ceph config set mgr mgr/prometheus/server_port 9283
+
+.. warning::
+
+    The ``scrape_interval`` of this module should always be set to match
+    Prometheus' scrape interval to work properly and not cause any issues.
+
+The scrape interval in the module is used for caching purposes
+and to determine when a cache is stale.
+
+It is not recommended to use a scrape interval below 10 seconds.  It is
+recommended to use 15 seconds as scrape interval, though, in some cases it
+might be useful to increase the scrape interval.
+
+To set a different scrape interval in the Prometheus module, set
+``scrape_interval`` to the desired value::
+
+    ceph config set mgr mgr/prometheus/scrape_interval 20
+
+On large clusters (>1000 OSDs), the time to fetch the metrics may become
+significant.  Without the cache, the Prometheus manager module could, especially
+in conjunction with multiple Prometheus instances, overload the manager and lead
+to unresponsive or crashing Ceph manager instances.  Hence, the cache is enabled
+by default.  This means that there is a possibility that the cache becomes
+stale.  The cache is considered stale when the time to fetch the metrics from
+Ceph exceeds the configured :confval:``mgr/prometheus/scrape_interval``.
+
+If that is the case, **a warning will be logged** and the module will either
+
+* respond with a 503 HTTP status code (service unavailable) or,
+* it will return the content of the cache, even though it might be stale.
+
+This behavior can be configured. By default, it will return a 503 HTTP status
+code (service unavailable). You can set other options using the ``ceph config
+set`` commands.
+
+To tell the module to respond with possibly stale data, set it to ``return``::
+
+    ceph config set mgr mgr/prometheus/stale_cache_strategy return
+
+To tell the module to respond with "service unavailable", set it to ``fail``::
+
+    ceph config set mgr mgr/prometheus/stale_cache_strategy fail
+
+If you are confident that you don't require the cache, you can disable it::
+
+    ceph config set mgr mgr/prometheus/cache false
+
+If you are using the prometheus module behind some kind of reverse proxy or
+loadbalancer, you can simplify discovering the active instance by switching
+to ``error``-mode::
+
+    ceph config set mgr mgr/prometheus/standby_behaviour error
+
+If set, the prometheus module will repond with a HTTP error when requesting ``/``
+from the standby instance. The default error code is 500, but you can configure
+the HTTP response code with::
+
+    ceph config set mgr mgr/prometheus/standby_error_status_code 503
+
+Valid error codes are between 400-599.
+
+To switch back to the default behaviour, simply set the config key to ``default``::
+
+    ceph config set mgr mgr/prometheus/standby_behaviour default
+
+.. _prometheus-rbd-io-statistics:
+
+Ceph Health Checks
+------------------
+
+The mgr/prometheus module also tracks and maintains a history of Ceph health checks,
+exposing them to the Prometheus server as discrete metrics. This allows Prometheus
+alert rules to be configured for specific health check events.
+
+The metrics take the following form;
+
+::
+
+    # HELP ceph_health_detail healthcheck status by type (0=inactive, 1=active)
+    # TYPE ceph_health_detail gauge
+    ceph_health_detail{name="OSDMAP_FLAGS",severity="HEALTH_WARN"} 0.0
+    ceph_health_detail{name="OSD_DOWN",severity="HEALTH_WARN"} 1.0
+    ceph_health_detail{name="PG_DEGRADED",severity="HEALTH_WARN"} 1.0
+
+The health check history is made available through the following commands;
+
+::
+
+    healthcheck history ls [--format {plain|json|json-pretty}]
+    healthcheck history clear
+
+The ``ls`` command provides an overview of the health checks that the cluster has
+encountered, or since the last ``clear`` command was issued. The example below;
+
+::
+
+    [ceph: root@c8-node1 /]# ceph healthcheck history ls
+    Healthcheck Name          First Seen (UTC)      Last seen (UTC)       Count  Active
+    OSDMAP_FLAGS              2021/09/16 03:17:47   2021/09/16 22:07:40       2    No
+    OSD_DOWN                  2021/09/17 00:11:59   2021/09/17 00:11:59       1   Yes
+    PG_DEGRADED               2021/09/17 00:11:59   2021/09/17 00:11:59       1   Yes
+    3 health check(s) listed
+
+
+RBD IO statistics
+-----------------
+
+The module can optionally collect RBD per-image IO statistics by enabling
+dynamic OSD performance counters. The statistics are gathered for all images
+in the pools that are specified in the ``mgr/prometheus/rbd_stats_pools``
+configuration parameter. The parameter is a comma or space separated list
+of ``pool[/namespace]`` entries. If the namespace is not specified the
+statistics are collected for all namespaces in the pool.
+
+Example to activate the RBD-enabled pools ``pool1``, ``pool2`` and ``poolN``::
+
+  ceph config set mgr mgr/prometheus/rbd_stats_pools "pool1,pool2,poolN"
+
+The module makes the list of all available images scanning the specified
+pools and namespaces and refreshes it periodically. The period is
+configurable via the ``mgr/prometheus/rbd_stats_pools_refresh_interval``
+parameter (in sec) and is 300 sec (5 minutes) by default. The module will
+force refresh earlier if it detects statistics from a previously unknown
+RBD image.
+
+Example to turn up the sync interval to 10 minutes::
+
+  ceph config set mgr mgr/prometheus/rbd_stats_pools_refresh_interval 600
+
+Statistic names and labels
+==========================
+
+The names of the stats are exactly as Ceph names them, with
+illegal characters ``.``, ``-`` and ``::`` translated to ``_``,
+and ``ceph_`` prefixed to all names.
+
+
+All *daemon* statistics have a ``ceph_daemon`` label such as "osd.123"
+that identifies the type and ID of the daemon they come from.  Some
+statistics can come from different types of daemon, so when querying
+e.g. an OSD's RocksDB stats, you would probably want to filter
+on ceph_daemon starting with "osd" to avoid mixing in the monitor
+rocksdb stats.
+
+
+The *cluster* statistics (i.e. those global to the Ceph cluster)
+have labels appropriate to what they report on.  For example,
+metrics relating to pools have a ``pool_id`` label.
+
+
+The long running averages that represent the histograms from core Ceph
+are represented by a pair of ``<name>_sum`` and ``<name>_count`` metrics.
+This is similar to how histograms are represented in `Prometheus <https://prometheus.io/docs/concepts/metric_types/#histogram>`_
+and they can also be treated `similarly <https://prometheus.io/docs/practices/histograms/>`_.
+
+Pool and OSD metadata series
+----------------------------
+
+Special series are output to enable displaying and querying on
+certain metadata fields.
+
+Pools have a ``ceph_pool_metadata`` field like this:
+
+::
+
+    ceph_pool_metadata{pool_id="2",name="cephfs_metadata_a"} 1.0
+
+OSDs have a ``ceph_osd_metadata`` field like this:
+
+::
+
+    ceph_osd_metadata{cluster_addr="172.21.9.34:6802/19096",device_class="ssd",ceph_daemon="osd.0",public_addr="172.21.9.34:6801/19096",weight="1.0"} 1.0
+
+
+Correlating drive statistics with node_exporter
+-----------------------------------------------
+
+The prometheus output from Ceph is designed to be used in conjunction
+with the generic host monitoring from the Prometheus node_exporter.
+
+To enable correlation of Ceph OSD statistics with node_exporter's
+drive statistics, special series are output like this:
+
+::
+
+    ceph_disk_occupation_human{ceph_daemon="osd.0", device="sdd", exported_instance="myhost"}
+
+To use this to get disk statistics by OSD ID, use either the ``and`` operator or
+the ``*`` operator in your prometheus query. All metadata metrics (like ``
+ceph_disk_occupation_human`` have the value 1 so they act neutral with ``*``. Using ``*``
+allows to use ``group_left`` and ``group_right`` grouping modifiers, so that
+the resulting metric has additional labels from one side of the query.
+
+See the
+`prometheus documentation`__ for more information about constructing queries.
+
+__ https://prometheus.io/docs/prometheus/latest/querying/basics
+
+The goal is to run a query like
+
+::
+
+    rate(node_disk_bytes_written[30s]) and
+    on (device,instance) ceph_disk_occupation_human{ceph_daemon="osd.0"}
+
+Out of the box the above query will not return any metrics since the ``instance`` labels of
+both metrics don't match. The ``instance`` label of ``ceph_disk_occupation_human``
+will be the currently active MGR node.
+
+The following two section outline two approaches to remedy this.
+
+.. note::
+
+    If you need to group on the `ceph_daemon` label instead of `device` and
+    `instance` labels, using `ceph_disk_occupation_human` may not work reliably.
+    It is advised that you use `ceph_disk_occupation` instead.
+
+    The difference is that `ceph_disk_occupation_human` may group several OSDs
+    into the value of a single `ceph_daemon` label in cases where multiple OSDs
+    share a disk.
+
+Use label_replace
+=================
+
+The ``label_replace`` function (cp.
+`label_replace documentation <https://prometheus.io/docs/prometheus/latest/querying/functions/#label_replace>`_)
+can add a label to, or alter a label of, a metric within a query.
+
+To correlate an OSD and its disks write rate, the following query can be used:
+
+::
+
+    label_replace(
+        rate(node_disk_bytes_written[30s]),
+        "exported_instance",
+        "$1",
+        "instance",
+        "(.*):.*"
+    ) and on (device, exported_instance) ceph_disk_occupation_human{ceph_daemon="osd.0"}
+
+Configuring Prometheus server
+=============================
+
+honor_labels
+------------
+
+To enable Ceph to output properly-labeled data relating to any host,
+use the ``honor_labels`` setting when adding the ceph-mgr endpoints
+to your prometheus configuration.
+
+This allows Ceph to export the proper ``instance`` label without prometheus
+overwriting it. Without this setting, Prometheus applies an ``instance`` label
+that includes the hostname and port of the endpoint that the series came from.
+Because Ceph clusters have multiple manager daemons, this results in an
+``instance`` label that changes spuriously when the active manager daemon
+changes.
+
+If this is undesirable a custom ``instance`` label can be set in the
+Prometheus target configuration: you might wish to set it to the hostname
+of your first mgr daemon, or something completely arbitrary like "ceph_cluster".
+
+node_exporter hostname labels
+-----------------------------
+
+Set your ``instance`` labels to match what appears in Ceph's OSD metadata
+in the ``instance`` field.  This is generally the short hostname of the node.
+
+This is only necessary if you want to correlate Ceph stats with host stats,
+but you may find it useful to do it in all cases in case you want to do
+the correlation in the future.
+
+Example configuration
+---------------------
+
+This example shows a single node configuration running ceph-mgr and
+node_exporter on a server called ``senta04``. Note that this requires one
+to add an appropriate and unique ``instance`` label to each ``node_exporter`` target.
+
+This is just an example: there are other ways to configure prometheus
+scrape targets and label rewrite rules.
+
+prometheus.yml
+~~~~~~~~~~~~~~
+
+::
+
+    global:
+      scrape_interval:     15s
+      evaluation_interval: 15s
+
+    scrape_configs:
+      - job_name: 'node'
+        file_sd_configs:
+          - files:
+            - node_targets.yml
+      - job_name: 'ceph'
+        honor_labels: true
+        file_sd_configs:
+          - files:
+            - ceph_targets.yml
+
+
+ceph_targets.yml
+~~~~~~~~~~~~~~~~
+
+
+::
+
+    [
+        {
+            "targets": [ "senta04.mydomain.com:9283" ],
+            "labels": {}
+        }
+    ]
+
+
+node_targets.yml
+~~~~~~~~~~~~~~~~
+
+::
+
+    [
+        {
+            "targets": [ "senta04.mydomain.com:9100" ],
+            "labels": {
+                "instance": "senta04"
+            }
+        }
+    ]
+
+
+Notes
+=====
+
+Counters and gauges are exported; currently histograms and long-running
+averages are not.  It's possible that Ceph's 2-D histograms could be
+reduced to two separate 1-D histograms, and that long-running averages
+could be exported as Prometheus' Summary type.
+
+Timestamps, as with many Prometheus exporters, are established by
+the server's scrape time (Prometheus expects that it is polling the
+actual counter process synchronously).  It is possible to supply a
+timestamp along with the stat report, but the Prometheus team strongly
+advises against this.  This means that timestamps will be delayed by
+an unpredictable amount; it's not clear if this will be problematic,
+but it's worth knowing about.
diff --git a/doc/mgr/restful.rst b/doc/mgr/restful.rst
new file mode 100644
index 000000000..d684399fc
--- /dev/null
+++ b/doc/mgr/restful.rst
@@ -0,0 +1,189 @@
+Restful Module
+==============
+
+RESTful module offers the REST API access to the status of the cluster
+over an SSL-secured connection.
+
+Enabling
+--------
+
+The *restful* module is enabled with::
+
+  ceph mgr module enable restful
+
+You will also need to configure an SSL certificate below before the
+API endpoint is available.  By default the module will accept HTTPS
+requests on port ``8003`` on all IPv4 and IPv6 addresses on the host.
+
+Securing
+--------
+
+All connections to *restful* are secured with SSL.  You can generate a
+self-signed certificate with the command::
+
+  ceph restful create-self-signed-cert
+
+Note that with a self-signed certificate most clients will need a flag
+to allow a connection and/or suppress warning messages.  For example,
+if the ``ceph-mgr`` daemon is on the same host,::
+
+  curl -k https://localhost:8003/
+
+To properly secure a deployment, a certificate that is signed by the
+organization's certificate authority should be used.  For example, a key pair
+can be generated with a command similar to::
+
+  openssl req -new -nodes -x509 \
+    -subj "/O=IT/CN=ceph-mgr-restful" \
+    -days 3650 -keyout restful.key -out restful.crt -extensions v3_ca
+
+The ``restful.crt`` should then be signed by your organization's CA
+(certificate authority).  Once that is done, you can set it with::
+
+  ceph config-key set mgr/restful/$name/crt -i restful.crt
+  ceph config-key set mgr/restful/$name/key -i restful.key
+
+where ``$name`` is the name of the ``ceph-mgr`` instance (usually the
+hostname). If all manager instances are to share the same certificate,
+you can leave off the ``$name`` portion::
+
+  ceph config-key set mgr/restful/crt -i restful.crt
+  ceph config-key set mgr/restful/key -i restful.key
+
+
+Configuring IP and port
+-----------------------
+
+Like any other RESTful API endpoint, *restful* binds to an IP and
+port.  By default, the currently active ``ceph-mgr`` daemon will bind
+to port 8003 and any available IPv4 or IPv6 address on the host.
+
+Since each ``ceph-mgr`` hosts its own instance of *restful*, it may
+also be necessary to configure them separately. The IP and port
+can be changed via the configuration key facility::
+
+  ceph config set mgr mgr/restful/$name/server_addr $IP
+  ceph config set mgr mgr/restful/$name/server_port $PORT
+
+where ``$name`` is the ID of the ceph-mgr daemon (usually the hostname).
+
+These settings can also be configured cluster-wide and not manager
+specific.  For example,::
+
+  ceph config set mgr mgr/restful/server_addr $IP
+  ceph config set mgr mgr/restful/server_port $PORT
+
+If the port is not configured, *restful* will bind to port ``8003``.
+If the address it not configured, the *restful* will bind to ``::``,
+which corresponds to all available IPv4 and IPv6 addresses.
+
+.. _creating-an-api-user:
+
+Creating an API User
+-----------------------
+
+To create an API user, please run the following command::
+
+  ceph restful create-key <username>
+
+Replace ``<username>`` with the desired name of the user. For example, to create a user named
+``api``::
+
+  $ ceph restful create-key api
+  52dffd92-a103-4a10-bfce-5b60f48f764e
+
+The UUID generated from ``ceph restful create-key api`` acts as the key for the user.
+
+To list all of your API keys, please run the following command::
+
+  ceph restful list-keys
+
+The ``ceph restful list-keys`` command will output in JSON::
+
+  {
+  	"api": "52dffd92-a103-4a10-bfce-5b60f48f764e"
+  }
+
+You can use ``curl`` in order to test your user with the API. Here is an example::
+
+  curl -k https://api:52dffd92-a103-4a10-bfce-5b60f48f764e@<ceph-mgr>:<port>/server
+
+In the case above, we are using ``GET`` to fetch information from the ``server`` endpoint.
+
+Load balancer
+-------------
+
+Please note that *restful* will *only* start on the manager which
+is active at that moment. Query the Ceph cluster status to see which
+manager is active (e.g., ``ceph mgr dump``).  In order to make the
+API available via a consistent URL regardless of which manager
+daemon is currently active, you may want to set up a load balancer
+front-end to direct traffic to whichever manager endpoint is
+available.
+
+Available methods
+-----------------
+
+You can navigate to the ``/doc`` endpoint for full list of available
+endpoints and HTTP methods implemented for each endpoint.
+
+For example, if you want to use the PATCH method of the ``/osd/<id>``
+endpoint to set the state ``up`` of the OSD id ``1``, you can use the
+following curl command::
+
+  echo -En '{"up": true}' | curl --request PATCH --data @- --silent --insecure --user <user> 'https://<ceph-mgr>:<port>/osd/1'
+
+or you can use python to do so::
+
+  $ python
+  >> import requests
+  >> result = requests.patch(
+         'https://<ceph-mgr>:<port>/osd/1',
+         json={"up": True},
+         auth=("<user>", "<password>")
+     )
+  >> print result.json()
+
+Some of the other endpoints implemented in the *restful* module include
+
+* ``/config/cluster``: **GET**
+* ``/config/osd``: **GET**, **PATCH**
+* ``/crush/rule``: **GET**
+* ``/mon``: **GET**
+* ``/osd``: **GET**
+* ``/pool``: **GET**, **POST**
+* ``/pool/<arg>``: **DELETE**, **GET**, **PATCH**
+* ``/request``: **DELETE**, **GET**, **POST**
+* ``/request/<arg>``: **DELETE**, **GET**
+* ``/server``: **GET**
+
+The ``/request`` endpoint
+-------------------------
+
+You can use the ``/request`` endpoint to poll the state of a request
+you scheduled with any **DELETE**, **POST** or **PATCH** method. These
+methods are by default asynchronous since it may take longer for them
+to finish execution. You can modify this behaviour by appending
+``?wait=1`` to the request url. The returned request will then always
+be completed.
+
+The **POST** method of the ``/request`` method provides a passthrough
+for the ceph mon commands as defined in ``src/mon/MonCommands.h``.
+Let's consider the following command::
+
+  COMMAND("osd ls " \
+          "name=epoch,type=CephInt,range=0,req=false", \
+          "show all OSD ids", "osd", "r", "cli,rest")
+
+The **prefix** is **osd ls**. The optional argument's name is **epoch**
+and it is of type ``CephInt``, i.e. ``integer``. This means that you
+need to do the following **POST** request to schedule the command::
+
+  $ python
+  >> import requests
+  >> result = requests.post(
+         'https://<ceph-mgr>:<port>/request',
+         json={'prefix': 'osd ls', 'epoch': 0},
+         auth=("<user>", "<password>")
+     )
+  >> print result.json()
diff --git a/doc/mgr/rook.rst b/doc/mgr/rook.rst
new file mode 100644
index 000000000..1ae369623
--- /dev/null
+++ b/doc/mgr/rook.rst
@@ -0,0 +1,39 @@
+
+.. _mgr-rook:
+
+====
+Rook
+====
+
+Rook (https://rook.io/) is an orchestration tool that can run Ceph inside
+a Kubernetes cluster.
+
+The ``rook`` module provides integration between Ceph's orchestrator framework
+(used by modules such as ``dashboard`` to control cluster services) and
+Rook.
+
+Orchestrator modules only provide services to other modules, which in turn
+provide user interfaces.  To try out the rook module, you might like
+to use the :ref:`Orchestrator CLI <orchestrator-cli-module>` module.
+
+Requirements
+------------
+
+- Running ceph-mon and ceph-mgr services that were set up with Rook in
+  Kubernetes.
+- Rook 0.9 or newer.
+
+Configuration
+-------------
+
+Because a Rook cluster's ceph-mgr daemon is running as a Kubernetes pod, 
+the ``rook`` module can connect to the Kubernetes API without any explicit
+configuration.
+
+Development
+-----------
+
+If you are a developer, please see :ref:`kubernetes-dev` for instructions
+on setting up a development environment to work with this.
+
+
diff --git a/doc/mgr/telegraf.rst b/doc/mgr/telegraf.rst
new file mode 100644
index 000000000..5944f7255
--- /dev/null
+++ b/doc/mgr/telegraf.rst
@@ -0,0 +1,88 @@
+===============
+Telegraf Module 
+===============
+The Telegraf module collects and sends statistics series to a Telegraf agent.
+
+The Telegraf agent can buffer, aggregate, parse and process the data before
+sending it to an output which can be InfluxDB, ElasticSearch and many more.
+
+Currently the only way to send statistics to Telegraf from this module is to
+use the socket listener. The module can send statistics over UDP, TCP or
+a UNIX socket.
+
+The Telegraf module was introduced in the 13.x *Mimic* release.
+
+--------
+Enabling 
+--------
+
+To enable the module, use the following command:
+
+::
+
+    ceph mgr module enable telegraf
+
+If you wish to subsequently disable the module, you can use the corresponding
+*disable* command:
+
+::
+
+    ceph mgr module disable telegraf
+
+-------------
+Configuration 
+-------------
+
+For the telegraf module to send statistics to a Telegraf agent it is
+required to configure the address to send the statistics to.
+
+Set configuration values using the following command:
+
+::
+
+    ceph telegraf config-set <key> <value>
+
+
+The most important settings are ``address`` and ``interval``.
+
+For example, a typical configuration might look like this:
+
+::
+
+    ceph telegraf config-set address udp://:8094
+    ceph telegraf config-set interval 10
+    
+The default values for these configuration keys are:
+
+- address: unixgram:///tmp/telegraf.sock
+- interval: 15
+
+----------------
+Socket Listener
+----------------
+The module only supports sending data to Telegraf through the socket listener
+of the Telegraf module using the Influx data format.
+
+A typical Telegraf configuration might be:
+
+
+    [[inputs.socket_listener]]
+    # service_address = "tcp://:8094"
+    # service_address = "tcp://127.0.0.1:http"
+    # service_address = "tcp4://:8094"
+    # service_address = "tcp6://:8094"
+    # service_address = "tcp6://[2001:db8::1]:8094"
+    service_address = "udp://:8094"
+    # service_address = "udp4://:8094"
+    # service_address = "udp6://:8094"
+    # service_address = "unix:///tmp/telegraf.sock"
+    # service_address = "unixgram:///tmp/telegraf.sock"
+    data_format = "influx"
+
+In this case the `address` configuration option for the module would need to be set
+to:
+
+  udp://:8094
+
+
+Refer to the Telegraf documentation for more configuration options.
diff --git a/doc/mgr/telemetry.rst b/doc/mgr/telemetry.rst
new file mode 100644
index 000000000..6eaaa5c44
--- /dev/null
+++ b/doc/mgr/telemetry.rst
@@ -0,0 +1,155 @@
+.. _telemetry:
+
+Telemetry Module
+================
+
+The telemetry module sends anonymous data about the cluster back to the Ceph
+developers to help understand how Ceph is used and what problems users may
+be experiencing.
+
+This data is visualized on `public dashboards <https://telemetry-public.ceph.com/>`_
+that allow the community to quickly see summary statistics on how many clusters
+are reporting, their total capacity and OSD count, and version distribution
+trends.
+
+Channels
+--------
+
+The telemetry report is broken down into several "channels", each with
+a different type of information.  Assuming telemetry has been enabled,
+individual channels can be turned on and off.  (If telemetry is off,
+the per-channel setting has no effect.)
+
+* **basic** (default: on): Basic information about the cluster
+
+    - capacity of the cluster
+    - number of monitors, managers, OSDs, MDSs, object gateways, or other daemons
+    - software version currently being used
+    - number and types of RADOS pools and CephFS file systems
+    - names of configuration options that have been changed from their
+      default (but *not* their values)
+
+* **crash** (default: on): Information about daemon crashes, including
+
+    - type of daemon
+    - version of the daemon
+    - operating system (OS distribution, kernel version)
+    - stack trace identifying where in the Ceph code the crash occurred
+
+* **device** (default: on): Information about device metrics, including
+
+    - anonymized SMART metrics
+
+* **ident** (default: off): User-provided identifying information about
+  the cluster
+
+    - cluster description
+    - contact email address
+
+The data being reported does *not* contain any sensitive
+data like pool names, object names, object contents, hostnames, or device
+serial numbers.
+
+It contains counters and statistics on how the cluster has been
+deployed, the version of Ceph, the distribution of the hosts and other
+parameters which help the project to gain a better understanding of
+the way Ceph is used.
+
+Data is sent secured to *https://telemetry.ceph.com*.
+
+Sample report
+-------------
+
+You can look at what data is reported at any time with the command::
+
+  ceph telemetry show
+
+To protect your privacy, device reports are generated separately, and data such
+as hostname and device serial number is anonymized. The device telemetry is
+sent to a different endpoint and does not associate the device data with a
+particular cluster. To see a preview of the device report use the command::
+
+  ceph telemetry show-device
+
+Please note: In order to generate the device report we use Smartmontools
+version 7.0 and up, which supports JSON output. 
+If you have any concerns about privacy with regard to the information included in
+this report, please contact the Ceph developers.
+
+Channels
+--------
+
+Individual channels can be enabled or disabled with::
+
+  ceph config set mgr mgr/telemetry/channel_ident false
+  ceph config set mgr mgr/telemetry/channel_basic false
+  ceph config set mgr mgr/telemetry/channel_crash false
+  ceph config set mgr mgr/telemetry/channel_device false
+  ceph telemetry show
+  ceph telemetry show-device
+
+Enabling Telemetry
+------------------
+
+To allow the *telemetry* module to start sharing data::
+
+  ceph telemetry on
+
+Please note: Telemetry data is licensed under the Community Data License
+Agreement - Sharing - Version 1.0 (https://cdla.io/sharing-1-0/). Hence,
+telemetry module can be enabled only after you add '--license sharing-1-0' to
+the 'ceph telemetry on' command.
+
+Telemetry can be disabled at any time with::
+
+  ceph telemetry off
+
+Interval
+--------
+
+The module compiles and sends a new report every 24 hours by default.
+You can adjust this interval with::
+
+  ceph config set mgr mgr/telemetry/interval 72    # report every three days
+
+Status
+--------
+
+The see the current configuration::
+
+  ceph telemetry status
+
+Manually sending telemetry
+--------------------------
+
+To ad hoc send telemetry data::
+
+  ceph telemetry send
+
+In case telemetry is not enabled (with 'ceph telemetry on'), you need to add
+'--license sharing-1-0' to 'ceph telemetry send' command.
+
+Sending telemetry through a proxy
+---------------------------------
+
+If the cluster cannot directly connect to the configured telemetry
+endpoint (default *telemetry.ceph.com*), you can configure a HTTP/HTTPS
+proxy server with::
+
+  ceph config set mgr mgr/telemetry/proxy https://10.0.0.1:8080
+
+You can also include a *user:pass* if needed::
+
+  ceph config set mgr mgr/telemetry/proxy https://ceph:telemetry@10.0.0.1:8080
+
+
+Contact and Description
+-----------------------
+
+A contact and description can be added to the report.  This is
+completely optional, and disabled by default.::
+
+  ceph config set mgr mgr/telemetry/contact 'John Doe <john.doe@example.com>'
+  ceph config set mgr mgr/telemetry/description 'My first Ceph cluster'
+  ceph config set mgr mgr/telemetry/channel_ident true
+
diff --git a/doc/mgr/zabbix.rst b/doc/mgr/zabbix.rst
new file mode 100644
index 000000000..e06714625
--- /dev/null
+++ b/doc/mgr/zabbix.rst
@@ -0,0 +1,153 @@
+Zabbix Module
+=============
+
+The Zabbix module actively sends information to a Zabbix server like:
+
+- Ceph status
+- I/O operations
+- I/O bandwidth
+- OSD status
+- Storage utilization
+
+Requirements
+------------
+
+The module requires that the *zabbix_sender* executable is present on *all*
+machines running ceph-mgr. It can be installed on most distributions using
+the package manager.
+
+Dependencies
+^^^^^^^^^^^^
+Installing zabbix_sender can be done under Ubuntu or CentOS using either apt
+or dnf.
+
+On Ubuntu Xenial:
+
+::
+
+    apt install zabbix-agent
+
+On Fedora:
+
+::
+
+    dnf install zabbix-sender
+
+
+Enabling
+--------
+You can enable the *zabbix* module with:
+
+::
+
+    ceph mgr module enable zabbix
+
+Configuration
+-------------
+
+Two configuration keys are vital for the module to work:
+
+- zabbix_host
+- identifier (optional)
+
+The parameter *zabbix_host* controls the hostname of the Zabbix server to which
+*zabbix_sender* will send the items. This can be a IP-Address if required by
+your installation.
+
+The *identifier* parameter controls the identifier/hostname to use as source
+when sending items to Zabbix. This should match the name of the *Host* in
+your Zabbix server.
+
+When the *identifier* parameter is not configured the ceph-<fsid> of the cluster
+will be used when sending data to Zabbix.
+
+This would for example be *ceph-c4d32a99-9e80-490f-bd3a-1d22d8a7d354*
+
+Additional configuration keys which can be configured and their default values:
+
+- zabbix_port: 10051
+- zabbix_sender: /usr/bin/zabbix_sender
+- interval: 60
+- discovery_interval: 100
+
+Configuration keys
+^^^^^^^^^^^^^^^^^^^
+
+Configuration keys can be set on any machine with the proper cephx credentials,
+these are usually Monitors where the *client.admin* key is present.
+
+::
+
+    ceph zabbix config-set <key> <value>
+
+For example:
+
+::
+
+    ceph zabbix config-set zabbix_host zabbix.localdomain
+    ceph zabbix config-set identifier ceph.eu-ams02.local
+
+The current configuration of the module can also be shown:
+
+::
+
+   ceph zabbix config-show
+
+
+Template
+^^^^^^^^
+A `template <https://raw.githubusercontent.com/ceph/ceph/master/src/pybind/mgr/zabbix/zabbix_template.xml>`_. 
+(XML) to be used on the Zabbix server can be found in the source directory of the module.
+
+This template contains all items and a few triggers. You can customize the triggers afterwards to fit your needs.
+
+
+Multiple Zabbix servers
+^^^^^^^^^^^^^^^^^^^^^^^
+It is possible to instruct zabbix module to send data to multiple Zabbix servers.
+
+Parameter *zabbix_host* can be set with multiple hostnames separated by commas.
+Hosnames (or IP adderesses) can be followed by colon and port number. If a port
+number is not present module will use the port number defined in *zabbix_port*.
+
+For example:
+
+::
+
+    ceph zabbix config-set zabbix_host "zabbix1,zabbix2:2222,zabbix3:3333"
+
+
+Manually sending data
+---------------------
+If needed the module can be asked to send data immediately instead of waiting for
+the interval.
+
+This can be done with this command:
+
+::
+
+    ceph zabbix send
+
+The module will now send its latest data to the Zabbix server.
+
+Items discovery is accomplished also via zabbix_sender, and runs every `discovery_interval * interval` seconds. If you wish to launch discovery 
+manually, this can be done with this command:
+
+::
+
+    ceph zabbix discovery
+
+
+Debugging
+---------
+
+Should you want to debug the Zabbix module increase the logging level for
+ceph-mgr and check the logs.
+
+::
+
+    [mgr]
+        debug mgr = 20
+
+With logging set to debug for the manager the module will print various logging
+lines prefixed with *mgr[zabbix]* for easy filtering.
diff --git a/doc/mon/README.txt b/doc/mon/README.txt
new file mode 100644
index 000000000..fa1bf791a
--- /dev/null
+++ b/doc/mon/README.txt
@@ -0,0 +1,27 @@
+paxos-call-chain.dot describes to some detail the call chain involved in the
+Paxos algorithm, paying special consideration to the messages involved.
+
+This information is not easily obtainable by Doxygen, as it does not follow
+the call chain when messages are involved, since it becomes an async workflow.
+
+To obtain the graph one should run
+
+  dot -T<format> paxos-call-chain.dot -o paxos-call-chain.<format>
+
+e.g.,
+
+  dot -Tps paxos-call-chain.dot -o paxos-call-chain.ps
+
+or
+
+  dot -Tpng paxos-call-chain.dot -o paxos-call-chain.png
+
+It should do the trick.
+
+Also, for future reference, we consider that:
+  - boxed nodes refer to the Leader;
+  - elliptical nodes refer to the Peon;
+  - diamond shaped nodes refer to state changes;
+  - dotted lines illustrate a message being sent from the Leader to the Peon,
+  or vice-versa.
+
diff --git a/doc/mon/paxos-call-chain.dot b/doc/mon/paxos-call-chain.dot
new file mode 100644
index 000000000..cdca200f0
--- /dev/null
+++ b/doc/mon/paxos-call-chain.dot
@@ -0,0 +1,82 @@
+digraph Paxos {
+  concentrate=true
+//  subgraph cluster0 {
+    collect -> state_recovering;
+    collect -> send_mmp_collect -> handle_collect [style=dotted];
+    handle_collect -> send_mmp_last;
+    send_mmp_last -> handle_last [style=dotted];
+    handle_last -> collect [label="collect(last_pn)"];
+    handle_last -> send_mmp_commit;
+    handle_last -> state_active;
+    handle_last -> begin;
+    handle_last -> extend_lease;
+
+//    color=grey;
+//    style=filled;
+//    label="Post-Election call chain";
+//  }
+
+  election_finished -> leader_init;
+  leader_init -> collect [label="collect(0)"];
+  send_mmp_commit -> handle_commit [style=dotted];
+
+  begin -> state_updating;
+  begin -> send_mmp_begin;
+  begin -> commit;
+  
+  send_mmp_begin -> handle_begin [style=dotted];
+  handle_begin -> state_updating;
+  handle_begin -> send_mmp_accept;
+  send_mmp_accept -> handle_accept [style=dotted];
+
+  handle_accept -> extend_lease;
+  handle_accept -> state_active;
+  handle_accept -> commit;
+
+  extend_lease -> send_mmp_lease;
+  send_mmp_lease -> handle_lease [style=dotted];
+  handle_lease -> state_active;
+  handle_lease -> send_mmp_lease_ack;
+  send_mmp_lease_ack -> handle_lease_ack [style=dotted];
+
+  commit -> send_mmp_commit;
+  commit -> "last_committed++" [shape=box];
+  send_mmp_commit -> handle_commit [style=dotted];
+
+  handle_commit -> store_state;
+
+  propose_pending -> propose_new_value;
+  propose_new_value -> begin;
+
+  election_finished [label="Election Finished", shape=box, bgcolor=grey, style=filled];
+  collect	    [label="collect()", shape=box];
+  commit	    [label="commit()", shape=box];
+  begin		    [label="begin()", shape=box];
+  extend_lease	    [label="extend_lease()", shape=box];
+  store_state	    [label="store_state()"]; // peon
+
+  propose_pending   [label="PaxosService::propose_pending()"];
+  propose_new_value [label="propose_new_value"];
+
+  send_mmp_collect  [label="send(OP_COLLECT)", shape=box];
+  send_mmp_last	    [label="send(OP_LAST)"];
+  send_mmp_commit   [label="send(OP_COMMIT)", shape=box];
+  send_mmp_begin    [label="send(OP_BEGIN)", shape=box];
+  send_mmp_accept   [label="send(OP_ACCEPT)"];
+  send_mmp_lease    [label="send(OP_LEASE)", shape=box];
+  send_mmp_lease_ack [label="send(OP_LEASE_ACK)"];
+
+  handle_collect    [label="handle_collect()"];
+  handle_last	    [label="handle_last()", shape=box];
+  handle_begin	    [label="handle_begin()"];
+  handle_accept	    [label="handle_accept()", shape=box];
+  handle_lease	    [label="handle_lease()"];
+  handle_lease_ack  [label="handle_lease_ack()", shape=box];
+  handle_commit	    [label="handle_commit()"];
+
+  leader_init	    [label="Paxos::leader_init()", shape=box];
+
+  state_recovering  [label="RECOVERING", shape=diamond];
+  state_active	    [label="ACTIVE", shape=diamond];
+  state_updating    [label="UPDATING", shape=diamond];
+}
diff --git a/doc/rados/api/index.rst b/doc/rados/api/index.rst
new file mode 100644
index 000000000..63bc7222d
--- /dev/null
+++ b/doc/rados/api/index.rst
@@ -0,0 +1,23 @@
+===========================
+ Ceph Storage Cluster APIs
+===========================
+
+The :term:`Ceph Storage Cluster` has a messaging layer protocol that enables
+clients to interact with a :term:`Ceph Monitor` and a :term:`Ceph OSD Daemon`.
+``librados`` provides this functionality to :term:`Ceph Client`\s in the form of
+a library.  All Ceph Clients either use ``librados`` or the same functionality
+encapsulated in ``librados`` to interact with the object store.  For example,
+``librbd`` and ``libcephfs`` leverage this functionality. You may use
+``librados`` to interact with Ceph directly (e.g., an application that talks to
+Ceph, your own interface to Ceph, etc.).
+
+
+.. toctree::
+   :maxdepth: 2 
+
+   Introduction to librados <librados-intro>
+   librados (C) <librados>
+   librados (C++) <libradospp>
+   librados (Python) <python>
+   libcephsqlite (SQLite) <libcephsqlite>
+   object class <objclass-sdk>
diff --git a/doc/rados/api/libcephsqlite.rst b/doc/rados/api/libcephsqlite.rst
new file mode 100644
index 000000000..76ab306bb
--- /dev/null
+++ b/doc/rados/api/libcephsqlite.rst
@@ -0,0 +1,438 @@
+.. _libcephsqlite:
+
+================
+ Ceph SQLite VFS
+================
+
+This `SQLite VFS`_ may be used for storing and accessing a `SQLite`_ database
+backed by RADOS. This allows you to fully decentralize your database using
+Ceph's object store for improved availability, accessibility, and use of
+storage.
+
+Note what this is not: a distributed SQL engine. SQLite on RADOS can be thought
+of like RBD as compared to CephFS: RBD puts a disk image on RADOS for the
+purposes of exclusive access by a machine and generally does not allow parallel
+access by other machines; on the other hand, CephFS allows fully distributed
+access to a file system from many client mounts. SQLite on RADOS is meant to be
+accessed by a single SQLite client database connection at a given time.  The
+database may be manipulated safely by multiple clients only in a serial fashion
+controlled by RADOS locks managed by the Ceph SQLite VFS.
+
+
+Usage
+^^^^^
+
+Normal unmodified applications (including the sqlite command-line toolset
+binary) may load the *ceph* VFS using the `SQLite Extension Loading API`_.
+
+.. code:: sql
+
+    .LOAD libcephsqlite.so
+
+or during the invocation of ``sqlite3``
+
+.. code:: sh
+
+   sqlite3 -cmd '.load libcephsqlite.so'
+
+A database file is formatted as a SQLite URI::
+
+    file:///<"*"poolid|poolname>:[namespace]/<dbname>?vfs=ceph
+
+The RADOS ``namespace`` is optional. Note the triple ``///`` in the path. The URI
+authority must be empty or localhost in SQLite. Only the path part of the URI
+is parsed. For this reason, the URI will not parse properly if you only use two
+``//``.
+
+A complete example of (optionally) creating a database and opening:
+
+.. code:: sh
+
+   sqlite3 -cmd '.load libcephsqlite.so' -cmd '.open file:///foo:bar/baz.db?vfs=ceph'
+
+Note you cannot specify the database file as the normal positional argument to
+``sqlite3``. This is because the ``.load libcephsqlite.so`` command is applied
+after opening the database, but opening the database depends on the extension
+being loaded first.
+
+An example passing the pool integer id and no RADOS namespace:
+
+.. code:: sh
+
+   sqlite3 -cmd '.load libcephsqlite.so' -cmd '.open file:///*2:/baz.db?vfs=ceph'
+
+Like other Ceph tools, the *ceph* VFS looks at some environment variables that
+help with configuring which Ceph cluster to communicate with and which
+credential to use. Here would be a typical configuration:
+
+.. code:: sh
+
+   export CEPH_CONF=/path/to/ceph.conf
+   export CEPH_KEYRING=/path/to/ceph.keyring
+   export CEPH_ARGS='--id myclientid'
+   ./runmyapp
+   # or
+   sqlite3 -cmd '.load libcephsqlite.so' -cmd '.open file:///foo:bar/baz.db?vfs=ceph'
+
+The default operation would look at the standard Ceph configuration file path
+using the ``client.admin`` user.
+
+
+User
+^^^^
+
+The *ceph* VFS requires a user credential with read access to the monitors, the
+ability to blocklist dead clients of the database, and access to the OSDs
+hosting the database. This can be done with authorizations as simply as:
+
+.. code:: sh
+
+   ceph auth get-or-create client.X mon 'allow r, allow command "osd blocklist" with blocklistop=add' osd 'allow rwx'
+
+.. note:: The terminology change from ``blacklist`` to ``blocklist``; older clusters may require using the old terms.
+
+You may also simplify using the ``simple-rados-client-with-blocklist`` profile:
+
+.. code:: sh
+
+   ceph auth get-or-create client.X mon 'profile simple-rados-client-with-blocklist' osd 'allow rwx'
+
+To learn why blocklisting is necessary, see :ref:`libcephsqlite-corrupt`.
+
+
+Page Size
+^^^^^^^^^
+
+SQLite allows configuring the page size prior to creating a new database. It is
+advisable to increase this config to 65536 (64K) when using RADOS backed
+databases to reduce the number of OSD reads/writes and thereby improve
+throughput and latency.
+
+.. code:: sql
+
+   PRAGMA page_size = 65536
+
+You may also try other values according to your application needs but note that
+64K is the max imposed by SQLite.
+
+
+Cache
+^^^^^
+
+The ceph VFS does not do any caching of reads or buffering of writes. Instead,
+and more appropriately, the SQLite page cache is used. You may find it is too small
+for most workloads and should therefore increase it significantly:
+
+
+.. code:: sql
+
+   PRAGMA cache_size = 4096
+
+Which will cache 4096 pages or 256MB (with 64K ``page_cache``).
+
+
+Journal Persistence
+^^^^^^^^^^^^^^^^^^^
+
+By default, SQLite deletes the journal for every transaction. This can be
+expensive as the *ceph* VFS must delete every object backing the journal for each
+transaction. For this reason, it is much faster and simpler to ask SQLite to
+**persist** the journal. In this mode, SQLite will invalidate the journal via a
+write to its header. This is done as:
+
+.. code:: sql
+
+   PRAGMA journal_mode = PERSIST
+
+The cost of this may be increased unused space according to the high-water size
+of the rollback journal (based on transaction type and size).
+
+
+Exclusive Lock Mode
+^^^^^^^^^^^^^^^^^^^
+
+SQLite operates in a ``NORMAL`` locking mode where each transaction requires
+locking the backing database file. This can add unnecessary overhead to
+transactions when you know there's only ever one user of the database at a
+given time. You can have SQLite lock the database once for the duration of the
+connection using:
+
+.. code:: sql
+
+   PRAGMA locking_mode = EXCLUSIVE
+
+This can more than **halve** the time taken to perform a transaction. Keep in
+mind this prevents other clients from accessing the database.
+
+In this locking mode, each write transaction to the database requires 3
+synchronization events: once to write to the journal, another to write to the
+database file, and a final write to invalidate the journal header (in
+``PERSIST`` journaling mode).
+
+
+WAL Journal
+^^^^^^^^^^^
+
+The `WAL Journal Mode`_ is only available when SQLite is operating in exclusive
+lock mode. This is because it requires shared memory communication with other
+readers and writers when in the ``NORMAL`` locking mode.
+
+As with local disk databases, WAL mode may significantly reduce small
+transaction latency. Testing has shown it can provide more than 50% speedup
+over persisted rollback journals in exclusive locking mode. You can expect
+around 150-250 transactions per second depending on size.
+
+
+Performance Notes
+^^^^^^^^^^^^^^^^^
+
+The filing backend for the database on RADOS is asynchronous as much as
+possible.  Still, performance can be anywhere from 3x-10x slower than a local
+database on SSD. Latency can be a major factor. It is advisable to be familiar
+with SQL transactions and other strategies for efficient database updates.
+Depending on the performance of the underlying pool, you can expect small
+transactions to take up to 30 milliseconds to complete. If you use the
+``EXCLUSIVE`` locking mode, it can be reduced further to 15 milliseconds per
+transaction. A WAL journal in ``EXCLUSIVE`` locking mode can further reduce
+this as low as ~2-5 milliseconds (or the time to complete a RADOS write; you
+won't get better than that!).
+
+There is no limit to the size of a SQLite database on RADOS imposed by the Ceph
+VFS. There are standard `SQLite Limits`_ to be aware of, notably the maximum
+database size of 281 TB. Large databases may or may not be performant on Ceph.
+Experimentation for your own use-case is advised.
+
+Be aware that read-heavy queries could take significant amounts of time as
+reads are necessarily synchronous (due to the VFS API). No readahead is yet
+performed by the VFS.
+
+
+Recommended Use-Cases
+^^^^^^^^^^^^^^^^^^^^^
+
+The original purpose of this module was to support saving relational or large
+data in RADOS which needs to span multiple objects. Many current applications
+with trivial state try to use RADOS omap storage on a single object but this
+cannot scale without striping data across multiple objects. Unfortunately, it
+is non-trivial to design a store spanning multiple objects which is consistent
+and also simple to use. SQLite can be used to bridge that gap.
+
+
+Parallel Access
+^^^^^^^^^^^^^^^
+
+The VFS does not yet support concurrent readers. All database access is protected
+by a single exclusive lock.
+
+
+Export or Extract Database out of RADOS
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The database is striped on RADOS and can be extracted using the RADOS cli toolset.
+
+.. code:: sh
+
+    rados --pool=foo --striper get bar.db local-bar.db
+    rados --pool=foo --striper get bar.db-journal local-bar.db-journal
+    sqlite3 local-bar.db ...
+
+Keep in mind the rollback journal is also striped and will need to be extracted
+as well if the database was in the middle of a transaction. If you're using
+WAL, that journal will need to be extracted as well.
+
+Keep in mind that extracting the database using the striper uses the same RADOS
+locks as those used by the *ceph* VFS. However, the journal file locks are not
+used by the *ceph* VFS (SQLite only locks the main database file) so there is a
+potential race with other SQLite clients when extracting both files. That could
+result in fetching a corrupt journal.
+
+Instead of manually extracting the files, it would be more advisable to use the
+`SQLite Backup`_ mechanism instead.
+
+
+Temporary Tables
+^^^^^^^^^^^^^^^^
+
+Temporary tables backed by the ceph VFS are not supported. The main reason for
+this is that the VFS lacks context about where it should put the database, i.e.
+which RADOS pool. The persistent database associated with the temporary
+database is not communicated via the SQLite VFS API.
+
+Instead, it's suggested to attach a secondary local or `In-Memory Database`_
+and put the temporary tables there. Alternatively, you may set a connection
+pragma:
+
+.. code:: sql
+
+   PRAGMA temp_store=memory
+
+
+.. _libcephsqlite-breaking-locks:
+
+Breaking Locks
+^^^^^^^^^^^^^^
+
+Access to the database file is protected by an exclusive lock on the first
+object stripe of the database. If the application fails without unlocking the
+database (e.g. a segmentation fault), the lock is not automatically unlocked,
+even if the client connection is blocklisted afterward. Eventually, the lock
+will timeout subject to the configurations::
+
+    cephsqlite_lock_renewal_timeout = 30000
+
+The timeout is in milliseconds. Once the timeout is reached, the OSD will
+expire the lock and allow clients to relock. When this occurs, the database
+will be recovered by SQLite and the in-progress transaction rolled back. The
+new client recovering the database will also blocklist the old client to
+prevent potential database corruption from rogue writes.
+
+The holder of the exclusive lock on the database will periodically renew the
+lock so it does not lose the lock. This is necessary for large transactions or
+database connections operating in ``EXCLUSIVE`` locking mode. The lock renewal
+interval is adjustable via::
+
+    cephsqlite_lock_renewal_interval = 2000
+
+This configuration is also in units of milliseconds.
+
+It is possible to break the lock early if you know the client is gone for good
+(e.g. blocklisted). This allows restoring database access to clients
+immediately. For example:
+
+.. code:: sh
+
+    $ rados --pool=foo --namespace bar lock info baz.db.0000000000000000 striper.lock
+    {"name":"striper.lock","type":"exclusive","tag":"","lockers":[{"name":"client.4463","cookie":"555c7208-db39-48e8-a4d7-3ba92433a41a","description":"SimpleRADOSStriper","expiration":"0.000000","addr":"127.0.0.1:0/1831418345"}]}
+
+    $ rados --pool=foo --namespace bar lock break baz.db.0000000000000000 striper.lock client.4463 --lock-cookie 555c7208-db39-48e8-a4d7-3ba92433a41a
+
+.. _libcephsqlite-corrupt:
+
+How to Corrupt Your Database
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+There is the usual reading on `How to Corrupt Your SQLite Database`_ that you
+should review before using this tool. To add to that, the most likely way you
+may corrupt your database is by a rogue process transiently losing network
+connectivity and then resuming its work. The exclusive RADOS lock it held will
+be lost but it cannot know that immediately. Any work it might do after
+regaining network connectivity could corrupt the database.
+
+The *ceph* VFS library defaults do not allow for this scenario to occur. The Ceph
+VFS will blocklist the last owner of the exclusive lock on the database if it
+detects incomplete cleanup.
+
+By blocklisting the old client, it's no longer possible for the old client to
+resume its work on the database when it returns (subject to blocklist
+expiration, 3600 seconds by default). To turn off blocklisting the prior client, change::
+
+    cephsqlite_blocklist_dead_locker = false
+
+Do NOT do this unless you know database corruption cannot result due to other
+guarantees. If this config is true (the default), the *ceph* VFS will cowardly
+fail if it cannot blocklist the prior instance (due to lack of authorization,
+for example).
+
+One example where out-of-band mechanisms exist to blocklist the last dead
+holder of the exclusive lock on the database is in the ``ceph-mgr``. The
+monitors are made aware of the RADOS connection used for the *ceph* VFS and will
+blocklist the instance during ``ceph-mgr`` failover. This prevents a zombie
+``ceph-mgr`` from continuing work and potentially corrupting the database. For
+this reason, it is not necessary for the *ceph* VFS to do the blocklist command
+in the new instance of the ``ceph-mgr`` (but it still does so, harmlessly).
+
+To blocklist the *ceph* VFS manually, you may see the instance address of the
+*ceph* VFS using the ``ceph_status`` SQL function:
+
+.. code:: sql
+
+    SELECT ceph_status();
+
+.. code::
+
+    {"id":788461300,"addr":"172.21.10.4:0/1472139388"}
+
+You may easily manipulate that information using the `JSON1 extension`_:
+
+.. code:: sql
+
+    SELECT json_extract(ceph_status(), '$.addr');
+
+.. code::
+
+   172.21.10.4:0/3563721180
+
+This is the address you would pass to the ceph blocklist command:
+
+.. code:: sh
+
+   ceph osd blocklist add 172.21.10.4:0/3082314560
+
+
+Performance Statistics
+^^^^^^^^^^^^^^^^^^^^^^
+
+The *ceph* VFS provides a SQLite function, ``ceph_perf``, for querying the
+performance statistics of the VFS. The data is from "performance counters" as
+in other Ceph services normally queried via an admin socket.
+
+.. code:: sql
+
+    SELECT ceph_perf();
+
+.. code::
+
+    {"libcephsqlite_vfs":{"op_open":{"avgcount":2,"sum":0.150001291,"avgtime":0.075000645},"op_delete":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"op_access":{"avgcount":1,"sum":0.003000026,"avgtime":0.003000026},"op_fullpathname":{"avgcount":1,"sum":0.064000551,"avgtime":0.064000551},"op_currenttime":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"opf_close":{"avgcount":1,"sum":0.000000000,"avgtime":0.000000000},"opf_read":{"avgcount":3,"sum":0.036000310,"avgtime":0.012000103},"opf_write":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"opf_truncate":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"opf_sync":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"opf_filesize":{"avgcount":2,"sum":0.000000000,"avgtime":0.000000000},"opf_lock":{"avgcount":1,"sum":0.158001360,"avgtime":0.158001360},"opf_unlock":{"avgcount":1,"sum":0.101000871,"avgtime":0.101000871},"opf_checkreservedlock":{"avgcount":1,"sum":0.002000017,"avgtime":0.002000017},"opf_filecontrol":{"avgcount":4,"sum":0.000000000,"avgtime":0.000000000},"opf_sectorsize":{"avgcount":0,"sum":0.000000000,"avgtime":0.000000000},"opf_devicecharacteristics":{"avgcount":4,"sum":0.000000000,"avgtime":0.000000000}},"libcephsqlite_striper":{"update_metadata":0,"update_allocated":0,"update_size":0,"update_version":0,"shrink":0,"shrink_bytes":0,"lock":1,"unlock":1}}
+
+You may easily manipulate that information using the `JSON1 extension`_:
+
+.. code:: sql
+
+    SELECT json_extract(ceph_perf(), '$.libcephsqlite_vfs.opf_sync.avgcount');
+
+.. code::
+
+    776
+
+That tells you the number of times SQLite has called the xSync method of the
+`SQLite IO Methods`_ of the VFS (for **all** open database connections in the
+process). You could analyze the performance stats before and after a number of
+queries to see the number of file system syncs required (this would just be
+proportional to the number of transactions). Alternatively, you may be more
+interested in the average latency to complete a write:
+
+.. code:: sql
+
+    SELECT json_extract(ceph_perf(), '$.libcephsqlite_vfs.opf_write');
+
+.. code::
+
+    {"avgcount":7873,"sum":0.675005797,"avgtime":0.000085736}
+
+Which would tell you there have been 7873 writes with an average
+time-to-complete of 85 microseconds. That clearly shows the calls are executed
+asynchronously. Returning to sync:
+
+.. code:: sql
+
+    SELECT json_extract(ceph_perf(), '$.libcephsqlite_vfs.opf_sync');
+
+.. code::
+
+    {"avgcount":776,"sum":4.802041199,"avgtime":0.006188197}
+
+6 milliseconds were spent on average executing a sync call. This gathers all of
+the asynchronous writes as well as an asynchronous update to the size of the
+striped file.
+
+
+.. _SQLite: https://sqlite.org/index.html
+.. _SQLite VFS: https://www.sqlite.org/vfs.html
+.. _SQLite Backup: https://www.sqlite.org/backup.html
+.. _SQLite Limits: https://www.sqlite.org/limits.html
+.. _SQLite Extension Loading API: https://sqlite.org/c3ref/load_extension.html
+.. _In-Memory Database: https://www.sqlite.org/inmemorydb.html
+.. _WAL Journal Mode: https://sqlite.org/wal.html
+.. _How to Corrupt Your SQLite Database: https://www.sqlite.org/howtocorrupt.html
+.. _JSON1 Extension: https://www.sqlite.org/json1.html
+.. _SQLite IO Methods: https://www.sqlite.org/c3ref/io_methods.html
diff --git a/doc/rados/api/librados-intro.rst b/doc/rados/api/librados-intro.rst
new file mode 100644
index 000000000..9bffa3114
--- /dev/null
+++ b/doc/rados/api/librados-intro.rst
@@ -0,0 +1,1052 @@
+==========================
+ Introduction to librados
+==========================
+
+The :term:`Ceph Storage Cluster` provides the basic storage service that allows
+:term:`Ceph` to uniquely deliver **object, block, and file storage** in one
+unified system. However, you are not limited to using the RESTful, block, or
+POSIX interfaces. Based upon :abbr:`RADOS (Reliable Autonomic Distributed Object
+Store)`, the ``librados`` API enables you to create your own interface to the
+Ceph Storage Cluster.
+
+The ``librados`` API enables you to interact with the two types of daemons in
+the Ceph Storage Cluster: 
+
+- The :term:`Ceph Monitor`, which maintains a master copy of the cluster map. 
+- The :term:`Ceph OSD Daemon` (OSD), which stores data as objects on a storage node.
+
+.. ditaa::
+            +---------------------------------+
+            |  Ceph Storage Cluster Protocol  |
+            |           (librados)            |
+            +---------------------------------+
+            +---------------+ +---------------+
+            |      OSDs     | |    Monitors   |
+            +---------------+ +---------------+
+
+This guide provides a high-level introduction to using ``librados``. 
+Refer to :doc:`../../architecture` for additional details of the Ceph
+Storage Cluster. To use the API, you need a running Ceph Storage Cluster. 
+See `Installation (Quick)`_ for details.
+
+
+Step 1: Getting librados
+========================
+
+Your client application must bind with ``librados`` to connect to the Ceph
+Storage Cluster. You must install ``librados`` and any required packages to
+write applications that use ``librados``. The ``librados`` API is written in
+C++, with additional bindings for C, Python, Java and PHP. 
+
+
+Getting librados for C/C++
+--------------------------
+
+To install ``librados`` development support files for C/C++ on Debian/Ubuntu
+distributions, execute the following:
+
+.. prompt:: bash $
+
+   sudo apt-get install librados-dev
+
+To install ``librados`` development support files for C/C++ on RHEL/CentOS
+distributions, execute the following:
+
+.. prompt:: bash $
+
+   sudo yum install librados2-devel
+
+Once you install ``librados`` for developers, you can find the required 
+headers for C/C++ under ``/usr/include/rados``:
+
+.. prompt:: bash $
+
+   ls /usr/include/rados
+
+
+Getting librados for Python
+---------------------------
+
+The ``rados`` module provides ``librados`` support to Python
+applications. The ``librados-dev`` package for Debian/Ubuntu
+and the ``librados2-devel`` package for RHEL/CentOS will install the
+``python-rados`` package for you. You may install ``python-rados``
+directly too.
+
+To install ``librados`` development support files for Python on Debian/Ubuntu
+distributions, execute the following:
+
+.. prompt:: bash $
+
+   sudo apt-get install python3-rados
+
+To install ``librados`` development support files for Python on RHEL/CentOS
+distributions, execute the following:
+
+.. prompt:: bash $
+
+   sudo yum install python-rados
+
+To install ``librados`` development support files for Python on SLE/openSUSE
+distributions, execute the following:
+
+.. prompt:: bash $
+
+   sudo zypper install python3-rados
+
+You can find the module under ``/usr/share/pyshared`` on Debian systems,
+or under ``/usr/lib/python*/site-packages`` on CentOS/RHEL systems.
+
+
+Getting librados for Java
+-------------------------
+
+To install ``librados`` for Java, you need to execute the following procedure:
+
+#. Install ``jna.jar``. For Debian/Ubuntu, execute:
+
+   .. prompt:: bash $
+
+      sudo apt-get install libjna-java
+
+   For CentOS/RHEL, execute:
+
+   .. prompt:: bash $
+
+      sudo yum install jna
+
+   The JAR files are located in ``/usr/share/java``.
+
+#. Clone the ``rados-java`` repository:
+
+   .. prompt:: bash $
+
+      git clone --recursive https://github.com/ceph/rados-java.git
+
+#. Build the ``rados-java`` repository:
+
+   .. prompt:: bash $
+
+      cd rados-java
+      ant
+
+   The JAR file is located under ``rados-java/target``.
+
+#. Copy the JAR for RADOS to a common location (e.g., ``/usr/share/java``) and 
+   ensure that it and the JNA JAR are in your JVM's classpath. For example:
+
+   .. prompt:: bash $
+
+      sudo cp target/rados-0.1.3.jar /usr/share/java/rados-0.1.3.jar
+      sudo ln -s /usr/share/java/jna-3.2.7.jar /usr/lib/jvm/default-java/jre/lib/ext/jna-3.2.7.jar  
+      sudo ln -s /usr/share/java/rados-0.1.3.jar  /usr/lib/jvm/default-java/jre/lib/ext/rados-0.1.3.jar
+
+To build the documentation, execute the following:
+
+.. prompt:: bash $
+
+   ant docs
+
+
+Getting librados for PHP
+-------------------------
+
+To install the ``librados`` extension for PHP, you need to execute the following procedure:
+
+#. Install php-dev. For Debian/Ubuntu, execute:
+
+   .. prompt:: bash $
+
+      sudo apt-get install php5-dev build-essential
+
+   For CentOS/RHEL, execute:
+
+   .. prompt:: bash $
+
+      sudo yum install php-devel
+
+#. Clone the ``phprados`` repository:
+
+   .. prompt:: bash $
+
+      git clone https://github.com/ceph/phprados.git
+
+#. Build ``phprados``:
+
+   .. prompt:: bash $
+
+      cd phprados
+      phpize
+      ./configure
+      make
+      sudo make install
+
+#. Enable ``phprados`` by adding the following line to ``php.ini``::
+   
+      extension=rados.so
+
+
+Step 2: Configuring a Cluster Handle
+====================================
+
+A :term:`Ceph Client`, via ``librados``, interacts directly with OSDs to store
+and retrieve data. To interact with OSDs, the client app must invoke
+``librados``  and connect to a Ceph Monitor. Once connected, ``librados``
+retrieves the  :term:`Cluster Map` from the Ceph Monitor. When the client app
+wants to read or write data, it creates an I/O context and binds to a
+:term:`Pool`. The pool has an associated :term:`CRUSH rule` that defines how it
+will place data in the storage cluster. Via the I/O context, the client 
+provides the object name to ``librados``, which takes the object name
+and the cluster map (i.e., the topology of the cluster) and `computes`_ the
+placement group and `OSD`_  for locating the data. Then the client application
+can read or write data. The client app doesn't need to learn about the topology
+of the cluster directly.
+
+.. ditaa::
+            +--------+  Retrieves  +---------------+
+            | Client |------------>|  Cluster Map  |
+            +--------+             +---------------+
+                 |
+                 v      Writes
+              /-----\
+              | obj |
+              \-----/
+                 |      To
+                 v
+            +--------+           +---------------+
+            |  Pool  |---------->| CRUSH Rule |
+            +--------+  Selects  +---------------+
+
+
+The Ceph Storage Cluster handle encapsulates the client configuration, including:
+
+- The `user ID`_ for ``rados_create()`` or user name for ``rados_create2()`` 
+  (preferred).
+- The :term:`cephx` authentication key
+- The monitor ID and IP address
+- Logging levels
+- Debugging levels
+
+Thus, the first steps in using the cluster from your app are to 1) create
+a cluster handle that your app will use to connect to the storage cluster,
+and then 2) use that handle to connect. To connect to the cluster, the
+app must supply a monitor address, a username and an authentication key
+(cephx is enabled by default).
+
+.. tip:: Talking to different Ceph Storage Clusters – or to the same cluster 
+   with different users – requires different cluster handles.
+
+RADOS provides a number of ways for you to set the required values. For
+the monitor and encryption key settings, an easy way to handle them is to ensure
+that your Ceph configuration file contains a ``keyring`` path to a keyring file
+and at least one monitor address (e.g., ``mon host``). For example::
+
+	[global]
+	mon host = 192.168.1.1
+	keyring = /etc/ceph/ceph.client.admin.keyring
+
+Once you create the handle, you can read a Ceph configuration file to configure
+the handle. You can also pass arguments to your app and parse them with the
+function for parsing command line arguments (e.g., ``rados_conf_parse_argv()``),
+or parse Ceph environment variables (e.g., ``rados_conf_parse_env()``). Some
+wrappers may not implement convenience methods, so you may need to implement
+these capabilities. The following diagram provides a high-level flow for the
+initial connection.
+
+
+.. ditaa::
+           +---------+     +---------+
+           | Client  |     | Monitor |
+           +---------+     +---------+
+                |               |
+                |-----+ create  |
+                |     | cluster |
+                |<----+ handle  |
+                |               |
+                |-----+ read    |
+                |     | config  |
+                |<----+ file    |
+                |               |
+                |    connect    |
+                |-------------->|
+                |               |
+                |<--------------|
+                |   connected   |
+                |               |
+
+
+Once connected, your app can invoke functions that affect the whole cluster
+with only the cluster handle. For example, once you have a cluster
+handle, you can:
+
+- Get cluster statistics
+- Use Pool Operation (exists, create, list, delete)
+- Get and set the configuration
+
+
+One of the powerful features of Ceph is the ability to bind to different pools.
+Each pool may have a different number of placement groups, object replicas and
+replication strategies. For example, a pool could be set up as a "hot" pool that
+uses SSDs for frequently used objects or a "cold" pool that uses erasure coding.
+
+The main difference in the various ``librados`` bindings is between C and
+the object-oriented bindings for C++, Java and Python. The object-oriented
+bindings use objects to represent cluster handles, IO Contexts, iterators,
+exceptions, etc.
+
+
+C Example
+---------
+
+For C, creating a simple cluster handle using the ``admin`` user, configuring
+it and connecting to the cluster might look something like this: 
+
+.. code-block:: c
+
+	#include <stdio.h>
+	#include <stdlib.h>
+	#include <string.h>
+	#include <rados/librados.h>
+
+	int main (int argc, const char **argv) 
+	{
+
+		/* Declare the cluster handle and required arguments. */
+		rados_t cluster;
+		char cluster_name[] = "ceph";
+		char user_name[] = "client.admin";
+		uint64_t flags = 0;
+	
+		/* Initialize the cluster handle with the "ceph" cluster name and the "client.admin" user */  
+		int err;
+		err = rados_create2(&cluster, cluster_name, user_name, flags);
+
+		if (err < 0) {
+			fprintf(stderr, "%s: Couldn't create the cluster handle! %s\n", argv[0], strerror(-err));
+			exit(EXIT_FAILURE);
+		} else {
+			printf("\nCreated a cluster handle.\n");
+		}
+
+
+		/* Read a Ceph configuration file to configure the cluster handle. */
+		err = rados_conf_read_file(cluster, "/etc/ceph/ceph.conf");
+		if (err < 0) {
+			fprintf(stderr, "%s: cannot read config file: %s\n", argv[0], strerror(-err));
+			exit(EXIT_FAILURE);
+		} else {
+			printf("\nRead the config file.\n");
+		}
+
+		/* Read command line arguments */
+		err = rados_conf_parse_argv(cluster, argc, argv);
+		if (err < 0) {
+			fprintf(stderr, "%s: cannot parse command line arguments: %s\n", argv[0], strerror(-err));
+			exit(EXIT_FAILURE);
+		} else {
+			printf("\nRead the command line arguments.\n");
+		}
+
+		/* Connect to the cluster */
+		err = rados_connect(cluster);
+		if (err < 0) {
+			fprintf(stderr, "%s: cannot connect to cluster: %s\n", argv[0], strerror(-err));
+			exit(EXIT_FAILURE);
+		} else {
+			printf("\nConnected to the cluster.\n");
+		}
+
+	}
+
+Compile your client and link to ``librados`` using ``-lrados``. For example:
+
+.. prompt:: bash $
+
+   gcc ceph-client.c -lrados -o ceph-client
+
+
+C++ Example
+-----------
+
+The Ceph project provides a C++ example in the ``ceph/examples/librados``
+directory. For C++, a simple cluster handle using the ``admin`` user requires
+you to initialize a ``librados::Rados`` cluster handle object:
+
+.. code-block:: c++
+
+	#include <iostream>
+	#include <string>
+	#include <rados/librados.hpp>
+
+	int main(int argc, const char **argv)
+	{
+
+		int ret = 0;
+
+		/* Declare the cluster handle and required variables. */	
+		librados::Rados cluster;
+		char cluster_name[] = "ceph";
+		char user_name[] = "client.admin";
+		uint64_t flags = 0; 
+	
+		/* Initialize the cluster handle with the "ceph" cluster name and "client.admin" user */ 
+		{
+			ret = cluster.init2(user_name, cluster_name, flags);
+			if (ret < 0) {
+				std::cerr << "Couldn't initialize the cluster handle! error " << ret << std::endl;
+				return EXIT_FAILURE;
+			} else {
+				std::cout << "Created a cluster handle." << std::endl;
+			}
+		}
+
+		/* Read a Ceph configuration file to configure the cluster handle. */	
+		{	
+			ret = cluster.conf_read_file("/etc/ceph/ceph.conf");	
+			if (ret < 0) {
+				std::cerr << "Couldn't read the Ceph configuration file! error " << ret << std::endl;
+				return EXIT_FAILURE;
+			} else {
+				std::cout << "Read the Ceph configuration file." << std::endl;
+			}
+		}
+		
+		/* Read command line arguments */
+		{
+			ret = cluster.conf_parse_argv(argc, argv);
+			if (ret < 0) {
+				std::cerr << "Couldn't parse command line options! error " << ret << std::endl;
+				return EXIT_FAILURE;
+			} else {
+				std::cout << "Parsed command line options." << std::endl;
+			}
+		}
+	
+		/* Connect to the cluster */
+		{
+			ret = cluster.connect();
+			if (ret < 0) {
+				std::cerr << "Couldn't connect to cluster! error " << ret << std::endl;
+				return EXIT_FAILURE;
+			} else {
+				std::cout << "Connected to the cluster." << std::endl;
+			}
+		}
+	
+		return 0;
+	}
+	
+
+Compile the source; then, link ``librados`` using ``-lrados``. 
+For example:
+
+.. prompt:: bash $
+
+   g++ -g -c ceph-client.cc -o ceph-client.o
+   g++ -g ceph-client.o -lrados -o ceph-client
+
+
+
+Python Example
+--------------
+
+Python uses the ``admin`` id and the ``ceph`` cluster name by default, and
+will read the standard ``ceph.conf`` file if the conffile parameter is
+set to the empty string. The Python binding converts C++ errors
+into exceptions.
+
+
+.. code-block:: python
+
+	import rados
+
+	try:
+		cluster = rados.Rados(conffile='')
+	except TypeError as e:
+		print 'Argument validation error: ', e
+		raise e
+		
+	print "Created cluster handle."
+
+	try:
+		cluster.connect()
+	except Exception as e:
+		print "connection error: ", e
+		raise e
+	finally:
+		print "Connected to the cluster."
+
+
+Execute the example to verify that it connects to your cluster:
+
+.. prompt:: bash $
+
+   python ceph-client.py
+
+
+Java Example
+------------
+
+Java requires you to specify the user ID (``admin``) or user name
+(``client.admin``), and uses the ``ceph`` cluster name by default . The Java
+binding converts C++-based errors into exceptions.
+
+.. code-block:: java
+
+	import com.ceph.rados.Rados;
+	import com.ceph.rados.RadosException;
+	
+	import java.io.File;
+	
+	public class CephClient {
+		public static void main (String args[]){
+	
+			try {
+				Rados cluster = new Rados("admin");
+				System.out.println("Created cluster handle.");
+	            
+				File f = new File("/etc/ceph/ceph.conf");
+				cluster.confReadFile(f);
+				System.out.println("Read the configuration file.");
+
+				cluster.connect();
+				System.out.println("Connected to the cluster.");            
+
+			} catch (RadosException e) {
+				System.out.println(e.getMessage() + ": " + e.getReturnValue());
+			}
+		}
+	}
+
+
+Compile the source; then, run it. If you have copied the JAR to
+``/usr/share/java`` and sym linked from your ``ext`` directory, you won't need
+to specify the classpath. For example:
+
+.. prompt:: bash $
+
+   javac CephClient.java
+   java CephClient
+
+
+PHP Example
+------------
+
+With the RADOS extension enabled in PHP you can start creating a new cluster handle very easily:
+
+.. code-block:: php
+
+	<?php
+
+	$r = rados_create();
+	rados_conf_read_file($r, '/etc/ceph/ceph.conf');
+	if (!rados_connect($r)) {
+		echo "Failed to connect to Ceph cluster";
+	} else {
+		echo "Successfully connected to Ceph cluster";
+	}
+
+
+Save this as rados.php and run the code:
+
+.. prompt:: bash $
+
+   php rados.php
+
+
+Step 3: Creating an I/O Context
+===============================
+
+Once your app has a cluster handle and a connection to a Ceph Storage Cluster,
+you may create an I/O Context and begin reading and writing data. An I/O Context
+binds the connection to a specific pool. The user must have appropriate
+`CAPS`_ permissions to access the specified pool. For example, a user with read
+access but not write access will only be able to read data. I/O Context 
+functionality includes:
+
+- Write/read data and extended attributes
+- List and iterate over objects and extended attributes
+- Snapshot pools, list snapshots, etc.
+
+
+.. ditaa::
+           +---------+     +---------+     +---------+
+           | Client  |     | Monitor |     |   OSD   |
+           +---------+     +---------+     +---------+
+                |               |               |
+                |-----+ create  |               |
+                |     | I/O     |               | 
+                |<----+ context |               |              
+                |               |               |
+                |  write data   |               |
+                |---------------+-------------->|
+                |               |               |
+                |  write ack    |               |
+                |<--------------+---------------|
+                |               |               |
+                |  write xattr  |               |
+                |---------------+-------------->|
+                |               |               |
+                |  xattr ack    |               |
+                |<--------------+---------------|
+                |               |               |
+                |   read data   |               |
+                |---------------+-------------->|
+                |               |               |
+                |   read ack    |               |
+                |<--------------+---------------|
+                |               |               |
+                |  remove data  |               |
+                |---------------+-------------->|
+                |               |               |
+                |  remove ack   |               |
+                |<--------------+---------------|
+
+
+
+RADOS enables you to interact both synchronously and asynchronously. Once your
+app has an I/O Context, read/write operations only require you to know the
+object/xattr name. The CRUSH algorithm encapsulated in ``librados`` uses the
+cluster map to identify the appropriate OSD. OSD daemons handle the replication,
+as described in `Smart Daemons Enable Hyperscale`_. The ``librados`` library also 
+maps objects to placement groups, as described in  `Calculating PG IDs`_.
+
+The following examples use the default ``data`` pool. However, you may also
+use the API to list pools, ensure they exist, or create and delete pools. For 
+the write operations, the examples illustrate how to use synchronous mode. For
+the read operations, the examples illustrate how to use asynchronous mode.
+
+.. important:: Use caution when deleting pools with this API. If you delete
+   a pool, the pool and ALL DATA in the pool will be lost.
+
+
+C Example
+---------
+
+
+.. code-block:: c
+
+	#include <stdio.h>
+	#include <stdlib.h>
+	#include <string.h>
+	#include <rados/librados.h>
+
+	int main (int argc, const char **argv) 
+	{
+		/* 
+		 * Continued from previous C example, where cluster handle and
+		 * connection are established. First declare an I/O Context. 
+		 */
+
+		rados_ioctx_t io;
+		char *poolname = "data";
+	
+		err = rados_ioctx_create(cluster, poolname, &io);
+		if (err < 0) {
+			fprintf(stderr, "%s: cannot open rados pool %s: %s\n", argv[0], poolname, strerror(-err));
+			rados_shutdown(cluster);
+			exit(EXIT_FAILURE);
+		} else {
+			printf("\nCreated I/O context.\n");
+		}
+
+		/* Write data to the cluster synchronously. */	
+		err = rados_write(io, "hw", "Hello World!", 12, 0);
+		if (err < 0) {
+			fprintf(stderr, "%s: Cannot write object \"hw\" to pool %s: %s\n", argv[0], poolname, strerror(-err));
+			rados_ioctx_destroy(io);
+			rados_shutdown(cluster);
+			exit(1);
+		} else {
+			printf("\nWrote \"Hello World\" to object \"hw\".\n");
+		}
+	
+		char xattr[] = "en_US";
+		err = rados_setxattr(io, "hw", "lang", xattr, 5);
+		if (err < 0) {
+			fprintf(stderr, "%s: Cannot write xattr to pool %s: %s\n", argv[0], poolname, strerror(-err));
+			rados_ioctx_destroy(io);
+			rados_shutdown(cluster);
+			exit(1);
+		} else {
+			printf("\nWrote \"en_US\" to xattr \"lang\" for object \"hw\".\n");
+		}
+	
+		/*
+		 * Read data from the cluster asynchronously. 
+		 * First, set up asynchronous I/O completion.
+		 */
+		rados_completion_t comp;
+		err = rados_aio_create_completion(NULL, NULL, NULL, &comp);
+		if (err < 0) {
+			fprintf(stderr, "%s: Could not create aio completion: %s\n", argv[0], strerror(-err));
+			rados_ioctx_destroy(io);
+			rados_shutdown(cluster);
+			exit(1);
+		} else {
+			printf("\nCreated AIO completion.\n");
+		}
+
+		/* Next, read data using rados_aio_read. */
+		char read_res[100];
+		err = rados_aio_read(io, "hw", comp, read_res, 12, 0);
+		if (err < 0) {
+			fprintf(stderr, "%s: Cannot read object. %s %s\n", argv[0], poolname, strerror(-err));
+			rados_ioctx_destroy(io);
+			rados_shutdown(cluster);
+			exit(1);
+		} else {
+			printf("\nRead object \"hw\". The contents are:\n %s \n", read_res);
+		}
+		
+		/* Wait for the operation to complete */
+		rados_aio_wait_for_complete(comp);
+		
+		/* Release the asynchronous I/O complete handle to avoid memory leaks. */
+		rados_aio_release(comp);		
+		
+	
+		char xattr_res[100];
+		err = rados_getxattr(io, "hw", "lang", xattr_res, 5);
+		if (err < 0) {
+			fprintf(stderr, "%s: Cannot read xattr. %s %s\n", argv[0], poolname, strerror(-err));
+			rados_ioctx_destroy(io);
+			rados_shutdown(cluster);
+			exit(1);
+		} else {
+			printf("\nRead xattr \"lang\" for object \"hw\". The contents are:\n %s \n", xattr_res);
+		}
+
+		err = rados_rmxattr(io, "hw", "lang");
+		if (err < 0) {
+			fprintf(stderr, "%s: Cannot remove xattr. %s %s\n", argv[0], poolname, strerror(-err));
+			rados_ioctx_destroy(io);
+			rados_shutdown(cluster);
+			exit(1);
+		} else {
+			printf("\nRemoved xattr \"lang\" for object \"hw\".\n");
+		}
+
+		err = rados_remove(io, "hw");
+		if (err < 0) {
+			fprintf(stderr, "%s: Cannot remove object. %s %s\n", argv[0], poolname, strerror(-err));
+			rados_ioctx_destroy(io);
+			rados_shutdown(cluster);
+			exit(1);
+		} else {
+			printf("\nRemoved object \"hw\".\n");
+		}
+
+	}
+
+
+
+C++ Example
+-----------
+
+
+.. code-block:: c++
+
+	#include <iostream>
+	#include <string>
+	#include <rados/librados.hpp>
+
+	int main(int argc, const char **argv)
+	{
+
+		/* Continued from previous C++ example, where cluster handle and
+		 * connection are established. First declare an I/O Context. 
+		 */
+
+		librados::IoCtx io_ctx;
+		const char *pool_name = "data";
+		
+		{
+			ret = cluster.ioctx_create(pool_name, io_ctx);
+			if (ret < 0) {
+				std::cerr << "Couldn't set up ioctx! error " << ret << std::endl;
+				exit(EXIT_FAILURE);
+			} else {
+				std::cout << "Created an ioctx for the pool." << std::endl;
+			}
+		}
+		
+
+		/* Write an object synchronously. */
+		{
+			librados::bufferlist bl;
+			bl.append("Hello World!");
+			ret = io_ctx.write_full("hw", bl);
+			if (ret < 0) {
+				std::cerr << "Couldn't write object! error " << ret << std::endl;
+				exit(EXIT_FAILURE);
+			} else {
+				std::cout << "Wrote new object 'hw' " << std::endl;
+			}
+		}
+		
+		
+		/*
+		 * Add an xattr to the object.
+		 */
+		{
+			librados::bufferlist lang_bl;
+			lang_bl.append("en_US");
+			ret = io_ctx.setxattr("hw", "lang", lang_bl);
+			if (ret < 0) {
+				std::cerr << "failed to set xattr version entry! error "
+				<< ret << std::endl;
+				exit(EXIT_FAILURE);
+			} else {
+				std::cout << "Set the xattr 'lang' on our object!" << std::endl;
+			}
+		}
+		
+		
+		/*
+		 * Read the object back asynchronously.
+		 */
+		{
+			librados::bufferlist read_buf;
+			int read_len = 4194304;
+
+			//Create I/O Completion.
+			librados::AioCompletion *read_completion = librados::Rados::aio_create_completion();
+			
+			//Send read request.
+			ret = io_ctx.aio_read("hw", read_completion, &read_buf, read_len, 0);
+			if (ret < 0) {
+				std::cerr << "Couldn't start read object! error " << ret << std::endl;
+				exit(EXIT_FAILURE);
+			}
+
+			// Wait for the request to complete, and check that it succeeded.
+			read_completion->wait_for_complete();
+			ret = read_completion->get_return_value();
+			if (ret < 0) {
+				std::cerr << "Couldn't read object! error " << ret << std::endl;
+				exit(EXIT_FAILURE);
+			} else {
+				std::cout << "Read object hw asynchronously with contents.\n"
+				<< read_buf.c_str() << std::endl;
+			}
+		}
+		
+		
+		/*
+		 * Read the xattr.
+		 */
+		{
+			librados::bufferlist lang_res;
+			ret = io_ctx.getxattr("hw", "lang", lang_res);
+			if (ret < 0) {
+				std::cerr << "failed to get xattr version entry! error "
+				<< ret << std::endl;
+				exit(EXIT_FAILURE);
+			} else {
+				std::cout << "Got the xattr 'lang' from object hw!"
+				<< lang_res.c_str() << std::endl;
+			}
+		}
+		
+		
+		/*
+		 * Remove the xattr.
+		 */
+		{
+			ret = io_ctx.rmxattr("hw", "lang");
+			if (ret < 0) {
+				std::cerr << "Failed to remove xattr! error "
+				<< ret << std::endl;
+				exit(EXIT_FAILURE);
+			} else {
+				std::cout << "Removed the xattr 'lang' from our object!" << std::endl;
+			}
+		}
+		
+		/*
+		 * Remove the object.
+		 */
+		{
+			ret = io_ctx.remove("hw");
+			if (ret < 0) {
+				std::cerr << "Couldn't remove object! error " << ret << std::endl;
+				exit(EXIT_FAILURE);
+			} else {
+				std::cout << "Removed object 'hw'." << std::endl;
+			}
+		}
+	}
+
+
+
+Python Example
+--------------
+
+.. code-block:: python
+
+	print "\n\nI/O Context and Object Operations"
+	print "================================="
+	
+	print "\nCreating a context for the 'data' pool"
+	if not cluster.pool_exists('data'):
+		raise RuntimeError('No data pool exists')
+	ioctx = cluster.open_ioctx('data')
+	
+	print "\nWriting object 'hw' with contents 'Hello World!' to pool 'data'."
+	ioctx.write("hw", "Hello World!")
+	print "Writing XATTR 'lang' with value 'en_US' to object 'hw'"
+	ioctx.set_xattr("hw", "lang", "en_US")
+	
+	
+	print "\nWriting object 'bm' with contents 'Bonjour tout le monde!' to pool 'data'."
+	ioctx.write("bm", "Bonjour tout le monde!")
+	print "Writing XATTR 'lang' with value 'fr_FR' to object 'bm'"
+	ioctx.set_xattr("bm", "lang", "fr_FR")
+	
+	print "\nContents of object 'hw'\n------------------------"
+	print ioctx.read("hw")
+	
+	print "\n\nGetting XATTR 'lang' from object 'hw'"
+	print ioctx.get_xattr("hw", "lang")
+	
+	print "\nContents of object 'bm'\n------------------------"
+	print ioctx.read("bm")
+	
+	print "Getting XATTR 'lang' from object 'bm'"
+	print ioctx.get_xattr("bm", "lang")
+	
+	
+	print "\nRemoving object 'hw'"
+	ioctx.remove_object("hw")
+	
+	print "Removing object 'bm'"
+	ioctx.remove_object("bm")
+
+
+Java-Example
+------------
+
+.. code-block:: java
+
+	import com.ceph.rados.Rados;
+	import com.ceph.rados.RadosException;
+
+	import java.io.File;
+	import com.ceph.rados.IoCTX;
+
+	public class CephClient {
+        	public static void main (String args[]){
+
+                	try {
+				Rados cluster = new Rados("admin");
+				System.out.println("Created cluster handle.");
+
+                        	File f = new File("/etc/ceph/ceph.conf");
+                        	cluster.confReadFile(f);
+                        	System.out.println("Read the configuration file.");
+
+                        	cluster.connect();
+                        	System.out.println("Connected to the cluster.");
+
+				IoCTX io = cluster.ioCtxCreate("data");
+
+				String oidone = "hw";
+				String contentone = "Hello World!";
+				io.write(oidone, contentone); 
+
+				String oidtwo = "bm";
+				String contenttwo = "Bonjour tout le monde!";
+				io.write(oidtwo, contenttwo); 
+
+				String[] objects = io.listObjects();
+                       		for (String object: objects)
+					System.out.println(object);
+
+				io.remove(oidone);
+				io.remove(oidtwo);
+
+				cluster.ioCtxDestroy(io);
+
+                	} catch (RadosException e) {
+                        	System.out.println(e.getMessage() + ": " + e.getReturnValue());
+                	}
+        	}
+	}
+
+
+PHP Example
+-----------
+
+.. code-block:: php
+
+	<?php
+
+	$io = rados_ioctx_create($r, "mypool");
+	rados_write_full($io, "oidOne", "mycontents");
+	rados_remove("oidOne");
+	rados_ioctx_destroy($io);
+
+
+Step 4: Closing Sessions
+========================
+
+Once your app finishes with the I/O Context and cluster handle, the app should
+close the connection and shutdown the handle. For asynchronous I/O, the app
+should also ensure that pending asynchronous operations have completed.
+
+
+C Example
+---------
+
+.. code-block:: c
+
+	rados_ioctx_destroy(io);
+	rados_shutdown(cluster);	
+
+
+C++ Example
+-----------
+
+.. code-block:: c++
+
+	io_ctx.close();
+	cluster.shutdown();
+
+
+Java Example
+--------------
+
+.. code-block:: java
+
+	cluster.ioCtxDestroy(io);
+	cluster.shutDown();
+	
+	
+Python Example
+--------------
+
+.. code-block:: python
+
+	print "\nClosing the connection."
+	ioctx.close()
+	
+	print "Shutting down the handle."
+	cluster.shutdown()
+
+PHP Example
+-----------
+
+.. code-block:: php
+
+	rados_shutdown($r);
+
+
+
+.. _user ID: ../../operations/user-management#command-line-usage
+.. _CAPS: ../../operations/user-management#authorization-capabilities
+.. _Installation (Quick): ../../../start
+.. _Smart Daemons Enable Hyperscale: ../../../architecture#smart-daemons-enable-hyperscale
+.. _Calculating PG IDs: ../../../architecture#calculating-pg-ids
+.. _computes: ../../../architecture#calculating-pg-ids
+.. _OSD: ../../../architecture#mapping-pgs-to-osds
diff --git a/doc/rados/api/librados.rst b/doc/rados/api/librados.rst
new file mode 100644
index 000000000..3e202bd4b
--- /dev/null
+++ b/doc/rados/api/librados.rst
@@ -0,0 +1,187 @@
+==============
+ Librados (C)
+==============
+
+.. highlight:: c
+
+`librados` provides low-level access to the RADOS service. For an
+overview of RADOS, see :doc:`../../architecture`.
+
+
+Example: connecting and writing an object
+=========================================
+
+To use `Librados`, you instantiate a :c:type:`rados_t` variable (a cluster handle) and
+call :c:func:`rados_create()` with a pointer to it::
+
+	int err;
+	rados_t cluster;
+
+	err = rados_create(&cluster, NULL);
+	if (err < 0) {
+		fprintf(stderr, "%s: cannot create a cluster handle: %s\n", argv[0], strerror(-err));
+		exit(1);
+	}
+
+Then you configure your :c:type:`rados_t` to connect to your cluster,
+either by setting individual values (:c:func:`rados_conf_set()`),
+using a configuration file (:c:func:`rados_conf_read_file()`), using
+command line options (:c:func:`rados_conf_parse_argv`), or an
+environment variable (:c:func:`rados_conf_parse_env()`)::
+
+	err = rados_conf_read_file(cluster, "/path/to/myceph.conf");
+	if (err < 0) {
+		fprintf(stderr, "%s: cannot read config file: %s\n", argv[0], strerror(-err));
+		exit(1);
+	}
+
+Once the cluster handle is configured, you can connect to the cluster with :c:func:`rados_connect()`::
+
+	err = rados_connect(cluster);
+	if (err < 0) {
+		fprintf(stderr, "%s: cannot connect to cluster: %s\n", argv[0], strerror(-err));
+		exit(1);
+	}
+
+Then you open an "IO context", a :c:type:`rados_ioctx_t`, with :c:func:`rados_ioctx_create()`::
+
+	rados_ioctx_t io;
+	char *poolname = "mypool";
+
+	err = rados_ioctx_create(cluster, poolname, &io);
+	if (err < 0) {
+		fprintf(stderr, "%s: cannot open rados pool %s: %s\n", argv[0], poolname, strerror(-err));
+		rados_shutdown(cluster);
+		exit(1);
+	}
+
+Note that the pool you try to access must exist.
+
+Then you can use the RADOS data manipulation functions, for example
+write into an object called ``greeting`` with
+:c:func:`rados_write_full()`::
+
+	err = rados_write_full(io, "greeting", "hello", 5);
+	if (err < 0) {
+		fprintf(stderr, "%s: cannot write pool %s: %s\n", argv[0], poolname, strerror(-err));
+		rados_ioctx_destroy(io);
+		rados_shutdown(cluster);
+		exit(1);
+	}
+
+In the end, you will want to close your IO context and connection to RADOS with :c:func:`rados_ioctx_destroy()` and :c:func:`rados_shutdown()`::
+
+	rados_ioctx_destroy(io);
+	rados_shutdown(cluster);
+
+
+Asynchronous IO
+===============
+
+When doing lots of IO, you often don't need to wait for one operation
+to complete before starting the next one. `Librados` provides
+asynchronous versions of several operations:
+
+* :c:func:`rados_aio_write`
+* :c:func:`rados_aio_append`
+* :c:func:`rados_aio_write_full`
+* :c:func:`rados_aio_read`
+
+For each operation, you must first create a
+:c:type:`rados_completion_t` that represents what to do when the
+operation is safe or complete by calling
+:c:func:`rados_aio_create_completion`. If you don't need anything
+special to happen, you can pass NULL::
+
+	rados_completion_t comp;
+	err = rados_aio_create_completion(NULL, NULL, NULL, &comp);
+	if (err < 0) {
+		fprintf(stderr, "%s: could not create aio completion: %s\n", argv[0], strerror(-err));
+		rados_ioctx_destroy(io);
+		rados_shutdown(cluster);
+		exit(1);
+	}
+
+Now you can call any of the aio operations, and wait for it to
+be in memory or on disk on all replicas::
+
+	err = rados_aio_write(io, "foo", comp, "bar", 3, 0);
+	if (err < 0) {
+		fprintf(stderr, "%s: could not schedule aio write: %s\n", argv[0], strerror(-err));
+		rados_aio_release(comp);
+		rados_ioctx_destroy(io);
+		rados_shutdown(cluster);
+		exit(1);
+	}
+	rados_aio_wait_for_complete(comp); // in memory
+	rados_aio_wait_for_safe(comp); // on disk
+
+Finally, we need to free the memory used by the completion with :c:func:`rados_aio_release`::
+
+	rados_aio_release(comp);
+
+You can use the callbacks to tell your application when writes are
+durable, or when read buffers are full. For example, if you wanted to
+measure the latency of each operation when appending to several
+objects, you could schedule several writes and store the ack and
+commit time in the corresponding callback, then wait for all of them
+to complete using :c:func:`rados_aio_flush` before analyzing the
+latencies::
+
+	typedef struct {
+		struct timeval start;
+		struct timeval ack_end;
+		struct timeval commit_end;
+	} req_duration;
+
+	void ack_callback(rados_completion_t comp, void *arg) {
+		req_duration *dur = (req_duration *) arg;
+		gettimeofday(&dur->ack_end, NULL);
+	}
+
+	void commit_callback(rados_completion_t comp, void *arg) {
+		req_duration *dur = (req_duration *) arg;
+		gettimeofday(&dur->commit_end, NULL);
+	}
+
+	int output_append_latency(rados_ioctx_t io, const char *data, size_t len, size_t num_writes) {
+		req_duration times[num_writes];
+		rados_completion_t comps[num_writes];
+		for (size_t i = 0; i < num_writes; ++i) {
+			gettimeofday(&times[i].start, NULL);
+			int err = rados_aio_create_completion((void*) &times[i], ack_callback, commit_callback, &comps[i]);
+			if (err < 0) {
+				fprintf(stderr, "Error creating rados completion: %s\n", strerror(-err));
+				return err;
+			}
+			char obj_name[100];
+			snprintf(obj_name, sizeof(obj_name), "foo%ld", (unsigned long)i);
+			err = rados_aio_append(io, obj_name, comps[i], data, len);
+			if (err < 0) {
+				fprintf(stderr, "Error from rados_aio_append: %s", strerror(-err));
+				return err;
+			}
+		}
+		// wait until all requests finish *and* the callbacks complete
+		rados_aio_flush(io);
+		// the latencies can now be analyzed
+		printf("Request # | Ack latency (s) | Commit latency (s)\n");
+		for (size_t i = 0; i < num_writes; ++i) {
+			// don't forget to free the completions
+			rados_aio_release(comps[i]);
+			struct timeval ack_lat, commit_lat;
+			timersub(&times[i].ack_end, &times[i].start, &ack_lat);
+			timersub(&times[i].commit_end, &times[i].start, &commit_lat);
+			printf("%9ld | %8ld.%06ld | %10ld.%06ld\n", (unsigned long) i, ack_lat.tv_sec, ack_lat.tv_usec, commit_lat.tv_sec, commit_lat.tv_usec);
+		}
+		return 0;
+	}
+
+Note that all the :c:type:`rados_completion_t` must be freed with :c:func:`rados_aio_release` to avoid leaking memory.
+
+
+API calls
+=========
+
+ .. autodoxygenfile:: rados_types.h
+ .. autodoxygenfile:: librados.h
diff --git a/doc/rados/api/libradospp.rst b/doc/rados/api/libradospp.rst
new file mode 100644
index 000000000..08483c8d4
--- /dev/null
+++ b/doc/rados/api/libradospp.rst
@@ -0,0 +1,9 @@
+==================
+ LibradosPP (C++)
+==================
+
+.. note:: The librados C++ API is not guaranteed to be API+ABI stable
+   between major releases. All applications using the librados C++ API must
+   be recompiled and relinked against a specific Ceph release.
+
+.. todo:: write me!
diff --git a/doc/rados/api/objclass-sdk.rst b/doc/rados/api/objclass-sdk.rst
new file mode 100644
index 000000000..6b1162fd4
--- /dev/null
+++ b/doc/rados/api/objclass-sdk.rst
@@ -0,0 +1,37 @@
+===========================
+SDK for Ceph Object Classes
+===========================
+
+`Ceph` can be extended by creating shared object classes called `Ceph Object 
+Classes`. The existing framework to build these object classes has dependencies 
+on the internal functionality of `Ceph`, which restricts users to build object 
+classes within the tree. The aim of this project is to create an independent 
+object class interface, which can be used to build object classes outside the 
+`Ceph` tree. This allows us to have two types of object classes, 1) those that 
+have in-tree dependencies and reside in the tree and 2) those that can make use 
+of the `Ceph Object Class SDK framework` and can be built outside of the `Ceph` 
+tree because they do not depend on any internal implementation of `Ceph`. This 
+project decouples object class development from Ceph and encourages creation 
+and distribution of object classes as packages.
+
+In order to demonstrate the use of this framework, we have provided an example 
+called ``cls_sdk``, which is a very simple object class that makes use of the 
+SDK framework. This object class resides in the ``src/cls`` directory. 
+
+Installing objclass.h
+---------------------
+
+The object class interface that enables out-of-tree development of object 
+classes resides in ``src/include/rados/`` and gets installed with `Ceph` 
+installation. After running ``make install``, you should be able to see it 
+in ``<prefix>/include/rados``. ::
+
+        ls /usr/local/include/rados
+
+Using the SDK example
+---------------------
+
+The ``cls_sdk`` object class resides in ``src/cls/sdk/``. This gets built and 
+loaded into Ceph, with the Ceph build process. You can run the 
+``ceph_test_cls_sdk`` unittest, which resides in ``src/test/cls_sdk/``, 
+to test this class.
diff --git a/doc/rados/api/python.rst b/doc/rados/api/python.rst
new file mode 100644
index 000000000..0c9cb9e98
--- /dev/null
+++ b/doc/rados/api/python.rst
@@ -0,0 +1,425 @@
+===================
+ Librados (Python)
+===================
+
+The ``rados`` module is a thin Python wrapper for ``librados``.
+
+Installation
+============
+
+To install Python libraries for Ceph, see `Getting librados for Python`_.
+
+
+Getting Started
+===============
+
+You can create your own Ceph client using Python. The following tutorial will
+show you how to import the Ceph Python module, connect to a Ceph cluster,  and
+perform object operations as a ``client.admin`` user.
+
+.. note:: To use the Ceph Python bindings, you must have access to a
+   running Ceph cluster. To set one up quickly, see `Getting Started`_.
+
+First, create a Python source file for your Ceph client. ::
+   :linenos:
+
+	sudo vim client.py
+
+
+Import the Module
+-----------------
+
+To use the ``rados`` module, import it into your source file.
+
+.. code-block:: python
+   :linenos:
+
+	import rados
+
+
+Configure a Cluster Handle
+--------------------------
+
+Before connecting to the Ceph Storage Cluster, create a cluster handle. By
+default, the cluster handle assumes a cluster named ``ceph`` (i.e., the default
+for deployment tools, and our Getting Started guides too),  and a
+``client.admin`` user name. You may change these defaults to suit your needs.
+
+To connect to the Ceph Storage Cluster, your application needs to know where to
+find the  Ceph Monitor. Provide this information to your application by
+specifying the path to your Ceph configuration file, which contains the location
+of the initial Ceph monitors.
+
+.. code-block:: python
+   :linenos:
+
+	import rados, sys
+
+	#Create Handle Examples.
+	cluster = rados.Rados(conffile='ceph.conf')
+	cluster = rados.Rados(conffile=sys.argv[1])
+	cluster = rados.Rados(conffile = 'ceph.conf', conf = dict (keyring = '/path/to/keyring'))
+
+Ensure that the ``conffile`` argument provides the path and file name of your
+Ceph configuration file. You may use the ``sys`` module to avoid hard-coding the
+Ceph configuration path and file name.
+
+Your Python client also requires a client keyring. For this example, we use the
+``client.admin`` key by default. If you would like to specify the keyring when
+creating the cluster handle, you may use the ``conf`` argument. Alternatively,
+you may specify the keyring path in your Ceph configuration file. For example,
+you may add something like the following line to your Ceph configuration file::
+
+	keyring = /path/to/ceph.client.admin.keyring
+
+For additional details on modifying your configuration via Python, see `Configuration`_.
+
+
+Connect to the Cluster
+----------------------
+
+Once you have a cluster handle configured, you may connect to the cluster.
+With a connection to the cluster, you may execute methods that return
+information about the cluster.
+
+.. code-block:: python
+   :linenos:
+   :emphasize-lines: 7
+
+	import rados, sys
+
+	cluster = rados.Rados(conffile='ceph.conf')
+	print "\nlibrados version: " + str(cluster.version())
+	print "Will attempt to connect to: " + str(cluster.conf_get('mon host'))
+
+	cluster.connect()
+	print "\nCluster ID: " + cluster.get_fsid()
+
+	print "\n\nCluster Statistics"
+	print "=================="
+	cluster_stats = cluster.get_cluster_stats()
+
+	for key, value in cluster_stats.iteritems():
+		print key, value
+
+
+By default, Ceph authentication is ``on``. Your application will need to know
+the location of the keyring. The ``python-ceph`` module doesn't have the default
+location, so you need to specify the keyring path. The easiest way to specify
+the keyring is to add it to the Ceph configuration file. The following Ceph
+configuration file example uses the ``client.admin`` keyring.
+
+.. code-block:: ini
+   :linenos:
+
+	[global]
+	# ... elided configuration
+	keyring=/path/to/keyring/ceph.client.admin.keyring
+
+
+Manage Pools
+------------
+
+When connected to the cluster, the ``Rados`` API allows you to manage pools. You
+can list pools, check for the existence of a pool, create a pool and delete a
+pool.
+
+.. code-block:: python
+   :linenos:
+   :emphasize-lines: 6, 13, 18, 25
+
+	print "\n\nPool Operations"
+	print "==============="
+
+	print "\nAvailable Pools"
+	print "----------------"
+	pools = cluster.list_pools()
+
+	for pool in pools:
+		print pool
+
+	print "\nCreate 'test' Pool"
+	print "------------------"
+	cluster.create_pool('test')
+
+	print "\nPool named 'test' exists: " + str(cluster.pool_exists('test'))
+	print "\nVerify 'test' Pool Exists"
+	print "-------------------------"
+	pools = cluster.list_pools()
+
+	for pool in pools:
+		print pool
+
+	print "\nDelete 'test' Pool"
+	print "------------------"
+	cluster.delete_pool('test')
+	print "\nPool named 'test' exists: " + str(cluster.pool_exists('test'))
+
+
+
+Input/Output Context
+--------------------
+
+Reading from and writing to the Ceph Storage Cluster requires an input/output
+context (ioctx). You can create an ioctx with the ``open_ioctx()`` or
+``open_ioctx2()`` method of the ``Rados`` class. The ``ioctx_name`` parameter
+is the name of the  pool and ``pool_id`` is the ID of the pool you wish to use.
+
+.. code-block:: python
+   :linenos:
+
+	ioctx = cluster.open_ioctx('data')
+
+
+or
+
+.. code-block:: python
+   :linenos:
+
+        ioctx = cluster.open_ioctx2(pool_id)
+
+
+Once you have an I/O context, you can read/write objects, extended attributes,
+and perform a number of other operations. After you complete operations, ensure
+that you close the connection. For example:
+
+.. code-block:: python
+   :linenos:
+
+	print "\nClosing the connection."
+	ioctx.close()
+
+
+Writing, Reading and Removing Objects
+-------------------------------------
+
+Once you create an I/O context, you can write objects to the cluster. If you
+write to an object that doesn't exist, Ceph creates it. If you write to an
+object that exists, Ceph overwrites it (except when you specify a range, and
+then it only overwrites the range). You may read objects (and object ranges)
+from the cluster. You may also remove objects from the cluster. For example:
+
+.. code-block:: python
+	:linenos:
+	:emphasize-lines: 2, 5, 8
+
+	print "\nWriting object 'hw' with contents 'Hello World!' to pool 'data'."
+	ioctx.write_full("hw", "Hello World!")
+
+	print "\n\nContents of object 'hw'\n------------------------\n"
+	print ioctx.read("hw")
+
+	print "\nRemoving object 'hw'"
+	ioctx.remove_object("hw")
+
+
+Writing and Reading XATTRS
+--------------------------
+
+Once you create an object, you can write extended attributes (XATTRs) to
+the object and read XATTRs from the object. For example:
+
+.. code-block:: python
+	:linenos:
+	:emphasize-lines: 2, 5
+
+	print "\n\nWriting XATTR 'lang' with value 'en_US' to object 'hw'"
+	ioctx.set_xattr("hw", "lang", "en_US")
+
+	print "\n\nGetting XATTR 'lang' from object 'hw'\n"
+	print ioctx.get_xattr("hw", "lang")
+
+
+Listing Objects
+---------------
+
+If you want to examine the list of objects in a pool, you may
+retrieve the list of objects and iterate over them with the object iterator.
+For example:
+
+.. code-block:: python
+	:linenos:
+	:emphasize-lines: 1, 6, 7
+
+	object_iterator = ioctx.list_objects()
+
+	while True :
+
+		try :
+			rados_object = object_iterator.next()
+			print "Object contents = " + rados_object.read()
+
+		except StopIteration :
+			break
+
+The ``Object`` class provides a file-like interface to an object, allowing
+you to read and write content and extended attributes. Object operations using
+the I/O context provide additional functionality and asynchronous capabilities.
+
+
+Cluster Handle API
+==================
+
+The ``Rados`` class provides an interface into the Ceph Storage Daemon.
+
+
+Configuration
+-------------
+
+The ``Rados`` class provides methods for getting and setting configuration
+values, reading the Ceph configuration file, and parsing arguments. You
+do not need to be connected to the Ceph Storage Cluster to invoke the following
+methods. See `Storage Cluster Configuration`_ for details on settings.
+
+.. currentmodule:: rados
+.. automethod:: Rados.conf_get(option)
+.. automethod:: Rados.conf_set(option, val)
+.. automethod:: Rados.conf_read_file(path=None)
+.. automethod:: Rados.conf_parse_argv(args)
+.. automethod:: Rados.version()
+
+
+Connection Management
+---------------------
+
+Once you configure your cluster handle, you may connect to the cluster, check
+the cluster ``fsid``, retrieve cluster statistics, and disconnect (shutdown)
+from the cluster. You may also assert that the cluster handle is in a particular
+state (e.g., "configuring", "connecting", etc.).
+
+.. automethod:: Rados.connect(timeout=0)
+.. automethod:: Rados.shutdown()
+.. automethod:: Rados.get_fsid()
+.. automethod:: Rados.get_cluster_stats()
+
+.. documented manually because it raises warnings because of *args usage in the
+.. signature
+
+.. py:class:: Rados
+
+   .. py:method:: require_state(*args)
+
+      Checks if the Rados object is in a special state
+
+      :param args: Any number of states to check as separate arguments
+      :raises: :class:`RadosStateError`
+
+
+Pool Operations
+---------------
+
+To use pool operation methods, you must connect to the Ceph Storage Cluster
+first.  You may list the available pools, create a pool, check to see if a pool
+exists,  and delete a pool.
+
+.. automethod:: Rados.list_pools()
+.. automethod:: Rados.create_pool(pool_name, crush_rule=None)
+.. automethod:: Rados.pool_exists()
+.. automethod:: Rados.delete_pool(pool_name)
+
+
+CLI Commands
+------------
+
+The Ceph CLI command is internally using the following librados Python binding methods.
+
+In order to send a command, choose the correct method and choose the correct target.
+
+.. automethod:: Rados.mon_command
+.. automethod:: Rados.osd_command
+.. automethod:: Rados.mgr_command
+.. automethod:: Rados.pg_command
+
+
+Input/Output Context API
+========================
+
+To write data to and read data from the Ceph Object Store, you must create
+an Input/Output context (ioctx). The `Rados` class provides `open_ioctx()`
+and `open_ioctx2()` methods. The remaining ``ioctx`` operations involve
+invoking methods of the `Ioctx` and other classes.
+
+.. automethod:: Rados.open_ioctx(ioctx_name)
+.. automethod:: Ioctx.require_ioctx_open()
+.. automethod:: Ioctx.get_stats()
+.. automethod:: Ioctx.get_last_version()
+.. automethod:: Ioctx.close()
+
+
+.. Pool Snapshots
+.. --------------
+
+.. The Ceph Storage Cluster allows you to make a snapshot of a pool's state.
+.. Whereas, basic pool operations only require a connection to the cluster,
+.. snapshots require an I/O context.
+
+.. Ioctx.create_snap(self, snap_name)
+.. Ioctx.list_snaps(self)
+.. SnapIterator.next(self)
+.. Snap.get_timestamp(self)
+.. Ioctx.lookup_snap(self, snap_name)
+.. Ioctx.remove_snap(self, snap_name)
+
+.. not published. This doesn't seem ready yet.
+
+Object Operations
+-----------------
+
+The Ceph Storage Cluster stores data as objects. You can read and write objects
+synchronously or asynchronously. You can read and write from offsets. An object
+has a name (or key) and data.
+
+
+.. automethod:: Ioctx.aio_write(object_name, to_write, offset=0, oncomplete=None, onsafe=None)
+.. automethod:: Ioctx.aio_write_full(object_name, to_write, oncomplete=None, onsafe=None)
+.. automethod:: Ioctx.aio_append(object_name, to_append, oncomplete=None, onsafe=None)
+.. automethod:: Ioctx.write(key, data, offset=0)
+.. automethod:: Ioctx.write_full(key, data)
+.. automethod:: Ioctx.aio_flush()
+.. automethod:: Ioctx.set_locator_key(loc_key)
+.. automethod:: Ioctx.aio_read(object_name, length, offset, oncomplete)
+.. automethod:: Ioctx.read(key, length=8192, offset=0)
+.. automethod:: Ioctx.stat(key)
+.. automethod:: Ioctx.trunc(key, size)
+.. automethod:: Ioctx.remove_object(key)
+
+
+Object Extended Attributes
+--------------------------
+
+You may set extended attributes (XATTRs) on an object. You can retrieve a list
+of objects or XATTRs and iterate over them.
+
+.. automethod:: Ioctx.set_xattr(key, xattr_name, xattr_value)
+.. automethod:: Ioctx.get_xattrs(oid)
+.. automethod:: XattrIterator.__next__()
+.. automethod:: Ioctx.get_xattr(key, xattr_name)
+.. automethod:: Ioctx.rm_xattr(key, xattr_name)
+
+
+
+Object Interface
+================
+
+From an I/O context, you can retrieve a list of objects from a pool and iterate
+over them. The object interface provide makes each object look like a file, and
+you may perform synchronous operations on the  objects. For asynchronous
+operations, you should use the I/O context methods.
+
+.. automethod:: Ioctx.list_objects()
+.. automethod:: ObjectIterator.__next__()
+.. automethod:: Object.read(length = 1024*1024)
+.. automethod:: Object.write(string_to_write)
+.. automethod:: Object.get_xattrs()
+.. automethod:: Object.get_xattr(xattr_name)
+.. automethod:: Object.set_xattr(xattr_name, xattr_value)
+.. automethod:: Object.rm_xattr(xattr_name)
+.. automethod:: Object.stat()
+.. automethod:: Object.remove()
+
+
+
+
+.. _Getting Started: ../../../start
+.. _Storage Cluster Configuration: ../../configuration
+.. _Getting librados for Python: ../librados-intro#getting-librados-for-python
diff --git a/doc/rados/command/list-inconsistent-obj.json b/doc/rados/command/list-inconsistent-obj.json
new file mode 100644
index 000000000..2bdc5f74c
--- /dev/null
+++ b/doc/rados/command/list-inconsistent-obj.json
@@ -0,0 +1,237 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "object",
+  "properties": {
+    "epoch": {
+      "description": "Scrub epoch",
+      "type": "integer"
+    },
+    "inconsistents": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "object": {
+            "description": "Identify a Ceph object",
+            "type": "object",
+            "properties": {
+              "name": {
+                "type": "string"
+              },
+              "nspace": {
+                "type": "string"
+              },
+              "locator": {
+                "type": "string"
+              },
+              "version": {
+                "type": "integer",
+                "minimum": 0
+              },
+              "snap": {
+                "oneOf": [
+                  {
+                    "type": "string",
+                    "enum": [ "head", "snapdir" ]
+                  },
+                  {
+                    "type": "integer",
+                    "minimum": 0
+                  }
+                ]
+              }
+            },
+            "required": [
+              "name",
+              "nspace",
+              "locator",
+              "version",
+              "snap"
+            ]
+          },
+          "selected_object_info": {
+              "type": "object",
+              "description": "Selected object information",
+              "additionalProperties": true
+          },
+          "union_shard_errors": {
+            "description": "Union of all shard errors",
+            "type": "array",
+            "items": {
+              "enum": [
+                "missing",
+                "stat_error",
+                "read_error",
+                "data_digest_mismatch_info",
+                "omap_digest_mismatch_info",
+                "size_mismatch_info",
+                "ec_hash_error",
+                "ec_size_error",
+                "info_missing",
+                "info_corrupted",
+                "obj_size_info_mismatch",
+                "snapset_missing",
+                "snapset_corrupted",
+                "hinfo_missing",
+                "hinfo_corrupted"
+              ]
+            },
+            "minItems": 0,
+            "uniqueItems": true
+          },
+          "errors": {
+            "description": "Errors related to the analysis of this object",
+            "type": "array",
+            "items": {
+              "enum": [
+                "object_info_inconsistency",
+                "data_digest_mismatch",
+                "omap_digest_mismatch",
+                "size_mismatch",
+                "attr_value_mismatch",
+                "attr_name_mismatch",
+                "snapset_inconsistency",
+                "hinfo_inconsistency",
+		"size_too_large"
+              ]
+            },
+            "minItems": 0,
+            "uniqueItems": true
+          },
+          "shards": {
+            "description": "All found or expected shards",
+            "type": "array",
+            "items": {
+              "description": "Information about a particular shard of object",
+              "type": "object",
+              "properties": {
+                "object_info": {
+                  "oneOf": [
+                    {
+                      "type": "string"
+                    },
+                    {
+                      "type": "object",
+                      "description": "Object information",
+                      "additionalProperties": true
+                    }
+                  ]
+                },
+                "snapset": {
+                  "oneOf": [
+                    {
+                      "type": "string"
+                    },
+                    {
+                      "type": "object",
+                      "description": "Snap set information",
+                      "additionalProperties": true
+                    }
+                  ]
+                },
+                "hashinfo": {
+                  "oneOf": [
+                    {
+                      "type": "string"
+                    },
+                    {
+                      "type": "object",
+                      "description": "Erasure code hash information",
+                      "additionalProperties": true
+                    }
+                  ]
+                },
+                "shard": {
+                  "type": "integer"
+                },
+                "osd": {
+                  "type": "integer"
+                },
+                "primary": {
+                  "type": "boolean"
+                },
+                "size": {
+                  "type": "integer"
+                },
+                "omap_digest": {
+                  "description": "Hex representation (e.g. 0x1abd1234)",
+                  "type": "string"
+                },
+                "data_digest": {
+                  "description": "Hex representation (e.g. 0x1abd1234)",
+                  "type": "string"
+                },
+                "errors": {
+                  "description": "Errors with this shard",
+                  "type": "array",
+                  "items": {
+                    "enum": [
+                      "missing",
+                      "stat_error",
+                      "read_error",
+                      "data_digest_mismatch_info",
+                      "omap_digest_mismatch_info",
+                      "size_mismatch_info",
+                      "ec_hash_error",
+                      "ec_size_error",
+                      "info_missing",
+                      "info_corrupted",
+                      "obj_size_info_mismatch",
+                      "snapset_missing",
+                      "snapset_corrupted",
+                      "hinfo_missing",
+                      "hinfo_corrupted"
+                    ]
+                  },
+                  "minItems": 0,
+                  "uniqueItems": true
+                },
+                "attrs": {
+                  "description": "If any shard's attr error is set then all attrs are here",
+                  "type": "array",
+                  "items": {
+                    "description": "Information about a particular shard of object",
+                    "type": "object",
+                    "properties": {
+                      "name": {
+                        "type": "string"
+                      },
+                      "value": {
+                        "type": "string"
+                      },
+                      "Base64": {
+                        "type": "boolean"
+                      }
+                    },
+                    "required": [
+                      "name",
+                      "value",
+                      "Base64"
+                    ],
+                    "additionalProperties": false
+                  }
+                }
+              },
+              "additionalProperties": false,
+              "required": [
+                "osd",
+                "primary",
+                "errors"
+              ]
+            }
+          }
+        },
+        "required": [
+          "object",
+          "union_shard_errors",
+          "errors",
+          "shards"
+        ]
+      }
+    }
+  },
+  "required": [
+    "epoch",
+    "inconsistents"
+  ]
+}
diff --git a/doc/rados/command/list-inconsistent-snap.json b/doc/rados/command/list-inconsistent-snap.json
new file mode 100644
index 000000000..55f1d53e9
--- /dev/null
+++ b/doc/rados/command/list-inconsistent-snap.json
@@ -0,0 +1,86 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "object",
+  "properties": {
+    "epoch": {
+      "description": "Scrub epoch",
+      "type": "integer"
+    },
+    "inconsistents": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "name": {
+            "type": "string"
+          },
+          "nspace": {
+            "type": "string"
+          },
+          "locator": {
+            "type": "string"
+          },
+          "snap": {
+            "oneOf": [
+              {
+                "type": "string",
+                "enum": [
+                  "head",
+                  "snapdir"
+                ]
+              },
+              {
+                "type": "integer",
+                "minimum": 0
+              }
+            ]
+          },
+          "errors": {
+            "description": "Errors for this object's snap",
+            "type": "array",
+            "items": {
+              "enum": [
+                "snapset_missing",
+                "snapset_corrupted",
+                "info_missing",
+                "info_corrupted",
+                "snapset_error",
+                "headless",
+                "size_mismatch",
+                "extra_clones",
+                "clone_missing"
+              ]
+            },
+            "minItems": 0,
+            "uniqueItems": true
+          },
+          "missing": {
+            "description": "List of missing clones if clone_missing error set",
+            "type": "array",
+            "items": {
+              "type": "integer"
+            }
+          },
+          "extra_clones": {
+            "description": "List of extra clones if extra_clones error set",
+            "type": "array",
+            "items": {
+              "type": "integer"
+            }
+          }
+        },
+        "required": [
+          "name",
+          "nspace",
+          "locator",
+          "snap",
+          "errors"
+        ]
+      }
+    }
+  },
+  "required": [
+    "epoch",
+    "inconsistents"
+  ]
+}
diff --git a/doc/rados/configuration/auth-config-ref.rst b/doc/rados/configuration/auth-config-ref.rst
new file mode 100644
index 000000000..5cc13ff6a
--- /dev/null
+++ b/doc/rados/configuration/auth-config-ref.rst
@@ -0,0 +1,362 @@
+========================
+ Cephx Config Reference
+========================
+
+The ``cephx`` protocol is enabled by default. Cryptographic authentication has
+some computational costs, though they should generally be quite low.  If the
+network environment connecting your client and server hosts is very safe and
+you cannot afford authentication, you can turn it off. **This is not generally
+recommended**.
+
+.. note:: If you disable authentication, you are at risk of a man-in-the-middle
+   attack altering your client/server messages, which could lead to disastrous
+   security effects.
+
+For creating users, see `User Management`_. For details on the architecture
+of Cephx, see `Architecture - High Availability Authentication`_.
+
+
+Deployment Scenarios
+====================
+
+There are two main scenarios for deploying a Ceph cluster, which impact
+how you initially configure Cephx. Most first time Ceph users use
+``cephadm`` to create a cluster (easiest). For clusters using
+other deployment tools (e.g., Chef, Juju, Puppet, etc.), you will need
+to use the manual procedures or configure your deployment tool to
+bootstrap your monitor(s).
+
+Manual Deployment
+-----------------
+
+When you deploy a cluster manually, you have to bootstrap the monitor manually
+and create the ``client.admin`` user and keyring. To bootstrap monitors, follow
+the steps in `Monitor Bootstrapping`_. The steps for monitor bootstrapping are
+the logical steps you must perform when using third party deployment tools like
+Chef, Puppet,  Juju, etc.
+
+
+Enabling/Disabling Cephx
+========================
+
+Enabling Cephx requires that you have deployed keys for your monitors,
+OSDs and metadata servers. If you are simply toggling Cephx on / off,
+you do not have to repeat the bootstrapping procedures.
+
+
+Enabling Cephx
+--------------
+
+When ``cephx`` is enabled, Ceph will look for the keyring in the default search
+path, which includes ``/etc/ceph/$cluster.$name.keyring``. You can override
+this location by adding a ``keyring`` option in the ``[global]`` section of
+your `Ceph configuration`_ file, but this is not recommended.
+
+Execute the following procedures to enable ``cephx`` on a cluster with
+authentication disabled. If you (or your deployment utility) have already
+generated the keys, you may skip the steps related to generating keys.
+
+#. Create a ``client.admin`` key, and save a copy of the key for your client
+   host
+
+   .. prompt:: bash $
+
+     ceph auth get-or-create client.admin mon 'allow *' mds 'allow *' mgr 'allow *' osd 'allow *' -o /etc/ceph/ceph.client.admin.keyring
+
+   **Warning:** This will clobber any existing
+   ``/etc/ceph/client.admin.keyring`` file. Do not perform this step if a
+   deployment tool has already done it for you. Be careful!
+
+#. Create a keyring for your monitor cluster and generate a monitor
+   secret key.
+
+   .. prompt:: bash $
+
+     ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'
+
+#. Copy the monitor keyring into a ``ceph.mon.keyring`` file in every monitor's
+   ``mon data`` directory. For example, to copy it to ``mon.a`` in cluster ``ceph``,
+   use the following
+
+   .. prompt:: bash $
+
+     cp /tmp/ceph.mon.keyring /var/lib/ceph/mon/ceph-a/keyring
+
+#. Generate a secret key for every MGR, where ``{$id}`` is the MGR letter
+
+   .. prompt:: bash $
+
+      ceph auth get-or-create mgr.{$id} mon 'allow profile mgr' mds 'allow *' osd 'allow *' -o /var/lib/ceph/mgr/ceph-{$id}/keyring
+
+#. Generate a secret key for every OSD, where ``{$id}`` is the OSD number
+
+   .. prompt:: bash $
+
+      ceph auth get-or-create osd.{$id} mon 'allow rwx' osd 'allow *' -o /var/lib/ceph/osd/ceph-{$id}/keyring
+
+#. Generate a secret key for every MDS, where ``{$id}`` is the MDS letter
+
+   .. prompt:: bash $
+
+      ceph auth get-or-create mds.{$id} mon 'allow rwx' osd 'allow *' mds 'allow *' mgr 'allow profile mds' -o /var/lib/ceph/mds/ceph-{$id}/keyring
+
+#. Enable ``cephx`` authentication by setting the following options in the
+   ``[global]`` section of your `Ceph configuration`_ file
+
+   .. code-block:: ini
+
+      auth_cluster_required = cephx
+      auth_service_required = cephx
+      auth_client_required = cephx
+
+
+#. Start or restart the Ceph cluster. See `Operating a Cluster`_ for details.
+
+For details on bootstrapping a monitor manually, see `Manual Deployment`_.
+
+
+
+Disabling Cephx
+---------------
+
+The following procedure describes how to disable Cephx. If your cluster
+environment is relatively safe, you can offset the computation expense of
+running authentication. **We do not recommend it.** However, it may be easier
+during setup and/or troubleshooting to temporarily disable authentication.
+
+#. Disable ``cephx`` authentication by setting the following options in the
+   ``[global]`` section of your `Ceph configuration`_ file
+
+   .. code-block:: ini
+
+      auth_cluster_required = none
+      auth_service_required = none
+      auth_client_required = none
+
+
+#. Start or restart the Ceph cluster. See `Operating a Cluster`_ for details.
+
+
+Configuration Settings
+======================
+
+Enablement
+----------
+
+
+``auth_cluster_required``
+
+:Description: If enabled, the Ceph Storage Cluster daemons (i.e., ``ceph-mon``,
+              ``ceph-osd``, ``ceph-mds`` and ``ceph-mgr``) must authenticate with
+              each other. Valid settings are ``cephx`` or ``none``.
+
+:Type: String
+:Required: No
+:Default: ``cephx``.
+
+
+``auth_service_required``
+
+:Description: If enabled, the Ceph Storage Cluster daemons require Ceph Clients
+              to authenticate with the Ceph Storage Cluster in order to access
+              Ceph services. Valid settings are ``cephx`` or ``none``.
+
+:Type: String
+:Required: No
+:Default: ``cephx``.
+
+
+``auth_client_required``
+
+:Description: If enabled, the Ceph Client requires the Ceph Storage Cluster to
+              authenticate with the Ceph Client. Valid settings are ``cephx``
+              or ``none``.
+
+:Type: String
+:Required: No
+:Default: ``cephx``.
+
+
+.. index:: keys; keyring
+
+Keys
+----
+
+When you run Ceph with authentication enabled, ``ceph`` administrative commands
+and Ceph Clients require authentication keys to access the Ceph Storage Cluster.
+
+The most common way to provide these keys to the ``ceph`` administrative
+commands and clients is to include a Ceph keyring under the ``/etc/ceph``
+directory. For Octopus and later releases using ``cephadm``, the filename
+is usually ``ceph.client.admin.keyring`` (or ``$cluster.client.admin.keyring``).
+If you include the keyring under the ``/etc/ceph`` directory, you don't need to
+specify a ``keyring`` entry in your Ceph configuration file.
+
+We recommend copying the Ceph Storage Cluster's keyring file to nodes where you
+will run administrative commands, because it contains the ``client.admin`` key.
+
+To perform this step manually, execute the following:
+
+.. prompt:: bash $
+
+   sudo scp {user}@{ceph-cluster-host}:/etc/ceph/ceph.client.admin.keyring /etc/ceph/ceph.client.admin.keyring
+
+.. tip:: Ensure the ``ceph.keyring`` file has appropriate permissions set
+   (e.g., ``chmod 644``) on your client machine.
+
+You may specify the key itself in the Ceph configuration file using the ``key``
+setting (not recommended), or a path to a keyfile using the ``keyfile`` setting.
+
+
+``keyring``
+
+:Description: The path to the keyring file.
+:Type: String
+:Required: No
+:Default: ``/etc/ceph/$cluster.$name.keyring,/etc/ceph/$cluster.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin``
+
+
+``keyfile``
+
+:Description: The path to a key file (i.e,. a file containing only the key).
+:Type: String
+:Required: No
+:Default: None
+
+
+``key``
+
+:Description: The key (i.e., the text string of the key itself). Not recommended.
+:Type: String
+:Required: No
+:Default: None
+
+
+Daemon Keyrings
+---------------
+
+Administrative users or deployment tools  (e.g., ``cephadm``) may generate
+daemon keyrings in the same way as generating user keyrings.  By default, Ceph
+stores daemons keyrings inside their data directory. The default keyring
+locations, and the capabilities necessary for the daemon to function, are shown
+below.
+
+``ceph-mon``
+
+:Location: ``$mon_data/keyring``
+:Capabilities: ``mon 'allow *'``
+
+``ceph-osd``
+
+:Location: ``$osd_data/keyring``
+:Capabilities: ``mgr 'allow profile osd' mon 'allow profile osd' osd 'allow *'``
+
+``ceph-mds``
+
+:Location: ``$mds_data/keyring``
+:Capabilities: ``mds 'allow' mgr 'allow profile mds' mon 'allow profile mds' osd 'allow rwx'``
+
+``ceph-mgr``
+
+:Location: ``$mgr_data/keyring``
+:Capabilities: ``mon 'allow profile mgr' mds 'allow *' osd 'allow *'``
+
+``radosgw``
+
+:Location: ``$rgw_data/keyring``
+:Capabilities: ``mon 'allow rwx' osd 'allow rwx'``
+
+
+.. note:: The monitor keyring (i.e., ``mon.``) contains a key but no
+   capabilities, and is not part of the cluster ``auth`` database.
+
+The daemon data directory locations default to directories of the form::
+
+  /var/lib/ceph/$type/$cluster-$id
+
+For example, ``osd.12`` would be::
+
+  /var/lib/ceph/osd/ceph-12
+
+You can override these locations, but it is not recommended.
+
+
+.. index:: signatures
+
+Signatures
+----------
+
+Ceph performs a signature check that provides some limited protection
+against messages being tampered with in flight (e.g., by a "man in the
+middle" attack).
+
+Like other parts of Ceph authentication, Ceph provides fine-grained control so
+you can enable/disable signatures for service messages between clients and
+Ceph, and so you can enable/disable signatures for messages between Ceph daemons.
+
+Note that even with signatures enabled data is not encrypted in
+flight.
+
+``cephx_require_signatures``
+
+:Description: If set to ``true``, Ceph requires signatures on all message
+              traffic between the Ceph Client and the Ceph Storage Cluster, and
+              between daemons comprising the Ceph Storage Cluster.
+
+	      Ceph Argonaut and Linux kernel versions prior to 3.19 do
+	      not support signatures; if such clients are in use this
+	      option can be turned off to allow them to connect.
+
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``cephx_cluster_require_signatures``
+
+:Description: If set to ``true``, Ceph requires signatures on all message
+              traffic between Ceph daemons comprising the Ceph Storage Cluster.
+
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``cephx_service_require_signatures``
+
+:Description: If set to ``true``, Ceph requires signatures on all message
+              traffic between Ceph Clients and the Ceph Storage Cluster.
+
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``cephx_sign_messages``
+
+:Description: If the Ceph version supports message signing, Ceph will sign
+              all messages so they are more difficult to spoof.
+
+:Type: Boolean
+:Default: ``true``
+
+
+Time to Live
+------------
+
+``auth_service_ticket_ttl``
+
+:Description: When the Ceph Storage Cluster sends a Ceph Client a ticket for
+              authentication, the Ceph Storage Cluster assigns the ticket a
+              time to live.
+
+:Type: Double
+:Default: ``60*60``
+
+
+.. _Monitor Bootstrapping: ../../../install/manual-deployment#monitor-bootstrapping
+.. _Operating a Cluster: ../../operations/operating
+.. _Manual Deployment: ../../../install/manual-deployment
+.. _Ceph configuration: ../ceph-conf
+.. _Architecture - High Availability Authentication: ../../../architecture#high-availability-authentication
+.. _User Management: ../../operations/user-management
diff --git a/doc/rados/configuration/bluestore-config-ref.rst b/doc/rados/configuration/bluestore-config-ref.rst
new file mode 100644
index 000000000..3bfc8e295
--- /dev/null
+++ b/doc/rados/configuration/bluestore-config-ref.rst
@@ -0,0 +1,482 @@
+==========================
+BlueStore Config Reference
+==========================
+
+Devices
+=======
+
+BlueStore manages either one, two, or (in certain cases) three storage
+devices.
+
+In the simplest case, BlueStore consumes a single (primary) storage device.
+The storage device is normally used as a whole, occupying the full device that
+is managed directly by BlueStore. This *primary device* is normally identified
+by a ``block`` symlink in the data directory.
+
+The data directory is a ``tmpfs`` mount which gets populated (at boot time, or
+when ``ceph-volume`` activates it) with all the common OSD files that hold
+information about the OSD, like: its identifier, which cluster it belongs to,
+and its private keyring.
+
+It is also possible to deploy BlueStore across one or two additional devices:
+
+* A *write-ahead log (WAL) device* (identified as ``block.wal`` in the data directory) can be
+  used for BlueStore's internal journal or write-ahead log. It is only useful
+  to use a WAL device if the device is faster than the primary device (e.g.,
+  when it is on an SSD and the primary device is an HDD).
+* A *DB device* (identified as ``block.db`` in the data directory) can be used
+  for storing BlueStore's internal metadata.  BlueStore (or rather, the
+  embedded RocksDB) will put as much metadata as it can on the DB device to
+  improve performance.  If the DB device fills up, metadata will spill back
+  onto the primary device (where it would have been otherwise).  Again, it is
+  only helpful to provision a DB device if it is faster than the primary
+  device.
+
+If there is only a small amount of fast storage available (e.g., less
+than a gigabyte), we recommend using it as a WAL device.  If there is
+more, provisioning a DB device makes more sense.  The BlueStore
+journal will always be placed on the fastest device available, so
+using a DB device will provide the same benefit that the WAL device
+would while *also* allowing additional metadata to be stored there (if
+it will fit).  This means that if a DB device is specified but an explicit
+WAL device is not, the WAL will be implicitly colocated with the DB on the faster
+device.
+
+A single-device (colocated) BlueStore OSD can be provisioned with:
+
+.. prompt:: bash $
+
+   ceph-volume lvm prepare --bluestore --data <device>
+
+To specify a WAL device and/or DB device:
+   
+.. prompt:: bash $
+
+   ceph-volume lvm prepare --bluestore --data <device> --block.wal <wal-device> --block.db <db-device>
+
+.. note:: ``--data`` can be a Logical Volume using  *vg/lv* notation. Other
+          devices can be existing logical volumes or GPT partitions.
+
+Provisioning strategies
+-----------------------
+Although there are multiple ways to deploy a BlueStore OSD (unlike Filestore
+which had just one), there are two common arrangements that should help clarify
+the deployment strategy:
+
+.. _bluestore-single-type-device-config:
+
+**block (data) only**
+^^^^^^^^^^^^^^^^^^^^^
+If all devices are the same type, for example all rotational drives, and
+there are no fast devices to use for metadata, it makes sense to specify the
+block device only and to not separate ``block.db`` or ``block.wal``. The
+:ref:`ceph-volume-lvm` command for a single ``/dev/sda`` device looks like:
+
+.. prompt:: bash $
+
+   ceph-volume lvm create --bluestore --data /dev/sda
+
+If logical volumes have already been created for each device, (a single LV
+using 100% of the device), then the :ref:`ceph-volume-lvm` call for an LV named
+``ceph-vg/block-lv`` would look like:
+
+.. prompt:: bash $
+
+   ceph-volume lvm create --bluestore --data ceph-vg/block-lv
+
+.. _bluestore-mixed-device-config:
+
+**block and block.db**
+^^^^^^^^^^^^^^^^^^^^^^
+If you have a mix of fast and slow devices (SSD / NVMe and rotational),
+it is recommended to place ``block.db`` on the faster device while ``block``
+(data) lives on the slower (spinning drive).
+
+You must create these volume groups and logical volumes manually as 
+the ``ceph-volume`` tool is currently not able to do so automatically.
+
+For the below example, let us assume four rotational (``sda``, ``sdb``, ``sdc``, and ``sdd``)
+and one (fast) solid state drive (``sdx``). First create the volume groups:
+
+.. prompt:: bash $
+
+   vgcreate ceph-block-0 /dev/sda
+   vgcreate ceph-block-1 /dev/sdb
+   vgcreate ceph-block-2 /dev/sdc
+   vgcreate ceph-block-3 /dev/sdd
+
+Now create the logical volumes for ``block``:
+
+.. prompt:: bash $
+
+   lvcreate -l 100%FREE -n block-0 ceph-block-0
+   lvcreate -l 100%FREE -n block-1 ceph-block-1
+   lvcreate -l 100%FREE -n block-2 ceph-block-2
+   lvcreate -l 100%FREE -n block-3 ceph-block-3
+
+We are creating 4 OSDs for the four slow spinning devices, so assuming a 200GB
+SSD in ``/dev/sdx`` we will create 4 logical volumes, each of 50GB:
+
+.. prompt:: bash $
+
+   vgcreate ceph-db-0 /dev/sdx
+   lvcreate -L 50GB -n db-0 ceph-db-0
+   lvcreate -L 50GB -n db-1 ceph-db-0
+   lvcreate -L 50GB -n db-2 ceph-db-0
+   lvcreate -L 50GB -n db-3 ceph-db-0
+
+Finally, create the 4 OSDs with ``ceph-volume``:
+
+.. prompt:: bash $
+
+   ceph-volume lvm create --bluestore --data ceph-block-0/block-0 --block.db ceph-db-0/db-0
+   ceph-volume lvm create --bluestore --data ceph-block-1/block-1 --block.db ceph-db-0/db-1
+   ceph-volume lvm create --bluestore --data ceph-block-2/block-2 --block.db ceph-db-0/db-2
+   ceph-volume lvm create --bluestore --data ceph-block-3/block-3 --block.db ceph-db-0/db-3
+
+These operations should end up creating four OSDs, with ``block`` on the slower
+rotational drives with a 50 GB logical volume (DB) for each on the solid state
+drive.
+
+Sizing
+======
+When using a :ref:`mixed spinning and solid drive setup
+<bluestore-mixed-device-config>` it is important to make a large enough
+``block.db`` logical volume for BlueStore. Generally, ``block.db`` should have
+*as large as possible* logical volumes.
+
+The general recommendation is to have ``block.db`` size in between 1% to 4%
+of ``block`` size. For RGW workloads, it is recommended that the ``block.db``
+size isn't smaller than 4% of ``block``, because RGW heavily uses it to store
+metadata (omap keys). For example, if the ``block`` size is 1TB, then ``block.db`` shouldn't
+be less than 40GB. For RBD workloads, 1% to 2% of ``block`` size is usually enough.
+
+In older releases, internal level sizes mean that the DB can fully utilize only
+specific partition / LV sizes that correspond to sums of L0, L0+L1, L1+L2,
+etc. sizes, which with default settings means roughly 3 GB, 30 GB, 300 GB, and
+so forth.  Most deployments will not substantially benefit from sizing to
+accommodate L3 and higher, though DB compaction can be facilitated by doubling
+these figures to 6GB, 60GB, and 600GB.
+
+Improvements in releases beginning with Nautilus 14.2.12 and Octopus 15.2.6
+enable better utilization of arbitrary DB device sizes, and the Pacific
+release brings experimental dynamic level support.  Users of older releases may
+thus wish to plan ahead by provisioning larger DB devices today so that their
+benefits may be realized with future upgrades.
+
+When *not* using a mix of fast and slow devices, it isn't required to create
+separate logical volumes for ``block.db`` (or ``block.wal``). BlueStore will
+automatically colocate these within the space of ``block``.
+
+
+Automatic Cache Sizing
+======================
+
+BlueStore can be configured to automatically resize its caches when TCMalloc
+is configured as the memory allocator and the ``bluestore_cache_autotune``
+setting is enabled.  This option is currently enabled by default.  BlueStore
+will attempt to keep OSD heap memory usage under a designated target size via
+the ``osd_memory_target`` configuration option.  This is a best effort
+algorithm and caches will not shrink smaller than the amount specified by
+``osd_memory_cache_min``.  Cache ratios will be chosen based on a hierarchy
+of priorities.  If priority information is not available, the
+``bluestore_cache_meta_ratio`` and ``bluestore_cache_kv_ratio`` options are
+used as fallbacks.
+
+Manual Cache Sizing
+===================
+
+The amount of memory consumed by each OSD for BlueStore caches is
+determined by the ``bluestore_cache_size`` configuration option.  If
+that config option is not set (i.e., remains at 0), there is a
+different default value that is used depending on whether an HDD or
+SSD is used for the primary device (set by the
+``bluestore_cache_size_ssd`` and ``bluestore_cache_size_hdd`` config
+options).
+
+BlueStore and the rest of the Ceph OSD daemon do the best they can
+to work within this memory budget.  Note that on top of the configured
+cache size, there is also memory consumed by the OSD itself, and
+some additional utilization due to memory fragmentation and other
+allocator overhead.
+
+The configured cache memory budget can be used in a few different ways:
+
+* Key/Value metadata (i.e., RocksDB's internal cache)
+* BlueStore metadata
+* BlueStore data (i.e., recently read or written object data)
+
+Cache memory usage is governed by the following options:
+``bluestore_cache_meta_ratio`` and ``bluestore_cache_kv_ratio``.
+The fraction of the cache devoted to data
+is governed by the effective bluestore cache size (depending on
+``bluestore_cache_size[_ssd|_hdd]`` settings and the device class of the primary
+device) as well as the meta and kv ratios.
+The data fraction can be calculated by
+``<effective_cache_size> * (1 - bluestore_cache_meta_ratio - bluestore_cache_kv_ratio)``
+
+Checksums
+=========
+
+BlueStore checksums all metadata and data written to disk.  Metadata
+checksumming is handled by RocksDB and uses `crc32c`. Data
+checksumming is done by BlueStore and can make use of `crc32c`,
+`xxhash32`, or `xxhash64`.  The default is `crc32c` and should be
+suitable for most purposes.
+
+Full data checksumming does increase the amount of metadata that
+BlueStore must store and manage.  When possible, e.g., when clients
+hint that data is written and read sequentially, BlueStore will
+checksum larger blocks, but in many cases it must store a checksum
+value (usually 4 bytes) for every 4 kilobyte block of data.
+
+It is possible to use a smaller checksum value by truncating the
+checksum to two or one byte, reducing the metadata overhead.  The
+trade-off is that the probability that a random error will not be
+detected is higher with a smaller checksum, going from about one in
+four billion with a 32-bit (4 byte) checksum to one in 65,536 for a
+16-bit (2 byte) checksum or one in 256 for an 8-bit (1 byte) checksum.
+The smaller checksum values can be used by selecting `crc32c_16` or
+`crc32c_8` as the checksum algorithm.
+
+The *checksum algorithm* can be set either via a per-pool
+``csum_type`` property or the global config option.  For example:
+
+.. prompt:: bash $
+
+   ceph osd pool set <pool-name> csum_type <algorithm>
+
+Inline Compression
+==================
+
+BlueStore supports inline compression using `snappy`, `zlib`, or
+`lz4`. Please note that the `lz4` compression plugin is not
+distributed in the official release.
+
+Whether data in BlueStore is compressed is determined by a combination
+of the *compression mode* and any hints associated with a write
+operation.  The modes are:
+
+* **none**: Never compress data.
+* **passive**: Do not compress data unless the write operation has a
+  *compressible* hint set.
+* **aggressive**: Compress data unless the write operation has an
+  *incompressible* hint set.
+* **force**: Try to compress data no matter what.
+
+For more information about the *compressible* and *incompressible* IO
+hints, see :c:func:`rados_set_alloc_hint`.
+
+Note that regardless of the mode, if the size of the data chunk is not
+reduced sufficiently it will not be used and the original
+(uncompressed) data will be stored.  For example, if the ``bluestore
+compression required ratio`` is set to ``.7`` then the compressed data
+must be 70% of the size of the original (or smaller).
+
+The *compression mode*, *compression algorithm*, *compression required
+ratio*, *min blob size*, and *max blob size* can be set either via a
+per-pool property or a global config option.  Pool properties can be
+set with:
+
+.. prompt:: bash $
+
+   ceph osd pool set <pool-name> compression_algorithm <algorithm>
+   ceph osd pool set <pool-name> compression_mode <mode>
+   ceph osd pool set <pool-name> compression_required_ratio <ratio>
+   ceph osd pool set <pool-name> compression_min_blob_size <size>
+   ceph osd pool set <pool-name> compression_max_blob_size <size>
+
+.. _bluestore-rocksdb-sharding:
+
+RocksDB Sharding
+================
+
+Internally BlueStore uses multiple types of key-value data,
+stored in RocksDB.  Each data type in BlueStore is assigned a
+unique prefix. Until Pacific all key-value data was stored in
+single RocksDB column family: 'default'.  Since Pacific,
+BlueStore can divide this data into multiple RocksDB column
+families. When keys have similar access frequency, modification
+frequency and lifetime, BlueStore benefits from better caching
+and more precise compaction. This improves performance, and also
+requires less disk space during compaction, since each column
+family is smaller and can compact independent of others.
+
+OSDs deployed in Pacific or later use RocksDB sharding by default.
+If Ceph is upgraded to Pacific from a previous version, sharding is off.
+
+To enable sharding and apply the Pacific defaults, stop an OSD and run
+
+    .. prompt:: bash #
+
+      ceph-bluestore-tool \
+        --path <data path> \
+        --sharding="m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L P" \
+        reshard
+
+
+Throttling
+==========
+
+SPDK Usage
+==================
+
+If you want to use the SPDK driver for NVMe devices, you must prepare your system.
+Refer to `SPDK document`__ for more details.
+
+.. __: http://www.spdk.io/doc/getting_started.html#getting_started_examples
+
+SPDK offers a script to configure the device automatically. Users can run the
+script as root:
+
+.. prompt:: bash $
+
+   sudo src/spdk/scripts/setup.sh
+
+You will need to specify the subject NVMe device's device selector with
+the "spdk:" prefix for ``bluestore_block_path``.
+
+For example, you can find the device selector of an Intel PCIe SSD with:
+
+.. prompt:: bash $
+
+   lspci -mm -n -D -d 8086:0953
+
+The device selector always has the form of ``DDDD:BB:DD.FF`` or ``DDDD.BB.DD.FF``.
+
+and then set::
+
+  bluestore_block_path = "spdk:trtype:PCIe traddr:0000:01:00.0"
+
+Where ``0000:01:00.0`` is the device selector found in the output of ``lspci``
+command above.
+
+You may also specify a remote NVMeoF target over the TCP transport as in the
+following example::
+
+  bluestore_block_path = "spdk:trtype:TCP traddr:10.67.110.197 trsvcid:4420 subnqn:nqn.2019-02.io.spdk:cnode1"
+
+To run multiple SPDK instances per node, you must specify the
+amount of dpdk memory in MB that each instance will use, to make sure each
+instance uses its own DPDK memory.
+
+In most cases, a single device can be used for data, DB, and WAL.  We describe
+this strategy as *colocating* these components. Be sure to enter the below
+settings to ensure that all IOs are issued through SPDK.::
+
+  bluestore_block_db_path = ""
+  bluestore_block_db_size = 0
+  bluestore_block_wal_path = ""
+  bluestore_block_wal_size = 0
+
+Otherwise, the current implementation will populate the SPDK map files with
+kernel file system symbols and will use the kernel driver to issue DB/WAL IO.
+
+Minimum Allocation Size
+========================
+
+There is a configured minimum amount of storage that BlueStore will allocate on
+an OSD.  In practice, this is the least amount of capacity that a RADOS object
+can consume.  The value of `bluestore_min_alloc_size` is derived from the
+value of `bluestore_min_alloc_size_hdd` or `bluestore_min_alloc_size_ssd`
+depending on the OSD's ``rotational`` attribute.  This means that when an OSD
+is created on an HDD, BlueStore will be initialized with the current value
+of `bluestore_min_alloc_size_hdd`, and SSD OSDs (including NVMe devices)
+with the value of `bluestore_min_alloc_size_ssd`.
+
+Through the Mimic release, the default values were 64KB and 16KB for rotational
+(HDD) and non-rotational (SSD) media respectively.  Octopus changed the default
+for SSD (non-rotational) media to 4KB, and Pacific changed the default for HDD
+(rotational) media to 4KB as well.
+
+These changes were driven by space amplification experienced by Ceph RADOS
+GateWay (RGW) deployments that host large numbers of small files
+(S3/Swift objects).
+
+For example, when an RGW client stores a 1KB S3 object, it is written to a
+single RADOS object.  With the default `min_alloc_size` value, 4KB of
+underlying drive space is allocated.  This means that roughly
+(4KB - 1KB) == 3KB is allocated but never used, which corresponds to 300%
+overhead or 25% efficiency. Similarly, a 5KB user object will be stored
+as one 4KB and one 1KB RADOS object, again stranding 4KB of device capcity,
+though in this case the overhead is a much smaller percentage.  Think of this
+in terms of the remainder from a modulus operation. The overhead *percentage*
+thus decreases rapidly as user object size increases.
+
+An easily missed additional subtlety is that this
+takes place for *each* replica.  So when using the default three copies of
+data (3R), a 1KB S3 object actually consumes roughly 9KB of storage device
+capacity.  If erasure coding (EC) is used instead of replication, the
+amplification may be even higher: for a ``k=4,m=2`` pool, our 1KB S3 object
+will allocate (6 * 4KB) = 24KB of device capacity.
+
+When an RGW bucket pool contains many relatively large user objects, the effect
+of this phenomenon is often negligible, but should be considered for deployments
+that expect a signficiant fraction of relatively small objects.
+
+The 4KB default value aligns well with conventional HDD and SSD devices.  Some
+new coarse-IU (Indirection Unit) QLC SSDs however perform and wear best
+when `bluestore_min_alloc_size_ssd`
+is set at OSD creation to match the device's IU:. 8KB, 16KB, or even 64KB.
+These novel storage drives allow one to achieve read performance competitive
+with conventional TLC SSDs and write performance faster than HDDs, with
+high density and lower cost than TLC SSDs.
+
+Note that when creating OSDs on these devices, one must carefully apply the
+non-default value only to appropriate devices, and not to conventional SSD and
+HDD devices.  This may be done through careful ordering of OSD creation, custom
+OSD device classes, and especially by the use of central configuration _masks_.
+
+Quincy and later releases add
+the `bluestore_use_optimal_io_size_for_min_alloc_size`
+option that enables automatic discovery of the appropriate value as each OSD is
+created.  Note that the use of ``bcache``, ``OpenCAS``, ``dmcrypt``,
+``ATA over Ethernet``, `iSCSI`, or other device layering / abstraction
+technologies may confound the determination of appropriate values. OSDs
+deployed on top of VMware storage have been reported to also
+sometimes report a ``rotational`` attribute that does not match the underlying
+hardware.
+
+We suggest inspecting such OSDs at startup via logs and admin sockets to ensure that
+behavior is appropriate.  Note that this also may not work as desired with
+older kernels.  You can check for this by examining the presence and value
+of ``/sys/block/<drive>/queue/optimal_io_size``.
+
+You may also inspect a given OSD:
+
+.. prompt:: bash #
+
+   ceph osd metadata osd.1701 | grep rotational
+
+This space amplification may manifest as an unusually high ratio of raw to
+stored data reported by ``ceph df``.  ``ceph osd df`` may also report
+anomalously high ``%USE`` / ``VAR`` values when
+compared to other, ostensibly identical OSDs.  A pool using OSDs with
+mismatched ``min_alloc_size`` values may experience unexpected balancer
+behavior as well.
+
+Note that this BlueStore attribute takes effect *only* at OSD creation; if
+changed later, a given OSD's behavior will not change unless / until it is
+destroyed and redeployed with the appropriate option value(s).  Upgrading
+to a later Ceph release will *not* change the value used by OSDs deployed
+under older releases or with other settings.
+
+DSA (Data Streaming Accelerator Usage)
+======================================
+
+If you want to use the DML library to drive DSA device for offloading
+read/write operations on Persist memory in Bluestore. You need to install
+`DML`_ and `idxd-config`_ library in your machine with SPR (Sapphire Rapids) CPU.
+
+.. _DML: https://github.com/intel/DML
+.. _idxd-config: https://github.com/intel/idxd-config
+
+After installing the DML software, you need to configure the shared
+work queues (WQs) with the following WQ configuration example via accel-config tool:
+
+.. prompt:: bash $
+
+   accel-config config-wq --group-id=1 --mode=shared --wq-size=16 --threshold=15 --type=user --name="MyApp1" --priority=10 --block-on-fault=1 dsa0/wq0.1
+   accel-config config-engine dsa0/engine0.1 --group-id=1
+   accel-config enable-device dsa0
+   accel-config enable-wq dsa0/wq0.1
diff --git a/doc/rados/configuration/ceph-conf.rst b/doc/rados/configuration/ceph-conf.rst
new file mode 100644
index 000000000..ad93598de
--- /dev/null
+++ b/doc/rados/configuration/ceph-conf.rst
@@ -0,0 +1,689 @@
+.. _configuring-ceph:
+
+==================
+ Configuring Ceph
+==================
+
+When Ceph services start, the initialization process activates a series
+of daemons that run in the background. A :term:`Ceph Storage Cluster` runs 
+at a minimum three types of daemons:
+
+- :term:`Ceph Monitor` (``ceph-mon``)
+- :term:`Ceph Manager` (``ceph-mgr``)
+- :term:`Ceph OSD Daemon` (``ceph-osd``)
+
+Ceph Storage Clusters that support the :term:`Ceph File System` also run at
+least one :term:`Ceph Metadata Server` (``ceph-mds``). Clusters that
+support :term:`Ceph Object Storage` run Ceph RADOS Gateway daemons
+(``radosgw``) as well.
+
+Each daemon has a number of configuration options, each of which has a
+default value.  You may adjust the behavior of the system by changing these
+configuration options.  Be careful to understand the consequences before
+overriding default values, as it is possible to significantly degrade the
+performance and stability of your cluster.  Also note that default values
+sometimes change between releases, so it is best to review the version of
+this documentation that aligns with your Ceph release.
+
+Option names
+============
+
+All Ceph configuration options have a unique name consisting of words
+formed with lower-case characters and connected with underscore
+(``_``) characters.
+
+When option names are specified on the command line, either underscore
+(``_``) or dash (``-``) characters can be used interchangeable (e.g.,
+``--mon-host`` is equivalent to ``--mon_host``).
+
+When option names appear in configuration files, spaces can also be
+used in place of underscore or dash.  We suggest, though, that for
+clarity and convenience you consistently use underscores, as we do
+throughout this documentation.
+
+Config sources
+==============
+
+Each Ceph daemon, process, and library will pull its configuration
+from several sources, listed below.  Sources later in the list will
+override those earlier in the list when both are present.
+
+- the compiled-in default value
+- the monitor cluster's centralized configuration database
+- a configuration file stored on the local host
+- environment variables
+- command line arguments
+- runtime overrides set by an administrator
+
+One of the first things a Ceph process does on startup is parse the
+configuration options provided via the command line, environment, and
+local configuration file.  The process will then contact the monitor
+cluster to retrieve configuration stored centrally for the entire
+cluster.  Once a complete view of the configuration is available, the
+daemon or process startup will proceed.
+
+.. _bootstrap-options:
+
+Bootstrap options
+-----------------
+
+Because some configuration options affect the process's ability to
+contact the monitors, authenticate, and retrieve the cluster-stored
+configuration, they may need to be stored locally on the node and set
+in a local configuration file.  These options include:
+
+  - ``mon_host``, the list of monitors for the cluster
+  - ``mon_host_override``, the list of monitors for the cluster to
+    **initially** contact when beginning a new instance of communication with the
+    Ceph cluster.  This overrides the known monitor list derived from MonMap
+    updates sent to older Ceph instances (like librados cluster handles).  It is
+    expected this option is primarily useful for debugging.
+  - ``mon_dns_srv_name`` (default: `ceph-mon`), the name of the DNS
+    SRV record to check to identify the cluster monitors via DNS
+  - ``mon_data``, ``osd_data``, ``mds_data``, ``mgr_data``, and
+    similar options that define which local directory the daemon
+    stores its data in.
+  - ``keyring``, ``keyfile``, and/or ``key``, which can be used to
+    specify the authentication credential to use to authenticate with
+    the monitor.  Note that in most cases the default keyring location
+    is in the data directory specified above.
+
+In the vast majority of cases the default values of these are
+appropriate, with the exception of the ``mon_host`` option that
+identifies the addresses of the cluster's monitors.  When DNS is used
+to identify monitors a local ceph configuration file can be avoided
+entirely.
+
+Skipping monitor config
+-----------------------
+
+Any process may be passed the option ``--no-mon-config`` to skip the
+step that retrieves configuration from the cluster monitors.  This is
+useful in cases where configuration is managed entirely via
+configuration files or where the monitor cluster is currently down but
+some maintenance activity needs to be done.
+
+
+.. _ceph-conf-file:
+
+
+Configuration sections
+======================
+
+Any given process or daemon has a single value for each configuration
+option.  However, values for an option may vary across different
+daemon types even daemons of the same type.  Ceph options that are
+stored in the monitor configuration database or in local configuration
+files are grouped into sections to indicate which daemons or clients
+they apply to.
+
+These sections include:
+
+``global``
+
+:Description: Settings under ``global`` affect all daemons and clients
+              in a Ceph Storage Cluster.
+
+:Example: ``log_file = /var/log/ceph/$cluster-$type.$id.log``
+
+``mon``
+
+:Description: Settings under ``mon`` affect all ``ceph-mon`` daemons in
+              the Ceph Storage Cluster, and override the same setting in 
+              ``global``.
+
+:Example: ``mon_cluster_log_to_syslog = true``
+
+
+``mgr``
+
+:Description: Settings in the ``mgr`` section affect all ``ceph-mgr`` daemons in
+              the Ceph Storage Cluster, and override the same setting in 
+              ``global``.
+
+:Example: ``mgr_stats_period = 10``
+
+``osd``
+
+:Description: Settings under ``osd`` affect all ``ceph-osd`` daemons in
+              the Ceph Storage Cluster, and override the same setting in
+              ``global``.
+
+:Example: ``osd_op_queue = wpq``
+
+``mds``
+
+:Description: Settings in the ``mds`` section affect all ``ceph-mds`` daemons in
+              the Ceph Storage Cluster, and override the same setting in
+              ``global``.
+
+:Example: ``mds_cache_memory_limit = 10G``
+
+``client``
+
+:Description: Settings under ``client`` affect all Ceph Clients
+              (e.g., mounted Ceph File Systems, mounted Ceph Block Devices,
+              etc.) as well as Rados Gateway (RGW) daemons.
+
+:Example: ``objecter_inflight_ops = 512``
+
+
+Sections may also specify an individual daemon or client name.  For example,
+``mon.foo``, ``osd.123``, and ``client.smith`` are all valid section names.
+
+
+Any given daemon will draw its settings from the global section, the
+daemon or client type section, and the section sharing its name.
+Settings in the most-specific section take precedence, so for example
+if the same option is specified in both ``global``, ``mon``, and
+``mon.foo`` on the same source (i.e., in the same configurationfile),
+the ``mon.foo`` value will be used.
+
+If multiple values of the same configuration option are specified in the same
+section, the last value wins.
+
+Note that values from the local configuration file always take
+precedence over values from the monitor configuration database,
+regardless of which section they appear in.
+
+
+.. _ceph-metavariables:
+
+Metavariables
+=============
+
+Metavariables simplify Ceph Storage Cluster configuration
+dramatically. When a metavariable is set in a configuration value,
+Ceph expands the metavariable into a concrete value at the time the
+configuration value is used. Ceph metavariables are similar to variable expansion in the Bash shell.
+
+Ceph supports the following metavariables: 
+
+``$cluster``
+
+:Description: Expands to the Ceph Storage Cluster name. Useful when running 
+              multiple Ceph Storage Clusters on the same hardware.
+
+:Example: ``/etc/ceph/$cluster.keyring``
+:Default: ``ceph``
+
+
+``$type``
+
+:Description: Expands to a daemon or process type (e.g., ``mds``, ``osd``, or ``mon``)
+
+:Example: ``/var/lib/ceph/$type``
+
+
+``$id``
+
+:Description: Expands to the daemon or client identifier. For
+              ``osd.0``, this would be ``0``; for ``mds.a``, it would
+              be ``a``.
+
+:Example: ``/var/lib/ceph/$type/$cluster-$id``
+
+
+``$host``
+
+:Description: Expands to the host name where the process is running.
+
+
+``$name``
+
+:Description: Expands to ``$type.$id``.
+:Example: ``/var/run/ceph/$cluster-$name.asok``
+
+``$pid``
+
+:Description: Expands to daemon pid.
+:Example: ``/var/run/ceph/$cluster-$name-$pid.asok``
+
+
+
+The Configuration File
+======================
+
+On startup, Ceph processes search for a configuration file in the
+following locations:
+
+#. ``$CEPH_CONF`` (*i.e.,* the path following the ``$CEPH_CONF``
+   environment variable)
+#. ``-c path/path``  (*i.e.,* the ``-c`` command line argument)
+#. ``/etc/ceph/$cluster.conf``
+#. ``~/.ceph/$cluster.conf``
+#. ``./$cluster.conf`` (*i.e.,* in the current working directory)
+#. On FreeBSD systems only, ``/usr/local/etc/ceph/$cluster.conf``
+
+where ``$cluster`` is the cluster's name (default ``ceph``).
+
+The Ceph configuration file uses an *ini* style syntax. You can add comment
+text after a pound sign (#) or a semi-colon (;).  For example:
+
+.. code-block:: ini
+
+	# <--A number (#) sign precedes a comment.
+	; A comment may be anything.
+	# Comments always follow a semi-colon (;) or a pound (#) on each line.
+	# The end of the line terminates a comment.
+	# We recommend that you provide comments in your configuration file(s).
+
+
+.. _ceph-conf-settings:
+
+Config file section names
+-------------------------
+
+The configuration file is divided into sections. Each section must begin with a
+valid configuration section name (see `Configuration sections`_, above)
+surrounded by square brackets. For example,
+
+.. code-block:: ini
+
+	[global]
+	debug_ms = 0
+	
+	[osd]
+	debug_ms = 1
+
+	[osd.1]
+	debug_ms = 10
+
+	[osd.2]
+	debug_ms = 10
+
+
+Config file option values
+-------------------------
+
+The value of a configuration option is a string. If it is too long to
+fit in a single line, you can put a backslash (``\``) at the end of line
+as the line continuation marker, so the value of the option will be
+the string after ``=`` in current line combined with the string in the next
+line::
+
+  [global]
+  foo = long long ago\
+  long ago
+
+In the example above, the value of "``foo``" would be "``long long ago long ago``".
+
+Normally, the option value ends with a new line, or a comment, like
+
+.. code-block:: ini
+
+    [global]
+    obscure_one = difficult to explain # I will try harder in next release
+    simpler_one = nothing to explain
+
+In the example above, the value of "``obscure one``" would be "``difficult to explain``";
+and the value of "``simpler one`` would be "``nothing to explain``".
+
+If an option value contains spaces, and we want to make it explicit, we
+could quote the value using single or double quotes, like
+
+.. code-block:: ini
+
+    [global]
+    line = "to be, or not to be"
+
+Certain characters are not allowed to be present in the option values directly.
+They are ``=``, ``#``, ``;`` and ``[``. If we have to, we need to escape them,
+like
+
+.. code-block:: ini
+
+    [global]
+    secret = "i love \# and \["
+
+Every configuration option is typed with one of the types below:
+
+``int``
+
+:Description: 64-bit signed integer, Some SI prefixes are supported, like "K", "M", "G",
+              "T", "P", "E", meaning, respectively, 10\ :sup:`3`, 10\ :sup:`6`,
+              10\ :sup:`9`, etc.  And "B" is the only supported unit. So, "1K", "1M", "128B" and "-1" are all valid
+              option values. Some times, a negative value implies "unlimited" when it comes to
+              an option for threshold or limit.
+:Example: ``42``, ``-1``
+
+``uint``
+
+:Description: It is almost identical to ``integer``. But a negative value will be rejected.
+:Example: ``256``, ``0``
+
+``str``
+
+:Description: Free style strings encoded in UTF-8, but some characters are not allowed. Please
+              reference the above notes for the details.
+:Example: ``"hello world"``, ``"i love \#"``, ``yet-another-name``
+
+``boolean``
+
+:Description: one of the two values ``true`` or ``false``. But an integer is also accepted,
+              where "0" implies ``false``, and any non-zero values imply ``true``.
+:Example: ``true``, ``false``, ``1``, ``0``
+
+``addr``
+
+:Description: a single address optionally prefixed with ``v1``, ``v2`` or ``any`` for the messenger
+              protocol. If the prefix is not specified, ``v2`` protocol is used. Please see
+              :ref:`address_formats` for more details.
+:Example: ``v1:1.2.3.4:567``, ``v2:1.2.3.4:567``, ``1.2.3.4:567``, ``2409:8a1e:8fb6:aa20:1260:4bff:fe92:18f5::567``, ``[::1]:6789``
+
+``addrvec``
+
+:Description: a set of addresses separated by ",". The addresses can be optionally quoted with ``[`` and ``]``.
+:Example: ``[v1:1.2.3.4:567,v2:1.2.3.4:568]``, ``v1:1.2.3.4:567,v1:1.2.3.14:567``  ``[2409:8a1e:8fb6:aa20:1260:4bff:fe92:18f5::567], [2409:8a1e:8fb6:aa20:1260:4bff:fe92:18f5::568]``
+
+``uuid``
+
+:Description: the string format of a uuid defined by `RFC4122 <https://www.ietf.org/rfc/rfc4122.txt>`_.
+              And some variants are also supported, for more details, see
+              `Boost document <https://www.boost.org/doc/libs/1_74_0/libs/uuid/doc/uuid.html#String%20Generator>`_.
+:Example: ``f81d4fae-7dec-11d0-a765-00a0c91e6bf6``
+
+``size``
+
+:Description: denotes a 64-bit unsigned integer. Both SI prefixes and IEC prefixes are
+              supported. And "B" is the only supported unit. A negative value will be
+              rejected.
+:Example: ``1Ki``, ``1K``, ``1KiB`` and ``1B``.
+
+``secs``
+
+:Description: denotes a duration of time. By default the unit is second if not specified.
+              Following units of time are supported:
+
+              * second: "s", "sec", "second", "seconds"
+              * minute: "m", "min", "minute", "minutes"
+              * hour: "hs", "hr", "hour", "hours"
+              * day: "d", "day", "days"
+              * week: "w", "wk", "week", "weeks"
+              * month: "mo", "month", "months"
+              * year: "y", "yr", "year", "years"
+:Example: ``1 m``, ``1m`` and ``1 week``
+
+.. _ceph-conf-database:
+
+Monitor configuration database
+==============================
+
+The monitor cluster manages a database of configuration options that
+can be consumed by the entire cluster, enabling streamlined central
+configuration management for the entire system.  The vast majority of
+configuration options can and should be stored here for ease of
+administration and transparency.
+
+A handful of settings may still need to be stored in local
+configuration files because they affect the ability to connect to the
+monitors, authenticate, and fetch configuration information.  In most
+cases this is limited to the ``mon_host`` option, although this can
+also be avoided through the use of DNS SRV records.
+
+Sections and masks
+------------------
+
+Configuration options stored by the monitor can live in a global
+section, daemon type section, or specific daemon section, just like
+options in a configuration file can.
+
+In addition, options may also have a *mask* associated with them to
+further restrict which daemons or clients the option applies to.
+Masks take two forms:
+
+#. ``type:location`` where *type* is a CRUSH property like `rack` or
+   `host`, and *location* is a value for that property.  For example,
+   ``host:foo`` would limit the option only to daemons or clients
+   running on a particular host.
+#. ``class:device-class`` where *device-class* is the name of a CRUSH
+   device class (e.g., ``hdd`` or ``ssd``).  For example,
+   ``class:ssd`` would limit the option only to OSDs backed by SSDs.
+   (This mask has no effect for non-OSD daemons or clients.)
+
+When setting a configuration option, the `who` may be a section name,
+a mask, or a combination of both separated by a slash (``/``)
+character.  For example, ``osd/rack:foo`` would mean all OSD daemons
+in the ``foo`` rack.
+
+When viewing configuration options, the section name and mask are
+generally separated out into separate fields or columns to ease readability.
+
+
+Commands
+--------
+
+The following CLI commands are used to configure the cluster:
+
+* ``ceph config dump`` will dump the entire configuration database for
+  the cluster.
+
+* ``ceph config get <who>`` will dump the configuration for a specific
+  daemon or client (e.g., ``mds.a``), as stored in the monitors'
+  configuration database.
+
+* ``ceph config set <who> <option> <value>`` will set a configuration
+  option in the monitors' configuration database.
+
+* ``ceph config show <who>`` will show the reported running
+  configuration for a running daemon.  These settings may differ from
+  those stored by the monitors if there are also local configuration
+  files in use or options have been overridden on the command line or
+  at run time.  The source of the option values is reported as part
+  of the output.
+
+* ``ceph config assimilate-conf -i <input file> -o <output file>``
+  will ingest a configuration file from *input file* and move any
+  valid options into the monitors' configuration database.  Any
+  settings that are unrecognized, invalid, or cannot be controlled by
+  the monitor will be returned in an abbreviated config file stored in
+  *output file*.  This command is useful for transitioning from legacy
+  configuration files to centralized monitor-based configuration.
+
+
+Help
+====
+
+You can get help for a particular option with:
+
+.. prompt:: bash $
+
+   ceph config help <option>
+
+Note that this will use the configuration schema that is compiled into the running monitors.  If you have a mixed-version cluster (e.g., during an upgrade), you might also want to query the option schema from a specific running daemon:
+
+.. prompt:: bash $
+
+   ceph daemon <name> config help [option]
+
+For example:
+
+.. prompt:: bash $
+
+   ceph config help log_file
+
+:: 
+
+  log_file - path to log file
+    (std::string, basic)
+    Default (non-daemon):
+    Default (daemon): /var/log/ceph/$cluster-$name.log
+    Can update at runtime: false
+    See also: [log_to_stderr,err_to_stderr,log_to_syslog,err_to_syslog]
+
+or:
+
+.. prompt:: bash $
+
+   ceph config help log_file -f json-pretty
+
+::
+
+  {
+      "name": "log_file",
+      "type": "std::string",
+      "level": "basic",
+      "desc": "path to log file",
+      "long_desc": "",
+      "default": "",
+      "daemon_default": "/var/log/ceph/$cluster-$name.log",
+      "tags": [],
+      "services": [],
+      "see_also": [
+          "log_to_stderr",
+          "err_to_stderr",
+          "log_to_syslog",
+          "err_to_syslog"
+      ],
+      "enum_values": [],
+      "min": "",
+      "max": "",
+      "can_update_at_runtime": false
+  }
+
+The ``level`` property can be any of `basic`, `advanced`, or `dev`.
+The `dev` options are intended for use by developers, generally for
+testing purposes, and are not recommended for use by operators.
+
+
+Runtime Changes
+===============
+
+In most cases, Ceph allows you to make changes to the configuration of
+a daemon at runtime. This capability is quite useful for
+increasing/decreasing logging output, enabling/disabling debug
+settings, and even for runtime optimization.
+
+Generally speaking, configuration options can be updated in the usual
+way via the ``ceph config set`` command.  For example, do enable the debug log level on a specific OSD:
+
+.. prompt:: bash $
+
+   ceph config set osd.123 debug_ms 20
+
+Note that if the same option is also customized in a local
+configuration file, the monitor setting will be ignored (it has a
+lower priority than the local config file).
+
+Override values
+---------------
+
+You can also temporarily set an option using the `tell` or `daemon`
+interfaces on the Ceph CLI.  These *override* values are ephemeral in
+that they only affect the running process and are discarded/lost if
+the daemon or process restarts.
+
+Override values can be set in two ways:
+
+#. From any host, we can send a message to a daemon over the network with:
+   
+   .. prompt:: bash $
+
+      ceph tell <name> config set <option> <value>
+
+   For example:
+   
+   .. prompt:: bash $
+
+      ceph tell osd.123 config set debug_osd 20
+
+   The `tell` command can also accept a wildcard for the daemon
+   identifier.  For example, to adjust the debug level on all OSD
+   daemons:
+   
+   .. prompt:: bash $
+
+      ceph tell osd.* config set debug_osd 20
+
+#. From the host the process is running on, we can connect directly to
+   the process via a socket in ``/var/run/ceph`` with:
+
+   .. prompt:: bash $
+
+      ceph daemon <name> config set <option> <value>
+
+   For example:
+   
+   .. prompt:: bash $
+
+      ceph daemon osd.4 config set debug_osd 20
+
+Note that in the ``ceph config show`` command output these temporary
+values will be shown with a source of ``override``.
+
+
+Viewing runtime settings
+========================
+
+You can see the current options set for a running daemon with the ``ceph config show`` command.  For example:
+
+.. prompt:: bash $
+
+   ceph config show osd.0
+
+will show you the (non-default) options for that daemon.  You can also look at a specific option with:
+
+.. prompt:: bash $
+
+   ceph config show osd.0 debug_osd
+
+or view all options (even those with default values) with:
+
+.. prompt:: bash $
+
+   ceph config show-with-defaults osd.0
+
+You can also observe settings for a running daemon by connecting to it from the local host via the admin socket.  For example:
+
+.. prompt:: bash $
+
+   ceph daemon osd.0 config show
+
+will dump all current settings:
+
+.. prompt:: bash $
+
+   ceph daemon osd.0 config diff
+
+will show only non-default settings (as well as where the value came from: a config file, the monitor, an override, etc.), and:
+
+.. prompt:: bash $
+
+   ceph daemon osd.0 config get debug_osd
+
+will report the value of a single option.
+
+
+
+Changes since Nautilus
+======================
+
+With the Octopus release We changed the way the configuration file is parsed.
+These changes are as follows:
+
+- Repeated configuration options are allowed, and no warnings will be printed.
+  The value of the last one is used, which means that the setting last in the file
+  is the one that takes effect. Before this change, we would print warning messages
+  when lines with duplicated options were encountered, like::
+
+    warning line 42: 'foo' in section 'bar' redefined
+
+- Invalid UTF-8 options were ignored with warning messages. But since Octopus,
+  they are treated as fatal errors.
+
+- Backslash ``\`` is used as the line continuation marker to combine the next
+  line with current one. Before Octopus, it was required to follow a backslash with
+  a non-empty line. But in Octopus, an empty line following a backslash is now allowed.
+
+- In the configuration file, each line specifies an individual configuration
+  option. The option's name and its value are separated with ``=``, and the
+  value may be quoted using single or double quotes. If an invalid
+  configuration is specified, we will treat it as an invalid configuration
+  file ::
+
+    bad option ==== bad value
+
+- Before Octopus, if no section name was specified in the configuration file,
+  all options would be set as though they were within the ``global`` section. This is
+  now discouraged. Since Octopus, only a single option is allowed for
+  configuration files without a section name.
diff --git a/doc/rados/configuration/common.rst b/doc/rados/configuration/common.rst
new file mode 100644
index 000000000..709c8bce2
--- /dev/null
+++ b/doc/rados/configuration/common.rst
@@ -0,0 +1,218 @@
+
+.. _ceph-conf-common-settings:
+
+Common Settings
+===============
+
+The `Hardware Recommendations`_ section provides some hardware guidelines for
+configuring a Ceph Storage Cluster. It is possible for a single :term:`Ceph
+Node` to run multiple daemons. For example, a single node with multiple drives
+may run one ``ceph-osd`` for each drive. Ideally, you will  have a node for a
+particular type of process. For example, some nodes may run ``ceph-osd``
+daemons, other nodes may run ``ceph-mds`` daemons, and still  other nodes may
+run ``ceph-mon`` daemons.
+
+Each node has a name identified by the ``host`` setting. Monitors also specify
+a network address and port (i.e., domain name or IP address) identified by the
+``addr`` setting.  A basic configuration file will typically specify only
+minimal settings for each instance of monitor daemons. For example:
+
+.. code-block:: ini
+
+	[global]
+	mon_initial_members = ceph1
+	mon_host = 10.0.0.1
+
+
+.. important:: The ``host`` setting is the short name of the node (i.e., not
+   an fqdn). It is **NOT** an IP address either.  Enter ``hostname -s`` on
+   the command line to retrieve the name of the node. Do not use ``host``
+   settings for anything other than initial monitors unless you are deploying
+   Ceph manually. You **MUST NOT** specify ``host`` under individual daemons
+   when using deployment tools like ``chef`` or ``cephadm``, as those tools
+   will enter the appropriate values for you in the cluster map.
+
+
+.. _ceph-network-config:
+
+Networks
+========
+
+See the `Network Configuration Reference`_ for a detailed discussion about
+configuring a network for use with Ceph.
+
+
+Monitors
+========
+
+Production Ceph clusters typically provision a minimum of three :term:`Ceph Monitor`
+daemons to ensure availability should a monitor instance crash. A minimum of
+three ensures that the Paxos algorithm can determine which version
+of the :term:`Ceph Cluster Map` is the most recent from a majority of Ceph
+Monitors in the quorum.
+
+.. note:: You may deploy Ceph with a single monitor, but if the instance fails,
+	       the lack of other monitors may interrupt data service availability.
+
+Ceph Monitors normally listen on port ``3300`` for the new v2 protocol, and ``6789`` for the old v1 protocol.
+
+By default, Ceph expects to store monitor data under the
+following path::
+
+	/var/lib/ceph/mon/$cluster-$id
+
+You or a deployment tool (e.g., ``cephadm``) must create the corresponding
+directory. With metavariables fully  expressed and a cluster named "ceph", the
+foregoing directory would evaluate to::
+
+	/var/lib/ceph/mon/ceph-a
+
+For additional details, see the `Monitor Config Reference`_.
+
+.. _Monitor Config Reference: ../mon-config-ref
+
+
+.. _ceph-osd-config:
+
+
+Authentication
+==============
+
+.. versionadded:: Bobtail 0.56
+
+For Bobtail (v 0.56) and beyond, you should expressly enable or disable
+authentication in the ``[global]`` section of your Ceph configuration file.
+
+.. code-block:: ini
+
+	auth_cluster_required = cephx
+	auth_service_required = cephx
+	auth_client_required = cephx
+
+Additionally, you should enable message signing. See `Cephx Config Reference`_ for details.
+
+.. _Cephx Config Reference: ../auth-config-ref
+
+
+.. _ceph-monitor-config:
+
+
+OSDs
+====
+
+Ceph production clusters typically deploy :term:`Ceph OSD Daemons` where one node
+has one OSD daemon running a Filestore on one storage device. The BlueStore back
+end is now default, but when using Filestore you specify a journal size. For example:
+
+.. code-block:: ini
+
+	[osd]
+	osd_journal_size = 10000
+
+	[osd.0]
+	host = {hostname} #manual deployments only.
+
+
+By default, Ceph expects to store a Ceph OSD Daemon's data at the
+following path::
+
+	/var/lib/ceph/osd/$cluster-$id
+
+You or a deployment tool (e.g., ``cephadm``) must create the corresponding
+directory. With metavariables fully expressed and a cluster named "ceph", this
+example would evaluate to::
+
+	/var/lib/ceph/osd/ceph-0
+
+You may override this path using the ``osd_data`` setting. We recommend not
+changing the default location. Create the default directory on your OSD host.
+
+.. prompt:: bash $
+
+	ssh {osd-host}
+	sudo mkdir /var/lib/ceph/osd/ceph-{osd-number}
+
+The ``osd_data`` path ideally leads to a mount point with a device that is
+separate from the device that contains the operating system and
+daemons. If an OSD is to use a device other than the OS device, prepare it for
+use with Ceph, and mount it to the directory you just created
+
+.. prompt:: bash $
+
+	ssh {new-osd-host}
+	sudo mkfs -t {fstype} /dev/{disk}
+	sudo mount -o user_xattr /dev/{hdd} /var/lib/ceph/osd/ceph-{osd-number}
+
+We recommend using the ``xfs`` file system when running
+:command:`mkfs`.  (``btrfs`` and ``ext4`` are not recommended and are no
+longer tested.)
+
+See the `OSD Config Reference`_ for additional configuration details.
+
+
+Heartbeats
+==========
+
+During runtime operations, Ceph OSD Daemons check up on other Ceph OSD Daemons
+and report their  findings to the Ceph Monitor. You do not have to provide any
+settings. However, if you have network latency issues, you may wish to modify
+the settings.
+
+See `Configuring Monitor/OSD Interaction`_ for additional details.
+
+
+.. _ceph-logging-and-debugging:
+
+Logs / Debugging
+================
+
+Sometimes you may encounter issues with Ceph that require
+modifying logging output and using Ceph's debugging. See `Debugging and
+Logging`_ for details on log rotation.
+
+.. _Debugging and Logging: ../../troubleshooting/log-and-debug
+
+
+Example ceph.conf
+=================
+
+.. literalinclude:: demo-ceph.conf
+   :language: ini
+
+.. _ceph-runtime-config:
+
+
+
+Running Multiple Clusters (DEPRECATED)
+======================================
+
+Each Ceph cluster has an internal name that is used as part of configuration
+and log file names as well as directory and mountpoint names.  This name
+defaults to "ceph".  Previous releases of Ceph allowed one to specify a custom
+name instead, for example "ceph2".  This was intended to faciliate running
+multiple logical clusters on the same physical hardware, but in practice this
+was rarely exploited and should no longer be attempted.  Prior documentation
+could also be misinterpreted as requiring unique cluster names in order to
+use ``rbd-mirror``.
+
+Custom cluster names are now considered deprecated and the ability to deploy
+them has already been removed from some tools, though existing custom name
+deployments continue to operate.  The ability to run and manage clusters with
+custom names may be progressively removed by future Ceph releases, so it is
+strongly recommended to deploy all new clusters with the default name "ceph".
+
+Some Ceph CLI commands accept an optional ``--cluster`` (cluster name) option. This
+option is present purely for backward compatibility and need not be accomodated
+by new tools and deployments.
+
+If you do need to allow multiple clusters to exist on the same host, please use
+:ref:`cephadm`, which uses containers to fully isolate each cluster.
+
+
+
+
+
+.. _Hardware Recommendations: ../../../start/hardware-recommendations
+.. _Network Configuration Reference: ../network-config-ref
+.. _OSD Config Reference: ../osd-config-ref
+.. _Configuring Monitor/OSD Interaction: ../mon-osd-interaction
diff --git a/doc/rados/configuration/demo-ceph.conf b/doc/rados/configuration/demo-ceph.conf
new file mode 100644
index 000000000..58bb7061f
--- /dev/null
+++ b/doc/rados/configuration/demo-ceph.conf
@@ -0,0 +1,31 @@
+[global]
+fsid = {cluster-id}
+mon_initial_ members = {hostname}[, {hostname}]
+mon_host = {ip-address}[, {ip-address}]
+
+#All clusters have a front-side public network.
+#If you have two network interfaces, you can configure a private / cluster 
+#network for RADOS object replication, heartbeats, backfill,
+#recovery, etc.
+public_network = {network}[, {network}]
+#cluster_network = {network}[, {network}] 
+
+#Clusters require authentication by default.
+auth_cluster_required = cephx
+auth_service_required = cephx
+auth_client_required = cephx
+
+#Choose reasonable numbers for journals, number of replicas
+#and placement groups.
+osd_journal_size = {n}
+osd_pool_default_size = {n}  # Write an object n times.
+osd_pool_default_min size = {n} # Allow writing n copy in a degraded state.
+osd_pool_default_pg num = {n}
+osd_pool_default_pgp num = {n}
+
+#Choose a reasonable crush leaf type.
+#0 for a 1-node cluster.
+#1 for a multi node cluster in a single rack
+#2 for a multi node, multi chassis cluster with multiple hosts in a chassis
+#3 for a multi node cluster with hosts across racks, etc.
+osd_crush_chooseleaf_type = {n}
\ No newline at end of file
diff --git a/doc/rados/configuration/filestore-config-ref.rst b/doc/rados/configuration/filestore-config-ref.rst
new file mode 100644
index 000000000..435a800a8
--- /dev/null
+++ b/doc/rados/configuration/filestore-config-ref.rst
@@ -0,0 +1,367 @@
+============================
+ Filestore Config Reference
+============================
+
+The Filestore back end is no longer the default when creating new OSDs,
+though Filestore OSDs are still supported.
+
+``filestore debug omap check``
+
+:Description: Debugging check on synchronization. Expensive. For debugging only.
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+.. index:: filestore; extended attributes
+
+Extended Attributes
+===================
+
+Extended Attributes (XATTRs) are important for Filestore OSDs.
+Some file systems have limits on the number of bytes that can be stored in XATTRs. 
+Additionally, in some cases, the file system may not be as fast as an alternative
+method of storing XATTRs. The following settings may help improve performance
+by using a method of storing XATTRs that is extrinsic to the underlying file system.
+
+Ceph XATTRs are stored as ``inline xattr``, using the XATTRs provided
+by the underlying file system, if it does not impose a size limit. If
+there is a size limit (4KB total on ext4, for instance), some Ceph
+XATTRs will be stored in a key/value database when either the
+``filestore_max_inline_xattr_size`` or ``filestore_max_inline_xattrs``
+threshold is reached.
+
+
+``filestore_max_inline_xattr_size``
+
+:Description: The maximum size of an XATTR stored in the file system (i.e., XFS,
+              Btrfs, EXT4, etc.) per object. Should not be larger than the
+              file system can handle. Default value of 0 means to use the value
+              specific to the underlying file system.
+:Type: Unsigned 32-bit Integer
+:Required: No
+:Default: ``0``
+
+
+``filestore_max_inline_xattr_size_xfs``
+
+:Description: The maximum size of an XATTR stored in the XFS file system.
+              Only used if ``filestore_max_inline_xattr_size`` == 0.
+:Type: Unsigned 32-bit Integer
+:Required: No
+:Default: ``65536``
+
+
+``filestore_max_inline_xattr_size_btrfs``
+
+:Description: The maximum size of an XATTR stored in the Btrfs file system.
+              Only used if ``filestore_max_inline_xattr_size`` == 0.
+:Type: Unsigned 32-bit Integer
+:Required: No
+:Default: ``2048``
+
+
+``filestore_max_inline_xattr_size_other``
+
+:Description: The maximum size of an XATTR stored in other file systems.
+              Only used if ``filestore_max_inline_xattr_size`` == 0.
+:Type: Unsigned 32-bit Integer
+:Required: No
+:Default: ``512``
+
+
+``filestore_max_inline_xattrs``
+
+:Description: The maximum number of XATTRs stored in the file system per object.
+              Default value of 0 means to use the value specific to the
+              underlying file system.
+:Type: 32-bit Integer
+:Required: No
+:Default: ``0``
+
+
+``filestore_max_inline_xattrs_xfs``
+
+:Description: The maximum number of XATTRs stored in the XFS file system per object.
+              Only used if ``filestore_max_inline_xattrs`` == 0.
+:Type: 32-bit Integer
+:Required: No
+:Default: ``10``
+
+
+``filestore_max_inline_xattrs_btrfs``
+
+:Description: The maximum number of XATTRs stored in the Btrfs file system per object.
+              Only used if ``filestore_max_inline_xattrs`` == 0.
+:Type: 32-bit Integer
+:Required: No
+:Default: ``10``
+
+
+``filestore_max_inline_xattrs_other``
+
+:Description: The maximum number of XATTRs stored in other file systems per object.
+              Only used if ``filestore_max_inline_xattrs`` == 0.
+:Type: 32-bit Integer
+:Required: No
+:Default: ``2``
+
+.. index:: filestore; synchronization
+
+Synchronization Intervals
+=========================
+
+Filestore needs to periodically quiesce writes and synchronize the
+file system, which creates a consistent commit point. It can then free journal
+entries up to the commit point. Synchronizing more frequently tends to reduce
+the time required to perform synchronization, and reduces the amount of data
+that needs to remain in the  journal. Less frequent synchronization allows the
+backing file system to coalesce small writes and metadata updates more
+optimally, potentially resulting in more efficient synchronization at the
+expense of potentially increasing tail latency.
+
+``filestore_max_sync_interval``
+
+:Description: The maximum interval in seconds for synchronizing Filestore.
+:Type: Double
+:Required: No
+:Default: ``5``
+
+
+``filestore_min_sync_interval``
+
+:Description: The minimum interval in seconds for synchronizing Filestore.
+:Type: Double
+:Required: No
+:Default: ``.01``
+
+
+.. index:: filestore; flusher
+
+Flusher
+=======
+
+The Filestore flusher forces data from large writes to be written out using
+``sync_file_range`` before the sync in order to (hopefully) reduce the cost of
+the eventual sync. In practice, disabling 'filestore_flusher' seems to improve
+performance in some cases.
+
+
+``filestore_flusher``
+
+:Description: Enables the filestore flusher.
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+.. deprecated:: v.65
+
+``filestore_flusher_max_fds``
+
+:Description: Sets the maximum number of file descriptors for the flusher.
+:Type: Integer
+:Required: No
+:Default: ``512``
+
+.. deprecated:: v.65
+
+``filestore_sync_flush``
+
+:Description: Enables the synchronization flusher. 
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+.. deprecated:: v.65
+
+``filestore_fsync_flushes_journal_data``
+
+:Description: Flush journal data during file system synchronization.
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+.. index:: filestore; queue
+
+Queue
+=====
+
+The following settings provide limits on the size of the Filestore queue.
+
+``filestore_queue_max_ops``
+
+:Description: Defines the maximum number of in progress operations the file store accepts before blocking on queuing new operations. 
+:Type: Integer
+:Required: No. Minimal impact on performance.
+:Default: ``50``
+
+
+``filestore_queue_max_bytes``
+
+:Description: The maximum number of bytes for an operation. 
+:Type: Integer
+:Required: No
+:Default: ``100 << 20``
+
+
+
+
+.. index:: filestore; timeouts
+
+Timeouts
+========
+
+
+``filestore_op_threads``
+
+:Description: The number of file system operation threads that execute in parallel. 
+:Type: Integer
+:Required: No
+:Default: ``2``
+
+
+``filestore_op_thread_timeout``
+
+:Description: The timeout for a file system operation thread (in seconds).
+:Type: Integer
+:Required: No
+:Default: ``60``
+
+
+``filestore_op_thread_suicide_timeout``
+
+:Description: The timeout for a commit operation before cancelling the commit (in seconds). 
+:Type: Integer
+:Required: No
+:Default: ``180``
+
+
+.. index:: filestore; btrfs
+
+B-Tree Filesystem
+=================
+
+
+``filestore_btrfs_snap``
+
+:Description: Enable snapshots for a ``btrfs`` filestore.
+:Type: Boolean
+:Required: No. Only used for ``btrfs``.
+:Default: ``true``
+
+
+``filestore_btrfs_clone_range``
+
+:Description: Enable cloning ranges for a ``btrfs`` filestore.
+:Type: Boolean
+:Required: No. Only used for ``btrfs``.
+:Default: ``true``
+
+
+.. index:: filestore; journal
+
+Journal
+=======
+
+
+``filestore_journal_parallel``
+
+:Description: Enables parallel journaling, default for Btrfs.
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``filestore_journal_writeahead``
+
+:Description: Enables writeahead journaling, default for XFS.
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``filestore_journal_trailing``
+
+:Description: Deprecated, never use.
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+Misc
+====
+
+
+``filestore_merge_threshold``
+
+:Description: Min number of files in a subdir before merging into parent
+              NOTE: A negative value means to disable subdir merging
+:Type: Integer
+:Required: No
+:Default: ``-10``
+
+
+``filestore_split_multiple``
+
+:Description:  ``(filestore_split_multiple * abs(filestore_merge_threshold) + (rand() % filestore_split_rand_factor)) * 16``
+               is the maximum number of files in a subdirectory before 
+               splitting into child directories.
+
+:Type: Integer
+:Required: No
+:Default: ``2``
+
+
+``filestore_split_rand_factor``
+
+:Description:  A random factor added to the split threshold to avoid
+               too many (expensive) Filestore splits occurring at once. See
+               ``filestore_split_multiple`` for details.
+               This can only be changed offline for an existing OSD,
+               via the ``ceph-objectstore-tool apply-layout-settings`` command.
+
+:Type: Unsigned 32-bit Integer
+:Required: No
+:Default: ``20``
+
+
+``filestore_update_to``
+
+:Description: Limits Filestore auto upgrade to specified version.
+:Type: Integer
+:Required: No
+:Default: ``1000``
+
+
+``filestore_blackhole``
+
+:Description: Drop any new transactions on the floor.
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``filestore_dump_file``
+
+:Description: File onto which store transaction dumps.
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``filestore_kill_at``
+
+:Description: inject a failure at the n'th opportunity
+:Type: String
+:Required: No
+:Default: ``false``
+
+
+``filestore_fail_eio``
+
+:Description: Fail/Crash on eio.
+:Type: Boolean
+:Required: No
+:Default: ``true``
+
diff --git a/doc/rados/configuration/general-config-ref.rst b/doc/rados/configuration/general-config-ref.rst
new file mode 100644
index 000000000..fc837c395
--- /dev/null
+++ b/doc/rados/configuration/general-config-ref.rst
@@ -0,0 +1,66 @@
+==========================
+ General Config Reference
+==========================
+
+
+``fsid``
+
+:Description: The file system ID. One per cluster.
+:Type: UUID
+:Required: No. 
+:Default: N/A. Usually generated by deployment tools.
+
+
+``admin_socket``
+
+:Description: The socket for executing administrative commands on a daemon, 
+              irrespective of whether Ceph Monitors have established a quorum.
+
+:Type: String
+:Required: No
+:Default: ``/var/run/ceph/$cluster-$name.asok`` 
+
+
+``pid_file``
+
+:Description: The file in which the mon, osd or mds will write its
+              PID.  For instance, ``/var/run/$cluster/$type.$id.pid``
+              will create /var/run/ceph/mon.a.pid for the ``mon`` with
+              id ``a`` running in the ``ceph`` cluster. The ``pid
+              file`` is removed when the daemon stops gracefully. If
+              the process is not daemonized (i.e. runs with the ``-f``
+              or ``-d`` option), the ``pid file`` is not created. 
+:Type: String
+:Required: No
+:Default: No
+
+
+``chdir``
+
+:Description: The directory Ceph daemons change to once they are 
+              up and running. Default ``/`` directory recommended.
+
+:Type: String
+:Required: No
+:Default: ``/``
+
+
+``max_open_files``
+
+:Description: If set, when the :term:`Ceph Storage Cluster` starts, Ceph sets 
+              the max open FDs at the OS level (i.e., the max # of file 
+              descriptors). A suitably large value prevents Ceph Daemons from running out
+              of file descriptors.
+
+:Type: 64-bit Integer
+:Required: No
+:Default: ``0``
+
+
+``fatal_signal_handlers``
+
+:Description: If set, we will install signal handlers for SEGV, ABRT, BUS, ILL,
+              FPE, XCPU, XFSZ, SYS signals to generate a useful log message
+
+:Type: Boolean
+:Default: ``true``
diff --git a/doc/rados/configuration/index.rst b/doc/rados/configuration/index.rst
new file mode 100644
index 000000000..414fcc6fa
--- /dev/null
+++ b/doc/rados/configuration/index.rst
@@ -0,0 +1,54 @@
+===============
+ Configuration
+===============
+
+Each Ceph process, daemon, or utility draws its configuration from several
+sources on startup. Such sources can include (1) a local configuration, (2) the
+monitors, (3) the command line, and (4) environment variables.
+
+Configuration options can be set globally so that they apply (1) to all
+daemons, (2) to all daemons or services of a particular type, or (3) to only a
+specific daemon, process, or client.
+
+.. raw:: html
+
+	<table cellpadding="10"><colgroup><col width="50%"><col width="50%"></colgroup><tbody valign="top"><tr><td><h3>Configuring the Object Store</h3>
+
+For general object store configuration, refer to the following:
+
+.. toctree::
+   :maxdepth: 1
+
+   Storage devices <storage-devices>
+   ceph-conf
+
+
+.. raw:: html 
+
+	</td><td><h3>Reference</h3>
+
+To optimize the performance of your cluster, refer to the following:
+
+.. toctree::
+   :maxdepth: 1
+
+   Common Settings <common>
+   Network Settings <network-config-ref>
+   Messenger v2 protocol <msgr2>
+   Auth Settings <auth-config-ref>
+   Monitor Settings <mon-config-ref>
+   mon-lookup-dns
+   Heartbeat Settings <mon-osd-interaction>
+   OSD Settings <osd-config-ref>
+   DmClock Settings <mclock-config-ref>
+   BlueStore Settings <bluestore-config-ref>
+   FileStore Settings <filestore-config-ref>
+   Journal Settings <journal-ref>
+   Pool, PG & CRUSH Settings <pool-pg-config-ref.rst>
+   Messaging Settings <ms-ref>
+   General Settings <general-config-ref>
+
+   
+.. raw:: html
+
+	</td></tr></tbody></table>
diff --git a/doc/rados/configuration/journal-ref.rst b/doc/rados/configuration/journal-ref.rst
new file mode 100644
index 000000000..71c74c606
--- /dev/null
+++ b/doc/rados/configuration/journal-ref.rst
@@ -0,0 +1,119 @@
+==========================
+ Journal Config Reference
+==========================
+
+.. index:: journal; journal configuration
+
+Filestore OSDs use a journal for two reasons: speed and consistency.  Note
+that since Luminous, the BlueStore OSD back end has been preferred and default.
+This information is provided for pre-existing OSDs and for rare situations where
+Filestore is preferred for new deployments.
+
+- **Speed:** The journal enables the Ceph OSD Daemon to commit small writes 
+  quickly. Ceph writes small, random i/o to the journal sequentially, which 
+  tends to speed up bursty workloads by allowing the backing file system more 
+  time to coalesce writes. The Ceph OSD Daemon's journal, however, can lead 
+  to spiky performance with short spurts of high-speed writes followed by 
+  periods without any write progress as the file system catches up to the 
+  journal.
+
+- **Consistency:** Ceph OSD Daemons require a file system interface that 
+  guarantees atomic compound operations. Ceph OSD Daemons write a description 
+  of the operation to the journal and apply the operation to the file system. 
+  This enables atomic updates to an object (for example, placement group 
+  metadata). Every few seconds--between ``filestore max sync interval`` and
+  ``filestore min sync interval``--the Ceph OSD Daemon stops writes and 
+  synchronizes the journal with the file system, allowing Ceph OSD Daemons to 
+  trim operations from the journal and reuse the space. On failure, Ceph 
+  OSD Daemons replay the journal starting after the last synchronization 
+  operation.
+
+Ceph OSD Daemons recognize the following journal settings: 
+
+
+``journal_dio``
+
+:Description: Enables direct i/o to the journal. Requires ``journal block 
+              align`` set to ``true``.
+              
+:Type: Boolean
+:Required: Yes when using ``aio``.
+:Default: ``true``
+
+
+
+``journal_aio``
+
+.. versionchanged:: 0.61 Cuttlefish
+
+:Description: Enables using ``libaio`` for asynchronous writes to the journal. 
+              Requires ``journal dio`` set to ``true``.
+
+:Type: Boolean 
+:Required: No.
+:Default: Version 0.61 and later, ``true``. Version 0.60 and earlier, ``false``.
+
+
+``journal_block_align``
+
+:Description: Block aligns write operations. Required for ``dio`` and ``aio``.
+:Type: Boolean
+:Required: Yes when using ``dio`` and ``aio``.
+:Default: ``true``
+
+
+``journal_max_write_bytes``
+
+:Description: The maximum number of bytes the journal will write at 
+              any one time.
+
+:Type: Integer
+:Required: No
+:Default: ``10 << 20``
+
+
+``journal_max_write_entries``
+
+:Description: The maximum number of entries the journal will write at 
+              any one time.
+
+:Type: Integer
+:Required: No
+:Default: ``100``
+
+
+``journal_queue_max_ops``
+
+:Description: The maximum number of operations allowed in the queue at 
+              any one time.
+
+:Type: Integer
+:Required: No
+:Default: ``500``
+
+
+``journal_queue_max_bytes``
+
+:Description: The maximum number of bytes allowed in the queue at 
+              any one time.
+
+:Type: Integer
+:Required: No
+:Default: ``10 << 20``
+
+
+``journal_align_min_size``
+
+:Description: Align data payloads greater than the specified minimum.
+:Type: Integer
+:Required: No
+:Default: ``64 << 10``
+
+
+``journal_zero_on_create``
+
+:Description: Causes the file store to overwrite the entire journal with 
+              ``0``'s during ``mkfs``.
+:Type: Boolean
+:Required: No
+:Default: ``false``
diff --git a/doc/rados/configuration/mclock-config-ref.rst b/doc/rados/configuration/mclock-config-ref.rst
new file mode 100644
index 000000000..579056895
--- /dev/null
+++ b/doc/rados/configuration/mclock-config-ref.rst
@@ -0,0 +1,395 @@
+========================
+ mClock Config Reference
+========================
+
+.. index:: mclock; configuration
+
+Mclock profiles mask the low level details from users, making it
+easier for them to configure mclock.
+
+The following input parameters are required for a mclock profile to configure
+the QoS related parameters:
+
+* total capacity (IOPS) of each OSD (determined automatically)
+
+* an mclock profile type to enable
+
+Using the settings in the specified profile, the OSD determines and applies the
+lower-level mclock and Ceph parameters. The parameters applied by the mclock
+profile make it possible to tune the QoS between client I/O, recovery/backfill
+operations, and other background operations (for example, scrub, snap trim, and
+PG deletion). These background activities are considered best-effort internal
+clients of Ceph.
+
+
+.. index:: mclock; profile definition
+
+mClock Profiles - Definition and Purpose
+========================================
+
+A mclock profile is *“a configuration setting that when applied on a running
+Ceph cluster enables the throttling of the operations(IOPS) belonging to
+different client classes (background recovery, scrub, snaptrim, client op,
+osd subop)�*.
+
+The mclock profile uses the capacity limits and the mclock profile type selected
+by the user to determine the low-level mclock resource control parameters.
+
+Depending on the profile type, lower-level mclock resource-control parameters
+and some Ceph-configuration parameters are transparently applied.
+
+The low-level mclock resource control parameters are the *reservation*,
+*limit*, and *weight* that provide control of the resource shares, as
+described in the :ref:`dmclock-qos` section.
+
+
+.. index:: mclock; profile types
+
+mClock Profile Types
+====================
+
+mclock profiles can be broadly classified into two types,
+
+- **Built-in**: Users can choose between the following built-in profile types:
+
+  - **high_client_ops** (*default*):
+    This profile allocates more reservation and limit to external-client ops
+    as compared to background recoveries and other internal clients within
+    Ceph. This profile is enabled by default.
+  - **high_recovery_ops**:
+    This profile allocates more reservation to background recoveries as
+    compared to external clients and other internal clients within Ceph. For
+    example, an admin may enable this profile temporarily to speed-up background
+    recoveries during non-peak hours.
+  - **balanced**:
+    This profile allocates equal reservation to client ops and background
+    recovery ops.
+
+- **Custom**: This profile gives users complete control over all the mclock
+  configuration parameters. Using this profile is not recommended without
+  a deep understanding of mclock and related Ceph-configuration options.
+
+.. note:: Across the built-in profiles, internal clients of mclock (for example
+          "scrub", "snap trim", and "pg deletion") are given slightly lower
+          reservations, but higher weight and no limit. This ensures that
+          these operations are able to complete quickly if there are no other
+          competing services.
+
+
+.. index:: mclock; built-in profiles
+
+mClock Built-in Profiles
+========================
+
+When a built-in profile is enabled, the mClock scheduler calculates the low
+level mclock parameters [*reservation*, *weight*, *limit*] based on the profile
+enabled for each client type. The mclock parameters are calculated based on
+the max OSD capacity provided beforehand. As a result, the following mclock
+config parameters cannot be modified when using any of the built-in profiles:
+
+- ``osd_mclock_scheduler_client_res``
+- ``osd_mclock_scheduler_client_wgt``
+- ``osd_mclock_scheduler_client_lim``
+- ``osd_mclock_scheduler_background_recovery_res``
+- ``osd_mclock_scheduler_background_recovery_wgt``
+- ``osd_mclock_scheduler_background_recovery_lim``
+- ``osd_mclock_scheduler_background_best_effort_res``
+- ``osd_mclock_scheduler_background_best_effort_wgt``
+- ``osd_mclock_scheduler_background_best_effort_lim``
+
+The following Ceph options will not be modifiable by the user:
+
+- ``osd_max_backfills``
+- ``osd_recovery_max_active``
+
+This is because the above options are internally modified by the mclock
+scheduler in order to maximize the impact of the set profile.
+
+By default, the *high_client_ops* profile is enabled to ensure that a larger
+chunk of the bandwidth allocation goes to client ops. Background recovery ops
+are given lower allocation (and therefore take a longer time to complete). But
+there might be instances that necessitate giving higher allocations to either
+client ops or recovery ops. In order to deal with such a situation, you can
+enable one of the alternate built-in profiles by following the steps mentioned
+in the next section.
+
+If any mClock profile (including "custom") is active, the following Ceph config
+sleep options will be disabled,
+
+- ``osd_recovery_sleep``
+- ``osd_recovery_sleep_hdd``
+- ``osd_recovery_sleep_ssd``
+- ``osd_recovery_sleep_hybrid``
+- ``osd_scrub_sleep``
+- ``osd_delete_sleep``
+- ``osd_delete_sleep_hdd``
+- ``osd_delete_sleep_ssd``
+- ``osd_delete_sleep_hybrid``
+- ``osd_snap_trim_sleep``
+- ``osd_snap_trim_sleep_hdd``
+- ``osd_snap_trim_sleep_ssd``
+- ``osd_snap_trim_sleep_hybrid``
+
+The above sleep options are disabled to ensure that mclock scheduler is able to
+determine when to pick the next op from its operation queue and transfer it to
+the operation sequencer. This results in the desired QoS being provided across
+all its clients.
+
+
+.. index:: mclock; enable built-in profile
+
+Steps to Enable mClock Profile
+==============================
+
+As already mentioned, the default mclock profile is set to *high_client_ops*.
+The other values for the built-in profiles include *balanced* and
+*high_recovery_ops*.
+
+If there is a requirement to change the default profile, then the option
+``osd_mclock_profile`` may be set during runtime by using the following
+command:
+
+  .. prompt:: bash #
+
+    ceph config set osd.N osd_mclock_profile <value>
+
+For example, to change the profile to allow faster recoveries on "osd.0", the
+following command can be used to switch to the *high_recovery_ops* profile:
+
+  .. prompt:: bash #
+
+    ceph config set osd.0 osd_mclock_profile high_recovery_ops
+
+.. note:: The *custom* profile is not recommended unless you are an advanced
+          user.
+
+And that's it! You are ready to run workloads on the cluster and check if the
+QoS requirements are being met.
+
+
+OSD Capacity Determination (Automated)
+======================================
+
+The OSD capacity in terms of total IOPS is determined automatically during OSD
+initialization. This is achieved by running the OSD bench tool and overriding
+the default value of ``osd_mclock_max_capacity_iops_[hdd, ssd]`` option
+depending on the device type. No other action/input is expected from the user
+to set the OSD capacity. You may verify the capacity of an OSD after the
+cluster is brought up by using the following command:
+
+  .. prompt:: bash #
+
+    ceph config show osd.N osd_mclock_max_capacity_iops_[hdd, ssd]
+
+For example, the following command shows the max capacity for "osd.0" on a Ceph
+node whose underlying device type is SSD:
+
+  .. prompt:: bash #
+
+    ceph config show osd.0 osd_mclock_max_capacity_iops_ssd
+
+
+Steps to Manually Benchmark an OSD (Optional)
+=============================================
+
+.. note:: These steps are only necessary if you want to override the OSD
+          capacity already determined automatically during OSD initialization.
+          Otherwise, you may skip this section entirely.
+
+.. tip:: If you have already determined the benchmark data and wish to manually
+         override the max osd capacity for an OSD, you may skip to section
+         `Specifying  Max OSD Capacity`_.
+
+
+Any existing benchmarking tool can be used for this purpose. In this case, the
+steps use the *Ceph OSD Bench* command described in the next section. Regardless
+of the tool/command used, the steps outlined further below remain the same.
+
+As already described in the :ref:`dmclock-qos` section, the number of
+shards and the bluestore's throttle parameters have an impact on the mclock op
+queues. Therefore, it is critical to set these values carefully in order to
+maximize the impact of the mclock scheduler.
+
+:Number of Operational Shards:
+  We recommend using the default number of shards as defined by the
+  configuration options ``osd_op_num_shards``, ``osd_op_num_shards_hdd``, and
+  ``osd_op_num_shards_ssd``. In general, a lower number of shards will increase
+  the impact of the mclock queues.
+
+:Bluestore Throttle Parameters:
+  We recommend using the default values as defined by
+  ``bluestore_throttle_bytes`` and ``bluestore_throttle_deferred_bytes``. But
+  these parameters may also be determined during the benchmarking phase as
+  described below.
+
+
+OSD Bench Command Syntax
+````````````````````````
+
+The :ref:`osd-subsystem` section describes the OSD bench command. The syntax
+used for benchmarking is shown below :
+
+.. prompt:: bash #
+
+  ceph tell osd.N bench [TOTAL_BYTES] [BYTES_PER_WRITE] [OBJ_SIZE] [NUM_OBJS]
+
+where,
+
+* ``TOTAL_BYTES``: Total number of bytes to write
+* ``BYTES_PER_WRITE``: Block size per write
+* ``OBJ_SIZE``: Bytes per object
+* ``NUM_OBJS``: Number of objects to write
+
+Benchmarking Test Steps Using OSD Bench
+```````````````````````````````````````
+
+The steps below use the default shards and detail the steps used to determine
+the correct bluestore throttle values (optional).
+
+#. Bring up your Ceph cluster and login to the Ceph node hosting the OSDs that
+   you wish to benchmark.
+#. Run a simple 4KiB random write workload on an OSD using the following
+   commands:
+
+   .. note:: Note that before running the test, caches must be cleared to get an
+             accurate measurement.
+
+   For example, if you are running the benchmark test on osd.0, run the following
+   commands:
+
+   .. prompt:: bash #
+
+     ceph tell osd.0 cache drop
+
+   .. prompt:: bash #
+
+     ceph tell osd.0 bench 12288000 4096 4194304 100
+
+#. Note the overall throughput(IOPS) obtained from the output of the osd bench
+   command. This value is the baseline throughput(IOPS) when the default
+   bluestore throttle options are in effect.
+#. If the intent is to determine the bluestore throttle values for your
+   environment, then set the two options, ``bluestore_throttle_bytes``
+   and ``bluestore_throttle_deferred_bytes`` to 32 KiB(32768 Bytes) each
+   to begin with. Otherwise, you may skip to the next section.
+#. Run the 4KiB random write test as before using OSD bench.
+#. Note the overall throughput from the output and compare the value
+   against the baseline throughput recorded in step 3.
+#. If the throughput doesn't match with the baseline, increment the bluestore
+   throttle options by 2x and repeat steps 5 through 7 until the obtained
+   throughput is very close to the baseline value.
+
+For example, during benchmarking on a machine with NVMe SSDs, a value of 256 KiB
+for both bluestore throttle and deferred bytes was determined to maximize the
+impact of mclock. For HDDs, the corresponding value was 40 MiB, where the
+overall throughput was roughly equal to the baseline throughput. Note that in
+general for HDDs, the bluestore throttle values are expected to be higher when
+compared to SSDs.
+
+
+Specifying  Max OSD Capacity
+````````````````````````````
+
+The steps in this section may be performed only if you want to override the
+max osd capacity automatically set during OSD initialization. The option
+``osd_mclock_max_capacity_iops_[hdd, ssd]`` for an OSD can be set by running the
+following command:
+
+  .. prompt:: bash #
+
+     ceph config set osd.N osd_mclock_max_capacity_iops_[hdd,ssd] <value>
+
+For example, the following command sets the max capacity for a specific OSD
+(say "osd.0") whose underlying device type is HDD to 350 IOPS:
+
+  .. prompt:: bash #
+
+    ceph config set osd.0 osd_mclock_max_capacity_iops_hdd 350
+
+Alternatively, you may specify the max capacity for OSDs within the Ceph
+configuration file under the respective [osd.N] section. See
+:ref:`ceph-conf-settings` for more details.
+
+
+.. index:: mclock; config settings
+
+mClock Config Options
+=====================
+
+``osd_mclock_profile``
+
+:Description: This sets the type of mclock profile to use for providing QoS
+              based on operations belonging to different classes (background
+              recovery, scrub, snaptrim, client op, osd subop). Once a built-in
+              profile is enabled, the lower level mclock resource control
+              parameters [*reservation, weight, limit*] and some Ceph
+              configuration parameters are set transparently. Note that the
+              above does not apply for the *custom* profile.
+
+:Type: String
+:Valid Choices: high_client_ops, high_recovery_ops, balanced, custom
+:Default: ``high_client_ops``
+
+``osd_mclock_max_capacity_iops_hdd``
+
+:Description: Max IOPS capacity (at 4KiB block size) to consider per OSD (for
+              rotational media)
+
+:Type: Float
+:Default: ``315.0``
+
+``osd_mclock_max_capacity_iops_ssd``
+
+:Description: Max IOPS capacity (at 4KiB block size) to consider per OSD (for
+              solid state media)
+
+:Type: Float
+:Default: ``21500.0``
+
+``osd_mclock_cost_per_io_usec``
+
+:Description: Cost per IO in microseconds to consider per OSD (overrides _ssd
+              and _hdd if non-zero)
+
+:Type: Float
+:Default: ``0.0``
+
+``osd_mclock_cost_per_io_usec_hdd``
+
+:Description: Cost per IO in microseconds to consider per OSD (for rotational
+              media)
+
+:Type: Float
+:Default: ``25000.0``
+
+``osd_mclock_cost_per_io_usec_ssd``
+
+:Description: Cost per IO in microseconds to consider per OSD (for solid state
+              media)
+
+:Type: Float
+:Default: ``50.0``
+
+``osd_mclock_cost_per_byte_usec``
+
+:Description: Cost per byte in microseconds to consider per OSD (overrides _ssd
+              and _hdd if non-zero)
+
+:Type: Float
+:Default: ``0.0``
+
+``osd_mclock_cost_per_byte_usec_hdd``
+
+:Description: Cost per byte in microseconds to consider per OSD (for rotational
+              media)
+
+:Type: Float
+:Default: ``5.2``
+
+``osd_mclock_cost_per_byte_usec_ssd``
+
+:Description: Cost per byte in microseconds to consider per OSD (for solid state
+              media)
+
+:Type: Float
+:Default: ``0.011``
diff --git a/doc/rados/configuration/mon-config-ref.rst b/doc/rados/configuration/mon-config-ref.rst
new file mode 100644
index 000000000..3b12af43d
--- /dev/null
+++ b/doc/rados/configuration/mon-config-ref.rst
@@ -0,0 +1,1243 @@
+.. _monitor-config-reference:
+
+==========================
+ Monitor Config Reference
+==========================
+
+Understanding how to configure a :term:`Ceph Monitor` is an important part of
+building a reliable :term:`Ceph Storage Cluster`. **All Ceph Storage Clusters
+have at least one monitor**. The monitor complement usually remains fairly
+consistent, but you can add, remove or replace a monitor in a cluster. See
+`Adding/Removing a Monitor`_ for details.
+
+
+.. index:: Ceph Monitor; Paxos
+
+Background
+==========
+
+Ceph Monitors maintain a "master copy" of the :term:`Cluster Map`, which means a
+:term:`Ceph Client` can determine the location of all Ceph Monitors, Ceph OSD
+Daemons, and Ceph Metadata Servers just by connecting to one Ceph Monitor and
+retrieving a current cluster map. Before Ceph Clients can read from or write to
+Ceph OSD Daemons or Ceph Metadata Servers, they must connect to a Ceph Monitor
+first. With a current copy of the cluster map and the CRUSH algorithm, a Ceph
+Client can compute the location for any object. The ability to compute object
+locations allows a Ceph Client to talk directly to Ceph OSD Daemons, which is a
+very important aspect of Ceph's high scalability and performance. See 
+`Scalability and High Availability`_ for additional details.
+
+The primary role of the Ceph Monitor is to maintain a master copy of the cluster
+map. Ceph Monitors also provide authentication and logging services. Ceph
+Monitors write all changes in the monitor services to a single Paxos instance,
+and Paxos writes the changes to a key/value store for strong consistency. Ceph
+Monitors can query the most recent version of the cluster map during sync
+operations. Ceph Monitors leverage the key/value store's snapshots and iterators
+(using leveldb) to perform store-wide synchronization.
+
+.. ditaa::
+ /-------------\               /-------------\
+ |   Monitor   | Write Changes |    Paxos    |
+ |   cCCC      +-------------->+   cCCC      |
+ |             |               |             |
+ +-------------+               \------+------/
+ |    Auth     |                      |
+ +-------------+                      | Write Changes
+ |    Log      |                      |
+ +-------------+                      v
+ | Monitor Map |               /------+------\
+ +-------------+               | Key / Value |
+ |   OSD Map   |               |    Store    |
+ +-------------+               |  cCCC       |
+ |   PG Map    |               \------+------/
+ +-------------+                      ^
+ |   MDS Map   |                      | Read Changes
+ +-------------+                      |
+ |    cCCC     |*---------------------+
+ \-------------/
+
+
+.. deprecated:: version 0.58
+
+In Ceph versions 0.58 and earlier, Ceph Monitors use a Paxos instance for
+each service and store the map as a file. 
+
+.. index:: Ceph Monitor; cluster map
+
+Cluster Maps
+------------
+
+The cluster map is a composite of maps, including the monitor map, the OSD map,
+the placement group map and the metadata server map. The cluster map tracks a
+number of important things: which processes are ``in`` the Ceph Storage Cluster;
+which processes that are ``in`` the Ceph Storage Cluster are ``up`` and running
+or ``down``; whether, the placement groups are ``active`` or ``inactive``, and
+``clean`` or in some other state; and, other details that reflect the current
+state of the cluster such as the total amount of storage space, and the amount
+of storage used.
+
+When there is a significant change in the state of the cluster--e.g., a Ceph OSD
+Daemon goes down, a placement group falls into a degraded state, etc.--the
+cluster map gets updated to reflect the current state of the cluster.
+Additionally, the Ceph Monitor also maintains a history of the prior states of
+the cluster. The monitor map, OSD map, placement group map and metadata server
+map each maintain a history of their map versions. We call each version an
+"epoch."
+
+When operating your Ceph Storage Cluster, keeping track of these states is an
+important part of your system administration duties. See `Monitoring a Cluster`_
+and `Monitoring OSDs and PGs`_ for additional details.
+
+.. index:: high availability; quorum
+
+Monitor Quorum
+--------------
+
+Our Configuring ceph section provides a trivial `Ceph configuration file`_ that
+provides for one monitor in the test cluster. A cluster will run fine with a
+single monitor; however, **a single monitor is a single-point-of-failure**. To
+ensure high availability in a production Ceph Storage Cluster, you should run
+Ceph with multiple monitors so that the failure of a single monitor **WILL NOT**
+bring down your entire cluster.
+
+When a Ceph Storage Cluster runs multiple Ceph Monitors for high availability,
+Ceph Monitors use `Paxos`_ to establish consensus about the master cluster map.
+A consensus requires a majority of monitors running to establish a quorum for
+consensus about the cluster map (e.g., 1; 2 out of 3; 3 out of 5; 4 out of 6;
+etc.).
+
+``mon force quorum join``
+
+:Description: Force monitor to join quorum even if it has been previously removed from the map
+:Type: Boolean
+:Default: ``False``
+
+.. index:: Ceph Monitor; consistency
+
+Consistency
+-----------
+
+When you add monitor settings to your Ceph configuration file, you need to be
+aware of some of the architectural aspects of Ceph Monitors. **Ceph imposes
+strict consistency requirements** for a Ceph monitor when discovering another
+Ceph Monitor within the cluster. Whereas, Ceph Clients and other Ceph daemons
+use the Ceph configuration file to discover monitors, monitors discover each
+other using the monitor map (monmap), not the Ceph configuration file.
+
+A Ceph Monitor always refers to the local copy of the monmap when discovering
+other Ceph Monitors in the Ceph Storage Cluster. Using the monmap instead of the
+Ceph configuration file avoids errors that could break the cluster (e.g., typos
+in ``ceph.conf`` when specifying a monitor address or port). Since monitors use
+monmaps for discovery and they share monmaps with clients and other Ceph
+daemons, **the monmap provides monitors with a strict guarantee that their
+consensus is valid.**
+
+Strict consistency also applies to updates to the monmap. As with any other
+updates on the Ceph Monitor, changes to the monmap always run through a
+distributed consensus algorithm called `Paxos`_. The Ceph Monitors must agree on
+each update to the monmap, such as adding or removing a Ceph Monitor, to ensure
+that each monitor in the quorum has the same version of the monmap. Updates to
+the monmap are incremental so that Ceph Monitors have the latest agreed upon
+version, and a set of previous versions. Maintaining a history enables a Ceph
+Monitor that has an older version of the monmap to catch up with the current
+state of the Ceph Storage Cluster.
+
+If Ceph Monitors were to discover each other through the Ceph configuration file
+instead of through the monmap, additional risks would be introduced because
+Ceph configuration files are not updated and distributed automatically. Ceph
+Monitors might inadvertently use an older Ceph configuration file, fail to
+recognize a Ceph Monitor, fall out of a quorum, or develop a situation where
+`Paxos`_ is not able to determine the current state of the system accurately.
+
+
+.. index:: Ceph Monitor; bootstrapping monitors
+
+Bootstrapping Monitors
+----------------------
+
+In most configuration and deployment cases, tools that deploy Ceph help
+bootstrap the Ceph Monitors by generating a monitor map for you (e.g.,
+``cephadm``, etc). A Ceph Monitor requires a few explicit
+settings:
+
+- **Filesystem ID**: The ``fsid`` is the unique identifier for your
+  object store. Since you can run multiple clusters on the same
+  hardware, you must specify the unique ID of the object store when
+  bootstrapping a monitor.  Deployment tools usually do this for you
+  (e.g., ``cephadm`` can call a tool like ``uuidgen``), but you
+  may specify the ``fsid`` manually too.
+  
+- **Monitor ID**: A monitor ID is a unique ID assigned to each monitor within 
+  the cluster. It is an alphanumeric value, and by convention the identifier 
+  usually follows an alphabetical increment (e.g., ``a``, ``b``, etc.). This 
+  can be set in a Ceph configuration file (e.g., ``[mon.a]``, ``[mon.b]``, etc.), 
+  by a deployment tool, or using the ``ceph`` commandline.
+
+- **Keys**: The monitor must have secret keys. A deployment tool such as 
+  ``cephadm`` usually does this for you, but you may
+  perform this step manually too. See `Monitor Keyrings`_ for details.
+
+For additional details on bootstrapping, see `Bootstrapping a Monitor`_.
+
+.. index:: Ceph Monitor; configuring monitors
+
+Configuring Monitors
+====================
+
+To apply configuration settings to the entire cluster, enter the configuration
+settings under ``[global]``. To apply configuration settings to all monitors in
+your cluster, enter the configuration settings under ``[mon]``. To apply
+configuration settings to specific monitors, specify the monitor instance 
+(e.g., ``[mon.a]``). By convention, monitor instance names use alpha notation.
+
+.. code-block:: ini
+
+	[global]
+
+	[mon]		
+		
+	[mon.a]
+		
+	[mon.b]
+		
+	[mon.c]
+
+
+Minimum Configuration
+---------------------
+
+The bare minimum monitor settings for a Ceph monitor via the Ceph configuration
+file include a hostname and a network address for each monitor. You can configure
+these under ``[mon]`` or under the entry for a specific monitor.
+
+.. code-block:: ini
+
+	[global]
+		mon host = 10.0.0.2,10.0.0.3,10.0.0.4
+
+.. code-block:: ini
+
+	[mon.a]
+		host = hostname1
+		mon addr = 10.0.0.10:6789
+
+See the `Network Configuration Reference`_ for details.
+
+.. note:: This minimum configuration for monitors assumes that a deployment 
+   tool generates the ``fsid`` and the ``mon.`` key for you.
+
+Once you deploy a Ceph cluster, you **SHOULD NOT** change the IP addresses of
+monitors. However, if you decide to change the monitor's IP address, you
+must follow a specific procedure. See `Changing a Monitor's IP Address`_ for
+details.
+
+Monitors can also be found by clients by using DNS SRV records. See `Monitor lookup through DNS`_ for details.
+
+Cluster ID
+----------
+
+Each Ceph Storage Cluster has a unique identifier (``fsid``). If specified, it
+usually appears under the ``[global]`` section of the configuration file.
+Deployment tools usually generate the ``fsid`` and store it in the monitor map,
+so the value may not appear in a configuration file. The ``fsid`` makes it
+possible to run daemons for multiple clusters on the same hardware.
+
+``fsid``
+
+:Description: The cluster ID. One per cluster.
+:Type: UUID
+:Required: Yes.
+:Default: N/A. May be generated by a deployment tool if not specified.
+
+.. note:: Do not set this value if you use a deployment tool that does
+   it for you.
+
+
+.. index:: Ceph Monitor; initial members
+
+Initial Members
+---------------
+
+We recommend running a production Ceph Storage Cluster with at least three Ceph
+Monitors to ensure high availability. When you run multiple monitors, you may
+specify the initial monitors that must be members of the cluster in order to
+establish a quorum. This may reduce the time it takes for your cluster to come
+online.
+
+.. code-block:: ini
+
+	[mon]		
+		mon_initial_members = a,b,c
+
+
+``mon_initial_members``
+
+:Description: The IDs of initial monitors in a cluster during startup. If 
+              specified, Ceph requires an odd number of monitors to form an 
+              initial quorum (e.g., 3). 
+
+:Type: String
+:Default: None
+
+.. note:: A *majority* of monitors in your cluster must be able to reach 
+   each other in order to establish a quorum. You can decrease the initial 
+   number of monitors to establish a quorum with this setting.
+
+.. index:: Ceph Monitor; data path
+
+Data
+----
+
+Ceph provides a default path where Ceph Monitors store data. For optimal
+performance in a production Ceph Storage Cluster, we recommend running Ceph
+Monitors on separate hosts and drives from Ceph OSD Daemons. As leveldb uses
+``mmap()`` for writing the data, Ceph Monitors flush their data from memory to disk
+very often, which can interfere with Ceph OSD Daemon workloads if the data
+store is co-located with the OSD Daemons.
+
+In Ceph versions 0.58 and earlier, Ceph Monitors store their data in plain files. This 
+approach allows users to inspect monitor data with common tools like ``ls``
+and ``cat``. However, this approach didn't provide strong consistency.
+
+In Ceph versions 0.59 and later, Ceph Monitors store their data as key/value
+pairs. Ceph Monitors require `ACID`_ transactions. Using a data store prevents
+recovering Ceph Monitors from running corrupted versions through Paxos, and it
+enables multiple modification operations in one single atomic batch, among other
+advantages.
+
+Generally, we do not recommend changing the default data location. If you modify
+the default location, we recommend that you make it uniform across Ceph Monitors
+by setting it in the ``[mon]`` section of the configuration file.
+
+
+``mon_data`` 
+
+:Description: The monitor's data location.
+:Type: String
+:Default: ``/var/lib/ceph/mon/$cluster-$id``
+
+
+``mon_data_size_warn``
+
+:Description: Raise ``HEALTH_WARN`` status when a monitor's data
+              store grows to be larger than this size, 15GB by default.
+
+:Type: Integer
+:Default: ``15*1024*1024*1024``
+
+
+``mon_data_avail_warn``
+
+:Description: Raise ``HEALTH_WARN`` status when the filesystem that houses a
+              monitor's data store reports that its available capacity is
+              less than or equal to this percentage .
+
+:Type: Integer
+:Default: ``30``
+
+
+``mon_data_avail_crit``
+
+:Description: Raise ``HEALTH_ERR`` status when the filesystem that houses a
+              monitor's data store reports that its available capacity is
+              less than or equal to this percentage.
+
+:Type: Integer
+:Default: ``5``
+
+``mon_warn_on_cache_pools_without_hit_sets``
+
+:Description: Raise ``HEALTH_WARN`` when a cache pool does not
+              have the ``hit_set_type`` value configured.
+              See :ref:`hit_set_type <hit_set_type>` for more
+              details.
+
+:Type: Boolean
+:Default: ``True``
+
+``mon_warn_on_crush_straw_calc_version_zero``
+
+:Description: Raise ``HEALTH_WARN`` when the CRUSH
+              ``straw_calc_version`` is zero. See
+              :ref:`CRUSH map tunables <crush-map-tunables>` for
+              details.
+
+:Type: Boolean
+:Default: ``True``
+
+
+``mon_warn_on_legacy_crush_tunables``
+
+:Description: Raise ``HEALTH_WARN`` when
+              CRUSH tunables are too old (older than ``mon_min_crush_required_version``)
+
+:Type: Boolean
+:Default: ``True``
+
+
+``mon_crush_min_required_version``
+
+:Description: The minimum tunable profile required by the cluster.
+              See
+              :ref:`CRUSH map tunables <crush-map-tunables>` for
+              details.
+
+:Type: String
+:Default: ``hammer``
+
+
+``mon_warn_on_osd_down_out_interval_zero``
+
+:Description: Raise ``HEALTH_WARN`` when
+              ``mon_osd_down_out_interval`` is zero. Having this option set to
+              zero on the leader acts much like the ``noout`` flag. It's hard
+              to figure out what's going wrong with clusters without the
+              ``noout`` flag set but acting like that just the same, so we
+              report a warning in this case.
+
+:Type: Boolean
+:Default: ``True``
+
+
+``mon_warn_on_slow_ping_ratio``
+
+:Description: Raise ``HEALTH_WARN`` when any heartbeat
+              between OSDs exceeds ``mon_warn_on_slow_ping_ratio``
+              of ``osd_heartbeat_grace``.  The default is 5%.
+:Type: Float
+:Default: ``0.05``
+
+
+``mon_warn_on_slow_ping_time``
+
+:Description: Override ``mon_warn_on_slow_ping_ratio`` with a specific value.
+              Raise ``HEALTH_WARN`` if any heartbeat
+              between OSDs exceeds ``mon_warn_on_slow_ping_time``
+              milliseconds.  The default is 0 (disabled).
+:Type: Integer
+:Default: ``0``
+
+
+``mon_warn_on_pool_no_redundancy``
+
+:Description: Raise ``HEALTH_WARN`` if any pool is
+              configured with no replicas.
+:Type: Boolean
+:Default: ``True``
+
+
+``mon_cache_target_full_warn_ratio``
+
+:Description: Position between pool's ``cache_target_full`` and
+              ``target_max_object`` where we start warning
+
+:Type: Float
+:Default: ``0.66``
+
+
+``mon_health_to_clog``
+
+:Description: Enable sending a health summary to the cluster log periodically.
+:Type: Boolean
+:Default: ``True``
+
+
+``mon_health_to_clog_tick_interval``
+
+:Description: How often (in seconds) the monitor sends a health summary to the cluster
+              log (a non-positive number disables). If current health summary
+              is empty or identical to the last time, monitor will not send it
+              to cluster log.
+
+:Type: Float
+:Default: ``60.0``
+
+
+``mon_health_to_clog_interval``
+
+:Description: How often (in seconds) the monitor sends a health summary to the cluster
+              log (a non-positive number disables). Monitors will always
+              send a summary to the cluster log whether or not it differs from
+              the previous summary.
+
+:Type: Integer
+:Default: ``3600``
+
+
+
+.. index:: Ceph Storage Cluster; capacity planning, Ceph Monitor; capacity planning
+
+.. _storage-capacity:
+
+Storage Capacity
+----------------
+
+When a Ceph Storage Cluster gets close to its maximum capacity
+(see``mon_osd_full ratio``), Ceph prevents you from writing to or reading from OSDs
+as a safety measure to prevent data loss. Therefore, letting a
+production Ceph Storage Cluster approach its full ratio is not a good practice,
+because it sacrifices high availability. The default full ratio is ``.95``, or
+95% of capacity. This a very aggressive setting for a test cluster with a small
+number of OSDs.
+
+.. tip:: When monitoring your cluster, be alert to warnings related to the 
+   ``nearfull`` ratio. This means that a failure of some OSDs could result
+   in a temporary service disruption if one or more OSDs fails. Consider adding
+   more OSDs to increase storage capacity.
+
+A common scenario for test clusters involves a system administrator removing an
+OSD from the Ceph Storage Cluster, watching the cluster rebalance, then removing
+another OSD, and another, until at least one OSD eventually reaches the full
+ratio and the cluster locks up. We recommend a bit of capacity
+planning even with a test cluster. Planning enables you to gauge how much spare
+capacity you will need in order to maintain high availability. Ideally, you want
+to plan for a series of Ceph OSD Daemon failures where the cluster can recover
+to an ``active+clean`` state without replacing those OSDs
+immediately. Cluster operation continues in the ``active+degraded`` state, but this
+is not ideal for normal operation and should be addressed promptly.
+
+The following diagram depicts a simplistic Ceph Storage Cluster containing 33
+Ceph Nodes with one OSD per host, each OSD reading from
+and writing to a 3TB drive. So this exemplary Ceph Storage Cluster has a maximum
+actual capacity of 99TB. With a ``mon osd full ratio`` of ``0.95``, if the Ceph
+Storage Cluster falls to 5TB of remaining capacity, the cluster will not allow
+Ceph Clients to read and write data. So the Ceph Storage Cluster's operating
+capacity is 95TB, not 99TB.
+
+.. ditaa::
+ +--------+  +--------+  +--------+  +--------+  +--------+  +--------+
+ | Rack 1 |  | Rack 2 |  | Rack 3 |  | Rack 4 |  | Rack 5 |  | Rack 6 |
+ | cCCC   |  | cF00   |  | cCCC   |  | cCCC   |  | cCCC   |  | cCCC   |
+ +--------+  +--------+  +--------+  +--------+  +--------+  +--------+
+ | OSD 1  |  | OSD 7  |  | OSD 13 |  | OSD 19 |  | OSD 25 |  | OSD 31 |
+ +--------+  +--------+  +--------+  +--------+  +--------+  +--------+
+ | OSD 2  |  | OSD 8  |  | OSD 14 |  | OSD 20 |  | OSD 26 |  | OSD 32 |
+ +--------+  +--------+  +--------+  +--------+  +--------+  +--------+
+ | OSD 3  |  | OSD 9  |  | OSD 15 |  | OSD 21 |  | OSD 27 |  | OSD 33 |
+ +--------+  +--------+  +--------+  +--------+  +--------+  +--------+
+ | OSD 4  |  | OSD 10 |  | OSD 16 |  | OSD 22 |  | OSD 28 |  | Spare  | 
+ +--------+  +--------+  +--------+  +--------+  +--------+  +--------+
+ | OSD 5  |  | OSD 11 |  | OSD 17 |  | OSD 23 |  | OSD 29 |  | Spare  |
+ +--------+  +--------+  +--------+  +--------+  +--------+  +--------+
+ | OSD 6  |  | OSD 12 |  | OSD 18 |  | OSD 24 |  | OSD 30 |  | Spare  |
+ +--------+  +--------+  +--------+  +--------+  +--------+  +--------+
+
+It is normal in such a cluster for one or two OSDs to fail. A less frequent but
+reasonable scenario involves a rack's router or power supply failing, which
+brings down multiple OSDs simultaneously (e.g., OSDs 7-12). In such a scenario,
+you should still strive for a cluster that can remain operational and achieve an
+``active + clean`` state--even if that means adding a few hosts with additional
+OSDs in short order. If your capacity utilization is too high, you may not lose
+data, but you could still sacrifice data availability while resolving an outage
+within a failure domain if capacity utilization of the cluster exceeds the full
+ratio. For this reason, we recommend at least some rough capacity planning.
+
+Identify two numbers for your cluster:
+
+#. The number of OSDs. 
+#. The total capacity of the cluster 
+
+If you divide the total capacity of your cluster by the number of OSDs in your
+cluster, you will find the mean average capacity of an OSD within your cluster.
+Consider multiplying that number by the number of OSDs you expect will fail
+simultaneously during normal operations (a relatively small number). Finally
+multiply the capacity of the cluster by the full ratio to arrive at a maximum
+operating capacity; then, subtract the number of amount of data from the OSDs
+you expect to fail to arrive at a reasonable full ratio. Repeat the foregoing
+process with a higher number of OSD failures (e.g., a rack of OSDs) to arrive at
+a reasonable number for a near full ratio.
+
+The following settings only apply on cluster creation and are then stored in
+the OSDMap. To clarify, in normal operation the values that are used by OSDs
+are those found in the OSDMap, not those in the configuration file or central
+config store.
+
+.. code-block:: ini
+
+	[global]
+		mon_osd_full_ratio = .80
+		mon_osd_backfillfull_ratio = .75
+		mon_osd_nearfull_ratio = .70
+
+
+``mon_osd_full_ratio`` 
+
+:Description: The threshold percentage of device space utilized before an OSD is 
+              considered ``full``.
+
+:Type: Float
+:Default: ``0.95``
+
+
+``mon_osd_backfillfull_ratio``
+
+:Description: The threshold percentage of device space utilized before an OSD is
+              considered too ``full`` to backfill.
+
+:Type: Float
+:Default: ``0.90``
+
+
+``mon_osd_nearfull_ratio`` 
+
+:Description: The threshold percentage of device space used before an OSD is 
+              considered ``nearfull``.
+
+:Type: Float
+:Default: ``0.85``
+
+
+.. tip:: If some OSDs are nearfull, but others have plenty of capacity, you 
+         may have an inaccurate CRUSH weight set for the nearfull OSDs.
+
+.. tip:: These settings only apply during cluster creation. Afterwards they need
+         to be changed in the OSDMap using ``ceph osd set-nearfull-ratio`` and
+         ``ceph osd set-full-ratio``
+
+.. index:: heartbeat
+
+Heartbeat
+---------
+
+Ceph monitors know about the cluster by requiring reports from each OSD, and by
+receiving reports from OSDs about the status of their neighboring OSDs. Ceph
+provides reasonable default settings for monitor/OSD interaction; however,  you
+may modify them as needed. See `Monitor/OSD Interaction`_ for details.
+
+
+.. index:: Ceph Monitor; leader, Ceph Monitor; provider, Ceph Monitor; requester, Ceph Monitor; synchronization
+
+Monitor Store Synchronization
+-----------------------------
+
+When you run a production cluster with multiple monitors (recommended), each
+monitor checks to see if a neighboring monitor has a more recent version of the
+cluster map (e.g., a map in a neighboring monitor with one or more epoch numbers
+higher than the most current epoch in the map of the instant monitor).
+Periodically, one monitor in the cluster may fall behind the other monitors to
+the point where it must leave the quorum, synchronize to retrieve the most
+current information about the cluster, and then rejoin the quorum. For the
+purposes of synchronization, monitors may assume one of three roles: 
+
+#. **Leader**: The `Leader` is the first monitor to achieve the most recent
+   Paxos version of the cluster map.
+
+#. **Provider**: The `Provider` is a monitor that has the most recent version
+   of the cluster map, but wasn't the first to achieve the most recent version.
+
+#. **Requester:** A `Requester` is a monitor that has fallen behind the leader
+   and must synchronize in order to retrieve the most recent information about
+   the cluster before it can rejoin the quorum.
+
+These roles enable a leader to delegate synchronization duties to a provider,
+which prevents synchronization requests from overloading the leader--improving
+performance. In the following diagram, the requester has learned that it has
+fallen behind the other monitors. The requester asks the leader to synchronize,
+and the leader tells the requester to synchronize with a provider.
+
+
+.. ditaa::
+           +-----------+          +---------+          +----------+
+           | Requester |          | Leader  |          | Provider |
+           +-----------+          +---------+          +----------+
+                  |                    |                     |
+                  |                    |                     |
+                  | Ask to Synchronize |                     |
+                  |------------------->|                     |
+                  |                    |                     |
+                  |<-------------------|                     |
+                  | Tell Requester to  |                     |
+                  | Sync with Provider |                     |
+                  |                    |                     |
+                  |               Synchronize                |
+                  |--------------------+-------------------->|
+                  |                    |                     |
+                  |<-------------------+---------------------|
+                  |        Send Chunk to Requester           |
+                  |         (repeat as necessary)            |
+                  |    Requester Acks Chuck to Provider      |
+                  |--------------------+-------------------->|
+                  |                    |
+                  |   Sync Complete    |
+                  |    Notification    |
+                  |------------------->|
+                  |                    |
+                  |<-------------------|
+                  |        Ack         |
+                  |                    |
+
+
+Synchronization always occurs when a new monitor joins the cluster. During
+runtime operations, monitors may receive updates to the cluster map at different
+times. This means the leader and provider roles may migrate from one monitor to
+another. If this happens while synchronizing (e.g., a provider falls behind the
+leader), the provider can terminate synchronization with a requester.
+
+Once synchronization is complete, Ceph performs trimming across the cluster. 
+Trimming requires that the placement groups are ``active+clean``.
+
+
+``mon_sync_timeout``
+
+:Description: Number of seconds the monitor will wait for the next update
+              message from its sync provider before it gives up and bootstrap
+              again.
+
+:Type: Double
+:Default: ``60.0``
+
+
+``mon_sync_max_payload_size``
+
+:Description: The maximum size for a sync payload (in bytes).
+:Type: 32-bit Integer
+:Default: ``1048576``
+
+
+``paxos_max_join_drift``
+
+:Description: The maximum Paxos iterations before we must first sync the
+              monitor data stores. When a monitor finds that its peer is too
+              far ahead of it, it will first sync with data stores before moving
+              on.
+
+:Type: Integer
+:Default: ``10``
+
+
+``paxos_stash_full_interval``
+
+:Description: How often (in commits) to stash a full copy of the PaxosService state.
+              Current this setting only affects ``mds``, ``mon``, ``auth`` and ``mgr``
+              PaxosServices.
+
+:Type: Integer
+:Default: ``25``
+
+
+``paxos_propose_interval``
+
+:Description: Gather updates for this time interval before proposing 
+              a map update.
+
+:Type: Double
+:Default: ``1.0``
+
+
+``paxos_min``
+
+:Description: The minimum number of Paxos states to keep around
+:Type: Integer
+:Default: ``500``
+
+
+``paxos_min_wait``
+
+:Description: The minimum amount of time to gather updates after a period of 
+              inactivity.
+
+:Type: Double
+:Default: ``0.05``
+
+
+``paxos_trim_min``
+
+:Description: Number of extra proposals tolerated before trimming
+:Type: Integer
+:Default: ``250``
+
+
+``paxos_trim_max``
+
+:Description: The maximum number of extra proposals to trim at a time
+:Type: Integer
+:Default: ``500``
+
+
+``paxos_service_trim_min``
+
+:Description: The minimum amount of versions to trigger a trim (0 disables it)
+:Type: Integer
+:Default: ``250``
+
+
+``paxos_service_trim_max``
+
+:Description: The maximum amount of versions to trim during a single proposal (0 disables it)
+:Type: Integer
+:Default: ``500``
+
+
+``paxos service trim max multiplier``
+
+:Description: The factor by which paxos service trim max will be multiplied
+              to get a new upper bound when trim sizes are high (0 disables it)
+:Type: Integer
+:Default: ``20``
+
+
+``mon mds force trim to``
+
+:Description: Force monitor to trim mdsmaps to this point (0 disables it.
+              dangerous, use with care)
+
+:Type: Integer
+:Default: ``0``
+
+
+``mon_osd_force_trim_to``
+
+:Description: Force monitor to trim osdmaps to this point, even if there is
+              PGs not clean at the specified epoch (0 disables it. dangerous,
+              use with care)
+
+:Type: Integer
+:Default: ``0``
+
+
+``mon_osd_cache_size``
+
+:Description: The size of osdmaps cache, not to rely on underlying store's cache
+:Type: Integer
+:Default: ``500``
+
+
+``mon_election_timeout``
+
+:Description: On election proposer, maximum waiting time for all ACKs in seconds.
+:Type: Float
+:Default: ``5.00``
+
+
+``mon_lease`` 
+
+:Description: The length (in seconds) of the lease on the monitor's versions.
+:Type: Float
+:Default: ``5.00``
+
+
+``mon_lease_renew_interval_factor``
+
+:Description: ``mon_lease`` \* ``mon_lease_renew_interval_factor`` will be the
+              interval for the Leader to renew the other monitor's leases. The
+              factor should be less than ``1.0``.
+
+:Type: Float
+:Default: ``0.60``
+
+
+``mon_lease_ack_timeout_factor``
+
+:Description: The Leader will wait ``mon_lease`` \* ``mon_lease_ack_timeout_factor``
+              for the Providers to acknowledge the lease extension.
+
+:Type: Float
+:Default: ``2.00``
+
+
+``mon_accept_timeout_factor``
+
+:Description: The Leader will wait ``mon_lease`` \* ``mon_accept_timeout_factor``
+              for the Requester(s) to accept a Paxos update. It is also used
+              during the Paxos recovery phase for similar purposes.
+
+:Type: Float
+:Default: ``2.00``
+
+
+``mon_min_osdmap_epochs`` 
+
+:Description: Minimum number of OSD map epochs to keep at all times.
+:Type: 32-bit Integer
+:Default: ``500``
+
+
+``mon_max_log_epochs`` 
+
+:Description: Maximum number of Log epochs the monitor should keep.
+:Type: 32-bit Integer
+:Default: ``500``
+
+
+
+.. index:: Ceph Monitor; clock
+
+Clock
+-----
+
+Ceph daemons pass critical messages to each other, which must be processed
+before daemons reach a timeout threshold. If the clocks in Ceph monitors
+are not synchronized, it can lead to a number of anomalies. For example:
+
+- Daemons ignoring received messages (e.g., timestamps outdated)
+- Timeouts triggered too soon/late when a message wasn't received in time.
+
+See `Monitor Store Synchronization`_ for details.
+
+
+.. tip:: You must configure NTP or PTP daemons on your Ceph monitor hosts to 
+         ensure that the monitor cluster operates with synchronized clocks.
+         It can be advantageous to have monitor hosts sync with each other
+         as well as with multiple quality upstream time sources.
+
+Clock drift may still be noticeable with NTP even though the discrepancy is not
+yet harmful. Ceph's clock drift / clock skew warnings may get triggered even 
+though NTP maintains a reasonable level of synchronization. Increasing your 
+clock drift may be tolerable under such circumstances; however, a number of 
+factors such as workload, network latency, configuring overrides to default 
+timeouts and the `Monitor Store Synchronization`_ settings may influence 
+the level of acceptable clock drift without compromising Paxos guarantees.
+
+Ceph provides the following tunable options to allow you to find 
+acceptable values.
+
+
+``mon_tick_interval`` 
+
+:Description: A monitor's tick interval in seconds. 
+:Type: 32-bit Integer
+:Default: ``5`` 
+
+
+``mon_clock_drift_allowed`` 
+
+:Description: The clock drift in seconds allowed between monitors.
+:Type: Float
+:Default: ``0.05``
+
+
+``mon_clock_drift_warn_backoff`` 
+
+:Description: Exponential backoff for clock drift warnings
+:Type: Float
+:Default: ``5.00``
+
+
+``mon_timecheck_interval``
+
+:Description: The time check interval (clock drift check) in seconds 
+              for the Leader.
+
+:Type: Float
+:Default: ``300.00``
+
+
+``mon_timecheck_skew_interval``
+
+:Description: The time check interval (clock drift check) in seconds when in
+              presence of a skew in seconds for the Leader.
+
+:Type: Float
+:Default: ``30.00``
+
+
+Client
+------
+
+``mon_client_hunt_interval``
+
+:Description: The client will try a new monitor every ``N`` seconds until it
+              establishes a connection.
+              
+:Type: Double
+:Default: ``3.00``
+
+
+``mon_client_ping_interval``
+
+:Description: The client will ping the monitor every ``N`` seconds.
+:Type: Double
+:Default: ``10.00``
+
+
+``mon_client_max_log_entries_per_message``
+
+:Description: The maximum number of log entries a monitor will generate 
+              per client message.
+
+:Type: Integer
+:Default: ``1000``
+
+
+``mon_client_bytes``
+
+:Description: The amount of client message data allowed in memory (in bytes).
+:Type: 64-bit Integer Unsigned
+:Default: ``100ul << 20``
+
+.. _pool-settings:
+
+Pool settings
+=============
+
+Since version v0.94 there is support for pool flags which allow or disallow changes to be made to pools.
+Monitors can also disallow removal of pools if appropriately configured. The inconvenience of this guardrail
+is far outweighed by the number of accidental pool (and thus data) deletions it prevents.
+
+``mon_allow_pool_delete``
+
+:Description: Should monitors allow pools to be removed, regardless of what the pool flags say?
+
+:Type: Boolean
+:Default: ``false``
+
+
+``osd_pool_default_ec_fast_read``
+
+:Description: Whether to turn on fast read on the pool or not. It will be used as
+              the default setting of newly created erasure coded pools if ``fast_read``
+              is not specified at create time.
+
+:Type: Boolean
+:Default: ``false``
+
+
+``osd_pool_default_flag_hashpspool``
+
+:Description: Set the hashpspool flag on new pools
+:Type: Boolean
+:Default: ``true``
+
+
+``osd_pool_default_flag_nodelete``
+
+:Description: Set the ``nodelete`` flag on new pools, which prevents pool removal.
+:Type: Boolean
+:Default: ``false``
+
+
+``osd_pool_default_flag_nopgchange``
+
+:Description: Set the ``nopgchange`` flag on new pools. Does not allow the number of PGs to be changed.
+:Type: Boolean
+:Default: ``false``
+
+
+``osd_pool_default_flag_nosizechange``
+
+:Description: Set the ``nosizechange`` flag on new pools. Does not allow the ``size`` to be changed.
+:Type: Boolean
+:Default: ``false``
+
+For more information about the pool flags see `Pool values`_.
+
+Miscellaneous
+=============
+
+``mon_max_osd``
+
+:Description: The maximum number of OSDs allowed in the cluster.
+:Type: 32-bit Integer
+:Default: ``10000``
+
+
+``mon_globalid_prealloc`` 
+
+:Description: The number of global IDs to pre-allocate for clients and daemons in the cluster.
+:Type: 32-bit Integer
+:Default: ``10000``
+
+
+``mon_subscribe_interval`` 
+
+:Description: The refresh interval (in seconds) for subscriptions. The 
+              subscription mechanism enables obtaining cluster maps 
+              and log information.
+
+:Type: Double
+:Default: ``86400.00`` 
+
+
+``mon_stat_smooth_intervals``
+
+:Description: Ceph will smooth statistics over the last ``N`` PG maps.
+:Type: Integer
+:Default: ``6``
+
+
+``mon_probe_timeout`` 
+
+:Description: Number of seconds the monitor will wait to find peers before bootstrapping.
+:Type: Double
+:Default: ``2.00``
+
+
+``mon_daemon_bytes``
+
+:Description: The message memory cap for metadata server and OSD messages (in bytes).
+:Type: 64-bit Integer Unsigned
+:Default: ``400ul << 20``
+
+
+``mon_max_log_entries_per_event``
+
+:Description: The maximum number of log entries per event. 
+:Type: Integer
+:Default: ``4096``
+
+
+``mon_osd_prime_pg_temp``
+
+:Description: Enables or disables priming the PGMap with the previous OSDs when an ``out``
+              OSD comes back into the cluster. With the ``true`` setting, clients
+              will continue to use the previous OSDs until the newly ``in`` OSDs for
+              a PG have peered.
+
+:Type: Boolean
+:Default: ``true``
+
+
+``mon_osd_prime pg temp max time``
+
+:Description: How much time in seconds the monitor should spend trying to prime the
+              PGMap when an out OSD comes back into the cluster.
+
+:Type: Float
+:Default: ``0.50``
+
+
+``mon_osd_prime_pg_temp_max_time_estimate``
+
+:Description: Maximum estimate of time spent on each PG before we prime all PGs
+              in parallel.
+
+:Type: Float
+:Default: ``0.25``
+
+
+``mon_mds_skip_sanity``
+
+:Description: Skip safety assertions on FSMap (in case of bugs where we want to
+              continue anyway). Monitor terminates if the FSMap sanity check
+              fails, but we can disable it by enabling this option.
+
+:Type: Boolean
+:Default: ``False``
+
+
+``mon_max_mdsmap_epochs``
+
+:Description: The maximum number of mdsmap epochs to trim during a single proposal.
+:Type: Integer
+:Default: ``500``
+
+
+``mon_config_key_max_entry_size``
+
+:Description: The maximum size of config-key entry (in bytes)
+:Type: Integer
+:Default: ``65536``
+
+
+``mon_scrub_interval``
+
+:Description: How often the monitor scrubs its store by comparing
+              the stored checksums with the computed ones for all stored
+              keys. (0 disables it. dangerous, use with care)
+
+:Type: Seconds
+:Default: ``1 day``
+
+
+``mon_scrub_max_keys``
+
+:Description: The maximum number of keys to scrub each time.
+:Type: Integer
+:Default: ``100``
+
+
+``mon_compact_on_start``
+
+:Description: Compact the database used as Ceph Monitor store on
+              ``ceph-mon`` start. A manual compaction helps to shrink the
+              monitor database and improve the performance of it if the regular
+              compaction fails to work.
+
+:Type: Boolean
+:Default: ``False``
+
+
+``mon_compact_on_bootstrap``
+
+:Description: Compact the database used as Ceph Monitor store
+              on bootstrap. Monitors probe each other to establish
+              a quorum after bootstrap. If a monitor times out before joining the
+              quorum, it will start over and bootstrap again.
+
+:Type: Boolean
+:Default: ``False``
+
+
+``mon_compact_on_trim``
+
+:Description: Compact a certain prefix (including paxos) when we trim its old states.
+:Type: Boolean
+:Default: ``True``
+
+
+``mon_cpu_threads``
+
+:Description: Number of threads for performing CPU intensive work on monitor.
+:Type: Integer
+:Default: ``4``
+
+
+``mon_osd_mapping_pgs_per_chunk``
+
+:Description: We calculate the mapping from placement group to OSDs in chunks.
+              This option specifies the number of placement groups per chunk.
+
+:Type: Integer
+:Default: ``4096``
+
+
+``mon_session_timeout``
+
+:Description: Monitor will terminate inactive sessions stay idle over this
+              time limit.
+
+:Type: Integer
+:Default: ``300``
+
+
+``mon_osd_cache_size_min``
+
+:Description: The minimum amount of bytes to be kept mapped in memory for osd
+               monitor caches.
+
+:Type: 64-bit Integer
+:Default: ``134217728``
+
+
+``mon_memory_target``
+
+:Description: The amount of bytes pertaining to OSD monitor caches and KV cache
+              to be kept mapped in memory with cache auto-tuning enabled.
+
+:Type: 64-bit Integer
+:Default: ``2147483648``
+
+
+``mon_memory_autotune``
+
+:Description: Autotune the cache memory used for OSD monitors and KV
+              database.
+
+:Type: Boolean
+:Default: ``True``
+
+
+.. _Paxos: https://en.wikipedia.org/wiki/Paxos_(computer_science)
+.. _Monitor Keyrings: ../../../dev/mon-bootstrap#secret-keys
+.. _Ceph configuration file: ../ceph-conf/#monitors
+.. _Network Configuration Reference: ../network-config-ref
+.. _Monitor lookup through DNS: ../mon-lookup-dns
+.. _ACID: https://en.wikipedia.org/wiki/ACID
+.. _Adding/Removing a Monitor: ../../operations/add-or-rm-mons
+.. _Monitoring a Cluster: ../../operations/monitoring
+.. _Monitoring OSDs and PGs: ../../operations/monitoring-osd-pg
+.. _Bootstrapping a Monitor: ../../../dev/mon-bootstrap
+.. _Changing a Monitor's IP Address: ../../operations/add-or-rm-mons#changing-a-monitor-s-ip-address
+.. _Monitor/OSD Interaction: ../mon-osd-interaction
+.. _Scalability and High Availability: ../../../architecture#scalability-and-high-availability
+.. _Pool values: ../../operations/pools/#set-pool-values
diff --git a/doc/rados/configuration/mon-lookup-dns.rst b/doc/rados/configuration/mon-lookup-dns.rst
new file mode 100644
index 000000000..c9bece004
--- /dev/null
+++ b/doc/rados/configuration/mon-lookup-dns.rst
@@ -0,0 +1,56 @@
+===============================
+Looking up Monitors through DNS
+===============================
+
+Since version 11.0.0 RADOS supports looking up Monitors through DNS.
+
+This way daemons and clients do not require a *mon host* configuration directive in their ceph.conf configuration file.
+
+Using DNS SRV TCP records clients are able to look up the monitors.
+
+This allows for less configuration on clients and monitors. Using a DNS update clients and daemons can be made aware of changes in the monitor topology.
+
+By default clients and daemons will look for the TCP service called *ceph-mon* which is configured by the *mon_dns_srv_name* configuration directive.
+
+
+``mon dns srv name``
+
+:Description: the service name used querying the DNS for the monitor hosts/addresses
+:Type: String
+:Default: ``ceph-mon``
+
+Example
+-------
+When the DNS search domain is set to *example.com* a DNS zone file might contain the following elements.
+
+First, create records for the Monitors, either IPv4 (A) or IPv6 (AAAA).
+
+::
+
+    mon1.example.com. AAAA 2001:db8::100
+    mon2.example.com. AAAA 2001:db8::200
+    mon3.example.com. AAAA 2001:db8::300
+
+::
+
+    mon1.example.com. A 192.168.0.1
+    mon2.example.com. A 192.168.0.2
+    mon3.example.com. A 192.168.0.3
+
+
+With those records now existing we can create the SRV TCP records with the name *ceph-mon* pointing to the three Monitors.
+
+::
+
+    _ceph-mon._tcp.example.com. 60 IN SRV 10 20 6789 mon1.example.com.
+    _ceph-mon._tcp.example.com. 60 IN SRV 10 30 6789 mon2.example.com.
+    _ceph-mon._tcp.example.com. 60 IN SRV 20 50 6789 mon3.example.com.
+
+Now all Monitors are running on port *6789*, with priorities 10, 10, 20 and weights 20, 30, 50 respectively.
+
+Monitor clients choose monitor by referencing the SRV records. If a cluster has multiple Monitor SRV records
+with the same priority value, clients and daemons will load balance the connections to Monitors in proportion
+to the values of the SRV weight fields.
+
+For the above example, this will result in approximate 40% of the clients and daemons connecting to mon1,
+60% of them connecting to mon2. However, if neither of them is reachable, then mon3 will be reconsidered as a fallback.
diff --git a/doc/rados/configuration/mon-osd-interaction.rst b/doc/rados/configuration/mon-osd-interaction.rst
new file mode 100644
index 000000000..727070491
--- /dev/null
+++ b/doc/rados/configuration/mon-osd-interaction.rst
@@ -0,0 +1,396 @@
+=====================================
+ Configuring Monitor/OSD Interaction
+=====================================
+
+.. index:: heartbeat
+
+After you have completed your initial Ceph configuration, you may deploy and run
+Ceph.  When you execute a command such as ``ceph health`` or ``ceph -s``,  the
+:term:`Ceph Monitor` reports on the current state of the :term:`Ceph Storage
+Cluster`. The Ceph Monitor knows about the Ceph Storage Cluster by requiring
+reports from each :term:`Ceph OSD Daemon`, and by receiving reports from Ceph
+OSD Daemons about the status of their neighboring Ceph OSD Daemons. If the Ceph
+Monitor doesn't receive reports, or if it receives reports of changes in the
+Ceph Storage Cluster, the Ceph Monitor updates the status of the :term:`Ceph
+Cluster Map`.
+
+Ceph provides reasonable default settings for Ceph Monitor/Ceph OSD Daemon
+interaction. However, you may override the defaults. The following sections
+describe how Ceph Monitors and Ceph OSD Daemons interact for the purposes of
+monitoring the Ceph Storage Cluster.
+
+.. index:: heartbeat interval
+
+OSDs Check Heartbeats
+=====================
+
+Each Ceph OSD Daemon checks the heartbeat of other Ceph OSD Daemons at random
+intervals less than every 6 seconds.  If a neighboring Ceph OSD Daemon doesn't
+show a heartbeat within a 20 second grace period, the Ceph OSD Daemon may
+consider the neighboring Ceph OSD Daemon ``down`` and report it back to a Ceph
+Monitor, which will update the Ceph Cluster Map. You may change this grace
+period by adding an ``osd heartbeat grace`` setting under the ``[mon]``
+and ``[osd]`` or ``[global]`` section of your Ceph configuration file,
+or by setting the value at runtime.
+
+
+.. ditaa::
+           +---------+          +---------+
+           |  OSD 1  |          |  OSD 2  |
+           +---------+          +---------+
+                |                    |
+                |----+ Heartbeat     |
+                |    | Interval      |
+                |<---+ Exceeded      |
+                |                    |
+                |       Check        |
+                |     Heartbeat      |
+                |------------------->|
+                |                    |
+                |<-------------------|
+                |   Heart Beating    |
+                |                    |
+                |----+ Heartbeat     |
+                |    | Interval      |
+                |<---+ Exceeded      |
+                |                    |
+                |       Check        |
+                |     Heartbeat      |
+                |------------------->|
+                |                    |
+                |----+ Grace         |
+                |    | Period        |
+                |<---+ Exceeded      |
+                |                    |
+                |----+ Mark          |
+                |    | OSD 2         |
+                |<---+ Down          |
+
+
+.. index:: OSD down report
+
+OSDs Report Down OSDs
+=====================
+
+By default, two Ceph OSD Daemons from different hosts must report to the Ceph
+Monitors that another Ceph OSD Daemon is ``down`` before the Ceph Monitors
+acknowledge that the reported Ceph OSD Daemon is ``down``. But there is chance
+that all the OSDs reporting the failure are hosted in a rack with a bad switch
+which has trouble connecting to another OSD. To avoid this sort of false alarm,
+we consider the peers reporting a failure a proxy for a potential "subcluster"
+over the overall cluster that is similarly laggy. This is clearly not true in
+all cases, but will sometimes help us localize the grace correction to a subset
+of the system that is unhappy. ``mon osd reporter subtree level`` is used to
+group the peers into the "subcluster" by their common ancestor type in CRUSH
+map. By default, only two reports from different subtree are required to report
+another Ceph OSD Daemon ``down``. You can change the number of reporters from
+unique subtrees and the common ancestor type required to report a Ceph OSD
+Daemon ``down`` to a Ceph Monitor by adding an ``mon osd min down reporters``
+and ``mon osd reporter subtree level`` settings  under the ``[mon]`` section of
+your Ceph configuration file, or by setting the value at runtime.
+
+
+.. ditaa::
+
+           +---------+     +---------+      +---------+
+           |  OSD 1  |     |  OSD 2  |      | Monitor |
+           +---------+     +---------+      +---------+
+                |               |                |
+                | OSD 3 Is Down |                |
+                |---------------+--------------->|
+                |               |                |
+                |               |                |
+                |               | OSD 3 Is Down  |
+                |               |--------------->|
+                |               |                |
+                |               |                |
+                |               |                |---------+ Mark
+                |               |                |         | OSD 3
+                |               |                |<--------+ Down
+
+
+.. index:: peering failure
+
+OSDs Report Peering Failure
+===========================
+
+If a Ceph OSD Daemon cannot peer with any of the Ceph OSD Daemons defined in its
+Ceph configuration file (or the cluster map), it will ping a Ceph Monitor for
+the most recent copy of the cluster map every 30 seconds. You can change the
+Ceph Monitor heartbeat interval by adding an ``osd mon heartbeat interval``
+setting under the ``[osd]`` section of your Ceph configuration file, or by
+setting the value at runtime.
+
+.. ditaa::
+
+           +---------+     +---------+     +-------+     +---------+
+           |  OSD 1  |     |  OSD 2  |     | OSD 3 |     | Monitor |
+           +---------+     +---------+     +-------+     +---------+
+                |               |              |              |
+                |  Request To   |              |              |
+                |     Peer      |              |              |
+                |-------------->|              |              |
+                |<--------------|              |              |
+                |    Peering                   |              |
+                |                              |              |
+                |  Request To                  |              |
+                |     Peer                     |              |
+                |----------------------------->|              |
+                |                                             |
+                |----+ OSD Monitor                            |
+                |    | Heartbeat                              |
+                |<---+ Interval Exceeded                      |
+                |                                             |
+                |         Failed to Peer with OSD 3           |
+                |-------------------------------------------->|
+                |<--------------------------------------------|
+                |          Receive New Cluster Map            |
+
+
+.. index:: OSD status
+
+OSDs Report Their Status
+========================
+
+If an Ceph OSD Daemon doesn't report to a Ceph Monitor, the Ceph Monitor will
+consider the Ceph OSD Daemon ``down`` after the  ``mon osd report timeout``
+elapses. A Ceph OSD Daemon sends a report to a Ceph Monitor when a reportable
+event such as a failure, a change in placement group stats, a change in
+``up_thru`` or when it boots within 5 seconds. You can change the Ceph OSD
+Daemon minimum report interval by adding an ``osd mon report interval``
+setting under the ``[osd]`` section of your Ceph configuration file, or by
+setting the value at runtime. A Ceph OSD Daemon sends a report to a Ceph
+Monitor every 120 seconds irrespective of whether any notable changes occur.
+You can change the Ceph Monitor report interval by adding an ``osd mon report
+interval max`` setting under the ``[osd]`` section of your Ceph configuration
+file, or by setting the value at runtime.
+
+
+.. ditaa::
+
+           +---------+          +---------+
+           |  OSD 1  |          | Monitor |
+           +---------+          +---------+
+                |                    |
+                |----+ Report Min    |
+                |    | Interval      |
+                |<---+ Exceeded      |
+                |                    |
+                |----+ Reportable    |
+                |    | Event         |
+                |<---+ Occurs        |
+                |                    |
+                |     Report To      |
+                |      Monitor       |
+                |------------------->|
+                |                    |
+                |----+ Report Max    |
+                |    | Interval      |
+                |<---+ Exceeded      |
+                |                    |
+                |     Report To      |
+                |      Monitor       |
+                |------------------->|
+                |                    |
+                |----+ Monitor       |
+                |    | Fails         |
+                |<---+               |
+                                     +----+ Monitor OSD
+                                     |    | Report Timeout
+                                     |<---+ Exceeded
+                                     |
+                                     +----+ Mark
+                                     |    | OSD 1
+                                     |<---+ Down
+
+
+
+
+Configuration Settings
+======================
+
+When modifying heartbeat settings, you should include them in the ``[global]``
+section of your configuration file.
+
+.. index:: monitor heartbeat
+
+Monitor Settings
+----------------
+
+``mon osd min up ratio``
+
+:Description: The minimum ratio of ``up`` Ceph OSD Daemons before Ceph will
+              mark Ceph OSD Daemons ``down``.
+
+:Type: Double
+:Default: ``.3``
+
+
+``mon osd min in ratio``
+
+:Description: The minimum ratio of ``in`` Ceph OSD Daemons before Ceph will
+              mark Ceph OSD Daemons ``out``.
+
+:Type: Double
+:Default: ``.75``
+
+
+``mon osd laggy halflife``
+
+:Description: The number of seconds laggy estimates will decay.
+:Type: Integer
+:Default: ``60*60``
+
+
+``mon osd laggy weight``
+
+:Description: The weight for new samples in laggy estimation decay.
+:Type: Double
+:Default: ``0.3``
+
+
+
+``mon osd laggy max interval``
+
+:Description: Maximum value of ``laggy_interval`` in laggy estimations (in seconds).
+              Monitor uses an adaptive approach to evaluate the ``laggy_interval`` of
+              a certain OSD. This value will be used to calculate the grace time for
+              that OSD.
+:Type: Integer
+:Default: 300
+
+``mon osd adjust heartbeat grace``
+
+:Description: If set to ``true``, Ceph will scale based on laggy estimations.
+:Type: Boolean
+:Default: ``true``
+
+
+``mon osd adjust down out interval``
+
+:Description: If set to ``true``, Ceph will scaled based on laggy estimations.
+:Type: Boolean
+:Default: ``true``
+
+
+``mon osd auto mark in``
+
+:Description: Ceph will mark any booting Ceph OSD Daemons as ``in``
+              the Ceph Storage Cluster.
+
+:Type: Boolean
+:Default: ``false``
+
+
+``mon osd auto mark auto out in``
+
+:Description: Ceph will mark booting Ceph OSD Daemons auto marked ``out``
+              of the Ceph Storage Cluster as ``in`` the cluster.
+
+:Type: Boolean
+:Default: ``true``
+
+
+``mon osd auto mark new in``
+
+:Description: Ceph will mark booting new Ceph OSD Daemons as ``in`` the
+              Ceph Storage Cluster.
+
+:Type: Boolean
+:Default: ``true``
+
+
+``mon osd down out interval``
+
+:Description: The number of seconds Ceph waits before marking a Ceph OSD Daemon
+              ``down`` and ``out`` if it doesn't respond.
+
+:Type: 32-bit Integer
+:Default: ``600``
+
+
+``mon osd down out subtree limit``
+
+:Description: The smallest :term:`CRUSH` unit type that Ceph will **not**
+              automatically mark out. For instance, if set to ``host`` and if
+              all OSDs of a host are down, Ceph will not automatically mark out
+              these OSDs.
+
+:Type: String
+:Default: ``rack``
+
+
+``mon osd report timeout``
+
+:Description: The grace period in seconds before declaring
+              unresponsive Ceph OSD Daemons ``down``.
+
+:Type: 32-bit Integer
+:Default: ``900``
+
+``mon osd min down reporters``
+
+:Description: The minimum number of Ceph OSD Daemons required to report a
+              ``down`` Ceph OSD Daemon.
+
+:Type: 32-bit Integer
+:Default: ``2``
+
+
+``mon_osd_reporter_subtree_level``
+
+:Description: In which level of parent bucket the reporters are counted. The OSDs
+              send failure reports to monitors if they find a peer that is not responsive.
+              Monitors mark the reported ``OSD`` out and then ``down`` after a grace period.
+:Type: String
+:Default: ``host``
+
+
+.. index:: OSD hearbeat
+
+OSD Settings
+------------
+
+``osd_heartbeat_interval``
+
+:Description: How often an Ceph OSD Daemon pings its peers (in seconds).
+:Type: 32-bit Integer
+:Default: ``6``
+
+
+``osd_heartbeat_grace``
+
+:Description: The elapsed time when a Ceph OSD Daemon hasn't shown a heartbeat
+              that the Ceph Storage Cluster considers it ``down``.
+              This setting must be set in both the [mon] and [osd] or [global]
+              sections so that it is read by both monitor and OSD daemons.
+:Type: 32-bit Integer
+:Default: ``20``
+
+
+``osd_mon_heartbeat_interval``
+
+:Description: How often the Ceph OSD Daemon pings a Ceph Monitor if it has no
+              Ceph OSD Daemon peers.
+
+:Type: 32-bit Integer
+:Default: ``30``
+
+
+``osd_mon_heartbeat_stat_stale``
+
+:Description: Stop reporting on heartbeat ping times which haven't been updated for
+              this many seconds.  Set to zero to disable this action.
+
+:Type: 32-bit Integer
+:Default: ``3600``
+
+
+``osd_mon_report_interval``
+
+:Description: The number of seconds a Ceph OSD Daemon may wait
+              from startup or another reportable event before reporting
+              to a Ceph Monitor.
+
+:Type: 32-bit Integer
+:Default: ``5``
diff --git a/doc/rados/configuration/ms-ref.rst b/doc/rados/configuration/ms-ref.rst
new file mode 100644
index 000000000..113bd0913
--- /dev/null
+++ b/doc/rados/configuration/ms-ref.rst
@@ -0,0 +1,133 @@
+===========
+ Messaging
+===========
+
+General Settings
+================
+
+``ms_tcp_nodelay``
+
+:Description: Disables Nagle's algorithm on messenger TCP sessions.
+:Type: Boolean
+:Required: No
+:Default: ``true``
+
+
+``ms_initial_backoff``
+
+:Description: The initial time to wait before reconnecting on a fault.
+:Type: Double
+:Required: No
+:Default: ``.2``
+
+
+``ms_max_backoff``
+
+:Description: The maximum time to wait before reconnecting on a fault.
+:Type: Double
+:Required: No
+:Default: ``15.0``
+
+
+``ms_nocrc``
+
+:Description: Disables CRC on network messages.  May increase performance if CPU limited.
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``ms_die_on_bad_msg``
+
+:Description: Debug option; do not configure.
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``ms_dispatch_throttle_bytes``
+
+:Description: Throttles total size of messages waiting to be dispatched.
+:Type: 64-bit Unsigned Integer
+:Required: No
+:Default: ``100 << 20``
+
+
+``ms_bind_ipv6``
+
+:Description: Enable to bind daemons to IPv6 addresses instead of IPv4. Not required if you specify a daemon or cluster IP.
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``ms_rwthread_stack_bytes``
+
+:Description: Debug option for stack size; do not configure.
+:Type: 64-bit Unsigned Integer
+:Required: No
+:Default: ``1024 << 10``
+
+
+``ms_tcp_read_timeout``
+
+:Description: Controls how long (in seconds) the messenger will wait before closing an idle connection.
+:Type: 64-bit Unsigned Integer
+:Required: No
+:Default: ``900``
+
+
+``ms_inject_socket_failures``
+
+:Description: Debug option; do not configure.
+:Type: 64-bit Unsigned Integer
+:Required: No
+:Default: ``0``
+
+Async messenger options
+=======================
+
+
+``ms_async_transport_type``
+
+:Description: Transport type used by Async Messenger. Can be ``posix``, ``dpdk``
+              or ``rdma``. Posix uses standard TCP/IP networking and is default. 
+              Other transports may be experimental and support may be limited.
+:Type: String
+:Required: No
+:Default: ``posix``
+
+
+``ms_async_op_threads``
+
+:Description: Initial number of worker threads used by each Async Messenger instance.
+              Should be at least equal to highest number of replicas, but you can
+              decrease it if you are low on CPU core count and/or you host a lot of
+              OSDs on single server.
+:Type: 64-bit Unsigned Integer
+:Required: No
+:Default: ``3``
+
+
+``ms_async_max_op_threads``
+
+:Description: Maximum number of worker threads used by each Async Messenger instance. 
+              Set to lower values when your machine has limited CPU count, and increase 
+              when your CPUs are underutilized (i. e. one or more of CPUs are
+              constantly on 100% load during I/O operations).
+:Type: 64-bit Unsigned Integer
+:Required: No
+:Default: ``5``
+
+
+``ms_async_send_inline``
+
+:Description: Send messages directly from the thread that generated them instead of
+              queuing and sending from Async Messenger thread. This option is known
+              to decrease performance on systems with a lot of CPU cores, so it's
+              disabled by default.
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
diff --git a/doc/rados/configuration/msgr2.rst b/doc/rados/configuration/msgr2.rst
new file mode 100644
index 000000000..3415b1f5f
--- /dev/null
+++ b/doc/rados/configuration/msgr2.rst
@@ -0,0 +1,233 @@
+.. _msgr2:
+
+Messenger v2
+============
+
+What is it
+----------
+
+The messenger v2 protocol, or msgr2, is the second major revision on
+Ceph's on-wire protocol.  It brings with it several key features:
+
+* A *secure* mode that encrypts all data passing over the network
+* Improved encapsulation of authentication payloads, enabling future
+  integration of new authentication modes like Kerberos
+* Improved earlier feature advertisement and negotiation, enabling
+  future protocol revisions
+
+Ceph daemons can now bind to multiple ports, allowing both legacy Ceph
+clients and new v2-capable clients to connect to the same cluster.
+
+By default, monitors now bind to the new IANA-assigned port ``3300``
+(ce4h or 0xce4) for the new v2 protocol, while also binding to the
+old default port ``6789`` for the legacy v1 protocol.
+
+.. _address_formats:
+
+Address formats
+---------------
+
+Prior to Nautilus, all network addresses were rendered like
+``1.2.3.4:567/89012`` where there was an IP address, a port, and a
+nonce to uniquely identify a client or daemon on the network.
+Starting with Nautilus, we now have three different address types:
+
+* **v2**: ``v2:1.2.3.4:578/89012`` identifies a daemon binding to a
+  port speaking the new v2 protocol
+* **v1**: ``v1:1.2.3.4:578/89012`` identifies a daemon binding to a
+  port speaking the legacy v1 protocol.  Any address that was
+  previously shown with any prefix is now shown as a ``v1:`` address.
+* **TYPE_ANY** ``any:1.2.3.4:578/89012`` identifies a client that can
+  speak either version of the protocol. Prior to nautilus, clients would appear as
+  ``1.2.3.4:0/123456``, where the port of 0 indicates they are clients
+  and do not accept incoming connections.  Starting with Nautilus,
+  these clients are now internally represented by a **TYPE_ANY**
+  address, and still shown with no prefix, because they may
+  connect to daemons using the v2 or v1 protocol, depending on what
+  protocol(s) the daemons are using.
+
+Because daemons now bind to multiple ports, they are now described by
+a vector of addresses instead of a single address.  For example,
+dumping the monitor map on a Nautilus cluster now includes lines
+like::
+
+  epoch 1
+  fsid 50fcf227-be32-4bcb-8b41-34ca8370bd16
+  last_changed 2019-02-25 11:10:46.700821
+  created 2019-02-25 11:10:46.700821
+  min_mon_release 14 (nautilus)
+  0: [v2:10.0.0.10:3300/0,v1:10.0.0.10:6789/0] mon.foo
+  1: [v2:10.0.0.11:3300/0,v1:10.0.0.11:6789/0] mon.bar
+  2: [v2:10.0.0.12:3300/0,v1:10.0.0.12:6789/0] mon.baz
+
+The bracketed list or vector of addresses means that the same daemon can be
+reached on multiple ports (and protocols).  Any client or other daemon
+connecting to that daemon will use the v2 protocol (listed first) if
+possible; otherwise it will back to the legacy v1 protocol.  Legacy
+clients will only see the v1 addresses and will continue to connect as
+they did before, with the v1 protocol.
+
+Starting in Nautilus, the ``mon_host`` configuration option and ``-m
+<mon-host>`` command line options support the same bracketed address
+vector syntax.
+
+
+Bind configuration options
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Two new configuration options control whether the v1 and/or v2
+protocol is used:
+
+  * ``ms_bind_msgr1`` [default: true] controls whether a daemon binds
+    to a port speaking the v1 protocol
+  * ``ms_bind_msgr2`` [default: true] controls whether a daemon binds
+    to a port speaking the v2 protocol
+
+Similarly, two options control whether IPv4 and IPv6 addresses are used:
+
+  * ``ms_bind_ipv4`` [default: true] controls whether a daemon binds
+    to an IPv4 address
+  * ``ms_bind_ipv6`` [default: false] controls whether a daemon binds
+    to an IPv6 address
+
+.. note:: The ability to bind to multiple ports has paved the way for
+   dual-stack IPv4 and IPv6 support.  That said, dual-stack support is
+   not yet tested as of Nautilus v14.2.0 and likely needs some
+   additional code changes to work correctly.
+
+Connection modes
+----------------
+
+The v2 protocol supports two connection modes:
+
+* *crc* mode provides:
+
+  - a strong initial authentication when the connection is established
+    (with cephx, mutual authentication of both parties with protection
+    from a man-in-the-middle or eavesdropper), and
+  - a crc32c integrity check to protect against bit flips due to flaky
+    hardware or cosmic rays
+
+  *crc* mode does *not* provide:
+
+  - secrecy (an eavesdropper on the network can see all
+    post-authentication traffic as it goes by) or
+  - protection from a malicious man-in-the-middle (who can deliberate
+    modify traffic as it goes by, as long as they are careful to
+    adjust the crc32c values to match)
+
+* *secure* mode provides:
+
+  - a strong initial authentication when the connection is established
+    (with cephx, mutual authentication of both parties with protection
+    from a man-in-the-middle or eavesdropper), and
+  - full encryption of all post-authentication traffic, including a
+    cryptographic integrity check.
+
+  In Nautilus, secure mode uses the `AES-GCM
+  <https://en.wikipedia.org/wiki/Galois/Counter_Mode>`_ stream cipher,
+  which is generally very fast on modern processors (e.g., faster than
+  a SHA-256 cryptographic hash).
+
+Connection mode configuration options
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+For most connections, there are options that control which modes are used:
+
+* ``ms_cluster_mode`` is the connection mode (or permitted modes) used
+  for intra-cluster communication between Ceph daemons.  If multiple
+  modes are listed, the modes listed first are preferred.
+* ``ms_service_mode`` is a list of permitted modes for clients to use
+  when connecting to the cluster.
+* ``ms_client_mode`` is a list of connection modes, in order of
+  preference, for clients to use (or allow) when talking to a Ceph
+  cluster.
+
+There are a parallel set of options that apply specifically to
+monitors, allowing administrators to set different (usually more
+secure) requirements on communication with the monitors.
+
+* ``ms_mon_cluster_mode`` is the connection mode (or permitted modes)
+  to use between monitors.
+* ``ms_mon_service_mode`` is a list of permitted modes for clients or
+  other Ceph daemons to use when connecting to monitors.
+* ``ms_mon_client_mode`` is a list of connection modes, in order of
+  preference, for clients or non-monitor daemons to use when
+  connecting to monitors.
+
+
+Transitioning from v1-only to v2-plus-v1
+----------------------------------------
+
+By default, ``ms_bind_msgr2`` is true starting with Nautilus 14.2.z.
+However, until the monitors start using v2, only limited services will
+start advertising v2 addresses.
+
+For most users, the monitors are binding to the default legacy port ``6789``
+for the v1 protocol.  When this is the case, enabling v2 is as simple as:
+
+.. prompt:: bash $
+
+   ceph mon enable-msgr2
+
+If the monitors are bound to non-standard ports, you will need to
+specify an additional port for v2 explicitly.  For example, if your
+monitor ``mon.a`` binds to ``1.2.3.4:1111``, and you want to add v2 on
+port ``1112``:
+
+.. prompt:: bash $
+
+   ceph mon set-addrs a [v2:1.2.3.4:1112,v1:1.2.3.4:1111]
+
+Once the monitors bind to v2, each daemon will start advertising a v2
+address when it is next restarted.
+
+
+.. _msgr2_ceph_conf:
+
+Updating ceph.conf and mon_host
+-------------------------------
+
+Prior to Nautilus, a CLI user or daemon will normally discover the
+monitors via the ``mon_host`` option in ``/etc/ceph/ceph.conf``.  The
+syntax for this option has expanded starting with Nautilus to allow
+support the new bracketed list format.  For example, an old line
+like::
+
+  mon_host = 10.0.0.1:6789,10.0.0.2:6789,10.0.0.3:6789
+
+Can be changed to::
+
+  mon_host = [v2:10.0.0.1:3300/0,v1:10.0.0.1:6789/0],[v2:10.0.0.2:3300/0,v1:10.0.0.2:6789/0],[v2:10.0.0.3:3300/0,v1:10.0.0.3:6789/0]
+
+However, when default ports are used (``3300`` and ``6789``), they can
+be omitted::
+
+  mon_host = 10.0.0.1,10.0.0.2,10.0.0.3
+
+Once v2 has been enabled on the monitors, ``ceph.conf`` may need to be
+updated to either specify no ports (this is usually simplest), or
+explicitly specify both the v2 and v1 addresses.  Note, however, that
+the new bracketed syntax is only understood by Nautilus and later, so
+do not make that change on hosts that have not yet had their ceph
+packages upgraded.
+
+When you are updating ``ceph.conf``, note the new ``ceph config
+generate-minimal-conf`` command (which generates a barebones config
+file with just enough information to reach the monitors) and the
+``ceph config assimilate-conf`` (which moves config file options into
+the monitors' configuration database) may be helpful.  For example,::
+
+  # ceph config assimilate-conf < /etc/ceph/ceph.conf
+  # ceph config generate-minimal-config > /etc/ceph/ceph.conf.new
+  # cat /etc/ceph/ceph.conf.new
+  # minimal ceph.conf for 0e5a806b-0ce5-4bc6-b949-aa6f68f5c2a3
+  [global]
+          fsid = 0e5a806b-0ce5-4bc6-b949-aa6f68f5c2a3
+          mon_host = [v2:10.0.0.1:3300/0,v1:10.0.0.1:6789/0]
+  # mv /etc/ceph/ceph.conf.new /etc/ceph/ceph.conf
+
+Protocol
+--------
+
+For a detailed description of the v2 wire protocol, see :ref:`msgr2-protocol`.
diff --git a/doc/rados/configuration/network-config-ref.rst b/doc/rados/configuration/network-config-ref.rst
new file mode 100644
index 000000000..97229a401
--- /dev/null
+++ b/doc/rados/configuration/network-config-ref.rst
@@ -0,0 +1,454 @@
+=================================
+ Network Configuration Reference
+=================================
+
+Network configuration is critical for building a high performance  :term:`Ceph
+Storage Cluster`. The Ceph Storage Cluster does not perform  request routing or
+dispatching on behalf of the :term:`Ceph Client`. Instead, Ceph Clients make
+requests directly to Ceph OSD Daemons. Ceph OSD Daemons perform data replication
+on behalf of Ceph Clients, which means replication and other factors impose
+additional loads on Ceph Storage Cluster networks.
+
+Our Quick Start configurations provide a trivial Ceph configuration file that
+sets monitor IP addresses and daemon host names only. Unless you specify a
+cluster network, Ceph assumes a single "public" network. Ceph functions just
+fine with a public network only, but you may see significant performance
+improvement with a second "cluster" network in a large cluster.
+
+It is possible to run a Ceph Storage Cluster with two networks: a public
+(client, front-side) network and a cluster (private, replication, back-side)
+network.  However, this approach
+complicates network configuration (both hardware and software) and does not usually
+have a significant impact on overall performance.  For this reason, we recommend
+that for resilience and capacity dual-NIC systems either active/active bond
+these interfaces or implemebnt a layer 3 multipath strategy with eg. FRR.
+
+If, despite the complexity, one still wishes to use two networks, each
+:term:`Ceph Node` will need to have more than one network interface or VLAN. See `Hardware
+Recommendations - Networks`_ for additional details.
+
+.. ditaa::
+                               +-------------+
+                               | Ceph Client |
+                               +----*--*-----+
+                                    |  ^
+                            Request |  : Response
+                                    v  |
+ /----------------------------------*--*-------------------------------------\
+ |                              Public Network                               |
+ \---*--*------------*--*-------------*--*------------*--*------------*--*---/
+     ^  ^            ^  ^             ^  ^            ^  ^            ^  ^
+     |  |            |  |             |  |            |  |            |  |
+     |  :            |  :             |  :            |  :            |  :
+     v  v            v  v             v  v            v  v            v  v
+ +---*--*---+    +---*--*---+     +---*--*---+    +---*--*---+    +---*--*---+
+ | Ceph MON |    | Ceph MDS |     | Ceph OSD |    | Ceph OSD |    | Ceph OSD |
+ +----------+    +----------+     +---*--*---+    +---*--*---+    +---*--*---+
+                                      ^  ^            ^  ^            ^  ^
+     The cluster network relieves     |  |            |  |            |  |
+     OSD replication and heartbeat    |  :            |  :            |  :
+     traffic from the public network. v  v            v  v            v  v
+ /------------------------------------*--*------------*--*------------*--*---\
+ |   cCCC                      Cluster Network                               |
+ \---------------------------------------------------------------------------/
+
+
+IP Tables
+=========
+
+By default, daemons `bind`_ to ports within the ``6800:7300`` range. You may
+configure this range at your discretion. Before configuring your IP tables,
+check the default ``iptables`` configuration.
+
+.. prompt:: bash $
+
+   sudo iptables -L
+
+Some Linux distributions include rules that reject all inbound requests
+except SSH from all network interfaces. For example:: 
+
+	REJECT all -- anywhere anywhere reject-with icmp-host-prohibited
+
+You will need to delete these rules on both your public and cluster networks
+initially, and replace them with appropriate rules when you are ready to 
+harden the ports on your Ceph Nodes.
+
+
+Monitor IP Tables
+-----------------
+
+Ceph Monitors listen on ports ``3300`` and ``6789`` by
+default. Additionally, Ceph Monitors always operate on the public
+network. When you add the rule using the example below, make sure you
+replace ``{iface}`` with the public network interface (e.g., ``eth0``,
+``eth1``, etc.), ``{ip-address}`` with the IP address of the public
+network and ``{netmask}`` with the netmask for the public network. :
+
+.. prompt:: bash $
+
+   sudo iptables -A INPUT -i {iface} -p tcp -s {ip-address}/{netmask} --dport 6789 -j ACCEPT
+
+
+MDS and Manager IP Tables
+-------------------------
+
+A :term:`Ceph Metadata Server` or :term:`Ceph Manager` listens on the first 
+available port on the public network beginning at port 6800. Note that this 
+behavior is not deterministic, so if you are running more than one OSD or MDS
+on the same host, or if you restart the daemons within a short window of time,
+the daemons will bind to higher ports. You should open the entire 6800-7300
+range by default.  When you add the rule using the example below, make sure
+you replace ``{iface}`` with the public network interface (e.g., ``eth0``,
+``eth1``, etc.), ``{ip-address}`` with the IP address of the public network
+and ``{netmask}`` with the netmask of the public network.
+
+For example:
+
+.. prompt:: bash $
+
+   sudo iptables -A INPUT -i {iface} -m multiport -p tcp -s {ip-address}/{netmask} --dports 6800:7300 -j ACCEPT
+
+
+OSD IP Tables
+-------------
+
+By default, Ceph OSD Daemons `bind`_ to the first available ports on a Ceph Node
+beginning at port 6800.  Note that this behavior is not deterministic, so if you
+are running more than one OSD or MDS on the same host, or if you restart the
+daemons within a short window of time, the daemons will bind to higher ports.
+Each Ceph OSD Daemon on a Ceph Node may use up to four ports:
+
+#. One for talking to clients and monitors.
+#. One for sending data to other OSDs.
+#. Two for heartbeating on each interface.
+
+.. ditaa::
+              /---------------\
+              |      OSD      |
+              |           +---+----------------+-----------+
+              |           | Clients & Monitors | Heartbeat |
+              |           +---+----------------+-----------+
+              |               |
+              |           +---+----------------+-----------+
+              |           | Data Replication   | Heartbeat |
+              |           +---+----------------+-----------+
+              | cCCC          |
+              \---------------/
+
+When a daemon fails and restarts without letting go of the port, the restarted
+daemon will bind to a new port. You should open the entire 6800-7300 port range
+to handle this possibility.
+
+If you set up separate public and cluster networks, you must add rules for both
+the public network and the cluster network, because clients will connect using
+the public network and other Ceph OSD Daemons will connect using the cluster
+network. When you add the rule using the example below, make sure you replace
+``{iface}`` with the network interface (e.g., ``eth0``, ``eth1``, etc.),
+``{ip-address}`` with the IP address and ``{netmask}`` with the netmask of the
+public or cluster network. For example:
+
+.. prompt:: bash $
+
+   sudo iptables -A INPUT -i {iface}  -m multiport -p tcp -s {ip-address}/{netmask} --dports 6800:7300 -j ACCEPT
+
+.. tip:: If you run Ceph Metadata Servers on the same Ceph Node as the 
+   Ceph OSD Daemons, you can consolidate the public network configuration step. 
+
+
+Ceph Networks
+=============
+
+To configure Ceph networks, you must add a network configuration to the
+``[global]`` section of the configuration file. Our 5-minute Quick Start
+provides a trivial Ceph configuration file that assumes one public network
+with client and server on the same network and subnet. Ceph functions just fine
+with a public network only. However, Ceph allows you to establish much more
+specific criteria, including multiple IP network and subnet masks for your
+public network. You can also establish a separate cluster network to handle OSD
+heartbeat, object replication and recovery traffic. Don't confuse the IP
+addresses you set in your configuration with the public-facing IP addresses
+network clients may use to access your service. Typical internal IP networks are
+often ``192.168.0.0`` or ``10.0.0.0``.
+
+.. tip:: If you specify more than one IP address and subnet mask for
+   either the public or the cluster network, the subnets within the network
+   must be capable of routing to each other. Additionally, make sure you
+   include each IP address/subnet in your IP tables and open ports for them
+   as necessary.
+
+.. note:: Ceph uses `CIDR`_ notation for subnets (e.g., ``10.0.0.0/24``).
+
+When you have configured your networks, you may restart your cluster or restart
+each daemon. Ceph daemons bind dynamically, so you do not have to restart the
+entire cluster at once if you change your network configuration.
+
+
+Public Network
+--------------
+
+To configure a public network, add the following option to the ``[global]``
+section of your Ceph configuration file. 
+
+.. code-block:: ini
+
+	[global]
+		# ... elided configuration
+		public_network = {public-network/netmask}
+
+.. _cluster-network:
+
+Cluster Network
+---------------
+
+If you declare a cluster network, OSDs will route heartbeat, object replication
+and recovery traffic over the cluster network. This may improve performance
+compared to using a single network. To configure a cluster network, add the
+following option to the ``[global]`` section of your Ceph configuration file. 
+
+.. code-block:: ini
+
+	[global]
+		# ... elided configuration
+		cluster_network = {cluster-network/netmask}
+
+We prefer that the cluster network is **NOT** reachable from the public network
+or the Internet for added security.
+
+IPv4/IPv6 Dual Stack Mode
+-------------------------
+
+If you want to run in an IPv4/IPv6 dual stack mode and want to define your public and/or
+cluster networks, then you need to specify both your IPv4 and IPv6 networks for each:
+
+.. code-block:: ini
+
+	[global]
+		# ... elided configuration
+		public_network = {IPv4 public-network/netmask}, {IPv6 public-network/netmask}
+
+This is so that Ceph can find a valid IP address for both address families.
+
+If you want just an IPv4 or an IPv6 stack environment, then make sure you set the `ms bind`
+options correctly.
+
+.. note::
+   Binding to IPv4 is enabled by default, so if you just add the option to bind to IPv6
+   you'll actually put yourself into dual stack mode. If you want just IPv6, then disable IPv4 and
+   enable IPv6. See `Bind`_ below.
+
+Ceph Daemons
+============
+
+Monitor daemons are each configured to bind to a specific IP address.  These
+addresses are normally configured by your deployment tool.  Other components
+in the Ceph cluster discover the monitors via the ``mon host`` configuration
+option, normally specified in the ``[global]`` section of the ``ceph.conf`` file.
+
+.. code-block:: ini
+
+     [global]
+         mon_host = 10.0.0.2, 10.0.0.3, 10.0.0.4
+
+The ``mon_host`` value can be a list of IP addresses or a name that is
+looked up via DNS.  In the case of a DNS name with multiple A or AAAA
+records, all records are probed in order to discover a monitor.  Once
+one monitor is reached, all other current monitors are discovered, so
+the ``mon host`` configuration option only needs to be sufficiently up
+to date such that a client can reach one monitor that is currently online.
+
+The MGR, OSD, and MDS daemons will bind to any available address and
+do not require any special configuration.  However, it is possible to
+specify a specific IP address for them to bind to with the ``public
+addr`` (and/or, in the case of OSD daemons, the ``cluster addr``)
+configuration option.  For example,
+
+.. code-block:: ini
+
+	[osd.0]
+		public addr = {host-public-ip-address}
+		cluster addr = {host-cluster-ip-address}
+
+.. topic:: One NIC OSD in a Two Network Cluster
+
+   Generally, we do not recommend deploying an OSD host with a single network interface in a 
+   cluster with two networks. However, you may accomplish this by forcing the 
+   OSD host to operate on the public network by adding a ``public_addr`` entry
+   to the ``[osd.n]`` section of the Ceph configuration file, where ``n`` 
+   refers to the ID of the OSD with one network interface. Additionally, the public
+   network and cluster network must be able to route traffic to each other, 
+   which we don't recommend for security reasons.
+
+
+Network Config Settings
+=======================
+
+Network configuration settings are not required. Ceph assumes a public network
+with all hosts operating on it unless you specifically configure a cluster 
+network.
+
+
+Public Network
+--------------
+
+The public network configuration allows you specifically define IP addresses
+and subnets for the public network. You may specifically assign static IP 
+addresses or override ``public_network`` settings using the ``public_addr``
+setting for a specific daemon.
+
+``public_network``
+
+:Description: The IP address and netmask of the public (front-side) network 
+              (e.g., ``192.168.0.0/24``). Set in ``[global]``. You may specify
+              comma-separated subnets.
+
+:Type: ``{ip-address}/{netmask} [, {ip-address}/{netmask}]``
+:Required: No
+:Default: N/A
+
+
+``public_addr``
+
+:Description: The IP address for the public (front-side) network. 
+              Set for each daemon.
+
+:Type: IP Address
+:Required: No
+:Default: N/A
+
+
+
+Cluster Network
+---------------
+
+The cluster network configuration allows you to declare a cluster network, and
+specifically define IP addresses and subnets for the cluster network. You may
+specifically assign static IP  addresses or override ``cluster_network``
+settings using the ``cluster_addr`` setting for specific OSD daemons.
+
+
+``cluster_network``
+
+:Description: The IP address and netmask of the cluster (back-side) network 
+              (e.g., ``10.0.0.0/24``).  Set in ``[global]``. You may specify
+              comma-separated subnets.
+
+:Type: ``{ip-address}/{netmask} [, {ip-address}/{netmask}]``
+:Required: No
+:Default: N/A
+
+
+``cluster_addr``
+
+:Description: The IP address for the cluster (back-side) network. 
+              Set for each daemon.
+
+:Type: Address
+:Required: No
+:Default: N/A
+
+
+Bind
+----
+
+Bind settings set the default port ranges Ceph OSD and MDS daemons use. The
+default range is ``6800:7300``. Ensure that your `IP Tables`_ configuration
+allows you to use the configured port range.
+
+You may also enable Ceph daemons to bind to IPv6 addresses instead of IPv4
+addresses.
+
+
+``ms_bind_port_min``
+
+:Description: The minimum port number to which an OSD or MDS daemon will bind.
+:Type: 32-bit Integer
+:Default: ``6800``
+:Required: No
+
+
+``ms_bind_port_max``
+
+:Description: The maximum port number to which an OSD or MDS daemon will bind.
+:Type: 32-bit Integer
+:Default: ``7300``
+:Required: No. 
+
+``ms_bind_ipv4``
+
+:Description: Enables Ceph daemons to bind to IPv4 addresses.
+:Type: Boolean
+:Default: ``true``
+:Required: No
+
+``ms_bind_ipv6``
+
+:Description: Enables Ceph daemons to bind to IPv6 addresses.
+:Type: Boolean
+:Default: ``false``
+:Required: No
+
+``public_bind_addr``
+
+:Description: In some dynamic deployments the Ceph MON daemon might bind
+              to an IP address locally that is different from the ``public_addr``
+              advertised to other peers in the network. The environment must ensure
+              that routing rules are set correctly. If ``public_bind_addr`` is set
+              the Ceph Monitor daemon will bind to it locally and use ``public_addr``
+              in the monmaps to advertise its address to peers. This behavior is limited
+              to the Monitor daemon.
+
+:Type: IP Address
+:Required: No
+:Default: N/A
+
+
+
+TCP
+---
+
+Ceph disables TCP buffering by default.
+
+
+``ms_tcp_nodelay``
+
+:Description: Ceph enables ``ms_tcp_nodelay`` so that each request is sent 
+              immediately (no buffering). Disabling `Nagle's algorithm`_
+              increases network traffic, which can introduce latency. If you 
+              experience large numbers of small packets, you may try 
+              disabling ``ms_tcp_nodelay``. 
+
+:Type: Boolean
+:Required: No
+:Default: ``true``
+
+
+``ms_tcp_rcvbuf``
+
+:Description: The size of the socket buffer on the receiving end of a network
+              connection. Disable by default.
+
+:Type: 32-bit Integer
+:Required: No
+:Default: ``0``
+
+
+``ms_tcp_read_timeout``
+
+:Description: If a client or daemon makes a request to another Ceph daemon and
+              does not drop an unused connection, the ``ms tcp read timeout`` 
+              defines the connection as idle after the specified number 
+              of seconds.
+
+:Type: Unsigned 64-bit Integer
+:Required: No
+:Default: ``900`` 15 minutes.
+
+
+
+.. _Scalability and High Availability: ../../../architecture#scalability-and-high-availability
+.. _Hardware Recommendations - Networks: ../../../start/hardware-recommendations#networks
+.. _hardware recommendations: ../../../start/hardware-recommendations
+.. _Monitor / OSD Interaction: ../mon-osd-interaction
+.. _Message Signatures: ../auth-config-ref#signatures
+.. _CIDR: https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing
+.. _Nagle's Algorithm: https://en.wikipedia.org/wiki/Nagle's_algorithm
diff --git a/doc/rados/configuration/osd-config-ref.rst b/doc/rados/configuration/osd-config-ref.rst
new file mode 100644
index 000000000..7f69b5e80
--- /dev/null
+++ b/doc/rados/configuration/osd-config-ref.rst
@@ -0,0 +1,1127 @@
+======================
+ OSD Config Reference
+======================
+
+.. index:: OSD; configuration
+
+You can configure Ceph OSD Daemons in the Ceph configuration file (or in recent
+releases, the central config store), but Ceph OSD
+Daemons can use the default values and a very minimal configuration. A minimal
+Ceph OSD Daemon configuration sets ``osd journal size`` (for Filestore), ``host``,  and
+uses default values for nearly everything else.
+
+Ceph OSD Daemons are numerically identified in incremental fashion, beginning
+with ``0`` using the following convention. ::
+
+	osd.0
+	osd.1
+	osd.2
+
+In a configuration file, you may specify settings for all Ceph OSD Daemons in
+the cluster by adding configuration settings to the ``[osd]`` section of your
+configuration file. To add settings directly to a specific Ceph OSD Daemon
+(e.g., ``host``), enter  it in an OSD-specific section of your configuration
+file. For example:
+
+.. code-block:: ini
+
+	[osd]
+		osd_journal_size = 5120
+
+	[osd.0]
+		host = osd-host-a
+
+	[osd.1]
+		host = osd-host-b
+
+
+.. index:: OSD; config settings
+
+General Settings
+================
+
+The following settings provide a Ceph OSD Daemon's ID, and determine paths to
+data and journals. Ceph deployment scripts typically generate the UUID
+automatically.
+
+.. warning:: **DO NOT** change the default paths for data or journals, as it
+             makes it more problematic to troubleshoot Ceph later.
+
+When using Filestore, the journal size should be at least twice the product of the expected drive
+speed multiplied by ``filestore_max_sync_interval``. However, the most common
+practice is to partition the journal drive (often an SSD), and mount it such
+that Ceph uses the entire partition for the journal.
+
+
+``osd_uuid``
+
+:Description: The universally unique identifier (UUID) for the Ceph OSD Daemon.
+:Type: UUID
+:Default: The UUID.
+:Note: The ``osd_uuid`` applies to a single Ceph OSD Daemon. The ``fsid``
+       applies to the entire cluster.
+
+
+``osd_data``
+
+:Description: The path to the OSDs data. You must create the directory when
+              deploying Ceph. You should mount a drive for OSD data at this
+              mount point. We do not recommend changing the default.
+
+:Type: String
+:Default: ``/var/lib/ceph/osd/$cluster-$id``
+
+
+``osd_max_write_size``
+
+:Description: The maximum size of a write in megabytes.
+:Type: 32-bit Integer
+:Default: ``90``
+
+
+``osd_max_object_size``
+
+:Description: The maximum size of a RADOS object in bytes.
+:Type: 32-bit Unsigned Integer
+:Default: 128MB
+
+
+``osd_client_message_size_cap``
+
+:Description: The largest client data message allowed in memory.
+:Type: 64-bit Unsigned Integer
+:Default: 500MB default. ``500*1024L*1024L``
+
+
+``osd_class_dir``
+
+:Description: The class path for RADOS class plug-ins.
+:Type: String
+:Default: ``$libdir/rados-classes``
+
+
+.. index:: OSD; file system
+
+File System Settings
+====================
+Ceph builds and mounts file systems which are used for Ceph OSDs.
+
+``osd_mkfs_options {fs-type}``
+
+:Description: Options used when creating a new Ceph Filestore OSD of type {fs-type}.
+
+:Type: String
+:Default for xfs: ``-f -i 2048``
+:Default for other file systems: {empty string}
+
+For example::
+  ``osd_mkfs_options_xfs = -f -d agcount=24``
+
+``osd_mount_options {fs-type}``
+
+:Description: Options used when mounting a Ceph Filestore OSD of type {fs-type}.
+
+:Type: String
+:Default for xfs: ``rw,noatime,inode64``
+:Default for other file systems: ``rw, noatime``
+
+For example::
+  ``osd_mount_options_xfs = rw, noatime, inode64, logbufs=8``
+
+
+.. index:: OSD; journal settings
+
+Journal Settings
+================
+
+This section applies only to the older Filestore OSD back end.  Since Luminous
+BlueStore has been default and preferred.
+
+By default, Ceph expects that you will provision a Ceph OSD Daemon's journal at
+the following path, which is usually a symlink to a device or partition::
+
+	/var/lib/ceph/osd/$cluster-$id/journal
+
+When using a single device type (for example, spinning drives), the journals
+should be *colocated*: the logical volume (or partition) should be in the same
+device as the ``data`` logical volume.
+
+When using a mix of fast (SSDs, NVMe) devices with slower ones (like spinning
+drives) it makes sense to place the journal on the faster device, while
+``data`` occupies the slower device fully.
+
+The default ``osd_journal_size`` value is 5120 (5 gigabytes), but it can be
+larger, in which case it will need to be set in the ``ceph.conf`` file.
+A value of 10 gigabytes is common in practice::
+
+	osd_journal_size = 10240
+
+
+``osd_journal``
+
+:Description: The path to the OSD's journal. This may be a path to a file or a
+              block device (such as a partition of an SSD). If it is a file,
+              you must create the directory to contain it. We recommend using a
+              separate fast device when the ``osd_data`` drive is an HDD.
+
+:Type: String
+:Default: ``/var/lib/ceph/osd/$cluster-$id/journal``
+
+
+``osd_journal_size``
+
+:Description: The size of the journal in megabytes.
+
+:Type: 32-bit Integer
+:Default: ``5120``
+
+
+See `Journal Config Reference`_ for additional details.
+
+
+Monitor OSD Interaction
+=======================
+
+Ceph OSD Daemons check each other's heartbeats and report to monitors
+periodically. Ceph can use default values in many cases. However, if your
+network has latency issues, you may need to adopt longer intervals. See
+`Configuring Monitor/OSD Interaction`_ for a detailed discussion of heartbeats.
+
+
+Data Placement
+==============
+
+See `Pool & PG Config Reference`_ for details.
+
+
+.. index:: OSD; scrubbing
+
+Scrubbing
+=========
+
+In addition to making multiple copies of objects, Ceph ensures data integrity by
+scrubbing placement groups. Ceph scrubbing is analogous to ``fsck`` on the
+object storage layer. For each placement group, Ceph generates a catalog of all
+objects and compares each primary object and its replicas to ensure that no
+objects are missing or mismatched. Light scrubbing (daily) checks the object
+size and attributes.  Deep scrubbing (weekly) reads the data and uses checksums
+to ensure data integrity.
+
+Scrubbing is important for maintaining data integrity, but it can reduce
+performance. You can adjust the following settings to increase or decrease
+scrubbing operations.
+
+
+``osd_max_scrubs``
+
+:Description: The maximum number of simultaneous scrub operations for
+              a Ceph OSD Daemon.
+
+:Type: 32-bit Int
+:Default: ``1``
+
+``osd_scrub_begin_hour``
+
+:Description: This restricts scrubbing to this hour of the day or later.
+              Use ``osd_scrub_begin_hour = 0`` and ``osd_scrub_end_hour = 0``
+              to allow scrubbing the entire day.  Along with ``osd_scrub_end_hour``, they define a time
+              window, in which the scrubs can happen.
+              But a scrub will be performed
+              no matter whether the time window allows or not, as long as the placement
+              group's scrub interval exceeds ``osd_scrub_max_interval``.
+:Type: Integer in the range of 0 to 23
+:Default: ``0``
+
+
+``osd_scrub_end_hour``
+
+:Description: This restricts scrubbing to the hour earlier than this.
+              Use ``osd_scrub_begin_hour = 0`` and ``osd_scrub_end_hour = 0`` to allow scrubbing
+              for the entire day.  Along with ``osd_scrub_begin_hour``, they define a time
+              window, in which the scrubs can happen. But a scrub will be performed
+              no matter whether the time window allows or not, as long as the placement
+              group's scrub interval exceeds ``osd_scrub_max_interval``.
+:Type: Integer in the range of 0 to 23
+:Default: ``0``
+
+
+``osd_scrub_begin_week_day``
+
+:Description: This restricts scrubbing to this day of the week or later.
+              0  = Sunday, 1 = Monday, etc. Use ``osd_scrub_begin_week_day = 0``
+              and ``osd_scrub_end_week_day = 0`` to allow scrubbing for the entire week.
+              Along with ``osd_scrub_end_week_day``, they define a time window in which
+              scrubs can happen. But a scrub will be performed
+              no matter whether the time window allows or not, when the PG's
+              scrub interval exceeds ``osd_scrub_max_interval``.
+:Type: Integer in the range of 0 to 6
+:Default: ``0``
+
+
+``osd_scrub_end_week_day``
+
+:Description: This restricts scrubbing to days of the week earlier than this.
+              0 = Sunday, 1 = Monday, etc.  Use ``osd_scrub_begin_week_day = 0``
+              and ``osd_scrub_end_week_day = 0`` to allow scrubbing for the entire week.
+              Along with ``osd_scrub_begin_week_day``, they define a time
+              window, in which the scrubs can happen. But a scrub will be performed
+              no matter whether the time window allows or not, as long as the placement
+              group's scrub interval exceeds ``osd_scrub_max_interval``.
+:Type: Integer in the range of 0 to 6
+:Default: ``0``
+
+
+``osd scrub during recovery``
+
+:Description: Allow scrub during recovery. Setting this to ``false`` will disable
+              scheduling new scrub (and deep--scrub) while there is active recovery.
+              Already running scrubs will be continued. This might be useful to reduce
+              load on busy clusters.
+:Type: Boolean
+:Default: ``false``
+
+
+``osd_scrub_thread_timeout``
+
+:Description: The maximum time in seconds before timing out a scrub thread.
+:Type: 32-bit Integer
+:Default: ``60``
+
+
+``osd_scrub_finalize_thread_timeout``
+
+:Description: The maximum time in seconds before timing out a scrub finalize
+              thread.
+
+:Type: 32-bit Integer
+:Default: ``10*60``
+
+
+``osd_scrub_load_threshold``
+
+:Description: The normalized maximum load. Ceph will not scrub when the system load
+              (as defined by ``getloadavg() / number of online CPUs``) is higher than this number.
+              Default is ``0.5``.
+
+:Type: Float
+:Default: ``0.5``
+
+
+``osd_scrub_min_interval``
+
+:Description: The minimal interval in seconds for scrubbing the Ceph OSD Daemon
+              when the Ceph Storage Cluster load is low.
+
+:Type: Float
+:Default: Once per day. ``24*60*60``
+
+.. _osd_scrub_max_interval:
+
+``osd_scrub_max_interval``
+
+:Description: The maximum interval in seconds for scrubbing the Ceph OSD Daemon
+              irrespective of cluster load.
+
+:Type: Float
+:Default: Once per week. ``7*24*60*60``
+
+
+``osd_scrub_chunk_min``
+
+:Description: The minimal number of object store chunks to scrub during single operation.
+              Ceph blocks writes to single chunk during scrub.
+
+:Type: 32-bit Integer
+:Default: 5
+
+
+``osd_scrub_chunk_max``
+
+:Description: The maximum number of object store chunks to scrub during single operation.
+
+:Type: 32-bit Integer
+:Default: 25
+
+
+``osd_scrub_sleep``
+
+:Description: Time to sleep before scrubbing the next group of chunks. Increasing this value will slow
+              down the overall rate of scrubbing so that client operations will be less impacted.
+
+:Type: Float
+:Default: 0
+
+
+``osd_deep_scrub_interval``
+
+:Description: The interval for "deep" scrubbing (fully reading all data). The
+              ``osd_scrub_load_threshold`` does not affect this setting.
+
+:Type: Float
+:Default: Once per week.  ``7*24*60*60``
+
+
+``osd_scrub_interval_randomize_ratio``
+
+:Description: Add a random delay to ``osd_scrub_min_interval`` when scheduling
+              the next scrub job for a PG. The delay is a random
+              value less than ``osd_scrub_min_interval`` \*
+              ``osd_scrub_interval_randomized_ratio``. The default setting
+              spreads scrubs throughout the allowed time
+              window of ``[1, 1.5]`` \* ``osd_scrub_min_interval``.
+:Type: Float
+:Default: ``0.5``
+
+``osd_deep_scrub_stride``
+
+:Description: Read size when doing a deep scrub.
+:Type: 32-bit Integer
+:Default: 512 KB. ``524288``
+
+
+``osd_scrub_auto_repair``
+
+:Description: Setting this to ``true`` will enable automatic PG repair when errors
+              are found by scrubs or deep-scrubs.  However, if more than
+              ``osd_scrub_auto_repair_num_errors`` errors are found a repair is NOT performed.
+:Type: Boolean
+:Default: ``false``
+
+
+``osd_scrub_auto_repair_num_errors``
+
+:Description: Auto repair will not occur if more than this many errors are found.
+:Type: 32-bit Integer
+:Default: ``5``
+
+
+.. index:: OSD; operations settings
+
+Operations
+==========
+
+ ``osd_op_queue``
+
+:Description: This sets the type of queue to be used for prioritizing ops
+              within each OSD. Both queues feature a strict sub-queue which is
+              dequeued before the normal queue. The normal queue is different
+              between implementations. The WeightedPriorityQueue (``wpq``)
+              dequeues operations in relation to their priorities to prevent
+              starvation of any queue. WPQ should help in cases where a few OSDs
+              are more overloaded than others. The new mClockQueue
+              (``mclock_scheduler``) prioritizes operations based on which class
+              they belong to (recovery, scrub, snaptrim, client op, osd subop).
+              See `QoS Based on mClock`_. Requires a restart.
+
+:Type: String
+:Valid Choices: wpq, mclock_scheduler
+:Default: ``wpq``
+
+
+``osd_op_queue_cut_off``
+
+:Description: This selects which priority ops will be sent to the strict
+              queue verses the normal queue. The ``low`` setting sends all
+              replication ops and higher to the strict queue, while the ``high``
+              option sends only replication acknowledgment ops and higher to
+              the strict queue. Setting this to ``high`` should help when a few
+              OSDs in the cluster are very busy especially when combined with
+              ``wpq`` in the ``osd_op_queue`` setting. OSDs that are very busy
+              handling replication traffic could starve primary client traffic
+              on these OSDs without these settings. Requires a restart.
+
+:Type: String
+:Valid Choices: low, high
+:Default: ``high``
+
+
+``osd_client_op_priority``
+
+:Description: The priority set for client operations.  This value is relative
+              to that of ``osd_recovery_op_priority`` below.  The default
+              strongly favors client ops over recovery.
+
+:Type: 32-bit Integer
+:Default: ``63``
+:Valid Range: 1-63
+
+
+``osd_recovery_op_priority``
+
+:Description: The priority of recovery operations vs client operations, if not specified by the
+              pool's ``recovery_op_priority``.  The default value prioritizes client
+              ops (see above) over recovery ops.  You may adjust the tradeoff of client
+              impact against the time to restore cluster health by lowering this value
+              for increased prioritization of client ops, or by increasing it to favor
+              recovery.
+
+:Type: 32-bit Integer
+:Default: ``3``
+:Valid Range: 1-63
+
+
+``osd_scrub_priority``
+
+:Description: The default work queue priority for scheduled scrubs when the
+              pool doesn't specify a value of ``scrub_priority``.  This can be
+              boosted to the value of ``osd_client_op_priority`` when scrubs are
+              blocking client operations.
+
+:Type: 32-bit Integer
+:Default: ``5``
+:Valid Range: 1-63
+
+
+``osd_requested_scrub_priority``
+
+:Description: The priority set for user requested scrub on the work queue.  If
+              this value were to be smaller than ``osd_client_op_priority`` it
+              can be boosted to the value of ``osd_client_op_priority`` when
+              scrub is blocking client operations.
+
+:Type: 32-bit Integer
+:Default: ``120``
+
+
+``osd_snap_trim_priority``
+
+:Description: The priority set for the snap trim work queue.
+
+:Type: 32-bit Integer
+:Default: ``5``
+:Valid Range: 1-63
+
+``osd_snap_trim_sleep``
+
+:Description: Time in seconds to sleep before next snap trim op.
+              Increasing this value will slow down snap trimming.
+              This option overrides backend specific variants.
+
+:Type: Float
+:Default: ``0``
+
+
+``osd_snap_trim_sleep_hdd``
+
+:Description: Time in seconds to sleep before next snap trim op
+              for HDDs.
+
+:Type: Float
+:Default: ``5``
+
+
+``osd_snap_trim_sleep_ssd``
+
+:Description: Time in seconds to sleep before next snap trim op
+              for SSD OSDs (including NVMe).
+
+:Type: Float
+:Default: ``0``
+
+
+``osd_snap_trim_sleep_hybrid``
+
+:Description: Time in seconds to sleep before next snap trim op
+              when OSD data is on an HDD and the OSD journal or WAL+DB is on an SSD.
+
+:Type: Float
+:Default: ``2``
+
+``osd_op_thread_timeout``
+
+:Description: The Ceph OSD Daemon operation thread timeout in seconds.
+:Type: 32-bit Integer
+:Default: ``15``
+
+
+``osd_op_complaint_time``
+
+:Description: An operation becomes complaint worthy after the specified number
+              of seconds have elapsed.
+
+:Type: Float
+:Default: ``30``
+
+
+``osd_op_history_size``
+
+:Description: The maximum number of completed operations to track.
+:Type: 32-bit Unsigned Integer
+:Default: ``20``
+
+
+``osd_op_history_duration``
+
+:Description: The oldest completed operation to track.
+:Type: 32-bit Unsigned Integer
+:Default: ``600``
+
+
+``osd_op_log_threshold``
+
+:Description: How many operations logs to display at once.
+:Type: 32-bit Integer
+:Default: ``5``
+
+
+.. _dmclock-qos:
+
+QoS Based on mClock
+-------------------
+
+Ceph's use of mClock is now more refined and can be used by following the
+steps as described in `mClock Config Reference`_.
+
+Core Concepts
+`````````````
+
+Ceph's QoS support is implemented using a queueing scheduler
+based on `the dmClock algorithm`_. This algorithm allocates the I/O
+resources of the Ceph cluster in proportion to weights, and enforces
+the constraints of minimum reservation and maximum limitation, so that
+the services can compete for the resources fairly. Currently the
+*mclock_scheduler* operation queue divides Ceph services involving I/O
+resources into following buckets:
+
+- client op: the iops issued by client
+- osd subop: the iops issued by primary OSD
+- snap trim: the snap trimming related requests
+- pg recovery: the recovery related requests
+- pg scrub: the scrub related requests
+
+And the resources are partitioned using following three sets of tags. In other
+words, the share of each type of service is controlled by three tags:
+
+#. reservation: the minimum IOPS allocated for the service.
+#. limitation: the maximum IOPS allocated for the service.
+#. weight: the proportional share of capacity if extra capacity or system
+   oversubscribed.
+
+In Ceph, operations are graded with "cost". And the resources allocated
+for serving various services are consumed by these "costs". So, for
+example, the more reservation a services has, the more resource it is
+guaranteed to possess, as long as it requires. Assuming there are 2
+services: recovery and client ops:
+
+- recovery: (r:1, l:5, w:1)
+- client ops: (r:2, l:0, w:9)
+
+The settings above ensure that the recovery won't get more than 5
+requests per second serviced, even if it requires so (see CURRENT
+IMPLEMENTATION NOTE below), and no other services are competing with
+it. But if the clients start to issue large amount of I/O requests,
+neither will they exhaust all the I/O resources. 1 request per second
+is always allocated for recovery jobs as long as there are any such
+requests. So the recovery jobs won't be starved even in a cluster with
+high load. And in the meantime, the client ops can enjoy a larger
+portion of the I/O resource, because its weight is "9", while its
+competitor "1". In the case of client ops, it is not clamped by the
+limit setting, so it can make use of all the resources if there is no
+recovery ongoing.
+
+CURRENT IMPLEMENTATION NOTE: the current implementation enforces the limit
+values. Therefore, if a service crosses the enforced limit, the op remains
+in the operation queue until the limit is restored.
+
+Subtleties of mClock
+````````````````````
+
+The reservation and limit values have a unit of requests per
+second. The weight, however, does not technically have a unit and the
+weights are relative to one another. So if one class of requests has a
+weight of 1 and another a weight of 9, then the latter class of
+requests should get 9 executed at a 9 to 1 ratio as the first class.
+However that will only happen once the reservations are met and those
+values include the operations executed under the reservation phase.
+
+Even though the weights do not have units, one must be careful in
+choosing their values due how the algorithm assigns weight tags to
+requests. If the weight is *W*, then for a given class of requests,
+the next one that comes in will have a weight tag of *1/W* plus the
+previous weight tag or the current time, whichever is larger. That
+means if *W* is sufficiently large and therefore *1/W* is sufficiently
+small, the calculated tag may never be assigned as it will get a value
+of the current time. The ultimate lesson is that values for weight
+should not be too large. They should be under the number of requests
+one expects to be serviced each second.
+
+Caveats
+```````
+
+There are some factors that can reduce the impact of the mClock op
+queues within Ceph. First, requests to an OSD are sharded by their
+placement group identifier. Each shard has its own mClock queue and
+these queues neither interact nor share information among them. The
+number of shards can be controlled with the configuration options
+``osd_op_num_shards``, ``osd_op_num_shards_hdd``, and
+``osd_op_num_shards_ssd``. A lower number of shards will increase the
+impact of the mClock queues, but may have other deleterious effects.
+
+Second, requests are transferred from the operation queue to the
+operation sequencer, in which they go through the phases of
+execution. The operation queue is where mClock resides and mClock
+determines the next op to transfer to the operation sequencer. The
+number of operations allowed in the operation sequencer is a complex
+issue. In general we want to keep enough operations in the sequencer
+so it's always getting work done on some operations while it's waiting
+for disk and network access to complete on other operations. On the
+other hand, once an operation is transferred to the operation
+sequencer, mClock no longer has control over it. Therefore to maximize
+the impact of mClock, we want to keep as few operations in the
+operation sequencer as possible. So we have an inherent tension.
+
+The configuration options that influence the number of operations in
+the operation sequencer are ``bluestore_throttle_bytes``,
+``bluestore_throttle_deferred_bytes``,
+``bluestore_throttle_cost_per_io``,
+``bluestore_throttle_cost_per_io_hdd``, and
+``bluestore_throttle_cost_per_io_ssd``.
+
+A third factor that affects the impact of the mClock algorithm is that
+we're using a distributed system, where requests are made to multiple
+OSDs and each OSD has (can have) multiple shards. Yet we're currently
+using the mClock algorithm, which is not distributed (note: dmClock is
+the distributed version of mClock).
+
+Various organizations and individuals are currently experimenting with
+mClock as it exists in this code base along with their modifications
+to the code base. We hope you'll share you're experiences with your
+mClock and dmClock experiments on the ``ceph-devel`` mailing list.
+
+
+``osd_push_per_object_cost``
+
+:Description: the overhead for serving a push op
+
+:Type: Unsigned Integer
+:Default: 1000
+
+
+``osd_recovery_max_chunk``
+
+:Description: the maximum total size of data chunks a recovery op can carry.
+
+:Type: Unsigned Integer
+:Default: 8 MiB
+
+
+``osd_mclock_scheduler_client_res``
+
+:Description: IO proportion reserved for each client (default).
+
+:Type: Unsigned Integer
+:Default: 1
+
+
+``osd_mclock_scheduler_client_wgt``
+
+:Description: IO share for each client (default) over reservation.
+
+:Type: Unsigned Integer
+:Default: 1
+
+
+``osd_mclock_scheduler_client_lim``
+
+:Description: IO limit for each client (default) over reservation.
+
+:Type: Unsigned Integer
+:Default: 999999
+
+
+``osd_mclock_scheduler_background_recovery_res``
+
+:Description: IO proportion reserved for background recovery (default).
+
+:Type: Unsigned Integer
+:Default: 1
+
+
+``osd_mclock_scheduler_background_recovery_wgt``
+
+:Description: IO share for each background recovery over reservation.
+
+:Type: Unsigned Integer
+:Default: 1
+
+
+``osd_mclock_scheduler_background_recovery_lim``
+
+:Description: IO limit for background recovery over reservation.
+
+:Type: Unsigned Integer
+:Default: 999999
+
+
+``osd_mclock_scheduler_background_best_effort_res``
+
+:Description: IO proportion reserved for background best_effort (default).
+
+:Type: Unsigned Integer
+:Default: 1
+
+
+``osd_mclock_scheduler_background_best_effort_wgt``
+
+:Description: IO share for each background best_effort over reservation.
+
+:Type: Unsigned Integer
+:Default: 1
+
+
+``osd_mclock_scheduler_background_best_effort_lim``
+
+:Description: IO limit for background best_effort over reservation.
+
+:Type: Unsigned Integer
+:Default: 999999
+
+.. _the dmClock algorithm: https://www.usenix.org/legacy/event/osdi10/tech/full_papers/Gulati.pdf
+
+
+.. index:: OSD; backfilling
+
+Backfilling
+===========
+
+When you add or remove Ceph OSD Daemons to a cluster, CRUSH will
+rebalance the cluster by moving placement groups to or from Ceph OSDs
+to restore balanced utilization. The process of migrating placement groups and
+the objects they contain can reduce the cluster's operational performance
+considerably. To maintain operational performance, Ceph performs this migration
+with 'backfilling', which allows Ceph to set backfill operations to a lower
+priority than requests to read or write data.
+
+
+``osd_max_backfills``
+
+:Description: The maximum number of backfills allowed to or from a single OSD.
+              Note that this is applied separately for read and write operations.
+:Type: 64-bit Unsigned Integer
+:Default: ``1``
+
+
+``osd_backfill_scan_min``
+
+:Description: The minimum number of objects per backfill scan.
+
+:Type: 32-bit Integer
+:Default: ``64``
+
+
+``osd_backfill_scan_max``
+
+:Description: The maximum number of objects per backfill scan.
+
+:Type: 32-bit Integer
+:Default: ``512``
+
+
+``osd_backfill_retry_interval``
+
+:Description: The number of seconds to wait before retrying backfill requests.
+:Type: Double
+:Default: ``10.0``
+
+.. index:: OSD; osdmap
+
+OSD Map
+=======
+
+OSD maps reflect the OSD daemons operating in the cluster. Over time, the
+number of map epochs increases. Ceph provides some settings to ensure that
+Ceph performs well as the OSD map grows larger.
+
+
+``osd_map_dedup``
+
+:Description: Enable removing duplicates in the OSD map.
+:Type: Boolean
+:Default: ``true``
+
+
+``osd_map_cache_size``
+
+:Description: The number of OSD maps to keep cached.
+:Type: 32-bit Integer
+:Default: ``50``
+
+
+``osd_map_message_max``
+
+:Description: The maximum map entries allowed per MOSDMap message.
+:Type: 32-bit Integer
+:Default: ``40``
+
+
+
+.. index:: OSD; recovery
+
+Recovery
+========
+
+When the cluster starts or when a Ceph OSD Daemon crashes and restarts, the OSD
+begins peering with other Ceph OSD Daemons before writes can occur.  See
+`Monitoring OSDs and PGs`_ for details.
+
+If a Ceph OSD Daemon crashes and comes back online, usually it will be out of
+sync with other Ceph OSD Daemons containing more recent versions of objects in
+the placement groups. When this happens, the Ceph OSD Daemon goes into recovery
+mode and seeks to get the latest copy of the data and bring its map back up to
+date. Depending upon how long the Ceph OSD Daemon was down, the OSD's objects
+and placement groups may be significantly out of date. Also, if a failure domain
+went down (e.g., a rack), more than one Ceph OSD Daemon may come back online at
+the same time. This can make the recovery process time consuming and resource
+intensive.
+
+To maintain operational performance, Ceph performs recovery with limitations on
+the number recovery requests, threads and object chunk sizes which allows Ceph
+perform well in a degraded state.
+
+
+``osd_recovery_delay_start``
+
+:Description: After peering completes, Ceph will delay for the specified number
+              of seconds before starting to recover RADOS objects.
+
+:Type: Float
+:Default: ``0``
+
+
+``osd_recovery_max_active``
+
+:Description: The number of active recovery requests per OSD at one time. More
+              requests will accelerate recovery, but the requests places an
+              increased load on the cluster.
+
+	      This value is only used if it is non-zero. Normally it
+	      is ``0``, which means that the ``hdd`` or ``ssd`` values
+	      (below) are used, depending on the type of the primary
+	      device backing the OSD.
+
+:Type: 32-bit Integer
+:Default: ``0``
+
+``osd_recovery_max_active_hdd``
+
+:Description: The number of active recovery requests per OSD at one time, if the
+	      primary device is rotational.
+
+:Type: 32-bit Integer
+:Default: ``3``
+
+``osd_recovery_max_active_ssd``
+
+:Description: The number of active recovery requests per OSD at one time, if the
+	      primary device is non-rotational (i.e., an SSD).
+
+:Type: 32-bit Integer
+:Default: ``10``
+
+
+``osd_recovery_max_chunk``
+
+:Description: The maximum size of a recovered chunk of data to push.
+:Type: 64-bit Unsigned Integer
+:Default: ``8 << 20``
+
+
+``osd_recovery_max_single_start``
+
+:Description: The maximum number of recovery operations per OSD that will be
+              newly started when an OSD is recovering.
+:Type: 64-bit Unsigned Integer
+:Default: ``1``
+
+
+``osd_recovery_thread_timeout``
+
+:Description: The maximum time in seconds before timing out a recovery thread.
+:Type: 32-bit Integer
+:Default: ``30``
+
+
+``osd_recover_clone_overlap``
+
+:Description: Preserves clone overlap during recovery. Should always be set
+              to ``true``.
+
+:Type: Boolean
+:Default: ``true``
+
+
+``osd_recovery_sleep``
+
+:Description: Time in seconds to sleep before the next recovery or backfill op.
+              Increasing this value will slow down recovery operation while
+              client operations will be less impacted.
+
+:Type: Float
+:Default: ``0``
+
+
+``osd_recovery_sleep_hdd``
+
+:Description: Time in seconds to sleep before next recovery or backfill op
+              for HDDs.
+
+:Type: Float
+:Default: ``0.1``
+
+
+``osd_recovery_sleep_ssd``
+
+:Description: Time in seconds to sleep before the next recovery or backfill op
+              for SSDs.
+
+:Type: Float
+:Default: ``0``
+
+
+``osd_recovery_sleep_hybrid``
+
+:Description: Time in seconds to sleep before the next recovery or backfill op
+              when OSD data is on HDD and OSD journal / WAL+DB is on SSD.
+
+:Type: Float
+:Default: ``0.025``
+
+
+``osd_recovery_priority``
+
+:Description: The default priority set for recovery work queue.  Not
+              related to a pool's ``recovery_priority``.
+
+:Type: 32-bit Integer
+:Default: ``5``
+
+
+Tiering
+=======
+
+``osd_agent_max_ops``
+
+:Description: The maximum number of simultaneous flushing ops per tiering agent
+              in the high speed mode.
+:Type: 32-bit Integer
+:Default: ``4``
+
+
+``osd_agent_max_low_ops``
+
+:Description: The maximum number of simultaneous flushing ops per tiering agent
+              in the low speed mode.
+:Type: 32-bit Integer
+:Default: ``2``
+
+See `cache target dirty high ratio`_ for when the tiering agent flushes dirty
+objects within the high speed mode.
+
+Miscellaneous
+=============
+
+
+``osd_snap_trim_thread_timeout``
+
+:Description: The maximum time in seconds before timing out a snap trim thread.
+:Type: 32-bit Integer
+:Default: ``1*60*60``
+
+
+``osd_backlog_thread_timeout``
+
+:Description: The maximum time in seconds before timing out a backlog thread.
+:Type: 32-bit Integer
+:Default: ``1*60*60``
+
+
+``osd_default_notify_timeout``
+
+:Description: The OSD default notification timeout (in seconds).
+:Type: 32-bit Unsigned Integer
+:Default: ``30``
+
+
+``osd_check_for_log_corruption``
+
+:Description: Check log files for corruption. Can be computationally expensive.
+:Type: Boolean
+:Default: ``false``
+
+
+``osd_remove_thread_timeout``
+
+:Description: The maximum time in seconds before timing out a remove OSD thread.
+:Type: 32-bit Integer
+:Default: ``60*60``
+
+
+``osd_command_thread_timeout``
+
+:Description: The maximum time in seconds before timing out a command thread.
+:Type: 32-bit Integer
+:Default: ``10*60``
+
+
+``osd_delete_sleep``
+
+:Description: Time in seconds to sleep before the next removal transaction. This
+              throttles the PG deletion process.
+
+:Type: Float
+:Default: ``0``
+
+
+``osd_delete_sleep_hdd``
+
+:Description: Time in seconds to sleep before the next removal transaction
+              for HDDs.
+
+:Type: Float
+:Default: ``5``
+
+
+``osd_delete_sleep_ssd``
+
+:Description: Time in seconds to sleep before the next removal transaction
+              for SSDs.
+
+:Type: Float
+:Default: ``0``
+
+
+``osd_delete_sleep_hybrid``
+
+:Description: Time in seconds to sleep before the next removal transaction
+              when OSD data is on HDD and OSD journal or WAL+DB is on SSD.
+
+:Type: Float
+:Default: ``1``
+
+
+``osd_command_max_records``
+
+:Description: Limits the number of lost objects to return.
+:Type: 32-bit Integer
+:Default: ``256``
+
+
+``osd_fast_fail_on_connection_refused``
+
+:Description: If this option is enabled, crashed OSDs are marked down
+              immediately by connected peers and MONs (assuming that the
+              crashed OSD host survives). Disable it to restore old
+              behavior, at the expense of possible long I/O stalls when
+              OSDs crash in the middle of I/O operations.
+:Type: Boolean
+:Default: ``true``
+
+
+
+.. _pool: ../../operations/pools
+.. _Configuring Monitor/OSD Interaction: ../mon-osd-interaction
+.. _Monitoring OSDs and PGs: ../../operations/monitoring-osd-pg#peering
+.. _Pool & PG Config Reference: ../pool-pg-config-ref
+.. _Journal Config Reference: ../journal-ref
+.. _cache target dirty high ratio: ../../operations/pools#cache-target-dirty-high-ratio
+.. _mClock Config Reference: ../mclock-config-ref
diff --git a/doc/rados/configuration/pool-pg-config-ref.rst b/doc/rados/configuration/pool-pg-config-ref.rst
new file mode 100644
index 000000000..b4b5df478
--- /dev/null
+++ b/doc/rados/configuration/pool-pg-config-ref.rst
@@ -0,0 +1,282 @@
+======================================
+ Pool, PG and CRUSH Config Reference
+======================================
+
+.. index:: pools; configuration
+
+When you create pools and set the number of placement groups (PGs) for each, Ceph
+uses default values when you don't specifically override the defaults. **We
+recommend** overriding some of the defaults. Specifically, we recommend setting
+a pool's replica size and overriding the default number of placement groups. You
+can specifically set these values when running `pool`_ commands. You can also
+override the defaults by adding new ones in the ``[global]`` section of  your
+Ceph configuration file.
+
+
+.. literalinclude:: pool-pg.conf
+   :language: ini
+
+
+``mon_max_pool_pg_num``
+
+:Description: The maximum number of placement groups per pool.
+:Type: Integer
+:Default: ``65536``
+
+
+``mon_pg_create_interval``
+
+:Description: Number of seconds between PG creation in the same
+              Ceph OSD Daemon.
+
+:Type: Float
+:Default: ``30.0``
+
+
+``mon_pg_stuck_threshold``
+
+:Description: Number of seconds after which PGs can be considered as
+              being stuck.
+
+:Type: 32-bit Integer
+:Default: ``300``
+
+``mon_pg_min_inactive``
+
+:Description: Raise ``HEALTH_ERR`` if the count of PGs that have been
+              inactive longer than the ``mon_pg_stuck_threshold`` exceeds this
+              setting. A non-positive number means disabled, never go into ERR.
+:Type: Integer
+:Default: ``1``
+
+
+``mon_pg_warn_min_per_osd``
+
+:Description: Raise ``HEALTH_WARN`` if the average number
+              of PGs per ``in`` OSD is under this number. A non-positive number
+              disables this.
+:Type: Integer
+:Default: ``30``
+
+
+``mon_pg_warn_min_objects``
+
+:Description: Do not warn if the total number of RADOS objects in cluster is below
+              this number
+:Type: Integer
+:Default: ``1000``
+
+
+``mon_pg_warn_min_pool_objects``
+
+:Description: Do not warn on pools whose RADOS object count is below this number
+:Type: Integer
+:Default: ``1000``
+
+
+``mon_pg_check_down_all_threshold``
+
+:Description: Percentage threshold of ``down`` OSDs above which we check all PGs
+              for stale ones.
+:Type: Float
+:Default: ``0.5``
+
+
+``mon_pg_warn_max_object_skew``
+
+:Description: Raise ``HEALTH_WARN`` if the average RADOS object count per PG
+              of any pool is greater than ``mon_pg_warn_max_object_skew`` times
+              the average RADOS object count per PG of all pools. Zero or a non-positive
+              number disables this. Note that this option applies to ``ceph-mgr`` daemons.
+:Type: Float
+:Default: ``10``
+
+
+``mon_delta_reset_interval``
+
+:Description: Seconds of inactivity before we reset the PG delta to 0. We keep
+              track of the delta of the used space of each pool, so, for
+              example, it would be easier for us to understand the progress of
+              recovery or the performance of cache tier. But if there's no
+              activity reported for a certain pool, we just reset the history of
+              deltas of that pool.
+:Type: Integer
+:Default: ``10``
+
+
+``mon_osd_max_op_age``
+
+:Description: Maximum op age before we get concerned (make it a power of 2).
+              ``HEALTH_WARN`` will be raised if a request has been blocked longer
+              than this limit.
+:Type: Float
+:Default: ``32.0``
+
+
+``osd_pg_bits``
+
+:Description: Placement group bits per Ceph OSD Daemon.
+:Type: 32-bit Integer
+:Default: ``6``
+
+
+``osd_pgp_bits``
+
+:Description: The number of bits per Ceph OSD Daemon for PGPs.
+:Type: 32-bit Integer
+:Default: ``6``
+
+
+``osd_crush_chooseleaf_type``
+
+:Description: The bucket type to use for ``chooseleaf`` in a CRUSH rule. Uses
+              ordinal rank rather than name.
+
+:Type: 32-bit Integer
+:Default: ``1``. Typically a host containing one or more Ceph OSD Daemons.
+
+
+``osd_crush_initial_weight``
+
+:Description: The initial CRUSH weight for newly added OSDs.
+
+:Type: Double
+:Default: ``the size of a newly added OSD in TB``. By default, the initial CRUSH
+          weight for a newly added OSD is set to its device size in TB.
+          See `Weighting Bucket Items`_ for details.
+
+
+``osd_pool_default_crush_rule``
+
+:Description: The default CRUSH rule to use when creating a replicated pool.
+:Type: 8-bit Integer
+:Default: ``-1``, which means "pick the rule with the lowest numerical ID and 
+          use that".  This is to make pool creation work in the absence of rule 0.
+
+
+``osd_pool_erasure_code_stripe_unit``
+
+:Description: Sets the default size, in bytes, of a chunk of an object
+              stripe for erasure coded pools. Every object of size S
+              will be stored as N stripes, with each data chunk
+              receiving ``stripe unit`` bytes. Each stripe of ``N *
+              stripe unit`` bytes will be encoded/decoded
+              individually. This option can is overridden by the
+              ``stripe_unit`` setting in an erasure code profile.
+
+:Type: Unsigned 32-bit Integer
+:Default: ``4096``
+
+
+``osd_pool_default_size``
+
+:Description: Sets the number of replicas for objects in the pool. The default
+              value is the same as
+              ``ceph osd pool set {pool-name} size {size}``.
+
+:Type: 32-bit Integer
+:Default: ``3``
+
+
+``osd_pool_default_min_size``
+
+:Description: Sets the minimum number of written replicas for objects in the
+             pool in order to acknowledge a write operation to the client.  If
+             minimum is not met, Ceph will not acknowledge the write to the
+             client, **which may result in data loss**. This setting ensures
+             a minimum number of replicas when operating in ``degraded`` mode.
+
+:Type: 32-bit Integer
+:Default: ``0``, which means no particular minimum. If ``0``,
+          minimum is ``size - (size / 2)``.
+
+
+``osd_pool_default_pg_num``
+
+:Description: The default number of placement groups for a pool. The default
+              value is the same as ``pg_num`` with ``mkpool``.
+
+:Type: 32-bit Integer
+:Default: ``32``
+
+
+``osd_pool_default_pgp_num``
+
+:Description: The default number of placement groups for placement for a pool.
+              The default value is the same as ``pgp_num`` with ``mkpool``.
+              PG and PGP should be equal (for now).
+
+:Type: 32-bit Integer
+:Default: ``8``
+
+
+``osd_pool_default_flags``
+
+:Description: The default flags for new pools.
+:Type: 32-bit Integer
+:Default: ``0``
+
+
+``osd_max_pgls``
+
+:Description: The maximum number of placement groups to list. A client
+              requesting a large number can tie up the Ceph OSD Daemon.
+
+:Type: Unsigned 64-bit Integer
+:Default: ``1024``
+:Note: Default should be fine.
+
+
+``osd_min_pg_log_entries``
+
+:Description: The minimum number of placement group logs to maintain
+              when trimming log files.
+
+:Type: 32-bit Int Unsigned
+:Default: ``250``
+
+
+``osd_max_pg_log_entries``
+
+:Description: The maximum number of placement group logs to maintain
+              when trimming log files.
+
+:Type: 32-bit Int Unsigned
+:Default: ``10000``
+
+
+``osd_default_data_pool_replay_window``
+
+:Description: The time (in seconds) for an OSD to wait for a client to replay
+              a request.
+
+:Type: 32-bit Integer
+:Default: ``45``
+
+``osd_max_pg_per_osd_hard_ratio``
+
+:Description: The ratio of number of PGs per OSD allowed by the cluster before the
+              OSD refuses to create new PGs. An OSD stops creating new PGs if the number
+              of PGs it serves exceeds
+              ``osd_max_pg_per_osd_hard_ratio`` \* ``mon_max_pg_per_osd``.
+
+:Type: Float
+:Default: ``2``
+
+``osd_recovery_priority``
+
+:Description: Priority of recovery in the work queue.
+
+:Type: Integer
+:Default: ``5``
+
+``osd_recovery_op_priority``
+
+:Description: Default priority used for recovery operations if pool doesn't override.
+
+:Type: Integer
+:Default: ``3``
+
+.. _pool: ../../operations/pools
+.. _Monitoring OSDs and PGs: ../../operations/monitoring-osd-pg#peering
+.. _Weighting Bucket Items: ../../operations/crush-map#weightingbucketitems
diff --git a/doc/rados/configuration/pool-pg.conf b/doc/rados/configuration/pool-pg.conf
new file mode 100644
index 000000000..34c3af9f0
--- /dev/null
+++ b/doc/rados/configuration/pool-pg.conf
@@ -0,0 +1,21 @@
+[global]
+
+	# By default, Ceph makes 3 replicas of RADOS objects. If you want to maintain four
+	# copies of an object the default value--a primary copy and three replica
+	# copies--reset the default values as shown in 'osd_pool_default_size'.
+	# If you want to allow Ceph to write a lesser number of copies in a degraded
+	# state, set 'osd_pool_default_min_size' to a number less than the
+	# 'osd_pool_default_size' value.
+
+	osd_pool_default_size = 3  # Write an object 3 times.
+	osd_pool_default_min_size = 2 # Allow writing two copies in a degraded state.
+
+	# Ensure you have a realistic number of placement groups. We recommend
+	# approximately 100 per OSD. E.g., total number of OSDs multiplied by 100
+	# divided by the number of replicas (i.e., osd pool default size). So for
+	# 10 OSDs and osd pool default size = 4, we'd recommend approximately
+	# (100 * 10) / 4 = 250.
+        # always use the nearest power of 2
+
+	osd_pool_default_pg_num = 256
+	osd_pool_default_pgp_num = 256
diff --git a/doc/rados/configuration/storage-devices.rst b/doc/rados/configuration/storage-devices.rst
new file mode 100644
index 000000000..8536d2cfa
--- /dev/null
+++ b/doc/rados/configuration/storage-devices.rst
@@ -0,0 +1,96 @@
+=================
+ Storage Devices
+=================
+
+There are two Ceph daemons that store data on devices:
+
+.. _rados_configuration_storage-devices_ceph_osd:
+
+* **Ceph OSDs** (Object Storage Daemons) store most of the data
+  in Ceph. Usually each OSD is backed by a single storage device.
+  This can be a traditional hard disk (HDD) or a solid state disk
+  (SSD). OSDs can also be backed by a combination of devices: for
+  example, a HDD for most data and an SSD (or partition of an
+  SSD) for some metadata. The number of OSDs in a cluster is
+  usually a function of the amount of data to be stored, the size
+  of each storage device, and the level and type of redundancy
+  specified (replication or erasure coding).
+* **Ceph Monitor** daemons manage critical cluster state. This
+  includes cluster membership and authentication information.
+  Small clusters require only a few gigabytes of storage to hold
+  the monitor database. In large clusters, however, the monitor
+  database can reach sizes of tens of gigabytes to hundreds of
+  gigabytes.  
+* **Ceph Manager** daemons run alongside monitor daemons, providing
+  additional monitoring and providing interfaces to external
+  monitoring and management systems.
+
+
+OSD Back Ends
+=============
+
+There are two ways that OSDs manage the data they store.  As of the Luminous
+12.2.z release, the default (and recommended) back end is *BlueStore*.  Prior
+to the Luminous release, the default (and only) back end was *Filestore*.
+
+.. _rados_config_storage_devices_bluestore:
+
+BlueStore
+---------
+
+<<<<<<< HEAD
+BlueStore is a special-purpose storage backend designed specifically
+for managing data on disk for Ceph OSD workloads.  It is motivated by
+experience supporting and managing OSDs using FileStore over the
+last ten years.  Key BlueStore features include:
+=======
+BlueStore is a special-purpose storage back end designed specifically for
+managing data on disk for Ceph OSD workloads.  BlueStore's design is based on
+a decade of experience of supporting and managing Filestore OSDs. 
+>>>>>>> 28abc6a9a59 (doc/rados: s/backend/back end/)
+
+* Direct management of storage devices.  BlueStore consumes raw block
+  devices or partitions.  This avoids any intervening layers of
+  abstraction (such as local file systems like XFS) that may limit
+  performance or add complexity.
+* Metadata management with RocksDB.  We embed RocksDB's key/value database
+  in order to manage internal metadata, such as the mapping from object
+  names to block locations on disk.
+* Full data and metadata checksumming.  By default all data and
+  metadata written to BlueStore is protected by one or more
+  checksums.  No data or metadata will be read from disk or returned
+  to the user without being verified.
+* Inline compression.  Data written may be optionally compressed
+  before being written to disk.
+* Multi-device metadata tiering.  BlueStore allows its internal
+  journal (write-ahead log) to be written to a separate, high-speed
+  device (like an SSD, NVMe, or NVDIMM) to increased performance.  If
+  a significant amount of faster storage is available, internal
+  metadata can also be stored on the faster device.
+* Efficient copy-on-write.  RBD and CephFS snapshots rely on a
+  copy-on-write *clone* mechanism that is implemented efficiently in
+  BlueStore.  This results in efficient IO both for regular snapshots
+  and for erasure coded pools (which rely on cloning to implement
+  efficient two-phase commits).
+
+For more information, see :doc:`bluestore-config-ref` and :doc:`/rados/operations/bluestore-migration`.
+
+FileStore
+---------
+
+FileStore is the legacy approach to storing objects in Ceph.  It
+relies on a standard file system (normally XFS) in combination with a
+key/value database (traditionally LevelDB, now RocksDB) for some
+metadata.
+
+FileStore is well-tested and widely used in production but suffers
+from many performance deficiencies due to its overall design and
+reliance on a traditional file system for storing object data.
+
+Although FileStore is generally capable of functioning on most
+POSIX-compatible file systems (including btrfs and ext4), we only
+recommend that XFS be used.  Both btrfs and ext4 have known bugs and
+deficiencies and their use may lead to data loss.  By default all Ceph
+provisioning tools will use XFS.
+
+For more information, see :doc:`filestore-config-ref`.
diff --git a/doc/rados/index.rst b/doc/rados/index.rst
new file mode 100644
index 000000000..5f3f112e8
--- /dev/null
+++ b/doc/rados/index.rst
@@ -0,0 +1,78 @@
+.. _rados-index:
+
+======================
+ Ceph Storage Cluster
+======================
+
+The :term:`Ceph Storage Cluster` is the foundation for all Ceph deployments.
+Based upon :abbr:`RADOS (Reliable Autonomic Distributed Object Store)`, Ceph
+Storage Clusters consist of two types of daemons: a :term:`Ceph OSD Daemon`
+(OSD) stores data as objects on a storage node; and a :term:`Ceph Monitor` (MON)
+maintains a master copy of the cluster map. A Ceph Storage Cluster may contain
+thousands of storage nodes. A minimal system will have at least one 
+Ceph Monitor and two Ceph OSD Daemons for data replication. 
+
+The Ceph File System, Ceph Object Storage and Ceph Block Devices read data from
+and write data to the Ceph Storage Cluster.
+
+.. raw:: html
+
+	<style type="text/css">div.body h3{margin:5px 0px 0px 0px;}</style>
+	<table cellpadding="10"><colgroup><col width="33%"><col width="33%"><col width="33%"></colgroup><tbody valign="top"><tr><td><h3>Config and Deploy</h3>
+
+Ceph Storage Clusters have a few required settings, but most configuration
+settings have default values. A typical deployment uses a deployment tool 
+to define a cluster and bootstrap a monitor. See `Deployment`_ for details 
+on ``cephadm.``
+
+.. toctree::
+	:maxdepth: 2
+
+	Configuration <configuration/index>
+	Deployment <../cephadm/index>
+
+.. raw:: html 
+
+	</td><td><h3>Operations</h3>
+
+Once you have deployed a Ceph Storage Cluster, you may begin operating 
+your cluster.
+
+.. toctree::
+	:maxdepth: 2
+	
+	
+	Operations <operations/index>
+
+.. toctree::
+	:maxdepth: 1
+
+	Man Pages <man/index>
+
+
+.. toctree:: 
+	:hidden:
+	
+	troubleshooting/index
+
+.. raw:: html 
+
+	</td><td><h3>APIs</h3>
+
+Most Ceph deployments use `Ceph Block Devices`_, `Ceph Object Storage`_ and/or the
+`Ceph File System`_. You  may also develop applications that talk directly to
+the Ceph Storage Cluster.
+
+.. toctree::
+	:maxdepth: 2
+
+	APIs <api/index>
+	
+.. raw:: html
+
+	</td></tr></tbody></table>
+
+.. _Ceph Block Devices: ../rbd/
+.. _Ceph File System: ../cephfs/
+.. _Ceph Object Storage: ../radosgw/
+.. _Deployment: ../cephadm/
diff --git a/doc/rados/man/index.rst b/doc/rados/man/index.rst
new file mode 100644
index 000000000..8311bfd5a
--- /dev/null
+++ b/doc/rados/man/index.rst
@@ -0,0 +1,31 @@
+=======================
+ Object Store Manpages
+=======================
+
+.. toctree:: 
+   :maxdepth: 1
+
+   ../../man/8/ceph-volume.rst
+   ../../man/8/ceph-volume-systemd.rst
+   ../../man/8/ceph.rst
+   ../../man/8/ceph-authtool.rst
+   ../../man/8/ceph-clsinfo.rst
+   ../../man/8/ceph-conf.rst
+   ../../man/8/ceph-debugpack.rst
+   ../../man/8/ceph-dencoder.rst
+   ../../man/8/ceph-mon.rst
+   ../../man/8/ceph-osd.rst
+   ../../man/8/ceph-kvstore-tool.rst
+   ../../man/8/ceph-run.rst
+   ../../man/8/ceph-syn.rst
+   ../../man/8/crushtool.rst
+   ../../man/8/librados-config.rst
+   ../../man/8/monmaptool.rst
+   ../../man/8/osdmaptool.rst
+   ../../man/8/rados.rst
+
+
+.. toctree::
+	:hidden:
+	
+	../../man/8/ceph-post-file.rst
diff --git a/doc/rados/operations/add-or-rm-mons.rst b/doc/rados/operations/add-or-rm-mons.rst
new file mode 100644
index 000000000..359fa7676
--- /dev/null
+++ b/doc/rados/operations/add-or-rm-mons.rst
@@ -0,0 +1,446 @@
+.. _adding-and-removing-monitors:
+
+==========================
+ Adding/Removing Monitors
+==========================
+
+When you have a cluster up and running, you may add or remove monitors
+from the cluster at runtime. To bootstrap a monitor, see `Manual Deployment`_
+or `Monitor Bootstrap`_.
+
+.. _adding-monitors:
+
+Adding Monitors
+===============
+
+Ceph monitors are lightweight processes that are the single source of truth
+for the cluster map. You can run a cluster with 1 monitor but we recommend at least 3 
+for a production cluster. Ceph monitors use a variation of the
+`Paxos`_ algorithm to establish consensus about maps and other critical
+information across the cluster. Due to the nature of Paxos, Ceph requires
+a majority of monitors to be active to establish a quorum (thus establishing
+consensus).
+
+It is advisable to run an odd number of monitors. An
+odd number of monitors is more resilient than an
+even number. For instance, with a two monitor deployment, no
+failures can be tolerated and still maintain a quorum; with three monitors,
+one failure can be tolerated; in a four monitor deployment, one failure can
+be tolerated; with five monitors, two failures can be tolerated.  This avoids
+the dreaded *split brain* phenomenon, and is why an odd number is best.
+In short, Ceph needs a majority of
+monitors to be active (and able to communicate with each other), but that
+majority can be achieved using a single monitor, or 2 out of 2 monitors,
+2 out of 3, 3 out of 4, etc.
+
+For small or non-critical deployments of multi-node Ceph clusters, it is
+advisable to deploy three monitors, and to increase the number of monitors
+to five for larger clusters or to survive a double failure.  There is rarely
+justification for seven or more.
+
+Since monitors are lightweight, it is possible to run them on the same 
+host as OSDs; however, we recommend running them on separate hosts,
+because `fsync` issues with the kernel may impair performance.
+Dedicated monitor nodes also minimize disruption since monitor and OSD
+daemons are not inactive at the same time when a node crashes or is
+taken down for maintenance.
+
+Dedicated
+monitor nodes also make for cleaner maintenance by avoiding both OSDs and
+a mon going down if a node is rebooted, taken down, or crashes.
+
+.. note:: A *majority* of monitors in your cluster must be able to 
+   reach each other in order to establish a quorum.
+
+Deploy your Hardware
+--------------------
+
+If you are adding a new host when adding a new monitor,  see `Hardware
+Recommendations`_ for details on minimum recommendations for monitor hardware.
+To add a monitor host to your cluster, first make sure you have an up-to-date
+version of Linux installed (typically Ubuntu 16.04 or RHEL 7). 
+
+Add your monitor host to a rack in your cluster, connect it to the network
+and ensure that it has network connectivity.
+
+.. _Hardware Recommendations: ../../../start/hardware-recommendations
+
+Install the Required Software
+-----------------------------
+
+For manually deployed clusters, you must install Ceph packages
+manually. See `Installing Packages`_ for details.
+You should configure SSH to a user with password-less authentication
+and root permissions.
+
+.. _Installing Packages: ../../../install/install-storage-cluster
+
+
+.. _Adding a Monitor (Manual):
+
+Adding a Monitor (Manual)
+-------------------------
+
+This procedure creates a ``ceph-mon`` data directory, retrieves the monitor map
+and monitor keyring, and adds a ``ceph-mon`` daemon to your cluster.  If
+this results in only two monitor daemons, you may add more monitors by
+repeating this procedure until you have a sufficient number of ``ceph-mon`` 
+daemons to achieve a quorum.
+
+At this point you should define your monitor's id.  Traditionally, monitors 
+have been named with single letters (``a``, ``b``, ``c``, ...), but you are 
+free to define the id as you see fit.  For the purpose of this document, 
+please take into account that ``{mon-id}`` should be the id you chose, 
+without the ``mon.`` prefix (i.e., ``{mon-id}`` should be the ``a`` 
+on ``mon.a``).
+
+#. Create the default directory on the machine that will host your 
+   new monitor:
+
+   .. prompt:: bash $
+
+	ssh {new-mon-host}
+	sudo mkdir /var/lib/ceph/mon/ceph-{mon-id}
+
+#. Create a temporary directory ``{tmp}`` to keep the files needed during 
+   this process. This directory should be different from the monitor's default 
+   directory created in the previous step, and can be removed after all the 
+   steps are executed:
+
+   .. prompt:: bash $
+
+	mkdir {tmp}
+
+#. Retrieve the keyring for your monitors, where ``{tmp}`` is the path to 
+   the retrieved keyring, and ``{key-filename}`` is the name of the file 
+   containing the retrieved monitor key:
+
+   .. prompt:: bash $
+
+      ceph auth get mon. -o {tmp}/{key-filename}
+
+#. Retrieve the monitor map, where ``{tmp}`` is the path to 
+   the retrieved monitor map, and ``{map-filename}`` is the name of the file 
+   containing the retrieved monitor map:
+
+   .. prompt:: bash $
+
+      ceph mon getmap -o {tmp}/{map-filename}
+
+#. Prepare the monitor's data directory created in the first step. You must 
+   specify the path to the monitor map so that you can retrieve the 
+   information about a quorum of monitors and their ``fsid``. You must also 
+   specify a path to the monitor keyring:
+   
+   .. prompt:: bash $
+
+      sudo ceph-mon -i {mon-id} --mkfs --monmap {tmp}/{map-filename} --keyring {tmp}/{key-filename}
+	
+
+#. Start the new monitor and it will automatically join the cluster.
+   The daemon needs to know which address to bind to, via either the
+   ``--public-addr {ip}`` or ``--public-network {network}`` argument.
+   For example:
+   
+   .. prompt:: bash $
+
+      ceph-mon -i {mon-id} --public-addr {ip:port}
+
+.. _removing-monitors:
+
+Removing Monitors
+=================
+
+When you remove monitors from a cluster, consider that Ceph monitors use 
+Paxos to establish consensus about the master cluster map. You must have 
+a sufficient number of monitors to establish a quorum for consensus about 
+the cluster map.
+
+.. _Removing a Monitor (Manual):
+
+Removing a Monitor (Manual)
+---------------------------
+
+This procedure removes a ``ceph-mon`` daemon from your cluster.   If this
+procedure results in only two monitor daemons, you may add or remove another
+monitor until you have a number of ``ceph-mon`` daemons that can achieve a 
+quorum.
+
+#. Stop the monitor:
+
+   .. prompt:: bash $
+
+      service ceph -a stop mon.{mon-id}
+	
+#. Remove the monitor from the cluster:
+
+   .. prompt:: bash $
+
+      ceph mon remove {mon-id}
+	
+#. Remove the monitor entry from ``ceph.conf``. 
+
+.. _rados-mon-remove-from-unhealthy: 
+
+Removing Monitors from an Unhealthy Cluster
+-------------------------------------------
+
+This procedure removes a ``ceph-mon`` daemon from an unhealthy
+cluster, for example a cluster where the monitors cannot form a
+quorum.
+
+
+#. Stop all ``ceph-mon`` daemons on all monitor hosts:
+
+   .. prompt:: bash $
+
+      ssh {mon-host}
+      systemctl stop ceph-mon.target
+
+   Repeat for all monitor hosts.
+
+#. Identify a surviving monitor and log in to that host:
+
+   .. prompt:: bash $
+
+      ssh {mon-host}
+
+#. Extract a copy of the monmap file:
+
+   .. prompt:: bash $
+
+      ceph-mon -i {mon-id} --extract-monmap {map-path}
+
+   In most cases, this command will be:
+
+   .. prompt:: bash $
+
+      ceph-mon -i `hostname` --extract-monmap /tmp/monmap
+
+#. Remove the non-surviving or problematic monitors.  For example, if
+   you have three monitors, ``mon.a``, ``mon.b``, and ``mon.c``, where
+   only ``mon.a`` will survive, follow the example below:
+
+   .. prompt:: bash $
+
+      monmaptool {map-path} --rm {mon-id}
+
+   For example,
+
+   .. prompt:: bash $
+
+      monmaptool /tmp/monmap --rm b
+      monmaptool /tmp/monmap --rm c
+	
+#. Inject the surviving map with the removed monitors into the
+   surviving monitor(s).  For example, to inject a map into monitor
+   ``mon.a``, follow the example below:
+
+   .. prompt:: bash $
+
+      ceph-mon -i {mon-id} --inject-monmap {map-path}
+
+   For example:
+
+   .. prompt:: bash $
+
+      ceph-mon -i a --inject-monmap /tmp/monmap
+
+#. Start only the surviving monitors.
+
+#. Verify the monitors form a quorum (``ceph -s``).
+
+#. You may wish to archive the removed monitors' data directory in
+   ``/var/lib/ceph/mon`` in a safe location, or delete it if you are
+   confident the remaining monitors are healthy and are sufficiently
+   redundant.
+
+.. _Changing a Monitor's IP address:
+
+Changing a Monitor's IP Address
+===============================
+
+.. important:: Existing monitors are not supposed to change their IP addresses.
+
+Monitors are critical components of a Ceph cluster, and they need to maintain a
+quorum for the whole system to work properly. To establish a quorum, the
+monitors need to discover each other. Ceph has strict requirements for
+discovering monitors.
+
+Ceph clients and other Ceph daemons use ``ceph.conf`` to discover monitors.
+However, monitors discover each other using the monitor map, not ``ceph.conf``.
+For example,  if you refer to `Adding a Monitor (Manual)`_ you will see that you
+need to obtain the current monmap for the cluster when creating a new monitor,
+as it is one of the required arguments of ``ceph-mon -i {mon-id} --mkfs``. The
+following sections explain the consistency requirements for Ceph monitors, and a
+few safe ways to change a monitor's IP address.
+
+
+Consistency Requirements
+------------------------
+
+A monitor always refers to the local copy of the monmap  when discovering other
+monitors in the cluster.  Using the monmap instead of ``ceph.conf`` avoids
+errors that could  break the cluster (e.g., typos in ``ceph.conf`` when
+specifying a monitor address or port). Since monitors use monmaps for discovery
+and they share monmaps with clients and other Ceph daemons, the monmap provides
+monitors with a strict guarantee that their consensus is valid.
+
+Strict consistency also applies to updates to the monmap. As with any other
+updates on the monitor, changes to the monmap always run through a distributed
+consensus algorithm called `Paxos`_. The monitors must agree on each update to
+the monmap, such as adding or removing a monitor, to ensure that each monitor in
+the quorum has the same version of the monmap. Updates to the monmap are
+incremental so that monitors have the latest agreed upon version, and a set of
+previous versions, allowing a monitor that has an older version of the monmap to
+catch up with the current state of the cluster.
+
+If monitors discovered each other through the Ceph configuration file instead of
+through the monmap, it would introduce additional risks because the Ceph
+configuration files are not updated and distributed automatically. Monitors
+might inadvertently use an older ``ceph.conf`` file, fail to recognize a
+monitor, fall out of a quorum, or develop a situation where `Paxos`_ is not able
+to determine the current state of the system accurately. Consequently,  making
+changes to an existing monitor's IP address must be done with  great care.
+
+
+Changing a Monitor's IP address (The Right Way)
+-----------------------------------------------
+
+Changing a monitor's IP address in ``ceph.conf`` only is not sufficient to
+ensure that other monitors in the cluster will receive the update.  To change a
+monitor's IP address, you must add a new monitor with the IP  address you want
+to use (as described in `Adding a Monitor (Manual)`_),  ensure that the new
+monitor successfully joins the  quorum; then, remove the monitor that uses the
+old IP address. Then, update the ``ceph.conf`` file to ensure that clients and
+other daemons know the IP address of the new monitor.
+
+For example, lets assume there are three monitors in place, such as :: 
+
+	[mon.a]
+		host = host01
+		addr = 10.0.0.1:6789
+	[mon.b]
+		host = host02
+		addr = 10.0.0.2:6789
+	[mon.c]
+		host = host03
+		addr = 10.0.0.3:6789
+
+To change ``mon.c`` to ``host04`` with the IP address  ``10.0.0.4``, follow the
+steps in `Adding a Monitor (Manual)`_ by adding a  new monitor ``mon.d``. Ensure
+that ``mon.d`` is  running before removing ``mon.c``, or it will break the
+quorum. Remove ``mon.c`` as described on  `Removing a Monitor (Manual)`_. Moving
+all three  monitors would thus require repeating this process as many times as
+needed.
+
+
+Changing a Monitor's IP address (The Messy Way)
+-----------------------------------------------
+
+There may come a time when the monitors must be moved to a different network,  a
+different part of the datacenter or a different datacenter altogether. While  it
+is possible to do it, the process becomes a bit more hazardous.
+
+In such a case, the solution is to generate a new monmap with updated IP
+addresses for all the monitors in the cluster, and inject the new map on each
+individual monitor.  This is not the most user-friendly approach, but we do not
+expect this to be something that needs to be done every other week.  As it is
+clearly stated on the top of this section, monitors are not supposed to change
+IP addresses.
+
+Using the previous monitor configuration as an example, assume you want to move
+all the  monitors from the ``10.0.0.x`` range to ``10.1.0.x``, and these
+networks  are unable to communicate.  Use the following procedure:
+
+#. Retrieve the monitor map, where ``{tmp}`` is the path to 
+   the retrieved monitor map, and ``{filename}`` is the name of the file 
+   containing the retrieved monitor map:
+
+   .. prompt:: bash $
+
+      ceph mon getmap -o {tmp}/{filename}
+
+#. The following example demonstrates the contents of the monmap:
+
+   .. prompt:: bash $
+
+      monmaptool --print {tmp}/{filename}
+
+   ::	
+
+	monmaptool: monmap file {tmp}/{filename}
+	epoch 1
+	fsid 224e376d-c5fe-4504-96bb-ea6332a19e61
+	last_changed 2012-12-17 02:46:41.591248
+	created 2012-12-17 02:46:41.591248
+	0: 10.0.0.1:6789/0 mon.a
+	1: 10.0.0.2:6789/0 mon.b
+	2: 10.0.0.3:6789/0 mon.c
+
+#. Remove the existing monitors:
+
+   .. prompt:: bash $
+
+      monmaptool --rm a --rm b --rm c {tmp}/{filename}
+	
+
+   ::
+
+	monmaptool: monmap file {tmp}/{filename}
+	monmaptool: removing a
+	monmaptool: removing b
+	monmaptool: removing c
+	monmaptool: writing epoch 1 to {tmp}/{filename} (0 monitors)
+
+#. Add the new monitor locations:
+
+   .. prompt:: bash $
+
+      monmaptool --add a 10.1.0.1:6789 --add b 10.1.0.2:6789 --add c 10.1.0.3:6789 {tmp}/{filename}
+
+
+   ::
+	
+      monmaptool: monmap file {tmp}/{filename}
+      monmaptool: writing epoch 1 to {tmp}/{filename} (3 monitors)
+
+#. Check new contents:
+
+   .. prompt:: bash $
+
+       monmaptool --print {tmp}/{filename}
+	
+   ::
+
+	monmaptool: monmap file {tmp}/{filename}
+	epoch 1
+	fsid 224e376d-c5fe-4504-96bb-ea6332a19e61
+	last_changed 2012-12-17 02:46:41.591248
+	created 2012-12-17 02:46:41.591248
+	0: 10.1.0.1:6789/0 mon.a
+	1: 10.1.0.2:6789/0 mon.b
+	2: 10.1.0.3:6789/0 mon.c
+
+At this point, we assume the monitors (and stores) are installed at the new
+location. The next step is to propagate the modified monmap to the new 
+monitors, and inject the modified monmap into each new monitor.
+
+#. First, make sure to stop all your monitors.  Injection must be done while 
+   the daemon is not running.
+
+#. Inject the monmap: 
+
+   .. prompt:: bash $
+
+      ceph-mon -i {mon-id} --inject-monmap {tmp}/{filename}
+
+#. Restart the monitors.
+
+After this step, migration to the new location is complete and 
+the monitors should operate successfully.
+
+
+.. _Manual Deployment: ../../../install/manual-deployment
+.. _Monitor Bootstrap: ../../../dev/mon-bootstrap
+.. _Paxos: https://en.wikipedia.org/wiki/Paxos_(computer_science)
diff --git a/doc/rados/operations/add-or-rm-osds.rst b/doc/rados/operations/add-or-rm-osds.rst
new file mode 100644
index 000000000..315552859
--- /dev/null
+++ b/doc/rados/operations/add-or-rm-osds.rst
@@ -0,0 +1,386 @@
+======================
+ Adding/Removing OSDs
+======================
+
+When you have a cluster up and running, you may add OSDs or remove OSDs
+from the cluster at runtime.
+
+Adding OSDs
+===========
+
+When you want to expand a cluster, you may add an OSD at runtime. With Ceph, an
+OSD is generally one Ceph ``ceph-osd`` daemon for one storage drive within a
+host machine. If your host has multiple storage drives, you may map one
+``ceph-osd`` daemon for each drive.
+
+Generally, it's a good idea to check the capacity of your cluster to see if you
+are reaching the upper end of its capacity. As your cluster reaches its ``near
+full`` ratio, you should add one or more OSDs to expand your cluster's capacity.
+
+.. warning:: Do not let your cluster reach its ``full ratio`` before
+   adding an OSD. OSD failures that occur after the cluster reaches
+   its ``near full`` ratio may cause the cluster to exceed its
+   ``full ratio``.
+
+Deploy your Hardware
+--------------------
+
+If you are adding a new host when adding a new OSD,  see `Hardware
+Recommendations`_ for details on minimum recommendations for OSD hardware. To
+add an OSD host to your cluster, first make sure you have an up-to-date version
+of Linux installed, and you have made some initial preparations for your
+storage drives.  See `Filesystem Recommendations`_ for details.
+
+Add your OSD host to a rack in your cluster, connect it to the network
+and ensure that it has network connectivity. See the `Network Configuration
+Reference`_ for details.
+
+.. _Hardware Recommendations: ../../../start/hardware-recommendations
+.. _Filesystem Recommendations: ../../configuration/filesystem-recommendations
+.. _Network Configuration Reference: ../../configuration/network-config-ref
+
+Install the Required Software
+-----------------------------
+
+For manually deployed clusters, you must install Ceph packages
+manually. See `Installing Ceph (Manual)`_ for details.
+You should configure SSH to a user with password-less authentication
+and root permissions.
+
+.. _Installing Ceph (Manual): ../../../install
+
+
+Adding an OSD (Manual)
+----------------------
+
+This procedure sets up a ``ceph-osd`` daemon, configures it to use one drive,
+and configures the cluster to distribute data to the OSD. If your host has
+multiple drives, you may add an OSD for each drive by repeating this procedure.
+
+To add an OSD, create a data directory for it, mount a drive to that directory,
+add the OSD to the cluster, and then add it to the CRUSH map.
+
+When you add the OSD to the CRUSH map, consider the weight you give to the new
+OSD. Hard drive capacity grows 40% per year, so newer OSD hosts may have larger
+hard drives than older hosts in the cluster (i.e., they may have greater
+weight).
+
+.. tip:: Ceph prefers uniform hardware across pools. If you are adding drives
+   of dissimilar size, you can adjust their weights. However, for best
+   performance, consider a CRUSH hierarchy with drives of the same type/size.
+
+#. Create the OSD. If no UUID is given, it will be set automatically when the
+   OSD starts up. The following command will output the OSD number, which you
+   will need for subsequent steps:
+
+   .. prompt:: bash $
+
+      ceph osd create [{uuid} [{id}]]
+
+   If the optional parameter {id} is given it will be used as the OSD id.
+   Note, in this case the command may fail if the number is already in use.
+
+   .. warning:: In general, explicitly specifying {id} is not recommended.
+      IDs are allocated as an array, and skipping entries consumes some extra
+      memory. This can become significant if there are large gaps and/or
+      clusters are large. If {id} is not specified, the smallest available is
+      used.
+
+#. Create the default directory on your new OSD:
+
+   .. prompt:: bash $
+
+      ssh {new-osd-host}
+      sudo mkdir /var/lib/ceph/osd/ceph-{osd-number}
+
+#. If the OSD is for a drive other than the OS drive, prepare it
+   for use with Ceph, and mount it to the directory you just created:
+
+   .. prompt:: bash $
+
+      ssh {new-osd-host}
+      sudo mkfs -t {fstype} /dev/{drive}
+      sudo mount -o user_xattr /dev/{hdd} /var/lib/ceph/osd/ceph-{osd-number}
+
+#. Initialize the OSD data directory:
+
+   .. prompt:: bash $
+
+      ssh {new-osd-host}
+      ceph-osd -i {osd-num} --mkfs --mkkey
+
+   The directory must be empty before you can run ``ceph-osd``.
+
+#. Register the OSD authentication key. The value of ``ceph`` for
+   ``ceph-{osd-num}`` in the path is the ``$cluster-$id``.  If your
+   cluster name differs from ``ceph``, use your cluster name instead:
+
+   .. prompt:: bash $
+
+      ceph auth add osd.{osd-num} osd 'allow *' mon 'allow rwx' -i /var/lib/ceph/osd/ceph-{osd-num}/keyring
+
+#. Add the OSD to the CRUSH map so that the OSD can begin receiving data. The
+   ``ceph osd crush add`` command allows you to add OSDs to the CRUSH hierarchy
+   wherever you wish. If you specify at least one bucket, the command
+   will place the OSD into the most specific bucket you specify, *and* it will
+   move that bucket underneath any other buckets you specify. **Important:** If
+   you specify only the root bucket, the command will attach the OSD directly
+   to the root, but CRUSH rules expect OSDs to be inside of hosts.
+
+   Execute the following:
+
+   .. prompt:: bash $
+
+      ceph osd crush add {id-or-name} {weight}  [{bucket-type}={bucket-name} ...]
+
+   You may also decompile the CRUSH map, add the OSD to the device list, add the
+   host as a bucket (if it's not already in the CRUSH map), add the device as an
+   item in the host, assign it a weight, recompile it and set it. See
+   `Add/Move an OSD`_ for details.
+
+
+.. _rados-replacing-an-osd:
+
+Replacing an OSD
+----------------
+
+.. note:: If the instructions in this section do not work for you, try the
+   instructions in the cephadm documentation: :ref:`cephadm-replacing-an-osd`.
+
+When disks fail, or if an administrator wants to reprovision OSDs with a new
+backend, for instance, for switching from FileStore to BlueStore, OSDs need to
+be replaced. Unlike `Removing the OSD`_, replaced OSD's id and CRUSH map entry
+need to be keep intact after the OSD is destroyed for replacement.
+
+#. Make sure it is safe to destroy the OSD:
+
+   .. prompt:: bash $
+
+      while ! ceph osd safe-to-destroy osd.{id} ; do sleep 10 ; done
+
+#. Destroy the OSD first:
+
+   .. prompt:: bash $
+
+      ceph osd destroy {id} --yes-i-really-mean-it
+
+#. Zap a disk for the new OSD, if the disk was used before for other purposes.
+   It's not necessary for a new disk:
+
+   .. prompt:: bash $
+
+      ceph-volume lvm zap /dev/sdX
+
+#. Prepare the disk for replacement by using the previously destroyed OSD id:
+
+   .. prompt:: bash $
+
+      ceph-volume lvm prepare --osd-id {id} --data /dev/sdX
+
+#. And activate the OSD:
+
+   .. prompt:: bash $
+
+      ceph-volume lvm activate {id} {fsid}
+
+Alternatively, instead of preparing and activating, the device can be recreated
+in one call, like:
+
+   .. prompt:: bash $
+
+      ceph-volume lvm create --osd-id {id} --data /dev/sdX
+
+
+Starting the OSD
+----------------
+
+After you add an OSD to Ceph, the OSD is in your configuration. However,
+it is not yet running. The OSD is ``down`` and ``in``. You must start
+your new OSD before it can begin receiving data. You may use
+``service ceph`` from your admin host or start the OSD from its host
+machine:
+
+   .. prompt:: bash $
+
+      sudo systemctl start ceph-osd@{osd-num}
+
+
+Once you start your OSD, it is ``up`` and ``in``.
+
+
+Observe the Data Migration
+--------------------------
+
+Once you have added your new OSD to the CRUSH map, Ceph  will begin rebalancing
+the server by migrating placement groups to your new OSD. You can observe this
+process with  the `ceph`_ tool. :
+
+   .. prompt:: bash $
+
+      ceph -w
+
+You should see the placement group states change from ``active+clean`` to
+``active, some degraded objects``, and finally ``active+clean`` when migration
+completes. (Control-c to exit.)
+
+.. _Add/Move an OSD: ../crush-map#addosd
+.. _ceph: ../monitoring
+
+
+
+Removing OSDs (Manual)
+======================
+
+When you want to reduce the size of a cluster or replace hardware, you may
+remove an OSD at runtime. With Ceph, an OSD is generally one Ceph ``ceph-osd``
+daemon for one storage drive within a host machine. If your host has multiple
+storage drives, you may need to remove one ``ceph-osd`` daemon for each drive.
+Generally, it's a good idea to check the capacity of your cluster to see if you
+are reaching the upper end of its capacity. Ensure that when you remove an OSD
+that your cluster is not at its ``near full`` ratio.
+
+.. warning:: Do not let your cluster reach its ``full ratio`` when
+   removing an OSD. Removing OSDs could cause the cluster to reach
+   or exceed its ``full ratio``.
+
+
+Take the OSD out of the Cluster
+-----------------------------------
+
+Before you remove an OSD, it is usually ``up`` and ``in``.  You need to take it
+out of the cluster so that Ceph can begin rebalancing and copying its data to
+other OSDs. :
+
+   .. prompt:: bash $
+
+      ceph osd out {osd-num}
+
+
+Observe the Data Migration
+--------------------------
+
+Once you have taken your OSD ``out`` of the cluster, Ceph  will begin
+rebalancing the cluster by migrating placement groups out of the OSD you
+removed. You can observe  this process with  the `ceph`_ tool. :
+
+   .. prompt:: bash $
+
+      ceph -w
+
+You should see the placement group states change from ``active+clean`` to
+``active, some degraded objects``, and finally ``active+clean`` when migration
+completes. (Control-c to exit.)
+
+.. note:: Sometimes, typically in a "small" cluster with few hosts (for
+   instance with a small testing cluster), the fact to take ``out`` the
+   OSD can spawn a CRUSH corner case where some PGs remain stuck in the
+   ``active+remapped`` state. If you are in this case, you should mark
+   the OSD ``in`` with:
+
+   .. prompt:: bash $
+
+      ceph osd in {osd-num}
+
+   to come back to the initial state and then, instead of marking ``out``
+   the OSD, set its weight to 0 with:
+
+   .. prompt:: bash $
+
+      ceph osd crush reweight osd.{osd-num} 0
+
+   After that, you can observe the data migration which should come to its
+   end. The difference between marking ``out`` the OSD and reweighting it
+   to 0 is that in the first case the weight of the bucket which contains
+   the OSD is not changed whereas in the second case the weight of the bucket
+   is updated (and decreased of the OSD weight). The reweight command could
+   be sometimes favoured in the case of a "small" cluster.
+
+
+
+Stopping the OSD
+----------------
+
+After you take an OSD out of the cluster, it may still be running.
+That is, the OSD may be ``up`` and ``out``. You must stop
+your OSD before you remove it from the configuration: 
+
+   .. prompt:: bash $
+
+      ssh {osd-host}
+      sudo systemctl stop ceph-osd@{osd-num}
+
+Once you stop your OSD, it is ``down``.
+
+
+Removing the OSD
+----------------
+
+This procedure removes an OSD from a cluster map, removes its authentication
+key, removes the OSD from the OSD map, and removes the OSD from the
+``ceph.conf`` file. If your host has multiple drives, you may need to remove an
+OSD for each drive by repeating this procedure.
+
+#. Let the cluster forget the OSD first. This step removes the OSD from the CRUSH
+   map, removes its authentication key. And it is removed from the OSD map as
+   well. Please note the :ref:`purge subcommand <ceph-admin-osd>` is introduced in Luminous, for older
+   versions, please see below:
+
+   .. prompt:: bash $
+
+      ceph osd purge {id} --yes-i-really-mean-it
+
+#. Navigate to the host where you keep the master copy of the cluster's
+   ``ceph.conf`` file:
+
+   .. prompt:: bash $
+
+      ssh {admin-host}
+      cd /etc/ceph
+      vim ceph.conf
+
+#. Remove the OSD entry from your ``ceph.conf`` file (if it exists)::
+
+	[osd.1]
+		host = {hostname}
+
+#. From the host where you keep the master copy of the cluster's ``ceph.conf``
+   file, copy the updated ``ceph.conf`` file to the ``/etc/ceph`` directory of
+   other hosts in your cluster.
+
+If your Ceph cluster is older than Luminous, instead of using ``ceph osd
+purge``, you need to perform this step manually:
+
+
+#. Remove the OSD from the CRUSH map so that it no longer receives data. You may
+   also decompile the CRUSH map, remove the OSD from the device list, remove the
+   device as an item in the host bucket or remove the host  bucket (if it's in the
+   CRUSH map and you intend to remove the host), recompile the map and set it.
+   See `Remove an OSD`_ for details:
+
+   .. prompt:: bash $
+
+      ceph osd crush remove {name}
+
+#. Remove the OSD authentication key:
+
+   .. prompt:: bash $
+
+      ceph auth del osd.{osd-num}
+
+   The value of ``ceph`` for ``ceph-{osd-num}`` in the path is the
+   ``$cluster-$id``.  If your cluster name differs from ``ceph``, use your
+   cluster name instead.
+
+#. Remove the OSD:
+
+   .. prompt:: bash $
+
+      ceph osd rm {osd-num}
+
+   for example:
+
+   .. prompt:: bash $
+
+      ceph osd rm 1
+
+.. _Remove an OSD: ../crush-map#removeosd
diff --git a/doc/rados/operations/balancer.rst b/doc/rados/operations/balancer.rst
new file mode 100644
index 000000000..b02a8914d
--- /dev/null
+++ b/doc/rados/operations/balancer.rst
@@ -0,0 +1,206 @@
+.. _balancer:
+
+Balancer
+========
+
+The *balancer* can optimize the placement of PGs across OSDs in
+order to achieve a balanced distribution, either automatically or in a
+supervised fashion.
+
+Status
+------
+
+The current status of the balancer can be checked at any time with:
+
+   .. prompt:: bash $
+
+      ceph balancer status
+
+
+Automatic balancing
+-------------------
+
+The automatic balancing feature is enabled by default in ``upmap``
+mode. Please refer to :ref:`upmap` for more details. The balancer can be
+turned off with:
+
+   .. prompt:: bash $
+
+      ceph balancer off
+
+The balancer mode can be changed to ``crush-compat`` mode, which is
+backward compatible with older clients, and will make small changes to
+the data distribution over time to ensure that OSDs are equally utilized.
+
+
+Throttling
+----------
+
+No adjustments will be made to the PG distribution if the cluster is
+degraded (e.g., because an OSD has failed and the system has not yet
+healed itself).
+
+When the cluster is healthy, the balancer will throttle its changes
+such that the percentage of PGs that are misplaced (i.e., that need to
+be moved) is below a threshold of (by default) 5%.  The
+``target_max_misplaced_ratio`` threshold can be adjusted with:
+
+   .. prompt:: bash $
+
+      ceph config set mgr target_max_misplaced_ratio .07   # 7%
+
+Set the number of seconds to sleep in between runs of the automatic balancer:
+
+   .. prompt:: bash $
+
+      ceph config set mgr mgr/balancer/sleep_interval 60
+
+Set the time of day to begin automatic balancing in HHMM format:
+
+   .. prompt:: bash $
+
+      ceph config set mgr mgr/balancer/begin_time 0000
+
+Set the time of day to finish automatic balancing in HHMM format:
+
+   .. prompt:: bash $
+
+      ceph config set mgr mgr/balancer/end_time 2359
+
+Restrict automatic balancing to this day of the week or later. 
+Uses the same conventions as crontab, 0 is Sunday, 1 is Monday, and so on:
+
+   .. prompt:: bash $
+
+      ceph config set mgr mgr/balancer/begin_weekday 0
+
+Restrict automatic balancing to this day of the week or earlier. 
+Uses the same conventions as crontab, 0 is Sunday, 1 is Monday, and so on:
+
+   .. prompt:: bash $
+
+      ceph config set mgr mgr/balancer/end_weekday 6
+
+Pool IDs to which the automatic balancing will be limited. 
+The default for this is an empty string, meaning all pools will be balanced. 
+The numeric pool IDs can be gotten with the :command:`ceph osd pool ls detail` command:
+
+   .. prompt:: bash $
+
+      ceph config set mgr mgr/balancer/pool_ids 1,2,3
+
+
+Modes
+-----
+
+There are currently two supported balancer modes:
+
+#. **crush-compat**.  The CRUSH compat mode uses the compat weight-set
+   feature (introduced in Luminous) to manage an alternative set of
+   weights for devices in the CRUSH hierarchy.  The normal weights
+   should remain set to the size of the device to reflect the target
+   amount of data that we want to store on the device.  The balancer
+   then optimizes the weight-set values, adjusting them up or down in
+   small increments, in order to achieve a distribution that matches
+   the target distribution as closely as possible.  (Because PG
+   placement is a pseudorandom process, there is a natural amount of
+   variation in the placement; by optimizing the weights we
+   counter-act that natural variation.)
+
+   Notably, this mode is *fully backwards compatible* with older
+   clients: when an OSDMap and CRUSH map is shared with older clients,
+   we present the optimized weights as the "real" weights.
+
+   The primary restriction of this mode is that the balancer cannot
+   handle multiple CRUSH hierarchies with different placement rules if
+   the subtrees of the hierarchy share any OSDs.  (This is normally
+   not the case, and is generally not a recommended configuration
+   because it is hard to manage the space utilization on the shared
+   OSDs.)
+
+#. **upmap**.  Starting with Luminous, the OSDMap can store explicit
+   mappings for individual OSDs as exceptions to the normal CRUSH
+   placement calculation.  These `upmap` entries provide fine-grained
+   control over the PG mapping.  This CRUSH mode will optimize the
+   placement of individual PGs in order to achieve a balanced
+   distribution.  In most cases, this distribution is "perfect," which
+   an equal number of PGs on each OSD (+/-1 PG, since they might not
+   divide evenly).
+
+   Note that using upmap requires that all clients be Luminous or newer.
+
+The default mode is ``upmap``.  The mode can be adjusted with:
+
+   .. prompt:: bash $
+
+      ceph balancer mode crush-compat
+
+Supervised optimization
+-----------------------
+
+The balancer operation is broken into a few distinct phases:
+
+#. building a *plan*
+#. evaluating the quality of the data distribution, either for the current PG distribution, or the PG distribution that would result after executing a *plan*
+#. executing the *plan*
+
+To evaluate and score the current distribution:
+
+   .. prompt:: bash $
+
+      ceph balancer eval
+
+You can also evaluate the distribution for a single pool with:
+
+   .. prompt:: bash $
+
+      ceph balancer eval <pool-name>
+
+Greater detail for the evaluation can be seen with:
+
+   .. prompt:: bash $
+
+      ceph balancer eval-verbose ...
+  
+The balancer can generate a plan, using the currently configured mode, with:
+
+   .. prompt:: bash $
+
+      ceph balancer optimize <plan-name>
+
+The name is provided by the user and can be any useful identifying string.  The contents of a plan can be seen with:
+
+   .. prompt:: bash $
+
+      ceph balancer show <plan-name>
+
+All plans can be shown with:
+
+   .. prompt:: bash $
+
+      ceph balancer ls
+
+Old plans can be discarded with:
+
+   .. prompt:: bash $
+
+      ceph balancer rm <plan-name>
+
+Currently recorded plans are shown as part of the status command:
+
+   .. prompt:: bash $
+
+      ceph balancer status
+
+The quality of the distribution that would result after executing a plan can be calculated with:
+
+   .. prompt:: bash $
+
+      ceph balancer eval <plan-name>
+
+Assuming the plan is expected to improve the distribution (i.e., it has a lower score than the current cluster state), the user can execute that plan with:
+
+   .. prompt:: bash $
+
+      ceph balancer execute <plan-name>
+
diff --git a/doc/rados/operations/bluestore-migration.rst b/doc/rados/operations/bluestore-migration.rst
new file mode 100644
index 000000000..1ac5f2b13
--- /dev/null
+++ b/doc/rados/operations/bluestore-migration.rst
@@ -0,0 +1,338 @@
+=====================
+ BlueStore Migration
+=====================
+
+Each OSD can run either BlueStore or FileStore, and a single Ceph
+cluster can contain a mix of both.  Users who have previously deployed
+FileStore are likely to want to transition to BlueStore in order to
+take advantage of the improved performance and robustness.  There are
+several strategies for making such a transition.
+
+An individual OSD cannot be converted in place in isolation, however:
+BlueStore and FileStore are simply too different for that to be
+practical.  "Conversion" will rely either on the cluster's normal
+replication and healing support or tools and strategies that copy OSD
+content from an old (FileStore) device to a new (BlueStore) one.
+
+
+Deploy new OSDs with BlueStore
+==============================
+
+Any new OSDs (e.g., when the cluster is expanded) can be deployed
+using BlueStore.  This is the default behavior so no specific change
+is needed.
+
+Similarly, any OSDs that are reprovisioned after replacing a failed drive
+can use BlueStore.
+
+Convert existing OSDs
+=====================
+
+Mark out and replace
+--------------------
+
+The simplest approach is to mark out each device in turn, wait for the
+data to replicate across the cluster, reprovision the OSD, and mark
+it back in again.  It is simple and easy to automate.  However, it requires
+more data migration than should be necessary, so it is not optimal.
+
+#. Identify a FileStore OSD to replace::
+
+     ID=<osd-id-number>
+     DEVICE=<disk-device>
+
+   You can tell whether a given OSD is FileStore or BlueStore with:
+
+   .. prompt:: bash $
+
+      ceph osd metadata $ID | grep osd_objectstore
+
+   You can get a current count of filestore vs bluestore with:
+
+   .. prompt:: bash $
+
+      ceph osd count-metadata osd_objectstore
+
+#. Mark the filestore OSD out:
+
+   .. prompt:: bash $
+
+      ceph osd out $ID
+
+#. Wait for the data to migrate off the OSD in question:
+
+   .. prompt:: bash $
+
+      while ! ceph osd safe-to-destroy $ID ; do sleep 60 ; done
+
+#. Stop the OSD:
+
+   .. prompt:: bash $
+
+      systemctl kill ceph-osd@$ID
+
+#. Make note of which device this OSD is using:
+
+   .. prompt:: bash $
+
+      mount | grep /var/lib/ceph/osd/ceph-$ID
+
+#. Unmount the OSD:
+
+   .. prompt:: bash $
+
+      umount /var/lib/ceph/osd/ceph-$ID
+
+#. Destroy the OSD data. Be *EXTREMELY CAREFUL* as this will destroy
+   the contents of the device; be certain the data on the device is
+   not needed (i.e., that the cluster is healthy) before proceeding:
+
+   .. prompt:: bash $
+
+      ceph-volume lvm zap $DEVICE
+
+#. Tell the cluster the OSD has been destroyed (and a new OSD can be
+   reprovisioned with the same ID):
+
+   .. prompt:: bash $
+
+      ceph osd destroy $ID --yes-i-really-mean-it
+
+#. Reprovision a BlueStore OSD in its place with the same OSD ID.
+   This requires you do identify which device to wipe based on what you saw
+   mounted above. BE CAREFUL! :
+
+   .. prompt:: bash $
+
+      ceph-volume lvm create --bluestore --data $DEVICE --osd-id $ID
+
+#. Repeat.
+
+You can allow the refilling of the replacement OSD to happen
+concurrently with the draining of the next OSD, or follow the same
+procedure for multiple OSDs in parallel, as long as you ensure the
+cluster is fully clean (all data has all replicas) before destroying
+any OSDs.  Failure to do so will reduce the redundancy of your data
+and increase the risk of (or potentially even cause) data loss.
+
+Advantages:
+
+* Simple.
+* Can be done on a device-by-device basis.
+* No spare devices or hosts are required.
+
+Disadvantages:
+
+* Data is copied over the network twice: once to some other OSD in the
+  cluster (to maintain the desired number of replicas), and then again
+  back to the reprovisioned BlueStore OSD.
+
+
+Whole host replacement
+----------------------
+
+If you have a spare host in the cluster, or have sufficient free space
+to evacuate an entire host in order to use it as a spare, then the
+conversion can be done on a host-by-host basis with each stored copy of
+the data migrating only once.
+
+First, you need have empty host that has no data.  There are two ways to do this: either by starting with a new, empty host that isn't yet part of the cluster, or by offloading data from an existing host that in the cluster.
+
+Use a new, empty host
+^^^^^^^^^^^^^^^^^^^^^
+
+Ideally the host should have roughly the
+same capacity as other hosts you will be converting (although it
+doesn't strictly matter). ::
+
+  NEWHOST=<empty-host-name>
+
+Add the host to the CRUSH hierarchy, but do not attach it to the root:
+
+.. prompt:: bash $
+
+   ceph osd crush add-bucket $NEWHOST host
+
+Make sure the ceph packages are installed.
+
+Use an existing host
+^^^^^^^^^^^^^^^^^^^^
+
+If you would like to use an existing host
+that is already part of the cluster, and there is sufficient free
+space on that host so that all of its data can be migrated off,
+then you can instead do::
+
+  OLDHOST=<existing-cluster-host-to-offload>
+
+.. prompt:: bash $ 
+   
+   ceph osd crush unlink $OLDHOST default
+
+where "default" is the immediate ancestor in the CRUSH map. (For
+smaller clusters with unmodified configurations this will normally
+be "default", but it might also be a rack name.)  You should now
+see the host at the top of the OSD tree output with no parent:
+
+.. prompt:: bash $
+
+   bin/ceph osd tree
+
+::
+
+  ID CLASS WEIGHT  TYPE NAME     STATUS REWEIGHT PRI-AFF
+  -5             0 host oldhost
+  10   ssd 1.00000     osd.10        up  1.00000 1.00000
+  11   ssd 1.00000     osd.11        up  1.00000 1.00000
+  12   ssd 1.00000     osd.12        up  1.00000 1.00000
+  -1       3.00000 root default
+  -2       3.00000     host foo
+   0   ssd 1.00000         osd.0     up  1.00000 1.00000
+   1   ssd 1.00000         osd.1     up  1.00000 1.00000
+   2   ssd 1.00000         osd.2     up  1.00000 1.00000
+  ...
+
+If everything looks good, jump directly to the "Wait for data
+migration to complete" step below and proceed from there to clean up
+the old OSDs.
+
+Migration process
+^^^^^^^^^^^^^^^^^
+
+If you're using a new host, start at step #1.  For an existing host,
+jump to step #5 below.
+
+#. Provision new BlueStore OSDs for all devices:
+
+   .. prompt:: bash $
+
+      ceph-volume lvm create --bluestore --data /dev/$DEVICE
+
+#. Verify OSDs join the cluster with:
+
+   .. prompt:: bash $
+
+      ceph osd tree
+
+   You should see the new host ``$NEWHOST`` with all of the OSDs beneath
+   it, but the host should *not* be nested beneath any other node in
+   hierarchy (like ``root default``).  For example, if ``newhost`` is
+   the empty host, you might see something like::
+
+     $ bin/ceph osd tree
+     ID CLASS WEIGHT  TYPE NAME     STATUS REWEIGHT PRI-AFF
+     -5             0 host newhost
+     10   ssd 1.00000     osd.10        up  1.00000 1.00000
+     11   ssd 1.00000     osd.11        up  1.00000 1.00000
+     12   ssd 1.00000     osd.12        up  1.00000 1.00000
+     -1       3.00000 root default
+     -2       3.00000     host oldhost1
+      0   ssd 1.00000         osd.0     up  1.00000 1.00000
+      1   ssd 1.00000         osd.1     up  1.00000 1.00000
+      2   ssd 1.00000         osd.2     up  1.00000 1.00000
+     ...
+
+#. Identify the first target host to convert :
+
+   .. prompt:: bash $
+
+      OLDHOST=<existing-cluster-host-to-convert>
+
+#. Swap the new host into the old host's position in the cluster:
+
+   .. prompt:: bash $
+
+      ceph osd crush swap-bucket $NEWHOST $OLDHOST
+
+   At this point all data on ``$OLDHOST`` will start migrating to OSDs
+   on ``$NEWHOST``.  If there is a difference in the total capacity of
+   the old and new hosts you may also see some data migrate to or from
+   other nodes in the cluster, but as long as the hosts are similarly
+   sized this will be a relatively small amount of data.
+
+#. Wait for data migration to complete:
+
+   .. prompt:: bash $
+
+      while ! ceph osd safe-to-destroy $(ceph osd ls-tree $OLDHOST); do sleep 60 ; done
+
+#. Stop all old OSDs on the now-empty ``$OLDHOST``:
+
+   .. prompt:: bash $
+
+      ssh $OLDHOST
+     systemctl kill ceph-osd.target
+     umount /var/lib/ceph/osd/ceph-*
+
+#. Destroy and purge the old OSDs:
+
+   .. prompt:: bash $
+
+      for osd in `ceph osd ls-tree $OLDHOST`; do
+         ceph osd purge $osd --yes-i-really-mean-it
+     done
+
+#. Wipe the old OSD devices. This requires you do identify which
+   devices are to be wiped manually (BE CAREFUL!). For each device:
+
+   .. prompt:: bash $
+
+      ceph-volume lvm zap $DEVICE
+
+#. Use the now-empty host as the new host, and repeat::
+
+     NEWHOST=$OLDHOST
+
+Advantages:
+
+* Data is copied over the network only once.
+* Converts an entire host's OSDs at once.
+* Can parallelize to converting multiple hosts at a time.
+* No spare devices are required on each host.
+
+Disadvantages:
+
+* A spare host is required.
+* An entire host's worth of OSDs will be migrating data at a time.  This
+  is like likely to impact overall cluster performance.
+* All migrated data still makes one full hop over the network.
+
+
+Per-OSD device copy
+-------------------
+
+A single logical OSD can be converted by using the ``copy`` function
+of ``ceph-objectstore-tool``.  This requires that the host have a free
+device (or devices) to provision a new, empty BlueStore OSD.  For
+example, if each host in your cluster has 12 OSDs, then you'd need a
+13th available device so that each OSD can be converted in turn before the
+old device is reclaimed to convert the next OSD.
+
+Caveats:
+
+* This strategy requires that a blank BlueStore OSD be prepared
+  without allocating a new OSD ID, something that the ``ceph-volume``
+  tool doesn't support.  More importantly, the setup of *dmcrypt* is
+  closely tied to the OSD identity, which means that this approach
+  does not work with encrypted OSDs.
+
+* The device must be manually partitioned.
+
+* Tooling not implemented!
+
+* Not documented!
+
+Advantages:
+
+* Little or no data migrates over the network during the conversion.
+
+Disadvantages:
+
+* Tooling not fully implemented.
+* Process not documented.
+* Each host must have a spare or empty device.
+* The OSD is offline during the conversion, which means new writes will
+  be written to only a subset of the OSDs.  This increases the risk of data
+  loss due to a subsequent failure.  (However, if there is a failure before
+  conversion is complete, the original FileStore OSD can be started to provide
+  access to its original data.)
diff --git a/doc/rados/operations/cache-tiering.rst b/doc/rados/operations/cache-tiering.rst
new file mode 100644
index 000000000..8056ace47
--- /dev/null
+++ b/doc/rados/operations/cache-tiering.rst
@@ -0,0 +1,552 @@
+===============
+ Cache Tiering
+===============
+
+A cache tier provides Ceph Clients with better I/O performance for a subset of
+the data stored in a backing storage tier. Cache tiering involves creating a
+pool of relatively fast/expensive storage devices (e.g., solid state drives)
+configured to act as a cache tier, and a backing pool of either erasure-coded
+or relatively slower/cheaper devices configured to act as an economical storage
+tier. The Ceph objecter handles where to place the objects and the tiering
+agent determines when to flush objects from the cache to the backing storage
+tier. So the cache tier and the backing storage tier are completely transparent 
+to Ceph clients.
+
+
+.. ditaa::
+           +-------------+
+           | Ceph Client |
+           +------+------+
+                  ^
+     Tiering is   |  
+    Transparent   |              Faster I/O
+        to Ceph   |           +---------------+
+     Client Ops   |           |               |   
+                  |    +----->+   Cache Tier  |
+                  |    |      |               |
+                  |    |      +-----+---+-----+
+                  |    |            |   ^ 
+                  v    v            |   |   Active Data in Cache Tier
+           +------+----+--+         |   |
+           |   Objecter   |         |   |
+           +-----------+--+         |   |
+                       ^            |   |   Inactive Data in Storage Tier
+                       |            v   |
+                       |      +-----+---+-----+
+                       |      |               |
+                       +----->|  Storage Tier |
+                              |               |
+                              +---------------+
+                                 Slower I/O
+
+
+The cache tiering agent handles the migration of data between the cache tier 
+and the backing storage tier automatically. However, admins have the ability to
+configure how this migration takes place by setting the ``cache-mode``. There are
+two main scenarios:
+
+- **writeback** mode: If the base tier and the cache tier are configured in
+  ``writeback`` mode, Ceph clients receive an ACK from the base tier every time
+  they write data to it. Then the cache tiering agent determines whether
+  ``osd_tier_default_cache_min_write_recency_for_promote`` has been set. If it
+  has been set and the data has been written more than a specified number of
+  times per interval, the data is promoted to the cache tier.
+
+  When Ceph clients need access to data stored in the base tier, the cache
+  tiering agent reads the data from the base tier and returns it to the client.
+  While data is being read from the base tier, the cache tiering agent consults
+  the value of ``osd_tier_default_cache_min_read_recency_for_promote`` and
+  decides whether to promote that data from the base tier to the cache tier.
+  When data has been promoted from the base tier to the cache tier, the Ceph
+  client is able to perform I/O operations on it using the cache tier. This is
+  well-suited for mutable data (for example, photo/video editing, transactional
+  data).
+
+- **readproxy** mode: This mode will use any objects that already
+  exist in the cache tier, but if an object is not present in the
+  cache the request will be proxied to the base tier.  This is useful
+  for transitioning from ``writeback`` mode to a disabled cache as it
+  allows the workload to function properly while the cache is drained,
+  without adding any new objects to the cache.
+
+Other cache modes are:
+
+- **readonly** promotes objects to the cache on read operations only; write
+  operations are forwarded to the base tier. This mode is intended for
+  read-only workloads that do not require consistency to be enforced by the
+  storage system. (**Warning**: when objects are updated in the base tier,
+  Ceph makes **no** attempt to sync these updates to the corresponding objects
+  in the cache. Since this mode is considered experimental, a
+  ``--yes-i-really-mean-it`` option must be passed in order to enable it.)
+
+- **none** is used to completely disable caching.
+
+
+A word of caution
+=================
+
+Cache tiering will *degrade* performance for most workloads.  Users should use
+extreme caution before using this feature.
+
+* *Workload dependent*: Whether a cache will improve performance is
+  highly dependent on the workload.  Because there is a cost
+  associated with moving objects into or out of the cache, it can only
+  be effective when there is a *large skew* in the access pattern in
+  the data set, such that most of the requests touch a small number of
+  objects.  The cache pool should be large enough to capture the
+  working set for your workload to avoid thrashing.
+
+* *Difficult to benchmark*: Most benchmarks that users run to measure
+  performance will show terrible performance with cache tiering, in
+  part because very few of them skew requests toward a small set of
+  objects, it can take a long time for the cache to "warm up," and
+  because the warm-up cost can be high.
+
+* *Usually slower*: For workloads that are not cache tiering-friendly,
+  performance is often slower than a normal RADOS pool without cache
+  tiering enabled.
+
+* *librados object enumeration*: The librados-level object enumeration
+  API is not meant to be coherent in the presence of the case.  If
+  your application is using librados directly and relies on object
+  enumeration, cache tiering will probably not work as expected.
+  (This is not a problem for RGW, RBD, or CephFS.)
+
+* *Complexity*: Enabling cache tiering means that a lot of additional
+  machinery and complexity within the RADOS cluster is being used.
+  This increases the probability that you will encounter a bug in the system
+  that other users have not yet encountered and will put your deployment at a
+  higher level of risk.
+
+Known Good Workloads
+--------------------
+
+* *RGW time-skewed*: If the RGW workload is such that almost all read
+  operations are directed at recently written objects, a simple cache
+  tiering configuration that destages recently written objects from
+  the cache to the base tier after a configurable period can work
+  well.
+
+Known Bad Workloads
+-------------------
+
+The following configurations are *known to work poorly* with cache
+tiering.
+
+* *RBD with replicated cache and erasure-coded base*: This is a common
+  request, but usually does not perform well.  Even reasonably skewed
+  workloads still send some small writes to cold objects, and because
+  small writes are not yet supported by the erasure-coded pool, entire
+  (usually 4 MB) objects must be migrated into the cache in order to
+  satisfy a small (often 4 KB) write.  Only a handful of users have
+  successfully deployed this configuration, and it only works for them
+  because their data is extremely cold (backups) and they are not in
+  any way sensitive to performance.
+
+* *RBD with replicated cache and base*: RBD with a replicated base
+  tier does better than when the base is erasure coded, but it is
+  still highly dependent on the amount of skew in the workload, and
+  very difficult to validate.  The user will need to have a good
+  understanding of their workload and will need to tune the cache
+  tiering parameters carefully.
+
+
+Setting Up Pools
+================
+
+To set up cache tiering, you must have two pools. One will act as the 
+backing storage and the other will act as the cache.
+
+
+Setting Up a Backing Storage Pool
+---------------------------------
+
+Setting up a backing storage pool typically involves one of two scenarios: 
+
+- **Standard Storage**: In this scenario, the pool stores multiple copies
+  of an object in the Ceph Storage Cluster.
+
+- **Erasure Coding:** In this scenario, the pool uses erasure coding to 
+  store data much more efficiently with a small performance tradeoff.
+
+In the standard storage scenario, you can setup a CRUSH rule to establish 
+the failure domain (e.g., osd, host, chassis, rack, row, etc.). Ceph OSD 
+Daemons perform optimally when all storage drives in the rule are of the 
+same size, speed (both RPMs and throughput) and type. See `CRUSH Maps`_ 
+for details on creating a rule. Once you have created a rule, create 
+a backing storage pool. 
+
+In the erasure coding scenario, the pool creation arguments will generate the
+appropriate rule automatically. See `Create a Pool`_ for details.
+
+In subsequent examples, we will refer to the backing storage pool 
+as ``cold-storage``.
+
+
+Setting Up a Cache Pool
+-----------------------
+
+Setting up a cache pool follows the same procedure as the standard storage
+scenario, but with this difference: the drives for the cache tier are typically
+high performance drives that reside in their own servers and have their own
+CRUSH rule.  When setting up such a rule, it should take account of the hosts
+that have the high performance drives while omitting the hosts that don't. See
+:ref:`CRUSH Device Class<crush-map-device-class>` for details.
+
+
+In subsequent examples, we will refer to the cache pool as ``hot-storage`` and
+the backing pool as ``cold-storage``.
+
+For cache tier configuration and default values, see 
+`Pools - Set Pool Values`_.
+
+
+Creating a Cache Tier
+=====================
+
+Setting up a cache tier involves associating a backing storage pool with
+a cache pool:
+
+.. prompt:: bash $
+
+   ceph osd tier add {storagepool} {cachepool}
+
+For example:
+
+.. prompt:: bash $
+
+   ceph osd tier add cold-storage hot-storage
+
+To set the cache mode, execute the following:
+
+.. prompt:: bash $
+
+   ceph osd tier cache-mode {cachepool} {cache-mode}
+
+For example:
+
+.. prompt:: bash $
+
+   ceph osd tier cache-mode hot-storage writeback
+
+The cache tiers overlay the backing storage tier, so they require one
+additional step: you must direct all client traffic from the storage pool to 
+the cache pool. To direct client traffic directly to the cache pool, execute 
+the following:
+
+.. prompt:: bash $
+
+   ceph osd tier set-overlay {storagepool} {cachepool}
+
+For example:
+
+.. prompt:: bash $
+
+   ceph osd tier set-overlay cold-storage hot-storage
+
+
+Configuring a Cache Tier
+========================
+
+Cache tiers have several configuration options. You may set
+cache tier configuration options with the following usage:
+
+.. prompt:: bash $
+
+   ceph osd pool set {cachepool} {key} {value}
+   
+See `Pools - Set Pool Values`_ for details.
+
+
+Target Size and Type
+--------------------
+
+Ceph's production cache tiers use a `Bloom Filter`_ for the ``hit_set_type``:
+
+.. prompt:: bash $
+
+   ceph osd pool set {cachepool} hit_set_type bloom
+
+For example:
+
+.. prompt:: bash $
+
+   ceph osd pool set hot-storage hit_set_type bloom
+
+The ``hit_set_count`` and ``hit_set_period`` define how many such HitSets to
+store, and how much time each HitSet should cover:
+
+.. prompt:: bash $
+
+   ceph osd pool set {cachepool} hit_set_count 12
+   ceph osd pool set {cachepool} hit_set_period 14400
+   ceph osd pool set {cachepool} target_max_bytes 1000000000000
+
+.. note:: A larger ``hit_set_count`` results in more RAM consumed by
+          the ``ceph-osd`` process.
+
+Binning accesses over time allows Ceph to determine whether a Ceph client
+accessed an object at least once, or more than once over a time period 
+("age" vs "temperature").
+
+The ``min_read_recency_for_promote`` defines how many HitSets to check for the
+existence of an object when handling a read operation. The checking result is
+used to decide whether to promote the object asynchronously. Its value should be
+between 0 and ``hit_set_count``. If it's set to 0, the object is always promoted.
+If it's set to 1, the current HitSet is checked. And if this object is in the
+current HitSet, it's promoted. Otherwise not. For the other values, the exact
+number of archive HitSets are checked. The object is promoted if the object is
+found in any of the most recent ``min_read_recency_for_promote`` HitSets.
+
+A similar parameter can be set for the write operation, which is
+``min_write_recency_for_promote``:
+
+.. prompt:: bash $
+
+   ceph osd pool set {cachepool} min_read_recency_for_promote 2
+   ceph osd pool set {cachepool} min_write_recency_for_promote 2
+
+.. note:: The longer the period and the higher the
+   ``min_read_recency_for_promote`` and
+   ``min_write_recency_for_promote``values, the more RAM the ``ceph-osd``
+   daemon consumes. In particular, when the agent is active to flush
+   or evict cache objects, all ``hit_set_count`` HitSets are loaded
+   into RAM.
+
+
+Cache Sizing
+------------
+
+The cache tiering agent performs two main functions: 
+
+- **Flushing:** The agent identifies modified (or dirty) objects and forwards
+  them to the storage pool for long-term storage.
+  
+- **Evicting:** The agent identifies objects that haven't been modified 
+  (or clean) and evicts the least recently used among them from the cache.
+
+
+Absolute Sizing
+~~~~~~~~~~~~~~~
+
+The cache tiering agent can flush or evict objects based upon the total number
+of bytes or the total number of objects. To specify a maximum number of bytes,
+execute the following:
+
+.. prompt:: bash $
+
+   ceph osd pool set {cachepool} target_max_bytes {#bytes}
+
+For example, to flush or evict at 1 TB, execute the following:
+
+.. prompt:: bash $
+
+   ceph osd pool set hot-storage target_max_bytes 1099511627776
+
+To specify the maximum number of objects, execute the following:
+
+.. prompt:: bash $
+
+   ceph osd pool set {cachepool} target_max_objects {#objects}
+
+For example, to flush or evict at 1M objects, execute the following:
+
+.. prompt:: bash $
+
+   ceph osd pool set hot-storage target_max_objects 1000000
+
+.. note:: Ceph is not able to determine the size of a cache pool automatically, so
+   the configuration on the absolute size is required here, otherwise the
+   flush/evict will not work. If you specify both limits, the cache tiering
+   agent will begin flushing or evicting when either threshold is triggered.
+
+.. note:: All client requests will be blocked only when  ``target_max_bytes`` or
+   ``target_max_objects`` reached
+
+Relative Sizing
+~~~~~~~~~~~~~~~
+
+The cache tiering agent can flush or evict objects relative to the size of the
+cache pool(specified by ``target_max_bytes`` / ``target_max_objects`` in
+`Absolute sizing`_).  When the cache pool consists of a certain percentage of
+modified (or dirty) objects, the cache tiering agent will flush them to the
+storage pool. To set the ``cache_target_dirty_ratio``, execute the following:
+
+.. prompt:: bash $
+
+   ceph osd pool set {cachepool} cache_target_dirty_ratio {0.0..1.0}
+
+For example, setting the value to ``0.4`` will begin flushing modified
+(dirty) objects when they reach 40% of the cache pool's capacity:
+
+.. prompt:: bash $
+
+   ceph osd pool set hot-storage cache_target_dirty_ratio 0.4
+
+When the dirty objects reaches a certain percentage of its capacity, flush dirty
+objects with a higher speed. To set the ``cache_target_dirty_high_ratio``:
+
+.. prompt:: bash $
+
+   ceph osd pool set {cachepool} cache_target_dirty_high_ratio {0.0..1.0}
+
+For example, setting the value to ``0.6`` will begin aggressively flush dirty
+objects when they reach 60% of the cache pool's capacity. obviously, we'd
+better set the value between dirty_ratio and full_ratio:
+
+.. prompt:: bash $
+
+   ceph osd pool set hot-storage cache_target_dirty_high_ratio 0.6
+
+When the cache pool reaches a certain percentage of its capacity, the cache
+tiering agent will evict objects to maintain free capacity. To set the 
+``cache_target_full_ratio``, execute the following:
+
+.. prompt:: bash $
+
+   ceph osd pool set {cachepool} cache_target_full_ratio {0.0..1.0}
+
+For example, setting the value to ``0.8`` will begin flushing unmodified
+(clean) objects when they reach 80% of the cache pool's capacity:
+
+.. prompt:: bash $
+
+   ceph osd pool set hot-storage cache_target_full_ratio 0.8
+
+
+Cache Age
+---------
+
+You can specify the minimum age of an object before the cache tiering agent 
+flushes a recently modified (or dirty) object to the backing storage pool:
+
+.. prompt:: bash $
+
+   ceph osd pool set {cachepool} cache_min_flush_age {#seconds}
+
+For example, to flush modified (or dirty) objects after 10 minutes, execute the
+following:
+
+.. prompt:: bash $
+
+   ceph osd pool set hot-storage cache_min_flush_age 600
+
+You can specify the minimum age of an object before it will be evicted from the
+cache tier:
+
+.. prompt:: bash $
+
+   ceph osd pool {cache-tier} cache_min_evict_age {#seconds}
+
+For example, to evict objects after 30 minutes, execute the following:
+
+.. prompt:: bash $
+
+   ceph osd pool set hot-storage cache_min_evict_age 1800
+
+
+Removing a Cache Tier
+=====================
+
+Removing a cache tier differs depending on whether it is a writeback 
+cache or a read-only cache.
+
+
+Removing a Read-Only Cache
+--------------------------
+
+Since a read-only cache does not have modified data, you can disable
+and remove it without losing any recent changes to objects in the cache. 
+
+#. Change the cache-mode to ``none`` to disable it.:
+
+   .. prompt:: bash 
+
+      ceph osd tier cache-mode {cachepool} none
+
+   For example:
+
+   .. prompt:: bash $
+
+      ceph osd tier cache-mode hot-storage none
+
+#. Remove the cache pool from the backing pool.:
+
+   .. prompt:: bash $
+
+      ceph osd tier remove {storagepool} {cachepool}
+
+   For example:
+
+   .. prompt:: bash $
+
+      ceph osd tier remove cold-storage hot-storage
+
+
+Removing a Writeback Cache
+--------------------------
+
+Since a writeback cache may have modified data, you must take steps to ensure 
+that you do not lose any recent changes to objects in the cache before you 
+disable and remove it.
+
+
+#. Change the cache mode to ``proxy`` so that new and modified objects will 
+   flush to the backing storage pool.:
+
+   .. prompt:: bash $
+
+      ceph osd tier cache-mode {cachepool} proxy
+
+   For example: 
+
+   .. prompt:: bash $
+
+      ceph osd tier cache-mode hot-storage proxy
+
+
+#. Ensure that the cache pool has been flushed. This may take a few minutes:
+
+   .. prompt:: bash $
+
+      rados -p {cachepool} ls
+
+   If the cache pool still has objects, you can flush them manually. 
+   For example:
+
+   .. prompt:: bash $
+
+      rados -p {cachepool} cache-flush-evict-all
+
+
+#. Remove the overlay so that clients will not direct traffic to the cache.:
+
+   .. prompt:: bash $
+
+      ceph osd tier remove-overlay {storagetier}
+
+   For example:
+
+   .. prompt:: bash $
+
+      ceph osd tier remove-overlay cold-storage
+
+
+#. Finally, remove the cache tier pool from the backing storage pool.:
+
+   .. prompt:: bash $
+
+      ceph osd tier remove {storagepool} {cachepool} 
+
+   For example:
+
+   .. prompt:: bash $
+
+      ceph osd tier remove cold-storage hot-storage
+
+
+.. _Create a Pool: ../pools#create-a-pool
+.. _Pools - Set Pool Values: ../pools#set-pool-values
+.. _Bloom Filter: https://en.wikipedia.org/wiki/Bloom_filter
+.. _CRUSH Maps: ../crush-map
+.. _Absolute Sizing: #absolute-sizing
diff --git a/doc/rados/operations/change-mon-elections.rst b/doc/rados/operations/change-mon-elections.rst
new file mode 100644
index 000000000..eba730bdc
--- /dev/null
+++ b/doc/rados/operations/change-mon-elections.rst
@@ -0,0 +1,88 @@
+.. _changing_monitor_elections:
+
+=====================================
+Configure Monitor Election Strategies
+=====================================
+
+By default, the monitors will use the ``classic`` mode. We
+recommend that you stay in this mode unless you have a very specific reason.
+
+If you want to switch modes BEFORE constructing the cluster, change
+the ``mon election default strategy`` option. This option is an integer value:
+
+* 1 for "classic"
+* 2 for "disallow"
+* 3 for "connectivity"
+
+Once your cluster is running, you can change strategies by running ::
+
+  $ ceph mon set election_strategy {classic|disallow|connectivity}
+
+Choosing a mode
+===============
+The modes other than classic provide different features. We recommend
+you stay in classic mode if you don't need the extra features as it is
+the simplest mode.
+
+The disallow Mode
+=================
+This mode lets you mark monitors as disallowd, in which case they will
+participate in the quorum and serve clients, but cannot be elected leader. You
+may wish to use this if you have some monitors which are known to be far away
+from clients.
+You can disallow a leader by running:
+
+.. prompt:: bash $
+
+   ceph mon add disallowed_leader {name}
+
+You can remove a monitor from the disallowed list, and allow it to become
+a leader again, by running:
+
+.. prompt:: bash $
+
+   ceph mon rm disallowed_leader {name}
+
+The list of disallowed_leaders is included when you run:
+
+.. prompt:: bash $
+
+   ceph mon dump
+
+The connectivity Mode
+=====================
+This mode evaluates connection scores provided by each monitor for its
+peers and elects the monitor with the highest score. This mode is designed
+to handle network partitioning or *net-splits*, which may happen if your cluster
+is stretched across multiple data centers or otherwise has a non-uniform
+or unbalanced network topology.
+
+This mode also supports disallowing monitors from being the leader
+using the same commands as above in disallow.
+
+Examining connectivity scores
+=============================
+The monitors maintain connection scores even if they aren't in
+the connectivity election mode. You can examine the scores a monitor
+has by running:
+
+.. prompt:: bash $
+
+   ceph daemon mon.{name} connection scores dump
+
+Scores for individual connections range from 0-1 inclusive, and also
+include whether the connection is considered alive or dead (determined by
+whether it returned its latest ping within the timeout).
+
+While this would be an unexpected occurrence, if for some reason you experience
+problems and troubleshooting makes you think your scores have become invalid,
+you can forget history and reset them by running:
+
+.. prompt:: bash $
+
+   ceph daemon mon.{name} connection scores reset
+
+While resetting scores has low risk (monitors will still quickly determine
+if a connection is alive or dead, and trend back to the previous scores if they
+were accurate!), it should also not be needed and is not recommended unless
+requested by your support team or a developer.
diff --git a/doc/rados/operations/control.rst b/doc/rados/operations/control.rst
new file mode 100644
index 000000000..d7a512618
--- /dev/null
+++ b/doc/rados/operations/control.rst
@@ -0,0 +1,601 @@
+.. index:: control, commands
+
+==================
+ Control Commands
+==================
+
+
+Monitor Commands
+================
+
+Monitor commands are issued using the ``ceph`` utility:
+
+.. prompt:: bash $
+
+   ceph [-m monhost] {command}
+
+The command is usually (though not always) of the form:
+
+.. prompt:: bash $
+
+   ceph {subsystem} {command}
+
+
+System Commands
+===============
+
+Execute the following to display the current cluster status.  :
+
+.. prompt:: bash $
+
+   ceph -s
+   ceph status
+
+Execute the following to display a running summary of cluster status
+and major events. :
+
+.. prompt:: bash $
+
+   ceph -w
+
+Execute the following to show the monitor quorum, including which monitors are
+participating and which one is the leader. :
+
+.. prompt:: bash $
+
+   ceph mon stat
+   ceph quorum_status
+
+Execute the following to query the status of a single monitor, including whether
+or not it is in the quorum. :
+
+.. prompt:: bash $
+
+   ceph tell mon.[id] mon_status
+
+where the value of ``[id]`` can be determined, e.g., from ``ceph -s``.
+
+
+Authentication Subsystem
+========================
+
+To add a keyring for an OSD, execute the following:
+
+.. prompt:: bash $
+
+   ceph auth add {osd} {--in-file|-i} {path-to-osd-keyring}
+
+To list the cluster's keys and their capabilities, execute the following:
+
+.. prompt:: bash $
+
+   ceph auth ls
+
+
+Placement Group Subsystem
+=========================
+
+To display the statistics for all placement groups (PGs), execute the following: 
+
+.. prompt:: bash $
+
+   ceph pg dump [--format {format}]
+
+The valid formats are ``plain`` (default), ``json`` ``json-pretty``, ``xml``, and ``xml-pretty``.
+When implementing monitoring and other tools, it is best to use ``json`` format.
+JSON parsing is more deterministic than the human-oriented ``plain``, and the layout is much
+less variable from release to release.  The ``jq`` utility can be invaluable when extracting
+data from JSON output.
+
+To display the statistics for all placement groups stuck in a specified state, 
+execute the following: 
+
+.. prompt:: bash $
+
+   ceph pg dump_stuck inactive|unclean|stale|undersized|degraded [--format {format}] [-t|--threshold {seconds}]
+
+
+``--format`` may be ``plain`` (default), ``json``, ``json-pretty``, ``xml``, or ``xml-pretty``.
+
+``--threshold`` defines how many seconds "stuck" is (default: 300)
+
+**Inactive** Placement groups cannot process reads or writes because they are waiting for an OSD
+with the most up-to-date data to come back.
+
+**Unclean** Placement groups contain objects that are not replicated the desired number
+of times. They should be recovering.
+
+**Stale** Placement groups are in an unknown state - the OSDs that host them have not
+reported to the monitor cluster in a while (configured by
+``mon_osd_report_timeout``).
+
+Delete "lost" objects or revert them to their prior state, either a previous version
+or delete them if they were just created. :
+
+.. prompt:: bash $
+
+   ceph pg {pgid} mark_unfound_lost revert|delete
+
+
+.. _osd-subsystem:
+
+OSD Subsystem
+=============
+
+Query OSD subsystem status. :
+
+.. prompt:: bash $
+
+   ceph osd stat
+
+Write a copy of the most recent OSD map to a file. See
+:ref:`osdmaptool <osdmaptool>`. :
+
+.. prompt:: bash $
+
+   ceph osd getmap -o file
+
+Write a copy of the crush map from the most recent OSD map to
+file. :
+
+.. prompt:: bash $
+
+   ceph osd getcrushmap -o file
+
+The foregoing is functionally equivalent to :
+
+.. prompt:: bash $
+
+   ceph osd getmap -o /tmp/osdmap
+   osdmaptool /tmp/osdmap --export-crush file
+
+Dump the OSD map. Valid formats for ``-f`` are ``plain``, ``json``, ``json-pretty``,
+``xml``, and ``xml-pretty``. If no ``--format`` option is given, the OSD map is 
+dumped as plain text.  As above, JSON format is best for tools, scripting, and other automation. :
+
+.. prompt:: bash $
+
+   ceph osd dump [--format {format}]
+
+Dump the OSD map as a tree with one line per OSD containing weight
+and state. :
+
+.. prompt:: bash $
+
+   ceph osd tree [--format {format}]
+
+Find out where a specific object is or would be stored in the system:
+
+.. prompt:: bash $
+
+   ceph osd map <pool-name> <object-name>
+
+Add or move a new item (OSD) with the given id/name/weight at the specified
+location. :
+
+.. prompt:: bash $
+
+   ceph osd crush set {id} {weight} [{loc1} [{loc2} ...]]
+
+Remove an existing item (OSD) from the CRUSH map. :
+
+.. prompt:: bash $
+
+   ceph osd crush remove {name}
+
+Remove an existing bucket from the CRUSH map. :
+
+.. prompt:: bash $
+
+   ceph osd crush remove {bucket-name}
+
+Move an existing bucket from one position in the hierarchy to another.  :
+
+.. prompt:: bash $
+
+   ceph osd crush move {id} {loc1} [{loc2} ...]
+
+Set the weight of the item given by ``{name}`` to ``{weight}``. :
+
+.. prompt:: bash $
+
+   ceph osd crush reweight {name} {weight}
+
+Mark an OSD as ``lost``. This may result in permanent data loss. Use with caution. :
+
+.. prompt:: bash $
+
+   ceph osd lost {id} [--yes-i-really-mean-it]
+
+Create a new OSD. If no UUID is given, it will be set automatically when the OSD
+starts up. :
+
+.. prompt:: bash $
+
+   ceph osd create [{uuid}]
+
+Remove the given OSD(s). :
+
+.. prompt:: bash $
+
+   ceph osd rm [{id}...]
+
+Query the current ``max_osd`` parameter in the OSD map. :
+
+.. prompt:: bash $
+
+   ceph osd getmaxosd
+
+Import the given crush map. :
+
+.. prompt:: bash $
+
+   ceph osd setcrushmap -i file
+
+Set the ``max_osd`` parameter in the OSD map. This defaults to 10000 now so
+most admins will never need to adjust this. :
+
+.. prompt:: bash $
+
+   ceph osd setmaxosd
+
+Mark OSD ``{osd-num}`` down. :
+
+.. prompt:: bash $
+
+   ceph osd down {osd-num}
+
+Mark OSD ``{osd-num}`` out of the distribution (i.e. allocated no data). :
+
+.. prompt:: bash $
+
+   ceph osd out {osd-num}
+
+Mark ``{osd-num}`` in the distribution (i.e. allocated data). :
+
+.. prompt:: bash $
+
+   ceph osd in {osd-num}
+
+Set or clear the pause flags in the OSD map. If set, no IO requests
+will be sent to any OSD. Clearing the flags via unpause results in
+resending pending requests. :
+
+.. prompt:: bash $
+
+   ceph osd pause
+   ceph osd unpause
+
+Set the override weight (reweight) of ``{osd-num}`` to ``{weight}``. Two OSDs with the
+same weight will receive roughly the same number of I/O requests and
+store approximately the same amount of data. ``ceph osd reweight``
+sets an override weight on the OSD. This value is in the range 0 to 1,
+and forces CRUSH to re-place (1-weight) of the data that would
+otherwise live on this drive. It does not change weights assigned
+to the buckets above the OSD in the crush map, and is a corrective
+measure in case the normal CRUSH distribution is not working out quite
+right. For instance, if one of your OSDs is at 90% and the others are
+at 50%, you could reduce this weight to compensate. :
+
+.. prompt:: bash $
+
+   ceph osd reweight {osd-num} {weight}
+
+Balance OSD fullness by reducing the override weight of OSDs which are
+overly utilized.  Note that these override aka ``reweight`` values
+default to 1.00000 and are relative only to each other; they not absolute.
+It is crucial to distinguish them from CRUSH weights, which reflect the
+absolute capacity of a bucket in TiB.  By default this command adjusts
+override weight on OSDs which have + or - 20% of the average utilization,
+but if you include a ``threshold`` that percentage will be used instead. :
+
+.. prompt:: bash $
+
+   ceph osd reweight-by-utilization [threshold [max_change [max_osds]]] [--no-increasing]
+
+To limit the step by which any OSD's reweight will be changed, specify
+``max_change`` which defaults to 0.05.  To limit the number of OSDs that will
+be adjusted, specify ``max_osds`` as well; the default is 4.  Increasing these
+parameters can speed leveling of OSD utilization, at the potential cost of
+greater impact on client operations due to more data moving at once.
+
+To determine which and how many PGs and OSDs will be affected by a given invocation
+you can test before executing. :
+
+.. prompt:: bash $
+
+   ceph osd test-reweight-by-utilization [threshold [max_change max_osds]] [--no-increasing]
+
+Adding ``--no-increasing`` to either command prevents increasing any
+override weights that are currently < 1.00000.  This can be useful when
+you are balancing in a hurry to remedy ``full`` or ``nearful`` OSDs or
+when some OSDs are being evacuated or slowly brought into service.
+
+Deployments utilizing Nautilus (or later revisions of Luminous and Mimic)
+that have no pre-Luminous cients may instead wish to instead enable the
+`balancer`` module for ``ceph-mgr``.
+
+Add/remove an IP address or CIDR range to/from the blocklist.
+When adding to the blocklist,
+you can specify how long it should be blocklisted in seconds; otherwise,
+it will default to 1 hour. A blocklisted address is prevented from
+connecting to any OSD. If you blocklist an IP or range containing an OSD, be aware
+that OSD will also be prevented from performing operations on its peers where it
+acts as a client. (This includes tiering and copy-from functionality.)
+
+If you want to blocklist a range (in CIDR format), you may do so by
+including the ``range`` keyword.
+
+These commands are mostly only useful for failure testing, as
+blocklists are normally maintained automatically and shouldn't need
+manual intervention. :
+
+.. prompt:: bash $
+
+   ceph osd blocklist ["range"] add ADDRESS[:source_port][/netmask_bits] [TIME]
+   ceph osd blocklist ["range"] rm ADDRESS[:source_port][/netmask_bits]
+
+Creates/deletes a snapshot of a pool. :
+
+.. prompt:: bash $
+
+   ceph osd pool mksnap {pool-name} {snap-name}
+   ceph osd pool rmsnap {pool-name} {snap-name}
+
+Creates/deletes/renames a storage pool. :
+
+.. prompt:: bash $
+
+   ceph osd pool create {pool-name} [pg_num [pgp_num]]
+   ceph osd pool delete {pool-name} [{pool-name} --yes-i-really-really-mean-it]
+   ceph osd pool rename {old-name} {new-name}
+
+Changes a pool setting. : 
+
+.. prompt:: bash $
+
+   ceph osd pool set {pool-name} {field} {value}
+
+Valid fields are:
+
+	* ``size``: Sets the number of copies of data in the pool.
+	* ``pg_num``: The placement group number.
+	* ``pgp_num``: Effective number when calculating pg placement.
+	* ``crush_rule``: rule number for mapping placement.
+
+Get the value of a pool setting. :
+
+.. prompt:: bash $
+
+   ceph osd pool get {pool-name} {field}
+
+Valid fields are:
+
+	* ``pg_num``: The placement group number.
+	* ``pgp_num``: Effective number of placement groups when calculating placement.
+
+
+Sends a scrub command to OSD ``{osd-num}``. To send the command to all OSDs, use ``*``. :
+
+.. prompt:: bash $
+
+   ceph osd scrub {osd-num}
+
+Sends a repair command to OSD.N. To send the command to all OSDs, use ``*``. :
+
+.. prompt:: bash $
+
+   ceph osd repair N
+
+Runs a simple throughput benchmark against OSD.N, writing ``TOTAL_DATA_BYTES``
+in write requests of ``BYTES_PER_WRITE`` each. By default, the test
+writes 1 GB in total in 4-MB increments.
+The benchmark is non-destructive and will not overwrite existing live
+OSD data, but might temporarily affect the performance of clients
+concurrently accessing the OSD. :
+
+.. prompt:: bash $
+
+   ceph tell osd.N bench [TOTAL_DATA_BYTES] [BYTES_PER_WRITE]
+
+To clear an OSD's caches between benchmark runs, use the 'cache drop' command :
+
+.. prompt:: bash $
+
+   ceph tell osd.N cache drop
+
+To get the cache statistics of an OSD, use the 'cache status' command :
+
+.. prompt:: bash $
+
+   ceph tell osd.N cache status
+
+MDS Subsystem
+=============
+
+Change configuration parameters on a running mds. :
+
+.. prompt:: bash $
+
+   ceph tell mds.{mds-id} config set {setting} {value}
+
+Example:
+
+.. prompt:: bash $
+
+   ceph tell mds.0 config set debug_ms 1
+
+Enables debug messages. :
+
+.. prompt:: bash $
+
+   ceph mds stat
+
+Displays the status of all metadata servers. :
+
+.. prompt:: bash $
+
+   ceph mds fail 0
+
+Marks the active MDS as failed, triggering failover to a standby if present.
+
+.. todo:: ``ceph mds`` subcommands missing docs: set, dump, getmap, stop, setmap
+
+
+Mon Subsystem
+=============
+
+Show monitor stats:
+
+.. prompt:: bash $
+
+   ceph mon stat
+
+::
+
+	e2: 3 mons at {a=127.0.0.1:40000/0,b=127.0.0.1:40001/0,c=127.0.0.1:40002/0}, election epoch 6, quorum 0,1,2 a,b,c
+
+
+The ``quorum`` list at the end lists monitor nodes that are part of the current quorum.
+
+This is also available more directly:
+
+.. prompt:: bash $
+
+   ceph quorum_status -f json-pretty
+	
+.. code-block:: javascript	
+
+	{
+	    "election_epoch": 6,
+	    "quorum": [
+		0,
+		1,
+		2
+	    ],
+	    "quorum_names": [
+		"a",
+		"b",
+		"c"
+	    ],
+	    "quorum_leader_name": "a",
+	    "monmap": {
+		"epoch": 2,
+		"fsid": "ba807e74-b64f-4b72-b43f-597dfe60ddbc",
+		"modified": "2016-12-26 14:42:09.288066",
+		"created": "2016-12-26 14:42:03.573585",
+		"features": {
+		    "persistent": [
+			"kraken"
+		    ],
+		    "optional": []
+		},
+		"mons": [
+		    {
+			"rank": 0,
+			"name": "a",
+			"addr": "127.0.0.1:40000\/0",
+			"public_addr": "127.0.0.1:40000\/0"
+		    },
+		    {
+			"rank": 1,
+			"name": "b",
+			"addr": "127.0.0.1:40001\/0",
+			"public_addr": "127.0.0.1:40001\/0"
+		    },
+		    {
+			"rank": 2,
+			"name": "c",
+			"addr": "127.0.0.1:40002\/0",
+			"public_addr": "127.0.0.1:40002\/0"
+		    }
+		]
+	    }
+	}
+	  
+
+The above will block until a quorum is reached.
+
+For a status of just a single monitor:
+
+.. prompt:: bash $
+
+   ceph tell mon.[name] mon_status
+	
+where the value of ``[name]`` can be taken from ``ceph quorum_status``. Sample
+output::
+	
+	{
+	    "name": "b",
+	    "rank": 1,
+	    "state": "peon",
+	    "election_epoch": 6,
+	    "quorum": [
+		0,
+		1,
+		2
+	    ],
+	    "features": {
+		"required_con": "9025616074522624",
+		"required_mon": [
+		    "kraken"
+		],
+		"quorum_con": "1152921504336314367",
+		"quorum_mon": [
+		    "kraken"
+		]
+	    },
+	    "outside_quorum": [],
+	    "extra_probe_peers": [],
+	    "sync_provider": [],
+	    "monmap": {
+		"epoch": 2,
+		"fsid": "ba807e74-b64f-4b72-b43f-597dfe60ddbc",
+		"modified": "2016-12-26 14:42:09.288066",
+		"created": "2016-12-26 14:42:03.573585",
+		"features": {
+		    "persistent": [
+			"kraken"
+		    ],
+		    "optional": []
+		},
+		"mons": [
+		    {
+			"rank": 0,
+			"name": "a",
+			"addr": "127.0.0.1:40000\/0",
+			"public_addr": "127.0.0.1:40000\/0"
+		    },
+		    {
+			"rank": 1,
+			"name": "b",
+			"addr": "127.0.0.1:40001\/0",
+			"public_addr": "127.0.0.1:40001\/0"
+		    },
+		    {
+			"rank": 2,
+			"name": "c",
+			"addr": "127.0.0.1:40002\/0",
+			"public_addr": "127.0.0.1:40002\/0"
+		    }
+		]
+	    }
+	}
+
+A dump of the monitor state:
+
+       .. prompt:: bash $
+
+	ceph mon dump
+
+  ::
+
+	dumped monmap epoch 2
+	epoch 2
+	fsid ba807e74-b64f-4b72-b43f-597dfe60ddbc
+	last_changed 2016-12-26 14:42:09.288066
+	created 2016-12-26 14:42:03.573585
+	0: 127.0.0.1:40000/0 mon.a
+	1: 127.0.0.1:40001/0 mon.b
+	2: 127.0.0.1:40002/0 mon.c
+
diff --git a/doc/rados/operations/crush-map-edits.rst b/doc/rados/operations/crush-map-edits.rst
new file mode 100644
index 000000000..18553e47d
--- /dev/null
+++ b/doc/rados/operations/crush-map-edits.rst
@@ -0,0 +1,747 @@
+Manually editing a CRUSH Map
+============================
+
+.. note:: Manually editing the CRUSH map is an advanced
+	  administrator operation.  All CRUSH changes that are
+	  necessary for the overwhelming majority of installations are
+	  possible via the standard ceph CLI and do not require manual
+	  CRUSH map edits.  If you have identified a use case where
+	  manual edits *are* necessary with recent Ceph releases, consider
+	  contacting the Ceph developers so that future versions of Ceph
+	  can obviate your corner case.
+
+To edit an existing CRUSH map:
+
+#. `Get the CRUSH map`_.
+#. `Decompile`_ the CRUSH map.
+#. Edit at least one of `Devices`_, `Buckets`_ and `Rules`_.
+#. `Recompile`_ the CRUSH map.
+#. `Set the CRUSH map`_.
+
+For details on setting the CRUSH map rule for a specific pool, see `Set
+Pool Values`_.
+
+.. _Get the CRUSH map: #getcrushmap
+.. _Decompile: #decompilecrushmap
+.. _Devices: #crushmapdevices
+.. _Buckets: #crushmapbuckets
+.. _Rules: #crushmaprules
+.. _Recompile: #compilecrushmap
+.. _Set the CRUSH map: #setcrushmap
+.. _Set Pool Values: ../pools#setpoolvalues
+
+.. _getcrushmap:
+
+Get a CRUSH Map
+---------------
+
+To get the CRUSH map for your cluster, execute the following:
+
+.. prompt:: bash $
+
+	ceph osd getcrushmap -o {compiled-crushmap-filename}
+
+Ceph will output (-o) a compiled CRUSH map to the filename you specified. Since
+the CRUSH map is in a compiled form, you must decompile it first before you can
+edit it.
+
+.. _decompilecrushmap:
+
+Decompile a CRUSH Map
+---------------------
+
+To decompile a CRUSH map, execute the following:
+
+.. prompt:: bash $
+
+	crushtool -d {compiled-crushmap-filename} -o {decompiled-crushmap-filename}
+
+.. _compilecrushmap:
+
+Recompile a CRUSH Map
+---------------------
+
+To compile a CRUSH map, execute the following:
+
+.. prompt:: bash $
+
+	crushtool -c {decompiled-crushmap-filename} -o {compiled-crushmap-filename}
+
+.. _setcrushmap:
+
+Set the CRUSH Map
+-----------------
+
+To set the CRUSH map for your cluster, execute the following:
+
+.. prompt:: bash $
+
+	ceph osd setcrushmap -i {compiled-crushmap-filename}
+
+Ceph will load (-i) a compiled CRUSH map from the filename you specified.
+
+Sections
+--------
+
+There are six main sections to a CRUSH Map.
+
+#. **tunables:** The preamble at the top of the map describes any *tunables*
+   that differ from the historical / legacy CRUSH behavior. These
+   correct for old bugs, optimizations, or other changes that have
+   been made over the years to improve CRUSH's behavior.
+
+#. **devices:** Devices are individual OSDs that store data.
+
+#. **types**: Bucket ``types`` define the types of buckets used in
+   your CRUSH hierarchy. Buckets consist of a hierarchical aggregation
+   of storage locations (e.g., rows, racks, chassis, hosts, etc.) and
+   their assigned weights.
+
+#. **buckets:** Once you define bucket types, you must define each node
+   in the hierarchy, its type, and which devices or other nodes it
+   contains.
+
+#. **rules:** Rules define policy about how data is distributed across
+   devices in the hierarchy.
+
+#. **choose_args:** Choose_args are alternative weights associated with
+   the hierarchy that have been adjusted to optimize data placement.  A single
+   choose_args map can be used for the entire cluster, or one can be
+   created for each individual pool.
+
+
+.. _crushmapdevices:
+
+CRUSH Map Devices
+-----------------
+
+Devices are individual OSDs that store data.  Usually one is defined here for each
+OSD daemon in your
+cluster.  Devices are identified by an ``id`` (a non-negative integer) and
+a ``name``, normally ``osd.N`` where ``N`` is the device id.
+
+.. _crush-map-device-class:
+
+Devices may also have a *device class* associated with them (e.g.,
+``hdd`` or ``ssd``), allowing them to be conveniently targeted by a
+crush rule.
+
+.. prompt:: bash #
+
+	devices
+
+::
+
+	device {num} {osd.name} [class {class}]
+
+For example:
+
+.. prompt:: bash #
+
+	devices
+
+::
+
+	device 0 osd.0 class ssd
+	device 1 osd.1 class hdd
+	device 2 osd.2
+	device 3 osd.3
+
+In most cases, each device maps to a single ``ceph-osd`` daemon.  This
+is normally a single storage device, a pair of devices (for example,
+one for data and one for a journal or metadata), or in some cases a
+small RAID device.
+
+CRUSH Map Bucket Types
+----------------------
+
+The second list in the CRUSH map defines 'bucket' types. Buckets facilitate
+a hierarchy of nodes and leaves. Node (or non-leaf) buckets typically represent
+physical locations in a hierarchy. Nodes aggregate other nodes or leaves.
+Leaf buckets represent ``ceph-osd`` daemons and their corresponding storage
+media.
+
+.. tip:: The term "bucket" used in the context of CRUSH means a node in
+   the hierarchy, i.e. a location or a piece of physical hardware. It
+   is a different concept from the term "bucket" when used in the
+   context of RADOS Gateway APIs.
+
+To add a bucket type to the CRUSH map, create a new line under your list of
+bucket types. Enter ``type`` followed by a unique numeric ID and a bucket name.
+By convention, there is one leaf bucket and it is ``type 0``;  however, you may
+give it any name you like (e.g., osd, disk, drive, storage)::
+
+	# types
+	type {num} {bucket-name}
+
+For example::
+
+	# types
+	type 0 osd
+	type 1 host
+	type 2 chassis
+	type 3 rack
+	type 4 row
+	type 5 pdu
+	type 6 pod
+	type 7 room
+	type 8 datacenter
+	type 9 zone
+	type 10 region
+	type 11 root
+
+
+
+.. _crushmapbuckets:
+
+CRUSH Map Bucket Hierarchy
+--------------------------
+
+The CRUSH algorithm distributes data objects among storage devices according
+to a per-device weight value, approximating a uniform probability distribution.
+CRUSH distributes objects and their replicas according to the hierarchical
+cluster map you define. Your CRUSH map represents the available storage
+devices and the logical elements that contain them.
+
+To map placement groups to OSDs across failure domains, a CRUSH map defines a
+hierarchical list of bucket types (i.e., under ``#types`` in the generated CRUSH
+map). The purpose of creating a bucket hierarchy is to segregate the
+leaf nodes by their failure domains, such as hosts, chassis, racks, power
+distribution units, pods, rows, rooms, and data centers. With the exception of
+the leaf nodes representing OSDs, the rest of the hierarchy is arbitrary, and
+you may define it according to your own needs.
+
+We recommend adapting your CRUSH map to your firm's hardware naming conventions
+and using instance names that reflect the physical hardware. Your naming
+practice can make it easier to administer the cluster and troubleshoot
+problems when an OSD and/or other hardware malfunctions and the administrator
+need access to physical hardware.
+
+In the following example, the bucket hierarchy has a leaf bucket named ``osd``,
+and two node buckets named ``host`` and ``rack`` respectively.
+
+.. ditaa::
+                           +-----------+
+                           | {o}rack   |
+                           |   Bucket  |
+                           +-----+-----+
+                                 |
+                 +---------------+---------------+
+                 |                               |
+           +-----+-----+                   +-----+-----+
+           | {o}host   |                   | {o}host   |
+           |   Bucket  |                   |   Bucket  |
+           +-----+-----+                   +-----+-----+
+                 |                               |
+         +-------+-------+               +-------+-------+
+         |               |               |               |
+   +-----+-----+   +-----+-----+   +-----+-----+   +-----+-----+
+   |    osd    |   |    osd    |   |    osd    |   |    osd    |
+   |   Bucket  |   |   Bucket  |   |   Bucket  |   |   Bucket  |
+   +-----------+   +-----------+   +-----------+   +-----------+
+
+.. note:: The higher numbered ``rack`` bucket type aggregates the lower
+   numbered ``host`` bucket type.
+
+Since leaf nodes reflect storage devices declared under the ``#devices`` list
+at the beginning of the CRUSH map, you do not need to declare them as bucket
+instances. The second lowest bucket type in your hierarchy usually aggregates
+the devices (i.e., it's usually the computer containing the storage media, and
+uses whatever term you prefer to describe it, such as  "node", "computer",
+"server," "host", "machine", etc.). In high density environments, it is
+increasingly common to see multiple hosts/nodes per chassis. You should account
+for chassis failure too--e.g., the need to pull a chassis if a node fails may
+result in bringing down numerous hosts/nodes and their OSDs.
+
+When declaring a bucket instance, you must specify its type, give it a unique
+name (string), assign it a unique ID expressed as a negative integer (optional),
+specify a weight relative to the total capacity/capability of its item(s),
+specify the bucket algorithm (usually ``straw2``), and the hash (usually ``0``,
+reflecting hash algorithm ``rjenkins1``). A bucket may have one or more items.
+The items may consist of node buckets or leaves. Items may have a weight that
+reflects the relative weight of the item.
+
+You may declare a node bucket with the following syntax::
+
+	[bucket-type] [bucket-name] {
+		id [a unique negative numeric ID]
+		weight [the relative capacity/capability of the item(s)]
+		alg [the bucket type: uniform | list | tree | straw | straw2 ]
+		hash [the hash type: 0 by default]
+		item [item-name] weight [weight]
+	}
+
+For example, using the diagram above, we would define two host buckets
+and one rack bucket. The OSDs are declared as items within the host buckets::
+
+	host node1 {
+		id -1
+		alg straw2
+		hash 0
+		item osd.0 weight 1.00
+		item osd.1 weight 1.00
+	}
+
+	host node2 {
+		id -2
+		alg straw2
+		hash 0
+		item osd.2 weight 1.00
+		item osd.3 weight 1.00
+	}
+
+	rack rack1 {
+		id -3
+		alg straw2
+		hash 0
+		item node1 weight 2.00
+		item node2 weight 2.00
+	}
+
+.. note:: In the foregoing example, note that the rack bucket does not contain
+   any OSDs. Rather it contains lower level host buckets, and includes the
+   sum total of their weight in the item entry.
+
+.. topic:: Bucket Types
+
+   Ceph supports five bucket types, each representing a tradeoff between
+   performance and reorganization efficiency. If you are unsure of which bucket
+   type to use, we recommend using a ``straw2`` bucket.  For a detailed
+   discussion of bucket types, refer to
+   `CRUSH - Controlled, Scalable, Decentralized Placement of Replicated Data`_,
+   and more specifically to **Section 3.4**. The bucket types are:
+
+	#. **uniform**: Uniform buckets aggregate devices with **exactly** the same
+	   weight. For example, when firms commission or decommission hardware, they
+	   typically do so with many machines that have exactly the same physical
+	   configuration (e.g., bulk purchases). When storage devices have exactly
+	   the same weight, you may use the ``uniform`` bucket type, which allows
+	   CRUSH to map replicas into uniform buckets in constant time. With
+	   non-uniform weights, you should use another bucket algorithm.
+
+	#. **list**: List buckets aggregate their content as linked lists. Based on
+	   the :abbr:`RUSH (Replication Under Scalable Hashing)` :sub:`P` algorithm,
+	   a list is a natural and intuitive choice for an **expanding cluster**:
+	   either an object is relocated to the newest device with some appropriate
+	   probability, or it remains on the older devices as before. The result is
+	   optimal data migration when items are added to the bucket. Items removed
+	   from the middle or tail of the list, however, can result in a signi�cant
+	   amount of unnecessary movement, making list buckets most suitable for
+	   circumstances in which they **never (or very rarely) shrink**.
+
+	#. **tree**: Tree buckets use a binary search tree. They are more efficient
+	   than list buckets when a bucket contains a larger set of items. Based on
+	   the :abbr:`RUSH (Replication Under Scalable Hashing)` :sub:`R` algorithm,
+	   tree buckets reduce the placement time to O(log :sub:`n`), making them
+	   suitable for managing much larger sets of devices or nested buckets.
+
+	#. **straw**: List and Tree buckets use a divide and conquer strategy
+	   in a way that either gives certain items precedence (e.g., those
+	   at the beginning of a list) or obviates the need to consider entire
+	   subtrees of items at all. That improves the performance of the replica
+	   placement process, but can also introduce suboptimal reorganization
+	   behavior when the contents of a bucket change due an addition, removal,
+	   or re-weighting of an item. The straw bucket type allows all items to
+	   fairly “compete� against each other for replica placement through a
+	   process analogous to a draw of straws.
+
+        #. **straw2**: Straw2 buckets improve Straw to correctly avoid any data
+           movement between items when neighbor weights change.
+
+           For example the weight of item A including adding it anew or removing
+           it completely, there will be data movement only to or from item A.
+
+.. topic:: Hash
+
+   Each bucket uses a hash algorithm. Currently, Ceph supports ``rjenkins1``.
+   Enter ``0`` as your hash setting to select ``rjenkins1``.
+
+
+.. _weightingbucketitems:
+
+.. topic:: Weighting Bucket Items
+
+   Ceph expresses bucket weights as doubles, which allows for fine
+   weighting. A weight is the relative difference between device capacities. We
+   recommend using ``1.00`` as the relative weight for a 1TB storage device.
+   In such a scenario, a weight of ``0.5`` would represent approximately 500GB,
+   and a weight of ``3.00`` would represent approximately 3TB. Higher level
+   buckets have a weight that is the sum total of the leaf items aggregated by
+   the bucket.
+
+   A bucket item weight is one dimensional, but you may also calculate your
+   item weights to reflect the performance of the storage drive. For example,
+   if you have many 1TB drives where some have relatively low data transfer
+   rate and the others have a relatively high data transfer rate, you may
+   weight them differently, even though they have the same capacity (e.g.,
+   a weight of 0.80 for the first set of drives with lower total throughput,
+   and 1.20 for the second set of drives with higher total throughput).
+
+
+.. _crushmaprules:
+
+CRUSH Map Rules
+---------------
+
+CRUSH maps support the notion of 'CRUSH rules', which are the rules that
+determine data placement for a pool. The default CRUSH map has a rule for each
+pool. For large clusters, you will likely create many pools where each pool may
+have its own non-default CRUSH rule.
+
+.. note:: In most cases, you will not need to modify the default rule. When
+   you create a new pool, by default the rule will be set to ``0``.
+
+
+CRUSH rules define placement and replication strategies or distribution policies
+that allow you to specify exactly how CRUSH places object replicas. For
+example, you might create a rule selecting a pair of targets for 2-way
+mirroring, another rule for selecting three targets in two different data
+centers for 3-way mirroring, and yet another rule for erasure coding over six
+storage devices. For a detailed discussion of CRUSH rules, refer to
+`CRUSH - Controlled, Scalable, Decentralized Placement of Replicated Data`_,
+and more specifically to **Section 3.2**.
+
+A rule takes the following form::
+
+	rule <rulename> {
+
+		id [a unique whole numeric ID]
+		type [ replicated | erasure ]
+		min_size <min-size>
+		max_size <max-size>
+		step take <bucket-name> [class <device-class>]
+		step [choose|chooseleaf] [firstn|indep] <N> type <bucket-type>
+		step emit
+	}
+
+
+``id``
+
+:Description: A unique whole number for identifying the rule.
+
+:Purpose: A component of the rule mask.
+:Type: Integer
+:Required: Yes
+:Default: 0
+
+
+``type``
+
+:Description: Describes a rule for either a storage drive (replicated)
+              or a RAID.
+
+:Purpose: A component of the rule mask.
+:Type: String
+:Required: Yes
+:Default: ``replicated``
+:Valid Values: Currently only ``replicated`` and ``erasure``
+
+``min_size``
+
+:Description: If a pool makes fewer replicas than this number, CRUSH will
+              **NOT** select this rule.
+
+:Type: Integer
+:Purpose: A component of the rule mask.
+:Required: Yes
+:Default: ``1``
+
+``max_size``
+
+:Description: If a pool makes more replicas than this number, CRUSH will
+              **NOT** select this rule.
+
+:Type: Integer
+:Purpose: A component of the rule mask.
+:Required: Yes
+:Default: 10
+
+
+``step take <bucket-name> [class <device-class>]``
+
+:Description: Takes a bucket name, and begins iterating down the tree.
+              If the ``device-class`` is specified, it must match
+              a class previously used when defining a device. All
+              devices that do not belong to the class are excluded.
+:Purpose: A component of the rule.
+:Required: Yes
+:Example: ``step take data``
+
+
+``step choose firstn {num} type {bucket-type}``
+
+:Description: Selects the number of buckets of the given type from within the
+	      current bucket. The number is usually the number of replicas in
+	      the pool (i.e., pool size).
+
+              - If ``{num} == 0``, choose ``pool-num-replicas`` buckets (all available).
+              - If ``{num} > 0 && < pool-num-replicas``, choose that many buckets.
+              - If ``{num} < 0``, it means ``pool-num-replicas - {num}``.
+
+:Purpose: A component of the rule.
+:Prerequisite: Follows ``step take`` or ``step choose``.
+:Example: ``step choose firstn 1 type row``
+
+
+``step chooseleaf firstn {num} type {bucket-type}``
+
+:Description: Selects a set of buckets of ``{bucket-type}`` and chooses a leaf
+              node (that is, an OSD) from the subtree of each bucket in the set of buckets.
+              The number of buckets in the set is usually the number of replicas in
+              the pool (i.e., pool size).
+
+              - If ``{num} == 0``, choose ``pool-num-replicas`` buckets (all available).
+              - If ``{num} > 0 && < pool-num-replicas``, choose that many buckets.
+              - If ``{num} < 0``, it means ``pool-num-replicas - {num}``.
+
+:Purpose: A component of the rule. Usage removes the need to select a device using two steps.
+:Prerequisite: Follows ``step take`` or ``step choose``.
+:Example: ``step chooseleaf firstn 0 type row``
+
+
+``step emit``
+
+:Description: Outputs the current value and empties the stack. Typically used
+              at the end of a rule, but may also be used to pick from different
+              trees in the same rule.
+
+:Purpose: A component of the rule.
+:Prerequisite: Follows ``step choose``.
+:Example: ``step emit``
+
+.. important:: A given CRUSH rule may be assigned to multiple pools, but it
+   is not possible for a single pool to have multiple CRUSH rules.
+
+``firstn`` versus ``indep``
+
+:Description: Controls the replacement strategy CRUSH uses when items (OSDs)
+	      are marked down in the CRUSH map. If this rule is to be used with
+	      replicated pools it should be ``firstn`` and if it's for
+	      erasure-coded pools it should be ``indep``.
+
+	      The reason has to do with how they behave when a
+	      previously-selected device fails. Let's say you have a PG stored
+	      on OSDs 1, 2, 3, 4, 5. Then 3 goes down.
+	      
+	      With the "firstn" mode, CRUSH simply adjusts its calculation to
+	      select 1 and 2, then selects 3 but discovers it's down, so it
+	      retries and selects 4 and 5, and then goes on to select a new
+	      OSD 6. So the final CRUSH mapping change is
+	      1, 2, 3, 4, 5 -> 1, 2, 4, 5, 6.
+
+	      But if you're storing an EC pool, that means you just changed the
+	      data mapped to OSDs 4, 5, and 6! So the "indep" mode attempts to
+	      not do that. You can instead expect it, when it selects the failed
+	      OSD 3, to try again and pick out 6, for a final transformation of:
+	      1, 2, 3, 4, 5 -> 1, 2, 6, 4, 5
+	      
+.. _crush-reclassify:
+
+Migrating from a legacy SSD rule to device classes
+--------------------------------------------------
+
+It used to be necessary to manually edit your CRUSH map and maintain a
+parallel hierarchy for each specialized device type (e.g., SSD) in order to
+write rules that apply to those devices.  Since the Luminous release,
+the *device class* feature has enabled this transparently.
+
+However, migrating from an existing, manually customized per-device map to
+the new device class rules in the trivial way will cause all data in the
+system to be reshuffled.
+
+The ``crushtool`` has a few commands that can transform a legacy rule
+and hierarchy so that you can start using the new class-based rules.
+There are three types of transformations possible:
+
+#. ``--reclassify-root <root-name> <device-class>``
+
+   This will take everything in the hierarchy beneath root-name and
+   adjust any rules that reference that root via a ``take
+   <root-name>`` to instead ``take <root-name> class <device-class>``.
+   It renumbers the buckets in such a way that the old IDs are instead
+   used for the specified class's "shadow tree" so that no data
+   movement takes place.
+
+   For example, imagine you have an existing rule like::
+
+     rule replicated_ruleset {
+        id 0
+        type replicated
+        min_size 1
+        max_size 10
+        step take default
+        step chooseleaf firstn 0 type rack
+        step emit
+     }
+
+   If you reclassify the root `default` as class `hdd`, the rule will
+   become::
+
+     rule replicated_ruleset {
+        id 0
+        type replicated
+        min_size 1
+        max_size 10
+        step take default class hdd
+        step chooseleaf firstn 0 type rack
+        step emit
+     }
+
+#. ``--set-subtree-class <bucket-name> <device-class>``
+
+   This will mark every device in the subtree rooted at *bucket-name*
+   with the specified device class.
+
+   This is normally used in conjunction with the ``--reclassify-root``
+   option to ensure that all devices in that root are labeled with the
+   correct class.  In some situations, however, some of those devices
+   (correctly) have a different class and we do not want to relabel
+   them.  In such cases, one can exclude the ``--set-subtree-class``
+   option.  This means that the remapping process will not be perfect,
+   since the previous rule distributed across devices of multiple
+   classes but the adjusted rules will only map to devices of the
+   specified *device-class*, but that often is an accepted level of
+   data movement when the number of outlier devices is small.
+
+#. ``--reclassify-bucket <match-pattern> <device-class> <default-parent>``
+
+   This will allow you to merge a parallel type-specific hierarchy with the normal hierarchy.  For example, many users have maps like::
+
+     host node1 {
+        id -2           # do not change unnecessarily
+        # weight 109.152
+        alg straw2
+        hash 0  # rjenkins1
+        item osd.0 weight 9.096
+        item osd.1 weight 9.096
+        item osd.2 weight 9.096
+        item osd.3 weight 9.096
+        item osd.4 weight 9.096
+        item osd.5 weight 9.096
+        ...
+     }
+
+     host node1-ssd {
+        id -10          # do not change unnecessarily
+        # weight 2.000
+        alg straw2
+        hash 0  # rjenkins1
+        item osd.80 weight 2.000
+	...
+     }
+
+     root default {
+        id -1           # do not change unnecessarily
+        alg straw2
+        hash 0  # rjenkins1
+        item node1 weight 110.967
+        ...
+     }
+
+     root ssd {
+        id -18          # do not change unnecessarily
+        # weight 16.000
+        alg straw2
+        hash 0  # rjenkins1
+        item node1-ssd weight 2.000
+	...
+     }
+
+   This function will reclassify each bucket that matches a
+   pattern.  The pattern can look like ``%suffix`` or ``prefix%``.
+   For example, in the above example, we would use the pattern
+   ``%-ssd``.  For each matched bucket, the remaining portion of the
+   name (that matches the ``%`` wildcard) specifies the *base bucket*.
+   All devices in the matched bucket are labeled with the specified
+   device class and then moved to the base bucket.  If the base bucket
+   does not exist (e.g., ``node12-ssd`` exists but ``node12`` does
+   not), then it is created and linked underneath the specified
+   *default parent* bucket.  In each case, we are careful to preserve
+   the old bucket IDs for the new shadow buckets to prevent data
+   movement.  Any rules with ``take`` steps referencing the old
+   buckets are adjusted.
+
+#. ``--reclassify-bucket <bucket-name> <device-class> <base-bucket>``
+
+   The same command can also be used without a wildcard to map a
+   single bucket.  For example, in the previous example, we want the
+   ``ssd`` bucket to be mapped to the ``default`` bucket.
+
+The final command to convert the map comprising the above fragments would be something like:
+
+.. prompt:: bash $
+
+  ceph osd getcrushmap -o original
+  crushtool -i original --reclassify \
+    --set-subtree-class default hdd \
+    --reclassify-root default hdd \
+    --reclassify-bucket %-ssd ssd default \
+    --reclassify-bucket ssd ssd default \
+    -o adjusted
+
+In order to ensure that the conversion is correct, there is a ``--compare`` command that will test a large sample of inputs against the CRUSH map and check that the same result is output. These inputs are controlled by the same options that apply to the ``--test`` command.  For the above example,:
+
+.. prompt:: bash $
+
+   crushtool -i original --compare adjusted
+
+::
+
+  rule 0 had 0/10240 mismatched mappings (0)
+  rule 1 had 0/10240 mismatched mappings (0)
+  maps appear equivalent
+
+If there were differences, the ratio of remapped inputs would be reported in
+the parentheses.
+
+When you are satisfied with the adjusted map, apply it to the cluster with a command of the form:
+
+.. prompt:: bash $
+
+   ceph osd setcrushmap -i adjusted
+
+Tuning CRUSH, the hard way
+--------------------------
+
+If you can ensure that all clients are running recent code, you can
+adjust the tunables by extracting the CRUSH map, modifying the values,
+and reinjecting it into the cluster.
+
+* Extract the latest CRUSH map:
+
+  .. prompt:: bash $
+
+	ceph osd getcrushmap -o /tmp/crush
+
+* Adjust tunables.  These values appear to offer the best behavior
+  for both large and small clusters we tested with.  You will need to
+  additionally specify the ``--enable-unsafe-tunables`` argument to
+  ``crushtool`` for this to work.  Please use this option with
+  extreme care.:
+
+  .. prompt:: bash $
+
+     crushtool -i /tmp/crush --set-choose-local-tries 0 --set-choose-local-fallback-tries 0 --set-choose-total-tries 50 -o /tmp/crush.new
+
+* Reinject modified map:
+
+  .. prompt:: bash $
+
+     ceph osd setcrushmap -i /tmp/crush.new
+
+Legacy values
+-------------
+
+For reference, the legacy values for the CRUSH tunables can be set
+with:
+
+.. prompt:: bash $
+
+   crushtool -i /tmp/crush --set-choose-local-tries 2 --set-choose-local-fallback-tries 5 --set-choose-total-tries 19 --set-chooseleaf-descend-once 0 --set-chooseleaf-vary-r 0 -o /tmp/crush.legacy
+
+Again, the special ``--enable-unsafe-tunables`` option is required.
+Further, as noted above, be careful running old versions of the
+``ceph-osd`` daemon after reverting to legacy values as the feature
+bit is not perfectly enforced.
+
+.. _CRUSH - Controlled, Scalable, Decentralized Placement of Replicated Data: https://ceph.io/assets/pdfs/weil-crush-sc06.pdf
diff --git a/doc/rados/operations/crush-map.rst b/doc/rados/operations/crush-map.rst
new file mode 100644
index 000000000..f22ebb24e
--- /dev/null
+++ b/doc/rados/operations/crush-map.rst
@@ -0,0 +1,1126 @@
+============
+ CRUSH Maps
+============
+
+The :abbr:`CRUSH (Controlled Replication Under Scalable Hashing)` algorithm
+determines how to store and retrieve data by computing storage locations.
+CRUSH empowers Ceph clients to communicate with OSDs directly rather than
+through a centralized server or broker. With an algorithmically determined
+method of storing and retrieving data, Ceph avoids a single point of failure, a
+performance bottleneck, and a physical limit to its scalability.
+
+CRUSH uses a map of your cluster (the CRUSH map) to pseudo-randomly
+map data to OSDs, distributing it across the cluster according to configured
+replication policy and failure domain.  For a detailed discussion of CRUSH, see
+`CRUSH - Controlled, Scalable, Decentralized Placement of Replicated Data`_
+
+CRUSH maps contain a list of :abbr:`OSDs (Object Storage Devices)`, a hierarchy
+of 'buckets' for aggregating devices and buckets, and
+rules that govern how CRUSH replicates data within the cluster's pools. By
+reflecting the underlying physical organization of the installation, CRUSH can
+model (and thereby address) the potential for correlated device failures.
+Typical factors include chassis, racks, physical proximity, a shared power
+source, and shared networking. By encoding this information into the cluster
+map, CRUSH placement
+policies distribute object replicas across failure domains while
+maintaining the desired distribution. For example, to address the
+possibility of concurrent failures, it may be desirable to ensure that data
+replicas are on devices using different shelves, racks, power supplies,
+controllers, and/or physical locations.
+
+When you deploy OSDs they are automatically added to the CRUSH map under a
+``host`` bucket named for the node on which they run.  This,
+combined with the configured CRUSH failure domain, ensures that replicas or
+erasure code shards are distributed across hosts and that a single host or other
+failure will not affect availability.  For larger clusters, administrators must
+carefully consider their choice of failure domain.  Separating replicas across racks,
+for example, is typical for mid- to large-sized clusters.
+
+
+CRUSH Location
+==============
+
+The location of an OSD within the CRUSH map's hierarchy is
+referred to as a ``CRUSH location``.  This location specifier takes the
+form of a list of key and value pairs.  For
+example, if an OSD is in a particular row, rack, chassis and host, and
+is part of the 'default' CRUSH root (which is the case for most
+clusters), its CRUSH location could be described as::
+
+  root=default row=a rack=a2 chassis=a2a host=a2a1
+
+Note:
+
+#. Note that the order of the keys does not matter.
+#. The key name (left of ``=``) must be a valid CRUSH ``type``.  By default
+   these include ``root``, ``datacenter``, ``room``, ``row``, ``pod``, ``pdu``,
+   ``rack``, ``chassis`` and ``host``.
+   These defined types suffice for almost all clusters, but can be customized
+   by modifying the CRUSH map.
+#. Not all keys need to be specified.  For example, by default, Ceph
+   automatically sets an ``OSD``'s location to be
+   ``root=default host=HOSTNAME`` (based on the output from ``hostname -s``).
+
+The CRUSH location for an OSD can be defined by adding the ``crush location``
+option in ``ceph.conf``.  Each time the OSD starts,
+it verifies it is in the correct location in the CRUSH map and, if it is not,
+it moves itself.  To disable this automatic CRUSH map management, add the
+following to your configuration file in the ``[osd]`` section::
+
+  osd crush update on start = false
+
+Note that in most cases you will not need to manually configure this.
+
+
+Custom location hooks
+---------------------
+
+A customized location hook can be used to generate a more complete
+CRUSH location on startup.  The CRUSH location is based on, in order
+of preference:
+
+#. A ``crush location`` option in ``ceph.conf``
+#. A default of ``root=default host=HOSTNAME`` where the hostname is
+   derived from the ``hostname -s`` command
+
+A script can be written to provide additional
+location fields (for example, ``rack`` or ``datacenter``) and the
+hook enabled via the config option::
+
+ crush location hook = /path/to/customized-ceph-crush-location
+
+This hook is passed several arguments (below) and should output a single line
+to ``stdout`` with the CRUSH location description.::
+
+  --cluster CLUSTER --id ID --type TYPE
+
+where the cluster name is typically ``ceph``, the ``id`` is the daemon
+identifier (e.g., the OSD number or daemon identifier), and the daemon
+type is ``osd``, ``mds``, etc.
+
+For example, a simple hook that additionally specifies a rack location
+based on a value in the file ``/etc/rack`` might be::
+
+  #!/bin/sh
+  echo "host=$(hostname -s) rack=$(cat /etc/rack) root=default"
+
+
+CRUSH structure
+===============
+
+The CRUSH map consists of a hierarchy that describes
+the physical topology of the cluster and a set of rules defining
+data placement policy.  The hierarchy has
+devices (OSDs) at the leaves, and internal nodes
+corresponding to other physical features or groupings: hosts, racks,
+rows, datacenters, and so on.  The rules describe how replicas are
+placed in terms of that hierarchy (e.g., 'three replicas in different
+racks').
+
+Devices
+-------
+
+Devices are individual OSDs that store data, usually one for each storage drive.
+Devices are identified by an ``id``
+(a non-negative integer) and a ``name``, normally ``osd.N`` where ``N`` is the device id.
+
+Since the Luminous release, devices may also have a *device class* assigned (e.g.,
+``hdd`` or ``ssd`` or ``nvme``), allowing them to be conveniently targeted by
+CRUSH rules.  This is especially useful when mixing device types within hosts.
+
+.. _crush_map_default_types:
+
+Types and Buckets
+-----------------
+
+A bucket is the CRUSH term for internal nodes in the hierarchy: hosts,
+racks, rows, etc.  The CRUSH map defines a series of *types* that are
+used to describe these nodes.  Default types include:
+
+- ``osd`` (or ``device``)
+- ``host``
+- ``chassis``
+- ``rack``
+- ``row``
+- ``pdu``
+- ``pod``
+- ``room``
+- ``datacenter``
+- ``zone``
+- ``region``
+- ``root``
+
+Most clusters use only a handful of these types, and others
+can be defined as needed.
+
+The hierarchy is built with devices (normally type ``osd``) at the
+leaves, interior nodes with non-device types, and a root node of type
+``root``.  For example,
+
+.. ditaa::
+
+                        +-----------------+
+                        |{o}root default  |
+                        +--------+--------+
+                                 |
+                 +---------------+---------------+
+                 |                               |
+          +------+------+                 +------+------+
+          |{o}host foo  |                 |{o}host bar  |
+          +------+------+                 +------+------+
+                 |                               |
+         +-------+-------+               +-------+-------+
+         |               |               |               |
+   +-----+-----+   +-----+-----+   +-----+-----+   +-----+-----+
+   |   osd.0   |   |   osd.1   |   |   osd.2   |   |   osd.3   |
+   +-----------+   +-----------+   +-----------+   +-----------+
+
+Each node (device or bucket) in the hierarchy has a *weight*
+that indicates the relative proportion of the total
+data that device or hierarchy subtree should store.  Weights are set
+at the leaves, indicating the size of the device, and automatically
+sum up the tree, such that the weight of the ``root`` node
+will be the total of all devices contained beneath it.  Normally
+weights are in units of terabytes (TB).
+
+You can get a simple view the of CRUSH hierarchy for your cluster,
+including weights, with:
+
+.. prompt:: bash $
+
+   ceph osd tree
+
+Rules
+-----
+
+CRUSH Rules define policy about how data is distributed across the devices
+in the hierarchy. They define placement and replication strategies or
+distribution policies that allow you to specify exactly how CRUSH
+places data replicas. For example, you might create a rule selecting
+a pair of targets for two-way mirroring, another rule for selecting
+three targets in two different data centers for three-way mirroring, and
+yet another rule for erasure coding (EC) across six storage devices. For a
+detailed discussion of CRUSH rules, refer to `CRUSH - Controlled,
+Scalable, Decentralized Placement of Replicated Data`_, and more
+specifically to **Section 3.2**.
+
+CRUSH rules can be created via the CLI by
+specifying the *pool type* they will be used for (replicated or
+erasure coded), the *failure domain*, and optionally a *device class*.
+In rare cases rules must be written by hand by manually editing the
+CRUSH map.
+
+You can see what rules are defined for your cluster with:
+
+.. prompt:: bash $
+
+   ceph osd crush rule ls
+
+You can view the contents of the rules with:
+
+.. prompt:: bash $
+
+   ceph osd crush rule dump
+
+Device classes
+--------------
+
+Each device can optionally have a *class* assigned.  By
+default, OSDs automatically set their class at startup to
+`hdd`, `ssd`, or `nvme` based on the type of device they are backed
+by.
+
+The device class for one or more OSDs can be explicitly set with:
+
+.. prompt:: bash $
+
+   ceph osd crush set-device-class <class> <osd-name> [...]
+
+Once a device class is set, it cannot be changed to another class
+until the old class is unset with:
+
+.. prompt:: bash $
+
+   ceph osd crush rm-device-class <osd-name> [...]
+
+This allows administrators to set device classes without the class
+being changed on OSD restart or by some other script.
+
+A placement rule that targets a specific device class can be created with:
+
+.. prompt:: bash $
+
+   ceph osd crush rule create-replicated <rule-name> <root> <failure-domain> <class>
+
+A pool can then be changed to use the new rule with:
+
+.. prompt:: bash $
+
+   ceph osd pool set <pool-name> crush_rule <rule-name>
+
+Device classes are implemented by creating a "shadow" CRUSH hierarchy
+for each device class in use that contains only devices of that class.
+CRUSH rules can then distribute data over the shadow hierarchy.
+This approach is fully backward compatible with
+old Ceph clients.  You can view the CRUSH hierarchy with shadow items
+with:
+
+.. prompt:: bash $
+
+   ceph osd crush tree --show-shadow
+
+For older clusters created before Luminous that relied on manually
+crafted CRUSH maps to maintain per-device-type hierarchies, there is a
+*reclassify* tool available to help transition to device classes
+without triggering data movement (see :ref:`crush-reclassify`).
+
+
+Weights sets
+------------
+
+A *weight set* is an alternative set of weights to use when
+calculating data placement.  The normal weights associated with each
+device in the CRUSH map are set based on the device size and indicate
+how much data we *should* be storing where.  However, because CRUSH is
+a "probabilistic" pseudorandom placement process, there is always some
+variation from this ideal distribution, in the same way that rolling a
+die sixty times will not result in rolling exactly 10 ones and 10
+sixes.  Weight sets allow the cluster to perform numerical optimization
+based on the specifics of your cluster (hierarchy, pools, etc.) to achieve
+a balanced distribution.
+
+There are two types of weight sets supported:
+
+ #. A **compat** weight set is a single alternative set of weights for
+    each device and node in the cluster.  This is not well-suited for
+    correcting for all anomalies (for example, placement groups for
+    different pools may be different sizes and have different load
+    levels, but will be mostly treated the same by the balancer).
+    However, compat weight sets have the huge advantage that they are
+    *backward compatible* with previous versions of Ceph, which means
+    that even though weight sets were first introduced in Luminous
+    v12.2.z, older clients (e.g., firefly) can still connect to the
+    cluster when a compat weight set is being used to balance data.
+ #. A **per-pool** weight set is more flexible in that it allows
+    placement to be optimized for each data pool.  Additionally,
+    weights can be adjusted for each position of placement, allowing
+    the optimizer to correct for a subtle skew of data toward devices
+    with small weights relative to their peers (and effect that is
+    usually only apparently in very large clusters but which can cause
+    balancing problems).
+
+When weight sets are in use, the weights associated with each node in
+the hierarchy is visible as a separate column (labeled either
+``(compat)`` or the pool name) from the command:
+
+.. prompt:: bash $
+
+   ceph osd tree
+
+When both *compat* and *per-pool* weight sets are in use, data
+placement for a particular pool will use its own per-pool weight set
+if present.  If not, it will use the compat weight set if present.  If
+neither are present, it will use the normal CRUSH weights.
+
+Although weight sets can be set up and manipulated by hand, it is
+recommended that the ``ceph-mgr`` *balancer* module be enabled to do so
+automatically when running Luminous or later releases.
+
+
+Modifying the CRUSH map
+=======================
+
+.. _addosd:
+
+Add/Move an OSD
+---------------
+
+.. note: OSDs are normally automatically added to the CRUSH map when
+         the OSD is created.  This command is rarely needed.
+
+To add or move an OSD in the CRUSH map of a running cluster:
+
+.. prompt:: bash $
+
+   ceph osd crush set {name} {weight} root={root} [{bucket-type}={bucket-name} ...]
+
+Where:
+
+``name``
+
+:Description: The full name of the OSD.
+:Type: String
+:Required: Yes
+:Example: ``osd.0``
+
+
+``weight``
+
+:Description: The CRUSH weight for the OSD, normally its size measure in terabytes (TB).
+:Type: Double
+:Required: Yes
+:Example: ``2.0``
+
+
+``root``
+
+:Description: The root node of the tree in which the OSD resides (normally ``default``)
+:Type: Key/value pair.
+:Required: Yes
+:Example: ``root=default``
+
+
+``bucket-type``
+
+:Description: You may specify the OSD's location in the CRUSH hierarchy.
+:Type: Key/value pairs.
+:Required: No
+:Example: ``datacenter=dc1 room=room1 row=foo rack=bar host=foo-bar-1``
+
+
+The following example adds ``osd.0`` to the hierarchy, or moves the
+OSD from a previous location:
+
+.. prompt:: bash $
+
+   ceph osd crush set osd.0 1.0 root=default datacenter=dc1 room=room1 row=foo rack=bar host=foo-bar-1
+
+
+Adjust OSD weight
+-----------------
+
+.. note: Normally OSDs automatically add themselves to the CRUSH map
+         with the correct weight when they are created. This command
+         is rarely needed.
+
+To adjust an OSD's CRUSH weight in the CRUSH map of a running cluster, execute
+the following:
+
+.. prompt:: bash $
+
+   ceph osd crush reweight {name} {weight}
+
+Where:
+
+``name``
+
+:Description: The full name of the OSD.
+:Type: String
+:Required: Yes
+:Example: ``osd.0``
+
+
+``weight``
+
+:Description: The CRUSH weight for the OSD.
+:Type: Double
+:Required: Yes
+:Example: ``2.0``
+
+
+.. _removeosd:
+
+Remove an OSD
+-------------
+
+.. note: OSDs are normally removed from the CRUSH as part of the
+   ``ceph osd purge`` command.  This command is rarely needed.
+
+To remove an OSD from the CRUSH map of a running cluster, execute the
+following:
+
+.. prompt:: bash $
+
+   ceph osd crush remove {name}
+
+Where:
+
+``name``
+
+:Description: The full name of the OSD.
+:Type: String
+:Required: Yes
+:Example: ``osd.0``
+
+
+Add a Bucket
+------------
+
+.. note: Buckets are implicitly created when an OSD is added
+   that specifies a ``{bucket-type}={bucket-name}`` as part of its
+   location,  if a bucket with that name does not already exist.  This
+   command is typically used when manually adjusting the structure of the
+   hierarchy after OSDs have been created.  One use is to move a
+   series of hosts underneath a new rack-level bucket; another is to
+   add new ``host`` buckets (OSD nodes) to a dummy ``root`` so that they don't
+   receive data until you're ready, at which time you would move them to the
+   ``default`` or other root as described below.
+
+To add a bucket in the CRUSH map of a running cluster, execute the
+``ceph osd crush add-bucket`` command:
+
+.. prompt:: bash $
+
+   ceph osd crush add-bucket {bucket-name} {bucket-type}
+
+Where:
+
+``bucket-name``
+
+:Description: The full name of the bucket.
+:Type: String
+:Required: Yes
+:Example: ``rack12``
+
+
+``bucket-type``
+
+:Description: The type of the bucket. The type must already exist in the hierarchy.
+:Type: String
+:Required: Yes
+:Example: ``rack``
+
+
+The following example adds the ``rack12`` bucket to the hierarchy:
+
+.. prompt:: bash $
+
+   ceph osd crush add-bucket rack12 rack
+
+Move a Bucket
+-------------
+
+To move a bucket to a different location or position in the CRUSH map
+hierarchy, execute the following:
+
+.. prompt:: bash $
+
+   ceph osd crush move {bucket-name} {bucket-type}={bucket-name}, [...]
+
+Where:
+
+``bucket-name``
+
+:Description: The name of the bucket to move/reposition.
+:Type: String
+:Required: Yes
+:Example: ``foo-bar-1``
+
+``bucket-type``
+
+:Description: You may specify the bucket's location in the CRUSH hierarchy.
+:Type: Key/value pairs.
+:Required: No
+:Example: ``datacenter=dc1 room=room1 row=foo rack=bar host=foo-bar-1``
+
+Remove a Bucket
+---------------
+
+To remove a bucket from the CRUSH hierarchy, execute the following:
+
+.. prompt:: bash $
+
+   ceph osd crush remove {bucket-name}
+
+.. note:: A bucket must be empty before removing it from the CRUSH hierarchy.
+
+Where:
+
+``bucket-name``
+
+:Description: The name of the bucket that you'd like to remove.
+:Type: String
+:Required: Yes
+:Example: ``rack12``
+
+The following example removes the ``rack12`` bucket from the hierarchy:
+
+.. prompt:: bash $
+
+   ceph osd crush remove rack12
+
+Creating a compat weight set
+----------------------------
+
+.. note: This step is normally done automatically by the ``balancer``
+   module when enabled.
+
+To create a *compat* weight set:
+
+.. prompt:: bash $
+
+   ceph osd crush weight-set create-compat
+
+Weights for the compat weight set can be adjusted with:
+
+.. prompt:: bash $
+
+   ceph osd crush weight-set reweight-compat {name} {weight}
+
+The compat weight set can be destroyed with:
+
+.. prompt:: bash $
+
+   ceph osd crush weight-set rm-compat
+
+Creating per-pool weight sets
+-----------------------------
+
+To create a weight set for a specific pool:
+
+.. prompt:: bash $
+
+   ceph osd crush weight-set create {pool-name} {mode}
+
+.. note:: Per-pool weight sets require that all servers and daemons
+          run Luminous v12.2.z or later.
+
+Where:
+
+``pool-name``
+
+:Description: The name of a RADOS pool
+:Type: String
+:Required: Yes
+:Example: ``rbd``
+
+``mode``
+
+:Description: Either ``flat`` or ``positional``.  A *flat* weight set
+	      has a single weight for each device or bucket.  A
+	      *positional* weight set has a potentially different
+	      weight for each position in the resulting placement
+	      mapping.  For example, if a pool has a replica count of
+	      3, then a positional weight set will have three weights
+	      for each device and bucket.
+:Type: String
+:Required: Yes
+:Example: ``flat``
+
+To adjust the weight of an item in a weight set:
+
+.. prompt:: bash $
+
+   ceph osd crush weight-set reweight {pool-name} {item-name} {weight [...]}
+
+To list existing weight sets:
+
+.. prompt:: bash $
+
+   ceph osd crush weight-set ls
+
+To remove a weight set:
+
+.. prompt:: bash $
+
+   ceph osd crush weight-set rm {pool-name}
+
+Creating a rule for a replicated pool
+-------------------------------------
+
+For a replicated pool, the primary decision when creating the CRUSH
+rule is what the failure domain is going to be.  For example, if a
+failure domain of ``host`` is selected, then CRUSH will ensure that
+each replica of the data is stored on a unique host.  If ``rack``
+is selected, then each replica will be stored in a different rack.
+What failure domain you choose primarily depends on the size and
+topology of your cluster.
+
+In most cases the entire cluster hierarchy is nested beneath a root node
+named ``default``.  If you have customized your hierarchy, you may
+want to create a rule nested at some other node in the hierarchy.  It
+doesn't matter what type is associated with that node (it doesn't have
+to be a ``root`` node).
+
+It is also possible to create a rule that restricts data placement to
+a specific *class* of device.  By default, Ceph OSDs automatically
+classify themselves as either ``hdd`` or ``ssd``, depending on the
+underlying type of device being used.  These classes can also be
+customized.
+
+To create a replicated rule:
+
+.. prompt:: bash $
+
+   ceph osd crush rule create-replicated {name} {root} {failure-domain-type} [{class}]
+
+Where:
+
+``name``
+
+:Description: The name of the rule
+:Type: String
+:Required: Yes
+:Example: ``rbd-rule``
+
+``root``
+
+:Description: The name of the node under which data should be placed.
+:Type: String
+:Required: Yes
+:Example: ``default``
+
+``failure-domain-type``
+
+:Description: The type of CRUSH nodes across which we should separate replicas.
+:Type: String
+:Required: Yes
+:Example: ``rack``
+
+``class``
+
+:Description: The device class on which data should be placed.
+:Type: String
+:Required: No
+:Example: ``ssd``
+
+Creating a rule for an erasure coded pool
+-----------------------------------------
+
+For an erasure-coded (EC) pool, the same basic decisions need to be made:
+what is the failure domain, which node in the
+hierarchy will data be placed under (usually ``default``), and will
+placement be restricted to a specific device class.  Erasure code
+pools are created a bit differently, however, because they need to be
+constructed carefully based on the erasure code being used.  For this reason,
+you must include this information in the *erasure code profile*.  A CRUSH
+rule will then be created from that either explicitly or automatically when
+the profile is used to create a pool.
+
+The erasure code profiles can be listed with:
+
+.. prompt:: bash $
+
+   ceph osd erasure-code-profile ls
+
+An existing profile can be viewed with:
+
+.. prompt:: bash $
+
+   ceph osd erasure-code-profile get {profile-name}
+
+Normally profiles should never be modified; instead, a new profile
+should be created and used when creating a new pool or creating a new
+rule for an existing pool.
+
+An erasure code profile consists of a set of key=value pairs.  Most of
+these control the behavior of the erasure code that is encoding data
+in the pool.  Those that begin with ``crush-``, however, affect the
+CRUSH rule that is created.
+
+The erasure code profile properties of interest are:
+
+ * **crush-root**: the name of the CRUSH node under which to place data [default: ``default``].
+ * **crush-failure-domain**: the CRUSH bucket type across which to distribute erasure-coded shards [default: ``host``].
+ * **crush-device-class**: the device class on which to place data [default: none, meaning all devices are used].
+ * **k** and **m** (and, for the ``lrc`` plugin, **l**): these determine the number of erasure code shards, affecting the resulting CRUSH rule.
+
+Once a profile is defined, you can create a CRUSH rule with:
+
+.. prompt:: bash $
+
+   ceph osd crush rule create-erasure {name} {profile-name}
+
+.. note: When creating a new pool, it is not actually necessary to
+   explicitly create the rule.  If the erasure code profile alone is
+   specified and the rule argument is left off then Ceph will create
+   the CRUSH rule automatically.
+
+Deleting rules
+--------------
+
+Rules that are not in use by pools can be deleted with:
+
+.. prompt:: bash $
+
+   ceph osd crush rule rm {rule-name}
+
+
+.. _crush-map-tunables:
+
+Tunables
+========
+
+Over time, we have made (and continue to make) improvements to the
+CRUSH algorithm used to calculate the placement of data.  In order to
+support the change in behavior, we have introduced a series of tunable
+options that control whether the legacy or improved variation of the
+algorithm is used.
+
+In order to use newer tunables, both clients and servers must support
+the new version of CRUSH.  For this reason, we have created
+``profiles`` that are named after the Ceph version in which they were
+introduced.  For example, the ``firefly`` tunables are first supported
+by the Firefly release, and will not work with older (e.g., Dumpling)
+clients.  Once a given set of tunables are changed from the legacy
+default behavior, the ``ceph-mon`` and ``ceph-osd`` will prevent older
+clients who do not support the new CRUSH features from connecting to
+the cluster.
+
+argonaut (legacy)
+-----------------
+
+The legacy CRUSH behavior used by Argonaut and older releases works
+fine for most clusters, provided there are not many OSDs that have
+been marked out.
+
+bobtail (CRUSH_TUNABLES2)
+-------------------------
+
+The ``bobtail`` tunable profile fixes a few key misbehaviors:
+
+ * For hierarchies with a small number of devices in the leaf buckets,
+   some PGs map to fewer than the desired number of replicas.  This
+   commonly happens for hierarchies with "host" nodes with a small
+   number (1-3) of OSDs nested beneath each one.
+
+ * For large clusters, some small percentages of PGs map to fewer than
+   the desired number of OSDs.  This is more prevalent when there are
+   mutiple hierarchy layers in use (e.g., ``row``, ``rack``, ``host``, ``osd``).
+
+ * When some OSDs are marked out, the data tends to get redistributed
+   to nearby OSDs instead of across the entire hierarchy.
+
+The new tunables are:
+
+ * ``choose_local_tries``: Number of local retries.  Legacy value is
+   2, optimal value is 0.
+
+ * ``choose_local_fallback_tries``: Legacy value is 5, optimal value
+   is 0.
+
+ * ``choose_total_tries``: Total number of attempts to choose an item.
+   Legacy value was 19, subsequent testing indicates that a value of
+   50 is more appropriate for typical clusters.  For extremely large
+   clusters, a larger value might be necessary.
+
+ * ``chooseleaf_descend_once``: Whether a recursive chooseleaf attempt
+   will retry, or only try once and allow the original placement to
+   retry.  Legacy default is 0, optimal value is 1.
+
+Migration impact:
+
+ * Moving from ``argonaut`` to ``bobtail`` tunables triggers a moderate amount
+   of data movement.  Use caution on a cluster that is already
+   populated with data.
+
+firefly (CRUSH_TUNABLES3)
+-------------------------
+
+The ``firefly`` tunable profile fixes a problem
+with ``chooseleaf`` CRUSH rule behavior that tends to result in PG
+mappings with too few results when too many OSDs have been marked out.
+
+The new tunable is:
+
+ * ``chooseleaf_vary_r``: Whether a recursive chooseleaf attempt will
+   start with a non-zero value of ``r``, based on how many attempts the
+   parent has already made.  Legacy default is ``0``, but with this value
+   CRUSH is sometimes unable to find a mapping.  The optimal value (in
+   terms of computational cost and correctness) is ``1``.
+
+Migration impact:
+
+ * For existing clusters that house lots of data, changing
+   from ``0`` to ``1`` will cause a lot of data to move; a value of ``4`` or ``5``
+   will allow CRUSH to still find a valid mapping but will cause less data
+   to move.
+
+straw_calc_version tunable (introduced with Firefly too)
+--------------------------------------------------------
+
+There were some problems with the internal weights calculated and
+stored in the CRUSH map for ``straw`` algorithm buckets.  Specifically, when
+there were items with a CRUSH weight of ``0``, or both a mix of different and
+unique weights, CRUSH would distribute data incorrectly (i.e.,
+not in proportion to the weights).
+
+The new tunable is:
+
+ * ``straw_calc_version``: A value of ``0`` preserves the old, broken
+   internal weight calculation; a value of ``1`` fixes the behavior.
+
+Migration impact:
+
+ * Moving to straw_calc_version ``1`` and then adjusting a straw bucket
+   (by adding, removing, or reweighting an item, or by using the
+   reweight-all command) can trigger a small to moderate amount of
+   data movement *if* the cluster has hit one of the problematic
+   conditions.
+
+This tunable option is special because it has absolutely no impact
+concerning the required kernel version in the client side.
+
+hammer (CRUSH_V4)
+-----------------
+
+The ``hammer`` tunable profile does not affect the
+mapping of existing CRUSH maps simply by changing the profile.  However:
+
+ * There is a new bucket algorithm (``straw2``) supported.  The new
+   ``straw2`` bucket algorithm fixes several limitations in the original
+   ``straw``.  Specifically, the old ``straw`` buckets would
+   change some mappings that should have changed when a weight was
+   adjusted, while ``straw2`` achieves the original goal of only
+   changing mappings to or from the bucket item whose weight has
+   changed.
+
+ * ``straw2`` is the default for any newly created buckets.
+
+Migration impact:
+
+ * Changing a bucket type from ``straw`` to ``straw2`` will result in
+   a reasonably small amount of data movement, depending on how much
+   the bucket item weights vary from each other.  When the weights are
+   all the same no data will move, and when item weights vary
+   significantly there will be more movement.
+
+jewel (CRUSH_TUNABLES5)
+-----------------------
+
+The ``jewel`` tunable profile improves the
+overall behavior of CRUSH such that significantly fewer mappings
+change when an OSD is marked out of the cluster.  This results in
+significantly less data movement.
+
+The new tunable is:
+
+ * ``chooseleaf_stable``: Whether a recursive chooseleaf attempt will
+   use a better value for an inner loop that greatly reduces the number
+   of mapping changes when an OSD is marked out.  The legacy value is ``0``,
+   while the new value of ``1`` uses the new approach.
+
+Migration impact:
+
+ * Changing this value on an existing cluster will result in a very
+   large amount of data movement as almost every PG mapping is likely
+   to change.
+
+
+
+
+Which client versions support CRUSH_TUNABLES
+--------------------------------------------
+
+ * argonaut series, v0.48.1 or later
+ * v0.49 or later
+ * Linux kernel version v3.6 or later (for the file system and RBD kernel clients)
+
+Which client versions support CRUSH_TUNABLES2
+---------------------------------------------
+
+ * v0.55 or later, including bobtail series (v0.56.x)
+ * Linux kernel version v3.9 or later (for the file system and RBD kernel clients)
+
+Which client versions support CRUSH_TUNABLES3
+---------------------------------------------
+
+ * v0.78 (firefly) or later
+ * Linux kernel version v3.15 or later (for the file system and RBD kernel clients)
+
+Which client versions support CRUSH_V4
+--------------------------------------
+
+ * v0.94 (hammer) or later
+ * Linux kernel version v4.1 or later (for the file system and RBD kernel clients)
+
+Which client versions support CRUSH_TUNABLES5
+---------------------------------------------
+
+ * v10.0.2 (jewel) or later
+ * Linux kernel version v4.5 or later (for the file system and RBD kernel clients)
+
+Warning when tunables are non-optimal
+-------------------------------------
+
+Starting with version v0.74, Ceph will issue a health warning if the
+current CRUSH tunables don't include all the optimal values from the
+``default`` profile (see below for the meaning of the ``default`` profile).
+To make this warning go away, you have two options:
+
+1. Adjust the tunables on the existing cluster.  Note that this will
+   result in some data movement (possibly as much as 10%).  This is the
+   preferred route, but should be taken with care on a production cluster
+   where the data movement may affect performance.  You can enable optimal
+   tunables with:
+
+   .. prompt:: bash $
+
+      ceph osd crush tunables optimal
+
+   If things go poorly (e.g., too much load) and not very much
+   progress has been made, or there is a client compatibility problem
+   (old kernel CephFS or RBD clients, or pre-Bobtail ``librados``
+   clients), you can switch back with:
+
+   .. prompt:: bash $
+
+      ceph osd crush tunables legacy
+
+2. You can make the warning go away without making any changes to CRUSH by
+   adding the following option to your ceph.conf ``[mon]`` section::
+
+      mon warn on legacy crush tunables = false
+
+   For the change to take effect, you will need to restart the monitors, or
+   apply the option to running monitors with:
+
+   .. prompt:: bash $
+
+      ceph tell mon.\* config set mon_warn_on_legacy_crush_tunables false
+
+
+A few important points
+----------------------
+
+ * Adjusting these values will result in the shift of some PGs between
+   storage nodes.  If the Ceph cluster is already storing a lot of
+   data, be prepared for some fraction of the data to move.
+ * The ``ceph-osd`` and ``ceph-mon`` daemons will start requiring the
+   feature bits of new connections as soon as they get
+   the updated map.  However, already-connected clients are
+   effectively grandfathered in, and will misbehave if they do not
+   support the new feature.
+ * If the CRUSH tunables are set to non-legacy values and then later
+   changed back to the default values, ``ceph-osd`` daemons will not be
+   required to support the feature.  However, the OSD peering process
+   requires examining and understanding old maps.  Therefore, you
+   should not run old versions of the ``ceph-osd`` daemon
+   if the cluster has previously used non-legacy CRUSH values, even if
+   the latest version of the map has been switched back to using the
+   legacy defaults.
+
+Tuning CRUSH
+------------
+
+The simplest way to adjust CRUSH tunables is by applying them in matched
+sets known as *profiles*.  As of the Octopus release these are:
+
+ * ``legacy``: the legacy behavior from argonaut and earlier.
+ * ``argonaut``: the legacy values supported by the original argonaut release
+ * ``bobtail``: the values supported by the bobtail release
+ * ``firefly``: the values supported by the firefly release
+ * ``hammer``: the values supported by the hammer release
+ * ``jewel``: the values supported by the jewel release
+ * ``optimal``: the best (i.e. optimal) values of the current version of Ceph
+ * ``default``: the default values of a new cluster installed from
+   scratch. These values, which depend on the current version of Ceph,
+   are hardcoded and are generally a mix of optimal and legacy values.
+   These values generally match the ``optimal`` profile of the previous
+   LTS release, or the most recent release for which we generally expect
+   most users to have up-to-date clients for.
+
+You can apply a profile to a running cluster with the command:
+
+.. prompt:: bash $
+
+   ceph osd crush tunables {PROFILE}
+
+Note that this may result in data movement, potentially quite a bit.  Study
+release notes and documentation carefully before changing the profile on a
+running cluster, and consider throttling recovery/backfill parameters to
+limit the impact of a bolus of backfill.
+
+.. _CRUSH - Controlled, Scalable, Decentralized Placement of Replicated Data: https://ceph.io/assets/pdfs/weil-crush-sc06.pdf
+
+
+Primary Affinity
+================
+
+When a Ceph Client reads or writes data, it first contacts the primary OSD in
+each affected PG's acting set. By default, the first OSD in the acting set is
+the primary.  For example, in the acting set ``[2, 3, 4]``, ``osd.2`` is
+listed first and thus is the primary (aka lead) OSD. Sometimes we know that an
+OSD is less well suited to act as the lead than are other OSDs (e.g., it has
+a slow drive or a slow controller). To prevent performance bottlenecks
+(especially on read operations) while maximizing utilization of your hardware,
+you can influence the selection of primary OSDs by adjusting primary affinity
+values, or by crafting a CRUSH rule that selects preferred OSDs first.
+
+Tuning primary OSD selection is mainly useful for replicated pools, because
+by default read operations are served from the primary OSD for each PG.
+For erasure coded (EC) pools, a way to speed up read operations is to enable
+**fast read** as described in :ref:`pool-settings`.
+
+A common scenario for primary affinity is when a cluster contains
+a mix of drive sizes, for example older racks with 1.9 TB SATA SSDS and newer racks with
+3.84TB SATA SSDs.  On average the latter will be assigned double the number of
+PGs and thus will serve double the number of write and read operations, thus
+they'll be busier than the former.  A rough assignment of primary affinity
+inversely proportional to OSD size won't be 100% optimal, but it can readily
+achieve a 15% improvement in overall read throughput by utilizing SATA
+interface bandwidth and CPU cycles more evenly.
+
+By default, all ceph OSDs have primary affinity of ``1``, which indicates that
+any OSD may act as a primary with equal probability.
+
+You can reduce a Ceph OSD's primary affinity so that CRUSH is less likely to
+choose the OSD as primary in a PG's acting set.:
+
+.. prompt:: bash $
+
+   ceph osd primary-affinity <osd-id> <weight>
+
+You may set an OSD's primary affinity to a real number in the range ``[0-1]``,
+where ``0`` indicates that the OSD may **NOT** be used as a primary and ``1``
+indicates that an OSD may be used as a primary.  When the weight is between
+these extremes, it is less likely that CRUSH will select that OSD as a primary.
+The process for selecting the lead OSD is more nuanced than a simple
+probability based on relative affinity values, but measurable results can be
+achieved even with first-order approximations of desirable values.
+
+Custom CRUSH Rules
+------------------
+
+There are occasional clusters that balance cost and performance by mixing SSDs
+and HDDs in the same replicated pool. By setting the primary affinity of HDD
+OSDs to ``0`` one can direct operations to the SSD in each acting set. An
+alternative is to define a CRUSH rule that always selects an SSD OSD as the
+first OSD, then selects HDDs for the remaining OSDs. Thus, each PG's acting
+set will contain exactly one SSD OSD as the primary with the balance on HDDs.
+
+For example, the CRUSH rule below::
+
+	rule mixed_replicated_rule {
+	        id 11
+	        type replicated
+	        min_size 1
+	        max_size 10
+	        step take default class ssd
+	        step chooseleaf firstn 1 type host
+	        step emit
+	        step take default class hdd
+	        step chooseleaf firstn 0 type host
+	        step emit
+	}
+
+chooses an SSD as the first OSD.  Note that for an ``N``-times replicated pool
+this rule selects ``N+1`` OSDs to guarantee that ``N`` copies are on different
+hosts, because the first SSD OSD might be co-located with any of the ``N`` HDD
+OSDs.
+
+This extra storage requirement can be avoided by placing SSDs and HDDs in
+different hosts with the tradeoff that hosts with SSDs will receive all client
+requests.  You may thus consider faster CPU(s) for SSD hosts and more modest
+ones for HDD nodes, since the latter will normally only service recovery
+operations.  Here the CRUSH roots ``ssd_hosts`` and ``hdd_hosts`` strictly
+must not contain the same servers::
+
+        rule mixed_replicated_rule_two {
+               id 1
+               type replicated
+               min_size 1
+               max_size 10
+               step take ssd_hosts class ssd
+               step chooseleaf firstn 1 type host
+               step emit
+               step take hdd_hosts class hdd
+               step chooseleaf firstn -1 type host
+               step emit
+        }
+
+
+Note also that on failure of an SSD, requests to a PG will be served temporarily
+from a (slower) HDD OSD until the PG's data has been replicated onto the replacement
+primary SSD OSD.
+
diff --git a/doc/rados/operations/data-placement.rst b/doc/rados/operations/data-placement.rst
new file mode 100644
index 000000000..bd9bd7ec7
--- /dev/null
+++ b/doc/rados/operations/data-placement.rst
@@ -0,0 +1,43 @@
+=========================
+ Data Placement Overview
+=========================
+
+Ceph stores, replicates and rebalances data objects across a RADOS cluster
+dynamically.  With many different users storing objects in different pools for
+different purposes on countless OSDs, Ceph operations require some data
+placement planning.  The main data placement planning concepts in Ceph include:
+
+- **Pools:** Ceph stores data within pools, which are logical groups for storing
+  objects. Pools manage the number of placement groups, the number of replicas,
+  and the CRUSH rule for the pool. To store data in a pool, you must have
+  an authenticated user with permissions for the pool. Ceph can snapshot pools.
+  See `Pools`_ for additional details.
+
+- **Placement Groups:** Ceph maps objects to placement groups (PGs).
+  Placement groups (PGs) are shards or fragments of a logical object pool
+  that place objects as a group into OSDs. Placement groups reduce the amount
+  of per-object metadata when Ceph stores the data in OSDs. A larger number of
+  placement groups (e.g., 100 per OSD) leads to better balancing. See
+  `Placement Groups`_ for additional details.
+
+- **CRUSH Maps:**  CRUSH is a big part of what allows Ceph to scale without
+  performance bottlenecks, without limitations to scalability, and without a
+  single point of failure. CRUSH maps provide the physical topology of the
+  cluster to the CRUSH algorithm to determine where the data for an object
+  and its replicas should be stored, and how to do so across failure domains
+  for added data safety among other things. See `CRUSH Maps`_ for additional
+  details.
+
+- **Balancer:** The balancer is a feature that will automatically optimize the
+  distribution of PGs across devices to achieve a balanced data distribution,
+  maximizing the amount of data that can be stored in the cluster and evenly
+  distributing the workload across OSDs.
+
+When you initially set up a test cluster, you can use the default values. Once
+you begin planning for a large Ceph cluster, refer to pools, placement groups
+and CRUSH for data placement operations.
+
+.. _Pools: ../pools
+.. _Placement Groups: ../placement-groups
+.. _CRUSH Maps: ../crush-map
+.. _Balancer: ../balancer
diff --git a/doc/rados/operations/devices.rst b/doc/rados/operations/devices.rst
new file mode 100644
index 000000000..1b6eaebde
--- /dev/null
+++ b/doc/rados/operations/devices.rst
@@ -0,0 +1,208 @@
+.. _devices:
+
+Device Management
+=================
+
+Ceph tracks which hardware storage devices (e.g., HDDs, SSDs) are consumed by
+which daemons, and collects health metrics about those devices in order to
+provide tools to predict and/or automatically respond to hardware failure.
+
+Device tracking
+---------------
+
+You can query which storage devices are in use with:
+
+.. prompt:: bash $
+
+   ceph device ls
+
+You can also list devices by daemon or by host:
+
+.. prompt:: bash $
+
+   ceph device ls-by-daemon <daemon>
+   ceph device ls-by-host <host>
+
+For any individual device, you can query information about its
+location and how it is being consumed with:
+
+.. prompt:: bash $
+
+   ceph device info <devid>
+
+Identifying physical devices
+----------------------------
+
+You can blink the drive LEDs on hardware enclosures to make the replacement of
+failed disks easy and less error-prone.  Use the following command::
+
+  device light on|off <devid> [ident|fault] [--force]
+
+The ``<devid>`` parameter is the device identification. You can obtain this
+information using the following command:
+
+.. prompt:: bash $
+
+   ceph device ls
+
+The ``[ident|fault]`` parameter is used to set the kind of light to blink.
+By default, the `identification` light is used.
+
+.. note::
+   This command needs the Cephadm or the Rook `orchestrator <https://docs.ceph.com/docs/master/mgr/orchestrator/#orchestrator-cli-module>`_ module enabled.
+   The orchestrator module enabled is shown by executing the following command:
+
+   .. prompt:: bash $
+
+      ceph orch status
+
+The command behind the scene to blink the drive LEDs is `lsmcli`. If you need
+to customize this command you can configure this via a Jinja2 template::
+
+   ceph config-key set mgr/cephadm/blink_device_light_cmd "<template>"
+   ceph config-key set mgr/cephadm/<host>/blink_device_light_cmd "lsmcli local-disk-{{ ident_fault }}-led-{{'on' if on else 'off'}} --path '{{ path or dev }}'"
+
+The Jinja2 template is rendered using the following arguments:
+
+* ``on``
+    A boolean value.
+* ``ident_fault``
+    A string containing `ident` or `fault`.
+* ``dev``
+    A string containing the device ID, e.g. `SanDisk_X400_M.2_2280_512GB_162924424784`.
+* ``path``
+    A string containing the device path, e.g. `/dev/sda`.
+
+.. _enabling-monitoring:
+
+Enabling monitoring
+-------------------
+
+Ceph can also monitor health metrics associated with your device.  For
+example, SATA hard disks implement a standard called SMART that
+provides a wide range of internal metrics about the device's usage and
+health, like the number of hours powered on, number of power cycles,
+or unrecoverable read errors.  Other device types like SAS and NVMe
+implement a similar set of metrics (via slightly different standards).
+All of these can be collected by Ceph via the ``smartctl`` tool.
+
+You can enable or disable health monitoring with:
+
+.. prompt:: bash $
+
+   ceph device monitoring on
+
+or:
+
+.. prompt:: bash $
+
+   ceph device monitoring off
+
+
+Scraping
+--------
+
+If monitoring is enabled, metrics will automatically be scraped at regular intervals.  That interval can be configured with:
+
+.. prompt:: bash $
+
+   ceph config set mgr mgr/devicehealth/scrape_frequency <seconds>
+
+The default is to scrape once every 24 hours.
+
+You can manually trigger a scrape of all devices with:
+   
+.. prompt:: bash $
+
+   ceph device scrape-health-metrics
+
+A single device can be scraped with:
+
+.. prompt:: bash $
+
+   ceph device scrape-health-metrics <device-id>
+
+Or a single daemon's devices can be scraped with:
+
+.. prompt:: bash $
+
+   ceph device scrape-daemon-health-metrics <who>
+
+The stored health metrics for a device can be retrieved (optionally
+for a specific timestamp) with:
+
+.. prompt:: bash $
+
+   ceph device get-health-metrics <devid> [sample-timestamp]
+
+Failure prediction
+------------------
+
+Ceph can predict life expectancy and device failures based on the
+health metrics it collects.  There are three modes:
+
+* *none*: disable device failure prediction.
+* *local*: use a pre-trained prediction model from the ceph-mgr daemon
+
+The prediction mode can be configured with:
+
+.. prompt:: bash $
+
+   ceph config set global device_failure_prediction_mode <mode>
+
+Prediction normally runs in the background on a periodic basis, so it
+may take some time before life expectancy values are populated.  You
+can see the life expectancy of all devices in output from:
+
+.. prompt:: bash $
+
+   ceph device ls
+
+You can also query the metadata for a specific device with:
+
+.. prompt:: bash $
+
+   ceph device info <devid>
+
+You can explicitly force prediction of a device's life expectancy with:
+
+.. prompt:: bash $
+
+   ceph device predict-life-expectancy <devid>
+
+If you are not using Ceph's internal device failure prediction but
+have some external source of information about device failures, you
+can inform Ceph of a device's life expectancy with:
+
+.. prompt:: bash $
+
+   ceph device set-life-expectancy <devid> <from> [<to>]
+
+Life expectancies are expressed as a time interval so that
+uncertainty can be expressed in the form of a wide interval. The
+interval end can also be left unspecified.
+
+Health alerts
+-------------
+
+The ``mgr/devicehealth/warn_threshold`` controls how soon an expected
+device failure must be before we generate a health warning.
+
+The stored life expectancy of all devices can be checked, and any
+appropriate health alerts generated, with:
+
+.. prompt:: bash $
+
+   ceph device check-health
+
+Automatic Mitigation
+--------------------
+
+If the ``mgr/devicehealth/self_heal`` option is enabled (it is by
+default), then for devices that are expected to fail soon the module
+will automatically migrate data away from them by marking the devices
+"out".
+
+The ``mgr/devicehealth/mark_out_threshold`` controls how soon an
+expected device failure must be before we automatically mark an osd
+"out".
diff --git a/doc/rados/operations/erasure-code-clay.rst b/doc/rados/operations/erasure-code-clay.rst
new file mode 100644
index 000000000..1cffa32f5
--- /dev/null
+++ b/doc/rados/operations/erasure-code-clay.rst
@@ -0,0 +1,240 @@
+================
+CLAY code plugin
+================
+
+CLAY (short for coupled-layer) codes are erasure codes designed to bring about significant savings 
+in terms of network bandwidth and disk IO when a failed node/OSD/rack is being repaired. Let:
+
+	d = number of OSDs contacted during repair
+
+If *jerasure* is configured with *k=8* and *m=4*, losing one OSD requires 
+reading from the *d=8* others to repair. And recovery of say a 1GiB needs
+a download of 8 X 1GiB = 8GiB of information.
+
+However, in the case of the *clay* plugin *d* is configurable within the limits:
+
+	k+1 <= d <= k+m-1 
+
+By default, the clay code plugin picks *d=k+m-1* as it provides the greatest savings in terms 
+of network bandwidth and disk IO. In the case of the *clay* plugin configured with 
+*k=8*, *m=4* and *d=11* when a single OSD fails, d=11 osds are contacted and 
+250MiB is downloaded from each of them, resulting in a total download of 11 X 250MiB = 2.75GiB 
+amount of information. More general parameters are provided below. The benefits are substantial 
+when the repair is carried out for a rack that stores information on the order of 
+Terabytes.
+
+	+-------------+---------------------------------------------------------+
+	| plugin      | total amount of disk IO                                 |
+	+=============+=========================================================+
+	|jerasure,isa | :math:`k S`                                             |
+	+-------------+---------------------------------------------------------+
+	| clay        | :math:`\frac{d S}{d - k + 1} = \frac{(k + m - 1) S}{m}` |
+	+-------------+---------------------------------------------------------+
+
+where *S* is the amount of data stored on a single OSD undergoing repair. In the table above, we have 
+used the largest possible value of *d* as this will result in the smallest amount of data download needed
+to achieve recovery from an OSD failure.
+
+Erasure-code profile examples
+=============================
+
+An example configuration that can be used to observe reduced bandwidth usage:
+
+.. prompt:: bash $
+
+   ceph osd erasure-code-profile set CLAYprofile \
+      plugin=clay \
+      k=4 m=2 d=5 \
+      crush-failure-domain=host
+   ceph osd pool create claypool erasure CLAYprofile
+
+
+Creating a clay profile
+=======================
+
+To create a new clay code profile:
+
+.. prompt:: bash $
+
+   ceph osd erasure-code-profile set {name} \
+        plugin=clay \
+        k={data-chunks} \
+        m={coding-chunks} \
+        [d={helper-chunks}] \
+        [scalar_mds={plugin-name}] \
+        [technique={technique-name}] \
+        [crush-failure-domain={bucket-type}] \
+        [crush-device-class={device-class}] \
+        [directory={directory}] \
+        [--force]
+
+Where:
+
+``k={data chunks}``
+
+:Description: Each object is split into **data-chunks** parts,
+              each of which is stored on a different OSD.
+
+:Type: Integer
+:Required: Yes.
+:Example: 4
+
+``m={coding-chunks}``
+
+:Description: Compute **coding chunks** for each object and store them
+              on different OSDs. The number of coding chunks is also
+              the number of OSDs that can be down without losing data.
+
+:Type: Integer
+:Required: Yes.
+:Example: 2
+
+``d={helper-chunks}``
+
+:Description: Number of OSDs requested to send data during recovery of
+              a single chunk. *d* needs to be chosen such that
+              k+1 <= d <= k+m-1. The larger the *d*, the better the savings.
+
+:Type: Integer
+:Required: No.
+:Default: k+m-1
+
+``scalar_mds={jerasure|isa|shec}``
+
+:Description: **scalar_mds** specifies the plugin that is used as a 
+             building block in the layered construction. It can be 
+             one of *jerasure*, *isa*, *shec*
+
+:Type: String
+:Required: No.
+:Default: jerasure
+
+``technique={technique}``
+
+:Description: **technique** specifies the technique that will be picked
+             within the 'scalar_mds' plugin specified. Supported techniques
+             are 'reed_sol_van', 'reed_sol_r6_op', 'cauchy_orig', 
+             'cauchy_good', 'liber8tion' for jerasure, 'reed_sol_van',
+             'cauchy' for isa and 'single', 'multiple' for shec.
+
+:Type: String
+:Required: No.
+:Default: reed_sol_van (for jerasure, isa), single (for shec)
+
+
+``crush-root={root}``
+
+:Description: The name of the crush bucket used for the first step of
+              the CRUSH rule. For instance **step take default**.
+
+:Type: String
+:Required: No.
+:Default: default
+
+
+``crush-failure-domain={bucket-type}``
+
+:Description: Ensure that no two chunks are in a bucket with the same
+              failure domain. For instance, if the failure domain is
+              **host** no two chunks will be stored on the same
+              host. It is used to create a CRUSH rule step such as **step
+              chooseleaf host**.
+
+:Type: String
+:Required: No.
+:Default: host
+
+``crush-device-class={device-class}``
+
+:Description: Restrict placement to devices of a specific class (e.g.,
+              ``ssd`` or ``hdd``), using the crush device class names
+              in the CRUSH map.
+
+:Type: String
+:Required: No.
+:Default:
+
+``directory={directory}``
+
+:Description: Set the **directory** name from which the erasure code
+              plugin is loaded.
+
+:Type: String
+:Required: No.
+:Default: /usr/lib/ceph/erasure-code
+
+``--force``
+
+:Description: Override an existing profile by the same name.
+
+:Type: String
+:Required: No.
+
+
+Notion of sub-chunks
+====================
+
+The Clay code is able to save in terms of disk IO, network bandwidth as it
+is a vector code and it is able to view and manipulate data within a chunk 
+at a finer granularity termed as a sub-chunk. The number of sub-chunks within 
+a chunk for a Clay code is given by:
+
+	sub-chunk count = :math:`q^{\frac{k+m}{q}}`, where :math:`q = d - k + 1`
+
+
+During repair of an OSD, the helper information requested
+from an available OSD is only a fraction of a chunk. In fact, the number
+of sub-chunks within a chunk that are accessed during repair is given by:
+
+	repair sub-chunk count = :math:`\frac{sub---chunk \: count}{q}`
+
+Examples
+--------
+
+#. For a configuration with *k=4*, *m=2*, *d=5*, the sub-chunk count is
+   8 and  the repair sub-chunk count is 4. Therefore, only half of a chunk is read 
+   during repair.
+#. When *k=8*, *m=4*, *d=11* the sub-chunk count is 64 and repair sub-chunk count
+   is 16. A quarter of a chunk is read from an available OSD for repair of a failed 
+   chunk.
+
+
+
+How to choose a configuration given a workload
+==============================================
+
+Only a few sub-chunks are read of all the sub-chunks within a chunk. These sub-chunks
+are not necessarily stored consecutively within a chunk. For best disk IO 
+performance, it is helpful to read contiguous data. For this reason, it is suggested that
+you choose stripe-size such that the sub-chunk size is sufficiently large.
+
+For a given stripe-size (that's fixed based on a workload), choose ``k``, ``m``, ``d`` such that:
+
+	sub-chunk size = :math:`\frac{stripe-size}{k sub-chunk count}` = 4KB, 8KB, 12KB ...
+
+#. For large size workloads for which the stripe size is large, it is easy to choose k, m, d.
+   For example consider a stripe-size of size 64MB, choosing *k=16*, *m=4* and *d=19* will
+   result in a sub-chunk count of 1024 and a sub-chunk size of 4KB.
+#. For small size workloads, *k=4*, *m=2* is a good configuration that provides both network
+   and disk IO benefits.
+
+Comparisons with LRC
+====================
+
+Locally Recoverable Codes (LRC) are also designed in order to save in terms of network
+bandwidth, disk IO during single OSD recovery. However, the focus in LRCs is to keep the
+number of OSDs contacted during repair (d) to be minimal, but this comes at the cost of storage overhead.
+The *clay* code has a storage overhead m/k. In the case of an *lrc*, it stores (k+m)/d parities in
+addition to the ``m`` parities resulting in a storage overhead (m+(k+m)/d)/k. Both *clay* and *lrc*
+can recover from the failure of any ``m`` OSDs.
+
+	+-----------------+----------------------------------+----------------------------------+
+	| Parameters      | disk IO, storage overhead (LRC)  | disk IO, storage overhead (CLAY) |
+	+=================+================+=================+==================================+
+	| (k=10, m=4)     | 7 * S, 0.6 (d=7)                 | 3.25 * S, 0.4 (d=13)             |
+	+-----------------+----------------------------------+----------------------------------+
+	| (k=16, m=4)     | 4 * S, 0.5625 (d=4)              | 4.75 * S, 0.25 (d=19)            |
+	+-----------------+----------------------------------+----------------------------------+
+
+
+where ``S`` is the amount of data stored of single OSD being recovered.
diff --git a/doc/rados/operations/erasure-code-isa.rst b/doc/rados/operations/erasure-code-isa.rst
new file mode 100644
index 000000000..9a43f89a2
--- /dev/null
+++ b/doc/rados/operations/erasure-code-isa.rst
@@ -0,0 +1,107 @@
+=======================
+ISA erasure code plugin
+=======================
+
+The *isa* plugin encapsulates the `ISA
+<https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version/>`_
+library.
+
+Create an isa profile
+=====================
+
+To create a new *isa* erasure code profile:
+
+.. prompt:: bash $
+
+   ceph osd erasure-code-profile set {name} \
+     plugin=isa \
+     technique={reed_sol_van|cauchy} \
+     [k={data-chunks}] \
+     [m={coding-chunks}] \
+     [crush-root={root}] \
+     [crush-failure-domain={bucket-type}] \
+     [crush-device-class={device-class}] \
+     [directory={directory}] \
+     [--force]
+
+Where:
+
+``k={data chunks}``
+
+:Description: Each object is split in **data-chunks** parts,
+              each stored on a different OSD.
+
+:Type: Integer
+:Required: No.
+:Default: 7
+
+``m={coding-chunks}``
+
+:Description: Compute **coding chunks** for each object and store them
+              on different OSDs. The number of coding chunks is also
+              the number of OSDs that can be down without losing data.
+
+:Type: Integer
+:Required: No.
+:Default: 3
+
+``technique={reed_sol_van|cauchy}``
+
+:Description: The ISA plugin comes in two `Reed Solomon
+              <https://en.wikipedia.org/wiki/Reed%E2%80%93Solomon_error_correction>`_
+              forms. If *reed_sol_van* is set, it is `Vandermonde
+              <https://en.wikipedia.org/wiki/Vandermonde_matrix>`_, if
+              *cauchy* is set, it is `Cauchy
+              <https://en.wikipedia.org/wiki/Cauchy_matrix>`_.
+
+:Type: String
+:Required: No.
+:Default: reed_sol_van
+
+``crush-root={root}``
+
+:Description: The name of the crush bucket used for the first step of
+              the CRUSH rule. For instance **step take default**.
+
+:Type: String
+:Required: No.
+:Default: default
+
+``crush-failure-domain={bucket-type}``
+
+:Description: Ensure that no two chunks are in a bucket with the same
+              failure domain. For instance, if the failure domain is
+              **host** no two chunks will be stored on the same
+              host. It is used to create a CRUSH rule step such as **step
+              chooseleaf host**.
+
+:Type: String
+:Required: No.
+:Default: host
+
+``crush-device-class={device-class}``
+
+:Description: Restrict placement to devices of a specific class (e.g.,
+              ``ssd`` or ``hdd``), using the crush device class names
+              in the CRUSH map.
+
+:Type: String
+:Required: No.
+:Default:
+
+``directory={directory}``
+
+:Description: Set the **directory** name from which the erasure code
+              plugin is loaded.
+
+:Type: String
+:Required: No.
+:Default: /usr/lib/ceph/erasure-code
+
+``--force``
+
+:Description: Override an existing profile by the same name.
+
+:Type: String
+:Required: No.
+
diff --git a/doc/rados/operations/erasure-code-jerasure.rst b/doc/rados/operations/erasure-code-jerasure.rst
new file mode 100644
index 000000000..553afa09d
--- /dev/null
+++ b/doc/rados/operations/erasure-code-jerasure.rst
@@ -0,0 +1,121 @@
+============================
+Jerasure erasure code plugin
+============================
+
+The *jerasure* plugin is the most generic and flexible plugin, it is
+also the default for Ceph erasure coded pools. 
+
+The *jerasure* plugin encapsulates the `Jerasure
+<http://jerasure.org>`_ library. It is
+recommended to read the *jerasure* documentation to get a better
+understanding of the parameters.
+
+Create a jerasure profile
+=========================
+
+To create a new *jerasure* erasure code profile:
+ 
+.. prompt:: bash $
+
+   ceph osd erasure-code-profile set {name} \
+     plugin=jerasure \
+     k={data-chunks} \
+     m={coding-chunks} \
+     technique={reed_sol_van|reed_sol_r6_op|cauchy_orig|cauchy_good|liberation|blaum_roth|liber8tion} \
+     [crush-root={root}] \
+     [crush-failure-domain={bucket-type}] \
+     [crush-device-class={device-class}] \
+     [directory={directory}] \
+     [--force]
+
+Where:
+
+``k={data chunks}``
+
+:Description: Each object is split in **data-chunks** parts,
+              each stored on a different OSD.
+
+:Type: Integer
+:Required: Yes.
+:Example: 4
+
+``m={coding-chunks}``
+
+:Description: Compute **coding chunks** for each object and store them
+              on different OSDs. The number of coding chunks is also
+              the number of OSDs that can be down without losing data.
+
+:Type: Integer
+:Required: Yes.
+:Example: 2
+
+``technique={reed_sol_van|reed_sol_r6_op|cauchy_orig|cauchy_good|liberation|blaum_roth|liber8tion}``
+
+:Description: The more flexible technique is *reed_sol_van* : it is
+              enough to set *k* and *m*. The *cauchy_good* technique
+              can be faster but you need to chose the *packetsize*
+              carefully. All of *reed_sol_r6_op*, *liberation*,
+              *blaum_roth*, *liber8tion* are *RAID6* equivalents in
+              the sense that they can only be configured with *m=2*. 
+
+:Type: String
+:Required: No.
+:Default: reed_sol_van
+
+``packetsize={bytes}``
+
+:Description: The encoding will be done on packets of *bytes* size at
+              a time. Choosing the right packet size is difficult. The
+              *jerasure* documentation contains extensive information
+              on this topic.
+
+:Type: Integer
+:Required: No.
+:Default: 2048
+
+``crush-root={root}``
+
+:Description: The name of the crush bucket used for the first step of
+              the CRUSH rule. For instance **step take default**.
+
+:Type: String
+:Required: No.
+:Default: default
+
+``crush-failure-domain={bucket-type}``
+
+:Description: Ensure that no two chunks are in a bucket with the same
+              failure domain. For instance, if the failure domain is
+              **host** no two chunks will be stored on the same
+              host. It is used to create a CRUSH rule step such as **step
+              chooseleaf host**.
+
+:Type: String
+:Required: No.
+:Default: host
+
+``crush-device-class={device-class}``
+
+:Description: Restrict placement to devices of a specific class (e.g.,
+              ``ssd`` or ``hdd``), using the crush device class names
+              in the CRUSH map.
+
+:Type: String
+:Required: No.
+
+``directory={directory}``
+
+:Description: Set the **directory** name from which the erasure code
+              plugin is loaded.
+
+:Type: String
+:Required: No.
+:Default: /usr/lib/ceph/erasure-code
+
+``--force``
+
+:Description: Override an existing profile by the same name.
+
+:Type: String
+:Required: No.
+
diff --git a/doc/rados/operations/erasure-code-lrc.rst b/doc/rados/operations/erasure-code-lrc.rst
new file mode 100644
index 000000000..5329603b9
--- /dev/null
+++ b/doc/rados/operations/erasure-code-lrc.rst
@@ -0,0 +1,388 @@
+======================================
+Locally repairable erasure code plugin
+======================================
+
+With the *jerasure* plugin, when an erasure coded object is stored on
+multiple OSDs, recovering from the loss of one OSD requires reading
+from *k* others. For instance if *jerasure* is configured with
+*k=8* and *m=4*, recovering from the loss of one OSD requires reading
+from eight others.
+
+The *lrc* erasure code plugin creates local parity chunks to enable
+recovery using fewer surviving OSDs. For instance if *lrc* is configured with
+*k=8*, *m=4* and *l=4*, it will create an additional parity chunk for
+every four OSDs. When a single OSD is lost, it can be recovered with
+only four OSDs instead of eight.
+
+Erasure code profile examples
+=============================
+
+Reduce recovery bandwidth between hosts
+---------------------------------------
+
+Although it is probably not an interesting use case when all hosts are
+connected to the same switch, reduced bandwidth usage can actually be
+observed.:
+
+.. prompt:: bash $
+
+   ceph osd erasure-code-profile set LRCprofile \
+      plugin=lrc \
+      k=4 m=2 l=3 \
+      crush-failure-domain=host
+   ceph osd pool create lrcpool erasure LRCprofile
+
+
+Reduce recovery bandwidth between racks
+---------------------------------------
+
+In Firefly the bandwidth reduction will only be observed if the primary
+OSD is in the same rack as the lost chunk.:
+
+.. prompt:: bash $
+
+   ceph osd erasure-code-profile set LRCprofile \
+      plugin=lrc \
+      k=4 m=2 l=3 \
+      crush-locality=rack \
+      crush-failure-domain=host
+   ceph osd pool create lrcpool erasure LRCprofile
+
+
+Create an lrc profile
+=====================
+
+To create a new lrc erasure code profile:
+
+.. prompt:: bash $
+
+   ceph osd erasure-code-profile set {name} \
+       plugin=lrc \
+       k={data-chunks} \
+       m={coding-chunks} \
+       l={locality} \
+       [crush-root={root}] \
+       [crush-locality={bucket-type}] \
+       [crush-failure-domain={bucket-type}] \
+       [crush-device-class={device-class}] \
+       [directory={directory}] \
+       [--force]
+
+Where:
+
+``k={data chunks}``
+
+:Description: Each object is split in **data-chunks** parts,
+              each stored on a different OSD.
+
+:Type: Integer
+:Required: Yes.
+:Example: 4
+
+``m={coding-chunks}``
+
+:Description: Compute **coding chunks** for each object and store them
+              on different OSDs. The number of coding chunks is also
+              the number of OSDs that can be down without losing data.
+
+:Type: Integer
+:Required: Yes.
+:Example: 2
+
+``l={locality}``
+
+:Description: Group the coding and data chunks into sets of size
+              **locality**. For instance, for **k=4** and **m=2**,
+              when **locality=3** two groups of three are created.
+              Each set can be recovered without reading chunks
+              from another set.
+
+:Type: Integer
+:Required: Yes.
+:Example: 3
+
+``crush-root={root}``
+
+:Description: The name of the crush bucket used for the first step of
+              the CRUSH rule. For instance **step take default**.
+
+:Type: String
+:Required: No.
+:Default: default
+
+``crush-locality={bucket-type}``
+
+:Description: The type of the CRUSH bucket in which each set of chunks
+              defined by **l** will be stored. For instance, if it is
+              set to **rack**, each group of **l** chunks will be
+              placed in a different rack. It is used to create a
+              CRUSH rule step such as **step choose rack**. If it is not
+              set, no such grouping is done.
+
+:Type: String
+:Required: No.
+
+``crush-failure-domain={bucket-type}``
+
+:Description: Ensure that no two chunks are in a bucket with the same
+              failure domain. For instance, if the failure domain is
+              **host** no two chunks will be stored on the same
+              host. It is used to create a CRUSH rule step such as **step
+              chooseleaf host**.
+
+:Type: String
+:Required: No.
+:Default: host
+
+``crush-device-class={device-class}``
+
+:Description: Restrict placement to devices of a specific class (e.g.,
+              ``ssd`` or ``hdd``), using the crush device class names
+              in the CRUSH map.
+
+:Type: String
+:Required: No.
+:Default:
+
+``directory={directory}``
+
+:Description: Set the **directory** name from which the erasure code
+              plugin is loaded.
+
+:Type: String
+:Required: No.
+:Default: /usr/lib/ceph/erasure-code
+
+``--force``
+
+:Description: Override an existing profile by the same name.
+
+:Type: String
+:Required: No.
+
+Low level plugin configuration
+==============================
+
+The sum of **k** and **m** must be a multiple of the **l** parameter.
+The low level configuration parameters however do not enforce this
+restriction and it may be advantageous to use them for specific
+purposes. It is for instance possible to define two groups, one with 4
+chunks and another with 3 chunks. It is also possible to recursively
+define locality sets, for instance datacenters and racks into
+datacenters. The **k/m/l** are implemented by generating a low level
+configuration.
+
+The *lrc* erasure code plugin recursively applies erasure code
+techniques so that recovering from the loss of some chunks only
+requires a subset of the available chunks, most of the time.
+
+For instance, when three coding steps are described as::
+
+   chunk nr    01234567
+   step 1      _cDD_cDD
+   step 2      cDDD____
+   step 3      ____cDDD
+
+where *c* are coding chunks calculated from the data chunks *D*, the
+loss of chunk *7* can be recovered with the last four chunks. And the
+loss of chunk *2* chunk can be recovered with the first four
+chunks.
+
+Erasure code profile examples using low level configuration
+===========================================================
+
+Minimal testing
+---------------
+
+It is strictly equivalent to using a *K=2* *M=1* erasure code profile. The *DD*
+implies *K=2*, the *c* implies *M=1* and the *jerasure* plugin is used
+by default.:
+
+.. prompt:: bash $
+   
+   ceph osd erasure-code-profile set LRCprofile \
+      plugin=lrc \
+      mapping=DD_ \
+      layers='[ [ "DDc", "" ] ]'
+   ceph osd pool create lrcpool erasure LRCprofile
+
+Reduce recovery bandwidth between hosts
+---------------------------------------
+
+Although it is probably not an interesting use case when all hosts are
+connected to the same switch, reduced bandwidth usage can actually be
+observed. It is equivalent to **k=4**, **m=2** and **l=3** although
+the layout of the chunks is different. **WARNING: PROMPTS ARE SELECTABLE**
+
+::
+
+   $ ceph osd erasure-code-profile set LRCprofile \
+        plugin=lrc \
+        mapping=__DD__DD \
+        layers='[
+                  [ "_cDD_cDD", "" ],
+                  [ "cDDD____", "" ],
+                  [ "____cDDD", "" ],
+                ]'
+   $ ceph osd pool create lrcpool erasure LRCprofile
+
+
+Reduce recovery bandwidth between racks
+---------------------------------------
+
+In Firefly the reduced bandwidth will only be observed if the primary OSD is in
+the same rack as the lost chunk. **WARNING: PROMPTS ARE SELECTABLE**
+
+::
+
+   $ ceph osd erasure-code-profile set LRCprofile \
+       plugin=lrc \
+       mapping=__DD__DD \
+       layers='[
+                 [ "_cDD_cDD", "" ],
+                 [ "cDDD____", "" ],
+                 [ "____cDDD", "" ],
+               ]' \
+       crush-steps='[
+                       [ "choose", "rack", 2 ],
+                       [ "chooseleaf", "host", 4 ],
+                      ]'
+  
+   $ ceph osd pool create lrcpool erasure LRCprofile
+
+Testing with different Erasure Code backends
+--------------------------------------------
+
+LRC now uses jerasure as the default EC backend. It is possible to
+specify the EC backend/algorithm on a per layer basis using the low
+level configuration. The second argument in layers='[ [ "DDc", "" ] ]'
+is actually an erasure code profile to be used for this level. The
+example below specifies the ISA backend with the cauchy technique to
+be used in the lrcpool.:
+
+.. prompt:: bash $
+
+   ceph osd erasure-code-profile set LRCprofile \
+      plugin=lrc \
+      mapping=DD_ \
+      layers='[ [ "DDc", "plugin=isa technique=cauchy" ] ]'
+   ceph osd pool create lrcpool erasure LRCprofile
+
+You could also use a different erasure code profile for each
+layer. **WARNING: PROMPTS ARE SELECTABLE**
+
+::
+
+   $ ceph osd erasure-code-profile set LRCprofile \
+        plugin=lrc \
+        mapping=__DD__DD \
+        layers='[
+                  [ "_cDD_cDD", "plugin=isa technique=cauchy" ],
+                  [ "cDDD____", "plugin=isa" ],
+                  [ "____cDDD", "plugin=jerasure" ],
+                ]'
+   $ ceph osd pool create lrcpool erasure LRCprofile
+
+
+
+Erasure coding and decoding algorithm
+=====================================
+
+The steps found in the layers description::
+
+   chunk nr    01234567
+
+   step 1      _cDD_cDD
+   step 2      cDDD____
+   step 3      ____cDDD
+
+are applied in order. For instance, if a 4K object is encoded, it will
+first go through *step 1* and be divided in four 1K chunks (the four
+uppercase D). They are stored in the chunks 2, 3, 6 and 7, in
+order. From these, two coding chunks are calculated (the two lowercase
+c). The coding chunks are stored in the chunks 1 and 5, respectively.
+
+The *step 2* re-uses the content created by *step 1* in a similar
+fashion and stores a single coding chunk *c* at position 0. The last four
+chunks, marked with an underscore (*_*) for readability, are ignored.
+
+The *step 3* stores a single coding chunk *c* at position 4. The three
+chunks created by *step 1* are used to compute this coding chunk,
+i.e. the coding chunk from *step 1* becomes a data chunk in *step 3*.
+
+If chunk *2* is lost::
+
+   chunk nr    01234567
+
+   step 1      _c D_cDD
+   step 2      cD D____
+   step 3      __ _cDDD
+
+decoding will attempt to recover it by walking the steps in reverse
+order: *step 3* then *step 2* and finally *step 1*.
+
+The *step 3* knows nothing about chunk *2* (i.e. it is an underscore)
+and is skipped.
+
+The coding chunk from *step 2*, stored in chunk *0*, allows it to
+recover the content of chunk *2*. There are no more chunks to recover
+and the process stops, without considering *step 1*.
+
+Recovering chunk *2* requires reading chunks *0, 1, 3* and writing
+back chunk *2*.
+
+If chunk *2, 3, 6* are lost::
+
+   chunk nr    01234567
+
+   step 1      _c  _c D
+   step 2      cD  __ _
+   step 3      __  cD D
+
+The *step 3* can recover the content of chunk *6*::
+
+   chunk nr    01234567
+
+   step 1      _c  _cDD
+   step 2      cD  ____
+   step 3      __  cDDD
+
+The *step 2* fails to recover and is skipped because there are two
+chunks missing (*2, 3*) and it can only recover from one missing
+chunk.
+
+The coding chunk from *step 1*, stored in chunk *1, 5*, allows it to
+recover the content of chunk *2, 3*::
+
+   chunk nr    01234567
+
+   step 1      _cDD_cDD
+   step 2      cDDD____
+   step 3      ____cDDD
+
+Controlling CRUSH placement
+===========================
+
+The default CRUSH rule provides OSDs that are on different hosts. For instance::
+
+   chunk nr    01234567
+
+   step 1      _cDD_cDD
+   step 2      cDDD____
+   step 3      ____cDDD
+
+needs exactly *8* OSDs, one for each chunk. If the hosts are in two
+adjacent racks, the first four chunks can be placed in the first rack
+and the last four in the second rack. So that recovering from the loss
+of a single OSD does not require using bandwidth between the two
+racks.
+
+For instance::
+
+   crush-steps='[ [ "choose", "rack", 2 ], [ "chooseleaf", "host", 4 ] ]'
+
+will create a rule that will select two crush buckets of type
+*rack* and for each of them choose four OSDs, each of them located in
+different buckets of type *host*.
+
+The CRUSH rule can also be manually crafted for finer control.
diff --git a/doc/rados/operations/erasure-code-profile.rst b/doc/rados/operations/erasure-code-profile.rst
new file mode 100644
index 000000000..45b071f8a
--- /dev/null
+++ b/doc/rados/operations/erasure-code-profile.rst
@@ -0,0 +1,126 @@
+.. _erasure-code-profiles:
+
+=====================
+Erasure code profiles
+=====================
+
+Erasure code is defined by a **profile** and is used when creating an
+erasure coded pool and the associated CRUSH rule.
+
+The **default** erasure code profile (which is created when the Ceph
+cluster is initialized) will split the data into 2 equal-sized chunks,
+and have 2 parity chunks of the same size. It will take as much space
+in the cluster as a 2-replica pool but can sustain the data loss of 2
+chunks out of 4. It is described as a profile with **k=2** and **m=2**,
+meaning the information is spread over four OSD (k+m == 4) and two of
+them can be lost.
+
+To improve redundancy without increasing raw storage requirements, a
+new profile can be created. For instance, a profile with **k=10** and
+**m=4** can sustain the loss of four (**m=4**) OSDs by distributing an
+object on fourteen (k+m=14) OSDs. The object is first divided in
+**10** chunks (if the object is 10MB, each chunk is 1MB) and **4**
+coding chunks are computed, for recovery (each coding chunk has the
+same size as the data chunk, i.e. 1MB). The raw space overhead is only
+40% and the object will not be lost even if four OSDs break at the
+same time.
+
+.. _list of available plugins:
+
+.. toctree::
+	:maxdepth: 1
+
+	erasure-code-jerasure
+	erasure-code-isa
+	erasure-code-lrc
+	erasure-code-shec
+	erasure-code-clay
+
+osd erasure-code-profile set
+============================
+
+To create a new erasure code profile::
+
+	ceph osd erasure-code-profile set {name} \
+             [{directory=directory}] \
+             [{plugin=plugin}] \
+             [{stripe_unit=stripe_unit}] \
+             [{key=value} ...] \
+             [--force]
+
+Where:
+
+``{directory=directory}``
+
+:Description: Set the **directory** name from which the erasure code
+              plugin is loaded.
+
+:Type: String
+:Required: No.
+:Default: /usr/lib/ceph/erasure-code
+
+``{plugin=plugin}``
+
+:Description: Use the erasure code **plugin** to compute coding chunks
+              and recover missing chunks. See the `list of available
+              plugins`_ for more information.
+
+:Type: String
+:Required: No.
+:Default: jerasure
+
+``{stripe_unit=stripe_unit}``
+
+:Description: The amount of data in a data chunk, per stripe. For
+              example, a profile with 2 data chunks and stripe_unit=4K
+              would put the range 0-4K in chunk 0, 4K-8K in chunk 1,
+              then 8K-12K in chunk 0 again. This should be a multiple
+              of 4K for best performance. The default value is taken
+              from the monitor config option
+              ``osd_pool_erasure_code_stripe_unit`` when a pool is
+              created.  The stripe_width of a pool using this profile
+              will be the number of data chunks multiplied by this
+              stripe_unit.
+
+:Type: String
+:Required: No.
+
+``{key=value}``
+
+:Description: The semantic of the remaining key/value pairs is defined
+              by the erasure code plugin.
+
+:Type: String
+:Required: No.
+
+``--force``
+
+:Description: Override an existing profile by the same name, and allow
+              setting a non-4K-aligned stripe_unit.
+
+:Type: String
+:Required: No.
+
+osd erasure-code-profile rm
+============================
+
+To remove an erasure code profile::
+
+	ceph osd erasure-code-profile rm {name}
+
+If the profile is referenced by a pool, the deletion will fail.
+
+osd erasure-code-profile get
+============================
+
+To display an erasure code profile::
+
+	ceph osd erasure-code-profile get {name}
+
+osd erasure-code-profile ls
+===========================
+
+To list the names of all erasure code profiles::
+
+	ceph osd erasure-code-profile ls
+
diff --git a/doc/rados/operations/erasure-code-shec.rst b/doc/rados/operations/erasure-code-shec.rst
new file mode 100644
index 000000000..4e8f59b0b
--- /dev/null
+++ b/doc/rados/operations/erasure-code-shec.rst
@@ -0,0 +1,145 @@
+========================
+SHEC erasure code plugin
+========================
+
+The *shec* plugin encapsulates the `multiple SHEC
+<http://tracker.ceph.com/projects/ceph/wiki/Shingled_Erasure_Code_(SHEC)>`_
+library. It allows ceph to recover data more efficiently than Reed Solomon codes.
+
+Create an SHEC profile
+======================
+
+To create a new *shec* erasure code profile:
+
+.. prompt:: bash $
+
+        ceph osd erasure-code-profile set {name} \
+             plugin=shec \
+             [k={data-chunks}] \
+             [m={coding-chunks}] \
+             [c={durability-estimator}] \
+             [crush-root={root}] \
+             [crush-failure-domain={bucket-type}] \
+             [crush-device-class={device-class}] \
+             [directory={directory}] \
+             [--force]
+
+Where:
+
+``k={data-chunks}``
+
+:Description: Each object is split in **data-chunks** parts,
+              each stored on a different OSD.
+
+:Type: Integer
+:Required: No.
+:Default: 4
+
+``m={coding-chunks}``
+
+:Description: Compute **coding-chunks** for each object and store them on
+              different OSDs. The number of **coding-chunks** does not necessarily
+              equal the number of OSDs that can be down without losing data.
+
+:Type: Integer
+:Required: No.
+:Default: 3
+
+``c={durability-estimator}``
+
+:Description: The number of parity chunks each of which includes each data chunk in its
+              calculation range. The number is used as a **durability estimator**.
+              For instance, if c=2, 2 OSDs can be down without losing data.
+
+:Type: Integer
+:Required: No.
+:Default: 2
+
+``crush-root={root}``
+
+:Description: The name of the crush bucket used for the first step of
+              the CRUSH rule. For instance **step take default**.
+
+:Type: String
+:Required: No.
+:Default: default
+
+``crush-failure-domain={bucket-type}``
+
+:Description: Ensure that no two chunks are in a bucket with the same
+              failure domain. For instance, if the failure domain is
+              **host** no two chunks will be stored on the same
+              host. It is used to create a CRUSH rule step such as **step
+              chooseleaf host**.
+
+:Type: String
+:Required: No.
+:Default: host
+
+``crush-device-class={device-class}``
+
+:Description: Restrict placement to devices of a specific class (e.g.,
+              ``ssd`` or ``hdd``), using the crush device class names
+              in the CRUSH map.
+
+:Type: String
+:Required: No.
+:Default:
+
+``directory={directory}``
+
+:Description: Set the **directory** name from which the erasure code
+              plugin is loaded.
+
+:Type: String
+:Required: No.
+:Default: /usr/lib/ceph/erasure-code
+
+``--force``
+
+:Description: Override an existing profile by the same name.
+
+:Type: String
+:Required: No.
+
+Brief description of SHEC's layouts
+===================================
+
+Space Efficiency
+----------------
+
+Space efficiency is a ratio of data chunks to all ones in a object and
+represented as k/(k+m).
+In order to improve space efficiency, you should increase k or decrease m:
+
+        space efficiency of SHEC(4,3,2) = :math:`\frac{4}{4+3}` = 0.57
+        SHEC(5,3,2) or SHEC(4,2,2) improves SHEC(4,3,2)'s space efficiency
+
+Durability
+----------
+
+The third parameter of SHEC (=c) is a durability estimator, which approximates
+the number of OSDs that can be down without losing data.
+
+``durability estimator of SHEC(4,3,2) = 2``
+
+Recovery Efficiency
+-------------------
+
+Describing calculation of recovery efficiency is beyond the scope of this document,
+but at least increasing m without increasing c achieves improvement of recovery efficiency.
+(However, we must pay attention to the sacrifice of space efficiency in this case.)
+
+``SHEC(4,2,2) -> SHEC(4,3,2) : achieves improvement of recovery efficiency``
+
+Erasure code profile examples
+=============================
+
+
+.. prompt:: bash $
+
+          ceph osd erasure-code-profile set SHECprofile \
+             plugin=shec \
+             k=8 m=4 c=3 \
+             crush-failure-domain=host
+          ceph osd pool create shecpool erasure SHECprofile
diff --git a/doc/rados/operations/erasure-code.rst b/doc/rados/operations/erasure-code.rst
new file mode 100644
index 000000000..1dea23c35
--- /dev/null
+++ b/doc/rados/operations/erasure-code.rst
@@ -0,0 +1,262 @@
+.. _ecpool:
+
+=============
+ Erasure code
+=============
+
+By default, Ceph `pools <../pools>`_ are created with the type "replicated". In
+replicated-type pools, every object is copied to multiple disks (this
+multiple copying is the "replication").
+
+In contrast, `erasure-coded <https://en.wikipedia.org/wiki/Erasure_code>`_
+pools use a method of data protection that is different from replication. In
+erasure coding, data is broken into fragments of two kinds: data blocks and
+parity blocks. If a drive fails or becomes corrupted, the parity blocks are
+used to rebuild the data. At scale, erasure coding saves space relative to
+replication.
+
+In this documentation, data blocks are referred to as "data chunks"
+and parity blocks are referred to as "encoding chunks".
+
+Erasure codes are also called "forward error correction codes". The
+first forward error correction code was developed in 1950 by Richard
+Hamming at Bell Laboratories.
+
+
+Creating a sample erasure coded pool
+------------------------------------
+
+The simplest erasure coded pool is equivalent to `RAID5
+<https://en.wikipedia.org/wiki/Standard_RAID_levels#RAID_5>`_ and
+requires at least three hosts:
+
+.. prompt:: bash $
+
+   ceph osd pool create ecpool erasure
+
+::
+
+   pool 'ecpool' created
+
+.. prompt:: bash $
+
+   echo ABCDEFGHI | rados --pool ecpool put NYAN -
+   rados --pool ecpool get NYAN -
+   
+::
+
+   ABCDEFGHI
+
+Erasure code profiles
+---------------------
+
+The default erasure code profile can sustain the loss of two OSDs. This erasure
+code profile is equivalent to a replicated pool of size three, but requires
+2TB to store 1TB of data instead of 3TB to store 1TB of data. The default
+profile can be displayed with this command:
+
+.. prompt:: bash $
+
+   ceph osd erasure-code-profile get default
+   
+::
+
+   k=2
+   m=2
+   plugin=jerasure
+   crush-failure-domain=host
+   technique=reed_sol_van
+
+.. note::
+   The default erasure-coded pool, the profile of which is displayed here, is
+   not the same as the simplest erasure-coded pool. 
+   
+   The default erasure-coded pool has two data chunks (k) and two coding chunks
+   (m). The profile of the default erasure-coded pool is "k=2 m=2".
+
+   The simplest erasure-coded pool has two data chunks (k) and one coding chunk
+   (m). The profile of the simplest erasure-coded pool is "k=2 m=1".
+
+Choosing the right profile is important because the profile cannot be modified
+after the pool is created. If you find that you need an erasure-coded pool with
+a profile different than the one you have created, you must create a new pool
+with a different (and presumably more carefully-considered) profile. When the
+new pool is created, all objects from the wrongly-configured pool must be moved
+to the newly-created pool. There is no way to alter the profile of a pool after its creation.
+
+The most important parameters of the profile are *K*, *M* and
+*crush-failure-domain* because they define the storage overhead and
+the data durability. For example, if the desired architecture must
+sustain the loss of two racks with a storage overhead of 67% overhead,
+the following profile can be defined:
+
+.. prompt:: bash $
+
+   ceph osd erasure-code-profile set myprofile \
+       k=3 \
+       m=2 \
+       crush-failure-domain=rack
+   ceph osd pool create ecpool erasure myprofile
+   echo ABCDEFGHI | rados --pool ecpool put NYAN -
+   rados --pool ecpool get NYAN -
+
+::
+
+    ABCDEFGHI
+
+The *NYAN* object will be divided in three (*K=3*) and two additional
+*chunks* will be created (*M=2*). The value of *M* defines how many
+OSD can be lost simultaneously without losing any data. The
+*crush-failure-domain=rack* will create a CRUSH rule that ensures
+no two *chunks* are stored in the same rack.
+
+.. ditaa::
+                            +-------------------+
+                       name |       NYAN        |
+                            +-------------------+
+                    content |     ABCDEFGHI     |
+                            +--------+----------+
+                                     |
+                                     |
+                                     v
+                              +------+------+
+              +---------------+ encode(3,2) +-----------+
+              |               +--+--+---+---+           |
+              |                  |  |   |               |
+              |          +-------+  |   +-----+         |
+              |          |          |         |         |
+           +--v---+   +--v---+   +--v---+  +--v---+  +--v---+
+     name  | NYAN |   | NYAN |   | NYAN |  | NYAN |  | NYAN |
+           +------+   +------+   +------+  +------+  +------+
+    shard  |  1   |   |  2   |   |  3   |  |  4   |  |  5   |
+           +------+   +------+   +------+  +------+  +------+
+  content  | ABC  |   | DEF  |   | GHI  |  | YXY  |  | QGC  |
+           +--+---+   +--+---+   +--+---+  +--+---+  +--+---+
+              |          |          |         |         |
+              |          |          v         |         |
+              |          |       +--+---+     |         |
+              |          |       | OSD1 |     |         |
+              |          |       +------+     |         |
+              |          |                    |         |
+              |          |       +------+     |         |
+              |          +------>| OSD2 |     |         |
+              |                  +------+     |         |
+              |                               |         |
+              |                  +------+     |         |
+              |                  | OSD3 |<----+         |
+              |                  +------+               |
+              |                                         |
+              |                  +------+               |
+              |                  | OSD4 |<--------------+
+              |                  +------+
+              |
+              |                  +------+
+              +----------------->| OSD5 |
+                                 +------+
+
+ 
+More information can be found in the `erasure code profiles
+<../erasure-code-profile>`_ documentation.
+
+
+Erasure Coding with Overwrites
+------------------------------
+
+By default, erasure coded pools only work with uses like RGW that
+perform full object writes and appends.
+
+Since Luminous, partial writes for an erasure coded pool may be
+enabled with a per-pool setting. This lets RBD and CephFS store their
+data in an erasure coded pool:
+
+.. prompt:: bash $
+
+    ceph osd pool set ec_pool allow_ec_overwrites true
+
+This can only be enabled on a pool residing on bluestore OSDs, since
+bluestore's checksumming is used to detect bitrot or other corruption
+during deep-scrub. In addition to being unsafe, using filestore with
+ec overwrites yields low performance compared to bluestore.
+
+Erasure coded pools do not support omap, so to use them with RBD and
+CephFS you must instruct them to store their data in an ec pool, and
+their metadata in a replicated pool. For RBD, this means using the
+erasure coded pool as the ``--data-pool`` during image creation:
+
+.. prompt:: bash $
+
+    rbd create --size 1G --data-pool ec_pool replicated_pool/image_name
+
+For CephFS, an erasure coded pool can be set as the default data pool during
+file system creation or via `file layouts <../../../cephfs/file-layouts>`_.
+
+
+Erasure coded pool and cache tiering
+------------------------------------
+
+Erasure coded pools require more resources than replicated pools and
+lack some functionalities such as omap. To overcome these
+limitations, one can set up a `cache tier <../cache-tiering>`_
+before the erasure coded pool.
+
+For instance, if the pool *hot-storage* is made of fast storage:
+
+.. prompt:: bash $
+
+   ceph osd tier add ecpool hot-storage
+   ceph osd tier cache-mode hot-storage writeback
+   ceph osd tier set-overlay ecpool hot-storage
+
+will place the *hot-storage* pool as tier of *ecpool* in *writeback*
+mode so that every write and read to the *ecpool* are actually using
+the *hot-storage* and benefit from its flexibility and speed.
+
+More information can be found in the `cache tiering
+<../cache-tiering>`_ documentation.
+
+Erasure coded pool recovery
+---------------------------
+If an erasure coded pool loses some shards, it must recover them from the others.
+This generally involves reading from the remaining shards, reconstructing the data, and
+writing it to the new peer.
+In Octopus, erasure coded pools can recover as long as there are at least *K* shards
+available. (With fewer than *K* shards, you have actually lost data!)
+
+Prior to Octopus, erasure coded pools required at least *min_size* shards to be
+available, even if *min_size* is greater than *K*. (We generally recommend min_size
+be *K+2* or more to prevent loss of writes and data.)
+This conservative decision was made out of an abundance of caution when designing the new pool
+mode but also meant pools with lost OSDs but no data loss were unable to recover and go active
+without manual intervention to change the *min_size*.
+
+Glossary
+--------
+
+*chunk*
+   when the encoding function is called, it returns chunks of the same
+   size. Data chunks which can be concatenated to reconstruct the original
+   object and coding chunks which can be used to rebuild a lost chunk.
+
+*K*
+   the number of data *chunks*, i.e. the number of *chunks* in which the
+   original object is divided. For instance if *K* = 2 a 10KB object
+   will be divided into *K* objects of 5KB each.
+
+*M*
+   the number of coding *chunks*, i.e. the number of additional *chunks*
+   computed by the encoding functions. If there are 2 coding *chunks*,
+   it means 2 OSDs can be out without losing data.
+
+
+Table of content
+----------------
+
+.. toctree::
+	:maxdepth: 1
+
+	erasure-code-profile
+	erasure-code-jerasure
+	erasure-code-isa
+	erasure-code-lrc
+	erasure-code-shec
+	erasure-code-clay
diff --git a/doc/rados/operations/health-checks.rst b/doc/rados/operations/health-checks.rst
new file mode 100644
index 000000000..a8fa8243f
--- /dev/null
+++ b/doc/rados/operations/health-checks.rst
@@ -0,0 +1,1549 @@
+.. _health-checks:
+
+=============
+Health checks
+=============
+
+Overview
+========
+
+There is a finite set of possible health messages that a Ceph cluster can
+raise -- these are defined as *health checks* which have unique identifiers.
+
+The identifier is a terse pseudo-human-readable (i.e. like a variable name)
+string.  It is intended to enable tools (such as UIs) to make sense of
+health checks, and present them in a way that reflects their meaning.
+
+This page lists the health checks that are raised by the monitor and manager
+daemons.  In addition to these, you may also see health checks that originate
+from MDS daemons (see :ref:`cephfs-health-messages`), and health checks
+that are defined by ceph-mgr python modules.
+
+Definitions
+===========
+
+Monitor
+-------
+
+DAEMON_OLD_VERSION
+__________________
+
+Warn if old version(s) of Ceph are running on any daemons.
+It will generate a health error if multiple versions are detected.
+This condition must exist for over mon_warn_older_version_delay (set to 1 week by default) in order for the
+health condition to be triggered.  This allows most upgrades to proceed
+without falsely seeing the warning.  If upgrade is paused for an extended
+time period, health mute can be used like this
+"ceph health mute DAEMON_OLD_VERSION --sticky".  In this case after
+upgrade has finished use "ceph health unmute DAEMON_OLD_VERSION".
+
+MON_DOWN
+________
+
+One or more monitor daemons is currently down.  The cluster requires a
+majority (more than 1/2) of the monitors in order to function.  When
+one or more monitors are down, clients may have a harder time forming
+their initial connection to the cluster as they may need to try more
+addresses before they reach an operating monitor.
+
+The down monitor daemon should generally be restarted as soon as
+possible to reduce the risk of a subsequen monitor failure leading to
+a service outage.
+
+MON_CLOCK_SKEW
+______________
+
+The clocks on the hosts running the ceph-mon monitor daemons are not
+sufficiently well synchronized.  This health alert is raised if the
+cluster detects a clock skew greater than ``mon_clock_drift_allowed``.
+
+This is best resolved by synchronizing the clocks using a tool like
+``ntpd`` or ``chrony``.
+
+If it is impractical to keep the clocks closely synchronized, the
+``mon_clock_drift_allowed`` threshold can also be increased, but this
+value must stay significantly below the ``mon_lease`` interval in
+order for monitor cluster to function properly.
+
+MON_MSGR2_NOT_ENABLED
+_____________________
+
+The ``ms_bind_msgr2`` option is enabled but one or more monitors is
+not configured to bind to a v2 port in the cluster's monmap.  This
+means that features specific to the msgr2 protocol (e.g., encryption)
+are not available on some or all connections.
+
+In most cases this can be corrected by issuing the command:
+
+.. prompt:: bash $
+
+   ceph mon enable-msgr2
+
+That command will change any monitor configured for the old default
+port 6789 to continue to listen for v1 connections on 6789 and also
+listen for v2 connections on the new default 3300 port.
+
+If a monitor is configured to listen for v1 connections on a non-standard port (not 6789), then the monmap will need to be modified manually.
+
+
+MON_DISK_LOW
+____________
+
+One or more monitors is low on disk space.  This alert triggers if the
+available space on the file system storing the monitor database
+(normally ``/var/lib/ceph/mon``), as a percentage, drops below
+``mon_data_avail_warn`` (default: 30%).
+
+This may indicate that some other process or user on the system is
+filling up the same file system used by the monitor.  It may also
+indicate that the monitors database is large (see ``MON_DISK_BIG``
+below).
+
+If space cannot be freed, the monitor's data directory may need to be
+moved to another storage device or file system (while the monitor
+daemon is not running, of course).
+
+
+MON_DISK_CRIT
+_____________
+
+One or more monitors is critically low on disk space.  This alert
+triggers if the available space on the file system storing the monitor
+database (normally ``/var/lib/ceph/mon``), as a percentage, drops
+below ``mon_data_avail_crit`` (default: 5%).  See ``MON_DISK_LOW``, above.
+
+MON_DISK_BIG
+____________
+
+The database size for one or more monitors is very large.  This alert
+triggers if the size of the monitor's database is larger than
+``mon_data_size_warn`` (default: 15 GiB).
+
+A large database is unusual, but may not necessarily indicate a
+problem.  Monitor databases may grow in size when there are placement
+groups that have not reached an ``active+clean`` state in a long time.
+
+This may also indicate that the monitor's database is not properly
+compacting, which has been observed with some older versions of
+leveldb and rocksdb.  Forcing a compaction with ``ceph daemon mon.<id>
+compact`` may shrink the on-disk size.
+
+This warning may also indicate that the monitor has a bug that is
+preventing it from pruning the cluster metadata it stores.  If the
+problem persists, please report a bug.
+
+The warning threshold may be adjusted with:
+
+.. prompt:: bash $
+
+   ceph config set global mon_data_size_warn <size>
+
+AUTH_INSECURE_GLOBAL_ID_RECLAIM
+_______________________________
+
+One or more clients or daemons are connected to the cluster that are
+not securely reclaiming their global_id (a unique number identifying
+each entity in the cluster) when reconnecting to a monitor.  The
+client is being permitted to connect anyway because the
+``auth_allow_insecure_global_id_reclaim`` option is set to true (which may
+be necessary until all ceph clients have been upgraded), and the
+``auth_expose_insecure_global_id_reclaim`` option set to ``true`` (which
+allows monitors to detect clients with insecure reclaim early by forcing them to
+reconnect right after they first authenticate).
+
+You can identify which client(s) are using unpatched ceph client code with:
+
+.. prompt:: bash $
+
+   ceph health detail
+
+Clients global_id reclaim rehavior can also seen in the
+``global_id_status`` field in the dump of clients connected to an
+individual monitor (``reclaim_insecure`` means the client is
+unpatched and is contributing to this health alert):
+
+.. prompt:: bash $
+
+   ceph tell mon.\* sessions
+
+We strongly recommend that all clients in the system are upgraded to a
+newer version of Ceph that correctly reclaims global_id values.  Once
+all clients have been updated, you can stop allowing insecure reconnections
+with:
+
+.. prompt:: bash $
+
+   ceph config set mon auth_allow_insecure_global_id_reclaim false
+
+If it is impractical to upgrade all clients immediately, you can silence
+this warning temporarily with:
+
+.. prompt:: bash $
+
+   ceph health mute AUTH_INSECURE_GLOBAL_ID_RECLAIM 1w   # 1 week
+
+Although we do NOT recommend doing so, you can also disable this warning
+indefinitely with:
+
+.. prompt:: bash $
+
+   ceph config set mon mon_warn_on_insecure_global_id_reclaim false
+
+AUTH_INSECURE_GLOBAL_ID_RECLAIM_ALLOWED
+_______________________________________
+
+Ceph is currently configured to allow clients to reconnect to monitors using
+an insecure process to reclaim their previous global_id because the setting
+``auth_allow_insecure_global_id_reclaim`` is set to ``true``.  It may be necessary to
+leave this setting enabled while existing Ceph clients are upgraded to newer
+versions of Ceph that correctly and securely reclaim their global_id.
+
+If the ``AUTH_INSECURE_GLOBAL_ID_RECLAIM`` health alert has not also been raised and
+the ``auth_expose_insecure_global_id_reclaim`` setting has not been disabled (it is
+on by default), then there are currently no clients connected that need to be
+upgraded, and it is safe to disallow insecure global_id reclaim with:
+
+.. prompt:: bash $
+
+   ceph config set mon auth_allow_insecure_global_id_reclaim false
+
+If there are still clients that need to be upgraded, then this alert can be
+silenced temporarily with:
+
+.. prompt:: bash $
+
+   ceph health mute AUTH_INSECURE_GLOBAL_ID_RECLAIM_ALLOWED 1w   # 1 week
+
+Although we do NOT recommend doing so, you can also disable this warning indefinitely
+with:
+
+.. prompt:: bash $
+
+   ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false
+
+
+Manager
+-------
+
+MGR_DOWN
+________
+
+All manager daemons are currently down.  The cluster should normally
+have at least one running manager (``ceph-mgr``) daemon.  If no
+manager daemon is running, the cluster's ability to monitor itself will
+be compromised, and parts of the management API will become
+unavailable (for example, the dashboard will not work, and most CLI
+commands that report metrics or runtime state will block).  However,
+the cluster will still be able to perform all IO operations and
+recover from failures.
+
+The down manager daemon should generally be restarted as soon as
+possible to ensure that the cluster can be monitored (e.g., so that
+the ``ceph -s`` information is up to date, and/or metrics can be
+scraped by Prometheus).
+
+
+MGR_MODULE_DEPENDENCY
+_____________________
+
+An enabled manager module is failing its dependency check.  This health check
+should come with an explanatory message from the module about the problem.
+
+For example, a module might report that a required package is not installed:
+install the required package and restart your manager daemons.
+
+This health check is only applied to enabled modules.  If a module is
+not enabled, you can see whether it is reporting dependency issues in
+the output of `ceph module ls`.
+
+
+MGR_MODULE_ERROR
+________________
+
+A manager module has experienced an unexpected error.  Typically,
+this means an unhandled exception was raised from the module's `serve`
+function.  The human readable description of the error may be obscurely
+worded if the exception did not provide a useful description of itself.
+
+This health check may indicate a bug: please open a Ceph bug report if you
+think you have encountered a bug.
+
+If you believe the error is transient, you may restart your manager
+daemon(s), or use `ceph mgr fail` on the active daemon to prompt
+a failover to another daemon.
+
+
+OSDs
+----
+
+OSD_DOWN
+________
+
+One or more OSDs are marked down.  The ceph-osd daemon may have been
+stopped, or peer OSDs may be unable to reach the OSD over the network.
+Common causes include a stopped or crashed daemon, a down host, or a
+network outage.
+
+Verify the host is healthy, the daemon is started, and network is
+functioning.  If the daemon has crashed, the daemon log file
+(``/var/log/ceph/ceph-osd.*``) may contain debugging information.
+
+OSD_<crush type>_DOWN
+_____________________
+
+(e.g. OSD_HOST_DOWN, OSD_ROOT_DOWN)
+
+All the OSDs within a particular CRUSH subtree are marked down, for example
+all OSDs on a host.
+
+OSD_ORPHAN
+__________
+
+An OSD is referenced in the CRUSH map hierarchy but does not exist.
+
+The OSD can be removed from the CRUSH hierarchy with:
+
+.. prompt:: bash $
+
+   ceph osd crush rm osd.<id>
+
+OSD_OUT_OF_ORDER_FULL
+_____________________
+
+The utilization thresholds for `nearfull`, `backfillfull`, `full`,
+and/or `failsafe_full` are not ascending.  In particular, we expect
+`nearfull < backfillfull`, `backfillfull < full`, and `full <
+failsafe_full`.
+
+The thresholds can be adjusted with:
+
+.. prompt:: bash $
+
+   ceph osd set-nearfull-ratio <ratio>
+   ceph osd set-backfillfull-ratio <ratio>
+   ceph osd set-full-ratio <ratio>
+
+
+OSD_FULL
+________
+
+One or more OSDs has exceeded the `full` threshold and is preventing
+the cluster from servicing writes.
+
+Utilization by pool can be checked with:
+
+.. prompt:: bash $
+
+   ceph df
+
+The currently defined `full` ratio can be seen with:
+
+.. prompt:: bash $
+
+   ceph osd dump | grep full_ratio
+
+A short-term workaround to restore write availability is to raise the full
+threshold by a small amount:
+
+.. prompt:: bash $
+
+   ceph osd set-full-ratio <ratio>
+
+New storage should be added to the cluster by deploying more OSDs or
+existing data should be deleted in order to free up space.
+
+OSD_BACKFILLFULL
+________________
+
+One or more OSDs has exceeded the `backfillfull` threshold, which will
+prevent data from being allowed to rebalance to this device.  This is
+an early warning that rebalancing may not be able to complete and that
+the cluster is approaching full.
+
+Utilization by pool can be checked with:
+
+.. prompt:: bash $
+
+   ceph df
+
+OSD_NEARFULL
+____________
+
+One or more OSDs has exceeded the `nearfull` threshold.  This is an early
+warning that the cluster is approaching full.
+
+Utilization by pool can be checked with:
+
+.. prompt:: bash $
+
+   ceph df
+
+OSDMAP_FLAGS
+____________
+
+One or more cluster flags of interest has been set.  These flags include:
+
+* *full* - the cluster is flagged as full and cannot serve writes
+* *pauserd*, *pausewr* - paused reads or writes
+* *noup* - OSDs are not allowed to start
+* *nodown* - OSD failure reports are being ignored, such that the
+  monitors will not mark OSDs `down`
+* *noin* - OSDs that were previously marked `out` will not be marked
+  back `in` when they start
+* *noout* - down OSDs will not automatically be marked out after the
+  configured interval
+* *nobackfill*, *norecover*, *norebalance* - recovery or data
+  rebalancing is suspended
+* *noscrub*, *nodeep_scrub* - scrubbing is disabled
+* *notieragent* - cache tiering activity is suspended
+
+With the exception of *full*, these flags can be set or cleared with:
+
+.. prompt:: bash $
+
+   ceph osd set <flag>
+   ceph osd unset <flag>
+
+OSD_FLAGS
+_________
+
+One or more OSDs or CRUSH {nodes,device classes} has a flag of interest set.
+These flags include:
+
+* *noup*: these OSDs are not allowed to start
+* *nodown*: failure reports for these OSDs will be ignored
+* *noin*: if these OSDs were previously marked `out` automatically
+  after a failure, they will not be marked in when they start
+* *noout*: if these OSDs are down they will not automatically be marked
+  `out` after the configured interval
+
+These flags can be set and cleared in batch with:
+
+.. prompt:: bash $
+
+   ceph osd set-group <flags> <who>
+   ceph osd unset-group <flags> <who>
+
+For example:
+
+.. prompt:: bash $
+
+   ceph osd set-group noup,noout osd.0 osd.1
+   ceph osd unset-group noup,noout osd.0 osd.1
+   ceph osd set-group noup,noout host-foo
+   ceph osd unset-group noup,noout host-foo
+   ceph osd set-group noup,noout class-hdd
+   ceph osd unset-group noup,noout class-hdd
+
+OLD_CRUSH_TUNABLES
+__________________
+
+The CRUSH map is using very old settings and should be updated.  The
+oldest tunables that can be used (i.e., the oldest client version that
+can connect to the cluster) without triggering this health warning is
+determined by the ``mon_crush_min_required_version`` config option.
+See :ref:`crush-map-tunables` for more information.
+
+OLD_CRUSH_STRAW_CALC_VERSION
+____________________________
+
+The CRUSH map is using an older, non-optimal method for calculating
+intermediate weight values for ``straw`` buckets.
+
+The CRUSH map should be updated to use the newer method
+(``straw_calc_version=1``).  See
+:ref:`crush-map-tunables` for more information.
+
+CACHE_POOL_NO_HIT_SET
+_____________________
+
+One or more cache pools is not configured with a *hit set* to track
+utilization, which will prevent the tiering agent from identifying
+cold objects to flush and evict from the cache.
+
+Hit sets can be configured on the cache pool with:
+
+.. prompt:: bash $
+
+   ceph osd pool set <poolname> hit_set_type <type>
+   ceph osd pool set <poolname> hit_set_period <period-in-seconds>
+   ceph osd pool set <poolname> hit_set_count <number-of-hitsets>
+   ceph osd pool set <poolname> hit_set_fpp <target-false-positive-rate>
+
+OSD_NO_SORTBITWISE
+__________________
+
+No pre-luminous v12.y.z OSDs are running but the ``sortbitwise`` flag has not
+been set.
+
+The ``sortbitwise`` flag must be set before luminous v12.y.z or newer
+OSDs can start.  You can safely set the flag with:
+
+.. prompt:: bash $
+
+   ceph osd set sortbitwise
+
+OSD_FILESTORE
+__________________
+
+Filestore has been deprecated, considering that Bluestore has been the default
+objectstore for quite some time. Warn if OSDs are running Filestore.
+
+The 'mclock_scheduler' is not supported for filestore OSDs. Therefore, the
+default 'osd_op_queue' is set to 'wpq' for filestore OSDs and is enforced
+even if the user attempts to change it.
+
+Filestore OSDs can be listed with:
+
+.. prompt:: bash $
+
+   ceph report | jq -c '."osd_metadata" | .[] | select(.osd_objectstore | contains("filestore")) | {id, osd_objectstore}'
+
+If it is not feasible to migrate Filestore OSDs to Bluestore immediately, you
+can silence this warning temporarily with:
+
+.. prompt:: bash $
+
+   ceph health mute OSD_FILESTORE
+
+POOL_FULL
+_________
+
+One or more pools has reached its quota and is no longer allowing writes.
+
+Pool quotas and utilization can be seen with:
+
+.. prompt:: bash $
+
+   ceph df detail
+
+You can either raise the pool quota with:
+
+.. prompt:: bash $
+
+   ceph osd pool set-quota <poolname> max_objects <num-objects>
+   ceph osd pool set-quota <poolname> max_bytes <num-bytes>
+
+or delete some existing data to reduce utilization.
+
+BLUEFS_SPILLOVER
+________________
+
+One or more OSDs that use the BlueStore backend have been allocated
+`db` partitions (storage space for metadata, normally on a faster
+device) but that space has filled, such that metadata has "spilled
+over" onto the normal slow device.  This isn't necessarily an error
+condition or even unexpected, but if the administrator's expectation
+was that all metadata would fit on the faster device, it indicates
+that not enough space was provided.
+
+This warning can be disabled on all OSDs with:
+
+.. prompt:: bash $
+
+   ceph config set osd bluestore_warn_on_bluefs_spillover false
+
+Alternatively, it can be disabled on a specific OSD with:
+
+.. prompt:: bash $
+
+   ceph config set osd.123 bluestore_warn_on_bluefs_spillover false
+
+To provide more metadata space, the OSD in question could be destroyed and
+reprovisioned.  This will involve data migration and recovery.
+
+It may also be possible to expand the LVM logical volume backing the
+`db` storage.  If the underlying LV has been expanded, the OSD daemon
+needs to be stopped and BlueFS informed of the device size change with:
+
+.. prompt:: bash $
+
+   ceph-bluestore-tool bluefs-bdev-expand --path /var/lib/ceph/osd/ceph-$ID
+
+BLUEFS_AVAILABLE_SPACE
+______________________
+
+To check how much space is free for BlueFS do:
+
+.. prompt:: bash $
+
+   ceph daemon osd.123 bluestore bluefs available
+
+This will output up to 3 values: `BDEV_DB free`, `BDEV_SLOW free` and
+`available_from_bluestore`. `BDEV_DB` and `BDEV_SLOW` report amount of space that
+has been acquired by BlueFS and is considered free. Value `available_from_bluestore`
+denotes ability of BlueStore to relinquish more space to BlueFS.
+It is normal that this value is different from amount of BlueStore free space, as
+BlueFS allocation unit is typically larger than BlueStore allocation unit.
+This means that only part of BlueStore free space will be acceptable for BlueFS.
+
+BLUEFS_LOW_SPACE
+_________________
+
+If BlueFS is running low on available free space and there is little
+`available_from_bluestore` one can consider reducing BlueFS allocation unit size.
+To simulate available space when allocation unit is different do:
+
+.. prompt:: bash $
+
+   ceph daemon osd.123 bluestore bluefs available <alloc-unit-size>
+
+BLUESTORE_FRAGMENTATION
+_______________________
+
+As BlueStore works free space on underlying storage will get fragmented.
+This is normal and unavoidable but excessive fragmentation will cause slowdown.
+To inspect BlueStore fragmentation one can do:
+
+.. prompt:: bash $
+
+   ceph daemon osd.123 bluestore allocator score block
+
+Score is given in [0-1] range.
+[0.0 .. 0.4] tiny fragmentation
+[0.4 .. 0.7] small, acceptable fragmentation
+[0.7 .. 0.9] considerable, but safe fragmentation
+[0.9 .. 1.0] severe fragmentation, may impact BlueFS ability to get space from BlueStore
+
+If detailed report of free fragments is required do:
+
+.. prompt:: bash $
+
+   ceph daemon osd.123 bluestore allocator dump block
+
+In case when handling OSD process that is not running fragmentation can be
+inspected with `ceph-bluestore-tool`.
+Get fragmentation score:
+
+.. prompt:: bash $
+
+   ceph-bluestore-tool --path /var/lib/ceph/osd/ceph-123 --allocator block free-score
+
+And dump detailed free chunks:
+
+.. prompt:: bash $
+
+   ceph-bluestore-tool --path /var/lib/ceph/osd/ceph-123 --allocator block free-dump
+
+BLUESTORE_LEGACY_STATFS
+_______________________
+
+In the Nautilus release, BlueStore tracks its internal usage
+statistics on a per-pool granular basis, and one or more OSDs have
+BlueStore volumes that were created prior to Nautilus.  If *all* OSDs
+are older than Nautilus, this just means that the per-pool metrics are
+not available.  However, if there is a mix of pre-Nautilus and
+post-Nautilus OSDs, the cluster usage statistics reported by ``ceph
+df`` will not be accurate.
+
+The old OSDs can be updated to use the new usage tracking scheme by stopping each OSD, running a repair operation, and the restarting it.  For example, if ``osd.123`` needed to be updated,:
+
+.. prompt:: bash $
+
+   systemctl stop ceph-osd@123
+   ceph-bluestore-tool repair --path /var/lib/ceph/osd/ceph-123
+   systemctl start ceph-osd@123
+
+This warning can be disabled with:
+
+.. prompt:: bash $
+
+   ceph config set global bluestore_warn_on_legacy_statfs false
+
+BLUESTORE_NO_PER_POOL_OMAP
+__________________________
+
+Starting with the Octopus release, BlueStore tracks omap space utilization
+by pool, and one or more OSDs have volumes that were created prior to
+Octopus.  If all OSDs are not running BlueStore with the new tracking
+enabled, the cluster will report and approximate value for per-pool omap usage
+based on the most recent deep-scrub.
+
+The old OSDs can be updated to track by pool by stopping each OSD,
+running a repair operation, and the restarting it.  For example, if
+``osd.123`` needed to be updated,:
+
+.. prompt:: bash $
+
+   systemctl stop ceph-osd@123
+   ceph-bluestore-tool repair --path /var/lib/ceph/osd/ceph-123
+   systemctl start ceph-osd@123
+
+This warning can be disabled with:
+
+.. prompt:: bash $
+
+   ceph config set global bluestore_warn_on_no_per_pool_omap false
+
+BLUESTORE_NO_PER_PG_OMAP
+__________________________
+
+Starting with the Pacific release, BlueStore tracks omap space utilization
+by PG, and one or more OSDs have volumes that were created prior to
+Pacific.  Per-PG omap enables faster PG removal when PGs migrate.
+
+The older OSDs can be updated to track by PG by stopping each OSD,
+running a repair operation, and the restarting it.  For example, if
+``osd.123`` needed to be updated,:
+
+.. prompt:: bash $
+
+   systemctl stop ceph-osd@123
+   ceph-bluestore-tool repair --path /var/lib/ceph/osd/ceph-123
+   systemctl start ceph-osd@123
+
+This warning can be disabled with:
+
+.. prompt:: bash $
+
+   ceph config set global bluestore_warn_on_no_per_pg_omap false
+
+
+BLUESTORE_DISK_SIZE_MISMATCH
+____________________________
+
+One or more OSDs using BlueStore has an internal inconsistency between the size
+of the physical device and the metadata tracking its size.  This can lead to
+the OSD crashing in the future.
+
+The OSDs in question should be destroyed and reprovisioned.  Care should be
+taken to do this one OSD at a time, and in a way that doesn't put any data at
+risk.  For example, if osd ``$N`` has the error:
+
+.. prompt:: bash $
+
+   ceph osd out osd.$N
+   while ! ceph osd safe-to-destroy osd.$N ; do sleep 1m ; done
+   ceph osd destroy osd.$N
+   ceph-volume lvm zap /path/to/device
+   ceph-volume lvm create --osd-id $N --data /path/to/device
+
+BLUESTORE_NO_COMPRESSION
+________________________
+
+One or more OSDs is unable to load a BlueStore compression plugin.
+This can be caused by a broken installation, in which the ``ceph-osd``
+binary does not match the compression plugins, or a recent upgrade
+that did not include a restart of the ``ceph-osd`` daemon.
+
+Verify that the package(s) on the host running the OSD(s) in question
+are correctly installed and that the OSD daemon(s) have been
+restarted.  If the problem persists, check the OSD log for any clues
+as to the source of the problem.
+
+BLUESTORE_SPURIOUS_READ_ERRORS
+______________________________
+
+One or more OSDs using BlueStore detects spurious read errors at main device.
+BlueStore has recovered from these errors by retrying disk reads.
+Though this might show some issues with underlying hardware, I/O subsystem,
+etc.
+Which theoretically might cause permanent data corruption.
+Some observations on the root cause can be found at 
+https://tracker.ceph.com/issues/22464
+
+This alert doesn't require immediate response but corresponding host might need
+additional attention, e.g. upgrading to the latest OS/kernel versions and
+H/W resource utilization monitoring.
+
+This warning can be disabled on all OSDs with:
+
+.. prompt:: bash $
+
+   ceph config set osd bluestore_warn_on_spurious_read_errors false
+
+Alternatively, it can be disabled on a specific OSD with:
+
+.. prompt:: bash $
+
+   ceph config set osd.123 bluestore_warn_on_spurious_read_errors false
+
+
+Device health
+-------------
+
+DEVICE_HEALTH
+_____________
+
+One or more devices is expected to fail soon, where the warning
+threshold is controlled by the ``mgr/devicehealth/warn_threshold``
+config option.
+
+This warning only applies to OSDs that are currently marked "in", so
+the expected response to this failure is to mark the device "out" so
+that data is migrated off of the device, and then to remove the
+hardware from the system.  Note that the marking out is normally done
+automatically if ``mgr/devicehealth/self_heal`` is enabled based on
+the ``mgr/devicehealth/mark_out_threshold``.
+
+Device health can be checked with:
+
+.. prompt:: bash $
+
+   ceph device info <device-id>
+
+Device life expectancy is set by a prediction model run by
+the mgr or an by external tool via the command:
+
+.. prompt:: bash $
+
+   ceph device set-life-expectancy <device-id> <from> <to>
+
+You can change the stored life expectancy manually, but that usually
+doesn't accomplish anything as whatever tool originally set it will
+probably set it again, and changing the stored value does not affect
+the actual health of the hardware device.
+
+DEVICE_HEALTH_IN_USE
+____________________
+
+One or more devices is expected to fail soon and has been marked "out"
+of the cluster based on ``mgr/devicehealth/mark_out_threshold``, but it
+is still participating in one more PGs.  This may be because it was
+only recently marked "out" and data is still migrating, or because data
+cannot be migrated off for some reason (e.g., the cluster is nearly
+full, or the CRUSH hierarchy is such that there isn't another suitable
+OSD to migrate the data too).
+
+This message can be silenced by disabling the self heal behavior
+(setting ``mgr/devicehealth/self_heal`` to false), by adjusting the
+``mgr/devicehealth/mark_out_threshold``, or by addressing what is
+preventing data from being migrated off of the ailing device.
+
+DEVICE_HEALTH_TOOMANY
+_____________________
+
+Too many devices is expected to fail soon and the
+``mgr/devicehealth/self_heal`` behavior is enabled, such that marking
+out all of the ailing devices would exceed the clusters
+``mon_osd_min_in_ratio`` ratio that prevents too many OSDs from being
+automatically marked "out".
+
+This generally indicates that too many devices in your cluster are
+expected to fail soon and you should take action to add newer
+(healthier) devices before too many devices fail and data is lost.
+
+The health message can also be silenced by adjusting parameters like
+``mon_osd_min_in_ratio`` or ``mgr/devicehealth/mark_out_threshold``,
+but be warned that this will increase the likelihood of unrecoverable
+data loss in the cluster.
+
+
+Data health (pools & placement groups)
+--------------------------------------
+
+PG_AVAILABILITY
+_______________
+
+Data availability is reduced, meaning that the cluster is unable to
+service potential read or write requests for some data in the cluster.
+Specifically, one or more PGs is in a state that does not allow IO
+requests to be serviced.  Problematic PG states include *peering*,
+*stale*, *incomplete*, and the lack of *active* (if those conditions do not clear
+quickly).
+
+Detailed information about which PGs are affected is available from:
+
+.. prompt:: bash $
+
+   ceph health detail
+
+In most cases the root cause is that one or more OSDs is currently
+down; see the discussion for ``OSD_DOWN`` above.
+
+The state of specific problematic PGs can be queried with:
+
+.. prompt:: bash $
+
+   ceph tell <pgid> query
+
+PG_DEGRADED
+___________
+
+Data redundancy is reduced for some data, meaning the cluster does not
+have the desired number of replicas for all data (for replicated
+pools) or erasure code fragments (for erasure coded pools).
+Specifically, one or more PGs:
+
+* has the *degraded* or *undersized* flag set, meaning there are not
+  enough instances of that placement group in the cluster;
+* has not had the *clean* flag set for some time.
+
+Detailed information about which PGs are affected is available from:
+
+.. prompt:: bash $
+
+   ceph health detail
+
+In most cases the root cause is that one or more OSDs is currently
+down; see the dicussion for ``OSD_DOWN`` above.
+
+The state of specific problematic PGs can be queried with:
+
+.. prompt:: bash $
+
+   ceph tell <pgid> query
+
+
+PG_RECOVERY_FULL
+________________
+
+Data redundancy may be reduced or at risk for some data due to a lack
+of free space in the cluster.  Specifically, one or more PGs has the
+*recovery_toofull* flag set, meaning that the
+cluster is unable to migrate or recover data because one or more OSDs
+is above the *full* threshold.
+
+See the discussion for *OSD_FULL* above for steps to resolve this condition.
+
+PG_BACKFILL_FULL
+________________
+
+Data redundancy may be reduced or at risk for some data due to a lack
+of free space in the cluster.  Specifically, one or more PGs has the
+*backfill_toofull* flag set, meaning that the
+cluster is unable to migrate or recover data because one or more OSDs
+is above the *backfillfull* threshold.
+
+See the discussion for *OSD_BACKFILLFULL* above for
+steps to resolve this condition.
+
+PG_DAMAGED
+__________
+
+Data scrubbing has discovered some problems with data consistency in
+the cluster.  Specifically, one or more PGs has the *inconsistent* or
+*snaptrim_error* flag is set, indicating an earlier scrub operation
+found a problem, or that the *repair* flag is set, meaning a repair
+for such an inconsistency is currently in progress.
+
+See :doc:`pg-repair` for more information.
+
+OSD_SCRUB_ERRORS
+________________
+
+Recent OSD scrubs have uncovered inconsistencies. This error is generally
+paired with *PG_DAMAGED* (see above).
+
+See :doc:`pg-repair` for more information.
+
+OSD_TOO_MANY_REPAIRS
+____________________
+
+When a read error occurs and another replica is available it is used to repair
+the error immediately, so that the client can get the object data.  Scrub
+handles errors for data at rest.  In order to identify possible failing disks
+that aren't seeing scrub errors, a count of read repairs is maintained.  If
+it exceeds a config value threshold *mon_osd_warn_num_repaired* default 10,
+this health warning is generated.
+
+LARGE_OMAP_OBJECTS
+__________________
+
+One or more pools contain large omap objects as determined by
+``osd_deep_scrub_large_omap_object_key_threshold`` (threshold for number of keys
+to determine a large omap object) or
+``osd_deep_scrub_large_omap_object_value_sum_threshold`` (the threshold for
+summed size (bytes) of all key values to determine a large omap object) or both.
+More information on the object name, key count, and size in bytes can be found
+by searching the cluster log for 'Large omap object found'. Large omap objects
+can be caused by RGW bucket index objects that do not have automatic resharding
+enabled. Please see :ref:`RGW Dynamic Bucket Index Resharding
+<rgw_dynamic_bucket_index_resharding>` for more information on resharding.
+
+The thresholds can be adjusted with:
+
+.. prompt:: bash $
+
+   ceph config set osd osd_deep_scrub_large_omap_object_key_threshold <keys>
+   ceph config set osd osd_deep_scrub_large_omap_object_value_sum_threshold <bytes>
+
+CACHE_POOL_NEAR_FULL
+____________________
+
+A cache tier pool is nearly full.  Full in this context is determined
+by the ``target_max_bytes`` and ``target_max_objects`` properties on
+the cache pool.  Once the pool reaches the target threshold, write
+requests to the pool may block while data is flushed and evicted
+from the cache, a state that normally leads to very high latencies and
+poor performance.
+
+The cache pool target size can be adjusted with:
+
+.. prompt:: bash $
+
+   ceph osd pool set <cache-pool-name> target_max_bytes <bytes>
+   ceph osd pool set <cache-pool-name> target_max_objects <objects>
+
+Normal cache flush and evict activity may also be throttled due to reduced
+availability or performance of the base tier, or overall cluster load.
+
+TOO_FEW_PGS
+___________
+
+The number of PGs in use in the cluster is below the configurable
+threshold of ``mon_pg_warn_min_per_osd`` PGs per OSD.  This can lead
+to suboptimal distribution and balance of data across the OSDs in
+the cluster, and similarly reduce overall performance.
+
+This may be an expected condition if data pools have not yet been
+created.
+
+The PG count for existing pools can be increased or new pools can be created.
+Please refer to :ref:`choosing-number-of-placement-groups` for more
+information.
+
+POOL_PG_NUM_NOT_POWER_OF_TWO
+____________________________
+
+One or more pools has a ``pg_num`` value that is not a power of two.
+Although this is not strictly incorrect, it does lead to a less
+balanced distribution of data because some PGs have roughly twice as
+much data as others.
+
+This is easily corrected by setting the ``pg_num`` value for the
+affected pool(s) to a nearby power of two:
+
+.. prompt:: bash $
+
+   ceph osd pool set <pool-name> pg_num <value>
+
+This health warning can be disabled with:
+
+.. prompt:: bash $
+
+   ceph config set global mon_warn_on_pool_pg_num_not_power_of_two false
+
+POOL_TOO_FEW_PGS
+________________
+
+One or more pools should probably have more PGs, based on the amount
+of data that is currently stored in the pool.  This can lead to
+suboptimal distribution and balance of data across the OSDs in the
+cluster, and similarly reduce overall performance.  This warning is
+generated if the ``pg_autoscale_mode`` property on the pool is set to
+``warn``.
+
+To disable the warning, you can disable auto-scaling of PGs for the
+pool entirely with:
+
+.. prompt:: bash $
+
+   ceph osd pool set <pool-name> pg_autoscale_mode off
+
+To allow the cluster to automatically adjust the number of PGs,:
+
+.. prompt:: bash $
+
+   ceph osd pool set <pool-name> pg_autoscale_mode on
+
+You can also manually set the number of PGs for the pool to the
+recommended amount with:
+
+.. prompt:: bash $
+
+   ceph osd pool set <pool-name> pg_num <new-pg-num>
+
+Please refer to :ref:`choosing-number-of-placement-groups` and
+:ref:`pg-autoscaler` for more information.
+
+TOO_MANY_PGS
+____________
+
+The number of PGs in use in the cluster is above the configurable
+threshold of ``mon_max_pg_per_osd`` PGs per OSD.  If this threshold is
+exceed the cluster will not allow new pools to be created, pool `pg_num` to
+be increased, or pool replication to be increased (any of which would lead to
+more PGs in the cluster).  A large number of PGs can lead
+to higher memory utilization for OSD daemons, slower peering after
+cluster state changes (like OSD restarts, additions, or removals), and
+higher load on the Manager and Monitor daemons.
+
+The simplest way to mitigate the problem is to increase the number of
+OSDs in the cluster by adding more hardware.  Note that the OSD count
+used for the purposes of this health check is the number of "in" OSDs,
+so marking "out" OSDs "in" (if there are any) can also help:
+
+.. prompt:: bash $
+
+   ceph osd in <osd id(s)>
+
+Please refer to :ref:`choosing-number-of-placement-groups` for more
+information.
+
+POOL_TOO_MANY_PGS
+_________________
+
+One or more pools should probably have more PGs, based on the amount
+of data that is currently stored in the pool.  This can lead to higher
+memory utilization for OSD daemons, slower peering after cluster state
+changes (like OSD restarts, additions, or removals), and higher load
+on the Manager and Monitor daemons.  This warning is generated if the
+``pg_autoscale_mode`` property on the pool is set to ``warn``.
+
+To disable the warning, you can disable auto-scaling of PGs for the
+pool entirely with:
+
+.. prompt:: bash $
+
+   ceph osd pool set <pool-name> pg_autoscale_mode off
+
+To allow the cluster to automatically adjust the number of PGs,:
+
+.. prompt:: bash $
+
+   ceph osd pool set <pool-name> pg_autoscale_mode on
+
+You can also manually set the number of PGs for the pool to the
+recommended amount with:
+
+.. prompt:: bash $
+
+   ceph osd pool set <pool-name> pg_num <new-pg-num>
+
+Please refer to :ref:`choosing-number-of-placement-groups` and
+:ref:`pg-autoscaler` for more information.
+
+POOL_TARGET_SIZE_BYTES_OVERCOMMITTED
+____________________________________
+
+One or more pools have a ``target_size_bytes`` property set to
+estimate the expected size of the pool,
+but the value(s) exceed the total available storage (either by
+themselves or in combination with other pools' actual usage).
+
+This is usually an indication that the ``target_size_bytes`` value for
+the pool is too large and should be reduced or set to zero with:
+
+.. prompt:: bash $
+
+   ceph osd pool set <pool-name> target_size_bytes 0
+
+For more information, see :ref:`specifying_pool_target_size`.
+
+POOL_HAS_TARGET_SIZE_BYTES_AND_RATIO
+____________________________________
+
+One or more pools have both ``target_size_bytes`` and
+``target_size_ratio`` set to estimate the expected size of the pool.
+Only one of these properties should be non-zero. If both are set,
+``target_size_ratio`` takes precedence and ``target_size_bytes`` is
+ignored.
+
+To reset ``target_size_bytes`` to zero:
+
+.. prompt:: bash $
+
+   ceph osd pool set <pool-name> target_size_bytes 0
+
+For more information, see :ref:`specifying_pool_target_size`.
+
+TOO_FEW_OSDS
+____________
+
+The number of OSDs in the cluster is below the configurable
+threshold of ``osd_pool_default_size``.
+
+SMALLER_PGP_NUM
+_______________
+
+One or more pools has a ``pgp_num`` value less than ``pg_num``.  This
+is normally an indication that the PG count was increased without
+also increasing the placement behavior.
+
+This is sometimes done deliberately to separate out the `split` step
+when the PG count is adjusted from the data migration that is needed
+when ``pgp_num`` is changed.
+
+This is normally resolved by setting ``pgp_num`` to match ``pg_num``,
+triggering the data migration, with:
+
+.. prompt:: bash $
+
+   ceph osd pool set <pool> pgp_num <pg-num-value>
+
+MANY_OBJECTS_PER_PG
+___________________
+
+One or more pools has an average number of objects per PG that is
+significantly higher than the overall cluster average.  The specific
+threshold is controlled by the ``mon_pg_warn_max_object_skew``
+configuration value.
+
+This is usually an indication that the pool(s) containing most of the
+data in the cluster have too few PGs, and/or that other pools that do
+not contain as much data have too many PGs.  See the discussion of
+*TOO_MANY_PGS* above.
+
+The threshold can be raised to silence the health warning by adjusting
+the ``mon_pg_warn_max_object_skew`` config option on the managers.
+
+The health warning will be silenced for a particular pool if
+``pg_autoscale_mode`` is set to ``on``.
+
+POOL_APP_NOT_ENABLED
+____________________
+
+A pool exists that contains one or more objects but has not been
+tagged for use by a particular application.
+
+Resolve this warning by labeling the pool for use by an application.  For
+example, if the pool is used by RBD,:
+
+.. prompt:: bash $
+
+   rbd pool init <poolname>
+
+If the pool is being used by a custom application 'foo', you can also label
+via the low-level command:
+
+.. prompt:: bash $
+
+   ceph osd pool application enable foo
+
+For more information, see :ref:`associate-pool-to-application`.
+
+POOL_FULL
+_________
+
+One or more pools has reached (or is very close to reaching) its
+quota.  The threshold to trigger this error condition is controlled by
+the ``mon_pool_quota_crit_threshold`` configuration option.
+
+Pool quotas can be adjusted up or down (or removed) with:
+
+.. prompt:: bash $
+
+   ceph osd pool set-quota <pool> max_bytes <bytes>
+   ceph osd pool set-quota <pool> max_objects <objects>
+
+Setting the quota value to 0 will disable the quota.
+
+POOL_NEAR_FULL
+______________
+
+One or more pools is approaching a configured fullness threshold.
+
+One threshold that can trigger this warning condition is the
+``mon_pool_quota_warn_threshold`` configuration option.
+
+Pool quotas can be adjusted up or down (or removed) with:
+
+.. prompt:: bash $
+
+   ceph osd pool set-quota <pool> max_bytes <bytes>
+   ceph osd pool set-quota <pool> max_objects <objects>
+
+Setting the quota value to 0 will disable the quota.
+
+Other thresholds that can trigger the above two warning conditions are
+``mon_osd_nearfull_ratio`` and ``mon_osd_full_ratio``.  Visit the
+:ref:`storage-capacity` and :ref:`no-free-drive-space` documents for details
+and resolution.
+
+OBJECT_MISPLACED
+________________
+
+One or more objects in the cluster is not stored on the node the
+cluster would like it to be stored on.  This is an indication that
+data migration due to some recent cluster change has not yet completed.
+
+Misplaced data is not a dangerous condition in and of itself; data
+consistency is never at risk, and old copies of objects are never
+removed until the desired number of new copies (in the desired
+locations) are present.
+
+OBJECT_UNFOUND
+______________
+
+One or more objects in the cluster cannot be found.  Specifically, the
+OSDs know that a new or updated copy of an object should exist, but a
+copy of that version of the object has not been found on OSDs that are
+currently online.
+
+Read or write requests to unfound objects will block.
+
+Ideally, a down OSD can be brought back online that has the more
+recent copy of the unfound object.  Candidate OSDs can be identified from the
+peering state for the PG(s) responsible for the unfound object:
+
+.. prompt:: bash $
+
+   ceph tell <pgid> query
+
+If the latest copy of the object is not available, the cluster can be
+told to roll back to a previous version of the object. See
+:ref:`failures-osd-unfound` for more information.
+
+SLOW_OPS
+________
+
+One or more OSD or monitor requests is taking a long time to process.  This can
+be an indication of extreme load, a slow storage device, or a software
+bug.
+
+The request queue for the daemon in question can be queried with the
+following command, executed from the daemon's host:
+
+.. prompt:: bash $
+
+   ceph daemon osd.<id> ops
+
+A summary of the slowest recent requests can be seen with:
+
+.. prompt:: bash $
+
+   ceph daemon osd.<id> dump_historic_ops
+
+The location of an OSD can be found with:
+
+.. prompt:: bash $
+
+   ceph osd find osd.<id>
+
+PG_NOT_SCRUBBED
+_______________
+
+One or more PGs has not been scrubbed recently.  PGs are normally scrubbed
+within every configured interval specified by
+:ref:`osd_scrub_max_interval <osd_scrub_max_interval>` globally. This
+interval can be overriden on per-pool basis with
+:ref:`scrub_max_interval <scrub_max_interval>`. The warning triggers when
+``mon_warn_pg_not_scrubbed_ratio`` percentage of interval has elapsed without a
+scrub since it was due.
+
+PGs will not scrub if they are not flagged as *clean*, which may
+happen if they are misplaced or degraded (see *PG_AVAILABILITY* and
+*PG_DEGRADED* above).
+
+You can manually initiate a scrub of a clean PG with::
+
+  ceph pg scrub <pgid>
+
+PG_NOT_DEEP_SCRUBBED
+____________________
+
+One or more PGs has not been deep scrubbed recently.  PGs are normally
+scrubbed every ``osd_deep_scrub_interval`` seconds, and this warning
+triggers when ``mon_warn_pg_not_deep_scrubbed_ratio`` percentage of interval has elapsed
+without a scrub since it was due.
+
+PGs will not (deep) scrub if they are not flagged as *clean*, which may
+happen if they are misplaced or degraded (see *PG_AVAILABILITY* and
+*PG_DEGRADED* above).
+
+You can manually initiate a scrub of a clean PG with:
+
+.. prompt:: bash $
+
+   ceph pg deep-scrub <pgid>
+
+
+PG_SLOW_SNAP_TRIMMING
+_____________________
+
+The snapshot trim queue for one or more PGs has exceeded the
+configured warning threshold.  This indicates that either an extremely
+large number of snapshots were recently deleted, or that the OSDs are
+unable to trim snapshots quickly enough to keep up with the rate of
+new snapshot deletions.
+
+The warning threshold is controlled by the
+``mon_osd_snap_trim_queue_warn_on`` option (default: 32768).
+
+This warning may trigger if OSDs are under excessive load and unable
+to keep up with their background work, or if the OSDs' internal
+metadata database is heavily fragmented and unable to perform.  It may
+also indicate some other performance issue with the OSDs.
+
+The exact size of the snapshot trim queue is reported by the
+``snaptrimq_len`` field of ``ceph pg ls -f json-detail``.
+
+Miscellaneous
+-------------
+
+RECENT_CRASH
+____________
+
+One or more Ceph daemons has crashed recently, and the crash has not
+yet been archived (acknowledged) by the administrator.  This may
+indicate a software bug, a hardware problem (e.g., a failing disk), or
+some other problem.
+
+New crashes can be listed with:
+
+.. prompt:: bash $
+
+   ceph crash ls-new
+
+Information about a specific crash can be examined with:
+
+.. prompt:: bash $
+
+   ceph crash info <crash-id>
+
+This warning can be silenced by "archiving" the crash (perhaps after
+being examined by an administrator) so that it does not generate this
+warning:
+
+.. prompt:: bash $
+
+   ceph crash archive <crash-id>
+
+Similarly, all new crashes can be archived with:
+
+.. prompt:: bash $
+
+   ceph crash archive-all
+
+Archived crashes will still be visible via ``ceph crash ls`` but not
+``ceph crash ls-new``.
+
+The time period for what "recent" means is controlled by the option
+``mgr/crash/warn_recent_interval`` (default: two weeks).
+
+These warnings can be disabled entirely with:
+
+.. prompt:: bash $
+
+   ceph config set mgr/crash/warn_recent_interval 0
+
+RECENT_MGR_MODULE_CRASH
+_______________________
+
+One or more ceph-mgr modules has crashed recently, and the crash as
+not yet been archived (acknowledged) by the administrator.  This
+generally indicates a software bug in one of the software modules run
+inside the ceph-mgr daemon.  Although the module that experienced the
+problem maybe be disabled as a result, the function of other modules
+is normally unaffected.
+
+As with the *RECENT_CRASH* health alert, the crash can be inspected with:
+
+.. prompt:: bash $
+
+   ceph crash info <crash-id>
+
+This warning can be silenced by "archiving" the crash (perhaps after
+being examined by an administrator) so that it does not generate this
+warning:
+
+.. prompt:: bash $
+
+   ceph crash archive <crash-id>
+
+Similarly, all new crashes can be archived with:
+
+.. prompt:: bash $
+
+   ceph crash archive-all
+
+Archived crashes will still be visible via ``ceph crash ls`` but not
+``ceph crash ls-new``.
+
+The time period for what "recent" means is controlled by the option
+``mgr/crash/warn_recent_interval`` (default: two weeks).
+
+These warnings can be disabled entirely with:
+
+.. prompt:: bash $
+
+   ceph config set mgr/crash/warn_recent_interval 0
+
+TELEMETRY_CHANGED
+_________________
+
+Telemetry has been enabled, but the contents of the telemetry report
+have changed since that time, so telemetry reports will not be sent.
+
+The Ceph developers periodically revise the telemetry feature to
+include new and useful information, or to remove information found to
+be useless or sensitive.  If any new information is included in the
+report, Ceph will require the administrator to re-enable telemetry to
+ensure they have an opportunity to (re)review what information will be
+shared.
+
+To review the contents of the telemetry report:
+
+.. prompt:: bash $
+
+   ceph telemetry show
+
+Note that the telemetry report consists of several optional channels
+that may be independently enabled or disabled.  For more information, see
+:ref:`telemetry`.
+
+To re-enable telemetry (and make this warning go away):
+
+.. prompt:: bash $
+
+   ceph telemetry on
+
+To disable telemetry (and make this warning go away):
+
+.. prompt:: bash $
+
+   ceph telemetry off
+
+AUTH_BAD_CAPS
+_____________
+
+One or more auth users has capabilities that cannot be parsed by the
+monitor.  This generally indicates that the user will not be
+authorized to perform any action with one or more daemon types.
+
+This error is mostly likely to occur after an upgrade if the
+capabilities were set with an older version of Ceph that did not
+properly validate their syntax, or if the syntax of the capabilities
+has changed.
+
+The user in question can be removed with:
+
+.. prompt:: bash $
+
+   ceph auth rm <entity-name>
+
+(This will resolve the health alert, but obviously clients will not be
+able to authenticate as that user.)
+
+Alternatively, the capabilities for the user can be updated with:
+
+.. prompt:: bash $
+
+   ceph auth <entity-name> <daemon-type> <caps> [<daemon-type> <caps> ...]
+
+For more information about auth capabilities, see :ref:`user-management`.
+
+OSD_NO_DOWN_OUT_INTERVAL
+________________________
+
+The ``mon_osd_down_out_interval`` option is set to zero, which means
+that the system will not automatically perform any repair or healing
+operations after an OSD fails.  Instead, an administrator (or some
+other external entity) will need to manually mark down OSDs as 'out'
+(i.e., via ``ceph osd out <osd-id>``) in order to trigger recovery.
+
+This option is normally set to five or ten minutes--enough time for a
+host to power-cycle or reboot.
+
+This warning can silenced by setting the
+``mon_warn_on_osd_down_out_interval_zero`` to false:
+
+.. prompt:: bash $
+
+   ceph config global mon mon_warn_on_osd_down_out_interval_zero false
+
+DASHBOARD_DEBUG
+_______________
+
+The Dashboard debug mode is enabled. This means, if there is an error
+while processing a REST API request, the HTTP error response contains
+a Python traceback. This behaviour should be disabled in production
+environments because such a traceback might contain and expose sensible
+information.
+
+The debug mode can be disabled with:
+
+.. prompt:: bash $
+
+   ceph dashboard debug disable
diff --git a/doc/rados/operations/index.rst b/doc/rados/operations/index.rst
new file mode 100644
index 000000000..2136918c7
--- /dev/null
+++ b/doc/rados/operations/index.rst
@@ -0,0 +1,98 @@
+.. _rados-operations:
+
+====================
+ Cluster Operations
+====================
+
+.. raw:: html
+
+	<table><colgroup><col width="50%"><col width="50%"></colgroup><tbody valign="top"><tr><td><h3>High-level Operations</h3>
+
+High-level cluster operations consist primarily of starting, stopping, and
+restarting a cluster with the ``ceph`` service;  checking the cluster's health;
+and, monitoring an operating cluster.
+
+.. toctree::
+	:maxdepth: 1
+
+	operating
+	health-checks
+	monitoring
+	monitoring-osd-pg
+	user-management
+	pg-repair
+
+.. raw:: html
+
+	</td><td><h3>Data Placement</h3>
+
+Once you have your cluster up and running, you may begin working with data
+placement. Ceph supports petabyte-scale data storage clusters, with storage
+pools and placement groups that distribute data across the cluster using Ceph's
+CRUSH algorithm.
+
+.. toctree::
+	:maxdepth: 1
+
+	data-placement
+	pools
+	erasure-code
+	cache-tiering
+	placement-groups
+	balancer
+	upmap
+	crush-map
+	crush-map-edits
+	stretch-mode
+	change-mon-elections
+
+
+
+.. raw:: html
+
+	</td></tr><tr><td><h3>Low-level Operations</h3>
+
+Low-level cluster operations consist of starting, stopping, and restarting a
+particular daemon within a cluster; changing the settings of a particular
+daemon or subsystem; and, adding a daemon to the cluster or removing a	daemon
+from the cluster. The most common use cases for low-level operations include
+growing or shrinking the Ceph cluster and replacing legacy or failed hardware
+with new hardware.
+
+.. toctree::
+	:maxdepth: 1
+
+	add-or-rm-osds
+	add-or-rm-mons
+	devices
+	bluestore-migration
+	Command Reference <control>
+
+
+
+.. raw:: html
+
+	</td><td><h3>Troubleshooting</h3>
+
+Ceph is still on the leading edge, so you may encounter situations that require
+you to evaluate your Ceph configuration and modify your logging and debugging
+settings to identify and remedy issues you are encountering with your cluster.
+
+.. toctree::
+	:maxdepth: 1
+
+	../troubleshooting/community
+	../troubleshooting/troubleshooting-mon
+	../troubleshooting/troubleshooting-osd
+	../troubleshooting/troubleshooting-pg
+	../troubleshooting/log-and-debug
+	../troubleshooting/cpu-profiling
+	../troubleshooting/memory-profiling
+
+
+
+
+.. raw:: html
+
+	</td></tr></tbody></table>
+
diff --git a/doc/rados/operations/monitoring-osd-pg.rst b/doc/rados/operations/monitoring-osd-pg.rst
new file mode 100644
index 000000000..3b997bfb4
--- /dev/null
+++ b/doc/rados/operations/monitoring-osd-pg.rst
@@ -0,0 +1,553 @@
+=========================
+ Monitoring OSDs and PGs
+=========================
+
+High availability and high reliability require a fault-tolerant approach to
+managing hardware and software issues. Ceph has no single point-of-failure, and
+can service requests for data in a "degraded" mode. Ceph's `data placement`_
+introduces a layer of indirection to ensure that data doesn't bind directly to
+particular OSD addresses. This means that tracking down system faults requires
+finding the `placement group`_ and the underlying OSDs at root of the problem.
+
+.. tip:: A fault in one part of the cluster may prevent you from accessing a 
+   particular object, but that doesn't mean that you cannot access other objects.
+   When you run into a fault, don't panic. Just follow the steps for monitoring
+   your OSDs and placement groups. Then, begin troubleshooting.
+
+Ceph is generally self-repairing. However, when problems persist, monitoring
+OSDs and placement groups will help you identify the problem.
+
+
+Monitoring OSDs
+===============
+
+An OSD's status is either in the cluster (``in``) or out of the cluster
+(``out``); and, it is either up and running (``up``), or it is down and not
+running (``down``). If an OSD is ``up``, it may be either ``in`` the cluster
+(you can read and write data) or it is ``out`` of the cluster.  If it was
+``in`` the cluster and recently moved ``out`` of the cluster, Ceph will migrate
+placement groups to other OSDs. If an OSD is ``out`` of the cluster, CRUSH will
+not assign placement groups to the OSD. If an OSD is ``down``, it should also be
+``out``.
+
+.. note:: If an OSD is ``down`` and ``in``, there is a problem and the cluster 
+   will not be in a healthy state.
+
+.. ditaa::
+
+           +----------------+        +----------------+
+           |                |        |                |
+           |   OSD #n In    |        |   OSD #n Up    |
+           |                |        |                |
+           +----------------+        +----------------+
+                   ^                         ^
+                   |                         |
+                   |                         |
+                   v                         v
+           +----------------+        +----------------+
+           |                |        |                |
+           |   OSD #n Out   |        |   OSD #n Down  |
+           |                |        |                |
+           +----------------+        +----------------+
+
+If you execute a command such as ``ceph health``, ``ceph -s`` or ``ceph -w``,
+you may notice that the cluster does not always echo back ``HEALTH OK``. Don't
+panic. With respect to OSDs, you should expect that the cluster will **NOT**
+echo   ``HEALTH OK`` in a few expected circumstances:
+
+#. You haven't started the cluster yet (it won't respond).
+#. You have just started or restarted the cluster and it's not ready yet,
+   because the placement groups are getting created and the OSDs are in
+   the process of peering.
+#. You just added or removed an OSD.
+#. You just have modified your cluster map.
+
+An important aspect of monitoring OSDs is to ensure that when the cluster
+is up and running that all OSDs that are ``in`` the cluster are ``up`` and
+running, too. To see if all OSDs are running, execute:
+
+.. prompt:: bash $
+
+	ceph osd stat
+
+The result should tell you the total number of OSDs (x),
+how many are ``up`` (y), how many are ``in`` (z) and the map epoch (eNNNN). ::
+
+	x osds: y up, z in; epoch: eNNNN
+
+If the number of OSDs that are ``in`` the cluster is more than the number of
+OSDs that are ``up``, execute the following command to identify the ``ceph-osd``
+daemons that are not running:
+
+.. prompt:: bash $
+
+	ceph osd tree
+
+:: 
+
+	#ID CLASS WEIGHT  TYPE NAME             STATUS REWEIGHT PRI-AFF
+	 -1       2.00000 pool openstack
+	 -3       2.00000 rack dell-2950-rack-A
+	 -2       2.00000 host dell-2950-A1
+	  0   ssd 1.00000      osd.0                up  1.00000 1.00000
+	  1   ssd 1.00000      osd.1              down  1.00000 1.00000
+
+.. tip:: The ability to search through a well-designed CRUSH hierarchy may help
+   you troubleshoot your cluster by identifying the physical locations faster.
+
+If an OSD is ``down``, start it:
+
+.. prompt:: bash $
+
+	sudo systemctl start ceph-osd@1
+
+See `OSD Not Running`_ for problems associated with OSDs that stopped, or won't
+restart.
+	
+
+PG Sets
+=======
+
+When CRUSH assigns placement groups to OSDs, it looks at the number of replicas
+for the pool and assigns the placement group to OSDs such that each replica of
+the placement group gets assigned to a different OSD. For example, if the pool
+requires three replicas of a placement group, CRUSH may assign them to
+``osd.1``, ``osd.2`` and ``osd.3`` respectively. CRUSH actually seeks a
+pseudo-random placement that will take into account failure domains you set in
+your `CRUSH map`_, so you will rarely see placement groups assigned to nearest
+neighbor OSDs in a large cluster.
+
+Ceph processes a client request using the **Acting Set**, which is the set of
+OSDs that will actually handle the requests since they have a full and working
+version of a placement group shard. The set of OSDs that should contain a shard
+of a particular placement group as the **Up Set**, i.e. where data is
+moved/copied to (or planned to be).
+
+In some cases, an OSD in the Acting Set is ``down`` or otherwise not able to
+service requests for objects in the placement group. When these situations
+arise, don't panic. Common examples include:
+
+- You added or removed an OSD. Then, CRUSH reassigned the placement group to 
+  other OSDs--thereby changing the composition of the Acting Set and spawning
+  the migration of data with a "backfill" process.
+- An OSD was ``down``, was restarted, and is now ``recovering``.
+- An OSD in the Acting Set is ``down`` or unable to service requests, 
+  and another OSD has temporarily assumed its duties.
+
+In most cases, the Up Set and the Acting Set are identical. When they are not,
+it may indicate that Ceph is migrating the PG (it's remapped), an OSD is
+recovering, or that there is a problem (i.e., Ceph usually echoes a "HEALTH
+WARN" state with a "stuck stale" message in such scenarios).
+
+To retrieve a list of placement groups, execute:
+
+.. prompt:: bash $
+
+	ceph pg dump
+	
+To view which OSDs are within the Acting Set or the Up Set for a given placement
+group, execute:
+
+.. prompt:: bash $
+
+	ceph pg map {pg-num}
+
+The result should tell you the osdmap epoch (eNNN), the placement group number
+({pg-num}), the OSDs in the Up Set (up[]), and the OSDs in the acting set
+(acting[])::
+
+	osdmap eNNN pg {raw-pg-num} ({pg-num}) -> up [0,1,2] acting [0,1,2]
+
+.. note:: If the Up Set and Acting Set do not match, this may be an indicator
+   that the cluster rebalancing itself or of a potential problem with 
+   the cluster.
+ 
+
+Peering
+=======
+
+Before you can write data to a placement group, it must be in an ``active``
+state, and it  **should** be in a ``clean`` state. For Ceph to determine the
+current state of a placement group, the primary OSD of the placement group
+(i.e., the first OSD in the acting set), peers with the secondary and tertiary
+OSDs to establish agreement on the current state of the placement group
+(assuming a pool with 3 replicas of the PG).
+
+
+.. ditaa::
+
+           +---------+     +---------+     +-------+
+           |  OSD 1  |     |  OSD 2  |     | OSD 3 |
+           +---------+     +---------+     +-------+
+                |               |              |
+                |  Request To   |              |
+                |     Peer      |              |             
+                |-------------->|              |
+                |<--------------|              |
+                |    Peering                   |
+                |                              |
+                |         Request To           |
+                |            Peer              | 
+                |----------------------------->|  
+                |<-----------------------------|
+                |          Peering             |
+
+The OSDs also report their status to the monitor. See `Configuring Monitor/OSD
+Interaction`_ for details. To troubleshoot peering issues, see `Peering
+Failure`_.
+
+
+Monitoring Placement Group States
+=================================
+
+If you execute a command such as ``ceph health``, ``ceph -s`` or ``ceph -w``,
+you may notice that the cluster does not always echo back ``HEALTH OK``. After
+you check to see if the OSDs are running, you should also check placement group
+states. You should expect that the cluster will **NOT** echo ``HEALTH OK`` in a
+number of placement group peering-related circumstances:
+
+#. You have just created a pool and placement groups haven't peered yet.
+#. The placement groups are recovering.
+#. You have just added an OSD to or removed an OSD from the cluster.
+#. You have just modified your CRUSH map and your placement groups are migrating.
+#. There is inconsistent data in different replicas of a placement group.
+#. Ceph is scrubbing a placement group's replicas.
+#. Ceph doesn't have enough storage capacity to complete backfilling operations.
+
+If one of the foregoing circumstances causes Ceph to echo ``HEALTH WARN``, don't
+panic. In many cases, the cluster will recover on its own. In some cases, you
+may need to take action. An important aspect of monitoring placement groups is
+to ensure that when the cluster is up and running that all placement groups are
+``active``, and preferably in the ``clean`` state. To see the status of all
+placement groups, execute:
+
+.. prompt:: bash $
+
+	ceph pg stat
+
+The result should tell you the total number of placement groups (x), how many
+placement groups are in a particular state such as ``active+clean`` (y) and the
+amount of data stored (z). ::
+
+	x pgs: y active+clean; z bytes data, aa MB used, bb GB / cc GB avail
+
+.. note:: It is common for Ceph to report multiple states for placement groups.
+
+In addition to the placement group states, Ceph will also echo back the amount of
+storage capacity used (aa), the amount of storage capacity remaining (bb), and the total
+storage capacity for the placement group. These numbers can be important in a
+few cases: 
+
+- You are reaching your ``near full ratio`` or ``full ratio``. 
+- Your data is not getting distributed across the cluster due to an 
+  error in your CRUSH configuration.
+
+
+.. topic:: Placement Group IDs
+
+   Placement group IDs consist of the pool number (not pool name) followed 
+   by a period (.) and the placement group ID--a hexadecimal number. You
+   can view pool numbers and their names from the output of ``ceph osd 
+   lspools``. For example, the first pool created corresponds to
+   pool number ``1``. A fully qualified placement group ID has the
+   following form::
+   
+   	{pool-num}.{pg-id}
+   
+   And it typically looks like this:: 
+   
+	1.1f
+   
+
+To retrieve a list of placement groups, execute the following:
+
+.. prompt:: bash $
+
+	ceph pg dump
+	
+You can also format the output in JSON format and save it to a file:
+
+.. prompt:: bash $
+
+	ceph pg dump -o {filename} --format=json
+
+To query a particular placement group, execute the following:
+
+.. prompt:: bash $
+
+	ceph pg {poolnum}.{pg-id} query
+	
+Ceph will output the query in JSON format.
+
+The following subsections describe the common pg states in detail.
+
+Creating
+--------
+
+When you create a pool, it will create the number of placement groups you
+specified.  Ceph will echo ``creating`` when it is creating one or more
+placement groups. Once they are created, the OSDs that are part of a placement
+group's Acting Set will peer. Once peering is complete, the placement group
+status should be ``active+clean``, which means a Ceph client can begin writing
+to the placement group.
+
+.. ditaa::
+
+       /-----------\       /-----------\       /-----------\
+       | Creating  |------>|  Peering  |------>|  Active   |
+       \-----------/       \-----------/       \-----------/
+
+Peering
+-------
+
+When Ceph is Peering a placement group, Ceph is bringing the OSDs that
+store the replicas of the placement group into **agreement about the state**
+of the objects and metadata in the placement group. When Ceph completes peering,
+this means that the OSDs that store the placement group agree about the current
+state of the placement group. However, completion of the peering process does
+**NOT** mean that each replica has the latest contents.
+
+.. topic:: Authoritative History
+
+   Ceph will **NOT** acknowledge a write operation to a client, until 
+   all OSDs of the acting set persist the write operation. This practice 
+   ensures that at least one member of the acting set will have a record 
+   of every acknowledged write operation since the last successful 
+   peering operation.
+   
+   With an accurate record of each acknowledged write operation, Ceph can 
+   construct and disseminate a new authoritative history of the placement 
+   group--a complete, and fully ordered set of operations that, if performed, 
+   would bring an OSD’s copy of a placement group up to date.
+
+
+Active
+------
+
+Once Ceph completes the peering process, a placement group may become
+``active``. The ``active`` state means that the data in the placement group is
+generally  available in the primary placement group and the replicas for read
+and write operations. 
+
+
+Clean 
+-----
+
+When a placement group is in the ``clean`` state, the primary OSD and the
+replica OSDs have successfully peered and there are no stray replicas for the
+placement group. Ceph replicated all objects in the placement group the correct 
+number of times.
+
+
+Degraded
+--------
+
+When a client writes an object to the primary OSD, the primary OSD is
+responsible for writing the replicas to the replica OSDs. After the primary OSD
+writes the object to storage, the placement group will remain in a ``degraded``
+state until the primary OSD has received an acknowledgement from the replica
+OSDs that Ceph created the replica objects successfully. 
+
+The reason a placement group can be ``active+degraded`` is that an OSD may be
+``active`` even though it doesn't hold all of the objects yet. If an OSD goes
+``down``, Ceph marks each placement group assigned to the OSD as ``degraded``.
+The OSDs must peer again when the OSD comes back online. However, a client can
+still write a new object to a ``degraded`` placement group if it is ``active``.
+
+If an OSD is ``down`` and the ``degraded`` condition persists, Ceph may mark the
+``down`` OSD as ``out`` of the cluster and remap the data from the ``down`` OSD
+to another OSD. The time between being marked ``down`` and being marked ``out``
+is controlled by ``mon osd down out interval``, which is set to ``600`` seconds
+by default.
+
+A placement group can also be ``degraded``, because Ceph cannot find one or more
+objects that Ceph thinks should be in the placement group. While you cannot
+read or write to unfound objects, you can still access all of the other objects
+in the ``degraded`` placement group.
+
+
+Recovering
+----------
+
+Ceph was designed for fault-tolerance at a scale where hardware and software
+problems are ongoing. When an OSD goes ``down``, its contents may fall behind
+the current state of other replicas in the placement groups. When the OSD is
+back ``up``, the contents of the placement groups must be updated to reflect the
+current state. During that time period, the OSD may reflect a ``recovering``
+state.
+
+Recovery is not always trivial, because a hardware failure might cause a
+cascading failure of multiple OSDs. For example, a network switch for a rack or
+cabinet may fail, which can cause the OSDs of a number of host machines to fall
+behind the current state  of the cluster. Each one of the OSDs must recover once
+the fault is resolved.
+
+Ceph provides a number of settings to balance the resource contention between
+new service requests and the need to recover data objects and restore the
+placement groups to the current state. The ``osd recovery delay start`` setting
+allows an OSD to restart, re-peer and even process some replay requests before
+starting the recovery process.  The ``osd
+recovery thread timeout`` sets a thread timeout, because multiple OSDs may fail,
+restart and re-peer at staggered rates. The ``osd recovery max active`` setting
+limits the  number of recovery requests an OSD will entertain simultaneously to
+prevent the OSD from failing to serve . The ``osd recovery max chunk`` setting
+limits the size of the recovered data chunks to prevent network congestion.
+
+
+Back Filling
+------------
+
+When a new OSD joins the cluster, CRUSH will reassign placement groups from OSDs
+in the cluster to the newly added OSD. Forcing the new OSD to accept the
+reassigned placement groups immediately can put excessive load on the new OSD.
+Back filling the OSD with the placement groups allows this process to begin in
+the background.  Once backfilling is complete, the new OSD will begin serving
+requests when it is ready.
+
+During the backfill operations, you may see one of several states:
+``backfill_wait`` indicates that a backfill operation is pending, but is not
+underway yet; ``backfilling`` indicates that a backfill operation is underway;
+and, ``backfill_toofull`` indicates that a backfill operation was requested,
+but couldn't be completed due to insufficient storage capacity. When a 
+placement group cannot be backfilled, it may be considered ``incomplete``.
+
+The ``backfill_toofull`` state may be transient.  It is possible that as PGs
+are moved around, space may become available.  The ``backfill_toofull`` is
+similar to ``backfill_wait`` in that as soon as conditions change
+backfill can proceed.
+
+Ceph provides a number of settings to manage the load spike associated with
+reassigning placement groups to an OSD (especially a new OSD). By default,
+``osd_max_backfills`` sets the maximum number of concurrent backfills to and from
+an OSD to 1. The ``backfill full ratio`` enables an OSD to refuse a
+backfill request if the OSD is approaching its full ratio (90%, by default) and
+change with ``ceph osd set-backfillfull-ratio`` command.
+If an OSD refuses a backfill request, the ``osd backfill retry interval``
+enables an OSD to retry the request (after 30 seconds, by default). OSDs can
+also set ``osd backfill scan min`` and ``osd backfill scan max`` to manage scan
+intervals (64 and 512, by default).
+
+
+Remapped
+--------
+
+When the Acting Set that services a placement group changes, the data migrates
+from the old acting set to the new acting set. It may take some time for a new
+primary OSD to service requests. So it may ask the old primary to continue to
+service requests until the placement group migration is complete. Once  data
+migration completes, the mapping uses the primary OSD of the new acting set.
+
+
+Stale
+-----
+
+While Ceph uses heartbeats to ensure that hosts and daemons are running, the
+``ceph-osd`` daemons may also get into a ``stuck`` state where they are not
+reporting statistics in a timely manner (e.g., a temporary network fault). By
+default, OSD daemons report their placement group, up through, boot and failure
+statistics every half second (i.e., ``0.5``),  which is more frequent than the
+heartbeat thresholds. If the **Primary OSD** of a placement group's acting set
+fails to report to the monitor or if other OSDs have reported the primary OSD
+``down``, the monitors will mark the placement group ``stale``.
+
+When you start your cluster, it is common to see the ``stale`` state until
+the peering process completes. After your cluster has been running for awhile, 
+seeing placement groups in the ``stale`` state indicates that the primary OSD
+for those placement groups is ``down`` or not reporting placement group statistics
+to the monitor.
+
+
+Identifying Troubled PGs
+========================
+
+As previously noted, a placement group is not necessarily problematic just 
+because its state is not ``active+clean``. Generally, Ceph's ability to self
+repair may not be working when placement groups get stuck. The stuck states
+include:
+
+- **Unclean**: Placement groups contain objects that are not replicated the 
+  desired number of times. They should be recovering.
+- **Inactive**: Placement groups cannot process reads or writes because they 
+  are waiting for an OSD with the most up-to-date data to come back ``up``.
+- **Stale**: Placement groups are in an unknown state, because the OSDs that 
+  host them have not reported to the monitor cluster in a while (configured 
+  by ``mon osd report timeout``).
+
+To identify stuck placement groups, execute the following:
+
+.. prompt:: bash $
+
+	ceph pg dump_stuck [unclean|inactive|stale|undersized|degraded]
+
+See `Placement Group Subsystem`_ for additional details. To troubleshoot
+stuck placement groups, see `Troubleshooting PG Errors`_.
+
+
+Finding an Object Location
+==========================
+
+To store object data in the Ceph Object Store, a Ceph client must: 
+
+#. Set an object name
+#. Specify a `pool`_
+
+The Ceph client retrieves the latest cluster map and the CRUSH algorithm
+calculates how to map the object to a `placement group`_, and then calculates
+how to assign the placement group to an OSD dynamically. To find the object
+location, all you need is the object name and the pool name. For example:
+
+.. prompt:: bash $
+
+	ceph osd map {poolname} {object-name} [namespace]
+
+.. topic:: Exercise: Locate an Object
+
+        As an exercise, let's create an object. Specify an object name, a path
+        to a test file containing some object data and a pool name using the
+        ``rados put`` command on the command line. For example:
+
+        .. prompt:: bash $
+   
+		rados put {object-name} {file-path} --pool=data   	
+		rados put test-object-1 testfile.txt --pool=data
+   
+        To verify that the Ceph Object Store stored the object, execute the
+        following:
+   
+        .. prompt:: bash $
+
+           rados -p data ls
+   
+	Now, identify the object location:
+
+        .. prompt:: bash $
+
+           ceph osd map {pool-name} {object-name}
+           ceph osd map data test-object-1
+   
+	Ceph should output the object's location. For example:: 
+   
+		osdmap e537 pool 'data' (1) object 'test-object-1' -> pg 1.d1743484 (1.4) -> up ([0,1], p0) acting ([0,1], p0)
+   
+        To remove the test object, simply delete it using the ``rados rm``
+        command.  For example:
+
+        .. prompt:: bash $
+   
+           rados rm test-object-1 --pool=data
+   
+
+As the cluster evolves, the object location may change dynamically. One benefit
+of Ceph's dynamic rebalancing is that Ceph relieves you from having to perform
+the migration manually. See the  `Architecture`_ section for details.
+
+.. _data placement: ../data-placement
+.. _pool: ../pools
+.. _placement group: ../placement-groups
+.. _Architecture: ../../../architecture
+.. _OSD Not Running: ../../troubleshooting/troubleshooting-osd#osd-not-running
+.. _Troubleshooting PG Errors: ../../troubleshooting/troubleshooting-pg#troubleshooting-pg-errors
+.. _Peering Failure: ../../troubleshooting/troubleshooting-pg#failures-osd-peering
+.. _CRUSH map: ../crush-map
+.. _Configuring Monitor/OSD Interaction: ../../configuration/mon-osd-interaction/
+.. _Placement Group Subsystem: ../control#placement-group-subsystem
diff --git a/doc/rados/operations/monitoring.rst b/doc/rados/operations/monitoring.rst
new file mode 100644
index 000000000..4df711d8b
--- /dev/null
+++ b/doc/rados/operations/monitoring.rst
@@ -0,0 +1,647 @@
+======================
+ Monitoring a Cluster
+======================
+
+Once you have a running cluster, you may use the ``ceph`` tool to monitor your
+cluster. Monitoring a cluster typically involves checking OSD status, monitor 
+status, placement group status and metadata server status.
+
+Using the command line
+======================
+
+Interactive mode
+----------------
+
+To run the ``ceph`` tool in interactive mode, type ``ceph`` at the command line
+with no arguments.  For example:
+
+.. prompt:: bash $
+
+	ceph
+
+.. prompt:: ceph>
+    :prompts: ceph>
+
+    health
+    status
+    quorum_status
+    mon stat
+
+Non-default paths
+-----------------
+
+If you specified non-default locations for your configuration or keyring,
+you may specify their locations:
+
+.. prompt:: bash $
+
+   ceph -c /path/to/conf -k /path/to/keyring health
+
+Checking a Cluster's Status
+===========================
+
+After you start your cluster, and before you start reading and/or
+writing data, check your cluster's status first.
+
+To check a cluster's status, execute the following:
+
+.. prompt:: bash $
+
+   ceph status
+	
+Or:
+
+.. prompt:: bash $
+
+   ceph -s
+
+In interactive mode, type ``status`` and press **Enter**:
+
+.. prompt:: ceph>
+    :prompts: ceph>
+   
+    ceph> status
+
+Ceph will print the cluster status. For example, a tiny Ceph demonstration
+cluster with one of each service may print the following:
+
+::
+
+  cluster:
+    id:     477e46f1-ae41-4e43-9c8f-72c918ab0a20
+    health: HEALTH_OK
+   
+  services:
+    mon: 3 daemons, quorum a,b,c
+    mgr: x(active)
+    mds: cephfs_a-1/1/1 up  {0=a=up:active}, 2 up:standby
+    osd: 3 osds: 3 up, 3 in
+  
+  data:
+    pools:   2 pools, 16 pgs
+    objects: 21 objects, 2.19K
+    usage:   546 GB used, 384 GB / 931 GB avail
+    pgs:     16 active+clean
+
+
+.. topic:: How Ceph Calculates Data Usage
+
+   The ``usage`` value reflects the *actual* amount of raw storage used. The 
+   ``xxx GB / xxx GB`` value means the amount available (the lesser number)
+   of the overall storage capacity of the cluster. The notional number reflects 
+   the size of the stored data before it is replicated, cloned or snapshotted.
+   Therefore, the amount of data actually stored typically exceeds the notional
+   amount stored, because Ceph creates replicas of the data and may also use 
+   storage capacity for cloning and snapshotting.
+
+
+Watching a Cluster
+==================
+
+In addition to local logging by each daemon, Ceph clusters maintain
+a *cluster log* that records high level events about the whole system.
+This is logged to disk on monitor servers (as ``/var/log/ceph/ceph.log`` by
+default), but can also be monitored via the command line.
+
+To follow the cluster log, use the following command:
+
+.. prompt:: bash $
+
+   ceph -w
+
+Ceph will print the status of the system, followed by each log message as it
+is emitted. For example:
+
+:: 
+
+  cluster:
+    id:     477e46f1-ae41-4e43-9c8f-72c918ab0a20
+    health: HEALTH_OK
+  
+  services:
+    mon: 3 daemons, quorum a,b,c
+    mgr: x(active)
+    mds: cephfs_a-1/1/1 up  {0=a=up:active}, 2 up:standby
+    osd: 3 osds: 3 up, 3 in
+  
+  data:
+    pools:   2 pools, 16 pgs
+    objects: 21 objects, 2.19K
+    usage:   546 GB used, 384 GB / 931 GB avail
+    pgs:     16 active+clean
+  
+  
+  2017-07-24 08:15:11.329298 mon.a mon.0 172.21.9.34:6789/0 23 : cluster [INF] osd.0 172.21.9.34:6806/20527 boot
+  2017-07-24 08:15:14.258143 mon.a mon.0 172.21.9.34:6789/0 39 : cluster [INF] Activating manager daemon x
+  2017-07-24 08:15:15.446025 mon.a mon.0 172.21.9.34:6789/0 47 : cluster [INF] Manager daemon x is now available
+
+
+In addition to using ``ceph -w`` to print log lines as they are emitted,
+use ``ceph log last [n]`` to see the most recent ``n`` lines from the cluster
+log.
+
+Monitoring Health Checks
+========================
+
+Ceph continuously runs various *health checks* against its own status.  When
+a health check fails, this is reflected in the output of ``ceph status`` (or
+``ceph health``).  In addition, messages are sent to the cluster log to
+indicate when a check fails, and when the cluster recovers.
+
+For example, when an OSD goes down, the ``health`` section of the status
+output may be updated as follows:
+
+::
+
+    health: HEALTH_WARN
+            1 osds down
+            Degraded data redundancy: 21/63 objects degraded (33.333%), 16 pgs unclean, 16 pgs degraded
+
+At this time, cluster log messages are also emitted to record the failure of the 
+health checks:
+
+::
+
+    2017-07-25 10:08:58.265945 mon.a mon.0 172.21.9.34:6789/0 91 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN)
+    2017-07-25 10:09:01.302624 mon.a mon.0 172.21.9.34:6789/0 94 : cluster [WRN] Health check failed: Degraded data redundancy: 21/63 objects degraded (33.333%), 16 pgs unclean, 16 pgs degraded (PG_DEGRADED)
+
+When the OSD comes back online, the cluster log records the cluster's return
+to a health state:
+
+::
+
+    2017-07-25 10:11:11.526841 mon.a mon.0 172.21.9.34:6789/0 109 : cluster [WRN] Health check update: Degraded data redundancy: 2 pgs unclean, 2 pgs degraded, 2 pgs undersized (PG_DEGRADED)
+    2017-07-25 10:11:13.535493 mon.a mon.0 172.21.9.34:6789/0 110 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2 pgs unclean, 2 pgs degraded, 2 pgs undersized)
+    2017-07-25 10:11:13.535577 mon.a mon.0 172.21.9.34:6789/0 111 : cluster [INF] Cluster is now healthy
+
+Network Performance Checks
+--------------------------
+
+Ceph OSDs send heartbeat ping messages amongst themselves to monitor daemon availability.  We
+also use the response times to monitor network performance.
+While it is possible that a busy OSD could delay a ping response, we can assume
+that if a network switch fails multiple delays will be detected between distinct pairs of OSDs.
+
+By default we will warn about ping times which exceed 1 second (1000 milliseconds).
+
+::
+
+    HEALTH_WARN Slow OSD heartbeats on back (longest 1118.001ms)
+
+The health detail will add the combination of OSDs are seeing the delays and by how much.  There is a limit of 10
+detail line items.
+
+::
+
+    [WRN] OSD_SLOW_PING_TIME_BACK: Slow OSD heartbeats on back (longest 1118.001ms)
+        Slow OSD heartbeats on back from osd.0 [dc1,rack1] to osd.1 [dc1,rack1] 1118.001 msec possibly improving
+        Slow OSD heartbeats on back from osd.0 [dc1,rack1] to osd.2 [dc1,rack2] 1030.123 msec
+        Slow OSD heartbeats on back from osd.2 [dc1,rack2] to osd.1 [dc1,rack1] 1015.321 msec
+        Slow OSD heartbeats on back from osd.1 [dc1,rack1] to osd.0 [dc1,rack1] 1010.456 msec
+
+To see even more detail and a complete dump of network performance information the ``dump_osd_network`` command can be used.  Typically, this would be
+sent to a mgr, but it can be limited to a particular OSD's interactions by issuing it to any OSD.  The current threshold which defaults to 1 second
+(1000 milliseconds) can be overridden as an argument in milliseconds.
+
+The following command will show all gathered network performance data by specifying a threshold of 0 and sending to the mgr.
+
+.. prompt:: bash $
+
+   ceph daemon /var/run/ceph/ceph-mgr.x.asok dump_osd_network 0
+
+::
+
+    {
+        "threshold": 0,
+        "entries": [
+            {
+                "last update": "Wed Sep  4 17:04:49 2019",
+                "stale": false,
+                "from osd": 2,
+                "to osd": 0,
+                "interface": "front",
+                "average": {
+                    "1min": 1.023,
+                    "5min": 0.860,
+                    "15min": 0.883
+                },
+                "min": {
+                    "1min": 0.818,
+                    "5min": 0.607,
+                    "15min": 0.607
+                },
+                "max": {
+                    "1min": 1.164,
+                    "5min": 1.173,
+                    "15min": 1.544
+                },
+                "last": 0.924
+            },
+            {
+                "last update": "Wed Sep  4 17:04:49 2019",
+                "stale": false,
+                "from osd": 2,
+                "to osd": 0,
+                "interface": "back",
+                "average": {
+                    "1min": 0.968,
+                    "5min": 0.897,
+                    "15min": 0.830
+                },
+                "min": {
+                    "1min": 0.860,
+                    "5min": 0.563,
+                    "15min": 0.502
+                },
+                "max": {
+                    "1min": 1.171,
+                    "5min": 1.216,
+                    "15min": 1.456
+                },
+                "last": 0.845
+            },
+            {
+                "last update": "Wed Sep  4 17:04:48 2019",
+                "stale": false,
+                "from osd": 0,
+                "to osd": 1,
+                "interface": "front",
+                "average": {
+                    "1min": 0.965,
+                    "5min": 0.811,
+                    "15min": 0.850
+                },
+                "min": {
+                    "1min": 0.650,
+                    "5min": 0.488,
+                    "15min": 0.466
+                },
+                "max": {
+                    "1min": 1.252,
+                    "5min": 1.252,
+                    "15min": 1.362
+                },
+            "last": 0.791
+        },
+        ...
+
+
+
+Muting health checks
+--------------------
+
+Health checks can be muted so that they do not affect the overall
+reported status of the cluster.  Alerts are specified using the health
+check code (see :ref:`health-checks`):
+
+.. prompt:: bash $
+
+   ceph health mute <code>
+
+For example, if there is a health warning, muting it will make the
+cluster report an overall status of ``HEALTH_OK``.  For example, to
+mute an ``OSD_DOWN`` alert,:
+
+.. prompt:: bash $
+
+   ceph health mute OSD_DOWN
+
+Mutes are reported as part of the short and long form of the ``ceph health`` command.
+For example, in the above scenario, the cluster would report:
+
+.. prompt:: bash $
+
+   ceph health
+
+::
+
+   HEALTH_OK (muted: OSD_DOWN)
+
+.. prompt:: bash $
+
+   ceph health detail
+
+::
+
+   HEALTH_OK (muted: OSD_DOWN)
+   (MUTED) OSD_DOWN 1 osds down
+       osd.1 is down
+
+A mute can be explicitly removed with:
+
+.. prompt:: bash $
+
+   ceph health unmute <code>
+
+For example:
+
+.. prompt:: bash $
+
+   ceph health unmute OSD_DOWN
+
+A health check mute may optionally have a TTL (time to live)
+associated with it, such that the mute will automatically expire
+after the specified period of time has elapsed.  The TTL is specified as an optional
+duration argument, e.g.:
+
+.. prompt:: bash $
+
+   ceph health mute OSD_DOWN 4h    # mute for 4 hours
+   ceph health mute MON_DOWN 15m   # mute for 15  minutes
+
+Normally, if a muted health alert is resolved (e.g., in the example
+above, the OSD comes back up), the mute goes away.  If the alert comes
+back later, it will be reported in the usual way.
+
+It is possible to make a mute "sticky" such that the mute will remain even if the
+alert clears.  For example:
+
+.. prompt:: bash $
+
+   ceph health mute OSD_DOWN 1h --sticky   # ignore any/all down OSDs for next hour
+
+Most health mutes also disappear if the extent of an alert gets worse.  For example,
+if there is one OSD down, and the alert is muted, the mute will disappear if one
+or more additional OSDs go down.  This is true for any health alert that involves
+a count indicating how much or how many of something is triggering the warning or
+error.
+
+
+Detecting configuration issues
+==============================
+
+In addition to the health checks that Ceph continuously runs on its
+own status, there are some configuration issues that may only be detected
+by an external tool.
+
+Use the `ceph-medic`_ tool to run these additional checks on your Ceph
+cluster's configuration.
+
+Checking a Cluster's Usage Stats
+================================
+
+To check a cluster's data usage and data distribution among pools, you can
+use the ``df`` option. It is similar to Linux ``df``. Execute 
+the following:
+
+.. prompt:: bash $
+
+   ceph df
+
+The output of ``ceph df`` looks like this::
+
+   CLASS     SIZE    AVAIL     USED  RAW USED  %RAW USED
+   ssd    202 GiB  200 GiB  2.0 GiB   2.0 GiB       1.00
+   TOTAL  202 GiB  200 GiB  2.0 GiB   2.0 GiB       1.00
+
+   --- POOLS ---
+   POOL                   ID  PGS   STORED   (DATA)   (OMAP)   OBJECTS     USED  (DATA)   (OMAP)   %USED  MAX AVAIL  QUOTA OBJECTS  QUOTA BYTES  DIRTY  USED COMPR  UNDER COMPR
+   device_health_metrics   1    1  242 KiB   15 KiB  227 KiB         4  251 KiB  24 KiB  227 KiB       0    297 GiB            N/A          N/A      4         0 B          0 B
+   cephfs.a.meta           2   32  6.8 KiB  6.8 KiB      0 B        22   96 KiB  96 KiB      0 B       0    297 GiB            N/A          N/A     22         0 B          0 B
+   cephfs.a.data           3   32      0 B      0 B      0 B         0      0 B     0 B      0 B       0     99 GiB            N/A          N/A      0         0 B          0 B
+   test                    4   32   22 MiB   22 MiB   50 KiB       248   19 MiB  19 MiB   50 KiB       0    297 GiB            N/A          N/A    248         0 B          0 B
+
+
+
+
+
+- **CLASS:** for example, "ssd" or "hdd"
+- **SIZE:** The amount of storage capacity managed by the cluster.
+- **AVAIL:** The amount of free space available in the cluster.
+- **USED:** The amount of raw storage consumed by user data (excluding
+  BlueStore's database)
+- **RAW USED:** The amount of raw storage consumed by user data, internal
+  overhead, or reserved capacity.
+- **%RAW USED:** The percentage of raw storage used. Use this number in
+  conjunction with the ``full ratio`` and ``near full ratio`` to ensure that 
+  you are not reaching your cluster's capacity. See `Storage Capacity`_ for 
+  additional details.
+
+
+**POOLS:**  
+
+The **POOLS** section of the output provides a list of pools and the notional 
+usage of each pool. The output from this section **DOES NOT** reflect replicas,
+clones or snapshots. For example, if you store an object with 1MB of data, the 
+notional usage will be 1MB, but the actual usage may be 2MB or more depending 
+on the number of replicas, clones and snapshots.  
+
+- **ID:** The number of the node within the pool.
+- **STORED:** actual amount of data user/Ceph has stored in a pool. This is
+  similar to the USED column in earlier versions of Ceph but the calculations
+  (for BlueStore!) are more precise (gaps are properly handled).
+
+  - **(DATA):** usage for RBD (RADOS Block Device), CephFS file data, and RGW
+    (RADOS Gateway) object data.
+  - **(OMAP):** key-value pairs. Used primarily by CephFS and RGW (RADOS
+    Gateway) for metadata storage.
+
+- **OBJECTS:** The notional number of objects stored per pool. "Notional" is
+  defined above in the paragraph immediately under "POOLS".
+- **USED:** The space allocated for a pool over all OSDs. This includes
+  replication, allocation granularity, and erasure-coding overhead. Compression
+  savings and object content gaps are also taken into account. BlueStore's
+  database is not included in this amount.
+
+  - **(DATA):** object usage for RBD (RADOS Block Device), CephFS file data, and RGW
+    (RADOS Gateway) object data.
+  - **(OMAP):** object key-value pairs. Used primarily by CephFS and RGW (RADOS
+    Gateway) for metadata storage.
+
+- **%USED:** The notional percentage of storage used per pool.
+- **MAX AVAIL:** An estimate of the notional amount of data that can be written
+  to this pool.
+- **QUOTA OBJECTS:** The number of quota objects.
+- **QUOTA BYTES:** The number of bytes in the quota objects.
+- **DIRTY:** The number of objects in the cache pool that have been written to
+  the cache pool but have not been flushed yet to the base pool. This field is
+  only available when cache tiering is in use.
+- **USED COMPR:** amount of space allocated for compressed data (i.e. this
+  includes comrpessed data plus all the allocation, replication and erasure
+  coding overhead).
+- **UNDER COMPR:** amount of data passed through compression (summed over all
+  replicas) and beneficial enough to be stored in a compressed form.
+
+
+.. note:: The numbers in the POOLS section are notional. They are not
+   inclusive of the number of replicas, snapshots or clones. As a result, the
+   sum of the USED and %USED amounts will not add up to the USED and %USED
+   amounts in the RAW section of the output.
+
+.. note:: The MAX AVAIL value is a complicated function of the replication
+   or erasure code used, the CRUSH rule that maps storage to devices, the
+   utilization of those devices, and the configured ``mon_osd_full_ratio``.
+
+
+Checking OSD Status
+===================
+
+You can check OSDs to ensure they are ``up`` and ``in`` by executing the
+following command:
+
+.. prompt:: bash #
+
+  ceph osd stat
+	
+Or: 
+
+.. prompt:: bash #
+
+  ceph osd dump
+	
+You can also check view OSDs according to their position in the CRUSH map by
+using the folloiwng command:
+
+.. prompt:: bash #
+
+   ceph osd tree
+
+Ceph will print out a CRUSH tree with a host, its OSDs, whether they are up
+and their weight:
+
+.. code-block:: bash
+
+   #ID CLASS WEIGHT  TYPE NAME             STATUS REWEIGHT PRI-AFF
+    -1       3.00000 pool default
+    -3       3.00000 rack mainrack
+    -2       3.00000 host osd-host
+     0   ssd 1.00000         osd.0             up  1.00000 1.00000
+     1   ssd 1.00000         osd.1             up  1.00000 1.00000
+     2   ssd 1.00000         osd.2             up  1.00000 1.00000
+
+For a detailed discussion, refer to `Monitoring OSDs and Placement Groups`_.
+
+Checking Monitor Status
+=======================
+
+If your cluster has multiple monitors (likely), you should check the monitor
+quorum status after you start the cluster and before reading and/or writing data. A
+quorum must be present when multiple monitors are running. You should also check
+monitor status periodically to ensure that they are running.
+
+To see display the monitor map, execute the following:
+
+.. prompt:: bash $
+
+   ceph mon stat
+	
+Or:
+
+.. prompt:: bash $
+
+   ceph mon dump
+	
+To check the quorum status for the monitor cluster, execute the following:
+	
+.. prompt:: bash $
+
+   ceph quorum_status
+
+Ceph will return the quorum status. For example, a Ceph  cluster consisting of
+three monitors may return the following:
+
+.. code-block:: javascript
+
+	{ "election_epoch": 10,
+	  "quorum": [
+	        0,
+	        1,
+	        2],
+	  "quorum_names": [
+		"a",
+		"b",
+		"c"],
+	  "quorum_leader_name": "a",
+	  "monmap": { "epoch": 1,
+	      "fsid": "444b489c-4f16-4b75-83f0-cb8097468898",
+	      "modified": "2011-12-12 13:28:27.505520",
+	      "created": "2011-12-12 13:28:27.505520",
+	      "features": {"persistent": [
+				"kraken",
+				"luminous",
+				"mimic"],
+		"optional": []
+	      },
+	      "mons": [
+	            { "rank": 0,
+	              "name": "a",
+	              "addr": "127.0.0.1:6789/0",
+		      "public_addr": "127.0.0.1:6789/0"},
+	            { "rank": 1,
+	              "name": "b",
+	              "addr": "127.0.0.1:6790/0",
+		      "public_addr": "127.0.0.1:6790/0"},
+	            { "rank": 2,
+	              "name": "c",
+	              "addr": "127.0.0.1:6791/0",
+		      "public_addr": "127.0.0.1:6791/0"}
+	           ]
+	  }
+	}
+
+Checking MDS Status
+===================
+
+Metadata servers provide metadata services for  CephFS. Metadata servers have
+two sets of states: ``up | down`` and ``active | inactive``. To ensure your
+metadata servers are ``up`` and ``active``,  execute the following:
+
+.. prompt:: bash $
+
+   ceph mds stat
+	
+To display details of the metadata cluster, execute the following:
+
+.. prompt:: bash $
+
+   ceph fs dump
+
+
+Checking Placement Group States
+===============================
+
+Placement groups map objects to OSDs. When you monitor your
+placement groups,  you will want them to be ``active`` and ``clean``. 
+For a detailed discussion, refer to `Monitoring OSDs and Placement Groups`_.
+
+.. _Monitoring OSDs and Placement Groups: ../monitoring-osd-pg
+
+.. _rados-monitoring-using-admin-socket:
+
+Using the Admin Socket
+======================
+
+The Ceph admin socket allows you to query a daemon via a socket interface. 
+By default, Ceph sockets reside under ``/var/run/ceph``. To access a daemon
+via the admin socket, login to the host running the daemon and use the 
+following command:
+
+.. prompt:: bash $
+
+   ceph daemon {daemon-name}
+   ceph daemon {path-to-socket-file}
+
+For example, the following are equivalent:
+
+.. prompt:: bash $
+
+   ceph daemon osd.0 foo
+   ceph daemon /var/run/ceph/ceph-osd.0.asok foo
+
+To view the available admin socket commands, execute the following command:
+
+.. prompt:: bash $
+
+   ceph daemon {daemon-name} help
+
+The admin socket command enables you to show and set your configuration at
+runtime. See `Viewing a Configuration at Runtime`_ for details.
+
+Additionally, you can set configuration values at runtime directly (i.e., the
+admin socket bypasses the monitor, unlike ``ceph tell {daemon-type}.{id}
+config set``, which relies on the monitor but doesn't require you to login
+directly to the host in question ).
+
+.. _Viewing a Configuration at Runtime: ../../configuration/ceph-conf#viewing-a-configuration-at-runtime
+.. _Storage Capacity: ../../configuration/mon-config-ref#storage-capacity
+.. _ceph-medic: http://docs.ceph.com/ceph-medic/master/
diff --git a/doc/rados/operations/operating.rst b/doc/rados/operations/operating.rst
new file mode 100644
index 000000000..134774ccb
--- /dev/null
+++ b/doc/rados/operations/operating.rst
@@ -0,0 +1,255 @@
+=====================
+ Operating a Cluster
+=====================
+
+.. index:: systemd; operating a cluster
+
+
+Running Ceph with systemd
+==========================
+
+For all distributions that support systemd (CentOS 7, Fedora, Debian
+Jessie 8 and later, SUSE), ceph daemons are now managed using native
+systemd files instead of the legacy sysvinit scripts.  For example:
+
+.. prompt:: bash $
+
+   sudo systemctl start ceph.target       # start all daemons
+   sudo systemctl status ceph-osd@12      # check status of osd.12
+
+To list the Ceph systemd units on a node, execute:
+
+.. prompt:: bash $
+
+   sudo systemctl status ceph\*.service ceph\*.target
+
+Starting all Daemons
+--------------------
+
+To start all daemons on a Ceph Node (irrespective of type), execute the
+following:
+
+.. prompt:: bash $
+
+   sudo systemctl start ceph.target
+
+
+Stopping all Daemons
+--------------------
+
+To stop all daemons on a Ceph Node (irrespective of type), execute the
+following:
+
+.. prompt:: bash $
+
+   sudo systemctl stop ceph\*.service ceph\*.target
+
+
+Starting all Daemons by Type
+----------------------------
+
+To start all daemons of a particular type on a Ceph Node, execute one of the
+following:
+
+.. prompt:: bash $
+
+   sudo systemctl start ceph-osd.target
+   sudo systemctl start ceph-mon.target
+   sudo systemctl start ceph-mds.target
+
+
+Stopping all Daemons by Type
+----------------------------
+
+To stop all daemons of a particular type on a Ceph Node, execute one of the
+following:
+
+.. prompt:: bash $
+
+   sudo systemctl stop ceph-mon\*.service ceph-mon.target
+   sudo systemctl stop ceph-osd\*.service ceph-osd.target
+   sudo systemctl stop ceph-mds\*.service ceph-mds.target
+
+
+Starting a Daemon
+-----------------
+
+To start a specific daemon instance on a Ceph Node, execute one of the
+following:
+
+.. prompt:: bash $
+
+   sudo systemctl start ceph-osd@{id}
+   sudo systemctl start ceph-mon@{hostname}
+   sudo systemctl start ceph-mds@{hostname}
+
+For example:
+
+.. prompt:: bash $
+
+   sudo systemctl start ceph-osd@1
+   sudo systemctl start ceph-mon@ceph-server
+   sudo systemctl start ceph-mds@ceph-server
+
+
+Stopping a Daemon
+-----------------
+
+To stop a specific daemon instance on a Ceph Node, execute one of the
+following:
+
+.. prompt:: bash $
+
+   sudo systemctl stop ceph-osd@{id}
+   sudo systemctl stop ceph-mon@{hostname}
+   sudo systemctl stop ceph-mds@{hostname}
+
+For example:
+
+.. prompt:: bash $
+
+   sudo systemctl stop ceph-osd@1
+   sudo systemctl stop ceph-mon@ceph-server
+   sudo systemctl stop ceph-mds@ceph-server
+
+
+.. index:: Upstart; operating a cluster
+
+Running Ceph with Upstart
+==========================
+
+Starting all Daemons
+--------------------
+
+To start all daemons on a Ceph Node (irrespective of type), execute the
+following:: 
+
+	sudo start ceph-all
+	
+
+Stopping all Daemons	
+--------------------
+
+To stop all daemons on a Ceph Node (irrespective of type), execute the
+following:: 
+
+	sudo stop ceph-all
+	
+
+Starting all Daemons by Type
+----------------------------
+
+To start all daemons of a particular type on a Ceph Node, execute one of the
+following:: 
+
+	sudo start ceph-osd-all
+	sudo start ceph-mon-all
+	sudo start ceph-mds-all
+
+
+Stopping all Daemons by Type
+----------------------------
+
+To stop all daemons of a particular type on a Ceph Node, execute one of the
+following::
+
+	sudo stop ceph-osd-all
+	sudo stop ceph-mon-all
+	sudo stop ceph-mds-all
+
+
+Starting a Daemon
+-----------------
+
+To start a specific daemon instance on a Ceph Node, execute one of the
+following:: 
+
+	sudo start ceph-osd id={id}
+	sudo start ceph-mon id={hostname}
+	sudo start ceph-mds id={hostname}
+
+For example:: 
+
+	sudo start ceph-osd id=1
+	sudo start ceph-mon id=ceph-server
+	sudo start ceph-mds id=ceph-server
+
+
+Stopping a Daemon
+-----------------
+
+To stop a specific daemon instance on a Ceph Node, execute one of the
+following:: 
+
+	sudo stop ceph-osd id={id}
+	sudo stop ceph-mon id={hostname}
+	sudo stop ceph-mds id={hostname}
+
+For example:: 
+
+	sudo stop ceph-osd id=1
+	sudo start ceph-mon id=ceph-server
+	sudo start ceph-mds id=ceph-server
+
+
+.. index:: sysvinit; operating a cluster
+
+Running Ceph with sysvinit
+==========================
+
+Each time you to **start**, **restart**, and  **stop** Ceph daemons (or your
+entire cluster) you must specify at least one option and one command. You may
+also specify a daemon type or a daemon instance. ::
+
+	{commandline} [options] [commands] [daemons]
+
+
+The ``ceph`` options include:
+
++-----------------+----------+-------------------------------------------------+
+| Option          | Shortcut | Description                                     |
++=================+==========+=================================================+
+| ``--verbose``   |  ``-v``  | Use verbose logging.                            |
++-----------------+----------+-------------------------------------------------+
+| ``--valgrind``  | ``N/A``  | (Dev and QA only) Use `Valgrind`_ debugging.    |
++-----------------+----------+-------------------------------------------------+
+| ``--allhosts``  |  ``-a``  | Execute on all nodes in ``ceph.conf.``          |
+|                 |          | Otherwise, it only executes on ``localhost``.   |
++-----------------+----------+-------------------------------------------------+
+| ``--restart``   | ``N/A``  | Automatically restart daemon if it core dumps.  |
++-----------------+----------+-------------------------------------------------+
+| ``--norestart`` | ``N/A``  | Don't restart a daemon if it core dumps.        |
++-----------------+----------+-------------------------------------------------+
+| ``--conf``      |  ``-c``  | Use an alternate configuration file.            |
++-----------------+----------+-------------------------------------------------+
+
+The ``ceph`` commands include:
+
++------------------+------------------------------------------------------------+
+| Command          | Description                                                |
++==================+============================================================+
+|    ``start``     | Start the daemon(s).                                       |
++------------------+------------------------------------------------------------+
+|    ``stop``      | Stop the daemon(s).                                        |
++------------------+------------------------------------------------------------+
+|  ``forcestop``   | Force the daemon(s) to stop. Same as ``kill -9``           |
++------------------+------------------------------------------------------------+
+|   ``killall``    | Kill all daemons of a particular type.                     | 
++------------------+------------------------------------------------------------+
+|  ``cleanlogs``   | Cleans out the log directory.                              |
++------------------+------------------------------------------------------------+
+| ``cleanalllogs`` | Cleans out **everything** in the log directory.            |
++------------------+------------------------------------------------------------+
+
+For subsystem operations, the ``ceph`` service can target specific daemon types
+by adding a particular daemon type for the ``[daemons]`` option. Daemon types
+include: 
+
+- ``mon``
+- ``osd``
+- ``mds``
+
+
+
+.. _Valgrind: http://www.valgrind.org/
+.. _initctl: http://manpages.ubuntu.com/manpages/raring/en/man8/initctl.8.html
diff --git a/doc/rados/operations/pg-concepts.rst b/doc/rados/operations/pg-concepts.rst
new file mode 100644
index 000000000..636d6bf9a
--- /dev/null
+++ b/doc/rados/operations/pg-concepts.rst
@@ -0,0 +1,102 @@
+==========================
+ Placement Group Concepts
+==========================
+
+When you execute commands like ``ceph -w``, ``ceph osd dump``, and other 
+commands related to placement groups, Ceph may return values using some
+of the following terms: 
+
+*Peering*
+   The process of bringing all of the OSDs that store
+   a Placement Group (PG) into agreement about the state
+   of all of the objects (and their metadata) in that PG.
+   Note that agreeing on the state does not mean that
+   they all have the latest contents.
+
+*Acting Set*
+   The ordered list of OSDs who are (or were as of some epoch)
+   responsible for a particular placement group.
+
+*Up Set*
+   The ordered list of OSDs responsible for a particular placement
+   group for a particular epoch according to CRUSH. Normally this
+   is the same as the *Acting Set*, except when the *Acting Set* has 
+   been explicitly overridden via ``pg_temp`` in the OSD Map.
+
+*Current Interval* or *Past Interval*
+   A sequence of OSD map epochs during which the *Acting Set* and *Up
+   Set* for particular placement group do not change.
+
+*Primary*
+   The member (and by convention first) of the *Acting Set*,
+   that is responsible for coordination peering, and is
+   the only OSD that will accept client-initiated
+   writes to objects in a placement group.
+
+*Replica*
+   A non-primary OSD in the *Acting Set* for a placement group
+   (and who has been recognized as such and *activated* by the primary).
+
+*Stray*
+   An OSD that is not a member of the current *Acting Set*, but
+   has not yet been told that it can delete its copies of a
+   particular placement group.
+
+*Recovery*
+   Ensuring that copies of all of the objects in a placement group
+   are on all of the OSDs in the *Acting Set*.  Once *Peering* has 
+   been performed, the *Primary* can start accepting write operations, 
+   and *Recovery* can proceed in the background.
+
+*PG Info* 
+   Basic metadata about the placement group's creation epoch, the version
+   for the most recent write to the placement group, *last epoch started*, 
+   *last epoch clean*, and the beginning of the *current interval*.  Any
+   inter-OSD communication about placement groups includes the *PG Info*, 
+   such that any OSD that knows a placement group exists (or once existed) 
+   also has a lower bound on *last epoch clean* or *last epoch started*.
+
+*PG Log*
+   A list of recent updates made to objects in a placement group.
+   Note that these logs can be truncated after all OSDs
+   in the *Acting Set* have acknowledged up to a certain
+   point.
+
+*Missing Set*
+   Each OSD notes update log entries and if they imply updates to
+   the contents of an object, adds that object to a list of needed
+   updates.  This list is called the *Missing Set* for that ``<OSD,PG>``.
+
+*Authoritative History*
+   A complete, and fully ordered set of operations that, if
+   performed, would bring an OSD's copy of a placement group
+   up to date.
+
+*Epoch*
+   A (monotonically increasing) OSD map version number
+
+*Last Epoch Start*
+   The last epoch at which all nodes in the *Acting Set*
+   for a particular placement group agreed on an
+   *Authoritative History*.  At this point, *Peering* is
+   deemed to have been successful.
+
+*up_thru*
+   Before a *Primary* can successfully complete the *Peering* process,
+   it must inform a monitor that is alive through the current
+   OSD map *Epoch* by having the monitor set its *up_thru* in the osd
+   map.  This helps *Peering* ignore previous *Acting Sets* for which
+   *Peering* never completed after certain sequences of failures, such as
+   the second interval below:
+
+   - *acting set* = [A,B]
+   - *acting set* = [A]
+   - *acting set* = [] very shortly after (e.g., simultaneous failure, but staggered detection)
+   - *acting set* = [B] (B restarts, A does not)
+
+*Last Epoch Clean*
+   The last *Epoch* at which all nodes in the *Acting set*
+   for a particular placement group were completely
+   up to date (both placement group logs and object contents).
+   At this point, *recovery* is deemed to have been
+   completed.
diff --git a/doc/rados/operations/pg-repair.rst b/doc/rados/operations/pg-repair.rst
new file mode 100644
index 000000000..f495530cc
--- /dev/null
+++ b/doc/rados/operations/pg-repair.rst
@@ -0,0 +1,81 @@
+============================
+Repairing PG inconsistencies
+============================
+Sometimes a placement group might become "inconsistent". To return the
+placement group to an active+clean state, you must first determine which
+of the placement groups has become inconsistent and then run the "pg
+repair" command on it. This page contains commands for diagnosing placement
+groups and the command for repairing placement groups that have become
+inconsistent.
+
+.. highlight:: console
+
+Commands for Diagnosing Placement-group Problems
+================================================
+The commands in this section provide various ways of diagnosing broken placement groups.
+
+The following command provides a high-level (low detail) overview of the health of the ceph cluster:
+
+.. prompt:: bash #
+
+   ceph health detail
+
+The following command provides more detail on the status of the placement groups:
+
+.. prompt:: bash #
+
+   ceph pg dump --format=json-pretty
+
+The following command lists inconsistent placement groups:
+
+.. prompt:: bash #
+
+   rados list-inconsistent-pg {pool}
+
+The following command lists inconsistent rados objects:
+
+.. prompt:: bash #
+
+   rados list-inconsistent-obj {pgid}
+
+The following command lists inconsistent snapsets in the given placement group:
+
+.. prompt:: bash #
+
+   rados list-inconsistent-snapset {pgid}
+
+
+Commands for Repairing Placement Groups
+=======================================
+The form of the command to repair a broken placement group is:
+
+.. prompt:: bash #
+
+   ceph pg repair {pgid}
+
+Where ``{pgid}`` is the id of the affected placement group.
+
+For example:
+
+.. prompt:: bash #
+
+   ceph pg repair 1.4
+
+More Information on Placement Group Repair
+==========================================
+Ceph stores and updates the checksums of objects stored in the cluster. When a scrub is performed on a placement group, the OSD attempts to choose an authoritative copy from among its replicas. Among all of the possible cases, only one case is consistent. After a deep scrub, Ceph calculates the checksum of an object read from the disk and compares it to the checksum previously recorded. If the current checksum and the previously recorded checksums do not match, that is an inconsistency. In the case of replicated pools, any mismatch between the checksum of any replica of an object and the checksum of the authoritative copy means that there is an inconsistency.
+
+The "pg repair" command attempts to fix inconsistencies of various kinds. If "pg repair" finds an inconsistent placement group, it attempts to overwrite the digest of the inconsistent copy with the digest of the authoritative copy. If "pg repair" finds an inconsistent replicated pool, it marks the inconsistent copy as missing. Recovery, in the case of replicated pools, is beyond the scope of "pg repair".
+
+For erasure coded and bluestore pools, Ceph will automatically repair if osd_scrub_auto_repair (configuration default "false") is set to true and at most osd_scrub_auto_repair_num_errors (configuration default 5) errors are found.
+
+"pg repair" will not solve every problem. Ceph does not automatically repair placement groups when inconsistencies are found in them.
+
+The checksum of an object or an omap is not always available. Checksums are calculated incrementally. If a replicated object is updated non-sequentially, the write operation involved in the update changes the object and invalidates its checksum. The whole object is not read while recalculating the checksum. "ceph pg repair" is able to repair things even when checksums are not available to it, as in the case of filestore. When replicated filestore pools are in question, users might prefer manual repair to "ceph pg repair". 
+
+The material in this paragraph is relevant for filestore, and bluestore has its own internal checksums. The matched-record checksum and the calculated checksum cannot prove that the authoritative copy is in fact authoritative. In the case that there is no checksum available, "pg repair" favors the data on the primary. this might or might not be the uncorrupted replica. This is why human intervention is necessary when an inconsistency is discovered. Human intervention sometimes means using the "ceph-objectstore-tool".
+
+External Links
+==============
+https://ceph.io/geen-categorie/ceph-manually-repair-object/ - This page contains a walkthrough of the repair of a placement group, and is recommended reading if you want to repair a placement
+group but have never done so.
diff --git a/doc/rados/operations/pg-states.rst b/doc/rados/operations/pg-states.rst
new file mode 100644
index 000000000..495229d92
--- /dev/null
+++ b/doc/rados/operations/pg-states.rst
@@ -0,0 +1,118 @@
+========================
+ Placement Group States
+========================
+
+When checking a cluster's status (e.g., running ``ceph -w`` or ``ceph -s``), 
+Ceph will report on the status of the placement groups. A placement group has 
+one or more states. The optimum state for placement groups in the placement group
+map is ``active + clean``. 
+
+*creating*
+  Ceph is still creating the placement group.
+
+*activating*
+  The placement group is peered but not yet active.
+
+*active*
+  Ceph will process requests to the placement group.
+
+*clean*
+  Ceph replicated all objects in the placement group the correct number of times.
+
+*down*
+  A replica with necessary data is down, so the placement group is offline.
+
+*laggy*
+  A replica is not acknowledging new leases from the primary in a timely fashion; IO is temporarily paused.
+
+*wait*
+  The set of OSDs for this PG has just changed and IO is temporarily paused until the previous interval's leases expire.
+
+*scrubbing*
+  Ceph is checking the placement group metadata for inconsistencies.
+
+*deep*
+  Ceph is checking the placement group data against stored checksums.
+
+*degraded*
+  Ceph has not replicated some objects in the placement group the correct number of times yet.
+
+*inconsistent*
+  Ceph detects inconsistencies in the one or more replicas of an object in the placement group
+  (e.g. objects are the wrong size, objects are missing from one replica *after* recovery finished, etc.).
+
+*peering*
+  The placement group is undergoing the peering process
+
+*repair*
+  Ceph is checking the placement group and repairing any inconsistencies it finds (if possible).
+
+*recovering*
+  Ceph is migrating/synchronizing objects and their replicas.
+
+*forced_recovery*
+  High recovery priority of that PG is enforced by user.
+
+*recovery_wait*
+  The placement group is waiting in line to start recover.
+
+*recovery_toofull*
+  A recovery operation is waiting because the destination OSD is over its
+  full ratio.
+
+*recovery_unfound*
+  Recovery stopped due to unfound objects.
+
+*backfilling*
+  Ceph is scanning and synchronizing the entire contents of a placement group
+  instead of inferring what contents need to be synchronized from the logs of
+  recent operations. Backfill is a special case of recovery.
+
+*forced_backfill*
+  High backfill priority of that PG is enforced by user.
+
+*backfill_wait*
+  The placement group is waiting in line to start backfill.
+
+*backfill_toofull*
+  A backfill operation is waiting because the destination OSD is over
+  the backfillfull ratio.
+
+*backfill_unfound*
+  Backfill stopped due to unfound objects.
+
+*incomplete*
+  Ceph detects that a placement group is missing information about
+  writes that may have occurred, or does not have any healthy
+  copies. If you see this state, try to start any failed OSDs that may
+  contain the needed information. In the case of an erasure coded pool
+  temporarily reducing min_size may allow recovery.
+
+*stale*
+  The placement group is in an unknown state - the monitors have not received
+  an update for it since the placement group mapping changed.
+
+*remapped*
+  The placement group is temporarily mapped to a different set of OSDs from what
+  CRUSH specified.
+
+*undersized*
+  The placement group has fewer copies than the configured pool replication level.
+
+*peered*
+  The placement group has peered, but cannot serve client IO due to not having
+  enough copies to reach the pool's configured min_size parameter.  Recovery
+  may occur in this state, so the pg may heal up to min_size eventually.
+
+*snaptrim*
+  Trimming snaps.
+
+*snaptrim_wait*
+  Queued to trim snaps.
+
+*snaptrim_error*
+  Error stopped trimming snaps.
+
+*unknown*
+  The ceph-mgr hasn't yet received any information about the PG's state from an
+  OSD since mgr started up.
diff --git a/doc/rados/operations/placement-groups.rst b/doc/rados/operations/placement-groups.rst
new file mode 100644
index 000000000..d51f8d76e
--- /dev/null
+++ b/doc/rados/operations/placement-groups.rst
@@ -0,0 +1,798 @@
+==================
+ Placement Groups
+==================
+
+.. _pg-autoscaler:
+
+Autoscaling placement groups
+============================
+
+Placement groups (PGs) are an internal implementation detail of how
+Ceph distributes data.  You can allow the cluster to either make
+recommendations or automatically tune PGs based on how the cluster is
+used by enabling *pg-autoscaling*.
+
+Each pool in the system has a ``pg_autoscale_mode`` property that can be set to ``off``, ``on``, or ``warn``.
+
+* ``off``: Disable autoscaling for this pool.  It is up to the administrator to choose an appropriate PG number for each pool.  Please refer to :ref:`choosing-number-of-placement-groups` for more information.
+* ``on``: Enable automated adjustments of the PG count for the given pool.
+* ``warn``: Raise health alerts when the PG count should be adjusted
+
+To set the autoscaling mode for an existing pool:
+
+.. prompt:: bash #
+
+   ceph osd pool set <pool-name> pg_autoscale_mode <mode>
+
+For example to enable autoscaling on pool ``foo``:
+
+.. prompt:: bash #
+
+   ceph osd pool set foo pg_autoscale_mode on
+
+You can also configure the default ``pg_autoscale_mode`` that is
+set on any pools that are subsequently created:
+
+.. prompt:: bash #
+
+   ceph config set global osd_pool_default_pg_autoscale_mode <mode>
+
+You can disable or enable the autoscaler for all pools with
+the ``noautoscale`` flag. By default this flag is set to  be ``off``,
+but you can turn it ``on`` by using the command:
+
+.. prompt:: bash $
+
+   ceph osd pool set noautoscale
+
+You can turn it ``off`` using the command:
+
+.. prompt:: bash #
+
+   ceph osd pool unset noautoscale
+
+To ``get`` the value of the flag use the command:
+
+.. prompt:: bash #
+
+   ceph osd pool get noautoscale
+
+Viewing PG scaling recommendations
+----------------------------------
+
+You can view each pool, its relative utilization, and any suggested changes to
+the PG count with this command:
+
+.. prompt:: bash #
+
+   ceph osd pool autoscale-status
+
+Output will be something like::
+
+   POOL    SIZE  TARGET SIZE  RATE  RAW CAPACITY   RATIO  TARGET RATIO  EFFECTIVE RATIO BIAS PG_NUM  NEW PG_NUM  AUTOSCALE BULK
+   a     12900M                3.0        82431M  0.4695                                          8         128  warn      True
+   c         0                 3.0        82431M  0.0000        0.2000           0.9884  1.0      1          64  warn      True
+   b         0        953.6M   3.0        82431M  0.0347                                          8              warn      False
+
+**SIZE** is the amount of data stored in the pool. **TARGET SIZE**, if
+present, is the amount of data the administrator has specified that
+they expect to eventually be stored in this pool.  The system uses
+the larger of the two values for its calculation.
+
+**RATE** is the multiplier for the pool that determines how much raw
+storage capacity is consumed.  For example, a 3 replica pool will
+have a ratio of 3.0, while a k=4,m=2 erasure coded pool will have a
+ratio of 1.5.
+
+**RAW CAPACITY** is the total amount of raw storage capacity on the
+OSDs that are responsible for storing this pool's (and perhaps other
+pools') data.  **RATIO** is the ratio of that total capacity that
+this pool is consuming (i.e., ratio = size * rate / raw capacity).
+
+**TARGET RATIO**, if present, is the ratio of storage that the
+administrator has specified that they expect this pool to consume
+relative to other pools with target ratios set.
+If both target size bytes and ratio are specified, the
+ratio takes precedence.
+
+**EFFECTIVE RATIO** is the target ratio after adjusting in two ways:
+
+1. subtracting any capacity expected to be used by pools with target size set
+2. normalizing the target ratios among pools with target ratio set so
+   they collectively target the rest of the space. For example, 4
+   pools with target_ratio 1.0 would have an effective ratio of 0.25.
+
+The system uses the larger of the actual ratio and the effective ratio
+for its calculation.
+
+**BIAS** is used as a multiplier to manually adjust a pool's PG based
+on prior information about how much PGs a specific pool is expected
+to have.
+
+**PG_NUM** is the current number of PGs for the pool (or the current
+number of PGs that the pool is working towards, if a ``pg_num``
+change is in progress).  **NEW PG_NUM**, if present, is what the
+system believes the pool's ``pg_num`` should be changed to.  It is
+always a power of 2, and will only be present if the "ideal" value
+varies from the current value by more than a factor of 3 by default.
+This factor can be be adjusted with:
+
+.. prompt:: bash #
+
+  ceph osd pool set threshold 2.0
+
+**AUTOSCALE**, is the pool ``pg_autoscale_mode``
+and will be either ``on``, ``off``, or ``warn``.
+
+The final column, **BULK** determines if the pool is ``bulk``
+and will be either ``True`` or ``False``. A ``bulk`` pool
+means that the pool is expected to be large and should start out
+with large amount of PGs for performance purposes. On the other hand,
+pools without the ``bulk`` flag are expected to be smaller e.g.,
+.mgr or meta pools.
+
+
+Automated scaling
+-----------------
+
+Allowing the cluster to automatically scale PGs based on usage is the
+simplest approach.  Ceph will look at the total available storage and
+target number of PGs for the whole system, look at how much data is
+stored in each pool, and try to apportion the PGs accordingly.  The
+system is relatively conservative with its approach, only making
+changes to a pool when the current number of PGs (``pg_num``) is more
+than 3 times off from what it thinks it should be.
+
+The target number of PGs per OSD is based on the
+``mon_target_pg_per_osd`` configurable (default: 100), which can be
+adjusted with:
+
+.. prompt:: bash #
+
+   ceph config set global mon_target_pg_per_osd 100
+
+The autoscaler analyzes pools and adjusts on a per-subtree basis.
+Because each pool may map to a different CRUSH rule, and each rule may
+distribute data across different devices, Ceph will consider
+utilization of each subtree of the hierarchy independently.  For
+example, a pool that maps to OSDs of class `ssd` and a pool that maps
+to OSDs of class `hdd` will each have optimal PG counts that depend on
+the number of those respective device types.
+
+In the case where a pool uses OSDs under two or more CRUSH roots, e.g., (shadow
+trees with both `ssd` and `hdd` devices), the autoscaler will
+issue a warning to the user in the manager log stating the name of the pool
+and the set of roots that overlap each other. The autoscaler will not
+scale any pools with overlapping roots because this can cause problems
+with the scaling process. We recommend making each pool belong to only
+one root (one OSD class) to get rid of the warning and ensure a successful
+scaling process.
+
+The autoscaler uses the `bulk` flag to determine which pool
+should start out with a full complement of PGs and only
+scales down when the usage ratio across the pool is not even.
+However, if the pool doesn't have the `bulk` flag, the pool will
+start out with minimal PGs and only when there is more usage in the pool.
+
+To create pool with `bulk` flag:
+
+.. prompt:: bash #
+
+   ceph osd pool create <pool-name> --bulk
+
+To set/unset `bulk` flag of existing pool:
+
+.. prompt:: bash #
+
+   ceph osd pool set <pool-name> bulk <true/false/1/0>
+
+To get `bulk` flag of existing pool:
+
+.. prompt:: bash #
+
+   ceph osd pool get <pool-name> bulk
+
+.. _specifying_pool_target_size:
+
+Specifying expected pool size
+-----------------------------
+
+When a cluster or pool is first created, it will consume a small
+fraction of the total cluster capacity and will appear to the system
+as if it should only need a small number of placement groups.
+However, in most cases cluster administrators have a good idea which
+pools are expected to consume most of the system capacity over time.
+By providing this information to Ceph, a more appropriate number of
+PGs can be used from the beginning, preventing subsequent changes in
+``pg_num`` and the overhead associated with moving data around when
+those adjustments are made.
+
+The *target size* of a pool can be specified in two ways: either in
+terms of the absolute size of the pool (i.e., bytes), or as a weight
+relative to other pools with a ``target_size_ratio`` set.
+
+For example:
+
+.. prompt:: bash #
+
+   ceph osd pool set mypool target_size_bytes 100T
+
+will tell the system that `mypool` is expected to consume 100 TiB of
+space.  Alternatively:
+
+.. prompt:: bash # 
+
+   ceph osd pool set mypool target_size_ratio 1.0
+
+will tell the system that `mypool` is expected to consume 1.0 relative
+to the other pools with ``target_size_ratio`` set. If `mypool` is the
+only pool in the cluster, this means an expected use of 100% of the
+total capacity. If there is a second pool with ``target_size_ratio``
+1.0, both pools would expect to use 50% of the cluster capacity.
+
+You can also set the target size of a pool at creation time with the optional ``--target-size-bytes <bytes>`` or ``--target-size-ratio <ratio>`` arguments to the ``ceph osd pool create`` command.
+
+Note that if impossible target size values are specified (for example,
+a capacity larger than the total cluster) then a health warning
+(``POOL_TARGET_SIZE_BYTES_OVERCOMMITTED``) will be raised.
+
+If both ``target_size_ratio`` and ``target_size_bytes`` are specified
+for a pool, only the ratio will be considered, and a health warning
+(``POOL_HAS_TARGET_SIZE_BYTES_AND_RATIO``) will be issued.
+
+Specifying bounds on a pool's PGs
+---------------------------------
+
+It is also possible to specify a minimum number of PGs for a pool.
+This is useful for establishing a lower bound on the amount of
+parallelism client will see when doing IO, even when a pool is mostly
+empty.  Setting the lower bound prevents Ceph from reducing (or
+recommending you reduce) the PG number below the configured number.
+
+You can set the minimum or maximum number of PGs for a pool with:
+
+.. prompt:: bash #
+
+   ceph osd pool set <pool-name> pg_num_min <num>
+   ceph osd pool set <pool-name> pg_num_max <num>
+
+You can also specify the minimum or maximum PG count at pool creation
+time with the optional ``--pg-num-min <num>`` or ``--pg-num-max
+<num>`` arguments to the ``ceph osd pool create`` command.
+
+.. _preselection:
+
+A preselection of pg_num
+========================
+
+When creating a new pool with:
+
+.. prompt:: bash #
+
+   ceph osd pool create {pool-name} [pg_num]
+
+it is optional to choose the value of ``pg_num``.  If you do not
+specify ``pg_num``, the cluster can (by default) automatically tune it
+for you based on how much data is stored in the pool (see above, :ref:`pg-autoscaler`).
+
+Alternatively, ``pg_num`` can be explicitly provided.  However,
+whether you specify a ``pg_num`` value or not does not affect whether
+the value is automatically tuned by the cluster after the fact.  To
+enable or disable auto-tuning:
+
+.. prompt:: bash #
+
+   ceph osd pool set {pool-name} pg_autoscale_mode (on|off|warn)
+
+The "rule of thumb" for PGs per OSD has traditionally be 100.  With
+the additional of the balancer (which is also enabled by default), a
+value of more like 50 PGs per OSD is probably reasonable.  The
+challenge (which the autoscaler normally does for you), is to:
+
+- have the PGs per pool proportional to the data in the pool, and
+- end up with 50-100 PGs per OSDs, after the replication or
+  erasuring-coding fan-out of each PG across OSDs is taken into
+  consideration
+
+How are Placement Groups used ?
+===============================
+
+A placement group (PG) aggregates objects within a pool because
+tracking object placement and object metadata on a per-object basis is
+computationally expensive--i.e., a system with millions of objects
+cannot realistically track placement on a per-object basis.
+
+.. ditaa::
+           /-----\  /-----\  /-----\  /-----\  /-----\
+           | obj |  | obj |  | obj |  | obj |  | obj |
+           \-----/  \-----/  \-----/  \-----/  \-----/
+              |        |        |        |        |
+              +--------+--------+        +---+----+
+              |                              |
+              v                              v
+   +-----------------------+      +-----------------------+
+   |  Placement Group #1   |      |  Placement Group #2   |
+   |                       |      |                       |
+   +-----------------------+      +-----------------------+
+               |                              |
+               +------------------------------+
+                             |
+                             v
+                  +-----------------------+
+                  |        Pool           |
+                  |                       |
+                  +-----------------------+
+
+The Ceph client will calculate which placement group an object should
+be in. It does this by hashing the object ID and applying an operation
+based on the number of PGs in the defined pool and the ID of the pool.
+See `Mapping PGs to OSDs`_ for details.
+
+The object's contents within a placement group are stored in a set of
+OSDs. For instance, in a replicated pool of size two, each placement
+group will store objects on two OSDs, as shown below.
+
+.. ditaa::
+   +-----------------------+      +-----------------------+
+   |  Placement Group #1   |      |  Placement Group #2   |
+   |                       |      |                       |
+   +-----------------------+      +-----------------------+
+        |             |               |             |
+        v             v               v             v
+   /----------\  /----------\    /----------\  /----------\
+   |          |  |          |    |          |  |          |
+   |  OSD #1  |  |  OSD #2  |    |  OSD #2  |  |  OSD #3  |
+   |          |  |          |    |          |  |          |
+   \----------/  \----------/    \----------/  \----------/
+
+
+Should OSD #2 fail, another will be assigned to Placement Group #1 and
+will be filled with copies of all objects in OSD #1. If the pool size
+is changed from two to three, an additional OSD will be assigned to
+the placement group and will receive copies of all objects in the
+placement group.
+
+Placement groups do not own the OSD; they share it with other
+placement groups from the same pool or even other pools. If OSD #2
+fails, the Placement Group #2 will also have to restore copies of
+objects, using OSD #3.
+
+When the number of placement groups increases, the new placement
+groups will be assigned OSDs. The result of the CRUSH function will
+also change and some objects from the former placement groups will be
+copied over to the new Placement Groups and removed from the old ones.
+
+Placement Groups Tradeoffs
+==========================
+
+Data durability and even distribution among all OSDs call for more
+placement groups but their number should be reduced to the minimum to
+save CPU and memory.
+
+.. _data durability:
+
+Data durability
+---------------
+
+After an OSD fails, the risk of data loss increases until the data it
+contained is fully recovered. Let's imagine a scenario that causes
+permanent data loss in a single placement group:
+
+- The OSD fails and all copies of the object it contains are lost.
+  For all objects within the placement group the number of replica
+  suddenly drops from three to two.
+
+- Ceph starts recovery for this placement group by choosing a new OSD
+  to re-create the third copy of all objects.
+
+- Another OSD, within the same placement group, fails before the new
+  OSD is fully populated with the third copy. Some objects will then
+  only have one surviving copies.
+
+- Ceph picks yet another OSD and keeps copying objects to restore the
+  desired number of copies.
+
+- A third OSD, within the same placement group, fails before recovery
+  is complete. If this OSD contained the only remaining copy of an
+  object, it is permanently lost.
+
+In a cluster containing 10 OSDs with 512 placement groups in a three
+replica pool, CRUSH will give each placement groups three OSDs. In the
+end, each OSDs will end up hosting (512 * 3) / 10 = ~150 Placement
+Groups. When the first OSD fails, the above scenario will therefore
+start recovery for all 150 placement groups at the same time.
+
+The 150 placement groups being recovered are likely to be
+homogeneously spread over the 9 remaining OSDs. Each remaining OSD is
+therefore likely to send copies of objects to all others and also
+receive some new objects to be stored because they became part of a
+new placement group.
+
+The amount of time it takes for this recovery to complete entirely
+depends on the architecture of the Ceph cluster. Let say each OSD is
+hosted by a 1TB SSD on a single machine and all of them are connected
+to a 10Gb/s switch and the recovery for a single OSD completes within
+M minutes. If there are two OSDs per machine using spinners with no
+SSD journal and a 1Gb/s switch, it will at least be an order of
+magnitude slower.
+
+In a cluster of this size, the number of placement groups has almost
+no influence on data durability. It could be 128 or 8192 and the
+recovery would not be slower or faster.
+
+However, growing the same Ceph cluster to 20 OSDs instead of 10 OSDs
+is likely to speed up recovery and therefore improve data durability
+significantly. Each OSD now participates in only ~75 placement groups
+instead of ~150 when there were only 10 OSDs and it will still require
+all 19 remaining OSDs to perform the same amount of object copies in
+order to recover. But where 10 OSDs had to copy approximately 100GB
+each, they now have to copy 50GB each instead. If the network was the
+bottleneck, recovery will happen twice as fast. In other words,
+recovery goes faster when the number of OSDs increases.
+
+If this cluster grows to 40 OSDs, each of them will only host ~35
+placement groups. If an OSD dies, recovery will keep going faster
+unless it is blocked by another bottleneck. However, if this cluster
+grows to 200 OSDs, each of them will only host ~7 placement groups. If
+an OSD dies, recovery will happen between at most of ~21 (7 * 3) OSDs
+in these placement groups: recovery will take longer than when there
+were 40 OSDs, meaning the number of placement groups should be
+increased.
+
+No matter how short the recovery time is, there is a chance for a
+second OSD to fail while it is in progress. In the 10 OSDs cluster
+described above, if any of them fail, then ~17 placement groups
+(i.e. ~150 / 9 placement groups being recovered) will only have one
+surviving copy. And if any of the 8 remaining OSD fail, the last
+objects of two placement groups are likely to be lost (i.e. ~17 / 8
+placement groups with only one remaining copy being recovered).
+
+When the size of the cluster grows to 20 OSDs, the number of Placement
+Groups damaged by the loss of three OSDs drops. The second OSD lost
+will degrade ~4 (i.e. ~75 / 19 placement groups being recovered)
+instead of ~17 and the third OSD lost will only lose data if it is one
+of the four OSDs containing the surviving copy. In other words, if the
+probability of losing one OSD is 0.0001% during the recovery time
+frame, it goes from 17 * 10 * 0.0001% in the cluster with 10 OSDs to 4 * 20 *
+0.0001% in the cluster with 20 OSDs.
+
+In a nutshell, more OSDs mean faster recovery and a lower risk of
+cascading failures leading to the permanent loss of a Placement
+Group. Having 512 or 4096 Placement Groups is roughly equivalent in a
+cluster with less than 50 OSDs as far as data durability is concerned.
+
+Note: It may take a long time for a new OSD added to the cluster to be
+populated with placement groups that were assigned to it. However
+there is no degradation of any object and it has no impact on the
+durability of the data contained in the Cluster.
+
+.. _object distribution:
+
+Object distribution within a pool
+---------------------------------
+
+Ideally objects are evenly distributed in each placement group. Since
+CRUSH computes the placement group for each object, but does not
+actually know how much data is stored in each OSD within this
+placement group, the ratio between the number of placement groups and
+the number of OSDs may influence the distribution of the data
+significantly.
+
+For instance, if there was a single placement group for ten OSDs in a
+three replica pool, only three OSD would be used because CRUSH would
+have no other choice. When more placement groups are available,
+objects are more likely to be evenly spread among them. CRUSH also
+makes every effort to evenly spread OSDs among all existing Placement
+Groups.
+
+As long as there are one or two orders of magnitude more Placement
+Groups than OSDs, the distribution should be even. For instance, 256
+placement groups for 3 OSDs, 512 or 1024 placement groups for 10 OSDs
+etc.
+
+Uneven data distribution can be caused by factors other than the ratio
+between OSDs and placement groups. Since CRUSH does not take into
+account the size of the objects, a few very large objects may create
+an imbalance. Let say one million 4K objects totaling 4GB are evenly
+spread among 1024 placement groups on 10 OSDs. They will use 4GB / 10
+= 400MB on each OSD. If one 400MB object is added to the pool, the
+three OSDs supporting the placement group in which the object has been
+placed will be filled with 400MB + 400MB = 800MB while the seven
+others will remain occupied with only 400MB.
+
+.. _resource usage:
+
+Memory, CPU and network usage
+-----------------------------
+
+For each placement group, OSDs and MONs need memory, network and CPU
+at all times and even more during recovery. Sharing this overhead by
+clustering objects within a placement group is one of the main reasons
+they exist.
+
+Minimizing the number of placement groups saves significant amounts of
+resources.
+
+.. _choosing-number-of-placement-groups:
+
+Choosing the number of Placement Groups
+=======================================
+
+.. note: It is rarely necessary to do this math by hand.  Instead, use the ``ceph osd pool autoscale-status`` command in combination with the ``target_size_bytes`` or ``target_size_ratio`` pool properties.  See :ref:`pg-autoscaler` for more information.
+
+If you have more than 50 OSDs, we recommend approximately 50-100
+placement groups per OSD to balance out resource usage, data
+durability and distribution. If you have less than 50 OSDs, choosing
+among the `preselection`_ above is best. For a single pool of objects,
+you can use the following formula to get a baseline
+
+  Total PGs = :math:`\frac{OSDs \times 100}{pool \: size}`
+
+Where **pool size** is either the number of replicas for replicated
+pools or the K+M sum for erasure coded pools (as returned by **ceph
+osd erasure-code-profile get**).
+
+You should then check if the result makes sense with the way you
+designed your Ceph cluster to maximize `data durability`_,
+`object distribution`_ and minimize `resource usage`_.
+
+The result should always be **rounded up to the nearest power of two**.
+
+Only a power of two will evenly balance the number of objects among
+placement groups. Other values will result in an uneven distribution of
+data across your OSDs. Their use should be limited to incrementally
+stepping from one power of two to another.
+
+As an example, for a cluster with 200 OSDs and a pool size of 3
+replicas, you would estimate your number of PGs as follows
+
+  :math:`\frac{200 \times 100}{3} = 6667`. Nearest power of 2: 8192
+
+When using multiple data pools for storing objects, you need to ensure
+that you balance the number of placement groups per pool with the
+number of placement groups per OSD so that you arrive at a reasonable
+total number of placement groups that provides reasonably low variance
+per OSD without taxing system resources or making the peering process
+too slow.
+
+For instance a cluster of 10 pools each with 512 placement groups on
+ten OSDs is a total of 5,120 placement groups spread over ten OSDs,
+that is 512 placement groups per OSD. That does not use too many
+resources. However, if 1,000 pools were created with 512 placement
+groups each, the OSDs will handle ~50,000 placement groups each and it
+would require significantly more resources and time for peering.
+
+You may find the `PGCalc`_ tool helpful.
+
+
+.. _setting the number of placement groups:
+
+Set the Number of Placement Groups
+==================================
+
+To set the number of placement groups in a pool, you must specify the
+number of placement groups at the time you create the pool.
+See `Create a Pool`_ for details.  Even after a pool is created you can also change the number of placement groups with:
+
+.. prompt:: bash # 
+
+   ceph osd pool set {pool-name} pg_num {pg_num}
+
+After you increase the number of placement groups, you must also
+increase the number of placement groups for placement (``pgp_num``)
+before your cluster will rebalance. The ``pgp_num`` will be the number of
+placement groups that will be considered for placement by the CRUSH
+algorithm. Increasing ``pg_num`` splits the placement groups but data
+will not be migrated to the newer placement groups until placement
+groups for placement, ie. ``pgp_num`` is increased. The ``pgp_num``
+should be equal to the ``pg_num``.  To increase the number of
+placement groups for placement, execute the following:
+
+.. prompt:: bash #
+
+   ceph osd pool set {pool-name} pgp_num {pgp_num}
+
+When decreasing the number of PGs, ``pgp_num`` is adjusted
+automatically for you.
+
+Get the Number of Placement Groups
+==================================
+
+To get the number of placement groups in a pool, execute the following:
+
+.. prompt:: bash #
+   
+   ceph osd pool get {pool-name} pg_num
+
+
+Get a Cluster's PG Statistics
+=============================
+
+To get the statistics for the placement groups in your cluster, execute the following:
+
+.. prompt:: bash #
+
+   ceph pg dump [--format {format}]
+
+Valid formats are ``plain`` (default) and ``json``.
+
+
+Get Statistics for Stuck PGs
+============================
+
+To get the statistics for all placement groups stuck in a specified state,
+execute the following:
+
+.. prompt:: bash #
+
+   ceph pg dump_stuck inactive|unclean|stale|undersized|degraded [--format <format>] [-t|--threshold <seconds>]
+
+**Inactive** Placement groups cannot process reads or writes because they are waiting for an OSD
+with the most up-to-date data to come up and in.
+
+**Unclean** Placement groups contain objects that are not replicated the desired number
+of times. They should be recovering.
+
+**Stale** Placement groups are in an unknown state - the OSDs that host them have not
+reported to the monitor cluster in a while (configured by ``mon_osd_report_timeout``).
+
+Valid formats are ``plain`` (default) and ``json``. The threshold defines the minimum number
+of seconds the placement group is stuck before including it in the returned statistics
+(default 300 seconds).
+
+
+Get a PG Map
+============
+
+To get the placement group map for a particular placement group, execute the following:
+
+.. prompt:: bash #
+
+   ceph pg map {pg-id}
+
+For example: 
+
+.. prompt:: bash #
+
+   ceph pg map 1.6c
+
+Ceph will return the placement group map, the placement group, and the OSD status:
+
+.. prompt:: bash #
+
+   osdmap e13 pg 1.6c (1.6c) -> up [1,0] acting [1,0]
+
+
+Get a PGs Statistics
+====================
+
+To retrieve statistics for a particular placement group, execute the following:
+
+.. prompt:: bash #
+
+   ceph pg {pg-id} query
+
+
+Scrub a Placement Group
+=======================
+
+To scrub a placement group, execute the following:
+
+.. prompt:: bash #
+
+   ceph pg scrub {pg-id}
+
+Ceph checks the primary and any replica nodes, generates a catalog of all objects
+in the placement group and compares them to ensure that no objects are missing
+or mismatched, and their contents are consistent.  Assuming the replicas all
+match, a final semantic sweep ensures that all of the snapshot-related object
+metadata is consistent. Errors are reported via logs.
+
+To scrub all placement groups from a specific pool, execute the following:
+
+.. prompt:: bash #
+
+   ceph osd pool scrub {pool-name}
+
+Prioritize backfill/recovery of a Placement Group(s)
+====================================================
+
+You may run into a situation where a bunch of placement groups will require
+recovery and/or backfill, and some particular groups hold data more important
+than others (for example, those PGs may hold data for images used by running
+machines and other PGs may be used by inactive machines/less relevant data).
+In that case, you may want to prioritize recovery of those groups so
+performance and/or availability of data stored on those groups is restored
+earlier. To do this (mark particular placement group(s) as prioritized during
+backfill or recovery), execute the following:
+
+.. prompt:: bash #
+
+   ceph pg force-recovery {pg-id} [{pg-id #2}] [{pg-id #3} ...]
+   ceph pg force-backfill {pg-id} [{pg-id #2}] [{pg-id #3} ...]
+
+This will cause Ceph to perform recovery or backfill on specified placement
+groups first, before other placement groups. This does not interrupt currently
+ongoing backfills or recovery, but causes specified PGs to be processed
+as soon as possible. If you change your mind or prioritize wrong groups,
+use:
+
+.. prompt:: bash #
+
+   ceph pg cancel-force-recovery {pg-id} [{pg-id #2}] [{pg-id #3} ...]
+   ceph pg cancel-force-backfill {pg-id} [{pg-id #2}] [{pg-id #3} ...]
+
+This will remove "force" flag from those PGs and they will be processed
+in default order. Again, this doesn't affect currently processed placement
+group, only those that are still queued.
+
+The "force" flag is cleared automatically after recovery or backfill of group
+is done.
+
+Similarly, you may use the following commands to force Ceph to perform recovery
+or backfill on all placement groups from a specified pool first:
+
+.. prompt:: bash #
+
+   ceph osd pool force-recovery {pool-name}
+   ceph osd pool force-backfill {pool-name}
+
+or:
+
+.. prompt:: bash #
+
+   ceph osd pool cancel-force-recovery {pool-name}
+   ceph osd pool cancel-force-backfill {pool-name}
+
+to restore to the default recovery or backfill priority if you change your mind.
+
+Note that these commands could possibly break the ordering of Ceph's internal
+priority computations, so use them with caution!
+Especially, if you have multiple pools that are currently sharing the same
+underlying OSDs, and some particular pools hold data more important than others,
+we recommend you use the following command to re-arrange all pools's
+recovery/backfill priority in a better order:
+
+.. prompt:: bash #
+
+   ceph osd pool set {pool-name} recovery_priority {value}
+
+For example, if you have 10 pools you could make the most important one priority 10,
+next 9, etc. Or you could leave most pools alone and have say 3 important pools
+all priority 1 or priorities 3, 2, 1 respectively.
+
+Revert Lost
+===========
+
+If the cluster has lost one or more objects, and you have decided to
+abandon the search for the lost data, you must mark the unfound objects
+as ``lost``.
+
+If all possible locations have been queried and objects are still
+lost, you may have to give up on the lost objects. This is
+possible given unusual combinations of failures that allow the cluster
+to learn about writes that were performed before the writes themselves
+are recovered.
+
+Currently the only supported option is "revert", which will either roll back to
+a previous version of the object or (if it was a new object) forget about it
+entirely. To mark the "unfound" objects as "lost", execute the following:
+
+.. prompt:: bash #
+
+   ceph pg {pg-id} mark_unfound_lost revert|delete
+
+.. important:: Use this feature with caution, because it may confuse
+   applications that expect the object(s) to exist.
+
+
+.. toctree::
+        :hidden:
+
+        pg-states
+        pg-concepts
+
+
+.. _Create a Pool: ../pools#createpool
+.. _Mapping PGs to OSDs: ../../../architecture#mapping-pgs-to-osds
+.. _pgcalc: https://old.ceph.com/pgcalc/
diff --git a/doc/rados/operations/pools.rst b/doc/rados/operations/pools.rst
new file mode 100644
index 000000000..b44c48460
--- /dev/null
+++ b/doc/rados/operations/pools.rst
@@ -0,0 +1,900 @@
+.. _rados_pools:
+
+=======
+ Pools
+=======
+Pools are logical partitions that are used to store objects.
+
+Pools provide:
+
+- **Resilience**: It is possible to set the number of OSDs that are allowed to
+  fail without any data being lost. If your cluster uses replicated pools, the
+  number of OSDs that can fail without data loss is the number of replicas.
+  For example: a typical configuration stores an object and two additional
+  copies (that is: ``size = 3``), but you can configure the number of replicas
+  on a per-pool basis. For `erasure coded pools <../erasure-code>`_, resilience
+  is defined as the number of coding chunks (for example, ``m = 2`` in the
+  **erasure code profile**).
+
+- **Placement Groups**: You can set the number of placement groups for the
+  pool. A typical configuration targets approximately 100 placement groups per
+  OSD, providing optimal balancing without consuming many computing resources.
+  When setting up multiple pools, be careful to set a reasonable number of
+  placement groups for each pool and for the cluster as a whole. Note that each
+  PG belongs to a specific pool: when multiple pools use the same OSDs, make
+  sure that the **sum** of PG replicas per OSD is in the desired PG per OSD
+  target range. Use the `pgcalc`_ tool to calculate the number of placement
+  groups to set for your pool.
+
+- **CRUSH Rules**: When data is stored in a pool, the placement of the object
+  and its replicas (or chunks, in the case of erasure-coded pools) in your
+  cluster is governed by CRUSH rules. Custom CRUSH rules can be created for a
+  pool if the default rule does not fit your use case.
+
+- **Snapshots**: The command ``ceph osd pool mksnap`` creates a snapshot of a
+  pool. 
+
+Pool Names
+==========
+
+Pool names beginning with ``.`` are reserved for use by Ceph's internal
+operations. Please do not create or manipulate pools with these names.
+
+List Pools
+==========
+
+To list your cluster's pools, execute:
+
+.. prompt:: bash $
+
+   ceph osd lspools
+
+.. _createpool:
+
+Create a Pool
+=============
+
+Before creating pools, refer to the `Pool, PG and CRUSH Config Reference`_.
+Ideally, you should override the default value for the number of placement
+groups in your Ceph configuration file, as the default is NOT ideal.
+For details on placement group numbers refer to `setting the number of placement groups`_
+
+.. note:: Starting with Luminous, all pools need to be associated to the
+   application using the pool. See `Associate Pool to Application`_ below for
+   more information.
+
+For example:
+
+.. prompt:: bash $
+
+	osd pool default pg num = 100
+	osd pool default pgp num = 100
+
+To create a pool, execute:
+
+.. prompt:: bash $
+
+	ceph osd pool create {pool-name} [{pg-num} [{pgp-num}]] [replicated] \
+             [crush-rule-name] [expected-num-objects]
+	ceph osd pool create {pool-name} [{pg-num} [{pgp-num}]]   erasure \
+             [erasure-code-profile] [crush-rule-name] [expected_num_objects] [--autoscale-mode=<on,off,warn>]
+
+Where:
+
+``{pool-name}``
+
+:Description: The name of the pool. It must be unique.
+:Type: String
+:Required: Yes.
+
+``{pg-num}``
+
+:Description: The total number of placement groups for the pool. See `Placement
+              Groups`_  for details on calculating a suitable number. The
+              default value ``8`` is NOT suitable for most systems.
+
+:Type: Integer
+:Required: Yes.
+:Default: 8
+
+``{pgp-num}``
+
+:Description: The total number of placement groups for placement purposes. This
+              **should be equal to the total number of placement groups**, except
+              for placement group splitting scenarios.
+
+:Type: Integer
+:Required: Yes. Picks up default or Ceph configuration value if not specified.
+:Default: 8
+
+``{replicated|erasure}``
+
+:Description: The pool type which may either be **replicated** to
+              recover from lost OSDs by keeping multiple copies of the
+              objects or **erasure** to get a kind of
+              `generalized RAID5 <../erasure-code>`_ capability.
+              The **replicated** pools require more
+              raw storage but implement all Ceph operations. The
+              **erasure** pools require less raw storage but only
+              implement a subset of the available operations.
+
+:Type: String
+:Required: No.
+:Default: replicated
+
+``[crush-rule-name]``
+
+:Description: The name of a CRUSH rule to use for this pool.  The specified
+              rule must exist.
+
+:Type: String
+:Required: No.
+:Default: For **replicated** pools it is the rule specified by the ``osd
+          pool default crush rule`` config variable.  This rule must exist.
+          For **erasure** pools it is ``erasure-code`` if the ``default``
+          `erasure code profile`_ is used or ``{pool-name}`` otherwise.  This
+          rule will be created implicitly if it doesn't exist already.
+
+
+``[erasure-code-profile=profile]``
+
+.. _erasure code profile: ../erasure-code-profile
+
+:Description: For **erasure** pools only. Use the `erasure code profile`_. It
+              must be an existing profile as defined by
+              **osd erasure-code-profile set**.
+
+:Type: String
+:Required: No.
+
+``--autoscale-mode=<on,off,warn>``
+
+:Description: Autoscale mode
+
+:Type: String
+:Required: No.
+:Default:  The default behavior is controlled by the ``osd pool default pg autoscale mode`` option.
+
+If you set the autoscale mode to ``on`` or ``warn``, you can let the system autotune or recommend changes to the number of placement groups in your pool based on actual usage.  If you leave it off, then you should refer to `Placement Groups`_ for more information.
+
+.. _Placement Groups: ../placement-groups
+
+``[expected-num-objects]``
+
+:Description: The expected number of objects for this pool. By setting this value (
+              together with a negative **filestore merge threshold**), the PG folder
+              splitting would happen at the pool creation time, to avoid the latency
+              impact to do a runtime folder splitting.
+
+:Type: Integer
+:Required: No.
+:Default: 0, no splitting at the pool creation time.
+
+.. _associate-pool-to-application:
+
+Associate Pool to Application
+=============================
+
+Pools need to be associated with an application before use. Pools that will be
+used with CephFS or pools that are automatically created by RGW are
+automatically associated. Pools that are intended for use with RBD should be
+initialized using the ``rbd`` tool (see `Block Device Commands`_ for more
+information).
+
+For other cases, you can manually associate a free-form application name to
+a pool.:
+
+.. prompt:: bash $
+
+   ceph osd pool application enable {pool-name} {application-name}
+
+.. note:: CephFS uses the application name ``cephfs``, RBD uses the
+   application name ``rbd``, and RGW uses the application name ``rgw``.
+
+Set Pool Quotas
+===============
+
+You can set pool quotas for the maximum number of bytes and/or the maximum
+number of objects per pool:
+
+.. prompt:: bash $
+
+   ceph osd pool set-quota {pool-name} [max_objects {obj-count}] [max_bytes {bytes}]
+
+For example:
+
+.. prompt:: bash $
+
+   ceph osd pool set-quota data max_objects 10000
+
+To remove a quota, set its value to ``0``.
+
+
+Delete a Pool
+=============
+
+To delete a pool, execute:
+
+.. prompt:: bash $
+
+   ceph osd pool delete {pool-name} [{pool-name} --yes-i-really-really-mean-it]
+
+
+To remove a pool the mon_allow_pool_delete flag must be set to true in the Monitor's
+configuration. Otherwise they will refuse to remove a pool.
+
+See `Monitor Configuration`_ for more information.
+
+.. _Monitor Configuration: ../../configuration/mon-config-ref
+
+If you created your own rules for a pool you created, you should consider
+removing them when you no longer need your pool:
+
+.. prompt:: bash $
+
+   ceph osd pool get {pool-name} crush_rule
+
+If the rule was "123", for example, you can check the other pools like so:
+
+.. prompt:: bash $
+
+	ceph osd dump | grep "^pool" | grep "crush_rule 123"
+
+If no other pools use that custom rule, then it's safe to delete that
+rule from the cluster.
+
+If you created users with permissions strictly for a pool that no longer
+exists, you should consider deleting those users too:
+
+
+.. prompt:: bash $
+
+	ceph auth ls | grep -C 5 {pool-name}
+	ceph auth del {user}
+
+
+Rename a Pool
+=============
+
+To rename a pool, execute:
+
+.. prompt:: bash $
+
+   ceph osd pool rename {current-pool-name} {new-pool-name}
+
+If you rename a pool and you have per-pool capabilities for an authenticated
+user, you must update the user's capabilities (i.e., caps) with the new pool
+name.
+
+Show Pool Statistics
+====================
+
+To show a pool's utilization statistics, execute:
+
+.. prompt:: bash $
+
+   rados df
+
+Additionally, to obtain I/O information for a specific pool or all, execute:
+
+.. prompt:: bash $
+
+   ceph osd pool stats [{pool-name}]
+
+
+Make a Snapshot of a Pool
+=========================
+
+To make a snapshot of a pool, execute:
+
+.. prompt:: bash $
+
+   ceph osd pool mksnap {pool-name} {snap-name}
+
+Remove a Snapshot of a Pool
+===========================
+
+To remove a snapshot of a pool, execute:
+
+.. prompt:: bash $
+
+   ceph osd pool rmsnap {pool-name} {snap-name}
+
+.. _setpoolvalues:
+
+
+Set Pool Values
+===============
+
+To set a value to a pool, execute the following:
+
+.. prompt:: bash $
+
+   ceph osd pool set {pool-name} {key} {value}
+
+You may set values for the following keys:
+
+.. _compression_algorithm:
+
+``compression_algorithm``
+
+:Description: Sets inline compression algorithm to use for underlying BlueStore. This setting overrides the `global setting <https://docs.ceph.com/en/latest/rados/configuration/bluestore-config-ref/#inline-compression>`__ of ``bluestore compression algorithm``.
+
+:Type: String
+:Valid Settings: ``lz4``, ``snappy``, ``zlib``, ``zstd``
+
+``compression_mode``
+
+:Description: Sets the policy for the inline compression algorithm for underlying BlueStore. This setting overrides the `global setting <http://docs.ceph.com/en/latest/rados/configuration/bluestore-config-ref/#inline-compression>`__ of ``bluestore compression mode``.
+
+:Type: String
+:Valid Settings: ``none``, ``passive``, ``aggressive``, ``force``
+
+``compression_min_blob_size``
+
+:Description: Chunks smaller than this are never compressed. This setting overrides the `global setting <http://docs.ceph.com/en/latest/rados/configuration/bluestore-config-ref/#inline-compression>`__ of ``bluestore compression min blob *``.
+
+:Type: Unsigned Integer
+
+``compression_max_blob_size``
+
+:Description: Chunks larger than this are broken into smaller blobs sizing
+              ``compression_max_blob_size`` before being compressed.
+
+:Type: Unsigned Integer
+
+.. _size:
+
+``size``
+
+:Description: Sets the number of replicas for objects in the pool.
+              See `Set the Number of Object Replicas`_ for further details.
+              Replicated pools only.
+
+:Type: Integer
+
+.. _min_size:
+
+``min_size``
+
+:Description: Sets the minimum number of replicas required for I/O.
+              See `Set the Number of Object Replicas`_ for further details.
+              In the case of Erasure Coded pools this should be set to a value
+              greater than 'k' since if we allow IO at the value 'k' there is no
+              redundancy and data will be lost in the event of a permanent OSD
+              failure. For more information see `Erasure Code
+              <../erasure-code>`_
+
+:Type: Integer
+:Version: ``0.54`` and above
+
+.. _pg_num:
+
+``pg_num``
+
+:Description: The effective number of placement groups to use when calculating
+              data placement.
+:Type: Integer
+:Valid Range: Superior to ``pg_num`` current value.
+
+.. _pgp_num:
+
+``pgp_num``
+
+:Description: The effective number of placement groups for placement to use
+              when calculating data placement.
+
+:Type: Integer
+:Valid Range: Equal to or less than ``pg_num``.
+
+.. _crush_rule:
+
+``crush_rule``
+
+:Description: The rule to use for mapping object placement in the cluster.
+:Type: String
+
+.. _allow_ec_overwrites:
+
+``allow_ec_overwrites``
+
+:Description: Whether writes to an erasure coded pool can update part
+              of an object, so cephfs and rbd can use it. See
+              `Erasure Coding with Overwrites`_ for more details.
+:Type: Boolean
+:Version: ``12.2.0`` and above
+
+.. _hashpspool:
+
+``hashpspool``
+
+:Description: Set/Unset HASHPSPOOL flag on a given pool.
+:Type: Integer
+:Valid Range: 1 sets flag, 0 unsets flag
+
+.. _nodelete:
+
+``nodelete``
+
+:Description: Set/Unset NODELETE flag on a given pool.
+:Type: Integer
+:Valid Range: 1 sets flag, 0 unsets flag
+:Version: Version ``FIXME``
+
+.. _nopgchange:
+
+``nopgchange``
+
+:Description: Set/Unset NOPGCHANGE flag on a given pool.
+:Type: Integer
+:Valid Range: 1 sets flag, 0 unsets flag
+:Version: Version ``FIXME``
+
+.. _nosizechange:
+
+``nosizechange``
+
+:Description: Set/Unset NOSIZECHANGE flag on a given pool.
+:Type: Integer
+:Valid Range: 1 sets flag, 0 unsets flag
+:Version: Version ``FIXME``
+
+.. _bulk:
+
+.. describe:: bulk
+
+   Set/Unset bulk flag on a given pool.
+
+   :Type: Boolean
+   :Valid Range: true/1 sets flag, false/0 unsets flag
+
+.. _write_fadvise_dontneed:
+
+``write_fadvise_dontneed``
+
+:Description: Set/Unset WRITE_FADVISE_DONTNEED flag on a given pool.
+:Type: Integer
+:Valid Range: 1 sets flag, 0 unsets flag
+
+.. _noscrub:
+
+``noscrub``
+
+:Description: Set/Unset NOSCRUB flag on a given pool.
+:Type: Integer
+:Valid Range: 1 sets flag, 0 unsets flag
+
+.. _nodeep-scrub:
+
+``nodeep-scrub``
+
+:Description: Set/Unset NODEEP_SCRUB flag on a given pool.
+:Type: Integer
+:Valid Range: 1 sets flag, 0 unsets flag
+
+.. _hit_set_type:
+
+``hit_set_type``
+
+:Description: Enables hit set tracking for cache pools.
+              See `Bloom Filter`_ for additional information.
+
+:Type: String
+:Valid Settings: ``bloom``, ``explicit_hash``, ``explicit_object``
+:Default: ``bloom``. Other values are for testing.
+
+.. _hit_set_count:
+
+``hit_set_count``
+
+:Description: The number of hit sets to store for cache pools. The higher
+              the number, the more RAM consumed by the ``ceph-osd`` daemon.
+
+:Type: Integer
+:Valid Range: ``1``. Agent doesn't handle > 1 yet.
+
+.. _hit_set_period:
+
+``hit_set_period``
+
+:Description: The duration of a hit set period in seconds for cache pools.
+              The higher the number, the more RAM consumed by the
+              ``ceph-osd`` daemon.
+
+:Type: Integer
+:Example: ``3600`` 1hr
+
+.. _hit_set_fpp:
+
+``hit_set_fpp``
+
+:Description: The false positive probability for the ``bloom`` hit set type.
+              See `Bloom Filter`_ for additional information.
+
+:Type: Double
+:Valid Range: 0.0 - 1.0
+:Default: ``0.05``
+
+.. _cache_target_dirty_ratio:
+
+``cache_target_dirty_ratio``
+
+:Description: The percentage of the cache pool containing modified (dirty)
+              objects before the cache tiering agent will flush them to the
+              backing storage pool.
+
+:Type: Double
+:Default: ``.4``
+
+.. _cache_target_dirty_high_ratio:
+
+``cache_target_dirty_high_ratio``
+
+:Description: The percentage of the cache pool containing modified (dirty)
+              objects before the cache tiering agent will flush them to the
+              backing storage pool with a higher speed.
+
+:Type: Double
+:Default: ``.6``
+
+.. _cache_target_full_ratio:
+
+``cache_target_full_ratio``
+
+:Description: The percentage of the cache pool containing unmodified (clean)
+              objects before the cache tiering agent will evict them from the
+              cache pool.
+
+:Type: Double
+:Default: ``.8``
+
+.. _target_max_bytes:
+
+``target_max_bytes``
+
+:Description: Ceph will begin flushing or evicting objects when the
+              ``max_bytes`` threshold is triggered.
+
+:Type: Integer
+:Example: ``1000000000000``  #1-TB
+
+.. _target_max_objects:
+
+``target_max_objects``
+
+:Description: Ceph will begin flushing or evicting objects when the
+              ``max_objects`` threshold is triggered.
+
+:Type: Integer
+:Example: ``1000000`` #1M objects
+
+
+``hit_set_grade_decay_rate``
+
+:Description: Temperature decay rate between two successive hit_sets
+:Type: Integer
+:Valid Range: 0 - 100
+:Default: ``20``
+
+
+``hit_set_search_last_n``
+
+:Description: Count at most N appearance in hit_sets for temperature calculation
+:Type: Integer
+:Valid Range: 0 - hit_set_count
+:Default: ``1``
+
+
+.. _cache_min_flush_age:
+
+``cache_min_flush_age``
+
+:Description: The time (in seconds) before the cache tiering agent will flush
+              an object from the cache pool to the storage pool.
+
+:Type: Integer
+:Example: ``600`` 10min
+
+.. _cache_min_evict_age:
+
+``cache_min_evict_age``
+
+:Description: The time (in seconds) before the cache tiering agent will evict
+              an object from the cache pool.
+
+:Type: Integer
+:Example: ``1800`` 30min
+
+.. _fast_read:
+
+``fast_read``
+
+:Description: On Erasure Coding pool, if this flag is turned on, the read request
+              would issue sub reads to all shards, and waits until it receives enough
+              shards to decode to serve the client. In the case of jerasure and isa
+              erasure plugins, once the first K replies return, client's request is
+              served immediately using the data decoded from these replies. This
+              helps to tradeoff some resources for better performance. Currently this
+              flag is only supported for Erasure Coding pool.
+
+:Type: Boolean
+:Defaults: ``0``
+
+.. _scrub_min_interval:
+
+``scrub_min_interval``
+
+:Description: The minimum interval in seconds for pool scrubbing when
+              load is low. If it is 0, the value osd_scrub_min_interval
+              from config is used.
+
+:Type: Double
+:Default: ``0``
+
+.. _scrub_max_interval:
+
+``scrub_max_interval``
+
+:Description: The maximum interval in seconds for pool scrubbing
+              irrespective of cluster load. If it is 0, the value
+              osd_scrub_max_interval from config is used.
+
+:Type: Double
+:Default: ``0``
+
+.. _deep_scrub_interval:
+
+``deep_scrub_interval``
+
+:Description: The interval in seconds for pool “deep� scrubbing. If it
+              is 0, the value osd_deep_scrub_interval from config is used.
+
+:Type: Double
+:Default: ``0``
+
+
+.. _recovery_priority:
+
+``recovery_priority``
+
+:Description: When a value is set it will increase or decrease the computed
+              reservation priority. This value must be in the range -10 to
+              10.  Use a negative priority for less important pools so they
+              have lower priority than any new pools.
+
+:Type: Integer
+:Default: ``0``
+
+
+.. _recovery_op_priority:
+
+``recovery_op_priority``
+
+:Description: Specify the recovery operation priority for this pool instead of ``osd_recovery_op_priority``.
+
+:Type: Integer
+:Default: ``0``
+
+
+Get Pool Values
+===============
+
+To get a value from a pool, execute the following:
+
+.. prompt:: bash $
+
+   ceph osd pool get {pool-name} {key}
+
+You may get values for the following keys:
+
+``size``
+
+:Description: see size_
+
+:Type: Integer
+
+``min_size``
+
+:Description: see min_size_
+
+:Type: Integer
+:Version: ``0.54`` and above
+
+``pg_num``
+
+:Description: see pg_num_
+
+:Type: Integer
+
+
+``pgp_num``
+
+:Description: see pgp_num_
+
+:Type: Integer
+:Valid Range: Equal to or less than ``pg_num``.
+
+
+``crush_rule``
+
+:Description: see crush_rule_
+
+
+``hit_set_type``
+
+:Description: see hit_set_type_
+
+:Type: String
+:Valid Settings: ``bloom``, ``explicit_hash``, ``explicit_object``
+
+``hit_set_count``
+
+:Description: see hit_set_count_
+
+:Type: Integer
+
+
+``hit_set_period``
+
+:Description: see hit_set_period_
+
+:Type: Integer
+
+
+``hit_set_fpp``
+
+:Description: see hit_set_fpp_
+
+:Type: Double
+
+
+``cache_target_dirty_ratio``
+
+:Description: see cache_target_dirty_ratio_
+
+:Type: Double
+
+
+``cache_target_dirty_high_ratio``
+
+:Description: see cache_target_dirty_high_ratio_
+
+:Type: Double
+
+
+``cache_target_full_ratio``
+
+:Description: see cache_target_full_ratio_
+
+:Type: Double
+
+
+``target_max_bytes``
+
+:Description: see target_max_bytes_
+
+:Type: Integer
+
+
+``target_max_objects``
+
+:Description: see target_max_objects_
+
+:Type: Integer
+
+
+``cache_min_flush_age``
+
+:Description: see cache_min_flush_age_
+
+:Type: Integer
+
+
+``cache_min_evict_age``
+
+:Description: see cache_min_evict_age_
+
+:Type: Integer
+
+
+``fast_read``
+
+:Description: see fast_read_
+
+:Type: Boolean
+
+
+``scrub_min_interval``
+
+:Description: see scrub_min_interval_
+
+:Type: Double
+
+
+``scrub_max_interval``
+
+:Description: see scrub_max_interval_
+
+:Type: Double
+
+
+``deep_scrub_interval``
+
+:Description: see deep_scrub_interval_
+
+:Type: Double
+
+
+``allow_ec_overwrites``
+
+:Description: see allow_ec_overwrites_
+
+:Type: Boolean
+
+
+``recovery_priority``
+
+:Description: see recovery_priority_
+
+:Type: Integer
+
+
+``recovery_op_priority``
+
+:Description: see recovery_op_priority_
+
+:Type: Integer
+
+
+Set the Number of Object Replicas
+=================================
+
+To set the number of object replicas on a replicated pool, execute the following:
+
+.. prompt:: bash $
+
+   ceph osd pool set {poolname} size {num-replicas}
+
+.. important:: The ``{num-replicas}`` includes the object itself.
+   If you want the object and two copies of the object for a total of
+   three instances of the object, specify ``3``.
+
+For example:
+
+.. prompt:: bash $
+
+   ceph osd pool set data size 3
+
+You may execute this command for each pool. **Note:** An object might accept
+I/Os in degraded mode with fewer than ``pool size`` replicas.  To set a minimum
+number of required replicas for I/O, you should use the ``min_size`` setting.
+For example:
+
+.. prompt:: bash $
+
+   ceph osd pool set data min_size 2
+
+This ensures that no object in the data pool will receive I/O with fewer than
+``min_size`` replicas.
+
+
+Get the Number of Object Replicas
+=================================
+
+To get the number of object replicas, execute the following:
+
+.. prompt:: bash $
+
+   ceph osd dump | grep 'replicated size'
+
+Ceph will list the pools, with the ``replicated size`` attribute highlighted.
+By default, ceph creates two replicas of an object (a total of three copies, or
+a size of 3).
+
+
+.. _pgcalc: https://old.ceph.com/pgcalc/
+.. _Pool, PG and CRUSH Config Reference: ../../configuration/pool-pg-config-ref
+.. _Bloom Filter: https://en.wikipedia.org/wiki/Bloom_filter
+.. _setting the number of placement groups: ../placement-groups#set-the-number-of-placement-groups
+.. _Erasure Coding with Overwrites: ../erasure-code#erasure-coding-with-overwrites
+.. _Block Device Commands: ../../../rbd/rados-rbd-cmds/#create-a-block-device-pool
diff --git a/doc/rados/operations/stretch-mode.rst b/doc/rados/operations/stretch-mode.rst
new file mode 100644
index 000000000..6b4c9ba8a
--- /dev/null
+++ b/doc/rados/operations/stretch-mode.rst
@@ -0,0 +1,215 @@
+.. _stretch_mode:
+
+================
+Stretch Clusters
+================
+
+
+Stretch Clusters
+================
+Ceph generally expects all parts of its network and overall cluster to be
+equally reliable, with failures randomly distributed across the CRUSH map.
+So you may lose a switch that knocks out a number of OSDs, but we expect
+the remaining OSDs and monitors to route around that.
+
+This is usually a good choice, but may not work well in some
+stretched cluster configurations where a significant part of your cluster
+is stuck behind a single network component. For instance, a single
+cluster which is located in multiple data centers, and you want to
+sustain the loss of a full DC.
+
+There are two standard configurations we've seen deployed, with either
+two or three data centers (or, in clouds, availability zones). With two
+zones, we expect each site to hold a copy of the data, and for a third
+site to have a tiebreaker monitor (this can be a VM or high-latency compared
+to the main sites) to pick a winner if the network connection fails and both
+DCs remain alive. For three sites, we expect a copy of the data and an equal
+number of monitors in each site.
+
+Note that the standard Ceph configuration will survive MANY failures of the
+network or data centers and it will never compromise data consistency.  If you
+bring back enough Ceph servers following a failure, it will recover. If you
+lose a data center, but can still form a quorum of monitors and have all the data
+available (with enough copies to satisfy pools' ``min_size``, or CRUSH rules
+that will re-replicate to meet it), Ceph will maintain availability.
+
+What can't it handle?
+
+Stretch Cluster Issues
+======================
+No matter what happens, Ceph will not compromise on data integrity
+and consistency. If there's a failure in your network or a loss of nodes and
+you can restore service, Ceph will return to normal functionality on its own.
+
+But there are scenarios where you lose data availibility despite having
+enough servers available to satisfy Ceph's consistency and sizing constraints, or
+where you may be surprised to not satisfy Ceph's constraints.
+The first important category of these failures resolve around inconsistent
+networks -- if there's a netsplit, Ceph may be unable to mark OSDs down and kick
+them out of the acting PG sets despite the primary being unable to replicate data.
+If this happens, IO will not be permitted, because Ceph can't satisfy its durability
+guarantees.
+
+The second important category of failures is when you think you have data replicated
+across data centers, but the constraints aren't sufficient to guarantee this.
+For instance, you might have data centers A and B, and your CRUSH rule targets 3 copies
+and places a copy in each data center with a ``min_size`` of 2. The PG may go active with
+2 copies in site A and no copies in site B, which means that if you then lose site A you
+have lost data and Ceph can't operate on it. This situation is surprisingly difficult
+to avoid with standard CRUSH rules.
+
+Stretch Mode
+============
+The new stretch mode is designed to handle the 2-site case. Three sites are
+just as susceptible to netsplit issues, but are much more tolerant of
+component availability outages than 2-site clusters are.
+
+To enter stretch mode, you must set the location of each monitor, matching
+your CRUSH map. For instance, to place ``mon.a`` in your first data center:
+
+.. prompt:: bash $
+
+   ceph mon set_location a datacenter=site1
+
+Next, generate a CRUSH rule which will place 2 copies in each data center. This
+will require editing the CRUSH map directly:
+
+.. prompt:: bash $
+
+   ceph osd getcrushmap > crush.map.bin
+   crushtool -d crush.map.bin -o crush.map.txt
+
+Now edit the ``crush.map.txt`` file to add a new rule. Here
+there is only one other rule, so this is ID 1, but you may need
+to use a different rule ID. We also have two datacenter buckets
+named ``site1`` and ``site2``::
+
+  rule stretch_rule {
+          id 1
+          type replicated
+          min_size 1
+          max_size 10
+          step take site1
+          step chooseleaf firstn 2 type host
+          step emit
+          step take site2
+          step chooseleaf firstn 2 type host
+          step emit
+  }
+
+Finally, inject the CRUSH map to make the rule available to the cluster:
+
+.. prompt:: bash $
+
+   crushtool -c crush.map.txt -o crush2.map.bin
+   ceph osd setcrushmap -i crush2.map.bin
+
+If you aren't already running your monitors in connectivity mode, do so with
+the instructions in `Changing Monitor Elections`_.
+
+.. _Changing Monitor elections: ../change-mon-elections
+
+And lastly, tell the cluster to enter stretch mode. Here, ``mon.e`` is the
+tiebreaker and we are splitting across data centers. ``mon.e`` should be also
+set a datacenter, that will differ from ``site1`` and ``site2``. For this
+purpose you can create another datacenter bucket named ```site3`` in your
+CRUSH and place ``mon.e`` there:
+
+.. prompt:: bash $
+
+   ceph mon set_location e datacenter=site3
+   ceph mon enable_stretch_mode e stretch_rule datacenter
+
+When stretch mode is enabled, the OSDs wlll only take PGs active when
+they peer across data centers (or whatever other CRUSH bucket type
+you specified), assuming both are alive. Pools will increase in size
+from the default 3 to 4, expecting 2 copies in each site. OSDs will only
+be allowed to connect to monitors in the same data center. New monitors
+will not be allowed to join the cluster if they do not specify a location.
+
+If all the OSDs and monitors from a data center become inaccessible
+at once, the surviving data center will enter a degraded stretch mode. This
+will issue a warning, reduce the min_size to 1, and allow
+the cluster to go active with data in the single remaining site. Note that
+we do not change the pool size, so you will also get warnings that the
+pools are too small -- but a special stretch mode flag will prevent the OSDs
+from creating extra copies in the remaining data center (so it will only keep
+2 copies, as before).
+
+When the missing data center comes back, the cluster will enter
+recovery stretch mode. This changes the warning and allows peering, but
+still only requires OSDs from the data center which was up the whole time.
+When all PGs are in a known state, and are neither degraded nor incomplete,
+the cluster transitions back to regular stretch mode, ends the warning,
+restores min_size to its starting value (2) and requires both sites to peer,
+and stops requiring the always-alive site when peering (so that you can fail
+over to the other site, if necessary).
+
+  
+Stretch Mode Limitations
+========================
+As implied by the setup, stretch mode only handles 2 sites with OSDs.
+
+While it is not enforced, you should run 2 monitors in each site plus
+a tiebreaker, for a total of 5. This is because OSDs can only connect
+to monitors in their own site when in stretch mode.
+
+You cannot use erasure coded pools with stretch mode. If you try, it will
+refuse, and it will not allow you to create EC pools once in stretch mode.
+
+You must create your own CRUSH rule which provides 2 copies in each site, and
+you must use 4 total copies with 2 in each site. If you have existing pools
+with non-default size/min_size, Ceph will object when you attempt to
+enable stretch mode.
+
+Because it runs with ``min_size 1`` when degraded, you should only use stretch
+mode with all-flash OSDs.  This minimizes the time needed to recover once
+connectivity is restored, and thus minimizes the potential for data loss.
+
+Hopefully, future development will extend this feature to support EC pools and
+running with more than 2 full sites.
+
+Other commands
+==============
+If your tiebreaker monitor fails for some reason, you can replace it. Turn on
+a new monitor and run:
+
+.. prompt:: bash $
+
+   ceph mon set_new_tiebreaker mon.<new_mon_name>
+
+This command will protest if the new monitor is in the same location as existing
+non-tiebreaker monitors. This command WILL NOT remove the previous tiebreaker
+monitor; you should do so yourself.
+
+Also in 16.2.7, if you are writing your own tooling for deploying Ceph, you can use a new
+``--set-crush-location`` option when booting monitors, instead of running
+``ceph mon set_location``. This option accepts only a single "bucket=loc" pair, eg
+``ceph-mon --set-crush-location 'datacenter=a'``, which must match the
+bucket type you specified when running ``enable_stretch_mode``.
+
+
+When in stretch degraded mode, the cluster will go into "recovery" mode automatically
+when the disconnected data center comes back. If that doesn't work, or you want to
+enable recovery mode early, you can invoke:
+
+.. prompt:: bash $
+
+   ceph osd force_recovery_stretch_mode --yes-i-really-mean-it
+
+But this command should not be necessary; it is included to deal with
+unanticipated situations.
+
+When in recovery mode, the cluster should go back into normal stretch mode
+when the PGs are healthy. If this doesn't happen, or you want to force the
+cross-data-center peering early and are willing to risk data downtime (or have
+verified separately that all the PGs can peer, even if they aren't fully
+recovered), you can invoke:
+
+.. prompt:: bash $
+
+   ceph osd force_healthy_stretch_mode --yes-i-really-mean-it
+
+This command should not be necessary; it is included to deal with
+unanticipated situations. But you might wish to invoke it to remove
+the ``HEALTH_WARN`` state which recovery mode generates.
diff --git a/doc/rados/operations/upmap.rst b/doc/rados/operations/upmap.rst
new file mode 100644
index 000000000..343adf2c4
--- /dev/null
+++ b/doc/rados/operations/upmap.rst
@@ -0,0 +1,105 @@
+.. _upmap:
+
+Using the pg-upmap
+==================
+
+Starting in Luminous v12.2.z there is a new *pg-upmap* exception table
+in the OSDMap that allows the cluster to explicitly map specific PGs to
+specific OSDs.  This allows the cluster to fine-tune the data
+distribution to, in most cases, perfectly distributed PGs across OSDs.
+
+The key caveat to this new mechanism is that it requires that all
+clients understand the new *pg-upmap* structure in the OSDMap.
+
+Enabling
+--------
+
+New clusters will have this module on by default. The cluster must only
+have luminous (and newer) clients. You can the turn the balancer off with:
+
+.. prompt:: bash $
+
+   ceph balancer off
+
+To allow use of the feature on existing clusters, you must tell the
+cluster that it only needs to support luminous (and newer) clients with:
+
+.. prompt:: bash $
+
+   ceph osd set-require-min-compat-client luminous
+
+This command will fail if any pre-luminous clients or daemons are
+connected to the monitors.  You can see what client versions are in
+use with:
+
+.. prompt:: bash $
+
+   ceph features
+
+Balancer module
+-----------------
+
+The `balancer` module for ceph-mgr will automatically balance
+the number of PGs per OSD.  See :ref:`balancer`
+
+
+Offline optimization
+--------------------
+
+Upmap entries are updated with an offline optimizer built into ``osdmaptool``.
+
+#. Grab the latest copy of your osdmap:
+
+   .. prompt:: bash $
+
+      ceph osd getmap -o om
+
+#. Run the optimizer:
+
+   .. prompt:: bash $
+
+      osdmaptool om --upmap out.txt [--upmap-pool <pool>] \ 
+      [--upmap-max <max-optimizations>] \ 
+      [--upmap-deviation <max-deviation>] \ 
+      [--upmap-active]
+
+   It is highly recommended that optimization be done for each pool
+   individually, or for sets of similarly-utilized pools.  You can
+   specify the ``--upmap-pool`` option multiple times.  "Similar pools"
+   means pools that are mapped to the same devices and store the same
+   kind of data (e.g., RBD image pools, yes; RGW index pool and RGW
+   data pool, no).
+
+   The ``max-optimizations`` value is the maximum number of upmap entries to
+   identify in the run.  The default is `10` like the ceph-mgr balancer module,
+   but you should use a larger number if you are doing offline optimization.
+   If it cannot find any additional changes to make it will stop early
+   (i.e., when the pool distribution is perfect).
+
+   The ``max-deviation`` value defaults to `5`.  If an OSD PG count
+   varies from the computed target number by less than or equal
+   to this amount it will be considered perfect.
+
+   The ``--upmap-active`` option simulates the behavior of the active
+   balancer in upmap mode.  It keeps cycling until the OSDs are balanced
+   and reports how many rounds and how long each round is taking.  The
+   elapsed time for rounds indicates the CPU load ceph-mgr will be
+   consuming when it tries to compute the next optimization plan.
+
+#. Apply the changes:
+
+   .. prompt:: bash $
+
+      source out.txt
+
+   The proposed changes are written to the output file ``out.txt`` in
+   the example above.  These are normal ceph CLI commands that can be
+   run to apply the changes to the cluster.
+
+
+The above steps can be repeated as many times as necessary to achieve
+a perfect distribution of PGs for each set of pools.
+
+You can see some (gory) details about what the tool is doing by
+passing ``--debug-osd 10`` and even more with ``--debug-crush 10``
+to ``osdmaptool``.
diff --git a/doc/rados/operations/user-management.rst b/doc/rados/operations/user-management.rst
new file mode 100644
index 000000000..78d77236d
--- /dev/null
+++ b/doc/rados/operations/user-management.rst
@@ -0,0 +1,823 @@
+.. _user-management:
+
+=================
+ User Management
+=================
+
+This document describes :term:`Ceph Client` users, and their authentication and
+authorization with the :term:`Ceph Storage Cluster`. Users are either
+individuals or system actors such as applications, which use Ceph clients to
+interact with the Ceph Storage Cluster daemons.
+
+.. ditaa::
+            +-----+
+            | {o} |
+            |     |
+            +--+--+       /---------\               /---------\
+               |          |  Ceph   |               |  Ceph   |
+            ---+---*----->|         |<------------->|         |
+               |     uses | Clients |               | Servers |
+               |          \---------/               \---------/
+            /--+--\
+            |     |
+            |     |
+             actor
+
+
+When Ceph runs with authentication and authorization enabled (enabled by
+default), you must specify a user name and a keyring containing the secret key
+of the specified user (usually via the command line). If you do not specify a
+user name, Ceph will use ``client.admin`` as the default user name. If you do
+not specify a keyring, Ceph will look for a keyring via the ``keyring`` setting
+in the Ceph configuration. For example, if you execute the ``ceph health``
+command without specifying a user or keyring:
+
+.. prompt:: bash $
+
+   ceph health
+
+Ceph interprets the command like this:
+
+.. prompt:: bash $
+
+   ceph -n client.admin --keyring=/etc/ceph/ceph.client.admin.keyring health
+
+Alternatively, you may use the ``CEPH_ARGS`` environment variable to avoid
+re-entry of the user name and secret.
+
+For details on configuring the Ceph Storage Cluster to use authentication,
+see `Cephx Config Reference`_. For details on the architecture of Cephx, see
+`Architecture - High Availability Authentication`_.
+
+Background
+==========
+
+Irrespective of the type of Ceph client (e.g., Block Device, Object Storage,
+Filesystem, native API, etc.), Ceph stores all data as objects within `pools`_.
+Ceph users must have access to pools in order to read and write data.
+Additionally, Ceph users must have execute permissions to use Ceph's
+administrative commands. The following concepts will help you understand Ceph
+user management.
+
+User
+----
+
+A user is either an individual or a system actor such as an application.
+Creating users allows you to control who (or what) can access your Ceph Storage
+Cluster, its pools, and the data within pools.
+
+Ceph has the notion of a ``type`` of user. For the purposes of user management,
+the type will always be ``client``. Ceph identifies users in period (.)
+delimited form consisting of the user type and the user ID: for example,
+``TYPE.ID``, ``client.admin``, or ``client.user1``. The reason for user typing
+is that Ceph Monitors, OSDs, and Metadata Servers also use the Cephx protocol,
+but they are not clients. Distinguishing the user type helps to distinguish
+between client users and other users--streamlining access control, user
+monitoring and traceability.
+
+Sometimes Ceph's user type may seem confusing, because the Ceph command line
+allows you to specify a user with or without the type, depending upon your
+command line usage. If you specify ``--user`` or ``--id``, you can omit the
+type. So ``client.user1`` can be entered simply as ``user1``. If you specify
+``--name`` or ``-n``, you must specify the type and name, such as
+``client.user1``. We recommend using the type and name as a best practice
+wherever possible.
+
+.. note:: A Ceph Storage Cluster user is not the same as a Ceph Object Storage
+   user or a Ceph File System user. The Ceph Object Gateway uses a Ceph Storage
+   Cluster user to communicate between the gateway daemon and the storage
+   cluster, but the gateway has its own user management functionality for end
+   users. The Ceph File System uses POSIX semantics. The user space associated
+   with the Ceph File System is not the same as a Ceph Storage Cluster user.
+
+
+
+Authorization (Capabilities)
+----------------------------
+
+Ceph uses the term "capabilities" (caps) to describe authorizing an
+authenticated user to exercise the functionality of the monitors, OSDs and
+metadata servers. Capabilities can also restrict access to data within a pool,
+a namespace within a pool, or a set of pools based on their application tags.
+A Ceph administrative user sets a user's capabilities when creating or updating
+a user.
+
+Capability syntax follows the form::
+
+	{daemon-type} '{cap-spec}[, {cap-spec} ...]'
+
+- **Monitor Caps:** Monitor capabilities include ``r``, ``w``, ``x`` access
+  settings or ``profile {name}``. For example::
+
+	mon 'allow {access-spec} [network {network/prefix}]'
+
+	mon 'profile {name}'
+
+  The ``{access-spec}`` syntax is as follows: ::
+
+        * | all | [r][w][x]
+
+  The optional ``{network/prefix}`` is a standard network name and
+  prefix length in CIDR notation (e.g., ``10.3.0.0/16``).  If present,
+  the use of this capability is restricted to clients connecting from
+  this network.
+
+- **OSD Caps:** OSD capabilities include ``r``, ``w``, ``x``, ``class-read``,
+  ``class-write`` access settings or ``profile {name}``. Additionally, OSD
+  capabilities also allow for pool and namespace settings. ::
+
+	osd 'allow {access-spec} [{match-spec}] [network {network/prefix}]'
+
+	osd 'profile {name} [pool={pool-name} [namespace={namespace-name}]] [network {network/prefix}]'
+
+  The ``{access-spec}`` syntax is either of the following: ::
+
+        * | all | [r][w][x] [class-read] [class-write]
+
+        class {class name} [{method name}]
+
+  The optional ``{match-spec}`` syntax is either of the following: ::
+
+        pool={pool-name} [namespace={namespace-name}] [object_prefix {prefix}]
+
+        [namespace={namespace-name}] tag {application} {key}={value}
+
+  The optional ``{network/prefix}`` is a standard network name and
+  prefix length in CIDR notation (e.g., ``10.3.0.0/16``).  If present,
+  the use of this capability is restricted to clients connecting from
+  this network.
+
+- **Manager Caps:** Manager (``ceph-mgr``) capabilities include
+  ``r``, ``w``, ``x`` access settings or ``profile {name}``. For example: ::
+
+	mgr 'allow {access-spec} [network {network/prefix}]'
+
+	mgr 'profile {name} [{key1} {match-type} {value1} ...] [network {network/prefix}]'
+
+  Manager capabilities can also be specified for specific commands,
+  all commands exported by a built-in manager service, or all commands
+  exported by a specific add-on module. For example: ::
+
+        mgr 'allow command "{command-prefix}" [with {key1} {match-type} {value1} ...] [network {network/prefix}]'
+
+        mgr 'allow service {service-name} {access-spec} [network {network/prefix}]'
+
+        mgr 'allow module {module-name} [with {key1} {match-type} {value1} ...] {access-spec} [network {network/prefix}]'
+
+  The ``{access-spec}`` syntax is as follows: ::
+
+        * | all | [r][w][x]
+
+  The ``{service-name}`` is one of the following: ::
+
+        mgr | osd | pg | py
+
+  The ``{match-type}`` is one of the following: ::
+
+        = | prefix | regex
+
+- **Metadata Server Caps:** For administrators, use ``allow *``.  For all
+  other users, such as CephFS clients, consult :doc:`/cephfs/client-auth`
+
+
+.. note:: The Ceph Object Gateway daemon (``radosgw``) is a client of the
+          Ceph Storage Cluster, so it is not represented as a Ceph Storage
+          Cluster daemon type.
+
+The following entries describe each access capability.
+
+``allow``
+
+:Description: Precedes access settings for a daemon. Implies ``rw``
+              for MDS only.
+
+
+``r``
+
+:Description: Gives the user read access. Required with monitors to retrieve
+              the CRUSH map.
+
+
+``w``
+
+:Description: Gives the user write access to objects.
+
+
+``x``
+
+:Description: Gives the user the capability to call class methods
+              (i.e., both read and write) and to conduct ``auth``
+              operations on monitors.
+
+
+``class-read``
+
+:Descriptions: Gives the user the capability to call class read methods.
+               Subset of ``x``.
+
+
+``class-write``
+
+:Description: Gives the user the capability to call class write methods.
+              Subset of ``x``.
+
+
+``*``, ``all``
+
+:Description: Gives the user read, write and execute permissions for a
+              particular daemon/pool, and the ability to execute
+              admin commands.
+
+The following entries describe valid capability profiles:
+
+``profile osd`` (Monitor only)
+
+:Description: Gives a user permissions to connect as an OSD to other OSDs or
+              monitors. Conferred on OSDs to enable OSDs to handle replication
+              heartbeat traffic and status reporting.
+
+
+``profile mds`` (Monitor only)
+
+:Description: Gives a user permissions to connect as a MDS to other MDSs or
+              monitors.
+
+
+``profile bootstrap-osd`` (Monitor only)
+
+:Description: Gives a user permissions to bootstrap an OSD. Conferred on
+              deployment tools such as ``ceph-volume``, ``cephadm``, etc.
+              so that they have permissions to add keys, etc. when
+              bootstrapping an OSD.
+
+
+``profile bootstrap-mds`` (Monitor only)
+
+:Description: Gives a user permissions to bootstrap a metadata server.
+              Conferred on deployment tools such as ``cephadm``, etc.
+              so they have permissions to add keys, etc. when bootstrapping
+              a metadata server.
+
+``profile bootstrap-rbd`` (Monitor only)
+
+:Description: Gives a user permissions to bootstrap an RBD user.
+              Conferred on deployment tools such as ``cephadm``, etc.
+              so they have permissions to add keys, etc. when bootstrapping
+              an RBD user.
+
+``profile bootstrap-rbd-mirror`` (Monitor only)
+
+:Description: Gives a user permissions to bootstrap an ``rbd-mirror`` daemon
+              user. Conferred on deployment tools such as ``cephadm``, etc.
+              so they have permissions to add keys, etc. when bootstrapping
+              an ``rbd-mirror`` daemon.
+
+``profile rbd`` (Manager, Monitor, and OSD)
+
+:Description: Gives a user permissions to manipulate RBD images. When used
+              as a Monitor cap, it provides the minimal privileges required
+              by an RBD client application; this includes the ability
+	      to blocklist other client users. When used as an OSD cap, it
+              provides read-write access to the specified pool to an
+	      RBD client application. The Manager cap supports optional
+              ``pool`` and ``namespace`` keyword arguments.
+
+``profile rbd-mirror`` (Monitor only)
+
+:Description: Gives a user permissions to manipulate RBD images and retrieve
+              RBD mirroring config-key secrets. It provides the minimal
+              privileges required for the ``rbd-mirror`` daemon.
+
+``profile rbd-read-only`` (Manager and OSD)
+
+:Description: Gives a user read-only permissions to RBD images. The Manager
+              cap supports optional ``pool`` and ``namespace`` keyword
+              arguments.
+
+``profile simple-rados-client`` (Monitor only)
+
+:Description: Gives a user read-only permissions for monitor, OSD, and PG data.
+              Intended for use by direct librados client applications.
+
+``profile simple-rados-client-with-blocklist`` (Monitor only)
+
+:Description: Gives a user read-only permissions for monitor, OSD, and PG data.
+              Intended for use by direct librados client applications. Also
+              includes permission to add blocklist entries to build HA
+              applications.
+
+``profile fs-client`` (Monitor only)
+
+:Description: Gives a user read-only permissions for monitor, OSD, PG, and MDS
+              data.  Intended for CephFS clients.
+
+``profile role-definer`` (Monitor and Auth)
+
+:Description: Gives a user **all** permissions for the auth subsystem, read-only
+              access to monitors, and nothing else.  Useful for automation
+              tools.  Do not assign this unless you really, **really** know what
+              you're doing as the security ramifications are substantial and
+              pervasive.
+
+``profile crash`` (Monitor only)
+
+:Description: Gives a user read-only access to monitors, used in conjunction
+              with the manager ``crash`` module when collecting daemon crash
+              dumps for later analysis.
+
+Pool
+----
+
+A pool is a logical partition where users store data.
+In Ceph deployments, it is common to create a pool as a logical partition for
+similar types of data. For example, when deploying Ceph as a backend for
+OpenStack, a typical deployment would have pools for volumes, images, backups
+and virtual machines, and users such as ``client.glance``, ``client.cinder``,
+etc.
+
+Application Tags
+----------------
+
+Access may be restricted to specific pools as defined by their application
+metadata. The ``*`` wildcard may be used for the ``key`` argument, the
+``value`` argument, or both. ``all`` is a synony for ``*``.
+
+Namespace
+---------
+
+Objects within a pool can be associated to a namespace--a logical group of
+objects within the pool. A user's access to a pool can be associated with a
+namespace such that reads and writes by the user take place only within the
+namespace. Objects written to a namespace within the pool can only be accessed
+by users who have access to the namespace.
+
+.. note:: Namespaces are primarily useful for applications written on top of
+   ``librados`` where the logical grouping can alleviate the need to create
+   different pools. Ceph Object Gateway (from ``luminous``) uses namespaces for various
+   metadata objects.
+
+The rationale for namespaces is that pools can be a computationally expensive
+method of segregating data sets for the purposes of authorizing separate sets
+of users. For example, a pool should have ~100 placement groups per OSD. So an
+exemplary cluster with 1000 OSDs would have 100,000 placement groups for one
+pool. Each pool would create another 100,000 placement groups in the exemplary
+cluster. By contrast, writing an object to a namespace simply associates the
+namespace to the object name with out the computational overhead of a separate
+pool. Rather than creating a separate pool for a user or set of users, you may
+use a namespace. **Note:** Only available using ``librados`` at this time.
+
+Access may be restricted to specific RADOS namespaces using the ``namespace``
+capability. Limited globbing of namespaces is supported; if the last character
+of the specified namespace is ``*``, then access is granted to any namespace
+starting with the provided argument.
+
+Managing Users
+==============
+
+User management functionality provides Ceph Storage Cluster administrators with
+the ability to create, update and delete users directly in the Ceph Storage
+Cluster.
+
+When you create or delete users in the Ceph Storage Cluster, you may need to
+distribute keys to clients so that they can be added to keyrings. See `Keyring
+Management`_ for details.
+
+List Users
+----------
+
+To list the users in your cluster, execute the following:
+
+.. prompt:: bash $
+
+	ceph auth ls
+
+Ceph will list out all users in your cluster. For example, in a two-node
+exemplary cluster, ``ceph auth ls`` will output something that looks like
+this::
+
+	installed auth entries:
+
+	osd.0
+		key: AQCvCbtToC6MDhAATtuT70Sl+DymPCfDSsyV4w==
+		caps: [mon] allow profile osd
+		caps: [osd] allow *
+	osd.1
+		key: AQC4CbtTCFJBChAAVq5spj0ff4eHZICxIOVZeA==
+		caps: [mon] allow profile osd
+		caps: [osd] allow *
+	client.admin
+		key: AQBHCbtT6APDHhAA5W00cBchwkQjh3dkKsyPjw==
+		caps: [mds] allow
+		caps: [mon] allow *
+		caps: [osd] allow *
+	client.bootstrap-mds
+		key: AQBICbtTOK9uGBAAdbe5zcIGHZL3T/u2g6EBww==
+		caps: [mon] allow profile bootstrap-mds
+	client.bootstrap-osd
+		key: AQBHCbtT4GxqORAADE5u7RkpCN/oo4e5W0uBtw==
+		caps: [mon] allow profile bootstrap-osd
+
+
+Note that the ``TYPE.ID`` notation for users applies such that ``osd.0`` is a
+user of type ``osd`` and its ID is ``0``, ``client.admin`` is a user of type
+``client`` and its ID is ``admin`` (i.e., the default ``client.admin`` user).
+Note also that each entry has a ``key: <value>`` entry, and one or more
+``caps:`` entries.
+
+You may use the ``-o {filename}`` option with ``ceph auth ls`` to
+save the output to a file.
+
+
+Get a User
+----------
+
+To retrieve a specific user, key and capabilities, execute the
+following:
+
+.. prompt:: bash $
+
+   ceph auth get {TYPE.ID}
+
+For example:
+
+.. prompt:: bash $
+
+   ceph auth get client.admin
+
+You may also use the ``-o {filename}`` option with ``ceph auth get`` to
+save the output to a file. Developers may also execute the following:
+
+.. prompt:: bash $
+
+   ceph auth export {TYPE.ID}
+
+The ``auth export`` command is identical to ``auth get``.
+
+Add a User
+----------
+
+Adding a user creates a username (i.e., ``TYPE.ID``), a secret key and
+any capabilities included in the command you use to create the user.
+
+A user's key enables the user to authenticate with the Ceph Storage Cluster.
+The user's capabilities authorize the user to read, write, or execute on Ceph
+monitors (``mon``), Ceph OSDs (``osd``) or Ceph Metadata  Servers (``mds``).
+
+There are a few ways to add a user:
+
+- ``ceph auth add``: This command is the canonical way to add a user. It
+  will create the user, generate a key and add any specified capabilities.
+
+- ``ceph auth get-or-create``: This command is often the most convenient way
+  to create a user, because it returns a keyfile format with the user name
+  (in brackets) and the key. If the user already exists, this command
+  simply returns the user name and key in the keyfile format. You may use the
+  ``-o {filename}`` option to save the output to a file.
+
+- ``ceph auth get-or-create-key``: This command is a convenient way to create
+  a user and return the user's key (only). This is useful for clients that
+  need the key only (e.g., libvirt). If the user already exists, this command
+  simply returns the key. You may use the ``-o {filename}`` option to save the
+  output to a file.
+
+When creating client users, you may create a user with no capabilities. A user
+with no capabilities is useless beyond mere authentication, because the client
+cannot retrieve the cluster map from the monitor. However, you can create a
+user with no capabilities if you wish to defer adding capabilities later using
+the ``ceph auth caps`` command.
+
+A typical user has at least read capabilities on the Ceph monitor and
+read and write capability on Ceph OSDs. Additionally, a user's OSD permissions
+are often restricted to accessing a particular pool:
+
+.. prompt:: bash $
+
+   ceph auth add client.john mon 'allow r' osd 'allow rw pool=liverpool'
+   ceph auth get-or-create client.paul mon 'allow r' osd 'allow rw pool=liverpool'
+   ceph auth get-or-create client.george mon 'allow r' osd 'allow rw pool=liverpool' -o george.keyring
+   ceph auth get-or-create-key client.ringo mon 'allow r' osd 'allow rw pool=liverpool' -o ringo.key
+
+
+.. important:: If you provide a user with capabilities to OSDs, but you DO NOT
+   restrict access to particular pools, the user will have access to ALL
+   pools in the cluster!
+
+
+.. _modify-user-capabilities:
+
+Modify User Capabilities
+------------------------
+
+The ``ceph auth caps`` command allows you to specify a user and change the
+user's capabilities. Setting new capabilities will overwrite current capabilities.
+To view current capabilities run ``ceph auth get USERTYPE.USERID``.  To add
+capabilities, you should also specify the existing capabilities when using the form:
+
+.. prompt:: bash $
+
+   ceph auth caps USERTYPE.USERID {daemon} 'allow [r|w|x|*|...] [pool={pool-name}] [namespace={namespace-name}]' [{daemon} 'allow [r|w|x|*|...] [pool={pool-name}] [namespace={namespace-name}]']
+
+For example:
+
+.. prompt:: bash $
+
+   ceph auth get client.john
+   ceph auth caps client.john mon 'allow r' osd 'allow rw pool=liverpool'
+   ceph auth caps client.paul mon 'allow rw' osd 'allow rwx pool=liverpool'
+   ceph auth caps client.brian-manager mon 'allow *' osd 'allow *'
+
+See `Authorization (Capabilities)`_ for additional details on capabilities.
+
+Delete a User
+-------------
+
+To delete a user, use ``ceph auth del``:
+
+.. prompt:: bash $
+
+   ceph auth del {TYPE}.{ID}
+
+Where ``{TYPE}`` is one of ``client``, ``osd``, ``mon``, or ``mds``,
+and ``{ID}`` is the user name or ID of the daemon.
+
+
+Print a User's Key
+------------------
+
+To print a user's authentication key to standard output, execute the following:
+
+.. prompt:: bash $
+
+   ceph auth print-key {TYPE}.{ID}
+
+Where ``{TYPE}`` is one of ``client``, ``osd``, ``mon``, or ``mds``,
+and ``{ID}`` is the user name or ID of the daemon.
+
+Printing a user's key is useful when you need to populate client
+software with a user's key  (e.g., libvirt):
+
+.. prompt:: bash $
+
+   mount -t ceph serverhost:/ mountpoint -o name=client.user,secret=`ceph auth print-key client.user`
+
+Import a User(s)
+----------------
+
+To import one or more users, use ``ceph auth import`` and
+specify a keyring:
+
+.. prompt:: bash $
+
+   ceph auth import -i /path/to/keyring
+
+For example:
+
+.. prompt:: bash $
+
+   sudo ceph auth import -i /etc/ceph/ceph.keyring
+
+
+.. note:: The Ceph storage cluster will add new users, their keys and their
+   capabilities and will update existing users, their keys and their
+   capabilities.
+
+Keyring Management
+==================
+
+When you access Ceph via a Ceph client, the Ceph client will look for a local
+keyring. Ceph presets the ``keyring`` setting with the following four keyring
+names by default so you don't have to set them in your Ceph configuration file
+unless you want to override the defaults (not recommended):
+
+- ``/etc/ceph/$cluster.$name.keyring``
+- ``/etc/ceph/$cluster.keyring``
+- ``/etc/ceph/keyring``
+- ``/etc/ceph/keyring.bin``
+
+The ``$cluster`` metavariable is your Ceph cluster name as defined by the
+name of the Ceph configuration file (i.e., ``ceph.conf`` means the cluster name
+is ``ceph``; thus, ``ceph.keyring``). The ``$name`` metavariable is the user
+type and user ID (e.g., ``client.admin``; thus, ``ceph.client.admin.keyring``).
+
+.. note:: When executing commands that read or write to ``/etc/ceph``, you may
+   need to use ``sudo`` to execute the command as ``root``.
+
+After you create a user (e.g., ``client.ringo``), you must get the key and add
+it to a keyring on a Ceph client so that the user can access the Ceph Storage
+Cluster.
+
+The `User Management`_ section details how to list, get, add, modify and delete
+users directly in the Ceph Storage Cluster. However, Ceph also provides the
+``ceph-authtool`` utility to allow you to manage keyrings from a Ceph client.
+
+Create a Keyring
+----------------
+
+When you use the procedures in the `Managing Users`_ section to create users,
+you need to provide user keys to the Ceph client(s) so that the Ceph client
+can retrieve the key for the specified user and authenticate with the Ceph
+Storage Cluster. Ceph Clients access keyrings to lookup a user name and
+retrieve the user's key.
+
+The ``ceph-authtool`` utility allows you to create a keyring. To create an
+empty keyring, use ``--create-keyring`` or ``-C``. For example:
+
+.. prompt:: bash $
+
+   ceph-authtool --create-keyring /path/to/keyring
+
+When creating a keyring with multiple users, we recommend using the cluster name
+(e.g., ``$cluster.keyring``) for the keyring filename and saving it in the
+``/etc/ceph`` directory so that the ``keyring`` configuration default setting
+will pick up the filename without requiring you to specify it in the local copy
+of your Ceph configuration file. For example, create ``ceph.keyring`` by
+executing the following:
+
+.. prompt:: bash $
+
+   sudo ceph-authtool -C /etc/ceph/ceph.keyring
+
+When creating a keyring with a single user, we recommend using the cluster name,
+the user type and the user name and saving it in the ``/etc/ceph`` directory.
+For example, ``ceph.client.admin.keyring`` for the ``client.admin`` user.
+
+To create a keyring in ``/etc/ceph``, you must do so as ``root``. This means
+the file will have ``rw`` permissions for the ``root`` user only, which is
+appropriate when the keyring contains administrator keys. However, if you
+intend to use the keyring for a particular user or group of users, ensure
+that you execute ``chown`` or ``chmod`` to establish appropriate keyring
+ownership and access.
+
+Add a User to a Keyring
+-----------------------
+
+When you  `Add a User`_ to the Ceph Storage Cluster, you can use the `Get a
+User`_ procedure to retrieve a user, key and capabilities and save the user to a
+keyring.
+
+When you only want to use one user per keyring, the `Get a User`_ procedure with
+the ``-o`` option will save the output in the keyring file format. For example,
+to create a keyring for the ``client.admin`` user, execute the following:
+
+.. prompt:: bash $
+
+   sudo ceph auth get client.admin -o /etc/ceph/ceph.client.admin.keyring
+
+Notice that we use the recommended file format for an individual user.
+
+When you want to import users to a keyring, you can use ``ceph-authtool``
+to specify the destination keyring and the source keyring.
+For example:
+
+.. prompt:: bash $
+
+   sudo ceph-authtool /etc/ceph/ceph.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring
+
+Create a User
+-------------
+
+Ceph provides the `Add a User`_ function to create a user directly in the Ceph
+Storage Cluster. However, you can also create a user, keys and capabilities
+directly on a Ceph client keyring. Then, you can import the user to the Ceph
+Storage Cluster. For example:
+
+.. prompt:: bash $
+
+   sudo ceph-authtool -n client.ringo --cap osd 'allow rwx' --cap mon 'allow rwx' /etc/ceph/ceph.keyring
+
+See `Authorization (Capabilities)`_ for additional details on capabilities.
+
+You can also create a keyring and add a new user to the keyring simultaneously.
+For example:
+
+.. prompt:: bash $
+
+   sudo ceph-authtool -C /etc/ceph/ceph.keyring -n client.ringo --cap osd 'allow rwx' --cap mon 'allow rwx' --gen-key
+
+In the foregoing scenarios, the new user ``client.ringo`` is only in the
+keyring. To add the new user to the Ceph Storage Cluster, you must still add
+the new user to the Ceph Storage Cluster:
+
+.. prompt:: bash $
+
+   sudo ceph auth add client.ringo -i /etc/ceph/ceph.keyring
+
+Modify a User
+-------------
+
+To modify the capabilities of a user record in a keyring, specify the keyring,
+and the user followed by the capabilities. For example:
+
+.. prompt:: bash $
+
+   sudo ceph-authtool /etc/ceph/ceph.keyring -n client.ringo --cap osd 'allow rwx' --cap mon 'allow rwx'
+
+To update the user to the Ceph Storage Cluster, you must update the user
+in the keyring to the user entry in the Ceph Storage Cluster:
+
+.. prompt:: bash $
+
+   sudo ceph auth import -i /etc/ceph/ceph.keyring
+
+See `Import a User(s)`_ for details on updating a Ceph Storage Cluster user
+from a keyring.
+
+You may also `Modify User Capabilities`_ directly in the cluster, store the
+results to a keyring file; then, import the keyring into your main
+``ceph.keyring`` file.
+
+Command Line Usage
+==================
+
+Ceph supports the following usage for user name and secret:
+
+``--id`` | ``--user``
+
+:Description: Ceph identifies users with a type and an ID (e.g., ``TYPE.ID`` or
+              ``client.admin``, ``client.user1``). The ``id``, ``name`` and
+              ``-n`` options enable you to specify the ID portion of the user
+              name (e.g., ``admin``, ``user1``, ``foo``, etc.). You can specify
+              the user with the ``--id`` and omit the type. For example,
+              to specify user ``client.foo`` enter the following:
+
+              .. prompt:: bash $
+
+                 ceph --id foo --keyring /path/to/keyring health
+                 ceph --user foo --keyring /path/to/keyring health
+
+
+``--name`` | ``-n``
+
+:Description: Ceph identifies users with a type and an ID (e.g., ``TYPE.ID`` or
+              ``client.admin``, ``client.user1``). The ``--name`` and ``-n``
+              options enables you to specify the fully qualified user name.
+              You must specify the user type (typically ``client``) with the
+              user ID. For example:
+
+              .. prompt:: bash $
+
+                 ceph --name client.foo --keyring /path/to/keyring health
+                 ceph -n client.foo --keyring /path/to/keyring health
+
+
+``--keyring``
+
+:Description: The path to the keyring containing one or more user name and
+              secret. The ``--secret`` option provides the same functionality,
+              but it does not work with Ceph RADOS Gateway, which uses
+              ``--secret`` for another purpose. You may retrieve a keyring with
+              ``ceph auth get-or-create`` and store it locally. This is a
+              preferred approach, because you can switch user names without
+              switching the keyring path. For example:
+
+              .. prompt:: bash $
+
+                 sudo rbd map --id foo --keyring /path/to/keyring mypool/myimage
+
+
+.. _pools: ../pools
+
+Limitations
+===========
+
+The ``cephx`` protocol authenticates Ceph clients and servers to each other.  It
+is not intended to handle authentication of human users or application programs
+run on their behalf.  If that effect is required to handle your access control
+needs, you must have another mechanism, which is likely to be specific to the
+front end used to access the Ceph object store.  This other mechanism has the
+role of ensuring that only acceptable users and programs are able to run on the
+machine that Ceph will permit to access its object store.
+
+The keys used to authenticate Ceph clients and servers are typically stored in
+a plain text file with appropriate permissions in a trusted host.
+
+.. important:: Storing keys in plaintext files has security shortcomings, but
+   they are difficult to avoid, given the basic authentication methods Ceph
+   uses in the background. Those setting up Ceph systems should be aware of
+   these shortcomings.
+
+In particular, arbitrary user machines, especially portable machines, should not
+be configured to interact directly with Ceph, since that mode of use would
+require the storage of a plaintext authentication key on an insecure machine.
+Anyone  who stole that machine or obtained surreptitious access to it could
+obtain the key that will allow them to authenticate their own machines to Ceph.
+
+Rather than permitting potentially insecure machines to access a Ceph object
+store directly,  users should be required to sign in to a trusted machine in
+your environment using a method  that provides sufficient security for your
+purposes.  That trusted machine will store the plaintext Ceph keys for the
+human users.  A future version of Ceph may address these particular
+authentication issues more fully.
+
+At the moment, none of the Ceph authentication protocols provide secrecy for
+messages in transit. Thus, an eavesdropper on the wire can hear and understand
+all data sent between clients and servers in Ceph, even if it cannot create or
+alter them. Further, Ceph does not include options to encrypt user data in the
+object store. Users can hand-encrypt and store their own data in the Ceph
+object store, of course, but Ceph provides no features to perform object
+encryption itself. Those storing sensitive data in Ceph should consider
+encrypting their data before providing it  to the Ceph system.
+
+
+.. _Architecture - High Availability Authentication: ../../../architecture#high-availability-authentication
+.. _Cephx Config Reference: ../../configuration/auth-config-ref
diff --git a/doc/rados/troubleshooting/community.rst b/doc/rados/troubleshooting/community.rst
new file mode 100644
index 000000000..f816584ae
--- /dev/null
+++ b/doc/rados/troubleshooting/community.rst
@@ -0,0 +1,28 @@
+====================
+ The Ceph Community
+====================
+
+The Ceph community is an excellent source of information and help. For
+operational issues with Ceph releases we recommend you `subscribe to the
+ceph-users email list`_. When you no longer want to receive emails, you can
+`unsubscribe from the ceph-users email list`_.
+
+You may also `subscribe to the ceph-devel email list`_. You should do so if
+your issue is:
+
+- Likely related to a bug
+- Related to a development release package
+- Related to a development testing package
+- Related to your own builds
+
+If you no longer want to receive emails from the ``ceph-devel`` email list, you
+may `unsubscribe from the ceph-devel email list`_.
+
+.. tip:: The Ceph community is growing rapidly, and community members can help
+   you if you provide them with detailed information about your problem. You
+   can attach the output of the ``ceph report`` command to help people understand your issues.
+
+.. _subscribe to the ceph-devel email list: mailto:dev-join@ceph.io
+.. _unsubscribe from the ceph-devel email list: mailto:dev-leave@ceph.io
+.. _subscribe to the ceph-users email list: mailto:ceph-users-join@ceph.io
+.. _unsubscribe from the ceph-users email list: mailto:ceph-users-leave@ceph.io
diff --git a/doc/rados/troubleshooting/cpu-profiling.rst b/doc/rados/troubleshooting/cpu-profiling.rst
new file mode 100644
index 000000000..159f7998d
--- /dev/null
+++ b/doc/rados/troubleshooting/cpu-profiling.rst
@@ -0,0 +1,67 @@
+===============
+ CPU Profiling
+===============
+
+If you built Ceph from source and compiled Ceph for use with `oprofile`_
+you can profile Ceph's CPU usage. See `Installing Oprofile`_ for details.
+
+
+Initializing oprofile
+=====================
+
+The first time you use ``oprofile`` you need to initialize it. Locate the
+``vmlinux`` image corresponding to the kernel you are now running. :: 
+
+	ls /boot
+	sudo opcontrol --init
+	sudo opcontrol --setup --vmlinux={path-to-image} --separate=library --callgraph=6
+
+
+Starting oprofile
+=================
+
+To start ``oprofile`` execute the following command:: 
+
+	opcontrol --start
+
+Once you start ``oprofile``, you may run some tests with Ceph. 
+
+
+Stopping oprofile
+=================
+
+To stop ``oprofile`` execute the following command:: 
+
+	opcontrol --stop
+	
+	
+Retrieving oprofile Results
+===========================
+
+To retrieve the top ``cmon`` results, execute the following command:: 
+
+	opreport -gal ./cmon | less	
+	
+
+To retrieve the top ``cmon`` results with call graphs attached, execute the
+following command:: 
+
+	opreport -cal ./cmon | less	
+	
+.. important:: After reviewing results, you should reset ``oprofile`` before
+   running it again. Resetting ``oprofile`` removes data from the session 
+   directory.
+
+
+Resetting oprofile
+==================
+
+To reset ``oprofile``, execute the following command:: 
+
+	sudo opcontrol --reset   
+   
+.. important:: You should reset ``oprofile`` after analyzing data so that 
+   you do not commingle results from different tests.
+
+.. _oprofile: http://oprofile.sourceforge.net/about/
+.. _Installing Oprofile: ../../../dev/cpu-profiler
diff --git a/doc/rados/troubleshooting/index.rst b/doc/rados/troubleshooting/index.rst
new file mode 100644
index 000000000..80d14f3ce
--- /dev/null
+++ b/doc/rados/troubleshooting/index.rst
@@ -0,0 +1,19 @@
+=================
+ Troubleshooting
+=================
+
+Ceph is still on the leading edge, so you may encounter situations that require 
+you to examine your configuration, modify your logging output, troubleshoot
+monitors and OSDs, profile memory and CPU usage, and reach out to the 
+Ceph community for help.
+
+.. toctree::
+   :maxdepth: 1
+   
+   community
+   log-and-debug
+   troubleshooting-mon
+   troubleshooting-osd
+   troubleshooting-pg
+   memory-profiling
+   cpu-profiling
diff --git a/doc/rados/troubleshooting/log-and-debug.rst b/doc/rados/troubleshooting/log-and-debug.rst
new file mode 100644
index 000000000..71170149b
--- /dev/null
+++ b/doc/rados/troubleshooting/log-and-debug.rst
@@ -0,0 +1,599 @@
+=======================
+ Logging and Debugging
+=======================
+
+Typically, when you add debugging to your Ceph configuration, you do so at
+runtime. You can also add Ceph debug logging to your Ceph configuration file if
+you are encountering issues when starting your cluster. You may view Ceph log
+files under ``/var/log/ceph`` (the default location).
+
+.. tip:: When debug output slows down your system, the latency can hide 
+   race conditions.
+
+Logging is resource intensive. If you are encountering a problem in a specific
+area of your cluster, enable logging for that area of the cluster. For example,
+if your OSDs are running fine, but your metadata servers are not, you should
+start by enabling debug logging for the specific metadata server instance(s)
+giving you trouble. Enable logging for each subsystem as needed.
+
+.. important:: Verbose logging can generate over 1GB of data per hour. If your 
+   OS disk reaches its capacity, the node will stop working.
+   
+If you enable or increase the rate of Ceph logging, ensure that you have
+sufficient disk space on your OS disk.  See `Accelerating Log Rotation`_ for
+details on rotating log files. When your system is running well, remove
+unnecessary debugging settings to ensure your cluster runs optimally. Logging
+debug output messages is relatively slow, and a waste of resources when
+operating your cluster.
+
+See `Subsystem, Log and Debug Settings`_ for details on available settings.
+
+Runtime
+=======
+
+If you would like to see the configuration settings at runtime, you must log
+in to a host with a running daemon and execute the following:: 
+
+	ceph daemon {daemon-name} config show | less
+
+For example,::
+
+  ceph daemon osd.0 config show | less
+
+To activate Ceph's debugging output (*i.e.*, ``dout()``) at runtime,  use the
+``ceph tell`` command to inject arguments into the runtime configuration:: 
+
+	ceph tell {daemon-type}.{daemon id or *} config set {name} {value}
+	
+Replace ``{daemon-type}`` with one of ``osd``, ``mon`` or ``mds``. You may apply
+the runtime setting to all daemons of a particular type with ``*``, or specify
+a specific daemon's ID. For example, to increase
+debug logging for a ``ceph-osd`` daemon named ``osd.0``, execute the following:: 
+
+	ceph tell osd.0 config set debug_osd 0/5
+
+The ``ceph tell`` command goes through the monitors. If you cannot bind to the
+monitor, you can still make the change by logging into the host of the daemon
+whose configuration you'd like to change using ``ceph daemon``.
+For example:: 
+
+	sudo ceph daemon osd.0 config set debug_osd 0/5
+
+See `Subsystem, Log and Debug Settings`_ for details on available settings.
+
+
+Boot Time
+=========
+
+To activate Ceph's debugging output (*i.e.*, ``dout()``) at boot time, you must
+add settings to your Ceph configuration file. Subsystems common to each daemon
+may be set under ``[global]`` in your configuration file. Subsystems for
+particular daemons are set under the daemon section in your configuration file
+(*e.g.*, ``[mon]``, ``[osd]``, ``[mds]``). For example::
+
+	[global]
+		debug ms = 1/5
+		
+	[mon]
+		debug mon = 20
+		debug paxos = 1/5
+		debug auth = 2
+		 
+ 	[osd]
+ 		debug osd = 1/5
+ 		debug filestore = 1/5
+ 		debug journal = 1
+ 		debug monc = 5/20
+ 		
+	[mds]
+		debug mds = 1
+		debug mds balancer = 1
+
+
+See `Subsystem, Log and Debug Settings`_ for details.
+
+
+Accelerating Log Rotation
+=========================
+
+If your OS disk is relatively full, you can accelerate log rotation by modifying
+the Ceph log rotation file at ``/etc/logrotate.d/ceph``. Add  a size setting
+after the rotation frequency to accelerate log rotation (via cronjob) if your
+logs exceed the size setting. For example, the  default setting looks like
+this::
+   
+	rotate 7
+  	weekly
+  	compress
+  	sharedscripts
+   	
+Modify it by adding a ``size`` setting. ::
+   
+  	rotate 7
+  	weekly
+  	size 500M
+  	compress
+  	sharedscripts
+
+Then, start the crontab editor for your user space. ::
+   
+  	crontab -e
+	
+Finally, add an entry to check the ``etc/logrotate.d/ceph`` file. ::
+   
+  	30 * * * * /usr/sbin/logrotate /etc/logrotate.d/ceph >/dev/null 2>&1
+
+The preceding example checks the ``etc/logrotate.d/ceph`` file every 30 minutes.
+
+
+Valgrind
+========
+
+Debugging may also require you to track down memory and threading issues. 
+You can run a single daemon, a type of daemon, or the whole cluster with 
+Valgrind. You should only use Valgrind when developing or debugging Ceph. 
+Valgrind is computationally expensive, and will slow down your system otherwise. 
+Valgrind messages are logged to ``stderr``. 
+
+
+Subsystem, Log and Debug Settings
+=================================
+
+In most cases, you will enable debug logging output via subsystems. 
+
+Ceph Subsystems
+---------------
+
+Each subsystem has a logging level for its output logs, and for its logs
+in-memory. You may set different values for each of these subsystems by setting
+a log file level and a memory level for debug logging. Ceph's logging levels
+operate on a scale of ``1`` to ``20``, where ``1`` is terse and ``20`` is
+verbose [#]_ . In general, the logs in-memory are not sent to the output log unless:
+
+- a fatal signal is raised or
+- an ``assert`` in source code is triggered or
+- upon requested. Please consult `document on admin socket <http://docs.ceph.com/en/latest/man/8/ceph/#daemon>`_ for more details.
+
+A debug logging setting can take a single value for the log level and the
+memory level, which sets them both as the same value. For example, if you
+specify ``debug ms = 5``, Ceph will treat it as a log level and a memory level
+of ``5``. You may also specify them separately. The first setting is the log
+level, and the second setting is the memory level.  You must separate them with
+a forward slash (/). For example, if you want to set the ``ms`` subsystem's
+debug logging level to ``1`` and its memory level to ``5``, you would specify it
+as ``debug ms = 1/5``. For example:
+
+
+
+.. code-block:: ini 
+
+	debug {subsystem} = {log-level}/{memory-level}
+	#for example
+	debug mds balancer = 1/20
+
+
+The following table provides a list of Ceph subsystems and their default log and
+memory levels. Once you complete your logging efforts, restore the subsystems
+to their default level or to a level suitable for normal operations.
+
+
++--------------------+-----------+--------------+
+| Subsystem          | Log Level | Memory Level |
++====================+===========+==============+
+| ``default``        |     0     |      5       |
++--------------------+-----------+--------------+
+| ``lockdep``        |     0     |      1       |
++--------------------+-----------+--------------+
+| ``context``        |     0     |      1       |
++--------------------+-----------+--------------+
+| ``crush``          |     1     |      1       |
++--------------------+-----------+--------------+
+| ``mds``            |     1     |      5       |
++--------------------+-----------+--------------+
+| ``mds balancer``   |     1     |      5       |
++--------------------+-----------+--------------+
+| ``mds locker``     |     1     |      5       |
++--------------------+-----------+--------------+
+| ``mds log``        |     1     |      5       |
++--------------------+-----------+--------------+
+| ``mds log expire`` |     1     |      5       |
++--------------------+-----------+--------------+
+| ``mds migrator``   |     1     |      5       |
++--------------------+-----------+--------------+
+| ``buffer``         |     0     |      1       |
++--------------------+-----------+--------------+
+| ``timer``          |     0     |      1       |
++--------------------+-----------+--------------+
+| ``filer``          |     0     |      1       |
++--------------------+-----------+--------------+
+| ``striper``        |     0     |      1       |
++--------------------+-----------+--------------+
+| ``objecter``       |     0     |      1       |
++--------------------+-----------+--------------+
+| ``rados``          |     0     |      5       |
++--------------------+-----------+--------------+
+| ``rbd``            |     0     |      5       |
++--------------------+-----------+--------------+
+| ``rbd mirror``     |     0     |      5       |
++--------------------+-----------+--------------+
+| ``rbd replay``     |     0     |      5       |
++--------------------+-----------+--------------+
+| ``journaler``      |     0     |      5       |
++--------------------+-----------+--------------+
+| ``objectcacher``   |     0     |      5       |
++--------------------+-----------+--------------+
+| ``client``         |     0     |      5       |
++--------------------+-----------+--------------+
+| ``osd``            |     1     |      5       |
++--------------------+-----------+--------------+
+| ``optracker``      |     0     |      5       |
++--------------------+-----------+--------------+
+| ``objclass``       |     0     |      5       |
++--------------------+-----------+--------------+
+| ``filestore``      |     1     |      3       |
++--------------------+-----------+--------------+
+| ``journal``        |     1     |      3       |
++--------------------+-----------+--------------+
+| ``ms``             |     0     |      5       |
++--------------------+-----------+--------------+
+| ``mon``            |     1     |      5       |
++--------------------+-----------+--------------+
+| ``monc``           |     0     |      10      |
++--------------------+-----------+--------------+
+| ``paxos``          |     1     |      5       |
++--------------------+-----------+--------------+
+| ``tp``             |     0     |      5       |
++--------------------+-----------+--------------+
+| ``auth``           |     1     |      5       |
++--------------------+-----------+--------------+
+| ``crypto``         |     1     |      5       |
++--------------------+-----------+--------------+
+| ``finisher``       |     1     |      1       |
++--------------------+-----------+--------------+
+| ``reserver``       |     1     |      1       |
++--------------------+-----------+--------------+
+| ``heartbeatmap``   |     1     |      5       |
++--------------------+-----------+--------------+
+| ``perfcounter``    |     1     |      5       |
++--------------------+-----------+--------------+
+| ``rgw``            |     1     |      5       |
++--------------------+-----------+--------------+
+| ``rgw sync``       |     1     |      5       |
++--------------------+-----------+--------------+
+| ``civetweb``       |     1     |      10      |
++--------------------+-----------+--------------+
+| ``javaclient``     |     1     |      5       |
++--------------------+-----------+--------------+
+| ``asok``           |     1     |      5       |
++--------------------+-----------+--------------+
+| ``throttle``       |     1     |      1       |
++--------------------+-----------+--------------+
+| ``refs``           |     0     |      0       |
++--------------------+-----------+--------------+
+| ``compressor``     |     1     |      5       |
++--------------------+-----------+--------------+
+| ``bluestore``      |     1     |      5       |
++--------------------+-----------+--------------+
+| ``bluefs``         |     1     |      5       |
++--------------------+-----------+--------------+
+| ``bdev``           |     1     |      3       |
++--------------------+-----------+--------------+
+| ``kstore``         |     1     |      5       |
++--------------------+-----------+--------------+
+| ``rocksdb``        |     4     |      5       |
++--------------------+-----------+--------------+
+| ``leveldb``        |     4     |      5       |
++--------------------+-----------+--------------+
+| ``memdb``          |     4     |      5       |
++--------------------+-----------+--------------+
+| ``fuse``           |     1     |      5       |
++--------------------+-----------+--------------+
+| ``mgr``            |     1     |      5       |
++--------------------+-----------+--------------+
+| ``mgrc``           |     1     |      5       |
++--------------------+-----------+--------------+
+| ``dpdk``           |     1     |      5       |
++--------------------+-----------+--------------+
+| ``eventtrace``     |     1     |      5       |
++--------------------+-----------+--------------+
+
+
+Logging Settings
+----------------
+
+Logging and debugging settings are not required in a Ceph configuration file,
+but you may override default settings as needed. Ceph supports the following
+settings:
+
+
+``log file``
+
+:Description: The location of the logging file for your cluster.
+:Type: String
+:Required: No
+:Default: ``/var/log/ceph/$cluster-$name.log``
+
+
+``log max new``
+
+:Description: The maximum number of new log files.
+:Type: Integer
+:Required: No
+:Default: ``1000``
+
+
+``log max recent``
+
+:Description: The maximum number of recent events to include in a log file.
+:Type: Integer
+:Required:  No
+:Default: ``10000``
+
+
+``log to file``
+
+:Description: Determines if logging messages should appear in a file.
+:Type: Boolean
+:Required: No
+:Default: ``true``
+
+
+``log to stderr``
+
+:Description: Determines if logging messages should appear in ``stderr``.
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``err to stderr``
+
+:Description: Determines if error messages should appear in ``stderr``.
+:Type: Boolean
+:Required: No
+:Default: ``true``
+
+
+``log to syslog``
+
+:Description: Determines if logging messages should appear in ``syslog``.
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``err to syslog``
+
+:Description: Determines if error messages should appear in ``syslog``.
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``log flush on exit``
+
+:Description: Determines if Ceph should flush the log files after exit.
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``clog to monitors``
+
+:Description: Determines if ``clog`` messages should be sent to monitors.
+:Type: Boolean
+:Required: No
+:Default: ``true``
+
+
+``clog to syslog``
+
+:Description: Determines if ``clog`` messages should be sent to syslog.
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``mon cluster log to syslog``
+
+:Description: Determines if the cluster log should be output to the syslog.
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``mon cluster log file``
+
+:Description: The locations of the cluster's log files. There are two channels in
+              Ceph: ``cluster`` and ``audit``. This option represents a mapping
+              from channels to log files, where the log entries of that
+              channel are sent to. The ``default`` entry is a fallback
+              mapping for channels not explicitly specified. So, the following
+              default setting will send cluster log to ``$cluster.log``, and
+              send audit log to ``$cluster.audit.log``, where ``$cluster`` will
+              be replaced with the actual cluster name.
+:Type: String
+:Required: No
+:Default: ``default=/var/log/ceph/$cluster.$channel.log,cluster=/var/log/ceph/$cluster.log``
+
+
+
+OSD
+---
+
+
+``osd debug drop ping probability``
+
+:Description: ?
+:Type: Double
+:Required: No
+:Default: 0
+
+
+``osd debug drop ping duration``
+
+:Description: 
+:Type: Integer
+:Required: No
+:Default: 0
+
+``osd debug drop pg create probability``
+
+:Description: 
+:Type: Integer
+:Required: No
+:Default: 0
+
+``osd debug drop pg create duration``
+
+:Description: ?
+:Type: Double
+:Required: No
+:Default: 1
+
+
+``osd min pg log entries``
+
+:Description: The minimum number of log entries for placement groups. 
+:Type: 32-bit Unsigned Integer
+:Required: No
+:Default: 250
+
+
+``osd op log threshold``
+
+:Description: How many op log messages to show up in one pass. 
+:Type: Integer
+:Required: No
+:Default: 5
+
+
+
+Filestore
+---------
+
+``filestore debug omap check``
+
+:Description: Debugging check on synchronization. This is an expensive operation.
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+MDS
+---
+
+
+``mds debug scatterstat``
+
+:Description: Ceph will assert that various recursive stat invariants are true 
+              (for developers only).
+
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``mds debug frag``
+
+:Description: Ceph will verify directory fragmentation invariants when 
+              convenient (developers only).
+
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``mds debug auth pins``
+
+:Description: The debug auth pin invariants (for developers only).
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``mds debug subtrees``
+
+:Description: The debug subtree invariants (for developers only).
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+
+RADOS Gateway
+-------------
+
+
+``rgw log nonexistent bucket``
+
+:Description: Should we log a non-existent buckets?
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``rgw log object name``
+
+:Description: Should an object's name be logged. // man date to see codes (a subset are supported)
+:Type: String
+:Required: No
+:Default: ``%Y-%m-%d-%H-%i-%n``
+
+
+``rgw log object name utc``
+
+:Description: Object log name contains UTC?
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``rgw enable ops log``
+
+:Description: Enables logging of every RGW operation.
+:Type: Boolean
+:Required: No
+:Default: ``true``
+
+
+``rgw enable usage log``
+
+:Description: Enable logging of RGW's bandwidth usage.
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+
+``rgw usage log flush threshold``
+
+:Description: Threshold to flush pending log data.
+:Type: Integer
+:Required: No
+:Default: ``1024``
+
+
+``rgw usage log tick interval``
+
+:Description: Flush pending log data every ``s`` seconds.
+:Type: Integer
+:Required: No
+:Default: 30
+
+
+``rgw intent log object name``
+
+:Description: 
+:Type: String
+:Required: No
+:Default: ``%Y-%m-%d-%i-%n``
+
+
+``rgw intent log object name utc``
+
+:Description: Include a UTC timestamp in the intent log object name.
+:Type: Boolean
+:Required: No
+:Default: ``false``
+
+.. [#] there are levels >20 in some rare cases and that they are extremely verbose.
diff --git a/doc/rados/troubleshooting/memory-profiling.rst b/doc/rados/troubleshooting/memory-profiling.rst
new file mode 100644
index 000000000..e2396e2fd
--- /dev/null
+++ b/doc/rados/troubleshooting/memory-profiling.rst
@@ -0,0 +1,142 @@
+==================
+ Memory Profiling
+==================
+
+Ceph MON, OSD and MDS can generate heap profiles using
+``tcmalloc``. To generate heap profiles, ensure you have
+``google-perftools`` installed::
+
+	sudo apt-get install google-perftools
+
+The profiler dumps output to your ``log file`` directory (i.e.,
+``/var/log/ceph``). See `Logging and Debugging`_ for details.
+To view the profiler logs with Google's performance tools, execute the
+following:: 
+
+    google-pprof --text {path-to-daemon}  {log-path/filename}
+
+For example::
+
+    $ ceph tell osd.0 heap start_profiler
+    $ ceph tell osd.0 heap dump
+    osd.0 tcmalloc heap stats:------------------------------------------------
+    MALLOC:        2632288 (    2.5 MiB) Bytes in use by application
+    MALLOC: +       499712 (    0.5 MiB) Bytes in page heap freelist
+    MALLOC: +       543800 (    0.5 MiB) Bytes in central cache freelist
+    MALLOC: +       327680 (    0.3 MiB) Bytes in transfer cache freelist
+    MALLOC: +      1239400 (    1.2 MiB) Bytes in thread cache freelists
+    MALLOC: +      1142936 (    1.1 MiB) Bytes in malloc metadata
+    MALLOC:   ------------
+    MALLOC: =      6385816 (    6.1 MiB) Actual memory used (physical + swap)
+    MALLOC: +            0 (    0.0 MiB) Bytes released to OS (aka unmapped)
+    MALLOC:   ------------
+    MALLOC: =      6385816 (    6.1 MiB) Virtual address space used
+    MALLOC:
+    MALLOC:            231              Spans in use
+    MALLOC:             56              Thread heaps in use
+    MALLOC:           8192              Tcmalloc page size
+    ------------------------------------------------
+    Call ReleaseFreeMemory() to release freelist memory to the OS (via madvise()).
+    Bytes released to the OS take up virtual address space but no physical memory.
+    $ google-pprof --text \
+                   /usr/bin/ceph-osd  \
+                   /var/log/ceph/ceph-osd.0.profile.0001.heap
+     Total: 3.7 MB
+     1.9  51.1%  51.1%      1.9  51.1% ceph::log::Log::create_entry
+     1.8  47.3%  98.4%      1.8  47.3% std::string::_Rep::_S_create
+     0.0   0.4%  98.9%      0.0   0.6% SimpleMessenger::add_accept_pipe
+     0.0   0.4%  99.2%      0.0   0.6% decode_message
+     ...
+
+Another heap dump on the same daemon will add another file. It is
+convenient to compare to a previous heap dump to show what has grown
+in the interval. For instance::
+
+    $ google-pprof --text --base out/osd.0.profile.0001.heap \
+          ceph-osd out/osd.0.profile.0003.heap
+     Total: 0.2 MB
+     0.1  50.3%  50.3%      0.1  50.3% ceph::log::Log::create_entry
+     0.1  46.6%  96.8%      0.1  46.6% std::string::_Rep::_S_create
+     0.0   0.9%  97.7%      0.0  26.1% ReplicatedPG::do_op
+     0.0   0.8%  98.5%      0.0   0.8% __gnu_cxx::new_allocator::allocate
+
+Refer to `Google Heap Profiler`_ for additional details.
+
+Once you have the heap profiler installed, start your cluster and
+begin using the heap profiler. You may enable or disable the heap
+profiler at runtime, or ensure that it runs continuously. For the
+following commandline usage, replace ``{daemon-type}`` with ``mon``,
+``osd`` or ``mds``, and replace ``{daemon-id}`` with the OSD number or
+the MON or MDS id.
+
+
+Starting the Profiler
+---------------------
+
+To start the heap profiler, execute the following:: 
+
+	ceph tell {daemon-type}.{daemon-id} heap start_profiler
+
+For example:: 
+
+	ceph tell osd.1 heap start_profiler
+
+Alternatively the profile can be started when the daemon starts
+running if the ``CEPH_HEAP_PROFILER_INIT=true`` variable is found in
+the environment.
+
+Printing Stats
+--------------
+
+To print out statistics, execute the following:: 
+
+	ceph  tell {daemon-type}.{daemon-id} heap stats
+
+For example:: 
+
+	ceph tell osd.0 heap stats
+
+.. note:: Printing stats does not require the profiler to be running and does
+   not dump the heap allocation information to a file.
+
+
+Dumping Heap Information
+------------------------
+
+To dump heap information, execute the following:: 
+
+	ceph tell {daemon-type}.{daemon-id} heap dump
+
+For example:: 
+
+	ceph tell mds.a heap dump
+
+.. note:: Dumping heap information only works when the profiler is running.
+
+
+Releasing Memory
+----------------
+
+To release memory that ``tcmalloc`` has allocated but which is not being used by
+the Ceph daemon itself, execute the following:: 
+
+	ceph tell {daemon-type}{daemon-id} heap release
+
+For example:: 
+
+	ceph tell osd.2 heap release
+
+
+Stopping the Profiler
+---------------------
+
+To stop the heap profiler, execute the following:: 
+
+	ceph tell {daemon-type}.{daemon-id} heap stop_profiler
+
+For example:: 
+
+	ceph tell osd.0 heap stop_profiler
+
+.. _Logging and Debugging: ../log-and-debug
+.. _Google Heap Profiler: http://goog-perftools.sourceforge.net/doc/heap_profiler.html
diff --git a/doc/rados/troubleshooting/troubleshooting-mon.rst b/doc/rados/troubleshooting/troubleshooting-mon.rst
new file mode 100644
index 000000000..dc575f761
--- /dev/null
+++ b/doc/rados/troubleshooting/troubleshooting-mon.rst
@@ -0,0 +1,613 @@
+.. _rados-troubleshooting-mon:
+
+=================================
+ Troubleshooting Monitors
+=================================
+
+.. index:: monitor, high availability
+
+When a cluster encounters monitor-related troubles there's a tendency to
+panic, and sometimes with good reason. Losing one or more monitors doesn't
+necessarily mean that your cluster is down, so long as a majority are up,
+running, and form a quorum.
+Regardless of how bad the situation is, the first thing you should do is to
+calm down, take a breath, and step through the below troubleshooting steps.
+
+
+Initial Troubleshooting
+========================
+
+
+**Are the monitors running?**
+
+  First of all, we need to make sure the monitor (*mon*) daemon processes
+  (``ceph-mon``) are running.  You would be amazed by how often Ceph admins
+  forget to start the mons, or to restart them after an upgrade. There's no
+  shame, but try to not lose a couple of hours looking for a deeper problem.
+  When running Kraken or later releases also ensure that the manager
+  daemons (``ceph-mgr``) are running, usually alongside each ``ceph-mon``.
+ 
+
+**Are you able to reach to the mon nodes?**
+
+  Doesn't happen often, but sometimes there are ``iptables`` rules that
+  block accesse to mon nodes or TCP ports. These may be leftovers from
+  prior stress-testing or rule development. Try SSHing into
+  the server and, if that succeeds, try connecting to the monitor's ports
+  (``tcp/3300`` and ``tcp/6789``) using a ``telnet``, ``nc``, or similar tools.
+
+**Does ceph -s run and obtain a reply from the cluster?**
+
+  If the answer is yes then your cluster is up and running.  One thing you
+  can take for granted is that the monitors will only answer to a ``status``
+  request if there is a formed quorum.  Also check that at least one ``mgr``
+  daemon is reported as running, ideally all of them.
+
+  If ``ceph -s`` hangs without obtaining a reply from the cluster
+  or showing ``fault`` messages, then it is likely that your monitors
+  are either down completely or just a fraction are up -- a fraction
+  insufficient to form a majority quorum.  This check will connect to an
+  arbitrary mon; in rare cases it may be illuminating to bind to specific
+  mons in sequence by adding e.g. ``-m mymon1`` to the command.
+
+**What if ceph -s doesn't come back?**
+
+  If you haven't gone through all the steps so far, please go back and do.
+
+  You can contact each monitor individually asking them for their status,
+  regardless of a quorum being formed. This can be achieved using
+  ``ceph tell mon.ID mon_status``, ID being the monitor's identifier. You should
+  perform this for each monitor in the cluster. In section `Understanding
+  mon_status`_ we will explain how to interpret the output of this command.
+
+  You may instead SSH into each mon node and query the daemon's admin socket.
+
+
+Using the monitor's admin socket
+=================================
+
+The admin socket allows you to interact with a given daemon directly using a
+Unix socket file. This file can be found in your monitor's ``run`` directory.
+By default, the admin socket will be kept in ``/var/run/ceph/ceph-mon.ID.asok``
+but this may be elsewhere if you have overridden the default directory. If you
+don't find it there, check your ``ceph.conf`` for an alternative path or
+run::
+
+  ceph-conf --name mon.ID --show-config-value admin_socket
+
+Bear in mind that the admin socket will be available only while the monitor
+daemon is running. When the monitor is properly shut down, the admin socket
+will be removed. If however the monitor is not running and the admin socket
+persists, it is likely that the monitor was improperly shut down.
+Regardless, if the monitor is not running, you will not be able to use the
+admin socket, with ``ceph`` likely returning ``Error 111: Connection Refused``.
+
+Accessing the admin socket is as simple as running ``ceph tell`` on the daemon
+you are interested in. For example::
+
+  ceph tell mon.<id> mon_status
+
+Under the hood, this passes the command ``help`` to the running MON daemon
+``<id>`` via its "admin socket", which is a file ending in ``.asok``
+somewhere under ``/var/run/ceph``. Once you know the full path to the file,
+you can even do this yourself::
+
+  ceph --admin-daemon <full_path_to_asok_file> <command>
+
+Using ``help`` as the command to the ``ceph`` tool will show you the
+supported commands available through the admin socket. Please take a look
+at ``config get``, ``config show``, ``mon stat`` and ``quorum_status``,
+as those can be enlightening when troubleshooting a monitor.
+
+
+Understanding mon_status
+=========================
+
+``mon_status`` can always be obtained via the admin socket. This command will
+output a multitude of information about the monitor, including the same output
+you would get with ``quorum_status``.
+
+Take the following example output of ``ceph tell mon.c mon_status``::
+
+  
+  { "name": "c",
+    "rank": 2,
+    "state": "peon",
+    "election_epoch": 38,
+    "quorum": [
+          1,
+          2],
+    "outside_quorum": [],
+    "extra_probe_peers": [],
+    "sync_provider": [],
+    "monmap": { "epoch": 3,
+        "fsid": "5c4e9d53-e2e1-478a-8061-f543f8be4cf8",
+        "modified": "2013-10-30 04:12:01.945629",
+        "created": "2013-10-29 14:14:41.914786",
+        "mons": [
+              { "rank": 0,
+                "name": "a",
+                "addr": "127.0.0.1:6789\/0"},
+              { "rank": 1,
+                "name": "b",
+                "addr": "127.0.0.1:6790\/0"},
+              { "rank": 2,
+                "name": "c",
+                "addr": "127.0.0.1:6795\/0"}]}}
+
+A couple of things are obvious: we have three monitors in the monmap (*a*, *b*
+and *c*), the quorum is formed by only two monitors, and *c* is in the quorum
+as a *peon*.
+
+Which monitor is out of the quorum?
+
+  The answer would be **a**.
+
+Why?
+
+  Take a look at the ``quorum`` set. We have two monitors in this set: *1*
+  and *2*. These are not monitor names. These are monitor ranks, as established
+  in the current monmap. We are missing the monitor with rank 0, and according
+  to the monmap that would be ``mon.a``.
+
+By the way, how are ranks established?
+
+  Ranks are (re)calculated whenever you add or remove monitors and follow a
+  simple rule: the **greater** the ``IP:PORT`` combination, the **lower** the
+  rank is. In this case, considering that ``127.0.0.1:6789`` is lower than all
+  the remaining ``IP:PORT`` combinations, ``mon.a`` has rank 0.
+
+Most Common Monitor Issues
+===========================
+
+Have Quorum but at least one Monitor is down
+---------------------------------------------
+
+When this happens, depending on the version of Ceph you are running,
+you should be seeing something similar to::
+
+      $ ceph health detail
+      [snip]
+      mon.a (rank 0) addr 127.0.0.1:6789/0 is down (out of quorum)
+
+How to troubleshoot this?
+
+  First, make sure ``mon.a`` is running.
+
+  Second, make sure you are able to connect to ``mon.a``'s node from the
+  other mon nodes. Check the TCP ports as well. Check ``iptables`` and
+  ``nf_conntrack`` on all nodes and ensure that you are not
+  dropping/rejecting connections.
+
+  If this initial troubleshooting doesn't solve your problems, then it's
+  time to go deeper.
+
+  First, check the problematic monitor's ``mon_status`` via the admin
+  socket as explained in `Using the monitor's admin socket`_ and
+  `Understanding mon_status`_.
+
+  If the monitor is out of the quorum, its state should be one of
+  ``probing``, ``electing`` or ``synchronizing``. If it happens to be either
+  ``leader`` or ``peon``, then the monitor believes to be in quorum, while
+  the remaining cluster is sure it is not; or maybe it got into the quorum
+  while we were troubleshooting the monitor, so check you ``ceph -s`` again
+  just to make sure. Proceed if the monitor is not yet in the quorum.
+
+What if the state is ``probing``?
+
+  This means the monitor is still looking for the other monitors. Every time
+  you start a monitor, the monitor will stay in this state for some time
+  while trying to connect the rest of the monitors specified in the ``monmap``.
+  The time a monitor will spend in this state can vary. For instance, when on
+  a single-monitor cluster (never do this in production),
+  the monitor will pass through the probing state almost instantaneously.
+  In a multi-monitor cluster, the monitors will stay in this state until they
+  find enough monitors to form a quorum -- this means that if you have 2 out
+  of 3 monitors down, the one remaining monitor will stay in this state
+  indefinitely until you bring one of the other monitors up.
+
+  If you have a quorum the starting daemon should be able to find the
+  other monitors quickly, as long as they can be reached. If your
+  monitor is stuck probing and you have gone through with all the communication
+  troubleshooting, then there is a fair chance that the monitor is trying
+  to reach the other monitors on a wrong address. ``mon_status`` outputs the
+  ``monmap`` known to the monitor: check if the other monitor's locations
+  match reality. If they don't, jump to
+  `Recovering a Monitor's Broken monmap`_; if they do, then it may be related
+  to severe clock skews amongst the monitor nodes and you should refer to
+  `Clock Skews`_ first, but if that doesn't solve your problem then it is
+  the time to prepare some logs and reach out to the community (please refer
+  to `Preparing your logs`_ on how to best prepare your logs).
+
+
+What if state is ``electing``?
+
+  This means the monitor is in the middle of an election. With recent Ceph
+  releases these typically complete quickly, but at times the monitors can
+  get stuck in what is known as an *election storm*. This can indicate
+  clock skew among the monitor nodes; jump to
+  `Clock Skews`_ for more information. If all your clocks are properly
+  synchronized, you should search the mailing lists and tracker.
+  This is not a state that is likely to persist and aside from
+  (*really*) old bugs there is not an obvious reason besides clock skews on
+  why this would happen.  Worst case, if there are enough surviving mons,
+  down the problematic one while you investigate.
+
+What if state is ``synchronizing``?
+
+  This means the monitor is catching up with the rest of the cluster in
+  order to join the quorum. Time to synchronize is a function of the size
+  of your monitor store and thus of cluster size and state, so if you have a
+  large or degraded cluster this may take a while.
+
+  If you notice that the monitor jumps from ``synchronizing`` to
+  ``electing`` and then back to ``synchronizing``, then you do have a
+  problem: the cluster state may be advancing (i.e., generating new maps)
+  too fast for the synchronization process to keep up. This was a more common
+  thing in early days (Cuttlefish), but since then the synchronization process
+  has been refactored and enhanced to avoid this dynamic. If you experience
+  this in later versions please let us know via a bug tracker. And bring some logs
+  (see `Preparing your logs`_).
+
+What if state is ``leader`` or ``peon``?
+
+  This should not happen:  famous last words.  If it does, however, it likely
+  has a lot to do with clock skew -- see `Clock Skews`_. If you are not
+  suffering from clock skew, then please prepare your logs (see
+  `Preparing your logs`_) and reach out to the community.
+
+
+Recovering a Monitor's Broken ``monmap``
+----------------------------------------
+
+This is how a ``monmap`` usually looks, depending on the number of
+monitors::
+
+
+      epoch 3
+      fsid 5c4e9d53-e2e1-478a-8061-f543f8be4cf8
+      last_changed 2013-10-30 04:12:01.945629
+      created 2013-10-29 14:14:41.914786
+      0: 127.0.0.1:6789/0 mon.a
+      1: 127.0.0.1:6790/0 mon.b
+      2: 127.0.0.1:6795/0 mon.c
+      
+This may not be what you have however. For instance, in some versions of
+early Cuttlefish there was a bug that could cause your ``monmap``
+to be nullified.  Completely filled with zeros. This means that not even
+``monmaptool`` would be able to make sense of cold, hard, inscrutable zeros.
+It's also possible to end up with a monitor with a severely outdated monmap,
+notably if the node has been down for months while you fight with your vendor's
+TAC.  The subject ``ceph-mon`` daemon might be unable to find the surviving
+monitors (e.g., say ``mon.c`` is down; you add a new monitor ``mon.d``,
+then remove ``mon.a``, then add a new monitor ``mon.e`` and remove
+``mon.b``; you will end up with a totally different monmap from the one
+``mon.c`` knows).
+
+In this situation you have two possible solutions:
+
+Scrap the monitor and redeploy
+
+  You should only take this route if you are positive that you won't
+  lose the information kept by that monitor; that you have other monitors
+  and that they are running just fine so that your new monitor is able
+  to synchronize from the remaining monitors. Keep in mind that destroying
+  a monitor, if there are no other copies of its contents, may lead to
+  loss of data.
+
+Inject a monmap into the monitor
+
+  Usually the safest path. You should grab the monmap from the remaining
+  monitors and inject it into the monitor with the corrupted/lost monmap.
+
+  These are the basic steps:
+
+  1. Is there a formed quorum? If so, grab the monmap from the quorum::
+
+      $ ceph mon getmap -o /tmp/monmap
+
+  2. No quorum? Grab the monmap directly from another monitor (this
+     assumes the monitor you are grabbing the monmap from has id ID-FOO
+     and has been stopped)::
+
+      $ ceph-mon -i ID-FOO --extract-monmap /tmp/monmap
+
+  3. Stop the monitor you are going to inject the monmap into.
+
+  4. Inject the monmap::
+
+      $ ceph-mon -i ID --inject-monmap /tmp/monmap
+
+  5. Start the monitor
+
+  Please keep in mind that the ability to inject monmaps is a powerful
+  feature that can cause havoc with your monitors if misused as it will
+  overwrite the latest, existing monmap kept by the monitor.
+
+
+Clock Skews
+------------
+
+Monitor operation can be severely affected by clock skew among the quorum's
+mons, as the PAXOS consensus algorithm requires tight time alignment.
+Skew can result in weird behavior with no obvious
+cause. To avoid such issues, you must run a clock synchronization tool
+on your monitor nodes:  ``Chrony`` or the legacy ``ntpd``.  Be sure to
+configure the mon nodes with the `iburst` option and multiple peers:
+
+* Each other
+* Internal ``NTP`` servers
+* Multiple external, public pool servers
+
+For good measure, *all* nodes in your cluster should also sync against
+internal and external servers, and perhaps even your mons.  ``NTP`` servers
+should run on bare metal; VM virtualized clocks are not suitable for steady
+timekeeping.  Visit `https://www.ntp.org <https://www.ntp.org>`_ for more info.  Your
+organization may already have quality internal ``NTP`` servers you can use.  
+Sources for ``NTP`` server appliances include:
+
+* Microsemi (formerly Symmetricom) `https://microsemi.com <https://www.microsemi.com/product-directory/3425-timing-synchronization>`_
+* EndRun `https://endruntechnologies.com <https://endruntechnologies.com/products/ntp-time-servers>`_
+* Netburner `https://www.netburner.com <https://www.netburner.com/products/network-time-server/pk70-ex-ntp-network-time-server>`_
+
+
+What's the maximum tolerated clock skew?
+
+  By default the monitors will allow clocks to drift up to 0.05 seconds (50 ms).
+
+
+Can I increase the maximum tolerated clock skew?
+
+  The maximum tolerated clock skew is configurable via the
+  ``mon-clock-drift-allowed`` option, and
+  although you *CAN* you almost certainly *SHOULDN'T*. The clock skew mechanism
+  is in place because clock-skewed monitors are liely to misbehave. We, as
+  developers and QA aficionados, are comfortable with the current default
+  value, as it will alert the user before the monitors get out hand. Changing
+  this value may cause unforeseen effects on the
+  stability of the monitors and overall cluster health.
+
+How do I know there's a clock skew?
+
+  The monitors will warn you via the cluster status ``HEALTH_WARN``. ``ceph health
+  detail`` or ``ceph status`` should show something like::
+
+      mon.c addr 10.10.0.1:6789/0 clock skew 0.08235s > max 0.05s (latency 0.0045s)
+
+  That means that ``mon.c`` has been flagged as suffering from a clock skew.
+
+  On releases beginning with Luminous you can issue the
+  ``ceph time-sync-status`` command to check status.  Note that the lead mon
+  is typically the one with the numerically lowest IP address.  It will always
+  show ``0``: the reported offsets of other mons are relative to
+  the lead mon, not to any external reference source.
+
+
+What should I do if there's a clock skew?
+
+  Synchronize your clocks. Running an NTP client may help. If you are already
+  using one and you hit this sort of issues, check if you are using some NTP
+  server remote to your network and consider hosting your own NTP server on
+  your network.  This last option tends to reduce the amount of issues with
+  monitor clock skews.
+
+
+Client Can't Connect or Mount
+------------------------------
+
+Check your IP tables. Some OS install utilities add a ``REJECT`` rule to
+``iptables``. The rule rejects all clients trying to connect to the host except
+for ``ssh``. If your monitor host's IP tables have such a ``REJECT`` rule in
+place, clients connecting from a separate node will fail to mount with a timeout
+error. You need to address ``iptables`` rules that reject clients trying to
+connect to Ceph daemons.  For example, you would need to address rules that look
+like this appropriately::
+
+	REJECT all -- anywhere anywhere reject-with icmp-host-prohibited
+
+You may also need to add rules to IP tables on your Ceph hosts to ensure
+that clients can access the ports associated with your Ceph monitors (i.e., port
+6789 by default) and Ceph OSDs (i.e., 6800 through 7300 by default). For
+example::
+
+	iptables -A INPUT -m multiport -p tcp -s {ip-address}/{netmask} --dports 6789,6800:7300 -j ACCEPT
+
+Monitor Store Failures
+======================
+
+Symptoms of store corruption
+----------------------------
+
+Ceph monitor stores the :term:`Cluster Map` in a key/value store such as LevelDB. If
+a monitor fails due to the key/value store corruption, following error messages
+might be found in the monitor log::
+
+  Corruption: error in middle of record
+
+or::
+
+  Corruption: 1 missing files; e.g.: /var/lib/ceph/mon/mon.foo/store.db/1234567.ldb
+
+Recovery using healthy monitor(s)
+---------------------------------
+
+If there are any survivors, we can always :ref:`replace <adding-and-removing-monitors>` the corrupted one with a
+new one. After booting up, the new joiner will sync up with a healthy
+peer, and once it is fully sync'ed, it will be able to serve the clients.
+
+.. _mon-store-recovery-using-osds:
+
+Recovery using OSDs
+-------------------
+
+But what if all monitors fail at the same time? Since users are encouraged to
+deploy at least three (and preferably five) monitors in a Ceph cluster, the chance of simultaneous
+failure is rare. But unplanned power-downs in a data center with improperly
+configured disk/fs settings could fail the underlying file system, and hence
+kill all the monitors. In this case, we can recover the monitor store with the
+information stored in OSDs.::
+
+  ms=/root/mon-store
+  mkdir $ms
+  
+  # collect the cluster map from stopped OSDs
+  for host in $hosts; do
+    rsync -avz $ms/. user@$host:$ms.remote
+    rm -rf $ms
+    ssh user@$host <<EOF
+      for osd in /var/lib/ceph/osd/ceph-*; do
+        ceph-objectstore-tool --data-path \$osd --no-mon-config --op update-mon-db --mon-store-path $ms.remote
+      done
+  EOF
+    rsync -avz user@$host:$ms.remote/. $ms
+  done
+  
+  # rebuild the monitor store from the collected map, if the cluster does not
+  # use cephx authentication, we can skip the following steps to update the
+  # keyring with the caps, and there is no need to pass the "--keyring" option.
+  # i.e. just use "ceph-monstore-tool $ms rebuild" instead
+  ceph-authtool /path/to/admin.keyring -n mon. \
+    --cap mon 'allow *'
+  ceph-authtool /path/to/admin.keyring -n client.admin \
+    --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *'
+  # add one or more ceph-mgr's key to the keyring. in this case, an encoded key
+  # for mgr.x is added, you can find the encoded key in
+  # /etc/ceph/${cluster}.${mgr_name}.keyring on the machine where ceph-mgr is
+  # deployed
+  ceph-authtool /path/to/admin.keyring --add-key 'AQDN8kBe9PLWARAAZwxXMr+n85SBYbSlLcZnMA==' -n mgr.x \
+    --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *'
+  # if your monitors' ids are not single characters like 'a', 'b', 'c', please
+  # specify them in the command line by passing them as arguments of the "--mon-ids"
+  # option. if you are not sure, please check your ceph.conf to see if there is any
+  # sections named like '[mon.foo]'. don't pass the "--mon-ids" option, if you are
+  # using DNS SRV for looking up monitors.
+  ceph-monstore-tool $ms rebuild -- --keyring /path/to/admin.keyring --mon-ids alpha beta gamma
+  
+  # make a backup of the corrupted store.db just in case!  repeat for
+  # all monitors.
+  mv /var/lib/ceph/mon/mon.foo/store.db /var/lib/ceph/mon/mon.foo/store.db.corrupted
+
+  # move rebuild store.db into place.  repeat for all monitors.
+  mv $ms/store.db /var/lib/ceph/mon/mon.foo/store.db
+  chown -R ceph:ceph /var/lib/ceph/mon/mon.foo/store.db
+
+The steps above
+
+#. collect the map from all OSD hosts,
+#. then rebuild the store,
+#. fill the entities in keyring file with appropriate caps
+#. replace the corrupted store on ``mon.foo`` with the recovered copy.
+
+Known limitations
+~~~~~~~~~~~~~~~~~
+
+Following information are not recoverable using the steps above:
+
+- **some added keyrings**: all the OSD keyrings added using ``ceph auth add`` command
+  are recovered from the OSD's copy. And the ``client.admin`` keyring is imported
+  using ``ceph-monstore-tool``. But the MDS keyrings and other keyrings are missing
+  in the recovered monitor store. You might need to re-add them manually.
+
+- **creating pools**: If any RADOS pools were in the process of being creating, that state is lost.  The recovery tool assumes that all pools have been created.  If there are PGs that are stuck in the 'unknown' after the recovery for a partially created pool, you can force creation of the *empty* PG with the ``ceph osd force-create-pg`` command.  Note that this will create an *empty* PG, so only do this if you know the pool is empty.
+
+- **MDS Maps**: the MDS maps are lost.
+
+
+
+Everything Failed! Now What?
+=============================
+
+Reaching out for help
+----------------------
+
+You can find us on IRC at #ceph and #ceph-devel at OFTC (server irc.oftc.net)
+and on ``ceph-devel@vger.kernel.org`` and ``ceph-users@lists.ceph.com``. Make
+sure you have grabbed your logs and have them ready if someone asks: the faster
+the interaction and lower the latency in response, the better chances everyone's
+time is optimized.
+
+
+Preparing your logs
+---------------------
+
+Monitor logs are, by default, kept in ``/var/log/ceph/ceph-mon.FOO.log*``. We
+may want them. However, your logs may not have the necessary information. If
+you don't find your monitor logs at their default location, you can check
+where they should be by running::
+
+  ceph-conf --name mon.FOO --show-config-value log_file
+
+The amount of information in the logs are subject to the debug levels being
+enforced by your configuration files. If you have not enforced a specific
+debug level then Ceph is using the default levels and your logs may not
+contain important information to track down you issue.
+A first step in getting relevant information into your logs will be to raise
+debug levels. In this case we will be interested in the information from the
+monitor.
+Similarly to what happens on other components, different parts of the monitor
+will output their debug information on different subsystems.
+
+You will have to raise the debug levels of those subsystems more closely
+related to your issue. This may not be an easy task for someone unfamiliar
+with troubleshooting Ceph. For most situations, setting the following options
+on your monitors will be enough to pinpoint a potential source of the issue::
+
+      debug mon = 10
+      debug ms = 1
+
+If we find that these debug levels are not enough, there's a chance we may
+ask you to raise them or even define other debug subsystems to obtain infos
+from -- but at least we started off with some useful information, instead
+of a massively empty log without much to go on with.
+
+Do I need to restart a monitor to adjust debug levels?
+------------------------------------------------------
+
+No. You may do it in one of two ways:
+
+You have quorum
+
+  Either inject the debug option into the monitor you want to debug::
+
+        ceph tell mon.FOO config set debug_mon 10/10
+
+  or into all monitors at once::
+
+        ceph tell mon.* config set debug_mon 10/10
+
+No quorum
+
+  Use the monitor's admin socket and directly adjust the configuration
+  options::
+
+      ceph daemon mon.FOO config set debug_mon 10/10
+
+
+Going back to default values is as easy as rerunning the above commands
+using the debug level ``1/10`` instead.  You can check your current
+values using the admin socket and the following commands::
+
+      ceph daemon mon.FOO config show
+
+or::
+
+      ceph daemon mon.FOO config get 'OPTION_NAME'
+
+
+Reproduced the problem with appropriate debug levels. Now what?
+----------------------------------------------------------------
+
+Ideally you would send us only the relevant portions of your logs.
+We realise that figuring out the corresponding portion may not be the
+easiest of tasks. Therefore, we won't hold it to you if you provide the
+full log, but common sense should be employed. If your log has hundreds
+of thousands of lines, it may get tricky to go through the whole thing,
+specially if we are not aware at which point, whatever your issue is,
+happened. For instance, when reproducing, keep in mind to write down
+current time and date and to extract the relevant portions of your logs
+based on that.
+
+Finally, you should reach out to us on the mailing lists, on IRC or file
+a new issue on the `tracker`_.
+
+.. _tracker: http://tracker.ceph.com/projects/ceph/issues/new
diff --git a/doc/rados/troubleshooting/troubleshooting-osd.rst b/doc/rados/troubleshooting/troubleshooting-osd.rst
new file mode 100644
index 000000000..cc852d73d
--- /dev/null
+++ b/doc/rados/troubleshooting/troubleshooting-osd.rst
@@ -0,0 +1,620 @@
+======================
+ Troubleshooting OSDs
+======================
+
+Before troubleshooting your OSDs, first check your monitors and network. If
+you execute ``ceph health`` or ``ceph -s`` on the command line and Ceph shows
+``HEALTH_OK``, it means that the monitors have a quorum.
+If you don't have a monitor quorum or if there are errors with the monitor
+status, `address the monitor issues first <../troubleshooting-mon>`_.
+Check your networks to ensure they
+are running properly, because networks may have a significant impact on OSD
+operation and performance. Look for dropped packets on the host side
+and CRC errors on the switch side.
+
+Obtaining Data About OSDs
+=========================
+
+A good first step in troubleshooting your OSDs is to obtain topology information in
+addition to the information you collected while `monitoring your OSDs`_
+(e.g., ``ceph osd tree``).
+
+
+Ceph Logs
+---------
+
+If you haven't changed the default path, you can find Ceph log files at
+``/var/log/ceph``::
+
+	ls /var/log/ceph
+
+If you don't see enough log detail you can change your logging level.  See
+`Logging and Debugging`_ for details to ensure that Ceph performs adequately
+under high logging volume.
+
+
+Admin Socket
+------------
+
+Use the admin socket tool to retrieve runtime information. For details, list
+the sockets for your Ceph daemons::
+
+	ls /var/run/ceph
+
+Then, execute the following, replacing ``{daemon-name}`` with an actual
+daemon (e.g., ``osd.0``)::
+
+  ceph daemon osd.0 help
+
+Alternatively, you can specify a ``{socket-file}`` (e.g., something in ``/var/run/ceph``)::
+
+  ceph daemon {socket-file} help
+
+The admin socket, among other things, allows you to:
+
+- List your configuration at runtime
+- Dump historic operations
+- Dump the operation priority queue state
+- Dump operations in flight
+- Dump perfcounters
+
+Display Freespace
+-----------------
+
+Filesystem issues may arise. To display your file system's free space, execute
+``df``. ::
+
+	df -h
+
+Execute ``df --help`` for additional usage.
+
+I/O Statistics
+--------------
+
+Use `iostat`_ to identify I/O-related issues. ::
+
+	iostat -x
+
+Diagnostic Messages
+-------------------
+
+To retrieve diagnostic messages from the kernel, use ``dmesg`` with ``less``, ``more``, ``grep``
+or ``tail``.  For example::
+
+	dmesg | grep scsi
+
+Stopping w/out Rebalancing
+==========================
+
+Periodically, you may need to perform maintenance on a subset of your cluster,
+or resolve a problem that affects a failure domain (e.g., a rack). If you do not
+want CRUSH to automatically rebalance the cluster as you stop OSDs for
+maintenance, set the cluster to ``noout`` first::
+
+	ceph osd set noout
+
+On Luminous or newer releases it is safer to set the flag only on affected OSDs.
+You can do this individually ::
+
+	ceph osd add-noout osd.0
+	ceph osd rm-noout  osd.0
+
+Or an entire CRUSH bucket at a time.  Say you're going to take down
+``prod-ceph-data1701`` to add RAM ::
+
+	ceph osd set-group noout prod-ceph-data1701
+
+Once the flag is set you can stop the OSDs and any other colocated Ceph
+services within the failure domain that requires maintenance work. ::
+
+	systemctl stop ceph\*.service ceph\*.target
+
+.. note:: Placement groups within the OSDs you stop will become ``degraded``
+   while you are addressing issues with within the failure domain.
+
+Once you have completed your maintenance, restart the OSDs and any other
+daemons.  If you rebooted the host as part of the maintenance, these should
+come back on their own without intervention. ::
+
+	sudo systemctl start ceph.target
+
+Finally, you must unset the cluster-wide``noout`` flag::
+
+	ceph osd unset noout
+	ceph osd unset-group noout prod-ceph-data1701
+
+Note that most Linux distributions that Ceph supports today employ ``systemd``
+for service management.  For other or older operating systems you may need
+to issue equivalent ``service`` or ``start``/``stop`` commands.
+
+.. _osd-not-running:
+
+OSD Not Running
+===============
+
+Under normal circumstances, simply restarting the ``ceph-osd`` daemon will
+allow it to rejoin the cluster and recover.
+
+An OSD Won't Start
+------------------
+
+If you start your cluster and an OSD won't start, check the following:
+
+- **Configuration File:** If you were not able to get OSDs running from
+  a new installation, check your configuration file to ensure it conforms
+  (e.g., ``host`` not ``hostname``, etc.).
+
+- **Check Paths:** Check the paths in your configuration, and the actual
+  paths themselves for data and metadata (journals, WAL, DB). If you separate the OSD data from
+  the metadata and there are errors in your configuration file or in the
+  actual mounts, you may have trouble starting OSDs. If you want to store the
+  metadata on a separate block device, you should partition or LVM your
+  drive and assign one partition per OSD.
+
+- **Check Max Threadcount:** If you have a node with a lot of OSDs, you may be
+  hitting the default maximum number of threads (e.g., usually 32k), especially
+  during recovery. You can increase the number of threads using ``sysctl`` to
+  see if increasing the maximum number of threads to the maximum possible
+  number of threads allowed (i.e.,  4194303) will help. For example::
+
+	sysctl -w kernel.pid_max=4194303
+
+  If increasing the maximum thread count resolves the issue, you can make it
+  permanent by including a ``kernel.pid_max`` setting in a file under ``/etc/sysctl.d`` or
+  within the master ``/etc/sysctl.conf`` file. For example::
+
+	kernel.pid_max = 4194303
+
+- **Check ``nf_conntrack``:** This connection tracking and limiting system
+  is the bane of many production Ceph clusters, and can be insidious in that
+  everything is fine at first. As cluster topology and client workload
+  grow, mysterious and intermittent connection failures and performance
+  glitches manifest, becoming worse over time and at certain times of day.
+  Check ``syslog`` history for table fillage events.  You can mitigate this
+  bother by raising ``nf_conntrack_max`` to a much higher value via ``sysctl``.
+  Be sure to raise ``nf_conntrack_buckets`` accordingly to
+  ``nf_conntrack_max / 4``, which may require action outside of ``sysctl`` e.g.
+  ``"echo 131072 > /sys/module/nf_conntrack/parameters/hashsize``
+  More interdictive but fussier is to blacklist the associated kernel modules
+  to disable processing altogether.  This is fragile in that the modules
+  vary among kernel versions, as does the order in which they must be listed.
+  Even when blacklisted there are situations in which ``iptables`` or ``docker``
+  may activate connection tracking anyway, so a "set and forget" strategy for
+  the tunables is advised.  On modern systems this will not consume appreciable
+  resources.
+
+- **Kernel Version:** Identify the kernel version and distribution you
+  are using. Ceph uses some third party tools by default, which may be
+  buggy or may conflict with certain distributions and/or kernel
+  versions (e.g., Google ``gperftools`` and ``TCMalloc``). Check the
+  `OS recommendations`_ and the release notes for each Ceph version
+  to ensure you have addressed any issues related to your kernel.
+
+- **Segment Fault:** If there is a segment fault, increase log levels
+  and start the problematic daemon(s) again. If segment faults recur,
+  search the Ceph bug tracker `https://tracker.ceph/com/projects/ceph <https://tracker.ceph.com/projects/ceph/>`_
+  and the ``dev`` and ``ceph-users`` mailing list archives `https://ceph.io/resources <https://ceph.io/resources>`_.
+  If this is truly a new and unique
+  failure, post to the ``dev`` email list and provide the specific Ceph
+  release being run, ``ceph.conf`` (with secrets XXX'd out),
+  your monitor status output and excerpts from your log file(s).
+
+An OSD Failed
+-------------
+
+When a ``ceph-osd`` process dies, surviving ``ceph-osd`` daemons will report
+to the mons that it appears down, which will in turn surface the new status
+via the ``ceph health`` command::
+
+	ceph health
+	HEALTH_WARN 1/3 in osds are down
+
+Specifically, you will get a warning whenever there are OSDs marked ``in``
+and ``down``.  You can identify which  are ``down`` with::
+
+	ceph health detail
+	HEALTH_WARN 1/3 in osds are down
+	osd.0 is down since epoch 23, last address 192.168.106.220:6800/11080
+
+or ::
+
+	ceph osd tree down
+
+If there is a drive
+failure or other fault preventing ``ceph-osd`` from functioning or
+restarting, an error message should be present in its log file under
+``/var/log/ceph``.
+
+If the daemon stopped because of a heartbeat failure or ``suicide timeout``,
+the underlying drive or filesystem may be unresponsive. Check ``dmesg``
+and `syslog`  output for drive or other kernel errors.  You may need to
+specify something like ``dmesg -T`` to get timestamps, otherwise it's
+easy to mistake old errors for new.
+
+If the problem is a software error (failed assertion or other
+unexpected error), search the archives and tracker as above, and
+report it to the `ceph-devel`_ email list if there's no clear fix or
+existing bug.
+
+.. _no-free-drive-space:
+
+No Free Drive Space
+-------------------
+
+Ceph prevents you from writing to a full OSD so that you don't lose data.
+In an operational cluster, you should receive a warning when your cluster's OSDs
+and pools approach the full ratio. The ``mon osd full ratio`` defaults to
+``0.95``, or 95% of capacity before it stops clients from writing data.
+The ``mon osd backfillfull ratio`` defaults to ``0.90``, or 90 % of
+capacity above which backfills will not start. The
+OSD nearfull ratio defaults to ``0.85``, or 85% of capacity
+when it generates a health warning.
+
+Note that individual OSDs within a cluster will vary in how much data Ceph
+allocates to them.  This utilization can be displayed for each OSD with ::
+
+	ceph osd df
+
+Overall cluster / pool fullness can be checked with ::
+
+	ceph df 
+
+Pay close attention to the **most full** OSDs, not the percentage of raw space
+used as reported by ``ceph df``.  It only takes one outlier OSD filling up to
+fail writes to its pool.  The space available to each pool as reported by
+``ceph df`` considers the ratio settings relative to the *most full* OSD that
+is part of a given pool.  The distribution can be flattened by progressively
+moving data from overfull or to underfull OSDs using the ``reweight-by-utilization``
+command.  With Ceph releases beginning with later revisions of Luminous one can also
+exploit the ``ceph-mgr`` ``balancer`` module to perform this task automatically
+and rather effectively.
+
+The ratios can be adjusted:
+
+::
+
+    ceph osd set-nearfull-ratio <float[0.0-1.0]>
+    ceph osd set-full-ratio <float[0.0-1.0]>
+    ceph osd set-backfillfull-ratio <float[0.0-1.0]>
+
+Full cluster issues can arise when an OSD fails either as a test or organically
+within small and/or very full or unbalanced cluster. When an OSD or node
+holds an outsize percentage of the cluster's data, the ``nearfull`` and ``full``
+ratios may be exceeded as a result of component failures or even natural growth.
+If you are testing how Ceph reacts to OSD failures on a small
+cluster, you should leave ample free disk space and consider temporarily
+lowering the OSD ``full ratio``, OSD ``backfillfull ratio`` and
+OSD ``nearfull ratio``
+
+Full ``ceph-osds`` will be reported by ``ceph health``::
+
+	ceph health
+	HEALTH_WARN 1 nearfull osd(s)
+
+Or::
+
+	ceph health detail
+	HEALTH_ERR 1 full osd(s); 1 backfillfull osd(s); 1 nearfull osd(s)
+	osd.3 is full at 97%
+	osd.4 is backfill full at 91%
+	osd.2 is near full at 87%
+
+The best way to deal with a full cluster is to add capacity via new OSDs, enabling
+the cluster to redistribute data to newly available storage.
+
+If you cannot start a legacy Filestore OSD because it is full, you may reclaim
+some space deleting a few placement group directories in the full OSD.
+
+.. important:: If you choose to delete a placement group directory on a full OSD,
+   **DO NOT** delete the same placement group directory on another full OSD, or
+   **YOU WILL LOSE DATA**. You **MUST** maintain at least one copy of your data on
+   at least one OSD.  This is a rare and extreme intervention, and is not to be
+   undertaken lightly.
+
+See `Monitor Config Reference`_ for additional details.
+
+OSDs are Slow/Unresponsive
+==========================
+
+A common issue involves slow or unresponsive OSDs. Ensure that you
+have eliminated other troubleshooting possibilities before delving into OSD
+performance issues. For example, ensure that your network(s) is working properly
+and your OSDs are running. Check to see if OSDs are throttling recovery traffic.
+
+.. tip:: Newer versions of Ceph provide better recovery handling by preventing
+   recovering OSDs from using up system resources so that ``up`` and ``in``
+   OSDs are not available or are otherwise slow.
+
+Networking Issues
+-----------------
+
+Ceph is a distributed storage system, so it relies upon networks for OSD peering
+and replication, recovery from faults, and periodic heartbeats. Networking
+issues can cause OSD latency and flapping OSDs. See `Flapping OSDs`_ for
+details.
+
+Ensure that Ceph processes and Ceph-dependent processes are connected and/or
+listening. ::
+
+	netstat -a | grep ceph
+	netstat -l | grep ceph
+	sudo netstat -p | grep ceph
+
+Check network statistics. ::
+
+	netstat -s
+
+Drive Configuration
+-------------------
+
+A SAS or SATA storage drive should only house one OSD; NVMe drives readily
+handle two or more. Read and write throughput can bottleneck if other processes
+share the drive, including journals / metadata, operating systems, Ceph monitors,
+`syslog` logs, other OSDs, and non-Ceph processes.
+
+Ceph acknowledges writes *after* journaling, so fast SSDs are an
+attractive option to accelerate the response time--particularly when
+using the ``XFS`` or ``ext4`` file systems for legacy Filestore OSDs.
+By contrast, the ``Btrfs``
+file system can write and journal simultaneously.  (Note, however, that
+we recommend against using ``Btrfs`` for production deployments.)
+
+.. note:: Partitioning a drive does not change its total throughput or
+   sequential read/write limits. Running a journal in a separate partition
+   may help, but you should prefer a separate physical drive.
+
+Bad Sectors / Fragmented Disk
+-----------------------------
+
+Check your drives for bad blocks, fragmentation, and other errors that can cause
+performance to drop substantially.  Invaluable tools include ``dmesg``, ``syslog``
+logs, and ``smartctl`` (from the ``smartmontools`` package).
+
+Co-resident Monitors/OSDs
+-------------------------
+
+Monitors are relatively lightweight processes, but they issue lots of
+``fsync()`` calls,
+which can interfere with other workloads, particularly if monitors run on the
+same drive as an OSD. Additionally, if you run monitors on the same host as
+OSDs, you may incur performance issues related to:
+
+- Running an older kernel (pre-3.0)
+- Running a kernel with no ``syncfs(2)`` syscall.
+
+In these cases, multiple OSDs running on the same host can drag each other down
+by doing lots of commits. That often leads to the bursty writes.
+
+Co-resident Processes
+---------------------
+
+Spinning up co-resident processes (convergence) such as a cloud-based solution, virtual
+machines and other applications that write data to Ceph while operating on the
+same hardware as OSDs can introduce significant OSD latency. Generally, we
+recommend optimizing hosts for use with Ceph and using other hosts for other
+processes. The practice of separating Ceph operations from other applications
+may help improve performance and may streamline troubleshooting and maintenance.
+
+Logging Levels
+--------------
+
+If you turned logging levels up to track an issue and then forgot to turn
+logging levels back down, the OSD may be putting a lot of logs onto the disk. If
+you intend to keep logging levels high, you may consider mounting a drive to the
+default path for logging (i.e., ``/var/log/ceph/$cluster-$name.log``).
+
+Recovery Throttling
+-------------------
+
+Depending upon your configuration, Ceph may reduce recovery rates to maintain
+performance or it may increase recovery rates to the point that recovery
+impacts OSD performance. Check to see if the OSD is recovering.
+
+Kernel Version
+--------------
+
+Check the kernel version you are running. Older kernels may not receive
+new backports that Ceph depends upon for better performance.
+
+Kernel Issues with SyncFS
+-------------------------
+
+Try running one OSD per host to see if performance improves. Old kernels
+might not have a recent enough version of ``glibc`` to support ``syncfs(2)``.
+
+Filesystem Issues
+-----------------
+
+Currently, we recommend deploying clusters with the BlueStore back end.
+When running a pre-Luminous release or if you have a specific reason to deploy
+OSDs with the previous Filestore backend, we recommend ``XFS``.
+
+We recommend against using ``Btrfs`` or ``ext4``.  The ``Btrfs`` filesystem has
+many attractive features, but bugs may lead to
+performance issues and spurious ENOSPC errors.  We do not recommend
+``ext4`` for Filestore OSDs because ``xattr`` limitations break support for long
+object names, which are needed for RGW.
+
+For more information, see `Filesystem Recommendations`_.
+
+.. _Filesystem Recommendations: ../configuration/filesystem-recommendations
+
+Insufficient RAM
+----------------
+
+We recommend a *minimum* of 4GB of RAM per OSD daemon and suggest rounding up
+from 6-8GB.  You may notice that during normal operations, ``ceph-osd``
+processes only use a fraction of that amount.
+Unused RAM makes it tempting to use the excess RAM for co-resident
+applications or to skimp on each node's memory capacity.  However,
+when OSDs experience recovery their memory utilization spikes. If
+there is insufficient RAM available, OSD performance will slow considerably
+and the daemons may even crash or be killed by the Linux ``OOM Killer``.
+
+Blocked Requests or Slow Requests
+---------------------------------
+
+If a ``ceph-osd`` daemon is slow to respond to a request, messages will be logged
+noting ops that are taking too long.  The warning threshold
+defaults to 30 seconds and is configurable via the ``osd op complaint time``
+setting.  When this happens, the cluster log will receive messages.
+
+Legacy versions of Ceph complain about ``old requests``::
+
+	osd.0 192.168.106.220:6800/18813 312 : [WRN] old request osd_op(client.5099.0:790 fatty_26485_object789 [write 0~4096] 2.5e54f643) v4 received at 2012-03-06 15:42:56.054801 currently waiting for sub ops
+
+New versions of Ceph complain about ``slow requests``::
+
+	{date} {osd.num} [WRN] 1 slow requests, 1 included below; oldest blocked for > 30.005692 secs
+	{date} {osd.num}  [WRN] slow request 30.005692 seconds old, received at {date-time}: osd_op(client.4240.0:8 benchmark_data_ceph-1_39426_object7 [write 0~4194304] 0.69848840) v4 currently waiting for subops from [610]
+
+Possible causes include:
+
+- A failing drive (check ``dmesg`` output)
+- A bug in the kernel file system (check ``dmesg`` output)
+- An overloaded cluster (check system load, iostat, etc.)
+- A bug in the ``ceph-osd`` daemon.
+
+Possible solutions:
+
+- Remove VMs from Ceph hosts
+- Upgrade kernel
+- Upgrade Ceph
+- Restart OSDs
+- Replace failed or failing components
+
+Debugging Slow Requests
+-----------------------
+
+If you run ``ceph daemon osd.<id> dump_historic_ops`` or ``ceph daemon osd.<id> dump_ops_in_flight``,
+you will see a set of operations and a list of events each operation went
+through. These are briefly described below.
+
+Events from the Messenger layer:
+
+- ``header_read``: When the messenger first started reading the message off the wire.
+- ``throttled``: When the messenger tried to acquire memory throttle space to read
+  the message into memory.
+- ``all_read``: When the messenger finished reading the message off the wire.
+- ``dispatched``: When the messenger gave the message to the OSD.
+- ``initiated``: This is identical to ``header_read``. The existence of both is a
+  historical oddity.
+
+Events from the OSD as it processes ops:
+
+- ``queued_for_pg``: The op has been put into the queue for processing by its PG.
+- ``reached_pg``: The PG has started doing the op.
+- ``waiting for \*``: The op is waiting for some other work to complete before it
+  can proceed (e.g. a new OSDMap; for its object target to scrub; for the PG to
+  finish peering; all as specified in the message).
+- ``started``: The op has been accepted as something the OSD should do and 
+  is now being performed.
+- ``waiting for subops from``: The op has been sent to replica OSDs.
+
+Events from ```Filestore```:
+
+- ``commit_queued_for_journal_write``: The op has been given to the FileStore.
+- ``write_thread_in_journal_buffer``: The op is in the journal's buffer and waiting
+  to be persisted (as the next disk write).
+- ``journaled_completion_queued``: The op was journaled to disk and its callback
+  queued for invocation.
+
+Events from the OSD after data has been given to underlying storage:
+
+- ``op_commit``: The op has been committed (i.e. written to journal) by the
+  primary OSD.
+- ``op_applied``: The op has been `write()'en <https://www.freebsd.org/cgi/man.cgi?write(2)>`_ to the backing FS (i.e.   applied in memory but not flushed out to disk) on the primary.
+- ``sub_op_applied``: ``op_applied``, but for a replica's "subop".
+- ``sub_op_committed``: ``op_commit``, but for a replica's subop (only for EC pools).
+- ``sub_op_commit_rec/sub_op_apply_rec from <X>``: The primary marks this when it
+  hears about the above, but for a particular replica (i.e. ``<X>``).
+- ``commit_sent``: We sent a reply back to the client (or primary OSD, for sub ops).
+
+Many of these events are seemingly redundant, but cross important boundaries in
+the internal code (such as passing data across locks into new threads).
+
+Flapping OSDs
+=============
+
+When OSDs peer and check heartbeats, they use the cluster (back-end)
+network when it's available. See `Monitor/OSD Interaction`_ for details.
+
+We have tradtionally recommended separate *public* (front-end) and *private*
+(cluster / back-end / replication) networks:
+
+#. Segregation of heartbeat and replication / recovery traffic (private)
+   from client and OSD <-> mon traffic (public).  This helps keep one
+   from DoS-ing the other, which could in turn result in a cascading failure.
+
+#. Additional throughput for both public and private traffic.
+
+When common networking technloogies were 100Mb/s and 1Gb/s, this separation
+was often critical.  With today's 10Gb/s, 40Gb/s, and 25/50/100Gb/s
+networks, the above capacity concerns are often diminished or even obviated.
+For example, if your OSD nodes have two network ports, dedicating one to
+the public and the other to the private network means no path redundancy.
+This degrades your ability to weather network maintenance and failures without
+significant cluster or client impact.  Consider instead using both links
+for just a public network:  with bonding (LACP) or equal-cost routing (e.g. FRR)
+you reap the benefits of increased throughput headroom, fault tolerance, and
+reduced OSD flapping.
+
+When a private network (or even a single host link) fails or degrades while the
+public network operates normally, OSDs may not handle this situation well. What
+happens is that OSDs use the public network to report each other ``down`` to
+the monitors, while marking themselves ``up``. The monitors then send out,
+again on the public network, an updated cluster map with affected OSDs marked
+`down`. These OSDs reply to the monitors "I'm not dead yet!", and the cycle
+repeats.  We call this scenario 'flapping`, and it can be difficult to isolate
+and remediate.  With no private network, this irksome dynamic is avoided:
+OSDs are generally either ``up`` or ``down`` without flapping.
+
+If something does cause OSDs to 'flap' (repeatedly getting marked ``down`` and
+then ``up`` again), you can force the monitors to halt the flapping by
+temporarily freezing their states::
+
+	ceph osd set noup      # prevent OSDs from getting marked up
+	ceph osd set nodown    # prevent OSDs from getting marked down
+
+These flags are recorded in the osdmap::
+
+	ceph osd dump | grep flags
+	flags no-up,no-down
+
+You can clear the flags with::
+
+	ceph osd unset noup
+	ceph osd unset nodown
+
+Two other flags are supported, ``noin`` and ``noout``, which prevent
+booting OSDs from being marked ``in`` (allocated data) or protect OSDs
+from eventually being marked ``out`` (regardless of what the current value for
+``mon osd down out interval`` is).
+
+.. note:: ``noup``, ``noout``, and ``nodown`` are temporary in the
+   sense that once the flags are cleared, the action they were blocking
+   should occur shortly after.  The ``noin`` flag, on the other hand,
+   prevents OSDs from being marked ``in`` on boot, and any daemons that
+   started while the flag was set will remain that way.
+
+.. note:: The causes and effects of flapping can be somewhat mitigated through
+   careful adjustments to the ``mon_osd_down_out_subtree_limit``,
+   ``mon_osd_reporter_subtree_level``, and ``mon_osd_min_down_reporters``.
+   Derivation of optimal settings depends on cluster size, topology, and the
+   Ceph  release in use. Their interactions are subtle and beyond the scope of
+   this document.
+
+
+.. _iostat: https://en.wikipedia.org/wiki/Iostat
+.. _Ceph Logging and Debugging: ../../configuration/ceph-conf#ceph-logging-and-debugging
+.. _Logging and Debugging: ../log-and-debug
+.. _Debugging and Logging: ../debug
+.. _Monitor/OSD Interaction: ../../configuration/mon-osd-interaction
+.. _Monitor Config Reference: ../../configuration/mon-config-ref
+.. _monitoring your OSDs: ../../operations/monitoring-osd-pg
+.. _subscribe to the ceph-devel email list: mailto:majordomo@vger.kernel.org?body=subscribe+ceph-devel
+.. _unsubscribe from the ceph-devel email list: mailto:majordomo@vger.kernel.org?body=unsubscribe+ceph-devel
+.. _subscribe to the ceph-users email list: mailto:ceph-users-join@lists.ceph.com
+.. _unsubscribe from the ceph-users email list: mailto:ceph-users-leave@lists.ceph.com
+.. _OS recommendations: ../../../start/os-recommendations
+.. _ceph-devel: ceph-devel@vger.kernel.org
diff --git a/doc/rados/troubleshooting/troubleshooting-pg.rst b/doc/rados/troubleshooting/troubleshooting-pg.rst
new file mode 100644
index 000000000..f5e5054ba
--- /dev/null
+++ b/doc/rados/troubleshooting/troubleshooting-pg.rst
@@ -0,0 +1,693 @@
+=====================
+ Troubleshooting PGs
+=====================
+
+Placement Groups Never Get Clean
+================================
+
+When you create a cluster and your cluster remains in ``active``,
+``active+remapped`` or ``active+degraded`` status and never achieves an
+``active+clean`` status, you likely have a problem with your configuration.
+
+You may need to review settings in the `Pool, PG and CRUSH Config Reference`_
+and make appropriate adjustments.
+
+As a general rule, you should run your cluster with more than one OSD and a
+pool size greater than 1 object replica.
+
+.. _one-node-cluster:
+
+One Node Cluster
+----------------
+
+Ceph no longer provides documentation for operating on a single node, because
+you would never deploy a system designed for distributed computing on a single
+node. Additionally, mounting client kernel modules on a single node containing a
+Ceph  daemon may cause a deadlock due to issues with the Linux kernel itself
+(unless you use VMs for the clients). You can experiment with Ceph in a 1-node
+configuration, in spite of the limitations as described herein.
+
+If you are trying to create a cluster on a single node, you must change the
+default of the ``osd crush chooseleaf type`` setting from ``1`` (meaning
+``host`` or ``node``) to ``0`` (meaning ``osd``) in your Ceph configuration
+file before you create your monitors and OSDs. This tells Ceph that an OSD
+can peer with another OSD on the same host. If you are trying to set up a
+1-node cluster and ``osd crush chooseleaf type`` is greater than ``0``,
+Ceph will try to peer the PGs of one OSD with the PGs of another OSD on
+another node, chassis, rack, row, or even datacenter depending on the setting.
+
+.. tip:: DO NOT mount kernel clients directly on the same node as your
+   Ceph Storage Cluster, because kernel conflicts can arise. However, you
+   can mount kernel clients within virtual machines (VMs) on a single node.
+
+If you are creating OSDs using a single disk, you must create directories
+for the data manually first.
+
+
+Fewer OSDs than Replicas
+------------------------
+
+If you have brought up two OSDs to an ``up`` and ``in`` state, but you still
+don't see ``active + clean`` placement groups, you may have an
+``osd pool default size`` set to greater than ``2``.
+
+There are a few ways to address this situation. If you want to operate your
+cluster in an ``active + degraded`` state with two replicas, you can set the
+``osd pool default min size`` to ``2`` so that you can write objects in
+an ``active + degraded`` state. You may also set the ``osd pool default size``
+setting to ``2`` so that you only have two stored replicas (the original and
+one replica), in which case the cluster should achieve an ``active + clean``
+state.
+
+.. note:: You can make the changes at runtime. If you make the changes in
+   your Ceph configuration file, you may need to restart your cluster.
+
+
+Pool Size = 1
+-------------
+
+If you have the ``osd pool default size`` set to ``1``, you will only have
+one copy of the object. OSDs rely on other OSDs to tell them which objects
+they should have. If a first OSD has a copy of an object and there is no
+second copy, then no second OSD can tell the first OSD that it should have
+that copy. For each placement group mapped to the first OSD (see
+``ceph pg dump``), you can force the first OSD to notice the placement groups
+it needs by running::
+
+   	ceph osd force-create-pg <pgid>
+
+
+CRUSH Map Errors
+----------------
+
+Another candidate for placement groups remaining unclean involves errors
+in your CRUSH map.
+
+
+Stuck Placement Groups
+======================
+
+It is normal for placement groups to enter states like "degraded" or "peering"
+following a failure.  Normally these states indicate the normal progression
+through the failure recovery process. However, if a placement group stays in one
+of these states for a long time this may be an indication of a larger problem.
+For this reason, the monitor will warn when placement groups get "stuck" in a
+non-optimal state.  Specifically, we check for:
+
+* ``inactive`` - The placement group has not been ``active`` for too long
+  (i.e., it hasn't been able to service read/write requests).
+
+* ``unclean`` - The placement group has not been ``clean`` for too long
+  (i.e., it hasn't been able to completely recover from a previous failure).
+
+* ``stale`` - The placement group status has not been updated by a ``ceph-osd``,
+  indicating that all nodes storing this placement group may be ``down``.
+
+You can explicitly list stuck placement groups with one of::
+
+	ceph pg dump_stuck stale
+	ceph pg dump_stuck inactive
+	ceph pg dump_stuck unclean
+
+For stuck ``stale`` placement groups, it is normally a matter of getting the
+right ``ceph-osd`` daemons running again.  For stuck ``inactive`` placement
+groups, it is usually a peering problem (see :ref:`failures-osd-peering`).  For
+stuck ``unclean`` placement groups, there is usually something preventing
+recovery from completing, like unfound objects (see
+:ref:`failures-osd-unfound`);
+
+
+
+.. _failures-osd-peering:
+
+Placement Group Down - Peering Failure
+======================================
+
+In certain cases, the ``ceph-osd`` `Peering` process can run into
+problems, preventing a PG from becoming active and usable.  For
+example, ``ceph health`` might report::
+
+	ceph health detail
+	HEALTH_ERR 7 pgs degraded; 12 pgs down; 12 pgs peering; 1 pgs recovering; 6 pgs stuck unclean; 114/3300 degraded (3.455%); 1/3 in osds are down
+	...
+	pg 0.5 is down+peering
+	pg 1.4 is down+peering
+	...
+	osd.1 is down since epoch 69, last address 192.168.106.220:6801/8651
+
+We can query the cluster to determine exactly why the PG is marked ``down`` with::
+
+	ceph pg 0.5 query
+
+.. code-block:: javascript
+
+ { "state": "down+peering",
+   ...
+   "recovery_state": [
+        { "name": "Started\/Primary\/Peering\/GetInfo",
+          "enter_time": "2012-03-06 14:40:16.169679",
+          "requested_info_from": []},
+        { "name": "Started\/Primary\/Peering",
+          "enter_time": "2012-03-06 14:40:16.169659",
+          "probing_osds": [
+                0,
+                1],
+          "blocked": "peering is blocked due to down osds",
+          "down_osds_we_would_probe": [
+                1],
+          "peering_blocked_by": [
+                { "osd": 1,
+                  "current_lost_at": 0,
+                  "comment": "starting or marking this osd lost may let us proceed"}]},
+        { "name": "Started",
+          "enter_time": "2012-03-06 14:40:16.169513"}
+    ]
+ }
+
+The ``recovery_state`` section tells us that peering is blocked due to
+down ``ceph-osd`` daemons, specifically ``osd.1``.  In this case, we can start that ``ceph-osd``
+and things will recover.
+
+Alternatively, if there is a catastrophic failure of ``osd.1`` (e.g., disk
+failure), we can tell the cluster that it is ``lost`` and to cope as
+best it can.
+
+.. important:: This is dangerous in that the cluster cannot
+   guarantee that the other copies of the data are consistent
+   and up to date.
+
+To instruct Ceph to continue anyway::
+
+	ceph osd lost 1
+
+Recovery will proceed.
+
+
+.. _failures-osd-unfound:
+
+Unfound Objects
+===============
+
+Under certain combinations of failures Ceph may complain about
+``unfound`` objects::
+
+	ceph health detail
+	HEALTH_WARN 1 pgs degraded; 78/3778 unfound (2.065%)
+	pg 2.4 is active+degraded, 78 unfound
+
+This means that the storage cluster knows that some objects (or newer
+copies of existing objects) exist, but it hasn't found copies of them.
+One example of how this might come about for a PG whose data is on ceph-osds
+1 and 2:
+
+* 1 goes down
+* 2 handles some writes, alone
+* 1 comes up
+* 1 and 2 repeer, and the objects missing on 1 are queued for recovery.
+* Before the new objects are copied, 2 goes down.
+
+Now 1 knows that these object exist, but there is no live ``ceph-osd`` who
+has a copy.  In this case, IO to those objects will block, and the
+cluster will hope that the failed node comes back soon; this is
+assumed to be preferable to returning an IO error to the user.
+
+First, you can identify which objects are unfound with::
+
+	ceph pg 2.4 list_unfound [starting offset, in json]
+
+.. code-block:: javascript
+
+  {
+    "num_missing": 1,
+    "num_unfound": 1,
+    "objects": [
+        {
+            "oid": {
+                "oid": "object",
+                "key": "",
+                "snapid": -2,
+                "hash": 2249616407,
+                "max": 0,
+                "pool": 2,
+                "namespace": ""
+            },
+            "need": "43'251",
+            "have": "0'0",
+            "flags": "none",
+            "clean_regions": "clean_offsets: [], clean_omap: 0, new_object: 1",
+            "locations": [
+                "0(3)",
+                "4(2)"
+            ]
+        }
+    ],
+    "state": "NotRecovering",
+    "available_might_have_unfound": true,
+    "might_have_unfound": [
+        {
+            "osd": "2(4)",
+            "status": "osd is down"
+        }
+    ],
+    "more": false
+  }
+
+If there are too many objects to list in a single result, the ``more``
+field will be true and you can query for more.  (Eventually the
+command line tool will hide this from you, but not yet.)
+
+Second, you can identify which OSDs have been probed or might contain
+data.
+
+At the end of the listing (before ``more`` is false), ``might_have_unfound`` is provided
+when ``available_might_have_unfound`` is true.  This is equivalent to the output
+of ``ceph pg #.# query``.  This eliminates the need to use ``query`` directly.
+The ``might_have_unfound`` information given behaves the same way as described below for ``query``.
+The only difference is that OSDs that have ``already probed`` status are ignored.
+
+Use of ``query``::
+
+	ceph pg 2.4 query
+
+.. code-block:: javascript
+
+   "recovery_state": [
+        { "name": "Started\/Primary\/Active",
+          "enter_time": "2012-03-06 15:15:46.713212",
+          "might_have_unfound": [
+                { "osd": 1,
+                  "status": "osd is down"}]},
+
+In this case, for example, the cluster knows that ``osd.1`` might have
+data, but it is ``down``.  The full range of possible states include:
+
+* already probed
+* querying
+* OSD is down
+* not queried (yet)
+
+Sometimes it simply takes some time for the cluster to query possible
+locations.
+
+It is possible that there are other locations where the object can
+exist that are not listed.  For example, if a ceph-osd is stopped and
+taken out of the cluster, the cluster fully recovers, and due to some
+future set of failures ends up with an unfound object, it won't
+consider the long-departed ceph-osd as a potential location to
+consider.  (This scenario, however, is unlikely.)
+
+If all possible locations have been queried and objects are still
+lost, you may have to give up on the lost objects. This, again, is
+possible given unusual combinations of failures that allow the cluster
+to learn about writes that were performed before the writes themselves
+are recovered.  To mark the "unfound" objects as "lost"::
+
+	ceph pg 2.5 mark_unfound_lost revert|delete
+
+This the final argument specifies how the cluster should deal with
+lost objects.
+
+The "delete" option will forget about them entirely.
+
+The "revert" option (not available for erasure coded pools) will
+either roll back to a previous version of the object or (if it was a
+new object) forget about it entirely.  Use this with caution, as it
+may confuse applications that expected the object to exist.
+
+
+Homeless Placement Groups
+=========================
+
+It is possible for all OSDs that had copies of a given placement groups to fail.
+If that's the case, that subset of the object store is unavailable, and the
+monitor will receive no status updates for those placement groups.  To detect
+this situation, the monitor marks any placement group whose primary OSD has
+failed as ``stale``.  For example::
+
+	ceph health
+	HEALTH_WARN 24 pgs stale; 3/300 in osds are down
+
+You can identify which placement groups are ``stale``, and what the last OSDs to
+store them were, with::
+
+	ceph health detail
+	HEALTH_WARN 24 pgs stale; 3/300 in osds are down
+	...
+	pg 2.5 is stuck stale+active+remapped, last acting [2,0]
+	...
+	osd.10 is down since epoch 23, last address 192.168.106.220:6800/11080
+	osd.11 is down since epoch 13, last address 192.168.106.220:6803/11539
+	osd.12 is down since epoch 24, last address 192.168.106.220:6806/11861
+
+If we want to get placement group 2.5 back online, for example, this tells us that
+it was last managed by ``osd.0`` and ``osd.2``.  Restarting those ``ceph-osd``
+daemons will allow the cluster to recover that placement group (and, presumably,
+many others).
+
+
+Only a Few OSDs Receive Data
+============================
+
+If you have many nodes in your cluster and only a few of them receive data,
+`check`_ the number of placement groups in your pool. Since placement groups get
+mapped to OSDs, a small number of placement groups will not distribute across
+your cluster. Try creating a pool with a placement group count that is a
+multiple of the number of OSDs. See `Placement Groups`_ for details. The default
+placement group count for pools is not useful, but you can change it `here`_.
+
+
+Can't Write Data
+================
+
+If your cluster is up, but some OSDs are down and you cannot write data,
+check to ensure that you have the minimum number of OSDs running for the
+placement group. If you don't have the minimum number of OSDs running,
+Ceph will not allow you to write data because there is no guarantee
+that Ceph can replicate your data. See ``osd pool default min size``
+in the `Pool, PG and CRUSH Config Reference`_ for details.
+
+
+PGs Inconsistent
+================
+
+If you receive an ``active + clean + inconsistent`` state, this may happen
+due to an error during scrubbing. As always, we can identify the inconsistent
+placement group(s) with::
+
+    $ ceph health detail
+    HEALTH_ERR 1 pgs inconsistent; 2 scrub errors
+    pg 0.6 is active+clean+inconsistent, acting [0,1,2]
+    2 scrub errors
+
+Or if you prefer inspecting the output in a programmatic way::
+
+    $ rados list-inconsistent-pg rbd
+    ["0.6"]
+
+There is only one consistent state, but in the worst case, we could have
+different inconsistencies in multiple perspectives found in more than one
+objects. If an object named ``foo`` in PG ``0.6`` is truncated, we will have::
+
+    $ rados list-inconsistent-obj 0.6 --format=json-pretty
+
+.. code-block:: javascript
+
+    {
+        "epoch": 14,
+        "inconsistents": [
+            {
+                "object": {
+                    "name": "foo",
+                    "nspace": "",
+                    "locator": "",
+                    "snap": "head",
+                    "version": 1
+                },
+                "errors": [
+                    "data_digest_mismatch",
+                    "size_mismatch"
+                ],
+                "union_shard_errors": [
+                    "data_digest_mismatch_info",
+                    "size_mismatch_info"
+                ],
+                "selected_object_info": "0:602f83fe:::foo:head(16'1 client.4110.0:1 dirty|data_digest|omap_digest s 968 uv 1 dd e978e67f od ffffffff alloc_hint [0 0 0])",
+                "shards": [
+                    {
+                        "osd": 0,
+                        "errors": [],
+                        "size": 968,
+                        "omap_digest": "0xffffffff",
+                        "data_digest": "0xe978e67f"
+                    },
+                    {
+                        "osd": 1,
+                        "errors": [],
+                        "size": 968,
+                        "omap_digest": "0xffffffff",
+                        "data_digest": "0xe978e67f"
+                    },
+                    {
+                        "osd": 2,
+                        "errors": [
+                            "data_digest_mismatch_info",
+                            "size_mismatch_info"
+                        ],
+                        "size": 0,
+                        "omap_digest": "0xffffffff",
+                        "data_digest": "0xffffffff"
+                    }
+                ]
+            }
+        ]
+    }
+
+In this case, we can learn from the output:
+
+* The only inconsistent object is named ``foo``, and it is its head that has
+  inconsistencies.
+* The inconsistencies fall into two categories:
+
+  * ``errors``: these errors indicate inconsistencies between shards without a
+    determination of which shard(s) are bad. Check for the ``errors`` in the
+    `shards` array, if available, to pinpoint the problem.
+
+    * ``data_digest_mismatch``: the digest of the replica read from OSD.2 is
+      different from the ones of OSD.0 and OSD.1
+    * ``size_mismatch``: the size of the replica read from OSD.2 is 0, while
+      the size reported by OSD.0 and OSD.1 is 968.
+  * ``union_shard_errors``: the union of all shard specific ``errors`` in
+    ``shards`` array. The ``errors`` are set for the given shard that has the
+    problem. They include errors like ``read_error``. The ``errors`` ending in
+    ``oi`` indicate a comparison with ``selected_object_info``. Look at the
+    ``shards`` array to determine which shard has which error(s).
+
+    * ``data_digest_mismatch_info``: the digest stored in the object-info is not
+      ``0xffffffff``, which is calculated from the shard read from OSD.2
+    * ``size_mismatch_info``: the size stored in the object-info is different
+      from the one read from OSD.2. The latter is 0.
+
+You can repair the inconsistent placement group by executing::
+
+	ceph pg repair {placement-group-ID}
+
+Which overwrites the `bad` copies with the `authoritative` ones. In most cases,
+Ceph is able to choose authoritative copies from all available replicas using
+some predefined criteria. But this does not always work. For example, the stored
+data digest could be missing, and the calculated digest will be ignored when
+choosing the authoritative copies. So, please use the above command with caution.
+
+If ``read_error`` is listed in the ``errors`` attribute of a shard, the
+inconsistency is likely due to disk errors. You might want to check your disk
+used by that OSD.
+
+If you receive ``active + clean + inconsistent`` states periodically due to
+clock skew, you may consider configuring your `NTP`_ daemons on your
+monitor hosts to act as peers. See `The Network Time Protocol`_ and Ceph
+`Clock Settings`_ for additional details.
+
+
+Erasure Coded PGs are not active+clean
+======================================
+
+When CRUSH fails to find enough OSDs to map to a PG, it will show as a
+``2147483647`` which is ITEM_NONE or ``no OSD found``. For instance::
+
+     [2,1,6,0,5,8,2147483647,7,4]
+
+Not enough OSDs
+---------------
+
+If the Ceph cluster only has 8 OSDs and the erasure coded pool needs
+9, that is what it will show. You can either create another erasure
+coded pool that requires less OSDs::
+
+     ceph osd erasure-code-profile set myprofile k=5 m=3
+     ceph osd pool create erasurepool erasure myprofile
+
+or add a new OSDs and the PG will automatically use them.
+
+CRUSH constraints cannot be satisfied
+-------------------------------------
+
+If the cluster has enough OSDs, it is possible that the CRUSH rule
+imposes constraints that cannot be satisfied. If there are 10 OSDs on
+two hosts and the CRUSH rule requires that no two OSDs from the
+same host are used in the same PG, the mapping may fail because only
+two OSDs will be found. You can check the constraint by displaying ("dumping")
+the rule::
+
+    $ ceph osd crush rule ls
+    [
+        "replicated_rule",
+        "erasurepool"]
+    $ ceph osd crush rule dump erasurepool
+    { "rule_id": 1,
+      "rule_name": "erasurepool",
+      "ruleset": 1,
+      "type": 3,
+      "min_size": 3,
+      "max_size": 20,
+      "steps": [
+            { "op": "take",
+              "item": -1,
+              "item_name": "default"},
+            { "op": "chooseleaf_indep",
+              "num": 0,
+              "type": "host"},
+            { "op": "emit"}]}
+
+
+You can resolve the problem by creating a new pool in which PGs are allowed
+to have OSDs residing on the same host with::
+
+     ceph osd erasure-code-profile set myprofile crush-failure-domain=osd
+     ceph osd pool create erasurepool erasure myprofile
+
+CRUSH gives up too soon
+-----------------------
+
+If the Ceph cluster has just enough OSDs to map the PG (for instance a
+cluster with a total of 9 OSDs and an erasure coded pool that requires
+9 OSDs per PG), it is possible that CRUSH gives up before finding a
+mapping. It can be resolved by:
+
+* lowering the erasure coded pool requirements to use less OSDs per PG
+  (that requires the creation of another pool as erasure code profiles
+  cannot be dynamically modified).
+
+* adding more OSDs to the cluster (that does not require the erasure
+  coded pool to be modified, it will become clean automatically)
+
+* use a handmade CRUSH rule that tries more times to find a good
+  mapping. This can be done by setting ``set_choose_tries`` to a value
+  greater than the default.
+
+You should first verify the problem with ``crushtool`` after
+extracting the crushmap from the cluster so your experiments do not
+modify the Ceph cluster and only work on a local files::
+
+    $ ceph osd crush rule dump erasurepool
+    { "rule_name": "erasurepool",
+      "ruleset": 1,
+      "type": 3,
+      "min_size": 3,
+      "max_size": 20,
+      "steps": [
+            { "op": "take",
+              "item": -1,
+              "item_name": "default"},
+            { "op": "chooseleaf_indep",
+              "num": 0,
+              "type": "host"},
+            { "op": "emit"}]}
+    $ ceph osd getcrushmap > crush.map
+    got crush map from osdmap epoch 13
+    $ crushtool -i crush.map --test --show-bad-mappings \
+       --rule 1 \
+       --num-rep 9 \
+       --min-x 1 --max-x $((1024 * 1024))
+    bad mapping rule 8 x 43 num_rep 9 result [3,2,7,1,2147483647,8,5,6,0]
+    bad mapping rule 8 x 79 num_rep 9 result [6,0,2,1,4,7,2147483647,5,8]
+    bad mapping rule 8 x 173 num_rep 9 result [0,4,6,8,2,1,3,7,2147483647]
+
+Where ``--num-rep`` is the number of OSDs the erasure code CRUSH
+rule needs, ``--rule`` is the value of the ``ruleset`` field
+displayed by ``ceph osd crush rule dump``.  The test will try mapping
+one million values (i.e. the range defined by ``[--min-x,--max-x]``)
+and must display at least one bad mapping. If it outputs nothing it
+means all mappings are successful and you can stop right there: the
+problem is elsewhere.
+
+The CRUSH rule can be edited by decompiling the crush map::
+
+    $ crushtool --decompile crush.map > crush.txt
+
+and adding the following line to the rule::
+
+    step set_choose_tries 100
+
+The relevant part of the ``crush.txt`` file should look something
+like::
+
+     rule erasurepool {
+             ruleset 1
+             type erasure
+             min_size 3
+             max_size 20
+             step set_chooseleaf_tries 5
+             step set_choose_tries 100
+             step take default
+             step chooseleaf indep 0 type host
+             step emit
+     }
+
+It can then be compiled and tested again::
+
+    $ crushtool --compile crush.txt -o better-crush.map
+
+When all mappings succeed, an histogram of the number of tries that
+were necessary to find all of them can be displayed with the
+``--show-choose-tries`` option of ``crushtool``::
+
+    $ crushtool -i better-crush.map --test --show-bad-mappings \
+       --show-choose-tries \
+       --rule 1 \
+       --num-rep 9 \
+       --min-x 1 --max-x $((1024 * 1024))
+    ...
+    11:        42
+    12:        44
+    13:        54
+    14:        45
+    15:        35
+    16:        34
+    17:        30
+    18:        25
+    19:        19
+    20:        22
+    21:        20
+    22:        17
+    23:        13
+    24:        16
+    25:        13
+    26:        11
+    27:        11
+    28:        13
+    29:        11
+    30:        10
+    31:         6
+    32:         5
+    33:        10
+    34:         3
+    35:         7
+    36:         5
+    37:         2
+    38:         5
+    39:         5
+    40:         2
+    41:         5
+    42:         4
+    43:         1
+    44:         2
+    45:         2
+    46:         3
+    47:         1
+    48:         0
+    ...
+    102:         0
+    103:         1
+    104:         0
+    ...
+
+It took 11 tries to map 42 PGs, 12 tries to map 44 PGs etc. The highest number of tries is the minimum value of ``set_choose_tries`` that prevents bad mappings (i.e. 103 in the above output because it did not take more than 103 tries for any PG to be mapped).
+
+.. _check: ../../operations/placement-groups#get-the-number-of-placement-groups
+.. _here: ../../configuration/pool-pg-config-ref
+.. _Placement Groups: ../../operations/placement-groups
+.. _Pool, PG and CRUSH Config Reference: ../../configuration/pool-pg-config-ref
+.. _NTP: https://en.wikipedia.org/wiki/Network_Time_Protocol
+.. _The Network Time Protocol: http://www.ntp.org/
+.. _Clock Settings: ../../configuration/mon-config-ref/#clock
+
+
diff --git a/doc/radosgw/STS.rst b/doc/radosgw/STS.rst
new file mode 100644
index 000000000..1678f86d5
--- /dev/null
+++ b/doc/radosgw/STS.rst
@@ -0,0 +1,298 @@
+===========
+STS in Ceph
+===========
+
+Secure Token Service is a web service in AWS that returns a set of temporary security credentials for authenticating federated users.
+The link to official AWS documentation can be found here: https://docs.aws.amazon.com/STS/latest/APIReference/Welcome.html.
+
+Ceph Object Gateway implements a subset of STS APIs that provide temporary credentials for identity and access management.
+These temporary credentials can be used to make subsequent S3 calls which will be authenticated by the STS engine in Ceph Object Gateway.
+Permissions of the temporary credentials can be further restricted via an IAM policy passed as a parameter to the STS APIs.
+
+STS REST APIs
+=============
+
+The following STS REST APIs have been implemented in Ceph Object Gateway:
+
+1. AssumeRole: Returns a set of temporary credentials that can be used for 
+cross-account access. The temporary credentials will have permissions that are
+allowed by both - permission policies attached with the Role and policy attached
+with the AssumeRole API.
+
+Parameters:
+    **RoleArn** (String/ Required): ARN of the Role to Assume.
+
+    **RoleSessionName** (String/ Required): An Identifier for the assumed role
+    session.
+
+    **Policy** (String/ Optional): An IAM Policy in JSON format.
+
+    **DurationSeconds** (Integer/ Optional): The duration in seconds of the session.
+    Its default value is 3600.
+
+    **ExternalId** (String/ Optional): A unique Id that might be used when a role is
+    assumed in another account.
+
+    **SerialNumber** (String/ Optional): The Id number of the MFA device associated
+    with the user making the AssumeRole call.
+
+    **TokenCode** (String/ Optional): The value provided by the MFA device, if the
+    trust policy of the role being assumed requires MFA.
+
+2. AssumeRoleWithWebIdentity: Returns a set of temporary credentials for users that
+have been authenticated by a web/mobile app by an OpenID Connect /OAuth2.0 Identity Provider.
+Currently Keycloak has been tested and integrated with RGW.
+
+Parameters:
+    **RoleArn** (String/ Required): ARN of the Role to Assume.
+
+    **RoleSessionName** (String/ Required): An Identifier for the assumed role
+    session.
+
+    **Policy** (String/ Optional): An IAM Policy in JSON format.
+
+    **DurationSeconds** (Integer/ Optional): The duration in seconds of the session.
+    Its default value is 3600.
+
+    **ProviderId** (String/ Optional): Fully qualified host component of the domain name
+    of the IDP. Valid only for OAuth2.0 tokens (not for OpenID Connect tokens).
+
+    **WebIdentityToken** (String/ Required): The OpenID Connect/ OAuth2.0 token, which the
+    application gets in return after authenticating its user with an IDP.
+
+Before invoking AssumeRoleWithWebIdentity, an OpenID Connect Provider entity (which the web application
+authenticates with), needs to be created in RGW.
+
+The trust between the IDP and the role is created by adding a condition to the role's trust policy, which
+allows access only to applications which satisfy the given condition.
+All claims of the JWT are supported in the condition of the role's trust policy.
+An example of a policy that uses the 'aud' claim in the condition is of the form::
+
+    "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Federated\":[\"arn:aws:iam:::oidc-provider/<URL of IDP>\"]},\"Action\":[\"sts:AssumeRoleWithWebIdentity\"],\"Condition\":{\"StringEquals\":{\"<URL of IDP> :app_id\":\"<aud>\"\}\}\}\]\}"
+
+The app_id in the condition above must match the 'aud' claim of the incoming token.
+
+An example of a policy that uses the 'sub' claim in the condition is of the form::
+
+    "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Federated\":[\"arn:aws:iam:::oidc-provider/<URL of IDP>\"]},\"Action\":[\"sts:AssumeRoleWithWebIdentity\"],\"Condition\":{\"StringEquals\":{\"<URL of IDP> :sub\":\"<sub>\"\}\}\}\]\}"
+
+Similarly, an example of a policy that uses 'azp' claim in the condition is of the form::
+
+    "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Federated\":[\"arn:aws:iam:::oidc-provider/<URL of IDP>\"]},\"Action\":[\"sts:AssumeRoleWithWebIdentity\"],\"Condition\":{\"StringEquals\":{\"<URL of IDP> :azp\":\"<azp>\"\}\}\}\]\}"
+
+A shadow user is created corresponding to every federated user. The user id is derived from the 'sub' field of the incoming web token.
+The user is created in a separate namespace - 'oidc' such that the user id doesn't clash with any other user ids in rgw. The format of the user id
+is - <tenant>$<user-namespace>$<sub> where user-namespace is 'oidc' for users that authenticate with oidc providers.
+
+RGW now supports Session tags that can be passed in the web token to AssumeRoleWithWebIdentity call. More information related to Session Tags can be found here
+:doc:`session-tags`.
+
+STS Configuration
+=================
+
+The following configurable options have to be added for STS integration::
+
+  [client.radosgw.gateway]
+  rgw sts key = {sts key for encrypting the session token}
+  rgw s3 auth use sts = true
+
+Note: By default, STS and S3 APIs co-exist in the same namespace, and both S3
+and STS APIs can be accessed via the same endpoint in Ceph Object Gateway.
+
+Examples
+========
+
+1. The following is an example of AssumeRole API call, which shows steps to create a role, assign a policy to it
+(that allows access to S3 resources), assuming a role to get temporary credentials and accessing s3 resources using
+those credentials. In this example, TESTER1 assumes a role created by TESTER, to access S3 resources owned by TESTER,
+according to the permission policy attached to the role.
+
+.. code-block:: console
+
+   radosgw-admin caps add --uid="TESTER" --caps="roles=*"
+
+2. The following is an example of the AssumeRole API call, which shows steps to create a role, assign a policy to it
+   (that allows access to S3 resources), assuming a role to get temporary credentials and accessing S3 resources using
+   those credentials. In this example, TESTER1 assumes a role created by TESTER, to access S3 resources owned by TESTER,
+   according to the permission policy attached to the role.
+
+.. code-block:: python
+
+    import boto3
+
+    iam_client = boto3.client('iam',
+    aws_access_key_id=<access_key of TESTER>,
+    aws_secret_access_key=<secret_key of TESTER>,
+    endpoint_url=<IAM URL>,
+    region_name=''
+    )
+
+    policy_document = "{\"Version\":\"2012-10-17\",\"Statement\":{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam:::user/TESTER1\"]},\"Action\":[\"sts:AssumeRole\"]}]}"
+
+    role_response = iam_client.create_role(
+    AssumeRolePolicyDocument=policy_document,
+    Path='/',
+    RoleName='S3Access',
+    )
+
+    role_policy = "{\"Version\":\"2012-10-17\",\"Statement\":{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"arn:aws:s3:::*\"}}"
+
+    response = iam_client.put_role_policy(
+    RoleName='S3Access',
+    PolicyName='Policy1',
+    PolicyDocument=role_policy
+    )
+
+    sts_client = boto3.client('sts',
+    aws_access_key_id=<access_key of TESTER1>,
+    aws_secret_access_key=<secret_key of TESTER1>,
+    endpoint_url=<STS URL>,
+    region_name='',
+    )
+
+    response = sts_client.assume_role(
+    RoleArn=role_response['Role']['Arn'],
+    RoleSessionName='Bob',
+    DurationSeconds=3600
+    )
+
+    s3client = boto3.client('s3',
+    aws_access_key_id = response['Credentials']['AccessKeyId'],
+    aws_secret_access_key = response['Credentials']['SecretAccessKey'],
+    aws_session_token = response['Credentials']['SessionToken'],
+    endpoint_url=<S3 URL>,
+    region_name='',)
+
+    bucket_name = 'my-bucket'
+    s3bucket = s3client.create_bucket(Bucket=bucket_name)
+    resp = s3client.list_buckets()
+
+2. The following is an example of AssumeRoleWithWebIdentity API call, where an external app that has users authenticated with
+an OpenID Connect/ OAuth2 IDP (Keycloak in this example), assumes a role to get back temporary credentials and access S3 resources
+according to permission policy of the role.
+
+.. code-block:: python
+
+    import boto3
+
+    iam_client = boto3.client('iam',
+    aws_access_key_id=<access_key of TESTER>,
+    aws_secret_access_key=<secret_key of TESTER>,
+    endpoint_url=<IAM URL>,
+    region_name=''
+    )
+
+    oidc_response = iam_client.create_open_id_connect_provider(
+        Url=<URL of the OpenID Connect Provider,
+        ClientIDList=[
+            <Client id registered with the IDP>
+        ],
+        ThumbprintList=[
+            <Thumbprint of the IDP>
+     ]
+    )
+
+    policy_document = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Federated\":[\"arn:aws:iam:::oidc-provider/localhost:8080/auth/realms/demo\"]},\"Action\":[\"sts:AssumeRoleWithWebIdentity\"],\"Condition\":{\"StringEquals\":{\"localhost:8080/auth/realms/demo:app_id\":\"customer-portal\"}}}]}"
+    role_response = iam_client.create_role(
+    AssumeRolePolicyDocument=policy_document,
+    Path='/',
+    RoleName='S3Access',
+    )
+
+    role_policy = "{\"Version\":\"2012-10-17\",\"Statement\":{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"arn:aws:s3:::*\"}}"
+
+    response = iam_client.put_role_policy(
+        RoleName='S3Access',
+        PolicyName='Policy1',
+        PolicyDocument=role_policy
+    )
+
+    sts_client = boto3.client('sts',
+    aws_access_key_id=<access_key of TESTER1>,
+    aws_secret_access_key=<secret_key of TESTER1>,
+    endpoint_url=<STS URL>,
+    region_name='',
+    )
+
+    response = client.assume_role_with_web_identity(
+    RoleArn=role_response['Role']['Arn'],
+    RoleSessionName='Bob',
+    DurationSeconds=3600,
+    WebIdentityToken=<Web Token>
+    )
+
+    s3client = boto3.client('s3',
+    aws_access_key_id = response['Credentials']['AccessKeyId'],
+    aws_secret_access_key = response['Credentials']['SecretAccessKey'],
+    aws_session_token = response['Credentials']['SessionToken'],
+    endpoint_url=<S3 URL>,
+    region_name='',)
+
+    bucket_name = 'my-bucket'
+    s3bucket = s3client.create_bucket(Bucket=bucket_name)
+    resp = s3client.list_buckets()
+
+How to obtain thumbprint of an OpenID Connect Provider IDP
+==========================================================
+1. Take the OpenID connect provider's URL and add /.well-known/openid-configuration
+to it to get the URL to get the IDP's configuration document. For example, if the URL
+of the IDP is http://localhost:8000/auth/realms/quickstart, then the URL to get the
+document from is http://localhost:8000/auth/realms/quickstart/.well-known/openid-configuration
+
+2. Use the following curl command to get the configuration document from the URL described
+in step 1::
+
+    curl -k -v \
+      -X GET \
+      -H "Content-Type: application/x-www-form-urlencoded" \
+      "http://localhost:8000/auth/realms/quickstart/.well-known/openid-configuration" \
+    | jq .
+
+ 3. From the response of step 2, use the value of "jwks_uri" to get the certificate of the IDP,
+ using the following code::
+     curl -k -v \
+      -X GET \
+      -H "Content-Type: application/x-www-form-urlencoded" \
+      "http://$KC_SERVER/$KC_CONTEXT/realms/$KC_REALM/protocol/openid-connect/certs" \
+      | jq .
+
+3. Copy the result of "x5c" in the response above, in a file certificate.crt, and add
+'-----BEGIN CERTIFICATE-----' at the beginning and "-----END CERTIFICATE-----"
+at the end.
+
+4. Use the following OpenSSL command to get the certificate thumbprint::
+
+    openssl x509 -in certificate.crt -fingerprint -noout
+
+5. The result of the above command in step 4, will be a SHA1 fingerprint, like the following::
+
+    SHA1 Fingerprint=F7:D7:B3:51:5D:D0:D3:19:DD:21:9A:43:A9:EA:72:7A:D6:06:52:87
+
+6.  Remove the colons from the result above to get the final thumbprint which can be as input
+while creating the OpenID Connect Provider entity in IAM::
+
+    F7D7B3515DD0D319DD219A43A9EA727AD6065287
+
+Roles in RGW
+============
+
+More information for role manipulation can be found here
+:doc:`role`.
+
+OpenID Connect Provider in RGW
+==============================
+
+More information for OpenID Connect Provider entity manipulation
+can be found here
+:doc:`oidc`.
+
+Keycloak integration with Radosgw
+=================================
+
+Steps for integrating Radosgw with Keycloak can be found here
+:doc:`keycloak`.
+
+STSLite
+=======
+STSLite has been built on STS, and documentation for the same can be found here
+:doc:`STSLite`.
diff --git a/doc/radosgw/STSLite.rst b/doc/radosgw/STSLite.rst
new file mode 100644
index 000000000..63164e48d
--- /dev/null
+++ b/doc/radosgw/STSLite.rst
@@ -0,0 +1,196 @@
+=========
+STS Lite
+=========
+
+Ceph Object Gateway provides support for a subset of Amazon Secure Token Service
+(STS) APIs. STS Lite is an extension of STS and builds upon one of its APIs to
+decrease the load on external IDPs like Keystone and LDAP.
+
+A set of temporary security credentials is returned after authenticating
+a set of AWS credentials with the external IDP. These temporary credentials can be used
+to make subsequent S3 calls which will be authenticated by the STS engine in Ceph,
+resulting in less load on the Keystone/ LDAP server.
+
+Temporary and limited privileged credentials can be obtained for a local user
+also using the STS Lite API.
+
+STS Lite REST APIs
+==================
+
+The following STS Lite REST API is part of STS Lite in Ceph Object Gateway:
+
+1. GetSessionToken: Returns a set of temporary credentials for a set of AWS
+credentials. After initial authentication with Keystone/ LDAP, the temporary
+credentials returned can be used to make subsequent S3 calls. The temporary
+credentials will have the same permission as that of the AWS credentials.
+
+Parameters:
+    **DurationSeconds** (Integer/ Optional): The duration in seconds for which the
+    credentials should remain valid. Its default value is 3600. Its default max
+    value is 43200 which is can be configured using rgw sts max session duration.
+
+    **SerialNumber** (String/ Optional): The Id number of the MFA device associated 
+    with the user making the GetSessionToken call.
+
+    **TokenCode** (String/ Optional): The value provided by the MFA device, if MFA is required.
+
+An administrative user needs to attach a policy to allow invocation of GetSessionToken API using its permanent
+credentials and to allow subsequent S3 operations invocation using only the temporary credentials returned
+by GetSessionToken.
+
+The user attaching the policy needs to have admin caps. For example::
+
+    radosgw-admin caps add --uid="TESTER" --caps="user-policy=*"
+
+The following is the policy that needs to be attached to a user 'TESTER1'::
+
+    user_policy = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Deny\",\"Action\":\"s3:*\",\"Resource\":[\"*\"],\"Condition\":{\"BoolIfExists\":{\"sts:authentication\":\"false\"}}},{\"Effect\":\"Allow\",\"Action\":\"sts:GetSessionToken\",\"Resource\":\"*\",\"Condition\":{\"BoolIfExists\":{\"sts:authentication\":\"false\"}}}]}"
+
+
+STS Lite Configuration
+======================
+
+The following configurable options are available for STS Lite integration::
+
+  [client.radosgw.gateway]
+  rgw sts key = {sts key for encrypting the session token}
+  rgw s3 auth use sts = true
+
+The above STS configurables can be used with the Keystone configurables if one
+needs to use STS Lite in conjunction with Keystone. The complete set of
+configurable options will be::
+
+  [client.radosgw.gateway]
+  rgw sts key = {sts key for encrypting/ decrypting the session token}
+  rgw s3 auth use sts = true
+
+  rgw keystone url = {keystone server url:keystone server admin port}
+  rgw keystone admin project = {keystone admin project name}
+  rgw keystone admin tenant = {keystone service tenant name}
+  rgw keystone admin domain = {keystone admin domain name}
+  rgw keystone api version = {keystone api version}
+  rgw keystone implicit tenants = {true for private tenant for each new user}
+  rgw keystone admin password = {keystone service tenant user name}
+  rgw keystone admin user = keystone service tenant user password}
+  rgw keystone accepted roles = {accepted user roles}
+  rgw keystone token cache size = {number of tokens to cache}
+  rgw s3 auth use keystone = true
+
+The details of the integrating ldap with Ceph Object Gateway can be found here:
+:doc:`keystone`
+
+The complete set of configurables to use STS Lite with LDAP are::
+
+  [client.radosgw.gateway]
+  rgw sts key = {sts key for encrypting/ decrypting the session token}
+  rgw s3 auth use sts = true
+
+  rgw_s3_auth_use_ldap = true
+  rgw_ldap_uri = {LDAP server to use}
+  rgw_ldap_binddn = {Distinguished Name (DN) of the service account}
+  rgw_ldap_secret = {password for the service account}
+  rgw_ldap_searchdn = {base in the directory information tree for searching users}
+  rgw_ldap_dnattr = {attribute being used in the constructed search filter to match a username}
+  rgw_ldap_searchfilter = {search filter}
+
+The details of the integrating ldap with Ceph Object Gateway can be found here:
+:doc:`ldap-auth`
+
+Note: By default, STS and S3 APIs co-exist in the same namespace, and both S3
+and STS APIs can be accessed via the same endpoint in Ceph Object Gateway.
+
+Example showing how to Use STS Lite with Keystone
+=================================================
+
+The following are the steps needed to use STS Lite with Keystone. Boto 3.x has
+been used to write an example code to show the integration of STS Lite with
+Keystone.
+
+1. Generate EC2 credentials :
+
+.. code-block:: javascript
+
+  openstack ec2 credentials create
+  +------------+--------------------------------------------------------+
+  | Field      | Value                                                  |
+  +------------+--------------------------------------------------------+
+  | access     | b924dfc87d454d15896691182fdeb0ef                       |
+  | links      | {u'self': u'http://192.168.0.15/identity/v3/users/     |
+  |            | 40a7140e424f493d8165abc652dc731c/credentials/          |
+  |            | OS-EC2/b924dfc87d454d15896691182fdeb0ef'}              |
+  | project_id | c703801dccaf4a0aaa39bec8c481e25a                       |
+  | secret     | 6a2142613c504c42a94ba2b82147dc28                       |
+  | trust_id   | None                                                   |
+  | user_id    | 40a7140e424f493d8165abc652dc731c                       |
+  +------------+--------------------------------------------------------+
+
+2. Use the credentials created in the step 1. to get back a set of temporary
+   credentials using GetSessionToken API.
+
+.. code-block:: python
+
+    import boto3
+ 
+    access_key = <ec2 access key>
+    secret_key = <ec2 secret key>
+
+    client = boto3.client('sts',
+    aws_access_key_id=access_key,
+    aws_secret_access_key=secret_key,
+    endpoint_url=<STS URL>,
+    region_name='',
+    )
+
+    response = client.get_session_token(
+        DurationSeconds=43200
+    )
+
+3. The temporary credentials obtained in step 2. can be used for making S3 calls:
+
+.. code-block:: python
+
+    s3client = boto3.client('s3',
+      aws_access_key_id = response['Credentials']['AccessKeyId'],
+      aws_secret_access_key = response['Credentials']['SecretAccessKey'],
+      aws_session_token = response['Credentials']['SessionToken'],
+      endpoint_url=<S3 URL>,
+      region_name='')
+
+    bucket = s3client.create_bucket(Bucket='my-new-shiny-bucket')
+    response = s3client.list_buckets()
+    for bucket in response["Buckets"]:
+        print "{name}\t{created}".format(
+                    name = bucket['Name'],
+                    created = bucket['CreationDate'],
+    )
+
+Similar steps can be performed for using GetSessionToken with LDAP.
+
+Limitations and Workarounds
+===========================
+
+1. Keystone currently supports only S3 requests, hence in order to successfully 
+authenticate an STS request, the following workaround needs to be added to boto
+to the following file - botocore/auth.py
+
+Lines 13-16 have been added as a workaround in the code block below:
+
+.. code-block:: python
+
+  class SigV4Auth(BaseSigner):
+    """
+    Sign a request with Signature V4.
+    """
+    REQUIRES_REGION = True
+
+    def __init__(self, credentials, service_name, region_name):
+        self.credentials = credentials
+        # We initialize these value here so the unit tests can have
+        # valid values.  But these will get overridden in ``add_auth``
+        # later for real requests.
+        self._region_name = region_name
+        if service_name == 'sts':
+            self._service_name = 's3'
+        else:
+            self._service_name = service_name
+
diff --git a/doc/radosgw/admin.rst b/doc/radosgw/admin.rst
new file mode 100644
index 000000000..5f47471ac
--- /dev/null
+++ b/doc/radosgw/admin.rst
@@ -0,0 +1,528 @@
+=============
+ Admin Guide
+=============
+
+Once you have your Ceph Object Storage service up and running, you may
+administer the service with user management, access controls, quotas 
+and usage tracking among other features.
+
+
+User Management
+===============
+
+Ceph Object Storage user management refers to users of the Ceph Object Storage
+service (i.e., not the Ceph Object Gateway as a user of the Ceph Storage
+Cluster). You must create a user, access key and secret to enable end users to
+interact with Ceph Object Gateway services.
+
+There are two user types: 
+
+- **User:** The term 'user' reflects a user of the S3 interface.
+
+- **Subuser:** The term 'subuser' reflects a user of the Swift interface. A subuser
+  is associated to a user .
+  
+.. ditaa::
+           +---------+
+           |   User  |
+           +----+----+  
+                |     
+                |     +-----------+
+                +-----+  Subuser  |
+                      +-----------+
+
+You can create, modify, view, suspend and remove users and subusers. In addition
+to user and subuser IDs, you may add a display name and an email address for a
+user.  You can specify a key and secret, or generate a key and secret
+automatically. When generating or specifying keys, note that user IDs correspond
+to an S3 key type and subuser IDs correspond to a swift key type. Swift keys
+also have access levels of ``read``, ``write``, ``readwrite`` and ``full``.
+
+
+Create a User
+-------------
+
+To create a user (S3 interface), execute the following::
+
+	radosgw-admin user create --uid={username} --display-name="{display-name}" [--email={email}]
+
+For example:: 	
+	
+  radosgw-admin user create --uid=johndoe --display-name="John Doe" --email=john@example.com
+  
+.. code-block:: javascript
+  
+  { "user_id": "johndoe",
+    "display_name": "John Doe",
+    "email": "john@example.com",
+    "suspended": 0,
+    "max_buckets": 1000,
+    "subusers": [],
+    "keys": [
+          { "user": "johndoe",
+            "access_key": "11BS02LGFB6AL6H1ADMW",
+            "secret_key": "vzCEkuryfn060dfee4fgQPqFrncKEIkh3ZcdOANY"}],
+    "swift_keys": [],
+    "caps": [],
+    "op_mask": "read, write, delete",
+    "default_placement": "",
+    "placement_tags": [],
+    "bucket_quota": { "enabled": false,
+        "max_size_kb": -1,
+        "max_objects": -1},
+    "user_quota": { "enabled": false,
+        "max_size_kb": -1,
+        "max_objects": -1},
+    "temp_url_keys": []}
+
+Creating a user also creates an ``access_key`` and ``secret_key`` entry for use
+with any S3 API-compatible client.  
+
+.. important:: Check the key output. Sometimes ``radosgw-admin``
+   generates a JSON escape (``\``) character, and some clients
+   do not know how to handle JSON escape characters. Remedies include 
+   removing the JSON escape character (``\``), encapsulating the string
+   in quotes, regenerating the key and ensuring that it 
+   does not have a JSON escape character or specify the key and secret 
+   manually.
+
+
+Create a Subuser
+----------------
+
+To create a subuser (Swift interface) for the user, you must specify the user ID
+(``--uid={username}``), a subuser ID and the access level for the subuser. ::
+
+  radosgw-admin subuser create --uid={uid} --subuser={uid} --access=[ read | write | readwrite | full ]
+
+For example::
+
+  radosgw-admin subuser create --uid=johndoe --subuser=johndoe:swift --access=full
+
+
+.. note:: ``full`` is not ``readwrite``, as it also includes the access control policy.
+
+.. code-block:: javascript
+
+  { "user_id": "johndoe",
+    "display_name": "John Doe",
+    "email": "john@example.com",
+    "suspended": 0,
+    "max_buckets": 1000,
+    "subusers": [
+          { "id": "johndoe:swift",
+            "permissions": "full-control"}],
+    "keys": [
+          { "user": "johndoe",
+            "access_key": "11BS02LGFB6AL6H1ADMW",
+            "secret_key": "vzCEkuryfn060dfee4fgQPqFrncKEIkh3ZcdOANY"}],
+    "swift_keys": [],
+    "caps": [],
+    "op_mask": "read, write, delete",
+    "default_placement": "",
+    "placement_tags": [],
+    "bucket_quota": { "enabled": false,
+        "max_size_kb": -1,
+        "max_objects": -1},
+    "user_quota": { "enabled": false,
+        "max_size_kb": -1,
+        "max_objects": -1},
+    "temp_url_keys": []}
+
+
+Get User Info
+-------------
+
+To get information about a user, you must specify ``user info`` and the user ID
+(``--uid={username}``) . :: 
+
+	radosgw-admin user info --uid=johndoe
+
+
+
+Modify User Info
+----------------
+
+To modify information about a user, you must specify the user ID (``--uid={username}``)
+and the attributes you want to modify. Typical modifications are to keys and secrets,
+email addresses, display names and access levels. For example:: 
+
+	radosgw-admin user modify --uid=johndoe --display-name="John E. Doe"
+
+To modify subuser values, specify ``subuser modify``, user ID and the subuser ID. For example::
+
+	radosgw-admin subuser modify --uid=johndoe --subuser=johndoe:swift --access=full
+
+
+User Enable/Suspend
+-------------------
+
+When you create a user, the user is enabled by default. However, you may suspend
+user  privileges and re-enable them at a later time. To suspend a user, specify
+``user suspend`` and the user ID. ::
+
+	radosgw-admin user suspend --uid=johndoe
+
+To re-enable a suspended user, specify ``user enable`` and the user ID. :: 
+
+	radosgw-admin user enable --uid=johndoe
+	
+.. note:: Disabling the user disables the subuser.
+
+
+Remove a User
+-------------
+
+When you remove a user, the user and subuser are removed from the system.
+However, you may remove just the subuser if you wish. To remove a user (and
+subuser), specify ``user rm`` and the user ID. ::
+
+	radosgw-admin user rm --uid=johndoe
+
+To remove the subuser only, specify ``subuser rm`` and the subuser ID. ::
+
+	radosgw-admin subuser rm --subuser=johndoe:swift
+
+
+Options include:
+
+- **Purge Data:** The ``--purge-data`` option purges all data associated 
+  to the UID.
+  
+- **Purge Keys:** The ``--purge-keys`` option purges all keys associated 
+  to the UID.
+
+
+Remove a Subuser
+----------------
+
+When you remove a sub user, you are removing access to the Swift interface. 
+The user will remain in the system. To remove the subuser, specify 
+``subuser rm`` and the subuser ID. ::
+
+	radosgw-admin subuser rm --subuser=johndoe:swift
+
+
+
+Options include:
+  
+- **Purge Keys:** The ``--purge-keys`` option purges all keys associated 
+  to the UID.
+
+
+Add / Remove a Key
+------------------------
+
+Both users and subusers require the key to access the S3 or Swift interface. To
+use S3, the user needs a key pair which is composed of an access key and a 
+secret key. On the other hand, to use Swift, the user typically needs a secret 
+key (password), and use it together with the associated user ID. You may create
+a key and either specify or generate the access key and/or secret key. You may 
+also remove a key. Options include:
+
+- ``--key-type=<type>`` specifies the key type. The options are: s3, swift
+- ``--access-key=<key>`` manually specifies an S3 access key.
+- ``--secret-key=<key>`` manually specifies a S3 secret key or a Swift secret key.
+- ``--gen-access-key`` automatically generates a random S3 access key.
+- ``--gen-secret`` automatically generates a random S3 secret key or a random Swift secret key.
+
+An example how to add a specified S3 key pair for a user. ::
+
+	radosgw-admin key create --uid=foo --key-type=s3 --access-key fooAccessKey --secret-key fooSecretKey
+
+.. code-block:: javascript
+
+  { "user_id": "foo",
+    "rados_uid": 0,
+    "display_name": "foo",
+    "email": "foo@example.com",
+    "suspended": 0,
+    "keys": [
+      { "user": "foo",
+        "access_key": "fooAccessKey",
+        "secret_key": "fooSecretKey"}],
+  }
+
+Note that you may create multiple S3 key pairs for a user.
+
+To attach a specified swift secret key for a subuser. ::
+
+	radosgw-admin key create --subuser=foo:bar --key-type=swift --secret-key barSecret
+
+.. code-block:: javascript
+
+  { "user_id": "foo",
+    "rados_uid": 0,
+    "display_name": "foo",
+    "email": "foo@example.com",
+    "suspended": 0,
+    "subusers": [
+       { "id": "foo:bar",
+         "permissions": "full-control"}],
+    "swift_keys": [
+      { "user": "foo:bar",
+        "secret_key": "asfghjghghmgm"}]}
+
+Note that a subuser can have only one swift secret key.
+
+Subusers can also be used with S3 APIs if the subuser is associated with a S3 key pair. ::	
+
+	radosgw-admin key create --subuser=foo:bar --key-type=s3 --access-key barAccessKey --secret-key barSecretKey
+	
+.. code-block:: javascript
+
+  { "user_id": "foo",
+    "rados_uid": 0,
+    "display_name": "foo",
+    "email": "foo@example.com",
+    "suspended": 0,
+    "subusers": [
+       { "id": "foo:bar",
+         "permissions": "full-control"}],
+    "keys": [
+      { "user": "foo:bar",
+        "access_key": "barAccessKey",
+        "secret_key": "barSecretKey"}],
+  }
+
+
+To remove a S3 key pair, specify the access key. :: 
+
+	radosgw-admin key rm --uid=foo --key-type=s3 --access-key=fooAccessKey 
+
+To remove the swift secret key. ::
+
+	radosgw-admin key rm --subuser=foo:bar --key-type=swift
+
+
+Add / Remove Admin Capabilities
+-------------------------------
+
+The Ceph Storage Cluster provides an administrative API that enables  users to
+execute administrative functions via the REST API. By default, users do NOT have
+access to this API. To enable a user to exercise  administrative functionality,
+provide the user with administrative capabilities.
+
+To add administrative capabilities to a user, execute the following:: 
+
+	radosgw-admin caps add --uid={uid} --caps={caps}
+
+
+You can add read, write or all capabilities to users, buckets, metadata and 
+usage (utilization). For example::
+
+	--caps="[users|buckets|metadata|usage|zone]=[*|read|write|read, write]"
+
+For example::
+
+	radosgw-admin caps add --uid=johndoe --caps="users=*;buckets=*"
+
+
+To remove administrative capabilities from a user, execute the following:: 
+
+	radosgw-admin caps rm --uid=johndoe --caps={caps}
+
+
+Quota Management
+================
+
+The Ceph Object Gateway enables you to set quotas on users and buckets owned by
+users. Quotas include the maximum number of objects in a bucket and the maximum
+storage size a bucket can hold.
+
+- **Bucket:** The ``--bucket`` option allows you to specify a quota for
+  buckets the user owns.
+
+- **Maximum Objects:** The ``--max-objects`` setting allows you to specify
+  the maximum number of objects. A negative value disables this setting.
+  
+- **Maximum Size:** The ``--max-size`` option allows you to specify a quota
+  size in B/K/M/G/T, where B is the default. A negative value disables this setting.
+  
+- **Quota Scope:** The ``--quota-scope`` option sets the scope for the quota.
+  The options are ``bucket`` and ``user``. Bucket quotas apply to buckets a 
+  user owns. User quotas apply to a user.
+
+
+Set User Quota
+--------------
+
+Before you enable a quota, you must first set the quota parameters.
+For example:: 
+
+	radosgw-admin quota set --quota-scope=user --uid=<uid> [--max-objects=<num objects>] [--max-size=<max size>]
+
+For example:: 
+
+	radosgw-admin quota set --quota-scope=user --uid=johndoe --max-objects=1024 --max-size=1024B
+
+
+A negative value for num objects and / or max size means that the
+specific quota attribute check is disabled.
+
+
+Enable/Disable User Quota
+-------------------------
+
+Once you set a user quota, you may enable it. For example:: 
+
+	radosgw-admin quota enable --quota-scope=user --uid=<uid>
+
+You may disable an enabled user quota. For example:: 
+
+	radosgw-admin quota disable --quota-scope=user --uid=<uid>
+
+
+Set Bucket Quota
+----------------
+
+Bucket quotas apply to the buckets owned by the specified ``uid``. They are
+independent of the user. ::
+
+	radosgw-admin quota set --uid=<uid> --quota-scope=bucket [--max-objects=<num objects>] [--max-size=<max size]
+
+A negative value for num objects and / or max size means that the
+specific quota attribute check is disabled.
+
+
+Enable/Disable Bucket Quota
+---------------------------
+
+Once you set a bucket quota, you may enable it. For example:: 
+
+	radosgw-admin quota enable --quota-scope=bucket --uid=<uid>
+
+You may disable an enabled bucket quota. For example:: 
+
+	radosgw-admin quota disable --quota-scope=bucket --uid=<uid>
+
+
+Get Quota Settings
+------------------
+
+You may access each user's quota settings via the user information
+API. To read user quota setting information with the CLI interface, 
+execute the following::
+
+	radosgw-admin user info --uid=<uid>
+
+
+Update Quota Stats
+------------------
+
+Quota stats get updated asynchronously. You can update quota
+statistics for all users and all buckets manually to retrieve
+the latest quota stats. ::
+
+	radosgw-admin user stats --uid=<uid> --sync-stats
+
+.. _rgw_user_usage_stats:
+
+Get User Usage Stats
+--------------------
+
+To see how much of the quota a user has consumed, execute the following::
+
+	radosgw-admin user stats --uid=<uid>
+
+.. note:: You should execute ``radosgw-admin user stats`` with the 
+   ``--sync-stats`` option to receive the latest data.
+
+Default Quotas
+--------------
+
+You can set default quotas in the config.  These defaults are used when
+creating a new user and have no effect on existing users. If the
+relevant default quota is set in config, then that quota is set on the
+new user, and that quota is enabled.  See ``rgw bucket default quota max objects``,
+``rgw bucket default quota max size``, ``rgw user default quota max objects``, and
+``rgw user default quota max size`` in `Ceph Object Gateway Config Reference`_
+
+Quota Cache
+-----------
+
+Quota statistics are cached on each RGW instance.  If there are multiple
+instances, then the cache can keep quotas from being perfectly enforced, as
+each instance will have a different view of quotas.  The options that control
+this are ``rgw bucket quota ttl``, ``rgw user quota bucket sync interval`` and
+``rgw user quota sync interval``.  The higher these values are, the more
+efficient quota operations are, but the more out-of-sync multiple instances
+will be.  The lower these values are, the closer to perfect enforcement
+multiple instances will achieve.  If all three are 0, then quota caching is
+effectively disabled, and multiple instances will have perfect quota
+enforcement.  See `Ceph Object Gateway Config Reference`_
+
+Reading / Writing Global Quotas
+-------------------------------
+
+You can read and write global quota settings in the period configuration. To
+view the global quota settings::
+
+	radosgw-admin global quota get
+
+The global quota settings can be manipulated with the ``global quota``
+counterparts of the ``quota set``, ``quota enable``, and ``quota disable``
+commands. ::
+
+	radosgw-admin global quota set --quota-scope bucket --max-objects 1024
+	radosgw-admin global quota enable --quota-scope bucket
+
+.. note:: In a multisite configuration, where there is a realm and period
+   present, changes to the global quotas must be committed using ``period
+   update --commit``. If there is no period present, the rados gateway(s) must
+   be restarted for the changes to take effect.
+
+
+Usage
+=====
+
+The Ceph Object Gateway logs usage for each user. You can track
+user usage within date ranges too.
+
+- Add ``rgw enable usage log = true`` in [client.rgw] section of ceph.conf and restart the radosgw service. 
+
+Options include: 
+
+- **Start Date:** The ``--start-date`` option allows you to filter usage
+  stats from a particular start date (**format:** ``yyyy-mm-dd[HH:MM:SS]``).
+
+- **End Date:** The ``--end-date`` option allows you to filter usage up
+  to a particular date (**format:** ``yyyy-mm-dd[HH:MM:SS]``). 
+  
+- **Log Entries:** The ``--show-log-entries`` option allows you to specify
+  whether or not to include log entries with the usage stats 
+  (options: ``true`` | ``false``).
+
+.. note:: You may specify time with minutes and seconds, but it is stored 
+   with 1 hour resolution.
+
+
+Show Usage
+----------
+
+To show usage statistics, specify the ``usage show``. To show usage for a
+particular user, you must specify a user ID. You may also specify a start date,
+end date, and whether or not to show log entries.::
+
+	radosgw-admin usage show --uid=johndoe --start-date=2012-03-01 --end-date=2012-04-01
+
+You may also show a summary of usage information for all users by omitting a user ID. ::
+
+	radosgw-admin usage show --show-log-entries=false
+
+
+Trim Usage
+----------
+
+With heavy use, usage logs can begin to take up storage space. You can trim
+usage logs for all users and for specific users. You may also specify date
+ranges for trim operations. ::
+
+	radosgw-admin usage trim --start-date=2010-01-01 --end-date=2010-12-31
+	radosgw-admin usage trim --uid=johndoe	
+	radosgw-admin usage trim --uid=johndoe --end-date=2013-12-31
+
+
+.. _radosgw-admin: ../../man/8/radosgw-admin/
+.. _Pool Configuration: ../../rados/configuration/pool-pg-config-ref/
+.. _Ceph Object Gateway Config Reference: ../config-ref/
diff --git a/doc/radosgw/adminops.rst b/doc/radosgw/adminops.rst
new file mode 100644
index 000000000..2dfd9f6d9
--- /dev/null
+++ b/doc/radosgw/adminops.rst
@@ -0,0 +1,1994 @@
+==================
+ Admin Operations
+==================
+
+An admin API request will be done on a URI that starts with the configurable 'admin'
+resource entry point. Authorization for the admin API duplicates the S3 authorization
+mechanism. Some operations require that the user holds special administrative capabilities.
+The response entity type (XML or JSON) may be specified as the 'format' option in the
+request and defaults to JSON if not specified.
+
+Get Usage
+=========
+
+Request bandwidth usage information.
+
+Note: this feature is disabled by default, can be enabled by setting ``rgw
+enable usage log = true`` in the appropriate section of ceph.conf. For changes
+in ceph.conf to take effect, radosgw process restart is needed.
+
+:caps: usage=read
+
+Syntax
+~~~~~~
+
+::
+
+	GET /{admin}/usage?format=json HTTP/1.1
+	Host: {fqdn}
+
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``uid``
+
+:Description: The user for which the information is requested. If not specified will apply to all users.
+:Type: String
+:Example: ``foo_user``
+:Required: No
+
+``start``
+
+:Description: Date and (optional) time that specifies the start time of the requested data.
+:Type: String
+:Example: ``2012-09-25 16:00:00``
+:Required: No
+
+``end``
+
+:Description: Date and (optional) time that specifies the end time of the requested data (non-inclusive).
+:Type: String
+:Example: ``2012-09-25 16:00:00``
+:Required: No
+
+
+``show-entries``
+
+:Description: Specifies whether data entries should be returned.
+:Type: Boolean
+:Example: True [True]
+:Required: No
+
+
+``show-summary``
+
+:Description: Specifies whether data summary should be returned.
+:Type: Boolean
+:Example: True [True]
+:Required: No
+
+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If successful, the response contains the requested information.
+
+``usage``
+
+:Description: A container for the usage information.
+:Type: Container
+
+``entries``
+
+:Description: A container for the usage entries information.
+:Type: Container
+
+``user``
+
+:Description: A container for the user data information.
+:Type: Container
+
+``owner``
+
+:Description: The name of the user that owns the buckets.
+:Type: String
+
+``bucket``
+
+:Description: The bucket name.
+:Type: String
+
+``time``
+
+:Description: Time lower bound for which data is being specified (rounded to the beginning of the first relevant hour).
+:Type: String
+
+``epoch``
+
+:Description: The time specified in seconds since 1/1/1970.
+:Type: String
+
+``categories``
+
+:Description: A container for stats categories.
+:Type: Container
+
+``entry``
+
+:Description: A container for stats entry.
+:Type: Container
+
+``category``
+
+:Description: Name of request category for which the stats are provided.
+:Type: String
+
+``bytes_sent``
+
+:Description: Number of bytes sent by the RADOS Gateway.
+:Type: Integer
+
+``bytes_received``
+
+:Description: Number of bytes received by the RADOS Gateway.
+:Type: Integer
+
+``ops``
+
+:Description: Number of operations.
+:Type: Integer
+
+``successful_ops``
+
+:Description: Number of successful operations.
+:Type: Integer
+
+``summary``
+
+:Description: A container for stats summary.
+:Type: Container
+
+``total``
+
+:Description: A container for stats summary aggregated total.
+:Type: Container
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+TBD.
+
+Trim Usage
+==========
+
+Remove usage information. With no dates specified, removes all usage
+information.
+
+Note: this feature is disabled by default, can be enabled by setting ``rgw
+enable usage log = true`` in the appropriate section of ceph.conf. For changes
+in ceph.conf to take effect, radosgw process restart is needed.
+
+:caps: usage=write
+
+Syntax
+~~~~~~
+
+::
+
+	DELETE /{admin}/usage?format=json HTTP/1.1
+	Host: {fqdn}
+
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``uid``
+
+:Description: The user for which the information is requested. If not specified will apply to all users.
+:Type: String
+:Example: ``foo_user``
+:Required: No
+
+``start``
+
+:Description: Date and (optional) time that specifies the start time of the requested data.
+:Type: String
+:Example: ``2012-09-25 16:00:00``
+:Required: No
+
+``end``
+
+:Description: Date and (optional) time that specifies the end time of the requested data (none inclusive).
+:Type: String
+:Example: ``2012-09-25 16:00:00``
+:Required: No
+
+
+``remove-all``
+
+:Description: Required when uid is not specified, in order to acknowledge multi user data removal.
+:Type: Boolean
+:Example: True [False]
+:Required: No
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+TBD.
+
+Get User Info
+=============
+
+Get user information.
+
+:caps: users=read
+
+
+Syntax
+~~~~~~
+
+::
+
+	GET /{admin}/user?format=json HTTP/1.1
+	Host: {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``uid``
+
+:Description: The user for which the information is requested.
+:Type: String
+:Example: ``foo_user``
+:Required: Yes
+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If successful, the response contains the user information.
+
+``user``
+
+:Description: A container for the user data information.
+:Type: Container
+
+``user_id``
+
+:Description: The user id.
+:Type: String
+:Parent: ``user``
+
+``display_name``
+
+:Description: Display name for the user.
+:Type: String
+:Parent: ``user``
+
+``suspended``
+
+:Description: True if the user is suspended.
+:Type: Boolean
+:Parent: ``user``
+
+``max_buckets``
+
+:Description: The maximum number of buckets to be owned by the user.
+:Type: Integer
+:Parent: ``user``
+
+``subusers``
+
+:Description: Subusers associated with this user account.
+:Type: Container
+:Parent: ``user``
+
+``keys``
+
+:Description: S3 keys associated with this user account.
+:Type: Container
+:Parent: ``user``
+
+``swift_keys``
+
+:Description: Swift keys associated with this user account.
+:Type: Container
+:Parent: ``user``
+
+``caps``
+
+:Description: User capabilities.
+:Type: Container
+:Parent: ``user``
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+None.
+
+Create User
+===========
+
+Create a new user. By default, a S3 key pair will be created automatically
+and returned in the response. If only one of ``access-key`` or ``secret-key``
+is provided, the omitted key will be automatically generated. By default, a
+generated key is added to the keyring without replacing an existing key pair.
+If ``access-key`` is specified and refers to an existing key owned by the user
+then it will be modified.
+
+.. versionadded:: Luminous
+
+A ``tenant`` may either be specified as a part of uid or as an additional
+request param.
+
+:caps: users=write
+
+Syntax
+~~~~~~
+
+::
+
+	PUT /{admin}/user?format=json HTTP/1.1
+	Host: {fqdn}
+
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``uid``
+
+:Description: The user ID to be created.
+:Type: String
+:Example: ``foo_user``
+:Required: Yes
+
+A tenant name may also specified as a part of ``uid``, by following the syntax
+``tenant$user``, refer to :ref:`Multitenancy <rgw-multitenancy>` for more details.
+
+``display-name``
+
+:Description: The display name of the user to be created.
+:Type: String
+:Example: ``foo user``
+:Required: Yes
+
+
+``email``
+
+:Description: The email address associated with the user.
+:Type: String
+:Example: ``foo@bar.com``
+:Required: No
+
+``key-type``
+
+:Description: Key type to be generated, options are: swift, s3 (default).
+:Type: String
+:Example: ``s3`` [``s3``]
+:Required: No
+
+``access-key``
+
+:Description: Specify access key.
+:Type: String
+:Example: ``ABCD0EF12GHIJ2K34LMN``
+:Required: No
+
+
+``secret-key``
+
+:Description: Specify secret key.
+:Type: String
+:Example: ``0AbCDEFg1h2i34JklM5nop6QrSTUV+WxyzaBC7D8``
+:Required: No
+
+``user-caps``
+
+:Description: User capabilities.
+:Type: String
+:Example: ``usage=read, write; users=read``
+:Required: No
+
+``generate-key``
+
+:Description: Generate a new key pair and add to the existing keyring.
+:Type: Boolean
+:Example: True [True]
+:Required: No
+
+``max-buckets``
+
+:Description: Specify the maximum number of buckets the user can own.
+:Type: Integer
+:Example: 500 [1000]
+:Required: No
+
+``suspended``
+
+:Description: Specify whether the user should be suspended.
+:Type: Boolean
+:Example: False [False]
+:Required: No
+
+.. versionadded:: Jewel
+
+``tenant``
+
+:Description: the Tenant under which a user is a part of.
+:Type: string
+:Example: tenant1
+:Required: No
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If successful, the response contains the user information.
+
+``user``
+
+:Description: A container for the user data information.
+:Type: Container
+
+``tenant``
+
+:Description: The tenant which user is a part of.
+:Type: String
+:Parent: ``user``
+
+``user_id``
+
+:Description: The user id.
+:Type: String
+:Parent: ``user``
+
+``display_name``
+
+:Description: Display name for the user.
+:Type: String
+:Parent: ``user``
+
+``suspended``
+
+:Description: True if the user is suspended.
+:Type: Boolean
+:Parent: ``user``
+
+``max_buckets``
+
+:Description: The maximum number of buckets to be owned by the user.
+:Type: Integer
+:Parent: ``user``
+
+``subusers``
+
+:Description: Subusers associated with this user account.
+:Type: Container
+:Parent: ``user``
+
+``keys``
+
+:Description: S3 keys associated with this user account.
+:Type: Container
+:Parent: ``user``
+
+``swift_keys``
+
+:Description: Swift keys associated with this user account.
+:Type: Container
+:Parent: ``user``
+
+``caps``
+
+:Description: User capabilities.
+:Type: Container
+:Parent: ``user``
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``UserExists``
+
+:Description: Attempt to create existing user.
+:Code: 409 Conflict
+
+``InvalidAccessKey``
+
+:Description: Invalid access key specified.
+:Code: 400 Bad Request
+
+``InvalidKeyType``
+
+:Description: Invalid key type specified.
+:Code: 400 Bad Request
+
+``InvalidSecretKey``
+
+:Description: Invalid secret key specified.
+:Code: 400 Bad Request
+
+``InvalidKeyType``
+
+:Description: Invalid key type specified.
+:Code: 400 Bad Request
+
+``KeyExists``
+
+:Description: Provided access key exists and belongs to another user.
+:Code: 409 Conflict
+
+``EmailExists``
+
+:Description: Provided email address exists.
+:Code: 409 Conflict
+
+``InvalidCapability``
+
+:Description: Attempt to grant invalid admin capability.
+:Code: 400 Bad Request
+
+
+Modify User
+===========
+
+Modify a user.
+
+:caps: users=write
+
+Syntax
+~~~~~~
+
+::
+
+	POST /{admin}/user?format=json HTTP/1.1
+	Host: {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``uid``
+
+:Description: The user ID to be modified.
+:Type: String
+:Example: ``foo_user``
+:Required: Yes
+
+``display-name``
+
+:Description: The display name of the user to be modified.
+:Type: String
+:Example: ``foo user``
+:Required: No
+
+``email``
+
+:Description: The email address to be associated with the user.
+:Type: String
+:Example: ``foo@bar.com``
+:Required: No
+
+``generate-key``
+
+:Description: Generate a new key pair and add to the existing keyring.
+:Type: Boolean
+:Example: True [False]
+:Required: No
+
+``access-key``
+
+:Description: Specify access key.
+:Type: String
+:Example: ``ABCD0EF12GHIJ2K34LMN``
+:Required: No
+
+``secret-key``
+
+:Description: Specify secret key.
+:Type: String
+:Example: ``0AbCDEFg1h2i34JklM5nop6QrSTUV+WxyzaBC7D8``
+:Required: No
+
+``key-type``
+
+:Description: Key type to be generated, options are: swift, s3 (default).
+:Type: String
+:Example: ``s3``
+:Required: No
+
+``user-caps``
+
+:Description: User capabilities.
+:Type: String
+:Example: ``usage=read, write; users=read``
+:Required: No
+
+``max-buckets``
+
+:Description: Specify the maximum number of buckets the user can own.
+:Type: Integer
+:Example: 500 [1000]
+:Required: No
+
+``suspended``
+
+:Description: Specify whether the user should be suspended.
+:Type: Boolean
+:Example: False [False]
+:Required: No
+
+``op-mask``
+
+:Description: The op-mask of the user to be modified.
+:Type: String
+:Example: ``read, write, delete, *``
+:Required: No
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If successful, the response contains the user information.
+
+``user``
+
+:Description: A container for the user data information.
+:Type: Container
+
+``user_id``
+
+:Description: The user id.
+:Type: String
+:Parent: ``user``
+
+``display_name``
+
+:Description: Display name for the user.
+:Type: String
+:Parent: ``user``
+
+
+``suspended``
+
+:Description: True if the user is suspended.
+:Type: Boolean
+:Parent: ``user``
+
+
+``max_buckets``
+
+:Description: The maximum number of buckets to be owned by the user.
+:Type: Integer
+:Parent: ``user``
+
+
+``subusers``
+
+:Description: Subusers associated with this user account.
+:Type: Container
+:Parent: ``user``
+
+
+``keys``
+
+:Description: S3 keys associated with this user account.
+:Type: Container
+:Parent: ``user``
+
+
+``swift_keys``
+
+:Description: Swift keys associated with this user account.
+:Type: Container
+:Parent: ``user``
+
+
+``caps``
+
+:Description: User capabilities.
+:Type: Container
+:Parent: ``user``
+
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``InvalidAccessKey``
+
+:Description: Invalid access key specified.
+:Code: 400 Bad Request
+
+``InvalidKeyType``
+
+:Description: Invalid key type specified.
+:Code: 400 Bad Request
+
+``InvalidSecretKey``
+
+:Description: Invalid secret key specified.
+:Code: 400 Bad Request
+
+``KeyExists``
+
+:Description: Provided access key exists and belongs to another user.
+:Code: 409 Conflict
+
+``EmailExists``
+
+:Description: Provided email address exists.
+:Code: 409 Conflict
+
+``InvalidCapability``
+
+:Description: Attempt to grant invalid admin capability.
+:Code: 400 Bad Request
+
+Remove User
+===========
+
+Remove an existing user.
+
+:caps: users=write
+
+Syntax
+~~~~~~
+
+::
+
+	DELETE /{admin}/user?format=json HTTP/1.1
+	Host: {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``uid``
+
+:Description: The user ID to be removed.
+:Type: String
+:Example: ``foo_user``
+:Required: Yes.
+
+``purge-data``
+
+:Description: When specified the buckets and objects belonging
+              to the user will also be removed.
+:Type: Boolean
+:Example: True
+:Required: No
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+None
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+None.
+
+Create Subuser
+==============
+
+Create a new subuser (primarily useful for clients using the Swift API).
+Note that in general for a subuser to be useful, it must be granted
+permissions by specifying ``access``. As with user creation if
+``subuser`` is specified without ``secret``, then a secret key will
+be automatically generated.
+
+:caps: users=write
+
+Syntax
+~~~~~~
+
+::
+
+	PUT /{admin}/user?subuser&format=json HTTP/1.1
+	Host {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``uid``
+
+:Description: The user ID under which a subuser is to  be created.
+:Type: String
+:Example: ``foo_user``
+:Required: Yes
+
+
+``subuser``
+
+:Description: Specify the subuser ID to be created.
+:Type: String
+:Example: ``sub_foo``
+:Required: Yes
+
+``secret-key``
+
+:Description: Specify secret key.
+:Type: String
+:Example: ``0AbCDEFg1h2i34JklM5nop6QrSTUV+WxyzaBC7D8``
+:Required: No
+
+``key-type``
+
+:Description: Key type to be generated, options are: swift (default), s3.
+:Type: String
+:Example: ``swift`` [``swift``]
+:Required: No
+
+``access``
+
+:Description: Set access permissions for sub-user, should be one
+              of ``read, write, readwrite, full``.
+:Type: String
+:Example: ``read``
+:Required: No
+
+``generate-secret``
+
+:Description: Generate the secret key.
+:Type: Boolean
+:Example: True [False]
+:Required: No
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If successful, the response contains the subuser information.
+
+
+``subusers``
+
+:Description: Subusers associated with the user account.
+:Type: Container
+
+``id``
+
+:Description: Subuser id.
+:Type: String
+:Parent: ``subusers``
+
+``permissions``
+
+:Description: Subuser access to user account.
+:Type: String
+:Parent: ``subusers``
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``SubuserExists``
+
+:Description: Specified subuser exists.
+:Code: 409 Conflict
+
+``InvalidKeyType``
+
+:Description: Invalid key type specified.
+:Code: 400 Bad Request
+
+``InvalidSecretKey``
+
+:Description: Invalid secret key specified.
+:Code: 400 Bad Request
+
+``InvalidAccess``
+
+:Description: Invalid subuser access specified.
+:Code: 400 Bad Request
+
+Modify Subuser
+==============
+
+Modify an existing subuser
+
+:caps: users=write
+
+Syntax
+~~~~~~
+
+::
+
+	POST /{admin}/user?subuser&format=json HTTP/1.1
+	Host {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``uid``
+
+:Description: The user ID under which the subuser is to be modified.
+:Type: String
+:Example: ``foo_user``
+:Required: Yes
+
+``subuser``
+
+:Description: The subuser ID to be modified.
+:Type: String
+:Example: ``sub_foo``
+:Required: Yes
+
+``generate-secret``
+
+:Description: Generate a new secret key for the subuser,
+              replacing the existing key.
+:Type: Boolean
+:Example: True [False]
+:Required: No
+
+``secret``
+
+:Description: Specify secret key.
+:Type: String
+:Example: ``0AbCDEFg1h2i34JklM5nop6QrSTUV+WxyzaBC7D8``
+:Required: No
+
+``key-type``
+
+:Description: Key type to be generated, options are: swift (default), s3 .
+:Type: String
+:Example: ``swift`` [``swift``]
+:Required: No
+
+``access``
+
+:Description: Set access permissions for sub-user, should be one
+              of ``read, write, readwrite, full``.
+:Type: String
+:Example: ``read``
+:Required: No
+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If successful, the response contains the subuser information.
+
+
+``subusers``
+
+:Description: Subusers associated with the user account.
+:Type: Container
+
+``id``
+
+:Description: Subuser id.
+:Type: String
+:Parent: ``subusers``
+
+``permissions``
+
+:Description: Subuser access to user account.
+:Type: String
+:Parent: ``subusers``
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``InvalidKeyType``
+
+:Description: Invalid key type specified.
+:Code: 400 Bad Request
+
+``InvalidSecretKey``
+
+:Description: Invalid secret key specified.
+:Code: 400 Bad Request
+
+``InvalidAccess``
+
+:Description: Invalid subuser access specified.
+:Code: 400 Bad Request
+
+Remove Subuser
+==============
+
+Remove an existing subuser
+
+:caps: users=write
+
+Syntax
+~~~~~~
+
+::
+
+	DELETE /{admin}/user?subuser&format=json HTTP/1.1
+	Host {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``uid``
+
+:Description: The user ID under which the subuser is to be removed.
+:Type: String
+:Example: ``foo_user``
+:Required: Yes
+
+
+``subuser``
+
+:Description: The subuser ID to be removed.
+:Type: String
+:Example: ``sub_foo``
+:Required: Yes
+
+``purge-keys``
+
+:Description: Remove keys belonging to the subuser.
+:Type: Boolean
+:Example: True [True]
+:Required: No
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+None.
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+None.
+
+Create Key
+==========
+
+Create a new key. If a ``subuser`` is specified then by default created keys
+will be swift type. If only one of ``access-key`` or ``secret-key`` is provided the
+committed key will be automatically generated, that is if only ``secret-key`` is
+specified then ``access-key`` will be automatically generated. By default, a
+generated key is added to the keyring without replacing an existing key pair.
+If ``access-key`` is specified and refers to an existing key owned by the user
+then it will be modified. The response is a container listing all keys of the same
+type as the key created. Note that when creating a swift key, specifying the option
+``access-key`` will have no effect. Additionally, only one swift key may be held by
+each user or subuser.
+
+:caps: users=write
+
+
+Syntax
+~~~~~~
+
+::
+
+	PUT /{admin}/user?key&format=json HTTP/1.1
+	Host {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``uid``
+
+:Description: The user ID to receive the new key.
+:Type: String
+:Example: ``foo_user``
+:Required: Yes
+
+``subuser``
+
+:Description: The subuser ID to receive the new key.
+:Type: String
+:Example: ``sub_foo``
+:Required: No
+
+``key-type``
+
+:Description: Key type to be generated, options are: swift, s3 (default).
+:Type: String
+:Example: ``s3`` [``s3``]
+:Required: No
+
+``access-key``
+
+:Description: Specify the access key.
+:Type: String
+:Example: ``AB01C2D3EF45G6H7IJ8K``
+:Required: No
+
+``secret-key``
+
+:Description: Specify the secret key.
+:Type: String
+:Example: ``0ab/CdeFGhij1klmnopqRSTUv1WxyZabcDEFgHij``
+:Required: No
+
+``generate-key``
+
+:Description: Generate a new key pair and add to the existing keyring.
+:Type: Boolean
+:Example: True [``True``]
+:Required: No
+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+``keys``
+
+:Description: Keys of type created associated with this user account.
+:Type: Container
+
+``user``
+
+:Description: The user account associated with the key.
+:Type: String
+:Parent: ``keys``
+
+``access-key``
+
+:Description: The access key.
+:Type: String
+:Parent: ``keys``
+
+``secret-key``
+
+:Description: The secret key
+:Type: String
+:Parent: ``keys``
+
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``InvalidAccessKey``
+
+:Description: Invalid access key specified.
+:Code: 400 Bad Request
+
+``InvalidKeyType``
+
+:Description: Invalid key type specified.
+:Code: 400 Bad Request
+
+``InvalidSecretKey``
+
+:Description: Invalid secret key specified.
+:Code: 400 Bad Request
+
+``InvalidKeyType``
+
+:Description: Invalid key type specified.
+:Code: 400 Bad Request
+
+``KeyExists``
+
+:Description: Provided access key exists and belongs to another user.
+:Code: 409 Conflict
+
+Remove Key
+==========
+
+Remove an existing key.
+
+:caps: users=write
+
+Syntax
+~~~~~~
+
+::
+
+	DELETE /{admin}/user?key&format=json HTTP/1.1
+	Host {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``access-key``
+
+:Description: The S3 access key belonging to the S3 key pair to remove.
+:Type: String
+:Example: ``AB01C2D3EF45G6H7IJ8K``
+:Required: Yes
+
+``uid``
+
+:Description: The user to remove the key from.
+:Type: String
+:Example: ``foo_user``
+:Required: No
+
+``subuser``
+
+:Description: The subuser to remove the key from.
+:Type: String
+:Example: ``sub_foo``
+:Required: No
+
+``key-type``
+
+:Description: Key type to be removed, options are: swift, s3.
+              NOTE: Required to remove swift key.
+:Type: String
+:Example: ``swift``
+:Required: No
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+None.
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+None.
+
+Get Bucket Info
+===============
+
+Get information about a subset of the existing buckets. If ``uid`` is specified
+without ``bucket`` then all buckets belonging to the user will be returned. If
+``bucket`` alone is specified, information for that particular bucket will be
+retrieved.
+
+:caps: buckets=read
+
+Syntax
+~~~~~~
+
+::
+
+	GET /{admin}/bucket?format=json HTTP/1.1
+	Host {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``bucket``
+
+:Description: The bucket to return info on.
+:Type: String
+:Example: ``foo_bucket``
+:Required: No
+
+``uid``
+
+:Description: The user to retrieve bucket information for.
+:Type: String
+:Example: ``foo_user``
+:Required: No
+
+``stats``
+
+:Description: Return bucket statistics.
+:Type: Boolean
+:Example: True [False]
+:Required: No
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If successful the request returns a buckets container containing
+the desired bucket information.
+
+``stats``
+
+:Description: Per bucket information.
+:Type: Container
+
+``buckets``
+
+:Description: Contains a list of one or more bucket containers.
+:Type: Container
+
+``bucket``
+
+:Description: Container for single bucket information.
+:Type: Container
+:Parent: ``buckets``
+
+``name``
+
+:Description: The name of the bucket.
+:Type: String
+:Parent: ``bucket``
+
+``pool``
+
+:Description: The pool the bucket is stored in.
+:Type: String
+:Parent: ``bucket``
+
+``id``
+
+:Description: The unique bucket id.
+:Type: String
+:Parent: ``bucket``
+
+``marker``
+
+:Description: Internal bucket tag.
+:Type: String
+:Parent: ``bucket``
+
+``owner``
+
+:Description: The user id of the bucket owner.
+:Type: String
+:Parent: ``bucket``
+
+``usage``
+
+:Description: Storage usage information.
+:Type: Container
+:Parent: ``bucket``
+
+``index``
+
+:Description: Status of bucket index.
+:Type: String
+:Parent: ``bucket``
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``IndexRepairFailed``
+
+:Description: Bucket index repair failed.
+:Code: 409 Conflict
+
+Check Bucket Index
+==================
+
+Check the index of an existing bucket. NOTE: to check multipart object
+accounting with ``check-objects``, ``fix`` must be set to True.
+
+:caps: buckets=write
+
+Syntax
+~~~~~~
+
+::
+
+	GET /{admin}/bucket?index&format=json HTTP/1.1
+	Host {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``bucket``
+
+:Description: The bucket to return info on.
+:Type: String
+:Example: ``foo_bucket``
+:Required: Yes
+
+``check-objects``
+
+:Description: Check multipart object accounting.
+:Type: Boolean
+:Example: True [False]
+:Required: No
+
+``fix``
+
+:Description: Also fix the bucket index when checking.
+:Type: Boolean
+:Example: False [False]
+:Required: No
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+``index``
+
+:Description: Status of bucket index.
+:Type: String
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``IndexRepairFailed``
+
+:Description: Bucket index repair failed.
+:Code: 409 Conflict
+
+Remove Bucket
+=============
+
+Delete an existing bucket.
+
+:caps: buckets=write
+
+Syntax
+~~~~~~
+
+::
+
+	DELETE /{admin}/bucket?format=json HTTP/1.1
+	Host {fqdn}
+
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``bucket``
+
+:Description: The bucket to remove.
+:Type: String
+:Example: ``foo_bucket``
+:Required: Yes
+
+``purge-objects``
+
+:Description: Remove a buckets objects before deletion.
+:Type: Boolean
+:Example: True [False]
+:Required: No
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+None.
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``BucketNotEmpty``
+
+:Description: Attempted to delete non-empty bucket.
+:Code: 409 Conflict
+
+``ObjectRemovalFailed``
+
+:Description: Unable to remove objects.
+:Code: 409 Conflict
+
+Unlink Bucket
+=============
+
+Unlink a bucket from a specified user. Primarily useful for changing
+bucket ownership.
+
+:caps: buckets=write
+
+Syntax
+~~~~~~
+
+::
+
+	POST /{admin}/bucket?format=json HTTP/1.1
+	Host {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``bucket``
+
+:Description: The bucket to unlink.
+:Type: String
+:Example: ``foo_bucket``
+:Required: Yes
+
+``uid``
+
+:Description: The user ID to unlink the bucket from.
+:Type: String
+:Example: ``foo_user``
+:Required: Yes
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+None.
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``BucketUnlinkFailed``
+
+:Description: Unable to unlink bucket from specified user.
+:Code: 409 Conflict
+
+Link Bucket
+===========
+
+Link a bucket to a specified user, unlinking the bucket from
+any previous user.
+
+:caps: buckets=write
+
+Syntax
+~~~~~~
+
+::
+
+	PUT /{admin}/bucket?format=json HTTP/1.1
+	Host {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``bucket``
+
+:Description: The bucket name to unlink.
+:Type: String
+:Example: ``foo_bucket``
+:Required: Yes
+
+``bucket-id``
+
+:Description: The bucket id to unlink.
+:Type: String
+:Example: ``dev.6607669.420``
+:Required: No
+
+``uid``
+
+:Description: The user ID to link the bucket to.
+:Type: String
+:Example: ``foo_user``
+:Required: Yes
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+``bucket``
+
+:Description: Container for single bucket information.
+:Type: Container
+
+``name``
+
+:Description: The name of the bucket.
+:Type: String
+:Parent: ``bucket``
+
+``pool``
+
+:Description: The pool the bucket is stored in.
+:Type: String
+:Parent: ``bucket``
+
+``id``
+
+:Description: The unique bucket id.
+:Type: String
+:Parent: ``bucket``
+
+``marker``
+
+:Description: Internal bucket tag.
+:Type: String
+:Parent: ``bucket``
+
+``owner``
+
+:Description: The user id of the bucket owner.
+:Type: String
+:Parent: ``bucket``
+
+``usage``
+
+:Description: Storage usage information.
+:Type: Container
+:Parent: ``bucket``
+
+``index``
+
+:Description: Status of bucket index.
+:Type: String
+:Parent: ``bucket``
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``BucketUnlinkFailed``
+
+:Description: Unable to unlink bucket from specified user.
+:Code: 409 Conflict
+
+``BucketLinkFailed``
+
+:Description: Unable to link bucket to specified user.
+:Code: 409 Conflict
+
+Remove Object
+=============
+
+Remove an existing object. NOTE: Does not require owner to be non-suspended.
+
+:caps: buckets=write
+
+Syntax
+~~~~~~
+
+::
+
+	DELETE /{admin}/bucket?object&format=json HTTP/1.1
+	Host {fqdn}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``bucket``
+
+:Description: The bucket containing the object to be removed.
+:Type: String
+:Example: ``foo_bucket``
+:Required: Yes
+
+``object``
+
+:Description: The object to remove.
+:Type: String
+:Example: ``foo.txt``
+:Required: Yes
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+None.
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``NoSuchObject``
+
+:Description: Specified object does not exist.
+:Code: 404 Not Found
+
+``ObjectRemovalFailed``
+
+:Description: Unable to remove objects.
+:Code: 409 Conflict
+
+
+
+Get Bucket or Object Policy
+===========================
+
+Read the policy of an object or bucket.
+
+:caps: buckets=read
+
+Syntax
+~~~~~~
+
+::
+
+	GET /{admin}/bucket?policy&format=json HTTP/1.1
+	Host {fqdn}
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``bucket``
+
+:Description: The bucket to read the policy from.
+:Type: String
+:Example: ``foo_bucket``
+:Required: Yes
+
+``object``
+
+:Description: The object to read the policy from.
+:Type: String
+:Example: ``foo.txt``
+:Required: No
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If successful, returns the object or bucket policy
+
+``policy``
+
+:Description: Access control policy.
+:Type: Container
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``IncompleteBody``
+
+:Description: Either bucket was not specified for a bucket policy request or bucket
+              and object were not specified for an object policy request.
+:Code: 400 Bad Request
+
+Add A User Capability
+=====================
+
+Add an administrative capability to a specified user.
+
+:caps: users=write
+
+Syntax
+~~~~~~
+
+::
+
+	PUT /{admin}/user?caps&format=json HTTP/1.1
+	Host {fqdn}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``uid``
+
+:Description: The user ID to add an administrative capability to.
+:Type: String
+:Example: ``foo_user``
+:Required: Yes
+
+``user-caps``
+
+:Description: The administrative capability to add to the user.
+:Type: String
+:Example: ``usage=read,write;user=write``
+:Required: Yes
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If successful, the response contains the user's capabilities.
+
+``user``
+
+:Description: A container for the user data information.
+:Type: Container
+:Parent: ``user``
+
+``user_id``
+
+:Description: The user id.
+:Type: String
+:Parent: ``user``
+
+``caps``
+
+:Description: User capabilities.
+:Type: Container
+:Parent: ``user``
+
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``InvalidCapability``
+
+:Description: Attempt to grant invalid admin capability.
+:Code: 400 Bad Request
+
+Example Request
+~~~~~~~~~~~~~~~
+
+::
+
+	PUT /{admin}/user?caps&user-caps=usage=read,write;user=write&format=json HTTP/1.1
+	Host: {fqdn}
+	Content-Type: text/plain
+	Authorization: {your-authorization-token}
+
+
+
+Remove A User Capability
+========================
+
+Remove an administrative capability from a specified user.
+
+:caps: users=write
+
+Syntax
+~~~~~~
+
+::
+
+	DELETE /{admin}/user?caps&format=json HTTP/1.1
+	Host {fqdn}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``uid``
+
+:Description: The user ID to remove an administrative capability from.
+:Type: String
+:Example: ``foo_user``
+:Required: Yes
+
+``user-caps``
+
+:Description: The administrative capabilities to remove from the user.
+:Type: String
+:Example: ``usage=read, write``
+:Required: Yes
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+If successful, the response contains the user's capabilities.
+
+``user``
+
+:Description: A container for the user data information.
+:Type: Container
+:Parent: ``user``
+
+``user_id``
+
+:Description: The user id.
+:Type: String
+:Parent: ``user``
+
+``caps``
+
+:Description: User capabilities.
+:Type: Container
+:Parent: ``user``
+
+
+Special Error Responses
+~~~~~~~~~~~~~~~~~~~~~~~
+
+``InvalidCapability``
+
+:Description: Attempt to remove an invalid admin capability.
+:Code: 400 Bad Request
+
+``NoSuchCap``
+
+:Description: User does not possess specified capability.
+:Code: 404 Not Found
+
+
+Quotas
+======
+
+The Admin Operations API enables you to set quotas on users and on bucket owned
+by users. See `Quota Management`_ for additional details. Quotas include the
+maximum number of objects in a bucket and the maximum storage size in megabytes.
+
+To view quotas, the user must have a ``users=read`` capability. To set,
+modify or disable a quota, the user must have ``users=write`` capability.
+See the `Admin Guide`_ for details.
+
+Valid parameters for quotas include:
+
+- **Bucket:** The ``bucket`` option allows you to specify a quota for
+  buckets owned by a user.
+
+- **Maximum Objects:** The ``max-objects`` setting allows you to specify
+  the maximum number of objects. A negative value disables this setting.
+
+- **Maximum Size:** The ``max-size`` option allows you to specify a quota
+  for the maximum number of bytes. The ``max-size-kb`` option allows you
+  to specify it in KiB. A negative value disables this setting.
+
+- **Quota Type:** The ``quota-type`` option sets the scope for the quota.
+  The options are ``bucket`` and ``user``.
+
+- **Enable/Disable Quota:** The ``enabled`` option specifies whether the
+  quota should be enabled. The value should be either 'True' or 'False'.
+
+Get User Quota
+~~~~~~~~~~~~~~
+
+To get a quota, the user must have ``users`` capability set with ``read``
+permission. ::
+
+	GET /admin/user?quota&uid=<uid>&quota-type=user
+
+
+Set User Quota
+~~~~~~~~~~~~~~
+
+To set a quota, the user must have ``users`` capability set with ``write``
+permission. ::
+
+	PUT /admin/user?quota&uid=<uid>&quota-type=user
+
+
+The content must include a JSON representation of the quota settings
+as encoded in the corresponding read operation.
+
+
+Get Bucket Quota
+~~~~~~~~~~~~~~~~
+
+To get a quota, the user must have ``users`` capability set with ``read``
+permission. ::
+
+	GET /admin/user?quota&uid=<uid>&quota-type=bucket
+
+
+Set Bucket Quota
+~~~~~~~~~~~~~~~~
+
+To set a quota, the user must have ``users`` capability set with ``write``
+permission. ::
+
+	PUT /admin/user?quota&uid=<uid>&quota-type=bucket
+
+The content must include a JSON representation of the quota settings
+as encoded in the corresponding read operation.
+
+
+Set Quota for an Individual Bucket
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To set a quota, the user must have ``buckets`` capability set with ``write``
+permission. ::
+
+	PUT /admin/bucket?quota&uid=<uid>&bucket=<bucket-name>&quota
+
+The content must include a JSON representation of the quota settings
+as mentioned in Set Bucket Quota section above.
+
+
+
+Standard Error Responses
+========================
+
+``AccessDenied``
+
+:Description: Access denied.
+:Code: 403 Forbidden
+
+``InternalError``
+
+:Description: Internal server error.
+:Code: 500 Internal Server Error
+
+``NoSuchUser``
+
+:Description: User does not exist.
+:Code: 404 Not Found
+
+``NoSuchBucket``
+
+:Description: Bucket does not exist.
+:Code: 404 Not Found
+
+``NoSuchKey``
+
+:Description: No such access key.
+:Code: 404 Not Found
+
+
+
+
+Binding libraries
+========================
+
+``Golang``
+
+ - `IrekFasikhov/go-rgwadmin`_
+ - `QuentinPerez/go-radosgw`_
+
+``Java``
+ 
+ - `twonote/radosgw-admin4j`_
+
+``Python``
+
+ - `UMIACS/rgwadmin`_
+ - `valerytschopp/python-radosgw-admin`_
+
+
+
+.. _Admin Guide: ../admin
+.. _Quota Management: ../admin#quota-management
+.. _IrekFasikhov/go-rgwadmin: https://github.com/IrekFasikhov/go-rgwadmin
+.. _QuentinPerez/go-radosgw: https://github.com/QuentinPerez/go-radosgw
+.. _twonote/radosgw-admin4j: https://github.com/twonote/radosgw-admin4j
+.. _UMIACS/rgwadmin: https://github.com/UMIACS/rgwadmin
+.. _valerytschopp/python-radosgw-admin: https://github.com/valerytschopp/python-radosgw-admin
+
diff --git a/doc/radosgw/api.rst b/doc/radosgw/api.rst
new file mode 100644
index 000000000..c01a3e579
--- /dev/null
+++ b/doc/radosgw/api.rst
@@ -0,0 +1,14 @@
+===============
+librgw (Python)
+===============
+
+.. highlight:: python
+
+The `rgw` python module provides file-like access to rgw.
+
+API Reference
+=============
+
+.. automodule:: rgw
+    :members: LibRGWFS, FileHandle
+
diff --git a/doc/radosgw/archive-sync-module.rst b/doc/radosgw/archive-sync-module.rst
new file mode 100644
index 000000000..b121ee6b1
--- /dev/null
+++ b/doc/radosgw/archive-sync-module.rst
@@ -0,0 +1,44 @@
+===================
+Archive Sync Module
+===================
+
+.. versionadded:: Nautilus
+
+This sync module leverages the versioning feature of the S3 objects in RGW to
+have an archive zone that captures the different versions of the S3 objects
+as they occur over time in the other zones.
+
+An archive zone allows to have a history of versions of S3 objects that can
+only be eliminated through the gateways associated with the archive zone.
+
+This functionality is useful to have a configuration where several
+non-versioned zones replicate their data and metadata through their zone
+gateways (mirror configuration) providing high availability to the end users,
+while the archive zone captures all the data updates and metadata for
+consolidate them as versions of S3 objects.
+
+Including an archive zone in a multizone configuration allows you to have the
+flexibility of an S3 object history in one only zone while saving the space
+that the replicas of the versioned S3 objects would consume in the rest of the
+zones.
+
+
+
+Archive Sync Tier Type Configuration
+------------------------------------
+
+How to Configure
+~~~~~~~~~~~~~~~~
+
+See `Multisite Configuration`_ for how to multisite config instructions. The
+archive sync module requires a creation of a new zone. The zone tier type needs
+to be defined as ``archive``:
+
+::
+
+    # radosgw-admin zone create --rgw-zonegroup={zone-group-name} \
+                                --rgw-zone={zone-name} \
+                                --endpoints={http://fqdn}[,{http://fqdn}]
+                                --tier-type=archive
+
+.. _Multisite Configuration: ../multisite
diff --git a/doc/radosgw/barbican.rst b/doc/radosgw/barbican.rst
new file mode 100644
index 000000000..a90d063fb
--- /dev/null
+++ b/doc/radosgw/barbican.rst
@@ -0,0 +1,123 @@
+==============================
+OpenStack Barbican Integration
+==============================
+
+OpenStack `Barbican`_ can be used as a secure key management service for
+`Server-Side Encryption`_.
+
+.. image:: ../images/rgw-encryption-barbican.png
+
+#. `Configure Keystone`_
+#. `Create a Keystone user`_
+#. `Configure the Ceph Object Gateway`_
+#. `Create a key in Barbican`_
+
+Configure Keystone
+==================
+
+Barbican depends on Keystone for authorization and access control of its keys.
+
+See `OpenStack Keystone Integration`_.
+
+Create a Keystone user
+======================
+
+Create a new user that will be used by the Ceph Object Gateway to retrieve
+keys.
+
+For example::
+
+   user = rgwcrypt-user
+   pass = rgwcrypt-password
+   tenant = rgwcrypt
+
+See OpenStack documentation for `Manage projects, users, and roles`_.
+
+Create a key in Barbican
+========================
+
+See Barbican documentation for `How to Create a Secret`_. Requests to
+Barbican must include a valid Keystone token in the ``X-Auth-Token`` header.
+
+.. note:: Server-side encryption keys must be 256-bit long and base64 encoded.
+
+Example request::
+
+   POST /v1/secrets HTTP/1.1
+   Host: barbican.example.com:9311
+   Accept: */*
+   Content-Type: application/json
+   X-Auth-Token: 7f7d588dd29b44df983bc961a6b73a10
+   Content-Length: 299
+   
+   {
+       "name": "my-key",
+       "expiration": "2016-12-28T19:14:44.180394",
+       "algorithm": "aes",
+       "bit_length": 256,
+       "mode": "cbc",
+       "payload": "6b+WOZ1T3cqZMxgThRcXAQBrS5mXKdDUphvpxptl9/4=",
+       "payload_content_type": "application/octet-stream",
+       "payload_content_encoding": "base64"
+   }
+
+Response::
+
+   {"secret_ref": "http://barbican.example.com:9311/v1/secrets/d1e7ef3b-f841-4b7c-90b2-b7d90ca2d723"}
+
+In the response, ``d1e7ef3b-f841-4b7c-90b2-b7d90ca2d723`` is the key id that
+can be used in any `SSE-KMS`_ request.
+
+This newly created key is not accessible by user ``rgwcrypt-user``. This
+privilege must be added with an ACL. See `How to Set/Replace ACL`_ for more
+details.
+
+Example request (assuming that the Keystone id of ``rgwcrypt-user`` is
+``906aa90bd8a946c89cdff80d0869460f``)::
+
+   PUT /v1/secrets/d1e7ef3b-f841-4b7c-90b2-b7d90ca2d723/acl HTTP/1.1
+   Host: barbican.example.com:9311
+   Accept: */*
+   Content-Type: application/json
+   X-Auth-Token: 7f7d588dd29b44df983bc961a6b73a10
+   Content-Length: 101
+
+   {
+       "read":{
+       "users":[ "906aa90bd8a946c89cdff80d0869460f" ],
+       "project-access": true
+       }
+   }
+
+Response::
+
+   {"acl_ref": "http://barbican.example.com:9311/v1/secrets/d1e7ef3b-f841-4b7c-90b2-b7d90ca2d723/acl"}
+
+Configure the Ceph Object Gateway
+=================================
+
+Edit the Ceph configuration file to enable Barbican as a KMS and add information
+about the Barbican server and Keystone user::
+
+   rgw crypt s3 kms backend = barbican
+   rgw barbican url = http://barbican.example.com:9311
+   rgw keystone barbican user = rgwcrypt-user
+   rgw keystone barbican password = rgwcrypt-password
+
+When using Keystone API version 2::
+
+   rgw keystone barbican tenant = rgwcrypt
+
+When using API version 3::
+
+   rgw keystone barbican project
+   rgw keystone barbican domain
+
+
+.. _Barbican: https://wiki.openstack.org/wiki/Barbican
+.. _Server-Side Encryption: ../encryption
+.. _OpenStack Keystone Integration: ../keystone
+.. _Manage projects, users, and roles: https://docs.openstack.org/admin-guide/cli-manage-projects-users-and-roles.html#create-a-user
+.. _How to Create a Secret: https://developer.openstack.org/api-guide/key-manager/secrets.html#how-to-create-a-secret
+.. _SSE-KMS: http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html
+.. _How to Set/Replace ACL: https://developer.openstack.org/api-guide/key-manager/acls.html#how-to-set-replace-acl
diff --git a/doc/radosgw/bucketpolicy.rst b/doc/radosgw/bucketpolicy.rst
new file mode 100644
index 000000000..5d6dddc74
--- /dev/null
+++ b/doc/radosgw/bucketpolicy.rst
@@ -0,0 +1,216 @@
+===============
+Bucket Policies
+===============
+
+.. versionadded:: Luminous
+
+The Ceph Object Gateway supports a subset of the Amazon S3 policy
+language applied to buckets.
+
+
+Creation and Removal
+====================
+
+Bucket policies are managed through standard S3 operations rather than
+radosgw-admin.
+
+For example, one may use s3cmd to set or delete a policy thus::
+
+  $ cat > examplepol
+  {
+    "Version": "2012-10-17",
+    "Statement": [{
+      "Effect": "Allow",
+      "Principal": {"AWS": ["arn:aws:iam::usfolks:user/fred:subuser"]},
+      "Action": "s3:PutObjectAcl",
+      "Resource": [
+        "arn:aws:s3:::happybucket/*"
+      ]
+    }]
+  }
+
+  $ s3cmd setpolicy examplepol s3://happybucket
+  $ s3cmd delpolicy s3://happybucket
+
+
+Limitations
+===========
+
+Currently, we support only the following actions:
+
+- s3:AbortMultipartUpload
+- s3:CreateBucket
+- s3:DeleteBucketPolicy
+- s3:DeleteBucket
+- s3:DeleteBucketWebsite
+- s3:DeleteObject
+- s3:DeleteObjectVersion
+- s3:DeleteReplicationConfiguration
+- s3:GetAccelerateConfiguration
+- s3:GetBucketAcl
+- s3:GetBucketCORS
+- s3:GetBucketLocation
+- s3:GetBucketLogging
+- s3:GetBucketNotification
+- s3:GetBucketPolicy
+- s3:GetBucketRequestPayment
+- s3:GetBucketTagging
+- s3:GetBucketVersioning
+- s3:GetBucketWebsite
+- s3:GetLifecycleConfiguration
+- s3:GetObjectAcl
+- s3:GetObject
+- s3:GetObjectTorrent
+- s3:GetObjectVersionAcl
+- s3:GetObjectVersion
+- s3:GetObjectVersionTorrent
+- s3:GetReplicationConfiguration
+- s3:IPAddress
+- s3:NotIpAddress
+- s3:ListAllMyBuckets
+- s3:ListBucketMultipartUploads
+- s3:ListBucket
+- s3:ListBucketVersions
+- s3:ListMultipartUploadParts
+- s3:PutAccelerateConfiguration
+- s3:PutBucketAcl
+- s3:PutBucketCORS
+- s3:PutBucketLogging
+- s3:PutBucketNotification
+- s3:PutBucketPolicy
+- s3:PutBucketRequestPayment
+- s3:PutBucketTagging
+- s3:PutBucketVersioning
+- s3:PutBucketWebsite
+- s3:PutLifecycleConfiguration
+- s3:PutObjectAcl
+- s3:PutObject
+- s3:PutObjectVersionAcl
+- s3:PutReplicationConfiguration
+- s3:RestoreObject
+
+We do not yet support setting policies on users, groups, or roles.
+
+We use the RGW ‘tenant’ identifier in place of the Amazon twelve-digit
+account ID. In the future we may allow you to assign an account ID to
+a tenant, but for now if you want to use policies between AWS S3 and
+RGW S3 you will have to use the Amazon account ID as the tenant ID when
+creating users.
+
+Under AWS, all tenants share a single namespace. RGW gives every
+tenant its own namespace of buckets. There may be an option to enable
+an AWS-like 'flat' bucket namespace in future versions. At present, to
+access a bucket belonging to another tenant, address it as
+"tenant:bucket" in the S3 request.
+
+In AWS, a bucket policy can grant access to another account, and that
+account owner can then grant access to individual users with user
+permissions. Since we do not yet support user, role, and group
+permissions, account owners will currently need to grant access
+directly to individual users, and granting an entire account access to
+a bucket grants access to all users in that account.
+
+Bucket policies do not yet support string interpolation.
+
+For all requests, condition keys we support are:
+- aws:CurrentTime
+- aws:EpochTime
+- aws:PrincipalType
+- aws:Referer
+- aws:SecureTransport
+- aws:SourceIp
+- aws:UserAgent
+- aws:username
+
+We support certain s3 condition keys for bucket and object requests.
+
+.. versionadded:: Mimic
+
+Bucket Related Operations
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
++-----------------------+----------------------+----------------+
+| Permission            | Condition Keys       | Comments       |
++-----------------------+----------------------+----------------+
+|                       | s3:x-amz-acl         |                |
+|                       | s3:x-amz-grant-<perm>|                |
+|s3:createBucket        | where perm is one of |                |
+|                       | read/write/read-acp  |                |
+|                       | write-acp/           |                |
+|                       | full-control         |                |
++-----------------------+----------------------+----------------+
+|                       | s3:prefix            |                |
+|                       +----------------------+----------------+
+| s3:ListBucket &       | s3:delimiter         |                |
+|                       +----------------------+----------------+
+| s3:ListBucketVersions | s3:max-keys          |                |
++-----------------------+----------------------+----------------+
+| s3:PutBucketAcl       | s3:x-amz-acl         |                |
+|                       | s3:x-amz-grant-<perm>|                |
++-----------------------+----------------------+----------------+
+
+.. _tag_policy:
+
+Object Related Operations
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
++-----------------------------+-----------------------------------------------+-------------------+
+|Permission                   |Condition Keys                                 | Comments          |
+|                             |                                               |                   |
++-----------------------------+-----------------------------------------------+-------------------+
+|                             |s3:x-amz-acl & s3:x-amz-grant-<perm>           |                   |
+|                             |                                               |                   |
+|                             +-----------------------------------------------+-------------------+
+|                             |s3:x-amz-copy-source                           |                   |
+|                             |                                               |                   |
+|                             +-----------------------------------------------+-------------------+
+|                             |s3:x-amz-server-side-encryption                |                   |
+|                             |                                               |                   |
+|                             +-----------------------------------------------+-------------------+
+|s3:PutObject                 |s3:x-amz-server-side-encryption-aws-kms-key-id |                   |
+|                             |                                               |                   |
+|                             +-----------------------------------------------+-------------------+
+|                             |s3:x-amz-metadata-directive                    |PUT & COPY to      |
+|                             |                                               |overwrite/preserve |
+|                             |                                               |metadata in COPY   |
+|                             |                                               |requests           |
+|                             +-----------------------------------------------+-------------------+
+|                             |s3:RequestObjectTag/<tag-key>                  |                   |
+|                             |                                               |                   |
++-----------------------------+-----------------------------------------------+-------------------+
+|s3:PutObjectAcl              |s3:x-amz-acl & s3-amz-grant-<perm>             |                   |
+|s3:PutObjectVersionAcl       |                                               |                   |
+|                             +-----------------------------------------------+-------------------+
+|                             |s3:ExistingObjectTag/<tag-key>                 |                   |
+|                             |                                               |                   |
++-----------------------------+-----------------------------------------------+-------------------+
+|                             |s3:RequestObjectTag/<tag-key>                  |                   |
+|s3:PutObjectTagging &        +-----------------------------------------------+-------------------+
+|s3:PutObjectVersionTagging   |s3:ExistingObjectTag/<tag-key>                 |                   |
+|                             |                                               |                   |
++-----------------------------+-----------------------------------------------+-------------------+
+|s3:GetObject &               |s3:ExistingObjectTag/<tag-key>                 |                   |
+|s3:GetObjectVersion          |                                               |                   |
++-----------------------------+-----------------------------------------------+-------------------+
+|s3:GetObjectAcl &            |s3:ExistingObjectTag/<tag-key>                 |                   |
+|s3:GetObjectVersionAcl       |                                               |                   |
++-----------------------------+-----------------------------------------------+-------------------+
+|s3:GetObjectTagging &        |s3:ExistingObjectTag/<tag-key>                 |                   |
+|s3:GetObjectVersionTagging   |                                               |                   |
++-----------------------------+-----------------------------------------------+-------------------+
+|s3:DeleteObjectTagging &     |s3:ExistingOBjectTag/<tag-key>                 |                   |
+|s3:DeleteObjectVersionTagging|                                               |                   |
++-----------------------------+-----------------------------------------------+-------------------+
+
+
+More may be supported soon as we integrate with the recently rewritten
+Authentication/Authorization subsystem.
+
+Swift
+=====
+
+There is no way to set bucket policies under Swift, but bucket
+policies that have been set govern Swift as well as S3 operations.
+
+Swift credentials are matched against Principals specified in a policy
+in a way specific to whatever backend is being used.
diff --git a/doc/radosgw/cloud-sync-module.rst b/doc/radosgw/cloud-sync-module.rst
new file mode 100644
index 000000000..bea7f441b
--- /dev/null
+++ b/doc/radosgw/cloud-sync-module.rst
@@ -0,0 +1,244 @@
+=========================
+Cloud Sync Module
+=========================
+
+.. versionadded:: Mimic
+
+This module syncs zone data to a remote cloud service. The sync is unidirectional; data is not synced back from the
+remote zone. The goal of this module is to enable syncing data to multiple cloud providers. The currently supported
+cloud providers are those that are compatible with AWS (S3).
+
+User credentials for the remote cloud object store service need to be configured. Since many cloud services impose limits
+on the number of buckets that each user can create, the mapping of source objects and buckets is configurable.
+It is possible to configure different targets to different buckets and bucket prefixes. Note that source ACLs will not
+be preserved. It is possible to map permissions of specific source users to specific destination users.
+
+Due to API limitations there is no way to preserve original object modification time and ETag. The cloud sync module 
+stores these as metadata attributes on the destination objects.
+
+
+
+Cloud Sync Tier Type Configuration
+-------------------------------------
+
+Trivial Configuration:
+~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+    {
+      "connection": {
+        "access_key": <access>,
+        "secret": <secret>,
+        "endpoint": <endpoint>,
+        "host_style": <path | virtual>,
+      },
+      "acls": [ { "type": <id | email | uri>,
+                  "source_id": <source_id>,
+                  "dest_id": <dest_id> } ... ],
+      "target_path": <target_path>,
+    }
+
+
+Non Trivial Configuration:
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+    {
+      "default": {
+        "connection": {
+            "access_key": <access>,
+            "secret": <secret>,
+            "endpoint": <endpoint>,
+            "host_style" <path | virtual>,
+        },
+        "acls": [
+        {
+          "type" : <id | email | uri>,   #  optional, default is id
+          "source_id": <id>,
+          "dest_id": <id>
+        } ... ]
+        "target_path": <path> # optional
+      },
+      "connections": [
+          {
+            "connection_id": <id>,
+            "access_key": <access>,
+            "secret": <secret>,
+            "endpoint": <endpoint>,
+            "host_style" <path | virtual>,  # optional
+          } ... ],
+      "acl_profiles": [
+          {
+            "acls_id": <id>, # acl mappings
+            "acls": [ {
+                "type": <id | email | uri>,
+                "source_id": <id>,
+                "dest_id": <id>
+              } ... ]
+          }
+      ],
+      "profiles": [
+          {
+           "source_bucket": <source>,
+           "connection_id": <connection_id>,
+           "acls_id": <mappings_id>,
+           "target_path": <dest>,          # optional
+          } ... ],
+    }
+
+
+.. Note:: Trivial configuration can coincide with the non-trivial one.
+
+
+* ``connection`` (container)
+
+Represents a connection to the remote cloud service. Contains ``conection_id`, ``access_key``,
+``secret``, ``endpoint``, and ``host_style``.
+
+* ``access_key`` (string)
+
+The remote cloud access key that will be used for a specific connection.
+
+* ``secret`` (string)
+
+The secret key for the remote cloud service.
+
+* ``endpoint`` (string)
+
+URL of remote cloud service endpoint.
+
+* ``host_style`` (path | virtual)
+
+Type of host style to be used when accessing remote cloud endpoint (default: ``path``).
+
+* ``acls`` (array)
+
+Contains a list of ``acl_mappings``.
+
+* ``acl_mapping`` (container)
+
+Each ``acl_mapping`` structure contains ``type``, ``source_id``, and ``dest_id``. These
+will define the ACL mutation that will be done on each object. An ACL mutation allows converting source
+user id to a destination id.
+
+* ``type`` (id | email | uri)
+
+ACL type: ``id`` defines user id, ``email`` defines user by email, and ``uri`` defines user by ``uri`` (group).
+
+* ``source_id`` (string)
+
+ID of user in the source zone.
+
+* ``dest_id`` (string)
+
+ID of user in the destination.
+
+* ``target_path`` (string)
+
+A string that defines how the target path is created. The target path specifies a prefix to which
+the source object name is appended. The target path configurable can include any of the following
+variables:
+- ``sid``: unique string that represents the sync instance ID
+- ``zonegroup``: the zonegroup name
+- ``zonegroup_id``: the zonegroup ID
+- ``zone``: the zone name
+- ``zone_id``: the zone id
+- ``bucket``: source bucket name
+- ``owner``: source bucket owner ID
+
+For example: ``target_path = rgwx-${zone}-${sid}/${owner}/${bucket}``
+
+
+* ``acl_profiles`` (array)
+
+An array of ``acl_profile``.
+
+* ``acl_profile`` (container)
+ 
+Each profile contains ``acls_id`` (string) that represents the profile, and ``acls`` array that
+holds a list of ``acl_mappings``.
+
+* ``profiles`` (array)
+
+A list of profiles. Each profile contains the following:
+- ``source_bucket``: either a bucket name, or a bucket prefix (if ends with ``*``) that defines the source bucket(s) for this profile
+- ``target_path``: as defined above
+- ``connection_id``: ID of the connection that will be used for this profile
+- ``acls_id``: ID of ACLs profile that will be used for this profile
+
+
+S3 Specific Configurables:
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Currently cloud sync will only work with backends that are compatible with AWS S3. There are
+a few configurables that can be used to tweak its behavior when accessing these cloud services:
+
+::
+
+    {
+      "multipart_sync_threshold": {object_size},
+      "multipart_min_part_size": {part_size}
+    }
+
+
+* ``multipart_sync_threshold`` (integer)
+
+Objects this size or larger will be synced to the cloud using multipart upload.
+
+* ``multipart_min_part_size`` (integer)
+
+Minimum parts size to use when syncing objects using multipart upload.
+
+
+How to Configure
+~~~~~~~~~~~~~~~~
+
+See :ref:`multisite` for how to multisite config instructions. The cloud sync module requires a creation of a new zone. The zone
+tier type needs to be defined as ``cloud``:
+
+::
+
+    # radosgw-admin zone create --rgw-zonegroup={zone-group-name} \
+                                --rgw-zone={zone-name} \
+                                --endpoints={http://fqdn}[,{http://fqdn}]
+                                --tier-type=cloud
+
+
+The tier configuration can be then done using the following command
+
+::
+
+    # radosgw-admin zone modify --rgw-zonegroup={zone-group-name} \
+                                --rgw-zone={zone-name} \
+                                --tier-config={key}={val}[,{key}={val}]
+
+The ``key`` in the configuration specifies the config variable that needs to be updated, and
+the ``val`` specifies its new value. Nested values can be accessed using period. For example:
+
+::
+
+    # radosgw-admin zone modify --rgw-zonegroup={zone-group-name} \
+                                --rgw-zone={zone-name} \
+                                --tier-config=connection.access_key={key},connection.secret={secret}
+
+
+Configuration array entries can be accessed by specifying the specific entry to be referenced enclosed
+in square brackets, and adding new array entry can be done by using `[]`. Index value of `-1` references
+the last entry in the array. At the moment it is not possible to create a new entry and reference it
+again at the same command.
+For example, creating a new profile for buckets starting with {prefix}:
+
+::
+
+    # radosgw-admin zone modify --rgw-zonegroup={zone-group-name} \
+                                --rgw-zone={zone-name} \
+                                --tier-config=profiles[].source_bucket={prefix}'*'
+
+    # radosgw-admin zone modify --rgw-zonegroup={zone-group-name} \
+                                --rgw-zone={zone-name} \
+                                --tier-config=profiles[-1].connection_id={conn_id},profiles[-1].acls_id={acls_id}
+
+
+An entry can be removed by using ``--tier-config-rm={key}``.
diff --git a/doc/radosgw/compression.rst b/doc/radosgw/compression.rst
new file mode 100644
index 000000000..23655f1dc
--- /dev/null
+++ b/doc/radosgw/compression.rst
@@ -0,0 +1,87 @@
+===========
+Compression
+===========
+
+.. versionadded:: Kraken
+
+The Ceph Object Gateway supports server-side compression of uploaded objects,
+using any of Ceph's existing compression plugins.
+
+
+Configuration
+=============
+
+Compression can be enabled on a storage class in the Zone's placement target
+by providing the ``--compression=<type>`` option to the command
+``radosgw-admin zone placement modify``.
+
+The compression ``type`` refers to the name of the compression plugin to use
+when writing new object data. Each compressed object remembers which plugin
+was used, so changing this setting does not hinder the ability to decompress
+existing objects, nor does it force existing objects to be recompressed.
+
+This compression setting applies to all new objects uploaded to buckets using
+this placement target. Compression can be disabled by setting the ``type`` to
+an empty string or ``none``.
+
+For example::
+
+  $ radosgw-admin zone placement modify \
+        --rgw-zone default \
+        --placement-id default-placement \
+        --storage-class STANDARD \
+        --compression zlib
+  {
+  ...
+      "placement_pools": [
+          {
+              "key": "default-placement",
+              "val": {
+                  "index_pool": "default.rgw.buckets.index",
+                  "storage_classes": {
+                      "STANDARD": {
+                          "data_pool": "default.rgw.buckets.data",
+                          "compression_type": "zlib"
+                      }
+                  },
+                  "data_extra_pool": "default.rgw.buckets.non-ec",
+                  "index_type": 0,
+              }
+          }
+      ],
+  ...
+  }
+
+.. note:: A ``default`` zone is created for you if you have not done any
+   previous `Multisite Configuration`_.
+
+
+Statistics
+==========
+
+While all existing commands and APIs continue to report object and bucket
+sizes based their uncompressed data, compression statistics for a given bucket
+are included in its ``bucket stats``::
+
+  $ radosgw-admin bucket stats --bucket=<name>
+  {
+  ...
+      "usage": {
+          "rgw.main": {
+              "size": 1075028,
+              "size_actual": 1331200,
+              "size_utilized": 592035,
+              "size_kb": 1050,
+              "size_kb_actual": 1300,
+              "size_kb_utilized": 579,
+              "num_objects": 104
+          }
+      },
+  ...
+  }
+
+The ``size_utilized`` and ``size_kb_utilized`` fields represent the total
+size of compressed data, in bytes and kilobytes respectively.
+
+
+.. _`Multisite Configuration`: ../multisite
diff --git a/doc/radosgw/config-ref.rst b/doc/radosgw/config-ref.rst
new file mode 100644
index 000000000..1ed6085a6
--- /dev/null
+++ b/doc/radosgw/config-ref.rst
@@ -0,0 +1,1161 @@
+======================================
+ Ceph Object Gateway Config Reference
+======================================
+
+The following settings may added to the Ceph configuration file (i.e., usually
+``ceph.conf``) under the ``[client.radosgw.{instance-name}]`` section. The
+settings may contain default values. If you do not specify each setting in the
+Ceph configuration file, the default value will be set automatically.
+
+Configuration variables set under the ``[client.radosgw.{instance-name}]``
+section will not apply to rgw or radosgw-admin commands without an instance-name
+specified in the command. Thus variables meant to be applied to all RGW
+instances or all radosgw-admin options can be put into the ``[global]`` or the
+``[client]`` section to avoid specifying ``instance-name``.
+
+``rgw_frontends``
+
+:Description: Configures the HTTP frontend(s). The configuration for multiple
+              frontends can be provided in a comma-delimited list. Each frontend
+              configuration may include a list of options separated by spaces,
+              where each option is in the form "key=value" or "key". See
+              `HTTP Frontends`_ for more on supported options.
+
+:Type: String
+:Default: ``beast port=7480``
+
+``rgw_data``
+
+:Description: Sets the location of the data files for Ceph RADOS Gateway.
+:Type: String
+:Default: ``/var/lib/ceph/radosgw/$cluster-$id``
+
+
+``rgw_enable_apis``
+
+:Description: Enables the specified APIs.
+
+              .. note:: Enabling the ``s3`` API is a requirement for
+                        any ``radosgw`` instance that is meant to
+                        participate in a `multi-site <../multisite>`_
+                        configuration.
+:Type: String
+:Default: ``s3, s3website, swift, swift_auth, admin, sts, iam, notifications`` All APIs.
+
+
+``rgw_cache_enabled``
+
+:Description: Whether the Ceph Object Gateway cache is enabled.
+:Type: Boolean
+:Default: ``true``
+
+
+``rgw_cache_lru_size``
+
+:Description: The number of entries in the Ceph Object Gateway cache.
+:Type: Integer
+:Default: ``10000``
+
+
+``rgw_socket_path``
+
+:Description: The socket path for the domain socket. ``FastCgiExternalServer``
+              uses this socket. If you do not specify a socket path, Ceph
+              Object Gateway will not run as an external server. The path you
+              specify here must be the same as the path specified in the
+              ``rgw.conf`` file.
+
+:Type: String
+:Default: N/A
+
+``rgw_fcgi_socket_backlog``
+
+:Description: The socket backlog for fcgi.
+:Type: Integer
+:Default: ``1024``
+
+``rgw_host``
+
+:Description: The host for the Ceph Object Gateway instance. Can be an IP
+              address or a hostname.
+
+:Type: String
+:Default: ``0.0.0.0``
+
+
+``rgw_port``
+
+:Description: Port the instance listens for requests. If not specified,
+              Ceph Object Gateway runs external FastCGI.
+
+:Type: String
+:Default: None
+
+
+``rgw_dns_name``
+
+:Description: The DNS name of the served domain. See also the ``hostnames`` setting within regions.
+:Type: String
+:Default: None
+
+
+``rgw_script_uri``
+
+:Description: The alternative value for the ``SCRIPT_URI`` if not set
+              in the request.
+
+:Type: String
+:Default: None
+
+
+``rgw_request_uri``
+
+:Description: The alternative value for the ``REQUEST_URI`` if not set
+              in the request.
+
+:Type: String
+:Default: None
+
+
+``rgw_print_continue``
+
+:Description: Enable ``100-continue`` if it is operational.
+:Type: Boolean
+:Default: ``true``
+
+
+``rgw_remote_addr_param``
+
+:Description: The remote address parameter. For example, the HTTP field
+              containing the remote address, or the ``X-Forwarded-For``
+              address if a reverse proxy is operational.
+
+:Type: String
+:Default: ``REMOTE_ADDR``
+
+
+``rgw_op_thread_timeout``
+
+:Description: The timeout in seconds for open threads.
+:Type: Integer
+:Default: 600
+
+
+``rgw_op_thread_suicide_timeout``
+
+:Description: The time ``timeout`` in seconds before a Ceph Object Gateway
+              process dies. Disabled if set to ``0``.
+
+:Type: Integer
+:Default: ``0``
+
+
+``rgw_thread_pool_size``
+
+:Description: The size of the thread pool.
+:Type: Integer
+:Default: 100 threads.
+
+
+``rgw_num_control_oids``
+
+:Description: The number of notification objects used for cache synchronization
+              between different ``rgw`` instances.
+
+:Type: Integer
+:Default: ``8``
+
+
+``rgw_init_timeout``
+
+:Description: The number of seconds before Ceph Object Gateway gives up on
+              initialization.
+
+:Type: Integer
+:Default: ``30``
+
+
+``rgw_mime_types_file``
+
+:Description: The path and location of the MIME-types file. Used for Swift
+              auto-detection of object types.
+
+:Type: String
+:Default: ``/etc/mime.types``
+
+
+``rgw_s3_success_create_obj_status``
+
+:Description: The alternate success status response for ``create-obj``.
+:Type: Integer
+:Default: ``0``
+
+
+``rgw_resolve_cname``
+
+:Description: Whether ``rgw`` should use DNS CNAME record of the request
+              hostname field (if hostname is not equal to ``rgw dns name``).
+
+:Type: Boolean
+:Default: ``false``
+
+
+``rgw_obj_stripe_size``
+
+:Description: The size of an object stripe for Ceph Object Gateway objects.
+              See `Architecture`_ for details on striping.
+
+:Type: Integer
+:Default: ``4 << 20``
+
+
+``rgw_extended_http_attrs``
+
+:Description: Add new set of attributes that could be set on an entity
+              (user, bucket or object). These extra attributes can be set
+              through HTTP header fields when putting the entity or modifying
+              it using POST method. If set, these attributes will return as
+              HTTP  fields when doing GET/HEAD on the entity.
+
+:Type: String
+:Default: None
+:Example: "content_foo, content_bar, x-foo-bar"
+
+
+``rgw_exit_timeout_secs``
+
+:Description: Number of seconds to wait for a process before exiting
+              unconditionally.
+
+:Type: Integer
+:Default: ``120``
+
+
+``rgw_get_obj_window_size``
+
+:Description: The window size in bytes for a single object request.
+:Type: Integer
+:Default: ``16 << 20``
+
+
+``rgw_get_obj_max_req_size``
+
+:Description: The maximum request size of a single get operation sent to the
+              Ceph Storage Cluster.
+
+:Type: Integer
+:Default: ``4 << 20``
+
+
+``rgw_relaxed_s3_bucket_names``
+
+:Description: Enables relaxed S3 bucket names rules for US region buckets.
+:Type: Boolean
+:Default: ``false``
+
+
+``rgw_list_buckets_max_chunk``
+
+:Description: The maximum number of buckets to retrieve in a single operation
+              when listing user buckets.
+
+:Type: Integer
+:Default: ``1000``
+
+
+``rgw_override_bucket_index_max_shards``
+
+:Description: Represents the number of shards for the bucket index object,
+              a value of zero indicates there is no sharding. It is not
+              recommended to set a value too large (e.g. thousand) as it
+              increases the cost for bucket listing.
+              This variable should be set in the client or global sections
+              so that it is automatically applied to radosgw-admin commands.
+
+:Type: Integer
+:Default: ``0``
+
+
+``rgw_curl_wait_timeout_ms``
+
+:Description: The timeout in milliseconds for certain ``curl`` calls.
+:Type: Integer
+:Default: ``1000``
+
+
+``rgw_copy_obj_progress``
+
+:Description: Enables output of object progress during long copy operations.
+:Type: Boolean
+:Default: ``true``
+
+
+``rgw_copy_obj_progress_every_bytes``
+
+:Description: The minimum bytes between copy progress output.
+:Type: Integer
+:Default: ``1024 * 1024``
+
+
+``rgw_admin_entry``
+
+:Description: The entry point for an admin request URL.
+:Type: String
+:Default: ``admin``
+
+
+``rgw_content_length_compat``
+
+:Description: Enable compatibility handling of FCGI requests with both ``CONTENT_LENGTH`` and ``HTTP_CONTENT_LENGTH`` set.
+:Type: Boolean
+:Default: ``false``
+
+
+``rgw_bucket_quota_ttl``
+
+:Description: The amount of time in seconds cached quota information is
+              trusted.  After this timeout, the quota information will be
+              re-fetched from the cluster.
+:Type: Integer
+:Default: ``600``
+
+
+``rgw_user_quota_bucket_sync_interval``
+
+:Description: The amount of time in seconds bucket quota information is
+              accumulated before syncing to the cluster.  During this time,
+              other RGW instances will not see the changes in bucket quota
+              stats from operations on this instance.
+:Type: Integer
+:Default: ``180``
+
+
+``rgw_user_quota_sync_interval``
+
+:Description: The amount of time in seconds user quota information is
+              accumulated before syncing to the cluster.  During this time,
+              other RGW instances will not see the changes in user quota stats
+              from operations on this instance.
+:Type: Integer
+:Default: ``180``
+
+
+``rgw_bucket_default_quota_max_objects``
+
+:Description: Default max number of objects per bucket. Set on new users,
+              if no other quota is specified. Has no effect on existing users.
+              This variable should be set in the client or global sections
+              so that it is automatically applied to radosgw-admin commands.
+:Type: Integer
+:Default: ``-1``
+
+
+``rgw_bucket_default_quota_max_size``
+
+:Description: Default max capacity per bucket, in bytes. Set on new users,
+              if no other quota is specified. Has no effect on existing users.
+:Type: Integer
+:Default: ``-1``
+
+
+``rgw_user_default_quota_max_objects``
+
+:Description: Default max number of objects for a user. This includes all
+              objects in all buckets owned by the user. Set on new users,
+              if no other quota is specified. Has no effect on existing users.
+:Type: Integer
+:Default: ``-1``
+
+
+``rgw_user_default_quota_max_size``
+
+:Description: The value for user max size quota in bytes set on new users,
+              if no other quota is specified.  Has no effect on existing users.
+:Type: Integer
+:Default: ``-1``
+
+
+``rgw_verify_ssl``
+
+:Description: Verify SSL certificates while making requests.
+:Type: Boolean
+:Default: ``true``
+
+Lifecycle Settings
+==================
+
+Bucket Lifecycle configuration can be used to manage your objects so they are stored
+effectively throughout their lifetime. In past releases Lifecycle processing was rate-limited
+by single threaded processing. With the Nautilus release this has been addressed and the
+Ceph Object Gateway now allows for parallel thread processing of bucket lifecycles across
+additional Ceph Object Gateway instances and replaces the in-order
+index shard enumeration with a random ordered sequence.
+
+There are two options in particular to look at when looking to increase the
+aggressiveness of lifecycle processing:
+
+``rgw_lc_max_worker``
+
+:Description: This option specifies the number of lifecycle worker threads
+              to run in parallel, thereby processing bucket and index
+              shards simultaneously.
+
+:Type: Integer
+:Default: ``3``
+
+``rgw_lc_max_wp_worker``
+
+:Description: This option specifies the number of threads in each lifecycle
+              workers work pool. This option can help accelerate processing each bucket.
+
+These values can be tuned based upon your specific workload to further increase the
+aggressiveness of lifecycle processing. For a workload with a larger number of buckets (thousands)
+you would look at increasing the ``rgw_lc_max_worker`` value from the default value of 3 whereas for a
+workload with a smaller number of buckets but higher number of objects (hundreds of thousands)
+per bucket you would consider decreasing ``rgw_lc_max_wp_worker`` from the default value of 3.
+
+:NOTE: When looking to tune either of these specific values please validate the
+       current Cluster performance and Ceph Object Gateway utilization before increasing.
+
+Garbage Collection Settings
+===========================
+
+The Ceph Object Gateway allocates storage for new objects immediately.
+
+The Ceph Object Gateway purges the storage space used for deleted and overwritten 
+objects in the Ceph Storage cluster some time after the gateway deletes the 
+objects from the bucket index. The process of purging the deleted object data 
+from the Ceph Storage cluster is known as Garbage Collection or GC.
+
+To view the queue of objects awaiting garbage collection, execute the following::
+
+  $ radosgw-admin gc list 
+
+  Note: specify ``--include-all`` to list all entries, including unexpired
+  
+Garbage collection is a background activity that may
+execute continuously or during times of low loads, depending upon how the
+administrator configures the Ceph Object Gateway. By default, the Ceph Object
+Gateway conducts GC operations continuously. Since GC operations are a normal
+part of Ceph Object Gateway operations, especially with object delete
+operations, objects eligible for garbage collection exist most of the time.
+
+Some workloads may temporarily or permanently outpace the rate of garbage
+collection activity. This is especially true of delete-heavy workloads, where
+many objects get stored for a short period of time and then deleted. For these
+types of workloads, administrators can increase the priority of garbage
+collection operations relative to other operations with the following
+configuration parameters.
+
+
+``rgw_gc_max_objs``
+
+:Description: The maximum number of objects that may be handled by
+              garbage collection in one garbage collection processing cycle.
+              Please do not change this value after the first deployment.
+
+:Type: Integer
+:Default: ``32``
+
+
+``rgw_gc_obj_min_wait``
+
+:Description: The minimum wait time before a deleted object may be removed
+              and handled by garbage collection processing.
+
+:Type: Integer
+:Default: ``2 * 3600``
+
+
+``rgw_gc_processor_max_time``
+
+:Description: The maximum time between the beginning of two consecutive garbage
+              collection processing cycles.
+
+:Type: Integer
+:Default: ``3600``
+
+
+``rgw_gc_processor_period``
+
+:Description: The cycle time for garbage collection processing.
+:Type: Integer
+:Default: ``3600``
+
+
+``rgw_gc_max_concurrent_io``
+
+:Description: The maximum number of concurrent IO operations that the RGW garbage
+              collection thread will use when purging old data.
+:Type: Integer
+:Default: ``10``
+
+
+:Tuning Garbage Collection for Delete Heavy Workloads:
+
+As an initial step towards tuning Ceph Garbage Collection to be more aggressive the following options are suggested to be increased from their default configuration values:
+
+``rgw_gc_max_concurrent_io = 20``
+``rgw_gc_max_trim_chunk = 64``
+
+:NOTE: Modifying these values requires a restart of the RGW service.
+
+Once these values have been increased from default please monitor for performance of the cluster during Garbage Collection to verify no adverse performance issues due to the increased values.
+
+Multisite Settings
+==================
+
+.. versionadded:: Jewel
+
+You may include the following settings in your Ceph configuration
+file under each ``[client.radosgw.{instance-name}]`` instance.
+
+
+``rgw_zone``
+
+:Description: The name of the zone for the gateway instance. If no zone is
+              set, a cluster-wide default can be configured with the command
+              ``radosgw-admin zone default``.
+:Type: String
+:Default: None
+
+
+``rgw_zonegroup``
+
+:Description: The name of the zonegroup for the gateway instance. If no
+              zonegroup is set, a cluster-wide default can be configured with
+              the command ``radosgw-admin zonegroup default``.
+:Type: String
+:Default: None
+
+
+``rgw_realm``
+
+:Description: The name of the realm for the gateway instance. If no realm is
+              set, a cluster-wide default can be configured with the command
+              ``radosgw-admin realm default``.
+:Type: String
+:Default: None
+
+
+``rgw_run_sync_thread``
+
+:Description: If there are other zones in the realm to sync from, spawn threads
+              to handle the sync of data and metadata.
+:Type: Boolean
+:Default: ``true``
+
+
+``rgw_data_log_window``
+
+:Description: The data log entries window in seconds.
+:Type: Integer
+:Default: ``30``
+
+
+``rgw_data_log_changes_size``
+
+:Description: The number of in-memory entries to hold for the data changes log.
+:Type: Integer
+:Default: ``1000``
+
+
+``rgw_data_log_obj_prefix``
+
+:Description: The object name prefix for the data log.
+:Type: String
+:Default: ``data_log``
+
+
+``rgw_data_log_num_shards``
+
+:Description: The number of shards (objects) on which to keep the
+              data changes log.
+
+:Type: Integer
+:Default: ``128``
+
+
+``rgw_md_log_max_shards``
+
+:Description: The maximum number of shards for the metadata log.
+:Type: Integer
+:Default: ``64``
+
+.. important:: The values of ``rgw_data_log_num_shards`` and
+   ``rgw_md_log_max_shards`` should not be changed after sync has
+   started.
+
+S3 Settings
+===========
+
+``rgw_s3_auth_use_ldap``
+
+:Description: Should S3 authentication use LDAP?
+:Type: Boolean
+:Default: ``false``
+
+
+Swift Settings
+==============
+
+``rgw_enforce_swift_acls``
+
+:Description: Enforces the Swift Access Control List (ACL) settings.
+:Type: Boolean
+:Default: ``true``
+
+
+``rgw_swift_token_expiration``
+
+:Description: The time in seconds for expiring a Swift token.
+:Type: Integer
+:Default: ``24 * 3600``
+
+
+``rgw_swift_url``
+
+:Description: The URL for the Ceph Object Gateway Swift API.
+:Type: String
+:Default: None
+
+
+``rgw_swift_url_prefix``
+
+:Description: The URL prefix for the Swift API, to distinguish it from
+              the S3 API endpoint. The default is ``swift``, which
+              makes the Swift API available at the URL
+              ``http://host:port/swift/v1`` (or
+              ``http://host:port/swift/v1/AUTH_%(tenant_id)s`` if
+              ``rgw swift account in url`` is enabled).
+
+              For compatibility, setting this configuration variable
+              to the empty string causes the default ``swift`` to be
+              used; if you do want an empty prefix, set this option to
+              ``/``.
+
+              .. warning:: If you set this option to ``/``, you must
+                           disable the S3 API by modifying ``rgw
+                           enable apis`` to exclude ``s3``. It is not
+                           possible to operate radosgw with ``rgw
+                           swift url prefix = /`` and simultaneously
+                           support both the S3 and Swift APIs. If you
+                           do need to support both APIs without
+                           prefixes, deploy multiple radosgw instances
+                           to listen on different hosts (or ports)
+                           instead, enabling some for S3 and some for
+                           Swift.
+:Default: ``swift``
+:Example: "/swift-testing"
+
+
+``rgw_swift_auth_url``
+
+:Description: Default URL for verifying v1 auth tokens (if not using internal
+              Swift auth).
+
+:Type: String
+:Default: None
+
+
+``rgw_swift_auth_entry``
+
+:Description: The entry point for a Swift auth URL.
+:Type: String
+:Default: ``auth``
+
+
+``rgw_swift_account_in_url``
+
+:Description: Whether or not the Swift account name should be included
+              in the Swift API URL.
+
+              If set to ``false`` (the default), then the Swift API
+              will listen on a URL formed like
+              ``http://host:port/<rgw_swift_url_prefix>/v1``, and the
+              account name (commonly a Keystone project UUID if
+              radosgw is configured with `Keystone integration
+              <../keystone>`_) will be inferred from request
+              headers.
+
+              If set to ``true``, the Swift API URL will be
+              ``http://host:port/<rgw_swift_url_prefix>/v1/AUTH_<account_name>``
+              (or
+              ``http://host:port/<rgw_swift_url_prefix>/v1/AUTH_<keystone_project_id>``)
+              instead, and the Keystone ``object-store`` endpoint must
+              accordingly be configured to include the
+              ``AUTH_%(tenant_id)s`` suffix.
+
+              You **must** set this option to ``true`` (and update the
+              Keystone service catalog) if you want radosgw to support
+              publicly-readable containers and `temporary URLs
+              <../swift/tempurl>`_.
+:Type: Boolean
+:Default: ``false``
+
+
+``rgw_swift_versioning_enabled``
+
+:Description: Enables the Object Versioning of OpenStack Object Storage API.
+              This allows clients to put the ``X-Versions-Location`` attribute
+              on containers that should be versioned. The attribute specifies
+              the name of container storing archived versions. It must be owned
+              by the same user that the versioned container due to access
+              control verification - ACLs are NOT taken into consideration.
+              Those containers cannot be versioned by the S3 object versioning
+              mechanism.
+
+	      A slightly different attribute, ``X-History-Location``, which is also understood by
+              `OpenStack Swift <https://docs.openstack.org/swift/latest/api/object_versioning.html>`_
+              for handling ``DELETE`` operations, is currently not supported.
+:Type: Boolean
+:Default: ``false``
+
+
+``rgw_trust_forwarded_https``
+
+:Description: When a proxy in front of radosgw is used for ssl termination, radosgw
+              does not know whether incoming http connections are secure. Enable
+              this option to trust the ``Forwarded`` and ``X-Forwarded-Proto`` headers
+              sent by the proxy when determining whether the connection is secure.
+              This is required for some features, such as server side encryption.
+              (Never enable this setting if you do not have a trusted proxy in front of
+              radosgw, or else malicious users will be able to set these headers in
+              any request.)
+:Type: Boolean
+:Default: ``false``
+
+
+
+Logging Settings
+================
+
+
+``rgw_log_nonexistent_bucket``
+
+:Description: Enables Ceph Object Gateway to log a request for a non-existent
+              bucket.
+
+:Type: Boolean
+:Default: ``false``
+
+
+``rgw_log_object_name``
+
+:Description: The logging format for an object name. See ma npage
+              :manpage:`date` for details about format specifiers.
+
+:Type: Date
+:Default: ``%Y-%m-%d-%H-%i-%n``
+
+
+``rgw_log_object_name_utc``
+
+:Description: Whether a logged object name includes a UTC time.
+              If ``false``, it uses the local time.
+
+:Type: Boolean
+:Default: ``false``
+
+
+``rgw_usage_max_shards``
+
+:Description: The maximum number of shards for usage logging.
+:Type: Integer
+:Default: ``32``
+
+
+``rgw_usage_max_user_shards``
+
+:Description: The maximum number of shards used for a single user's
+              usage logging.
+
+:Type: Integer
+:Default: ``1``
+
+
+``rgw_enable_ops_log``
+
+:Description: Enable logging for each successful Ceph Object Gateway operation.
+:Type: Boolean
+:Default: ``false``
+
+
+``rgw_enable_usage_log``
+
+:Description: Enable the usage log.
+:Type: Boolean
+:Default: ``false``
+
+
+``rgw_ops_log_rados``
+
+:Description: Whether the operations log should be written to the
+              Ceph Storage Cluster backend.
+
+:Type: Boolean
+:Default: ``true``
+
+
+``rgw_ops_log_socket_path``
+
+:Description: The Unix domain socket for writing operations logs.
+:Type: String
+:Default: None
+
+
+``rgw_ops_log_file_path``
+
+:Description: The file for writing operations logs.
+:Type: String
+:Default: None
+
+
+``rgw_ops_log_data_backlog``
+
+:Description: The maximum data backlog data size for operations logs written
+              to a Unix domain socket.
+
+:Type: Integer
+:Default: ``5 << 20``
+
+
+``rgw_usage_log_flush_threshold``
+
+:Description: The number of dirty merged entries in the usage log before
+              flushing synchronously.
+
+:Type: Integer
+:Default: 1024
+
+
+``rgw_usage_log_tick_interval``
+
+:Description: Flush pending usage log data every ``n`` seconds.
+:Type: Integer
+:Default: ``30``
+
+
+``rgw_log_http_headers``
+
+:Description: Comma-delimited list of HTTP headers to include with ops
+	      log entries.  Header names are case insensitive, and use
+	      the full header name with words separated by underscores.
+
+:Type: String
+:Default: None
+:Example: "http_x_forwarded_for, http_x_special_k"
+
+
+``rgw_intent_log_object_name``
+
+:Description: The logging format for the intent log object name. See the manpage
+              :manpage:`date` for details about format specifiers.
+
+:Type: Date
+:Default: ``%Y-%m-%d-%i-%n``
+
+
+``rgw_intent_log_object_name_utc``
+
+:Description: Whether the intent log object name uses UTC time.
+              If ``false``, it uses the local time.
+
+:Type: Boolean
+:Default: ``false``
+
+
+
+Keystone Settings
+=================
+
+
+``rgw_keystone_url``
+
+:Description: The URL for the Keystone server.
+:Type: String
+:Default: None
+
+
+``rgw_keystone_api_version``
+
+:Description: The version (2 or 3) of OpenStack Identity API that should be
+              used for communication with the Keystone server.
+:Type: Integer
+:Default: ``2``
+
+
+``rgw_keystone_admin_domain``
+
+:Description: The name of OpenStack domain with admin privilege when using
+              OpenStack Identity API v3.
+:Type: String
+:Default: None
+
+
+``rgw_keystone_admin_project``
+
+:Description: The name of OpenStack project with admin privilege when using
+              OpenStack Identity API v3. If left unspecified, value of
+              ``rgw keystone admin tenant`` will be used instead.
+:Type: String
+:Default: None
+
+
+``rgw_keystone_admin_token``
+
+:Description: The Keystone admin token (shared secret). In Ceph RGW
+              authentication with the admin token has priority over
+              authentication with the admin credentials
+              (``rgw_keystone_admin_user``, ``rgw_keystone_admin_password``,
+              ``rgw_keystone_admin_tenant``, ``rgw_keystone_admin_project``,
+              ``rgw_keystone_admin_domain``). The Keystone admin token
+              has been deprecated, but can be used to integrate with
+              older environments.  It is preferred to instead configure
+              ``rgw_keystone_admin_token_path`` to avoid exposing the token.
+:Type: String
+:Default: None
+
+``rgw_keystone_admin_token_path``
+
+:Description: Path to a file containing the Keystone admin token
+	      (shared secret).  In Ceph RadosGW authentication with
+	      the admin token has priority over authentication with
+	      the admin credentials
+              (``rgw_keystone_admin_user``, ``rgw_keystone_admin_password``,
+              ``rgw_keystone_admin_tenant``, ``rgw_keystone_admin_project``,
+              ``rgw_keystone_admin_domain``).
+              The Keystone admin token has been deprecated, but can be
+              used to integrate with older environments.
+:Type: String
+:Default: None
+
+``rgw_keystone_admin_tenant``
+
+:Description: The name of OpenStack tenant with admin privilege (Service Tenant) when
+              using OpenStack Identity API v2
+:Type: String
+:Default: None
+
+
+``rgw_keystone_admin_user``
+
+:Description: The name of OpenStack user with admin privilege for Keystone
+              authentication (Service User) when using OpenStack Identity API v2
+:Type: String
+:Default: None
+
+
+``rgw_keystone_admin_password``
+
+:Description: The password for OpenStack admin user when using OpenStack
+              Identity API v2.  It is preferred to instead configure
+              ``rgw_keystone_admin_password_path`` to avoid exposing the token.
+:Type: String
+:Default: None
+
+``rgw_keystone_admin_password_path``
+
+:Description: Path to a file containing the password for OpenStack
+              admin user when using OpenStack Identity API v2.
+:Type: String
+:Default: None
+
+
+``rgw_keystone_accepted_roles``
+
+:Description: The roles required to serve requests.
+:Type: String
+:Default: ``Member, admin``
+
+
+``rgw_keystone_token_cache_size``
+
+:Description: The maximum number of entries in each Keystone token cache.
+:Type: Integer
+:Default: ``10000``
+
+
+``rgw_keystone_revocation_interval``
+
+:Description: The number of seconds between token revocation checks.
+:Type: Integer
+:Default: ``15 * 60``
+
+
+``rgw_keystone_verify_ssl``
+
+:Description: Verify SSL certificates while making token requests to keystone.
+:Type: Boolean
+:Default: ``true``
+
+
+Server-side encryption Settings
+===============================
+
+``rgw_crypt_s3_kms_backend``
+
+:Description: Where the SSE-KMS encryption keys are stored. Supported KMS
+              systems are OpenStack Barbican (``barbican``, the default) and
+              HashiCorp Vault (``vault``).
+:Type: String
+:Default: None
+
+
+Barbican Settings
+=================
+
+``rgw_barbican_url``
+
+:Description: The URL for the Barbican server.
+:Type: String
+:Default: None
+
+``rgw_keystone_barbican_user``
+
+:Description: The name of the OpenStack user with access to the `Barbican`_
+              secrets used for `Encryption`_.
+:Type: String
+:Default: None
+
+``rgw_keystone_barbican_password``
+
+:Description: The password associated with the `Barbican`_ user.
+:Type: String
+:Default: None
+
+``rgw_keystone_barbican_tenant``
+
+:Description: The name of the OpenStack tenant associated with the `Barbican`_
+              user when using OpenStack Identity API v2.
+:Type: String
+:Default: None
+
+``rgw_keystone_barbican_project``
+
+:Description: The name of the OpenStack project associated with the `Barbican`_
+              user when using OpenStack Identity API v3.
+:Type: String
+:Default: None
+
+``rgw_keystone_barbican_domain``
+
+:Description: The name of the OpenStack domain associated with the `Barbican`_
+              user when using OpenStack Identity API v3.
+:Type: String
+:Default: None
+
+
+HashiCorp Vault Settings
+========================
+
+``rgw_crypt_vault_auth``
+
+:Description: Type of authentication method to be used. The only method
+              currently supported is ``token``.
+:Type: String
+:Default: ``token``
+
+``rgw_crypt_vault_token_file``
+
+:Description: If authentication method is ``token``, provide a path to the token
+              file, which should be readable only by Rados Gateway.
+:Type: String
+:Default: None
+
+``rgw_crypt_vault_addr``
+
+:Description: Vault server base address, e.g. ``http://vaultserver:8200``.
+:Type: String
+:Default: None
+
+``rgw_crypt_vault_prefix``
+
+:Description: The Vault secret URL prefix, which can be used to restrict access
+              to a particular subset of the secret space, e.g. ``/v1/secret/data``.
+:Type: String
+:Default: None
+
+``rgw_crypt_vault_secret_engine``
+
+:Description: Vault Secret Engine to be used to retrieve encryption keys: choose
+              between kv-v2, transit.
+:Type: String
+:Default: None
+
+``rgw_crypt_vault_namespace``
+
+:Description: If set, Vault Namespace provides tenant isolation for teams and individuals
+              on the same Vault Enterprise instance, e.g. ``acme/tenant1``
+:Type: String
+:Default: None
+
+
+QoS settings
+------------
+
+.. versionadded:: Nautilus
+
+The ``civetweb`` frontend has a threading model that uses a thread per
+connection and hence is automatically throttled by ``rgw_thread_pool_size``
+configurable when it comes to accepting connections. The newer ``beast`` frontend is
+not restricted by the thread pool size when it comes to accepting new
+connections, so a scheduler abstraction is introduced in the Nautilus release
+to support future methods of scheduling requests.
+
+Currently the scheduler defaults to a throttler which throttles the active
+connections to a configured limit. QoS based on mClock is currently in an
+*experimental* phase and not recommended for production yet. Current
+implementation of *dmclock_client* op queue divides RGW Ops on admin, auth
+(swift auth, sts) metadata & data requests.
+
+
+``rgw_max_concurrent_requests``
+
+:Description: Maximum number of concurrent HTTP requests that the Beast front end
+              will process. Tuning this can help to limit memory usage under
+              heavy load.
+:Type: Integer
+:Default: 1024
+
+
+``rgw_scheduler_type``
+
+:Description: The RGW scheduler to use. Valid values are ``throttler` and
+              ``dmclock``. Currently defaults to ``throttler`` which throttles Beast
+              frontend requests. ``dmclock` is *experimental* and requires the
+              ``dmclock`` to be included in the ``experimental_feature_enabled``
+              configuration option.
+
+
+The options below tune the experimental dmclock scheduler. For
+additional reading on dmclock, see :ref:`dmclock-qos`. `op_class` for the flags below is
+one of ``admin``, ``auth``, ``metadata``, or ``data``.
+
+``rgw_dmclock_<op_class>_res``
+
+:Description: The mclock reservation for `op_class` requests
+:Type: float
+:Default: 100.0
+
+``rgw_dmclock_<op_class>_wgt``
+
+:Description: The mclock weight for `op_class` requests
+:Type: float
+:Default: 1.0
+
+``rgw_dmclock_<op_class>_lim``
+
+:Description: The mclock limit for `op_class` requests
+:Type: float
+:Default: 0.0
+
+
+
+.. _Architecture: ../../architecture#data-striping
+.. _Pool Configuration: ../../rados/configuration/pool-pg-config-ref/
+.. _Cluster Pools: ../../rados/operations/pools
+.. _Rados cluster handles: ../../rados/api/librados-intro/#step-2-configuring-a-cluster-handle
+.. _Barbican: ../barbican
+.. _Encryption: ../encryption
+.. _HTTP Frontends: ../frontends
diff --git a/doc/radosgw/dynamicresharding.rst b/doc/radosgw/dynamicresharding.rst
new file mode 100644
index 000000000..a7698e7bb
--- /dev/null
+++ b/doc/radosgw/dynamicresharding.rst
@@ -0,0 +1,235 @@
+.. _rgw_dynamic_bucket_index_resharding:
+
+===================================
+RGW Dynamic Bucket Index Resharding
+===================================
+
+.. versionadded:: Luminous
+
+A large bucket index can lead to performance problems. In order
+to address this problem we introduced bucket index sharding.
+Until Luminous, changing the number of bucket shards (resharding)
+needed to be done offline. Starting with Luminous we support
+online bucket resharding.
+
+Each bucket index shard can handle its entries efficiently up until
+reaching a certain threshold number of entries. If this threshold is
+exceeded the system can suffer from performance issues. The dynamic
+resharding feature detects this situation and automatically increases
+the number of shards used by the bucket index, resulting in a
+reduction of the number of entries in each bucket index shard. This
+process is transparent to the user.
+
+By default dynamic bucket index resharding can only increase the
+number of bucket index shards to 1999, although this upper-bound is a
+configuration parameter (see Configuration below). When
+possible, the process chooses a prime number of bucket index shards to
+spread the number of bucket index entries across the bucket index
+shards more evenly.
+
+The detection process runs in a background process that periodically
+scans all the buckets. A bucket that requires resharding is added to
+the resharding queue and will be scheduled to be resharded later. The
+reshard thread runs in the background and execute the scheduled
+resharding tasks, one at a time.
+
+Multisite
+=========
+
+Dynamic resharding is not supported in a multisite environment.
+
+
+Configuration
+=============
+
+Enable/Disable dynamic bucket index resharding:
+
+- ``rgw_dynamic_resharding``:  true/false, default: true
+
+Configuration options that control the resharding process:
+
+- ``rgw_max_objs_per_shard``: maximum number of objects per bucket index shard before resharding is triggered, default: 100000 objects
+
+- ``rgw_max_dynamic_shards``: maximum number of shards that dynamic bucket index resharding can increase to, default: 1999
+
+- ``rgw_reshard_bucket_lock_duration``: duration, in seconds, of lock on bucket obj during resharding, default: 360 seconds (i.e., 6 minutes)
+
+- ``rgw_reshard_thread_interval``: maximum time, in seconds, between rounds of resharding queue processing, default: 600 seconds (i.e., 10 minutes)
+
+- ``rgw_reshard_num_logs``: number of shards for the resharding queue, default: 16
+
+Admin commands
+==============
+
+Add a bucket to the resharding queue
+------------------------------------
+
+::
+
+   # radosgw-admin reshard add --bucket <bucket_name> --num-shards <new number of shards>
+
+List resharding queue
+---------------------
+
+::
+
+   # radosgw-admin reshard list
+
+Process tasks on the resharding queue
+-------------------------------------
+
+::
+
+   # radosgw-admin reshard process
+
+Bucket resharding status
+------------------------
+
+::
+
+   # radosgw-admin reshard status --bucket <bucket_name>
+
+The output is a json array of 3 objects (reshard_status, new_bucket_instance_id, num_shards) per shard.
+
+For example, the output at different Dynamic Resharding stages is shown below:
+
+``1. Before resharding occurred:``
+::
+
+  [
+    {
+        "reshard_status": "not-resharding",
+        "new_bucket_instance_id": "",
+        "num_shards": -1
+    }
+  ]
+
+``2. During resharding:``
+::
+
+  [
+    {
+        "reshard_status": "in-progress",
+        "new_bucket_instance_id": "1179f470-2ebf-4630-8ec3-c9922da887fd.8652.1",
+        "num_shards": 2
+    },
+    {
+        "reshard_status": "in-progress",
+        "new_bucket_instance_id": "1179f470-2ebf-4630-8ec3-c9922da887fd.8652.1",
+        "num_shards": 2
+    }
+  ]
+
+``3, After resharding completed:``
+::
+
+  [
+    {
+        "reshard_status": "not-resharding",
+        "new_bucket_instance_id": "",
+        "num_shards": -1
+    },
+    {
+        "reshard_status": "not-resharding",
+        "new_bucket_instance_id": "",
+        "num_shards": -1
+    }
+  ]
+
+
+Cancel pending bucket resharding
+--------------------------------
+
+Note: Ongoing bucket resharding operations cannot be cancelled. ::
+
+   # radosgw-admin reshard cancel --bucket <bucket_name>
+
+Manual immediate bucket resharding
+----------------------------------
+
+::
+
+   # radosgw-admin bucket reshard --bucket <bucket_name> --num-shards <new number of shards>
+
+When choosing a number of shards, the administrator should keep a
+number of items in mind. Ideally the administrator is aiming for no
+more than 100000 entries per shard, now and through some future point
+in time.
+
+Additionally, bucket index shards that are prime numbers tend to work
+better in evenly distributing bucket index entries across the
+shards. For example, 7001 bucket index shards is better than 7000
+since the former is prime. A variety of web sites have lists of prime
+numbers; search for "list of prime numbers" withy your favorite web
+search engine to locate some web sites.
+
+Troubleshooting
+===============
+
+Clusters prior to Luminous 12.2.11 and Mimic 13.2.5 left behind stale bucket
+instance entries, which were not automatically cleaned up. The issue also affected
+LifeCycle policies, which were not applied to resharded buckets anymore. Both of
+these issues can be worked around using a couple of radosgw-admin commands.
+
+Stale instance management
+-------------------------
+
+List the stale instances in a cluster that are ready to be cleaned up.
+
+::
+
+   # radosgw-admin reshard stale-instances list
+
+Clean up the stale instances in a cluster. Note: cleanup of these
+instances should only be done on a single site cluster.
+
+::
+
+   # radosgw-admin reshard stale-instances rm
+
+
+Lifecycle fixes
+---------------
+
+For clusters that had resharded instances, it is highly likely that the old
+lifecycle processes would have flagged and deleted lifecycle processing as the
+bucket instance changed during a reshard. While this is fixed for newer clusters
+(from Mimic 13.2.6 and Luminous 12.2.12), older buckets that had lifecycle policies and
+that have undergone resharding will have to be manually fixed.
+
+The command to do so is:
+
+::
+
+   # radosgw-admin lc reshard fix --bucket {bucketname}
+
+
+As a convenience wrapper, if the ``--bucket`` argument is dropped then this
+command will try and fix lifecycle policies for all the buckets in the cluster.
+
+Object Expirer fixes
+--------------------
+
+Objects subject to Swift object expiration on older clusters may have
+been dropped from the log pool and never deleted after the bucket was
+resharded. This would happen if their expiration time was before the
+cluster was upgraded, but if their expiration was after the upgrade
+the objects would be correctly handled. To manage these expire-stale
+objects, radosgw-admin provides two subcommands.
+
+Listing:
+
+::
+
+   # radosgw-admin objects expire-stale list --bucket {bucketname}
+
+Displays a list of object names and expiration times in JSON format.
+
+Deleting:
+
+::
+
+   # radosgw-admin objects expire-stale rm --bucket {bucketname}
+
+
+Initiates deletion of such objects, displaying a list of object names, expiration times, and deletion status in JSON format.
diff --git a/doc/radosgw/elastic-sync-module.rst b/doc/radosgw/elastic-sync-module.rst
new file mode 100644
index 000000000..60c806e87
--- /dev/null
+++ b/doc/radosgw/elastic-sync-module.rst
@@ -0,0 +1,181 @@
+=========================
+ElasticSearch Sync Module
+=========================
+
+.. versionadded:: Kraken
+
+.. note::
+     As of 31 May 2020, only Elasticsearch 6 and lower are supported. ElasticSearch 7 is not supported.
+
+This sync module writes the metadata from other zones to `ElasticSearch`_. As of
+luminous this is a json of data fields we currently store in ElasticSearch.
+
+::
+
+   {
+        "_index" : "rgw-gold-ee5863d6",
+        "_type" : "object",
+        "_id" : "34137443-8592-48d9-8ca7-160255d52ade.34137.1:object1:null",
+        "_score" : 1.0,
+        "_source" : {
+          "bucket" : "testbucket123",
+          "name" : "object1",
+          "instance" : "null",
+          "versioned_epoch" : 0,
+          "owner" : {
+            "id" : "user1",
+            "display_name" : "user1"
+          },
+          "permissions" : [
+            "user1"
+          ],
+          "meta" : {
+            "size" : 712354,
+            "mtime" : "2017-05-04T12:54:16.462Z",
+            "etag" : "7ac66c0f148de9519b8bd264312c4d64"
+          }
+        }
+      }
+
+
+
+ElasticSearch tier type configurables
+-------------------------------------
+
+* ``endpoint``
+
+Specifies the Elasticsearch server endpoint to access
+
+* ``num_shards`` (integer)
+
+The number of shards that Elasticsearch will be configured with on
+data sync initialization. Note that this cannot be changed after init.
+Any change here requires rebuild of the Elasticsearch index and reinit
+of the data sync process.
+
+* ``num_replicas`` (integer)
+
+The number of the replicas that Elasticsearch will be configured with
+on data sync initialization.
+
+* ``explicit_custom_meta`` (true | false)
+
+Specifies whether all user custom metadata will be indexed, or whether
+user will need to configure (at the bucket level) what custom
+metadata entries should be indexed. This is false by default
+
+* ``index_buckets_list`` (comma separated list of strings)
+
+If empty, all buckets will be indexed. Otherwise, only buckets
+specified here will be indexed. It is possible to provide bucket
+prefixes (e.g., foo\*), or bucket suffixes (e.g., \*bar).
+
+* ``approved_owners_list`` (comma separated list of strings)
+
+If empty, buckets of all owners will be indexed (subject to other
+restrictions), otherwise, only buckets owned by specified owners will
+be indexed. Suffixes and prefixes can also be provided.
+
+* ``override_index_path`` (string)
+
+if not empty, this string will be used as the elasticsearch index
+path. Otherwise the index path will be determined and generated on
+sync initialization.
+
+
+End user metadata queries
+-------------------------
+
+.. versionadded:: Luminous
+
+Since the ElasticSearch cluster now stores object metadata, it is important that
+the ElasticSearch endpoint is not exposed to the public and only accessible to
+the cluster administrators. For exposing metadata queries to the end user itself
+this poses a problem since we'd want the user to only query their metadata and
+not of any other users, this would require the ElasticSearch cluster to
+authenticate users in a way similar to RGW does which poses a problem.
+
+As of Luminous RGW in the metadata master zone can now service end user
+requests. This allows for not exposing the elasticsearch endpoint in public and
+also solves the authentication and authorization problem since RGW itself can
+authenticate the end user requests. For this purpose RGW introduces a new query
+in the bucket APIs that can service elasticsearch requests. All these requests
+must be sent to the metadata master zone.
+
+Syntax
+~~~~~~
+
+Get an elasticsearch query
+``````````````````````````
+
+::
+
+   GET /{bucket}?query={query-expr}
+
+request params:
+ - max-keys: max number of entries to return
+ - marker: pagination marker
+
+``expression := [(]<arg> <op> <value> [)][<and|or> ...]``
+
+op is one of the following:
+<, <=, ==, >=, >
+
+For example ::
+
+  GET /?query=name==foo
+
+Will return all the indexed keys that user has read permission to, and
+are named 'foo'.
+
+The output will be a list of keys in XML that is similar to the S3
+list buckets response.
+
+Configure custom metadata fields
+````````````````````````````````
+
+Define which custom metadata entries should be indexed (under the
+specified bucket), and what are the types of these keys. If explicit
+custom metadata indexing is configured, this is needed so that rgw
+will index the specified custom metadata values. Otherwise it is
+needed in cases where the indexed metadata keys are of a type other
+than string.
+
+::
+
+   POST /{bucket}?mdsearch
+   x-amz-meta-search: <key [; type]> [, ...]
+
+Multiple metadata fields must be comma separated, a type can be forced for a
+field with a `;`. The currently allowed types are string(default), integer and
+date
+
+eg. if you want to index a custom object metadata x-amz-meta-year as int,
+x-amz-meta-date as type date and x-amz-meta-title as string, you'd do
+
+::
+
+   POST /mybooks?mdsearch
+   x-amz-meta-search: x-amz-meta-year;int, x-amz-meta-release-date;date, x-amz-meta-title;string
+
+
+Delete custom metadata configuration
+````````````````````````````````````
+
+Delete custom metadata bucket configuration.
+
+::
+
+   DELETE /<bucket>?mdsearch
+
+Get custom metadata configuration
+`````````````````````````````````
+
+Retrieve custom metadata bucket configuration.
+
+::
+
+   GET /<bucket>?mdsearch
+
+
+.. _`Elasticsearch`: https://github.com/elastic/elasticsearch
diff --git a/doc/radosgw/encryption.rst b/doc/radosgw/encryption.rst
new file mode 100644
index 000000000..07fd60ac5
--- /dev/null
+++ b/doc/radosgw/encryption.rst
@@ -0,0 +1,68 @@
+==========
+Encryption
+==========
+
+.. versionadded:: Luminous
+
+The Ceph Object Gateway supports server-side encryption of uploaded objects,
+with 3 options for the management of encryption keys. Server-side encryption
+means that the data is sent over HTTP in its unencrypted form, and the Ceph
+Object Gateway stores that data in the Ceph Storage Cluster in encrypted form.
+
+.. note:: Requests for server-side encryption must be sent over a secure HTTPS
+          connection to avoid sending secrets in plaintext. If a proxy is used
+          for SSL termination, ``rgw trust forwarded https`` must be enabled
+          before forwarded requests will be trusted as secure.
+
+.. note:: Server-side encryption keys must be 256-bit long and base64 encoded.
+
+Customer-Provided Keys
+======================
+
+In this mode, the client passes an encryption key along with each request to
+read or write encrypted data. It is the client's responsibility to manage those
+keys and remember which key was used to encrypt each object.
+
+This is implemented in S3 according to the `Amazon SSE-C`_ specification.
+
+As all key management is handled by the client, no special configuration is
+needed to support this encryption mode.
+
+Key Management Service
+======================
+
+This mode allows keys to be stored in a secure key management service and
+retrieved on demand by the Ceph Object Gateway to serve requests to encrypt
+or decrypt data.
+
+This is implemented in S3 according to the `Amazon SSE-KMS`_ specification.
+
+In principle, any key management service could be used here.  Currently
+integration with `Barbican`_, `Vault`_, and `KMIP`_ are implemented.
+
+See `OpenStack Barbican Integration`_, `HashiCorp Vault Integration`_,
+and `KMIP Integration`_.
+
+Automatic Encryption (for testing only)
+=======================================
+
+A ``rgw crypt default encryption key`` can be set in ceph.conf to force the
+encryption of all objects that do not otherwise specify an encryption mode.
+
+The configuration expects a base64-encoded 256 bit key. For example::
+
+  rgw crypt default encryption key = 4YSmvJtBv0aZ7geVgAsdpRnLBEwWSWlMIGnRS8a9TSA=
+
+.. important:: This mode is for diagnostic purposes only! The ceph configuration
+   file is not a secure method for storing encryption keys. Keys that are
+   accidentally exposed in this way should be considered compromised.
+
+
+.. _Amazon SSE-C: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
+.. _Amazon SSE-KMS: http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html
+.. _Barbican: https://wiki.openstack.org/wiki/Barbican
+.. _Vault: https://www.vaultproject.io/docs/
+.. _KMIP: http://www.oasis-open.org/committees/kmip/
+.. _OpenStack Barbican Integration: ../barbican
+.. _HashiCorp Vault Integration: ../vault
+.. _KMIP Integration: ../kmip
diff --git a/doc/radosgw/frontends.rst b/doc/radosgw/frontends.rst
new file mode 100644
index 000000000..274cdce87
--- /dev/null
+++ b/doc/radosgw/frontends.rst
@@ -0,0 +1,226 @@
+.. _rgw_frontends:
+
+==============
+HTTP Frontends
+==============
+
+.. contents::
+
+The Ceph Object Gateway supports two embedded HTTP frontend libraries
+that can be configured with ``rgw_frontends``. See `Config Reference`_
+for details about the syntax.
+
+Beast
+=====
+
+.. versionadded:: Mimic
+
+The ``beast`` frontend uses the Boost.Beast library for HTTP parsing
+and the Boost.Asio library for asynchronous network i/o.
+
+Options
+-------
+
+``port`` and ``ssl_port``
+
+:Description: Sets the ipv4 & ipv6 listening port number. Can be specified multiple
+              times as in ``port=80 port=8000``.
+:Type: Integer
+:Default: ``80``
+
+
+``endpoint`` and ``ssl_endpoint``
+
+:Description: Sets the listening address in the form ``address[:port]``, where
+              the address is an IPv4 address string in dotted decimal form, or
+              an IPv6 address in hexadecimal notation surrounded by square
+              brackets. Specifying a IPv6 endpoint would listen to v6 only. The
+              optional port defaults to 80 for ``endpoint`` and 443 for
+              ``ssl_endpoint``. Can be specified multiple times as in
+              ``endpoint=[::1] endpoint=192.168.0.100:8000``.
+
+:Type: Integer
+:Default: None
+
+
+``ssl_certificate``
+
+:Description: Path to the SSL certificate file used for SSL-enabled endpoints.
+              If path is prefixed with ``config://``, the certificate will be
+              pulled from the ceph monitor ``config-key`` database.
+
+:Type: String
+:Default: None
+
+
+``ssl_private_key``
+
+:Description: Optional path to the private key file used for SSL-enabled
+              endpoints. If one is not given, the ``ssl_certificate`` file
+              is used as the private key.
+              If path is prefixed with ``config://``, the certificate will be
+              pulled from the ceph monitor ``config-key`` database.
+
+:Type: String
+:Default: None
+
+``ssl_options``
+
+:Description: Optional colon separated list of ssl context options:
+
+              ``default_workarounds`` Implement various bug workarounds.
+
+              ``no_compression`` Disable compression.
+
+              ``no_sslv2`` Disable SSL v2.
+
+              ``no_sslv3`` Disable SSL v3.
+
+              ``no_tlsv1`` Disable TLS v1.
+
+              ``no_tlsv1_1`` Disable TLS v1.1.
+
+              ``no_tlsv1_2`` Disable TLS v1.2.
+
+              ``single_dh_use`` Always create a new key when using tmp_dh parameters.
+
+:Type: String
+:Default: ``no_sslv2:no_sslv3:no_tlsv1:no_tlsv1_1``
+
+``ssl_ciphers``
+
+:Description: Optional list of one or more cipher strings separated by colons.
+              The format of the string is described in openssl's ciphers(1)
+              manual.
+
+:Type: String
+:Default: None
+
+``tcp_nodelay``
+
+:Description: If set the socket option will disable Nagle's algorithm on 
+              the connection which means that packets will be sent as soon 
+              as possible instead of waiting for a full buffer or timeout to occur.
+
+              ``1`` Disable Nagel's algorithm for all sockets.
+
+              ``0`` Keep the default: Nagel's algorithm enabled.
+
+:Type: Integer (0 or 1)
+:Default: 0
+
+``max_connection_backlog``
+
+:Description: Optional value to define the maximum size for the queue of
+              connections waiting to be accepted. If not configured, the value
+              from ``boost::asio::socket_base::max_connections`` will be used.
+
+:Type: Integer
+:Default: None
+
+``request_timeout_ms``
+
+:Description: The amount of time in milliseconds that Beast will wait
+              for more incoming data or outgoing data before giving up.
+              Setting this value to 0 will disable timeout.
+
+:Type: Integer
+:Default: ``65000``
+
+
+Civetweb
+========
+
+.. versionadded:: Firefly
+.. deprecated:: Pacific
+
+The ``civetweb`` frontend uses the Civetweb HTTP library, which is a
+fork of Mongoose.
+
+
+Options
+-------
+
+``port``
+
+:Description: Sets the listening port number. For SSL-enabled ports, add an
+              ``s`` suffix like ``443s``. To bind a specific IPv4 or IPv6
+              address, use the form ``address:port``. Multiple endpoints
+              can either be separated by ``+`` as in ``127.0.0.1:8000+443s``,
+              or by providing multiple options as in ``port=8000 port=443s``.
+
+:Type: String
+:Default: ``7480``
+
+
+``num_threads``
+
+:Description: Sets the number of threads spawned by Civetweb to handle
+              incoming HTTP connections. This effectively limits the number
+              of concurrent connections that the frontend can service.
+
+:Type: Integer
+:Default: ``rgw_thread_pool_size``
+
+
+``request_timeout_ms``
+
+:Description: The amount of time in milliseconds that Civetweb will wait
+              for more incoming data before giving up.
+
+:Type: Integer
+:Default: ``30000``
+
+
+``ssl_certificate``
+
+:Description: Path to the SSL certificate file used for SSL-enabled ports.
+
+:Type: String
+:Default: None
+
+``access_log_file``
+
+:Description: Path to a file for access logs. Either full path, or relative
+			  to the current working directory. If absent (default), then
+			  accesses are not logged.
+
+:Type: String
+:Default: ``EMPTY``
+
+
+``error_log_file``
+
+:Description: Path to a file for error logs. Either full path, or relative
+			  to the current working directory. If absent (default), then
+			  errors are not logged.
+
+:Type: String
+:Default: ``EMPTY``
+
+
+The following is an example of the ``/etc/ceph/ceph.conf`` file with some of these options set: ::
+ 
+ [client.rgw.gateway-node1]
+ rgw_frontends = civetweb request_timeout_ms=30000 error_log_file=/var/log/radosgw/civetweb.error.log access_log_file=/var/log/radosgw/civetweb.access.log
+
+A complete list of supported options can be found in the `Civetweb User Manual`_.
+
+
+Generic Options
+===============
+
+Some frontend options are generic and supported by all frontends:
+
+``prefix``
+
+:Description: A prefix string that is inserted into the URI of all
+              requests. For example, a swift-only frontend could supply
+              a uri prefix of ``/swift``.
+
+:Type: String
+:Default: None
+
+
+.. _Civetweb User Manual: https://civetweb.github.io/civetweb/UserManual.html
+.. _Config Reference: ../config-ref
diff --git a/doc/radosgw/index.rst b/doc/radosgw/index.rst
new file mode 100644
index 000000000..efdb12d28
--- /dev/null
+++ b/doc/radosgw/index.rst
@@ -0,0 +1,85 @@
+.. _object-gateway:
+
+=====================
+ Ceph Object Gateway
+=====================
+
+:term:`Ceph Object Gateway` is an object storage interface built on top of
+``librados``. It provides a RESTful gateway between applications and Ceph
+Storage Clusters. :term:`Ceph Object Storage` supports two interfaces:
+
+#. **S3-compatible:** Provides object storage functionality with an interface
+   that is compatible with a large subset of the Amazon S3 RESTful API.
+
+#. **Swift-compatible:** Provides object storage functionality with an interface
+   that is compatible with a large subset of the OpenStack Swift API.
+
+Ceph Object Storage uses the Ceph Object Gateway daemon (``radosgw``), an HTTP
+server designed for interacting with a Ceph Storage Cluster. The Ceph Object
+Gateway provides interfaces that are compatible with both Amazon S3 and
+OpenStack Swift, and it has its own user management. Ceph Object Gateway can
+store data in the same Ceph Storage Cluster in which data from Ceph File System
+clients and Ceph Block Device clients is stored. The S3 API and the Swift API
+share a common namespace, which makes it possible to write data to a Ceph
+Storage Cluster with one API and then retrieve that data with the other API.
+
+.. ditaa::
+
+            +------------------------+ +------------------------+
+            |   S3 compatible API    | |  Swift compatible API  |
+            +------------------------+-+------------------------+
+            |                      radosgw                      |
+            +---------------------------------------------------+
+            |                      librados                     |
+            +------------------------+-+------------------------+
+            |          OSDs          | |        Monitors        |
+            +------------------------+ +------------------------+
+
+.. note:: Ceph Object Storage does **NOT** use the Ceph Metadata Server.
+
+
+.. toctree::
+   :maxdepth: 1
+
+   HTTP Frontends <frontends>
+   Pool Placement and Storage Classes <placement>
+   Multisite Configuration <multisite>
+   Multisite Sync Policy Configuration <multisite-sync-policy>
+   Configuring Pools <pools>
+   Config Reference <config-ref>
+   Admin Guide <admin>
+   S3 API <s3>
+   Data caching and CDN <rgw-cache.rst>
+   Swift API <swift>
+   Admin Ops API <adminops>
+   Python binding <api>
+   Export over NFS <nfs>
+   OpenStack Keystone Integration <keystone>
+   OpenStack Barbican Integration <barbican>
+   HashiCorp Vault Integration <vault>
+   KMIP Integration <kmip>
+   Open Policy Agent Integration <opa>
+   Multi-tenancy <multitenancy>
+   Compression <compression>
+   LDAP Authentication <ldap-auth>
+   Server-Side Encryption <encryption>
+   Bucket Policy <bucketpolicy>
+   Dynamic bucket index resharding <dynamicresharding>
+   Multi factor authentication <mfa>
+   Sync Modules <sync-modules>
+   Bucket Notifications <notifications>
+   Data Layout in RADOS <layout>
+   STS <STS>
+   STS Lite <STSLite>
+   Keycloak <keycloak>
+   Session Tags <session-tags>
+   Role <role>
+   Orphan List and Associated Tooling <orphans>
+   OpenID Connect Provider <oidc>
+   troubleshooting
+   Manpage radosgw <../../man/8/radosgw>
+   Manpage radosgw-admin <../../man/8/radosgw-admin>
+   QAT Acceleration for Encryption and Compression <qat-accel>
+   S3-select <s3select>
+   Lua Scripting <lua-scripting>
+
diff --git a/doc/radosgw/keycloak.rst b/doc/radosgw/keycloak.rst
new file mode 100644
index 000000000..534c4733a
--- /dev/null
+++ b/doc/radosgw/keycloak.rst
@@ -0,0 +1,127 @@
+=================================
+Keycloak integration with RadosGW
+=================================
+
+Keycloak can be setup as an OpenID Connect Identity Provider, which can be used by mobile/ web apps
+to authenticate their users. The Web token returned as a result of authentication can be used by the
+mobile/ web app to call AssumeRoleWithWebIdentity to get back a set of temporary S3 credentials,
+which can be used by the app to make S3 calls.
+
+Setting up Keycloak
+====================
+
+Installing and bringing up Keycloak can be found here: https://www.keycloak.org/docs/latest/server_installation/.
+
+Configuring Keycloak to talk to RGW
+===================================
+
+The following configurables have to be added for RGW to talk to Keycloak::
+
+  [client.radosgw.gateway]
+  rgw sts key = {sts key for encrypting/ decrypting the session token}
+  rgw s3 auth use sts = true
+
+Example showing how to fetch a web token from Keycloak
+======================================================
+
+Several examples of apps authenticating with Keycloak are given here: https://github.com/keycloak/keycloak-quickstarts/blob/latest/docs/getting-started.md
+Taking the example of app-profile-jee-jsp app given in the link above, its client id and client secret, can be used to fetch the
+access token (web token) for an application using grant type 'client_credentials' as given below::
+
+    KC_REALM=demo
+    KC_CLIENT=<client id>
+    KC_CLIENT_SECRET=<client secret>
+    KC_SERVER=<host>:8080
+    KC_CONTEXT=auth
+
+    # Request Tokens for credentials
+    KC_RESPONSE=$( \
+    curl -k -v -X POST \
+    -H "Content-Type: application/x-www-form-urlencoded" \
+    -d "scope=openid" \
+    -d "grant_type=client_credentials" \
+    -d "client_id=$KC_CLIENT" \
+    -d "client_secret=$KC_CLIENT_SECRET" \
+    "http://$KC_SERVER/$KC_CONTEXT/realms/$KC_REALM/protocol/openid-connect/token" \
+    | jq .
+    )
+
+    KC_ACCESS_TOKEN=$(echo $KC_RESPONSE| jq -r .access_token)
+
+An access token can also be fetched for a particular user with grant type 'password', using client id, client secret, username and its password
+as given below::
+
+    KC_REALM=demo
+    KC_USERNAME=<username>
+    KC_PASSWORD=<userpassword>
+    KC_CLIENT=<client id>
+    KC_CLIENT_SECRET=<client secret>
+    KC_SERVER=<host>:8080
+    KC_CONTEXT=auth
+
+   # Request Tokens for credentials
+    KC_RESPONSE=$( \
+    curl -k -v -X POST \
+    -H "Content-Type: application/x-www-form-urlencoded" \
+    -d "scope=openid" \
+    -d "grant_type=password" \
+    -d "client_id=$KC_CLIENT" \
+    -d "client_secret=$KC_CLIENT_SECRET" \
+    -d "username=$KC_USERNAME" \
+    -d "password=$KC_PASSWORD" \
+    "http://$KC_SERVER/$KC_CONTEXT/realms/$KC_REALM/protocol/openid-connect/token" \
+    | jq .
+    )
+
+    KC_ACCESS_TOKEN=$(echo $KC_RESPONSE| jq -r .access_token)
+
+
+KC_ACCESS_TOKEN can be used to invoke AssumeRoleWithWebIdentity as given in
+:doc:`STS`.
+
+Attaching tags to a user in Keycloak
+====================================
+
+We need to create a user in keycloak, and add tags to it as its attributes.
+
+Add a user as shown below:
+
+.. image:: ../images/keycloak-adduser.png
+   :align: center
+
+Add user details as shown below:
+
+.. image:: ../images/keycloak-userdetails.png
+   :align: center
+
+Add user credentials as shown below:
+
+.. image:: ../images/keycloak-usercredentials.png
+   :align: center
+
+Add tags to the 'attributes' tab of the user as shown below:
+
+.. image:: ../images/keycloak-usertags.png
+   :align: center
+
+Add a protocol mapper for the user attribute to a client as shown below:
+
+.. image:: ../images/keycloak-userclientmapper.png
+   :align: center
+
+
+After following the steps shown above, the tag 'Department' will appear in the JWT (web token), under 'https://aws.amazon.com/tags' namespace.
+The tags can be verified using token introspection of the JWT. The command to introspect a token using client id and client secret is shown below::
+
+    KC_REALM=demo
+    KC_CLIENT=<client id>
+    KC_CLIENT_SECRET=<client secret>
+    KC_SERVER=<host>:8080
+    KC_CONTEXT=auth
+
+    curl -k -v \
+    -X POST \
+    -u "$KC_CLIENT:$KC_CLIENT_SECRET" \
+    -d "token=$KC_ACCESS_TOKEN" \
+    "http://$KC_SERVER/$KC_CONTEXT/realms/$KC_REALM/protocol/openid-connect/token/introspect" \
+    | jq .
diff --git a/doc/radosgw/keystone.rst b/doc/radosgw/keystone.rst
new file mode 100644
index 000000000..628810ad3
--- /dev/null
+++ b/doc/radosgw/keystone.rst
@@ -0,0 +1,160 @@
+=====================================
+ Integrating with OpenStack Keystone
+=====================================
+
+It is possible to integrate the Ceph Object Gateway with Keystone, the OpenStack
+identity service. This sets up the gateway to accept Keystone as the users
+authority. A user that Keystone authorizes to access the gateway will also be
+automatically created on the Ceph Object Gateway (if didn't exist beforehand). A
+token that Keystone validates will be considered as valid by the gateway.
+
+The following configuration options are available for Keystone integration::
+
+	[client.radosgw.gateway]
+	rgw keystone api version = {keystone api version}
+	rgw keystone url = {keystone server url:keystone server admin port}
+	rgw keystone admin token = {keystone admin token}
+	rgw keystone admin token path = {path to keystone admin token} #preferred
+	rgw keystone accepted roles = {accepted user roles}
+	rgw keystone token cache size = {number of tokens to cache}
+	rgw keystone implicit tenants = {true for private tenant for each new user}
+
+It is also possible to configure a Keystone service tenant, user & password for
+Keystone (for v2.0 version of the OpenStack Identity API), similar to the way
+OpenStack services tend to be configured, this avoids the need for setting the
+shared secret ``rgw keystone admin token`` in the configuration file, which is
+recommended to be disabled in production environments. The service tenant
+credentials should have admin privileges, for more details refer the `OpenStack
+Keystone documentation`_, which explains the process in detail. The requisite
+configuration options for are::
+
+   rgw keystone admin user = {keystone service tenant user name}
+   rgw keystone admin password = {keystone service tenant user password}
+   rgw keystone admin password = {keystone service tenant user password path} # preferred
+   rgw keystone admin tenant = {keystone service tenant name}
+
+
+A Ceph Object Gateway user is mapped into a Keystone ``tenant``. A Keystone user
+has different roles assigned to it on possibly more than a single tenant. When
+the Ceph Object Gateway gets the ticket, it looks at the tenant, and the user
+roles that are assigned to that ticket, and accepts/rejects the request
+according to the ``rgw keystone accepted roles`` configurable.
+
+For a v3 version of the OpenStack Identity API you should replace
+``rgw keystone admin tenant`` with::
+
+   rgw keystone admin domain = {keystone admin domain name}
+   rgw keystone admin project = {keystone admin project name}
+
+For compatibility with previous versions of ceph, it is also
+possible to set ``rgw keystone implicit tenants`` to either
+``s3`` or ``swift``.  This has the effect of splitting
+the identity space such that the indicated protocol will
+only use implicit tenants, and the other protocol will
+never use implicit tenants.  Some older versions of ceph
+only supported implicit tenants with swift.
+
+Ocata (and later)
+-----------------
+
+Keystone itself needs to be configured to point to the Ceph Object Gateway as an
+object-storage endpoint::
+
+  openstack service create --name=swift \
+                           --description="Swift Service" \
+                           object-store
+  +-------------+----------------------------------+
+  | Field       | Value                            |
+  +-------------+----------------------------------+
+  | description | Swift Service                    |
+  | enabled     | True                             |
+  | id          | 37c4c0e79571404cb4644201a4a6e5ee |
+  | name        | swift                            |
+  | type        | object-store                     |
+  +-------------+----------------------------------+
+
+  openstack endpoint create --region RegionOne \
+       --publicurl   "http://radosgw.example.com:8080/swift/v1" \
+       --adminurl    "http://radosgw.example.com:8080/swift/v1" \
+       --internalurl "http://radosgw.example.com:8080/swift/v1" \
+       swift
+  +--------------+------------------------------------------+
+  | Field        | Value                                    |
+  +--------------+------------------------------------------+
+  | adminurl     | http://radosgw.example.com:8080/swift/v1 |
+  | id           | e4249d2b60e44743a67b5e5b38c18dd3         |
+  | internalurl  | http://radosgw.example.com:8080/swift/v1 |
+  | publicurl    | http://radosgw.example.com:8080/swift/v1 |
+  | region       | RegionOne                                |
+  | service_id   | 37c4c0e79571404cb4644201a4a6e5ee         |
+  | service_name | swift                                    |
+  | service_type | object-store                             |
+  +--------------+------------------------------------------+
+
+  $ openstack endpoint show object-store
+  +--------------+------------------------------------------+
+  | Field        | Value                                    |
+  +--------------+------------------------------------------+
+  | adminurl     | http://radosgw.example.com:8080/swift/v1 |
+  | enabled      | True                                     |
+  | id           | e4249d2b60e44743a67b5e5b38c18dd3         |
+  | internalurl  | http://radosgw.example.com:8080/swift/v1 |
+  | publicurl    | http://radosgw.example.com:8080/swift/v1 |
+  | region       | RegionOne                                |
+  | service_id   | 37c4c0e79571404cb4644201a4a6e5ee         |
+  | service_name | swift                                    |
+  | service_type | object-store                             |
+  +--------------+------------------------------------------+
+
+.. note:: If your radosgw ``ceph.conf`` sets the configuration option
+	  ``rgw swift account in url = true``, your ``object-store``
+	  endpoint URLs must be set to include the suffix
+	  ``/v1/AUTH_%(tenant_id)s`` (instead of just ``/v1``).
+
+The Keystone URL is the Keystone admin RESTful API URL. The admin token is the
+token that is configured internally in Keystone for admin requests.
+
+OpenStack Keystone may be terminated with a self signed ssl certificate, in
+order for radosgw to interact with Keystone in such a case, you could either
+install Keystone's ssl certificate in the node running radosgw. Alternatively
+radosgw could be made to not verify the ssl certificate at all (similar to
+OpenStack clients with a ``--insecure`` switch) by setting the value of the
+configurable ``rgw keystone verify ssl`` to false.
+
+
+.. _OpenStack Keystone documentation: http://docs.openstack.org/developer/keystone/configuringservices.html#setting-up-projects-users-and-roles
+
+Cross Project(Tenant) Access
+----------------------------
+
+In order to let a project (earlier called a 'tenant') access buckets belonging to a different project, the following config option needs to be enabled::
+
+   rgw swift account in url = true
+
+The Keystone object-store endpoint must accordingly be configured to include the AUTH_%(project_id)s suffix::
+
+   openstack endpoint create --region RegionOne \
+       --publicurl   "http://radosgw.example.com:8080/swift/v1/AUTH_$(project_id)s" \
+       --adminurl    "http://radosgw.example.com:8080/swift/v1/AUTH_$(project_id)s" \
+       --internalurl "http://radosgw.example.com:8080/swift/v1/AUTH_$(project_id)s" \
+       swift
+  +--------------+--------------------------------------------------------------+
+  | Field        | Value                                                        |
+  +--------------+--------------------------------------------------------------+
+  | adminurl     | http://radosgw.example.com:8080/swift/v1/AUTH_$(project_id)s |
+  | id           | e4249d2b60e44743a67b5e5b38c18dd3                             |
+  | internalurl  | http://radosgw.example.com:8080/swift/v1/AUTH_$(project_id)s |
+  | publicurl    | http://radosgw.example.com:8080/swift/v1/AUTH_$(project_id)s |
+  | region       | RegionOne                                                    |
+  | service_id   | 37c4c0e79571404cb4644201a4a6e5ee                             |
+  | service_name | swift                                                        |
+  | service_type | object-store                                                 |
+  +--------------+--------------------------------------------------------------+
+
+Keystone integration with the S3 API
+------------------------------------
+
+It is possible to use Keystone for authentication even when using the
+S3 API (with AWS-like access and secret keys), if the ``rgw s3 auth
+use keystone`` option is set. For details, see
+:doc:`s3/authentication`.
diff --git a/doc/radosgw/kmip.rst b/doc/radosgw/kmip.rst
new file mode 100644
index 000000000..988897121
--- /dev/null
+++ b/doc/radosgw/kmip.rst
@@ -0,0 +1,219 @@
+================
+KMIP Integration
+================
+
+`KMIP`_ can be used as a secure key management service for
+`Server-Side Encryption`_ (SSE-KMS).
+
+.. ditaa::
+
+           +---------+       +---------+        +------+      +-------+
+           |  Client |       | RadosGW |        | KMIP |      |  OSD  |
+           +---------+       +---------+        +------+      +-------+
+                | create secret   |                 |             |
+                | key for key ID  |                 |             |
+                |-----------------+---------------->|             |
+                |                 |                 |             |
+                | upload object   |                 |             |
+                | with key ID     |                 |             |
+                |---------------->| request secret  |             |
+                |                 | key for key ID  |             |
+                |                 |---------------->|             |
+                |                 |<----------------|             |
+                |                 | return secret   |             |
+                |                 | key             |             |
+                |                 |                 |             |
+                |                 | encrypt object  |             |
+                |                 | with secret key |             |
+                |                 |--------------+  |             |
+                |                 |              |  |             |
+                |                 |<-------------+  |             |
+                |                 |                 |             |
+                |                 | store encrypted |             |
+                |                 | object          |             |
+                |                 |------------------------------>|
+
+#. `Setting KMIP Access for Ceph`_
+#. `Creating Keys in KMIP`_
+#. `Configure the Ceph Object Gateway`_
+#. `Upload object`_
+
+Before you can use KMIP with ceph, you will need to do three things.
+You will need to associate ceph with client information in KMIP,
+and configure ceph to use that client information.
+You will also need to create 1 or more keys in KMIP.
+
+Setting KMIP Access for Ceph
+============================
+
+Setting up Ceph in KMIP is very dependent on the mechanism(s) supported
+by your implementation of KMIP.  Two implementations are described
+here,
+
+1. `IBM Security Guardium Key Lifecycle Manager (SKLM)`__.  This is a well
+   supported commercial product.
+
+__ SKLM_
+
+2. PyKMIP_.  This is a small python project, suitable for experimental
+   and testing use only.
+
+Using IBM SKLM
+--------------
+
+IBM SKLM__ supports client authentication using certificates.
+Certificates may either be self-signed certificates created,
+for instance, using openssl, or certificates may be created
+using SKLM.  Ceph should then be configured (see below) to
+use KMIP and an attempt made to use it.  This will fail,
+but it will leave an "untrusted client device certificate" in SKLM.
+This can be then upgraded to a registered client using the web
+interface to complete the registration process.
+
+__ SKLM_
+
+Find untrusted clients under ``Advanced Configuration``,
+``Client Device Communication Certificates``.  Select
+``Modify SSL/KMIP Certificates for Clients``, then toggle the flag
+``allow the server to trust this certificate and communicate...``.
+
+Using PyKMIP 
+------------
+
+PyKMIP_ has no special registration process, it simply
+trusts the certificate.  However, the certificate has to
+be issued by a certificate authority that is trusted by
+pykmip.  PyKMIP also prefers that the certificate contain
+an extension for "extended key usage".  However, that
+can be defeated by specifying ``enable_tls_client_auth=False``
+in the server configuration.
+
+Creating Keys in KMIP
+=====================
+
+Some KMIP implementations come with a web interface or other
+administrative tools to create and manage keys.  Refer to your
+documentation on that if you wish to use it.  The KMIP protocol can also
+be used to create and manage keys.  PyKMIP comes with a python client
+library that can be used this way.
+
+In preparation to using the pykmip client, you'll need to have a valid
+kmip client key & certificate, such as the one you created for ceph.
+
+Next, you'll then need to download and install it::
+
+  virtualenv $HOME/my-kmip-env
+  source $HOME/my-kmip-env/bin/activate
+  pip install pykmip
+
+Then you'll need to prepare a configuration file
+for the client, something like this::
+
+   cat <<EOF >$HOME/my-kmip-configuration
+   [client]
+   host={hostname}
+   port=5696
+   certfile={clientcert}
+   keyfile={clientkey}
+   ca_certs={clientca}
+   ssl_version=PROTOCOL_TLSv1_2
+   EOF
+
+You will need to replace {hostname} with the name of your kmip host,
+also replace {clientcert} {clientkey} and {clientca} with pathnames to
+a suitable pem encoded certificate, such as the one you created for
+ceph to use.
+
+Now, you can run this python script directly from
+the shell::
+
+  python
+  from kmip.pie import client
+  from kmip import enums
+  import ssl
+  import os
+  import sys
+  import json
+  c = client.ProxyKmipClient(config_file=os.environ['HOME']+"/my-kmip-configuration")
+
+  while True:
+    l=sys.stdin.readline()
+    keyname=l.strip()
+    if keyname == "": break
+    with c:
+      key_id = c.create(
+	  enums.CryptographicAlgorithm.AES,
+	  256,
+	  operation_policy_name='default',
+	  name=keyname,
+	  cryptographic_usage_mask=[
+	      enums.CryptographicUsageMask.ENCRYPT,
+	      enums.CryptographicUsageMask.DECRYPT
+	  ]
+      )
+      c.activate(key_id)
+      attrs = c.get_attributes(uid=key_id)
+      r = {}
+      for a in attrs[1]:
+       r[str(a.attribute_name)] = str(a.attribute_value)
+      print (json.dumps(r))
+
+If this is all entered at the shell prompt, python will
+prompt with ">>>" then "..." until the script is read in,
+after which it will read and process names with no prompt
+until a blank line or end of file (^D) is given it, or
+an error occurs.  Of course you can turn this into a regular
+python script if you prefer.
+
+Configure the Ceph Object Gateway
+=================================
+
+Edit the Ceph configuration file to enable Vault as a KMS backend for
+server-side encryption::
+
+  rgw crypt s3 kms backend = kmip
+  rgw crypt kmip ca path: /etc/ceph/kmiproot.crt
+  rgw crypt kmip client cert: /etc/ceph/kmip-client.crt
+  rgw crypt kmip client key: /etc/ceph/private/kmip-client.key
+  rgw crypt kmip kms key template: pykmip-$keyid
+
+You may need to change the paths above to match where
+you actually want to store kmip certificate data.
+
+The kmip key template describes how ceph will modify
+the name given to it before it looks it up
+in kmip.  The default is just "$keyid".
+If you don't want ceph to see all your kmip
+keys, you can use this to limit ceph to just the
+designated subset of your kmip key namespace.
+
+Upload object
+=============
+
+When uploading an object to the Gateway, provide the SSE key ID in the request.
+As an example, for the kv engine, using the AWS command-line client::
+
+  aws --endpoint=http://radosgw:8000 s3 cp plaintext.txt \
+  s3://mybucket/encrypted.txt --sse=aws:kms --sse-kms-key-id mybucketkey
+  
+As an example, for the transit engine, using the AWS command-line client::
+
+  aws --endpoint=http://radosgw:8000 s3 cp plaintext.txt \
+  s3://mybucket/encrypted.txt --sse=aws:kms --sse-kms-key-id mybucketkey
+
+The Object Gateway will fetch the key from Vault, encrypt the object and store
+it in the bucket. Any request to download the object will make the Gateway
+automatically retrieve the correspondent key from Vault and decrypt the object.
+
+Note that the secret will be fetched from kmip using a name constructed
+from the key template, replacing ``$keyid`` with the key provided.
+
+With the ceph configuration given above,
+radosgw would fetch the secret from::
+
+  pykmip-mybucketkey
+
+.. _Server-Side Encryption: ../encryption
+.. _KMIP: http://www.oasis-open.org/committees/kmip/
+.. _SKLM: https://www.ibm.com/products/ibm-security-key-lifecycle-manager
+.. _PyKMIP: https://pykmip.readthedocs.io/en/latest/
diff --git a/doc/radosgw/layout.rst b/doc/radosgw/layout.rst
new file mode 100644
index 000000000..37c3a72cd
--- /dev/null
+++ b/doc/radosgw/layout.rst
@@ -0,0 +1,207 @@
+===========================
+ Rados Gateway Data Layout
+===========================
+
+Although the source code is the ultimate guide, this document helps
+new developers to get up to speed with the implementation details.
+
+Introduction
+------------
+
+Swift offers something called a *container*, which we use interchangeably with
+the term *bucket*, so we say that RGW's buckets implement Swift containers.
+
+This document does not consider how RGW operates on these structures,
+e.g. the use of encode() and decode() methods for serialization and so on.
+
+Conceptual View
+---------------
+
+Although RADOS only knows about pools and objects with their xattrs and
+omap[1], conceptually RGW organizes its data into three different kinds:
+metadata, bucket index, and data.
+
+Metadata
+^^^^^^^^
+
+We have 3 'sections' of metadata: 'user', 'bucket', and 'bucket.instance'.
+You can use the following commands to introspect metadata entries: ::
+
+    $ radosgw-admin metadata list
+    $ radosgw-admin metadata list bucket
+    $ radosgw-admin metadata list bucket.instance
+    $ radosgw-admin metadata list user
+
+    $ radosgw-admin metadata get bucket:<bucket>
+    $ radosgw-admin metadata get bucket.instance:<bucket>:<bucket_id>
+    $ radosgw-admin metadata get user:<user>   # get or set
+    
+Some variables have been used in above commands, they are:
+
+- user: Holds user information
+- bucket: Holds a mapping between bucket name and bucket instance id
+- bucket.instance: Holds bucket instance information[2]
+
+Every metadata entry is kept on a single RADOS object. See below for implementation details.
+
+Note that the metadata is not indexed. When listing a metadata section we do a
+RADOS ``pgls`` operation on the containing pool.
+
+Bucket Index
+^^^^^^^^^^^^
+
+It's a different kind of metadata, and kept separately. The bucket index holds
+a key-value map in RADOS objects. By default it is a single RADOS object per
+bucket, but it is possible since Hammer to shard that map over multiple RADOS
+objects. The map itself is kept in omap, associated with each RADOS object.
+The key of each omap is the name of the objects, and the value holds some basic
+metadata of that object -- metadata that shows up when listing the bucket.
+Also, each omap holds a header, and we keep some bucket accounting metadata
+in that header (number of objects, total size, etc.).
+
+Note that we also hold other information in the bucket index, and it's kept in
+other key namespaces. We can hold the bucket index log there, and for versioned
+objects there is more information that we keep on other keys.
+
+Data
+^^^^
+
+Objects data is kept in one or more RADOS objects for each rgw object.
+
+Object Lookup Path
+------------------
+
+When accessing objects, REST APIs come to RGW with three parameters:
+account information (access key in S3 or account name in Swift),
+bucket or container name, and object name (or key). At present, RGW only
+uses account information to find out the user ID and for access control.
+Only the bucket name and object key are used to address the object in a pool.
+
+The user ID in RGW is a string, typically the actual user name from the user
+credentials and not a hashed or mapped identifier.
+
+When accessing a user's data, the user record is loaded from an object
+"<user_id>" in pool "default.rgw.meta" with namespace "users.uid".
+
+Bucket names are represented in the pool "default.rgw.meta" with namespace
+"root". Bucket record is
+loaded in order to obtain so-called marker, which serves as a bucket ID.
+
+The object is located in pool "default.rgw.buckets.data".
+Object name is "<marker>_<key>",
+for example "default.7593.4_image.png", where the marker is "default.7593.4"
+and the key is "image.png". Since these concatenated names are not parsed,
+only passed down to RADOS, the choice of the separator is not important and
+causes no ambiguity. For the same reason, slashes are permitted in object
+names (keys).
+
+It is also possible to create multiple data pools and make it so that
+different users buckets will be created in different RADOS pools by default,
+thus providing the necessary scaling. The layout and naming of these pools
+is controlled by a 'policy' setting.[3]
+
+An RGW object may consist of several RADOS objects, the first of which
+is the head that contains the metadata, such as manifest, ACLs, content type,
+ETag, and user-defined metadata. The metadata is stored in xattrs.
+The head may also contain up to 512 kilobytes of object data, for efficiency
+and atomicity. The manifest describes how each object is laid out in RADOS
+objects.
+
+Bucket and Object Listing
+-------------------------
+
+Buckets that belong to a given user are listed in an omap of an object named
+"<user_id>.buckets" (for example, "foo.buckets") in pool "default.rgw.meta"
+with namespace "users.uid".
+These objects are accessed when listing buckets, when updating bucket
+contents, and updating and retrieving bucket statistics (e.g. for quota).
+
+See the user-visible, encoded class 'cls_user_bucket_entry' and its
+nested class 'cls_user_bucket' for the values of these omap entires.
+
+These listings are kept consistent with buckets in pool ".rgw".
+
+Objects that belong to a given bucket are listed in a bucket index,
+as discussed in sub-section 'Bucket Index' above. The default naming
+for index objects is ".dir.<marker>" in pool "default.rgw.buckets.index".
+
+Footnotes
+---------
+
+[1] Omap is a key-value store, associated with an object, in a way similar
+to how Extended Attributes associate with a POSIX file. An object's omap
+is not physically located in the object's storage, but its precise
+implementation is invisible and immaterial to RADOS Gateway.
+In Hammer, one LevelDB is used to store omap in each OSD.
+
+[2] Before the Dumpling release, the 'bucket.instance' metadata did not
+exist and the 'bucket' metadata contained its information. It is possible
+to encounter such buckets in old installations.
+
+[3] The pool names have been changed starting with the Infernalis release.
+If you are looking at an older setup, some details may be different. In
+particular there was a different pool for each of the namespaces that are
+now being used inside the default.root.meta pool.
+
+Appendix: Compendium
+--------------------
+
+Known pools:
+
+.rgw.root
+  Unspecified region, zone, and global information records, one per object.
+
+<zone>.rgw.control
+  notify.<N>
+
+<zone>.rgw.meta
+  Multiple namespaces with different kinds of metadata:
+
+  namespace: root
+    <bucket>
+    .bucket.meta.<bucket>:<marker>   # see put_bucket_instance_info()
+
+    The tenant is used to disambiguate buckets, but not bucket instances.
+    Example::
+
+      .bucket.meta.prodtx:test%25star:default.84099.6
+      .bucket.meta.testcont:default.4126.1
+      .bucket.meta.prodtx:testcont:default.84099.4
+      prodtx/testcont
+      prodtx/test%25star
+      testcont
+
+  namespace: users.uid
+    Contains _both_ per-user information (RGWUserInfo) in "<user>" objects
+    and per-user lists of buckets in omaps of "<user>.buckets" objects.
+    The "<user>" may contain the tenant if non-empty, for example::
+
+      prodtx$prodt
+      test2.buckets
+      prodtx$prodt.buckets
+      test2
+
+  namespace: users.email
+    Unimportant
+
+  namespace: users.keys
+    47UA98JSTJZ9YAN3OS3O
+
+    This allows ``radosgw`` to look up users by their access keys during authentication.
+
+  namespace: users.swift
+    test:tester
+
+<zone>.rgw.buckets.index
+  Objects are named ".dir.<marker>", each contains a bucket index.
+  If the index is sharded, each shard appends the shard index after
+  the marker.
+
+<zone>.rgw.buckets.data
+  default.7593.4__shadow_.488urDFerTYXavx4yAd-Op8mxehnvTI_1
+  <marker>_<key>
+
+An example of a marker would be "default.16004.1" or "default.7593.4".
+The current format is "<zone>.<instance_id>.<bucket_id>". But once
+generated, a marker is not parsed again, so its format may change
+freely in the future.
diff --git a/doc/radosgw/ldap-auth.rst b/doc/radosgw/ldap-auth.rst
new file mode 100644
index 000000000..486d0c623
--- /dev/null
+++ b/doc/radosgw/ldap-auth.rst
@@ -0,0 +1,167 @@
+===================
+LDAP Authentication
+===================
+
+.. versionadded:: Jewel
+
+You can delegate the Ceph Object Gateway authentication to an LDAP server.
+
+How it works
+============
+
+The Ceph Object Gateway extracts the users LDAP credentials from a token. A
+search filter is constructed with the user name. The Ceph Object Gateway uses
+the configured service account to search the directory for a matching entry. If
+an entry is found, the Ceph Object Gateway attempts to bind to the found
+distinguished name with the password from the token. If the credentials are
+valid, the bind will succeed, and the Ceph Object Gateway will grant access and
+radosgw-user will be created with the provided username.
+
+You can limit the allowed users by setting the base for the search to a
+specific organizational unit or by specifying a custom search filter, for
+example requiring specific group membership, custom object classes, or
+attributes.
+
+The LDAP credentials must be available on the server to perform the LDAP
+authentication. Make sure to set the ``rgw`` log level low enough to hide the
+base-64-encoded credentials / access tokens.
+
+Requirements
+============
+
+- **LDAP or Active Directory:** A running LDAP instance accessible by the Ceph
+  Object Gateway
+- **Service account:** LDAP credentials to be used by the Ceph Object Gateway
+  with search permissions
+- **User account:** At least one user account in the LDAP directory
+- **Do not overlap LDAP and local users:** You should not use the same user
+  names for local users and for users being authenticated by using LDAP. The
+  Ceph Object Gateway cannot distinguish them and it treats them as the same
+  user.
+
+Sanity checks
+=============
+
+Use the ``ldapsearch`` utility to verify the service account or the LDAP connection:
+
+::
+
+  # ldapsearch -x -D "uid=ceph,ou=system,dc=example,dc=com" -W \
+  -H ldaps://example.com -b "ou=users,dc=example,dc=com" 'uid=*' dn
+
+.. note:: Make sure to use the same LDAP parameters like in the Ceph configuration file to
+          eliminate possible problems.
+
+Configuring the Ceph Object Gateway to use LDAP authentication
+==============================================================
+
+The following parameters in the Ceph configuration file are related to the LDAP
+authentication:
+
+- ``rgw_s3_auth_use_ldap``: Set this to ``true`` to enable S3 authentication with LDAP
+- ``rgw_ldap_uri``:  Specifies the LDAP server to use. Make sure to use the
+  ``ldaps://<fqdn>:<port>`` parameter to not transmit clear text credentials
+  over the wire.
+- ``rgw_ldap_binddn``: The Distinguished Name (DN) of the service account used
+  by the Ceph Object Gateway
+- ``rgw_ldap_secret``: Path to file containing credentials for ``rgw_ldap_binddn``
+- ``rgw_ldap_searchdn``: Specifies the base in the directory information tree
+  for searching users. This might be your users organizational unit or some
+  more specific Organizational Unit (OU).
+- ``rgw_ldap_dnattr``: The attribute being used in the constructed search
+  filter to match a username. Depending on your Directory Information Tree
+  (DIT) this would probably be ``uid`` or ``cn``. The generated filter string
+  will be, e.g., ``cn=some_username``.
+- ``rgw_ldap_searchfilter``: If not specified, the Ceph Object Gateway
+  automatically constructs the search filter with the ``rgw_ldap_dnattr``
+  setting. Use this parameter to narrow the list of allowed users in very
+  flexible ways. Consult the *Using a custom search filter to limit user access
+  section* for details
+
+Using a custom search filter to limit user access
+=================================================
+
+There are two ways to use the ``rgw_search_filter`` parameter:
+
+Specifying a partial filter to further limit the constructed search filter
+--------------------------------------------------------------------------
+
+An example for a partial filter:
+
+::
+
+  "objectclass=inetorgperson"
+
+The Ceph Object Gateway will generate the search filter as usual with the
+user name from the token and the value of ``rgw_ldap_dnattr``. The constructed
+filter is then combined with the partial filter from the ``rgw_search_filter``
+attribute. Depending on the user name and the settings the final search filter
+might become:
+
+::
+
+  "(&(uid=hari)(objectclass=inetorgperson))"
+
+So user ``hari`` will only be granted access if he is found in the LDAP
+directory, has an object class of ``inetorgperson``, and did specify a valid
+password.
+
+Specifying a complete filter
+----------------------------
+
+A complete filter must contain a ``@USERNAME@`` token which will be substituted
+with the user name during the authentication attempt. The ``rgw_ldap_dnattr``
+parameter is not used anymore in this case. For example, to limit valid users
+to a specific group, use the following filter:
+
+::
+
+  "(&(uid=@USERNAME@)(memberOf=cn=ceph-users,ou=groups,dc=mycompany,dc=com))"
+
+.. note:: Using the ``memberOf`` attribute in LDAP searches requires server side
+          support from you specific LDAP server implementation.
+
+Generating an access token for LDAP authentication
+==================================================
+
+The ``radosgw-token`` utility generates the access token based on the LDAP
+user name and password. It will output a base-64 encoded string which is the
+access token.
+
+::
+
+  # export RGW_ACCESS_KEY_ID="<username>"
+  # export RGW_SECRET_ACCESS_KEY="<password>"
+  # radosgw-token --encode
+
+.. important:: The access token is a base-64 encoded JSON struct and contains
+               the LDAP credentials as a clear text.
+
+Alternatively, users can also generate the token manually by base-64-encoding
+this JSON snippet, if they do not have the ``radosgw-token`` tool installed.
+
+::
+
+  {
+    "RGW_TOKEN": {
+      "version": 1,
+      "type": "ldap",
+      "id": "your_username",
+      "key": "your_clear_text_password_here"
+    }
+  }
+
+Using the access token
+======================
+
+Use your favorite S3 client and specify the token as the access key in your
+client or environment variables.
+
+::
+
+  # export AWS_ACCESS_KEY_ID=<base64-encoded token generated by radosgw-token>
+  # export AWS_SECRET_ACCESS_KEY="" # define this with an empty string, otherwise tools might complain about missing env variables.
+
+.. important:: The access token is a base-64 encoded JSON struct and contains
+               the LDAP credentials as a clear text. DO NOT share it unless
+               you want to share your clear text password!
diff --git a/doc/radosgw/lua-scripting.rst b/doc/radosgw/lua-scripting.rst
new file mode 100644
index 000000000..0f385c46f
--- /dev/null
+++ b/doc/radosgw/lua-scripting.rst
@@ -0,0 +1,393 @@
+=============
+Lua Scripting
+=============
+
+.. versionadded:: Pacific
+
+.. contents::
+
+This feature allows users to upload Lua scripts to different context in the radosgw. The two supported context are "preRequest" that will execute a script before the
+operation was taken, and "postRequest" that will execute after each operation is taken. Script may be uploaded to address requests for users of a specific tenant.
+The script can access fields in the request and modify some fields. All Lua language features can be used in the script.
+
+By default, all lua standard libraries are available in the script, however, in order to allow for other lua modules to be used in the script, we support adding packages to an allowlist:
+
+  - All packages in the allowlist are being re-installed using the luarocks package manager on radosgw restart. Therefore a restart is needed for adding or removing of packages to take effect 
+  - To add a package that contains C source code that needs to be compiled, use the `--allow-compilation` flag. In this case a C compiler needs to be available on the host
+  - Lua packages are installed in, and used from, a directory local to the radosgw. Meaning that lua packages in the allowlist are separated from any lua packages available on the host.
+    By default, this directory would be `/tmp/luarocks/<entity name>`. Its prefix part (`/tmp/luarocks/`) could be set to a different location via the `rgw_luarocks_location` configuration parameter. 
+    Note that this parameter should not be set to one of the default locations where luarocks install packages (e.g. `$HOME/.luarocks`, `/usr/lib64/lua`, `/usr/share/lua`)
+	
+
+.. toctree::
+   :maxdepth: 1
+
+
+Script Management via CLI
+-------------------------
+
+To upload a script:
+   
+::
+   
+   # radosgw-admin script put --infile={lua-file} --context={preRequest|postRequest} [--tenant={tenant-name}]
+
+
+To print the content of the script to standard output:
+
+::
+   
+   # radosgw-admin script get --context={preRequest|postRequest} [--tenant={tenant-name}]
+
+
+To remove the script:
+
+::
+   
+   # radosgw-admin script rm --context={preRequest|postRequest} [--tenant={tenant-name}]
+
+
+Package Management via CLI
+--------------------------
+
+To add a package to the allowlist:
+
+::
+
+  # radosgw-admin script-package add --package={package name} [--allow-compilation]
+
+
+To remove a package from the allowlist:
+
+::
+
+  # radosgw-admin script-package rm --package={package name}
+
+
+To print the list of packages in the allowlist:
+
+::
+
+  # radosgw-admin script-package list
+
+
+Context Free Functions
+----------------------
+Debug Log
+~~~~~~~~~
+The ``RGWDebugLog()`` function accepts a string and prints it to the debug log with priority 20.
+Each log message is prefixed ``Lua INFO:``. This function has no return value.
+
+Request Fields
+-----------------
+
+.. warning:: This feature is experimental. Fields may be removed or renamed in the future.
+
+.. note::
+
+    - Although Lua is a case-sensitive language, field names provided by the radosgw are case-insensitive. Function names remain case-sensitive.
+    - Fields marked "optional" can have a nil value.
+    - Fields marked as "iterable" can be used by the pairs() function and with the # length operator.
+    - All table fields can be used with the bracket operator ``[]``.
+    - ``time`` fields are strings with the following format: ``%Y-%m-%d %H:%M:%S``.
+
+
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| Field                                              | Type     | Description                                                  | Iterable | Writeable | Optional |
++====================================================+==========+==============================================================+==========+===========+==========+
+| ``Request.RGWOp``                                  | string   | radosgw operation                                            | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.DecodedURI``                             | string   | decoded URI                                                  | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.ContentLength``                          | integer  | size of the request                                          | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.GenericAttributes``                      | table    | string to string generic attributes map                      | yes      | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Response``                               | table    | response to the request                                      | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Response.HTTPStatusCode``                | integer  | HTTP status code                                             | no       | yes       | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Response.HTTPStatus``                    | string   | HTTP status text                                             | no       | yes       | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Response.RGWCode``                       | integer  | radosgw error code                                           | no       | yes       | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Response.Message``                       | string   | response message                                             | no       | yes       | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.SwiftAccountName``                       | string   | swift account name                                           | no       | no        | yes      |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket``                                 | table    | info on the bucket                                           | no       | no        | yes      |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.Tenant``                          | string   | tenant of the bucket                                         | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.Name``                            | string   | bucket name                                                  | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.Marker``                          | string   | bucket marker (initial id)                                   | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.Id``                              | string   | bucket id                                                    | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.Count``                           | integer  | number of objects in the bucket                              | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.Size``                            | integer  | total size of objects in the bucket                          | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.ZoneGroupId``                     | string   | zone group of the bucket                                     | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.CreationTime``                    | time     | creation time of the bucket                                  | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.MTime``                           | time     | modification time of the bucket                              | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.Quota``                           | table    | bucket quota                                                 | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.Quota.MaxSize``                   | integer  | bucket quota max size                                        | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.Quota.MaxObjects``                | integer  | bucket quota max number of objects                           | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Reques.Bucket.Quota.Enabled``                    | boolean  | bucket quota is enabled                                      | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.Quota.Rounded``                   | boolean  | bucket quota is rounded to 4K                                | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.PlacementRule``                   | table    | bucket placement rule                                        | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.PlacementRule.Name``              | string   | bucket placement rule name                                   | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.PlacementRule.StorageClass``      | string   | bucket placement rule storage class                          | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.User``                            | table    | bucket owner                                                 | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.User.Tenant``                     | string   | bucket owner tenant                                          | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Bucket.User.Id``                         | string   | bucket owner id                                              | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Object``                                 | table    | info on the object                                           | no       | no        | yes      |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Object.Name``                            | string   | object name                                                  | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Object.Instance``                        | string   | object version                                               | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Object.Id``                              | string   | object id                                                    | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Object.Size``                            | integer  | object size                                                  | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Object.MTime``                           | time     | object mtime                                                 | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.CopyFrom``                               | table    | information on copy operation                                | no       | no        | yes      |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.CopyFrom.Tenant``                        | string   | tenant of the object copied from                             | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.CopyFrom.Bucket``                        | string   | bucket of the object copied from                             | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.CopyFrom.Object``                        | table    | object copied from. See: ``Request.Object``                  | no       | no        | yes      |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.ObjectOwner``                            | table    | object owner                                                 | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.ObjectOwner.DisplayName``                | string   | object owner display name                                    | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.ObjectOwner.User``                       | table    | object user. See: ``Request.Bucket.User``                    | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.ZoneGroup.Name``                         | string   | name of zone group                                           | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.ZoneGroup.Endpoint``                     | string   | endpoint of zone group                                       | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.UserAcl``                                | table    | user ACL                                                     | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.UserAcl.Owner``                          | table    | user ACL owner. See: ``Request.ObjectOwner``                 | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.UserAcl.Grants``                         | table    | user ACL map of string to grant                              | yes      | no        | no       |
+|                                                    |          | note: grants without an Id are not presented when iterated   |          |           |          |
+|                                                    |          | and only one of them can be accessed via brackets            |          |           |          |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.UserAcl.Grants["<name>"]``               | table    | user ACL grant                                               | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.UserAcl.Grants["<name>"].Type``          | integer  | user ACL grant type                                          | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.UserAcl.Grants["<name>"].User``          | table    | user ACL grant user                                          | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.UserAcl.Grants["<name>"].User.Tenant``   | table    | user ACL grant user tenant                                   | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.UserAcl.Grants["<name>"].User.Id``       | table    | user ACL grant user id                                       | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.UserAcl.Grants["<name>"].GroupType``     | integer  | user ACL grant group type                                    | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.UserAcl.Grants["<name>"].Referer``       | string   | user ACL grant referer                                       | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.BucketAcl``                              | table    | bucket ACL. See: ``Request.UserAcl``                         | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.ObjectAcl``                              | table    | object ACL. See: ``Request.UserAcl``                         | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Environment``                            | table    | string to string environment map                             | yes      | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Policy``                                 | table    | policy                                                       | no       | no        | yes      |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Policy.Text``                            | string   | policy text                                                  | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Policy.Id``                              | string   | policy Id                                                    | no       | no        | yes      |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Policy.Statements``                      | table    | list of string statements                                    | yes      | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.UserPolicies``                           | table    | list of user policies                                        | yes      | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.UserPolicies[<index>]``                  | table    | user policy. See: ``Request.Policy``                         | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.RGWId``                                  | string   | radosgw host id: ``<host>-<zone>-<zonegroup>``               | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.HTTP``                                   | table    | HTTP header                                                  | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.HTTP.Parameters``                        | table    | string to string parameter map                               | yes      | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.HTTP.Resources``                         | table    | string to string resource map                                | yes      | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.HTTP.Metadata``                          | table    | string to string metadata map                                | yes      | yes       | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.HTTP.Host``                              | string   | host name                                                    | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.HTTP.Method``                            | string   | HTTP method                                                  | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.HTTP.URI``                               | string   | URI                                                          | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.HTTP.QueryString``                       | string   | HTTP query string                                            | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.HTTP.Domain``                            | string   | domain name                                                  | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Time``                                   | time     | request time                                                 | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Dialect``                                | string   | "S3" or "Swift"                                              | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Id``                                     | string   | request Id                                                   | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.TransactionId``                          | string   | transaction Id                                               | no       | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+| ``Request.Tags``                                   | table    | object tags map                                              | yes      | no        | no       |
++----------------------------------------------------+----------+--------------------------------------------------------------+----------+-----------+----------+
+
+Request Functions
+--------------------
+Operations Log
+~~~~~~~~~~~~~~
+The ``Request.Log()`` function prints the requests into the operations log. This function has no parameters. It returns 0 for success and an error code if it fails.
+
+Lua Code Samples
+----------------
+- Print information on source and destination objects in case of copy:
+
+.. code-block:: lua
+ 
+  function print_object(object)
+    RGWDebugLog("  Name: " .. object.Name)
+    RGWDebugLog("  Instance: " .. object.Instance)
+    RGWDebugLog("  Id: " .. object.Id)
+    RGWDebugLog("  Size: " .. object.Size)
+    RGWDebugLog("  MTime: " .. object.MTime)
+  end
+
+  if Request.CopyFrom and Request.Object and Request.CopyFrom.Object then
+    RGWDebugLog("copy from object:")
+    print_object(Request.CopyFrom.Object)
+    RGWDebugLog("to object:")
+    print_object(Request.Object)
+  end
+
+- Print ACLs via a "generic function":
+
+.. code-block:: lua
+
+  function print_owner(owner)
+    RGWDebugLog("Owner:")
+    RGWDebugLog("  Dispaly Name: " .. owner.DisplayName)
+    RGWDebugLog("  Id: " .. owner.User.Id)
+    RGWDebugLog("  Tenanet: " .. owner.User.Tenant)
+  end
+
+  function print_acl(acl_type)
+    index = acl_type .. "ACL"
+    acl = Request[index]
+    if acl then
+      RGWDebugLog(acl_type .. "ACL Owner")
+      print_owner(acl.Owner)
+      RGWDebugLog("  there are " .. #acl.Grants .. " grant for owner")
+      for k,v in pairs(acl.Grants) do
+        RGWDebugLog("    Grant Key: " .. k)
+        RGWDebugLog("    Grant Type: " .. v.Type)
+        RGWDebugLog("    Grant Group Type: " .. v.GroupType)
+        RGWDebugLog("    Grant Referer: " .. v.Referer)
+        RGWDebugLog("    Grant User Tenant: " .. v.User.Tenant)
+        RGWDebugLog("    Grant User Id: " .. v.User.Id)
+      end
+    else
+      RGWDebugLog("no " .. acl_type .. " ACL in request: " .. Request.Id)
+    end 
+  end
+
+  print_acl("User")
+  print_acl("Bucket")
+  print_acl("Object")
+
+- Use of operations log only in case of errors:
+
+.. code-block:: lua
+  
+  if Request.Response.HTTPStatusCode ~= 200 then
+    RGWDebugLog("request is bad, use ops log")
+    rc = Request.Log()
+    RGWDebugLog("ops log return code: " .. rc)
+  end
+
+- Set values into the error message:
+
+.. code-block:: lua
+
+  if Request.Response.HTTPStatusCode == 500 then
+    Request.Response.Message = "<Message> something bad happened :-( </Message>"
+  end
+
+- Add metadata to objects that was not originally sent by the client:
+
+In the `preRequest` context we should add:
+
+.. code-block:: lua
+
+  if Request.RGWOp == 'put_obj' then
+    Request.HTTP.Metadata["x-amz-meta-mydata"] = "my value"
+  end
+
+In the `postRequest` context we look at the metadata:
+
+.. code-block:: lua
+
+  RGWDebugLog("number of metadata entries is: " .. #Request.HTTP.Metadata)
+  for k, v in pairs(Request.HTTP.Metadata) do
+    RGWDebugLog("key=" .. k .. ", " .. "value=" .. v)
+  end
+ 
+- Use modules to create Unix socket based, JSON encoded, "access log":
+
+First we should add the following packages to the allowlist:
+
+::
+
+  # radosgw-admin script-package add --package=luajson
+  # radosgw-admin script-package add --package=luasocket --allow-compilation
+
+
+Then, do a restart for the radosgw and upload the following script to the `postRequest` context:
+
+.. code-block:: lua
+
+  if Request.RGWOp == "get_obj" then
+    local json = require("json")
+    local socket = require("socket")
+    local unix = require("socket.unix")
+    local s = assert(unix())
+    E = {}
+
+    msg = {bucket = (Request.Bucket or (Request.CopyFrom or E).Bucket).Name,
+      time = Request.Time,
+      operation = Request.RGWOp,
+      http_status = Request.Response.HTTPStatusCode,
+      error_code = Request.Response.HTTPStatus,
+      object_size = Request.Object.Size,
+      trans_id = Request.TransactionId}
+
+    assert(s:connect("/tmp/socket"))
+    assert(s:send(json.encode(msg).."\n"))
+    assert(s:close())
+  end
+
diff --git a/doc/radosgw/mfa.rst b/doc/radosgw/mfa.rst
new file mode 100644
index 000000000..0cbead85f
--- /dev/null
+++ b/doc/radosgw/mfa.rst
@@ -0,0 +1,102 @@
+.. _rgw_mfa:
+
+==========================================
+RGW Support for Multifactor Authentication
+==========================================
+
+.. versionadded:: Mimic
+
+The S3 multifactor authentication (MFA) feature allows
+users to require the use of one-time password when removing
+objects on certain buckets. The buckets need to be configured
+with versioning and MFA enabled which can be done through
+the S3 api.
+
+Time-based one time password tokens can be assigned to a user
+through radosgw-admin. Each token has a secret seed, and a serial
+id that is assigned to it. Tokens are added to the user, can
+be listedm removed, and can also be re-synchronized.
+
+Multisite
+=========
+
+While the MFA IDs are set on the user's metadata, the
+actual MFA one time password configuration resides in the local zone's
+osds. Therefore, in a multi-site environment it is advisable to use
+different tokens for different zones.
+
+
+Terminology
+=============
+
+-``TOTP``: Time-based One Time Password
+
+-``token serial``: a string that represents the ID of a TOTP token
+
+-``token seed``: the secret seed that is used to calculate the TOTP
+
+-``totp seconds``: the time resolution that is being used for TOTP generation
+
+-``totp window``: the number of TOTP tokens that are checked before and after the current token when validating token
+
+-``totp pin``: the valid value of a TOTP token at a certain time
+
+
+Admin commands
+==============
+
+Create a new MFA TOTP token
+------------------------------------
+
+::
+
+   # radosgw-admin mfa create --uid=<user-id> \
+                              --totp-serial=<serial> \
+                              --totp-seed=<seed> \
+                              [ --totp-seed-type=<hex|base32> ] \
+                              [ --totp-seconds=<num-seconds> ] \
+                              [ --totp-window=<twindow> ]
+
+List MFA TOTP tokens
+---------------------
+
+::
+
+   # radosgw-admin mfa list --uid=<user-id>
+
+
+Show MFA TOTP token
+------------------------------------
+
+::
+
+   # radosgw-admin mfa get --uid=<user-id> --totp-serial=<serial>
+
+
+Delete MFA TOTP token
+------------------------
+
+::
+
+   # radosgw-admin mfa remove --uid=<user-id> --totp-serial=<serial>
+
+
+Check MFA TOTP token
+--------------------------------
+
+Test a TOTP token pin, needed for validating that TOTP functions correctly. ::
+
+   # radosgw-admin mfa check --uid=<user-id> --totp-serial=<serial> \
+                             --totp-pin=<pin>
+
+
+Re-sync MFA TOTP token
+--------------------------------
+
+In order to re-sync the TOTP token (in case of time skew). This requires
+feeding two consecutive pins: the previous pin, and the current pin. ::
+
+   # radosgw-admin mfa resync --uid=<user-id> --totp-serial=<serial> \
+                              --totp-pin=<prev-pin> --totp=pin=<current-pin>
+
+
diff --git a/doc/radosgw/multisite-sync-policy.rst b/doc/radosgw/multisite-sync-policy.rst
new file mode 100644
index 000000000..56342473c
--- /dev/null
+++ b/doc/radosgw/multisite-sync-policy.rst
@@ -0,0 +1,714 @@
+=====================
+Multisite Sync Policy
+=====================
+
+.. versionadded:: Octopus
+
+Multisite bucket-granularity sync policy provides fine grained control of data movement between buckets in different zones. It extends the zone sync mechanism. Previously buckets were being treated symmetrically, that is -- each (data) zone holds a mirror of that bucket that should be the same as all the other zones. Whereas leveraging the bucket-granularity sync policy is possible for buckets to diverge, and a bucket can pull data from other buckets (ones that don't share its name or its ID) in different zone.  The sync process was assuming therefore that the bucket sync source and the bucket sync destination were always referring to the same bucket, now that is not the case anymore.
+
+The sync policy supersedes the old zonegroup coarse configuration (sync_from*). The sync policy can be configured at the zonegroup level (and if it is configured it replaces the old style config), but it can also be configured at the bucket level.
+
+In the sync policy multiple groups that can contain lists of data-flow configurations can be defined, as well as lists of pipe configurations. The data-flow defines the flow of data between the different zones. It can define symmetrical data flow, in which multiple zones sync data from each other, and it can define directional data flow, in which the data moves in one way from one zone to another.
+
+A pipe defines the actual buckets that can use these data flows, and the properties that are associated with it (for example: source object prefix).
+
+A sync policy group can be in 3 states:
+
++----------------------------+----------------------------------------+
+|  Value                     | Description                            |
++============================+========================================+
+| ``enabled``                | sync is allowed and enabled            |
++----------------------------+----------------------------------------+
+| ``allowed``                | sync is allowed                        |
++----------------------------+----------------------------------------+
+| ``forbidden``              | sync (as defined by this group) is not |
+|                            | allowed and can override other groups  |
++----------------------------+----------------------------------------+
+
+A policy can be defined at the bucket level. A bucket level sync policy inherits the data flow of the zonegroup policy, and can only define a subset of what the zonegroup allows.
+
+A wildcard zone, and a wildcard bucket parameter in the policy defines all relevant zones, or all relevant buckets. In the context of a bucket policy it means the current bucket instance.  A disaster recovery configuration where entire zones are mirrored doesn't require configuring anything on the buckets. However, for a fine grained bucket sync it would be better to configure the pipes to be synced by allowing (status=allowed) them at the zonegroup level (e.g., using wildcards), but only enable the specific sync at the bucket level (status=enabled). If needed, the policy at the bucket level can limit the data movement to specific relevant zones.
+
+.. important:: Any changes to the zonegroup policy needs to be applied on the
+               zonegroup master zone, and require period update and commit. Changes
+               to the bucket policy needs to be applied on the zonegroup master
+               zone. The changes are dynamically handled by rgw.
+
+
+S3 Replication API
+~~~~~~~~~~~~~~~~~~
+
+The S3 bucket replication api has also been implemented, and allows users to create replication rules between different buckets. Note though that while the AWS replication feature allows bucket replication within the same zone, rgw does not allow it at the moment.  However, the rgw api also added a new 'Zone' array that allows users to select to what zones the specific bucket will be synced.
+
+
+Sync Policy Control Reference
+=============================
+
+
+Get Sync Policy
+~~~~~~~~~~~~~~~
+
+To retrieve the current zonegroup sync policy, or a specific bucket policy:
+
+::
+
+     # radosgw-admin sync policy get [--bucket=<bucket>]
+
+
+Create Sync Policy Group
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+To create a sync policy group:
+
+::
+
+      # radosgw-admin sync group create [--bucket=<bucket>]                      \
+                                        --group-id=<group-id>                    \
+                                        --status=<enabled | allowed | forbidden> \
+                                       
+      
+Modify Sync Policy Group
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+To modify a sync policy group:
+
+::
+
+      # radosgw-admin sync group modify [--bucket=<bucket>]                      \
+                                        --group-id=<group-id>                    \
+                                        --status=<enabled | allowed | forbidden> \
+
+
+Show Sync Policy Group
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+To show a sync policy group:
+
+::
+
+      # radosgw-admin sync group get [--bucket=<bucket>]       \
+                                        --group-id=<group-id>
+                                       
+
+Remove Sync Policy Group
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+To remove a sync policy group:
+
+::
+
+      # radosgw-admin sync group remove [--bucket=<bucket>]    \
+                                        --group-id=<group-id>
+
+
+
+Create Sync Flow
+~~~~~~~~~~~~~~~~
+
+- To create or update directional sync flow:
+
+::
+
+      # radosgw-admin sync group flow create [--bucket=<bucket>]          \
+                                             --group-id=<group-id>        \
+                                             --flow-id=<flow-id>          \
+                                             --flow-type=directional      \
+                                             --source-zone=<source_zone>  \
+                                             --dest-zone=<dest_zone>
+
+
+- To create or update symmetrical sync flow:
+
+::
+
+      # radosgw-admin sync group flow create [--bucket=<bucket>]          \
+                                             --group-id=<group-id>        \
+                                             --flow-id=<flow-id>          \
+                                             --flow-type=symmetrical      \
+                                             --zones=<zones>
+
+
+Where zones are a comma separated list of all the zones that need to add to the flow.
+
+
+Remove Sync Flow Zones
+~~~~~~~~~~~~~~~~~~~~~~
+
+- To remove directional sync flow:
+
+::
+
+      # radosgw-admin sync group flow remove [--bucket=<bucket>]          \
+                                             --group-id=<group-id>        \
+                                             --flow-id=<flow-id>          \
+                                             --flow-type=directional      \
+                                             --source-zone=<source_zone>  \
+                                             --dest-zone=<dest_zone>
+
+
+- To remove specific zones from symmetrical sync flow:
+
+::
+
+      # radosgw-admin sync group flow remove [--bucket=<bucket>]          \
+                                             --group-id=<group-id>        \
+                                             --flow-id=<flow-id>          \
+                                             --flow-type=symmetrical      \
+                                             --zones=<zones>
+
+
+Where zones are a comma separated list of all zones to remove from the flow.
+
+                                             
+- To remove symmetrical sync flow:
+
+::
+
+      # radosgw-admin sync group flow remove [--bucket=<bucket>]          \
+                                             --group-id=<group-id>        \
+                                             --flow-id=<flow-id>          \
+                                             --flow-type=symmetrical
+
+
+Create Sync Pipe
+~~~~~~~~~~~~~~~~
+
+To create sync group pipe, or update its parameters:
+
+
+::
+
+      # radosgw-admin sync group pipe create [--bucket=<bucket>]                      \
+                                             --group-id=<group-id>                    \
+                                             --pipe-id=<pipe-id>                      \
+                                             --source-zones=<source_zones>            \
+                                             [--source-bucket=<source_buckets>]       \
+                                             [--source-bucket-id=<source_bucket_id>]  \
+                                             --dest-zones=<dest_zones>                \
+                                             [--dest-bucket=<dest_buckets>]           \
+                                             [--dest-bucket-id=<dest_bucket_id>]      \
+                                             [--prefix=<source_prefix>]               \
+                                             [--prefix-rm]                            \
+                                             [--tags-add=<tags>]                      \
+                                             [--tags-rm=<tags>]                       \
+                                             [--dest-owner=<owner>]                   \
+                                             [--storage-class=<storage_class>]        \
+                                             [--mode=<system | user>]                 \
+                                             [--uid=<user_id>]
+
+
+Zones are either a list of zones, or '*' (wildcard). Wildcard zones mean any zone that matches the sync flow rules.
+Buckets are either a bucket name, or '*' (wildcard). Wildcard bucket means the current bucket
+Prefix can be defined to filter source objects.
+Tags are passed by a comma separated list of 'key=value'.
+Destination owner can be set to force a destination owner of the objects. If user mode is selected, only the destination bucket owner can be set.
+Destinatino storage class can also be condfigured.
+User id can be set for user mode, and will be the user under which the sync operation will be executed (for permissions validation).
+
+
+Remove Sync Pipe
+~~~~~~~~~~~~~~~~
+
+To remove specific sync group pipe params, or the entire pipe:
+
+
+::
+
+      # radosgw-admin sync group pipe remove [--bucket=<bucket>]                     \
+                                             --group-id=<group-id>                   \
+                                             --pipe-id=<pipe-id>                     \
+                                             [--source-zones=<source_zones>]         \
+                                             [--source-bucket=<source_buckets>]      \
+                                             [--source-bucket-id=<source_bucket_id>] \
+                                             [--dest-zones=<dest_zones>]             \
+                                             [--dest-bucket=<dest_buckets>]          \
+                                             [--dest-bucket-id=<dest_bucket_id>]
+
+
+Sync Info
+~~~~~~~~~
+
+To get information about the expected sync sources and targets (as defined by the sync policy):
+
+::
+
+      # radosgw-admin sync info [--bucket=<bucket>]             \
+                                [--effective-zone-name=<zone>]
+
+
+Since a bucket can define a policy that defines data movement from it towards a different bucket at a different zone, when the policy is created we also generate a list of bucket dependencies that are used as hints when a sync of any particular bucket happens. The fact that a bucket references another bucket does not mean it actually syncs to/from it, as the data flow might not permit it.  
+
+
+Examples
+========
+
+The system in these examples includes 3 zones: ``us-east`` (the master zone), ``us-west``, ``us-west-2``.
+
+Example 1: Two Zones, Complete Mirror
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This is similar to older (pre ``Octopus``) sync capabilities, but being done via the new sync policy engine. Note that changes to the zonegroup sync policy require a period update and commit.
+
+
+::
+
+      [us-east] $ radosgw-admin sync group create --group-id=group1 --status=allowed
+      [us-east] $ radosgw-admin sync group flow create --group-id=group1 \
+                                --flow-id=flow-mirror --flow-type=symmetrical \
+                                --zones=us-east,us-west
+      [us-east] $ radosgw-admin sync group pipe create --group-id=group1 \
+                                --pipe-id=pipe1 --source-zones='*' \
+                                --source-bucket='*' --dest-zones='*' \
+                                --dest-bucket='*'
+      [us-east] $ radosgw-admin sync group modify --group-id=group1 --status=enabled
+      [us-east] $ radosgw-admin period update --commit
+
+      $ radosgw-admin sync info --bucket=buck
+      {
+          "sources": [
+              {
+                  "id": "pipe1",
+                  "source": {
+                      "zone": "us-west",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                  "dest": {
+                      "zone": "us-east",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                  "params": {
+      ...
+                  }
+              }
+          ],
+          "dests": [
+              {
+                  "id": "pipe1",
+                  "source": {
+                      "zone": "us-east",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                  "dest": {
+                      "zone": "us-west",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                 ...
+              }
+          ],
+          ...
+          }
+      }
+
+
+Note that the "id" field in the output above reflects the pipe rule
+that generated that entry, a single rule can generate multiple sync
+entries as can be seen in the example.
+
+::
+
+      [us-west] $ radosgw-admin sync info --bucket=buck
+      {
+          "sources": [
+              {
+                  "id": "pipe1",
+                  "source": {
+                      "zone": "us-east",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                  "dest": {
+                      "zone": "us-west",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                  ...
+              }
+          ],
+          "dests": [
+              {
+                  "id": "pipe1",
+                  "source": {
+                      "zone": "us-west",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                  "dest": {
+                      "zone": "us-east",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                 ...
+              }
+          ],
+          ...
+      }
+
+
+
+Example 2: Directional, Entire Zone Backup
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Also similar to older sync capabilities. In here we add a third zone, ``us-west-2`` that will be a replica of ``us-west``, but data will not be replicated back from it.
+
+::
+
+      [us-east] $ radosgw-admin sync group flow create --group-id=group1 \
+                                --flow-id=us-west-backup --flow-type=directional \
+                                --source-zone=us-west --dest-zone=us-west-2
+      [us-east] $ radosgw-admin period update --commit
+
+
+Note that us-west has two dests:
+
+::
+
+      [us-west] $ radosgw-admin sync info --bucket=buck
+      {
+          "sources": [
+              {
+                  "id": "pipe1",
+                  "source": {
+                      "zone": "us-east",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                  "dest": {
+                      "zone": "us-west",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                 ...
+              }
+          ],
+          "dests": [
+              {
+                  "id": "pipe1",
+                  "source": {
+                      "zone": "us-west",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                  "dest": {
+                      "zone": "us-east",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                 ...
+              },
+              {
+                  "id": "pipe1",
+                  "source": {
+                      "zone": "us-west",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                  "dest": {
+                      "zone": "us-west-2",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                 ...
+              }
+          ],
+          ...
+      }
+
+
+Whereas us-west-2 has only source and no destinations:
+
+::
+
+      [us-west-2] $ radosgw-admin sync info --bucket=buck
+      {
+          "sources": [
+              {
+                  "id": "pipe1",
+                  "source": {
+                      "zone": "us-west",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                  "dest": {
+                      "zone": "us-west-2",
+                      "bucket": "buck:115b12b3-....4409.1"
+                  },
+                 ...
+              }
+          ],
+          "dests": [],
+          ...
+      }
+
+      
+      
+Example 3: Mirror a Specific Bucket
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Using the same group configuration, but this time switching it to ``allowed`` state, which means that sync is allowed but not enabled.
+
+::
+
+      [us-east] $ radosgw-admin sync group modify --group-id=group1 --status=allowed
+      [us-east] $ radosgw-admin period update --commit
+
+
+And we will create a bucket level policy rule for existing bucket ``buck2``. Note that the bucket needs to exist before being able to set this policy, and admin commands that modify bucket policies need to run on the master zone, however, they do not require period update.  There is no need to change the data flow, as it is inherited from the zonegroup policy. A bucket policy flow will only be a subset of the flow defined in the zonegroup policy. Same goes for pipes, although a bucket policy can enable pipes that are not enabled (albeit not forbidden) at the zonegroup policy.
+
+::
+
+      [us-east] $ radosgw-admin sync group create --bucket=buck2 \
+                                --group-id=buck2-default --status=enabled
+
+      [us-east] $ radosgw-admin sync group pipe create --bucket=buck2 \
+                                --group-id=buck2-default --pipe-id=pipe1 \
+                                --source-zones='*' --dest-zones='*'
+
+
+
+Example 4: Limit Bucket Sync To Specific Zones
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This will only sync ``buck3`` to ``us-east`` (from any zone that flow allows to sync into ``us-east``).
+
+::
+
+      [us-east] $ radosgw-admin sync group create --bucket=buck3 \
+                                --group-id=buck3-default --status=enabled
+
+      [us-east] $ radosgw-admin sync group pipe create --bucket=buck3 \
+                                --group-id=buck3-default --pipe-id=pipe1 \
+                                --source-zones='*' --dest-zones=us-east
+
+
+
+Example 5: Sync From a Different Bucket
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Note that bucket sync only works (currently) across zones and not within the same zone.
+
+Set ``buck4`` to pull data from ``buck5``:
+
+::
+
+      [us-east] $ radosgw-admin sync group create --bucket=buck4 '
+                                --group-id=buck4-default --status=enabled
+
+      [us-east] $ radosgw-admin sync group pipe create --bucket=buck4 \
+                                --group-id=buck4-default --pipe-id=pipe1 \
+                                --source-zones='*' --source-bucket=buck5 \
+                                --dest-zones='*'
+
+
+can also limit it to specific zones, for example the following will
+only sync data originated in us-west:
+
+::
+
+      [us-east] $ radosgw-admin sync group pipe modify --bucket=buck4 \
+                                --group-id=buck4-default --pipe-id=pipe1 \
+                                --source-zones=us-west --source-bucket=buck5 \
+                                --dest-zones='*'
+
+
+Checking the sync info for ``buck5`` on ``us-west`` is interesting:
+
+::
+
+      [us-west] $ radosgw-admin sync info --bucket=buck5
+      {
+          "sources": [],
+          "dests": [],
+          "hints": {
+              "sources": [],
+              "dests": [
+                  "buck4:115b12b3-....14433.2"
+              ]
+          },
+          "resolved-hints-1": {
+              "sources": [],
+              "dests": [
+                  {
+                      "id": "pipe1",
+                      "source": {
+                          "zone": "us-west",
+                          "bucket": "buck5"
+                      },
+                      "dest": {
+                          "zone": "us-east",
+                          "bucket": "buck4:115b12b3-....14433.2"
+                      },
+                      ...
+                  },
+                  {
+                      "id": "pipe1",
+                      "source": {
+                          "zone": "us-west",
+                          "bucket": "buck5"
+                      },
+                      "dest": {
+                          "zone": "us-west-2",
+                          "bucket": "buck4:115b12b3-....14433.2"
+                      },
+                      ...
+                  }
+              ]
+          },
+          "resolved-hints": {
+              "sources": [],
+              "dests": []
+          }
+      }
+
+
+Note that there are resolved hints, which means that the bucket ``buck5`` found about ``buck4`` syncing from it indirectly, and not from its own policy (the policy for ``buck5`` itself is empty).
+
+
+Example 6: Sync To Different Bucket
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The same mechanism can work for configuring data to be synced to (vs.  synced from as in the previous example). Note that internally data is still pulled from the source at the destination zone:
+
+Set ``buck6`` to "push" data to ``buck5``:
+
+::
+
+      [us-east] $ radosgw-admin sync group create --bucket=buck6 \
+                                --group-id=buck6-default --status=enabled
+
+      [us-east] $ radosgw-admin sync group pipe create --bucket=buck6 \
+                                --group-id=buck6-default --pipe-id=pipe1 \
+                                --source-zones='*' --source-bucket='*' \
+                                --dest-zones='*' --dest-bucket=buck5
+
+
+A wildcard bucket name means the current bucket in the context of bucket sync policy.
+
+Combined with the configuration in Example 5, we can now write data to ``buck6`` on ``us-east``, data will sync to ``buck5`` on ``us-west``, and from there it will be distributed to ``buck4`` on ``us-east``, and on ``us-west-2``.
+
+Example 7: Source Filters
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Sync from ``buck8`` to ``buck9``, but only objects that start with ``foo/``:
+
+::
+
+      [us-east] $ radosgw-admin sync group create --bucket=buck8 \
+                                --group-id=buck8-default --status=enabled
+
+      [us-east] $ radosgw-admin sync group pipe create --bucket=buck8 \
+                                --group-id=buck8-default --pipe-id=pipe-prefix \
+                                --prefix=foo/ --source-zones='*' --dest-zones='*' \
+                                --dest-bucket=buck9
+
+
+Also sync from ``buck8`` to ``buck9`` any object that has the tags ``color=blue`` or ``color=red``:
+
+::
+
+      [us-east] $ radosgw-admin sync group pipe create --bucket=buck8 \
+                                --group-id=buck8-default --pipe-id=pipe-tags \
+                                --tags-add=color=blue,color=red --source-zones='*' \
+                                --dest-zones='*' --dest-bucket=buck9
+
+
+And we can check the expected sync in ``us-east`` (for example):
+
+::
+
+      [us-east] $ radosgw-admin sync info --bucket=buck8
+      {
+          "sources": [],
+          "dests": [
+              {
+                  "id": "pipe-prefix",
+                  "source": {
+                      "zone": "us-east",
+                      "bucket": "buck8:115b12b3-....14433.5"
+                  },
+                  "dest": {
+                      "zone": "us-west",
+                      "bucket": "buck9"
+                  },
+                  "params": {
+                      "source": {
+                          "filter": {
+                              "prefix": "foo/",
+                              "tags": []
+                          }
+                      },
+                      ...
+                  }
+              },
+              {
+                  "id": "pipe-tags",
+                  "source": {
+                      "zone": "us-east",
+                      "bucket": "buck8:115b12b3-....14433.5"
+                  },
+                  "dest": {
+                      "zone": "us-west",
+                      "bucket": "buck9"
+                  },
+                  "params": {
+                      "source": {
+                          "filter": {
+                              "tags": [
+                                  {
+                                      "key": "color",
+                                      "value": "blue"
+                                  },
+                                  {
+                                      "key": "color",
+                                      "value": "red"
+                                  }
+                              ]
+                          }
+                      },
+                      ...
+                  }
+              }
+          ],
+          ...
+      }
+
+
+Note that there aren't any sources, only two different destinations (one for each configuration). When the sync process happens it will select the relevant rule for each object it syncs.
+
+Prefixes and tags can be combined, in which object will need to have both in order to be synced. The priority param can also be passed, and it can be used to determine when there are multiple different rules that are matched (and have the same source and destination), to determine which of the rules to be used.
+
+
+Example 8: Destination Params: Storage Class
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Storage class of the destination objects can be configured:
+
+::
+
+      [us-east] $ radosgw-admin sync group create --bucket=buck10 \
+                                --group-id=buck10-default --status=enabled
+
+      [us-east] $ radosgw-admin sync group pipe create --bucket=buck10 \
+                                --group-id=buck10-default \
+                                --pipe-id=pipe-storage-class \
+                                --source-zones='*' --dest-zones=us-west-2 \
+                                --storage-class=CHEAP_AND_SLOW
+
+
+Example 9: Destination Params: Destination Owner Translation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Set the destination objects owner as the destination bucket owner.
+This requires specifying the uid of the destination bucket:
+
+::
+
+      [us-east] $ radosgw-admin sync group create --bucket=buck11 \
+                                --group-id=buck11-default --status=enabled
+
+      [us-east] $ radosgw-admin sync group pipe create --bucket=buck11 \
+                                --group-id=buck11-default --pipe-id=pipe-dest-owner \
+                                --source-zones='*' --dest-zones='*' \
+                                --dest-bucket=buck12 --dest-owner=joe
+
+Example 10: Destination Params: User Mode
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+User mode makes sure that the user has permissions to both read the objects, and write to the destination bucket. This requires that the uid of the user (which in its context the operation executes) is specified.
+
+::
+
+      [us-east] $ radosgw-admin sync group pipe modify --bucket=buck11 \
+                                --group-id=buck11-default --pipe-id=pipe-dest-owner \
+                                --mode=user --uid=jenny
+
+
+
diff --git a/doc/radosgw/multisite.rst b/doc/radosgw/multisite.rst
new file mode 100644
index 000000000..a4a89d268
--- /dev/null
+++ b/doc/radosgw/multisite.rst
@@ -0,0 +1,1537 @@
+.. _multisite:
+
+==========
+Multi-Site
+==========
+
+.. versionadded:: Jewel
+
+A single zone configuration typically consists of one zone group containing one
+zone and one or more `ceph-radosgw` instances where you may load-balance gateway
+client requests between the instances. In a single zone configuration, typically
+multiple gateway instances point to a single Ceph storage cluster. However, Kraken
+supports several multi-site configuration options for the Ceph Object Gateway:
+
+- **Multi-zone:** A more advanced configuration consists of one zone group and
+  multiple zones, each zone with one or more `ceph-radosgw` instances. Each zone
+  is backed by its own Ceph Storage Cluster. Multiple zones in a zone group
+  provides disaster recovery for the zone group should one of the zones experience
+  a significant failure. In Kraken, each zone is active and may receive write
+  operations. In addition to disaster recovery, multiple active zones may also
+  serve as a foundation for content delivery networks.
+
+- **Multi-zone-group:** Formerly called 'regions', Ceph Object Gateway can also
+  support multiple zone groups, each zone group with one or more zones. Objects
+  stored to zones in one zone group within the same realm as another zone
+  group will share a global object namespace, ensuring unique object IDs across
+  zone groups and zones.
+
+- **Multiple Realms:** In Kraken, the Ceph Object Gateway supports the notion
+  of realms, which can be a single zone group or multiple zone groups and
+  a globally unique namespace for the realm. Multiple realms provide the ability
+  to support numerous configurations and namespaces.
+
+Replicating object data between zones within a zone group looks something
+like this:
+
+.. image:: ../images/zone-sync2.png
+   :align: center
+
+For additional details on setting up a cluster, see `Ceph Object Gateway for
+Production <https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/3/html/ceph_object_gateway_for_production/index/>`__.
+
+Functional Changes from Infernalis
+==================================
+
+In Kraken, you can configure each Ceph Object Gateway to
+work in an active-active zone configuration, allowing for writes to
+non-master zones.
+
+The multi-site configuration is stored within a container called a
+"realm." The realm stores zone groups, zones, and a time "period" with
+multiple epochs for tracking changes to the configuration. In Kraken,
+the ``ceph-radosgw`` daemons handle the synchronization,
+eliminating the need for a separate synchronization agent. Additionally,
+the new approach to synchronization allows the Ceph Object Gateway to
+operate with an "active-active" configuration instead of
+"active-passive".
+
+Requirements and Assumptions
+============================
+
+A multi-site configuration requires at least two Ceph storage clusters,
+preferably given a distinct cluster name. At least two Ceph object
+gateway instances, one for each Ceph storage cluster.
+
+This guide assumes at least two Ceph storage clusters are in geographically
+separate locations; however, the configuration can work on the same
+site. This guide also assumes two Ceph object gateway servers named
+``rgw1`` and ``rgw2``.
+
+.. important:: Running a single Ceph storage cluster is NOT recommended unless you have 
+               low latency WAN connections.
+
+A multi-site configuration requires a master zone group and a master
+zone. Additionally, each zone group requires a master zone. Zone groups
+may have one or more secondary or non-master zones.
+
+In this guide, the ``rgw1`` host will serve as the master zone of the
+master zone group; and, the ``rgw2`` host will serve as the secondary zone
+of the master zone group.
+
+See `Pools`_ for instructions on creating and tuning pools for Ceph
+Object Storage.
+
+See `Sync Policy Config`_ for instructions on defining fine grained bucket sync
+policy rules.
+
+.. _master-zone-label:
+
+Configuring a Master Zone
+=========================
+
+All gateways in a multi-site configuration will retrieve their
+configuration from a ``ceph-radosgw`` daemon on a host within the master
+zone group and master zone. To configure your gateways in a multi-site
+configuration, choose a ``ceph-radosgw`` instance to configure the
+master zone group and master zone.
+
+Create a Realm
+--------------
+
+A realm contains the multi-site configuration of zone groups and zones
+and also serves to enforce a globally unique namespace within the realm.
+
+Create a new realm for the multi-site configuration by opening a command
+line interface on a host identified to serve in the master zone group
+and zone. Then, execute the following:
+
+.. prompt:: bash #
+
+   radosgw-admin realm create --rgw-realm={realm-name} [--default]
+
+For example:
+
+.. prompt:: bash #
+
+   radosgw-admin realm create --rgw-realm=movies --default
+
+If the cluster will have a single realm, specify the ``--default`` flag.
+If ``--default`` is specified, ``radosgw-admin`` will use this realm by
+default. If ``--default`` is not specified, adding zone-groups and zones
+requires specifying either the ``--rgw-realm`` flag or the
+``--realm-id`` flag to identify the realm when adding zone groups and
+zones.
+
+After creating the realm, ``radosgw-admin`` will echo back the realm
+configuration. For example:
+
+::
+
+    {
+        "id": "0956b174-fe14-4f97-8b50-bb7ec5e1cf62",
+        "name": "movies",
+        "current_period": "1950b710-3e63-4c41-a19e-46a715000980",
+        "epoch": 1
+    }
+
+.. note:: Ceph generates a unique ID for the realm, which allows the renaming
+          of a realm if the need arises.
+
+Create a Master Zone Group
+--------------------------
+
+A realm must have at least one zone group, which will serve as the
+master zone group for the realm.
+
+Create a new master zone group for the multi-site configuration by
+opening a command line interface on a host identified to serve in the
+master zone group and zone. Then, execute the following:
+
+.. prompt:: bash #
+
+   radosgw-admin zonegroup create --rgw-zonegroup={name} --endpoints={url} [--rgw-realm={realm-name}|--realm-id={realm-id}] --master --default
+
+For example:
+
+.. prompt:: bash #
+
+   radosgw-admin zonegroup create --rgw-zonegroup=us --endpoints=http://rgw1:80 --rgw-realm=movies --master --default
+
+If the realm will only have a single zone group, specify the
+``--default`` flag. If ``--default`` is specified, ``radosgw-admin``
+will use this zone group by default when adding new zones. If
+``--default`` is not specified, adding zones will require either the
+``--rgw-zonegroup`` flag or the ``--zonegroup-id`` flag to identify the
+zone group when adding or modifying zones.
+
+After creating the master zone group, ``radosgw-admin`` will echo back
+the zone group configuration. For example:
+
+::
+
+    {
+        "id": "f1a233f5-c354-4107-b36c-df66126475a6",
+        "name": "us",
+        "api_name": "us",
+        "is_master": "true",
+        "endpoints": [
+            "http:\/\/rgw1:80"
+        ],
+        "hostnames": [],
+        "hostnames_s3webzone": [],
+        "master_zone": "",
+        "zones": [],
+        "placement_targets": [],
+        "default_placement": "",
+        "realm_id": "0956b174-fe14-4f97-8b50-bb7ec5e1cf62"
+    }
+
+Create a Master Zone
+--------------------
+
+.. important:: Zones must be created on a Ceph Object Gateway node that will be
+               within the zone.
+
+Create a new master zone for the multi-site configuration by opening a
+command line interface on a host identified to serve in the master zone
+group and zone. Then, execute the following:
+
+.. prompt:: bash #
+
+   radosgw-admin zone create --rgw-zonegroup={zone-group-name} \
+                               --rgw-zone={zone-name} \
+                               --master --default \
+                               --endpoints={http://fqdn}[,{http://fqdn}]
+
+For example:
+
+.. prompt:: bash #
+
+   radosgw-admin zone create --rgw-zonegroup=us --rgw-zone=us-east \
+                               --master --default \
+                               --endpoints={http://fqdn}[,{http://fqdn}]
+
+
+.. note:: The ``--access-key`` and ``--secret`` aren’t specified. These
+          settings will be added to the zone once the user is created in the
+          next section.
+
+.. important:: The following steps assume a multi-site configuration using newly
+               installed systems that aren’t storing data yet. DO NOT DELETE the
+               ``default`` zone and its pools if you are already using it to store
+               data, or the data will be deleted and unrecoverable.
+
+Delete Default Zone Group and Zone
+----------------------------------
+
+Delete the ``default`` zone if it exists. Make sure to remove it from
+the default zone group first.
+
+.. prompt:: bash #
+
+    radosgw-admin zonegroup delete --rgw-zonegroup=default --rgw-zone=default
+    radosgw-admin period update --commit
+    radosgw-admin zone delete --rgw-zone=default
+    radosgw-admin period update --commit
+    radosgw-admin zonegroup delete --rgw-zonegroup=default
+    radosgw-admin period update --commit
+
+Finally, delete the ``default`` pools in your Ceph storage cluster if
+they exist.
+
+.. important:: The following step assumes a multi-site configuration using newly
+               installed systems that aren’t currently storing data. DO NOT DELETE
+               the ``default`` zone group if you are already using it to store
+               data.
+
+.. prompt:: bash #
+
+   ceph osd pool rm default.rgw.control default.rgw.control --yes-i-really-really-mean-it
+   ceph osd pool rm default.rgw.data.root default.rgw.data.root --yes-i-really-really-mean-it
+   ceph osd pool rm default.rgw.gc default.rgw.gc --yes-i-really-really-mean-it
+   ceph osd pool rm default.rgw.log default.rgw.log --yes-i-really-really-mean-it
+   ceph osd pool rm default.rgw.users.uid default.rgw.users.uid --yes-i-really-really-mean-it
+
+Create a System User
+--------------------
+
+The ``ceph-radosgw`` daemons must authenticate before pulling realm and
+period information. In the master zone, create a system user to
+facilitate authentication between daemons.
+
+
+.. prompt:: bash #
+
+   radosgw-admin user create --uid="{user-name}" --display-name="{Display Name}" --system
+
+For example:
+
+.. prompt:: bash #
+
+   radosgw-admin user create --uid="synchronization-user" --display-name="Synchronization User" --system
+
+Make a note of the ``access_key`` and ``secret_key``, as the secondary
+zones will require them to authenticate with the master zone.
+
+Finally, add the system user to the master zone.
+
+.. prompt:: bash #
+
+   radosgw-admin zone modify --rgw-zone={zone-name} --access-key={access-key} --secret={secret}
+   radosgw-admin period update --commit
+
+Update the Period
+-----------------
+
+After updating the master zone configuration, update the period.
+
+.. prompt:: bash #
+
+   radosgw-admin period update --commit
+
+.. note:: Updating the period changes the epoch, and ensures that other zones
+          will receive the updated configuration.
+
+Update the Ceph Configuration File
+----------------------------------
+
+Update the Ceph configuration file on master zone hosts by adding the
+``rgw_zone`` configuration option and the name of the master zone to the
+instance entry.
+
+::
+
+    [client.rgw.{instance-name}]
+    ...
+    rgw_zone={zone-name}
+
+For example:
+
+::
+
+    [client.rgw.rgw1]
+    host = rgw1
+    rgw frontends = "civetweb port=80"
+    rgw_zone=us-east
+
+Start the Gateway
+-----------------
+
+On the object gateway host, start and enable the Ceph Object Gateway
+service:
+
+.. prompt:: bash #
+
+   systemctl start ceph-radosgw@rgw.`hostname -s`
+   systemctl enable ceph-radosgw@rgw.`hostname -s`
+
+.. _secondary-zone-label:
+
+Configure Secondary Zones
+=========================
+
+Zones within a zone group replicate all data to ensure that each zone
+has the same data. When creating the secondary zone, execute all of the
+following operations on a host identified to serve the secondary zone.
+
+.. note:: To add a third zone, follow the same procedures as for adding the
+          secondary zone. Use different zone name.
+
+.. important:: You must execute metadata operations, such as user creation, on a
+               host within the master zone. The master zone and the secondary zone
+               can receive bucket operations, but the secondary zone redirects
+               bucket operations to the master zone. If the master zone is down,
+               bucket operations will fail.
+
+Pull the Realm
+--------------
+
+Using the URL path, access key and secret of the master zone in the
+master zone group, pull the realm configuration to the host. To pull a
+non-default realm, specify the realm using the ``--rgw-realm`` or
+``--realm-id`` configuration options.
+
+.. prompt:: bash #
+
+   radosgw-admin realm pull --url={url-to-master-zone-gateway} --access-key={access-key} --secret={secret}
+
+.. note:: Pulling the realm also retrieves the remote's current period
+          configuration, and makes it the current period on this host as well.
+
+If this realm is the default realm or the only realm, make the realm the
+default realm.
+
+.. prompt:: bash #
+
+   radosgw-admin realm default --rgw-realm={realm-name}
+
+Create a Secondary Zone
+-----------------------
+
+.. important:: Zones must be created on a Ceph Object Gateway node that will be
+               within the zone.
+
+Create a secondary zone for the multi-site configuration by opening a
+command line interface on a host identified to serve the secondary zone.
+Specify the zone group ID, the new zone name and an endpoint for the
+zone. **DO NOT** use the ``--master`` or ``--default`` flags. In Kraken,
+all zones run in an active-active configuration by
+default; that is, a gateway client may write data to any zone and the
+zone will replicate the data to all other zones within the zone group.
+If the secondary zone should not accept write operations, specify the
+``--read-only`` flag to create an active-passive configuration between
+the master zone and the secondary zone. Additionally, provide the
+``access_key`` and ``secret_key`` of the generated system user stored in
+the master zone of the master zone group. Execute the following:
+
+.. prompt:: bash #
+
+   radosgw-admin zone create --rgw-zonegroup={zone-group-name} \
+                                --rgw-zone={zone-name} \
+                                --access-key={system-key} --secret={secret} \
+                                --endpoints=http://{fqdn}:80 \
+                                [--read-only]
+
+For example:
+
+.. prompt:: bash #
+
+   radosgw-admin zone create --rgw-zonegroup=us --rgw-zone=us-west \
+                                --access-key={system-key} --secret={secret} \
+                                --endpoints=http://rgw2:80
+
+.. important:: The following steps assume a multi-site configuration using newly
+               installed systems that aren’t storing data. **DO NOT DELETE** the
+               ``default`` zone and its pools if you are already using it to store
+               data, or the data will be lost and unrecoverable.
+
+Delete the default zone if needed.
+
+.. prompt:: bash #
+
+   radosgw-admin zone delete --rgw-zone=default
+
+Finally, delete the default pools in your Ceph storage cluster if
+needed.
+
+.. prompt:: bash #
+
+   ceph osd pool rm default.rgw.control default.rgw.control --yes-i-really-really-mean-it
+   ceph osd pool rm default.rgw.data.root default.rgw.data.root --yes-i-really-really-mean-it
+   ceph osd pool rm default.rgw.gc default.rgw.gc --yes-i-really-really-mean-it
+   ceph osd pool rm default.rgw.log default.rgw.log --yes-i-really-really-mean-it
+   ceph osd pool rm default.rgw.users.uid default.rgw.users.uid --yes-i-really-really-mean-it
+
+Update the Ceph Configuration File
+----------------------------------
+
+Update the Ceph configuration file on the secondary zone hosts by adding
+the ``rgw_zone`` configuration option and the name of the secondary zone
+to the instance entry.
+
+::
+
+    [client.rgw.{instance-name}]
+    ...
+    rgw_zone={zone-name}
+
+For example:
+
+::
+
+    [client.rgw.rgw2]
+    host = rgw2
+    rgw frontends = "civetweb port=80"
+    rgw_zone=us-west
+
+Update the Period
+-----------------
+
+After updating the master zone configuration, update the period.
+
+.. prompt:: bash #
+
+   radosgw-admin period update --commit
+
+.. note:: Updating the period changes the epoch, and ensures that other zones
+          will receive the updated configuration.
+
+Start the Gateway
+-----------------
+
+On the object gateway host, start and enable the Ceph Object Gateway
+service:
+
+.. prompt:: bash #
+
+   systemctl start ceph-radosgw@rgw.`hostname -s`
+   systemctl enable ceph-radosgw@rgw.`hostname -s`
+
+Check Synchronization Status
+----------------------------
+
+Once the secondary zone is up and running, check the synchronization
+status. Synchronization copies users and buckets created in the master
+zone to the secondary zone.
+
+.. prompt:: bash #
+
+   radosgw-admin sync status
+
+The output will provide the status of synchronization operations. For
+example:
+
+::
+
+    realm f3239bc5-e1a8-4206-a81d-e1576480804d (earth)
+        zonegroup c50dbb7e-d9ce-47cc-a8bb-97d9b399d388 (us)
+             zone 4c453b70-4a16-4ce8-8185-1893b05d346e (us-west)
+    metadata sync syncing
+                  full sync: 0/64 shards
+                  metadata is caught up with master
+                  incremental sync: 64/64 shards
+        data sync source: 1ee9da3e-114d-4ae3-a8a4-056e8a17f532 (us-east)
+                          syncing
+                          full sync: 0/128 shards
+                          incremental sync: 128/128 shards
+                          data is caught up with source
+
+.. note:: Secondary zones accept bucket operations; however, secondary zones
+          redirect bucket operations to the master zone and then synchronize
+          with the master zone to receive the result of the bucket operations.
+          If the master zone is down, bucket operations executed on the
+          secondary zone will fail, but object operations should succeed.
+
+
+Maintenance
+===========
+
+Checking the Sync Status
+------------------------
+
+Information about the replication status of a zone can be queried with:
+
+.. prompt:: bash $
+
+   radosgw-admin sync status
+
+::
+
+            realm b3bc1c37-9c44-4b89-a03b-04c269bea5da (earth)
+        zonegroup f54f9b22-b4b6-4a0e-9211-fa6ac1693f49 (us)
+             zone adce11c9-b8ed-4a90-8bc5-3fc029ff0816 (us-2)
+            metadata sync syncing
+                  full sync: 0/64 shards
+                  incremental sync: 64/64 shards
+                  metadata is behind on 1 shards
+                  oldest incremental change not applied: 2017-03-22 10:20:00.0.881361s
+        data sync source: 341c2d81-4574-4d08-ab0f-5a2a7b168028 (us-1)
+                          syncing
+                          full sync: 0/128 shards
+                          incremental sync: 128/128 shards
+                          data is caught up with source
+                  source: 3b5d1a3f-3f27-4e4a-8f34-6072d4bb1275 (us-3)
+                          syncing
+                          full sync: 0/128 shards
+                          incremental sync: 128/128 shards
+                          data is caught up with source
+
+Changing the Metadata Master Zone
+---------------------------------
+
+.. important:: Care must be taken when changing which zone is the metadata
+   master. If a zone has not finished syncing metadata from the current
+   master zone, it will be unable to serve any remaining entries when
+   promoted to master and those changes will be lost. For this reason,
+   waiting for a zone's ``radosgw-admin sync status`` to catch up on
+   metadata sync before promoting it to master is recommended.
+
+Similarly, if changes to metadata are being processed by the current master
+zone while another zone is being promoted to master, those changes are
+likely to be lost. To avoid this, shutting down any ``radosgw`` instances
+on the previous master zone is recommended. After promoting another zone,
+its new period can be fetched with ``radosgw-admin period pull`` and the
+gateway(s) can be restarted.
+
+To promote a zone (for example, zone ``us-2`` in zonegroup ``us``) to metadata
+master, run the following commands on that zone:
+
+.. prompt:: bash $
+
+   radosgw-admin zone modify --rgw-zone=us-2 --master
+   radosgw-admin zonegroup modify --rgw-zonegroup=us --master
+   radosgw-admin period update --commit
+
+This will generate a new period, and the radosgw instance(s) in zone ``us-2``
+will send this period to other zones.
+
+Failover and Disaster Recovery
+==============================
+
+If the master zone should fail, failover to the secondary zone for
+disaster recovery.
+
+1. Make the secondary zone the master and default zone. For example:
+
+   .. prompt:: bash #
+
+      radosgw-admin zone modify --rgw-zone={zone-name} --master --default
+
+   By default, Ceph Object Gateway will run in an active-active
+   configuration. If the cluster was configured to run in an
+   active-passive configuration, the secondary zone is a read-only zone.
+   Remove the ``--read-only`` status to allow the zone to receive write
+   operations. For example:
+
+   .. prompt:: bash #
+
+      radosgw-admin zone modify --rgw-zone={zone-name} --master --default \
+                                   --read-only=false
+
+2. Update the period to make the changes take effect.
+
+   .. prompt:: bash #
+
+      radosgw-admin period update --commit
+
+3. Finally, restart the Ceph Object Gateway.
+
+   .. prompt:: bash #
+
+      systemctl restart ceph-radosgw@rgw.`hostname -s`
+
+If the former master zone recovers, revert the operation.
+
+1. From the recovered zone, pull the latest realm configuration
+   from the current master zone:
+
+   .. prompt:: bash #
+
+      radosgw-admin realm pull --url={url-to-master-zone-gateway} \
+                                  --access-key={access-key} --secret={secret}
+
+2. Make the recovered zone the master and default zone.
+
+   .. prompt:: bash #
+
+      radosgw-admin zone modify --rgw-zone={zone-name} --master --default
+
+3. Update the period to make the changes take effect.
+
+   .. prompt:: bash #
+
+      radosgw-admin period update --commit
+
+4. Then, restart the Ceph Object Gateway in the recovered zone.
+
+   .. prompt:: bash #
+
+       systemctl restart ceph-radosgw@rgw.`hostname -s`
+
+5. If the secondary zone needs to be a read-only configuration, update
+   the secondary zone.
+
+   .. prompt:: bash #
+
+      radosgw-admin zone modify --rgw-zone={zone-name} --read-only
+
+6. Update the period to make the changes take effect.
+
+   .. prompt:: bash #
+
+      radosgw-admin period update --commit
+
+7. Finally, restart the Ceph Object Gateway in the secondary zone.
+
+   .. prompt:: bash #
+
+      systemctl restart ceph-radosgw@rgw.`hostname -s`
+
+.. _rgw-multisite-migrate-from-single-site:
+
+Migrating a Single Site System to Multi-Site
+============================================
+
+To migrate from a single site system with a ``default`` zone group and
+zone to a multi site system, use the following steps:
+
+1. Create a realm. Replace ``<name>`` with the realm name.
+
+   .. prompt:: bash #
+
+      radosgw-admin realm create --rgw-realm=<name> --default
+
+2. Rename the default zone and zonegroup. Replace ``<name>`` with the
+   zonegroup or zone name.
+
+   .. prompt:: bash #
+
+      radosgw-admin zonegroup rename --rgw-zonegroup default --zonegroup-new-name=<name>
+      radosgw-admin zone rename --rgw-zone default --zone-new-name us-east-1 --rgw-zonegroup=<name>
+
+3. Configure the master zonegroup. Replace ``<name>`` with the realm or
+   zonegroup name. Replace ``<fqdn>`` with the fully qualified domain
+   name(s) in the zonegroup.
+
+   .. prompt:: bash #
+
+      radosgw-admin zonegroup modify --rgw-realm=<name> --rgw-zonegroup=<name> --endpoints http://<fqdn>:80 --master --default
+
+4. Configure the master zone. Replace ``<name>`` with the realm,
+   zonegroup or zone name. Replace ``<fqdn>`` with the fully qualified
+   domain name(s) in the zonegroup.
+
+   .. prompt:: bash #
+
+      radosgw-admin zone modify --rgw-realm=<name> --rgw-zonegroup=<name> \
+                                --rgw-zone=<name> --endpoints http://<fqdn>:80 \
+                                --access-key=<access-key> --secret=<secret-key> \
+                                --master --default
+
+5. Create a system user. Replace ``<user-id>`` with the username.
+   Replace ``<display-name>`` with a display name. It may contain
+   spaces.
+
+   .. prompt:: bash #
+
+      radosgw-admin user create --uid=<user-id> \
+      --display-name="<display-name>" \ 
+      --access-key=<access-key> \ 
+      --secret=<secret-key> --system
+
+6. Commit the updated configuration.
+
+   .. prompt:: bash #
+
+      radosgw-admin period update --commit
+
+7. Finally, restart the Ceph Object Gateway.
+
+   .. prompt:: bash #
+
+      systemctl restart ceph-radosgw@rgw.`hostname -s`
+
+After completing this procedure, proceed to `Configure a Secondary
+Zone <#configure-secondary-zones>`__ to create a secondary zone
+in the master zone group.
+
+
+Multi-Site Configuration Reference
+==================================
+
+The following sections provide additional details and command-line
+usage for realms, periods, zone groups and zones.
+
+Realms
+------
+
+A realm represents a globally unique namespace consisting of one or more
+zonegroups containing one or more zones, and zones containing buckets,
+which in turn contain objects. A realm enables the Ceph Object Gateway
+to support multiple namespaces and their configuration on the same
+hardware.
+
+A realm contains the notion of periods. Each period represents the state
+of the zone group and zone configuration in time. Each time you make a
+change to a zonegroup or zone, update the period and commit it.
+
+By default, the Ceph Object Gateway does not create a realm
+for backward compatibility with Infernalis and earlier releases.
+However, as a best practice, we recommend creating realms for new
+clusters.
+
+Create a Realm
+~~~~~~~~~~~~~~
+
+To create a realm, execute ``realm create`` and specify the realm name.
+If the realm is the default, specify ``--default``.
+
+.. prompt:: bash #
+
+   radosgw-admin realm create --rgw-realm={realm-name} [--default]
+
+For example:
+
+.. prompt:: bash #
+
+   radosgw-admin realm create --rgw-realm=movies --default
+
+By specifying ``--default``, the realm will be called implicitly with
+each ``radosgw-admin`` call unless ``--rgw-realm`` and the realm name
+are explicitly provided.
+
+Make a Realm the Default
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+One realm in the list of realms should be the default realm. There may
+be only one default realm. If there is only one realm and it wasn’t
+specified as the default realm when it was created, make it the default
+realm. Alternatively, to change which realm is the default, execute:
+
+.. prompt:: bash #
+
+   radosgw-admin realm default --rgw-realm=movies
+
+.. note:: When the realm is default, the command line assumes
+   ``--rgw-realm=<realm-name>`` as an argument.
+
+Delete a Realm
+~~~~~~~~~~~~~~
+
+To delete a realm, execute ``realm delete`` and specify the realm name.
+
+.. prompt:: bash #
+
+   radosgw-admin realm rm --rgw-realm={realm-name}
+
+For example:
+
+.. prompt:: bash #
+   
+   radosgw-admin realm rm --rgw-realm=movies
+
+Get a Realm
+~~~~~~~~~~~
+
+To get a realm, execute ``realm get`` and specify the realm name.
+
+.. prompt:: bash #
+
+   radosgw-admin realm get --rgw-realm=<name>
+
+For example:
+
+.. prompt:: bash #
+
+   radosgw-admin realm get --rgw-realm=movies [> filename.json]
+
+The CLI will echo a JSON object with the realm properties.
+
+::
+
+    {
+        "id": "0a68d52e-a19c-4e8e-b012-a8f831cb3ebc",
+        "name": "movies",
+        "current_period": "b0c5bbef-4337-4edd-8184-5aeab2ec413b",
+        "epoch": 1
+    }
+
+Use ``>`` and an output file name to output the JSON object to a file.
+
+Set a Realm
+~~~~~~~~~~~
+
+To set a realm, execute ``realm set``, specify the realm name, and
+``--infile=`` with an input file name.
+
+.. prompt:: bash #
+
+   radosgw-admin realm set --rgw-realm=<name> --infile=<infilename>
+
+For example:
+
+.. prompt:: bash #
+
+   radosgw-admin realm set --rgw-realm=movies --infile=filename.json
+
+List Realms
+~~~~~~~~~~~
+
+To list realms, execute ``realm list``.
+
+.. prompt:: bash #
+
+   radosgw-admin realm list
+
+List Realm Periods
+~~~~~~~~~~~~~~~~~~
+
+To list realm periods, execute ``realm list-periods``.
+
+.. prompt:: bash #
+
+   radosgw-admin realm list-periods
+
+Pull a Realm
+~~~~~~~~~~~~
+
+To pull a realm from the node containing the master zone group and
+master zone to a node containing a secondary zone group or zone, execute
+``realm pull`` on the node that will receive the realm configuration.
+
+.. prompt:: bash #
+
+   radosgw-admin realm pull --url={url-to-master-zone-gateway} --access-key={access-key} --secret={secret}
+
+Rename a Realm
+~~~~~~~~~~~~~~
+
+A realm is not part of the period. Consequently, renaming the realm is
+only applied locally, and will not get pulled with ``realm pull``. When
+renaming a realm with multiple zones, run the command on each zone. To
+rename a realm, execute the following:
+
+.. prompt:: bash #
+
+   radosgw-admin realm rename --rgw-realm=<current-name> --realm-new-name=<new-realm-name>
+
+.. note:: DO NOT use ``realm set`` to change the ``name`` parameter. That
+          changes the internal name only. Specifying ``--rgw-realm`` would
+          still use the old realm name.
+
+Zone Groups
+-----------
+
+The Ceph Object Gateway supports multi-site deployments and a global
+namespace by using the notion of zone groups. Formerly called a region
+in Infernalis, a zone group defines the geographic location of one or more Ceph
+Object Gateway instances within one or more zones.
+
+Configuring zone groups differs from typical configuration procedures,
+because not all of the settings end up in a Ceph configuration file. You
+can list zone groups, get a zone group configuration, and set a zone
+group configuration.
+
+Create a Zone Group
+~~~~~~~~~~~~~~~~~~~
+
+Creating a zone group consists of specifying the zone group name.
+Creating a zone assumes it will live in the default realm unless
+``--rgw-realm=<realm-name>`` is specified. If the zonegroup is the
+default zonegroup, specify the ``--default`` flag. If the zonegroup is
+the master zonegroup, specify the ``--master`` flag. For example:
+
+.. prompt:: bash #
+
+   radosgw-admin zonegroup create --rgw-zonegroup=<name> [--rgw-realm=<name>][--master] [--default]
+
+
+.. note:: Use ``zonegroup modify --rgw-zonegroup=<zonegroup-name>`` to modify
+          an existing zone group’s settings.
+
+Make a Zone Group the Default
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+One zonegroup in the list of zonegroups should be the default zonegroup.
+There may be only one default zonegroup. If there is only one zonegroup
+and it wasn’t specified as the default zonegroup when it was created,
+make it the default zonegroup. Alternatively, to change which zonegroup
+is the default, execute:
+
+.. prompt:: bash #
+
+   radosgw-admin zonegroup default --rgw-zonegroup=comedy
+
+.. note:: When the zonegroup is default, the command line assumes
+          ``--rgw-zonegroup=<zonegroup-name>`` as an argument.
+
+Then, update the period:
+
+.. prompt:: bash #
+
+   radosgw-admin period update --commit
+
+Add a Zone to a Zone Group
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To add a zone to a zonegroup, execute the following:
+
+.. prompt:: bash #
+
+   radosgw-admin zonegroup add --rgw-zonegroup=<name> --rgw-zone=<name>
+
+Then, update the period:
+
+.. prompt:: bash #
+
+   radosgw-admin period update --commit
+
+Remove a Zone from a Zone Group
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To remove a zone from a zonegroup, execute the following:
+
+.. prompt:: bash #
+
+   radosgw-admin zonegroup remove --rgw-zonegroup=<name> --rgw-zone=<name>
+
+Then, update the period:
+
+.. prompt:: bash #
+
+   radosgw-admin period update --commit
+
+Rename a Zone Group
+~~~~~~~~~~~~~~~~~~~
+
+To rename a zonegroup, execute the following:
+
+
+.. prompt:: bash #
+
+   radosgw-admin zonegroup rename --rgw-zonegroup=<name> --zonegroup-new-name=<name>
+
+Then, update the period:
+
+.. prompt:: bash #
+   
+   radosgw-admin period update --commit
+
+Delete a Zone Group
+~~~~~~~~~~~~~~~~~~~
+
+To delete a zonegroup, execute the following:
+
+.. prompt:: bash #
+   
+   radosgw-admin zonegroup delete --rgw-zonegroup=<name>
+
+Then, update the period:
+
+.. prompt:: bash #
+   
+   radosgw-admin period update --commit
+
+List Zone Groups
+~~~~~~~~~~~~~~~~
+
+A Ceph cluster contains a list of zone groups. To list the zone groups,
+execute:
+
+.. prompt:: bash #
+   
+   radosgw-admin zonegroup list
+
+The ``radosgw-admin`` returns a JSON formatted list of zone groups.
+
+::
+
+    {
+        "default_info": "90b28698-e7c3-462c-a42d-4aa780d24eda",
+        "zonegroups": [
+            "us"
+        ]
+    }
+
+Get a Zone Group Map
+~~~~~~~~~~~~~~~~~~~~
+
+To list the details of each zone group, execute:
+
+.. prompt:: bash #
+   
+   radosgw-admin zonegroup-map get
+
+.. note:: If you receive a ``failed to read zonegroup map`` error, run
+          ``radosgw-admin zonegroup-map update`` as ``root`` first.
+
+Get a Zone Group
+~~~~~~~~~~~~~~~~
+
+To view the configuration of a zone group, execute:
+
+.. prompt:: bash #
+   
+   dosgw-admin zonegroup get [--rgw-zonegroup=<zonegroup>]
+
+The zone group configuration looks like this:
+
+::
+
+    {
+        "id": "90b28698-e7c3-462c-a42d-4aa780d24eda",
+        "name": "us",
+        "api_name": "us",
+        "is_master": "true",
+        "endpoints": [
+            "http:\/\/rgw1:80"
+        ],
+        "hostnames": [],
+        "hostnames_s3website": [],
+        "master_zone": "9248cab2-afe7-43d8-a661-a40bf316665e",
+        "zones": [
+            {
+                "id": "9248cab2-afe7-43d8-a661-a40bf316665e",
+                "name": "us-east",
+                "endpoints": [
+                    "http:\/\/rgw1"
+                ],
+                "log_meta": "true",
+                "log_data": "true",
+                "bucket_index_max_shards": 0,
+                "read_only": "false"
+            },
+            {
+                "id": "d1024e59-7d28-49d1-8222-af101965a939",
+                "name": "us-west",
+                "endpoints": [
+                    "http:\/\/rgw2:80"
+                ],
+                "log_meta": "false",
+                "log_data": "true",
+                "bucket_index_max_shards": 0,
+                "read_only": "false"
+            }
+        ],
+        "placement_targets": [
+            {
+                "name": "default-placement",
+                "tags": []
+            }
+        ],
+        "default_placement": "default-placement",
+        "realm_id": "ae031368-8715-4e27-9a99-0c9468852cfe"
+    }
+
+Set a Zone Group
+~~~~~~~~~~~~~~~~
+
+Defining a zone group consists of creating a JSON object, specifying at
+least the required settings:
+
+1. ``name``: The name of the zone group. Required.
+
+2. ``api_name``: The API name for the zone group. Optional.
+
+3. ``is_master``: Determines if the zone group is the master zone group.
+   Required. **note:** You can only have one master zone group.
+
+4. ``endpoints``: A list of all the endpoints in the zone group. For
+   example, you may use multiple domain names to refer to the same zone
+   group. Remember to escape the forward slashes (``\/``). You may also
+   specify a port (``fqdn:port``) for each endpoint. Optional.
+
+5. ``hostnames``: A list of all the hostnames in the zone group. For
+   example, you may use multiple domain names to refer to the same zone
+   group. Optional. The ``rgw dns name`` setting will automatically be
+   included in this list. You should restart the gateway daemon(s) after
+   changing this setting.
+
+6. ``master_zone``: The master zone for the zone group. Optional. Uses
+   the default zone if not specified. **note:** You can only have one
+   master zone per zone group.
+
+7. ``zones``: A list of all zones within the zone group. Each zone has a
+   name (required), a list of endpoints (optional), and whether or not
+   the gateway will log metadata and data operations (false by default).
+
+8. ``placement_targets``: A list of placement targets (optional). Each
+   placement target contains a name (required) for the placement target
+   and a list of tags (optional) so that only users with the tag can use
+   the placement target (i.e., the user’s ``placement_tags`` field in
+   the user info).
+
+9. ``default_placement``: The default placement target for the object
+   index and object data. Set to ``default-placement`` by default. You
+   may also set a per-user default placement in the user info for each
+   user.
+
+To set a zone group, create a JSON object consisting of the required
+fields, save the object to a file (e.g., ``zonegroup.json``); then,
+execute the following command:
+
+.. prompt:: bash #
+   
+   radosgw-admin zonegroup set --infile zonegroup.json
+
+Where ``zonegroup.json`` is the JSON file you created.
+
+.. important:: The ``default`` zone group ``is_master`` setting is ``true`` by
+               default. If you create a new zone group and want to make it the
+               master zone group, you must either set the ``default`` zone group
+               ``is_master`` setting to ``false``, or delete the ``default`` zone
+               group.
+
+Finally, update the period:
+
+.. prompt:: bash #
+   
+   radosgw-admin period update --commit
+
+Set a Zone Group Map
+~~~~~~~~~~~~~~~~~~~~
+
+Setting a zone group map consists of creating a JSON object consisting
+of one or more zone groups, and setting the ``master_zonegroup`` for the
+cluster. Each zone group in the zone group map consists of a key/value
+pair, where the ``key`` setting is equivalent to the ``name`` setting
+for an individual zone group configuration, and the ``val`` is a JSON
+object consisting of an individual zone group configuration.
+
+You may only have one zone group with ``is_master`` equal to ``true``,
+and it must be specified as the ``master_zonegroup`` at the end of the
+zone group map. The following JSON object is an example of a default
+zone group map.
+
+::
+
+    {
+        "zonegroups": [
+            {
+                "key": "90b28698-e7c3-462c-a42d-4aa780d24eda",
+                "val": {
+                    "id": "90b28698-e7c3-462c-a42d-4aa780d24eda",
+                    "name": "us",
+                    "api_name": "us",
+                    "is_master": "true",
+                    "endpoints": [
+                        "http:\/\/rgw1:80"
+                    ],
+                    "hostnames": [],
+                    "hostnames_s3website": [],
+                    "master_zone": "9248cab2-afe7-43d8-a661-a40bf316665e",
+                    "zones": [
+                        {
+                            "id": "9248cab2-afe7-43d8-a661-a40bf316665e",
+                            "name": "us-east",
+                            "endpoints": [
+                                "http:\/\/rgw1"
+                            ],
+                            "log_meta": "true",
+                            "log_data": "true",
+                            "bucket_index_max_shards": 0,
+                            "read_only": "false"
+                        },
+                        {
+                            "id": "d1024e59-7d28-49d1-8222-af101965a939",
+                            "name": "us-west",
+                            "endpoints": [
+                                "http:\/\/rgw2:80"
+                            ],
+                            "log_meta": "false",
+                            "log_data": "true",
+                            "bucket_index_max_shards": 0,
+                            "read_only": "false"
+                        }
+                    ],
+                    "placement_targets": [
+                        {
+                            "name": "default-placement",
+                            "tags": []
+                        }
+                    ],
+                    "default_placement": "default-placement",
+                    "realm_id": "ae031368-8715-4e27-9a99-0c9468852cfe"
+                }
+            }
+        ],
+        "master_zonegroup": "90b28698-e7c3-462c-a42d-4aa780d24eda",
+        "bucket_quota": {
+            "enabled": false,
+            "max_size_kb": -1,
+            "max_objects": -1
+        },
+        "user_quota": {
+            "enabled": false,
+            "max_size_kb": -1,
+            "max_objects": -1
+        }
+    }
+
+To set a zone group map, execute the following:
+
+.. prompt:: bash #
+   
+   radosgw-admin zonegroup-map set --infile zonegroupmap.json
+
+Where ``zonegroupmap.json`` is the JSON file you created. Ensure that
+you have zones created for the ones specified in the zone group map.
+Finally, update the period.
+
+.. prompt:: bash #
+   
+   radosgw-admin period update --commit
+
+Zones
+-----
+
+Ceph Object Gateway supports the notion of zones. A zone defines a
+logical group consisting of one or more Ceph Object Gateway instances.
+
+Configuring zones differs from typical configuration procedures, because
+not all of the settings end up in a Ceph configuration file. You can
+list zones, get a zone configuration and set a zone configuration.
+
+Create a Zone
+~~~~~~~~~~~~~
+
+To create a zone, specify a zone name. If it is a master zone, specify
+the ``--master`` option. Only one zone in a zone group may be a master
+zone. To add the zone to a zonegroup, specify the ``--rgw-zonegroup``
+option with the zonegroup name.
+
+.. prompt:: bash #
+   
+   radosgw-admin zone create --rgw-zone=<name> \
+                    [--zonegroup=<zonegroup-name]\
+                    [--endpoints=<endpoint>[,<endpoint>] \
+                    [--master] [--default] \
+                    --access-key $SYSTEM_ACCESS_KEY --secret $SYSTEM_SECRET_KEY
+
+Then, update the period:
+
+.. prompt:: bash #
+   
+   radosgw-admin period update --commit
+
+Delete a Zone
+~~~~~~~~~~~~~
+
+To delete zone, first remove it from the zonegroup.
+
+.. prompt:: bash #
+   
+   radosgw-admin zonegroup remove --zonegroup=<name>\
+                                     --zone=<name>
+
+Then, update the period:
+
+.. prompt:: bash #
+   
+   radosgw-admin period update --commit
+
+Next, delete the zone. Execute the following:
+
+.. prompt:: bash #
+   
+   radosgw-admin zone delete --rgw-zone<name>
+
+Finally, update the period:
+
+.. prompt:: bash #
+   
+   radosgw-admin period update --commit
+
+.. important:: Do not delete a zone without removing it from a zone group first.
+               Otherwise, updating the period will fail.
+
+If the pools for the deleted zone will not be used anywhere else,
+consider deleting the pools. Replace ``<del-zone>`` in the example below
+with the deleted zone’s name.
+
+.. important:: Only delete the pools with prepended zone names. Deleting the root
+               pool, such as, ``.rgw.root`` will remove all of the system’s
+               configuration.
+
+.. important:: Once the pools are deleted, all of the data within them are deleted
+               in an unrecoverable manner. Only delete the pools if the pool
+               contents are no longer needed.
+
+.. prompt:: bash #
+   
+   ceph osd pool rm <del-zone>.rgw.control <del-zone>.rgw.control --yes-i-really-really-mean-it
+   ceph osd pool rm <del-zone>.rgw.meta <del-zone>.rgw.meta --yes-i-really-really-mean-it
+   ceph osd pool rm <del-zone>.rgw.log <del-zone>.rgw.log --yes-i-really-really-mean-it
+   ceph osd pool rm <del-zone>.rgw.otp <del-zone>.rgw.otp --yes-i-really-really-mean-it
+   ceph osd pool rm <del-zone>.rgw.buckets.index <del-zone>.rgw.buckets.index --yes-i-really-really-mean-it
+   ceph osd pool rm <del-zone>.rgw.buckets.non-ec <del-zone>.rgw.buckets.non-ec --yes-i-really-really-mean-it
+   ceph osd pool rm <del-zone>.rgw.buckets.data <del-zone>.rgw.buckets.data --yes-i-really-really-mean-it
+
+Modify a Zone
+~~~~~~~~~~~~~
+
+To modify a zone, specify the zone name and the parameters you wish to
+modify.
+
+.. prompt:: bash #
+   
+   radosgw-admin zone modify [options]
+
+Where ``[options]``:
+
+- ``--access-key=<key>``
+- ``--secret/--secret-key=<key>``
+- ``--master``
+- ``--default``
+- ``--endpoints=<list>``
+
+Then, update the period:
+
+.. prompt:: bash #
+   
+   radosgw-admin period update --commit
+
+List Zones
+~~~~~~~~~~
+
+As ``root``, to list the zones in a cluster, execute:
+
+.. prompt:: bash #
+   
+   radosgw-admin zone list
+
+Get a Zone
+~~~~~~~~~~
+
+As ``root``, to get the configuration of a zone, execute:
+
+.. prompt:: bash #
+   
+   radosgw-admin zone get [--rgw-zone=<zone>]
+
+The ``default`` zone looks like this:
+
+::
+
+    { "domain_root": ".rgw",
+      "control_pool": ".rgw.control",
+      "gc_pool": ".rgw.gc",
+      "log_pool": ".log",
+      "intent_log_pool": ".intent-log",
+      "usage_log_pool": ".usage",
+      "user_keys_pool": ".users",
+      "user_email_pool": ".users.email",
+      "user_swift_pool": ".users.swift",
+      "user_uid_pool": ".users.uid",
+      "system_key": { "access_key": "", "secret_key": ""},
+      "placement_pools": [
+          {  "key": "default-placement",
+             "val": { "index_pool": ".rgw.buckets.index",
+                      "data_pool": ".rgw.buckets"}
+          }
+        ]
+      }
+
+Set a Zone
+~~~~~~~~~~
+
+Configuring a zone involves specifying a series of Ceph Object Gateway
+pools. For consistency, we recommend using a pool prefix that is the
+same as the zone name. See
+`Pools <http://docs.ceph.com/en/latest/rados/operations/pools/#pools>`__
+for details of configuring pools.
+
+To set a zone, create a JSON object consisting of the pools, save the
+object to a file (e.g., ``zone.json``); then, execute the following
+command, replacing ``{zone-name}`` with the name of the zone:
+
+.. prompt:: bash #
+   
+   radosgw-admin zone set --rgw-zone={zone-name} --infile zone.json
+
+Where ``zone.json`` is the JSON file you created.
+
+Then, as ``root``, update the period:
+
+.. prompt:: bash #
+   
+   radosgw-admin period update --commit
+
+Rename a Zone
+~~~~~~~~~~~~~
+
+To rename a zone, specify the zone name and the new zone name.
+
+.. prompt:: bash #
+   
+   radosgw-admin zone rename --rgw-zone=<name> --zone-new-name=<name>
+
+Then, update the period:
+
+.. prompt:: bash #
+   
+   radosgw-admin period update --commit
+
+Zone Group and Zone Settings
+----------------------------
+
+When configuring a default zone group and zone, the pool name includes
+the zone name. For example:
+
+-  ``default.rgw.control``
+
+To change the defaults, include the following settings in your Ceph
+configuration file under each ``[client.radosgw.{instance-name}]``
+instance.
+
++-------------------------------------+-----------------------------------+---------+-----------------------+
+| Name                                | Description                       | Type    | Default               |
++=====================================+===================================+=========+=======================+
+| ``rgw_zone``                        | The name of the zone for the      | String  | None                  |
+|                                     | gateway instance.                 |         |                       |
++-------------------------------------+-----------------------------------+---------+-----------------------+
+| ``rgw_zonegroup``                   | The name of the zone group for    | String  | None                  |
+|                                     | the gateway instance.             |         |                       |
++-------------------------------------+-----------------------------------+---------+-----------------------+
+| ``rgw_zonegroup_root_pool``         | The root pool for the zone group. | String  | ``.rgw.root``         |
++-------------------------------------+-----------------------------------+---------+-----------------------+
+| ``rgw_zone_root_pool``              | The root pool for the zone.       | String  | ``.rgw.root``         |
++-------------------------------------+-----------------------------------+---------+-----------------------+
+| ``rgw_default_zone_group_info_oid`` | The OID for storing the default   | String  | ``default.zonegroup`` |
+|                                     | zone group. We do not recommend   |         |                       |
+|                                     | changing this setting.            |         |                       |
++-------------------------------------+-----------------------------------+---------+-----------------------+
+
+
+Zone Features
+=============
+
+Some multisite features require support from all zones before they can be enabled. Each zone lists its ``supported_features``, and each zonegroup lists its ``enabled_features``. Before a feature can be enabled in the zonegroup, it must be supported by all of its zones.
+
+On creation of new zones and zonegroups, all known features are supported/enabled. After upgrading an existing multisite configuration, however, new features must be enabled manually.
+
+Supported Features
+------------------
+
++---------------------------+---------+
+| Feature                   | Release |
++===========================+=========+
+| :ref:`feature_resharding` | Quincy  |
++---------------------------+---------+
+
+.. _feature_resharding:
+
+resharding
+~~~~~~~~~~
+
+Allows buckets to be resharded in a multisite configuration without interrupting the replication of their objects. When ``rgw_dynamic_resharding`` is enabled, it runs on each zone independently, and zones may choose different shard counts for the same bucket. When buckets are resharded manually with ``radosgw-admin bucket reshard``, only that zone's bucket is modified. A zone feature should only be marked as supported after all of its radosgws and osds have upgraded.
+
+
+Commands
+-----------------
+
+Add support for a zone feature
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+On the cluster that contains the given zone:
+
+.. prompt:: bash $
+
+   radosgw-admin zone modify --rgw-zone={zone-name} --enable-feature={feature-name}
+   radosgw-admin period update --commit
+
+
+Remove support for a zone feature
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+On the cluster that contains the given zone:
+
+.. prompt:: bash $
+
+   radosgw-admin zone modify --rgw-zone={zone-name} --disable-feature={feature-name}
+   radosgw-admin period update --commit
+
+Enable a zonegroup feature
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+On any cluster in the realm:
+
+.. prompt:: bash $
+
+   radosgw-admin zonegroup modify --rgw-zonegroup={zonegroup-name} --enable-feature={feature-name}
+   radosgw-admin period update --commit
+
+Disable a zonegroup feature
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+On any cluster in the realm:
+
+.. prompt:: bash $
+
+   radosgw-admin zonegroup modify --rgw-zonegroup={zonegroup-name} --disable-feature={feature-name}
+   radosgw-admin period update --commit
+
+
+.. _`Pools`: ../pools
+.. _`Sync Policy Config`: ../multisite-sync-policy
diff --git a/doc/radosgw/multitenancy.rst b/doc/radosgw/multitenancy.rst
new file mode 100644
index 000000000..0cca50d96
--- /dev/null
+++ b/doc/radosgw/multitenancy.rst
@@ -0,0 +1,169 @@
+.. _rgw-multitenancy:
+
+=================
+RGW Multi-tenancy
+=================
+
+.. versionadded:: Jewel
+
+The multi-tenancy feature allows to use buckets and users of the same
+name simultaneously by segregating them under so-called ``tenants``.
+This may be useful, for instance, to permit users of Swift API to
+create buckets with easily conflicting names such as "test" or "trove".
+
+From the Jewel release onward, each user and bucket lies under a tenant.
+For compatibility, a "legacy" tenant with an empty name is provided.
+Whenever a bucket is referred without an explicit tenant, an implicit
+tenant is used, taken from the user performing the operation. Since
+the pre-existing users are under the legacy tenant, they continue
+to create and access buckets as before. The layout of objects in RADOS
+is extended in a compatible way, ensuring a smooth upgrade to Jewel.
+
+Administering Users With Explicit Tenants
+=========================================
+
+Tenants as such do not have any operations on them. They appear and
+disappear as needed, when users are administered. In order to create,
+modify, and remove users with explicit tenants, either an additional
+option --tenant is supplied, or a syntax "<tenant>$<user>" is used
+in the parameters of the radosgw-admin command.
+
+Examples
+--------
+
+Create a user testx$tester to be accessed with S3::
+
+  # radosgw-admin --tenant testx --uid tester --display-name "Test User" --access_key TESTER --secret test123 user create
+
+Create a user testx$tester to be accessed with Swift::
+
+  # radosgw-admin --tenant testx --uid tester --display-name "Test User" --subuser tester:test --key-type swift --access full user create
+  # radosgw-admin --subuser 'testx$tester:test' --key-type swift --secret test123
+
+.. note:: The subuser with explicit tenant has to be quoted in the shell.
+
+   Tenant names may contain only alphanumeric characters and underscores.
+
+Accessing Buckets with Explicit Tenants
+=======================================
+
+When a client application accesses buckets, it always operates with
+credentials of a particular user. As mentioned above, every user belongs
+to a tenant. Therefore, every operation has an implicit tenant in its
+context, to be used if no tenant is specified explicitly. Thus a complete
+compatibility is maintained with previous releases, as long as the
+referred buckets and referring user belong to the same tenant.
+In other words, anything unusual occurs when accessing another tenant's
+buckets *only*.
+
+Extensions employed to specify an explicit tenant differ according
+to the protocol and authentication system used.
+
+S3
+--
+
+In case of S3, a colon character is used to separate tenant and bucket.
+Thus a sample URL would be::
+
+  https://ep.host.dom/tenant:bucket
+
+Here's a simple Python sample:
+
+.. code-block:: python
+   :linenos:
+
+	from boto.s3.connection import S3Connection, OrdinaryCallingFormat
+	c = S3Connection(
+		aws_access_key_id="TESTER",
+		aws_secret_access_key="test123",
+		host="ep.host.dom",
+		calling_format = OrdinaryCallingFormat())
+	bucket = c.get_bucket("test5b:testbucket")
+
+Note that it's not possible to supply an explicit tenant using
+a hostname. Hostnames cannot contain colons, or any other separators
+that are not already valid in bucket names. Using a period creates an
+ambiguous syntax. Therefore, the bucket-in-URL-path format has to be
+used.
+
+Due to the fact that the native S3 API does not deal with
+multi-tenancy and radosgw's implementation does, things get a bit
+involved when dealing with signed URLs and public read ACLs.
+
+* A **signed URL** does contain the ``AWSAccessKeyId`` query
+  parameters, from which radosgw is able to discern the correct user
+  and tenant owning the bucket. In other words, an application
+  generating signed URLs should be able to take just the un-prefixed
+  bucket name, and produce a signed URL that itself contains the
+  bucket name without the tenant prefix. However, it is *possible* to
+  include the prefix if you so choose.
+
+  Thus, accessing a signed URL of an object ``bar`` in a container
+  ``foo`` belonging to the tenant ``7188e165c0ae4424ac68ae2e89a05c50``
+  would be possible either via
+  ``http://<host>:<port>/foo/bar?AWSAccessKeyId=b200fb6634c547199e436a0f93c0c46e&Expires=1542890806&Signature=eok6CYQC%2FDwmQQmqvY5jTg6ehXU%3D``,
+  or via
+  ``http://<host>:<port>/7188e165c0ae4424ac68ae2e89a05c50:foo/bar?AWSAccessKeyId=b200fb6634c547199e436a0f93c0c46e&Expires=1542890806&Signature=eok6CYQC%2FDwmQQmqvY5jTg6ehXU%3D``,
+  depending on whether or not the tenant prefix was passed in on
+  signature generation.
+
+* A bucket with a **public read ACL** is meant to be read by an HTTP
+  client *without* including any query parameters that would allow
+  radosgw to discern tenants. Thus, publicly readable objects must
+  always be accessed using the bucket name with the tenant prefix.
+
+  Thus, if you set a public read ACL on an object ``bar`` in a
+  container ``foo`` belonging to the tenant
+  ``7188e165c0ae4424ac68ae2e89a05c50``, you would need to access that
+  object via the public URL
+  ``http://<host>:<port>/7188e165c0ae4424ac68ae2e89a05c50:foo/bar``.
+
+Swift with built-in authenticator
+---------------------------------
+
+TBD -- not in test_multen.py yet
+
+Swift with Keystone
+-------------------
+
+In the default configuration, although native Swift has inherent
+multi-tenancy, radosgw does not enable multi-tenancy for the Swift
+API. This is to ensure that a setup with legacy buckets --- that is,
+buckets that were created before radosgw supported multitenancy ---,
+those buckets retain their dual-API capability to be queried and
+modified using either S3 or Swift.
+
+If you want to enable multitenancy for Swift, particularly if your
+users only ever authenticate against OpenStack Keystone, you should
+enable Keystone-based multitenancy with the following ``ceph.conf``
+configuration option::
+
+  rgw keystone implicit tenants = true
+
+Once you enable this option, any newly connecting user (whether they
+are using the Swift API, or Keystone-authenticated S3) will prompt
+radosgw to create a user named ``<tenant_id>$<tenant_id``, where
+``<tenant_id>`` is a Keystone tenant (project) UUID --- for example,
+``7188e165c0ae4424ac68ae2e89a05c50$7188e165c0ae4424ac68ae2e89a05c50``.
+
+Whenever that user then creates an Swift container, radosgw internally
+translates the given container name into
+``<tenant_id>/<container_name>``, such as
+``7188e165c0ae4424ac68ae2e89a05c50/foo``. This ensures that if there
+are two or more different tenants all creating a container named
+``foo``, radosgw is able to transparently discern them by their tenant
+prefix.
+
+It is also possible to limit the effects of implicit tenants
+to only apply to swift or s3, by setting ``rgw keystone implicit tenants``
+to either ``s3`` or ``swift``.  This will likely primarily
+be of use to users who had previously used implicit tenants
+with older versions of ceph, where implicit tenants
+only applied to the swift protocol.
+
+Notes and known issues
+----------------------
+
+Just to be clear, it is not possible to create buckets in other
+tenants at present. The owner of newly created bucket is extracted
+from authentication information.
diff --git a/doc/radosgw/nfs.rst b/doc/radosgw/nfs.rst
new file mode 100644
index 000000000..0a506599a
--- /dev/null
+++ b/doc/radosgw/nfs.rst
@@ -0,0 +1,371 @@
+===
+NFS
+===
+
+.. versionadded:: Jewel
+
+.. note:: Only the NFSv4 protocol is supported when using a cephadm or rook based deployment.
+
+Ceph Object Gateway namespaces can be exported over the file-based
+NFSv4 protocols, alongside traditional HTTP access
+protocols (S3 and Swift).
+
+In particular, the Ceph Object Gateway can now be configured to
+provide file-based access when embedded in the NFS-Ganesha NFS server.
+
+The simplest and preferred way of managing nfs-ganesha clusters and rgw exports
+is using ``ceph nfs ...`` commands. See :doc:`/mgr/nfs` for more details.
+
+librgw
+======
+
+The librgw.so shared library (Unix) provides a loadable interface to
+Ceph Object Gateway services, and instantiates a full Ceph Object Gateway
+instance on initialization.
+
+In turn, librgw.so exports rgw_file, a stateful API for file-oriented
+access to RGW buckets and objects.  The API is general, but its design
+is strongly influenced by the File System Abstraction Layer (FSAL) API
+of NFS-Ganesha, for which it has been primarily designed.
+
+A set of Python bindings is also provided.
+
+Namespace Conventions
+=====================
+
+The implementation conforms to Amazon Web Services (AWS) hierarchical
+namespace conventions which map UNIX-style path names onto S3 buckets
+and objects.
+
+The top level of the attached namespace consists of S3 buckets,
+represented as NFS directories. Files and directories subordinate to
+buckets are each represented as objects, following S3 prefix and
+delimiter conventions, with '/' being the only supported path
+delimiter [#]_.
+
+For example, if an NFS client has mounted an RGW namespace at "/nfs",
+then a file "/nfs/mybucket/www/index.html" in the NFS namespace
+corresponds to an RGW object "www/index.html" in a bucket/container
+"mybucket."
+
+Although it is generally invisible to clients, the NFS namespace is
+assembled through concatenation of the corresponding paths implied by
+the objects in the namespace.  Leaf objects, whether files or
+directories, will always be materialized in an RGW object of the
+corresponding key name, "<name>" if a file, "<name>/" if a directory.
+Non-leaf directories (e.g., "www" above) might only be implied by
+their appearance in the names of one or more leaf objects. Directories
+created within NFS or directly operated on by an NFS client (e.g., via
+an attribute-setting operation such as chown or chmod) always have a
+leaf object representation used to store materialized attributes such
+as Unix ownership and permissions.
+
+Supported Operations
+====================
+
+The RGW NFS interface supports most operations on files and
+directories, with the following restrictions:
+
+- Links, including symlinks, are not supported.
+- NFS ACLs are not supported.
+
+  + Unix user and group ownership and permissions *are* supported.
+
+- Directories may not be moved/renamed.
+
+  + Files may be moved between directories.
+
+- Only full, sequential *write* I/O is supported
+
+  + i.e., write operations are constrained to be **uploads**.
+  + Many typical I/O operations such as editing files in place will necessarily fail as they perform non-sequential stores.
+  + Some file utilities *apparently* writing sequentially (e.g., some versions of GNU tar) may fail due to infrequent non-sequential stores.
+  + When mounting via NFS, sequential application I/O can generally be constrained to be written sequentially to the NFS server via a synchronous mount option (e.g. -osync in Linux).
+  + NFS clients which cannot mount synchronously (e.g., MS Windows) will not be able to upload files.
+
+Security
+========
+
+The RGW NFS interface provides a hybrid security model with the
+following characteristics:
+
+- NFS protocol security is provided by the NFS-Ganesha server, as negotiated by the NFS server and clients
+
+  + e.g., clients can by trusted (AUTH_SYS), or required to present Kerberos user credentials (RPCSEC_GSS)
+  + RPCSEC_GSS wire security can be integrity only (krb5i) or integrity and privacy (encryption, krb5p)
+  + various NFS-specific security and permission rules are available
+
+    * e.g., root-squashing
+
+- a set of RGW/S3 security credentials (unknown to NFS) is associated with each RGW NFS mount (i.e., NFS-Ganesha EXPORT)
+
+  + all RGW object operations performed via the NFS server will be performed by the RGW user associated with the credentials stored in the export being accessed (currently only RGW and RGW LDAP credentials are supported)
+
+    * additional RGW authentication types such as Keystone are not currently supported
+
+Manually configuring an NFS-Ganesha Instance
+============================================
+
+Each NFS RGW instance is an NFS-Ganesha server instance *embeddding*
+a full Ceph RGW instance.
+
+Therefore, the RGW NFS configuration includes Ceph and Ceph Object
+Gateway-specific configuration in a local ceph.conf, as well as
+NFS-Ganesha-specific configuration in the NFS-Ganesha config file,
+ganesha.conf.
+
+ceph.conf
+---------
+
+Required ceph.conf configuration for RGW NFS includes:
+
+* valid [client.radosgw.{instance-name}] section
+* valid values for minimal instance configuration, in particular, an installed and correct ``keyring``
+
+Other config variables are optional, front-end-specific and front-end
+selection variables (e.g., ``rgw data`` and ``rgw frontends``) are
+optional and in some cases ignored.
+
+A small number of config variables (e.g., ``rgw_nfs_namespace_expire_secs``)
+are unique to RGW NFS.
+
+ganesha.conf
+------------
+
+A strictly minimal ganesha.conf for use with RGW NFS includes one
+EXPORT block with embedded FSAL block of type RGW::
+
+   EXPORT
+   {
+        Export_ID={numeric-id};
+        Path = "/";
+        Pseudo = "/";
+        Access_Type = RW;
+        SecType = "sys";
+        NFS_Protocols = 4;
+        Transport_Protocols = TCP;
+
+        # optional, permit unsquashed access by client "root" user
+        #Squash = No_Root_Squash;
+
+        FSAL {
+                Name = RGW;
+                User_Id = {s3-user-id};
+                Access_Key_Id ="{s3-access-key}";
+                Secret_Access_Key = "{s3-secret}";
+        }
+   }
+
+``Export_ID`` must have an integer value, e.g., "77"
+
+``Path`` (for RGW) should be "/"
+
+``Pseudo`` defines an NFSv4 pseudo root name (NFSv4 only)
+
+``SecType = sys;`` allows clients to attach without Kerberos
+authentication
+
+``Squash = No_Root_Squash;`` enables the client root user to override
+permissions (Unix convention).  When root-squashing is enabled,
+operations attempted by the root user are performed as if by the local
+"nobody" (and "nogroup") user on the NFS-Ganesha server
+
+The RGW FSAL additionally supports RGW-specific configuration
+variables in the RGW config section::
+
+ RGW {
+     cluster = "{cluster name, default 'ceph'}";
+     name = "client.rgw.{instance-name}";
+     ceph_conf = "/opt/ceph-rgw/etc/ceph/ceph.conf";
+     init_args = "-d --debug-rgw=16";
+ }
+
+``cluster`` sets a Ceph cluster name (must match the cluster being exported)
+
+``name`` sets an RGW instance name (must match the cluster being exported)
+
+``ceph_conf`` gives a path to a non-default ceph.conf file to use
+
+
+Other useful NFS-Ganesha configuration:
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Any EXPORT block which should support NFSv3 should include version 3
+in the NFS_Protocols setting. Additionally, NFSv3 is the last major
+version to support the UDP transport. To enable UDP, include it in the
+Transport_Protocols setting. For example::
+
+ EXPORT {
+  ...
+    NFS_Protocols = 3,4;
+    Transport_Protocols = UDP,TCP;
+  ...
+ }
+
+One important family of options pertains to interaction with the Linux
+idmapping service, which is used to normalize user and group names
+across systems.  Details of idmapper integration are not provided here.
+
+With Linux NFS clients, NFS-Ganesha can be configured
+to accept client-supplied numeric user and group identifiers with
+NFSv4, which by default stringifies these--this may be useful in small
+setups and for experimentation::
+
+ NFSV4 {
+     Allow_Numeric_Owners = true;
+     Only_Numeric_Owners = true;
+ }
+
+Troubleshooting
+~~~~~~~~~~~~~~~
+
+NFS-Ganesha configuration problems are usually debugged by running the
+server with debugging options, controlled by the LOG config section.
+
+NFS-Ganesha log messages are grouped into various components, logging
+can be enabled separately for each component. Valid values for
+component logging include::
+
+  *FATAL* critical errors only
+  *WARN* unusual condition
+  *DEBUG* mildly verbose trace output
+  *FULL_DEBUG* verbose trace output
+
+Example::
+  
+  LOG {
+
+	Components {
+		MEMLEAKS = FATAL;
+		FSAL = FATAL;
+		NFSPROTO = FATAL;
+		NFS_V4 = FATAL;
+		EXPORT = FATAL;
+		FILEHANDLE = FATAL;
+		DISPATCH = FATAL;
+		CACHE_INODE = FATAL;
+		CACHE_INODE_LRU = FATAL;
+		HASHTABLE = FATAL;
+		HASHTABLE_CACHE = FATAL;
+		DUPREQ = FATAL;
+		INIT = DEBUG;
+		MAIN = DEBUG;
+		IDMAPPER = FATAL;
+		NFS_READDIR = FATAL;
+		NFS_V4_LOCK = FATAL;
+		CONFIG = FATAL;
+		CLIENTID = FATAL;
+		SESSIONS = FATAL;
+		PNFS = FATAL;
+		RW_LOCK = FATAL;
+		NLM = FATAL;
+		RPC = FATAL;
+		NFS_CB = FATAL;
+		THREAD = FATAL;
+		NFS_V4_ACL = FATAL;
+		STATE = FATAL;
+		FSAL_UP = FATAL;
+		DBUS = FATAL;
+	}
+	# optional: redirect log output
+ #	Facility {
+ #		name = FILE;
+ #		destination = "/tmp/ganesha-rgw.log";
+ #		enable = active;
+	}
+ }
+
+Running Multiple NFS Gateways
+=============================
+
+Each NFS-Ganesha instance acts as a full gateway endpoint, with the
+limitation that currently an NFS-Ganesha instance cannot be configured
+to export HTTP services. As with ordinary gateway instances, any
+number of NFS-Ganesha instances can be started, exporting the same or
+different resources from the cluster. This enables the clustering of
+NFS-Ganesha instances. However, this does not imply high availability.
+
+When regular gateway instances and NFS-Ganesha instances overlap the
+same data resources, they will be accessible from both the standard S3
+API and through the NFS-Ganesha instance as exported. You can
+co-locate the NFS-Ganesha instance with a Ceph Object Gateway instance
+on the same host. 
+
+RGW vs RGW NFS
+==============
+
+Exporting an NFS namespace and other RGW namespaces (e.g., S3 or Swift
+via the Civetweb HTTP front-end) from the same program instance is
+currently not supported.
+
+When adding objects and buckets outside of NFS, those objects will
+appear in the NFS namespace in the time set by
+``rgw_nfs_namespace_expire_secs``, which defaults to 300 seconds (5 minutes).
+Override the default value for ``rgw_nfs_namespace_expire_secs`` in the
+Ceph configuration file to change the refresh rate.
+
+If exporting Swift containers that do not conform to valid S3 bucket
+naming requirements, set ``rgw_relaxed_s3_bucket_names`` to true in the
+[client.radosgw] section of the Ceph configuration file. For example,
+if a Swift container name contains underscores, it is not a valid S3
+bucket name and will be rejected unless ``rgw_relaxed_s3_bucket_names``
+is set to true.
+
+Configuring NFSv4 clients
+=========================
+
+To access the namespace, mount the configured NFS-Ganesha export(s)
+into desired locations in the local POSIX namespace. As noted, this
+implementation has a few unique restrictions:
+
+- NFS 4.1 and higher protocol flavors are preferred
+
+  + NFSv4 OPEN and CLOSE operations are used to track upload transactions
+
+- To upload data successfully, clients must preserve write ordering
+
+  + on Linux and many Unix NFS clients, use the -osync mount option
+
+Conventions for mounting NFS resources are platform-specific. The
+following conventions work on Linux and some Unix platforms:
+
+From the command line::
+
+  mount -t nfs -o nfsvers=4.1,noauto,soft,sync,proto=tcp <ganesha-host-name>:/ <mount-point>
+
+In /etc/fstab::
+
+<ganesha-host-name>:/ <mount-point> nfs noauto,soft,nfsvers=4.1,sync,proto=tcp 0 0
+
+Specify the NFS-Ganesha host name and the path to the mount point on
+the client.
+
+Configuring NFSv3 Clients
+=========================
+
+Linux clients can be configured to mount with NFSv3 by supplying
+``nfsvers=3`` and ``noacl`` as mount options. To use UDP as the
+transport, add ``proto=udp`` to the mount options. However, TCP is the
+preferred transport::
+
+  <ganesha-host-name>:/ <mount-point> nfs noauto,noacl,soft,nfsvers=3,sync,proto=tcp 0 0
+
+Configure the NFS Ganesha EXPORT block Protocols setting with version
+3 and the Transports setting with UDP if the mount will use version 3 with UDP.
+
+NFSv3 Semantics
+---------------
+
+Since NFSv3 does not communicate client OPEN and CLOSE operations to
+file servers, RGW NFS cannot use these operations to mark the
+beginning and ending of file upload transactions. Instead, RGW NFS
+starts a new upload when the first write is sent to a file at offset
+0, and finalizes the upload when no new writes to the file have been
+seen for a period of time, by default, 10 seconds. To change this
+timeout, set an alternate value for ``rgw_nfs_write_completion_interval_s``
+in the RGW section(s) of the Ceph configuration file. 
+
+References
+==========
+
+.. [#] http://docs.aws.amazon.com/AmazonS3/latest/dev/ListingKeysHierarchy.html
diff --git a/doc/radosgw/notifications.rst b/doc/radosgw/notifications.rst
new file mode 100644
index 000000000..a294e6c15
--- /dev/null
+++ b/doc/radosgw/notifications.rst
@@ -0,0 +1,543 @@
+====================
+Bucket Notifications
+====================
+
+.. versionadded:: Nautilus
+
+.. contents::
+
+Bucket notifications provide a mechanism for sending information out of radosgw
+when certain events happen on the bucket. Notifications can be sent to HTTP
+endpoints, AMQP0.9.1 endpoints, and Kafka endpoints.
+
+The `PubSub Module`_ (and *not* the bucket-notification mechanism) should be
+used for events stored in Ceph. 
+
+A user can create topics. A topic entity is defined by its name and is "per
+tenant". A user can associate its topics (via notification configuration) only
+with buckets it owns.
+
+A notification entity must be created in order to send event notifications for
+a specific bucket. A notification entity can be created either for a subset
+of event types or for all event types (which is the default). The
+notification may also filter out events based on matches of the prefixes and
+suffixes of (1) the keys, (2) the metadata attributes attached to the object,
+or (3) the object tags. Regular-expression matching can also be used on these
+to create filters. There can be multiple notifications for any specific topic,
+and the same topic can used for multiple notifications.
+
+REST API has been defined so as to provide configuration and control interfaces
+for the bucket notification mechanism. This API is similar to the one defined
+as the S3-compatible API of the `PubSub Module`_.
+
+.. toctree::
+   :maxdepth: 1
+
+   S3 Bucket Notification Compatibility <s3-notification-compatibility>
+
+.. note:: To enable bucket notifications API, the `rgw_enable_apis` configuration parameter should contain: "notifications".
+
+Notification Reliability
+------------------------
+
+Notifications can be sent synchronously or asynchronously. This section
+describes the latency and reliability that you should expect for synchronous
+and asynchronous notifications.
+
+Synchronous Notifications
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Notifications can be sent synchronously, as part of the operation that
+triggered them. In this mode, the operation is acknowledged (acked) only after
+the notification is sent to the topic's configured endpoint. This means that
+the round trip time of the notification (the time it takes to send the
+notification to the topic's endpoint plus the time it takes to receive the
+acknowledgement) is added to the latency of the operation itself.
+
+.. note:: The original triggering operation is considered successful even if
+   the notification fails with an error, cannot be delivered, or times out.
+
+Asynchronous Notifications
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Notifications can be sent asynchronously. They are committed into persistent
+storage and then asynchronously sent to the topic's configured endpoint. In
+this case, the only latency added to the original operation is the latency
+added when the notification is committed to persistent storage.
+
+.. note:: If the notification fails with an error, cannot be delivered, or
+   times out, it is retried until it is successfully acknowledged.
+
+.. tip:: To minimize the latency added by asynchronous notification, we 
+   recommended placing the "log" pool on fast media.
+
+
+Topic Management via CLI
+------------------------
+
+Configuration of all topics, associated with a tenant, could be fetched using the following command:
+
+.. prompt:: bash #
+
+   radosgw-admin topic list [--tenant={tenant}]
+
+
+Configuration of a specific topic could be fetched using:
+
+.. prompt:: bash #
+
+   radosgw-admin topic get --topic={topic-name} [--tenant={tenant}]
+
+
+And removed using:
+
+.. prompt:: bash #
+
+   radosgw-admin topic rm --topic={topic-name} [--tenant={tenant}]
+
+
+Notification Performance Stats
+------------------------------
+The same counters are shared between the pubsub sync module and the bucket notification mechanism.
+
+- ``pubsub_event_triggered``: running counter of events with at least one topic associated with them
+- ``pubsub_event_lost``: running counter of events that had topics associated with them but that were not pushed to any of the endpoints
+- ``pubsub_push_ok``: running counter, for all notifications, of events successfully pushed to their endpoint
+- ``pubsub_push_fail``: running counter, for all notifications, of events failed to be pushed to their endpoint
+- ``pubsub_push_pending``: gauge value of events pushed to an endpoint but not acked or nacked yet
+
+.. note::
+
+    ``pubsub_event_triggered`` and ``pubsub_event_lost`` are incremented per event, while:
+    ``pubsub_push_ok``, ``pubsub_push_fail``, are incremented per push action on each notification
+
+Bucket Notification REST API
+----------------------------
+
+Topics
+~~~~~~
+
+.. note::
+
+    In all topic actions, the parameters are URL-encoded and sent in the
+    message body using this content type:
+    ``application/x-www-form-urlencoded``.
+   
+
+Create a Topic
+``````````````
+
+This creates a new topic. Provide the topic with push endpoint parameters,
+which will be used later when a notification is created. A response is
+generated. A successful response includes the the topic's `ARN
+<https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html>`_
+(the "Amazon Resource Name", a unique identifier used to reference the topic).
+To update a topic, use the same command that you used to create it (but when
+updating, use the name of an existing topic and different endpoint values).
+
+.. tip:: Any notification already associated with the topic must be re-created
+   in order for the topic to update.
+
+.. note:: For rabbitmq, ``push-endpoint`` (with a hyphen in the middle) must be
+   changed to ``push_endpoint`` (with an underscore in the middle).
+
+::
+
+   POST
+
+   Action=CreateTopic
+   &Name=<topic-name>
+   [&Attributes.entry.1.key=amqp-exchange&Attributes.entry.1.value=<exchange>]
+   [&Attributes.entry.2.key=amqp-ack-level&Attributes.entry.2.value=none|broker|routable]
+   [&Attributes.entry.3.key=verify-ssl&Attributes.entry.3.value=true|false]
+   [&Attributes.entry.4.key=kafka-ack-level&Attributes.entry.4.value=none|broker]
+   [&Attributes.entry.5.key=use-ssl&Attributes.entry.5.value=true|false]
+   [&Attributes.entry.6.key=ca-location&Attributes.entry.6.value=<file path>]
+   [&Attributes.entry.7.key=OpaqueData&Attributes.entry.7.value=<opaque data>]
+   [&Attributes.entry.8.key=push-endpoint&Attributes.entry.8.value=<endpoint>]
+   [&Attributes.entry.9.key=persistent&Attributes.entry.9.value=true|false]
+
+Request parameters:
+
+- push-endpoint: This is the URI of an endpoint to send push notifications to.
+- OpaqueData: Opaque data is set in the topic configuration and added to all
+  notifications that are triggered by the topic.
+- persistent: This indicates whether notifications to this endpoint are
+  persistent (=asynchronous) or not persistent. (This is "false" by default.)
+
+- HTTP endpoint
+
+ - URI: ``http[s]://<fqdn>[:<port]``
+ - port: This defaults to 80 for HTTP and 443 for HTTPS.
+ - verify-ssl: This indicates whether the server certificate is validated by
+   the client. (This is "true" by default.)
+ - cloudevents: This indicates whether the HTTP header should contain
+   attributes according to the `S3 CloudEvents Spec`_. (This is "false" by
+   default.)
+
+- AMQP0.9.1 endpoint
+
+ - URI: ``amqp[s]://[<user>:<password>@]<fqdn>[:<port>][/<vhost>]``
+ - user/password: This defaults to "guest/guest".
+ - user/password: This must be provided only over HTTPS. Topic creation
+   requests will otherwise be rejected.
+ - port: This defaults to 5672 for unencrypted connections and 5671 for
+   SSL-encrypted connections.
+ - vhost: This defaults to "/".
+ - verify-ssl: This indicates whether the server certificate is validated by
+   the client. (This is "true" by default.)
+ - If ``ca-location`` is provided and a secure connection is used, the
+   specified CA will be used to authenticate the broker. The default CA will
+   not be used.  
+ - amqp-exchange: The exchanges must exist and must be able to route messages
+   based on topics. This parameter is mandatory.  Different topics that point
+   to the same endpoint must use the same exchange.
+ - amqp-ack-level: No end2end acking is required. Messages may persist in the
+   broker before being delivered to their final destinations. Three ack methods
+   exist:
+
+  - "none": The message is considered "delivered" if it is sent to the broker.
+  - "broker": The message is considered "delivered" if it is acked by the broker (default).
+  - "routable": The message is considered "delivered" if the broker can route to a consumer.
+
+.. tip:: The topic-name (see :ref:`radosgw-create-a-topic`) is used for the
+   AMQP topic ("routing key" for a topic exchange).
+
+- Kafka endpoint
+
+ - URI: ``kafka://[<user>:<password>@]<fqdn>[:<port]``
+ - ``use-ssl``: If this is set to "true", a secure connection is used to
+   connect to the broker. (This is "false" by default.)
+ - ``ca-location``: If this is provided and a secure connection is used, the
+   specified CA will be used insted of the default CA to authenticate the
+   broker. 
+ - user/password: This must be provided only over HTTPS. Topic creation
+   requests will otherwise be rejected.
+ - user/password: This must be provided along with ``use-ssl``. Connections to
+   the broker will otherwise fail.
+ - port: This defaults to 9092.
+ - kafka-ack-level: No end2end acking is required. Messages may persist in the
+   broker before being delivered to their final destinations. Two ack methods
+   exist:
+
+  - "none": Messages are considered "delivered" if sent to the broker.
+  - "broker": Messages are considered "delivered" if acked by the broker. (This
+    is the default.)
+
+.. note::
+
+    - The key-value pair of a specific parameter need not reside in the same
+      line as the parameter, and need not appear in any specific order, but it
+      must use the same index.
+    - Attribute indexing need not be sequential and need not start from any
+      specific value.
+    - `AWS Create Topic`_ provides a detailed explanation of the endpoint
+      attributes format. In our case, however, different keys and values are
+      used.
+
+The response has the following format:
+
+::
+
+    <CreateTopicResponse xmlns="https://sns.amazonaws.com/doc/2010-03-31/">
+        <CreateTopicResult>
+            <TopicArn></TopicArn>
+        </CreateTopicResult>
+        <ResponseMetadata>
+            <RequestId></RequestId>
+        </ResponseMetadata>
+    </CreateTopicResponse>
+
+The topic `ARN
+<https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html>`_
+in the response has the following format:
+
+::
+
+   arn:aws:sns:<zone-group>:<tenant>:<topic>
+
+Get Topic Attributes
+````````````````````
+
+This returns information about a specific topic. This includes push-endpoint
+information, if provided.
+
+::
+
+   POST
+
+   Action=GetTopicAttributes
+   &TopicArn=<topic-arn>
+
+The response has the following format:
+
+::
+
+    <GetTopicAttributesResponse>
+        <GetTopicAttributesRersult>
+            <Attributes>
+                <entry>
+                    <key>User</key>
+                    <value></value>
+                </entry> 
+                <entry>
+                    <key>Name</key>
+                    <value></value>
+                </entry> 
+                <entry>
+                    <key>EndPoint</key>
+                    <value></value>
+                </entry> 
+                <entry>
+                    <key>TopicArn</key>
+                    <value></value>
+                </entry> 
+                <entry>
+                    <key>OpaqueData</key>
+                    <value></value>
+                </entry> 
+            </Attributes>
+        </GetTopicAttributesResult>
+        <ResponseMetadata>
+            <RequestId></RequestId>
+        </ResponseMetadata>
+    </GetTopicAttributesResponse>
+
+- User: the name of the user that created the topic.
+- Name: the name of the topic.
+- EndPoint: The JSON-formatted endpoint parameters, including:
+   - EndpointAddress: The push-endpoint URL.
+   - EndpointArgs: The push-endpoint args.
+   - EndpointTopic: The topic name to be sent to the endpoint (can be different
+     than the above topic name).
+   - HasStoredSecret: This is "true" if the endpoint URL contains user/password 
+     information. In this case, the request must be made over HTTPS. The "topic
+     get" request will otherwise be rejected.
+   - Persistent: This is "true" if the topic is persistent.
+- TopicArn: topic `ARN
+  <https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html>`_.
+- OpaqueData: The opaque data set on the topic.
+
+Get Topic Information
+`````````````````````
+
+This returns information about a specific topic. This includes push-endpoint
+information, if provided.  Note that this API is now deprecated in favor of the
+AWS compliant `GetTopicAttributes` API.
+
+::
+
+   POST
+
+   Action=GetTopic
+   &TopicArn=<topic-arn>
+
+The response has the following format:
+
+::
+
+    <GetTopicResponse>
+        <GetTopicRersult>
+            <Topic>
+                <User></User>
+                <Name></Name>
+                <EndPoint>
+                    <EndpointAddress></EndpointAddress>
+                    <EndpointArgs></EndpointArgs>
+                    <EndpointTopic></EndpointTopic>
+                    <HasStoredSecret></HasStoredSecret>
+                    <Persistent></Persistent>
+                </EndPoint>
+                <TopicArn></TopicArn>
+                <OpaqueData></OpaqueData>
+            </Topic>
+        </GetTopicResult>
+        <ResponseMetadata>
+            <RequestId></RequestId>
+        </ResponseMetadata>
+    </GetTopicResponse>
+
+- User: The name of the user that created the topic.
+- Name: The name of the topic.
+- EndpointAddress: The push-endpoint URL.
+- EndpointArgs: The push-endpoint args.
+- EndpointTopic: The topic name to be sent to the endpoint (which can be
+  different than the above topic name).
+- HasStoredSecret: This is "true" if the endpoint URL contains user/password
+  information. In this case, the request must be made over HTTPS. The "topic
+  get" request will otherwise be rejected.
+- Persistent: "true" if topic is persistent.
+- TopicArn: topic `ARN
+  <https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html>`_.
+- OpaqueData: the opaque data set on the topic.
+
+Delete Topic
+````````````
+
+::
+
+   POST
+
+   Action=DeleteTopic
+   &TopicArn=<topic-arn>
+
+This deletes the specified topic.
+
+.. note::
+
+  - Deleting an unknown notification (for example, double delete) is not
+    considered an error.
+  - Deleting a topic does not automatically delete all notifications associated
+    with it.
+
+The response has the following format:
+
+::
+
+    <DeleteTopicResponse xmlns="https://sns.amazonaws.com/doc/2010-03-31/">
+        <ResponseMetadata>
+            <RequestId></RequestId>
+        </ResponseMetadata>
+    </DeleteTopicResponse>
+
+List Topics
+```````````
+
+List all topics associated with a tenant.
+
+::
+
+   POST
+
+   Action=ListTopics
+
+The response has the following format:
+
+::
+
+    <ListTopicdResponse xmlns="https://sns.amazonaws.com/doc/2010-03-31/">
+        <ListTopicsRersult>
+            <Topics>
+                <member>
+                    <User></User>
+                    <Name></Name>
+                    <EndPoint>
+                        <EndpointAddress></EndpointAddress>
+                        <EndpointArgs></EndpointArgs>
+                        <EndpointTopic></EndpointTopic>
+                    </EndPoint>
+                    <TopicArn></TopicArn>
+                    <OpaqueData></OpaqueData>
+                </member>
+            </Topics>
+        </ListTopicsResult>
+        <ResponseMetadata>
+            <RequestId></RequestId>
+        </ResponseMetadata>
+    </ListTopicsResponse>
+
+- If the endpoint URL contains user/password information in any part of the
+  topic, the request must be made over HTTPS. The "topic list" request will
+  otherwise be rejected.
+
+Notifications
+~~~~~~~~~~~~~
+
+Detailed under: `Bucket Operations`_.
+
+.. note::
+
+    - "Abort Multipart Upload" request does not emit a notification
+    - Both "Initiate Multipart Upload" and "POST Object" requests will emit an ``s3:ObjectCreated:Post`` notification
+
+Events
+~~~~~~
+
+Events are in JSON format (regardless of the actual endpoint), and share the
+same structure as S3-compatible events that are pushed or pulled using the
+pubsub sync module. For example:
+
+::
+
+   {"Records":[
+       {
+           "eventVersion":"2.1",
+           "eventSource":"ceph:s3",
+           "awsRegion":"us-east-1",
+           "eventTime":"2019-11-22T13:47:35.124724Z",
+           "eventName":"ObjectCreated:Put",
+           "userIdentity":{
+               "principalId":"tester"
+           },
+           "requestParameters":{
+               "sourceIPAddress":""
+           },
+           "responseElements":{
+               "x-amz-request-id":"503a4c37-85eb-47cd-8681-2817e80b4281.5330.903595",
+               "x-amz-id-2":"14d2-zone1-zonegroup1"
+           },
+           "s3":{
+               "s3SchemaVersion":"1.0",
+               "configurationId":"mynotif1",
+               "bucket":{
+                   "name":"mybucket1",
+                   "ownerIdentity":{
+                       "principalId":"tester"
+                   },
+                   "arn":"arn:aws:s3:us-east-1::mybucket1",
+                   "id":"503a4c37-85eb-47cd-8681-2817e80b4281.5332.38"
+               },
+               "object":{
+                   "key":"myimage1.jpg",
+                   "size":"1024",
+                   "eTag":"37b51d194a7513e45b56f6524f2d51f2",
+                   "versionId":"",
+                   "sequencer": "F7E6D75DC742D108",
+                   "metadata":[],
+                   "tags":[]
+               }
+           },
+           "eventId":"",
+           "opaqueData":"me@example.com"
+       }
+   ]}
+
+- awsRegion: The zonegroup.
+- eventTime: The timestamp, indicating when the event was triggered.
+- eventName: For the list of supported events see: `S3 Notification
+  Compatibility`_. Note that eventName values do not start with the `s3:`
+  prefix.
+- userIdentity.principalId: The user that triggered the change.
+- requestParameters.sourceIPAddress: not supported
+- responseElements.x-amz-request-id: The request ID of the original change.
+- responseElements.x_amz_id_2: The RGW on which the change was made.
+- s3.configurationId: The notification ID that created the event.
+- s3.bucket.name: The name of the bucket.
+- s3.bucket.ownerIdentity.principalId: The owner of the bucket.
+- s3.bucket.arn: The ARN of the bucket.
+- s3.bucket.id: The ID of the bucket. (This is an extension to the S3
+  notification API.)
+- s3.object.key: The object key.
+- s3.object.size: The object size.
+- s3.object.eTag: The object etag.
+- s3.object.versionId: The object version, if the bucket is versioned. When a
+  copy is made, it includes the version of the target object. When a delete
+  marker is created, it includes the version of the delete marker.
+- s3.object.sequencer: The monotonically-increasing identifier of the "change
+  per object" (hexadecimal format).
+- s3.object.metadata: Any metadata set on the object that is sent as
+  ``x-amz-meta-`` (that is, any metadata set on the object that is sent as an
+  extension to the S3 notification API).
+- s3.object.tags: Any tags set on the object. (This is an extension to the S3
+  notification API.)
+- s3.eventId: The unique ID of the event, which could be used for acking. (This
+  is an extension to the S3 notification API.)
+- s3.opaqueData: This means that "opaque data" is set in the topic configuration
+  and is added to all notifications triggered by the topic. (This is an
+  extension to the S3 notification API.)
+
+.. _PubSub Module : ../pubsub-module
+.. _S3 Notification Compatibility: ../s3-notification-compatibility
+.. _AWS Create Topic: https://docs.aws.amazon.com/sns/latest/api/API_CreateTopic.html
+.. _Bucket Operations: ../s3/bucketops
+.. _S3 CloudEvents Spec: https://github.com/cloudevents/spec/blob/main/cloudevents/adapters/aws-s3.md
diff --git a/doc/radosgw/oidc.rst b/doc/radosgw/oidc.rst
new file mode 100644
index 000000000..46593f1d8
--- /dev/null
+++ b/doc/radosgw/oidc.rst
@@ -0,0 +1,97 @@
+===============================
+ OpenID Connect Provider in RGW
+===============================
+
+An entity describing the OpenID Connect Provider needs to be created in RGW, in order to establish trust between the two.
+
+REST APIs for Manipulating an OpenID Connect Provider
+=====================================================
+
+The following REST APIs can be used for creating and managing an OpenID Connect Provider entity in RGW.
+
+In order to invoke the REST admin APIs, a user with admin caps needs to be created.
+
+.. code-block:: javascript
+
+  radosgw-admin --uid TESTER --display-name "TestUser" --access_key TESTER --secret test123 user create
+  radosgw-admin caps add --uid="TESTER" --caps="oidc-provider=*"
+
+
+CreateOpenIDConnectProvider
+---------------------------------
+
+Create an OpenID Connect Provider entity in RGW
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``ClientIDList.member.N``
+
+:Description: List of Client Ids that needs access to S3 resources.
+:Type: Array of Strings
+
+``ThumbprintList.member.N``
+
+:Description: List of OpenID Connect IDP's server certificates' thumbprints. A maximum of 5 thumbprints are allowed.
+:Type: Array of Strings
+
+``Url``
+
+:Description: URL of the IDP.
+:Type: String
+
+
+Example::
+  POST "<hostname>?Action=Action=CreateOpenIDConnectProvider
+    &ThumbprintList.list.1=F7D7B3515DD0D319DD219A43A9EA727AD6065287
+    &ClientIDList.list.1=app-profile-jsp
+    &Url=http://localhost:8080/auth/realms/quickstart
+
+
+DeleteOpenIDConnectProvider
+---------------------------
+
+Deletes an OpenID Connect Provider entity in RGW
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``OpenIDConnectProviderArn``
+
+:Description: ARN of the IDP which is returned by the Create API.
+:Type: String
+
+Example::
+  POST "<hostname>?Action=Action=DeleteOpenIDConnectProvider
+    &OpenIDConnectProviderArn=arn:aws:iam:::oidc-provider/localhost:8080/auth/realms/quickstart
+
+
+GetOpenIDConnectProvider
+---------------------------
+
+Gets information about an IDP.
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``OpenIDConnectProviderArn``
+
+:Description: ARN of the IDP which is returned by the Create API.
+:Type: String
+
+Example::
+  POST "<hostname>?Action=Action=GetOpenIDConnectProvider
+    &OpenIDConnectProviderArn=arn:aws:iam:::oidc-provider/localhost:8080/auth/realms/quickstart
+
+ListOpenIDConnectProviders
+--------------------------
+
+Lists information about all IDPs
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+None
+
+Example::
+  POST "<hostname>?Action=Action=ListOpenIDConnectProviders
diff --git a/doc/radosgw/opa.rst b/doc/radosgw/opa.rst
new file mode 100644
index 000000000..f1b76b5ef
--- /dev/null
+++ b/doc/radosgw/opa.rst
@@ -0,0 +1,72 @@
+==============================
+Open Policy Agent Integration
+==============================
+
+Open Policy Agent (OPA) is a lightweight general-purpose policy engine
+that can be co-located with a service. OPA can be integrated as a
+sidecar, host-level daemon, or library.
+
+Services can offload policy decisions to OPA by executing queries. Hence,
+policy enforcement can be decoupled from policy decisions.
+
+Configure OPA
+=============
+
+To configure OPA, load custom policies into OPA that control the resources users
+are allowed to access. Relevant data or context can also be loaded into OPA to make decisions.
+
+Policies and data can be loaded into OPA in the following ways::
+  * OPA's RESTful APIs
+  * OPA's *bundle* feature that downloads policies and data from remote HTTP servers
+  * Filesystem
+
+Configure the Ceph Object Gateway
+=================================
+
+The following configuration options are available for OPA integration::
+
+     rgw use opa authz = {use opa server to authorize client requests}
+     rgw opa url = {opa server url:opa server port}
+     rgw opa token = {opa bearer token}
+     rgw opa verify ssl = {verify opa server ssl certificate}
+
+How does the RGW-OPA integration work
+=====================================
+
+After a user is authenticated, OPA can be used to check if the user is authorized
+to perform the given action on the resource. OPA responds with an allow or deny
+decision which is sent back to the RGW which enforces the decision.
+
+Example request::
+
+   POST /v1/data/ceph/authz HTTP/1.1
+   Host: opa.example.com:8181
+   Content-Type: application/json
+   
+   {
+       "input": {
+           "method": "GET",
+           "subuser": "subuser",
+           "user_info": {
+               "user_id": "john",
+               "display_name": "John"  
+           },
+           "bucket_info": {
+               "bucket": {
+                   "name": "Testbucket",
+                   "bucket_id": "testbucket" 
+               },
+               "owner": "john" 
+           }             
+       }
+   }
+
+Response::
+
+   {"result": true}
+
+The above is a sample request sent to OPA which contains information about the
+user, resource and the action to be performed on the resource. Based on the polices
+and data loaded into OPA, it will verify whether the request should be allowed or denied.
+In the sample request, RGW makes a POST request to the endpoint */v1/data/ceph/authz*,
+where *ceph* is the package name and *authz* is the rule name.
diff --git a/doc/radosgw/orphans.rst b/doc/radosgw/orphans.rst
new file mode 100644
index 000000000..9a77d60de
--- /dev/null
+++ b/doc/radosgw/orphans.rst
@@ -0,0 +1,115 @@
+==================================
+Orphan List and Associated Tooling
+==================================
+
+.. version added:: Luminous
+
+.. contents::
+
+Orphans are RADOS objects that are left behind after their associated
+RGW objects are removed. Normally these RADOS objects are removed
+automatically, either immediately or through a process known as
+"garbage collection". Over the history of RGW, however, there may have
+been bugs that prevented these RADOS objects from being deleted, and
+these RADOS objects may be consuming space on the Ceph cluster without
+being of any use. From the perspective of RGW, we call such RADOS
+objects "orphans".
+
+Orphans Find -- DEPRECATED
+--------------------------
+
+The `radosgw-admin` tool has/had three subcommands to help manage
+orphans, however these subcommands are (or will soon be)
+deprecated. These subcommands are:
+
+::
+   # radosgw-admin orphans find ...
+   # radosgw-admin orphans finish ...
+   # radosgw-admin orphans list-jobs ...
+
+There are two key problems with these subcommands, however. First,
+these subcommands have not been actively maintained and therefore have
+not tracked RGW as it has evolved in terms of features and updates. As
+a result the confidence that these subcommands can accurately identify
+true orphans is presently low.
+
+Second, these subcommands store intermediate results on the cluster
+itself. This can be problematic when cluster administrators are
+confronting insufficient storage space and want to remove orphans as a
+means of addressing the issue. The intermediate results could strain
+the existing cluster storage capacity even further.
+
+For these reasons "orphans find" has been deprecated.
+
+Orphan List
+-----------
+
+Because "orphans find" has been deprecated, RGW now includes an
+additional tool -- 'rgw-orphan-list'. When run it will list the
+available pools and prompt the user to enter the name of the data
+pool. At that point the tool will, perhaps after an extended period of
+time, produce a local file containing the RADOS objects from the
+designated pool that appear to be orphans. The administrator is free
+to examine this file and the decide on a course of action, perhaps
+removing those RADOS objects from the designated pool.
+
+All intermediate results are stored on the local file system rather
+than the Ceph cluster. So running the 'rgw-orphan-list' tool should
+have no appreciable impact on the amount of cluster storage consumed.
+
+WARNING: Experimental Status
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The 'rgw-orphan-list' tool is new and therefore currently considered
+experimental. The list of orphans produced should be "sanity checked"
+before being used for a large delete operation.
+
+WARNING: Specifying a Data Pool
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If a pool other than an RGW data pool is specified, the results of the
+tool will be erroneous. All RADOS objects found on such a pool will
+falsely be designated as orphans.
+
+WARNING: Unindexed Buckets
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+RGW allows for unindexed buckets, that is buckets that do not maintain
+an index of their contents. This is not a typical configuration, but
+it is supported. Because the 'rgw-orphan-list' tool uses the bucket
+indices to determine what RADOS objects should exist, objects in the
+unindexed buckets will falsely be listed as orphans.
+
+
+RADOS List
+----------
+
+One of the sub-steps in computing a list of orphans is to map each RGW
+object into its corresponding set of RADOS objects. This is done using
+a subcommand of 'radosgw-admin'.
+
+::
+   # radosgw-admin bucket radoslist [--bucket={bucket-name}]
+
+The subcommand will produce a list of RADOS objects that support all
+of the RGW objects. If a bucket is specified then the subcommand will
+only produce a list of RADOS objects that correspond back the RGW
+objects in the specified bucket.
+
+Note: Shared Bucket Markers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Some administrators will be aware of the coding schemes used to name
+the RADOS objects that correspond to RGW objects, which include a
+"marker" unique to a given bucket.
+
+RADOS objects that correspond with the contents of one RGW bucket,
+however, may contain a marker that specifies a different bucket. This
+behavior is a consequence of the "shallow copy" optimization used by
+RGW. When larger objects are copied from bucket to bucket, only the
+"head" objects are actually copied, and the tail objects are
+shared. Those shared objects will contain the marker of the original
+bucket.
+
+.. _Data Layout in RADOS : ../layout
+.. _Pool Placement and Storage Classes : ../placement
diff --git a/doc/radosgw/placement.rst b/doc/radosgw/placement.rst
new file mode 100644
index 000000000..4931cc87d
--- /dev/null
+++ b/doc/radosgw/placement.rst
@@ -0,0 +1,250 @@
+==================================
+Pool Placement and Storage Classes
+==================================
+
+.. contents::
+
+Placement Targets
+=================
+
+.. versionadded:: Jewel
+
+Placement targets control which `Pools`_ are associated with a particular
+bucket. A bucket's placement target is selected on creation, and cannot be
+modified. The ``radosgw-admin bucket stats`` command will display its
+``placement_rule``.
+
+The zonegroup configuration contains a list of placement targets with an
+initial target named ``default-placement``. The zone configuration then maps
+each zonegroup placement target name onto its local storage. This zone
+placement information includes the ``index_pool`` name for the bucket index,
+the ``data_extra_pool`` name for metadata about incomplete multipart uploads,
+and a ``data_pool`` name for each storage class.
+
+.. _storage_classes:
+
+Storage Classes
+===============
+
+.. versionadded:: Nautilus
+
+Storage classes are used to customize the placement of object data. S3 Bucket
+Lifecycle rules can automate the transition of objects between storage classes.
+
+Storage classes are defined in terms of placement targets. Each zonegroup
+placement target lists its available storage classes with an initial class
+named ``STANDARD``. The zone configuration is responsible for providing a
+``data_pool`` pool name for each of the zonegroup's storage classes.
+
+Zonegroup/Zone Configuration
+============================
+
+Placement configuration is performed with ``radosgw-admin`` commands on
+the zonegroups and zones.
+
+The zonegroup placement configuration can be queried with:
+
+::
+
+  $ radosgw-admin zonegroup get
+  {
+      "id": "ab01123f-e0df-4f29-9d71-b44888d67cd5",
+      "name": "default",
+      "api_name": "default",
+      ...
+      "placement_targets": [
+          {
+              "name": "default-placement",
+              "tags": [],
+              "storage_classes": [
+                  "STANDARD"
+              ]
+          }
+      ],
+      "default_placement": "default-placement",
+      ...
+  }
+
+The zone placement configuration can be queried with:
+
+::
+
+  $ radosgw-admin zone get
+  {
+      "id": "557cdcee-3aae-4e9e-85c7-2f86f5eddb1f",
+      "name": "default",
+      "domain_root": "default.rgw.meta:root",
+      ...
+      "placement_pools": [
+          {
+              "key": "default-placement",
+              "val": {
+                  "index_pool": "default.rgw.buckets.index",
+                  "storage_classes": {
+                      "STANDARD": {
+                          "data_pool": "default.rgw.buckets.data"
+                      }
+                  },
+                  "data_extra_pool": "default.rgw.buckets.non-ec",
+                  "index_type": 0
+              }
+          }
+      ],
+      ...
+  }
+
+.. note:: If you have not done any previous `Multisite Configuration`_,
+          a ``default`` zone and zonegroup are created for you, and changes
+          to the zone/zonegroup will not take effect until the Ceph Object
+          Gateways are restarted. If you have created a realm for multisite,
+          the zone/zonegroup changes will take effect once the changes are
+          committed with ``radosgw-admin period update --commit``.
+
+Adding a Placement Target
+-------------------------
+
+To create a new placement target named ``temporary``, start by adding it to
+the zonegroup:
+
+::
+
+  $ radosgw-admin zonegroup placement add \
+        --rgw-zonegroup default \
+        --placement-id temporary
+
+Then provide the zone placement info for that target:
+
+::
+
+  $ radosgw-admin zone placement add \
+        --rgw-zone default \
+        --placement-id temporary \
+        --data-pool default.rgw.temporary.data \
+        --index-pool default.rgw.temporary.index \
+        --data-extra-pool default.rgw.temporary.non-ec
+
+Adding a Storage Class
+----------------------
+
+To add a new storage class named ``GLACIER`` to the ``default-placement`` target,
+start by adding it to the zonegroup:
+
+::
+
+  $ radosgw-admin zonegroup placement add \
+        --rgw-zonegroup default \
+        --placement-id default-placement \
+        --storage-class GLACIER
+
+Then provide the zone placement info for that storage class:
+
+::
+
+  $ radosgw-admin zone placement add \
+        --rgw-zone default \
+        --placement-id default-placement \
+        --storage-class GLACIER \
+        --data-pool default.rgw.glacier.data \
+        --compression lz4
+
+Customizing Placement
+=====================
+
+Default Placement
+-----------------
+
+By default, new buckets will use the zonegroup's ``default_placement`` target.
+This zonegroup setting can be changed with:
+
+::
+
+  $ radosgw-admin zonegroup placement default \
+        --rgw-zonegroup default \
+        --placement-id new-placement
+
+User Placement
+--------------
+
+A Ceph Object Gateway user can override the zonegroup's default placement
+target by setting a non-empty ``default_placement`` field in the user info.
+Similarly, the ``default_storage_class`` can override the ``STANDARD``
+storage class applied to objects by default.
+
+::
+
+  $ radosgw-admin user info --uid testid
+  {
+      ...
+      "default_placement": "",
+      "default_storage_class": "",
+      "placement_tags": [],
+      ...
+  }
+
+If a zonegroup's placement target contains any ``tags``, users will be unable
+to create buckets with that placement target unless their user info contains
+at least one matching tag in its ``placement_tags`` field. This can be useful
+to restrict access to certain types of storage.
+
+The ``radosgw-admin`` command can modify these fields directly with:
+
+::
+
+  $ radosgw-admin user modify \
+        --uid <user-id> \
+        --placement-id <default-placement-id> \
+        --storage-class <default-storage-class> \
+        --tags <tag1,tag2>
+
+.. _s3_bucket_placement:
+
+S3 Bucket Placement
+-------------------
+
+When creating a bucket with the S3 protocol, a placement target can be
+provided as part of the LocationConstraint to override the default placement
+targets from the user and zonegroup.
+
+Normally, the LocationConstraint must match the zonegroup's ``api_name``:
+
+::
+
+  <LocationConstraint>default</LocationConstraint>
+
+A custom placement target can be added to the ``api_name`` following a colon:
+
+::
+
+  <LocationConstraint>default:new-placement</LocationConstraint>
+
+Swift Bucket Placement
+----------------------
+
+When creating a bucket with the Swift protocol, a placement target can be
+provided in the HTTP header ``X-Storage-Policy``:
+
+::
+
+  X-Storage-Policy: new-placement
+
+Using Storage Classes
+=====================
+
+All placement targets have a ``STANDARD`` storage class which is applied to
+new objects by default. The user can override this default with its
+``default_storage_class``.
+
+To create an object in a non-default storage class, provide that storage class
+name in an HTTP header with the request. The S3 protocol uses the
+``X-Amz-Storage-Class`` header, while the Swift protocol uses the
+``X-Object-Storage-Class`` header.
+
+When using AWS S3 SDKs such as python boto3, it is important that the non-default
+storage class will be called as one on of the AWS S3 allowed storage classes, or else the SDK
+will drop the request and raise an exception.
+
+S3 Object Lifecycle Management can then be used to move object data between
+storage classes using ``Transition`` actions.
+
+.. _`Pools`: ../pools
+.. _`Multisite Configuration`: ../multisite
diff --git a/doc/radosgw/pools.rst b/doc/radosgw/pools.rst
new file mode 100644
index 000000000..bb1246c1f
--- /dev/null
+++ b/doc/radosgw/pools.rst
@@ -0,0 +1,57 @@
+=====
+Pools
+=====
+
+The Ceph Object Gateway uses several pools for its various storage needs,
+which are listed in the Zone object (see ``radosgw-admin zone get``). A
+single zone named ``default`` is created automatically with pool names
+starting with ``default.rgw.``, but a `Multisite Configuration`_ will have
+multiple zones.
+
+Tuning
+======
+
+When ``radosgw`` first tries to operate on a zone pool that does not
+exist, it will create that pool with the default values from
+``osd pool default pg num`` and ``osd pool default pgp num``. These defaults
+are sufficient for some pools, but others (especially those listed in
+``placement_pools`` for the bucket index and data) will require additional
+tuning. We recommend using the `Ceph Placement Group’s per Pool
+Calculator <https://old.ceph.com/pgcalc/>`__ to calculate a suitable number of
+placement groups for these pools. See
+`Pools <http://docs.ceph.com/en/latest/rados/operations/pools/#pools>`__
+for details on pool creation.
+
+.. _radosgw-pool-namespaces:
+
+Pool Namespaces
+===============
+
+.. versionadded:: Luminous
+
+Pool names particular to a zone follow the naming convention
+``{zone-name}.pool-name``. For example, a zone named ``us-east`` will
+have the following pools:
+
+-  ``.rgw.root``
+
+-  ``us-east.rgw.control``
+
+-  ``us-east.rgw.meta``
+
+-  ``us-east.rgw.log``
+
+-  ``us-east.rgw.buckets.index``
+
+-  ``us-east.rgw.buckets.data``
+
+The zone definitions list several more pools than that, but many of those
+are consolidated through the use of rados namespaces. For example, all of
+the following pool entries use namespaces of the ``us-east.rgw.meta`` pool::
+
+    "user_keys_pool": "us-east.rgw.meta:users.keys",
+    "user_email_pool": "us-east.rgw.meta:users.email",
+    "user_swift_pool": "us-east.rgw.meta:users.swift",
+    "user_uid_pool": "us-east.rgw.meta:users.uid",
+
+.. _`Multisite Configuration`: ../multisite
diff --git a/doc/radosgw/pubsub-module.rst b/doc/radosgw/pubsub-module.rst
new file mode 100644
index 000000000..8a45f4105
--- /dev/null
+++ b/doc/radosgw/pubsub-module.rst
@@ -0,0 +1,646 @@
+================== 
+PubSub Sync Module
+==================
+
+.. versionadded:: Nautilus
+
+.. contents::
+
+This sync module provides a publish and subscribe mechanism for the object store modification
+events. Events are published into predefined topics. Topics can be subscribed to, and events
+can be pulled from them. Events need to be acked. Also, events will expire and disappear
+after a period of time. 
+
+A push notification mechanism exists too, currently supporting HTTP,
+AMQP0.9.1 and Kafka endpoints. In this case, the events are pushed to an endpoint on top of storing them in Ceph. If events should only be pushed to an endpoint
+and do not need to be stored in Ceph, the `Bucket Notification`_ mechanism should be used instead of pubsub sync module. 
+
+A user can create different topics. A topic entity is defined by its name and is per tenant. A
+user can only associate its topics (via notification configuration) with buckets it owns.
+
+In order to publish events for specific bucket a notification entity needs to be created. A
+notification can be created on a subset of event types, or for all event types (default).
+There can be multiple notifications for any specific topic, and the same topic could be used for multiple notifications.
+
+A subscription to a topic can also be defined. There can be multiple subscriptions for any
+specific topic.
+
+REST API has been defined to provide configuration and control interfaces for the pubsub
+mechanisms. This API has two flavors, one is S3-compatible and one is not. The two flavors can be used
+together, although it is recommended to use the S3-compatible one. 
+The S3-compatible API is similar to the one used in the bucket notification mechanism.
+
+Events are stored as RGW objects in a special bucket, under a special user (pubsub control user). Events cannot
+be accessed directly, but need to be pulled and acked using the new REST API.
+
+.. toctree::
+   :maxdepth: 1
+
+   S3 Bucket Notification Compatibility <s3-notification-compatibility>
+
+.. note:: To enable bucket notifications API, the `rgw_enable_apis` configuration parameter should contain: "notifications".
+
+PubSub Zone Configuration
+-------------------------
+
+The pubsub sync module requires the creation of a new zone in a :ref:`multisite` environment...
+First, a master zone must exist (see: :ref:`master-zone-label`), 
+then a secondary zone should be created (see :ref:`secondary-zone-label`).
+In the creation of the secondary zone, its tier type must be set to ``pubsub``:
+
+::
+
+   # radosgw-admin zone create --rgw-zonegroup={zone-group-name} \
+                               --rgw-zone={zone-name} \
+                               --endpoints={http://fqdn}[,{http://fqdn}] \
+                               --sync-from-all=0 \
+                               --sync-from={master-zone-name} \
+                               --tier-type=pubsub
+
+
+PubSub Zone Configuration Parameters
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+                                
+::
+
+   {
+       "tenant": <tenant>,             # default: <empty>
+       "uid": <uid>,                   # default: "pubsub"
+       "data_bucket_prefix": <prefix>  # default: "pubsub-"
+       "data_oid_prefix": <prefix>     #
+       "events_retention_days": <days> # default: 7
+   }
+
+* ``tenant`` (string)
+
+The tenant of the pubsub control user.
+
+* ``uid`` (string)
+
+The uid of the pubsub control user.
+
+* ``data_bucket_prefix`` (string)
+
+The prefix of the bucket name that will be created to store events for specific topic.
+
+* ``data_oid_prefix`` (string)
+
+The oid prefix for the stored events.
+
+* ``events_retention_days`` (integer)
+
+How many days to keep events that weren't acked.
+
+Configuring Parameters via CLI
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The tier configuration could be set using the following command:
+
+::
+
+   # radosgw-admin zone modify --rgw-zonegroup={zone-group-name} \
+                                --rgw-zone={zone-name} \
+                                --tier-config={key}={val}[,{key}={val}]
+
+Where the ``key`` in the configuration specifies the configuration variable that needs to be updated (from the list above), and
+the ``val`` specifies its new value. For example, setting the pubsub control user ``uid`` to ``user_ps``:
+
+::
+
+   # radosgw-admin zone modify --rgw-zonegroup={zone-group-name} \
+                                --rgw-zone={zone-name} \
+                                --tier-config=uid=pubsub
+
+A configuration field can be removed by using ``--tier-config-rm={key}``.
+
+
+Topic and Subscription Management via CLI
+-----------------------------------------
+
+Configuration of all topics, associated with a tenant, could be fetched using the following command:
+   
+::
+   
+   # radosgw-admin topic list [--tenant={tenant}]
+
+
+Configuration of a specific topic could be fetched using:
+
+::
+   
+   # radosgw-admin topic get --topic={topic-name} [--tenant={tenant}]
+
+
+And removed using:
+
+::
+   
+   # radosgw-admin topic rm --topic={topic-name} [--tenant={tenant}]
+
+
+Configuration of a subscription could be fetched using:
+
+::
+   
+   # radosgw-admin subscription get --subscription={topic-name} [--tenant={tenant}]
+
+And removed using:
+
+::
+   
+   # radosgw-admin subscription rm --subscription={topic-name} [--tenant={tenant}]
+
+
+To fetch all of the events stored in a subcription, use:
+
+::
+   
+   # radosgw-admin subscription pull --subscription={topic-name} [--marker={last-marker}] [--tenant={tenant}]
+
+
+To ack (and remove) an event from a subscription, use:
+
+::
+   
+   # radosgw-admin subscription ack --subscription={topic-name} --event-id={event-id} [--tenant={tenant}]
+
+
+PubSub Performance Stats
+-------------------------
+Same counters are shared between the pubsub sync module and the notification mechanism.
+
+- ``pubsub_event_triggered``: running counter of events with at lease one topic associated with them
+- ``pubsub_event_lost``: running counter of events that had topics and subscriptions associated with them but that were not stored or pushed to any of the subscriptions
+- ``pubsub_store_ok``: running counter, for all subscriptions, of stored events 
+- ``pubsub_store_fail``: running counter, for all subscriptions, of events failed to be stored
+- ``pubsub_push_ok``: running counter, for all subscriptions, of events successfully pushed to their endpoint
+- ``pubsub_push_fail``: running counter, for all subscriptions, of events failed to be pushed to their endpoint
+- ``pubsub_push_pending``: gauge value of events pushed to an endpoint but not acked or nacked yet
+
+.. note:: 
+
+    ``pubsub_event_triggered`` and ``pubsub_event_lost`` are incremented per event, while: 
+    ``pubsub_store_ok``, ``pubsub_store_fail``, ``pubsub_push_ok``, ``pubsub_push_fail``, are incremented per store/push action on each subscriptions.
+
+PubSub REST API
+---------------
+
+.. tip:: PubSub REST calls, and only them, should be sent to an RGW which belong to a PubSub zone
+
+Topics
+~~~~~~
+ 
+.. _radosgw-create-a-topic:
+
+Create a Topic
+``````````````
+
+This will create a new topic. Topic creation is needed both for both flavors of the API.
+Optionally the topic could be provided with push endpoint parameters that would be used later
+when an S3-compatible notification is created.
+Upon successful request, the response will include the topic ARN that could be later used to reference this topic in an S3-compatible notification request. 
+To update a topic, use the same command used for topic creation, with the topic name of an existing topic and different endpoint values.
+
+.. tip:: Any S3-compatible notification already associated with the topic needs to be re-created for the topic update to take effect 
+
+::
+
+   PUT /topics/<topic-name>[?OpaqueData=<opaque data>][&push-endpoint=<endpoint>[&amqp-exchange=<exchange>][&amqp-ack-level=none|broker|routable][&verify-ssl=true|false][&kafka-ack-level=none|broker][&use-ssl=true|false][&ca-location=<file path>]]
+
+Request parameters:
+
+- push-endpoint: URI of an endpoint to send push notification to
+- OpaqueData: opaque data is set in the topic configuration and added to all notifications triggered by the ropic
+
+The endpoint URI may include parameters depending with the type of endpoint:
+
+- HTTP endpoint 
+
+ - URI: ``http[s]://<fqdn>[:<port]``
+ - port defaults to: 80/443 for HTTP/S accordingly
+ - verify-ssl: indicate whether the server certificate is validated by the client or not ("true" by default)
+
+- AMQP0.9.1 endpoint
+
+ - URI: ``amqp[s]://[<user>:<password>@]<fqdn>[:<port>][/<vhost>]``
+ - user/password defaults to: guest/guest
+ - user/password may only be provided over HTTPS. Topic creation request will be rejected if not
+ - port defaults to: 5672/5671 for unencrypted/SSL-encrypted connections
+ - vhost defaults to: "/"
+ - verify-ssl: indicate whether the server certificate is validated by the client or not ("true" by default)
+ - if ``ca-location`` is provided, and secure connection is used, the specified CA will be used, instead of the default one, to authenticate the broker
+ - amqp-exchange: the exchanges must exist and be able to route messages based on topics (mandatory parameter for AMQP0.9.1). Different topics pointing to the same endpoint must use the same exchange
+ - amqp-ack-level: no end2end acking is required, as messages may persist in the broker before delivered into their final destination. Three ack methods exist:
+
+  - "none": message is considered "delivered" if sent to broker
+  - "broker": message is considered "delivered" if acked by broker (default)
+  - "routable": message is considered "delivered" if broker can route to a consumer
+
+.. tip:: The topic-name (see :ref:`radosgw-create-a-topic`) is used for the AMQP topic ("routing key" for a topic exchange)
+
+- Kafka endpoint 
+
+ - URI: ``kafka://[<user>:<password>@]<fqdn>[:<port]``
+ - if ``use-ssl`` is set to "true", secure connection will be used for connecting with the broker ("false" by default)
+ - if ``ca-location`` is provided, and secure connection is used, the specified CA will be used, instead of the default one, to authenticate the broker
+ - user/password may only be provided over HTTPS. Topic creation request will be rejected if not
+ - user/password may only be provided together with ``use-ssl``, connection to the broker would fail if not
+ - port defaults to: 9092
+ - kafka-ack-level: no end2end acking is required, as messages may persist in the broker before delivered into their final destination. Two ack methods exist:
+
+  - "none": message is considered "delivered" if sent to broker
+  - "broker": message is considered "delivered" if acked by broker (default)
+
+The topic ARN in the response will have the following format:
+
+::
+
+   arn:aws:sns:<zone-group>:<tenant>:<topic>
+
+Get Topic Information
+`````````````````````
+
+Returns information about specific topic. This includes subscriptions to that topic, and push-endpoint information, if provided.
+
+::
+
+   GET /topics/<topic-name>
+
+Response will have the following format (JSON):
+
+::
+
+   {
+       "topic":{
+           "user":"",
+           "name":"",
+           "dest":{
+               "bucket_name":"",
+               "oid_prefix":"",
+               "push_endpoint":"",
+               "push_endpoint_args":"",
+               "push_endpoint_topic":"",
+               "stored_secret":"",
+               "persistent":""
+           },
+           "arn":""
+           "opaqueData":""
+       },
+       "subs":[]
+   }             
+
+- topic.user: name of the user that created the topic
+- name: name of the topic
+- dest.bucket_name: not used
+- dest.oid_prefix: not used
+- dest.push_endpoint: in case of S3-compliant notifications, this value will be used as the push-endpoint URL
+- if push-endpoint URL contain user/password information, request must be made over HTTPS. Topic get request will be rejected if not 
+- dest.push_endpoint_args: in case of S3-compliant notifications, this value will be used as the push-endpoint args
+- dest.push_endpoint_topic: in case of S3-compliant notifications, this value will hold the topic name as sent to the endpoint (may be different than the internal topic name)
+- topic.arn: topic ARN
+- subs: list of subscriptions associated with this topic
+
+Delete Topic
+````````````
+
+::
+
+   DELETE /topics/<topic-name>
+
+Delete the specified topic.
+
+List Topics
+```````````
+
+List all topics associated with a tenant.
+
+::
+
+   GET /topics
+ 
+- if push-endpoint URL contain user/password information, in any of the topic, request must be made over HTTPS. Topic list request will be rejected if not 
+
+S3-Compliant Notifications
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Detailed under: `Bucket Operations`_.
+
+.. note:: 
+
+    - Notification creation will also create a subscription for pushing/pulling events
+    - The generated subscription's name will have the same as the notification Id, and could be used later to fetch and ack events with the subscription API.
+    - Notification deletion will deletes all generated subscriptions
+    - In case that bucket deletion implicitly deletes the notification, 
+      the associated subscription will not be deleted automatically (any events of the deleted bucket could still be access),
+      and will have to be deleted explicitly with the subscription deletion API
+    - Filtering based on metadata (which is an extension to S3) is not supported, and such rules will be ignored
+    - Filtering based on tags (which is an extension to S3) is not supported, and such rules will be ignored
+
+
+Non S3-Compliant Notifications
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Create a Notification
+`````````````````````
+
+This will create a publisher for a specific bucket into a topic.
+
+::
+
+   PUT /notifications/bucket/<bucket>?topic=<topic-name>[&events=<event>[,<event>]]
+
+Request parameters:
+
+- topic-name: name of topic
+- event: event type (string), one of: ``OBJECT_CREATE``, ``OBJECT_DELETE``, ``DELETE_MARKER_CREATE``
+ 
+Delete Notification Information
+```````````````````````````````
+
+Delete publisher from a specific bucket into a specific topic.
+
+::
+
+   DELETE /notifications/bucket/<bucket>?topic=<topic-name>
+
+Request parameters:
+
+- topic-name: name of topic
+
+.. note:: When the bucket is deleted, any notification defined on it is also deleted
+
+List Notifications
+``````````````````
+
+List all topics with associated events defined on a bucket.
+
+::
+
+   GET /notifications/bucket/<bucket>
+
+Response will have the following format (JSON):
+
+::
+
+   {"topics":[
+      {
+         "topic":{
+            "user":"",
+            "name":"",
+            "dest":{
+               "bucket_name":"",
+               "oid_prefix":"",
+               "push_endpoint":"",
+               "push_endpoint_args":"",
+               "push_endpoint_topic":""
+            }
+            "arn":""
+         },
+         "events":[]
+      }
+    ]}            
+
+Subscriptions
+~~~~~~~~~~~~~
+
+Create a Subscription
+`````````````````````
+
+Creates a new subscription.
+
+::
+
+   PUT /subscriptions/<sub-name>?topic=<topic-name>[?push-endpoint=<endpoint>[&amqp-exchange=<exchange>][&amqp-ack-level=none|broker|routable][&verify-ssl=true|false][&kafka-ack-level=none|broker][&ca-location=<file path>]]
+
+Request parameters:
+
+- topic-name: name of topic
+- push-endpoint: URI of endpoint to send push notification to
+
+The endpoint URI may include parameters depending with the type of endpoint:
+
+- HTTP endpoint 
+
+ - URI: ``http[s]://<fqdn>[:<port]``
+ - port defaults to: 80/443 for HTTP/S accordingly
+ - verify-ssl: indicate whether the server certificate is validated by the client or not ("true" by default)
+
+- AMQP0.9.1 endpoint
+
+ - URI: ``amqp://[<user>:<password>@]<fqdn>[:<port>][/<vhost>]``
+ - user/password defaults to : guest/guest
+ - port defaults to: 5672
+ - vhost defaults to: "/"
+ - amqp-exchange: the exchanges must exist and be able to route messages based on topics (mandatory parameter for AMQP0.9.1)
+ - amqp-ack-level: no end2end acking is required, as messages may persist in the broker before delivered into their final destination. Three ack methods exist:
+
+  - "none": message is considered "delivered" if sent to broker
+  - "broker": message is considered "delivered" if acked by broker (default)
+  - "routable": message is considered "delivered" if broker can route to a consumer
+
+- Kafka endpoint 
+
+ - URI: ``kafka://[<user>:<password>@]<fqdn>[:<port]``
+ - if ``ca-location`` is provided, secure connection will be used for connection with the broker
+ - user/password may only be provided over HTTPS. Topic creation request will be rejected if not
+ - user/password may only be provided together with ``ca-location``. Topic creation request will be rejected if not
+ - port defaults to: 9092
+ - kafka-ack-level: no end2end acking is required, as messages may persist in the broker before delivered into their final destination. Two ack methods exist:
+
+  - "none": message is considered "delivered" if sent to broker
+  - "broker": message is considered "delivered" if acked by broker (default)
+
+
+Get Subscription Information
+````````````````````````````
+
+Returns information about specific subscription.
+
+::
+
+   GET /subscriptions/<sub-name>
+
+Response will have the following format (JSON):
+
+::
+
+   {
+       "user":"",
+       "name":"",
+       "topic":"",
+       "dest":{
+           "bucket_name":"",
+           "oid_prefix":"",
+           "push_endpoint":"",
+           "push_endpoint_args":"",
+           "push_endpoint_topic":""
+       }
+       "s3_id":""
+   }             
+
+- user: name of the user that created the subscription
+- name: name of the subscription
+- topic: name of the topic the subscription is associated with
+- dest.bucket_name: name of the bucket storing the events
+- dest.oid_prefix: oid prefix for the events stored in the bucket
+- dest.push_endpoint: in case of S3-compliant notifications, this value will be used as the push-endpoint URL
+- if push-endpoint URL contain user/password information, request must be made over HTTPS. Topic get request will be rejected if not 
+- dest.push_endpoint_args: in case of S3-compliant notifications, this value will be used as the push-endpoint args
+- dest.push_endpoint_topic: in case of S3-compliant notifications, this value will hold the topic name as sent to the endpoint (may be different than the internal topic name)
+- s3_id: in case of S3-compliant notifications, this will hold the notification name that created the subscription
+
+Delete Subscription
+```````````````````
+
+Removes a subscription.
+
+::
+
+   DELETE /subscriptions/<sub-name>
+
+Events
+~~~~~~
+
+Pull Events
+```````````
+
+Pull events sent to a specific subscription.
+
+::
+
+   GET /subscriptions/<sub-name>?events[&max-entries=<max-entries>][&marker=<marker>]
+
+Request parameters:
+
+- marker: pagination marker for list of events, if not specified will start from the oldest
+- max-entries: max number of events to return
+
+The response will hold information on the current marker and whether there are more events not fetched:
+
+::
+
+   {"next_marker":"","is_truncated":"",...}
+
+
+The actual content of the response is depended with how the subscription was created.
+In case that the subscription was created via an S3-compatible notification, 
+the events will have an S3-compatible record format (JSON):
+
+::
+
+   {"Records":[  
+       {
+           "eventVersion":"2.1"
+           "eventSource":"aws:s3",
+           "awsRegion":"",
+           "eventTime":"",
+           "eventName":"",
+           "userIdentity":{  
+               "principalId":""
+           },
+           "requestParameters":{
+               "sourceIPAddress":""
+           },
+           "responseElements":{
+               "x-amz-request-id":"",
+               "x-amz-id-2":""
+           },
+           "s3":{
+               "s3SchemaVersion":"1.0",
+               "configurationId":"",
+               "bucket":{
+                   "name":"",
+                   "ownerIdentity":{
+                       "principalId":""
+                   },
+                   "arn":"",
+                   "id":""
+               },
+               "object":{
+                   "key":"",
+                   "size":"0",
+                   "eTag":"",
+                   "versionId":"",
+                   "sequencer":"",
+                   "metadata":[],
+                   "tags":[]
+               }
+           },
+           "eventId":"",
+           "opaqueData":"",
+       }
+   ]}
+
+- awsRegion: zonegroup
+- eventTime: timestamp indicating when the event was triggered
+- eventName: either ``s3:ObjectCreated:``, or ``s3:ObjectRemoved:``
+- userIdentity: not supported 
+- requestParameters: not supported
+- responseElements: not supported
+- s3.configurationId: notification ID that created the subscription for the event
+- s3.bucket.name: name of the bucket
+- s3.bucket.ownerIdentity.principalId: owner of the bucket
+- s3.bucket.arn: ARN of the bucket
+- s3.bucket.id: Id of the bucket (an extension to the S3 notification API)
+- s3.object.key: object key
+- s3.object.size: not supported
+- s3.object.eTag: object etag
+- s3.object.version: object version in case of versioned bucket
+- s3.object.sequencer: monotonically increasing identifier of the change per object (hexadecimal format)
+- s3.object.metadata: not supported (an extension to the S3 notification API)
+- s3.object.tags: not supported (an extension to the S3 notification API)
+- s3.eventId: unique ID of the event, that could be used for acking (an extension to the S3 notification API)
+- s3.opaqueData: opaque data is set in the topic configuration and added to all notifications triggered by the ropic (an extension to the S3 notification API)
+
+In case that the subscription was not created via a non S3-compatible notification, 
+the events will have the following event format (JSON):
+
+::
+
+    {"events":[
+       {
+           "id":"",
+           "event":"",
+           "timestamp":"",
+           "info":{
+               "attrs":{
+                   "mtime":""
+               },
+               "bucket":{
+                   "bucket_id":"",
+                   "name":"",
+                   "tenant":""
+               },
+               "key":{
+                   "instance":"",
+                   "name":""
+               }
+           }
+       }
+   ]}
+
+- id: unique ID of the event, that could be used for acking
+- event: one of: ``OBJECT_CREATE``, ``OBJECT_DELETE``, ``DELETE_MARKER_CREATE``
+- timestamp: timestamp indicating when the event was sent
+- info.attrs.mtime: timestamp indicating when the event was triggered
+- info.bucket.bucket_id: id of the bucket
+- info.bucket.name: name of the bucket
+- info.bucket.tenant: tenant the bucket belongs to
+- info.key.instance: object version in case of versioned bucket
+- info.key.name: object key
+
+Ack Event
+`````````
+
+Ack event so that it can be removed from the subscription history.
+
+::
+
+   POST /subscriptions/<sub-name>?ack&event-id=<event-id>
+
+Request parameters:
+
+- event-id: id of event to be acked
+
+.. _Bucket Notification : ../notifications
+.. _Bucket Operations: ../s3/bucketops
diff --git a/doc/radosgw/qat-accel.rst b/doc/radosgw/qat-accel.rst
new file mode 100644
index 000000000..4e661c63d
--- /dev/null
+++ b/doc/radosgw/qat-accel.rst
@@ -0,0 +1,94 @@
+===============================================
+QAT Acceleration for Encryption and Compression
+===============================================
+
+Intel QAT (QuickAssist Technology) can provide extended accelerated encryption
+and compression services by offloading the actual encryption and compression
+request(s) to the hardware QuickAssist accelerators, which are more efficient
+in terms of cost and power than general purpose CPUs for those specific
+compute-intensive workloads.
+
+See `QAT Support for Compression`_ and `QAT based Encryption for RGW`_.
+
+
+QAT in the Software Stack 
+=========================
+
+Application developers can access QuickAssist features through the QAT API.
+The QAT API is the top-level API for QuickAssist technology, and enables easy
+interfacing between the customer application and the QuickAssist acceleration
+driver.
+
+The QAT API accesses the QuickAssist driver, which in turn drives the
+QuickAssist Accelerator hardware. The QuickAssist driver is responsible for
+exposing the acceleration services to the application software.
+
+A user can write directly to the QAT API, or the use of QAT can be done via
+frameworks that have been enabled by others including Intel (for example, zlib*,
+OpenSSL* libcrypto*, and the Linux* Kernel Crypto Framework).
+
+QAT Environment Setup
+=====================
+1. QuickAssist Accelerator hardware is necessary to make use of accelerated
+   encryption and compression services. And QAT driver in kernel space have to
+   be loaded to drive the hardware.
+
+The driver package can be downloaded from `Intel Quickassist Technology`_.
+
+2. The implementation for QAT based encryption is directly base on QAT API which
+   is included the driver package. But QAT support for compression depends on
+   QATzip project, which is a user space library which builds on top of the QAT
+   API. Currently, QATzip speeds up gzip compression and decompression at the
+   time of writing.
+
+See `QATzip`_.
+
+Implementation
+==============
+1. QAT based Encryption for RGW 
+
+`OpenSSL support for RGW encryption`_ has been merged into Ceph, and Intel also
+provides one `QAT Engine`_ for OpenSSL. So, theoretically speaking, QAT based
+encryption in Ceph can be directly supported through OpenSSl+QAT Engine.
+
+But the QAT Engine for OpenSSL currently supports chained operations only, and
+so Ceph will not be able to utilize QAT hardware feature for crypto operations
+based on OpenSSL crypto plugin. As a result, one QAT plugin based on native
+QAT API is added into crypto framework.
+
+2. QAT Support for Compression
+
+As mentioned above, QAT support for compression is based on QATzip library in
+user space, which is designed to take full advantage of the performance provided
+by QuickAssist Technology. Unlike QAT based encryption, QAT based compression
+is supported through a tool class for QAT acceleration rather than a compressor
+plugin. The common tool class can transparently accelerate the existing compression
+types, but only zlib compressor can be supported at the time of writing. So
+user is allowed to use it to speed up zlib compressor as long as the QAT
+hardware is available and QAT is capable to handle it.
+
+Configuration
+=============
+1. QAT based Encryption for RGW 
+
+Edit the Ceph configuration file to make use of QAT based crypto plugin::
+
+    plugin crypto accelerator = crypto_qat
+
+2. QAT Support for Compression
+
+One CMake option have to be used to trigger QAT based compression::
+
+    -DWITH_QATZIP=ON
+
+Edit the Ceph configuration file to enable QAT support for compression::
+
+    qat compressor enabled=true
+
+
+.. _QAT Support for Compression: https://github.com/ceph/ceph/pull/19714
+.. _QAT based Encryption for RGW: https://github.com/ceph/ceph/pull/19386
+.. _Intel Quickassist Technology: https://01.org/intel-quickassist-technology
+.. _QATzip: https://github.com/intel/QATzip
+.. _OpenSSL support for RGW encryption: https://github.com/ceph/ceph/pull/15168
+.. _QAT Engine: https://github.com/intel/QAT_Engine
diff --git a/doc/radosgw/rgw-cache.rst b/doc/radosgw/rgw-cache.rst
new file mode 100644
index 000000000..9b4c96f1e
--- /dev/null
+++ b/doc/radosgw/rgw-cache.rst
@@ -0,0 +1,150 @@
+==========================
+RGW Data caching and CDN
+==========================
+
+.. versionadded:: Octopus
+
+.. contents::
+
+This feature adds to RGW the ability to securely cache objects and offload the workload from the cluster, using Nginx.
+After an object is accessed the first time it will be stored in the Nginx cache directory.
+When data is already cached, it need not be fetched from RGW. A permission check will be made against RGW to ensure the requesting user has access.
+This feature is based on some Nginx modules, ngx_http_auth_request_module, https://github.com/kaltura/nginx-aws-auth-module, Openresty for Lua capabilities.
+
+Currently, this feature will cache only AWSv4 requests (only s3 requests), caching-in the output of the 1st GET request
+and caching-out on subsequent GET requests, passing thru transparently PUT,POST,HEAD,DELETE and COPY requests.
+
+
+The feature introduces 2 new APIs: Auth and Cache.
+
+New APIs
+-------------------------
+
+There are 2 new APIs for this feature:
+
+Auth API - The cache uses this to validate that a user can access the cached data
+
+Cache API - Adds the ability to override securely Range header, that way Nginx can use it is own smart cache on top of S3:
+https://www.nginx.com/blog/smart-efficient-byte-range-caching-nginx/
+Using this API gives the ability to read ahead objects when clients asking a specific range from the object.
+On subsequent accesses to the cached object, Nginx will satisfy requests for already-cached ranges from the cache. Uncached ranges will be read from RGW (and cached).
+
+Auth API
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This API Validates a specific authenticated access being made to the cache, using RGW's knowledge of the client credentials and stored access policy.
+Returns success if the encapsulated request would be granted.
+
+Cache API
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This API is meant to allow changing signed Range headers using a privileged user, cache user.
+
+Creating cache user
+
+::
+
+$ radosgw-admin user create --uid=<uid for cache user> --display-name="cache user" --caps="amz-cache=read"
+
+This user can send to the RGW the Cache API header ``X-Amz-Cache``, this header contains the headers from the original request(before changing the Range header).
+It means that ``X-Amz-Cache`` built from several headers.
+The headers that are building the ``X-Amz-Cache`` header are separated by char with ASCII code 177 and the header name and value are separated by char ASCII code 178.
+The RGW will check that the cache user is an authorized user and if it is a cache user,
+if yes it will use the ``X-Amz-Cache`` to revalidate that the user has permissions, using the headers from the X-Amz-Cache.
+During this flow, the RGW will override the Range header.
+
+
+Using Nginx with RGW
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Download the source of Openresty:
+
+::
+
+$ wget https://openresty.org/download/openresty-1.15.8.3.tar.gz
+
+git clone the AWS auth Nginx module:
+
+::
+
+$ git clone https://github.com/kaltura/nginx-aws-auth-module
+
+untar the openresty package:
+
+::
+
+$ tar xvzf openresty-1.15.8.3.tar.gz
+$ cd openresty-1.15.8.3
+
+Compile openresty, Make sure that you have pcre lib and openssl lib:
+
+::
+
+$ sudo yum install pcre-devel openssl-devel gcc curl zlib-devel nginx
+$ ./configure --add-module=<the nginx-aws-auth-module dir> --with-http_auth_request_module --with-http_slice_module --conf-path=/etc/nginx/nginx.conf
+$ gmake -j $(nproc)
+$ sudo gmake install
+$ sudo ln -sf /usr/local/openresty/bin/openresty /usr/bin/nginx
+
+Put in-place your Nginx configuration files and edit them according to your environment:
+
+All Nginx conf files are under: https://github.com/ceph/ceph/tree/master/examples/rgw-cache
+
+`nginx.conf` should go to `/etc/nginx/nginx.conf`
+
+`nginx-lua-file.lua` should go to `/etc/nginx/nginx-lua-file.lua`
+
+`nginx-default.conf` should go to `/etc/nginx/conf.d/nginx-default.conf`
+
+The parameters that are most likely to require adjustment according to the environment are located in the file `nginx-default.conf`
+
+Modify the example values of *proxy_cache_path* and *max_size* at:
+
+::
+
+ proxy_cache_path /data/cache levels=2:2:2 keys_zone=mycache:999m max_size=20G inactive=1d use_temp_path=off;
+
+
+And modify the example *server* values to point to the RGWs URIs:
+
+::
+
+ server rgw1:8000 max_fails=2 fail_timeout=5s;
+ server rgw2:8000 max_fails=2 fail_timeout=5s;
+ server rgw3:8000 max_fails=2 fail_timeout=5s;
+
+| It is important to substitute the *access key* and *secret key* located in the `nginx.conf` with those belong to the user with the `amz-cache` caps
+| for example, create the `cache` user as following:
+
+::
+
+ radosgw-admin user create --uid=cacheuser --display-name="cache user" --caps="amz-cache=read" --access-key <access> --secret <secret>
+
+It is possible to use Nginx slicing which is a better method for streaming purposes.
+
+For using slice you should use `nginx-slicing.conf` and not `nginx-default.conf`
+
+Further information about Nginx slicing:
+
+https://docs.nginx.com/nginx/admin-guide/content-cache/content-caching/#byte-range-caching
+
+
+If you do not want to use the prefetch caching, It is possible to replace `nginx-default.conf` with `nginx-noprefetch.conf`
+Using `noprefetch` means that if the client is sending range request of 0-4095 and then 0-4096 Nginx will cache those requests separately, So it will need to fetch those requests twice.
+
+
+Run Nginx(openresty):
+
+::
+
+$ sudo systemctl restart nginx
+
+Appendix
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+**A note about performance:** In certain instances like development environment, disabling the authentication by commenting the following line in `nginx-default.conf`:
+
+::
+
+ #auth_request /authentication;
+
+may (depending on the hardware) increases the performance significantly as it forgoes the auth API calls to radosgw.
diff --git a/doc/radosgw/role.rst b/doc/radosgw/role.rst
new file mode 100644
index 000000000..2e2697f57
--- /dev/null
+++ b/doc/radosgw/role.rst
@@ -0,0 +1,523 @@
+======
+ Role
+======
+
+A role is similar to a user and has permission policies attached to it, that determine what a role can or can not do. A role can be assumed by any identity that needs it. If a user assumes a role, a set of dynamically created temporary credentials are returned to the user. A role can be used to delegate access to users, applications, services that do not have permissions to access some s3 resources.
+
+The following radosgw-admin commands can be used to create/ delete/ update a role and permissions asscociated with a role.
+
+Create a Role
+-------------
+
+To create a role, execute the following::
+
+	radosgw-admin role create --role-name={role-name} [--path=="{path to the role}"] [--assume-role-policy-doc={trust-policy-document}]
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``role-name``
+
+:Description: Name of the role.
+:Type: String
+
+``path``
+
+:Description: Path to the role. The default value is a slash(/).
+:Type: String
+
+``assume-role-policy-doc``
+
+:Description: The trust relationship policy document that grants an entity permission to assume the role.
+:Type: String
+
+For example:: 	
+	
+  radosgw-admin role create --role-name=S3Access1 --path=/application_abc/component_xyz/ --assume-role-policy-doc=\{\"Version\":\"2012-10-17\",\"Statement\":\[\{\"Effect\":\"Allow\",\"Principal\":\{\"AWS\":\[\"arn:aws:iam:::user/TESTER\"\]\},\"Action\":\[\"sts:AssumeRole\"\]\}\]\}
+  
+.. code-block:: javascript
+  
+  {
+    "id": "ca43045c-082c-491a-8af1-2eebca13deec",
+    "name": "S3Access1",
+    "path": "/application_abc/component_xyz/",
+    "arn": "arn:aws:iam:::role/application_abc/component_xyz/S3Access1",
+    "create_date": "2018-10-17T10:18:29.116Z",
+    "max_session_duration": 3600,
+    "assume_role_policy_document": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam:::user/TESTER\"]},\"Action\":[\"sts:AssumeRole\"]}]}"
+  }
+
+
+Delete a Role
+-------------
+
+To delete a role, execute the following::
+
+	radosgw-admin role rm --role-name={role-name}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``role-name``
+
+:Description: Name of the role.
+:Type: String
+
+For example:: 	
+	
+  radosgw-admin role rm --role-name=S3Access1
+
+Note: A role can be deleted only when it doesn't have any permission policy attached to it.
+
+Get a Role
+----------
+
+To get information about a role, execute the following::
+
+	radosgw-admin role get --role-name={role-name}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``role-name``
+
+:Description: Name of the role.
+:Type: String
+
+For example:: 	
+	
+  radosgw-admin role get --role-name=S3Access1
+  
+.. code-block:: javascript
+  
+  {
+    "id": "ca43045c-082c-491a-8af1-2eebca13deec",
+    "name": "S3Access1",
+    "path": "/application_abc/component_xyz/",
+    "arn": "arn:aws:iam:::role/application_abc/component_xyz/S3Access1",
+    "create_date": "2018-10-17T10:18:29.116Z",
+    "max_session_duration": 3600,
+    "assume_role_policy_document": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam:::user/TESTER\"]},\"Action\":[\"sts:AssumeRole\"]}]}"
+  }
+
+
+List Roles
+----------
+
+To list roles with a specified path prefix, execute the following::
+
+	radosgw-admin role list [--path-prefix ={path prefix}]
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``path-prefix``
+
+:Description: Path prefix for filtering roles. If this is not specified, all roles are listed.
+:Type: String
+
+For example:: 	
+	
+  radosgw-admin role list --path-prefix="/application"
+  
+.. code-block:: javascript
+  
+  [
+    {
+        "id": "3e1c0ff7-8f2b-456c-8fdf-20f428ba6a7f",
+        "name": "S3Access1",
+        "path": "/application_abc/component_xyz/",
+        "arn": "arn:aws:iam:::role/application_abc/component_xyz/S3Access1",
+        "create_date": "2018-10-17T10:32:01.881Z",
+        "max_session_duration": 3600,
+        "assume_role_policy_document": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam:::user/TESTER\"]},\"Action\":[\"sts:AssumeRole\"]}]}"
+    }
+  ]
+
+
+Update Assume Role Policy Document of a role
+--------------------------------------------
+
+To modify a role's assume role policy document, execute the following::
+
+	radosgw-admin role modify --role-name={role-name} --assume-role-policy-doc={trust-policy-document}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``role-name``
+
+:Description: Name of the role.
+:Type: String
+
+``assume-role-policy-doc``
+
+:Description: The trust relationship policy document that grants an entity permission to assume the role.
+:Type: String
+
+For example::
+
+  radosgw-admin role modify --role-name=S3Access1 --assume-role-policy-doc=\{\"Version\":\"2012-10-17\",\"Statement\":\[\{\"Effect\":\"Allow\",\"Principal\":\{\"AWS\":\[\"arn:aws:iam:::user/TESTER2\"\]\},\"Action\":\[\"sts:AssumeRole\"\]\}\]\}
+
+.. code-block:: javascript
+
+  {
+    "id": "ca43045c-082c-491a-8af1-2eebca13deec",
+    "name": "S3Access1",
+    "path": "/application_abc/component_xyz/",
+    "arn": "arn:aws:iam:::role/application_abc/component_xyz/S3Access1",
+    "create_date": "2018-10-17T10:18:29.116Z",
+    "max_session_duration": 3600,
+    "assume_role_policy_document": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam:::user/TESTER2\"]},\"Action\":[\"sts:AssumeRole\"]}]}"
+  }
+
+
+In the above example, we are modifying the Principal from TESTER to TESTER2 in its assume role policy document.
+
+Add/ Update a Policy attached to a Role
+---------------------------------------
+
+To add or update the inline policy attached to a role, execute the following::
+
+	radosgw-admin role policy put --role-name={role-name} --policy-name={policy-name} --policy-doc={permission-policy-doc}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``role-name``
+
+:Description: Name of the role.
+:Type: String
+
+``policy-name``
+
+:Description: Name of the policy.
+:Type: String
+
+``policy-doc``
+
+:Description: The Permission policy document.
+:Type: String
+
+For example::
+
+  radosgw-admin role-policy put --role-name=S3Access1 --policy-name=Policy1 --policy-doc=\{\"Version\":\"2012-10-17\",\"Statement\":\[\{\"Effect\":\"Allow\",\"Action\":\[\"s3:*\"\],\"Resource\":\"arn:aws:s3:::example_bucket\"\}\]\}
+
+In the above example, we are attaching a policy 'Policy1' to role 'S3Access1', which allows all s3 actions on 'example_bucket'.
+
+List Permission Policy Names attached to a Role
+-----------------------------------------------
+
+To list the names of permission policies attached to a role, execute the following::
+
+	radosgw-admin role policy get --role-name={role-name}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``role-name``
+
+:Description: Name of the role.
+:Type: String
+
+For example::
+
+  radosgw-admin role-policy list --role-name=S3Access1
+
+.. code-block:: javascript
+
+  [
+    "Policy1"
+  ]
+
+
+Get Permission Policy attached to a Role
+----------------------------------------
+
+To get a specific permission policy attached to a role, execute the following::
+
+	radosgw-admin role policy get --role-name={role-name} --policy-name={policy-name}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``role-name``
+
+:Description: Name of the role.
+:Type: String
+
+``policy-name``
+
+:Description: Name of the policy.
+:Type: String
+
+For example::
+
+  radosgw-admin role-policy get --role-name=S3Access1 --policy-name=Policy1
+
+.. code-block:: javascript
+
+  {
+    "Permission policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"s3:*\"],\"Resource\":\"arn:aws:s3:::example_bucket\"}]}"
+  }
+
+
+Delete Policy attached to a Role
+--------------------------------
+
+To delete permission policy attached to a role, execute the following::
+
+	radosgw-admin role policy rm --role-name={role-name} --policy-name={policy-name}
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``role-name``
+
+:Description: Name of the role.
+:Type: String
+
+``policy-name``
+
+:Description: Name of the policy.
+:Type: String
+
+For example::
+
+  radosgw-admin role-policy get --role-name=S3Access1 --policy-name=Policy1
+
+
+REST APIs for Manipulating a Role
+=================================
+
+In addition to the above radosgw-admin commands, the following REST APIs can be used for manipulating a role. For the request parameters and their explanations, refer to the sections above.
+
+In order to invoke the REST admin APIs, a user with admin caps needs to be created.
+
+.. code-block:: javascript
+
+  radosgw-admin --uid TESTER --display-name "TestUser" --access_key TESTER --secret test123 user create
+  radosgw-admin caps add --uid="TESTER" --caps="roles=*"
+
+
+Create a Role
+-------------
+
+Example::
+  POST "<hostname>?Action=CreateRole&RoleName=S3Access&Path=/application_abc/component_xyz/&AssumeRolePolicyDocument=\{\"Version\":\"2012-10-17\",\"Statement\":\[\{\"Effect\":\"Allow\",\"Principal\":\{\"AWS\":\[\"arn:aws:iam:::user/TESTER\"\]\},\"Action\":\[\"sts:AssumeRole\"\]\}\]\}"
+
+.. code-block:: XML
+
+  <role>
+    <id>8f41f4e0-7094-4dc0-ac20-074a881ccbc5</id>
+    <name>S3Access</name>
+    <path>/application_abc/component_xyz/</path>
+    <arn>arn:aws:iam:::role/application_abc/component_xyz/S3Access</arn>
+    <create_date>2018-10-23T07:43:42.811Z</create_date>
+    <max_session_duration>3600</max_session_duration>
+    <assume_role_policy_document>{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/TESTER"]},"Action":["sts:AssumeRole"]}]}</assume_role_policy_document>
+  </role>
+
+
+Delete a Role
+-------------
+
+Example::
+  POST "<hostname>?Action=DeleteRole&RoleName=S3Access"
+
+Note: A role can be deleted only when it doesn't have any permission policy attached to it.
+
+Get a Role
+----------
+
+Example::
+  POST "<hostname>?Action=GetRole&RoleName=S3Access"
+
+.. code-block:: XML
+
+  <role>
+    <id>8f41f4e0-7094-4dc0-ac20-074a881ccbc5</id>
+    <name>S3Access</name>
+    <path>/application_abc/component_xyz/</path>
+    <arn>arn:aws:iam:::role/application_abc/component_xyz/S3Access</arn>
+    <create_date>2018-10-23T07:43:42.811Z</create_date>
+    <max_session_duration>3600</max_session_duration>
+    <assume_role_policy_document>{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/TESTER"]},"Action":["sts:AssumeRole"]}]}</assume_role_policy_document>
+  </role>
+
+
+List Roles
+----------
+
+Example::
+  POST "<hostname>?Action=ListRoles&RoleName=S3Access&PathPrefix=/application"
+
+.. code-block:: XML
+
+  <role>
+    <id>8f41f4e0-7094-4dc0-ac20-074a881ccbc5</id>
+    <name>S3Access</name>
+    <path>/application_abc/component_xyz/</path>
+    <arn>arn:aws:iam:::role/application_abc/component_xyz/S3Access</arn>
+    <create_date>2018-10-23T07:43:42.811Z</create_date>
+    <max_session_duration>3600</max_session_duration>
+    <assume_role_policy_document>{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/TESTER"]},"Action":["sts:AssumeRole"]}]}</assume_role_policy_document>
+  </role>
+
+
+Update Assume Role Policy Document
+----------------------------------
+
+Example::
+  POST "<hostname>?Action=UpdateAssumeRolePolicy&RoleName=S3Access&PolicyDocument=\{\"Version\":\"2012-10-17\",\"Statement\":\[\{\"Effect\":\"Allow\",\"Principal\":\{\"AWS\":\[\"arn:aws:iam:::user/TESTER2\"\]\},\"Action\":\[\"sts:AssumeRole\"\]\}\]\}"
+
+Add/ Update a Policy attached to a Role
+---------------------------------------
+
+Example::
+  POST "<hostname>?Action=PutRolePolicy&RoleName=S3Access&PolicyName=Policy1&PolicyDocument=\{\"Version\":\"2012-10-17\",\"Statement\":\[\{\"Effect\":\"Allow\",\"Action\":\[\"s3:CreateBucket\"\],\"Resource\":\"arn:aws:s3:::example_bucket\"\}\]\}"
+
+List Permission Policy Names attached to a Role
+-----------------------------------------------
+
+Example::
+  POST "<hostname>?Action=ListRolePolicies&RoleName=S3Access"
+
+.. code-block:: XML
+
+  <PolicyNames>
+    <member>Policy1</member>
+  </PolicyNames>
+
+
+Get Permission Policy attached to a Role
+----------------------------------------
+
+Example::
+  POST "<hostname>?Action=GetRolePolicy&RoleName=S3Access&PolicyName=Policy1"
+
+.. code-block:: XML
+
+  <GetRolePolicyResult>
+    <PolicyName>Policy1</PolicyName>
+    <RoleName>S3Access</RoleName>
+    <Permission_policy>{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:CreateBucket"],"Resource":"arn:aws:s3:::example_bucket"}]}</Permission_policy>
+  </GetRolePolicyResult>
+
+
+Delete Policy attached to a Role
+--------------------------------
+
+Example::
+  POST "<hostname>?Action=DeleteRolePolicy&RoleName=S3Access&PolicyName=Policy1"
+
+Tag a role
+----------
+A role can have multivalued tags attached to it. These tags can be passed in as part of CreateRole REST API also.
+AWS does not support multi-valued role tags.
+
+Example::
+  POST "<hostname>?Action=TagRole&RoleName=S3Access&Tags.member.1.Key=Department&Tags.member.1.Value=Engineering"
+
+.. code-block:: XML
+
+  <TagRoleResponse>
+    <ResponseMetadata>
+      <RequestId>tx000000000000000000004-00611f337e-1027-default</RequestId>
+    </ResponseMetadata>
+  </TagRoleResponse>
+
+
+List role tags
+--------------
+Lists the tags attached to a role.
+
+Example::
+  POST "<hostname>?Action=ListRoleTags&RoleName=S3Access"
+
+.. code-block:: XML
+
+  <ListRoleTagsResponse>
+    <ListRoleTagsResult>
+      <Tags>
+        <member>
+          <Key>Department</Key>
+          <Value>Engineering</Value>
+        </member>
+      </Tags>
+    </ListRoleTagsResult>
+    <ResponseMetadata>
+      <RequestId>tx000000000000000000005-00611f337e-1027-default</RequestId>
+    </ResponseMetadata>
+  </ListRoleTagsResponse>
+
+Delete role tags
+----------------
+Delete a tag/ tags attached to a role.
+
+Example::
+  POST "<hostname>?Action=UntagRoles&RoleName=S3Access&TagKeys.member.1=Department"
+
+.. code-block:: XML
+
+  <UntagRoleResponse>
+    <ResponseMetadata>
+      <RequestId>tx000000000000000000007-00611f337e-1027-default</RequestId>
+    </ResponseMetadata>
+  </UntagRoleResponse>
+
+
+Sample code for tagging, listing tags and untagging a role
+----------------------------------------------------------
+
+The following is sample code for adding tags to role, listing tags and untagging a role using boto3.
+
+.. code-block:: python
+
+    import boto3
+
+    access_key = 'TESTER'
+    secret_key = 'test123'
+
+    iam_client = boto3.client('iam',
+    aws_access_key_id=access_key,
+    aws_secret_access_key=secret_key,
+    endpoint_url='http://s3.us-east.localhost:8000',
+    region_name=''
+    )
+
+    policy_document = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Federated\":[\"arn:aws:iam:::oidc-provider/localhost:8080/auth/realms/quickstart\"]},\"Action\":[\"sts:AssumeRoleWithWebIdentity\"],\"Condition\":{\"StringEquals\":{\"localhost:8080/auth/realms/quickstart:sub\":\"user1\"}}}]}"
+
+    print ("\n Creating Role with tags\n")
+    tags_list = [
+        {'Key':'Department','Value':'Engineering'}
+    ]
+    role_response = iam_client.create_role(
+        AssumeRolePolicyDocument=policy_document,
+        Path='/',
+        RoleName='S3Access',
+        Tags=tags_list,
+    )
+
+    print ("Adding tags to role\n")
+    response = iam_client.tag_role(
+                RoleName='S3Access',
+                Tags= [
+                        {'Key':'CostCenter','Value':'123456'}
+                    ]
+                )
+    print ("Listing role tags\n")
+    response = iam_client.list_role_tags(
+                RoleName='S3Access'
+                )
+    print (response)
+    print ("Untagging role\n")
+    response = iam_client.untag_role(
+        RoleName='S3Access',
+        TagKeys=[
+            'Department',
+        ]
+    )
+
+
+
diff --git a/doc/radosgw/s3-notification-compatibility.rst b/doc/radosgw/s3-notification-compatibility.rst
new file mode 100644
index 000000000..09054bed3
--- /dev/null
+++ b/doc/radosgw/s3-notification-compatibility.rst
@@ -0,0 +1,139 @@
+=====================================
+S3 Bucket Notifications Compatibility
+=====================================
+
+Ceph's `Bucket Notifications`_ and `PubSub Module`_ APIs follow `AWS S3 Bucket Notifications API`_. However, some differences exist, as listed below.
+
+
+.. note:: 
+
+    Compatibility is different depending on which of the above mechanism is used
+
+Supported Destination
+---------------------
+
+AWS supports: **SNS**, **SQS** and **Lambda** as possible destinations (AWS internal destinations). 
+Currently, we support: **HTTP/S**, **Kafka** and **AMQP**. And also support pulling and acking of events stored in Ceph (as an intrenal destination).
+
+We are using the **SNS** ARNs to represent the **HTTP/S**, **Kafka** and **AMQP** destinations.
+
+Notification Configuration XML
+------------------------------
+
+Following tags (and the tags inside them) are not supported:
+
++-----------------------------------+----------------------------------------------+
+| Tag                               | Remaks                                       |
++===================================+==============================================+
+| ``<QueueConfiguration>``          | not needed, we treat all destinations as SNS |
++-----------------------------------+----------------------------------------------+
+| ``<CloudFunctionConfiguration>``  | not needed, we treat all destinations as SNS |
++-----------------------------------+----------------------------------------------+
+
+REST API Extension 
+------------------
+
+Ceph's bucket notification API has the following extensions:
+
+- Deletion of a specific notification, or all notifications on a bucket, using the ``DELETE`` verb
+
+ - In S3, all notifications are deleted when the bucket is deleted, or when an empty notification is set on the bucket
+
+- Getting the information on a specific notification (when more than one exists on a bucket)
+
+  - In S3, it is only possible to fetch all notifications on a bucket
+
+- In addition to filtering based on prefix/suffix of object keys we support:
+
+  - Filtering based on regular expression matching
+
+  - Filtering based on metadata attributes attached to the object
+
+  - Filtering based on object tags
+
+- Each one of the additional filters extends the S3 API and using it will require extension of the client SDK (unless you are using plain HTTP). 
+
+- Filtering overlapping is allowed, so that same event could be sent as different notification
+
+
+Unsupported Fields in the Event Record
+--------------------------------------
+
+The records sent for bucket notification follow format described in: `Event Message Structure`_.
+However, the following fields may be sent empty, under the different deployment options (Notification/PubSub):
+
++----------------------------------------+--------------+---------------+------------------------------------------------------------+
+| Field                                  | Notification | PubSub        | Description                                                |
++========================================+==============+===============+============================================================+
+| ``userIdentity.principalId``           | Supported    | Not Supported | The identity of the user that triggered the event          |
++----------------------------------------+--------------+---------------+------------------------------------------------------------+
+| ``requestParameters.sourceIPAddress``  |         Not Supported        | The IP address of the client that triggered the event      |
++----------------------------------------+--------------+---------------+------------------------------------------------------------+
+| ``requestParameters.x-amz-request-id`` | Supported    | Not Supported | The request id that triggered the event                    |
++----------------------------------------+--------------+---------------+------------------------------------------------------------+
+| ``requestParameters.x-amz-id-2``       | Supported    | Not Supported | The IP address of the RGW on which the event was triggered |
++----------------------------------------+--------------+---------------+------------------------------------------------------------+
+| ``s3.object.size``                     | Supported    | Not Supported | The size of the object                                     |
++----------------------------------------+--------------+---------------+------------------------------------------------------------+
+
+Event Types
+-----------
+
++----------------------------------------------+-----------------+-------------------------------------------+
+| Event                                        | Notification    | PubSub                                    |
++==============================================+=================+===========================================+
+| ``s3:ObjectCreated:*``                       | Supported                                                   |
++----------------------------------------------+-----------------+-------------------------------------------+
+| ``s3:ObjectCreated:Put``                     | Supported       | Supported at ``s3:ObjectCreated:*`` level |
++----------------------------------------------+-----------------+-------------------------------------------+
+| ``s3:ObjectCreated:Post``                    | Supported       | Not Supported                             |
++----------------------------------------------+-----------------+-------------------------------------------+
+| ``s3:ObjectCreated:Copy``                    | Supported       | Supported at ``s3:ObjectCreated:*`` level |
++----------------------------------------------+-----------------+-------------------------------------------+
+| ``s3:ObjectCreated:CompleteMultipartUpload`` | Supported       | Supported at ``s3:ObjectCreated:*`` level |
++----------------------------------------------+-----------------+-------------------------------------------+
+| ``s3:ObjectRemoved:*``                       | Supported       | Supported only the specific events below  |
++----------------------------------------------+-----------------+-------------------------------------------+
+| ``s3:ObjectRemoved:Delete``                  | Supported                                                   |
++----------------------------------------------+-----------------+-------------------------------------------+
+| ``s3:ObjectRemoved:DeleteMarkerCreated``     | Supported                                                   |
++----------------------------------------------+-----------------+-------------------------------------------+
+| ``s3:ObjectRestore:Post``                    | Not applicable to Ceph                                      |
++----------------------------------------------+-----------------+-------------------------------------------+
+| ``s3:ObjectRestore:Complete``                | Not applicable to Ceph                                      |
++----------------------------------------------+-----------------+-------------------------------------------+
+| ``s3:ReducedRedundancyLostObject``           | Not applicable to Ceph                                      |
++----------------------------------------------+-----------------+-------------------------------------------+
+
+.. note:: 
+
+   The ``s3:ObjectRemoved:DeleteMarkerCreated`` event presents information on the latest version of the object
+
+.. note::
+
+   In case of multipart upload, an ``ObjectCreated:CompleteMultipartUpload`` notification will be sent at the end of the process.
+
+Topic Configuration
+-------------------
+In the case of bucket notifications, the topics management API will be derived from `AWS Simple Notification Service API`_. 
+Note that most of the API is not applicable to Ceph, and only the following actions are implemented:
+
+ - ``CreateTopic``
+ - ``DeleteTopic``
+ - ``ListTopics``
+
+We also have the following extensions to topic configuration: 
+
+ - In ``GetTopic`` we allow fetching a specific topic, instead of all user topics
+ - In ``CreateTopic``
+
+  - we allow setting endpoint attributes
+  - we allow setting opaque data thta will be sent to the endpoint in the notification
+
+
+.. _AWS Simple Notification Service API: https://docs.aws.amazon.com/sns/latest/api/API_Operations.html
+.. _AWS S3 Bucket Notifications API: https://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html
+.. _Event Message Structure: https://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html
+.. _`PubSub Module`: ../pubsub-module
+.. _`Bucket Notifications`: ../notifications
+.. _`boto3 SDK filter extensions`: https://github.com/ceph/ceph/tree/master/examples/boto3
diff --git a/doc/radosgw/s3.rst b/doc/radosgw/s3.rst
new file mode 100644
index 000000000..d68863d6b
--- /dev/null
+++ b/doc/radosgw/s3.rst
@@ -0,0 +1,102 @@
+============================
+ Ceph Object Gateway S3 API
+============================
+
+Ceph supports a RESTful API that is compatible with the basic data access model of the `Amazon S3 API`_.
+
+API
+---
+
+.. toctree::
+   :maxdepth: 1
+
+   Common <s3/commons>
+   Authentication <s3/authentication>
+   Service Ops <s3/serviceops>
+   Bucket Ops <s3/bucketops>
+   Object Ops <s3/objectops>
+   C++ <s3/cpp>
+   C# <s3/csharp>
+   Java <s3/java>
+   Perl <s3/perl>
+   PHP <s3/php>
+   Python <s3/python>
+   Ruby <s3/ruby>
+
+
+Features Support
+----------------
+
+The following table describes the support status for current Amazon S3 functional features:
+
++---------------------------------+-----------------+----------------------------------------+
+| Feature                         | Status          | Remarks                                |
++=================================+=================+========================================+
+| **List Buckets**                | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Delete Bucket**               | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Create Bucket**               | Supported       | Different set of canned ACLs           |
++---------------------------------+-----------------+----------------------------------------+
+| **Bucket Lifecycle**            | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Bucket Replication**          | Partial         | Only permitted across zones            |
++---------------------------------+-----------------+----------------------------------------+
+| **Policy (Buckets, Objects)**   | Supported       | ACLs & bucket policies are supported   |
++---------------------------------+-----------------+----------------------------------------+
+| **Bucket Website**              | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Bucket ACLs (Get, Put)**      | Supported       | Different set of canned ACLs           |
++---------------------------------+-----------------+----------------------------------------+
+| **Bucket Location**             | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Bucket Notification**         | Supported       | See `S3 Notification Compatibility`_   |
++---------------------------------+-----------------+----------------------------------------+
+| **Bucket Object Versions**      | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Get Bucket Info (HEAD)**      | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Bucket Request Payment**      | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Put Object**                  | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Delete Object**               | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Get Object**                  | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Object ACLs (Get, Put)**      | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Get Object Info (HEAD)**      | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **POST Object**                 | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Copy Object**                 | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Multipart Uploads**           | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Object Tagging**              | Supported       | See :ref:`tag_policy` for Policy verbs |
++---------------------------------+-----------------+----------------------------------------+
+| **Bucket Tagging**              | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Storage Class**               | Supported       | See :ref:`storage_classes`             |
++---------------------------------+-----------------+----------------------------------------+
+
+Unsupported Header Fields
+-------------------------
+
+The following common request header fields are not supported:
+
++----------------------------+------------+
+| Name                       | Type       |
++============================+============+
+| **Server**                 | Response   |
++----------------------------+------------+
+| **x-amz-delete-marker**    | Response   |
++----------------------------+------------+
+| **x-amz-id-2**             | Response   |
++----------------------------+------------+
+| **x-amz-version-id**       | Response   |
++----------------------------+------------+
+
+.. _Amazon S3 API: http://docs.aws.amazon.com/AmazonS3/latest/API/APIRest.html
+.. _S3 Notification Compatibility: ../s3-notification-compatibility
diff --git a/doc/radosgw/s3/authentication.rst b/doc/radosgw/s3/authentication.rst
new file mode 100644
index 000000000..10143290d
--- /dev/null
+++ b/doc/radosgw/s3/authentication.rst
@@ -0,0 +1,231 @@
+=========================
+ Authentication and ACLs
+=========================
+
+Requests to the RADOS Gateway (RGW) can be either authenticated or 
+unauthenticated. RGW assumes unauthenticated requests are sent by an anonymous 
+user. RGW supports canned ACLs.
+
+Authentication
+--------------
+Authenticating a request requires including an access key and a Hash-based 
+Message Authentication Code (HMAC) in the request before it is sent to the 
+RGW server. RGW uses an S3-compatible authentication approach. 
+
+::
+
+	HTTP/1.1
+	PUT /buckets/bucket/object.mpeg
+	Host: cname.domain.com
+	Date: Mon, 2 Jan 2012 00:01:01 +0000
+	Content-Encoding: mpeg	
+	Content-Length: 9999999
+
+	Authorization: AWS {access-key}:{hash-of-header-and-secret}
+
+In the foregoing example, replace ``{access-key}`` with the value for your access 
+key ID followed by a colon (``:``). Replace ``{hash-of-header-and-secret}`` with 
+a hash of the header string and the secret corresponding to the access key ID.
+
+To generate the hash of the header string and secret, you must:
+
+#. Get the value of the header string.
+#. Normalize the request header string into canonical form. 
+#. Generate an HMAC using a SHA-1 hashing algorithm.
+   See `RFC 2104`_ and `HMAC`_ for details.
+#. Encode the ``hmac`` result as base-64.
+
+To normalize the header into canonical form: 
+
+#. Get all fields beginning with ``x-amz-``.
+#. Ensure that the fields are all lowercase.
+#. Sort the fields lexicographically. 
+#. Combine multiple instances of the same field name into a 
+   single field and separate the field values with a comma.
+#. Replace white space and line breaks in field values with a single space.
+#. Remove white space before and after colons.
+#. Append a new line after each field.
+#. Merge the fields back into the header.
+
+Replace the ``{hash-of-header-and-secret}`` with the base-64 encoded HMAC string.
+
+Authentication against OpenStack Keystone
+-----------------------------------------
+
+In a radosgw instance that is configured with authentication against
+OpenStack Keystone, it is possible to use Keystone as an authoritative
+source for S3 API authentication. To do so, you must set:
+
+* the ``rgw keystone`` configuration options explained in :doc:`../keystone`,
+* ``rgw s3 auth use keystone = true``.
+
+In addition, a user wishing to use the S3 API must obtain an AWS-style
+access key and secret key. They can do so with the ``openstack ec2
+credentials create`` command::
+
+  $ openstack --os-interface public ec2 credentials create
+  +------------+---------------------------------------------------------------------------------------------------------------------------------------------+
+  | Field      | Value                                                                                                                                       |
+  +------------+---------------------------------------------------------------------------------------------------------------------------------------------+
+  | access     | c921676aaabbccdeadbeef7e8b0eeb2c                                                                                                            |
+  | links      | {u'self': u'https://auth.example.com:5000/v3/users/7ecbebaffeabbddeadbeefa23267ccbb24/credentials/OS-EC2/c921676aaabbccdeadbeef7e8b0eeb2c'} |
+  | project_id | 5ed51981aab4679851adeadbeef6ebf7                                                                                                            |
+  | secret     | ********************************                                                                                                            |
+  | trust_id   | None                                                                                                                                        |
+  | user_id    | 7ecbebaffeabbddeadbeefa23267cc24                                                                                                            |
+  +------------+---------------------------------------------------------------------------------------------------------------------------------------------+
+
+The thus-generated access and secret key can then be used for S3 API
+access to radosgw.
+
+.. note:: Consider that most production radosgw deployments
+          authenticating against OpenStack Keystone are also set up
+          for :doc:`../multitenancy`, for which special
+          considerations apply with respect to S3 signed URLs and
+          public read ACLs.
+
+Access Control Lists (ACLs)
+---------------------------
+
+RGW supports S3-compatible ACL functionality. An ACL is a list of access grants
+that specify which operations a user can perform on a bucket or on an object.
+Each grant has a different meaning when applied to a bucket versus applied to 
+an object:
+
++------------------+--------------------------------------------------------+----------------------------------------------+
+| Permission       | Bucket                                                 | Object                                       |
++==================+========================================================+==============================================+
+| ``READ``         | Grantee can list the objects in the bucket.            | Grantee can read the object.                 |
++------------------+--------------------------------------------------------+----------------------------------------------+
+| ``WRITE``        | Grantee can write or delete objects in the bucket.     | N/A                                          |
++------------------+--------------------------------------------------------+----------------------------------------------+
+| ``READ_ACP``     | Grantee can read bucket ACL.                           | Grantee can read the object ACL.             |
++------------------+--------------------------------------------------------+----------------------------------------------+
+| ``WRITE_ACP``    | Grantee can write bucket ACL.                          | Grantee can write to the object ACL.         |
++------------------+--------------------------------------------------------+----------------------------------------------+
+| ``FULL_CONTROL`` | Grantee has full permissions for object in the bucket. | Grantee can read or write to the object ACL. |
++------------------+--------------------------------------------------------+----------------------------------------------+
+
+Internally, S3 operations are mapped to ACL permissions thus:
+
++---------------------------------------+---------------+
+| Operation                             | Permission    |
++=======================================+===============+
+| ``s3:GetObject``                      | ``READ``      |
++---------------------------------------+---------------+
+| ``s3:GetObjectTorrent``               | ``READ``      |
++---------------------------------------+---------------+
+| ``s3:GetObjectVersion``               | ``READ``      |
++---------------------------------------+---------------+
+| ``s3:GetObjectVersionTorrent``        | ``READ``      |
++---------------------------------------+---------------+
+| ``s3:GetObjectTagging``               | ``READ``      |
++---------------------------------------+---------------+
+| ``s3:GetObjectVersionTagging``        | ``READ``      |
++---------------------------------------+---------------+
+| ``s3:ListAllMyBuckets``               | ``READ``      |
++---------------------------------------+---------------+
+| ``s3:ListBucket``                     | ``READ``      |
++---------------------------------------+---------------+
+| ``s3:ListBucketMultipartUploads``     | ``READ``      |
++---------------------------------------+---------------+
+| ``s3:ListBucketVersions``             | ``READ``      |
++---------------------------------------+---------------+
+| ``s3:ListMultipartUploadParts``       | ``READ``      |
++---------------------------------------+---------------+
+| ``s3:AbortMultipartUpload``           | ``WRITE``     |
++---------------------------------------+---------------+
+| ``s3:CreateBucket``                   | ``WRITE``     |
++---------------------------------------+---------------+
+| ``s3:DeleteBucket``                   | ``WRITE``     |
++---------------------------------------+---------------+
+| ``s3:DeleteObject``                   | ``WRITE``     |
++---------------------------------------+---------------+
+| ``s3:s3DeleteObjectVersion``          | ``WRITE``     |
++---------------------------------------+---------------+
+| ``s3:PutObject``                      | ``WRITE``     |
++---------------------------------------+---------------+
+| ``s3:PutObjectTagging``               | ``WRITE``     |
++---------------------------------------+---------------+
+| ``s3:PutObjectVersionTagging``        | ``WRITE``     |
++---------------------------------------+---------------+
+| ``s3:DeleteObjectTagging``            | ``WRITE``     |
++---------------------------------------+---------------+
+| ``s3:DeleteObjectVersionTagging``     | ``WRITE``     |
++---------------------------------------+---------------+
+| ``s3:RestoreObject``                  | ``WRITE``     |
++---------------------------------------+---------------+
+| ``s3:GetAccelerateConfiguration``     | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetBucketAcl``                   | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetBucketCORS``                  | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetBucketLocation``              | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetBucketLogging``               | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetBucketNotification``          | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetBucketPolicy``                | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetBucketRequestPayment``        | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetBucketTagging``               | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetBucketVersioning``            | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetBucketWebsite``               | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetLifecycleConfiguration``      | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetObjectAcl``                   | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetObjectVersionAcl``            | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:GetReplicationConfiguration``    | ``READ_ACP``  |
++---------------------------------------+---------------+
+| ``s3:DeleteBucketPolicy``             | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:DeleteBucketWebsite``            | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:DeleteReplicationConfiguration`` | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutAccelerateConfiguration``     | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutBucketAcl``                   | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutBucketCORS``                  | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutBucketLogging``               | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutBucketNotification``          | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutBucketPolicy``                | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutBucketRequestPayment``        | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutBucketTagging``               | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutPutBucketVersioning``         | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutBucketWebsite``               | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutLifecycleConfiguration``      | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutObjectAcl``                   | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutObjectVersionAcl``            | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+| ``s3:PutReplicationConfiguration``    | ``WRITE_ACP`` |
++---------------------------------------+---------------+
+
+Some mappings, (e.g. ``s3:CreateBucket`` to ``WRITE``) are not
+applicable to S3 operation, but are required to allow Swift and S3 to
+access the same resources when things like Swift user ACLs are in
+play. This is one of the many reasons that you should use S3 bucket
+policies rather than S3 ACLs when possible.
+
+
+.. _RFC 2104: http://www.ietf.org/rfc/rfc2104.txt
+.. _HMAC: https://en.wikipedia.org/wiki/HMAC
diff --git a/doc/radosgw/s3/bucketops.rst b/doc/radosgw/s3/bucketops.rst
new file mode 100644
index 000000000..378eb5f04
--- /dev/null
+++ b/doc/radosgw/s3/bucketops.rst
@@ -0,0 +1,706 @@
+===================
+ Bucket Operations
+===================
+
+PUT Bucket
+----------
+Creates a new bucket. To create a bucket, you must have a user ID and a valid AWS Access Key ID to authenticate requests. You may not
+create buckets as an anonymous user.
+
+Constraints
+~~~~~~~~~~~
+In general, bucket names should follow domain name constraints.
+
+- Bucket names must be unique.
+- Bucket names cannot be formatted as IP address.
+- Bucket names can be between 3 and 63 characters long.
+- Bucket names must not contain uppercase characters or underscores.
+- Bucket names must start with a lowercase letter or number.
+- Bucket names must be a series of one or more labels. Adjacent labels are separated by a single period (.). Bucket names can contain lowercase letters, numbers, and hyphens. Each label must start and end with a lowercase letter or a number.
+
+.. note:: The above constraints are relaxed if the option 'rgw_relaxed_s3_bucket_names' is set to true except that the bucket names must still be unique, cannot be formatted as IP address and can contain letters, numbers, periods, dashes and underscores for up to 255 characters long.
+
+Syntax
+~~~~~~
+
+::
+
+    PUT /{bucket} HTTP/1.1
+    Host: cname.domain.com
+    x-amz-acl: public-read-write
+
+    Authorization: AWS {access-key}:{hash-of-header-and-secret}
+
+Parameters
+~~~~~~~~~~
+
+
++---------------+----------------------+-----------------------------------------------------------------------------+------------+
+| Name          | Description          | Valid Values                                                                | Required   |
++===============+======================+=============================================================================+============+
+| ``x-amz-acl`` | Canned ACLs.         | ``private``, ``public-read``, ``public-read-write``, ``authenticated-read`` | No         |
++---------------+----------------------+-----------------------------------------------------------------------------+------------+
+| ``x-amz-bucket-object-lock-enabled`` | Enable object lock on bucket. | ``true``, ``false``                         | No         |
++--------------------------------------+-------------------------------+---------------------------------------------+------------+
+
+Request Entities
+~~~~~~~~~~~~~~~~
+
++-------------------------------+-----------+----------------------------------------------------------------+
+| Name                          | Type      | Description                                                    |
++===============================+===========+================================================================+
+| ``CreateBucketConfiguration`` | Container | A container for the bucket configuration.                      |
++-------------------------------+-----------+----------------------------------------------------------------+
+| ``LocationConstraint``        | String    | A zonegroup api name, with optional :ref:`s3_bucket_placement` |
++-------------------------------+-----------+----------------------------------------------------------------+
+
+
+HTTP Response
+~~~~~~~~~~~~~
+
+If the bucket name is unique, within constraints and unused, the operation will succeed.
+If a bucket with the same name already exists and the user is the bucket owner, the operation will succeed.
+If the bucket name is already in use, the operation will fail.
+
++---------------+-----------------------+----------------------------------------------------------+
+| HTTP Status   | Status Code           | Description                                              |
++===============+=======================+==========================================================+
+| ``409``       | BucketAlreadyExists   | Bucket already exists under different user's ownership.  |
++---------------+-----------------------+----------------------------------------------------------+
+
+DELETE Bucket
+-------------
+
+Deletes a bucket. You can reuse bucket names following a successful bucket removal.
+
+Syntax
+~~~~~~
+
+::
+
+    DELETE /{bucket} HTTP/1.1
+    Host: cname.domain.com
+
+    Authorization: AWS {access-key}:{hash-of-header-and-secret}
+
+HTTP Response
+~~~~~~~~~~~~~
+
++---------------+---------------+------------------+
+| HTTP Status   | Status Code   | Description      |
++===============+===============+==================+
+| ``204``       | No Content    | Bucket removed.  |
++---------------+---------------+------------------+
+
+GET Bucket
+----------
+Returns a list of bucket objects.
+
+Syntax
+~~~~~~
+
+::
+
+    GET /{bucket}?max-keys=25 HTTP/1.1
+    Host: cname.domain.com
+
+Parameters
+~~~~~~~~~~
+
++---------------------+-----------+-------------------------------------------------------------------------------------------------+
+| Name                | Type      | Description                                                                                     |
++=====================+===========+=================================================================================================+
+| ``prefix``          | String    | Only returns objects that contain the specified prefix.                                         |
++---------------------+-----------+-------------------------------------------------------------------------------------------------+
+| ``delimiter``       | String    | The delimiter between the prefix and the rest of the object name.                               |
++---------------------+-----------+-------------------------------------------------------------------------------------------------+
+| ``marker``          | String    | A beginning index for the list of objects returned.                                             |
++---------------------+-----------+-------------------------------------------------------------------------------------------------+
+| ``max-keys``        | Integer   | The maximum number of keys to return. Default is 1000.                                          |
++---------------------+-----------+-------------------------------------------------------------------------------------------------+
+| ``allow-unordered`` | Boolean   | Non-standard extension. Allows results to be returned unordered. Cannot be used with delimiter. |
++---------------------+-----------+-------------------------------------------------------------------------------------------------+
+
+HTTP Response
+~~~~~~~~~~~~~
+
++---------------+---------------+--------------------+
+| HTTP Status   | Status Code   | Description        |
++===============+===============+====================+
+| ``200``       | OK            | Buckets retrieved  |
++---------------+---------------+--------------------+
+
+Bucket Response Entities
+~~~~~~~~~~~~~~~~~~~~~~~~
+``GET /{bucket}`` returns a container for buckets with the following fields.
+
++------------------------+-----------+----------------------------------------------------------------------------------+
+| Name                   | Type      | Description                                                                      |
++========================+===========+==================================================================================+
+| ``ListBucketResult``   | Entity    | The container for the list of objects.                                           |
++------------------------+-----------+----------------------------------------------------------------------------------+
+| ``Name``               | String    | The name of the bucket whose contents will be returned.                          |
++------------------------+-----------+----------------------------------------------------------------------------------+
+| ``Prefix``             | String    | A prefix for the object keys.                                                    |
++------------------------+-----------+----------------------------------------------------------------------------------+
+| ``Marker``             | String    | A beginning index for the list of objects returned.                              |
++------------------------+-----------+----------------------------------------------------------------------------------+
+| ``MaxKeys``            | Integer   | The maximum number of keys returned.                                             |
++------------------------+-----------+----------------------------------------------------------------------------------+
+| ``Delimiter``          | String    | If set, objects with the same prefix will appear in the ``CommonPrefixes`` list. |
++------------------------+-----------+----------------------------------------------------------------------------------+
+| ``IsTruncated``        | Boolean   | If ``true``, only a subset of the bucket's contents were returned.               |
++------------------------+-----------+----------------------------------------------------------------------------------+
+| ``CommonPrefixes``     | Container | If multiple objects contain the same prefix, they will appear in this list.      |
++------------------------+-----------+----------------------------------------------------------------------------------+
+
+Object Response Entities
+~~~~~~~~~~~~~~~~~~~~~~~~
+The ``ListBucketResult`` contains objects, where each object is within a ``Contents`` container.
+
++------------------------+-----------+------------------------------------------+
+| Name                   | Type      | Description                              |
++========================+===========+==========================================+
+| ``Contents``           | Object    | A container for the object.              |
++------------------------+-----------+------------------------------------------+
+| ``Key``                | String    | The object's key.                        |
++------------------------+-----------+------------------------------------------+
+| ``LastModified``       | Date      | The object's last-modified date/time.    |
++------------------------+-----------+------------------------------------------+
+| ``ETag``               | String    | An MD-5 hash of the object. (entity tag) |
++------------------------+-----------+------------------------------------------+
+| ``Size``               | Integer   | The object's size.                       |
++------------------------+-----------+------------------------------------------+
+| ``StorageClass``       | String    | Should always return ``STANDARD``.       |
++------------------------+-----------+------------------------------------------+
+| ``Type``               | String    | ``Appendable`` or ``Normal``.            |
++------------------------+-----------+------------------------------------------+
+
+Get Bucket Location
+-------------------
+Retrieves the bucket's region. The user needs to be the bucket owner
+to call this. A bucket can be constrained to a region by providing
+``LocationConstraint`` during a PUT request.
+
+Syntax
+~~~~~~
+Add the ``location`` subresource to bucket resource as shown below
+
+::
+
+   GET /{bucket}?location HTTP/1.1
+   Host: cname.domain.com
+
+   Authorization: AWS {access-key}:{hash-of-header-and-secret}
+
+Response Entities
+~~~~~~~~~~~~~~~~~~~~~~~~
+
++------------------------+-----------+------------------------------------------+
+| Name                   | Type      | Description                              |
++========================+===========+==========================================+
+| ``LocationConstraint`` | String    | The region where bucket resides, empty   |
+|                        |           | string for default region                |
++------------------------+-----------+------------------------------------------+
+
+
+
+Get Bucket ACL
+--------------
+Retrieves the bucket access control list. The user needs to be the bucket
+owner or to have been granted ``READ_ACP`` permission on the bucket.
+
+Syntax
+~~~~~~
+Add the ``acl`` subresource to the bucket request as shown below.
+
+::
+
+    GET /{bucket}?acl HTTP/1.1
+    Host: cname.domain.com
+
+    Authorization: AWS {access-key}:{hash-of-header-and-secret}
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| Name                      | Type        | Description                                                                                  |
++===========================+=============+==============================================================================================+
+| ``AccessControlPolicy``   | Container   | A container for the response.                                                                |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``AccessControlList``     | Container   | A container for the ACL information.                                                         |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Owner``                 | Container   | A container for the bucket owner's ``ID`` and ``DisplayName``.                               |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``ID``                    | String      | The bucket owner's ID.                                                                       |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``DisplayName``           | String      | The bucket owner's display name.                                                             |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Grant``                 | Container   | A container for ``Grantee`` and ``Permission``.                                              |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Grantee``               | Container   | A container for the ``DisplayName`` and ``ID`` of the user receiving a grant of permission.  |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Permission``            | String      | The permission given to the ``Grantee`` bucket.                                              |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+
+PUT Bucket ACL
+--------------
+Sets an access control to an existing bucket. The user needs to be the bucket
+owner or to have been granted ``WRITE_ACP`` permission on the bucket.
+
+Syntax
+~~~~~~
+Add the ``acl`` subresource to the bucket request as shown below.
+
+::
+
+    PUT /{bucket}?acl HTTP/1.1
+
+Request Entities
+~~~~~~~~~~~~~~~~
+
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| Name                      | Type        | Description                                                                                  |
++===========================+=============+==============================================================================================+
+| ``AccessControlPolicy``   | Container   | A container for the request.                                                                 |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``AccessControlList``     | Container   | A container for the ACL information.                                                         |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Owner``                 | Container   | A container for the bucket owner's ``ID`` and ``DisplayName``.                               |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``ID``                    | String      | The bucket owner's ID.                                                                       |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``DisplayName``           | String      | The bucket owner's display name.                                                             |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Grant``                 | Container   | A container for ``Grantee`` and ``Permission``.                                              |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Grantee``               | Container   | A container for the ``DisplayName`` and ``ID`` of the user receiving a grant of permission.  |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Permission``            | String      | The permission given to the ``Grantee`` bucket.                                              |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+
+List Bucket Multipart Uploads
+-----------------------------
+
+``GET /?uploads`` returns a list of the current in-progress multipart uploads--i.e., the application initiates a multipart upload, but
+the service hasn't completed all the uploads yet.
+
+Syntax
+~~~~~~
+
+::
+
+    GET /{bucket}?uploads HTTP/1.1
+
+Parameters
+~~~~~~~~~~
+
+You may specify parameters for ``GET /{bucket}?uploads``, but none of them are required.
+
++------------------------+-----------+--------------------------------------------------------------------------------------+
+| Name                   | Type      | Description                                                                          |
++========================+===========+======================================================================================+
+| ``prefix``             | String    | Returns in-progress uploads whose keys contains the specified prefix.                |
++------------------------+-----------+--------------------------------------------------------------------------------------+
+| ``delimiter``          | String    | The delimiter between the prefix and the rest of the object name.                    |
++------------------------+-----------+--------------------------------------------------------------------------------------+
+| ``key-marker``         | String    | The beginning marker for the list of uploads.                                        |
++------------------------+-----------+--------------------------------------------------------------------------------------+
+| ``max-keys``           | Integer   | The maximum number of in-progress uploads. The default is 1000.                      |
++------------------------+-----------+--------------------------------------------------------------------------------------+
+| ``max-uploads``        | Integer   | The maximum number of multipart uploads. The range from 1-1000. The default is 1000. |
++------------------------+-----------+--------------------------------------------------------------------------------------+
+| ``upload-id-marker``   | String    | Ignored if ``key-marker`` is not specified. Specifies the ``ID`` of first            |
+|                        |           | upload to list in lexicographical order at or following the ``ID``.                  |
++------------------------+-----------+--------------------------------------------------------------------------------------+
+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| Name                                    | Type        | Description                                                                                              |
++=========================================+=============+==========================================================================================================+
+| ``ListMultipartUploadsResult``          | Container   | A container for the results.                                                                             |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``ListMultipartUploadsResult.Prefix``   | String      | The prefix specified by the ``prefix`` request parameter (if any).                                       |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Bucket``                              | String      | The bucket that will receive the bucket contents.                                                        |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``KeyMarker``                           | String      | The key marker specified by the ``key-marker`` request parameter (if any).                               |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``UploadIdMarker``                      | String      | The marker specified by the ``upload-id-marker`` request parameter (if any).                             |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``NextKeyMarker``                       | String      | The key marker to use in a subsequent request if ``IsTruncated`` is ``true``.                            |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``NextUploadIdMarker``                  | String      | The upload ID marker to use in a subsequent request if ``IsTruncated`` is ``true``.                      |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``MaxUploads``                          | Integer     | The max uploads specified by the ``max-uploads`` request parameter.                                      |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Delimiter``                           | String      | If set, objects with the same prefix will appear in the ``CommonPrefixes`` list.                         |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``IsTruncated``                         | Boolean     | If ``true``, only a subset of the bucket's upload contents were returned.                                |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Upload``                              | Container   | A container for ``Key``, ``UploadId``, ``InitiatorOwner``, ``StorageClass``, and ``Initiated`` elements. |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Key``                                 | String      | The key of the object once the multipart upload is complete.                                             |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``UploadId``                            | String      | The ``ID`` that identifies the multipart upload.                                                         |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Initiator``                           | Container   | Contains the ``ID`` and ``DisplayName`` of the user who initiated the upload.                            |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``DisplayName``                         | String      | The initiator's display name.                                                                            |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``ID``                                  | String      | The initiator's ID.                                                                                      |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Owner``                               | Container   | A container for the ``ID`` and ``DisplayName`` of the user who owns the uploaded object.                 |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``StorageClass``                        | String      | The method used to store the resulting object. ``STANDARD`` or ``REDUCED_REDUNDANCY``                    |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Initiated``                           | Date        | The date and time the user initiated the upload.                                                         |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``CommonPrefixes``                      | Container   | If multiple objects contain the same prefix, they will appear in this list.                              |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``CommonPrefixes.Prefix``               | String      | The substring of the key after the prefix as defined by the ``prefix`` request parameter.                |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+
+ENABLE/SUSPEND BUCKET VERSIONING
+--------------------------------
+
+``PUT /?versioning`` This subresource set the versioning state of an existing bucket. To set the versioning state, you must be the bucket owner.
+
+You can set the versioning state with one of the following values:
+
+- Enabled : Enables versioning for the objects in the bucket, All objects added to the bucket receive a unique version ID.
+- Suspended : Disables versioning for the objects in the bucket, All objects added to the bucket receive the version ID null.
+
+If the versioning state has never been set on a bucket, it has no versioning state; a GET versioning request does not return a versioning state value.
+
+Syntax
+~~~~~~
+
+::
+
+    PUT  /{bucket}?versioning  HTTP/1.1
+
+REQUEST ENTITIES
+~~~~~~~~~~~~~~~~
+
++-----------------------------+-----------+---------------------------------------------------------------------------+
+| Name                        | Type      | Description                                                               |
++=============================+===========+===========================================================================+
+| ``VersioningConfiguration`` | Container | A container for the request.                                              |
++-----------------------------+-----------+---------------------------------------------------------------------------+
+| ``Status``                  | String    | Sets the versioning state of the bucket.  Valid Values: Suspended/Enabled |
++-----------------------------+-----------+---------------------------------------------------------------------------+
+
+PUT BUCKET OBJECT LOCK
+--------------------------------
+
+Places an Object Lock configuration on the specified bucket. The rule specified in the Object Lock configuration will be
+applied by default to every new object placed in the specified bucket.
+
+Syntax
+~~~~~~
+
+::
+
+    PUT /{bucket}?object-lock HTTP/1.1
+
+Request Entities
+~~~~~~~~~~~~~~~~
+
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| Name                        | Type        | Description                                                                            | Required |
++=============================+=============+========================================================================================+==========+
+| ``ObjectLockConfiguration`` | Container   | A container for the request.                                                           |   Yes    |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| ``ObjectLockEnabled``       | String      | Indicates whether this bucket has an Object Lock configuration enabled.                |   Yes    |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| ``Rule``                    | Container   | The Object Lock rule in place for the specified bucket.                                |   No     |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| ``DefaultRetention``        | Container   | The default retention period applied to new objects placed in the specified bucket.    |   No     |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| ``Mode``                    | String      | The default Object Lock retention mode. Valid Values:  GOVERNANCE/COMPLIANCE           |   Yes    |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| ``Days``                    | Integer     | The number of days specified for the default retention period.                         |   No     |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| ``Years``                   | Integer     | The number of years specified for the default retention period.                        |   No     |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+
+HTTP Response
+~~~~~~~~~~~~~
+
+If the bucket object lock is not enabled when creating the bucket, the operation will fail.
+
++---------------+-----------------------+----------------------------------------------------------+
+| HTTP Status   | Status Code           | Description                                              |
++===============+=======================+==========================================================+
+| ``400``       | MalformedXML          | The XML is not well-formed                               |
++---------------+-----------------------+----------------------------------------------------------+
+| ``409``       | InvalidBucketState    | The bucket object lock is not enabled                    |
++---------------+-----------------------+----------------------------------------------------------+
+
+GET BUCKET OBJECT LOCK
+--------------------------------
+
+Gets the Object Lock configuration for a bucket. The rule specified in the Object Lock configuration will be applied by
+default to every new object placed in the specified bucket.
+
+Syntax
+~~~~~~
+
+::
+
+    GET /{bucket}?object-lock HTTP/1.1
+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| Name                        | Type        | Description                                                                            | Required |
++=============================+=============+========================================================================================+==========+
+| ``ObjectLockConfiguration`` | Container   | A container for the request.                                                           |   Yes    |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| ``ObjectLockEnabled``       | String      | Indicates whether this bucket has an Object Lock configuration enabled.                |   Yes    |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| ``Rule``                    | Container   | The Object Lock rule in place for the specified bucket.                                |   No     |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| ``DefaultRetention``        | Container   | The default retention period applied to new objects placed in the specified bucket.    |   No     |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| ``Mode``                    | String      | The default Object Lock retention mode. Valid Values:  GOVERNANCE/COMPLIANCE           |   Yes    |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| ``Days``                    | Integer     | The number of days specified for the default retention period.                         |   No     |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+| ``Years``                   | Integer     | The number of years specified for the default retention period.                        |   No     |
++-----------------------------+-------------+----------------------------------------------------------------------------------------+----------+
+
+Create Notification
+-------------------
+
+Create a publisher for a specific bucket into a topic.
+
+Syntax
+~~~~~~
+
+::
+
+    PUT /<bucket name>?notification HTTP/1.1
+
+
+Request Entities
+~~~~~~~~~~~~~~~~
+
+Parameters are XML encoded in the body of the request, in the following format:
+
+::
+
+   <NotificationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+       <TopicConfiguration>
+           <Id></Id>
+           <Topic></Topic>
+           <Event></Event>
+           <Filter>
+               <S3Key>
+                   <FilterRule>
+                       <Name></Name>
+                       <Value></Value>
+                   </FilterRule>
+        	    </S3Key>
+                <S3Metadata>
+                    <FilterRule>
+                        <Name></Name>
+                        <Value></Value>
+                    </FilterRule>
+                </S3Metadata>
+                <S3Tags>
+                    <FilterRule>
+                        <Name></Name>
+                        <Value></Value>
+                    </FilterRule>
+                </S3Tags>
+            </Filter>
+       </TopicConfiguration>
+   </NotificationConfiguration>
+
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| Name                          | Type      | Description                                                                          | Required |
++===============================+===========+======================================================================================+==========+
+| ``NotificationConfiguration`` | Container | Holding list of ``TopicConfiguration`` entities                                      | Yes      |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``TopicConfiguration``        | Container | Holding ``Id``, ``Topic`` and list of ``Event`` entities                             | Yes      |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``Id``                        | String    | Name of the notification                                                             | Yes      |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``Topic``                     | String    | Topic ARN. Topic must be created beforehand                                          | Yes      |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``Event``                     | String    | List of supported events see: `S3 Notification Compatibility`_.  Multiple ``Event``  | No       |
+|                               |           | entities can be used. If omitted, all events are handled                             |          |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``Filter``                    | Container | Holding ``S3Key``, ``S3Metadata`` and ``S3Tags`` entities                            | No       |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``S3Key``                     | Container | Holding a list of ``FilterRule`` entities, for filtering based on object key.        | No       |
+|                               |           | At most, 3 entities may be in the list, with ``Name`` be ``prefix``, ``suffix`` or   |          |
+|                               |           | ``regex``. All filter rules in the list must match for the filter to match.          |          |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``S3Metadata``                | Container | Holding a list of ``FilterRule`` entities, for filtering based on object metadata.   | No       |
+|                               |           | All filter rules in the list must match the metadata defined on the object. However, |          |
+|                               |           | the object still match if it has other metadata entries not listed in the filter.    |          |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``S3Tags``                    | Container | Holding a list of ``FilterRule`` entities, for filtering based on object tags.       | No       |
+|                               |           | All filter rules in the list must match the tags defined on the object. However,     |          |
+|                               |           | the object still match it it has other tags not listed in the filter.                |          |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``S3Key.FilterRule``          | Container | Holding ``Name`` and ``Value`` entities. ``Name`` would  be: ``prefix``, ``suffix``  | Yes      |
+|                               |           | or ``regex``. The ``Value`` would hold the key prefix, key suffix or a regular       |          |
+|                               |           | expression for matching the key, accordingly.                                        |          |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``S3Metadata.FilterRule``     | Container | Holding ``Name`` and ``Value`` entities. ``Name`` would be the name of the metadata  | Yes      |
+|                               |           | attribute (e.g. ``x-amz-meta-xxx``). The ``Value`` would be the expected value for   |          | 
+|                               |           | this attribute.                                                                      |          |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``S3Tags.FilterRule``         | Container | Holding ``Name`` and ``Value`` entities. ``Name`` would be the tag key,              |  Yes     |
+|                               |           | and ``Value`` would be the tag value.                                                |          | 
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+
+
+HTTP Response
+~~~~~~~~~~~~~
+
++---------------+-----------------------+----------------------------------------------------------+
+| HTTP Status   | Status Code           | Description                                              |
++===============+=======================+==========================================================+
+| ``400``       | MalformedXML          | The XML is not well-formed                               |
++---------------+-----------------------+----------------------------------------------------------+
+| ``400``       | InvalidArgument       | Missing Id; Missing/Invalid Topic ARN; Invalid Event     |
++---------------+-----------------------+----------------------------------------------------------+
+| ``404``       | NoSuchBucket          | The bucket does not exist                                |
++---------------+-----------------------+----------------------------------------------------------+
+| ``404``       | NoSuchKey             | The topic does not exist                                 |
++---------------+-----------------------+----------------------------------------------------------+
+
+
+Delete Notification
+-------------------
+
+Delete a specific, or all, notifications from a bucket.
+
+.. note:: 
+
+    - Notification deletion is an extension to the S3 notification API
+    - When the bucket is deleted, any notification defined on it is also deleted 
+    - Deleting an unknown notification (e.g. double delete) is not considered an error
+
+Syntax
+~~~~~~
+
+::
+
+    DELETE /bucket?notification[=<notification-id>] HTTP/1.1
+
+
+Parameters
+~~~~~~~~~~
+
++------------------------+-----------+----------------------------------------------------------------------------------------+
+| Name                   | Type      | Description                                                                            |
++========================+===========+========================================================================================+
+| ``notification-id``    | String    | Name of the notification. If not provided, all notifications on the bucket are deleted |
++------------------------+-----------+----------------------------------------------------------------------------------------+
+
+HTTP Response
+~~~~~~~~~~~~~
+
++---------------+-----------------------+----------------------------------------------------------+
+| HTTP Status   | Status Code           | Description                                              |
++===============+=======================+==========================================================+
+| ``404``       | NoSuchBucket          | The bucket does not exist                                |
++---------------+-----------------------+----------------------------------------------------------+
+
+Get/List Notification
+---------------------
+
+Get a specific notification, or list all notifications configured on a bucket.
+
+Syntax
+~~~~~~
+
+::
+
+    GET /bucket?notification[=<notification-id>] HTTP/1.1 
+
+
+Parameters
+~~~~~~~~~~
+
++------------------------+-----------+----------------------------------------------------------------------------------------+
+| Name                   | Type      | Description                                                                            |
++========================+===========+========================================================================================+
+| ``notification-id``    | String    | Name of the notification. If not provided, all notifications on the bucket are listed  |
++------------------------+-----------+----------------------------------------------------------------------------------------+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+Response is XML encoded in the body of the request, in the following format:
+
+::
+
+   <NotificationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+       <TopicConfiguration>
+           <Id></Id>
+           <Topic></Topic>
+           <Event></Event>
+           <Filter>
+               <S3Key>
+                   <FilterRule>
+                       <Name></Name>
+                       <Value></Value>
+                   </FilterRule>
+        	    </S3Key>
+                <S3Metadata>
+                    <FilterRule>
+                        <Name></Name>
+                        <Value></Value>
+                    </FilterRule>
+                </S3Metadata>
+                <S3Tags>
+                    <FilterRule>
+                        <Name></Name>
+                        <Value></Value>
+                    </FilterRule>
+                </S3Tags>
+            </Filter>
+       </TopicConfiguration>
+   </NotificationConfiguration>
+
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| Name                          | Type      | Description                                                                          | Required |
++===============================+===========+======================================================================================+==========+
+| ``NotificationConfiguration`` | Container | Holding list of ``TopicConfiguration`` entities                                      | Yes      |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``TopicConfiguration``        | Container | Holding ``Id``, ``Topic`` and list of ``Event`` entities                             | Yes      |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``Id``                        | String    | Name of the notification                                                             | Yes      |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``Topic``                     | String    | Topic ARN                                                                            | Yes      |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``Event``                     | String    | Handled event. Multiple ``Event`` entities may exist                                 | Yes      |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+| ``Filter``                    | Container | Holding the filters configured for this notification                                 | No       |
++-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+
+
+HTTP Response
+~~~~~~~~~~~~~
+
++---------------+-----------------------+----------------------------------------------------------+
+| HTTP Status   | Status Code           | Description                                              |
++===============+=======================+==========================================================+
+| ``404``       | NoSuchBucket          | The bucket does not exist                                |
++---------------+-----------------------+----------------------------------------------------------+
+| ``404``       | NoSuchKey             | The notification does not exist (if provided)            |
++---------------+-----------------------+----------------------------------------------------------+
+
+.. _S3 Notification Compatibility: ../../s3-notification-compatibility
diff --git a/doc/radosgw/s3/commons.rst b/doc/radosgw/s3/commons.rst
new file mode 100644
index 000000000..ca848bc27
--- /dev/null
+++ b/doc/radosgw/s3/commons.rst
@@ -0,0 +1,111 @@
+=================
+ Common Entities
+=================
+
+.. toctree::
+   :maxdepth: -1
+
+Bucket and Host Name
+--------------------
+There are two different modes of accessing the buckets. The first (preferred) method
+identifies the bucket as the top-level directory in the URI. ::
+
+	GET /mybucket HTTP/1.1
+	Host: cname.domain.com
+
+The second method identifies the bucket via a virtual bucket host name. For example::
+
+	GET / HTTP/1.1
+	Host: mybucket.cname.domain.com
+
+To configure virtual hosted buckets, you can either set ``rgw_dns_name = cname.domain.com`` in ceph.conf, or add ``cname.domain.com`` to the list of ``hostnames`` in your zonegroup configuration. See `Ceph Object Gateway - Multisite Configuration`_ for more on zonegroups.
+
+.. tip:: We prefer the first method, because the second method requires expensive domain certification and DNS wild cards.
+
+Common Request Headers
+----------------------
+
++--------------------+------------------------------------------+
+| Request Header     | Description                              |
++====================+==========================================+
+| ``CONTENT_LENGTH`` | Length of the request body.              |
++--------------------+------------------------------------------+
+| ``DATE``           | Request time and date (in UTC).          |
++--------------------+------------------------------------------+
+| ``HOST``           | The name of the host server.             |
++--------------------+------------------------------------------+
+| ``AUTHORIZATION``  | Authorization token.                     |
++--------------------+------------------------------------------+
+
+Common Response Status
+----------------------
+
++---------------+-----------------------------------+
+| HTTP Status   | Response Code                     |
++===============+===================================+
+| ``100``       | Continue                          |
++---------------+-----------------------------------+
+| ``200``       | Success                           |
++---------------+-----------------------------------+
+| ``201``       | Created                           |
++---------------+-----------------------------------+
+| ``202``       | Accepted                          |
++---------------+-----------------------------------+
+| ``204``       | NoContent                         |
++---------------+-----------------------------------+
+| ``206``       | Partial content                   |
++---------------+-----------------------------------+
+| ``304``       | NotModified                       |
++---------------+-----------------------------------+
+| ``400``       | InvalidArgument                   |
++---------------+-----------------------------------+
+| ``400``       | InvalidDigest                     |
++---------------+-----------------------------------+
+| ``400``       | BadDigest                         |
++---------------+-----------------------------------+
+| ``400``       | InvalidBucketName                 |
++---------------+-----------------------------------+
+| ``400``       | InvalidObjectName                 |
++---------------+-----------------------------------+
+| ``400``       | UnresolvableGrantByEmailAddress   |
++---------------+-----------------------------------+
+| ``400``       | InvalidPart                       |
++---------------+-----------------------------------+
+| ``400``       | InvalidPartOrder                  |
++---------------+-----------------------------------+
+| ``400``       | RequestTimeout                    |
++---------------+-----------------------------------+
+| ``400``       | EntityTooLarge                    |
++---------------+-----------------------------------+
+| ``403``       | AccessDenied                      |
++---------------+-----------------------------------+
+| ``403``       | UserSuspended                     |
++---------------+-----------------------------------+
+| ``403``       | RequestTimeTooSkewed              |
++---------------+-----------------------------------+
+| ``404``       | NoSuchKey                         |
++---------------+-----------------------------------+
+| ``404``       | NoSuchBucket                      |
++---------------+-----------------------------------+
+| ``404``       | NoSuchUpload                      |
++---------------+-----------------------------------+
+| ``405``       | MethodNotAllowed                  |
++---------------+-----------------------------------+
+| ``408``       | RequestTimeout                    |
++---------------+-----------------------------------+
+| ``409``       | BucketAlreadyExists               |
++---------------+-----------------------------------+
+| ``409``       | BucketNotEmpty                    |
++---------------+-----------------------------------+
+| ``411``       | MissingContentLength              |
++---------------+-----------------------------------+
+| ``412``       | PreconditionFailed                |
++---------------+-----------------------------------+
+| ``416``       | InvalidRange                      |
++---------------+-----------------------------------+
+| ``422``       | UnprocessableEntity               |
++---------------+-----------------------------------+
+| ``500``       | InternalError                     |
++---------------+-----------------------------------+
+
+.. _`Ceph Object Gateway - Multisite Configuration`: ../../multisite
diff --git a/doc/radosgw/s3/cpp.rst b/doc/radosgw/s3/cpp.rst
new file mode 100644
index 000000000..089c9c53a
--- /dev/null
+++ b/doc/radosgw/s3/cpp.rst
@@ -0,0 +1,337 @@
+.. _cpp:
+
+C++ S3 Examples
+===============
+
+Setup
+-----
+
+The following contains includes and globals that will be used in later examples:
+
+.. code-block:: cpp
+
+	#include "libs3.h"
+	#include <stdlib.h>
+	#include <iostream>
+	#include <fstream>
+
+	const char access_key[] = "ACCESS_KEY";
+	const char secret_key[] = "SECRET_KEY";
+	const char host[] = "HOST";
+	const char sample_bucket[] = "sample_bucket";
+	const char sample_key[] = "hello.txt";
+	const char sample_file[] = "resource/hello.txt";
+	const char *security_token = NULL;
+	const char *auth_region = NULL;
+
+	S3BucketContext bucketContext =
+	{
+		host,
+		sample_bucket,
+		S3ProtocolHTTP,
+		S3UriStylePath,
+		access_key,
+		secret_key,
+		security_token,
+		auth_region
+	};
+
+	S3Status responsePropertiesCallback(
+			const S3ResponseProperties *properties,
+			void *callbackData)
+	{
+		return S3StatusOK;
+	}
+
+	static void responseCompleteCallback(
+			S3Status status,
+			const S3ErrorDetails *error,
+			void *callbackData)
+	{
+		return;
+	}
+
+	S3ResponseHandler responseHandler =
+	{
+		&responsePropertiesCallback,
+		&responseCompleteCallback
+	};
+
+
+Creating (and Closing) a Connection
+-----------------------------------
+
+This creates a connection so that you can interact with the server.
+
+.. code-block:: cpp
+
+	S3_initialize("s3", S3_INIT_ALL, host);
+	// Do stuff...
+	S3_deinitialize();
+
+
+Listing Owned Buckets
+---------------------
+
+This gets a list of Buckets that you own.
+This also prints out the bucket name, owner ID, and display name
+for each bucket.
+
+.. code-block:: cpp
+
+	static S3Status listServiceCallback(
+			const char *ownerId,
+			const char *ownerDisplayName,
+			const char *bucketName,
+			int64_t creationDate, void *callbackData)
+	{
+		bool *header_printed = (bool*) callbackData;
+		if (!*header_printed) {
+			*header_printed = true;
+			printf("%-22s", "       Bucket");
+			printf("  %-20s  %-12s", "     Owner ID", "Display Name");
+			printf("\n");
+			printf("----------------------");
+			printf("  --------------------" "  ------------");
+			printf("\n");
+		}
+
+		printf("%-22s", bucketName);
+		printf("  %-20s  %-12s", ownerId ? ownerId : "", ownerDisplayName ? ownerDisplayName : "");
+		printf("\n");
+
+		return S3StatusOK;
+	}
+
+	S3ListServiceHandler listServiceHandler =
+	{
+		responseHandler,
+		&listServiceCallback
+	};
+	bool header_printed = false;
+	S3_list_service(S3ProtocolHTTP, access_key, secret_key, security_token, host, 
+                    auth_region, NULL, 0, &listServiceHandler, &header_printed);
+
+
+Creating a Bucket
+-----------------
+
+This creates a new bucket.
+
+.. code-block:: cpp
+
+	S3_create_bucket(S3ProtocolHTTP, access_key, secret_key, NULL, host, sample_bucket, S3CannedAclPrivate, NULL, NULL, &responseHandler, NULL);
+
+
+Listing a Bucket's Content
+--------------------------
+
+This gets a list of objects in the bucket.
+This also prints out each object's name, the file size, and
+last modified date.
+
+.. code-block:: cpp
+
+	static S3Status listBucketCallback(
+			int isTruncated,
+			const char *nextMarker,
+			int contentsCount,
+			const S3ListBucketContent *contents,
+			int commonPrefixesCount,
+			const char **commonPrefixes,
+			void *callbackData)
+	{
+		printf("%-22s",	"      Object Name");
+		printf("  %-5s  %-20s", "Size", "   Last Modified");
+		printf("\n");
+		printf("----------------------");
+		printf("  -----" "  --------------------");
+		printf("\n");
+
+	    for (int i = 0; i < contentsCount; i++) {
+		char timebuf[256];
+			char sizebuf[16];
+		const S3ListBucketContent *content = &(contents[i]);
+			time_t t = (time_t) content->lastModified;
+
+			strftime(timebuf, sizeof(timebuf), "%Y-%m-%dT%H:%M:%SZ", gmtime(&t));
+			sprintf(sizebuf, "%5llu", (unsigned long long) content->size);
+			printf("%-22s  %s  %s\n", content->key, sizebuf, timebuf);
+	    }
+
+	    return S3StatusOK;
+	}
+
+	S3ListBucketHandler listBucketHandler =
+	{
+		responseHandler,
+		&listBucketCallback
+	};
+	S3_list_bucket(&bucketContext, NULL, NULL, NULL, 0, NULL, 0, &listBucketHandler, NULL);
+
+The output will look something like this::
+
+   myphoto1.jpg	251262	2011-08-08T21:35:48.000Z
+   myphoto2.jpg	262518	2011-08-08T21:38:01.000Z
+
+
+Deleting a Bucket
+-----------------
+
+.. note::
+
+   The Bucket must be empty! Otherwise it won't work!
+
+.. code-block:: cpp
+
+	S3_delete_bucket(S3ProtocolHTTP, S3UriStylePath, access_key, secret_key, 0, host, sample_bucket, NULL, NULL, 0, &responseHandler, NULL);
+
+
+Creating an Object (from a file)
+--------------------------------
+
+This creates a file ``hello.txt``.
+
+.. code-block:: cpp
+
+	#include <sys/stat.h>
+	typedef struct put_object_callback_data
+	{
+	    FILE *infile;
+	    uint64_t contentLength;
+	} put_object_callback_data;
+
+
+	static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData)
+	{
+	    put_object_callback_data *data = (put_object_callback_data *) callbackData;
+
+	    int ret = 0;
+
+	    if (data->contentLength) {
+		int toRead = ((data->contentLength > (unsigned) bufferSize) ? (unsigned) bufferSize : data->contentLength);
+			ret = fread(buffer, 1, toRead, data->infile);
+	    }
+	    data->contentLength -= ret;
+	    return ret;
+	}
+
+	put_object_callback_data data;
+	struct stat statbuf;
+	if (stat(sample_file, &statbuf) == -1) {
+		fprintf(stderr, "\nERROR: Failed to stat file %s: ", sample_file);
+		perror(0);
+		exit(-1);
+	}
+
+	int contentLength = statbuf.st_size;
+	data.contentLength = contentLength;
+
+	if (!(data.infile = fopen(sample_file, "r"))) {
+		fprintf(stderr, "\nERROR: Failed to open input file %s: ", sample_file);
+		perror(0);
+		exit(-1);
+	}
+
+	S3PutObjectHandler putObjectHandler =
+	{
+		responseHandler,
+		&putObjectDataCallback
+	};
+
+	S3_put_object(&bucketContext, sample_key, contentLength, NULL, NULL, 0, &putObjectHandler, &data);
+	fclose(data.infile);
+
+
+Download an Object (to a file)
+------------------------------
+
+This downloads a file and prints the contents.
+
+.. code-block:: cpp
+
+	static S3Status getObjectDataCallback(int bufferSize, const char *buffer, void *callbackData)
+	{
+		FILE *outfile = (FILE *) callbackData;
+		size_t wrote = fwrite(buffer, 1, bufferSize, outfile);
+		return ((wrote < (size_t) bufferSize) ? S3StatusAbortedByCallback : S3StatusOK);
+	}
+
+	S3GetObjectHandler getObjectHandler =
+	{
+		responseHandler,
+		&getObjectDataCallback
+	};
+	FILE *outfile = stdout;
+	S3_get_object(&bucketContext, sample_key, NULL, 0, 0, NULL, 0, &getObjectHandler, outfile);
+
+
+Delete an Object
+----------------
+
+This deletes an object.
+
+.. code-block:: cpp
+
+	S3ResponseHandler deleteResponseHandler =
+	{
+		0,
+		&responseCompleteCallback
+	};
+	S3_delete_object(&bucketContext, sample_key, 0, 0, &deleteResponseHandler, 0);
+
+
+Change an Object's ACL
+----------------------
+
+This changes an object's ACL to grant full control to another user.
+
+
+.. code-block:: cpp
+
+	#include <string.h>
+	char ownerId[] = "owner";
+	char ownerDisplayName[] = "owner";
+	char granteeId[] = "grantee";
+	char granteeDisplayName[] = "grantee";
+
+	S3AclGrant grants[] = {
+		{
+			S3GranteeTypeCanonicalUser,
+			{{}},
+			S3PermissionFullControl
+		},
+		{
+			S3GranteeTypeCanonicalUser,
+			{{}},
+			S3PermissionReadACP
+		},
+		{
+			S3GranteeTypeAllUsers,
+			{{}},
+			S3PermissionRead
+		}
+	};
+
+	strncpy(grants[0].grantee.canonicalUser.id, ownerId, S3_MAX_GRANTEE_USER_ID_SIZE);
+	strncpy(grants[0].grantee.canonicalUser.displayName, ownerDisplayName, S3_MAX_GRANTEE_DISPLAY_NAME_SIZE);
+
+	strncpy(grants[1].grantee.canonicalUser.id, granteeId, S3_MAX_GRANTEE_USER_ID_SIZE);
+	strncpy(grants[1].grantee.canonicalUser.displayName, granteeDisplayName, S3_MAX_GRANTEE_DISPLAY_NAME_SIZE);
+
+	S3_set_acl(&bucketContext, sample_key, ownerId, ownerDisplayName, 3, grants, 0, &responseHandler, 0);
+
+
+Generate Object Download URL (signed)
+-------------------------------------
+
+This generates a signed download URL that will be valid for 5 minutes.
+
+.. code-block:: cpp
+
+	#include <time.h>
+	char buffer[S3_MAX_AUTHENTICATED_QUERY_STRING_SIZE];
+	int64_t expires = time(NULL) + 60 * 5; // Current time + 5 minutes
+
+	S3_generate_authenticated_query_string(buffer, &bucketContext, sample_key, expires, NULL, "GET");
+
diff --git a/doc/radosgw/s3/csharp.rst b/doc/radosgw/s3/csharp.rst
new file mode 100644
index 000000000..af1c6e4b5
--- /dev/null
+++ b/doc/radosgw/s3/csharp.rst
@@ -0,0 +1,199 @@
+.. _csharp:
+
+C# S3 Examples
+==============
+
+Creating a Connection
+---------------------
+
+This creates a connection so that you can interact with the server.
+
+.. code-block:: csharp
+
+	using System;
+	using Amazon;
+	using Amazon.S3;
+	using Amazon.S3.Model;
+
+	string accessKey = "put your access key here!";
+	string secretKey = "put your secret key here!";
+
+	AmazonS3Config config = new AmazonS3Config();
+	config.ServiceURL = "objects.dreamhost.com";
+
+	AmazonS3Client s3Client = new AmazonS3Client(
+		accessKey,
+		secretKey,
+		config
+		);
+
+
+Listing Owned Buckets
+---------------------
+
+This gets a list of Buckets that you own.
+This also prints out the bucket name and creation date of each bucket.
+
+.. code-block:: csharp
+
+	ListBucketsResponse response = client.ListBuckets();
+	foreach (S3Bucket b in response.Buckets)
+	{
+		Console.WriteLine("{0}\t{1}", b.BucketName, b.CreationDate);
+	}
+
+The output will look something like this::
+
+   mahbuckat1	2011-04-21T18:05:39.000Z
+   mahbuckat2	2011-04-21T18:05:48.000Z
+   mahbuckat3	2011-04-21T18:07:18.000Z
+
+
+Creating a Bucket
+-----------------
+This creates a new bucket called ``my-new-bucket``
+
+.. code-block:: csharp
+
+	PutBucketRequest request = new PutBucketRequest();
+	request.BucketName = "my-new-bucket";
+	client.PutBucket(request);
+
+Listing a Bucket's Content
+--------------------------
+
+This gets a list of objects in the bucket.
+This also prints out each object's name, the file size, and last
+modified date.
+
+.. code-block:: csharp
+
+	ListObjectsRequest request = new ListObjectsRequest();
+	request.BucketName = "my-new-bucket";
+	ListObjectsResponse response = client.ListObjects(request);
+	foreach (S3Object o in response.S3Objects)
+	{
+		Console.WriteLine("{0}\t{1}\t{2}", o.Key, o.Size, o.LastModified);
+	}
+
+The output will look something like this::
+
+   myphoto1.jpg	251262	2011-08-08T21:35:48.000Z
+   myphoto2.jpg	262518	2011-08-08T21:38:01.000Z
+
+
+Deleting a Bucket
+-----------------
+
+.. note::
+
+   The Bucket must be empty! Otherwise it won't work!
+
+.. code-block:: csharp
+
+	DeleteBucketRequest request = new DeleteBucketRequest();
+	request.BucketName = "my-new-bucket";
+	client.DeleteBucket(request);
+
+
+Forced Delete for Non-empty Buckets
+-----------------------------------
+
+.. attention::
+
+   not available
+
+
+Creating an Object
+------------------
+
+This creates a file ``hello.txt`` with the string ``"Hello World!"``
+
+.. code-block:: csharp
+
+	PutObjectRequest request = new PutObjectRequest();
+	request.BucketName  = "my-new-bucket";
+	request.Key         = "hello.txt";
+	request.ContentType = "text/plain";
+	request.ContentBody = "Hello World!";
+	client.PutObject(request);
+
+
+Change an Object's ACL
+----------------------
+
+This makes the object ``hello.txt`` to be publicly readable, and
+``secret_plans.txt`` to be private.
+
+.. code-block:: csharp
+
+	PutACLRequest request = new PutACLRequest();
+	request.BucketName = "my-new-bucket";
+	request.Key        = "hello.txt";
+	request.CannedACL  = S3CannedACL.PublicRead;
+	client.PutACL(request);
+
+	PutACLRequest request2 = new PutACLRequest();
+	request2.BucketName = "my-new-bucket";
+	request2.Key        = "secret_plans.txt";
+	request2.CannedACL  = S3CannedACL.Private;
+	client.PutACL(request2);
+
+
+Download an Object (to a file)
+------------------------------
+
+This downloads the object ``perl_poetry.pdf`` and saves it in
+``C:\Users\larry\Documents``
+
+.. code-block:: csharp
+
+	GetObjectRequest request = new GetObjectRequest();
+	request.BucketName = "my-new-bucket";
+	request.Key        = "perl_poetry.pdf";
+	GetObjectResponse response = client.GetObject(request);
+	response.WriteResponseStreamToFile("C:\\Users\\larry\\Documents\\perl_poetry.pdf");
+
+
+Delete an Object
+----------------
+
+This deletes the object ``goodbye.txt``
+
+.. code-block:: csharp
+
+	DeleteObjectRequest request = new DeleteObjectRequest();
+	request.BucketName = "my-new-bucket";
+	request.Key        = "goodbye.txt";
+	client.DeleteObject(request);
+
+
+Generate Object Download URLs (signed and unsigned)
+---------------------------------------------------
+
+This generates an unsigned download URL for ``hello.txt``. This works
+because we made ``hello.txt`` public by setting the ACL above.
+This then generates a signed download URL for ``secret_plans.txt`` that
+will work for 1 hour. Signed download URLs will work for the time
+period even if the object is private (when the time period is up, the
+URL will stop working).
+
+.. note::
+
+   The C# S3 Library does not have a method for generating unsigned
+   URLs, so the following example only shows generating signed URLs.
+
+.. code-block:: csharp
+
+	GetPreSignedUrlRequest request = new GetPreSignedUrlRequest();
+	request.BucketName = "my-bucket-name";
+	request.Key        = "secret_plans.txt";
+	request.Expires    = DateTime.Now.AddHours(1);
+	request.Protocol   = Protocol.HTTP;
+	string url = client.GetPreSignedURL(request);
+	Console.WriteLine(url);
+
+The output of this will look something like::
+
+   http://objects.dreamhost.com/my-bucket-name/secret_plans.txt?Signature=XXXXXXXXXXXXXXXXXXXXXXXXXXX&Expires=1316027075&AWSAccessKeyId=XXXXXXXXXXXXXXXXXXX
+
diff --git a/doc/radosgw/s3/java.rst b/doc/radosgw/s3/java.rst
new file mode 100644
index 000000000..057c09c2c
--- /dev/null
+++ b/doc/radosgw/s3/java.rst
@@ -0,0 +1,212 @@
+.. _java:
+
+Java S3 Examples
+================
+
+Setup
+-----
+
+The following examples may require some or all of the following java
+classes to be imported:
+
+.. code-block:: java
+
+	import java.io.ByteArrayInputStream;
+	import java.io.File;
+	import java.util.List;
+	import com.amazonaws.auth.AWSCredentials;
+	import com.amazonaws.auth.BasicAWSCredentials;
+	import com.amazonaws.util.StringUtils;
+	import com.amazonaws.services.s3.AmazonS3;
+	import com.amazonaws.services.s3.AmazonS3Client;
+	import com.amazonaws.services.s3.model.Bucket;
+	import com.amazonaws.services.s3.model.CannedAccessControlList;
+	import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
+	import com.amazonaws.services.s3.model.GetObjectRequest;
+	import com.amazonaws.services.s3.model.ObjectListing;
+	import com.amazonaws.services.s3.model.ObjectMetadata;
+	import com.amazonaws.services.s3.model.S3ObjectSummary;
+
+
+If you are just testing the Ceph Object Storage services, consider
+using HTTP protocol instead of HTTPS protocol. 
+
+First, import the ``ClientConfiguration`` and ``Protocol`` classes. 
+
+.. code-block:: java
+
+	import com.amazonaws.ClientConfiguration;
+	import com.amazonaws.Protocol;
+
+
+Then, define the client configuration, and add the client configuration
+as an argument for the S3 client.
+
+.. code-block:: java
+
+	AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
+			 	
+	ClientConfiguration clientConfig = new ClientConfiguration();
+	clientConfig.setProtocol(Protocol.HTTP);
+			
+	AmazonS3 conn = new AmazonS3Client(credentials, clientConfig);
+	conn.setEndpoint("endpoint.com");
+
+
+Creating a Connection
+---------------------
+
+This creates a connection so that you can interact with the server.
+
+.. code-block:: java
+
+	String accessKey = "insert your access key here!";
+	String secretKey = "insert your secret key here!";
+
+	AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
+	AmazonS3 conn = new AmazonS3Client(credentials);
+	conn.setEndpoint("objects.dreamhost.com");
+
+
+Listing Owned Buckets
+---------------------
+
+This gets a list of Buckets that you own.
+This also prints out the bucket name and creation date of each bucket.
+
+.. code-block:: java
+
+	List<Bucket> buckets = conn.listBuckets();
+	for (Bucket bucket : buckets) {
+		System.out.println(bucket.getName() + "\t" +
+			StringUtils.fromDate(bucket.getCreationDate()));
+	}
+
+The output will look something like this::
+
+   mahbuckat1	2011-04-21T18:05:39.000Z
+   mahbuckat2	2011-04-21T18:05:48.000Z
+   mahbuckat3	2011-04-21T18:07:18.000Z
+
+
+Creating a Bucket
+-----------------
+
+This creates a new bucket called ``my-new-bucket``
+
+.. code-block:: java
+
+	Bucket bucket = conn.createBucket("my-new-bucket");
+
+
+Listing a Bucket's Content
+--------------------------
+This gets a list of objects in the bucket.
+This also prints out each object's name, the file size, and last
+modified date.
+
+.. code-block:: java
+
+	ObjectListing objects = conn.listObjects(bucket.getName());
+	do {
+		for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
+			System.out.println(objectSummary.getKey() + "\t" +
+				objectSummary.getSize() + "\t" +
+				StringUtils.fromDate(objectSummary.getLastModified()));
+		}
+		objects = conn.listNextBatchOfObjects(objects);
+	} while (objects.isTruncated());
+
+The output will look something like this::
+
+   myphoto1.jpg	251262	2011-08-08T21:35:48.000Z
+   myphoto2.jpg	262518	2011-08-08T21:38:01.000Z
+
+
+Deleting a Bucket
+-----------------
+
+.. note::
+   The Bucket must be empty! Otherwise it won't work!
+
+.. code-block:: java
+
+	conn.deleteBucket(bucket.getName());
+
+
+Forced Delete for Non-empty Buckets
+-----------------------------------
+.. attention::
+   not available
+
+
+Creating an Object
+------------------
+
+This creates a file ``hello.txt`` with the string ``"Hello World!"``
+
+.. code-block:: java
+
+	ByteArrayInputStream input = new ByteArrayInputStream("Hello World!".getBytes());
+	conn.putObject(bucket.getName(), "hello.txt", input, new ObjectMetadata());
+
+
+Change an Object's ACL
+----------------------
+
+This makes the object ``hello.txt`` to be publicly readable, and
+``secret_plans.txt`` to be private.
+
+.. code-block:: java
+
+	conn.setObjectAcl(bucket.getName(), "hello.txt", CannedAccessControlList.PublicRead);
+	conn.setObjectAcl(bucket.getName(), "secret_plans.txt", CannedAccessControlList.Private);
+
+
+Download an Object (to a file)
+------------------------------
+
+This downloads the object ``perl_poetry.pdf`` and saves it in
+``/home/larry/documents``
+
+.. code-block:: java
+
+	conn.getObject(
+		new GetObjectRequest(bucket.getName(), "perl_poetry.pdf"),
+		new File("/home/larry/documents/perl_poetry.pdf")
+	);
+
+
+Delete an Object
+----------------
+
+This deletes the object ``goodbye.txt``
+
+.. code-block:: java
+
+	conn.deleteObject(bucket.getName(), "goodbye.txt");
+
+
+Generate Object Download URLs (signed and unsigned)
+---------------------------------------------------
+
+This generates an unsigned download URL for ``hello.txt``. This works
+because we made ``hello.txt`` public by setting the ACL above.
+This then generates a signed download URL for ``secret_plans.txt`` that
+will work for 1 hour. Signed download URLs will work for the time
+period even if the object is private (when the time period is up, the
+URL will stop working).
+
+.. note::
+   The java library does not have a method for generating unsigned
+   URLs, so the example below just generates a signed URL.
+
+.. code-block:: java
+
+	GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucket.getName(), "secret_plans.txt");
+	System.out.println(conn.generatePresignedUrl(request));
+
+The output will look something like this::
+
+   https://my-bucket-name.objects.dreamhost.com/secret_plans.txt?Signature=XXXXXXXXXXXXXXXXXXXXXXXXXXX&Expires=1316027075&AWSAccessKeyId=XXXXXXXXXXXXXXXXXXX
+
diff --git a/doc/radosgw/s3/objectops.rst b/doc/radosgw/s3/objectops.rst
new file mode 100644
index 000000000..2ac52607f
--- /dev/null
+++ b/doc/radosgw/s3/objectops.rst
@@ -0,0 +1,558 @@
+Object Operations
+=================
+
+Put Object
+----------
+Adds an object to a bucket. You must have write permissions on the bucket to perform this operation.
+
+
+Syntax
+~~~~~~
+
+::
+
+    PUT /{bucket}/{object} HTTP/1.1
+
+Request Headers
+~~~~~~~~~~~~~~~
+
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+| Name                 | Description                                | Valid Values                                                                  | Required   |
++======================+============================================+===============================================================================+============+
+| **content-md5**      | A base64 encoded MD-5 hash of the message. | A string. No defaults or constraints.                                         | No         |
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+| **content-type**     | A standard MIME type.                      | Any MIME type. Default: ``binary/octet-stream``                               | No         |
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+| **x-amz-meta-<...>** | User metadata.  Stored with the object.    | A string up to 8kb. No defaults.                                              | No         |
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+| **x-amz-acl**        | A canned ACL.                              | ``private``, ``public-read``, ``public-read-write``, ``authenticated-read``   | No         |
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+
+
+Copy Object
+-----------
+To copy an object, use ``PUT`` and specify a destination bucket and the object name.
+
+Syntax
+~~~~~~
+
+::
+
+    PUT /{dest-bucket}/{dest-object} HTTP/1.1
+    x-amz-copy-source: {source-bucket}/{source-object}
+
+Request Headers
+~~~~~~~~~~~~~~~
+
++--------------------------------------+-------------------------------------------------+------------------------+------------+
+| Name                                 | Description                                     | Valid Values           | Required   |
++======================================+=================================================+========================+============+
+| **x-amz-copy-source**                | The source bucket name + object name.           | {bucket}/{obj}         | Yes        |
++--------------------------------------+-------------------------------------------------+------------------------+------------+
+| **x-amz-acl**                        | A canned ACL.                                   | ``private``,           | No         |
+|                                      |                                                 | ``public-read``,       |            |
+|                                      |                                                 | ``public-read-write``, |            |
+|                                      |                                                 | ``authenticated-read`` |            |
++--------------------------------------+-------------------------------------------------+------------------------+------------+
+| **x-amz-copy-if-modified-since**     |  Copies only if modified since the timestamp.   |   Timestamp            | No         |
++--------------------------------------+-------------------------------------------------+------------------------+------------+
+| **x-amz-copy-if-unmodified-since**   |  Copies only if unmodified since the timestamp. |   Timestamp            | No         |
++--------------------------------------+-------------------------------------------------+------------------------+------------+
+| **x-amz-copy-if-match**              |  Copies only if object ETag matches ETag.       |   Entity Tag           | No         |
++--------------------------------------+-------------------------------------------------+------------------------+------------+
+| **x-amz-copy-if-none-match**         |  Copies only if object ETag doesn't match.      |   Entity Tag           | No         |
++--------------------------------------+-------------------------------------------------+------------------------+------------+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
++------------------------+-------------+-----------------------------------------------+
+| Name                   | Type        | Description                                   |
++========================+=============+===============================================+
+| **CopyObjectResult**   | Container   |  A container for the response elements.       |
++------------------------+-------------+-----------------------------------------------+
+| **LastModified**       | Date        |  The last modified date of the source object. |
++------------------------+-------------+-----------------------------------------------+
+| **Etag**               | String      |  The ETag of the new object.                  |
++------------------------+-------------+-----------------------------------------------+
+
+Remove Object
+-------------
+
+Removes an object. Requires WRITE permission set on the containing bucket.
+
+Syntax
+~~~~~~
+
+::
+
+    DELETE /{bucket}/{object} HTTP/1.1
+
+
+
+Get Object
+----------
+Retrieves an object from a bucket within RADOS. 
+
+Syntax
+~~~~~~
+
+::
+
+    GET /{bucket}/{object} HTTP/1.1
+
+Request Headers
+~~~~~~~~~~~~~~~
+
++---------------------------+------------------------------------------------+--------------------------------+------------+
+| Name                      | Description                                    | Valid Values                   | Required   |
++===========================+================================================+================================+============+
+| **range**                 | The range of the object to retrieve.           | Range: bytes=beginbyte-endbyte | No         |
++---------------------------+------------------------------------------------+--------------------------------+------------+
+| **if-modified-since**     | Gets only if modified since the timestamp.     | Timestamp                      | No         |
++---------------------------+------------------------------------------------+--------------------------------+------------+
+| **if-unmodified-since**   | Gets only if not modified since the timestamp. | Timestamp                      | No         |
++---------------------------+------------------------------------------------+--------------------------------+------------+
+| **if-match**              | Gets only if object ETag matches ETag.         | Entity Tag                     | No         |
++---------------------------+------------------------------------------------+--------------------------------+------------+
+| **if-none-match**         | Gets only if object ETag matches ETag.         | Entity Tag                     | No         |
++---------------------------+------------------------------------------------+--------------------------------+------------+
+
+Response Headers
+~~~~~~~~~~~~~~~~
+
++-------------------+--------------------------------------------------------------------------------------------+
+| Name              | Description                                                                                |
++===================+============================================================================================+
+| **Content-Range** | Data range, will only be returned if the range header field was specified in the request   |
++-------------------+--------------------------------------------------------------------------------------------+
+
+Get Object Info
+---------------
+
+Returns information about object. This request will return the same
+header information as with the Get Object request, but will include
+the metadata only, not the object data payload.
+
+Syntax
+~~~~~~
+
+::
+
+    HEAD /{bucket}/{object} HTTP/1.1
+
+Request Headers
+~~~~~~~~~~~~~~~
+
++---------------------------+------------------------------------------------+--------------------------------+------------+
+| Name                      | Description                                    | Valid Values                   | Required   |
++===========================+================================================+================================+============+
+| **range**                 | The range of the object to retrieve.           | Range: bytes=beginbyte-endbyte | No         |
++---------------------------+------------------------------------------------+--------------------------------+------------+
+| **if-modified-since**     | Gets only if modified since the timestamp.     | Timestamp                      | No         |
++---------------------------+------------------------------------------------+--------------------------------+------------+
+| **if-unmodified-since**   | Gets only if not modified since the timestamp. | Timestamp                      | No         |
++---------------------------+------------------------------------------------+--------------------------------+------------+
+| **if-match**              | Gets only if object ETag matches ETag.         | Entity Tag                     | No         |
++---------------------------+------------------------------------------------+--------------------------------+------------+
+| **if-none-match**         | Gets only if object ETag matches ETag.         | Entity Tag                     | No         |
++---------------------------+------------------------------------------------+--------------------------------+------------+
+
+Get Object ACL
+--------------
+
+Syntax
+~~~~~~
+
+::
+
+    GET /{bucket}/{object}?acl HTTP/1.1
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| Name                      | Type        | Description                                                                                  |
++===========================+=============+==============================================================================================+
+| ``AccessControlPolicy``   | Container   | A container for the response.                                                                |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``AccessControlList``     | Container   | A container for the ACL information.                                                         |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Owner``                 | Container   | A container for the object owner's ``ID`` and ``DisplayName``.                               |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``ID``                    | String      | The object owner's ID.                                                                       |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``DisplayName``           | String      | The object owner's display name.                                                             |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Grant``                 | Container   | A container for ``Grantee`` and ``Permission``.                                              |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Grantee``               | Container   | A container for the ``DisplayName`` and ``ID`` of the user receiving a grant of permission.  |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Permission``            | String      | The permission given to the ``Grantee`` object.                                              |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+
+
+
+Set Object ACL
+--------------
+
+Syntax
+~~~~~~
+
+::
+
+    PUT /{bucket}/{object}?acl
+
+Request Entities
+~~~~~~~~~~~~~~~~
+
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| Name                      | Type        | Description                                                                                  |
++===========================+=============+==============================================================================================+
+| ``AccessControlPolicy``   | Container   | A container for the response.                                                                |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``AccessControlList``     | Container   | A container for the ACL information.                                                         |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Owner``                 | Container   | A container for the object owner's ``ID`` and ``DisplayName``.                               |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``ID``                    | String      | The object owner's ID.                                                                       |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``DisplayName``           | String      | The object owner's display name.                                                             |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Grant``                 | Container   | A container for ``Grantee`` and ``Permission``.                                              |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Grantee``               | Container   | A container for the ``DisplayName`` and ``ID`` of the user receiving a grant of permission.  |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+| ``Permission``            | String      | The permission given to the ``Grantee`` object.                                              |
++---------------------------+-------------+----------------------------------------------------------------------------------------------+
+
+
+
+Initiate Multi-part Upload
+--------------------------
+
+Initiate a multi-part upload process.
+
+Syntax
+~~~~~~
+
+::
+
+    POST /{bucket}/{object}?uploads
+
+Request Headers
+~~~~~~~~~~~~~~~
+
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+| Name                 | Description                                | Valid Values                                                                  | Required   |
++======================+============================================+===============================================================================+============+
+| **content-md5**      | A base64 encoded MD-5 hash of the message. | A string. No defaults or constraints.                                         | No         |
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+| **content-type**     | A standard MIME type.                      | Any MIME type. Default: ``binary/octet-stream``                               | No         |
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+| **x-amz-meta-<...>** | User metadata.  Stored with the object.    | A string up to 8kb. No defaults.                                              | No         |
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+| **x-amz-acl**        | A canned ACL.                              | ``private``, ``public-read``, ``public-read-write``, ``authenticated-read``   | No         |
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| Name                                    | Type        | Description                                                                                              |
++=========================================+=============+==========================================================================================================+
+| ``InitiatedMultipartUploadsResult``     | Container   | A container for the results.                                                                             |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Bucket``                              | String      | The bucket that will receive the object contents.                                                        |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Key``                                 | String      | The key specified by the ``key`` request parameter (if any).                                             |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``UploadId``                            | String      | The ID specified by the ``upload-id`` request parameter identifying the multipart upload (if any).       |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+
+
+Multipart Upload Part
+---------------------
+
+Syntax
+~~~~~~
+
+::
+
+    PUT /{bucket}/{object}?partNumber=&uploadId= HTTP/1.1
+
+HTTP Response
+~~~~~~~~~~~~~
+
+The following HTTP response may be returned:
+
++---------------+----------------+--------------------------------------------------------------------------+
+| HTTP Status   | Status Code    | Description                                                              |
++===============+================+==========================================================================+
+| **404**       | NoSuchUpload   | Specified upload-id does not match any initiated upload on this object   |
++---------------+----------------+--------------------------------------------------------------------------+
+
+List Multipart Upload Parts
+---------------------------
+
+Syntax
+~~~~~~
+
+::
+
+    GET /{bucket}/{object}?uploadId=123 HTTP/1.1
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| Name                                    | Type        | Description                                                                                              |
++=========================================+=============+==========================================================================================================+
+| ``ListPartsResult``                     | Container   | A container for the results.                                                                             |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Bucket``                              | String      | The bucket that will receive the object contents.                                                        |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Key``                                 | String      | The key specified by the ``key`` request parameter (if any).                                             |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``UploadId``                            | String      | The ID specified by the ``upload-id`` request parameter identifying the multipart upload (if any).       |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Initiator``                           | Container   | Contains the ``ID`` and ``DisplayName`` of the user who initiated the upload.                            |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``ID``                                  | String      | The initiator's ID.                                                                                      |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``DisplayName``                         | String      | The initiator's display name.                                                                            |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Owner``                               | Container   | A container for the ``ID`` and ``DisplayName`` of the user who owns the uploaded object.                 |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``StorageClass``                        | String      | The method used to store the resulting object. ``STANDARD`` or ``REDUCED_REDUNDANCY``                    |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``PartNumberMarker``                    | String      | The part marker to use in a subsequent request if ``IsTruncated`` is ``true``. Precedes the list.        |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``NextPartNumberMarker``                | String      | The next part marker to use in a subsequent request if ``IsTruncated`` is ``true``. The end of the list. |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``MaxParts``                            | Integer     | The max parts allowed in the response as specified by the ``max-parts`` request parameter.               |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``IsTruncated``                         | Boolean     | If ``true``, only a subset of the object's upload contents were returned.                                |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Part``                                | Container   | A container for ``LastModified``, ``PartNumber``, ``ETag`` and ``Size`` elements.                        |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``LastModified``                        | Date        | Date and time at which the part was uploaded.                                                            |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``PartNumber``                          | Integer     | The identification number of the part.                                                                   |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``ETag``                                | String      | The part's entity tag.                                                                                   |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+| ``Size``                                | Integer     | The size of the uploaded part.                                                                           |
++-----------------------------------------+-------------+----------------------------------------------------------------------------------------------------------+
+
+
+
+Complete Multipart Upload
+-------------------------
+Assembles uploaded parts and creates a new object, thereby completing a multipart upload.
+
+Syntax
+~~~~~~
+
+::
+
+    POST /{bucket}/{object}?uploadId= HTTP/1.1
+
+Request Entities
+~~~~~~~~~~~~~~~~
+
++----------------------------------+-------------+-----------------------------------------------------+----------+
+| Name                             | Type        | Description                                         | Required |
++==================================+=============+=====================================================+==========+
+| ``CompleteMultipartUpload``      | Container   | A container consisting of one or more parts.        | Yes      |
++----------------------------------+-------------+-----------------------------------------------------+----------+
+| ``Part``                         | Container   | A container for the ``PartNumber`` and ``ETag``.    | Yes      |
++----------------------------------+-------------+-----------------------------------------------------+----------+
+| ``PartNumber``                   | Integer     | The identifier of the part.                         | Yes      |
++----------------------------------+-------------+-----------------------------------------------------+----------+
+| ``ETag``                         | String      | The part's entity tag.                              | Yes      |
++----------------------------------+-------------+-----------------------------------------------------+----------+
+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
++-------------------------------------+-------------+-------------------------------------------------------+
+| Name                                | Type        | Description                                           |
++=====================================+=============+=======================================================+
+| **CompleteMultipartUploadResult**   | Container   | A container for the response.                         |
++-------------------------------------+-------------+-------------------------------------------------------+
+| **Location**                        | URI         | The resource identifier (path) of the new object.     |
++-------------------------------------+-------------+-------------------------------------------------------+
+| **Bucket**                          | String      | The name of the bucket that contains the new object.  |
++-------------------------------------+-------------+-------------------------------------------------------+
+| **Key**                             | String      | The object's key.                                     |
++-------------------------------------+-------------+-------------------------------------------------------+
+| **ETag**                            | String      | The entity tag of the new object.                     |
++-------------------------------------+-------------+-------------------------------------------------------+
+
+Abort Multipart Upload
+----------------------
+
+Syntax
+~~~~~~
+
+::
+
+    DELETE /{bucket}/{object}?uploadId= HTTP/1.1
+
+
+
+Append Object
+-------------
+Append data to an object. You must have write permissions on the bucket to perform this operation.
+It is used to upload files in appending mode. The type of the objects created by the Append Object
+operation is Appendable Object, and the type of the objects uploaded with the Put Object operation is Normal Object.
+**Append Object can't be used if bucket versioning is enabled or suspended.**
+**Synced object will become normal in multisite, but you can still append to the original object.**
+**Compression and encryption features are disabled for Appendable objects.**
+
+Syntax
+~~~~~~
+
+::
+
+    PUT /{bucket}/{object}?append&position= HTTP/1.1
+
+Request Headers
+~~~~~~~~~~~~~~~
+
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+| Name                 | Description                                | Valid Values                                                                  | Required   |
++======================+============================================+===============================================================================+============+
+| **content-md5**      | A base64 encoded MD-5 hash of the message. | A string. No defaults or constraints.                                         | No         |
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+| **content-type**     | A standard MIME type.                      | Any MIME type. Default: ``binary/octet-stream``                               | No         |
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+| **x-amz-meta-<...>** | User metadata.  Stored with the object.    | A string up to 8kb. No defaults.                                              | No         |
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+| **x-amz-acl**        | A canned ACL.                              | ``private``, ``public-read``, ``public-read-write``, ``authenticated-read``   | No         |
++----------------------+--------------------------------------------+-------------------------------------------------------------------------------+------------+
+
+Response Headers
+~~~~~~~~~~~~~~~~
+
++--------------------------------+------------------------------------------------------------------+
+| Name                           | Description                                                      |
++================================+==================================================================+
+| **x-rgw-next-append-position** | Next position to append object                                   |
++--------------------------------+------------------------------------------------------------------+
+
+HTTP Response
+~~~~~~~~~~~~~
+
+The following HTTP response may be returned:
+
++---------------+----------------------------+---------------------------------------------------+
+| HTTP Status   | Status Code                | Description                                       |
++===============+============================+===================================================+
+| **409**       | PositionNotEqualToLength   | Specified position does not match object length   |
++---------------+----------------------------+---------------------------------------------------+
+| **409**       | ObjectNotAppendable        | Specified object can not be appended              |
++---------------+----------------------------+---------------------------------------------------+
+| **409**       | InvalidBucketstate         | Bucket versioning is enabled or suspended         |
++---------------+----------------------------+---------------------------------------------------+
+
+
+Put Object Retention
+--------------------
+Places an Object Retention configuration on an object.
+
+Syntax
+~~~~~~
+
+::
+
+    PUT /{bucket}/{object}?retention&versionId= HTTP/1.1
+
+Request Entities
+~~~~~~~~~~~~~~~~
+
++---------------------+-------------+-------------------------------------------------------------------------------+------------+
+| Name                | Type        | Description                                                                   |  Required  |
++=====================+=============+===============================================================================+============+
+| ``Retention``       | Container   | A container for the request.                                                  |    Yes     |
++---------------------+-------------+-------------------------------------------------------------------------------+------------+
+| ``Mode``            | String      | Retention mode for the specified object. Valid Values:  GOVERNANCE/COMPLIANCE |    Yes     |
++---------------------+-------------+--------------------------------------------------------------------------------------------+
+| ``RetainUntilDate`` | Timestamp   | Retention date. Format: 2020-01-05T00:00:00.000Z                              |    Yes     |
++---------------------+-------------+--------------------------------------------------------------------------------------------+
+
+
+Get Object Retention
+--------------------
+Gets an Object Retention configuration on an object.
+
+
+Syntax
+~~~~~~
+
+::
+
+    GET /{bucket}/{object}?retention&versionId= HTTP/1.1
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
++---------------------+-------------+-------------------------------------------------------------------------------+------------+
+| Name                | Type        | Description                                                                   |  Required  |
++=====================+=============+===============================================================================+============+
+| ``Retention``       | Container   | A container for the request.                                                  |    Yes     |
++---------------------+-------------+-------------------------------------------------------------------------------+------------+
+| ``Mode``            | String      | Retention mode for the specified object. Valid Values:  GOVERNANCE/COMPLIANCE |    Yes     |
++---------------------+-------------+--------------------------------------------------------------------------------------------+
+| ``RetainUntilDate`` | Timestamp   | Retention date. Format: 2020-01-05T00:00:00.000Z                              |    Yes     |
++---------------------+-------------+--------------------------------------------------------------------------------------------+
+
+
+Put Object Legal Hold
+---------------------
+Applies a Legal Hold configuration to the specified object.
+
+Syntax
+~~~~~~
+
+::
+
+    PUT /{bucket}/{object}?legal-hold&versionId= HTTP/1.1
+
+Request Entities
+~~~~~~~~~~~~~~~~
+
++----------------+-------------+----------------------------------------------------------------------------------------+------------+
+| Name           | Type        | Description                                                                            |  Required  |
++================+=============+========================================================================================+============+
+| ``LegalHold``  | Container   | A container for the request.                                                           |    Yes     |
++----------------+-------------+----------------------------------------------------------------------------------------+------------+
+| ``Status``     | String      | Indicates whether the specified object has a Legal Hold in place. Valid Values: ON/OFF |    Yes     |
++----------------+-------------+----------------------------------------------------------------------------------------+------------+
+
+
+Get Object Legal Hold
+---------------------
+Gets an object's current Legal Hold status.
+
+Syntax
+~~~~~~
+
+::
+
+    GET /{bucket}/{object}?legal-hold&versionId= HTTP/1.1
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
++----------------+-------------+----------------------------------------------------------------------------------------+------------+
+| Name           | Type        | Description                                                                            |  Required  |
++================+=============+========================================================================================+============+
+| ``LegalHold``  | Container   | A container for the request.                                                           |    Yes     |
++----------------+-------------+----------------------------------------------------------------------------------------+------------+
+| ``Status``     | String      | Indicates whether the specified object has a Legal Hold in place. Valid Values: ON/OFF |    Yes     |
++----------------+-------------+----------------------------------------------------------------------------------------+------------+
+
diff --git a/doc/radosgw/s3/perl.rst b/doc/radosgw/s3/perl.rst
new file mode 100644
index 000000000..f12e5c698
--- /dev/null
+++ b/doc/radosgw/s3/perl.rst
@@ -0,0 +1,192 @@
+.. _perl:
+
+Perl S3 Examples
+================
+
+Creating a Connection
+---------------------
+
+This creates a connection so that you can interact with the server.
+
+.. code-block:: perl
+
+	use Amazon::S3;
+	my $access_key = 'put your access key here!';
+	my $secret_key = 'put your secret key here!';
+
+	my $conn = Amazon::S3->new({
+		aws_access_key_id     => $access_key,
+		aws_secret_access_key => $secret_key,
+		host                  => 'objects.dreamhost.com',
+		secure                => 1,
+		retry                 => 1,
+	});
+
+
+Listing Owned Buckets
+---------------------
+
+This gets a list of `Amazon::S3::Bucket`_ objects that you own.
+We'll also print out the bucket name and creation date of each bucket.
+
+.. code-block:: perl
+
+	my @buckets = @{$conn->buckets->{buckets} || []};
+	foreach my $bucket (@buckets) {
+		print $bucket->bucket . "\t" . $bucket->creation_date . "\n";
+	}
+
+The output will look something like this::
+
+   mahbuckat1	2011-04-21T18:05:39.000Z
+   mahbuckat2	2011-04-21T18:05:48.000Z
+   mahbuckat3	2011-04-21T18:07:18.000Z
+
+
+Creating a Bucket
+-----------------
+
+This creates a new bucket called ``my-new-bucket``
+
+.. code-block:: perl
+
+	my $bucket = $conn->add_bucket({ bucket => 'my-new-bucket' });
+
+
+Listing a Bucket's Content
+--------------------------
+
+This gets a list of hashes with info about each object in the bucket.
+We'll also print out each object's name, the file size, and last
+modified date.
+
+.. code-block:: perl
+
+	my @keys = @{$bucket->list_all->{keys} || []};
+	foreach my $key (@keys) {
+		print "$key->{key}\t$key->{size}\t$key->{last_modified}\n";
+	}
+
+The output will look something like this::
+
+   myphoto1.jpg	251262	2011-08-08T21:35:48.000Z
+   myphoto2.jpg	262518	2011-08-08T21:38:01.000Z
+
+
+Deleting a Bucket
+-----------------
+
+.. note::
+   The Bucket must be empty! Otherwise it won't work!
+
+.. code-block:: perl
+
+	$conn->delete_bucket($bucket);
+
+
+Forced Delete for Non-empty Buckets
+-----------------------------------
+
+.. attention::
+
+   not available in the `Amazon::S3`_ perl module
+
+
+Creating an Object
+------------------
+
+This creates a file ``hello.txt`` with the string ``"Hello World!"``
+
+.. code-block:: perl
+
+	$bucket->add_key(
+		'hello.txt', 'Hello World!',
+		{ content_type => 'text/plain' },
+	);
+
+
+Change an Object's ACL
+----------------------
+
+This makes the object ``hello.txt`` to be publicly readable and
+``secret_plans.txt`` to be private.
+
+.. code-block:: perl
+
+	$bucket->set_acl({
+		key       => 'hello.txt',
+		acl_short => 'public-read',
+	});
+	$bucket->set_acl({
+		key       => 'secret_plans.txt',
+		acl_short => 'private',
+	});
+
+
+Download an Object (to a file)
+------------------------------
+
+This downloads the object ``perl_poetry.pdf`` and saves it in
+``/home/larry/documents/``
+
+.. code-block:: perl
+
+	$bucket->get_key_filename('perl_poetry.pdf', undef,
+		'/home/larry/documents/perl_poetry.pdf');
+
+
+Delete an Object
+----------------
+
+This deletes the object ``goodbye.txt``
+
+.. code-block:: perl
+
+	$bucket->delete_key('goodbye.txt');
+
+Generate Object Download URLs (signed and unsigned)
+---------------------------------------------------
+This generates an unsigned download URL for ``hello.txt``. This works
+because we made ``hello.txt`` public by setting the ACL above.
+Then this generates a signed download URL for ``secret_plans.txt`` that
+will work for 1 hour. Signed download URLs will work for the time
+period even if the object is private (when the time period is up, the
+URL will stop working).
+
+.. note::
+   The `Amazon::S3`_ module does not have a way to generate download
+   URLs, so we are going to be using another module instead. Unfortunately,
+   most modules for generating these URLs assume that you are using Amazon,
+   so we have had to go with using a more obscure module, `Muck::FS::S3`_. This
+   should be the same as Amazon's sample S3 perl module, but this sample
+   module is not in CPAN. So, you can either use CPAN to install
+   `Muck::FS::S3`_, or install Amazon's sample S3 module manually. If you go
+   the manual route, you can remove ``Muck::FS::`` from the example below.
+
+.. code-block:: perl
+
+	use Muck::FS::S3::QueryStringAuthGenerator;
+	my $generator = Muck::FS::S3::QueryStringAuthGenerator->new(
+		$access_key,
+		$secret_key,
+		0, # 0 means use 'http'. set this to 1 for 'https'
+		'objects.dreamhost.com',
+	);
+
+	my $hello_url = $generator->make_bare_url($bucket->bucket, 'hello.txt');
+	print $hello_url . "\n";
+
+	$generator->expires_in(3600); # 1 hour = 3600 seconds
+	my $plans_url = $generator->get($bucket->bucket, 'secret_plans.txt');
+	print $plans_url . "\n";
+
+The output will look something like this::
+
+   http://objects.dreamhost.com:80/my-bucket-name/hello.txt
+   http://objects.dreamhost.com:80/my-bucket-name/secret_plans.txt?Signature=XXXXXXXXXXXXXXXXXXXXXXXXXXX&Expires=1316027075&AWSAccessKeyId=XXXXXXXXXXXXXXXXXXX
+
+
+.. _`Amazon::S3`: http://search.cpan.org/~tima/Amazon-S3-0.441/lib/Amazon/S3.pm
+.. _`Amazon::S3::Bucket`: http://search.cpan.org/~tima/Amazon-S3-0.441/lib/Amazon/S3/Bucket.pm
+.. _`Muck::FS::S3`: http://search.cpan.org/~mike/Muck-0.02/
+
diff --git a/doc/radosgw/s3/php.rst b/doc/radosgw/s3/php.rst
new file mode 100644
index 000000000..4878a3489
--- /dev/null
+++ b/doc/radosgw/s3/php.rst
@@ -0,0 +1,214 @@
+.. _php:
+
+PHP S3 Examples
+===============
+
+Installing AWS PHP SDK
+----------------------
+
+This installs AWS PHP SDK using composer (see here_ how to install composer).
+
+.. _here: https://getcomposer.org/download/
+  
+.. code-block:: bash
+
+	$ composer install aws/aws-sdk-php
+
+Creating a Connection
+---------------------
+
+This creates a connection so that you can interact with the server.
+
+.. note::
+
+   The client initialization requires a region so we use ``''``.
+
+.. code-block:: php
+
+	<?php
+	
+	use Aws\S3\S3Client;
+	
+	define('AWS_KEY', 'place access key here');
+	define('AWS_SECRET_KEY', 'place secret key here');
+	$ENDPOINT = 'http://objects.dreamhost.com';
+
+	// require the amazon sdk from your composer vendor dir
+	require __DIR__.'/vendor/autoload.php';
+
+	// Instantiate the S3 class and point it at the desired host
+	$client = new S3Client([
+	    'region' => '',
+	    'version' => '2006-03-01',
+	    'endpoint' => $ENDPOINT,
+	    'credentials' => [
+                'key' => AWS_KEY,
+		'secret' => AWS_SECRET_KEY
+	    ],
+	    // Set the S3 class to use objects.dreamhost.com/bucket
+	    // instead of bucket.objects.dreamhost.com
+	    'use_path_style_endpoint' => true
+	]);
+
+Listing Owned Buckets
+---------------------
+This gets a ``AWS\Result`` instance that is more convenient to visit using array access way.
+This also prints out the bucket name and creation date of each bucket.
+
+.. code-block:: php
+
+	<?php
+	$listResponse = $client->listBuckets();
+	$buckets = $listResponse['Buckets'];
+	foreach ($buckets as $bucket) {
+    	    echo $bucket['Name'] . "\t" . $bucket['CreationDate'] . "\n";
+	}
+
+The output will look something like this::
+
+   mahbuckat1	2011-04-21T18:05:39.000Z
+   mahbuckat2	2011-04-21T18:05:48.000Z
+   mahbuckat3	2011-04-21T18:07:18.000Z
+
+
+Creating a Bucket
+-----------------
+
+This creates a new bucket called ``my-new-bucket`` and returns a
+``AWS\Result`` object.
+
+.. code-block:: php
+
+	<?php
+	$client->createBucket(['Bucket' => 'my-new-bucket']);
+
+
+List a Bucket's Content
+-----------------------
+
+This gets a ``AWS\Result`` instance that is more convenient to visit using array access way.
+This then prints out each object's name, the file size, and last modified date.
+
+.. code-block:: php
+
+	<?php
+	$objectsListResponse = $client->listObjects(['Bucket' => $bucketname]);
+	$objects = $objectsListResponse['Contents'] ?? [];
+	foreach ($objects as $object) {
+    	    echo $object['Key'] . "\t" . $object['Size'] . "\t" . $object['LastModified'] . "\n";
+	}
+
+.. note::
+
+   If there are more than 1000 objects in this bucket,
+   you need to check $objectsListResponse['isTruncated']
+   and run again with the name of the last key listed.
+   Keep doing this until isTruncated is not true.
+
+The output will look something like this if the bucket has some files::
+
+   myphoto1.jpg	251262	2011-08-08T21:35:48.000Z
+   myphoto2.jpg	262518	2011-08-08T21:38:01.000Z
+
+
+Deleting a Bucket
+-----------------
+
+This deletes the bucket called ``my-old-bucket`` and returns a
+``AWS\Result`` object
+
+.. note::
+
+   The Bucket must be empty! Otherwise it won't work!
+
+.. code-block:: php
+
+	<?php
+	$client->deleteBucket(['Bucket' => 'my-old-bucket']);
+
+
+Creating an Object
+------------------
+
+This creates an object ``hello.txt`` with the string ``"Hello World!"``
+
+.. code-block:: php
+
+	<?php
+	$client->putObject([
+    	    'Bucket' => 'my-bucket-name',
+    	    'Key' => 'hello.txt',
+    	    'Body' => "Hello World!"
+	]);
+
+
+Change an Object's ACL
+----------------------
+
+This makes the object ``hello.txt`` to be publicly readable and
+``secret_plans.txt`` to be private.
+
+.. code-block:: php
+
+	<?php
+	$client->putObjectAcl([
+    	    'Bucket' => 'my-bucket-name',
+    	    'Key' => 'hello.txt',
+    	    'ACL' => 'public-read'
+	]);
+	$client->putObjectAcl([
+    	    'Bucket' => 'my-bucket-name',
+    	    'Key' => 'secret_plans.txt',
+    	    'ACL' => 'private'
+	]);
+
+
+Delete an Object
+----------------
+
+This deletes the object ``goodbye.txt``
+
+.. code-block:: php
+
+	<?php
+	$client->deleteObject(['Bucket' => 'my-bucket-name', 'Key' => 'goodbye.txt']);
+
+
+Download an Object (to a file)
+------------------------------
+
+This downloads the object ``poetry.pdf`` and saves it in
+``/home/larry/documents/``
+
+.. code-block:: php
+
+	<?php
+	$object = $client->getObject(['Bucket' => 'my-bucket-name', 'Key' => 'poetry.pdf']);
+	file_put_contents('/home/larry/documents/poetry.pdf', $object['Body']->getContents());
+
+Generate Object Download URLs (signed and unsigned)
+---------------------------------------------------
+
+This generates an unsigned download URL for ``hello.txt``.
+This works because we made ``hello.txt`` public by setting
+the ACL above. This then generates a signed download URL
+for ``secret_plans.txt`` that will work for 1 hour.
+Signed download URLs will work for the time period even
+if the object is private (when the time period is up,
+the URL will stop working).
+
+.. code-block:: php
+
+	<?php
+	$hello_url = $client->getObjectUrl('my-bucket-name', 'hello.txt');
+	echo $hello_url."\n";
+	
+	$secret_plans_cmd = $client->getCommand('GetObject', ['Bucket' => 'my-bucket-name', 'Key' => 'secret_plans.txt']);
+	$request = $client->createPresignedRequest($secret_plans_cmd, '+1 hour');
+	echo $request->getUri()."\n";
+
+The output of this will look something like::
+
+   http://objects.dreamhost.com/my-bucket-name/hello.txt
+   http://objects.dreamhost.com/my-bucket-name/secret_plans.txt?X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=sandboxAccessKey%2F20190116%2F%2Fs3%2Faws4_request&X-Amz-Date=20190116T125520Z&X-Amz-SignedHeaders=host&X-Amz-Expires=3600&X-Amz-Signature=61921f07c73d7695e47a2192cf55ae030f34c44c512b2160bb5a936b2b48d923
+
diff --git a/doc/radosgw/s3/python.rst b/doc/radosgw/s3/python.rst
new file mode 100644
index 000000000..4b9faef11
--- /dev/null
+++ b/doc/radosgw/s3/python.rst
@@ -0,0 +1,186 @@
+.. _python:
+
+Python S3 Examples
+==================
+
+Creating a Connection
+---------------------
+
+This creates a connection so that you can interact with the server.
+
+.. code-block:: python
+
+	import boto
+	import boto.s3.connection
+	access_key = 'put your access key here!'
+	secret_key = 'put your secret key here!'
+
+	conn = boto.connect_s3(
+		aws_access_key_id = access_key,
+		aws_secret_access_key = secret_key,
+		host = 'objects.dreamhost.com',
+                #is_secure=False,               # uncomment if you are not using ssl
+		calling_format = boto.s3.connection.OrdinaryCallingFormat(),
+		)
+
+
+Listing Owned Buckets
+---------------------
+
+This gets a list of Buckets that you own.
+This also prints out the bucket name and creation date of each bucket.
+
+.. code-block:: python
+
+	for bucket in conn.get_all_buckets():
+		print "{name}\t{created}".format(
+			name = bucket.name,
+			created = bucket.creation_date,
+		)
+
+The output will look something like this::
+
+   mahbuckat1	2011-04-21T18:05:39.000Z
+   mahbuckat2	2011-04-21T18:05:48.000Z
+   mahbuckat3	2011-04-21T18:07:18.000Z
+
+
+Creating a Bucket
+-----------------
+
+This creates a new bucket called ``my-new-bucket``
+
+.. code-block:: python
+
+	bucket = conn.create_bucket('my-new-bucket')
+
+
+Listing a Bucket's Content
+--------------------------
+
+This gets a list of objects in the bucket.
+This also prints out each object's name, the file size, and last
+modified date.
+
+.. code-block:: python
+
+	for key in bucket.list():
+		print "{name}\t{size}\t{modified}".format(
+			name = key.name,
+			size = key.size,
+			modified = key.last_modified,
+			)
+
+The output will look something like this::
+
+   myphoto1.jpg	251262	2011-08-08T21:35:48.000Z
+   myphoto2.jpg	262518	2011-08-08T21:38:01.000Z
+
+
+Deleting a Bucket
+-----------------
+
+.. note::
+
+   The Bucket must be empty! Otherwise it won't work!
+
+.. code-block:: python
+
+	conn.delete_bucket(bucket.name)
+
+
+Forced Delete for Non-empty Buckets
+-----------------------------------
+
+.. attention::
+
+   not available in python
+
+
+Creating an Object
+------------------
+
+This creates a file ``hello.txt`` with the string ``"Hello World!"``
+
+.. code-block:: python
+
+	key = bucket.new_key('hello.txt')
+	key.set_contents_from_string('Hello World!')
+
+
+Change an Object's ACL
+----------------------
+
+This makes the object ``hello.txt`` to be publicly readable, and
+``secret_plans.txt`` to be private.
+
+.. code-block:: python
+
+	hello_key = bucket.get_key('hello.txt')
+	hello_key.set_canned_acl('public-read')
+	plans_key = bucket.get_key('secret_plans.txt')
+	plans_key.set_canned_acl('private')
+
+
+Download an Object (to a file)
+------------------------------
+
+This downloads the object ``perl_poetry.pdf`` and saves it in
+``/home/larry/documents/``
+
+.. code-block:: python
+
+	key = bucket.get_key('perl_poetry.pdf')
+	key.get_contents_to_filename('/home/larry/documents/perl_poetry.pdf')
+
+
+Delete an Object
+----------------
+
+This deletes the object ``goodbye.txt``
+
+.. code-block:: python
+
+	bucket.delete_key('goodbye.txt')
+
+
+Generate Object Download URLs (signed and unsigned)
+---------------------------------------------------
+
+This generates an unsigned download URL for ``hello.txt``. This works
+because we made ``hello.txt`` public by setting the ACL above.
+This then generates a signed download URL for ``secret_plans.txt`` that
+will work for 1 hour. Signed download URLs will work for the time
+period even if the object is private (when the time period is up, the
+URL will stop working).
+
+.. code-block:: python
+
+	hello_key = bucket.get_key('hello.txt')
+	hello_url = hello_key.generate_url(0, query_auth=False, force_http=True)
+	print hello_url
+
+	plans_key = bucket.get_key('secret_plans.txt')
+	plans_url = plans_key.generate_url(3600, query_auth=True, force_http=True)
+	print plans_url
+
+The output of this will look something like::
+
+   http://objects.dreamhost.com/my-bucket-name/hello.txt
+   http://objects.dreamhost.com/my-bucket-name/secret_plans.txt?Signature=XXXXXXXXXXXXXXXXXXXXXXXXXXX&Expires=1316027075&AWSAccessKeyId=XXXXXXXXXXXXXXXXXXX
+
+Using S3 API Extensions 
+-----------------------
+
+To use the boto3 client to tests the RadosGW extensions to the S3 API, the `extensions file`_ should be placed under: ``~/.aws/models/s3/2006-03-01/`` directory.
+For example, unordered list of objects could be fetched using:
+
+.. code-block:: python
+
+    print conn.list_objects(Bucket='my-new-bucket', AllowUnordered=True)
+
+
+Without the extensions file, in the above example, boto3 would complain that the ``AllowUnordered`` argument is invalid.
+
+
+.. _extensions file: https://github.com/ceph/ceph/blob/master/examples/boto3/service-2.sdk-extras.json
diff --git a/doc/radosgw/s3/ruby.rst b/doc/radosgw/s3/ruby.rst
new file mode 100644
index 000000000..435b3c630
--- /dev/null
+++ b/doc/radosgw/s3/ruby.rst
@@ -0,0 +1,364 @@
+.. _ruby:
+
+Ruby `AWS::SDK`_ Examples (aws-sdk gem ~>2)
+===========================================
+
+Settings
+---------------------
+
+You can setup the connection on global way:
+
+.. code-block:: ruby
+
+	Aws.config.update(
+		endpoint: 'https://objects.dreamhost.com.',
+		access_key_id: 'my-access-key',
+		secret_access_key: 'my-secret-key',
+		force_path_style: true, 
+		region: 'us-east-1'
+	)
+
+
+and instantiate a client object:
+
+.. code-block:: ruby
+
+    	s3_client = Aws::S3::Client.new
+
+Listing Owned Buckets
+---------------------
+
+This gets a list of buckets that you own.
+This also prints out the bucket name and creation date of each bucket.
+
+.. code-block:: ruby
+
+	s3_client.list_buckets.buckets.each do |bucket|
+		puts "#{bucket.name}\t#{bucket.creation_date}"
+	end
+
+The output will look something like this::
+
+   mahbuckat1	2011-04-21T18:05:39.000Z
+   mahbuckat2	2011-04-21T18:05:48.000Z
+   mahbuckat3	2011-04-21T18:07:18.000Z
+
+
+Creating a Bucket
+-----------------
+
+This creates a new bucket called ``my-new-bucket``
+
+.. code-block:: ruby
+
+	s3_client.create_bucket(bucket: 'my-new-bucket')
+
+If you want a private bucket: 
+
+`acl` option accepts: # private, public-read, public-read-write, authenticated-read
+
+.. code-block:: ruby
+
+	s3_client.create_bucket(bucket: 'my-new-bucket', acl: 'private')
+
+
+Listing a Bucket's Content
+--------------------------
+
+This gets a list of hashes with the contents of each object
+This also prints out each object's name, the file size, and last
+modified date.
+
+.. code-block:: ruby
+
+	s3_client.get_objects(bucket: 'my-new-bucket').contents.each do |object|
+		puts "#{object.key}\t#{object.size}\t#{object.last-modified}"
+	end
+
+The output will look something like this if the bucket has some files::
+
+   myphoto1.jpg	251262	2011-08-08T21:35:48.000Z
+   myphoto2.jpg	262518	2011-08-08T21:38:01.000Z
+
+
+Deleting a Bucket
+-----------------
+.. note::
+   The Bucket must be empty! Otherwise it won't work!
+
+.. code-block:: ruby
+
+	s3_client.delete_bucket(bucket: 'my-new-bucket')
+
+
+Forced Delete for Non-empty Buckets
+-----------------------------------
+First, you need to clear the bucket:
+
+.. code-block:: ruby
+
+	Aws::S3::Bucket.new('my-new-bucket', client: s3_client).clear!
+	
+after, you can destroy the bucket
+
+.. code-block:: ruby
+
+	s3_client.delete_bucket(bucket: 'my-new-bucket')
+
+
+Creating an Object
+------------------
+
+This creates a file ``hello.txt`` with the string ``"Hello World!"``
+
+.. code-block:: ruby
+
+	s3_client.put_object(
+		key: 'hello.txt',
+		body: 'Hello World!',
+		bucket: 'my-new-bucket',
+		content_type: 'text/plain'
+	)
+
+
+Change an Object's ACL
+----------------------
+
+This makes the object ``hello.txt`` to be publicly readable, and ``secret_plans.txt``
+to be private.
+
+.. code-block:: ruby
+
+	s3_client.put_object_acl(bucket: 'my-new-bucket', key: 'hello.txt', acl: 'public-read')
+
+	s3_client.put_object_acl(bucket: 'my-new-bucket', key: 'private.txt', acl: 'private')
+
+
+Download an Object (to a file)
+------------------------------
+
+This downloads the object ``poetry.pdf`` and saves it in
+``/home/larry/documents/``
+
+.. code-block:: ruby
+
+	s3_client.get_object(bucket: 'my-new-bucket', key: 'poetry.pdf', response_target: '/home/larry/documents/poetry.pdf')
+
+
+Delete an Object
+----------------
+
+This deletes the object ``goodbye.txt``
+
+.. code-block:: ruby
+
+	s3_client.delete_object(key: 'goodbye.txt', bucket: 'my-new-bucket')
+
+
+Generate Object Download URLs (signed and unsigned)
+---------------------------------------------------
+
+This generates an unsigned download URL for ``hello.txt``. This works
+because we made ``hello.txt`` public by setting the ACL above.
+This then generates a signed download URL for ``secret_plans.txt`` that
+will work for 1 hour. Signed download URLs will work for the time
+period even if the object is private (when the time period is up, the
+URL will stop working).
+
+.. code-block:: ruby
+
+	puts Aws::S3::Object.new(
+		key: 'hello.txt',
+		bucket_name: 'my-new-bucket',
+		client: s3_client
+	).public_url
+
+	puts Aws::S3::Object.new(
+		key: 'secret_plans.txt',
+		bucket_name: 'hermes_ceph_gem',
+		client: s3_client
+	).presigned_url(:get, expires_in: 60 * 60)
+
+The output of this will look something like::
+
+   http://objects.dreamhost.com/my-bucket-name/hello.txt
+   http://objects.dreamhost.com/my-bucket-name/secret_plans.txt?Signature=XXXXXXXXXXXXXXXXXXXXXXXXXXX&Expires=1316027075&AWSAccessKeyId=XXXXXXXXXXXXXXXXXXX
+
+.. _`AWS::SDK`: http://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html
+
+
+
+Ruby `AWS::S3`_ Examples (aws-s3 gem)
+=====================================
+
+Creating a Connection
+---------------------
+
+This creates a connection so that you can interact with the server.
+
+.. code-block:: ruby
+
+	AWS::S3::Base.establish_connection!(
+		:server            => 'objects.dreamhost.com',
+		:use_ssl           => true,
+		:access_key_id     => 'my-access-key',
+		:secret_access_key => 'my-secret-key'
+	)
+
+
+Listing Owned Buckets
+---------------------
+
+This gets a list of `AWS::S3::Bucket`_ objects that you own.
+This also prints out the bucket name and creation date of each bucket.
+
+.. code-block:: ruby
+
+	AWS::S3::Service.buckets.each do |bucket|
+		puts "#{bucket.name}\t#{bucket.creation_date}"
+	end
+
+The output will look something like this::
+
+   mahbuckat1	2011-04-21T18:05:39.000Z
+   mahbuckat2	2011-04-21T18:05:48.000Z
+   mahbuckat3	2011-04-21T18:07:18.000Z
+
+
+Creating a Bucket
+-----------------
+
+This creates a new bucket called ``my-new-bucket``
+
+.. code-block:: ruby
+
+	AWS::S3::Bucket.create('my-new-bucket')
+
+
+Listing a Bucket's Content
+--------------------------
+
+This gets a list of hashes with the contents of each object
+This also prints out each object's name, the file size, and last
+modified date.
+
+.. code-block:: ruby
+
+	new_bucket = AWS::S3::Bucket.find('my-new-bucket')
+	new_bucket.each do |object|
+		puts "#{object.key}\t#{object.about['content-length']}\t#{object.about['last-modified']}"
+	end
+
+The output will look something like this if the bucket has some files::
+
+   myphoto1.jpg	251262	2011-08-08T21:35:48.000Z
+   myphoto2.jpg	262518	2011-08-08T21:38:01.000Z
+
+
+Deleting a Bucket
+-----------------
+.. note::
+   The Bucket must be empty! Otherwise it won't work!
+
+.. code-block:: ruby
+
+	AWS::S3::Bucket.delete('my-new-bucket')
+
+
+Forced Delete for Non-empty Buckets
+-----------------------------------
+
+.. code-block:: ruby
+
+	AWS::S3::Bucket.delete('my-new-bucket', :force => true)
+
+
+Creating an Object
+------------------
+
+This creates a file ``hello.txt`` with the string ``"Hello World!"``
+
+.. code-block:: ruby
+
+	AWS::S3::S3Object.store(
+		'hello.txt',
+		'Hello World!',
+		'my-new-bucket',
+		:content_type => 'text/plain'
+	)
+
+
+Change an Object's ACL
+----------------------
+
+This makes the object ``hello.txt`` to be publicly readable, and ``secret_plans.txt``
+to be private.
+
+.. code-block:: ruby
+
+	policy = AWS::S3::S3Object.acl('hello.txt', 'my-new-bucket')
+	policy.grants = [ AWS::S3::ACL::Grant.grant(:public_read) ]
+	AWS::S3::S3Object.acl('hello.txt', 'my-new-bucket', policy)
+
+	policy = AWS::S3::S3Object.acl('secret_plans.txt', 'my-new-bucket')
+	policy.grants = []
+	AWS::S3::S3Object.acl('secret_plans.txt', 'my-new-bucket', policy)
+
+
+Download an Object (to a file)
+------------------------------
+
+This downloads the object ``poetry.pdf`` and saves it in
+``/home/larry/documents/``
+
+.. code-block:: ruby
+
+	open('/home/larry/documents/poetry.pdf', 'w') do |file|
+		AWS::S3::S3Object.stream('poetry.pdf', 'my-new-bucket') do |chunk|
+			file.write(chunk)
+		end
+	end
+
+
+Delete an Object
+----------------
+
+This deletes the object ``goodbye.txt``
+
+.. code-block:: ruby
+
+	AWS::S3::S3Object.delete('goodbye.txt', 'my-new-bucket')
+
+
+Generate Object Download URLs (signed and unsigned)
+---------------------------------------------------
+
+This generates an unsigned download URL for ``hello.txt``. This works
+because we made ``hello.txt`` public by setting the ACL above.
+This then generates a signed download URL for ``secret_plans.txt`` that
+will work for 1 hour. Signed download URLs will work for the time
+period even if the object is private (when the time period is up, the
+URL will stop working).
+
+.. code-block:: ruby
+
+	puts AWS::S3::S3Object.url_for(
+		'hello.txt',
+		'my-new-bucket',
+		:authenticated => false
+	)
+
+	puts AWS::S3::S3Object.url_for(
+		'secret_plans.txt',
+		'my-new-bucket',
+		:expires_in => 60 * 60
+	)
+
+The output of this will look something like::
+
+   http://objects.dreamhost.com/my-bucket-name/hello.txt
+   http://objects.dreamhost.com/my-bucket-name/secret_plans.txt?Signature=XXXXXXXXXXXXXXXXXXXXXXXXXXX&Expires=1316027075&AWSAccessKeyId=XXXXXXXXXXXXXXXXXXX
+
+.. _`AWS::S3`: http://amazon.rubyforge.org/
+.. _`AWS::S3::Bucket`: http://amazon.rubyforge.org/doc/
+
diff --git a/doc/radosgw/s3/serviceops.rst b/doc/radosgw/s3/serviceops.rst
new file mode 100644
index 000000000..54b6ca375
--- /dev/null
+++ b/doc/radosgw/s3/serviceops.rst
@@ -0,0 +1,69 @@
+Service Operations
+==================
+
+List Buckets
+------------
+``GET /`` returns a list of buckets created by the user making the request. ``GET /`` only
+returns buckets created by an authenticated user. You cannot make an anonymous request.
+
+Syntax
+~~~~~~
+::
+
+	GET / HTTP/1.1
+	Host: cname.domain.com
+
+	Authorization: AWS {access-key}:{hash-of-header-and-secret}
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
++----------------------------+-------------+-----------------------------------------------------------------+
+| Name                       | Type        | Description                                                     |
++============================+=============+=================================================================+
+| ``Buckets``                | Container   | Container for list of buckets.                                  |
++----------------------------+-------------+-----------------------------------------------------------------+
+| ``Bucket``                 | Container   | Container for bucket information.                               |
++----------------------------+-------------+-----------------------------------------------------------------+
+| ``Name``                   | String      | Bucket name.                                                    |
++----------------------------+-------------+-----------------------------------------------------------------+
+| ``CreationDate``           | Date        | UTC time when the bucket was created.                           |
++----------------------------+-------------+-----------------------------------------------------------------+
+| ``ListAllMyBucketsResult`` | Container   | A container for the result.                                     |
++----------------------------+-------------+-----------------------------------------------------------------+
+| ``Owner``                  | Container   | A container for the bucket owner's ``ID`` and ``DisplayName``.  |
++----------------------------+-------------+-----------------------------------------------------------------+
+| ``ID``                     | String      | The bucket owner's ID.                                          |
++----------------------------+-------------+-----------------------------------------------------------------+
+| ``DisplayName``            | String      | The bucket owner's display name.                                |
++----------------------------+-------------+-----------------------------------------------------------------+
+
+
+Get Usage Stats
+---------------
+
+Gets usage stats per user, similar to the admin command :ref:`rgw_user_usage_stats`.
+
+Syntax
+~~~~~~
+::
+
+	GET /?usage HTTP/1.1
+	Host: cname.domain.com
+
+	Authorization: AWS {access-key}:{hash-of-header-and-secret}
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
++----------------------------+-------------+-----------------------------------------------------------------+
+| Name                       | Type        | Description                                                     |
++============================+=============+=================================================================+
+| ``Summary``                | Container   | Summary of total stats by user.                                 |
++----------------------------+-------------+-----------------------------------------------------------------+
+| ``TotalBytes``             | Integer     | Bytes used by user                                              |
++----------------------------+-------------+-----------------------------------------------------------------+
+| ``TotalBytesRounded``      | Integer     | Bytes rounded to the nearest 4k boundary                        |
++----------------------------+-------------+-----------------------------------------------------------------+
+| ``TotalEntries``           | Integer     | Total object entries                                            |
++----------------------------+-------------+-----------------------------------------------------------------+
diff --git a/doc/radosgw/s3select.rst b/doc/radosgw/s3select.rst
new file mode 100644
index 000000000..fc7ae339c
--- /dev/null
+++ b/doc/radosgw/s3select.rst
@@ -0,0 +1,267 @@
+===============
+ Ceph s3 select 
+===============
+
+.. contents::
+
+Overview
+--------
+
+    | The purpose of the **s3 select** engine is to create an efficient pipe between user client and storage nodes (the engine should be close as possible to storage).
+    | It enables selection of a restricted subset of (structured) data stored in an S3 object using an SQL-like syntax.
+    | It also enables for higher level analytic-applications (such as SPARK-SQL) , using that feature to improve their latency and throughput.
+
+    | For example, a s3-object of several GB (CSV file), a user needs to extract a single column which filtered by another column.
+    | As the following query:
+    | ``select customer-id from s3Object where age>30 and age<65;``
+
+    | Currently the whole s3-object must retrieve from OSD via RGW before filtering and extracting data.
+    | By "pushing down" the query into OSD , it's possible to save a lot of network and CPU(serialization / deserialization).
+
+    | **The bigger the object, and the more accurate the query, the better the performance**.
+ 
+Basic workflow
+--------------
+    
+    | S3-select query is sent to RGW via `AWS-CLI <https://docs.aws.amazon.com/cli/latest/reference/s3api/select-object-content.html>`_
+
+    | It passes the authentication and permission process as an incoming message (POST).
+    | **RGWSelectObj_ObjStore_S3::send_response_data** is the “entry point�, it handles each fetched chunk according to input object-key.
+    | **send_response_data** is first handling the input query, it extracts the query and other CLI parameters.
+   
+    | Per each new fetched chunk (~4m), RGW executes s3-select query on it.    
+    | The current implementation supports CSV objects and since chunks are randomly “cutting� the CSV rows in the middle, those broken-lines (first or last per chunk) are skipped while processing the query.   
+    | Those “broken� lines are stored and later merged with the next broken-line (belong to the next chunk), and finally processed.
+   
+    | Per each processed chunk an output message is formatted according to `AWS specification <https://docs.aws.amazon.com/AmazonS3/latest/API/archive-RESTObjectSELECTContent.html#archive-RESTObjectSELECTContent-responses>`_ and sent back to the client.
+    | RGW supports the following response: ``{:event-type,records} {:content-type,application/octet-stream} {:message-type,event}``.
+    | For aggregation queries the last chunk should be identified as the end of input, following that the s3-select-engine initiates end-of-process and produces an aggregate result.  
+
+        
+Basic functionalities
+~~~~~~~~~~~~~~~~~~~~~
+
+    | **S3select** has a definite set of functionalities that should be implemented (if we wish to stay compliant with AWS), currently only a portion of it is implemented.
+    
+    | The implemented software architecture supports basic arithmetic expressions, logical and compare expressions, including nested function calls and casting operators, that alone enables the user reasonable flexibility. 
+    | review the below s3-select-feature-table_.
+
+
+Error Handling
+~~~~~~~~~~~~~~
+
+    | Any error occurs while the input query processing, i.e. parsing phase or execution phase, is returned to client as response error message.
+
+    | Fatal severity (attached to the exception) will end query execution immediately, other error severity are counted, upon reaching 100, it ends query execution with an error message.
+
+
+
+
+.. _s3-select-feature-table:
+
+Features Support
+----------------
+
+  | Currently only part of `AWS select command <https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-glacier-select-sql-reference-select.html>`_ is implemented, table bellow describes what is currently supported.
+  | The following table describes the current implementation for s3-select functionalities:
+
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| Feature                         | Detailed        | Example                                                               |
++=================================+=================+=======================================================================+
+| Arithmetic operators            | ^ * / + - ( )   | select (int(_1)+int(_2))*int(_9) from stdin;                          |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+|                                 |                 | select ((1+2)*3.14) ^ 2 from stdin;                                   |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| Compare operators               | > < >= <= == != | select _1,_2 from stdin where (int(1)+int(_3))>int(_5);               |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| logical operator                | AND OR          | select count(*) from stdin where int(1)>123 and int(_5)<200;          |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| casting operator                | int(expression) | select int(_1),int( 1.2 + 3.4) from stdin;                            |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+|                                 |float(expression)| select float(1.2) from stdin;                                         |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+|                                 | timestamp(...)  | select timestamp("1999:10:10-12:23:44") from stdin;                   |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| Aggregation Function            | sum             | select sum(int(_1)) from stdin;                                       |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| Aggregation Function            | min             | select min( int(_1) * int(_5) ) from stdin;                           |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| Aggregation Function            | max             | select max(float(_1)),min(int(_5)) from stdin;                        |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| Aggregation Function            | count           | select count(*) from stdin where (int(1)+int(_3))>int(_5);            |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| Timestamp Functions             | extract         | select count(*) from stdin where                                      |
+|                                 |                 | extract("year",timestamp(_2)) > 1950                                  |    
+|                                 |                 | and extract("year",timestamp(_1)) < 1960;                             |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| Timestamp Functions             | dateadd         | select count(0) from stdin where                                      |
+|                                 |                 | datediff("year",timestamp(_1),dateadd("day",366,timestamp(_1))) == 1; |  
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| Timestamp Functions             | datediff        | select count(0) from stdin where                                      |  
+|                                 |                 | datediff("month",timestamp(_1),timestamp(_2))) == 2;                  | 
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| Timestamp Functions             | utcnow          | select count(0) from stdin where                                      |
+|                                 |                 | datediff("hours",utcnow(),dateadd("day",1,utcnow())) == 24 ;          |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| String Functions                | substr          | select count(0) from stdin where                                      |
+|                                 |                 | int(substr(_1,1,4))>1950 and int(substr(_1,1,4))<1960;                |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| alias support                   |                 |  select int(_1) as a1, int(_2) as a2 , (a1+a2) as a3                  | 
+|                                 |                 |  from stdin where a3>100 and a3<300;                                  |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+
+s3-select function interfaces
+-----------------------------
+
+Timestamp functions
+~~~~~~~~~~~~~~~~~~~
+    | The `timestamp functionalities <https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-glacier-select-sql-reference-date.html>`_ is partially implemented.
+    | the casting operator( ``timestamp( string )`` ), converts string to timestamp basic type.
+    | Currently it can convert the following pattern ``yyyy:mm:dd hh:mi:dd``
+
+    | ``extract( date-part , timestamp)`` : function return integer according to date-part extract from input timestamp.
+    | supported date-part : year,month,week,day.
+
+    | ``dateadd(date-part , integer,timestamp)`` : function return timestamp, a calculation results of input timestamp and date-part.
+    | supported data-part : year,month,day.
+
+    | ``datediff(date-part,timestamp,timestamp)`` : function return an integer, a calculated result for difference between 2 timestamps according to date-part.
+    | supported date-part : year,month,day,hours.  
+
+
+    | ``utcnow()`` : return timestamp of current time.
+
+Aggregation functions
+~~~~~~~~~~~~~~~~~~~~~
+
+    | ``count()`` : return integer according to number of rows matching condition(if such exist).
+
+    | ``sum(expression)`` : return a summary of expression per all rows matching condition(if such exist).
+
+    | ``max(expression)`` : return the maximal result for all expressions matching condition(if such exist).
+
+    | ``min(expression)`` : return the minimal result for all expressions matching condition(if such exist).
+
+String functions
+~~~~~~~~~~~~~~~~
+
+    | ``substr(string,from,to)`` : return a string extract from input string according to from,to inputs.
+
+
+Alias
+~~~~~
+    | **Alias** programming-construct is an essential part of s3-select language, it enables much better programming especially with objects containing many columns or in the case of complex queries.
+    
+    | Upon parsing the statement containing alias construct, it replaces alias with reference to correct projection column, on query execution time the reference is evaluated as any other expression.
+
+    | There is a risk that self(or cyclic) reference may occur causing stack-overflow(endless-loop), for that concern upon evaluating an alias, it is validated for cyclic reference.
+    
+    | Alias also maintains result-cache, meaning upon using the same alias more than once, it’s not evaluating the same expression again(it will return the same result),instead it uses the result from cache.
+
+    | Of Course, per each new row the cache is invalidated.
+
+Sending Query to RGW
+--------------------
+
+   | Any http-client can send s3-select request to RGW, it must be compliant with `AWS Request syntax <https://docs.aws.amazon.com/AmazonS3/latest/API/API_SelectObjectContent.html#API_SelectObjectContent_RequestSyntax>`_.
+
+
+
+   | Sending s3-select request to RGW using AWS cli, should follow `AWS command reference <https://docs.aws.amazon.com/cli/latest/reference/s3api/select-object-content.html>`_.
+   | bellow is an example for it.
+
+::
+
+ aws --endpoint-url http://localhost:8000 s3api select-object-content 
+  --bucket {BUCKET-NAME}  
+  --expression-type 'SQL'     
+  --input-serialization 
+  '{"CSV": {"FieldDelimiter": "," , "QuoteCharacter": "\"" , "RecordDelimiter" : "\n" , "QuoteEscapeCharacter" : "\\" , "FileHeaderInfo": "USE" }, "CompressionType": "NONE"}' 
+  --output-serialization '{"CSV": {}}' 
+  --key {OBJECT-NAME} 
+  --expression "select count(0) from stdin where int(_1)<10;" output.csv
+
+Syntax
+~~~~~~
+
+    | **Input serialization** (Implemented), it let the user define the CSV definitions; the default values are {\\n} for row-delimiter {,} for field delimiter, {"} for quote, {\\} for escape characters.
+    | it handle the **csv-header-info**, the first row in input object containing the schema.
+    | **Output serialization** is currently not implemented, the same for **compression-type**.
+
+    | s3-select engine contain a CSV parser, which parse s3-objects as follows.   
+    | - each row ends with row-delimiter.
+    | - field-separator separates between adjacent columns, successive field separator define NULL column.
+    | - quote-character overrides field separator, meaning , field separator become as any character between quotes.
+    | - escape character disables any special characters, except for row delimiter.
+    
+    | Below are examples for CSV parsing rules.
+
+
+CSV parsing behavior
+--------------------
+
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+| Feature                         | Description     | input ==> tokens                                                      |
++=================================+=================+=======================================================================+
+|     NULL                        | successive      | ,,1,,2,    ==> {null}{null}{1}{null}{2}{null}                         |
+|                                 | field delimiter |                                                                       |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+|     QUOTE                       | quote character | 11,22,"a,b,c,d",last ==> {11}{22}{"a,b,c,d"}{last}                    |
+|                                 | overrides       |                                                                       |
+|                                 | field delimiter |                                                                       |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+|     Escape                      | escape char     | 11,22,str=\\"abcd\\"\\,str2=\\"123\\",last                            |
+|                                 | overrides       | ==> {11}{22}{str="abcd",str2="123"}{last}                             |
+|                                 | meta-character. |                                                                       |
+|                                 | escape removed  |                                                                       |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+|     row delimiter               | no close quote, | 11,22,a="str,44,55,66                                                 |
+|                                 | row delimiter is| ==> {11}{22}{a="str,44,55,66}                                         |
+|                                 | closing line    |                                                                       |
++---------------------------------+-----------------+-----------------------------------------------------------------------+
+|     csv header info             | FileHeaderInfo  | "**USE**" value means each token on first line is column-name,        |
+|                                 | tag             | "**IGNORE**" value means to skip the first line                       |
++---------------------------------+-----------------+-----------------------------------------------------------------------+       
+
+
+BOTO3
+-----
+
+ | using BOTO3 is "natural" and easy due to AWS-cli support. 
+
+::
+
+
+ def run_s3select(bucket,key,query,column_delim=",",row_delim="\n",quot_char='"',esc_char='\\',csv_header_info="NONE"):
+    s3 = boto3.client('s3',
+        endpoint_url=endpoint,
+        aws_access_key_id=access_key,
+        region_name=region_name,
+        aws_secret_access_key=secret_key)
+        
+
+
+    r = s3.select_object_content(
+        Bucket=bucket,
+        Key=key,
+        ExpressionType='SQL',
+        InputSerialization = {"CSV": {"RecordDelimiter" : row_delim, "FieldDelimiter" : column_delim,"QuoteEscapeCharacter": esc_char, "QuoteCharacter": quot_char, "FileHeaderInfo": csv_header_info}, "CompressionType": "NONE"},
+        OutputSerialization = {"CSV": {}},
+        Expression=query,)
+
+    result = ""
+    for event in r['Payload']:
+        if 'Records' in event:
+            records = event['Records']['Payload'].decode('utf-8')
+            result += records
+
+    return result
+
+
+
+
+  run_s3select(
+  "my_bucket",
+  "my_csv_object",
+  "select int(_1) as a1, int(_2) as a2 , (a1+a2) as a3 from stdin where a3>100 and a3<300;")
+
diff --git a/doc/radosgw/session-tags.rst b/doc/radosgw/session-tags.rst
new file mode 100644
index 000000000..abfb89e87
--- /dev/null
+++ b/doc/radosgw/session-tags.rst
@@ -0,0 +1,424 @@
+=======================================================
+Session tags for Attribute Based Access Control in STS
+=======================================================
+
+Session tags are key-value pairs that can be passed while federating a user (currently it
+is only supported as part of the web token passed to AssumeRoleWithWebIdentity). The session
+tags are passed along as aws:PrincipalTag in the session credentials (temporary credentials)
+that is returned back by STS. These Principal Tags consists of the session tags that come in
+as part of the web token and the tags that are attached to the role being assumed. Please note
+that the tags have to be always specified in the following namespace: https://aws.amazon.com/tags.
+
+An example of the session tags that are passed in by the IDP in the web token is as follows:
+
+.. code-block:: python
+
+    {
+        "jti": "947960a3-7e91-4027-99f6-da719b0d4059",
+        "exp": 1627438044,
+        "nbf": 0,
+        "iat": 1627402044,
+        "iss": "http://localhost:8080/auth/realms/quickstart",
+        "aud": "app-profile-jsp",
+        "sub": "test",
+        "typ": "ID",
+        "azp": "app-profile-jsp",
+        "auth_time": 0,
+        "session_state": "3a46e3e7-d198-4a64-8b51-69682bcfc670",
+        "preferred_username": "test",
+        "email_verified": false,
+        "acr": "1",
+        "https://aws.amazon.com/tags": [
+            {
+                "principal_tags": {
+                    "Department": [
+                        "Engineering",
+                        "Marketing"
+                    ]
+                }
+            }
+        ],
+        "client_id": "app-profile-jsp",
+        "username": "test",
+        "active": true
+    }
+
+Steps to configure Keycloak to pass tags in the web token are described here:doc:`keycloak`.
+
+The trust policy must have 'sts:TagSession' permission if the web token passed in by the federated user contains session tags, otherwise
+the AssumeRoleWithWebIdentity action will fail. An example of the trust policy with sts:TagSession is as follows:
+
+.. code-block:: python
+
+    {
+	    "Version":"2012-10-17",
+	    "Statement":[
+	    {
+	        "Effect":"Allow",
+	        "Action":["sts:AssumeRoleWithWebIdentity","sts:TagSession"],
+	        "Principal":{"Federated":["arn:aws:iam:::oidc-provider/localhost:8080/auth/realms/quickstart"]},
+	        "Condition":{"StringEquals":{"localhost:8080/auth/realms/quickstart:sub":"test"}}
+	    }]
+	}
+
+Tag Keys
+========
+
+The following are the tag keys that can be used in the role's trust policy or the role's permission policy:
+
+1. aws:RequestTag: This key is used to compare the key-value pair passed in the request with the key-value pair
+in the role's trust policy. In case of AssumeRoleWithWebIdentity, the session tags that are passed by the idp
+in the web token can be used as aws:RequestTag in the role's trust policy based on which a federated user can be
+allowed to assume a role.
+
+An example of a role trust policy that uses aws:RequestTag is as follows:
+
+.. code-block:: python
+
+    {
+	    "Version":"2012-10-17",
+	    "Statement":[
+	    {
+	        "Effect":"Allow",
+	        "Action":["sts:AssumeRoleWithWebIdentity","sts:TagSession"],
+	        "Principal":{"Federated":["arn:aws:iam:::oidc-provider/localhost:8080/auth/realms/quickstart"]},
+	        "Condition":{"StringEquals":{"aws:RequestTag/Department":"Engineering"}}
+	    }]
+	}
+
+2. aws:PrincipalTag: This key is used to compare the key-value pair attached to the principal with the key-value pair
+in the policy. In case of AssumeRoleWithWebIdentity, the session tags that are passed by the idp in the web token appear
+as Principal tags in the temporary credentials once a user has been authenticated, and these tags can be used as
+aws:PrincipalTag in the role's permission policy.
+
+An example of a role permission policy that uses aws:PrincipalTag is as follows:
+
+.. code-block:: python
+
+    {
+	    "Version":"2012-10-17",
+	    "Statement":[
+	    {
+	        "Effect":"Allow",
+	        "Action":["s3:*"],
+            "Resource":["arn:aws:s3::t1tenant:my-test-bucket","arn:aws:s3::t1tenant:my-test-bucket/*],"+
+	        "Condition":{"StringEquals":{"aws:PrincipalTag/Department":"Engineering"}}
+	    }]
+	}
+
+3. iam:ResourceTag: This key is used to compare the key-value pair attached to the resource with the key-value pair
+in the policy. In case of AssumeRoleWithWebIdentity, tags attached to the role can be used to compare with that in
+the trust policy to allow a user to assume a role.
+RGW now supports REST APIs for tagging, listing tags and untagging actions on a role. More information related to
+role tagging can be found here :doc:`role`.
+
+An example of a role's trust policy that uses aws:ResourceTag is as follows:
+
+.. code-block:: python
+
+    {
+	    "Version":"2012-10-17",
+	    "Statement":[
+	    {
+	        "Effect":"Allow",
+	        "Action":["sts:AssumeRoleWithWebIdentity","sts:TagSession"],
+	        "Principal":{"Federated":["arn:aws:iam:::oidc-provider/localhost:8080/auth/realms/quickstart"]},
+	        "Condition":{"StringEquals":{"iam:ResourceTag/Department":"Engineering"}}
+	    }]
+	}
+
+For the above to work, you need to attach 'Department=Engineering' tag to the role.
+
+4. aws:TagKeys: This key is used to compare tags in the request with the tags in the policy. In case of
+AssumeRoleWithWebIdentity this can be used to check the tag keys in a role's trust policy before a user
+is allowed to assume a role.
+This can also be used in the role's permission policy.
+
+An example of a role's trust policy that uses aws:TagKeys is as follows:
+
+.. code-block:: python
+
+    {
+	    "Version":"2012-10-17",
+	    "Statement":[
+	    {
+	        "Effect":"Allow",
+	        "Action":["sts:AssumeRoleWithWebIdentity","sts:TagSession"],
+	        "Principal":{"Federated":["arn:aws:iam:::oidc-provider/localhost:8080/auth/realms/quickstart"]},
+	        "Condition":{"ForAllValues:StringEquals":{"aws:TagKeys":["Marketing,Engineering"]}}
+	    }]
+	}
+
+'ForAllValues:StringEquals' tests whether every tag key in the request is a subset of the tag keys in the policy. So the above
+condition restricts the tag keys passed in the request.
+
+5. s3:ResourceTag: This key is used to compare tags present on the s3 resource (bucket or object) with the tags in
+the role's permission policy.
+
+An example of a role's permission policy that uses s3:ResourceTag is as follows:
+
+.. code-block:: python
+
+    {
+        "Version":"2012-10-17",
+        "Statement":[
+        {
+            "Effect":"Allow",
+            "Action":["s3:PutBucketTagging"],
+            "Resource":["arn:aws:s3::t1tenant:my-test-bucket\","arn:aws:s3::t1tenant:my-test-bucket/*"]
+        },
+        {
+            "Effect":"Allow",
+            "Action":["s3:*"],
+            "Resource":["*"],
+            "Condition":{"StringEquals":{"s3:ResourceTag/Department":\"Engineering"}}
+        }
+    }
+
+For the above to work, you need to attach 'Department=Engineering' tag to the bucket (and on the object too) on which you want this policy
+to be applied.
+
+More examples of policies using tags
+====================================
+
+1. To assume a role by matching the tags in the incoming request with the tag attached to the role.
+aws:RequestTag is the incoming tag in the JWT (access token) and iam:ResourceTag is the tag attached to the role being assumed:
+
+.. code-block:: python
+
+    {
+	    "Version":"2012-10-17",
+	    "Statement":[
+	    {
+	        "Effect":"Allow",
+	        "Action":["sts:AssumeRoleWithWebIdentity","sts:TagSession"],
+	        "Principal":{"Federated":["arn:aws:iam:::oidc-provider/localhost:8080/auth/realms/quickstart"]},
+	        "Condition":{"StringEquals":{"aws:RequestTag/Department":"${iam:ResourceTag/Department}"}}
+	    }]
+	}
+
+2. To evaluate a role's permission policy by matching principal tags with s3 resource tags.
+aws:PrincipalTag is the tag passed in along with the temporary credentials and s3:ResourceTag is the tag attached to
+the s3 resource (object/ bucket):
+
+.. code-block:: python
+
+
+    {
+        "Version":"2012-10-17",
+        "Statement":[
+        {
+            "Effect":"Allow",
+            "Action":["s3:PutBucketTagging"],
+            "Resource":["arn:aws:s3::t1tenant:my-test-bucket\","arn:aws:s3::t1tenant:my-test-bucket/*"]
+        },
+        {
+            "Effect":"Allow",
+            "Action":["s3:*"],
+            "Resource":["*"],
+            "Condition":{"StringEquals":{"s3:ResourceTag/Department":"${aws:PrincipalTag/Department}"}}
+        }
+    }
+
+Properties of Session Tags
+==========================
+
+1. Session Tags can be multi-valued. (Multi-valued session tags are not supported in AWS)
+2. A maximum of 50 session tags are allowed to be passed in by the IDP.
+3. The maximum size of a key allowed is 128 characters.
+4. The maximum size of a value allowed is 256 characters.
+5. The tag or the value can not start with "aws:".
+
+s3 Resource Tags
+================
+
+As stated above 's3:ResourceTag' key can be used for authorizing an s3 operation in RGW (this is not allowed in AWS).
+
+s3:ResourceTag is a key used to refer to tags that have been attached to an object or a bucket. Tags can be attached to an object or
+a bucket using REST APIs available for the same.
+
+The following table shows which s3 resource tag type (bucket/object) are supported for authorizing a particular operation.
+
++-----------------------------------+-------------------+
+| Operation                         | Tag type          |
++===================================+===================+
+| **GetObject**                     | Object tags       |
+| **GetObjectTags**                 |                   |
+| **DeleteObjectTags**              |                   |
+| **DeleteObject**                  |                   |
+| **PutACLs**                       |                   |
+| **InitMultipart**                 |                   |
+| **AbortMultipart**                |                   |
+| **ListMultipart**                 |                   |
+| **GetAttrs**                      |                   |
+| **PutObjectRetention**            |                   |
+| **GetObjectRetention**            |                   |
+| **PutObjectLegalHold**            |                   |
+| **GetObjectLegalHold**            |                   |
++-----------------------------------+-------------------+
+| **PutObjectTags**                 | Bucket tags       |
+| **GetBucketTags**                 |                   |
+| **PutBucketTags**                 |                   |
+| **DeleteBucketTags**              |                   |
+| **GetBucketReplication**          |                   |
+| **DeleteBucketReplication**       |                   |
+| **GetBucketVersioning**           |                   |
+| **SetBucketVersioning**           |                   |
+| **GetBucketWebsite**              |                   |
+| **SetBucketWebsite**              |                   |
+| **DeleteBucketWebsite**           |                   |
+| **StatBucket**                    |                   |
+| **ListBucket**                    |                   |
+| **GetBucketLogging**              |                   |
+| **GetBucketLocation**             |                   |
+| **DeleteBucket**                  |                   |
+| **GetLC**                         |                   |
+| **PutLC**                         |                   |
+| **DeleteLC**                      |                   |
+| **GetCORS**                       |                   |
+| **PutCORS**                       |                   |
+| **GetRequestPayment**             |                   |
+| **SetRequestPayment**             |                   |
+| **PutBucketPolicy**               |                   |
+| **GetBucketPolicy**               |                   |
+| **DeleteBucketPolicy**            |                   |
+| **PutBucketObjectLock**           |                   |
+| **GetBucketObjectLock**           |                   |
+| **GetBucketPolicyStatus**         |                   |
+| **PutBucketPublicAccessBlock**    |                   |
+| **GetBucketPublicAccessBlock**    |                   |
+| **DeleteBucketPublicAccessBlock** |                   |
++-----------------------------------+-------------------+
+| **GetACLs**                       | Bucket tags for   |
+| **PutACLs**                       | bucket ACLs       |
+|                                   | Object tags for   |
+|                                   | object ACLs       |
++-----------------------------------+-------------------+
+| **PutObject**                     | Object tags of    |
+| **CopyObject**                    | source object     |
+|                                   | Bucket tags of    |
+|                                   | destination bucket|
++-----------------------------------+-------------------+
+
+
+Sample code demonstrating usage of session tags
+===============================================
+
+The following is a sample code for tagging a role, a bucket, an object in it and using tag keys in a role's
+trust policy and its permission policy, assuming that a tag 'Department=Engineering' is passed in the
+JWT (access token) by the IDP
+
+.. code-block:: python
+
+    # -*- coding: utf-8 -*-
+
+    import boto3
+    import json
+    from nose.tools import eq_ as eq
+
+    access_key = 'TESTER'
+    secret_key = 'test123'
+    endpoint = 'http://s3.us-east.localhost:8000'
+
+    s3client = boto3.client('s3',
+    aws_access_key_id = access_key,
+    aws_secret_access_key = secret_key,
+    endpoint_url = endpoint,
+    region_name='',)
+
+    s3res = boto3.resource('s3',
+            aws_access_key_id = access_key,
+            aws_secret_access_key = secret_key,
+            endpoint_url = endpoint,
+            region_name='',)
+
+    iam_client = boto3.client('iam',
+    aws_access_key_id=access_key,
+    aws_secret_access_key=secret_key,
+    endpoint_url=endpoint,
+    region_name=''
+    )
+
+    bucket_name = 'test-bucket'
+    s3bucket = s3client.create_bucket(Bucket=bucket_name)
+
+    bucket_tagging = s3res.BucketTagging(bucket_name)
+    Set_Tag = bucket_tagging.put(Tagging={'TagSet':[{'Key':'Department', 'Value': 'Engineering'}]})
+    try:
+        response = iam_client.create_open_id_connect_provider(
+            Url='http://localhost:8080/auth/realms/quickstart',
+            ClientIDList=[
+                'app-profile-jsp',
+                'app-jee-jsp'
+            ],
+            ThumbprintList=[
+                'F7D7B3515DD0D319DD219A43A9EA727AD6065287'
+        ]
+        )
+    except ClientError as e:
+        print ("Provider already exists")
+
+    policy_document = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Federated\":[\"arn:aws:iam:::oidc-provider/localhost:8080/auth/realms/quickstart\"]},\"Action\":[\"sts:AssumeRoleWithWebIdentity\",\"sts:TagSession\"],\"Condition\":{\"StringEquals\":{\"aws:RequestTag/Department\":\"${iam:ResourceTag/Department}\"}}}]}"
+    role_response = ""
+
+    print ("\n Getting Role \n")
+
+    try:
+        role_response = iam_client.get_role(
+            RoleName='S3Access'
+        )
+        print (role_response)
+    except ClientError as e:
+        if e.response['Code'] == 'NoSuchEntity':
+            print ("\n Creating Role \n")
+            tags_list = [
+                {'Key':'Department','Value':'Engineering'},
+            ]
+            role_response = iam_client.create_role(
+                AssumeRolePolicyDocument=policy_document,
+                Path='/',
+                RoleName='S3Access',
+                Tags=tags_list,
+            )
+            print (role_response)
+        else:
+            print("Unexpected error: %s" % e)
+
+    role_policy = "{\"Version\":\"2012-10-17\",\"Statement\":{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"arn:aws:s3:::*\",\"Condition\":{\"StringEquals\":{\"s3:ResourceTag/Department\":[\"${aws:PrincipalTag/Department}\"]}}}}"
+
+    response = iam_client.put_role_policy(
+                RoleName='S3Access',
+                PolicyName='Policy1',
+                PolicyDocument=role_policy
+            )
+
+    sts_client = boto3.client('sts',
+    aws_access_key_id='abc',
+    aws_secret_access_key='def',
+    endpoint_url = endpoint,
+    region_name = '',
+    )
+
+
+    print ("\n Assuming Role with Web Identity\n")
+    response = sts_client.assume_role_with_web_identity(
+    RoleArn=role_response['Role']['Arn'],
+    RoleSessionName='Bob',
+    DurationSeconds=900,
+    WebIdentityToken='<web-token>')
+
+    s3client2 = boto3.client('s3',
+    aws_access_key_id = response['Credentials']['AccessKeyId'],
+    aws_secret_access_key = response['Credentials']['SecretAccessKey'],
+    aws_session_token = response['Credentials']['SessionToken'],
+    endpoint_url='http://s3.us-east.localhost:8000',
+    region_name='',)
+
+    bucket_body = 'this is a test file'
+    tags = 'Department=Engineering'
+    key = "test-1.txt"
+    s3_put_obj = s3client2.put_object(Body=bucket_body, Bucket=bucket_name, Key=key, Tagging=tags)
+    eq(s3_put_obj['ResponseMetadata']['HTTPStatusCode'],200)
+
+    s3_get_obj = s3client2.get_object(Bucket=bucket_name, Key=key)
+    eq(s3_get_obj['ResponseMetadata']['HTTPStatusCode'],200)
\ No newline at end of file
diff --git a/doc/radosgw/swift.rst b/doc/radosgw/swift.rst
new file mode 100644
index 000000000..2cb2dde67
--- /dev/null
+++ b/doc/radosgw/swift.rst
@@ -0,0 +1,77 @@
+===============================
+ Ceph Object Gateway Swift API
+===============================
+
+Ceph supports a RESTful API that is compatible with the basic data access model of the `Swift API`_.
+
+API
+---
+
+.. toctree::
+   :maxdepth: 1
+
+   Authentication <swift/auth>
+   Service Ops <swift/serviceops>
+   Container Ops <swift/containerops>
+   Object Ops <swift/objectops>
+   Temp URL Ops <swift/tempurl>
+   Tutorial <swift/tutorial>
+   Java <swift/java>
+   Python <swift/python>
+   Ruby <swift/ruby>
+
+
+Features Support
+----------------
+
+The following table describes the support status for current Swift functional features:
+
++---------------------------------+-----------------+----------------------------------------+
+| Feature                         | Status          | Remarks                                |
++=================================+=================+========================================+
+| **Authentication**              | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Get Account Metadata**        | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Swift ACLs**                  | Supported       | Supports a subset of Swift ACLs        |
++---------------------------------+-----------------+----------------------------------------+
+| **List Containers**             | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Delete Container**            | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Create Container**            | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Get Container Metadata**      | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Update Container Metadata**   | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Delete Container Metadata**   | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **List Objects**                | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Static Website**              | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Create Object**               | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Create Large Object**         | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Delete Object**               | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Get Object**                  | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Copy Object**                 | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Get Object Metadata**         | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Update Object Metadata**      | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Expiring Objects**            | Supported       |                                        |
++---------------------------------+-----------------+----------------------------------------+
+| **Temporary URLs**              | Partial Support | No support for container-level keys    |
++---------------------------------+-----------------+----------------------------------------+
+| **Object Versioning**           | Partial Support | No support for ``X-History-Location``  |
++---------------------------------+-----------------+----------------------------------------+
+| **CORS**                        | Not Supported   |                                        |
++---------------------------------+-----------------+----------------------------------------+
+
+.. _Swift API: https://developer.openstack.org/api-ref/object-store/index.html
diff --git a/doc/radosgw/swift/auth.rst b/doc/radosgw/swift/auth.rst
new file mode 100644
index 000000000..12d6b23ff
--- /dev/null
+++ b/doc/radosgw/swift/auth.rst
@@ -0,0 +1,82 @@
+================
+ Authentication
+================
+
+Swift API requests that require authentication must contain an
+``X-Storage-Token`` authentication token in the request header. 
+The token may be retrieved from RADOS Gateway, or from another authenticator.
+To obtain a token from RADOS Gateway, you must create a user. For example:: 
+
+    sudo radosgw-admin user create --subuser="{username}:{subusername}" --uid="{username}" 
+    --display-name="{Display Name}" --key-type=swift --secret="{password}" --access=full
+
+For details on RADOS Gateway administration, see `radosgw-admin`_. 
+
+.. _radosgw-admin: ../../../man/8/radosgw-admin/ 
+
+.. note::
+  For those used to the Swift API this is implementing the Swift auth v1.0 API, as such
+  `{username}` above is generally equivalent to a Swift `account` and `{subusername}`
+  is a user under that account.
+
+Auth Get
+--------
+
+To authenticate a user, make a request containing an ``X-Auth-User`` and a
+``X-Auth-Key`` in the header.
+
+Syntax
+~~~~~~
+
+::
+
+    GET /auth HTTP/1.1
+    Host: swift.radosgwhost.com
+    X-Auth-User: johndoe
+    X-Auth-Key: R7UUOLFDI2ZI9PRCQ53K
+
+Request Headers
+~~~~~~~~~~~~~~~
+
+``X-Auth-User`` 
+
+:Description: The key RADOS GW username to authenticate.
+:Type: String
+:Required: Yes
+
+``X-Auth-Key`` 
+
+:Description: The key associated to a RADOS GW username.
+:Type: String
+:Required: Yes
+
+
+Response Headers
+~~~~~~~~~~~~~~~~
+
+The response from the server should include an ``X-Auth-Token`` value. The 
+response may also contain a ``X-Storage-Url`` that provides the 
+``{api version}/{account}`` prefix that is specified in other requests
+throughout the API documentation.
+
+
+``X-Storage-Token`` 
+
+:Description: The authorization token for the ``X-Auth-User`` specified in the request.
+:Type: String
+
+
+``X-Storage-Url`` 
+
+:Description: The URL and ``{api version}/{account}`` path for the user.
+:Type: String
+
+A typical response looks like this:: 
+
+	HTTP/1.1 204 No Content
+	Date: Mon, 16 Jul 2012 11:05:33 GMT
+  	Server: swift
+  	X-Storage-Url: https://swift.radosgwhost.com/v1/ACCT-12345
+	X-Auth-Token: UOlCCC8TahFKlWuv9DB09TWHF0nDjpPElha0kAa
+	Content-Length: 0
+    Content-Type: text/plain; charset=UTF-8
diff --git a/doc/radosgw/swift/containerops.rst b/doc/radosgw/swift/containerops.rst
new file mode 100644
index 000000000..17ef4658b
--- /dev/null
+++ b/doc/radosgw/swift/containerops.rst
@@ -0,0 +1,341 @@
+======================
+ Container Operations
+======================
+
+A container is a mechanism for storing data objects. An account may
+have many containers, but container names must be unique. This API enables a 
+client to create a container, set access controls and metadata, 
+retrieve a container's contents, and delete a container. Since this API 
+makes requests related to information in a particular user's account, all 
+requests in this API must be authenticated unless a container's access control
+is deliberately made publicly accessible (i.e., allows anonymous requests).
+
+.. note:: The Amazon S3 API uses the term 'bucket' to describe a data container.
+   When you hear someone refer to a 'bucket' within the Swift API, the term 
+   'bucket' may be construed as the equivalent of the term 'container.'
+   
+One facet of object storage is that it does not support hierarchical paths
+or directories. Instead, it supports one level consisting of one or more
+containers, where each container may have objects. The RADOS Gateway's
+Swift-compatible API supports the notion of 'pseudo-hierarchical containers,'
+which is a means of using object naming to emulate a container (or directory)
+hierarchy without actually implementing one in the storage system. You may 
+name objects with pseudo-hierarchical names 
+(e.g., photos/buildings/empire-state.jpg), but container names cannot
+contain a forward slash (``/``) character.
+
+
+Create a Container
+==================
+
+To create a new container, make a ``PUT`` request with the API version, account,
+and the name of the new container. The container name must be unique, must not
+contain a forward-slash (/) character, and should be less than 256 bytes. You 
+may include access control headers and metadata headers in the request. The 
+operation is idempotent; that is, if you make a request to create a container
+that already exists, it will return with a HTTP 202 return code, but will not
+create another container.
+
+
+Syntax
+~~~~~~
+
+::
+
+	PUT /{api version}/{account}/{container} HTTP/1.1
+	Host: {fqdn}
+	X-Auth-Token: {auth-token}
+	X-Container-Read: {comma-separated-uids}
+	X-Container-Write: {comma-separated-uids}
+	X-Container-Meta-{key}: {value}
+
+
+Headers
+~~~~~~~
+
+``X-Container-Read``
+
+:Description: The user IDs with read permissions for the container. 
+:Type: Comma-separated string values of user IDs.
+:Required: No
+
+``X-Container-Write``
+
+:Description: The user IDs with write permissions for the container.
+:Type: Comma-separated string values of user IDs.
+:Required: No
+
+``X-Container-Meta-{key}``
+
+:Description:  A user-defined meta data key that takes an arbitrary string value.
+:Type: String
+:Required: No
+
+
+HTTP Response
+~~~~~~~~~~~~~
+
+If a container with the same name already exists, and the user is the
+container owner then the operation will succeed. Otherwise the operation
+will fail.
+
+``409``
+
+:Description: The container already exists under a different user's ownership.
+:Status Code: ``BucketAlreadyExists``
+
+
+
+
+List a Container's Objects
+==========================
+
+To list the objects within a container, make a ``GET`` request with the with the 
+API version, account, and the name of the container.  You can specify query 
+parameters to filter the full list, or leave out the parameters to return a list 
+of the first 10,000 object names stored in the container.
+
+
+Syntax
+~~~~~~
+
+::
+
+   GET /{api version}/{container} HTTP/1.1
+  	Host: {fqdn}
+	X-Auth-Token: {auth-token}
+
+
+Parameters
+~~~~~~~~~~
+
+``format``
+
+:Description: Defines the format of the result. 
+:Type: String
+:Valid Values: ``json`` | ``xml``
+:Required: No
+
+``prefix``
+
+:Description: Limits the result set to objects beginning with the specified prefix.
+:Type: String
+:Required: No
+
+``marker``
+
+:Description: Returns a list of results greater than the marker value.
+:Type: String
+:Required: No
+
+``limit``
+
+:Description: Limits the number of results to the specified value.
+:Type: Integer
+:Valid Range: 0 - 10,000
+:Required: No
+
+``delimiter``
+
+:Description: The delimiter between the prefix and the rest of the object name.
+:Type: String
+:Required: No
+
+``path``
+
+:Description: The pseudo-hierarchical path of the objects.
+:Type: String
+:Required: No
+
+``allow_unordered``
+
+:Description: Allows the results to be returned unordered to reduce computation overhead. Cannot be used with ``delimiter``.
+:Type: Boolean
+:Required: No
+:Non-Standard Extension: Yes
+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+``container``
+
+:Description: The container. 
+:Type: Container
+
+``object``
+
+:Description: An object within the container.
+:Type: Container
+
+``name``
+
+:Description: The name of an object within the container.
+:Type: String
+
+``hash``
+
+:Description: A hash code of the object's contents.
+:Type: String
+
+``last_modified``
+
+:Description: The last time the object's contents were modified.
+:Type: Date
+
+``content_type``
+
+:Description: The type of content within the object.
+:Type: String
+
+
+
+Update a Container's ACLs
+=========================
+
+When a user creates a container, the user has read and write access to the
+container by default. To allow other users to read a container's contents or
+write to a container, you must specifically enable the user. 
+You may also specify ``*`` in the ``X-Container-Read`` or ``X-Container-Write``
+settings, which effectively enables all users to either read from or write
+to the container. Setting ``*`` makes the container public. That is it 
+enables anonymous users to either read from or write to the container.
+
+.. note:: If you are planning to expose public read ACL functionality
+	  for the Swift API, it is strongly recommended to include the
+	  Swift account name in the endpoint definition, so as to most
+	  closely emulate the behavior of native OpenStack Swift. To
+	  do so, set the ``ceph.conf`` configuration option ``rgw
+	  swift account in url = true``, and update your Keystone
+	  endpoint to the URL suffix ``/v1/AUTH_%(tenant_id)s``
+	  (instead of just ``/v1``).
+
+
+Syntax
+~~~~~~
+
+::
+
+   POST /{api version}/{account}/{container} HTTP/1.1
+   Host: {fqdn}
+	X-Auth-Token: {auth-token}
+	X-Container-Read: *
+	X-Container-Write: {uid1}, {uid2}, {uid3}
+
+Request Headers
+~~~~~~~~~~~~~~~
+
+``X-Container-Read``
+
+:Description: The user IDs with read permissions for the container. 
+:Type: Comma-separated string values of user IDs.
+:Required: No
+
+``X-Container-Write``
+
+:Description: The user IDs with write permissions for the container.
+:Type: Comma-separated string values of user IDs.
+:Required: No
+
+
+Add/Update Container Metadata
+=============================
+
+To add metadata to a container, make a ``POST`` request with the API version, 
+account, and container name. You must have write permissions on the 
+container to add or update metadata.
+
+Syntax
+~~~~~~
+
+::
+
+   POST /{api version}/{account}/{container} HTTP/1.1
+   Host: {fqdn}
+	X-Auth-Token: {auth-token}
+	X-Container-Meta-Color: red
+	X-Container-Meta-Taste: salty
+	
+Request Headers
+~~~~~~~~~~~~~~~
+
+``X-Container-Meta-{key}``
+
+:Description:  A user-defined meta data key that takes an arbitrary string value.
+:Type: String
+:Required: No
+
+
+Enable Object Versioning for a Container
+========================================
+
+To enable object versioning a container, make a ``POST`` request with
+the API version, account, and container name. You must have write
+permissions on the container to add or update metadata.
+
+.. note:: Object versioning support is not enabled in radosgw by
+	  default; you must set ``rgw swift versioning enabled =
+	  true`` in ``ceph.conf`` to enable this feature.
+
+Syntax
+~~~~~~
+
+::
+
+   POST /{api version}/{account}/{container} HTTP/1.1
+   Host: {fqdn}
+	X-Auth-Token: {auth-token}
+	X-Versions-Location: {archive-container}
+
+Request Headers
+~~~~~~~~~~~~~~~
+
+``X-Versions-Location``
+
+:Description: The name of a container (the "archive container") that
+	      will be used to store versions of the objects in the
+	      container that the ``POST`` request is made on (the
+	      "current container"). The archive container need not
+	      exist at the time it is being referenced, but once
+	      ``X-Versions-Location`` is set on the current container,
+	      and object versioning is thus enabled, the archive
+	      container must exist before any further objects are
+	      updated or deleted in the current container.
+
+	      .. note:: ``X-Versions-Location`` is the only
+                        versioning-related header that radosgw
+                        interprets. ``X-History-Location``, supported
+                        by native OpenStack Swift, is currently not
+                        supported by radosgw.
+:Type: String
+:Required: No (if this header is passed with an empty value, object
+	   versioning on the current container is disabled, but the
+	   archive container continues to exist.)
+
+
+Delete a Container
+==================
+
+To delete a container, make a ``DELETE`` request with the API version, account,
+and the name of the container. The container must be empty. If you'd like to check 
+if the container is empty, execute a ``HEAD`` request against the container. Once 
+you have successfully removed the container, you will be able to reuse the container name.
+
+Syntax
+~~~~~~
+
+::
+
+	DELETE /{api version}/{account}/{container} HTTP/1.1
+	Host: {fqdn}
+	X-Auth-Token: {auth-token}    
+
+
+HTTP Response
+~~~~~~~~~~~~~
+
+``204``
+
+:Description: The container was removed.
+:Status Code: ``NoContent``
+
diff --git a/doc/radosgw/swift/java.rst b/doc/radosgw/swift/java.rst
new file mode 100644
index 000000000..8977a3b16
--- /dev/null
+++ b/doc/radosgw/swift/java.rst
@@ -0,0 +1,175 @@
+.. _java_swift:
+
+=====================
+ Java Swift Examples
+=====================
+
+Setup
+=====
+
+The following examples may require some or all of the following Java
+classes to be imported:
+
+.. code-block:: java
+
+       import org.javaswift.joss.client.factory.AccountConfig;
+       import org.javaswift.joss.client.factory.AccountFactory;
+       import org.javaswift.joss.client.factory.AuthenticationMethod;
+       import org.javaswift.joss.model.Account;
+       import org.javaswift.joss.model.Container;
+       import org.javaswift.joss.model.StoredObject;
+       import java.io.File;
+       import java.io.IOException;
+       import java.util.*;
+
+
+Create a Connection
+===================
+
+This creates a connection so that you can interact with the server:
+
+.. code-block:: java
+
+       String username = "USERNAME";
+       String password = "PASSWORD";
+       String authUrl  = "https://radosgw.endpoint/auth/1.0";
+
+       AccountConfig config = new AccountConfig();
+       config.setUsername(username);
+       config.setPassword(password);
+       config.setAuthUrl(authUrl);
+       config.setAuthenticationMethod(AuthenticationMethod.BASIC);
+       Account account = new AccountFactory(config).createAccount();
+
+
+Create a Container
+==================
+
+This creates a new container called ``my-new-container``:
+
+.. code-block:: java
+
+       Container container = account.getContainer("my-new-container");
+       container.create();
+
+
+Create an Object
+================
+
+This creates an object ``foo.txt`` from the file named ``foo.txt`` in 
+the container ``my-new-container``:
+
+.. code-block:: java
+
+       Container container = account.getContainer("my-new-container");
+       StoredObject object = container.getObject("foo.txt");
+       object.uploadObject(new File("foo.txt"));
+
+
+Add/Update Object Metadata
+==========================
+
+This adds the metadata key-value pair ``key``:``value`` to the object named
+``foo.txt`` in the container ``my-new-container``:
+
+.. code-block:: java
+
+       Container container = account.getContainer("my-new-container");
+       StoredObject object = container.getObject("foo.txt");
+       Map<String, Object> metadata = new TreeMap<String, Object>();
+       metadata.put("key", "value");
+       object.setMetadata(metadata);
+
+
+List Owned Containers
+=====================
+
+This gets a list of Containers that you own.
+This also prints out the container name.
+
+.. code-block:: java
+
+       Collection<Container> containers = account.list();
+       for (Container currentContainer : containers) {
+           System.out.println(currentContainer.getName());
+       }
+
+The output will look something like this::
+
+	mahbuckat1
+	mahbuckat2
+	mahbuckat3
+
+
+List a Container's Content
+==========================
+
+This gets a list of objects in the container ``my-new-container``; and, it also 
+prints out each object's name, the file size, and last modified date:
+
+.. code-block:: java
+
+       Container container = account.getContainer("my-new-container");
+       Collection<StoredObject> objects = container.list();
+       for (StoredObject currentObject : objects) {
+           System.out.println(currentObject.getName());
+       }
+
+The output will look something like this::
+
+   myphoto1.jpg
+   myphoto2.jpg
+
+
+Retrieve an Object's Metadata
+=============================
+
+This retrieves metadata and gets the MIME type for an object named ``foo.txt``
+in a container named ``my-new-container``:
+
+.. code-block:: java
+
+       Container container = account.getContainer("my-new-container");
+       StoredObject object = container.getObject("foo.txt");
+       Map<String, Object> returnedMetadata = object.getMetadata();
+       for (String name : returnedMetadata.keySet()) {
+           System.out.println("META / "+name+": "+returnedMetadata.get(name));
+       }
+
+
+Retrieve an Object
+==================
+
+This downloads the object ``foo.txt`` in the container ``my-new-container`` 
+and saves it in ``./outfile.txt``:
+
+.. code-block:: java
+
+       Container container = account.getContainer("my-new-container");
+       StoredObject object = container.getObject("foo.txt");
+       object.downloadObject(new File("outfile.txt"));
+
+
+Delete an Object
+================
+
+This deletes the object ``goodbye.txt`` in the container "my-new-container":
+
+.. code-block:: java
+
+       Container container = account.getContainer("my-new-container");
+       StoredObject object = container.getObject("foo.txt");
+       object.delete();
+
+
+Delete a Container
+==================
+
+This deletes a container named "my-new-container": 
+
+.. code-block:: java
+
+       Container container = account.getContainer("my-new-container");
+       container.delete();
+	
+.. note:: The container must be empty! Otherwise it won't work!
diff --git a/doc/radosgw/swift/objectops.rst b/doc/radosgw/swift/objectops.rst
new file mode 100644
index 000000000..fc8d21967
--- /dev/null
+++ b/doc/radosgw/swift/objectops.rst
@@ -0,0 +1,271 @@
+===================
+ Object Operations
+===================
+
+An object is a container for storing data and metadata. A container may
+have many objects, but the object names must be unique. This API enables a 
+client to create an object, set access controls and metadata, retrieve an 
+object's data and metadata, and delete an object. Since this API makes requests 
+related to information in a particular user's account, all requests in this API 
+must be authenticated unless the container or object's access control is 
+deliberately made publicly accessible (i.e., allows anonymous requests).
+
+
+Create/Update an Object
+=======================
+
+To create a new object, make a ``PUT`` request with the API version, account,
+container name and the name of the new object. You must have write permission
+on the container to create or update an object. The object name must be 
+unique within the container. The ``PUT`` request is not idempotent, so if you
+do not use a unique name, the request will update the object. However, you may
+use pseudo-hierarchical syntax in your object name to distinguish it from 
+another object of the same name if it is under a different pseudo-hierarchical 
+directory. You may include access control headers and metadata headers in the 
+request.
+
+
+Syntax
+~~~~~~
+
+::
+
+   PUT /{api version}/{account}/{container}/{object} HTTP/1.1 
+	Host: {fqdn}
+	X-Auth-Token: {auth-token}
+
+
+Request Headers
+~~~~~~~~~~~~~~~
+
+``ETag``
+
+:Description: An MD5 hash of the object's contents. Recommended. 
+:Type: String
+:Required: No
+
+
+``Content-Type``
+
+:Description: The type of content the object contains.
+:Type: String
+:Required: No
+
+
+``Transfer-Encoding``
+
+:Description: Indicates whether the object is part of a larger aggregate object.
+:Type: String
+:Valid Values: ``chunked``
+:Required: No
+
+
+Copy an Object
+==============
+
+Copying an object allows you to make a server-side copy of an object, so that
+you don't have to download it and upload it under another container/name.
+To copy the contents of one object to another object, you may make either a
+``PUT`` request or a ``COPY`` request with the API version, account, and the 
+container name. For a ``PUT`` request, use the destination container and object
+name in the request, and the source container and object in the request header.
+For a ``Copy`` request, use the source container and object in the request, and
+the destination container and object in the request header. You must have write 
+permission on the container to copy an object. The destination object name must be 
+unique within the container. The request is not idempotent, so if you do not use 
+a unique name, the request will update the destination object. However, you may 
+use pseudo-hierarchical syntax in your object name to distinguish the destination 
+object from the source object of the same name if it is under a different 
+pseudo-hierarchical directory. You may include access control headers and metadata 
+headers in the request.
+
+Syntax
+~~~~~~
+
+::
+
+	PUT /{api version}/{account}/{dest-container}/{dest-object} HTTP/1.1
+	X-Copy-From: {source-container}/{source-object}
+	Host: {fqdn}
+	X-Auth-Token: {auth-token}
+
+
+or alternatively:
+
+::
+
+	COPY /{api version}/{account}/{source-container}/{source-object} HTTP/1.1
+	Destination: {dest-container}/{dest-object}
+
+Request Headers
+~~~~~~~~~~~~~~~
+
+``X-Copy-From``
+
+:Description: Used with a ``PUT`` request to define the source container/object path.
+:Type: String
+:Required: Yes, if using ``PUT``
+
+
+``Destination``
+
+:Description: Used with a ``COPY`` request to define the destination container/object path.
+:Type: String
+:Required: Yes, if using ``COPY``
+
+
+``If-Modified-Since``
+
+:Description: Only copies if modified since the date/time of the source object's ``last_modified`` attribute.
+:Type: Date
+:Required: No
+
+
+``If-Unmodified-Since``
+
+:Description: Only copies if not modified since the date/time of the source object's ``last_modified`` attribute.
+:Type: Date
+:Required: No
+
+``Copy-If-Match``
+
+:Description: Copies only if the ETag in the request matches the source object's ETag.
+:Type: ETag.
+:Required: No
+
+
+``Copy-If-None-Match``
+
+:Description: Copies only if the ETag in the request does not match the source object's ETag.
+:Type: ETag.
+:Required: No
+
+
+Delete an Object
+================
+
+To delete an object, make a ``DELETE`` request with the API version, account,
+container and object name. You must have write permissions on the container to delete
+an object within it. Once you have successfully deleted the object, you will be able to 
+reuse the object name.
+
+Syntax
+~~~~~~
+
+::
+
+	DELETE /{api version}/{account}/{container}/{object} HTTP/1.1
+	Host: {fqdn}
+	X-Auth-Token: {auth-token}
+
+
+Get an Object
+=============
+
+To retrieve an object, make a ``GET`` request with the API version, account,
+container and object name. You must have read permissions on the container to
+retrieve an object within it.
+
+Syntax
+~~~~~~
+
+::
+
+	GET /{api version}/{account}/{container}/{object} HTTP/1.1
+	Host: {fqdn}
+	X-Auth-Token: {auth-token}
+
+
+
+Request Headers
+~~~~~~~~~~~~~~~
+
+``range``
+
+:Description: To retrieve a subset of an object's contents, you may specify a byte range.
+:Type: Date
+:Required: No
+
+
+``If-Modified-Since``
+
+:Description: Only copies if modified since the date/time of the source object's ``last_modified`` attribute.
+:Type: Date
+:Required: No
+
+
+``If-Unmodified-Since``
+
+:Description: Only copies if not modified since the date/time of the source object's ``last_modified`` attribute.
+:Type: Date
+:Required: No
+
+``Copy-If-Match``
+
+:Description: Copies only if the ETag in the request matches the source object's ETag.
+:Type: ETag.
+:Required: No
+
+
+``Copy-If-None-Match``
+
+:Description: Copies only if the ETag in the request does not match the source object's ETag.
+:Type: ETag.
+:Required: No
+
+
+
+Response Headers
+~~~~~~~~~~~~~~~~
+
+``Content-Range``
+
+:Description: The range of the subset of object contents. Returned only if the range header field was specified in the request  
+
+
+Get Object Metadata
+===================
+
+To retrieve an object's metadata, make a ``HEAD`` request with the API version, 
+account, container and object name. You must have read permissions on the 
+container to retrieve metadata from an object within the container. This request
+returns the same header information as the request for the object itself, but
+it does not return the object's data.
+
+Syntax
+~~~~~~
+
+::
+
+	HEAD /{api version}/{account}/{container}/{object} HTTP/1.1
+	Host: {fqdn}
+	X-Auth-Token: {auth-token}
+
+
+
+Add/Update Object Metadata
+==========================
+
+To add metadata to an object, make a ``POST`` request with the API version, 
+account, container and object name. You must have write permissions on the 
+parent container to add or update metadata.
+
+
+Syntax
+~~~~~~
+
+::
+
+	POST /{api version}/{account}/{container}/{object} HTTP/1.1
+	Host: {fqdn}
+	X-Auth-Token: {auth-token}
+
+Request Headers
+~~~~~~~~~~~~~~~
+
+``X-Object-Meta-{key}``
+
+:Description:  A user-defined meta data key that takes an arbitrary string value.
+:Type: String
+:Required: No
+
diff --git a/doc/radosgw/swift/python.rst b/doc/radosgw/swift/python.rst
new file mode 100644
index 000000000..28d92d7cc
--- /dev/null
+++ b/doc/radosgw/swift/python.rst
@@ -0,0 +1,114 @@
+.. _python_swift:
+
+=====================
+Python Swift Examples
+=====================
+
+Create a Connection
+===================
+
+This creates a connection so that you can interact with the server:
+
+.. code-block:: python
+
+	import swiftclient
+	user = 'account_name:username'
+	key = 'your_api_key'
+
+	conn = swiftclient.Connection(
+		user=user,
+		key=key,
+		authurl='https://objects.dreamhost.com/auth',
+	)
+
+
+Create a Container
+==================
+
+This creates a new container called ``my-new-container``:
+
+.. code-block:: python
+
+	container_name = 'my-new-container'
+	conn.put_container(container_name)
+
+
+Create an Object
+================
+
+This creates a file ``hello.txt`` from the file named ``my_hello.txt``:
+
+.. code-block:: python
+
+	with open('hello.txt', 'r') as hello_file:
+		conn.put_object(container_name, 'hello.txt',
+						contents= hello_file.read(),
+						content_type='text/plain')
+
+
+List Owned Containers
+=====================
+
+This gets a list of containers that you own, and prints out the container name:
+
+.. code-block:: python
+
+	for container in conn.get_account()[1]:
+		print container['name']
+
+The output will look something like this::
+
+   mahbuckat1
+   mahbuckat2
+   mahbuckat3
+
+List a Container's Content
+==========================
+
+This gets a list of objects in the container, and prints out each
+object's name, the file size, and last modified date:
+
+.. code-block:: python
+
+	for data in conn.get_container(container_name)[1]:
+		print '{0}\t{1}\t{2}'.format(data['name'], data['bytes'], data['last_modified'])
+
+The output will look something like this::
+
+   myphoto1.jpg	251262	2011-08-08T21:35:48.000Z
+   myphoto2.jpg	262518	2011-08-08T21:38:01.000Z
+
+
+Retrieve an Object
+==================
+
+This downloads the object ``hello.txt`` and saves it in
+``./my_hello.txt``:
+
+.. code-block:: python
+
+	obj_tuple = conn.get_object(container_name, 'hello.txt')
+	with open('my_hello.txt', 'w') as my_hello:
+		my_hello.write(obj_tuple[1])
+
+
+Delete an Object
+================
+
+This deletes the object ``hello.txt``:
+
+.. code-block:: python
+
+	conn.delete_object(container_name, 'hello.txt')
+
+Delete a Container
+==================
+
+.. note::
+
+   The container must be empty! Otherwise the request won't work!
+
+.. code-block:: python
+
+	conn.delete_container(container_name)
+
diff --git a/doc/radosgw/swift/ruby.rst b/doc/radosgw/swift/ruby.rst
new file mode 100644
index 000000000..a20b66d88
--- /dev/null
+++ b/doc/radosgw/swift/ruby.rst
@@ -0,0 +1,119 @@
+.. _ruby_swift:
+
+=====================
+ Ruby Swift Examples
+=====================
+
+Create a Connection
+===================
+
+This creates a connection so that you can interact with the server:
+
+.. code-block:: ruby
+
+	require 'cloudfiles'
+	username = 'account_name:user_name'
+	api_key  = 'your_secret_key'
+
+	conn = CloudFiles::Connection.new(
+		:username => username,
+		:api_key  => api_key,
+		:auth_url => 'http://objects.dreamhost.com/auth'
+	)
+
+
+Create a Container
+==================
+
+This creates a new container called ``my-new-container``
+
+.. code-block:: ruby
+
+	container = conn.create_container('my-new-container')
+
+
+Create an Object
+================
+
+This creates a file ``hello.txt`` from the file named ``my_hello.txt``
+
+.. code-block:: ruby
+
+	obj = container.create_object('hello.txt')
+	obj.load_from_filename('./my_hello.txt')
+	obj.content_type = 'text/plain'
+
+
+
+List Owned Containers
+=====================
+
+This gets a list of Containers that you own, and also prints out 
+the container name:
+
+.. code-block:: ruby
+
+	conn.containers.each do |container|
+		puts container
+	end
+
+The output will look something like this::
+
+   mahbuckat1
+   mahbuckat2
+   mahbuckat3
+
+
+List a Container's Contents
+===========================
+
+This gets a list of objects in the container, and prints out each 
+object's name, the file size, and last modified date:
+
+.. code-block:: ruby
+
+	require 'date'  # not necessary in the next version
+
+	container.objects_detail.each do |name, data|
+		puts "#{name}\t#{data[:bytes]}\t#{data[:last_modified]}"
+	end
+
+The output will look something like this::
+
+   myphoto1.jpg	251262	2011-08-08T21:35:48.000Z
+   myphoto2.jpg	262518	2011-08-08T21:38:01.000Z
+
+
+
+Retrieve an Object
+==================
+
+This downloads the object ``hello.txt`` and saves it in
+``./my_hello.txt``:
+
+.. code-block:: ruby
+
+	obj = container.object('hello.txt')
+	obj.save_to_filename('./my_hello.txt')
+
+
+Delete an Object
+================
+
+This deletes the object ``goodbye.txt``:
+
+.. code-block:: ruby
+
+	container.delete_object('goodbye.txt')
+	
+
+Delete a Container
+==================
+
+.. note::
+
+   The container must be empty! Otherwise the request won't work!
+
+.. code-block:: ruby
+
+	container.delete_container('my-new-container')
diff --git a/doc/radosgw/swift/serviceops.rst b/doc/radosgw/swift/serviceops.rst
new file mode 100644
index 000000000..a00f3d807
--- /dev/null
+++ b/doc/radosgw/swift/serviceops.rst
@@ -0,0 +1,76 @@
+====================
+ Service Operations
+====================
+
+To retrieve data about our Swift-compatible service, you may execute ``GET`` 
+requests using the ``X-Storage-Url`` value retrieved during authentication. 
+
+List Containers
+===============
+
+A ``GET`` request that specifies the API version and the account will return
+a list of containers for a particular user account. Since the request returns
+a particular user's containers, the request requires an authentication token.
+The request cannot be made anonymously.
+
+Syntax
+~~~~~~
+
+::
+
+	GET /{api version}/{account} HTTP/1.1
+	Host: {fqdn}
+	X-Auth-Token: {auth-token}
+
+
+
+Request Parameters
+~~~~~~~~~~~~~~~~~~
+
+``limit``
+
+:Description: Limits the number of results to the specified value.
+:Type: Integer
+:Required: No
+
+``format``
+
+:Description: Defines the format of the result. 
+:Type: String
+:Valid Values: ``json`` | ``xml``
+:Required: No
+
+
+``marker``
+
+:Description: Returns a list of results greater than the marker value.
+:Type: String
+:Required: No
+
+
+
+Response Entities
+~~~~~~~~~~~~~~~~~
+
+The response contains a list of containers, or returns with an HTTP
+204 response code
+
+``account``
+
+:Description: A list for account information.
+:Type: Container
+
+``container``
+
+:Description: The list of containers.
+:Type: Container
+
+``name``
+
+:Description: The name of a container.
+:Type: String
+
+``bytes``
+
+:Description: The size of the container.
+:Type: Integer
\ No newline at end of file
diff --git a/doc/radosgw/swift/tempurl.rst b/doc/radosgw/swift/tempurl.rst
new file mode 100644
index 000000000..79b392de6
--- /dev/null
+++ b/doc/radosgw/swift/tempurl.rst
@@ -0,0 +1,102 @@
+====================
+ Temp URL Operations
+====================
+
+To allow temporary access (for eg for `GET` requests) to objects
+without the need to share credentials, temp url functionality is
+supported by swift endpoint of radosgw. For this functionality,
+initially the value of `X-Account-Meta-Temp-URL-Key` and optionally
+`X-Account-Meta-Temp-URL-Key-2` should be set. The Temp URL
+functionality relies on a HMAC-SHA1 signature against these secret
+keys.
+
+.. note:: If you are planning to expose Temp URL functionality for the
+	  Swift API, it is strongly recommended to include the Swift
+	  account name in the endpoint definition, so as to most
+	  closely emulate the behavior of native OpenStack Swift. To
+	  do so, set the ``ceph.conf`` configuration option ``rgw
+	  swift account in url = true``, and update your Keystone
+	  endpoint to the URL suffix ``/v1/AUTH_%(tenant_id)s``
+	  (instead of just ``/v1``).
+
+
+POST Temp-URL Keys
+==================
+
+A ``POST`` request to the Swift account with the required key will set
+the secret temp URL key for the account, against which temporary URL
+access can be provided to accounts. Up to two keys are supported, and
+signatures are checked against both the keys, if present, so that keys
+can be rotated without invalidating the temporary URLs.
+
+.. note:: Native OpenStack Swift also supports the option to set
+          temporary URL keys at the container level, issuing a
+          ``POST`` or ``PUT`` request against a container that sets
+          ``X-Container-Meta-Temp-URL-Key`` or
+          ``X-Container-Meta-Temp-URL-Key-2``. This functionality is
+          not supported in radosgw; temporary URL keys can only be set
+          and used at the account level.
+
+Syntax
+~~~~~~
+
+::
+
+	POST /{api version}/{account} HTTP/1.1
+	Host: {fqdn}
+	X-Auth-Token: {auth-token}
+
+Request Headers
+~~~~~~~~~~~~~~~
+
+``X-Account-Meta-Temp-URL-Key``
+
+:Description: A user-defined key that takes an arbitrary string value.
+:Type: String
+:Required: Yes
+
+``X-Account-Meta-Temp-URL-Key-2``
+
+:Description: A user-defined key that takes an arbitrary string value.
+:Type: String
+:Required: No
+
+
+GET Temp-URL Objects
+====================
+
+Temporary URL uses a cryptographic HMAC-SHA1 signature, which includes
+the following elements:
+
+#. The value of the Request method, "GET" for instance
+#. The expiry time, in format of seconds since the epoch, ie Unix time
+#. The request path starting from "v1" onwards
+
+The above items are normalized with newlines appended between them,
+and a HMAC is generated using the SHA-1 hashing algorithm against one
+of the Temp URL Keys posted earlier.
+
+A sample python script to demonstrate the above is given below:
+
+
+.. code-block:: python
+
+   import hmac
+   from hashlib import sha1
+   from time import time
+
+   method = 'GET'
+   host = 'https://objectstore.example.com/swift'
+   duration_in_seconds = 300  # Duration for which the url is valid
+   expires = int(time() + duration_in_seconds)
+   path = '/v1/your-bucket/your-object'
+   key = 'secret'
+   hmac_body = '%s\n%s\n%s' % (method, expires, path)
+   sig = hmac.new(key, hmac_body, sha1).hexdigest()
+   rest_uri = "{host}{path}?temp_url_sig={sig}&temp_url_expires={expires}".format(
+		host=host, path=path, sig=sig, expires=expires)
+   print rest_uri
+
+   # Example Output
+   # https://objectstore.example.com/swift/v1/your-bucket/your-object?temp_url_sig=ff4657876227fc6025f04fcf1e82818266d022c6&temp_url_expires=1423200992
+
diff --git a/doc/radosgw/swift/tutorial.rst b/doc/radosgw/swift/tutorial.rst
new file mode 100644
index 000000000..5d2889b19
--- /dev/null
+++ b/doc/radosgw/swift/tutorial.rst
@@ -0,0 +1,62 @@
+==========
+ Tutorial
+==========
+
+The Swift-compatible API tutorials follow a simple container-based object 
+lifecycle. The first step requires you to setup a connection between your 
+client and the RADOS Gateway server. Then, you may follow a natural 
+container and object lifecycle, including adding and retrieving object 
+metadata. See example code for the following languages:
+
+- `Java`_
+- `Python`_
+- `Ruby`_
+
+
+.. ditaa::
+
+           +----------------------------+        +-----------------------------+
+           |                            |        |                             |
+           |    Create a Connection     |------->|      Create a Container     |
+           |                            |        |                             |
+           +----------------------------+        +-----------------------------+
+                                                                |
+                         +--------------------------------------+ 
+                         |
+                         v
+           +----------------------------+        +-----------------------------+
+           |                            |        |                             |
+           |     Create an Object       |------->| Add/Update Object Metadata  |
+           |                            |        |                             |
+           +----------------------------+        +-----------------------------+
+                                                                |
+                         +--------------------------------------+ 
+                         |
+                         v
+           +----------------------------+        +-----------------------------+
+           |                            |        |                             |
+           |   List Owned Containers    |------->| List a Container's Contents |
+           |                            |        |                             |
+           +----------------------------+        +-----------------------------+
+                                                                |
+                         +--------------------------------------+ 
+                         |
+                         v
+           +----------------------------+        +-----------------------------+
+           |                            |        |                             |
+           | Get an Object's Metadata   |------->|     Retrieve an Object      |
+           |                            |        |                             |
+           +----------------------------+        +-----------------------------+
+                                                                |
+                         +--------------------------------------+ 
+                         |
+                         v
+           +----------------------------+        +-----------------------------+
+           |                            |        |                             |
+           |      Delete an Object      |------->|      Delete a Container     |
+           |                            |        |                             |
+           +----------------------------+        +-----------------------------+
+
+.. _Java: ../java
+.. _Python: ../python
+.. _Ruby: ../ruby
diff --git a/doc/radosgw/sync-modules.rst b/doc/radosgw/sync-modules.rst
new file mode 100644
index 000000000..ca201e90a
--- /dev/null
+++ b/doc/radosgw/sync-modules.rst
@@ -0,0 +1,99 @@
+============
+Sync Modules
+============
+
+.. versionadded:: Kraken
+
+The :ref:`multisite` functionality of RGW introduced in Jewel allowed the ability to
+create multiple zones and mirror data and metadata between them. ``Sync Modules``
+are built atop of the multisite framework that allows for forwarding data and
+metadata to a different external tier. A sync module allows for a set of actions
+to be performed whenever a change in data occurs (metadata ops like bucket or
+user creation etc. are also regarded as changes in data). As the rgw multisite
+changes are eventually consistent at remote sites, changes are propagated
+asynchronously. This would allow for unlocking use cases such as backing up the
+object storage to an external cloud cluster or a custom backup solution using
+tape drives, indexing metadata in ElasticSearch etc.
+
+A sync module configuration is local to a zone. The sync module determines
+whether the zone exports data or can only consume data that was modified in
+another zone. As of luminous the supported sync plugins are `elasticsearch`_,
+``rgw``, which is the default sync plugin that synchronises data between the
+zones and ``log`` which is a trivial sync plugin that logs the metadata
+operation that happens in the remote zones. The following docs are written with
+the example of a zone using `elasticsearch sync module`_, the process would be similar
+for configuring any sync plugin
+
+.. toctree::
+   :maxdepth: 1
+
+   ElasticSearch Sync Module <elastic-sync-module>
+   Cloud Sync Module <cloud-sync-module>
+   PubSub Module <pubsub-module>
+   Archive Sync Module <archive-sync-module>
+
+.. note ``rgw`` is the default sync plugin and there is no need to explicitly
+   configure this
+
+Requirements and Assumptions
+----------------------------
+
+Let us assume a simple multisite configuration as described in the :ref:`multisite`
+docs, of 2 zones ``us-east`` and ``us-west``, let's add a third zone
+``us-east-es`` which is a zone that only processes metadata from the other
+sites. This zone can be in the same or a different ceph cluster as ``us-east``.
+This zone would only consume metadata from other zones and RGWs in this zone
+will not serve any end user requests directly.
+
+
+Configuring Sync Modules
+------------------------
+
+Create the third zone similar to the :ref:`multisite` docs, for example
+
+::
+
+   # radosgw-admin zone create --rgw-zonegroup=us --rgw-zone=us-east-es \
+   --access-key={system-key} --secret={secret} --endpoints=http://rgw-es:80
+
+
+
+A sync module can be configured for this zone via the following
+
+::
+
+   # radosgw-admin zone modify --rgw-zone={zone-name} --tier-type={tier-type} --tier-config={set of key=value pairs}
+
+
+For example in the ``elasticsearch`` sync module
+
+::
+
+   # radosgw-admin zone modify --rgw-zone={zone-name} --tier-type=elasticsearch \
+                               --tier-config=endpoint=http://localhost:9200,num_shards=10,num_replicas=1
+
+
+For the various supported tier-config options refer to the `elasticsearch sync module`_ docs
+
+Finally update the period
+
+
+::
+
+    # radosgw-admin period update --commit
+
+
+Now start the radosgw in the zone
+
+::
+
+    # systemctl start ceph-radosgw@rgw.`hostname -s`
+    # systemctl enable ceph-radosgw@rgw.`hostname -s`
+
+
+
+.. _`elasticsearch sync module`: ../elastic-sync-module
+.. _`elasticsearch`: ../elastic-sync-module
+.. _`cloud sync module`: ../cloud-sync-module
+.. _`pubsub module`: ../pubsub-module
+.. _`archive sync module`: ../archive-sync-module
diff --git a/doc/radosgw/troubleshooting.rst b/doc/radosgw/troubleshooting.rst
new file mode 100644
index 000000000..4a084e82a
--- /dev/null
+++ b/doc/radosgw/troubleshooting.rst
@@ -0,0 +1,208 @@
+=================
+ Troubleshooting
+=================
+
+
+The Gateway Won't Start
+=======================
+
+If you cannot start the gateway (i.e., there is no existing ``pid``), 
+check to see if there is an existing ``.asok`` file from another 
+user. If an ``.asok`` file from another user exists and there is no
+running ``pid``, remove the ``.asok`` file and try to start the
+process again. This may occur when you start the process as a ``root`` user and 
+the startup script is trying to start the process as a 
+``www-data`` or ``apache`` user and an existing ``.asok`` is 
+preventing the script from starting the daemon.
+
+The radosgw init script (/etc/init.d/radosgw) also has a verbose argument that
+can provide some insight as to what could be the issue::
+
+  /etc/init.d/radosgw start -v
+
+or ::
+
+  /etc/init.d radosgw start --verbose
+
+HTTP Request Errors
+===================
+
+Examining the access and error logs for the web server itself is
+probably the first step in identifying what is going on.  If there is
+a 500 error, that usually indicates a problem communicating with the
+``radosgw`` daemon.  Ensure the daemon is running, its socket path is
+configured, and that the web server is looking for it in the proper
+location.
+
+
+Crashed ``radosgw`` process
+===========================
+
+If the ``radosgw`` process dies, you will normally see a 500 error
+from the web server (apache, nginx, etc.).  In that situation, simply
+restarting radosgw will restore service.
+
+To diagnose the cause of the crash, check the log in ``/var/log/ceph``
+and/or the core file (if one was generated).
+
+
+Blocked ``radosgw`` Requests
+============================
+
+If some (or all) radosgw requests appear to be blocked, you can get
+some insight into the internal state of the ``radosgw`` daemon via
+its admin socket.  By default, there will be a socket configured to
+reside in ``/var/run/ceph``, and the daemon can be queried with::
+
+ ceph daemon /var/run/ceph/client.rgw help
+ 
+ help                list available commands
+ objecter_requests   show in-progress osd requests
+ perfcounters_dump   dump perfcounters value
+ perfcounters_schema dump perfcounters schema
+ version             get protocol version
+
+Of particular interest::
+
+ ceph daemon /var/run/ceph/client.rgw objecter_requests
+ ...
+
+will dump information about current in-progress requests with the
+RADOS cluster.  This allows one to identify if any requests are blocked
+by a non-responsive OSD.  For example, one might see::
+
+  { "ops": [
+        { "tid": 1858,
+          "pg": "2.d2041a48",
+          "osd": 1,
+          "last_sent": "2012-03-08 14:56:37.949872",
+          "attempts": 1,
+          "object_id": "fatty_25647_object1857",
+          "object_locator": "@2",
+          "snapid": "head",
+          "snap_context": "0=[]",
+          "mtime": "2012-03-08 14:56:37.949813",
+          "osd_ops": [
+                "write 0~4096"]},
+        { "tid": 1873,
+          "pg": "2.695e9f8e",
+          "osd": 1,
+          "last_sent": "2012-03-08 14:56:37.970615",
+          "attempts": 1,
+          "object_id": "fatty_25647_object1872",
+          "object_locator": "@2",
+          "snapid": "head",
+          "snap_context": "0=[]",
+          "mtime": "2012-03-08 14:56:37.970555",
+          "osd_ops": [
+                "write 0~4096"]}],
+  "linger_ops": [],
+  "pool_ops": [],
+  "pool_stat_ops": [],
+  "statfs_ops": []}
+
+In this dump, two requests are in progress.  The ``last_sent`` field is
+the time the RADOS request was sent.  If this is a while ago, it suggests
+that the OSD is not responding.  For example, for request 1858, you could
+check the OSD status with::
+
+ ceph pg map 2.d2041a48
+ 
+ osdmap e9 pg 2.d2041a48 (2.0) -> up [1,0] acting [1,0]
+
+This tells us to look at ``osd.1``, the primary copy for this PG::
+
+ ceph daemon osd.1 ops
+ { "num_ops": 651,
+  "ops": [
+        { "description": "osd_op(client.4124.0:1858 fatty_25647_object1857 [write 0~4096] 2.d2041a48)",
+          "received_at": "1331247573.344650",
+          "age": "25.606449",
+          "flag_point": "waiting for sub ops",
+          "client_info": { "client": "client.4124",
+              "tid": 1858}},
+ ...
+
+The ``flag_point`` field indicates that the OSD is currently waiting
+for replicas to respond, in this case ``osd.0``.
+
+
+Java S3 API Troubleshooting
+===========================
+
+
+Peer Not Authenticated
+----------------------
+
+You may receive an error that looks like this:: 
+
+     [java] INFO: Unable to execute HTTP request: peer not authenticated
+
+The Java SDK for S3 requires a valid certificate from a recognized certificate
+authority, because it uses HTTPS by default. If you are just testing the Ceph
+Object Storage services, you can resolve this problem in a few ways:  
+
+#. Prepend the IP address or hostname with ``http://``. For example, change this::
+
+	conn.setEndpoint("myserver");
+
+   To:: 
+
+	conn.setEndpoint("http://myserver")
+
+#. After setting your credentials, add a client configuration and set the 
+   protocol to ``Protocol.HTTP``. :: 
+
+			AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
+			
+			ClientConfiguration clientConfig = new ClientConfiguration();
+			clientConfig.setProtocol(Protocol.HTTP);
+			
+			AmazonS3 conn = new AmazonS3Client(credentials, clientConfig);
+
+
+
+405 MethodNotAllowed
+--------------------
+
+If you receive an 405 error, check to see if you have the S3 subdomain set up correctly. 
+You will need to have a wild card setting in your DNS record for subdomain functionality
+to work properly.
+
+Also, check to ensure that the default site is disabled. ::
+
+     [java] Exception in thread "main" Status Code: 405, AWS Service: Amazon S3, AWS Request ID: null, AWS Error Code: MethodNotAllowed, AWS Error Message: null, S3 Extended Request ID: null
+  
+  
+  
+Numerous objects in default.rgw.meta pool
+=========================================
+
+Clusters created prior to *jewel* have a metadata archival feature enabled by default, using the ``default.rgw.meta`` pool.
+This archive keeps all old versions of user and bucket metadata, resulting in large numbers of objects in the ``default.rgw.meta`` pool.
+
+Disabling the Metadata Heap
+---------------------------
+
+Users who want to disable this feature going forward should set the ``metadata_heap`` field to an empty string ``""``::
+
+  $ radosgw-admin zone get --rgw-zone=default > zone.json
+  [edit zone.json, setting "metadata_heap": ""]
+  $ radosgw-admin zone set --rgw-zone=default --infile=zone.json
+  $ radosgw-admin period update --commit
+
+This will stop new metadata from being written to the ``default.rgw.meta`` pool, but does not remove any existing objects or pool.
+
+Cleaning the Metadata Heap Pool
+-------------------------------
+
+Clusters created prior to *jewel* normally use ``default.rgw.meta`` only for the metadata archival feature.
+
+However, from *luminous* onwards, radosgw uses :ref:`Pool Namespaces <radosgw-pool-namespaces>` within ``default.rgw.meta`` for an entirely different purpose, that is, to store ``user_keys`` and other critical metadata.
+
+Users should check zone configuration before proceeding any cleanup procedures::
+
+  $ radosgw-admin zone get --rgw-zone=default | grep default.rgw.meta
+  [should not match any strings]
+
+Having confirmed that the pool is not used for any purpose, users may safely delete all objects in the ``default.rgw.meta`` pool, or optionally, delete the entire pool itself.
diff --git a/doc/radosgw/vault.rst b/doc/radosgw/vault.rst
new file mode 100644
index 000000000..0f3cb8fd1
--- /dev/null
+++ b/doc/radosgw/vault.rst
@@ -0,0 +1,482 @@
+===========================
+HashiCorp Vault Integration
+===========================
+
+HashiCorp `Vault`_ can be used as a secure key management service for
+`Server-Side Encryption`_ (SSE-KMS).
+
+.. ditaa::
+
+           +---------+       +---------+        +-------+     +-------+
+           |  Client |       | RadosGW |        | Vault |     |  OSD  |
+           +---------+       +---------+        +-------+     +-------+
+                | create secret   |                 |             |
+                | key for key ID  |                 |             |
+                |-----------------+---------------->|             |
+                |                 |                 |             |
+                | upload object   |                 |             |
+                | with key ID     |                 |             |
+                |---------------->| request secret  |             |
+                |                 | key for key ID  |             |
+                |                 |---------------->|             |
+                |                 |<----------------|             |
+                |                 | return secret   |             |
+                |                 | key             |             |
+                |                 |                 |             |
+                |                 | encrypt object  |             |
+                |                 | with secret key |             |
+                |                 |--------------+  |             |
+                |                 |              |  |             |
+                |                 |<-------------+  |             |
+                |                 |                 |             |
+                |                 | store encrypted |             |
+                |                 | object          |             |
+                |                 |------------------------------>|
+
+#. `Vault secrets engines`_
+#. `Vault authentication`_
+#. `Vault namespaces`_
+#. `Create a key in Vault`_
+#. `Configure the Ceph Object Gateway`_
+#. `Upload object`_
+
+Some examples below use the Vault command line utility to interact with
+Vault. You may need to set the following environment variable with the correct
+address of your Vault server to use this utility::
+
+  export VAULT_ADDR='https://vault-server-fqdn:8200'
+
+Vault secrets engines
+=====================
+
+Vault provides several secrets engines, which can store, generate, and encrypt
+data. Currently, the Object Gateway supports:
+
+- `KV secrets engine`_ version 2
+- `Transit engine`_
+
+KV secrets engine
+-----------------
+
+The KV secrets engine is used to store arbitrary key/value secrets in Vault. To
+enable the KV engine version 2 in Vault, use the following command::
+
+  vault secrets enable -path secret kv-v2
+
+The Object Gateway can be configured to use the KV engine version 2 with the
+following setting::
+
+  rgw crypt vault secret engine = kv
+
+Transit secrets engine
+----------------------
+
+The transit engine handles cryptographic functions on data in-transit. To enable
+it in Vault, use the following command::
+
+  vault secrets enable transit
+
+The Object Gateway can be configured to use the transit engine with the
+following setting::
+
+  rgw crypt vault secret engine = transit
+
+Vault authentication
+====================
+
+Vault supports several authentication mechanisms. Currently, the Object
+Gateway can be configured to authenticate to Vault using the
+`Token authentication method`_ or a `Vault agent`_.
+
+Most tokens in Vault have limited lifetimes and powers.  The only
+sort of Vault token that does not have a lifetime are root tokens.
+For all other tokens, it is necesary to periodically refresh them,
+either by performing initial authentication, or by renewing the token.
+Ceph does not have any logic to perform either operation.
+The simplest best way to use Vault tokens with ceph is to
+also run the Vault agent and have it refresh the token file.
+When the Vault agent is used in this mode, file system permissions
+can be used to restrict who has the use of tokens.
+
+Instead of having Vault agent refresh a token file, it can be told
+to act as a proxy server.  In this mode, Vault will add a token when
+necessary and add it to requests passed to it before forwarding them on
+to the real server.  Vault agent will still handle token renewal just
+as it would when storing a token in the filesystem.  In this mode, it
+is necessary to properly secure the network path rgw uses to reach the
+Vault agent, such as having the Vault agent listen only to localhost.
+
+Token policies for the object gateway
+-------------------------------------
+
+All Vault tokens have powers as specified by the polices attached
+to that token.  Multiple policies may be associated with one
+token.  You should only use the policy necessary for your
+configuration.
+
+When using the kv secret engine with the object gateway::
+
+  vault policy write rgw-kv-policy -<<EOF
+    path "secret/data/*" {
+      capabilities = ["read"]
+    }
+  EOF
+
+When using the transit secret engine with the object gateway::
+
+  vault policy write rgw-transit-policy -<<EOF
+    path "transit/keys/*" {
+      capabilities = [ "create", "update" ]
+      denied_parameters = {"exportable" = [], "allow_plaintext_backup" = [] }
+    }
+
+    path "transit/keys/*" {
+      capabilities = ["read", "delete"]
+    }
+
+    path "transit/keys/" {
+      capabilities = ["list"]
+    }
+
+    path "transit/keys/+/rotate" {
+      capabilities = [ "update" ]
+    }
+
+    path "transit/*" {
+      capabilities = [ "update" ]
+    }
+  EOF
+
+If you had previously used an older version of ceph with the
+transit secret engine, you might need the following policy::
+
+  vault policy write old-rgw-transit-policy -<<EOF
+    path "transit/export/encryption-key/*" {
+      capabilities = ["read"]
+    }
+  EOF
+
+
+Token authentication
+--------------------
+
+.. note: Never use root tokens with ceph in production environments.
+
+The token authentication method expects a Vault token to be present in a
+plaintext file. The Object Gateway can be configured to use token authentication
+with the following settings::
+
+  rgw crypt vault auth = token
+  rgw crypt vault token file = /run/.rgw-vault-token
+  rgw crypt vault addr = https://vault-server-fqdn:8200
+
+Adjust these settinsg to match your configuration.
+For security reasons, the token file must be readable by the Object Gateway
+only.
+
+You might set up vault agent as follows::
+
+  vault write auth/approle/role/rgw-ap \
+    token_policies=rgw-transit-policy,default \
+    token_max_ttl=60m
+
+Change the policy here to match your configuration.
+
+Get the role-id::
+
+  vault read auth/approle/role/rgw-ap/role-id -format=json | \
+    jq -r .data.role_id
+
+Store the output in some file, such as /usr/local/etc/vault/.rgw-ap-role-id
+
+Get the secret-id::
+
+  vault read auth/approle/role/rgw-ap/role-id -format=json | \
+    jq -r .data.role_id
+
+Store the output in some file, such as /usr/local/etc/vault/.rgw-ap-secret-id
+
+Create configuration for the Vault agent, such as::
+
+  pid_file = "/run/rgw-vault-agent-pid"
+  auto_auth {
+    method "AppRole" {
+      mount_path = "auth/approle"
+      config = {
+        role_id_file_path ="/usr/local/etc/vault/.rgw-ap-role-id"
+        secret_id_file_path ="/usr/local/etc/vault/.rgw-ap-secret-id"
+        remove_secret_id_file_after_reading ="false"
+      }
+    }
+    sink "file" {
+      config = {
+        path = "/run/.rgw-vault-token"
+      }
+    }
+  }
+  vault {
+    address = "https://vault-server-fqdn:8200"
+  }
+
+Then use systemctl or another method of your choice to run
+a persistent daemon with the following arguments::
+
+    /usr/local/bin/vault agent -config=/usr/local/etc/vault/rgw-agent.hcl
+
+Once the vault agent is running, the token file should be populated
+with a valid token.
+
+Vault agent
+-----------
+
+The Vault agent is a client daemon that provides authentication to Vault and
+manages token renewal and caching. It typically runs on the same host as the
+Object Gateway. With a Vault agent, it is possible to use other Vault
+authentication mechanism such as AppRole, AWS, Certs, JWT, and Azure.
+
+The Object Gateway can be configured to use a Vault agent with the following
+settings::
+
+  rgw crypt vault auth = agent
+  rgw crypt vault addr = http://127.0.0.1:8100
+
+You might set up vault agent as follows::
+
+  vault write auth/approle/role/rgw-ap \
+    token_policies=rgw-transit-policy,default \
+    token_max_ttl=60m
+
+Change the policy here to match your configuration.
+
+Get the role-id:
+  vault read auth/approle/role/rgw-ap/role-id -format=json | \
+    jq -r .data.role_id
+
+Store the output in some file, such as /usr/local/etc/vault/.rgw-ap-role-id
+
+Get the secret-id:
+  vault read auth/approle/role/rgw-ap/role-id -format=json | \
+    jq -r .data.role_id
+
+Store the output in some file, such as /usr/local/etc/vault/.rgw-ap-secret-id
+
+Create configuration for the Vault agent, such as::
+
+  pid_file = "/run/rgw-vault-agent-pid"
+  auto_auth {
+    method "AppRole" {
+      mount_path = "auth/approle"
+      config = {
+        role_id_file_path ="/usr/local/etc/vault/.rgw-ap-role-id"
+        secret_id_file_path ="/usr/local/etc/vault/.rgw-ap-secret-id"
+        remove_secret_id_file_after_reading ="false"
+      }
+    }
+  }
+  cache {
+    use_auto_auth_token = true
+  }
+  listener "tcp" {
+    address = "127.0.0.1:8100"
+    tls_disable = true
+  }
+  vault {
+    address = "https://vault-server-fqdn:8200"
+  }
+
+Then use systemctl or another method of your choice to run
+a persistent daemon with the following arguments::
+
+    /usr/local/bin/vault agent -config=/usr/local/etc/vault/rgw-agent.hcl
+
+Once the vault agent is running, you should find it listening
+to port 8100 on localhost, and you should be able to interact
+with it using the vault command.
+
+Vault namespaces
+================
+
+In the Enterprise version, Vault supports the concept of `namespaces`_, which
+allows centralized management for teams within an organization while ensuring
+that those teams operate within isolated environments known as tenants.
+
+The Object Gateway can be configured to access Vault within a particular
+namespace using the following configuration setting::
+
+  rgw crypt vault namespace = tenant1
+
+Create a key in Vault
+=====================
+
+.. note:: Keys for server-side encryption must be 256-bit long and base-64
+   encoded.
+
+Using the KV engine
+-------------------
+
+A key for server-side encryption can be created in the KV version 2 engine using
+the command line utility, as in the following example::
+
+  vault kv put secret/myproject/mybucketkey key=$(openssl rand -base64 32)
+
+Sample output::
+
+  ====== Metadata ======
+  Key              Value
+  ---              -----
+  created_time     2019-08-29T17:01:09.095824999Z
+  deletion_time    n/a
+  destroyed        false
+  version          1
+
+Note that in the KV secrets engine, secrets are stored as key-value pairs, and
+the Gateway expects the key name to be ``key``, i.e. the secret must be in the
+form ``key=<secret key>``.
+
+Using the Transit engine
+------------------------
+
+Keys created for use with the Transit engine should no longer be marked
+exportable.  They can be created with::
+
+  vault write -f transit/keys/mybucketkey
+
+The command above creates a keyring, which contains a key of type
+``aes256-gcm96`` by default. To verify that the key was correctly created, use
+the following command::
+
+  vault read transit/mybucketkey
+
+Sample output::
+
+  Key     Value
+  ---     -----
+  derived                   false
+  exportable                false
+  name                      mybucketkey
+  type                      aes256-gcm96
+
+Configure the Ceph Object Gateway
+=================================
+
+Edit the Ceph configuration file to enable Vault as a KMS backend for
+server-side encryption::
+
+  rgw crypt s3 kms backend = vault
+
+Choose the Vault authentication method, e.g.::
+
+  rgw crypt vault auth = token
+  rgw crypt vault token file = /run/.rgw-vault-token
+  rgw crypt vault addr = https://vault-server-fqdn:8200
+
+Or::
+
+  rgw crypt vault auth = agent
+  rgw crypt vault addr = http://localhost:8100
+
+Choose the secrets engine::
+
+  rgw crypt vault secret engine = kv
+
+Or::
+
+  rgw crypt vault secret engine = transit
+
+Optionally, set the Vault namespace where encryption keys will be fetched from::
+
+  rgw crypt vault namespace = tenant1
+
+Finally, the URLs where the Gateway will retrieve encryption keys from Vault can
+be restricted by setting a path prefix. For instance, the Gateway can be
+restricted to fetch KV keys as follows::
+
+  rgw crypt vault prefix = /v1/secret/data
+
+Or, when using the transit secret engine::
+
+  rgw crypt vault prefix = /v1/transit
+
+In the example above, the Gateway would only fetch transit encryption keys under
+``https://vault-server:8200/v1/transit``.
+
+You can use custom ssl certs to authenticate with vault with help of
+following options::
+
+  rgw crypt vault verify ssl = true
+  rgw crypt vault ssl cacert = /etc/ceph/vault.ca
+  rgw crypt vault ssl clientcert = /etc/ceph/vault.crt
+  rgw crypt vault ssl clientkey = /etc/ceph/vault.key
+
+where vault.ca is CA certificate and vault.key/vault.crt are private key and ssl
+ceritificate generated for RGW to access the vault server. It highly recommended to
+set this option true, setting false is very dangerous and need to avoid since this
+runs in very secured enviroments.
+
+Transit engine compatibility support
+------------------------------------
+The transit engine has compatibility support for previous
+versions of ceph, which used the transit engine as a simple key store.
+
+There is a a "compat" option which can be given to the transit
+engine to configure the compatibility support,
+
+To entirely disable backwards support, use::
+
+  rgw crypt vault secret engine = transit compat=0
+
+This will be the default in future verisons. and is safe to use
+for new installs using the current version.
+
+This is the normal default with the current version::
+
+  rgw crypt vault secret engine = transit compat=1
+
+This enables the new engine for newly created objects,
+but still allows the old engine to be used for old objects.
+In order to access old and new objects, the vault token given
+to ceph must have both the old and new transit policies.
+
+To force use of only the old engine, use::
+
+  rgw crypt vault secret engine = transit compat=2
+
+This mode is automatically selected if the vault prefix
+ends in export/encryption-key, which was the previously
+documented setting.
+
+Upload object
+=============
+
+When uploading an object to the Gateway, provide the SSE key ID in the request.
+As an example, for the kv engine, using the AWS command-line client::
+
+  aws --endpoint=http://radosgw:8000 s3 cp plaintext.txt s3://mybucket/encrypted.txt --sse=aws:kms --sse-kms-key-id myproject/mybucketkey
+  
+As an example, for the transit engine (new flavor), using the AWS command-line client::
+
+  aws --endpoint=http://radosgw:8000 s3 cp plaintext.txt s3://mybucket/encrypted.txt --sse=aws:kms --sse-kms-key-id mybucketkey
+
+The Object Gateway will fetch the key from Vault, encrypt the object and store
+it in the bucket. Any request to download the object will make the Gateway
+automatically retrieve the correspondent key from Vault and decrypt the object.
+
+Note that the secret will be fetched from Vault using a URL constructed by
+concatenating the base address (``rgw crypt vault addr``), the (optional)
+URL prefix (``rgw crypt vault prefix``), and finally the key ID. 
+
+In the kv engine example above, the Gateway would fetch the secret from::
+
+  http://vaultserver:8200/v1/secret/data/myproject/mybucketkey
+
+In the transit engine example above, the Gateway would encrypt the secret using this key::
+
+  http://vaultserver:8200/v1/transit/mybucketkey
+
+.. _Server-Side Encryption: ../encryption
+.. _Vault: https://www.vaultproject.io/docs/
+.. _Token authentication method: https://www.vaultproject.io/docs/auth/token.html
+.. _Vault agent: https://www.vaultproject.io/docs/agent/index.html
+.. _KV Secrets engine: https://www.vaultproject.io/docs/secrets/kv/
+.. _Transit engine: https://www.vaultproject.io/docs/secrets/transit
+.. _namespaces: https://www.vaultproject.io/docs/enterprise/namespaces/index.html
diff --git a/doc/rbd/api/index.rst b/doc/rbd/api/index.rst
new file mode 100644
index 000000000..27bb4485d
--- /dev/null
+++ b/doc/rbd/api/index.rst
@@ -0,0 +1,8 @@
+========================
+ Ceph Block Device APIs
+========================
+
+.. toctree::
+   :maxdepth: 2
+
+   librbd (Python) <librbdpy>
diff --git a/doc/rbd/api/librbdpy.rst b/doc/rbd/api/librbdpy.rst
new file mode 100644
index 000000000..981235f87
--- /dev/null
+++ b/doc/rbd/api/librbdpy.rst
@@ -0,0 +1,83 @@
+================
+ Librbd (Python)
+================
+
+.. highlight:: python
+
+The `rbd` python module provides file-like access to RBD images.
+
+
+Example: Creating and writing to an image
+=========================================
+
+To use `rbd`, you must first connect to RADOS and open an IO
+context::
+
+    cluster = rados.Rados(conffile='my_ceph.conf')
+    cluster.connect()
+    ioctx = cluster.open_ioctx('mypool')
+
+Then you instantiate an :class:rbd.RBD object, which you use to create the
+image::
+
+    rbd_inst = rbd.RBD()
+    size = 4 * 1024**3  # 4 GiB
+    rbd_inst.create(ioctx, 'myimage', size)
+
+To perform I/O on the image, you instantiate an :class:rbd.Image object::
+
+    image = rbd.Image(ioctx, 'myimage')
+    data = 'foo' * 200
+    image.write(data, 0)
+
+This writes 'foo' to the first 600 bytes of the image. Note that data
+cannot be :type:unicode - `Librbd` does not know how to deal with
+characters wider than a :c:type:char.
+
+In the end, you will want to close the image, the IO context and the connection to RADOS::
+
+    image.close()
+    ioctx.close()
+    cluster.shutdown()
+
+To be safe, each of these calls would need to be in a separate :finally
+block::
+
+    cluster = rados.Rados(conffile='my_ceph_conf')
+    try:
+        cluster.connect()
+        ioctx = cluster.open_ioctx('my_pool')
+        try:
+            rbd_inst = rbd.RBD()
+            size = 4 * 1024**3  # 4 GiB
+            rbd_inst.create(ioctx, 'myimage', size)
+            image = rbd.Image(ioctx, 'myimage')
+            try:
+                data = 'foo' * 200
+                image.write(data, 0)
+            finally:
+                image.close()
+        finally:
+            ioctx.close()
+    finally:
+        cluster.shutdown()
+
+This can be cumbersome, so the :class:`Rados`, :class:`Ioctx`, and
+:class:`Image` classes can be used as context managers that close/shutdown
+automatically (see :pep:`343`). Using them as context managers, the
+above example becomes::
+
+    with rados.Rados(conffile='my_ceph.conf') as cluster:
+        with cluster.open_ioctx('mypool') as ioctx:
+            rbd_inst = rbd.RBD()
+            size = 4 * 1024**3  # 4 GiB
+            rbd_inst.create(ioctx, 'myimage', size)
+            with rbd.Image(ioctx, 'myimage') as image:
+                data = 'foo' * 200
+                image.write(data, 0)
+
+API Reference
+=============
+
+.. automodule:: rbd
+    :members: RBD, Image, SnapIterator
diff --git a/doc/rbd/disk.conf b/doc/rbd/disk.conf
new file mode 100644
index 000000000..3db9b8a11
--- /dev/null
+++ b/doc/rbd/disk.conf
@@ -0,0 +1,8 @@
+<disk type='network' device='disk'>
+	<source protocol='rbd' name='poolname/imagename'>
+		<host name='{fqdn}' port='6789'/>
+		<host name='{fqdn}' port='6790'/>
+		<host name='{fqdn}' port='6791'/>
+	</source>
+	<target dev='vda' bus='virtio'/>
+</disk>
diff --git a/doc/rbd/index.rst b/doc/rbd/index.rst
new file mode 100644
index 000000000..4a8029bba
--- /dev/null
+++ b/doc/rbd/index.rst
@@ -0,0 +1,72 @@
+===================
+ Ceph Block Device
+===================
+
+.. index:: Ceph Block Device; introduction
+
+A block is a sequence of bytes (often 512).
+Block-based storage interfaces are a mature and common way to store data on
+media including HDDs, SSDs, CDs, floppy disks, and even tape.
+The ubiquity of block device interfaces is a perfect fit for interacting
+with mass data storage including Ceph.
+
+Ceph block devices are thin-provisioned, resizable, and store data striped over
+multiple OSDs.  Ceph block devices leverage
+:abbr:`RADOS (Reliable Autonomic Distributed Object Store)` capabilities
+including snapshotting, replication and strong consistency. Ceph block
+storage clients communicate with Ceph clusters through kernel modules or
+the ``librbd`` library.
+
+.. ditaa::
+
+            +------------------------+ +------------------------+
+            |     Kernel Module      | |        librbd          |
+            +------------------------+-+------------------------+
+            |                   RADOS Protocol                  |
+            +------------------------+-+------------------------+
+            |          OSDs          | |        Monitors        |
+            +------------------------+ +------------------------+
+
+.. note:: Kernel modules can use Linux page caching. For ``librbd``-based
+   applications, Ceph supports `RBD Caching`_.
+
+Ceph's block devices deliver high performance with vast scalability to
+`kernel modules`_, or to :abbr:`KVMs (kernel virtual machines)` such as `QEMU`_, and
+cloud-based computing systems like `OpenStack`_ and `CloudStack`_ that rely on
+libvirt and QEMU to integrate with Ceph block devices. You can use the same cluster
+to operate the :ref:`Ceph RADOS Gateway <object-gateway>`, the
+:ref:`Ceph File System <ceph-file-system>`, and Ceph block devices simultaneously.
+
+.. important:: To use Ceph Block Devices, you must have access to a running
+   Ceph cluster.
+
+.. toctree::
+        :maxdepth: 1
+
+	Basic Commands <rados-rbd-cmds>
+
+.. toctree::
+        :maxdepth: 2
+
+        Operations <rbd-operations>
+
+.. toctree::
+	:maxdepth: 2
+
+        Integrations <rbd-integrations>
+
+.. toctree::
+	:maxdepth: 2
+
+	Manpages <man/index>
+
+.. toctree::
+	:maxdepth: 2
+
+	APIs <api/index>
+
+.. _RBD Caching: ./rbd-config-ref/
+.. _kernel modules: ./rbd-ko/
+.. _QEMU: ./qemu-rbd/
+.. _OpenStack: ./rbd-openstack
+.. _CloudStack: ./rbd-cloudstack
diff --git a/doc/rbd/iscsi-initiator-esx.rst b/doc/rbd/iscsi-initiator-esx.rst
new file mode 100644
index 000000000..8bed6f2a2
--- /dev/null
+++ b/doc/rbd/iscsi-initiator-esx.rst
@@ -0,0 +1,105 @@
+------------------------------
+iSCSI Initiator for VMware ESX
+------------------------------
+
+**Prerequisite:**
+
+-  VMware ESX 6.5 or later using Virtual Machine compatibility 6.5 with VMFS 6.
+
+**iSCSI Discovery and Multipath Device Setup:**
+
+The following instructions will use the default vSphere web client and esxcli.
+
+#. Enable Software iSCSI
+
+   .. image:: ../images/esx_web_client_storage_main.png
+      :align: center
+
+   Click on "Storage" from "Navigator", and select the "Adapters" tab.
+   From there right click "Configure iSCSI".
+
+#. Set Initiator Name
+
+   .. image:: ../images/esx_config_iscsi_main.png
+      :align: center
+
+   If the initiator name in the "Name & alias" section is not the same name
+   used when creating the client during gwcli setup or the initiator name used
+   in the ansible client_connections client variable, then ssh to the ESX
+   host and run the following esxcli commands to change the name.
+
+   Get the adapter name for Software iSCSI:
+
+   ::
+
+       > esxcli iscsi adapter list
+       > Adapter  Driver     State   UID            Description
+       > -------  ---------  ------  -------------  ----------------------
+       > vmhba64  iscsi_vmk  online  iscsi.vmhba64  iSCSI Software Adapter
+
+   In this example the software iSCSI adapter is vmhba64 and the initiator
+   name is iqn.1994-05.com.redhat:rh7-client:
+
+    ::
+
+        > esxcli iscsi adapter set -A vmhba64 -n iqn.1994-05.com.redhat:rh7-client
+
+#. Setup CHAP
+
+   .. image:: ../images/esx_chap.png
+      :align: center
+
+   Expand the CHAP authentication section, select "Do not use CHAP unless
+   required by target" and enter the CHAP credentials used in the gwcli
+   auth command or ansible client_connections credentials variable.
+
+   The Mutual CHAP authentication section should have "Do not use CHAP"
+   selected.
+
+   Warning: There is a bug in the web client where the requested CHAP
+   settings are not always used initially. On the iSCSI gateway kernel
+   logs you will see the error:
+
+    ::
+
+       > kernel: CHAP user or password not set for Initiator ACL
+       > kernel: Security negotiation failed.
+       > kernel: iSCSI Login negotiation failed.
+
+   To workaround this set the CHAP settings with the esxcli command. Here
+   authname is the username and secret is the password used in previous
+   examples:
+
+   ::
+
+       > esxcli iscsi adapter auth chap set --direction=uni --authname=myiscsiusername --secret=myiscsipassword --level=discouraged -A vmhba64
+
+#. Configure iSCSI Settings
+
+   .. image:: ../images/esx_iscsi_recov_timeout.png
+      :align: center
+
+   Expand Advanced settings and set the "RecoveryTimeout" to 25.
+
+#. Set the discovery address
+
+   .. image:: ../images/esx_config_iscsi_main.png
+      :align: center
+
+   In the Dynamic targets section, click "Add dynamic target" and under
+   Addresses add one of the gateway IP addresses added during the iSCSI
+   gateway setup stage in the gwcli section or an IP set in the ansible
+   gateway_ip_list variable. Only one address needs to be added as the gateways
+   have been setup so all the iSCSI portals are returned during discovery.
+
+   Finally, click the "Save configuration" button. In the Devices tab, you
+   should see the RBD image.
+
+   The LUN should be automatically configured and using the ALUA SATP and
+   MRU PSP. Other SATPs and PSPs must not be used. This can be verified with
+   the esxcli command:
+
+   ::
+
+       > esxcli storage nmp path list -d eui.your_devices_id
+
diff --git a/doc/rbd/iscsi-initiator-linux.rst b/doc/rbd/iscsi-initiator-linux.rst
new file mode 100644
index 000000000..ba374c40c
--- /dev/null
+++ b/doc/rbd/iscsi-initiator-linux.rst
@@ -0,0 +1,91 @@
+-------------------------
+iSCSI Initiator for Linux
+-------------------------
+
+**Prerequisite:**
+
+-  Package ``iscsi-initiator-utils``
+
+-  Package ``device-mapper-multipath``
+
+**Installing:**
+
+Install the iSCSI initiator and multipath tools:
+
+   ::
+
+       # yum install iscsi-initiator-utils
+       # yum install device-mapper-multipath
+
+**Configuring:**
+
+#. Create the default ``/etc/multipath.conf`` file and enable the
+   ``multipathd`` service:
+
+   ::
+
+       # mpathconf --enable --with_multipathd y
+
+#. Add the following to ``/etc/multipath.conf`` file:
+
+   ::
+
+       devices {
+               device {
+                       vendor                 "LIO-ORG"
+                       hardware_handler       "1 alua"
+                       path_grouping_policy   "failover"
+                       path_selector          "queue-length 0"
+                       failback               60
+                       path_checker           tur
+                       prio                   alua
+                       prio_args              exclusive_pref_bit
+                       fast_io_fail_tmo       25
+                       no_path_retry          queue
+               }
+       }
+
+#. Restart the ``multipathd`` service:
+
+   ::
+
+       # systemctl reload multipathd
+
+**iSCSI Discovery and Setup:**
+
+#. If CHAP was setup on the iSCSI gateway, provide a CHAP username and
+   password by updating the ``/etc/iscsi/iscsid.conf`` file accordingly.
+
+#. Discover the target portals:
+
+   ::
+
+       # iscsiadm -m discovery -t st -p 192.168.56.101
+       192.168.56.101:3260,1 iqn.2003-01.org.linux-iscsi.rheln1
+       192.168.56.102:3260,2 iqn.2003-01.org.linux-iscsi.rheln1
+
+#. Login to target:
+
+   ::
+
+       # iscsiadm -m node -T iqn.2003-01.org.linux-iscsi.rheln1 -l
+
+**Multipath IO Setup:**
+
+The multipath daemon (``multipathd``), will set up devices automatically
+based on the ``multipath.conf`` settings. Running the ``multipath``
+command show devices setup in a failover configuration with a priority
+group for each path.
+
+::
+
+    # multipath -ll
+    mpathbt (360014059ca317516a69465c883a29603) dm-1 LIO-ORG ,IBLOCK
+    size=1.0G features='0' hwhandler='1 alua' wp=rw
+    |-+- policy='queue-length 0' prio=50 status=active
+    | `- 28:0:0:1 sde  8:64  active ready running
+    `-+- policy='queue-length 0' prio=10 status=enabled
+      `- 29:0:0:1 sdc  8:32  active ready running
+
+You should now be able to use the RBD image like you would a normal
+multipath’d iSCSI disk.
diff --git a/doc/rbd/iscsi-initiator-win.rst b/doc/rbd/iscsi-initiator-win.rst
new file mode 100644
index 000000000..e76b03d18
--- /dev/null
+++ b/doc/rbd/iscsi-initiator-win.rst
@@ -0,0 +1,102 @@
+-------------------------------------
+iSCSI Initiator for Microsoft Windows
+-------------------------------------
+
+**Prerequisite:**
+
+-  Microsoft Windows Server 2016 or later
+
+**iSCSI Initiator, Discovery and Setup:**
+
+#. Install the iSCSI initiator driver and MPIO tools.
+
+#. Launch the MPIO program, click on the "Discover Multi-Paths" tab, check the
+   "Add support for iSCSI devices� box, and click "Add". This will require a
+   reboot.
+
+#. On the iSCSI Initiator Properties window, on the "Discovery" tab, add a target
+   portal. Enter the IP address or DNS name and Port of the Ceph iSCSI gateway.
+
+#. On the “Targets� tab, select the target and click on “Connect�.
+
+#. On the “Connect To Target� window, select the “Enable multi-path� option, and
+   click the “Advanced� button.
+
+#. Under the "Connet using" section, select a “Target portal IP� . Select the
+   “Enable CHAP login on� and enter the "Name" and "Target secret" values from the
+   Ceph iSCSI Ansible client credentials section, and click OK.
+
+#. Repeat steps 5 and 6 for each target portal defined when setting up
+   the iSCSI gateway.
+
+**Multipath IO Setup:**
+
+Configuring the MPIO load balancing policy, setting the timeout and
+retry options are using PowerShell with the ``mpclaim`` command. The
+rest is done in the iSCSI Initiator tool.
+
+.. note::
+  It is recommended to increase the ``PDORemovePeriod`` option to 120
+  seconds from PowerShell. This value might need to be adjusted based
+  on the application. When all paths are down, and 120 seconds
+  expires, the operating system will start failing IO requests.
+
+::
+
+    Set-MPIOSetting -NewPDORemovePeriod 120
+
+::
+
+    mpclaim.exe -l -m 1
+
+::
+
+    mpclaim -s -m
+    MSDSM-wide Load Balance Policy: Fail Over Only
+
+#. Using the iSCSI Initiator tool, from the “Targets� tab, click on
+   the “Devices...� button.
+
+#. From the Devices window, select a disk and click the
+   “MPIO...� button.
+
+#. On the "Device Details" window the paths to each target portal is
+   displayed. If using the ``ceph-ansible`` setup method, the
+   iSCSI gateway will use ALUA to tell the iSCSI initiator which path
+   and iSCSI gateway should be used as the primary path. The Load
+   Balancing Policy “Fail Over Only� must be selected
+
+::
+
+    mpclaim -s -d $MPIO_DISK_ID
+
+.. note::
+  For the ``ceph-ansible`` setup method, there will be one
+  Active/Optimized path which is the path to the iSCSI gateway node
+  that owns the LUN, and there will be an Active/Unoptimized path for
+  each other iSCSI gateway node.
+
+**Tuning:**
+
+Consider using the following registry settings:
+
+-  Windows Disk Timeout
+
+   ::
+
+       HKEY_LOCAL_MACHINE\System\CurrentControlSet\Services\Disk
+
+   ::
+
+       TimeOutValue = 65
+
+-  Microsoft iSCSI Initiator Driver
+
+   ::
+
+       HKEY_LOCAL_MACHINE\\SYSTEM\CurrentControlSet\Control\Class\{4D36E97B-E325-11CE-BFC1-08002BE10318}\<Instance_Number>\Parameters
+
+   ::
+   
+       LinkDownTime = 25
+       SRBTimeoutDelta = 15
diff --git a/doc/rbd/iscsi-initiators.rst b/doc/rbd/iscsi-initiators.rst
new file mode 100644
index 000000000..67447789d
--- /dev/null
+++ b/doc/rbd/iscsi-initiators.rst
@@ -0,0 +1,23 @@
+--------------------------------
+Configuring the iSCSI Initiators
+--------------------------------
+
+- `iSCSI Initiator for Linux <../iscsi-initiator-linux>`_
+
+- `iSCSI Initiator for Microsoft Windows <../iscsi-initiator-win>`_
+
+- `iSCSI Initiator for VMware ESX <../iscsi-initiator-esx>`_
+
+    .. warning::
+
+        Applications that use SCSI persistent group reservations (PGR) and
+        SCSI 2 based reservations are not supported when exporting a RBD image
+        through more than one iSCSI gateway.
+
+.. toctree::
+  :maxdepth: 1
+  :hidden:
+
+  Linux <iscsi-initiator-linux>
+  Microsoft Windows <iscsi-initiator-win>
+  VMware ESX <iscsi-initiator-esx>
diff --git a/doc/rbd/iscsi-monitoring.rst b/doc/rbd/iscsi-monitoring.rst
new file mode 100644
index 000000000..da94bad87
--- /dev/null
+++ b/doc/rbd/iscsi-monitoring.rst
@@ -0,0 +1,85 @@
+------------------------------
+Monitoring Ceph iSCSI gateways
+------------------------------
+
+Ceph provides a tool for iSCSI gateway environments
+to monitor performance of exported RADOS Block Device (RBD) images.
+
+The ``gwtop`` tool is a ``top``-like tool that displays aggregated
+performance metrics of RBD images that are exported to clients over
+iSCSI. The metrics are sourced from a Performance Metrics Domain Agent
+(PMDA). Information from the Linux-IO target (LIO) PMDA is used to list
+each exported RBD image, the connected client, and its associated I/O
+metrics.
+
+**Requirements:**
+
+-  A running Ceph iSCSI gateway
+
+**Installing:**
+
+#. As ``root``, install the ``ceph-iscsi-tools`` package on each iSCSI
+   gateway node:
+
+   ::
+
+       # yum install ceph-iscsi-tools
+
+#. As ``root``, install the performance co-pilot package on each iSCSI
+   gateway node:
+
+   ::
+
+       # yum install pcp
+
+#. As ``root``, install the LIO PMDA package on each iSCSI gateway node:
+
+   ::
+
+       # yum install pcp-pmda-lio
+
+#. As ``root``, enable and start the performance co-pilot service on
+   each iSCSI gateway node:
+
+   ::
+
+       # systemctl enable pmcd
+       # systemctl start pmcd
+
+#. As ``root``, register the ``pcp-pmda-lio`` agent:
+
+   ::
+
+       cd /var/lib/pcp/pmdas/lio
+       ./Install
+
+By default, ``gwtop`` assumes the iSCSI gateway configuration object is
+stored in a RADOS object called ``gateway.conf`` in the ``rbd`` pool.
+This configuration defines the iSCSI gateways to contact for gathering
+the performance statistics. This can be overridden by using either the
+``-g`` or ``-c`` flags. See ``gwtop --help`` for more details.
+
+The LIO configuration determines which type of performance statistics to
+extract from performance co-pilot. When ``gwtop`` starts it looks at the
+LIO configuration, and if it find user-space disks, then ``gwtop``
+selects the LIO collector automatically.
+
+**Example ``gwtop`` Outputs**
+
+::
+
+    gwtop  2/2 Gateways   CPU% MIN:  4 MAX:  5    Network Total In:    2M  Out:    3M   10:20:00
+    Capacity:   8G    Disks:   8   IOPS:  503   Clients:  1   Ceph: HEALTH_OK          OSDs:   3
+    Pool.Image       Src    Size     iops     rMB/s     wMB/s   Client
+    iscsi.t1703             500M        0      0.00      0.00
+    iscsi.testme1           500M        0      0.00      0.00
+    iscsi.testme2           500M        0      0.00      0.00
+    iscsi.testme3           500M        0      0.00      0.00
+    iscsi.testme5           500M        0      0.00      0.00
+    rbd.myhost_1      T       4G      504      1.95      0.00   rh460p(CON)
+    rbd.test_2                1G        0      0.00      0.00
+    rbd.testme              500M        0      0.00      0.00
+
+In the *Client* column, ``(CON)`` means the iSCSI initiator (client) is
+currently logged into the iSCSI gateway. If ``-multi-`` is displayed,
+then multiple clients are mapped to the single RBD image.
diff --git a/doc/rbd/iscsi-overview.rst b/doc/rbd/iscsi-overview.rst
new file mode 100644
index 000000000..879083c3f
--- /dev/null
+++ b/doc/rbd/iscsi-overview.rst
@@ -0,0 +1,57 @@
+.. _ceph-iscsi:
+
+==================
+Ceph iSCSI Gateway
+==================
+
+The iSCSI Gateway presents a Highly Available (HA) iSCSI target that exports
+RADOS Block Device (RBD) images as SCSI disks. The iSCSI protocol allows
+clients (initiators) to send SCSI commands to storage devices (targets) over a
+TCP/IP network, enabling clients without native Ceph client support to access
+Ceph block storage.  
+
+Each iSCSI gateway exploits the Linux IO target kernel subsystem (LIO) to
+provide iSCSI protocol support. LIO utilizes userspace passthrough (TCMU) to
+interact with Ceph's librbd library and expose RBD images to iSCSI clients.
+With Ceph’s iSCSI gateway you can provision a fully integrated block-storage
+infrastructure with all the features and benefits of a conventional Storage
+Area Network (SAN).
+
+.. ditaa::
+                  Cluster Network (optional)
+                 +-------------------------------------------+
+                 |             |               |             |
+             +-------+     +-------+       +-------+     +-------+
+             |       |     |       |       |       |     |       |
+             | OSD 1 |     | OSD 2 |       | OSD 3 |     | OSD N |
+             |    {s}|     |    {s}|       |    {s}|     |    {s}|
+             +-------+     +-------+       +-------+     +-------+
+                 |             |               |             |
+      +--------->|             |  +---------+  |             |<---------+
+      :          |             |  |   RBD   |  |             |          :
+      |          +----------------|  Image  |----------------+          |
+      |           Public Network  |    {d}  |                           |
+      |                           +---------+                           |
+      |                                                                 |
+      |                      +-------------------+                      |
+      |   +--------------+   | iSCSI Initiators  |   +--------------+   |
+      |   | iSCSI GW     |   |   +-----------+   |   | iSCSI GW     |   |
+      +-->|  RBD Module  |<--+   | Various   |   +-->|  RBD Module  |<--+
+          |              |   |   | Operating |   |   |              |
+          +--------------+   |   | Systems   |   |   +--------------+
+                             |   +-----------+   |
+                             +-------------------+
+
+.. warning::
+
+   The iSCSI gateway is in maintenance as of November 2022. This means that
+   it is no longer in active development and will not be updated to add
+   new features. 
+
+.. toctree::
+  :maxdepth: 1
+
+  Requirements <iscsi-requirements>
+  Configuring the iSCSI Target <iscsi-targets>
+  Configuring the iSCSI Initiators <iscsi-initiators>
+  Monitoring the iSCSI Gateways <iscsi-monitoring>
diff --git a/doc/rbd/iscsi-requirements.rst b/doc/rbd/iscsi-requirements.rst
new file mode 100644
index 000000000..50dfc2a27
--- /dev/null
+++ b/doc/rbd/iscsi-requirements.rst
@@ -0,0 +1,51 @@
+==========================
+iSCSI Gateway Requirements
+==========================
+
+It is recommended to provision two to four iSCSI gateway nodes to
+realize a highly available Ceph iSCSI gateway solution.
+
+For hardware recommendations, see :ref:`hardware-recommendations` .
+
+.. note::
+    On iSCSI gateway nodes the memory footprint is a function of
+    of the RBD images mapped and can grow to be large. Plan memory
+    requirements accordingly based on the number RBD images to be mapped.
+
+There are no specific iSCSI gateway options for the Ceph Monitors or
+OSDs, but it is important to lower the default heartbeat interval for
+detecting down OSDs to reduce the possibility of initiator timeouts.
+The following configuration options are suggested::
+
+        [osd]
+        osd heartbeat grace = 20
+        osd heartbeat interval = 5
+
+-   Updating Running State From a Ceph Monitor Node
+
+   ::
+
+       ceph tell <daemon_type>.<id> config set <parameter_name> <new_value>
+
+   ::
+
+       ceph tell osd.* config set osd_heartbeat_grace 20
+       ceph tell osd.* config set osd_heartbeat_interval 5
+
+-  Updating Running State On Each OSD Node
+
+   ::
+
+       ceph daemon <daemon_type>.<id> config set osd_client_watch_timeout 15
+
+   ::
+
+       ceph daemon osd.0 config set osd_heartbeat_grace 20
+       ceph daemon osd.0 config set osd_heartbeat_interval 5
+
+For more details on setting Ceph's configuration options, see
+:ref:`configuring-ceph`.  Be sure to persist these settings in
+``/etc/ceph.conf`` or, on Mimic and later releases, in the
+centralized config store.
+
+
diff --git a/doc/rbd/iscsi-target-ansible.rst b/doc/rbd/iscsi-target-ansible.rst
new file mode 100644
index 000000000..d38e3bd01
--- /dev/null
+++ b/doc/rbd/iscsi-target-ansible.rst
@@ -0,0 +1,234 @@
+==========================================
+Configuring the iSCSI Target using Ansible
+==========================================
+
+The Ceph iSCSI gateway is the iSCSI target node and also a Ceph client
+node. The Ceph iSCSI gateway can be provisioned on dedicated node
+or be colocated on a Ceph Object Store Disk (OSD) node. The following steps will
+install and configure the Ceph iSCSI gateway for basic operation.
+
+**Requirements:**
+
+-  A running Ceph Luminous (12.2.x) cluster or newer
+
+-  Red Hat Enterprise Linux/CentOS 7.5 (or newer); Linux kernel v4.16 (or newer)
+
+-  The ``ceph-iscsi`` package installed on all the iSCSI gateway nodes
+
+**Installation:**
+
+#. On the Ansible installer node, which could be either the administration node
+   or a dedicated deployment node, perform the following steps:
+
+   #. As ``root``, install the ``ceph-ansible`` package:
+
+      ::
+
+          # yum install ceph-ansible
+
+   #. Add an entry in ``/etc/ansible/hosts`` file for the gateway group:
+
+      ::
+
+          [iscsigws]
+          ceph-igw-1
+          ceph-igw-2
+
+.. note::
+  If co-locating the iSCSI gateway with an OSD node, then add the OSD node to the
+  ``[iscsigws]`` section.
+
+**Configuration:**
+
+The ``ceph-ansible`` package places a file in the ``/usr/share/ceph-ansible/group_vars/``
+directory called ``iscsigws.yml.sample``. Create a copy of this sample file named
+``iscsigws.yml``. Review the following Ansible variables and descriptions,
+and update accordingly. See the ``iscsigws.yml.sample`` for a full list of
+advanced variables.
+
++--------------------------------------+--------------------------------------+
+| Variable                             | Meaning/Purpose                      |
++======================================+======================================+
+| ``seed_monitor``                     | Each gateway needs access to the     |
+|                                      | ceph cluster for rados and rbd       |
+|                                      | calls. This means the iSCSI gateway  |
+|                                      | must have an appropriate             |
+|                                      | ``/etc/ceph/`` directory defined.    |
+|                                      | The ``seed_monitor`` host is used to |
+|                                      | populate the iSCSI gateway’s         |
+|                                      | ``/etc/ceph/`` directory.            |
++--------------------------------------+--------------------------------------+
+| ``cluster_name``                     | Define a custom storage cluster      |
+|                                      | name.                                |
++--------------------------------------+--------------------------------------+
+| ``gateway_keyring``                  | Define a custom keyring name.        |
++--------------------------------------+--------------------------------------+
+| ``deploy_settings``                  | If set to ``true``, then deploy the  |
+|                                      | settings when the playbook is ran.   |
++--------------------------------------+--------------------------------------+
+| ``perform_system_checks``            | This is a boolean value that checks  |
+|                                      | for multipath and lvm configuration  |
+|                                      | settings on each gateway. It must be |
+|                                      | set to true for at least the first   |
+|                                      | run to ensure multipathd and lvm are |
+|                                      | configured properly.                 |
++--------------------------------------+--------------------------------------+
+| ``api_user``                         | The user name for the API. The       |
+|                                      | default is `admin`.                  |
++--------------------------------------+--------------------------------------+
+| ``api_password``                     | The password for using the API. The  |
+|                                      | default is `admin`.                  |
++--------------------------------------+--------------------------------------+
+| ``api_port``                         | The TCP port number for using the    |
+|                                      | API. The default is `5000`.          |
++--------------------------------------+--------------------------------------+
+| ``api_secure``                       | True if TLS must be used. The        |
+|                                      | default is `false`. If true the user |
+|                                      | must create the necessary            |
+|                                      | certificate and key files. See the   |
+|                                      | gwcli man file for details.          |
++--------------------------------------+--------------------------------------+
+| ``trusted_ip_list``                  | A list of IPv4 or IPv6 addresses     |
+|                                      | who have access to the API. By       |
+|                                      | default, only the iSCSI gateway      |
+|                                      | nodes have access.                   |
++--------------------------------------+--------------------------------------+
+
+**Deployment:**
+
+Perform the followint steps on the Ansible installer node.
+
+#. As ``root``, execute the Ansible playbook:
+
+   ::
+
+       # cd /usr/share/ceph-ansible
+       # ansible-playbook site.yml --limit iscsigws
+
+   .. note::
+    The Ansible playbook will handle RPM dependencies, setting up daemons,
+    and installing gwcli so it can be used to create iSCSI targets and export
+    RBD images as LUNs. In past versions, ``iscsigws.yml`` could define the
+    iSCSI target and other objects like clients, images and LUNs, but this is
+    no longer supported.
+
+#. Verify the configuration from an iSCSI gateway node:
+
+   ::
+
+       # gwcli ls
+
+   .. note::
+    See the `Configuring the iSCSI Target using the Command Line Interface`_
+    section to create gateways, LUNs, and clients using the `gwcli` tool.
+
+   .. important::
+    Attempting to use the ``targetcli`` tool to change the configuration will
+    cause problems including ALUA misconfiguration and path failover
+    issues. There is the potential to corrupt data, to have mismatched
+    configuration across iSCSI gateways, and to have mismatched WWN information,
+    leading to client multipath problems.
+
+**Service Management:**
+
+The ``ceph-iscsi`` package installs the configuration management
+logic and a Systemd service called ``rbd-target-api``. When the Systemd
+service is enabled, the ``rbd-target-api`` will start at boot time and
+will restore the Linux IO state. The Ansible playbook disables the
+target service during the deployment. Below are the outcomes of when
+interacting with the ``rbd-target-api`` Systemd service.
+
+::
+
+    # systemctl <start|stop|restart|reload> rbd-target-api
+
+-  ``reload``
+
+   A reload request will force ``rbd-target-api`` to reread the
+   configuration and apply it to the current running environment. This
+   is normally not required, since changes are deployed in parallel from
+   Ansible to all iSCSI gateway nodes
+
+-  ``stop``
+
+   A stop request will close the gateway’s portal interfaces, dropping
+   connections to clients and wipe the current LIO configuration from
+   the kernel. This returns the iSCSI gateway to a clean state. When
+   clients are disconnected, active I/O is rescheduled to the other
+   iSCSI gateways by the client side multipathing layer.
+
+**Removing the Configuration:**
+
+The ``ceph-ansible`` package provides an Ansible playbook to
+remove the iSCSI gateway configuration and related RBD images. The
+Ansible playbook is ``/usr/share/ceph-ansible/purge_gateways.yml``. When
+this Ansible playbook is ran a prompted for the type of purge to
+perform:
+
+*lio* :
+
+In this mode the LIO configuration is purged on all iSCSI gateways that
+are defined. Disks that were created are left untouched within the Ceph
+storage cluster.
+
+*all* :
+
+When ``all`` is chosen, the LIO configuration is removed together with
+**all** RBD images that were defined within the iSCSI gateway
+environment, other unrelated RBD images will not be removed. Ensure the
+correct mode is chosen, this operation will delete data.
+
+.. warning::
+  A purge operation is destructive action against your iSCSI gateway
+  environment.
+
+.. warning::
+  A purge operation will fail, if RBD images have snapshots or clones
+  and are exported through the Ceph iSCSI gateway.
+
+::
+
+    [root@rh7-iscsi-client ceph-ansible]# ansible-playbook purge_gateways.yml
+    Which configuration elements should be purged? (all, lio or abort) [abort]: all
+
+
+    PLAY [Confirm removal of the iSCSI gateway configuration] *********************
+
+
+    GATHERING FACTS ***************************************************************
+    ok: [localhost]
+
+
+    TASK: [Exit playbook if user aborted the purge] *******************************
+    skipping: [localhost]
+
+
+    TASK: [set_fact ] *************************************************************
+    ok: [localhost]
+
+
+    PLAY [Removing the gateway configuration] *************************************
+
+
+    GATHERING FACTS ***************************************************************
+    ok: [ceph-igw-1]
+    ok: [ceph-igw-2]
+
+
+    TASK: [igw_purge | purging the gateway configuration] *************************
+    changed: [ceph-igw-1]
+    changed: [ceph-igw-2]
+
+
+    TASK: [igw_purge | deleting configured rbd devices] ***************************
+    changed: [ceph-igw-1]
+    changed: [ceph-igw-2]
+
+
+    PLAY RECAP ********************************************************************
+    ceph-igw-1                 : ok=3    changed=2    unreachable=0    failed=0
+    ceph-igw-2                 : ok=3    changed=2    unreachable=0    failed=0
+    localhost                  : ok=2    changed=0    unreachable=0    failed=0
+
+
+.. _Configuring the iSCSI Target using the Command Line Interface: ../iscsi-target-cli
diff --git a/doc/rbd/iscsi-target-cli-manual-install.rst b/doc/rbd/iscsi-target-cli-manual-install.rst
new file mode 100644
index 000000000..ccc422e0d
--- /dev/null
+++ b/doc/rbd/iscsi-target-cli-manual-install.rst
@@ -0,0 +1,190 @@
+==============================
+Manual ceph-iscsi Installation
+==============================
+
+**Requirements**
+
+To complete the installation of ceph-iscsi, there are 4 steps:
+
+1. Install common packages from your Linux distribution's software repository
+2. Install Git to fetch the remaining packages directly from their Git repositories
+3. Ensure a compatible kernel is used
+4. Install all the components of ceph-iscsi and start associated daemons:
+
+   -  tcmu-runner
+   -  rtslib-fb
+   -  configshell-fb
+   -  targetcli-fb
+   -  ceph-iscsi
+
+
+1. Install Common Packages
+==========================
+
+The following packages will be used by ceph-iscsi and target tools.
+They must be installed from your Linux distribution's software repository
+on each machine that will be a iSCSI gateway:
+
+-  libnl3
+-  libkmod
+-  librbd1
+-  pyparsing
+-  python kmod
+-  python pyudev
+-  python gobject
+-  python urwid
+-  python pyparsing
+-  python rados
+-  python rbd
+-  python netifaces
+-  python crypto
+-  python requests
+-  python flask
+-  pyOpenSSL
+
+
+2. Install Git
+==============
+
+In order to install all the packages needed to run iSCSI with Ceph, you need to download them directly from their repository by using Git.
+On CentOS/RHEL execute:
+
+::
+
+   > sudo yum install git
+
+On Debian/Ubuntu execute:
+
+::
+
+   > sudo apt install git
+   
+To know more about Git and how it works, please, visit https://git-scm.com
+
+
+3. Ensure a compatible kernel is used
+=====================================
+
+Ensure you use a supported kernel that contains the required Ceph iSCSI patches:
+
+-  all Linux distribution with a kernel v4.16 or newer, or
+-  Red Hat Enterprise Linux or CentOS 7.5 or later (in these distributions ceph-iscsi support is backported)
+
+If you are already using a compatible kernel, you can go to next step.
+However, if you are NOT using a compatible kernel then check your distro's
+documentation for specific instructions on how to build this kernel. The only
+Ceph iSCSI specific requirements are that the following build options must be
+enabled:
+
+    ::
+    
+       CONFIG_TARGET_CORE=m
+       CONFIG_TCM_USER2=m
+       CONFIG_ISCSI_TARGET=m
+
+
+4. Install ceph-iscsi
+========================================================
+
+Finally, the remaining tools can be fetched directly from their Git repositories and their associated services started
+
+
+tcmu-runner
+-----------
+
+   Installation:
+
+   ::
+
+       > git clone https://github.com/open-iscsi/tcmu-runner
+       > cd tcmu-runner
+
+   Run the following command to install all the needed dependencies:
+
+   ::
+
+       > ./extra/install_dep.sh   
+   
+   Now you can build the tcmu-runner.
+   To do so, use the following build command:
+
+   ::
+
+       > cmake -Dwith-glfs=false -Dwith-qcow=false -DSUPPORT_SYSTEMD=ON -DCMAKE_INSTALL_PREFIX=/usr
+       > make install
+
+   Enable and start the daemon:
+
+   ::
+
+       > systemctl daemon-reload
+       > systemctl enable tcmu-runner
+       > systemctl start tcmu-runner
+
+
+rtslib-fb
+---------
+
+   Installation:
+
+   ::
+
+       > git clone https://github.com/open-iscsi/rtslib-fb.git
+       > cd rtslib-fb
+       > python setup.py install
+
+configshell-fb
+--------------
+
+   Installation:
+
+   ::
+
+       > git clone https://github.com/open-iscsi/configshell-fb.git
+       > cd configshell-fb
+       > python setup.py install
+
+targetcli-fb
+------------
+
+   Installation:
+
+   ::
+
+       > git clone https://github.com/open-iscsi/targetcli-fb.git
+       > cd targetcli-fb
+       > python setup.py install
+       > mkdir /etc/target
+       > mkdir /var/target
+
+   .. warning:: The ceph-iscsi tools assume they are managing all targets
+      on the system. If targets have been setup and are being managed by
+      targetcli the target service must be disabled.
+
+ceph-iscsi
+-----------------
+
+   Installation:
+
+   ::
+
+       > git clone https://github.com/ceph/ceph-iscsi.git
+       > cd ceph-iscsi
+       > python setup.py install --install-scripts=/usr/bin
+       > cp usr/lib/systemd/system/rbd-target-gw.service /lib/systemd/system
+       > cp usr/lib/systemd/system/rbd-target-api.service /lib/systemd/system
+
+   Enable and start the daemon:
+
+   ::
+
+       > systemctl daemon-reload
+       > systemctl enable rbd-target-gw
+       > systemctl start rbd-target-gw
+       > systemctl enable rbd-target-api
+       > systemctl start rbd-target-api
+
+Installation is complete. Proceed to the setup section in the
+`main ceph-iscsi CLI page`_.
+
+.. _`main ceph-iscsi CLI page`: ../iscsi-target-cli
diff --git a/doc/rbd/iscsi-target-cli.rst b/doc/rbd/iscsi-target-cli.rst
new file mode 100644
index 000000000..d888a34b0
--- /dev/null
+++ b/doc/rbd/iscsi-target-cli.rst
@@ -0,0 +1,244 @@
+=============================================================
+Configuring the iSCSI Target using the Command Line Interface
+=============================================================
+
+The Ceph iSCSI gateway is both an iSCSI target  and a Ceph client;
+think of it as a "translator" between Ceph's RBD interface
+and the iSCSI standard. The Ceph iSCSI gateway can run on a
+standalone node or be colocated with other daemons eg. on
+a Ceph Object Store Disk (OSD) node.  When co-locating, ensure
+that sufficient CPU and memory are available to share.
+The following steps install and configure the Ceph iSCSI gateway for basic operation.
+
+**Requirements:**
+
+-  A running Ceph Luminous or later storage cluster
+
+-  Red Hat Enterprise Linux/CentOS 7.5 (or newer); Linux kernel v4.16 (or newer)
+
+-  The following packages must be installed from your Linux distribution's software repository:
+
+   -  ``targetcli-2.1.fb47`` or newer package
+
+   -  ``python-rtslib-2.1.fb68`` or newer package
+
+   -  ``tcmu-runner-1.4.0`` or newer package
+
+   -  ``ceph-iscsi-3.2`` or newer package
+
+     .. important::
+        If previous versions of these packages exist, then they must
+        be removed first before installing the newer versions.
+
+Do the following steps on the Ceph iSCSI gateway node before proceeding
+to the *Installing* section:
+
+#. If the Ceph iSCSI gateway is not colocated on an OSD node, then copy
+   the Ceph configuration files, located in ``/etc/ceph/``, from a
+   running Ceph node in the storage cluster to the iSCSI Gateway node.
+   The Ceph configuration files must exist on the iSCSI gateway node
+   under ``/etc/ceph/``.
+
+#. Install and configure the `Ceph Command-line Interface`_
+
+#. If needed, open TCP ports 3260 and 5000 on the firewall.
+
+   .. note::
+      Access to port 5000 should be restricted to a trusted internal network or
+      only the individual hosts where ``gwcli`` is used or ``ceph-mgr`` daemons
+      are running.
+
+#. Create a new or use an existing RADOS Block Device (RBD).
+
+**Installing:**
+
+If you are using the upstream ceph-iscsi package follow the
+`manual install instructions`_.
+
+.. _`manual install instructions`: ../iscsi-target-cli-manual-install
+
+.. toctree::
+   :hidden:
+
+   iscsi-target-cli-manual-install
+
+For rpm based instructions execute the following commands:
+
+#. As ``root``, on all iSCSI gateway nodes, install the
+   ``ceph-iscsi`` package:
+
+   ::
+
+       # yum install ceph-iscsi
+
+#. As ``root``, on all iSCSI gateway nodes, install the ``tcmu-runner``
+   package:
+
+   ::
+
+       # yum install tcmu-runner
+
+**Setup:**
+
+#. gwcli requires a pool with the name ``rbd``, so it can store metadata
+   like the iSCSI configuration. To check if this pool has been created
+   run:
+
+   ::
+
+       # ceph osd lspools
+
+   If it does not exist instructions for creating pools can be found on the
+   `RADOS pool operations page
+   <http://docs.ceph.com/en/latest/rados/operations/pools/>`_.
+
+#. As ``root``, on a iSCSI gateway node, create a file named
+   ``iscsi-gateway.cfg`` in the ``/etc/ceph/`` directory:
+
+   ::
+
+       # touch /etc/ceph/iscsi-gateway.cfg
+
+   #. Edit the ``iscsi-gateway.cfg`` file and add the following lines:
+
+      ::
+
+          [config]
+          # Name of the Ceph storage cluster. A suitable Ceph configuration file allowing
+          # access to the Ceph storage cluster from the gateway node is required, if not
+          # colocated on an OSD node.
+          cluster_name = ceph
+
+          # Place a copy of the ceph cluster's admin keyring in the gateway's /etc/ceph
+          # drectory and reference the filename here
+          gateway_keyring = ceph.client.admin.keyring
+
+
+          # API settings.
+          # The API supports a number of options that allow you to tailor it to your
+          # local environment. If you want to run the API under https, you will need to
+          # create cert/key files that are compatible for each iSCSI gateway node, that is
+          # not locked to a specific node. SSL cert and key files *must* be called
+          # 'iscsi-gateway.crt' and 'iscsi-gateway.key' and placed in the '/etc/ceph/' directory
+          # on *each* gateway node. With the SSL files in place, you can use 'api_secure = true'
+          # to switch to https mode.
+
+          # To support the API, the bare minimum settings are:
+          api_secure = false
+
+          # Additional API configuration options are as follows, defaults shown.
+          # api_user = admin
+          # api_password = admin
+          # api_port = 5001
+          # trusted_ip_list = 192.168.0.10,192.168.0.11
+
+      .. note::
+        trusted_ip_list is a list of IP addresses on each iSCSI gateway that
+        will be used for management operations like target creation, LUN
+        exporting, etc. The IP can be the same that will be used for iSCSI
+        data, like READ/WRITE commands to/from the RBD image, but using
+        separate IPs is recommended.
+
+      .. important::
+        The ``iscsi-gateway.cfg`` file must be identical on all iSCSI gateway nodes.
+
+   #. As ``root``, copy the ``iscsi-gateway.cfg`` file to all iSCSI
+      gateway nodes.
+
+#. As ``root``, on all iSCSI gateway nodes, enable and start the API
+   service:
+
+   ::
+
+       # systemctl daemon-reload
+       
+       # systemctl enable rbd-target-gw
+       # systemctl start rbd-target-gw
+
+       # systemctl enable rbd-target-api
+       # systemctl start rbd-target-api
+
+
+**Configuring:**
+
+gwcli will create and configure the iSCSI target and RBD images and copy the
+configuration across the gateways setup in the last section. Lower level
+tools including targetcli and rbd can be used to query the local configuration,
+but should not be used to modify it. This next section will demonstrate how
+to create a iSCSI target and export a RBD image as LUN 0.
+
+#. As ``root``, on a iSCSI gateway node, start the iSCSI gateway
+   command-line interface:
+
+   ::
+
+       # gwcli
+
+#. Go to iscsi-targets and create a target with the name
+   iqn.2003-01.com.redhat.iscsi-gw:iscsi-igw:
+
+   ::
+
+       > /> cd /iscsi-target
+       > /iscsi-target>  create iqn.2003-01.com.redhat.iscsi-gw:iscsi-igw
+
+#. Create the iSCSI gateways. The IPs used below are the ones that will be
+   used for iSCSI data like READ and WRITE commands. They can be the
+   same IPs used for management operations listed in trusted_ip_list,
+   but it is recommended that different IPs are used.
+
+   ::
+
+       > /iscsi-target> cd iqn.2003-01.com.redhat.iscsi-gw:iscsi-igw/gateways
+       > /iscsi-target...-igw/gateways>  create ceph-gw-1 10.172.19.21
+       > /iscsi-target...-igw/gateways>  create ceph-gw-2 10.172.19.22
+
+   If not using RHEL/CentOS or using an upstream or ceph-iscsi-test kernel,
+   the skipchecks=true argument must be used. This will avoid the Red Hat kernel
+   and rpm checks:
+
+   ::
+
+       > /iscsi-target> cd iqn.2003-01.com.redhat.iscsi-gw:iscsi-igw/gateways
+       > /iscsi-target...-igw/gateways>  create ceph-gw-1 10.172.19.21 skipchecks=true
+       > /iscsi-target...-igw/gateways>  create ceph-gw-2 10.172.19.22 skipchecks=true
+
+#. Add a RBD image with the name disk_1 in the pool rbd:
+
+   ::
+
+       > /iscsi-target...-igw/gateways> cd /disks
+       > /disks> create pool=rbd image=disk_1 size=90G
+
+#. Create a client with the initiator name iqn.1994-05.com.redhat:rh7-client:
+
+   ::
+
+       > /disks> cd /iscsi-target/iqn.2003-01.com.redhat.iscsi-gw:iscsi-igw/hosts
+       > /iscsi-target...eph-igw/hosts>  create iqn.1994-05.com.redhat:rh7-client
+
+#. Set the client's CHAP username to myiscsiusername and password to
+   myiscsipassword:
+
+   ::
+
+       > /iscsi-target...at:rh7-client>  auth username=myiscsiusername password=myiscsipassword
+
+   .. warning::
+      CHAP must always be configured. Without CHAP, the target will
+      reject any login requests.
+
+#. Add the disk to the client:
+
+   ::
+
+       > /iscsi-target...at:rh7-client> disk add rbd/disk_1
+
+The next step is to configure the iSCSI initiators.
+
+.. _`Ceph Command-line Interface`: ../../start/quick-rbd/#install-ceph
+
+.. toctree::
+   :hidden:
+
+   ../../start/quick-rbd
diff --git a/doc/rbd/iscsi-targets.rst b/doc/rbd/iscsi-targets.rst
new file mode 100644
index 000000000..d2a035283
--- /dev/null
+++ b/doc/rbd/iscsi-targets.rst
@@ -0,0 +1,27 @@
+=============
+iSCSI Targets
+=============
+
+Traditionally, block-level access to a Ceph storage cluster has been
+limited to QEMU and ``librbd``, which is a key enabler for adoption
+within OpenStack environments. Starting with the Ceph Luminous release,
+block-level access is expanding to offer standard iSCSI support allowing
+wider platform usage, and potentially opening new use cases.
+
+-  Red Hat Enterprise Linux/CentOS 7.5 (or newer); Linux kernel v4.16 (or newer)
+
+-  A working Ceph Storage cluster, deployed with ``ceph-ansible`` or using the command-line interface
+
+-  iSCSI gateways nodes, which can either be colocated with OSD nodes or on dedicated nodes
+
+-  Separate network subnets for iSCSI front-end traffic and Ceph back-end traffic
+
+A choice of using Ansible or the command-line interface are the
+available deployment methods for installing and configuring the Ceph
+iSCSI gateway:
+
+.. toctree::
+  :maxdepth: 1
+
+  Using Ansible <iscsi-target-ansible>
+  Using the Command Line Interface <iscsi-target-cli>
diff --git a/doc/rbd/libvirt.rst b/doc/rbd/libvirt.rst
new file mode 100644
index 000000000..e3523f8a8
--- /dev/null
+++ b/doc/rbd/libvirt.rst
@@ -0,0 +1,323 @@
+=================================
+ Using libvirt with Ceph RBD
+=================================
+
+.. index:: Ceph Block Device; livirt
+
+The ``libvirt`` library creates a virtual machine abstraction layer between 
+hypervisor interfaces and the software applications that use them. With 
+``libvirt``, developers and system administrators can focus on a common 
+management framework, common API, and common shell interface (i.e., ``virsh``)
+to many different hypervisors, including: 
+
+- QEMU/KVM
+- XEN
+- LXC
+- VirtualBox
+- etc.
+
+Ceph block devices support QEMU/KVM. You can use Ceph block devices with
+software that interfaces with ``libvirt``. The following stack diagram
+illustrates how ``libvirt`` and QEMU use Ceph block devices via ``librbd``. 
+
+
+.. ditaa::
+
+            +---------------------------------------------------+
+            |                     libvirt                       |
+            +------------------------+--------------------------+
+                                     |
+                                     | configures
+                                     v
+            +---------------------------------------------------+
+            |                       QEMU                        |
+            +---------------------------------------------------+
+            |                      librbd                       |
+            +---------------------------------------------------+
+            |                     librados                      |
+            +------------------------+-+------------------------+
+            |          OSDs          | |        Monitors        |
+            +------------------------+ +------------------------+
+
+
+The most common ``libvirt`` use case involves providing Ceph block devices to
+cloud solutions like OpenStack or CloudStack. The cloud solution uses
+``libvirt`` to  interact with QEMU/KVM, and QEMU/KVM interacts with Ceph block
+devices via  ``librbd``. See `Block Devices and OpenStack`_ and `Block Devices
+and CloudStack`_ for details. See `Installation`_ for installation details.
+
+You can also use Ceph block devices with ``libvirt``, ``virsh`` and the
+``libvirt`` API. See `libvirt Virtualization API`_ for details.
+
+
+To create VMs that use Ceph block devices, use the procedures in the following
+sections. In the exemplary embodiment, we have used ``libvirt-pool`` for the pool
+name, ``client.libvirt`` for the user name, and ``new-libvirt-image`` for  the
+image name. You may use any value you like, but ensure you replace those values
+when executing commands in the subsequent procedures.
+
+
+Configuring Ceph
+================
+
+To configure Ceph for use with ``libvirt``, perform the following steps:
+
+#. `Create a pool`_. The following example uses the 
+   pool name ``libvirt-pool``.::
+
+	ceph osd pool create libvirt-pool
+
+   Verify the pool exists. :: 
+
+	ceph osd lspools
+
+#. Use the ``rbd`` tool to initialize the pool for use by RBD::
+
+        rbd pool init <pool-name>
+
+#. `Create a Ceph User`_ (or use ``client.admin`` for version 0.9.7 and
+   earlier). The following example uses the Ceph user name ``client.libvirt``
+   and references ``libvirt-pool``. ::
+
+	ceph auth get-or-create client.libvirt mon 'profile rbd' osd 'profile rbd pool=libvirt-pool'
+	
+   Verify the name exists. :: 
+   
+	ceph auth ls
+
+   **NOTE**: ``libvirt`` will access Ceph using the ID ``libvirt``, 
+   not the Ceph name ``client.libvirt``. See `User Management - User`_ and 
+   `User Management - CLI`_ for a detailed explanation of the difference 
+   between ID and name.	
+
+#. Use QEMU to `create an image`_ in your RBD pool. 
+   The following example uses the image name ``new-libvirt-image``
+   and references ``libvirt-pool``. ::
+
+	qemu-img create -f rbd rbd:libvirt-pool/new-libvirt-image 2G
+
+   Verify the image exists. :: 
+
+	rbd -p libvirt-pool ls
+
+   **NOTE:** You can also use `rbd create`_ to create an image, but we
+   recommend ensuring that QEMU is working properly.
+
+.. tip:: Optionally, if you wish to enable debug logs and the admin socket for
+   this client, you can add the following section to ``/etc/ceph/ceph.conf``::
+
+	[client.libvirt]
+	log file = /var/log/ceph/qemu-guest-$pid.log
+	admin socket = /var/run/ceph/$cluster-$type.$id.$pid.$cctid.asok
+
+   The ``client.libvirt`` section name should match the cephx user you created
+   above.  
+   If SELinux or AppArmor is enabled, note that this could prevent the client
+   process (qemu via libvirt) from doing some operations, such as writing logs
+   or operate the images or admin socket to the destination locations (``/var/
+   log/ceph`` or ``/var/run/ceph``). Additionally, make sure that the libvirt
+   and qemu users have appropriate access to the specified directory.
+
+
+Preparing the VM Manager
+========================
+
+You may use ``libvirt`` without a VM manager, but you may find it simpler to
+create your first domain with ``virt-manager``. 
+
+#. Install a virtual machine manager. See `KVM/VirtManager`_ for details. ::
+
+	sudo apt-get install virt-manager
+
+#. Download an OS image (if necessary).
+
+#. Launch the virtual machine manager. :: 
+
+	sudo virt-manager
+
+
+
+Creating a VM
+=============
+
+To create a VM with ``virt-manager``, perform the following steps:
+
+#. Press the **Create New Virtual Machine** button. 
+
+#. Name the new virtual machine domain. In the exemplary embodiment, we
+   use the name ``libvirt-virtual-machine``. You may use any name you wish,
+   but ensure you replace ``libvirt-virtual-machine`` with the name you 
+   choose in subsequent commandline and configuration examples. :: 
+
+	libvirt-virtual-machine
+
+#. Import the image. ::
+
+	/path/to/image/recent-linux.img
+
+   **NOTE:** Import a recent image. Some older images may not rescan for 
+   virtual devices properly.
+   
+#. Configure and start the VM.
+
+#. You may use ``virsh list`` to verify the VM domain exists. ::
+
+	sudo virsh list
+
+#. Login to the VM (root/root)
+
+#. Stop the VM before configuring it for use with Ceph.
+
+
+Configuring the VM
+==================
+
+When configuring the VM for use with Ceph, it is important  to use ``virsh``
+where appropriate. Additionally, ``virsh`` commands often require root
+privileges  (i.e., ``sudo``) and will not return appropriate results or notify
+you that root privileges are required. For a reference of ``virsh``
+commands, refer to `Virsh Command Reference`_.
+
+
+#. Open the configuration file with ``virsh edit``. :: 
+
+	sudo virsh edit {vm-domain-name}
+
+   Under ``<devices>`` there should be a ``<disk>`` entry. :: 
+
+	<devices>
+		<emulator>/usr/bin/kvm</emulator>
+		<disk type='file' device='disk'>
+			<driver name='qemu' type='raw'/>
+			<source file='/path/to/image/recent-linux.img'/>
+			<target dev='vda' bus='virtio'/>
+			<address type='drive' controller='0' bus='0' unit='0'/>
+		</disk>
+
+
+   Replace ``/path/to/image/recent-linux.img`` with the path to the OS image.
+   The minimum kernel for using the faster ``virtio`` bus is 2.6.25. See 
+   `Virtio`_ for details.
+
+   **IMPORTANT:** Use ``sudo virsh edit`` instead of a text editor. If you edit 
+   the configuration file under ``/etc/libvirt/qemu`` with a text editor, 
+   ``libvirt`` may not recognize the change. If there is a discrepancy between 
+   the contents of the XML file under ``/etc/libvirt/qemu`` and the result of 
+   ``sudo virsh dumpxml {vm-domain-name}``, then your VM may not work 
+   properly.
+   
+
+#. Add the Ceph RBD image you created as a ``<disk>`` entry. :: 
+
+	<disk type='network' device='disk'>
+		<source protocol='rbd' name='libvirt-pool/new-libvirt-image'>
+			<host name='{monitor-host}' port='6789'/>
+		</source>
+		<target dev='vdb' bus='virtio'/>
+	</disk>
+
+   Replace ``{monitor-host}`` with the name of your host, and replace the 
+   pool and/or image name as necessary. You may add multiple ``<host>`` 
+   entries for your Ceph monitors. The ``dev`` attribute is the logical
+   device name that will appear under the ``/dev`` directory of your 
+   VM. The optional ``bus`` attribute indicates the type of disk device to 
+   emulate. The valid settings are driver specific (e.g., "ide", "scsi", 
+   "virtio", "xen", "usb" or "sata").
+   
+   See `Disks`_ for details of the ``<disk>`` element, and its child elements
+   and attributes.
+	
+#. Save the file.
+
+#. If your Ceph Storage Cluster has `Ceph Authentication`_ enabled (it does by 
+   default), you must generate a secret. :: 
+
+	cat > secret.xml <<EOF
+	<secret ephemeral='no' private='no'>
+		<usage type='ceph'>
+			<name>client.libvirt secret</name>
+		</usage>
+	</secret>
+	EOF
+
+#. Define the secret. ::
+
+	sudo virsh secret-define --file secret.xml
+	{uuid of secret}
+
+#. Get the ``client.libvirt`` key and save the key string to a file. ::
+
+	ceph auth get-key client.libvirt | sudo tee client.libvirt.key
+
+#. Set the UUID of the secret. :: 
+
+	sudo virsh secret-set-value --secret {uuid of secret} --base64 $(cat client.libvirt.key) && rm client.libvirt.key secret.xml
+
+   You must also set the secret manually by adding the following ``<auth>`` 
+   entry to the ``<disk>`` element you entered earlier (replacing the
+   ``uuid`` value with the result from the command line example above). ::
+
+	sudo virsh edit {vm-domain-name}
+
+   Then, add ``<auth></auth>`` element to the domain configuration file::
+
+	...
+	</source>
+	<auth username='libvirt'>
+		<secret type='ceph' uuid='{uuid of secret}'/>
+	</auth>
+	<target ... 
+
+
+   **NOTE:** The exemplary ID is ``libvirt``, not the Ceph name 
+   ``client.libvirt`` as generated at step 2 of `Configuring Ceph`_. Ensure 
+   you use the ID component of the Ceph name you generated. If for some reason 
+   you need to regenerate the secret, you will have to execute 
+   ``sudo virsh secret-undefine {uuid}`` before executing 
+   ``sudo virsh secret-set-value`` again.
+
+
+Summary
+=======
+
+Once you have configured the VM for use with Ceph, you can start the VM.
+To verify that the VM and Ceph are communicating, you may perform the
+following procedures.
+
+
+#. Check to see if Ceph is running:: 
+
+	ceph health
+
+#. Check to see if the VM is running. :: 
+
+	sudo virsh list
+
+#. Check to see if the VM is communicating with Ceph. Replace 
+   ``{vm-domain-name}`` with the name of your VM domain:: 
+
+	sudo virsh qemu-monitor-command --hmp {vm-domain-name} 'info block'
+
+#. Check to see if the device from ``<target dev='vdb' bus='virtio'/>`` exists::
+   
+       virsh domblklist {vm-domain-name} --details
+
+If everything looks okay, you may begin using the Ceph block device 
+within your VM.
+
+
+.. _Installation: ../../install
+.. _libvirt Virtualization API: http://www.libvirt.org
+.. _Block Devices and OpenStack: ../rbd-openstack
+.. _Block Devices and CloudStack: ../rbd-cloudstack
+.. _Create a pool: ../../rados/operations/pools#create-a-pool
+.. _Create a Ceph User: ../../rados/operations/user-management#add-a-user
+.. _create an image: ../qemu-rbd#creating-images-with-qemu
+.. _Virsh Command Reference: http://www.libvirt.org/virshcmdref.html
+.. _KVM/VirtManager: https://help.ubuntu.com/community/KVM/VirtManager
+.. _Ceph Authentication: ../../rados/configuration/auth-config-ref
+.. _Disks: http://www.libvirt.org/formatdomain.html#elementsDisks
+.. _rbd create: ../rados-rbd-cmds#creating-a-block-device-image
+.. _User Management - User: ../../rados/operations/user-management#user
+.. _User Management - CLI: ../../rados/operations/user-management#command-line-usage
+.. _Virtio: http://www.linux-kvm.org/page/Virtio
diff --git a/doc/rbd/man/index.rst b/doc/rbd/man/index.rst
new file mode 100644
index 000000000..110273acc
--- /dev/null
+++ b/doc/rbd/man/index.rst
@@ -0,0 +1,16 @@
+============================
+ Ceph Block Device Manpages
+============================
+
+.. toctree::
+   :maxdepth: 1
+
+   rbd <../../man/8/rbd>
+   rbd-fuse <../../man/8/rbd-fuse>
+   rbd-nbd <../../man/8/rbd-nbd>
+   rbd-ggate <../../man/8/rbd-ggate>
+   rbd-map <../../man/8/rbdmap>
+   ceph-rbdnamer <../../man/8/ceph-rbdnamer>
+   rbd-replay-prep <../../man/8/rbd-replay-prep>
+   rbd-replay <../../man/8/rbd-replay>
+   rbd-replay-many <../../man/8/rbd-replay-many>
diff --git a/doc/rbd/qemu-rbd.rst b/doc/rbd/qemu-rbd.rst
new file mode 100644
index 000000000..f5925d6c3
--- /dev/null
+++ b/doc/rbd/qemu-rbd.rst
@@ -0,0 +1,220 @@
+========================
+ QEMU and Block Devices
+========================
+
+.. index:: Ceph Block Device; QEMU KVM
+
+The most frequent Ceph Block Device use case involves providing block device
+images to virtual machines. For example, a user may create  a "golden" image
+with an OS and any relevant software in an ideal configuration. Then the user
+takes a snapshot of the image. Finally the user clones the snapshot (potentially
+many times). See `Snapshots`_ for details. The ability to make copy-on-write
+clones of a snapshot means that Ceph can provision block device images to
+virtual machines quickly, because the client doesn't have to download the entire
+image each time it spins up a new virtual machine.
+
+
+.. ditaa::
+
+            +---------------------------------------------------+
+            |                       QEMU                        |
+            +---------------------------------------------------+
+            |                      librbd                       |
+            +---------------------------------------------------+
+            |                     librados                      |
+            +------------------------+-+------------------------+
+            |          OSDs          | |        Monitors        |
+            +------------------------+ +------------------------+
+
+
+Ceph Block Devices attach to QEMU virtual machines. For details on
+QEMU, see  `QEMU Open Source Processor Emulator`_. For QEMU documentation, see
+`QEMU Manual`_. For installation details, see `Installation`_.
+
+.. important:: To use Ceph Block Devices with QEMU, you must have access to a 
+   running Ceph cluster.
+
+
+Usage
+=====
+
+The QEMU command line expects you to specify the Ceph pool and image name. You
+may also specify a snapshot. 
+
+QEMU will assume that Ceph configuration resides in the default
+location (e.g., ``/etc/ceph/$cluster.conf``) and that you are executing
+commands as the default ``client.admin`` user unless you expressly specify
+another Ceph configuration file path or another user. When specifying a user,
+QEMU uses the ``ID`` rather than the full ``TYPE:ID``. See `User Management -
+User`_ for details. Do not prepend the client type (i.e., ``client.``) to the
+beginning of the user  ``ID``, or you will receive an authentication error. You
+should have the key for the ``admin`` user or the key of another user you
+specify with the ``:id={user}`` option in a keyring file stored in default path
+(i.e., ``/etc/ceph`` or the local directory with appropriate file ownership and
+permissions. Usage takes the following form::
+
+	qemu-img {command} [options] rbd:{pool-name}/{image-name}[@snapshot-name][:option1=value1][:option2=value2...]
+
+For example, specifying the ``id`` and ``conf`` options might look like the following:: 
+
+	qemu-img {command} [options] rbd:glance-pool/maipo:id=glance:conf=/etc/ceph/ceph.conf
+
+.. tip:: Configuration values containing ``:``, ``@``, or ``=`` can be escaped with a
+         leading ``\`` character.
+
+
+Creating Images with QEMU
+=========================
+
+You can create a block device image from QEMU. You must specify ``rbd``,  the
+pool name, and the name of the image you wish to create. You must also specify
+the size of the image. ::
+
+	qemu-img create -f raw rbd:{pool-name}/{image-name} {size}
+
+For example::
+
+	qemu-img create -f raw rbd:data/foo 10G
+
+.. important:: The ``raw`` data format is really the only sensible
+   ``format`` option to use with RBD. Technically, you could use other
+   QEMU-supported formats (such as ``qcow2`` or ``vmdk``), but doing
+   so would add additional overhead, and would also render the volume
+   unsafe for virtual machine live migration when caching (see below)
+   is enabled.
+
+
+Resizing Images with QEMU
+=========================
+
+You can resize a block device image from QEMU. You must specify ``rbd``, 
+the pool name, and the name of the image you wish to resize. You must also
+specify the size of the image. ::
+
+	qemu-img resize rbd:{pool-name}/{image-name} {size}
+
+For example::
+
+	qemu-img resize rbd:data/foo 10G
+
+
+Retrieving Image Info with QEMU
+===============================
+
+You can retrieve block device image information from QEMU. You must 
+specify ``rbd``, the pool name, and the name of the image. ::
+
+	qemu-img info rbd:{pool-name}/{image-name}
+
+For example::
+
+	qemu-img info rbd:data/foo
+
+
+Running QEMU with RBD
+=====================
+
+QEMU can pass a block device from the host on to a guest, but since
+QEMU 0.15, there's no need to map an image as a block device on
+the host. Instead, QEMU attaches an image as a virtual block
+device directly via ``librbd``. This strategy increases performance
+by avoiding context switches and taking advantage of `RBD caching`_.
+
+You can use ``qemu-img`` to convert existing virtual machine images to Ceph
+block device images. For example, if you have a qcow2 image, you could run::
+
+    qemu-img convert -f qcow2 -O raw debian_squeeze.qcow2 rbd:data/squeeze
+
+To run a virtual machine booting from that image, you could run::
+
+    qemu -m 1024 -drive format=raw,file=rbd:data/squeeze
+
+`RBD caching`_ can significantly improve performance.
+Since QEMU 1.2, QEMU's cache options control ``librbd`` caching::
+
+    qemu -m 1024 -drive format=rbd,file=rbd:data/squeeze,cache=writeback
+
+If you have an older version of QEMU, you can set the ``librbd`` cache
+configuration (like any Ceph configuration option) as part of the
+'file' parameter::
+
+    qemu -m 1024 -drive format=raw,file=rbd:data/squeeze:rbd_cache=true,cache=writeback
+
+.. important:: If you set rbd_cache=true, you must set cache=writeback
+   or risk data loss. Without cache=writeback, QEMU will not send
+   flush requests to librbd. If QEMU exits uncleanly in this
+   configuration, file systems on top of rbd can be corrupted.
+
+.. _RBD caching: ../rbd-config-ref/#rbd-cache-config-settings
+
+
+.. index:: Ceph Block Device; discard trim and libvirt
+
+Enabling Discard/TRIM
+=====================
+
+Since Ceph version 0.46 and QEMU version 1.1, Ceph Block Devices support the
+discard operation. This means that a guest can send TRIM requests to let a Ceph
+block device reclaim unused space. This can be enabled in the guest by mounting
+``ext4`` or ``XFS`` with the ``discard`` option.
+
+For this to be available to the guest, it must be explicitly enabled
+for the block device. To do this, you must specify a
+``discard_granularity`` associated with the drive::
+
+    qemu -m 1024 -drive format=raw,file=rbd:data/squeeze,id=drive1,if=none \
+         -device driver=ide-hd,drive=drive1,discard_granularity=512
+
+Note that this uses the IDE driver. The virtio driver does not
+support discard.
+
+If using libvirt, edit your libvirt domain's configuration file using ``virsh
+edit`` to include the ``xmlns:qemu`` value. Then, add a ``qemu:commandline``
+block as a child of that domain. The following example shows how to set two
+devices with ``qemu id=`` to different ``discard_granularity`` values.
+
+.. code-block:: xml
+
+	<domain type='kvm' xmlns:qemu='http://libvirt.org/schemas/domain/qemu/1.0'>
+		<qemu:commandline>
+			<qemu:arg value='-set'/>
+			<qemu:arg value='block.scsi0-0-0.discard_granularity=4096'/>
+			<qemu:arg value='-set'/>
+			<qemu:arg value='block.scsi0-0-1.discard_granularity=65536'/>
+		</qemu:commandline>
+	</domain>
+
+
+.. index:: Ceph Block Device; cache options
+
+QEMU Cache Options
+==================
+
+QEMU's cache options correspond to the following Ceph `RBD Cache`_ settings.
+
+Writeback::
+
+   rbd_cache = true
+
+Writethrough::
+
+   rbd_cache = true
+   rbd_cache_max_dirty = 0
+
+None:: 
+
+   rbd_cache = false
+
+QEMU's cache settings override Ceph's cache settings (including settings that
+are explicitly set in the Ceph configuration file).
+
+.. note:: Prior to QEMU v2.4.0, if you explicitly set `RBD Cache`_ settings
+   in the Ceph configuration file, your Ceph settings override the QEMU cache
+   settings.
+
+.. _QEMU Open Source Processor Emulator: http://wiki.qemu.org/Main_Page
+.. _QEMU Manual: http://wiki.qemu.org/Manual
+.. _RBD Cache: ../rbd-config-ref/
+.. _Snapshots: ../rbd-snapshot/
+.. _Installation: ../../install
+.. _User Management - User: ../../rados/operations/user-management#user
diff --git a/doc/rbd/rados-rbd-cmds.rst b/doc/rbd/rados-rbd-cmds.rst
new file mode 100644
index 000000000..0bbcb2611
--- /dev/null
+++ b/doc/rbd/rados-rbd-cmds.rst
@@ -0,0 +1,326 @@
+=============================
+ Basic Block Device Commands
+=============================
+
+.. index:: Ceph Block Device; image management
+
+The ``rbd`` command enables you to create, list, introspect and remove block
+device images. You can also use it to clone images, create snapshots,
+rollback an image to a snapshot, view a snapshot, etc. For details on using
+the ``rbd`` command, see `RBD – Manage RADOS Block Device (RBD) Images`_ for
+details. 
+
+.. important:: To use Ceph Block Device commands, you must have access to 
+   a running Ceph cluster.
+
+Create a Block Device Pool
+==========================
+
+#. Use the ``ceph`` tool to `create a pool`_.
+
+#. Use the ``rbd`` tool to initialize the pool for use by RBD:
+
+   .. prompt:: bash $
+
+      rbd pool init <pool-name>
+
+   .. note:: The ``rbd`` tool assumes a default pool name of 'rbd' if no pool
+      name is specified in the command.
+
+
+Create a Block Device User
+==========================
+
+Unless otherwise specified, the ``rbd`` command uses the Ceph user ID ``admin``
+to access the Ceph cluster. The ``admin`` Ceph user ID allows full
+administrative access to the cluster. We recommend that you acess the Ceph
+cluster with a Ceph user ID that has fewer permissions than the ``admin`` Ceph
+user ID does. We call this non-``admin`` Ceph user ID a "block device user" or
+"Ceph user".
+
+To `create a Ceph user`_, use the ``ceph auth get-or-create`` command to
+specify the Ceph user ID name, monitor caps (capabilities), and OSD caps
+(capabilities):
+
+.. prompt:: bash $
+
+   ceph auth get-or-create client.{ID} mon 'profile rbd' osd 'profile {profile name} [pool={pool-name}][, profile ...]' mgr 'profile rbd [pool={pool-name}]'
+
+For example: to create a Ceph user ID named ``qemu`` that has read-write access
+to the pool ``vms`` and read-only access to the pool ``images``, run the
+following command:
+
+.. prompt:: bash $
+
+   ceph auth get-or-create client.qemu mon 'profile rbd' osd 'profile rbd pool=vms, profile rbd-read-only pool=images' mgr 'profile rbd pool=images'
+
+The output from the ``ceph auth get-or-create`` command is the keyring for the
+specified Ceph user ID, which can be written to
+``/etc/ceph/ceph.client.{ID}.keyring``.
+
+.. note:: Specify the Ceph user ID by providing the ``--id {id} argument when
+   using the ``rbd`` command. This argument is optional. 
+
+Creating a Block Device Image
+=============================
+
+Before you can add a block device to a node, you must create an image for it in
+the :term:`Ceph Storage Cluster`. To create a block device image, run a command of this form: 
+
+.. prompt:: bash $
+
+   rbd create --size {megabytes} {pool-name}/{image-name}
+
+For example, to create a 1GB image named ``bar`` that stores information in a
+pool named ``swimmingpool``, run this command:
+
+.. prompt:: bash $
+
+   rbd create --size 1024 swimmingpool/bar
+
+If you don't specify a pool when you create an image, then the image will be
+stored in the default pool ``rbd``. For example, if you ran this command, you
+would create a 1GB image named ``foo`` that is stored in the default pool
+``rbd``:
+
+.. prompt:: bash $
+
+   rbd create --size 1024 foo
+
+.. note:: You must create a pool before you can specify it as a source. See
+   `Storage Pools`_ for details.
+
+Listing Block Device Images
+===========================
+
+To list block devices in the ``rbd`` pool, run the following command:
+
+.. prompt:: bash $
+
+   rbd ls
+
+.. note:: ``rbd`` is the default pool name, and ``rbd ls`` lists the commands
+   in the default pool.
+
+To list block devices in a particular pool, run the following command, but
+replace ``{poolname}`` with the name of the pool: 
+
+.. prompt:: bash $
+
+   rbd ls {poolname}
+	
+For example:
+
+.. prompt:: bash $
+
+   rbd ls swimmingpool
+
+To list "deferred delete" block devices in the ``rbd`` pool, run the
+following command:
+
+.. prompt:: bash $
+
+   rbd trash ls
+
+To list "deferred delete" block devices in a particular pool, run the 
+following command, but replace ``{poolname}`` with the name of the pool:
+
+.. prompt:: bash $
+
+   rbd trash ls {poolname}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd trash ls swimmingpool
+
+Retrieving Image Information
+============================
+
+To retrieve information from a particular image, run the following command, but
+replace ``{image-name}`` with the name for the image:
+
+.. prompt:: bash $
+
+   rbd info {image-name}
+	
+For example:
+
+.. prompt:: bash $
+
+   rbd info foo
+	
+To retrieve information from an image within a pool, run the following command,
+but replace ``{image-name}`` with the name of the image and replace
+``{pool-name}`` with the name of the pool:
+
+.. prompt:: bash $
+
+   rbd info {pool-name}/{image-name}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd info swimmingpool/bar
+
+.. note:: Other naming conventions are possible, and might conflict with the
+   naming convention described here. For example, ``userid/<uuid>`` is a
+   possible name for an RBD image, and such a name might (at the least) be
+   confusing. 
+
+Resizing a Block Device Image
+=============================
+
+:term:`Ceph Block Device` images are thin provisioned. They don't actually use
+any physical storage until you begin saving data to them. However, they do have
+a maximum capacity that you set with the ``--size`` option. If you want to
+increase (or decrease) the maximum size of a Ceph Block Device image, run one
+of the following commands:
+
+Increasing the Size of a Block Device Image
+-------------------------------------------
+
+.. prompt:: bash $
+
+   rbd resize --size 2048 foo
+
+Decreasing the Size of a Block Device Image
+-------------------------------------------
+
+.. prompt:: bash $
+
+   rbd resize --size 2048 foo --allow-shrink
+
+
+Removing a Block Device Image
+=============================
+
+To remove a block device, run the following command, but replace
+``{image-name}`` with the name of the image you want to remove:
+
+.. prompt:: bash $
+
+   rbd rm {image-name}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd rm foo
+
+Removing a Block Device from a Pool
+-----------------------------------
+
+To remove a block device from a pool, run the following command but replace
+``{image-name}`` with the name of the image to be removed, and replace
+``{pool-name}`` with the name of the pool from which the image is to be
+removed:
+
+.. prompt:: bash $
+
+   rbd rm {pool-name}/{image-name}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd rm swimmingpool/bar
+
+"Defer Deleting" a Block Device from a Pool
+-------------------------------------------
+
+To defer delete a block device from a pool (which entails moving it to the
+"trash" and deleting it later), run the following command but replace
+``{image-name}`` with the name of the image to be moved to the trash and
+replace ``{pool-name}`` with the name of the pool:
+
+.. prompt:: bash $
+
+   rbd trash mv {pool-name}/{image-name}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd trash mv swimmingpool/bar
+
+Removing a Deferred Block Device from a Pool
+--------------------------------------------
+
+To remove a deferred block device from a pool, run the following command but
+replace ``{image-}`` with the ID of the image to be removed, and replace
+``{pool-name}`` with the name of the pool from which the image is to be
+removed:
+
+.. prompt:: bash $
+
+   rbd trash rm {pool-name}/{image-}
+
+For example:
+
+.. prompt:: bash $
+   
+   rbd trash rm swimmingpool/2bf4474b0dc51
+
+.. note::
+
+  * You can move an image to the trash even if it has snapshot(s) or is
+    actively in use by clones. However, you cannot remove it from the trash
+    under those conditions.
+
+  * You can use ``--expires-at`` to set the deferment time (default is
+    ``now``). If the deferment time has not yet arrived, you cannot remove the
+    image unless you use ``--force``.
+
+Restoring a Block Device Image
+==============================
+
+To restore a deferred delete block device in the rbd pool, run the 
+following command but replace ``{image-id}`` with the ID of the image:
+
+.. prompt:: bash $
+
+   rbd trash restore {image-id}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd trash restore 2bf4474b0dc51
+
+Restoring a Block Device Image in a Specific Pool
+-------------------------------------------------
+
+To restore a deferred delete block device in a particular pool, run the
+following command but replace ``{image-id}`` with the ID of the image and
+replace ``{pool-name}`` with the name of the pool:
+
+.. prompt:: bash $
+
+  rbd trash restore {pool-name}/{image-id}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd trash restore swimmingpool/2bf4474b0dc51
+
+
+Renaming an Image While Restoring It
+------------------------------------
+
+You can also use ``--image`` to rename the image while restoring it. 
+
+For example:
+
+.. prompt:: bash $
+   
+   rbd trash restore swimmingpool/2bf4474b0dc51 --image new-name
+
+
+.. _create a pool: ../../rados/operations/pools/#create-a-pool
+.. _Storage Pools: ../../rados/operations/pools
+.. _RBD – Manage RADOS Block Device (RBD) Images: ../../man/8/rbd/
+.. _create a Ceph user: ../../rados/operations/user-management#add-a-user
diff --git a/doc/rbd/rbd-cloudstack.rst b/doc/rbd/rbd-cloudstack.rst
new file mode 100644
index 000000000..1b961234b
--- /dev/null
+++ b/doc/rbd/rbd-cloudstack.rst
@@ -0,0 +1,157 @@
+=============================
+ Block Devices and CloudStack
+=============================
+
+You may use Ceph Block Device images with CloudStack 4.0 and higher through
+``libvirt``, which configures the QEMU interface to ``librbd``. Ceph stripes
+block device images as objects across the cluster, which means that large Ceph
+Block Device images have better performance than a standalone server!
+
+To use Ceph Block Devices with CloudStack 4.0 and higher, you must install QEMU,
+``libvirt``, and CloudStack first. We recommend using a separate physical host
+for your CloudStack installation. CloudStack recommends a minimum of 4GB of RAM
+and a dual-core processor, but more CPU and RAM will perform better. The
+following diagram depicts the CloudStack/Ceph technology stack.
+
+
+.. ditaa::
+
+            +---------------------------------------------------+
+            |                   CloudStack                      |
+            +---------------------------------------------------+
+            |                     libvirt                       |
+            +------------------------+--------------------------+
+                                     |
+                                     | configures
+                                     v
+            +---------------------------------------------------+
+            |                       QEMU                        |
+            +---------------------------------------------------+
+            |                      librbd                       |
+            +---------------------------------------------------+
+            |                     librados                      |
+            +------------------------+-+------------------------+
+            |          OSDs          | |        Monitors        |
+            +------------------------+ +------------------------+
+
+.. important:: To use Ceph Block Devices with CloudStack, you must have  
+   access to a running Ceph Storage Cluster.
+
+CloudStack integrates with Ceph's block devices to provide CloudStack with a
+back end for CloudStack's Primary Storage. The instructions below detail the
+setup for CloudStack Primary Storage.
+
+.. note:: We recommend installing with Ubuntu 14.04 or later so that 
+   you can use package installation instead of having to compile 
+   libvirt from source.
+
+Installing and configuring QEMU for use with CloudStack doesn't require any
+special handling. Ensure that you have a running Ceph Storage Cluster. Install
+QEMU and configure it for use with Ceph; then, install ``libvirt`` version
+0.9.13 or higher (you may need to compile from source) and ensure it is running
+with Ceph.
+
+
+.. note:: Ubuntu 14.04 and CentOS 7.2 will have ``libvirt`` with RBD storage
+   pool support enabled by default.
+
+.. index:: pools; CloudStack
+
+Create a Pool
+=============
+
+By default, Ceph block devices use the ``rbd`` pool. Create a pool for
+CloudStack NFS Primary Storage. Ensure your Ceph cluster is running, then create
+the pool. ::
+
+   ceph osd pool create cloudstack
+   
+See `Create a Pool`_ for details on specifying the number of placement groups
+for your pools, and `Placement Groups`_ for details on the number of placement
+groups you should set for your pools.
+
+A newly created pool must be initialized prior to use. Use the ``rbd`` tool
+to initialize the pool::
+
+        rbd pool init cloudstack
+
+Create a Ceph User
+==================
+
+To access the Ceph cluster we require a Ceph user which has the correct
+credentials to access the ``cloudstack`` pool we just created. Although we could
+use ``client.admin`` for this, it's recommended to create a user with only
+access to the ``cloudstack`` pool. ::
+
+  ceph auth get-or-create client.cloudstack mon 'profile rbd' osd 'profile rbd pool=cloudstack'
+
+Use the information returned by the command in the next step when adding the 
+Primary Storage.
+
+See `User Management`_ for additional details.
+
+Add Primary Storage
+===================
+
+To add a Ceph block device as Primary Storage, the steps include: 
+
+#. Log in to the CloudStack UI.
+#. Click **Infrastructure** on the left side navigation bar. 
+#. Select **View All** under **Primary Storage**.
+#. Click the **Add Primary Storage** button on the top right hand side.
+#. Fill in the following information, according to your infrastructure setup:
+
+   - Scope (i.e. Cluster or Zone-Wide).
+   
+   - Zone.
+   
+   - Pod.
+   
+   - Cluster.
+   
+   - Name of Primary Storage.
+   
+   - For **Protocol**, select ``RBD``.
+   
+   - For **Provider**, select the appropriate provider type (i.e. DefaultPrimary, SolidFire, SolidFireShared, or CloudByte).  Depending on the provider chosen, fill out the information pertinent to your setup.
+   
+#. Add cluster information (``cephx`` is supported).
+
+   - For **RADOS Monitor**, provide the IP address of a Ceph monitor node.
+   
+   - For **RADOS Pool**, provide the name of an RBD pool.
+   
+   - For **RADOS User**, provide a user that has sufficient rights to the RBD pool. Note: Do not include the ``client.`` part of the user.
+   
+   - For **RADOS Secret**, provide the secret the user's secret.
+   
+   - **Storage Tags** are optional. Use tags at your own discretion. For more information about storage tags in CloudStack, refer to `Storage Tags`_.
+   
+#. Click **OK**.
+
+Create a Disk Offering
+======================
+
+To create a new disk offering, refer to `Create a New Disk Offering`_.
+Create a disk offering so that it matches the ``rbd`` tag.
+The ``StoragePoolAllocator`` will choose the  ``rbd``
+pool when searching for a suitable storage pool. If the disk offering doesn't
+match the ``rbd`` tag, the ``StoragePoolAllocator`` may select the pool you
+created (e.g., ``cloudstack``).
+
+
+Limitations
+===========
+
+- CloudStack will only bind to one monitor (You can however create a Round Robin DNS record over multiple monitors)
+
+
+
+.. _Create a Pool: ../../rados/operations/pools#createpool
+.. _Placement Groups: ../../rados/operations/placement-groups
+.. _Install and Configure QEMU: ../qemu-rbd
+.. _Install and Configure libvirt: ../libvirt
+.. _KVM Hypervisor Host Installation: http://docs.cloudstack.apache.org/en/latest/installguide/hypervisor/kvm.html
+.. _Storage Tags: http://docs.cloudstack.apache.org/en/latest/adminguide/storage.html#storage-tags
+.. _Create a New Disk Offering: http://docs.cloudstack.apache.org/en/latest/adminguide/service_offerings.html#creating-a-new-disk-offering
+.. _User Management: ../../rados/operations/user-management
diff --git a/doc/rbd/rbd-config-ref.rst b/doc/rbd/rbd-config-ref.rst
new file mode 100644
index 000000000..777894cd6
--- /dev/null
+++ b/doc/rbd/rbd-config-ref.rst
@@ -0,0 +1,476 @@
+=======================
+ Config Settings
+=======================
+
+See `Block Device`_ for additional details.
+
+Generic IO Settings
+===================
+
+``rbd_compression_hint``
+
+:Description: Hint to send to the OSDs on write operations. If set to 
+              ``compressible`` and the OSD ``bluestore_compression_mode``
+              setting is ``passive``, the OSD will attempt to compress data
+              If set to ``incompressible`` and the OSD compression setting
+              is ``aggressive``, the OSD will not attempt to compress data.
+:Type: Enum
+:Required: No
+:Default: ``none``
+:Values: ``none``, ``compressible``, ``incompressible``
+
+
+``rbd_read_from_replica_policy``
+
+:Description: Policy for determining which OSD will receive read operations.
+              If set to ``default``, each PG's primary OSD will always be used
+              for read operations. If set to ``balance``, read operations will
+              be sent to a randomly selected OSD within the replica set. If set
+              to ``localize``, read operations will be sent to the closest OSD
+              as determined by the CRUSH map. Note: this feature requires the
+              cluster to be configured with a minimum compatible OSD release of
+              Octopus.
+:Type: Enum
+:Required: No
+:Default: ``default``
+:Values: ``default``, ``balance``, ``localize``
+
+Cache Settings
+=======================
+
+.. sidebar:: Kernel Caching
+
+	The kernel driver for Ceph block devices can use the Linux page cache to
+	improve performance.
+
+The user space implementation of the Ceph block device (i.e., ``librbd``) cannot
+take advantage of the Linux page cache, so it includes its own in-memory
+caching, called "RBD caching." RBD caching behaves just like well-behaved hard
+disk caching.  When the OS sends a barrier or a flush request, all dirty data is
+written to the OSDs. This means that using write-back caching is just as safe as
+using a well-behaved physical hard disk with a VM that properly sends flushes
+(i.e. Linux kernel >= 2.6.32). The cache uses a Least Recently Used (LRU)
+algorithm, and in write-back mode it can coalesce contiguous requests for
+better throughput.
+
+The librbd cache is enabled by default and supports three different cache
+policies: write-around, write-back, and write-through. Writes return
+immediately under both the write-around and write-back policies, unless there
+are more than ``rbd_cache_max_dirty`` unwritten bytes to the storage cluster.
+The write-around policy differs from the write-back policy in that it does
+not attempt to service read requests from the cache, unlike the write-back
+policy, and is therefore faster for high performance write workloads. Under the
+write-through policy, writes return only when the data is on disk on all
+replicas, but reads may come from the cache.
+
+Prior to receiving a flush request, the cache behaves like a write-through cache
+to ensure safe operation for older operating systems that do not send flushes to
+ensure crash consistent behavior.
+
+If the librbd cache is disabled, writes and
+reads go directly to the storage cluster, and writes return only when the data
+is on disk on all replicas.
+
+.. note::
+   The cache is in memory on the client, and each RBD image has
+   its own.  Since the cache is local to the client, there's no coherency
+   if there are others accessing the image. Running GFS or OCFS on top of
+   RBD will not work with caching enabled.
+
+
+Option settings for RBD should be set in the ``[client]``
+section of your configuration file or the central config store. These settings
+include:
+
+``rbd_cache``
+
+:Description: Enable caching for RADOS Block Device (RBD).
+:Type: Boolean
+:Required: No
+:Default: ``true``
+
+
+``rbd_cache_policy``
+
+:Description: Select the caching policy for librbd.
+:Type: Enum
+:Required: No
+:Default: ``writearound``
+:Values: ``writearound``, ``writeback``, ``writethrough``
+
+
+``rbd_cache_writethrough_until_flush``
+
+:Description: Start out in ``writethrough`` mode, and switch to ``writeback``
+              after the first flush request is received. Enabling is a
+              conservative but safe strategy in case VMs running on RBD volumes
+              are too old to send flushes, like the ``virtio`` driver in Linux
+              kernels older than 2.6.32.
+:Type: Boolean
+:Required: No
+:Default: ``true``
+
+
+``rbd_cache_size``
+
+:Description: The per-volume RBD client cache size in bytes.
+:Type: 64-bit Integer
+:Required: No
+:Default: ``32 MiB``
+:Policies: write-back and write-through
+
+
+``rbd_cache_max_dirty``
+
+:Description: The ``dirty`` limit in bytes at which the cache triggers write-back.  If ``0``, uses write-through caching.
+:Type: 64-bit Integer
+:Required: No
+:Constraint: Must be less than ``rbd_cache_size``.
+:Default: ``24 MiB``
+:Policies: write-around and write-back
+
+
+``rbd_cache_target_dirty``
+
+:Description: The ``dirty target`` before the cache begins writing data to the data storage. Does not block writes to the cache.
+:Type: 64-bit Integer
+:Required: No
+:Constraint: Must be less than ``rbd_cache_max_dirty``.
+:Default: ``16 MiB``
+:Policies: write-back
+
+
+``rbd_cache_max_dirty_age``
+
+:Description: The number of seconds dirty data is in the cache before writeback starts. 
+:Type: Float
+:Required: No
+:Default: ``1.0``
+:Policies: write-back
+
+
+.. _Block Device: ../../rbd
+
+
+Read-ahead Settings
+=======================
+
+librbd supports read-ahead/prefetching to optimize small, sequential reads.
+This should normally be handled by the guest OS in the case of a VM,
+but boot loaders may not issue efficient reads. Read-ahead is automatically
+disabled if caching is disabled or if the policy is write-around.
+
+
+``rbd_readahead_trigger_requests``
+
+:Description: Number of sequential read requests necessary to trigger read-ahead.
+:Type: Integer
+:Required: No
+:Default: ``10``
+
+
+``rbd_readahead_max_bytes``
+
+:Description: Maximum size of a read-ahead request.  If zero, read-ahead is disabled.
+:Type: 64-bit Integer
+:Required: No
+:Default: ``512 KiB``
+
+
+``rbd_readahead_disable_after_bytes``
+
+:Description: After this many bytes have been read from an RBD image, read-ahead
+              is disabled for that image until it is closed.  This allows the
+              guest OS to take over read-ahead once it is booted.  If zero,
+              read-ahead stays enabled.
+:Type: 64-bit Integer
+:Required: No
+:Default: ``50 MiB``
+
+
+Image Features
+==============
+
+RBD supports advanced features which can be specified via the command line when
+creating images or the default features can be configured via
+``rbd_default_features = <sum of feature numeric values>`` or
+``rbd_default_features = <comma-delimited list of CLI values>``.
+
+``Layering``
+
+:Description: Layering enables cloning.
+:Internal value: 1
+:CLI value: layering
+:Added in: v0.52 (Bobtail)
+:KRBD support: since v3.10
+:Default: yes
+
+``Striping v2``
+
+:Description: Striping spreads data across multiple objects. Striping helps with
+              parallelism for sequential read/write workloads.
+:Internal value: 2
+:CLI value: striping
+:Added in: v0.55 (Bobtail)
+:KRBD support: since v3.10 (default striping only, "fancy" striping added in v4.17)
+:Default: yes
+
+``Exclusive locking``
+
+:Description: When enabled, it requires a client to acquire a lock on an object
+              before making a write. Exclusive lock should only be enabled when
+              a single client is accessing an image at any given time.
+:Internal value: 4
+:CLI value: exclusive-lock
+:Added in: v0.92 (Hammer)
+:KRBD support: since v4.9
+:Default: yes
+
+``Object map``
+
+:Description: Object map support depends on exclusive lock support. Block
+              devices are thin provisioned, which means that they only store
+              data that actually has been written, ie. they are *sparse*. Object
+              map support helps track which objects actually exist (have data
+              stored on a device). Enabling object map support speeds up I/O
+              operations for cloning, importing and exporting a sparsely
+              populated image, and deleting.
+:Internal value: 8
+:CLI value: object-map
+:Added in: v0.93 (Hammer)
+:KRBD support: since v5.3
+:Default: yes
+
+
+``Fast-diff``
+
+:Description: Fast-diff support depends on object map support and exclusive lock
+              support. It adds another property to the object map, which makes
+              it much faster to generate diffs between snapshots of an image.
+              It is also much faster to calculate the actual data usage of a
+              snapshot or volume (``rbd du``).
+:Internal value: 16
+:CLI value: fast-diff
+:Added in: v9.0.1 (Infernalis)
+:KRBD support: since v5.3
+:Default: yes
+
+
+``Deep-flatten``
+
+:Description: Deep-flatten enables ``rbd flatten`` to work on all  snapshots of
+              an image, in addition to the image itself. Without it, snapshots
+              of an image will still rely on the parent, so the parent cannot be
+              deleted until the snapshots are first deleted. Deep-flatten makes
+              a parent independent of its clones, even if they have snapshots,
+              at the expense of using additional OSD device space.
+:Internal value: 32
+:CLI value: deep-flatten
+:Added in: v9.0.2 (Infernalis)
+:KRBD support: since v5.1
+:Default: yes
+
+
+``Journaling``
+
+:Description: Journaling support depends on exclusive lock support. Journaling
+              records all modifications to an image in the order they occur. RBD
+              mirroring can utilize the journal to replicate a crash-consistent
+              image to a remote cluster.  It is best to let ``rbd-mirror``
+              manage this feature only as needed, as enabling it long term may
+              result in substantial additional OSD space consumption.
+:Internal value: 64
+:CLI value: journaling
+:Added in: v10.0.1 (Jewel)
+:KRBD support: no
+:Default: no
+
+
+``Data pool``
+
+:Description: On erasure-coded pools, the image data block objects need to be stored on a separate pool from the image metadata.
+:Internal value: 128
+:Added in: v11.1.0 (Kraken)
+:KRBD support: since v4.11
+:Default: no
+
+
+``Operations``
+
+:Description: Used to restrict older clients from performing certain maintenance operations against an image (e.g. clone, snap create).
+:Internal value: 256
+:Added in: v13.0.2 (Mimic)
+:KRBD support: since v4.16
+
+
+``Migrating``
+
+:Description: Used to restrict older clients from opening an image when it is in migration state.
+:Internal value: 512
+:Added in: v14.0.1 (Nautilus)
+:KRBD support: no
+
+``Non-primary``
+
+:Description: Used to restrict changes to non-primary images using snapshot-based mirroring.
+:Internal value: 1024
+:Added in: v15.2.0 (Octopus)
+:KRBD support: no
+
+
+QOS Settings
+============
+
+librbd supports limiting per-image IO, controlled by the following
+settings.
+
+``rbd_qos_iops_limit``
+
+:Description: The desired limit of IO operations per second.
+:Type: Unsigned Integer
+:Required: No
+:Default: ``0``
+
+
+``rbd_qos_bps_limit``
+
+:Description: The desired limit of IO bytes per second.
+:Type: Unsigned Integer
+:Required: No
+:Default: ``0``
+
+
+``rbd_qos_read_iops_limit``
+
+:Description: The desired limit of read operations per second.
+:Type: Unsigned Integer
+:Required: No
+:Default: ``0``
+
+
+``rbd_qos_write_iops_limit``
+
+:Description: The desired limit of write operations per second.
+:Type: Unsigned Integer
+:Required: No
+:Default: ``0``
+
+
+``rbd_qos_read_bps_limit``
+
+:Description: The desired limit of read bytes per second.
+:Type: Unsigned Integer
+:Required: No
+:Default: ``0``
+
+
+``rbd_qos_writ_bps_limit``
+
+:Description: The desired limit of write bytes per second.
+:Type: Unsigned Integer
+:Required: No
+:Default: ``0``
+
+
+``rbd_qos_iops_burst``
+
+:Description: The desired burst limit of IO operations.
+:Type: Unsigned Integer
+:Required: No
+:Default: ``0``
+
+
+``rbd_qos_bps_burst``
+
+:Description: The desired burst limit of IO bytes.
+:Type: Unsigned Integer
+:Required: No
+:Default: ``0``
+
+
+``rbd_qos_read_iops_burst``
+
+:Description: The desired burst limit of read operations.
+:Type: Unsigned Integer
+:Required: No
+:Default: ``0``
+
+
+``rbd_qos_write_iops_burst``
+
+:Description: The desired burst limit of write operations.
+:Type: Unsigned Integer
+:Required: No
+:Default: ``0``
+
+
+``rbd_qos_read_bps_burst``
+
+:Description: The desired burst limit of read bytes per second.
+:Type: Unsigned Integer
+:Required: No
+:Default: ``0``
+
+
+``rbd_qos_write_bps_burst``
+
+:Description: The desired burst limit of write bytes per second.
+:Type: Unsigned Integer
+:Required: No
+:Default: ``0``
+
+
+``rbd_qos_iops_burst_seconds``
+
+:Description: The desired burst duration in seconds of IO operations.
+:Type: Unsigned Integer
+:Required: No
+:Default: ``1``
+
+
+``rbd_qos_bps_burst_seconds``
+
+:Description: The desired burst duration in seconds.
+:Type: Unsigned Integer
+:Required: No
+:Default: ``1``
+
+
+``rbd_qos_read_iops_burst_seconds``
+
+:Description: The desired burst duration in seconds of read operations.
+:Type: Unsigned Integer
+:Required: No
+:Default: ``1``
+
+
+``rbd_qos_write_iops_burst_seconds``
+
+:Description: The desired burst duration in seconds of write operations.
+:Type: Unsigned Integer
+:Required: No
+:Default: ``1``
+
+
+``rbd_qos_read_bps_burst_seconds``
+
+:Description: The desired burst duration in seconds of read bytes.
+:Type: Unsigned Integer
+:Required: No
+:Default: ``1``
+
+
+``rbd_qos_write_bps_burst_seconds``
+
+:Description: The desired burst duration in seconds of write bytes.
+:Type: Unsigned Integer
+:Required: No
+:Default: ``1``
+
+
+``rbd_qos_schedule_tick_min``
+
+:Description: The minimum schedule tick (in milliseconds) for QoS.
+:Type: Unsigned Integer
+:Required: No
+:Default: ``50``
diff --git a/doc/rbd/rbd-encryption.rst b/doc/rbd/rbd-encryption.rst
new file mode 100644
index 000000000..0628f3781
--- /dev/null
+++ b/doc/rbd/rbd-encryption.rst
@@ -0,0 +1,148 @@
+======================
+ Image Encryption
+======================
+
+.. index:: Ceph Block Device; encryption
+
+Starting with the Pacific release, image-level encryption can be handled
+internally by RBD clients. This means you can set a secret key that will be
+used to encrypt a specific RBD image. This page describes the scope of the
+RBD encryption feature.
+
+.. note::
+   The ``krbd`` kernel module does not support encryption at this time.
+
+.. note::
+   External tools (e.g. dm-crypt, QEMU) can be used as well to encrypt
+   an RBD image, and the feature set and limitation set for that use may be
+   different than described here.
+
+Encryption Format
+=================
+
+By default, RBD images are not encrypted. To encrypt an RBD image, it needs to
+be formatted to one of the supported encryption formats. The format operation
+persists encryption metadata to the image. The encryption metadata usually
+includes information such as the encryption format and version, cipher
+algorithm and mode specification, as well as information used to secure the
+encryption key. The encryption key itself is protected by a user-kept secret
+(usually a passphrase), which is never persisted. The basic encryption format
+operation will require specifying the encryption format and a secret.
+
+Some of the encryption metadata may be stored as part of the image data,
+typically an encryption header will be written to the beginning of the raw
+image data. This means that the effective image size of the encrypted image may
+be lower than the raw image size. See the `Supported Formats`_ section for more
+details.
+
+.. note::
+   Currently only flat images (i.e. not cloned) can be formatted.
+   Clones of an encrypted image are inherently encrypted using the same format
+   and secret.
+
+.. note::
+   Any data written to the image prior to its format may become unreadable,
+   though it may still occupy storage resources.
+
+.. note::
+   Images with the `journal feature`_ enabled cannot be formatted and encrypted
+   by RBD clients.
+
+Encryption Load
+=================
+
+Formatting an image is a necessary pre-requisite for enabling encryption.
+However, formatted images will still be treated as raw unencrypted images by
+all of the RBD APIs. In particular, an encrypted RBD image can be opened
+by the same APIs as any other image, and raw unencrypted data can be
+read / written. Such raw IOs may risk the integrity of the encryption format,
+for example by overriding encryption metadata located at the beginning of the
+image.
+
+In order to safely perform encrypted IO on the formatted image, an additional
+*encryption load* operation should be applied after opening the image. The
+encryption load operation requires supplying the encryption format and a secret
+for unlocking the encryption key. Following a successful encryption load
+operation, all IOs for the opened image will be encrypted / decrypted.
+For a cloned image, this includes IOs for ancestor images as well. The
+encryption key will be stored in-memory by the RBD client until the image is
+closed.
+
+.. note::
+   Once encryption has been loaded, no other encryption load / format
+   operations can be applied to the context of the opened image.
+
+.. note::
+   Once encryption has been loaded, API calls for retrieving the image size
+   using the opened image context will return the effective image size.
+
+.. note::
+   Encryption load can be automatically applied when mounting RBD images as
+   block devices via `rbd-nbd`_.
+
+Supported Formats
+=================
+
+LUKS
+~~~~~~~
+
+Both LUKS1 and LUKS2 are supported. The data layout is fully compliant with the
+LUKS specification. Thus, images formatted by RBD can be loaded using external
+LUKS-supporting tools such as dm-crypt or QEMU. Furthermore, existing LUKS
+data, created outside of RBD, can be imported (by copying the raw LUKS data
+into the image) and loaded by RBD encryption.
+
+.. note::
+   The LUKS formats are supported on Linux-based systems only.
+
+.. note::
+   Currently, only AES-128 and AES-256 encryption algorithms are supported.
+   Additionally, xts-plain64 is currently the only supported encryption mode.
+
+To use the LUKS format, start by formatting the image::
+
+    $ rbd encryption format {pool-name}/{image-name} {luks1|luks2} {passphrase-file} [–cipher-alg {aes-128 | aes-256}]
+
+The encryption format operation generates a LUKS header and writes it to the
+beginning of the image. The header is appended with a single keyslot holding a
+randomly-generated encryption key, and is protected by the passphrase read from
+`passphrase-file`.
+
+.. note::
+   If the content of `passphrase-file` ends with a newline character, it will
+   be stripped off.
+
+By default, AES-256 in xts-plain64 mode (which is the current recommended mode,
+and the usual default for other tools) will be used. The format operation
+allows selecting AES-128 as well. Adding / removing passphrases is currently
+not supported by RBD, but can be applied to the raw RBD data using compatible
+tools such as cryptsetup.
+
+The LUKS header size can vary (upto 136MiB in LUKS2), but is usually upto
+16MiB, depending on the version of `libcryptsetup` installed. For optimal
+performance, the encryption format will set the data offset to be aligned with
+the image object size. For example expect a minimum overhead of 8MiB if using
+an imageconfigured with an 8MiB object size.
+
+In LUKS1, sectors, which are the minimal encryption units, are fixed at 512
+bytes. LUKS2 supports larger sectors, and for better performance we set
+the default sector size to the maximum of 4KiB. Writes which are either smaller
+than a sector, or are not aligned to a sector start, will trigger a guarded
+read-modify-write chain on the client, with a considerable latency penalty.
+A batch of such unaligned writes can lead to IO races which will further
+deteriorate performance. Thus it is advisable to avoid using RBD encryption
+in cases where incoming writes cannot be guaranteed to be sector-aligned.
+
+To mount a LUKS-encrypted image run::
+
+    $ rbd -p {pool-name} device map -t nbd -o encryption-format={luks1|luks2},encryption-passphrase-file={passphrase-file}
+
+Note that for security reasons, both the encryption format and encryption load
+operations are CPU-intensive, and may take a few seconds to complete. For the
+encryption operations of actual image IO, assuming AES-NI is enabled,
+a relative small microseconds latency should be added, as well as a small
+increase in CPU utilization.
+
+.. _journal feature: ../rbd-mirroring/#enable-image-journaling-feature
+.. _Supported Formats: #supported-formats
+.. _rbd-nbd: ../../man/8/rbd-nbd
diff --git a/doc/rbd/rbd-exclusive-locks.rst b/doc/rbd/rbd-exclusive-locks.rst
new file mode 100644
index 000000000..03609c74f
--- /dev/null
+++ b/doc/rbd/rbd-exclusive-locks.rst
@@ -0,0 +1,86 @@
+.. _rbd-exclusive-locks:
+
+====================
+ RBD Exclusive Locks
+====================
+
+.. index:: Ceph Block Device; RBD exclusive locks; exclusive-lock
+
+Exclusive locks are mechanisms designed to prevent multiple processes from
+accessing the same Rados Block Device (RBD) in an uncoordinated fashion.
+Exclusive locks are used heavily in virtualization (where they prevent VMs from
+clobbering each other's writes) and in RBD mirroring (where they are a
+prerequisite for journaling).
+
+By default, exclusive locks are enabled on newly created images. This default
+can be overridden via the ``rbd_default_features`` configuration option or the
+``--image-feature`` option for ``rbd create``.
+
+.. note::
+   Many image features, including ``object-map`` and ``fast-diff``, depend upon
+   exclusive locking. Disabling the ``exclusive-lock`` feature will negatively
+   affect the performance of some operations.
+
+In order to ensure proper exclusive locking operations, any client using an RBD
+image whose ``exclusive-lock`` feature is enabled must have a CephX identity
+whose capabilities include ``profile rbd``.
+
+Exclusive locking is mostly transparent to the user.
+
+#. Whenever any ``librbd`` client process or kernel RBD client
+   starts using an RBD image on which exclusive locking has been
+   enabled, it obtains an exclusive lock on the image before the first
+   write.
+
+#. Whenever any such client process terminates gracefully, the process
+   relinquishes the lock automatically.
+
+#. This graceful termination enables another, subsequent, process to acquire
+   the lock and to write to the image.
+
+.. note::
+   It is possible for two or more concurrently running processes to open the
+   image and to read from it. The client acquires the exclusive lock only when
+   attempting to write to the image. To disable transparent lock transitions
+   between multiple clients, the client must acquire the lock by using the
+   ``RBD_LOCK_MODE_EXCLUSIVE`` flag.
+
+
+Blacklisting
+============
+
+Sometimes a client process (or, in case of a krbd client, a client node's
+kernel) that previously held an exclusive lock on an image does not terminate
+gracefully, but dies abruptly. This may be because the client process received
+a ``KILL`` or ``ABRT`` signal, or because the client node underwent a hard
+reboot or suffered a power failure. In cases like this, the exclusive lock is
+never gracefully released. This means that any new process that starts and
+attempts to use the device must break the previously held exclusive lock.
+
+However, a process (or kernel thread) may hang or merely lose network
+connectivity to the Ceph cluster for some amount of time. In that case,
+breaking the lock would be potentially catastrophic: the hung process or
+connectivity issue could resolve itself and the original process might then
+compete with one that started in the interim, thus accessing RBD data in an
+uncoordinated and destructive manner.
+
+In the event that a lock cannot be acquired in the standard graceful manner,
+the overtaking process not only breaks the lock but also blocklists the
+previous lock holder. This is negotiated between the new client process and the
+Ceph Monitor. 
+
+* Upon receiving the blocklist request, the monitor instructs the relevant OSDs
+  to no longer serve requests from the old client process;
+* after the associated OSD map update is complete, the new client can break the
+  previously held lock;
+* after the new client has acquired the lock, it can commence writing
+  to the image.
+
+Blocklisting is thus a form of storage-level resource `fencing`_.
+
+In order for blocklisting to work, the client must have the ``osd
+blocklist`` capability. This capability is included in the ``profile
+rbd`` capability profile, which should be set generally on all Ceph
+:ref:`client identities <user-management>` using RBD.
+
+.. _fencing: https://en.wikipedia.org/wiki/Fencing_(computing)
diff --git a/doc/rbd/rbd-integrations.rst b/doc/rbd/rbd-integrations.rst
new file mode 100644
index 000000000..50d788d53
--- /dev/null
+++ b/doc/rbd/rbd-integrations.rst
@@ -0,0 +1,15 @@
+=========================================
+ Ceph Block Device 3rd Party Integration
+=========================================
+
+.. toctree::
+   :maxdepth: 1
+
+   Kernel Modules <rbd-ko>
+   QEMU <qemu-rbd>
+   libvirt <libvirt>
+   Kubernetes <rbd-kubernetes>
+   OpenStack <rbd-openstack>
+   CloudStack <rbd-cloudstack>
+   LIO iSCSI Gateway <iscsi-overview>
+   Windows <rbd-windows>
diff --git a/doc/rbd/rbd-ko.rst b/doc/rbd/rbd-ko.rst
new file mode 100644
index 000000000..70c407839
--- /dev/null
+++ b/doc/rbd/rbd-ko.rst
@@ -0,0 +1,59 @@
+==========================
+ Kernel Module Operations
+==========================
+
+.. index:: Ceph Block Device; kernel module
+
+.. important:: To use kernel module operations, you must have a running Ceph cluster.
+
+Get a List of Images
+====================
+
+To mount a block device image, first return a list of the images. ::
+
+	rbd list
+
+Map a Block Device
+==================
+
+Use ``rbd`` to map an image name to a kernel module. You must specify the 
+image name, the pool name, and the user name. ``rbd`` will load RBD kernel
+module on your behalf if it's not already loaded. ::
+
+  sudo rbd device map {pool-name}/{image-name} --id {user-name}
+
+For example:: 
+
+  sudo rbd device map rbd/myimage --id admin
+ 
+If you use `cephx`_ authentication, you must also specify a secret.  It may come
+from a keyring or a file containing the secret. ::
+
+  sudo rbd device map rbd/myimage --id admin --keyring /path/to/keyring
+  sudo rbd device map rbd/myimage --id admin --keyfile /path/to/file
+
+
+Show Mapped Block Devices
+=========================
+
+To show block device images mapped to kernel modules with the ``rbd``,
+specify ``device list`` arguments. ::
+
+	rbd device list
+
+
+Unmapping a Block Device
+========================
+
+To unmap a block device image with the ``rbd`` command, specify the
+``device unmap`` arguments and the device name (i.e., by convention the
+same as the block device image name). :: 
+
+	sudo rbd device unmap /dev/rbd/{poolname}/{imagename}
+
+For example::
+
+	sudo rbd device unmap /dev/rbd/rbd/foo
+
+
+.. _cephx: ../../rados/operations/user-management/
diff --git a/doc/rbd/rbd-kubernetes.rst b/doc/rbd/rbd-kubernetes.rst
new file mode 100644
index 000000000..caaf77d64
--- /dev/null
+++ b/doc/rbd/rbd-kubernetes.rst
@@ -0,0 +1,314 @@
+==============================
+ Block Devices and Kubernetes
+==============================
+
+You may use Ceph Block Device images with Kubernetes v1.13 and later through
+`ceph-csi`_, which dynamically provisions RBD images to back Kubernetes
+`volumes`_ and maps these RBD images as block devices (optionally mounting
+a file system contained within the image) on worker nodes running
+`pods`_ that reference an RBD-backed volume. Ceph stripes block device images as
+objects across the cluster, which means that large Ceph Block Device images have
+better performance than a standalone server!
+
+To use Ceph Block Devices with Kubernetes v1.13 and higher, you must install
+and configure ``ceph-csi`` within your Kubernetes environment. The following
+diagram depicts the Kubernetes/Ceph technology stack.
+
+.. ditaa::
+            +---------------------------------------------------+
+            |                   Kubernetes                      |
+            +---------------------------------------------------+
+            |                    ceph--csi                      |
+            +------------------------+--------------------------+
+                                     |
+                                     | configures
+                                     v
+            +------------------------+ +------------------------+
+            |                        | |        rbd--nbd        |
+            |     Kernel Modules     | +------------------------+
+            |                        | |         librbd         |
+            +------------------------+-+------------------------+
+            |                   RADOS Protocol                  |
+            +------------------------+-+------------------------+
+            |          OSDs          | |        Monitors        |
+            +------------------------+ +------------------------+
+
+
+.. important::
+   ``ceph-csi`` uses the RBD kernel modules by default which may not support all
+   Ceph `CRUSH tunables`_ or `RBD image features`_.
+
+Create a Pool
+=============
+
+By default, Ceph block devices use the ``rbd`` pool. Create a pool for
+Kubernetes volume storage. Ensure your Ceph cluster is running, then create
+the pool. ::
+
+        $ ceph osd pool create kubernetes
+
+See `Create a Pool`_ for details on specifying the number of placement groups
+for your pools, and `Placement Groups`_ for details on the number of placement
+groups you should set for your pools.
+
+A newly created pool must be initialized prior to use. Use the ``rbd`` tool
+to initialize the pool::
+
+        $ rbd pool init kubernetes
+
+Configure ceph-csi
+==================
+
+Setup Ceph Client Authentication
+--------------------------------
+
+Create a new user for Kubernetes and `ceph-csi`. Execute the following and
+record the generated key::
+
+    $ ceph auth get-or-create client.kubernetes mon 'profile rbd' osd 'profile rbd pool=kubernetes' mgr 'profile rbd pool=kubernetes'
+    [client.kubernetes]
+        key = AQD9o0Fd6hQRChAAt7fMaSZXduT3NWEqylNpmg==
+
+Generate `ceph-csi` `ConfigMap`
+-------------------------------
+
+The `ceph-csi` requires a `ConfigMap` object stored in Kubernetes to define the
+the Ceph monitor addresses for the Ceph cluster. Collect both the Ceph cluster
+unique `fsid` and the monitor addresses::
+
+        $ ceph mon dump
+        <...>
+        fsid b9127830-b0cc-4e34-aa47-9d1a2e9949a8
+        <...>
+        0: [v2:192.168.1.1:3300/0,v1:192.168.1.1:6789/0] mon.a
+        1: [v2:192.168.1.2:3300/0,v1:192.168.1.2:6789/0] mon.b
+        2: [v2:192.168.1.3:3300/0,v1:192.168.1.3:6789/0] mon.c
+
+.. note::
+   ``ceph-csi`` currently only supports the `legacy V1 protocol`_.
+
+Generate a `csi-config-map.yaml` file similar to the example below, substituting
+the `fsid` for "clusterID", and the monitor addresses for "monitors"::
+
+        $ cat <<EOF > csi-config-map.yaml
+        ---
+        apiVersion: v1
+        kind: ConfigMap
+        data:
+          config.json: |-
+            [
+              {
+                "clusterID": "b9127830-b0cc-4e34-aa47-9d1a2e9949a8",
+                "monitors": [
+                  "192.168.1.1:6789",
+                  "192.168.1.2:6789",
+                  "192.168.1.3:6789"
+                ]
+              }
+            ]
+        metadata:
+          name: ceph-csi-config
+        EOF
+
+Once generated, store the new `ConfigMap` object in Kubernetes::
+
+        $ kubectl apply -f csi-config-map.yaml
+
+Generate `ceph-csi` cephx `Secret`
+----------------------------------
+
+`ceph-csi` requires the cephx credentials for communicating with the Ceph
+cluster. Generate a `csi-rbd-secret.yaml` file similar to the example below,
+using the newly created Kubernetes user id and cephx key::
+
+        $ cat <<EOF > csi-rbd-secret.yaml
+        ---
+        apiVersion: v1
+        kind: Secret
+        metadata:
+          name: csi-rbd-secret
+          namespace: default
+        stringData:
+          userID: kubernetes
+          userKey: AQD9o0Fd6hQRChAAt7fMaSZXduT3NWEqylNpmg==
+        EOF
+
+Once generated, store the new `Secret` object in Kubernetes::
+
+        $ kubectl apply -f csi-rbd-secret.yaml
+
+Configure `ceph-csi` Plugins
+----------------------------
+
+Create the required `ServiceAccount` and RBAC `ClusterRole`/`ClusterRoleBinding`
+Kubernetes objects. These objects do not necessarily need to be customized for
+your Kubernetes environment and therefore can be used as-is from the `ceph-csi`
+deployment YAMLs::
+
+        $ kubectl apply -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml
+        $ kubectl apply -f https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-nodeplugin-rbac.yaml
+
+Finally, create the `ceph-csi` provisioner and node plugins. With the
+possible exception of the `ceph-csi` container release version, these objects do
+not necessarily need to be customized for your Kubernetes environment and
+therefore can be used as-is from the `ceph-csi` deployment YAMLs::
+
+        $ wget https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml
+        $ kubectl apply -f csi-rbdplugin-provisioner.yaml
+        $ wget https://raw.githubusercontent.com/ceph/ceph-csi/master/deploy/rbd/kubernetes/csi-rbdplugin.yaml
+        $ kubectl apply -f csi-rbdplugin.yaml
+
+.. important::
+   The provisioner and node plugin YAMLs will, by default, pull the development
+   release of the `ceph-csi` container (quay.io/cephcsi/cephcsi:canary).
+   The YAMLs should be updated to use a release version container for
+   production workloads.
+
+Using Ceph Block Devices
+========================
+
+Create a `StorageClass`
+-----------------------
+
+The Kubernetes `StorageClass` defines a class of storage. Multiple `StorageClass`
+objects can be created to map to different quality-of-service levels (i.e. NVMe
+vs HDD-based pools) and features.
+
+For example, to create a `ceph-csi` `StorageClass` that maps to the `kubernetes`
+pool created above, the following YAML file can be used after ensuring that the
+"clusterID" property matches your Ceph cluster's `fsid`::
+
+        $ cat <<EOF > csi-rbd-sc.yaml
+        ---
+        apiVersion: storage.k8s.io/v1
+        kind: StorageClass
+        metadata:
+           name: csi-rbd-sc
+        provisioner: rbd.csi.ceph.com
+        parameters:
+           clusterID: b9127830-b0cc-4e34-aa47-9d1a2e9949a8
+           pool: kubernetes
+           csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret
+           csi.storage.k8s.io/provisioner-secret-namespace: default
+           csi.storage.k8s.io/node-stage-secret-name: csi-rbd-secret
+           csi.storage.k8s.io/node-stage-secret-namespace: default
+        reclaimPolicy: Delete
+        mountOptions:
+           - discard
+        EOF
+        $ kubectl apply -f csi-rbd-sc.yaml
+
+Create a `PersistentVolumeClaim`
+--------------------------------
+
+A `PersistentVolumeClaim` is a request for abstract storage resources by a user.
+The `PersistentVolumeClaim` would then be associated to a `Pod` resource to
+provision a `PersistentVolume`, which would be backed by a Ceph block image.
+An optional `volumeMode` can be included to select between a mounted file system
+(default) or raw block device-based volume.
+
+Using `ceph-csi`, specifying `Filesystem` for `volumeMode` can support both
+`ReadWriteOnce` and `ReadOnlyMany` `accessMode` claims, and specifying `Block`
+for `volumeMode` can support `ReadWriteOnce`, `ReadWriteMany`, and
+`ReadOnlyMany` `accessMode` claims.
+
+For example, to create a block-based `PersistentVolumeClaim` that utilizes
+the `ceph-csi`-based `StorageClass` created above, the following YAML can be
+used to request raw block storage from the `csi-rbd-sc` `StorageClass`::
+
+        $ cat <<EOF > raw-block-pvc.yaml
+        ---
+        apiVersion: v1
+        kind: PersistentVolumeClaim
+        metadata:
+          name: raw-block-pvc
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          volumeMode: Block
+          resources:
+            requests:
+              storage: 1Gi
+          storageClassName: csi-rbd-sc
+        EOF
+        $ kubectl apply -f raw-block-pvc.yaml
+
+The following demonstrates and example of binding the above
+`PersistentVolumeClaim` to a `Pod` resource as a raw block device::
+
+        $ cat <<EOF > raw-block-pod.yaml
+        ---
+        apiVersion: v1
+        kind: Pod
+        metadata:
+          name: pod-with-raw-block-volume
+        spec:
+          containers:
+            - name: fc-container
+              image: fedora:26
+              command: ["/bin/sh", "-c"]
+              args: ["tail -f /dev/null"]
+              volumeDevices:
+                - name: data
+                  devicePath: /dev/xvda
+          volumes:
+            - name: data
+              persistentVolumeClaim:
+                claimName: raw-block-pvc
+        EOF
+        $ kubectl apply -f raw-block-pod.yaml
+
+To create a file-system-based `PersistentVolumeClaim` that utilizes the
+`ceph-csi`-based `StorageClass` created above, the following YAML can be used to
+request a mounted file system (backed by an RBD image) from the `csi-rbd-sc`
+`StorageClass`::
+
+        $ cat <<EOF > pvc.yaml
+        ---
+        apiVersion: v1
+        kind: PersistentVolumeClaim
+        metadata:
+          name: rbd-pvc
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          volumeMode: Filesystem
+          resources:
+            requests:
+              storage: 1Gi
+          storageClassName: csi-rbd-sc
+        EOF
+        $ kubectl apply -f pvc.yaml
+
+The following demonstrates and example of binding the above
+`PersistentVolumeClaim` to a `Pod` resource as a mounted file system::
+
+        $ cat <<EOF > pod.yaml
+        ---
+        apiVersion: v1
+        kind: Pod
+        metadata:
+          name: csi-rbd-demo-pod
+        spec:
+          containers:
+            - name: web-server
+              image: nginx
+              volumeMounts:
+                - name: mypvc
+                  mountPath: /var/lib/www/html
+          volumes:
+            - name: mypvc
+              persistentVolumeClaim:
+                claimName: rbd-pvc
+                readOnly: false
+        EOF
+        $ kubectl apply -f pod.yaml
+
+.. _ceph-csi: https://github.com/ceph/ceph-csi/
+.. _volumes: https://kubernetes.io/docs/concepts/storage/volumes/
+.. _pods: https://kubernetes.io/docs/concepts/workloads/pods/pod-overview/
+.. _Create a Pool: ../../rados/operations/pools#createpool
+.. _Placement Groups: ../../rados/operations/placement-groups
+.. _CRUSH tunables: ../../rados/operations/crush-map/#tunables
+.. _RBD image features: ../rbd-config-ref/#image-features
+.. _legacy V1 protocol: ../../rados/configuration/msgr2/#address-formats
diff --git a/doc/rbd/rbd-live-migration.rst b/doc/rbd/rbd-live-migration.rst
new file mode 100644
index 000000000..c3e09193d
--- /dev/null
+++ b/doc/rbd/rbd-live-migration.rst
@@ -0,0 +1,367 @@
+======================
+ Image Live-Migration
+======================
+
+.. index:: Ceph Block Device; live-migration
+
+RBD images can be live-migrated between different pools within the same cluster;
+between different image formats and layouts; or from external data sources.
+When started, the source will be deep-copied to the destination image, pulling
+all snapshot history while preserving the sparse allocation of data where
+possible.
+
+By default, when live-migrating RBD images within the same Ceph cluster, the
+source image will be marked read-only and all clients will instead redirect
+IOs to the new target image. In addition, this mode can optionally preserve the
+link to the source image's parent to preserve sparseness, or it can flatten the
+image during the migration to remove the dependency on the source image's
+parent.
+
+The live-migration process can also be used in an import-only mode where the
+source image remains unmodified and the target image can be linked to an
+external data source such as a backing file, HTTP(s) file, or S3 object.
+
+The live-migration copy process can safely run in the background while the new
+target image is in use. There is currently a requirement to temporarily stop
+using the source image before preparing a migration when not using the
+import-only mode of operation. This helps to ensure that the client using the
+image is updated to point to the new target image.
+
+.. note::
+   Image live-migration requires the Ceph Nautilus release or later. Support for
+   external data sources requires the Ceph Pacific release of later. The
+   ``krbd`` kernel module does not support live-migration at this time.
+
+
+.. ditaa::
+
+           +-------------+               +-------------+
+           | {s} c999    |               | {s}         |
+           |  Live       | Target refers |  Live       |
+           |  migration  |<-------------*|  migration  |
+           |  source     |  to Source    |  target     |
+           |             |               |             |
+           | (read only) |               | (writable)  |
+           +-------------+               +-------------+
+
+               Source                        Target
+
+The live-migration process is comprised of three steps:
+
+#. **Prepare Migration:** The initial step creates the new target image and
+   links the target image to the source. When not configured in the import-only
+   mode, the source image will also be linked to the target image and marked
+   read-only.
+
+   Similar to `layered images`_, attempts to read uninitialized data extents
+   within the target image will internally redirect the read to the source
+   image, and writes to uninitialized extents within the target will internally
+   deep-copy the overlapping source image block to the target image.
+
+
+#. **Execute Migration:** This is a background operation that deep-copies all
+   initialized blocks from the source image to the target. This step can be
+   run while clients are actively using the new target image.
+
+
+#. **Finish Migration:** Once the background migration process has completed,
+   the migration can be committed or aborted. Committing the migration will
+   remove the cross-links between the source and target images, and will
+   remove the source image if not configured in the import-only mode. Aborting
+   the migration will remove the cross-links, and will remove the target image.
+
+Prepare Migration
+=================
+
+The default live-migration process for images within the same Ceph cluster is
+initiated by running the `rbd migration prepare` command, providing the source
+and target images::
+
+    $ rbd migration prepare migration_source [migration_target]
+
+The `rbd migration prepare` command accepts all the same layout optionals as the
+`rbd create` command, which allows changes to the immutable image on-disk
+layout. The `migration_target` can be skipped if the goal is only to change the
+on-disk layout, keeping the original image name.
+
+All clients using the source image must be stopped prior to preparing a
+live-migration. The prepare step will fail if it finds any running clients with
+the image open in read/write mode. Once the prepare step is complete, the
+clients can be restarted using the new target image name. Attempting to restart
+the clients using the source image name will result in failure.
+
+The `rbd status` command will show the current state of the live-migration::
+
+    $ rbd status migration_target
+    Watchers: none
+    Migration:
+        	source: rbd/migration_source (5e2cba2f62e)
+        	destination: rbd/migration_target (5e2ed95ed806)
+        	state: prepared
+
+Note that the source image will be moved to the RBD trash to avoid mistaken
+usage during the migration process::
+
+    $ rbd info migration_source
+    rbd: error opening image migration_source: (2) No such file or directory
+    $ rbd trash ls --all
+    5e2cba2f62e migration_source
+
+
+Prepare Import-Only Migration
+=============================
+
+The import-only live-migration process is initiated by running the same
+`rbd migration prepare` command, but adding the `--import-only` optional
+and providing a JSON-encoded ``source-spec`` to describe how to access
+the source image data. This ``source-spec`` can either be passed
+directly via the `--source-spec` optional, or via a file or STDIN via the
+`--source-spec-path` optional::
+
+        $ rbd migration prepare --import-only --source-spec "<JSON>" migration_target
+
+The `rbd migration prepare` command accepts all the same layout optionals as the
+`rbd create` command.
+
+The `rbd status` command will show the current state of the live-migration::
+
+        $ rbd status migration_target
+        Watchers: none
+        Migration:
+	        source: {"stream":{"file_path":"/mnt/image.raw","type":"file"},"type":"raw"}
+        	destination: rbd/migration_target (ac69113dc1d7)
+	        state: prepared
+
+The general format for the ``source-spec`` JSON is as follows::
+
+        {
+            "type": "<format-type>",
+            <format unique parameters>
+            "stream": {
+                "type": "<stream-type>",
+                <stream unique parameters>
+            }
+        }
+
+The following formats are currently supported: ``native``, ``qcow``, and
+``raw``. The following streams are currently supported: ``file``, ``http``, and
+``s3``.
+
+Formats
+~~~~~~~
+
+The ``native`` format can be used to describe a native RBD image within a
+Ceph cluster as the source image. Its ``source-spec`` JSON is encoded
+as follows::
+
+        {
+            "type": "native",
+            "pool_name": "<pool-name>",
+            ["pool_id": <pool-id>,] (optional alternative to "pool_name")
+            ["pool_namespace": "<pool-namespace",] (optional)
+            "image_name": "<image-name>",
+            ["image_id": "<image-id>",] (optional if image in trash)
+            "snap_name": "<snap-name>",
+            ["snap_id": "<snap-id>",] (optional alternative to "snap_name")
+        }
+
+Note that the ``native`` format does not include the ``stream`` object since
+it utilizes native Ceph operations. For example, to import from the image
+``rbd/ns1/image1@snap1``, the ``source-spec`` could be encoded as::
+
+        {
+            "type": "native",
+            "pool_name": "rbd",
+            "pool_namespace": "ns1",
+            "image_name": "image1",
+            "snap_name": "snap1"
+        }
+
+The ``qcow`` format can be used to describe a QCOW (QEMU copy-on-write) block
+device. Both the QCOW (v1) and QCOW2 formats are currently supported with the
+exception of advanced features such as compression, encryption, backing
+files, and external data files. Support for these missing features may be added
+in a future release. The ``qcow`` format data can be linked to any supported
+stream source described below. For example, its base ``source-spec`` JSON is
+encoded as follows::
+
+        {
+            "type": "qcow",
+            "stream": {
+                <stream unique parameters>
+            }
+        }
+
+The ``raw`` format can be used to describe a thick-provisioned, raw block device
+export (i.e. `rbd export --export-format 1 <snap-spec>`). The ``raw`` format
+data can be linked to any supported stream source described below. For example,
+its base ``source-spec`` JSON is encoded as follows::
+
+        {
+            "type": "raw",
+            "stream": {
+                <stream unique parameters for HEAD, non-snapshot revision>
+            },
+            "snapshots": [
+                {
+                    "type": "raw",
+                    "name": "<snapshot-name>",
+                    "stream": {
+                        <stream unique parameters for snapshot>
+                    }
+                },
+            ] (optional oldest to newest ordering of snapshots)
+        }
+
+The inclusion of the ``snapshots`` array is optional and currently only supports
+thick-provisioned ``raw`` snapshot exports.
+
+Additional formats such as RBD export-format v2 and RBD export-diff
+snapshots will be added in a future release.
+
+Streams
+~~~~~~~
+
+The ``file`` stream can be used to import from a locally accessible POSIX file
+source. Its ``source-spec`` JSON is encoded as follows::
+
+        {
+            <format unique parameters>
+            "stream": {
+                "type": "file",
+                "file_path": "<file-path>"
+            }
+        }
+
+For example, to import a raw-format image from a file located at
+"/mnt/image.raw", its ``source-spec`` JSON is encoded as follows::
+
+        {
+            "type": "raw",
+            "stream": {
+                "type": "file",
+                "file_path": "/mnt/image.raw"
+            }
+        }
+
+The ``http`` stream can be used to import from a remote HTTP or HTTPS web
+server. Its ``source-spec`` JSON is encoded as follows::
+
+        {
+            <format unique parameters>
+            "stream": {
+                "type": "http",
+                "url": "<url-path>"
+            }
+        }
+
+For example, to import a raw-format image from a file located at
+``http://download.ceph.com/image.raw``, its ``source-spec`` JSON is encoded
+as follows::
+
+        {
+            "type": "raw",
+            "stream": {
+                "type": "http",
+                "url": "http://download.ceph.com/image.raw"
+            }
+        }
+
+The ``s3`` stream can be used to import from a remote S3 bucket. Its
+``source-spec`` JSON is encoded as follows::
+
+        {
+            <format unique parameters>
+            "stream": {
+                "type": "s3",
+                "url": "<url-path>",
+                "access_key": "<access-key>",
+                "secret_key": "<secret-key>"
+            }
+        }
+
+For example, to import a raw-format image from a file located at
+`http://s3.ceph.com/bucket/image.raw`, its ``source-spec`` JSON is encoded
+as follows::
+
+        {
+            "type": "raw",
+            "stream": {
+                "type": "s3",
+                "url": "http://s3.ceph.com/bucket/image.raw",
+                "access_key": "NX5QOQKC6BH2IDN8HC7A",
+                "secret_key": "LnEsqNNqZIpkzauboDcLXLcYaWwLQ3Kop0zAnKIn"
+            }
+        }
+
+.. note::
+  The ``access_key`` and ``secret_key`` parameters support storing the keys in
+  the MON config-key store by prefixing the key values with ``config://``
+  followed by the path in the MON config-key store to the value. Values can be
+  stored in the config-key store via ``ceph config-key set <key-path> <value>``
+  (e.g. ``ceph config-key set rbd/s3/access_key NX5QOQKC6BH2IDN8HC7A``).
+
+Execute Migration
+=================
+
+After preparing the live-migration, the image blocks from the source image
+must be copied to the target image. This is accomplished by running the
+`rbd migration execute` command::
+
+    $ rbd migration execute migration_target
+    Image migration: 100% complete...done.
+
+The `rbd status` command will also provide feedback on the progress of the
+migration block deep-copy process::
+
+    $ rbd status migration_target
+    Watchers:
+    	watcher=1.2.3.4:0/3695551461 client.123 cookie=123
+    Migration:
+        	source: rbd/migration_source (5e2cba2f62e)
+        	destination: rbd/migration_target (5e2ed95ed806)
+        	state: executing (32% complete)
+
+
+Commit Migration
+================
+
+Once the live-migration has completed deep-copying all data blocks from the
+source image to the target, the migration can be committed::
+
+    $ rbd status migration_target
+    Watchers: none
+    Migration:
+        	source: rbd/migration_source (5e2cba2f62e)
+        	destination: rbd/migration_target (5e2ed95ed806)
+        	state: executed
+    $ rbd migration commit migration_target
+    Commit image migration: 100% complete...done.
+
+If the `migration_source` image is a parent of one or more clones, the `--force`
+option will need to be specified after ensuring all descendent clone images are
+not in use.
+
+Committing the live-migration will remove the cross-links between the source
+and target images, and will remove the source image::
+
+    $ rbd trash list --all
+
+
+Abort Migration
+===============
+
+If you wish to revert the prepare or execute step, run the `rbd migration abort`
+command to revert the migration process::
+
+        $ rbd migration abort migration_target
+        Abort image migration: 100% complete...done.
+
+Aborting the migration will result in the target image being deleted and access
+to the original source image being restored::
+
+        $ rbd ls
+        migration_source
+
+
+.. _layered images: ../rbd-snapshot/#layering
diff --git a/doc/rbd/rbd-mirroring.rst b/doc/rbd/rbd-mirroring.rst
new file mode 100644
index 000000000..d5d1191d6
--- /dev/null
+++ b/doc/rbd/rbd-mirroring.rst
@@ -0,0 +1,529 @@
+===============
+ RBD Mirroring
+===============
+
+.. index:: Ceph Block Device; mirroring
+
+RBD images can be asynchronously mirrored between two Ceph clusters. This
+capability is available in two modes:
+
+* **Journal-based**: This mode uses the RBD journaling image feature to ensure
+  point-in-time, crash-consistent replication between clusters. Every write to
+  the RBD image is first recorded to the associated journal before modifying the
+  actual image. The remote cluster will read from this associated journal and
+  replay the updates to its local copy of the image. Since each write to the
+  RBD image will result in two writes to the Ceph cluster, expect write
+  latencies to nearly double while using the RBD journaling image feature.
+
+* **Snapshot-based**: This mode uses periodically scheduled or manually
+  created RBD image mirror-snapshots to replicate crash-consistent RBD images
+  between clusters. The remote cluster will determine any data or metadata
+  updates between two mirror-snapshots and copy the deltas to its local copy of
+  the image. With the help of the RBD ``fast-diff`` image feature, updated data
+  blocks can be quickly determined without the need to scan the full RBD image.
+  Since this mode is not as fine-grained as journaling, the complete delta 
+  between two snapshots will need to be synced prior to use during a failover
+  scenario. Any partially applied set of deltas will be rolled back at moment
+  of failover.
+
+.. note:: journal-based mirroring requires the Ceph Jewel release or later;
+   snapshot-based mirroring requires the Ceph Octopus release or later.
+
+Mirroring is configured on a per-pool basis within peer clusters and can be
+configured on a specific subset of images within the pool.  You can also mirror
+all images within a given pool when using journal-based
+mirroring. Mirroring is configured using the ``rbd`` command. The
+``rbd-mirror`` daemon is responsible for pulling image updates from the remote
+peer cluster and applying them to the image within the local cluster.
+
+Depending on the desired needs for replication, RBD mirroring can be configured
+for either one- or two-way replication:
+
+* **One-way Replication**: When data is only mirrored from a primary cluster to
+  a secondary cluster, the ``rbd-mirror`` daemon runs only on the secondary
+  cluster.
+
+* **Two-way Replication**: When data is mirrored from primary images on one
+  cluster to non-primary images on another cluster (and vice-versa), the
+  ``rbd-mirror`` daemon runs on both clusters.
+
+.. important:: Each instance of the ``rbd-mirror`` daemon must be able to
+   connect to both the local and remote Ceph clusters simultaneously (i.e.
+   all monitor and OSD hosts). Additionally, the network must have sufficient
+   bandwidth between the two data centers to handle mirroring workload.
+
+Pool Configuration
+==================
+
+The following procedures demonstrate how to perform the basic administrative
+tasks to configure mirroring using the ``rbd`` command. Mirroring is
+configured on a per-pool basis.
+
+These pool configuration steps should be performed on both peer clusters. These
+procedures assume that both clusters, named "site-a" and "site-b", are accessible
+from a single host for clarity.
+
+See the `rbd`_ manpage for additional details of how to connect to different
+Ceph clusters.
+
+.. note:: The cluster name in the following examples corresponds to a Ceph
+   configuration file of the same name (e.g. /etc/ceph/site-b.conf).  See the
+   `ceph-conf`_ documentation for how to configure multiple clusters.  Note
+   that ``rbd-mirror`` does **not** require the source and destination clusters
+   to have unique internal names; both can and should call themselves ``ceph``.
+   The config `files` that ``rbd-mirror`` needs for local and remote clusters
+   can be named arbitrarily, and containerizing the daemon is one strategy
+   for maintaining them outside of ``/etc/ceph`` to avoid confusion.
+
+Enable Mirroring
+----------------
+
+To enable mirroring on a pool with ``rbd``, issue the ``mirror pool enable``
+subcommand with the pool name, and the mirroring mode::
+
+        rbd mirror pool enable {pool-name} {mode}
+
+The mirroring mode can either be ``image`` or ``pool``:
+
+* **image**: When configured in ``image`` mode, mirroring must
+  `explicitly enabled`_ on each image.
+* **pool** (default):  When configured in ``pool`` mode, all images in the pool
+  with the journaling feature enabled are mirrored.
+
+For example::
+
+        $ rbd --cluster site-a mirror pool enable image-pool image
+        $ rbd --cluster site-b mirror pool enable image-pool image
+
+Disable Mirroring
+-----------------
+
+To disable mirroring on a pool with ``rbd``, specify the ``mirror pool disable``
+command and the pool name::
+
+        rbd mirror pool disable {pool-name}
+
+When mirroring is disabled on a pool in this way, mirroring will also be
+disabled on any images (within the pool) for which mirroring was enabled
+explicitly.
+
+For example::
+
+        $ rbd --cluster site-a mirror pool disable image-pool
+        $ rbd --cluster site-b mirror pool disable image-pool
+
+Bootstrap Peers
+---------------
+
+In order for the ``rbd-mirror`` daemon to discover its peer cluster, the peer
+must be registered and a user account must be created.
+This process can be automated with ``rbd`` and the
+``mirror pool peer bootstrap create`` and ``mirror pool peer bootstrap import``
+commands.
+
+To manually create a new bootstrap token with ``rbd``, issue the
+``mirror pool peer bootstrap create`` subcommand, a pool name, and an
+optional friendly site name to describe the local cluster::
+
+        rbd mirror pool peer bootstrap create [--site-name {local-site-name}] {pool-name}
+
+The output of ``mirror pool peer bootstrap create`` will be a token that should
+be provided to the ``mirror pool peer bootstrap import`` command. For example,
+on site-a::
+
+        $ rbd --cluster site-a mirror pool peer bootstrap create --site-name site-a image-pool
+        eyJmc2lkIjoiOWY1MjgyZGItYjg5OS00NTk2LTgwOTgtMzIwYzFmYzM5NmYzIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFBUnczOWQwdkhvQmhBQVlMM1I4RmR5dHNJQU50bkFTZ0lOTVE9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMS4zOjY4MjAsdjE6MTkyLjE2OC4xLjM6NjgyMV0ifQ==
+
+To manually import the bootstrap token created by another cluster with ``rbd``,
+specify the ``mirror pool peer bootstrap import`` command, the pool name, a file
+path to the created token (or '-' to read from standard input), along with an
+optional friendly site name to describe the local cluster and a mirroring
+direction (defaults to rx-tx for bidirectional mirroring, but can also be set
+to rx-only for unidirectional mirroring)::
+
+        rbd mirror pool peer bootstrap import [--site-name {local-site-name}] [--direction {rx-only or rx-tx}] {pool-name} {token-path}
+
+For example, on site-b::
+
+        $ cat <<EOF > token
+        eyJmc2lkIjoiOWY1MjgyZGItYjg5OS00NTk2LTgwOTgtMzIwYzFmYzM5NmYzIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFBUnczOWQwdkhvQmhBQVlMM1I4RmR5dHNJQU50bkFTZ0lOTVE9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMS4zOjY4MjAsdjE6MTkyLjE2OC4xLjM6NjgyMV0ifQ==
+        EOF
+        $ rbd --cluster site-b mirror pool peer bootstrap import --site-name site-b image-pool token
+
+Add Cluster Peer Manually
+-------------------------
+
+Cluster peers can be specified manually if desired or if the above bootstrap
+commands are not available with the currently installed Ceph release.
+
+The remote ``rbd-mirror`` daemon will need access to the local cluster to
+perform mirroring. A new local Ceph user should be created for the remote
+daemon to use. To `create a Ceph user`_, with ``ceph`` specify the
+``auth get-or-create`` command, user name, monitor caps, and OSD caps::
+
+        ceph auth get-or-create client.rbd-mirror-peer mon 'profile rbd' osd 'profile rbd'
+
+The resulting keyring should be copied to the other cluster's ``rbd-mirror``
+daemon hosts if not using the Ceph monitor ``config-key`` store described below.
+
+To manually add a mirroring peer Ceph cluster with ``rbd``, specify the
+``mirror pool peer add`` command, the pool name, and a cluster specification::
+
+        rbd mirror pool peer add {pool-name} {client-name}@{cluster-name}
+
+For example::
+
+        $ rbd --cluster site-a mirror pool peer add image-pool client.rbd-mirror-peer@site-b
+        $ rbd --cluster site-b mirror pool peer add image-pool client.rbd-mirror-peer@site-a
+
+By default, the ``rbd-mirror`` daemon needs to have access to a Ceph
+configuration file located at ``/etc/ceph/{cluster-name}.conf`` that provides
+the addresses of the peer cluster's monitors, in addition to a keyring for
+``{client-name}`` located in the default or configured keyring search paths
+(e.g. ``/etc/ceph/{cluster-name}.{client-name}.keyring``).
+
+Alternatively, the peer cluster's monitor and/or client key can be securely
+stored within the local Ceph monitor ``config-key`` store. To specify the
+peer cluster connection attributes when adding a mirroring peer, use the
+``--remote-mon-host`` and ``--remote-key-file`` optionals. For example::
+
+        $ cat <<EOF > remote-key-file
+        AQAeuZdbMMoBChAAcj++/XUxNOLFaWdtTREEsw==
+        EOF
+        $ rbd --cluster site-a mirror pool peer add image-pool client.rbd-mirror-peer@site-b --remote-mon-host 192.168.1.1,192.168.1.2 --remote-key-file remote-key-file
+        $ rbd --cluster site-a mirror pool info image-pool --all
+        Mode: pool
+        Peers: 
+          UUID                                 NAME   CLIENT                 MON_HOST                KEY                                      
+          587b08db-3d33-4f32-8af8-421e77abb081 site-b client.rbd-mirror-peer 192.168.1.1,192.168.1.2 AQAeuZdbMMoBChAAcj++/XUxNOLFaWdtTREEsw== 
+
+Remove Cluster Peer
+-------------------
+
+To remove a mirroring peer Ceph cluster with ``rbd``, specify the
+``mirror pool peer remove`` command, the pool name, and the peer UUID
+(available from the ``rbd mirror pool info`` command)::
+
+        rbd mirror pool peer remove {pool-name} {peer-uuid}
+
+For example::
+
+        $ rbd --cluster site-a mirror pool peer remove image-pool 55672766-c02b-4729-8567-f13a66893445
+        $ rbd --cluster site-b mirror pool peer remove image-pool 60c0e299-b38f-4234-91f6-eed0a367be08
+
+Data Pools
+----------
+
+When creating images in the destination cluster, ``rbd-mirror`` selects a data
+pool as follows:
+
+#. If the destination cluster has a default data pool configured (with the
+   ``rbd_default_data_pool`` configuration option), it will be used.
+#. Otherwise, if the source image uses a separate data pool, and a pool with the
+   same name exists on the destination cluster, that pool will be used.
+#. If neither of the above is true, no data pool will be set.
+
+Image Configuration
+===================
+
+Unlike pool configuration, image configuration only needs to be performed
+against a single mirroring peer Ceph cluster.
+
+Mirrored RBD images are designated as either primary or non-primary. This is a
+property of the image and not the pool. Images that are designated as
+non-primary cannot be modified.
+
+Images are automatically promoted to primary when mirroring is first enabled on
+an image (either implicitly if the pool mirror mode was ``pool`` and the image
+has the journaling image feature enabled, or `explicitly enabled`_ by the
+``rbd`` command if the pool mirror mode was ``image``).
+
+Enable Image Mirroring
+----------------------
+
+If mirroring is configured in ``image`` mode for the image's pool, then it
+is necessary to explicitly enable mirroring for each image within the pool.
+To enable mirroring for a specific image with ``rbd``, specify the
+``mirror image enable`` command along with the pool, image name, and mode::
+
+        rbd mirror image enable {pool-name}/{image-name} {mode}
+
+The mirror image mode can either be ``journal`` or ``snapshot``:
+
+* **journal** (default): When configured in ``journal`` mode, mirroring will
+  utilize the RBD journaling image feature to replicate the image contents. If
+  the RBD journaling image feature is not yet enabled on the image, it will be
+  automatically enabled.
+
+* **snapshot**:  When configured in ``snapshot`` mode, mirroring will utilize
+  RBD image mirror-snapshots to replicate the image contents. Once enabled, an
+  initial mirror-snapshot will automatically be created. Additional RBD image
+  `mirror-snapshots`_ can be created by the ``rbd`` command.
+
+For example::
+
+        $ rbd --cluster site-a mirror image enable image-pool/image-1 snapshot
+        $ rbd --cluster site-a mirror image enable image-pool/image-2 journal
+
+Enable Image Journaling Feature
+-------------------------------
+
+RBD journal-based mirroring uses the RBD image journaling feature to ensure that
+the replicated image always remains crash-consistent. When using the ``image``
+mirroring mode, the journaling feature will be automatically enabled when
+mirroring is enabled on the image. When using the ``pool`` mirroring mode,
+before an image can be mirrored to a peer cluster, the RBD image journaling
+feature must be enabled. The feature can be enabled at image creation time by
+providing the ``--image-feature exclusive-lock,journaling`` option to the
+``rbd`` command.
+
+Alternatively, the journaling feature can be dynamically enabled on
+pre-existing RBD images. To enable journaling with ``rbd``, specify
+the ``feature enable`` command, the pool and image name, and the feature name::
+
+        rbd feature enable {pool-name}/{image-name} {feature-name}
+
+For example::
+
+        $ rbd --cluster site-a feature enable image-pool/image-1 journaling
+
+.. note:: The journaling feature is dependent on the exclusive-lock feature. If
+   the exclusive-lock feature is not already enabled, it should be enabled prior
+   to enabling the journaling feature.
+
+.. tip:: You can enable journaling on all new images by default by adding
+   ``rbd default features = 125`` to your Ceph configuration file.
+
+.. tip:: ``rbd-mirror`` tunables are set by default to values suitable for
+   mirroring an entire pool.  When using ``rbd-mirror`` to migrate single
+   volumes been clusters you may achieve substantial performance gains
+   by setting ``rbd_mirror_journal_max_fetch_bytes=33554432`` and
+   ``rbd_journal_max_payload_bytes=8388608`` within the ``[client]`` config
+   section of the local or centralized configuration.  Note that these
+   settings may allow ``rbd-mirror`` to present a substantial write workload
+   to the destination cluster:  monitor cluster performance closely during
+   migrations and test carefuly before running multiple migrations in parallel.
+
+Create Image Mirror-Snapshots
+-----------------------------
+
+When using snapshot-based mirroring, mirror-snapshots will need to be created
+whenever it is desired to mirror the changed contents of the RBD image. To
+create a mirror-snapshot manually with ``rbd``, specify the
+``mirror image snapshot`` command along with the pool and image name::
+
+        rbd mirror image snapshot {pool-name}/{image-name}
+
+For example::
+
+        $ rbd --cluster site-a mirror image snapshot image-pool/image-1
+
+By default up to ``5`` mirror-snapshots will be created per-image. The most
+recent mirror-snapshot is automatically pruned if the limit is reached.
+The limit can be overridden via the ``rbd_mirroring_max_mirroring_snapshots``
+configuration option if required. Additionally, mirror-snapshots are
+automatically deleted when the image is removed or when mirroring is disabled.
+
+Mirror-snapshots can also be automatically created on a periodic basis if
+mirror-snapshot schedules are defined. The mirror-snapshot can be scheduled
+globally, per-pool, or per-image levels. Multiple mirror-snapshot schedules can
+be defined at any level, but only the most-specific snapshot schedules that
+match an individual mirrored image will run.
+
+To create a mirror-snapshot schedule with ``rbd``, specify the
+``mirror snapshot schedule add`` command along with an optional pool or
+image name; interval; and optional start time::
+
+        rbd mirror snapshot schedule add [--pool {pool-name}] [--image {image-name}] {interval} [{start-time}]
+
+The ``interval`` can be specified in days, hours, or minutes using ``d``, ``h``,
+``m`` suffix respectively. The optional ``start-time`` can be specified using
+the ISO 8601 time format. For example::
+
+        $ rbd --cluster site-a mirror snapshot schedule add --pool image-pool 24h 14:00:00-05:00
+        $ rbd --cluster site-a mirror snapshot schedule add --pool image-pool --image image1 6h
+
+To remove a mirror-snapshot schedules with ``rbd``, specify the
+``mirror snapshot schedule remove`` command with options that match the
+corresponding ``add`` schedule command.
+
+To list all snapshot schedules for a specific level (global, pool, or image)
+with ``rbd``, specify the ``mirror snapshot schedule ls`` command along with
+an optional pool or image name. Additionally, the ``--recursive`` option can
+be specified to list all schedules at the specified level and below. For
+example::
+
+        $ rbd --cluster site-a mirror schedule ls --pool image-pool --recursive
+        POOL        NAMESPACE IMAGE  SCHEDULE                            
+        image-pool  -         -      every 1d starting at 14:00:00-05:00 
+        image-pool            image1 every 6h                            
+
+To view the status for when the next snapshots will be created for
+snapshot-based mirroring RBD images with ``rbd``, specify the
+``mirror snapshot schedule status`` command along with an optional pool or
+image name::
+
+        rbd mirror snapshot schedule status [--pool {pool-name}] [--image {image-name}]
+
+For example::
+
+        $ rbd --cluster site-a mirror schedule status
+        SCHEDULE TIME       IMAGE             
+        2020-02-26 18:00:00 image-pool/image1 
+
+Disable Image Mirroring
+-----------------------
+
+To disable mirroring for a specific image with ``rbd``, specify the
+``mirror image disable`` command along with the pool and image name::
+
+        rbd mirror image disable {pool-name}/{image-name}
+
+For example::
+
+        $ rbd --cluster site-a mirror image disable image-pool/image-1
+
+Image Promotion and Demotion
+----------------------------
+
+In a failover scenario where the primary designation needs to be moved to the
+image in the peer Ceph cluster, access to the primary image should be stopped
+(e.g. power down the VM or remove the associated drive from a VM), demote the
+current primary image, promote the new primary image, and resume access to the
+image on the alternate cluster.
+
+.. note:: RBD only provides the necessary tools to facilitate an orderly
+   failover of an image. An external mechanism is required to coordinate the
+   full failover process (e.g. closing the image before demotion).
+
+To demote a specific image to non-primary with ``rbd``, specify the
+``mirror image demote`` command along with the pool and image name::
+
+        rbd mirror image demote {pool-name}/{image-name}
+
+For example::
+
+        $ rbd --cluster site-a mirror image demote image-pool/image-1
+
+To demote all primary images within a pool to non-primary with ``rbd``, specify
+the ``mirror pool demote`` command along with the pool name::
+
+        rbd mirror pool demote {pool-name}
+
+For example::
+
+        $ rbd --cluster site-a mirror pool demote image-pool
+
+To promote a specific image to primary with ``rbd``, specify the
+``mirror image promote`` command along with the pool and image name::
+
+        rbd mirror image promote [--force] {pool-name}/{image-name}
+
+For example::
+
+        $ rbd --cluster site-b mirror image promote image-pool/image-1
+
+To promote all non-primary images within a pool to primary with ``rbd``, specify
+the ``mirror pool promote`` command along with the pool name::
+
+        rbd mirror pool promote [--force] {pool-name}
+
+For example::
+
+        $ rbd --cluster site-a mirror pool promote image-pool
+
+.. tip:: Since the primary / non-primary status is per-image, it is possible to
+   have two clusters split the IO load and stage failover / failback.
+
+.. note:: Promotion can be forced using the ``--force`` option. Forced
+   promotion is needed when the demotion cannot be propagated to the peer
+   Ceph cluster (e.g. Ceph cluster failure, communication outage). This will
+   result in a split-brain scenario between the two peers and the image will no
+   longer be in-sync until a `force resync command`_ is issued.
+
+Force Image Resync
+------------------
+
+If a split-brain event is detected by the ``rbd-mirror`` daemon, it will not
+attempt to mirror the affected image until corrected. To resume mirroring for an
+image, first `demote the image`_ determined to be out-of-date and then request a
+resync to the primary image. To request an image resync with ``rbd``, specify
+the ``mirror image resync`` command along with the pool and image name::
+
+        rbd mirror image resync {pool-name}/{image-name}
+
+For example::
+
+        $ rbd mirror image resync image-pool/image-1
+
+.. note:: The ``rbd`` command only flags the image as requiring a resync. The
+   local cluster's ``rbd-mirror`` daemon process is responsible for performing
+   the resync asynchronously.
+
+Mirror Status
+=============
+
+The peer cluster replication status is stored for every primary mirrored image.
+This status can be retrieved using the ``mirror image status`` and
+``mirror pool status`` commands.
+
+To request the mirror image status with ``rbd``, specify the
+``mirror image status`` command along with the pool and image name::
+
+        rbd mirror image status {pool-name}/{image-name}
+
+For example::
+
+        $ rbd mirror image status image-pool/image-1
+
+To request the mirror pool summary status with ``rbd``, specify the
+``mirror pool status`` command along with the pool name::
+
+        rbd mirror pool status {pool-name}
+
+For example::
+
+        $ rbd mirror pool status image-pool
+
+.. note:: Adding ``--verbose`` option to the ``mirror pool status`` command will
+   additionally output status details for every mirroring image in the pool.
+
+rbd-mirror Daemon
+=================
+
+The two ``rbd-mirror`` daemons are responsible for watching image journals on
+the remote, peer cluster and replaying the journal events against the local
+cluster. The RBD image journaling feature records all modifications to the
+image in the order they occur. This ensures that a crash-consistent mirror of
+the remote image is available locally.
+
+The ``rbd-mirror`` daemon is available within the optional ``rbd-mirror``
+distribution package.
+
+.. important:: Each ``rbd-mirror`` daemon requires the ability to connect
+   to both clusters simultaneously.
+.. warning:: Pre-Luminous releases: only run a single ``rbd-mirror`` daemon per
+   Ceph cluster.
+
+Each ``rbd-mirror`` daemon should use a unique Ceph user ID. To
+`create a Ceph user`_, with ``ceph`` specify the ``auth get-or-create``
+command, user name, monitor caps, and OSD caps::
+
+  ceph auth get-or-create client.rbd-mirror.{unique id} mon 'profile rbd-mirror' osd 'profile rbd'
+
+The ``rbd-mirror`` daemon can be managed by ``systemd`` by specifying the user
+ID as the daemon instance::
+
+  systemctl enable ceph-rbd-mirror@rbd-mirror.{unique id}
+
+The ``rbd-mirror`` can also be run in foreground by ``rbd-mirror`` command::
+
+  rbd-mirror -f --log-file={log_path}
+
+.. _rbd: ../../man/8/rbd
+.. _ceph-conf: ../../rados/configuration/ceph-conf/#running-multiple-clusters
+.. _explicitly enabled: #enable-image-mirroring
+.. _force resync command: #force-image-resync
+.. _demote the image: #image-promotion-and-demotion
+.. _create a Ceph user: ../../rados/operations/user-management#add-a-user
+.. _mirror-snapshots: #create-image-mirror-snapshots
diff --git a/doc/rbd/rbd-openstack.rst b/doc/rbd/rbd-openstack.rst
new file mode 100644
index 000000000..3f1b85f30
--- /dev/null
+++ b/doc/rbd/rbd-openstack.rst
@@ -0,0 +1,395 @@
+=============================
+ Block Devices and OpenStack
+=============================
+
+.. index:: Ceph Block Device; OpenStack
+
+You can attach Ceph Block Device images to OpenStack instances through ``libvirt``,
+which configures the QEMU interface to ``librbd``. Ceph stripes block volumes
+across multiple OSDs within the cluster, which means that large volumes can
+realize better performance than local drives on a standalone server!
+
+To use Ceph Block Devices with OpenStack, you must install QEMU, ``libvirt``,
+and OpenStack first. We recommend using a separate physical node for your
+OpenStack installation. OpenStack recommends a minimum of 8GB of RAM and a
+quad-core processor. The following diagram depicts the OpenStack/Ceph
+technology stack.
+
+
+.. ditaa::
+
+            +---------------------------------------------------+
+            |                    OpenStack                      |
+            +---------------------------------------------------+
+            |                     libvirt                       |
+            +------------------------+--------------------------+
+                                     |
+                                     | configures
+                                     v
+            +---------------------------------------------------+
+            |                       QEMU                        |
+            +---------------------------------------------------+
+            |                      librbd                       |
+            +---------------------------------------------------+
+            |                     librados                      |
+            +------------------------+-+------------------------+
+            |          OSDs          | |        Monitors        |
+            +------------------------+ +------------------------+
+
+.. important:: To use Ceph Block Devices with OpenStack, you must have
+   access to a running Ceph Storage Cluster.
+
+Three parts of OpenStack integrate with Ceph's block devices:
+
+- **Images**: OpenStack Glance manages images for VMs. Images are immutable.
+  OpenStack treats images as binary blobs and downloads them accordingly.
+
+- **Volumes**: Volumes are block devices. OpenStack uses volumes to boot VMs,
+  or to attach volumes to running VMs. OpenStack manages volumes using
+  Cinder services.
+
+- **Guest Disks**: Guest disks are guest operating system disks. By default,
+  when you boot a virtual machine, its disk appears as a file on the file system
+  of the hypervisor (usually under ``/var/lib/nova/instances/<uuid>/``). Prior
+  to OpenStack Havana, the only way to boot a VM in Ceph was to use the
+  boot-from-volume functionality of Cinder. However, now it is possible to boot
+  every virtual machine inside Ceph directly without using Cinder, which is
+  advantageous because it allows you to perform maintenance operations easily
+  with the live-migration process. Additionally, if your hypervisor dies it is
+  also convenient to trigger ``nova evacuate`` and reinstate the virtual machine
+  elsewhere almost seamlessly. In doing so,
+  :ref:`exclusive locks <rbd-exclusive-locks>` prevent multiple
+  compute nodes from concurrently accessing the guest disk.
+
+
+You can use OpenStack Glance to store images as Ceph Block Devices, and you
+can use Cinder to boot a VM using a copy-on-write clone of an image.
+
+The instructions below detail the setup for Glance, Cinder and Nova, although
+they do not have to be used together. You may store images in Ceph block devices
+while running VMs using a local disk, or vice versa.
+
+.. important:: Using QCOW2 for hosting a virtual machine disk is NOT recommended.
+   If you want to boot virtual machines in Ceph (ephemeral backend or boot
+   from volume), please use the ``raw`` image format within Glance.
+
+.. index:: pools; OpenStack
+
+Create a Pool
+=============
+
+By default, Ceph block devices live within the ``rbd`` pool. You may use any
+suitable pool by specifying it explicitly. We recommend creating a pool for
+Cinder and a pool for Glance. Ensure your Ceph cluster is running, then create the pools. ::
+
+    ceph osd pool create volumes
+    ceph osd pool create images
+    ceph osd pool create backups
+    ceph osd pool create vms
+
+See `Create a Pool`_ for detail on specifying the number of placement groups for
+your pools, and `Placement Groups`_ for details on the number of placement
+groups you should set for your pools.
+
+Newly created pools must be initialized prior to use. Use the ``rbd`` tool
+to initialize the pools::
+
+        rbd pool init volumes
+        rbd pool init images
+        rbd pool init backups
+        rbd pool init vms
+
+.. _Create a Pool: ../../rados/operations/pools#createpool
+.. _Placement Groups: ../../rados/operations/placement-groups
+
+
+Configure OpenStack Ceph Clients
+================================
+
+The nodes running ``glance-api``, ``cinder-volume``, ``nova-compute`` and
+``cinder-backup`` act as Ceph clients. Each requires the ``ceph.conf`` file::
+
+  ssh {your-openstack-server} sudo tee /etc/ceph/ceph.conf </etc/ceph/ceph.conf
+
+
+Install Ceph client packages
+----------------------------
+
+On the ``glance-api`` node, you will need the Python bindings for ``librbd``::
+
+  sudo apt-get install python-rbd
+  sudo yum install python-rbd
+
+On the ``nova-compute``, ``cinder-backup`` and on the ``cinder-volume`` node,
+use both the Python bindings and the client command line tools::
+
+  sudo apt-get install ceph-common
+  sudo yum install ceph-common
+
+
+Setup Ceph Client Authentication
+--------------------------------
+
+If you have `cephx authentication`_ enabled, create a new user for Nova/Cinder
+and Glance. Execute the following::
+
+    ceph auth get-or-create client.glance mon 'profile rbd' osd 'profile rbd pool=images' mgr 'profile rbd pool=images'
+    ceph auth get-or-create client.cinder mon 'profile rbd' osd 'profile rbd pool=volumes, profile rbd pool=vms, profile rbd-read-only pool=images' mgr 'profile rbd pool=volumes, profile rbd pool=vms'
+    ceph auth get-or-create client.cinder-backup mon 'profile rbd' osd 'profile rbd pool=backups' mgr 'profile rbd pool=backups'
+
+Add the keyrings for ``client.cinder``, ``client.glance``, and
+``client.cinder-backup`` to the appropriate nodes and change their ownership::
+
+  ceph auth get-or-create client.glance | ssh {your-glance-api-server} sudo tee /etc/ceph/ceph.client.glance.keyring
+  ssh {your-glance-api-server} sudo chown glance:glance /etc/ceph/ceph.client.glance.keyring
+  ceph auth get-or-create client.cinder | ssh {your-volume-server} sudo tee /etc/ceph/ceph.client.cinder.keyring
+  ssh {your-cinder-volume-server} sudo chown cinder:cinder /etc/ceph/ceph.client.cinder.keyring
+  ceph auth get-or-create client.cinder-backup | ssh {your-cinder-backup-server} sudo tee /etc/ceph/ceph.client.cinder-backup.keyring
+  ssh {your-cinder-backup-server} sudo chown cinder:cinder /etc/ceph/ceph.client.cinder-backup.keyring
+
+Nodes running ``nova-compute`` need the keyring file for the ``nova-compute``
+process::
+
+  ceph auth get-or-create client.cinder | ssh {your-nova-compute-server} sudo tee /etc/ceph/ceph.client.cinder.keyring
+
+They also need to store the secret key of the ``client.cinder`` user in
+``libvirt``. The libvirt process needs it to access the cluster while attaching
+a block device from Cinder.
+
+Create a temporary copy of the secret key on the nodes running
+``nova-compute``::
+
+  ceph auth get-key client.cinder | ssh {your-compute-node} tee client.cinder.key
+
+Then, on the compute nodes, add the secret key to ``libvirt`` and remove the
+temporary copy of the key::
+
+  uuidgen
+  457eb676-33da-42ec-9a8c-9293d545c337
+
+  cat > secret.xml <<EOF
+  <secret ephemeral='no' private='no'>
+    <uuid>457eb676-33da-42ec-9a8c-9293d545c337</uuid>
+    <usage type='ceph'>
+      <name>client.cinder secret</name>
+    </usage>
+  </secret>
+  EOF
+  sudo virsh secret-define --file secret.xml
+  Secret 457eb676-33da-42ec-9a8c-9293d545c337 created
+  sudo virsh secret-set-value --secret 457eb676-33da-42ec-9a8c-9293d545c337 --base64 $(cat client.cinder.key) && rm client.cinder.key secret.xml
+
+Save the uuid of the secret for configuring ``nova-compute`` later.
+
+.. important:: You don't necessarily need the UUID on all the compute nodes.
+   However from a platform consistency perspective, it's better to keep the
+   same UUID.
+
+.. _cephx authentication: ../../rados/configuration/auth-config-ref/#enabling-disabling-cephx
+
+
+Configure OpenStack to use Ceph
+===============================
+
+Configuring Glance
+------------------
+
+Glance can use multiple back ends to store images. To use Ceph block devices by
+default, configure Glance like the following.
+
+
+Kilo and after
+~~~~~~~~~~~~~~
+
+Edit ``/etc/glance/glance-api.conf`` and add under the ``[glance_store]`` section::
+
+    [glance_store]
+    stores = rbd
+    default_store = rbd
+    rbd_store_pool = images
+    rbd_store_user = glance
+    rbd_store_ceph_conf = /etc/ceph/ceph.conf
+    rbd_store_chunk_size = 8
+
+For more information about the configuration options available in Glance please refer to the OpenStack Configuration Reference: http://docs.openstack.org/.
+
+Enable copy-on-write cloning of images
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Note that this exposes the back end location via Glance's API, so the endpoint
+with this option enabled should not be publicly accessible.
+
+Any OpenStack version except Mitaka
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you want to enable copy-on-write cloning of images, also add under the ``[DEFAULT]`` section::
+
+    show_image_direct_url = True
+
+Disable cache management (any OpenStack version)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Disable the Glance cache management to avoid images getting cached under ``/var/lib/glance/image-cache/``,
+assuming your configuration file has ``flavor = keystone+cachemanagement``::
+
+    [paste_deploy]
+    flavor = keystone
+
+Image properties
+~~~~~~~~~~~~~~~~
+
+We recommend to use the following properties for your images:
+
+- ``hw_scsi_model=virtio-scsi``: add the virtio-scsi controller and get better performance and support for discard operation
+- ``hw_disk_bus=scsi``: connect every cinder block devices to that controller
+- ``hw_qemu_guest_agent=yes``: enable the QEMU guest agent
+- ``os_require_quiesce=yes``: send fs-freeze/thaw calls through the QEMU guest agent
+
+
+Configuring Cinder
+------------------
+
+OpenStack requires a driver to interact with Ceph block devices. You must also
+specify the pool name for the block device. On your OpenStack node, edit
+``/etc/cinder/cinder.conf`` by adding::
+
+    [DEFAULT]
+    ...
+    enabled_backends = ceph
+    glance_api_version = 2
+    ...
+    [ceph]
+    volume_driver = cinder.volume.drivers.rbd.RBDDriver
+    volume_backend_name = ceph
+    rbd_pool = volumes
+    rbd_ceph_conf = /etc/ceph/ceph.conf
+    rbd_flatten_volume_from_snapshot = false
+    rbd_max_clone_depth = 5
+    rbd_store_chunk_size = 4
+    rados_connect_timeout = -1
+
+If you are using `cephx authentication`_, also configure the user and uuid of
+the secret you added to ``libvirt`` as documented earlier::
+
+    [ceph]
+    ...
+    rbd_user = cinder
+    rbd_secret_uuid = 457eb676-33da-42ec-9a8c-9293d545c337
+
+Note that if you are configuring multiple cinder back ends,
+``glance_api_version = 2`` must be in the ``[DEFAULT]`` section.
+
+
+Configuring Cinder Backup
+-------------------------
+
+OpenStack Cinder Backup requires a specific daemon so don't forget to install it.
+On your Cinder Backup node, edit ``/etc/cinder/cinder.conf`` and add::
+
+    backup_driver = cinder.backup.drivers.ceph
+    backup_ceph_conf = /etc/ceph/ceph.conf
+    backup_ceph_user = cinder-backup
+    backup_ceph_chunk_size = 134217728
+    backup_ceph_pool = backups
+    backup_ceph_stripe_unit = 0
+    backup_ceph_stripe_count = 0
+    restore_discard_excess_bytes = true
+
+
+Configuring Nova to attach Ceph RBD block device
+------------------------------------------------
+
+In order to attach Cinder devices (either normal block or by issuing a boot
+from volume), you must tell Nova (and libvirt) which user and UUID to refer to
+when attaching the device. libvirt will refer to this user when connecting and
+authenticating with the Ceph cluster. ::
+
+    [libvirt]
+    ...
+    rbd_user = cinder
+    rbd_secret_uuid = 457eb676-33da-42ec-9a8c-9293d545c337
+
+These two flags are also used by the Nova ephemeral back end.
+
+
+Configuring Nova
+----------------
+
+In order to boot virtual machines directly from Ceph volumes, you must
+configure the ephemeral backend for Nova.
+
+It is recommended to enable the RBD cache in your Ceph configuration file; this
+has been enabled by default since the Giant release. Moreover, enabling the
+client admin socket allows the collection of metrics and can be invaluable
+for troubleshooting.
+
+This socket can be accessed on the hypvervisor (Nova compute) node::
+
+    ceph daemon /var/run/ceph/ceph-client.cinder.19195.32310016.asok help
+
+To enable RBD cache and admin sockets, ensure that on each hypervisor's
+``ceph.conf`` contains::
+
+    [client]
+        rbd cache = true
+        rbd cache writethrough until flush = true
+        admin socket = /var/run/ceph/guests/$cluster-$type.$id.$pid.$cctid.asok
+        log file = /var/log/qemu/qemu-guest-$pid.log
+        rbd concurrent management ops = 20
+
+Configure permissions for these directories::
+
+    mkdir -p /var/run/ceph/guests/ /var/log/qemu/
+    chown qemu:libvirtd /var/run/ceph/guests /var/log/qemu/
+
+Note that user ``qemu`` and group ``libvirtd`` can vary depending on your system.
+The provided example works for RedHat based systems.
+
+.. tip:: If your virtual machine is already running you can simply restart it to enable the admin socket
+
+
+Restart OpenStack
+=================
+
+To activate the Ceph block device driver and load the block device pool name
+into the configuration, you must restart the related OpenStack services.
+For Debian based systems execute these commands on the appropriate nodes::
+
+    sudo glance-control api restart
+    sudo service nova-compute restart
+    sudo service cinder-volume restart
+    sudo service cinder-backup restart
+
+For Red Hat based systems execute::
+
+    sudo service openstack-glance-api restart
+    sudo service openstack-nova-compute restart
+    sudo service openstack-cinder-volume restart
+    sudo service openstack-cinder-backup restart
+
+Once OpenStack is up and running, you should be able to create a volume
+and boot from it.
+
+
+Booting from a Block Device
+===========================
+
+You can create a volume from an image using the Cinder command line tool::
+
+    cinder create --image-id {id of image} --display-name {name of volume} {size of volume}
+
+You can use `qemu-img`_ to convert from one format to another. For example::
+
+    qemu-img convert -f {source-format} -O {output-format} {source-filename} {output-filename}
+    qemu-img convert -f qcow2 -O raw precise-cloudimg.img precise-cloudimg.raw
+
+When Glance and Cinder are both using Ceph block devices, the image is a
+copy-on-write clone, so new volumes are created quickly. In the OpenStack
+dashboard, you can boot from that volume by performing the following steps:
+
+#. Launch a new instance.
+#. Choose the image associated to the copy-on-write clone.
+#. Select 'boot from volume'.
+#. Select the volume you created.
+
+.. _qemu-img: ../qemu-rbd/#running-qemu-with-rbd
diff --git a/doc/rbd/rbd-operations.rst b/doc/rbd/rbd-operations.rst
new file mode 100644
index 000000000..cc27b5103
--- /dev/null
+++ b/doc/rbd/rbd-operations.rst
@@ -0,0 +1,16 @@
+==============================
+ Ceph Block Device Operations
+==============================
+
+.. toctree::
+   :maxdepth: 1
+
+   Snapshots<rbd-snapshot>
+   Exclusive Locking <rbd-exclusive-locks>
+   Mirroring <rbd-mirroring>
+   Live-Migration <rbd-live-migration>
+   Persistent Read-only Cache <rbd-persistent-read-only-cache>
+   Persistent Write-back Cache <rbd-persistent-write-back-cache>
+   Encryption <rbd-encryption>
+   Config Settings (librbd) <rbd-config-ref/>
+   RBD Replay <rbd-replay>
diff --git a/doc/rbd/rbd-persistent-read-only-cache.rst b/doc/rbd/rbd-persistent-read-only-cache.rst
new file mode 100644
index 000000000..5bef7f592
--- /dev/null
+++ b/doc/rbd/rbd-persistent-read-only-cache.rst
@@ -0,0 +1,201 @@
+===============================
+ RBD Persistent Read-only Cache
+===============================
+
+.. index:: Ceph Block Device; Persistent Read-only Cache
+
+Shared, Read-only Parent Image Cache
+====================================
+
+`Cloned RBD images`_ usually modify only a small fraction of the parent
+image. For example, in a VDI use-case, VMs are cloned from the same
+base image and initially differ only by hostname and IP address. During
+booting, all of these VMs read portions of the same parent
+image data. If we have a local cache of the parent
+image, this speeds up reads on the caching host.  We also achieve
+reduction of client-to-cluster network traffic.
+RBD cache must be explicitly enabled in
+``ceph.conf``. The ``ceph-immutable-object-cache`` daemon is responsible for
+caching the parent content on the local disk, and future reads on that data
+will be serviced from the local cache.
+
+.. note:: RBD shared read-only parent image cache requires the Ceph Nautilus release or later.
+
+.. ditaa::
+
+            +--------------------------------------------------------+
+            |                         QEMU                           |
+            +--------------------------------------------------------+
+            |                librbd (cloned images)                  |
+            +-------------------+-+----------------------------------+
+            |      librados     | |  ceph--immutable--object--cache  |
+            +-------------------+ +----------------------------------+
+            |      OSDs/Mons    | |     local cached parent image    |
+            +-------------------+ +----------------------------------+
+
+
+Enable RBD Shared Read-only Parent Image Cache
+----------------------------------------------
+
+To enable RBD shared read-only parent image cache, the following Ceph settings
+need to added in the ``[client]`` `section`_ of your ``ceph.conf`` file::
+
+        rbd parent cache enabled = true
+        rbd plugins = parent_cache
+
+Immutable Object Cache Daemon
+=============================
+
+Introduction and Generic Settings
+---------------------------------
+
+The ``ceph-immutable-object-cache`` daemon is responsible for caching parent
+image content within its local caching directory. Using SSDs as the underlying
+storage is recommended because doing so provides better performance. 
+
+The key components of the daemon are:
+
+#. **Domain socket based IPC:** The daemon listens on a local domain socket at 
+   startup and waits for connections from librbd clients.
+
+#. **LRU based promotion/demotion policy:** The daemon maintains in-memory
+   statistics of cache hits for each cache file. It demotes the cold cache
+   if capacity reaches the configured threshold.
+
+#. **File-based caching store:** The daemon maintains a simple file-based cache
+   store. On promotion, the RADOS objects are fetched from RADOS cluster and
+   stored in the local caching directory.
+
+When each cloned RBD image is opened, ``librbd`` tries to connect to the cache
+daemon through its Unix domain socket. After ``librbd`` is successfully
+connected, it coordinates with the daemon upon every subsequent read. In the
+case of an uncached read, the daemon promotes the RADOS object to the local
+caching directory and the next read of the object is serviced from the cache.
+The daemon maintains simple LRU statistics, which are used to evict cold cache
+files when required (for example, when the cache is at capacity and under
+pressure). 
+
+Here are some important cache configuration settings:
+
+``immutable_object_cache_sock``
+
+:Description: The path to the domain socket used for communication between
+              librbd clients and the ceph-immutable-object-cache daemon.
+:Type: String
+:Required: No
+:Default: ``/var/run/ceph/immutable_object_cache_sock``
+
+
+``immutable_object_cache_path``
+
+:Description: The immutable object cache data directory.
+:Type: String
+:Required: No
+:Default: ``/tmp/ceph_immutable_object_cache``
+
+
+``immutable_object_cache_max_size``
+
+:Description: The max size for immutable cache.
+:Type: Size
+:Required: No
+:Default: ``1G``
+
+
+``immutable_object_cache_watermark``
+
+:Description: The high-water mark for the cache. The value is between (0, 1).
+              If the cache size reaches this threshold the daemon will start
+              to delete cold cache based on LRU statistics.
+:Type: Float
+:Required: No
+:Default: ``0.9``
+
+The ``ceph-immutable-object-cache`` daemon is available within the optional
+``ceph-immutable-object-cache`` distribution package.
+
+.. important:: ``ceph-immutable-object-cache`` daemon requires the ability to
+   connect RADOS clusters.
+
+Running the Immutable Object Cache Daemon
+-----------------------------------------
+
+``ceph-immutable-object-cache`` daemon should use a unique Ceph user ID.
+To `create a Ceph user`_, with ``ceph`` specify the ``auth get-or-create``
+command, user name, monitor caps, and OSD caps::
+
+  ceph auth get-or-create client.ceph-immutable-object-cache.{unique id} mon 'allow r' osd 'profile rbd-read-only'
+
+The ``ceph-immutable-object-cache`` daemon can be managed by ``systemd`` by specifying the user
+ID as the daemon instance::
+
+  systemctl enable ceph-immutable-object-cache@ceph-immutable-object-cache.{unique id}
+
+The ``ceph-immutable-object-cache`` can also be run in foreground by ``ceph-immutable-object-cache`` command::
+
+  ceph-immutable-object-cache -f --log-file={log_path}
+
+QOS Settings
+------------
+
+The immutable object cache supports throttling, controlled by the following settings:
+
+``immutable_object_cache_qos_schedule_tick_min``
+
+:Description: Minimum schedule tick for immutable object cache.
+:Type: Milliseconds
+:Required: No
+:Default: ``50``
+
+
+``immutable_object_cache_qos_iops_limit``
+
+:Description: The desired immutable object cache IO operations limit per second.
+:Type: Unsigned Integer
+:Required: No
+:Default: ``0``
+
+
+``immutable_object_cache_qos_iops_burst``
+
+:Description: The desired burst limit of immutable object cache IO operations.
+:Type: Unsigned Integer
+:Required: No
+:Default: ``0``
+
+
+``immutable_object_cache_qos_iops_burst_seconds``
+
+:Description: The desired burst duration in seconds of immutable object cache IO operations.
+:Type: Seconds
+:Required: No
+:Default: ``1``
+
+
+``immutable_object_cache_qos_bps_limit``
+
+:Description: The desired immutable object cache IO bytes limit per second.
+:Type: Unsigned Integer
+:Required: No
+:Default: ``0``
+
+
+``immutable_object_cache_qos_bps_burst``
+
+:Description: The desired burst limit of immutable object cache IO bytes.
+:Type: Unsigned Integer
+:Required: No
+:Default: ``0``
+
+
+``immutable_object_cache_qos_bps_burst_seconds``
+
+:Description: The desired burst duration in seconds of immutable object cache IO bytes.
+:Type: Seconds
+:Required: No
+:Default: ``1``
+
+.. _Cloned RBD Images: ../rbd-snapshot/#layering
+.. _section: ../../rados/configuration/ceph-conf/#configuration-sections
+.. _create a Ceph user: ../../rados/operations/user-management#add-a-user
+
diff --git a/doc/rbd/rbd-persistent-write-back-cache.rst b/doc/rbd/rbd-persistent-write-back-cache.rst
new file mode 100644
index 000000000..8066014e9
--- /dev/null
+++ b/doc/rbd/rbd-persistent-write-back-cache.rst
@@ -0,0 +1,133 @@
+================================
+ RBD Persistent Write-back Cache
+================================
+
+.. index:: Ceph Block Device; Persistent Write-back Cache
+
+Persistent Write-back Cache
+===========================
+
+The persistent write-back cache provides a persistent, fault-tolerant write-back
+cache for librbd-based RBD clients.
+
+This cache uses a log-ordered write-back design which maintains checkpoints
+internally so that writes that get flushed back to the cluster are always
+crash consistent. Even if the client cache is lost entirely, the disk image is
+still consistent but the data will appear to be stale.
+
+This cache can be used with PMEM or SSD as a cache device.
+
+Usage
+=====
+
+The persistent write-back cache manages the cache data in a persistent device.
+It looks for and creates cache files in a configured directory, and then caches
+data in the file.
+
+The persistent write-back cache can't be enabled without the exclusive-lock
+feature. It tries to enable the write-back cache only when the exclusive lock
+is acquired.
+
+The cache provides two different persistence modes. In persistent-on-write mode,
+the writes are completed only when they are persisted to the cache device and
+will be readable after a crash. In persistent-on-flush mode, the writes are
+completed as soon as it no longer needs the caller's data buffer to complete
+the writes, but does not guarantee that writes will be readable after a crash.
+The data is persisted to the cache device when a flush request is received.
+
+Initially it defaults to the persistent-on-write mode and it switches to
+persistent-on-flush mode after the first flush request is received.
+
+Enable Persistent Write-back Cache
+========================================
+
+To enable the persistent write-back cache, the following Ceph settings
+need to be enabled.::
+
+        rbd persistent cache mode = {cache-mode}
+        rbd plugins = pwl_cache
+
+Value of {cache-mode} can be ``rwl``, ``ssd`` or ``disabled``. By default the
+cache is disabled.
+
+Here are some cache configuration settings:
+
+- ``rbd_persistent_cache_path`` A file folder to cache data. This folder must
+  have DAX enabled (see `DAX`_) when using ``rwl`` mode to avoid performance
+  degradation.
+
+- ``rbd_persistent_cache_size`` The cache size per image. The minimum cache
+  size is 1 GB.
+
+The above configurations can be set per-host, per-pool, per-image etc. Eg, to
+set per-host, add the overrides to the appropriate `section`_ in the host's
+``ceph.conf`` file. To set per-pool, per-image, etc, please refer to the
+``rbd config`` `commands`_.
+
+Cache Status
+------------
+
+The persistent write-back cache is enabled when the exclusive lock is acquired,
+and it is closed when the exclusive lock is released. To check the cache status,
+users may use the command ``rbd status``.  ::
+
+        rbd status {pool-name}/{image-name}
+
+The status of the cache is shown, including present, clean, cache size and the
+location as well as some basic metrics.
+
+For example::
+
+        $ rbd status rbd/foo
+        Watchers:
+                watcher=10.10.0.102:0/1061883624 client.25496 cookie=140338056493088
+        Persistent cache state:
+                host: sceph9
+                path: /mnt/nvme0/rbd-pwl.rbd.101e5824ad9a.pool
+                size: 1 GiB
+                mode: ssd
+                stats_timestamp: Sun Apr 10 13:26:32 2022
+                present: true   empty: false    clean: false
+                allocated: 509 MiB
+                cached: 501 MiB
+                dirty: 338 MiB
+                free: 515 MiB
+                hits_full: 1450 / 61%
+                hits_partial: 0 / 0%
+                misses: 924
+                hit_bytes: 192 MiB / 66%
+                miss_bytes: 97 MiB
+
+Flush Cache
+-----------
+
+To flush a cache file with ``rbd``, specify the ``persistent-cache flush``
+command, the pool name and the image name.  ::
+
+        rbd persistent-cache flush {pool-name}/{image-name}
+
+If the application dies unexpectedly, this command can also be used to flush
+the cache back to OSDs.
+
+For example::
+
+        $ rbd persistent-cache flush rbd/foo
+
+Invalidate Cache
+----------------
+
+To invalidate (discard) a cache file with ``rbd``, specify the
+``persistent-cache invalidate`` command, the pool name and the image name.  ::
+
+        rbd persistent-cache invalidate {pool-name}/{image-name}
+
+The command removes the cache metadata of the corresponding image, disables
+the cache feature and deletes the local cache file if it exists.
+
+For example::
+
+        $ rbd persistent-cache invalidate rbd/foo
+
+.. _section: ../../rados/configuration/ceph-conf/#configuration-sections
+.. _commands: ../../man/8/rbd#commands
+.. _DAX: https://www.kernel.org/doc/Documentation/filesystems/dax.txt
diff --git a/doc/rbd/rbd-replay.rst b/doc/rbd/rbd-replay.rst
new file mode 100644
index 000000000..b1fc4973f
--- /dev/null
+++ b/doc/rbd/rbd-replay.rst
@@ -0,0 +1,42 @@
+===================
+ RBD Replay
+===================
+
+.. index:: Ceph Block Device; RBD Replay
+
+RBD Replay is a set of tools for capturing and replaying RADOS Block Device
+(RBD) workloads. To capture an RBD workload, ``lttng-tools`` must be installed
+on the client, and ``librbd`` on the client must be the v0.87 (Giant) release 
+or later. To replay an RBD workload, ``librbd`` on the client must be the Giant
+release or later.
+
+Capture and replay takes three steps:
+
+#. Capture the trace.  Make sure to capture ``pthread_id`` context::
+
+    mkdir -p traces
+    lttng create -o traces librbd
+    lttng enable-event -u 'librbd:*'
+    lttng add-context -u -t pthread_id
+    lttng start
+    # run RBD workload here
+    lttng stop
+
+#. Process the trace with `rbd-replay-prep`_::
+
+    rbd-replay-prep traces/ust/uid/*/* replay.bin
+
+#. Replay the trace with `rbd-replay`_. Use read-only until you know 
+   it's doing what you want::
+
+    rbd-replay --read-only replay.bin
+
+.. important:: ``rbd-replay`` will destroy data by default.  Do not use against 
+   an image you wish to keep, unless you use the ``--read-only`` option.
+
+The replayed workload does not have to be against the same RBD image or even the
+same cluster as the captured workload. To account for differences, you may need
+to use the ``--pool`` and ``--map-image`` options of ``rbd-replay``.
+
+.. _rbd-replay: ../../man/8/rbd-replay
+.. _rbd-replay-prep: ../../man/8/rbd-replay-prep
diff --git a/doc/rbd/rbd-snapshot.rst b/doc/rbd/rbd-snapshot.rst
new file mode 100644
index 000000000..765a1555b
--- /dev/null
+++ b/doc/rbd/rbd-snapshot.rst
@@ -0,0 +1,367 @@
+===========
+ Snapshots
+===========
+
+.. index:: Ceph Block Device; snapshots
+
+A snapshot is a read-only logical copy of an image at a particular point in
+time: a checkpoint. One of the advanced features of Ceph block devices is that
+you can create snapshots of images to retain point-in-time state history.  Ceph
+also supports snapshot layering, which allows you to clone images (for example,
+VM images) quickly and easily. Ceph block device snapshots are managed using
+the ``rbd`` command and several higher-level interfaces, including `QEMU`_,
+`libvirt`_, `OpenStack`_, and `CloudStack`_.
+
+.. important:: To use RBD snapshots, you must have a running Ceph cluster.
+
+.. note:: Because RBD is unaware of any file system within an image (volume),
+   snapshots are merely `crash-consistent` unless they are coordinated within
+   the mounting (attaching) operating system. We therefore recommend that you
+   pause or stop I/O before taking a snapshot.  
+   
+   If the volume contains a file system, the file system should be in an
+   internally consistent state before a snapshot is taken. Snapshots taken
+   without write quiescing could need an `fsck` pass before they are mounted
+   again. To quiesce I/O you can use `fsfreeze` command. See the `fsfreeze(8)`
+   man page for more details. 
+   
+   For virtual machines, `qemu-guest-agent` can be used to automatically freeze
+   file systems when creating a snapshot.
+
+.. ditaa::
+
+           +------------+         +-------------+
+           | {s}        |         | {s} c999    |
+           |   Active   |<-------*|   Snapshot  |
+           |   Image    |         |   of Image  |
+           | (stop i/o) |         | (read only) |
+           +------------+         +-------------+
+
+
+Cephx Notes
+===========
+
+When `cephx`_ authentication is enabled (it is by default), you must specify a
+user name or ID and a path to the keyring containing the corresponding key. See
+:ref:`User Management <user-management>` for details. 
+
+.. prompt:: bash $
+
+	rbd --id {user-ID} --keyring /path/to/secret [commands]
+	rbd --name {username} --keyring /path/to/secret [commands]
+
+For example:
+
+.. prompt:: bash $
+
+	rbd --id admin --keyring /etc/ceph/ceph.keyring [commands]
+	rbd --name client.admin --keyring /etc/ceph/ceph.keyring [commands]
+
+.. tip:: Add the user and secret to the ``CEPH_ARGS`` environment variable to
+   avoid re-entry of these parameters.
+
+
+Snapshot Basics
+===============
+
+The following procedures demonstrate how to create, list, and remove
+snapshots using the ``rbd`` command.
+
+Create Snapshot
+---------------
+
+To create a snapshot, use the ``rbd snap create`` command and specify the pool
+name, the image name, and the snap name:
+
+.. prompt:: bash $
+
+   rbd snap create {pool-name}/{image-name}@{snap-name}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd snap create rbd/foo@snapname
+	
+
+List Snapshots
+--------------
+
+To list the snapshots of an image, use the ``rbd snap ls`` command and specify
+the pool name and the image name:
+
+.. prompt:: bash $
+
+   rbd snap ls {pool-name}/{image-name}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd snap ls rbd/foo
+
+
+Roll back Snapshot
+------------------
+
+To roll back to a snapshot,  use the ``rbd snap rollback`` command and specify
+the pool name, the image name, and the snap name:
+
+.. prompt:: bash $
+
+   rbd snap rollback {pool-name}/{image-name}@{snap-name}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd snap rollback rbd/foo@snapname
+
+
+.. note:: Rolling back an image to a snapshot means overwriting the current
+   version of the image with data from a snapshot. The time it takes to execute
+   a rollback increases with the size of the image. It is **faster to clone**
+   from a snapshot **than to roll back** an image to a snapshot. Cloning from a
+   snapshot is the preferred method of returning to a pre-existing state.
+
+
+Delete a Snapshot
+-----------------
+
+To delete a snapshot, use the ``rbd snap rm`` command and specify the pool
+name, the image name, and the snap name:
+
+.. prompt:: bash $
+
+   rbd snap rm {pool-name}/{image-name}@{snap-name}
+	
+For example:
+
+.. prompt:: bash $
+
+   rbd snap rm rbd/foo@snapname
+
+
+.. note:: Ceph OSDs delete data asynchronously, so deleting a snapshot  does
+   not immediately free up the capacity of the underlying OSDs. This process is
+   known as "snaptrim", and is referred to as such in ``ceph status`` output.
+
+Purge Snapshots
+---------------
+
+To delete all snapshots, use the ``rbd snap purge`` command and specify the
+pool name and the image name:
+
+.. prompt:: bash $
+
+   rbd snap purge {pool-name}/{image-name}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd snap purge rbd/foo
+
+
+.. index:: Ceph Block Device; snapshot layering
+
+Layering
+========
+
+Ceph supports the ability to create many copy-on-write (COW) clones of a block
+device snapshot. Snapshot layering enables Ceph block device clients to create
+images very quickly. For example, you might create a block device image with a
+Linux VM written to it, snapshot the image, protect the snapshot, and create as
+many copy-on-write clones as you like. A snapshot is read-only, so cloning a
+snapshot simplifies semantics, making it possible to create clones rapidly.
+
+
+.. ditaa::
+
+           +-------------+              +-------------+
+           | {s} c999    |              | {s}         |
+           |  Snapshot   | Child refers |  COW Clone  |
+           |  of Image   |<------------*| of Snapshot |
+           |             |  to Parent   |             |
+           | (read only) |              | (writable)  |
+           +-------------+              +-------------+
+           
+               Parent                        Child
+
+.. note:: The terms "parent" and "child" refer to a Ceph block device snapshot
+   (parent) and the corresponding image cloned from the snapshot (child).
+   These terms are important for the command line usage below.
+   
+Each cloned image (child) stores a reference to its parent image, which enables
+the cloned image to open the parent snapshot and read it.
+
+A copy-on-write clone of a snapshot behaves exactly like any other Ceph
+block device image. You can read to, write from, clone, and resize cloned
+images. There are no special restrictions with cloned images. However, the
+copy-on-write clone of a snapshot depends on the snapshot, so you must 
+protect the snapshot before you clone it. The diagram below depicts this
+process.
+
+.. note:: Ceph supports the cloning of only "RBD format 2" images (that is,
+   images created without specifying ``--image-format 1``). The Linux kernel
+   client supports cloned images beginning with the 3.10 release.
+
+Getting Started with Layering
+-----------------------------
+
+Ceph block device layering is a simple process. You must have an image. You
+must create a snapshot of the image. You must protect the snapshot. After you
+have performed these steps, you can begin cloning the snapshot.
+
+.. ditaa::
+
+           +----------------------------+        +-----------------------------+
+           |                            |        |                             |
+           | Create Block Device Image  |------->|      Create a Snapshot      |
+           |                            |        |                             |
+           +----------------------------+        +-----------------------------+
+                                                                |
+                         +--------------------------------------+ 
+                         |
+                         v
+           +----------------------------+        +-----------------------------+
+           |                            |        |                             |
+           |   Protect the Snapshot     |------->|     Clone the Snapshot      |
+           |                            |        |                             |
+           +----------------------------+        +-----------------------------+
+
+
+The cloned image has a reference to the parent snapshot, and includes the pool
+ID, the image ID, and the snapshot ID. The inclusion of the pool ID means that
+you may clone snapshots from one pool to images in another pool.
+
+#. **Image Template:** A common use case for block device layering is to create
+   a base image and a snapshot that serves as a template for clones. For
+   example: a user may create an image for a Linux distribution (for example,
+   Ubuntu 22.04) and create a snapshot of it. The user may occasionally update
+   the image and create a new snapshot (by using such commands as ``sudo
+   apt-get update``, ``sudo apt-get upgrade``, or ``sudo apt-get dist-upgrade``
+   followed by ``rbd snap create``). As the image matures, the user can clone
+   any one of the snapshots.
+
+#. **Extended Template:** A more advanced use case includes extending a
+   template image to provide more information than a base image. For
+   example, a user may clone an image (for example, a VM template) and install
+   other software (for example, a database, a content management system, an
+   analytics system) and then snapshot the extended image, which may itself be
+   updated just like the base image.
+
+#. **Template Pool:** One way to use block device layering is to create a pool
+   that contains (1) base images that act as templates and (2) snapshots of
+   those templates. You may then extend read-only privileges to users so that
+   they may clone the snapshots even though they do not have permissions that
+   allow them to write or execute within the pool.
+
+#. **Image Migration/Recovery:** One way to use block device layering is to
+   migrate or recover data from one pool into another pool.
+
+Protecting a Snapshot
+---------------------
+
+Clones access the parent snapshots. All clones would break if a user
+inadvertently deleted the parent snapshot. To prevent data loss, you must 
+protect the snapshot before you can clone it:
+
+.. prompt:: bash $
+
+   rbd snap protect {pool-name}/{image-name}@{snapshot-name}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd snap protect rbd/foo@snapname
+
+.. note:: You cannot delete a protected snapshot.
+
+Cloning a Snapshot
+------------------
+
+To clone a snapshot, specify the parent pool, the parent image, and the parent
+snapshot; and also the child pool together with the image name. You must
+protect the snapshot before you can clone it:
+
+.. prompt:: bash $
+
+   rbd clone {pool-name}/{parent-image-name}@{snap-name} {pool-name}/{child-image-name}
+	
+For example:
+
+.. prompt:: bash $
+
+   rbd clone rbd/foo@snapname rbd/bar
+	
+
+.. note:: You may clone a snapshot from one pool to an image in another pool.
+   For example, you may maintain read-only images and snapshots as templates in
+   one pool, and writeable clones in another pool.
+
+Unprotecting a Snapshot
+-----------------------
+
+Before you can delete a snapshot, you must first unprotect it. Additionally,
+you may *NOT* delete snapshots that have references from clones. You must
+flatten or delete each clone of a snapshot before you can unprotect the
+snapshot:
+
+.. prompt:: bash $
+
+   rbd snap unprotect {pool-name}/{image-name}@{snapshot-name}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd snap unprotect rbd/foo@snapname
+
+
+Listing Children of a Snapshot
+------------------------------
+
+To list the children of a snapshot, use the ``rbd children`` command and
+specify the pool name, the image name, and the snap name:
+
+.. prompt:: bash $
+
+   rbd children {pool-name}/{image-name}@{snapshot-name}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd children rbd/foo@snapname
+
+
+Flattening a Cloned Image
+-------------------------
+
+Cloned images retain a reference to the parent snapshot. When you remove the
+reference to the parent snapshot from the clone, you effectively "flatten" the
+clone by copying the data stored in the snapshot to the clone.  The time it
+takes to flatten a clone increases with the size of the snapshot.  To delete a
+snapshot, you must first flatten the child images (or delete them):
+
+.. prompt:: bash $
+
+   rbd flatten {pool-name}/{image-name}
+
+For example:
+
+.. prompt:: bash $
+
+   rbd flatten rbd/bar
+
+.. note:: Since a flattened image contains all the data stored in the snapshot,
+   a flattened image takes up more storage space than a layered clone does.
+
+
+.. _cephx: ../../rados/configuration/auth-config-ref/
+.. _QEMU: ../qemu-rbd/
+.. _OpenStack: ../rbd-openstack/
+.. _CloudStack: ../rbd-cloudstack/
+.. _libvirt: ../libvirt/
diff --git a/doc/rbd/rbd-windows.rst b/doc/rbd/rbd-windows.rst
new file mode 100644
index 000000000..271e85592
--- /dev/null
+++ b/doc/rbd/rbd-windows.rst
@@ -0,0 +1,158 @@
+==============
+RBD on Windows
+==============
+
+The ``rbd`` command can be used to create, remove, import, export, map or
+unmap images exactly like it would on Linux. Make sure to check the
+`RBD basic commands`_ guide.
+
+``librbd.dll`` is also available for applications that can natively use Ceph.
+
+Please check the `installation guide`_ to get started.
+
+Windows service
+===============
+On Windows, ``rbd-wnbd`` daemons are managed by a centralized service. This allows
+decoupling the daemons from the Windows session from which they originate. At
+the same time, the service is responsible of recreating persistent mappings,
+usually when the host boots.
+
+Note that only one such service may run per host.
+
+By default, all image mappings are persistent. Non-persistent mappings can be
+requested using the ``-onon-persistent`` ``rbd`` flag.
+
+Persistent mappings are recreated when the service starts, unless explicitly
+unmapped. The service disconnects the mappings when being stopped. This also
+allows adjusting the Windows service start order so that RBD images can be
+mapped before starting services that may depend on it, such as VMMS.
+
+In order to be able to reconnect the images, ``rbd-wnbd`` stores mapping
+information in the Windows registry at the following location:
+``SYSTEM\CurrentControlSet\Services\rbd-wnbd``.
+
+The following command can be used to configure the service. Please update
+the ``rbd-wnbd.exe`` path accordingly::
+
+    New-Service -Name "ceph-rbd" `
+                -Description "Ceph RBD Mapping Service" `
+                -BinaryPathName "c:\ceph\rbd-wnbd.exe service" `
+                -StartupType Automatic
+
+Note that the Ceph MSI installer takes care of creating the ``ceph-rbd``
+Windows service.
+
+Usage
+=====
+
+Integration
+-----------
+
+RBD images can be exposed to the OS and host Windows partitions or they can be
+attached to Hyper-V VMs in the same way as iSCSI disks.
+
+Starting with Openstack Wallaby, the Nova Hyper-V driver can attach RBD Cinder
+volumes to Hyper-V VMs.
+
+Mapping images
+--------------
+
+The workflow and CLI is similar to the Linux counterpart, with a few
+notable differences:
+
+* device paths cannot be requested. The disk number and path will be picked by
+  Windows. If a device path is provided by the used when mapping an image, it
+  will be used as an identifier, which can also be used when unmapping the
+  image.
+* the ``show`` command was added, which describes a specific mapping.
+  This can be used for retrieving the disk path.
+* the ``service`` command was added, allowing ``rbd-wnbd`` to run as a Windows service.
+  All mappings are by default perisistent, being recreated when the service
+  stops, unless explicitly unmapped. The service disconnects the mappings
+  when being stopped.
+* the ``list`` command also includes a ``status`` column.
+
+The purpose of the ``service`` mode is to ensure that mappings survive reboots
+and that the Windows service start order can be adjusted so that RBD images can
+be mapped before starting services that may depend on it, such as VMMS.
+
+The mapped images can either be consumed by the host directly or exposed to
+Hyper-V VMs.
+
+Hyper-V VM disks
+----------------
+
+The following sample imports an RBD image and boots a Hyper-V VM using it::
+
+    # Feel free to use any other image. This one is convenient to use for
+    # testing purposes because it's very small (~15MB) and the login prompt
+    # prints the pre-configured password.
+    wget http://download.cirros-cloud.net/0.5.1/cirros-0.5.1-x86_64-disk.img `
+         -OutFile cirros-0.5.1-x86_64-disk.img
+
+    # We'll need to make sure that the imported images are raw (so no qcow2 or vhdx).
+    # You may get qemu-img from https://cloudbase.it/qemu-img-windows/
+    # You can add the extracted location to $env:Path or update the path accordingly.
+    qemu-img convert -O raw cirros-0.5.1-x86_64-disk.img cirros-0.5.1-x86_64-disk.raw
+
+    rbd import cirros-0.5.1-x86_64-disk.raw
+    # Let's give it a hefty 100MB size.
+    rbd resize cirros-0.5.1-x86_64-disk.raw --size=100MB
+
+    rbd device map cirros-0.5.1-x86_64-disk.raw
+
+    # Let's have a look at the mappings.
+    rbd device list
+    Get-Disk
+
+    $mappingJson = rbd-wnbd show cirros-0.5.1-x86_64-disk.raw --format=json
+    $mappingJson = $mappingJson | ConvertFrom-Json
+
+    $diskNumber = $mappingJson.disk_number
+
+    New-VM -VMName BootFromRBD -MemoryStartupBytes 512MB
+    # The disk must be turned offline before it can be passed to Hyper-V VMs
+    Set-Disk -Number $diskNumber -IsOffline $true
+    Add-VMHardDiskDrive -VMName BootFromRBD -DiskNumber $diskNumber
+    Start-VM -VMName BootFromRBD
+
+Windows partitions
+------------------
+
+The following sample creates an empty RBD image, attaches it to the host and
+initializes a partition::
+
+    rbd create blank_image --size=1G
+    rbd device map blank_image -onon-persistent
+
+    $mappingJson = rbd-wnbd show blank_image --format=json
+    $mappingJson = $mappingJson | ConvertFrom-Json
+
+    $diskNumber = $mappingJson.disk_number
+
+    # The disk must be online before creating or accessing partitions.
+    Set-Disk -Number $diskNumber -IsOffline $false
+
+    # Initialize the disk, partition it and create a fileystem.
+    Get-Disk -Number $diskNumber | `
+        Initialize-Disk -PassThru | `
+        New-Partition -AssignDriveLetter -UseMaximumSize | `
+        Format-Volume -Force -Confirm:$false
+
+Limitations
+-----------
+
+At the moment, the Microsoft Failover Cluster can't use WNBD disks as
+Cluster Shared Volumes (CSVs) underlying storage. The main reason is that
+``WNBD`` and ``rbd-wnbd`` don't support the *SCSI Persistent Reservations*
+feature yet.
+
+Troubleshooting
+===============
+
+Please consult the `Windows troubleshooting`_ page.
+
+.. _Windows troubleshooting: ../../install/windows-troubleshooting
+.. _installation guide: ../../install/windows-install
+.. _RBD basic commands: ../rados-rbd-cmds
+.. _WNBD driver: https://github.com/cloudbase/wnbd
diff --git a/doc/releases/releases.yml b/doc/releases/releases.yml
new file mode 100644
index 000000000..ae7767156
--- /dev/null
+++ b/doc/releases/releases.yml
@@ -0,0 +1,448 @@
+#
+# there are two sections
+#
+# releases: ... for named releases
+# development: ... for dev releases
+#
+# by default a `version` is interpreted as a sphinx reference when rendered (see
+# schedule.rst for the existing tags such as `_13.2.2`). If a version should not
+# be treated as a reference, add `skip_ref: True` as a key/value pair sibling to
+# the `version` key.
+#
+# If a version might represent an actual number (e.g. 0.80) quote it.
+#
+releases:
+  pacific:
+    target_eol: 2023-06-01
+    releases:
+      - version: 16.2.10
+        released: 2022-07-21
+      - version: 16.2.9
+        released: 2022-05-19
+      - version: 16.2.8
+        released: 2022-05-16
+      - version: 16.2.7
+        released: 2021-12-14
+      - version: 16.2.6
+        released: 2021-09-16
+      - version: 16.2.5
+        released: 2021-07-08
+      - version: 16.2.4
+        released: 2021-05-13
+      - version: 16.2.3
+        released: 2021-05-06
+      - version: 16.2.2
+        released: 2021-05-05
+      - version: 16.2.1
+        released: 2021-04-19
+      - version: 16.2.0
+        released: 2021-03-31
+
+  octopus:
+    target_eol: 2022-06-01
+    releases:
+      - version: 15.2.7
+        released: 2020-11-30
+      - version: 15.2.6
+        released: 2020-11-18
+      - version: 15.2.5
+        released: 2020-09-16
+      - version: 15.2.4
+        released: 2020-06-30
+      - version: 15.2.3
+        released: 2020-05-29
+      - version: 15.2.2
+        released: 2020-05-18
+      - version: 15.2.1
+        released: 2020-04-09
+      - version: 15.2.0
+        released: 2020-03-23
+
+  nautilus:
+    target_eol: 2021-06-01
+    releases:
+      - version: 14.2.14
+        released: 2020-11-18
+      - version: 14.2.13
+        released: 2020-11-02
+      - version: 14.2.12
+        released: 2020-09-21
+      - version: 14.2.11
+        released: 2020-08-11
+      - version: 14.2.10
+        released: 2020-06-26
+      - version: 14.2.9
+        released: 2020-04-15
+      - version: 14.2.8
+        released: 2020-03-03
+      - version: 14.2.7
+        released: 2020-01-31
+      - version: 14.2.6
+        released: 2020-01-09
+      - version: 14.2.5
+        released: 2019-12-10
+      - version: 14.2.4
+        released: 2019-09-17
+      - version: 14.2.3
+        released: 2019-09-04
+      - version: 14.2.2
+        released: 2019-07-17
+      - version: 14.2.1
+        released: 2019-04-29
+      - version: 14.2.0
+        released: 2019-03-19
+
+  mimic:
+    target_eol: 2020-06-01
+    actual_eol: 2020-07-22
+    releases:
+      - version: 13.2.10
+        released: 2020-04-23
+      - version: 13.2.9
+        released: 2020-04-16
+      - version: 13.2.8
+        released: 2019-12-13
+      - version: 13.2.7
+        released: 2019-11-25
+      - version: 13.2.6
+        released: 2019-06-04
+      - version: 13.2.5
+        released: 2019-03-13
+      - version: 13.2.4
+        released: 2019-01-07
+      - version: 13.2.3
+        released: 2019-01-07
+      - version: 13.2.2
+        released: 2018-09-01
+      - version: 13.2.1
+        released: 2018-07-01
+      - version: 13.2.0
+        released: 2018-06-01
+  
+  luminous:
+    target_eol: 2019-06-01
+    actual_eol: 2020-03-01
+    releases:
+      - version: 12.2.13
+        released: 2020-01-31
+      - version: 12.2.12
+        released: 2019-04-12
+      - version: 12.2.11
+        released: 2019-01-31
+      - version: 12.2.10
+        released: 2018-11-27
+      - version: 12.2.9
+        released: 2018-11-01
+      - version: 12.2.8
+        released: 2018-09-01
+      - version: 12.2.7
+        released: 2018-07-01
+      - version: 12.2.6
+        released: 2018-07-01
+      - version: 12.2.5
+        released: 2018-04-01
+      - version: 12.2.4
+        released: 2018-02-01
+      - version: 12.2.3
+        released: 2018-02-01
+      - version: 12.2.2
+        released: 2017-12-01
+      - version: 12.2.1
+        released: 2017-09-01
+      - version: 12.2.0
+        released: 2017-08-01
+  
+  kraken:
+    target_eol: 2017-08-01
+    actual_eol: 2017-08-01
+    releases:
+      - version: 11.2.1
+        released: 2017-08-01
+      - version: 11.2.0
+        released: 2017-01-01
+
+  jewel:
+    target_eol: 2018-06-01
+    actual_eol: 2018-07-01
+    releases:
+      - version: 10.2.11
+        released: 2018-07-11
+      - version: 10.2.10
+        released: 2017-10-01
+      - version: 10.2.9
+        released: 2017-07-01
+      - version: 10.2.8
+        released: 2017-07-01
+      - version: 10.2.7
+        released: 2017-04-01
+      - version: 10.2.6
+        released: 2017-03-01
+      - version: 10.2.5
+        released: 2016-12-01
+      - version: 10.2.4
+        released: 2016-12-01
+      - version: 10.2.3
+        released: 2016-09-01
+      - version: 10.2.2
+        released: 2016-06-01
+      - version: 10.2.1
+        released: 2016-05-01
+      - version: 10.2.0
+        released: 2016-04-01
+
+  infernalis:
+    actual_eol: 2016-04-01
+    releases:
+      - version: 9.2.1
+        released: 2016-02-01
+      - version: 9.2.0
+        released: 2015-11-01
+
+  hammer:
+    target_eol: 2017-05-01
+    actual_eol: 2017-08-01
+    releases:
+      - version: 0.94.10
+        released: 2017-02-01
+      - version: 0.94.9
+        released: 2016-08-01
+      - version: 0.94.8
+        released: 2016-08-01
+      - version: 0.94.7
+        released: 2016-05-01
+      - version: 0.94.6
+        released: 2016-02-01
+      - version: 0.94.5
+        released: 2015-10-01
+      - version: 0.94.4
+        released: 2015-10-01
+      - version: 0.94.3
+        released: 2015-08-01
+      - version: 0.94.2
+        released: 2015-06-01
+      - version: 0.94.1
+        released: 2015-04-01
+      - version: '0.94'
+        released: 2015-04-01
+
+  giant:
+    actual_eol: 2015-04-01
+    releases:
+      - version: 0.87.2
+        released: 2015-04-01
+      - version: 0.87.1
+        released: 2015-02-01
+      - version: '0.87'
+        released: 2014-10-01
+
+  firefly:
+    target_eol: 2016-01-01
+    actual_eol: 2016-04-01
+    releases:
+      - version: 0.80.11
+        released: 2015-11-01
+      - version: 0.80.10
+        released: 2015-07-01
+      - version: 0.80.9
+        released: 2015-03-01
+      - version: 0.80.8
+        released: 2015-01-01
+      - version: 0.80.7
+        released: 2014-10-01
+      - version: 0.80.6
+        released: 2014-10-01
+      - version: 0.80.5
+        released: 2014-07-01
+      - version: 0.80.4
+        released: 2014-07-01
+      - version: 0.80.3
+        released: 2014-07-01
+      - version: 0.80.2
+        released: 2014-07-01
+      - version: 0.80.1
+        released: 2014-05-01
+      - version: '0.80'
+        released: 2014-05-01
+
+  emperor:
+    actual_eol: 2014-05-01
+    releases:
+      - version: 0.72.2
+        released: 2013-12-01
+      - version: 0.72.1
+        released: 2013-11-01
+      - version: '0.72'
+        released: 2013-11-01
+
+  dumpling:
+    target_eol: 2015-03-01
+    actual_eol: 2015-05-01
+    releases:
+      - version: 0.67.11
+        released: 2014-09-01
+      - version: 0.67.10
+        released: 2014-08-01
+      - version: 0.67.9
+        released: 2014-05-01
+      - version: 0.67.8
+        released: 2014-05-01
+      - version: 0.67.7
+        released: 2014-02-01
+      - version: 0.67.6
+        released: 2014-02-01
+      - version: 0.67.5
+        released: 2013-12-01
+      - version: 0.67.4
+        released: 2013-10-01
+      - version: 0.67.3
+        released: 2013-09-01
+      - version: 0.67.2
+        released: 2013-08-01
+      - version: 0.67.1
+        released: 2013-08-01
+      - version: '0.67'
+        released: 2013-08-01
+
+development:
+  releases:
+    - version: 15.1.1
+      released: 2020-03-13
+      skip_ref: true
+    - version: 15.1.0
+      released: 2020-01-29
+      skip_ref: true
+    - version: 15.0.0
+      released: 2019-04-03
+      skip_ref: true        
+    - version: 14.1.1
+      released:  2019-03-11
+      skip_ref: true        
+    - version: 14.1.0
+      released:  2019-02-22
+      skip_ref: true  
+    - version: 14.0.1
+      released:  2018-11-05
+      skip_ref: true
+    - version: 14.0.0
+      released:  2018-05-06
+      skip_ref: true      
+    - version: 13.1.0
+      released: 2018-05-01
+      skip_ref: true
+    - version: 13.0.2
+      released: 2018-04-01
+      skip_ref: true
+    - version: 13.0.1
+      released: 2018-02-01
+      skip_ref: true
+    - version: 13.0.0
+      released: 2017-08-01
+      skip_ref: true
+    - version: 12.1.4
+      released: 2017-08-01
+      skip_ref: true
+    - version: 12.1.3
+      released: 2017-08-01
+      skip_ref: true
+    - version: 12.1.2
+      released: 2017-08-01
+      skip_ref: true
+    - version: 12.1.1
+      released: 2017-07-01
+      skip_ref: true
+    - version: 12.1.0
+      released: 2017-06-01
+      skip_ref: true
+    - version: 12.0.3
+      released: 2017-05-01
+      skip_ref: true
+    - version: 12.0.2
+      released: 2017-04-01
+      skip_ref: true
+    - version: 12.0.1
+      released: 2017-03-01
+      skip_ref: true
+    - version: 12.0.0
+      released: 2017-02-01
+      skip_ref: true
+    - version: 11.1.1
+      released: 2017-01-01
+      skip_ref: true
+    - version: 11.1.0
+      released: 2016-12-01
+      skip_ref: true
+    - version: 11.0.2
+      released: 2016-10-01
+    - version: 11.0.1
+      released: 2016-10-01
+      skip_ref: true
+    - version: 11.0.0
+      released: 2016-06-01
+      skip_ref: true
+    - version: 10.1.2
+      released: 2016-04-01
+    - version: 10.1.1
+      released: 2016-04-01
+    - version: 10.1.0
+      released: 2016-03-01
+    - version: 10.0.5
+      released: 2016-03-01
+    - version: 10.0.4
+      released: 2016-03-01
+      skip_ref: true
+    - version: 10.0.3
+      released: 2016-02-01
+    - version: 10.0.2
+      released: 2016-01-01
+    - version: 10.0.1
+      released: 2015-12-01
+    - version: 10.0.0
+      released: 2015-11-01
+    - version: 9.1.0
+      released: 2015-10-01
+    - version: 9.0.3
+      released: 2015-08-01
+    - version: 9.0.2
+      released: 2015-07-01
+    - version: 9.0.1
+      released: 2015-06-01
+    - version: 9.0.0
+      released: 2015-05-01
+    - version: '0.93'
+      released: 2015-02-01
+    - version: '0.92'
+      released: 2015-02-01
+    - version: '0.91'
+      released: 2015-01-01
+    - version: '0.90'
+      released: 2014-12-01
+    - version: '0.89'
+      released: 2014-12-01
+    - version: '0.88'
+      released: 2014-11-01
+    - version: '0.86'
+      released: 2014-10-01
+    - version: '0.85'
+      released: 2014-09-01
+    - version: '0.84'
+      released: 2014-08-01
+    - version: '0.83'
+      released: 2014-07-01
+    - version: '0.82'
+      released: 2014-06-01
+    - version: '0.81'
+      released: 2014-06-01
+    - version: '0.79'
+      released: 2014-04-01
+    - version: '0.78'
+      released: 2014-03-01
+    - version: '0.77'
+      released: 2014-02-01
+    - version: '0.76'
+      released: 2014-01-01
+    - version: '0.75'
+      released: 2014-01-01
+    - version: '0.74'
+      released: 2013-12-01
+    - version: '0.73'
+      released: 2013-12-01
diff --git a/doc/scripts/gen_state_diagram.py b/doc/scripts/gen_state_diagram.py
new file mode 100755
index 000000000..b084f8a74
--- /dev/null
+++ b/doc/scripts/gen_state_diagram.py
@@ -0,0 +1,242 @@
+#!/usr/bin/python3
+
+import itertools
+import re
+import sys
+
+
+def do_filter(generator):
+    return acc_lines(remove_multiline_comments(to_char(remove_single_line_comments(generator))))
+
+
+def acc_lines(generator):
+    current = ""
+    for i in generator:
+        current += i
+        if i == ';' or \
+            i == '{' or \
+            i == '}':
+            yield current.lstrip("\n")
+            current = ""
+
+
+def to_char(generator):
+    for line in generator:
+        for char in line:
+            if char != '\n':
+                yield char
+            else:
+                yield ' '
+
+
+def remove_single_line_comments(generator):
+    for i in generator:
+        if len(i) and i[0] == '#':
+            continue
+        yield re.sub(r'//.*', '', i)
+
+
+def remove_multiline_comments(generator):
+    saw = ""
+    in_comment = False
+    for char in generator:
+        if in_comment:
+            if saw == "*":
+                if char == "/":
+                    in_comment = False
+                saw = ""
+            if char == "*":
+                saw = "*"
+            continue
+        if saw == "/":
+            if char == '*':
+                in_comment = True
+                saw = ""
+                continue
+            else:
+                yield saw
+                saw = ""
+        if char == '/':
+            saw = "/"
+            continue
+        yield char
+
+
+class StateMachineRenderer(object):
+    def __init__(self):
+        self.states = {} # state -> parent
+        self.machines = {} # state-> initial
+        self.edges = {} # event -> [(state, state)]
+
+        self.context = [] # [(context, depth_encountered)]
+        self.context_depth = 0
+        self.state_contents = {}
+        self.subgraphnum = 0
+        self.clusterlabel = {}
+
+        self.color_palette = itertools.cycle([
+            "#000000",  # black
+            "#1e90ff",  # dodgerblue
+            "#ff0000",  # red
+            "#0000ff",  # blue
+            "#ffa500",  # orange
+            "#40e0d0",  # turquoise
+            "#c71585",  # mediumvioletred
+        ])
+
+    def __str__(self):
+        return f'''-------------------
+
+ states: {self.states}
+
+ machines: {self.machines}
+
+ edges: {self.edges}
+
+ context: {self.context}
+
+ state_contents: {self.state_contents}
+
+--------------------'''
+
+    def read_input(self, input_lines):
+        previous_line = None
+        for line in input_lines:
+            self.get_state(line)
+            self.get_event(line)
+            # pass two lines at a time to get the context so that regexes can
+            # match on split signatures
+            self.get_context(line, previous_line)
+            previous_line = line
+
+    def get_context(self, line, previous_line):
+        match = re.search(r"(\w+::)*::(?P<tag>\w+)::\w+\(const (?P<event>\w+)", line)
+        if match is None and previous_line is not None:
+            # it is possible that we need to match on the previous line as well, so join
+            # them to make them one line and try and get this matching
+            joined_line = ' '.join([previous_line, line])
+            match = re.search(r"(\w+::)*::(?P<tag>\w+)::\w+\(\s*const (?P<event>\w+)", joined_line)
+        if match is not None:
+            self.context.append((match.group('tag'), self.context_depth, match.group('event')))
+        if '{' in line:
+            self.context_depth += 1
+        if '}' in line:
+            self.context_depth -= 1
+            while len(self.context) and self.context[-1][1] == self.context_depth:
+                self.context.pop()
+
+    def get_state(self, line):
+        if "boost::statechart::state_machine" in line:
+            tokens = re.search(
+                r"boost::statechart::state_machine<\s*(\w*),\s*(\w*)\s*>",
+                line)
+            if tokens is None:
+                raise Exception("Error: malformed state_machine line: " + line)
+            self.machines[tokens.group(1)] = tokens.group(2)
+            self.context.append((tokens.group(1), self.context_depth, ""))
+            return
+        if "boost::statechart::state" in line:
+            tokens = re.search(
+                r"boost::statechart::state<\s*(\w*),\s*(\w*)\s*,?\s*(\w*)\s*>",
+                line)
+            if tokens is None:
+                raise Exception("Error: malformed state line: " + line)
+            self.states[tokens.group(1)] = tokens.group(2)
+            if tokens.group(2) not in self.state_contents.keys():
+                self.state_contents[tokens.group(2)] = []
+            self.state_contents[tokens.group(2)].append(tokens.group(1))
+            if tokens.group(3):
+                self.machines[tokens.group(1)] = tokens.group(3)
+            self.context.append((tokens.group(1), self.context_depth, ""))
+            return
+
+    def get_event(self, line):
+        if "boost::statechart::transition" in line:
+            for i in re.finditer(r'boost::statechart::transition<\s*([\w:]*)\s*,\s*(\w*)\s*>',
+                                 line):
+                if i.group(1) not in self.edges.keys():
+                    self.edges[i.group(1)] = []
+                if not self.context:
+                    raise Exception("no context at line: " + line)
+                self.edges[i.group(1)].append((self.context[-1][0], i.group(2)))
+        i = re.search("return\s+transit<\s*(\w*)\s*>()", line)
+        if i is not None:
+            if not self.context:
+                raise Exception("no context at line: " + line)
+            if not self.context[-1][2]:
+                raise Exception("no event in context at line: " + line)
+            if self.context[-1][2] not in self.edges.keys():
+                self.edges[self.context[-1][2]] = []
+            self.edges[self.context[-1][2]].append((self.context[-1][0], i.group(1)))
+
+    def emit_dot(self, output):
+        top_level = []
+        for state in self.machines.keys():
+            if state not in self.states.keys():
+                top_level.append(state)
+        print('Top Level States: ', top_level, file=sys.stderr)
+        print('digraph G {', file=output)
+        print('\tsize="7,7"', file=output)
+        print('\tcompound=true;', file=output)
+        for i in self.emit_state(top_level[0]):
+            print('\t' + i, file=output)
+        for i in self.edges.keys():
+            for j in self.emit_event(i):
+                print(j, file=output)
+        print('}', file=output)
+
+    def emit_state(self, state):
+        if state in self.state_contents.keys():
+            self.clusterlabel[state] = "cluster%s" % (str(self.subgraphnum),)
+            yield "subgraph cluster%s {" % (str(self.subgraphnum),)
+            self.subgraphnum += 1
+            yield """\tlabel = "%s";""" % (state,)
+            yield """\tcolor = "black";"""
+
+            if state in self.machines.values():
+                yield """\tstyle = "filled";"""
+                yield """\tfillcolor = "lightgrey";"""
+
+            for j in self.state_contents[state]:
+                for i in self.emit_state(j):
+                    yield "\t"+i
+            yield "}"
+        else:
+            found = False
+            for (k, v) in self.machines.items():
+                if v == state:
+                    yield state+"[shape=Mdiamond style=filled fillcolor=lightgrey];"
+                    found = True
+                    break
+            if not found:
+                yield state+";"
+
+    def emit_event(self, event):
+        def append(app):
+            retval = "["
+            for i in app:
+                retval += (i + ",")
+            retval += "]"
+            return retval
+
+        for (fro, to) in self.edges[event]:
+            color = next(self.color_palette)
+            appendix = ['label="%s"' % (event,),
+                        'color="%s"' % (color,),
+                        'fontcolor="%s"' % (color,)]
+            if fro in self.machines.keys():
+                appendix.append("ltail=%s" % (self.clusterlabel[fro],))
+                while fro in self.machines.keys():
+                    fro = self.machines[fro]
+            if to in self.machines.keys():
+                appendix.append("lhead=%s" % (self.clusterlabel[to],))
+                while to in self.machines.keys():
+                    to = self.machines[to]
+            yield("%s -> %s %s;" % (fro, to, append(appendix)))
+
+
+if __name__ == '__main__':
+    INPUT_GENERATOR = do_filter(line for line in sys.stdin)
+    RENDERER = StateMachineRenderer()
+    RENDERER.read_input(INPUT_GENERATOR)
+    RENDERER.emit_dot(output=sys.stdout)
diff --git a/doc/security/CVE-2021-20288.rst b/doc/security/CVE-2021-20288.rst
new file mode 100644
index 000000000..fa3b073cb
--- /dev/null
+++ b/doc/security/CVE-2021-20288.rst
@@ -0,0 +1,183 @@
+.. _CVE-2021-20288:
+
+CVE-2021-20288: Unauthorized global_id reuse in cephx
+=====================================================
+
+* `NIST information page <https://nvd.nist.gov/vuln/detail/CVE-2021-20288>`_
+
+Summary
+-------
+
+Ceph was not ensuring that reconnecting/renewing clients were
+presenting an existing ticket when reclaiming their global_id value.
+An attacker that was able to authenticate could claim a global_id in
+use by a different client and potentially disrupt
+other cluster services.
+
+Background
+----------
+
+Each authenticated client or daemon in Ceph is assigned a numeric
+global_id identifier. That value is assumed to be unique across the
+cluster.  When clients reconnect to the monitor (e.g., due to a
+network disconnection) or renew their ticket, they are supposed to
+present their old ticket to prove prior possession of their global_id
+so that it can be reclaimed and thus remain constant over the lifetime
+of that client instance.
+
+Ceph was not correctly checking that the old ticket was valid, allowing
+an arbitrary global_id to be reclaimed, even if it was in use by another
+active client in the system.
+
+Attacker Requirements
+---------------------
+
+Any potential attacker must:
+
+* have a valid authentication key for the cluster
+* know or guess the global_id of another client
+* run a modified version of the Ceph client code to reclaim another client's global_id
+* construct appropriate client messages or requests to disrupt service or exploit
+  Ceph daemon assumptions about global_id uniqueness
+
+Impact
+------
+
+Confidentiality Impact
+______________________
+
+None
+
+Integrity Impact
+________________
+
+Partial.  An attacker could potentially exploit assumptions around
+global_id uniqueness to disrupt other clients' access or disrupt
+Ceph daemons.
+
+Availability Impact
+___________________
+
+High.  An attacker could potentially exploit assumptions around
+global_id uniqueness to disrupt other clients' access or disrupt
+Ceph daemons.
+
+Access Complexity
+_________________
+
+High.  The client must make use of modified client code in order to
+exploit specific assumptions in the behavior of other Ceph daemons.
+
+Authentication
+______________
+
+Yes.  The attacker must also be authenticated and have access to the
+same services as a client it is wishing to impersonate or disrupt.
+
+Gained Access
+_____________
+
+Partial.  An attacker can partially impersonate another client.
+
+Affected versions
+-----------------
+
+All prior versions of Ceph monitors fail to ensure that global_id reclaim
+attempts are authentic.
+
+In addition, all user-space daemons and clients starting from Luminous v12.2.0
+were failing to securely reclaim their global_id following commit a2eb6ae3fb57
+("mon/monclient: hunt for multiple monitor in parallel").
+
+All versions of the Linux kernel client properly authenticate.
+
+Fixed versions
+--------------
+
+* Pacific v16.2.1 (and later)
+* Octopus v15.2.11 (and later)
+* Nautilus v14.2.20 (and later)
+
+
+Fix details
+-----------
+
+#. Patched monitors now properly require that clients securely reclaim
+   their global_id when the ``auth_allow_insecure_global_id_reclaim``
+   is ``false``.  Initially, by default, this option is set to
+   ``true`` so that existing clients can continue to function without
+   disruption until all clients have been upgraded.  When this option
+   is set to false, then an unpatched client will not be able to reconnect
+   to the cluster after an intermittent network disruption breaking
+   its connect to a monitor, or be able to renew its authentication
+   ticket when it times out (by default, after 72 hours).
+
+   Patched monitors raise the ``AUTH_INSECURE_GLOBAL_ID_RECLAIM_ALLOWED``
+   health alert if ``auth_allow_insecure_global_id_reclaim`` is enabled.
+   This health alert can be muted with::
+
+     ceph health mute AUTH_INSECURE_GLOBAL_ID_RECLAIM_ALLOWED 1w
+
+   Although it is not recommended, the alert can also be disabled with::
+
+     ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false
+
+#. Patched monitors can disconnect new clients right after they have
+   authenticated (forcing them to reconnect and reclaim) in order to
+   determine whether they securely reclaim global_ids.  This allows
+   the cluster and users to discover quickly whether clients would be
+   affected by requiring secure global_id reclaim: most clients will
+   report an authentication error immediately.  This behavior can be
+   disabled by setting ``auth_expose_insecure_global_id_reclaim`` to
+   ``false``::
+
+     ceph config set mon auth_expose_insecure_global_id_reclaim false
+
+#. Patched monitors will raise the ``AUTH_INSECURE_GLOBAL_ID_RECLAIM`` health
+   alert for any clients or daemons that are not securely reclaiming their
+   global_id.  These clients should be upgraded before disabling the
+   ``auth_allow_insecure_global_id_reclaim`` option to avoid disrupting
+   client access.
+
+   By default (if ``auth_expose_insecure_global_id_reclaim`` has not
+   been disabled), clients' failure to securely reclaim global_id will
+   immediately be exposed and raise this health alert.
+   However, if ``auth_expose_insecure_global_id_reclaim`` has been
+   disabled, this alert will not be triggered for a client until it is
+   forced to reconnect to a monitor (e.g., due to a network disruption)
+   or the client renews its authentication ticket (by default, after
+   72 hours).
+
+#. The default time-to-live (TTL) for authentication tickets has been increased
+   from 12 hours to 72 hours.  Because we previously were not ensuring that
+   a client's prior ticket was valid when reclaiming their global_id, a client
+   could tolerate a network outage that lasted longer than the ticket TTL and still
+   reclaim its global_id.  Once the cluster starts requiring secure global_id reclaim,
+   a client that is disconnected for longer than the TTL may fail to reclaim its global_id,
+   fail to reauthenticate, and be unable to continue communicating with the cluster
+   until it is restarted.  The default TTL was increased to minimize the impact of this
+   change on users.
+
+
+Recommendations
+---------------
+
+#. Users should upgrade to a patched version of Ceph at their earliest
+   convenience.
+
+#. Users should upgrade any unpatched clients at their earliest
+   convenience.  By default, these clients can be easily identified by
+   checking the ``ceph health detail`` output for the
+   ``AUTH_INSECURE_GLOBAL_ID_RECLAIM`` alert.
+
+#. If all clients cannot be upgraded immediately, the health alerts can be
+   temporarily muted with::
+
+     ceph health mute AUTH_INSECURE_GLOBAL_ID_RECLAIM 1w  # 1 week
+     ceph health mute AUTH_INSECURE_GLOBAL_ID_RECLAIM_ALLOWED 1w  # 1 week
+
+#. After all clients have been updated and the ``AUTH_INSECURE_GLOBAL_ID_RECLAIM``
+   alert is no longer present, the cluster should be set to prevent insecure
+   global_id reclaim with::
+
+     ceph config set mon auth_allow_insecure_global_id_reclaim false
diff --git a/doc/security/CVE-2021-3509.rst b/doc/security/CVE-2021-3509.rst
new file mode 100644
index 000000000..7e865e9b2
--- /dev/null
+++ b/doc/security/CVE-2021-3509.rst
@@ -0,0 +1,28 @@
+.. _CVE-2021-3509:
+
+CVE-2021-3509: Dashboard XSS via token cookie
+=============================================
+
+* `NIST information page <https://nvd.nist.gov/vuln/detail/CVE-2021-3509>`_
+
+The Ceph Dashboard was vulnerable to an XSS attack that could expose the authentication
+cookie to other sites.
+
+
+Affected versions
+-----------------
+
+* Octopus v15.2.0 and later
+
+Fixed versions
+--------------
+
+* Pacific v16.2.4 (and later)
+* Octopus v15.2.12 (and later)
+* Nautilus v14.2.21 (and later)
+
+
+Recommendations
+---------------
+
+All users of the Ceph dashboard should upgrade.
diff --git a/doc/security/CVE-2021-3524.rst b/doc/security/CVE-2021-3524.rst
new file mode 100644
index 000000000..4d627c071
--- /dev/null
+++ b/doc/security/CVE-2021-3524.rst
@@ -0,0 +1,30 @@
+.. _CVE-2021-3524:
+
+CVE-2021-3524: HTTP header injects via CORS in RGW
+==================================================
+
+* `NIST information page <https://nvd.nist.gov/vuln/detail/CVE-2021-3524>`_
+
+A flaw was found in the radosgw.  The vulnerability is related to the
+injection of HTTP headers via a CORS ExposeHeader tag. The \r
+character in the ExposeHeader tag in the CORS configuration file
+generates a header injection in the response when the CORS request is
+made.
+
+Fixed versions
+--------------
+
+* Pacific v16.2.4 (and later)
+* Octopus v15.2.12 (and later)
+* Nautilus v14.2.21 (and later)
+
+Recommendations
+---------------
+
+All users of Ceph object storage (RGW) should upgrade.
+
+Acknowledgements
+----------------
+
+Red Hat would like to thank Sergey Bobrov (Kaspersky) for reporting this issue.
+
diff --git a/doc/security/CVE-2021-3531.rst b/doc/security/CVE-2021-3531.rst
new file mode 100644
index 000000000..907cb4764
--- /dev/null
+++ b/doc/security/CVE-2021-3531.rst
@@ -0,0 +1,28 @@
+.. _CVE-2021-3531:
+
+CVE-2021-3531: Swift API denial of service
+==========================================
+
+* `NIST information page <https://nvd.nist.gov/vuln/detail/CVE-2021-3531>`_
+
+Unauthenticated users of the Swift API can trigger a server-side assertion with a
+malformed URL, leading to a denial of service.
+
+
+Affected versions
+-----------------
+
+* Nautilus v14.2.0 and later
+
+Fixed versions
+--------------
+
+* Pacific v16.2.4 (and later)
+* Octopus v15.2.12 (and later)
+* Nautilus v14.2.21 (and later)
+
+
+Recommendations
+---------------
+
+All users of Ceph object storage (RGW) should upgrade.
diff --git a/doc/security/CVE-2022-0670.rst b/doc/security/CVE-2022-0670.rst
new file mode 100644
index 000000000..e7863d1d9
--- /dev/null
+++ b/doc/security/CVE-2022-0670.rst
@@ -0,0 +1,43 @@
+.. _CVE-2022-0670:
+
+CVE-2022-0670: Native-CephFS Manila Path-restriction bypass
+===========================================================
+
+Summary
+-------
+
+Users who were running OpenStack Manila to export native CephFS and who
+upgraded their Ceph cluster from Nautilus (or earlier) to a later
+major version were vulnerable to an attack by malicious users. The
+vulnerability allowed users to obtain access to arbitrary portions of
+the CephFS filesystem hierarchy instead of being properly restricted
+to their own subvolumes. The vulnerability is due to a bug in the
+"volumes" plugin in Ceph Manager. This plugin is responsible for
+managing Ceph File System subvolumes, which are used by OpenStack
+Manila services as a way to provide shares to Manila users.
+
+Again, this vulnerability impacts only OpenStack Manila clusters that 
+provided native CephFS access to their users.
+
+Affected versions
+-----------------
+
+Any version of Ceph running OpenStack Manila that was upgraded from Nautilus
+or earlier.
+
+Fixed versions
+--------------
+
+* Quincy v17.2.2 (and later)
+* Pacific v16.2.10 (and later)
+* Octopus fix is forthcoming
+
+Recommendations
+---------------
+
+#. Users should upgrade to a patched version of Ceph at their earliest
+   convenience.
+
+#. Administrators who are
+   concerned they may have been impacted should audit the CephX keys in
+   their cluster for proper path restrictions.
diff --git a/doc/security/cves.rst b/doc/security/cves.rst
new file mode 100644
index 000000000..8bbccbf64
--- /dev/null
+++ b/doc/security/cves.rst
@@ -0,0 +1,114 @@
+
+Past vulnerabilities
+====================
+
++------------+-------------------+-------------+---------------------------------------------+
+| Published  | CVE               | Severity    | Summary                                     |
++------------+-------------------+-------------+---------------------------------------------+
+| 2022-07-21 | `CVE-2022-0670`_  | Medium      | Native-CephFS Manila Path-restriction bypass|
++------------+-------------------+-------------+---------------------------------------------+
+| 2021-05-13 | `CVE-2021-3531`_  | Medium      | Swift API denial of service                 |
++------------+-------------------+-------------+---------------------------------------------+
+| 2021-05-13 | `CVE-2021-3524`_  | Medium      | HTTP header injects via CORS in RGW         |
++------------+-------------------+-------------+---------------------------------------------+
+| 2021-05-13 | `CVE-2021-3509`_  | High        | Dashboard XSS via token cookie              |
++------------+-------------------+-------------+---------------------------------------------+
+| 2021-04-14 | `CVE-2021-20288`_ | High        | Unauthorized global_id reuse in cephx       |
++------------+-------------------+-------------+---------------------------------------------+
+| 2020-12-18 | `CVE-2020-27781`_ | 7.1 High    | CephFS creds read/modified by Manila users  |
++------------+-------------------+-------------+---------------------------------------------+
+| 2021-01-08 | `CVE-2020-25678`_ | 4.9 Medium  | mgr module passwords in clear text          |
++------------+-------------------+-------------+---------------------------------------------+
+| 2020-12-07 | `CVE-2020-25677`_ | 5.5 Medium  | ceph-ansible iscsi-gateway.conf perm        |
++------------+-------------------+-------------+---------------------------------------------+
+| 2020-11-23 | `CVE-2020-25660`_ | 8.8 High    | Cephx replay vulnerability                  |
++------------+-------------------+-------------+---------------------------------------------+
+| 2020-04-22 | `CVE-2020-12059`_ | 7.5 High    | malformed POST could crash RGW              |
++------------+-------------------+-------------+---------------------------------------------+
+| 2020-06-26 | `CVE-2020-10753`_ | 6.5 Medium  | HTTP header injects via CORS in RGW         |
++------------+-------------------+-------------+---------------------------------------------+
+| 2020-06-22 | `CVE-2020-10736`_ | 8.0 High    | authorization bypass in mon and mgr         |
++------------+-------------------+-------------+---------------------------------------------+
+| 2020-04-23 | `CVE-2020-1760`_  | 6.1 Medium  | potential RGW XSS attack                    |
++------------+-------------------+-------------+---------------------------------------------+
+| 2020-04-13 | `CVE-2020-1759`_  | 6.8 Medium  | Cephx nonce reuse in secure mode            |
++------------+-------------------+-------------+---------------------------------------------+
+| 2020-02-07 | `CVE-2020-1700`_  | 6.5 Medium  | RGW disconnects leak sockets, can DoS       |
++------------+-------------------+-------------+---------------------------------------------+
+| 2020-04-21 | `CVE-2020-1699`_  | 7.5 High    | Dashboard path traversal flaw               |
++------------+-------------------+-------------+---------------------------------------------+
+| 2019-12-23 | `CVE-2019-19337`_ | 6.5 Medium  | RGW DoS via malformed headers               |
++------------+-------------------+-------------+---------------------------------------------+
+| 2019-11-08 | `CVE-2019-10222`_ | 7.5 High    | Invalid HTTP headers could crash RGW        |
++------------+-------------------+-------------+---------------------------------------------+
+| 2019-03-27 | `CVE-2019-3821`_  | 7.5 High    | RGW file descriptors could be exhausted     |
++------------+-------------------+-------------+---------------------------------------------+
+| 2019-01-28 | `CVE-2018-16889`_ | 7.5 High    | encryption keys logged in plaintext         |
++------------+-------------------+-------------+---------------------------------------------+
+| 2019-01-15 | `CVE-2018-16846`_ | 6.5 Medium  | authenticated RGW users can cause DoS       |
++------------+-------------------+-------------+---------------------------------------------+
+| 2019-01-15 | `CVE-2018-14662`_ | 5.7 Medium  | read-only users could steal dm-crypt keys   |
++------------+-------------------+-------------+---------------------------------------------+
+| 2018-07-10 | `CVE-2018-10861`_ | 8.1 High    | authenticated user can create/delete pools  |
++------------+-------------------+-------------+---------------------------------------------+
+| 2018-03-19 | `CVE-2018-7262`_  | 7.5 High    | malformed headers can cause RGW DoS         |
++------------+-------------------+-------------+---------------------------------------------+
+| 2018-07-10 | `CVE-2018-1129`_  | 6.5 Medium  | network MITM can tamper with messages       |
++------------+-------------------+-------------+---------------------------------------------+
+| 2018-07-10 | `CVE-2018-1128`_  | 7.5 High    | Cephx replay vulnerability                  |
++------------+-------------------+-------------+---------------------------------------------+
+| 2018-07-27 | `CVE-2017-7519`_  | 4.4 Medium  | libradosstriper unvalidated format string   |
++------------+-------------------+-------------+---------------------------------------------+
+| 2018-08-01 | `CVE-2016-9579`_  | 7.6 High    | potential RGW XSS attack                    |
++------------+-------------------+-------------+---------------------------------------------+
+| 2018-07-31 | `CVE-2016-8626`_  | 6.5 Medium  | malformed POST can DoS RGW                  |
++------------+-------------------+-------------+---------------------------------------------+
+| 2016-10-03 | `CVE-2016-7031`_  | 7.5 High    | RGW unauthorized bucket listing             |
++------------+-------------------+-------------+---------------------------------------------+
+| 2016-07-12 | `CVE-2016-5009`_  | 6.5 Medium  | mon command handler DoS                     |
++------------+-------------------+-------------+---------------------------------------------+
+| 2016-12-03 | `CVE-2015-5245`_  |             | RGW header injection                        |
++------------+-------------------+-------------+---------------------------------------------+
+
+.. toctree::
+   :hidden:
+   :maxdepth: 0
+
+    CVE-2022-0670 <CVE-2022-0670.rst>
+    CVE-2021-3531 <CVE-2021-3531.rst>
+    CVE-2021-3524 <CVE-2021-3524.rst>
+    CVE-2021-3509 <CVE-2021-3509.rst>
+    CVE-2021-20288 <CVE-2021-20288.rst>
+
+.. _CVE-2022-0670: ../CVE-2022-0670
+.. _CVE-2021-3531: ../CVE-2021-3531
+.. _CVE-2021-3524: ../CVE-2021-3524
+.. _CVE-2021-3509: ../CVE-2021-3509
+.. _CVE-2021-20288: ../CVE-2021-20288
+.. _CVE-2020-27781: https://nvd.nist.gov/vuln/detail/CVE-2020-27781
+.. _CVE-2020-25678: https://nvd.nist.gov/vuln/detail/CVE-2020-25678
+.. _CVE-2020-25677: https://nvd.nist.gov/vuln/detail/CVE-2020-25677
+.. _CVE-2020-25660: https://nvd.nist.gov/vuln/detail/CVE-2020-25660
+.. _CVE-2020-12059: https://nvd.nist.gov/vuln/detail/CVE-2020-12059
+.. _CVE-2020-10753: https://nvd.nist.gov/vuln/detail/CVE-2020-10753
+.. _CVE-2020-10736: https://nvd.nist.gov/vuln/detail/CVE-2020-10736
+.. _CVE-2020-1760: https://nvd.nist.gov/vuln/detail/CVE-2020-1760
+.. _CVE-2020-1759: https://nvd.nist.gov/vuln/detail/CVE-2020-1759
+.. _CVE-2020-1700: https://nvd.nist.gov/vuln/detail/CVE-2020-1700
+.. _CVE-2020-1699: https://nvd.nist.gov/vuln/detail/CVE-2020-1699
+.. _CVE-2019-19337: https://nvd.nist.gov/vuln/detail/CVE-2019-19337
+.. _CVE-2019-10222: https://nvd.nist.gov/vuln/detail/CVE-2019-10222
+.. _CVE-2019-3821: https://nvd.nist.gov/vuln/detail/CVE-2019-3821
+.. _CVE-2018-16889: https://nvd.nist.gov/vuln/detail/CVE-2018-16889
+.. _CVE-2018-16846: https://nvd.nist.gov/vuln/detail/CVE-2018-16846
+.. _CVE-2018-14662: https://nvd.nist.gov/vuln/detail/CVE-2018-14662
+.. _CVE-2018-10861: https://nvd.nist.gov/vuln/detail/CVE-2018-10861
+.. _CVE-2018-7262: https://nvd.nist.gov/vuln/detail/CVE-2018-7262
+.. _CVE-2018-1129: https://nvd.nist.gov/vuln/detail/CVE-2018-1129
+.. _CVE-2018-1128: https://nvd.nist.gov/vuln/detail/CVE-2018-1128
+.. _CVE-2017-7519: https://nvd.nist.gov/vuln/detail/CVE-2017-7519
+.. _CVE-2016-9579: https://nvd.nist.gov/vuln/detail/CVE-2016-9579
+.. _CVE-2016-8626: https://nvd.nist.gov/vuln/detail/CVE-2016-8626
+.. _CVE-2016-7031: https://nvd.nist.gov/vuln/detail/CVE-2016-7031
+.. _CVE-2016-5009: https://nvd.nist.gov/vuln/detail/CVE-2016-5009
+.. _CVE-2015-5245: https://nvd.nist.gov/vuln/detail/CVE-2015-5245
diff --git a/doc/security/index.rst b/doc/security/index.rst
new file mode 100644
index 000000000..682d73d24
--- /dev/null
+++ b/doc/security/index.rst
@@ -0,0 +1,44 @@
+:orphan:
+
+==========
+ Security
+==========
+
+.. toctree::
+   :maxdepth: 1
+
+   Past Vulnerabilities / CVEs <cves>
+   Vulnerability Management Process <process>
+
+Reporting a vulnerability
+=========================
+
+To report a vulnerability, please send email to `security@ceph.io
+<security@ceph.io>`_.
+
+* Please do not file a public ceph tracker issue for a vulnerability.
+* We urge reporters to provide as much information as is practicable
+  (a reproducer, versions affected, fix if available, etc.), as this
+  can speed up the process considerably.
+* Please let us know to whom credit should be given and with what
+  affiliations.
+* If this issue is not yet disclosed publicly and you have any
+  disclosure date in mind, please share the same along with the
+  report.
+
+Although you are not required to, you may encrypt your message using 
+the following GPG key:
+
+**6EEF26FFD4093B99: Ceph Security Team (security@ceph.io)**
+
+| **Download:** `MIT PGP Public Key Server <https://pgp.mit.edu/pks/lookup?op=vindex&search=0x6EEF26FFD4093B99>`_
+| **Fingerprint:** A527 D019 21F9 7178 C232 66C1 6EEF 26FF D409 3B99
+
+
+Supported versions
+==================
+
+Security updates are applied only to the current `Active Releases`_.
+
+
+.. _Active Releases: https://docs.ceph.com/en/latest/releases/#active-releases
diff --git a/doc/security/process.rst b/doc/security/process.rst
new file mode 100644
index 000000000..c8a2031ca
--- /dev/null
+++ b/doc/security/process.rst
@@ -0,0 +1,48 @@
+Vulnerability Management Process
+================================
+
+#. The report will be acknowledged within three business days.
+#. The team will investigate the reported issue and will update the email
+   thread with relevant information. The team may ask for additional
+   information regarding the reported issue.
+#. If the team does not confirm the report, no further action will be
+   taken and the issue will be closed.
+#. If the report is confirmed by Ceph team members, a unique CVE identifier
+   will be assigned to the report and then shared with the reporter. The Ceph
+   security team will start working on a fix. 
+#. If a reporter has no disclosure date in mind, a Ceph security team
+   member will coordinate a release date (CRD) with the list members
+   and share the mutually agreed disclosure date with the reporter.
+#. The vulnerability disclosure / release date is set excluding Friday and
+   holiday periods.
+#. Embargoes are preferred for Critical and High impact
+   issues. Embargo should not be held for more than 90 days from the
+   date of vulnerability confirmation, except under unusual
+   circumstances. For Low and Moderate issues with limited impact and
+   an easy workaround or where an issue that is already public, a
+   standard patch release process will be followed to fix the
+   vulnerability once CVE is assigned.
+#. Fixes for issues of "Medium" and "Low" severity will be released as part of
+   the next standard release cycle. List members will receive seven days of
+   advance notice prior to the release date of these fixes. The details of the
+   CVE fix will be included in the release notes, and the release notes will be
+   linked in the public announcement.
+#. Commits will be handled in a private repository for review and
+   testing and a new patch version will be released from this private
+   repository.
+#. If a vulnerability is unintentionally already fixed in the public
+   repository, a few days are given to downstream stakeholders/vendors
+   to prepare for updating before the public disclosure.
+#. An announcement will be made disclosing the vulnerability. The
+   fastest place to receive security announcements is via the
+   `ceph-announce@ceph.io <ceph-announce@ceph.io>`_ or
+   `oss-security@lists.openwall.com <oss-security@lists.openwall.com>`_ mailing
+   lists.  (These lists are low-traffic).
+
+If the report is considered embargoed, we ask you to not disclose the
+vulnerability before it has been fixed and announced, unless you
+received a response from the Ceph security team that you can do
+so. This holds true until the public disclosure date that was agreed
+upon by the list. Thank you for improving the security of Ceph and its
+ecosystem. Your efforts and responsible disclosure are greatly
+appreciated and will be acknowledged.
diff --git a/doc/start/ceph.conf b/doc/start/ceph.conf
new file mode 100644
index 000000000..f3d558eb0
--- /dev/null
+++ b/doc/start/ceph.conf
@@ -0,0 +1,3 @@
+[global]
+	# list your monitors here
+	mon host = {mon-host-1}, {mon-host-2}
diff --git a/doc/start/documenting-ceph.rst b/doc/start/documenting-ceph.rst
new file mode 100644
index 000000000..93b266d16
--- /dev/null
+++ b/doc/start/documenting-ceph.rst
@@ -0,0 +1,793 @@
+==================
+ Documenting Ceph
+==================
+
+The **easiest way** to help the Ceph project is to contribute to the
+documentation. As the Ceph user base grows and the development pace quickens, an
+increasing number of people are updating the documentation and adding new
+information. Even small contributions like fixing spelling errors or clarifying
+instructions will help the Ceph project immensely.
+
+The Ceph documentation source resides in the ``ceph/doc`` directory of the Ceph
+repository, and Python Sphinx renders the source into HTML and manpages. The
+http://ceph.com/docs link currently displays the  ``master`` branch by default,
+but you may view documentation for older branches (e.g., ``argonaut``) or future
+branches (e.g., ``next``) as well as work-in-progress branches by substituting
+``master`` with the branch name you prefer.
+
+Another way to suggest a documentation correction is to make a pull request.
+The instructions for making a pull request against the Ceph documentation are
+in the section :ref:`making_contributions`.
+
+If this is your first time making an improvement to the documentation or
+if you have noticed a small mistake (such as a spelling error or a typo),
+it will be easier to send an email than to make a pull request. You will
+be credited for the improvement unless you instruct Ceph Upstream
+Documentation not to credit you.
+
+Location of the Documentation in the Repository
+===============================================
+
+The Ceph documentation source is in the ``ceph/doc`` directory of the Ceph
+repository. Python Sphinx renders the source into HTML and manpages. 
+
+Viewing Old Ceph Documentation
+==============================
+The https://docs.ceph.com link displays the latest release branch by default
+(for example, if "Quincy" is the most recent release, then by default
+https://docs.ceph.com displays the documentation for Quincy), but you can view
+the documentation for older versions of Ceph (for example, ``pacific``) by
+replacing the version name in the url (for example, ``quincy`` in
+`https://docs.ceph.com/en/pacific <https://docs.ceph.com/en/quincy>`_) with the
+branch name you prefer (for example, ``pacific``, to create a URL that reads
+`https://docs.ceph.com/en/pacific/ <https://docs.ceph.com/en/pacific/>`_).
+
+.. _making_contributions:
+
+Making Contributions
+====================
+
+Making a documentation contribution generally involves the same procedural
+sequence as making a code contribution, except that you must build documentation
+source instead of compiling program source. The sequence includes the following
+steps:
+
+#. `Get the Source`_
+#. `Select a Branch`_
+#. `Make a Change`_
+#. `Build the Source`_
+#. `Commit the Change`_
+#. `Push the Change`_
+#. `Make a Pull Request`_
+#. `Notify Us`_
+
+Get the Source
+--------------
+
+Ceph documentation lives in the Ceph repository right alongside the Ceph source
+code under the ``ceph/doc`` directory. For details on github and Ceph,
+see :ref:`Get Involved`.
+
+The most common way to make contributions is to use the `Fork and Pull`_
+approach. You must:
+
+#. Install git locally. For Debian/Ubuntu, execute:
+
+   .. prompt:: bash $
+
+	sudo apt-get install git
+
+   For Fedora, execute:
+
+   .. prompt:: bash $
+
+	sudo yum install git
+
+   For CentOS/RHEL, execute:
+
+   .. prompt:: bash $
+
+	sudo yum install git
+
+#. Ensure your ``.gitconfig`` file has your name and email address. :
+
+   .. code-block:: ini
+
+	[user]
+	   email = {your-email-address}
+	   name = {your-name}
+
+   For example:
+
+   .. prompt:: bash $
+
+	git config --global user.name "John Doe"
+	git config --global user.email johndoe@example.com
+
+
+#. Create a  `github`_ account (if you don't have one).
+
+#. Fork the Ceph project. See https://github.com/ceph/ceph.
+
+#. Clone your fork of the Ceph project to your local host.
+
+
+Ceph organizes documentation into an information architecture primarily by its
+main components.
+
+- **Ceph Storage Cluster:** The Ceph Storage Cluster documentation resides
+  under the ``doc/rados`` directory.
+  
+- **Ceph Block Device:** The Ceph Block Device documentation resides under
+  the ``doc/rbd`` directory.
+  
+- **Ceph Object Storage:** The Ceph Object Storage documentation resides under
+  the ``doc/radosgw`` directory.
+
+- **Ceph File System:** The Ceph File System documentation resides under the 
+  ``doc/cephfs`` directory.
+  
+- **Installation (Quick):** Quick start documentation resides under the
+  ``doc/start`` directory.
+  
+- **Installation (Manual):** Manual installation documentation resides under
+  the ``doc/install`` directory.
+  
+- **Manpage:** Manpage source resides under the ``doc/man`` directory.
+
+- **Developer:** Developer documentation resides under the ``doc/dev`` 
+  directory.
+
+- **Images:** If you include images such as JPEG or PNG files, you should 
+  store them under the ``doc/images`` directory.
+
+
+Select a Branch
+---------------
+
+When you make small changes to the documentation, such as fixing typographical
+errors or clarifying explanations, use the ``main`` branch (default). You
+should also use the ``main`` branch when making contributions to features that
+are in the current release. ``main`` is the most commonly used branch. :
+
+.. prompt:: bash $
+
+	git checkout main
+
+When you make changes to documentation that affect an upcoming release, use 
+the ``next`` branch. ``next`` is the second most commonly used branch. :
+
+.. prompt:: bash $
+
+	git checkout next
+
+When you are making substantial contributions such as new features that are not
+yet in the current release; if your contribution is related to an issue with a
+tracker ID; or, if you want to see your documentation rendered on the Ceph.com
+website before it gets merged into the ``main`` branch, you should create a
+branch. To distinguish branches that include only documentation updates, we
+prepend them with ``wip-doc`` by convention, following the form
+``wip-doc-{your-branch-name}``. If the branch relates to an issue filed in
+http://tracker.ceph.com/issues, the branch name incorporates the issue number.
+For example, if a documentation branch is a fix for issue #4000, the branch name
+should be ``wip-doc-4000`` by convention and the relevant tracker URL will be
+http://tracker.ceph.com/issues/4000.
+
+.. note:: Please do not mingle documentation contributions and source code
+   contributions in a single commit. When you keep documentation
+   commits separate from source code commits, it simplifies the review
+   process. We highly recommend that any pull request that adds a feature or
+   a configuration option should also include a documentation commit that
+   describes the changes.
+
+Before you create your branch name, ensure that it doesn't already exist in the
+local or remote repository. :
+
+.. prompt:: bash $
+
+	git branch -a | grep wip-doc-{your-branch-name}
+
+If it doesn't exist, create your branch:
+
+.. prompt:: bash $
+
+	git checkout -b wip-doc-{your-branch-name}
+
+
+Make a Change
+-------------
+
+Modifying a document involves opening a reStructuredText file, changing
+its contents, and saving the changes. See `Documentation Style Guide`_ for
+details on syntax requirements.
+
+Adding a document involves creating a new reStructuredText file within the
+``doc`` directory tree with a ``*.rst``
+extension. You must also include a reference to the document: a hyperlink
+or a table of contents entry. The ``index.rst`` file of a top-level directory
+usually contains a TOC, where you can add the new file name. All documents must
+have a title. See `Headings`_ for details.
+
+Your new document doesn't get tracked by ``git`` automatically. When you want 
+to add the document to the repository,  you must use ``git add 
+{path-to-filename}``. For example, from the top level  directory of the
+repository, adding an ``example.rst`` file to the ``rados`` subdirectory would
+look like this:
+
+.. prompt:: bash $
+
+	git add doc/rados/example.rst
+
+Deleting a document involves removing it from the repository with ``git rm
+{path-to-filename}``. For example:
+
+.. prompt:: bash $
+
+	git rm doc/rados/example.rst
+
+You must also remove any reference to a deleted document from other documents.
+
+
+Build the Source
+----------------
+
+To build the documentation, navigate to the ``ceph`` repository directory:
+
+
+.. prompt:: bash $
+
+	cd ceph
+
+.. note::
+   The directory that contains ``build-doc`` and ``serve-doc`` must be included
+   in the ``PATH`` environment variable in order for these commands to work.
+
+
+To build the documentation on Debian/Ubuntu, Fedora, or CentOS/RHEL, execute:
+
+.. prompt:: bash $
+
+	admin/build-doc
+
+To scan for the reachability of external links, execute:
+
+.. prompt:: bash $
+
+	admin/build-doc linkcheck
+
+Executing ``admin/build-doc`` will create a ``build-doc`` directory under
+``ceph``.  You may need to create a directory under ``ceph/build-doc`` for
+output of Javadoc files:
+
+.. prompt:: bash $
+
+	mkdir -p output/html/api/libcephfs-java/javadoc
+
+The build script ``build-doc`` will produce an output of errors and warnings.
+You MUST fix errors in documents you modified before committing a change, and
+you SHOULD fix warnings that are related to syntax you modified.
+
+.. important:: You must validate ALL HYPERLINKS. If a hyperlink is broken,
+   it automatically breaks the build!
+
+Once you build the documentation set, you may start an HTTP server at
+``http://localhost:8080/`` to view it:
+
+.. prompt:: bash $
+
+	admin/serve-doc
+
+You can also navigate to ``build-doc/output`` to inspect the built documents.
+There should be an ``html`` directory and a ``man`` directory containing
+documentation in HTML and manpage formats respectively.
+
+Build the Source (First Time)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Ceph uses Python Sphinx, which is generally distribution agnostic. The first
+time you build Ceph documentation, it will generate a doxygen XML tree, which
+is a bit time consuming.
+
+Python Sphinx does have some dependencies that vary across distributions. The
+first time you build the documentation, the script will notify you if you do not
+have the dependencies installed. To run Sphinx and build documentation successfully,
+the following packages are required:
+
+.. raw:: html
+
+	<style type="text/css">div.body h3{margin:5px 0px 0px 0px;}</style>
+	<table cellpadding="10"><colgroup><col width="30%"><col width="30%"><col width="30%"></colgroup><tbody valign="top"><tr><td><h3>Debian/Ubuntu</h3>
+
+- gcc
+- python3-dev
+- python3-pip
+- python3-sphinx
+- pytnon3-venv
+- libxml2-dev
+- libxslt1-dev
+- doxygen
+- graphviz
+- ant
+- ditaa
+
+.. raw:: html
+
+	</td><td><h3>Fedora</h3>
+
+- gcc
+- python-devel
+- python-pip
+- python-docutils
+- python-jinja2
+- python-pygments
+- python-sphinx
+- libxml2-devel
+- libxslt1-devel
+- doxygen
+- graphviz
+- ant
+- ditaa
+
+.. raw:: html
+
+	</td><td><h3>CentOS/RHEL</h3>
+
+- gcc
+- python-devel
+- python-pip
+- python-docutils
+- python-jinja2
+- python-pygments
+- python-sphinx
+- libxml2-dev
+- libxslt1-dev
+- doxygen
+- graphviz
+- ant
+
+.. raw:: html
+
+	</td></tr></tbody></table>
+
+
+Install each dependency that is not installed on your host. For Debian/Ubuntu
+distributions, execute the following:
+
+.. prompt:: bash $
+
+	sudo apt-get install gcc python-dev python-pip libxml2-dev libxslt-dev doxygen graphviz ant ditaa
+	sudo apt-get install python-sphinx
+
+For Fedora distributions, execute the following:
+
+.. prompt:: bash $
+
+   sudo yum install gcc python-devel python-pip libxml2-devel libxslt-devel doxygen graphviz ant
+   sudo pip install html2text
+   sudo yum install python-jinja2 python-pygments python-docutils python-sphinx
+   sudo yum install jericho-html ditaa
+
+For CentOS/RHEL distributions, it is recommended to have ``epel`` (Extra
+Packages for Enterprise Linux) repository as it provides some extra packages
+which are not available in the default repository. To install ``epel``, execute
+the following:
+
+.. prompt:: bash $
+
+        sudo yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+
+For CentOS/RHEL distributions, execute the following:
+
+.. prompt:: bash $
+
+	sudo yum install gcc python-devel python-pip libxml2-devel libxslt-devel doxygen graphviz ant
+	sudo pip install html2text
+
+For CentOS/RHEL distributions, the remaining python packages are not available
+in the default and ``epel`` repositories. So, use http://rpmfind.net/ to find
+the packages. Then, download them from a mirror and install them. For example:
+
+.. prompt:: bash $
+
+	wget http://rpmfind.net/linux/centos/7/os/x86_64/Packages/python-jinja2-2.7.2-2.el7.noarch.rpm
+	sudo yum install python-jinja2-2.7.2-2.el7.noarch.rpm
+	wget http://rpmfind.net/linux/centos/7/os/x86_64/Packages/python-pygments-1.4-9.el7.noarch.rpm
+	sudo yum install python-pygments-1.4-9.el7.noarch.rpm
+	wget http://rpmfind.net/linux/centos/7/os/x86_64/Packages/python-docutils-0.11-0.2.20130715svn7687.el7.noarch.rpm
+	sudo yum install python-docutils-0.11-0.2.20130715svn7687.el7.noarch.rpm
+	wget http://rpmfind.net/linux/centos/7/os/x86_64/Packages/python-sphinx-1.1.3-11.el7.noarch.rpm
+	sudo yum install python-sphinx-1.1.3-11.el7.noarch.rpm
+
+Ceph documentation makes extensive use of `ditaa`_, which is not presently built
+for CentOS/RHEL7. You must install ``ditaa`` if you are making changes to
+``ditaa`` diagrams so that you can verify that they render properly before you
+commit new or modified ``ditaa`` diagrams. You may retrieve compatible required
+packages for CentOS/RHEL distributions and install them manually. To run
+``ditaa`` on CentOS/RHEL7, following dependencies are required:
+
+- jericho-html
+- jai-imageio-core
+- batik
+
+Use http://rpmfind.net/ to find compatible ``ditaa`` and the dependencies.
+Then, download them from a mirror and install them. For example:
+
+.. prompt:: bash $
+
+	wget http://rpmfind.net/linux/fedora/linux/releases/22/Everything/x86_64/os/Packages/j/jericho-html-3.3-4.fc22.noarch.rpm
+	sudo yum install jericho-html-3.3-4.fc22.noarch.rpm
+	wget http://rpmfind.net/linux/centos/7/os/x86_64/Packages/jai-imageio-core-1.2-0.14.20100217cvs.el7.noarch.rpm
+	sudo yum install jai-imageio-core-1.2-0.14.20100217cvs.el7.noarch.rpm
+	wget http://rpmfind.net/linux/centos/7/os/x86_64/Packages/batik-1.8-0.12.svn1230816.el7.noarch.rpm
+	sudo yum install batik-1.8-0.12.svn1230816.el7.noarch.rpm
+	wget http://rpmfind.net/linux/fedora/linux/releases/22/Everything/x86_64/os/Packages/d/ditaa-0.9-13.r74.fc21.noarch.rpm
+	sudo yum install ditaa-0.9-13.r74.fc21.noarch.rpm
+
+Once you have installed all these packages, build the documentation by following
+the steps given in `Build the Source`_.
+
+
+Commit the Change
+-----------------
+
+Ceph documentation commits are simple, but follow a strict convention:
+
+- A commit SHOULD have 1 file per commit (it simplifies rollback). You MAY
+  commit multiple files with related changes. Unrelated changes SHOULD NOT
+  be put into the same commit.
+- A commit MUST have a comment.
+- A commit comment MUST be prepended with ``doc:``. (strict)
+- The comment summary MUST be one line only. (strict)
+- Additional comments MAY follow a blank line after the summary, 
+  but should be terse.
+- A commit MAY include ``Fixes: #{bug number}``.
+- Commits MUST include ``Signed-off-by: Firstname Lastname <email>``. (strict)
+
+.. tip:: Follow the foregoing convention particularly where it says 
+   ``(strict)`` or you will be asked to modify your commit to comply with 
+   this convention.
+
+The following is a common commit comment (preferred):: 
+
+	doc: Fixes a spelling error and a broken hyperlink.
+	
+	Signed-off-by: John Doe <john.doe@gmail.com>
+
+
+The following comment includes a reference to a bug. :: 
+
+	doc: Fixes a spelling error and a broken hyperlink.
+
+	Fixes: #1234
+	
+	Signed-off-by: John Doe <john.doe@gmail.com>
+
+
+The following comment includes a terse sentence following the comment summary.
+There is a carriage return between the summary line and the description:: 
+
+	doc: Added mon setting to monitor config reference
+	
+	Describes 'mon setting', which is a new setting added
+	to config_opts.h.
+	
+	Signed-off-by: John Doe <john.doe@gmail.com>
+
+
+To commit changes, execute the following:
+
+.. prompt:: bash $
+
+	git commit -a
+	
+
+An easy way to manage your documentation commits is to use visual tools for
+``git``. For example, ``gitk`` provides a graphical interface for viewing the
+repository history, and ``git-gui`` provides a graphical interface for viewing
+your uncommitted changes, staging them for commit, committing the changes and
+pushing them to your forked Ceph repository.
+
+
+For Debian/Ubuntu, execute:
+
+.. prompt:: bash $
+
+	sudo apt-get install gitk git-gui
+
+For Fedora/CentOS/RHEL, execute:
+
+.. prompt:: bash $
+
+	sudo yum install gitk git-gui
+
+Then, execute:
+
+.. prompt:: bash $
+
+	cd {git-ceph-repo-path}
+	gitk
+	
+Finally, select **File->Start git gui** to activate the graphical user interface.
+
+
+Push the Change
+---------------
+
+Once you have one or more commits, you must push them from the local copy of the
+repository to ``github``. A graphical tool like ``git-gui`` provides a user
+interface for pushing to the repository. If you created a branch previously:
+
+.. prompt:: bash $
+
+	git push origin wip-doc-{your-branch-name}
+
+Otherwise:
+
+.. prompt:: bash $
+
+	git push
+
+
+Make a Pull Request
+-------------------
+
+As noted earlier, you can make documentation contributions using the `Fork and
+Pull`_ approach.
+
+
+
+Notify Us
+---------
+
+After you make a pull request, please email ceph-docs@redhat.com.
+
+
+
+Documentation Style Guide
+=========================
+
+One objective of the Ceph documentation project is to ensure the readability of
+the documentation in both native restructuredText format and its rendered
+formats such as HTML. Navigate to your Ceph repository and view a document in
+its native format. You may notice that it is generally as legible in a terminal
+as it is in its rendered HTML format. Additionally, you may also notice that
+diagrams in ``ditaa`` format also render reasonably well in text mode. :
+
+.. prompt:: bash $
+
+	less doc/architecture.rst
+
+Review the following style guides to maintain this consistency.
+
+
+Headings
+--------
+
+#. **Document Titles:** Document titles use the ``=`` character overline and 
+   underline with a leading and trailing space on the title text line. 
+   See `Document Title`_ for details.
+
+#. **Section Titles:** Section tiles use the ``=`` character underline with no
+   leading or trailing spaces for text. Two carriage returns should precede a 
+   section title (unless an inline reference precedes it). See `Sections`_ for
+   details.
+
+#. **Subsection Titles:** Subsection titles use the ``_`` character underline 
+   with no leading or trailing spaces for text.  Two carriage returns should 
+   precede a subsection title (unless an inline reference precedes it).
+
+
+Text Body
+---------
+
+As a general rule, we prefer text to wrap at column 80 so that it is legible in
+a command line interface without leading or trailing white space. Where
+possible, we prefer to maintain this convention with text, lists, literal text
+(exceptions allowed), tables, and ``ditaa`` graphics.
+
+#. **Paragraphs**: Paragraphs have a leading and a trailing carriage return, 
+   and should be 80 characters wide or less so that the documentation can be 
+   read in native format in a command line terminal.
+
+#. **Literal Text:** To create an example of literal text (e.g., command line
+   usage), terminate the preceding paragraph with ``::`` or enter a carriage
+   return to create an empty line after the preceding paragraph; then, enter
+   ``::`` on a separate line followed by another empty line. Then, begin the
+   literal text with tab indentation (preferred) or space indentation of 3 
+   characters.
+
+#. **Indented Text:** Indented text such as bullet points 
+   (e.g., ``- some text``) may span multiple lines. The text of subsequent
+   lines should begin at the same character position as the text of the
+   indented text (less numbers, bullets, etc.).
+
+   Indented text may include literal text examples. Whereas, text indentation
+   should be done with spaces, literal text examples should be indented with
+   tabs. This convention enables you to add an additional indented paragraph
+   following a literal example by leaving a blank line and beginning the
+   subsequent paragraph with space indentation.
+
+#. **Numbered Lists:** Numbered lists should use autonumbering by starting
+   a numbered indent with ``#.`` instead of the actual number so that
+   numbered paragraphs can be repositioned without requiring manual 
+   renumbering.
+
+#. **Code Examples:** Ceph supports the use of the 
+   ``.. code-block::<language>`` role, so that you can add highlighting to 
+   source examples. This is preferred for source code. However, use of this 
+   tag will cause autonumbering to restart at 1 if it is used as an example 
+   within a numbered list. See `Showing code examples`_ for details.
+
+
+Paragraph Level Markup
+----------------------
+
+The Ceph project uses `paragraph level markup`_ to highlight points.
+
+#. **Tip:** Use the ``.. tip::`` directive to provide additional information
+   that assists the reader or steers the reader away from trouble.
+
+#. **Note**: Use the ``.. note::`` directive to highlight an important point.
+
+#. **Important:** Use the ``.. important::`` directive to highlight important
+   requirements or caveats (e.g., anything that could lead to data loss). Use
+   this directive sparingly, because it renders in red.
+
+#. **Version Added:** Use the ``.. versionadded::`` directive for new features
+   or configuration settings so that users know the minimum release for using
+   a feature.
+   
+#. **Version Changed:** Use the ``.. versionchanged::`` directive for changes
+   in usage or configuration settings.
+
+#. **Deprecated:** Use the ``.. deprecated::`` directive when CLI usage, 
+   a feature or a configuration setting is no longer preferred or will be 
+   discontinued.
+
+#. **Topic:** Use the ``.. topic::`` directive to encapsulate text that is
+   outside the main flow of the document. See the `topic directive`_ for
+   additional details.
+
+
+Table of Contents (TOC) and Hyperlinks
+---------------------------------------
+
+The documents in the Ceph documentation suite follow certain conventions that
+are explained in this section.
+
+Every document (every ``.rst`` file) in the Sphinx-controlled Ceph
+documentation suite must be linked either (1) from another document in the
+documentation suite or (2) from a table of contents (TOC). If any document in
+the documentation suite is not linked in this way, the ``build-doc`` script
+generates warnings when it tries to build the documentation. 
+
+The Ceph project uses the ``.. toctree::`` directive. See `The TOC tree`_ for
+details. When rendering a table of contents (TOC), specify the ``:maxdepth:``
+parameter so that the rendered TOC is not too long.
+
+Use the ``:ref:`` syntax where a link target contains a specific unique
+identifier (for example, ``.. _unique-target-id:``). A link to the section
+designated by ``.. _unique-target-id:`` looks like this:
+``:ref:`unique-target-id```. If this convention is followed, the links within
+the ``.rst`` source files will work even if the source files are moved within
+the ``ceph/doc`` directory. See `Cross referencing arbitrary locations`_ for
+details.
+
+.. _start_external_hyperlink_example:
+
+External Hyperlink Example
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+It is also possible to create a link to a section of the documentation and to
+have custom text appear in the body of the link. This is useful when it is more
+important to preserve the text of the sentence containing the link than it is
+to refer explicitly to the title of the section being linked to.
+
+For example, RST that links to the Sphinx Python Document Generator homepage
+and generates a sentence reading "Click here to learn more about Python
+Sphinx." looks like this: 
+
+::
+
+    ``Click `here <https://www.sphinx-doc.org>`_ to learn more about Python
+    Sphinx.`` 
+
+And here it is, rendered:
+
+Click `here <https://www.sphinx-doc.org>`_ to learn more about Python Sphinx. 
+
+Pay special attention to the underscore after the backtick. If you forget to
+include it and this is your first day working with RST, there's a chance that
+you'll spend all day wondering what went wrong without realizing that you
+omitted that underscore. Also, pay special attention to the space between the
+substitution text (in this case, "here") and the less-than bracket that sets
+the explicit link apart from the substition text. The link will not render
+properly without this space.
+
+Linking Customs
+~~~~~~~~~~~~~~~
+
+By a custom established when Ceph was still being developed by Inktank,
+contributors to the documentation of the Ceph project preferred to use the
+convention of putting ``.. _Link Text: ../path`` links at the bottom of the
+document and linking to them using references of the form ``:ref:`path```. This
+convention was preferred because it made the documents more readable in a
+command line interface. As of 2023, though, we have no preference for one over
+the other. Use whichever convention makes the text easier to read.
+
+Quirks of ReStructured Text
+---------------------------
+
+External Links
+~~~~~~~~~~~~~~
+
+.. _external_link_with_inline_text:
+
+This is the formula for links to addresses external to the Ceph documentation:
+
+::
+
+   `inline text <http:www.foo.com>`_
+
+.. note:: Do not fail to include the space between the inline text and the
+   less-than sign. 
+   
+   Do not fail to include the underscore after the final backtick.
+
+   To link to addresses that are external to the Ceph documentation, include a
+   space between the inline text and the angle bracket that precedes the
+   external address. This is precisely the opposite of :ref:`the convention for
+   inline text that links to a location inside the Ceph
+   documentation<internal_link_with_inline_text>`. If this seems inconsistent
+   and confusing to you, then you're right. It is inconsistent and confusing.
+
+See also ":ref:`External Hyperlink Example<start_external_hyperlink_example>`".
+
+Internal Links
+~~~~~~~~~~~~~~
+
+To link to a section in the Ceph documentation, you must (1) define a target
+link before the section and then (2) link to that target from another location
+in the documentation. Here are the formulas for targets and links to those
+targets:
+
+Target::
+
+   .. _target:
+
+   Title of Targeted Section
+   =========================
+
+   Lorem ipsum...
+
+Link to target::
+
+   :ref:`target`
+
+.. _internal_link_with_inline_text:
+
+Link to target with inline text::
+
+   :ref:`inline text<target>`
+
+.. note:: 
+
+   There is no space between "inline text" and the angle bracket that
+   immediately follows it. This is precisely the opposite of :ref:`the
+   convention for inline text that links to a location outside of the Ceph
+   documentation<external_link_with_inline_text>`. If this seems inconsistent
+   and confusing to you, then you're right. It is inconsistent and confusing.
+
+
+.. _Python Sphinx: https://www.sphinx-doc.org
+.. _restructuredText: http://docutils.sourceforge.net/rst.html
+.. _Fork and Pull: https://help.github.com/articles/using-pull-requests
+.. _github: http://github.com
+.. _ditaa: http://ditaa.sourceforge.net/
+.. _Document Title: http://docutils.sourceforge.net/docs/user/rst/quickstart.html#document-title-subtitle
+.. _Sections: http://docutils.sourceforge.net/docs/user/rst/quickstart.html#sections
+.. _Cross referencing arbitrary locations: http://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#role-ref
+.. _The TOC tree: http://sphinx-doc.org/markup/toctree.html
+.. _Showing code examples: http://sphinx-doc.org/markup/code.html
+.. _paragraph level markup: http://sphinx-doc.org/markup/para.html
+.. _topic directive: http://docutils.sourceforge.net/docs/ref/rst/directives.html#topic
diff --git a/doc/start/get-involved.rst b/doc/start/get-involved.rst
new file mode 100644
index 000000000..bc83f9899
--- /dev/null
+++ b/doc/start/get-involved.rst
@@ -0,0 +1,91 @@
+.. _Get Involved:
+
+=====================================
+ Get Involved in the Ceph Community!
+=====================================
+
+These are exciting times in the Ceph community! Get involved!
+
++----------------------+-------------------------------------------------+-----------------------------------------------+
+|Channel               | Description                                     | Contact Info                                  |
++======================+=================================================+===============================================+
+| **Blog**             | Check the Ceph Blog_ periodically to keep track | http://ceph.com/community/blog/               |
+|                      | of Ceph progress and important announcements.   |                                               |
++----------------------+-------------------------------------------------+-----------------------------------------------+
+| **Planet Ceph**      | Check the blog aggregation on Planet Ceph for   | https://ceph.com/category/planet/             |
+|                      | interesting stories, information and            |                                               |
+|                      | experiences from the community.                 |                                               |
++----------------------+-------------------------------------------------+-----------------------------------------------+
+| **Wiki**             | Check the Ceph Wiki is a source for more        | http://wiki.ceph.com/                         |
+|                      | community and development related topics. You   |                                               |
+|                      | can find there information about blueprints,    |                                               |
+|                      | meetups, the Ceph Developer Summits and more.   |                                               |
++----------------------+-------------------------------------------------+-----------------------------------------------+
+| **IRC**              | As you delve into Ceph, you may have questions  |                                               |
+|                      | or feedback for the Ceph development team. Ceph | - **Domain:**                                 |
+|                      | developers are often available on the ``#ceph`` |     ``irc.oftc.net``                          |
+|                      | IRC channel particularly during daytime hours   | - **Channels:**                               |
+|                      | in the US Pacific Standard Time zone.           |     ``#ceph``,                                |
+|                      | While ``#ceph`` is a good starting point for    |     ``#ceph-devel``,                          |
+|                      | cluster operators and users, there is also      |     ``#ceph-dashboard``                       |
+|                      | ``#ceph-devel`` and ``#ceph-dashboard``         |                                               |
+|                      | dedicated for Ceph developers.                  |                                               |
++----------------------+-------------------------------------------------+-----------------------------------------------+
+| **User List**        | Ask and answer user-related questions by        |                                               |
+|                      | subscribing to the email list at                | - `User Subscribe`_                           |
+|                      | ceph-users@ceph.io. You can opt out of the email| - `User Unsubscribe`_                         |
+|                      | list at any time by unsubscribing. A simple     | - `User Archives`_                            |
+|                      | email is all it takes!                          |                                               |
++----------------------+-------------------------------------------------+-----------------------------------------------+
+| **Devel List**       | Keep in touch with developer activity by        |                                               |
+|                      | subscribing to the email list at dev@ceph.io.   | - `Devel Subscribe`_                          |
+|                      | You can opt out of the email list at any time by| - `Devel Unsubscribe`_                        |
+|                      | unsubscribing. A simple email is all it takes!  | - `Devel Archives`_                           |
++----------------------+-------------------------------------------------+-----------------------------------------------+
+| **Commit List**      | Subscribe to ceph-commit@ceph.com to get        |                                               |
+|                      | commit notifications via email. You can opt out | - `Commit Subscribe`_                         |
+|                      | of the email list at any time by unsubscribing. | - `Commit Unsubscribe`_                       |
+|                      | A simple email is all it takes!                 | - `Mailing list archives`_                    |
++----------------------+-------------------------------------------------+-----------------------------------------------+
+| **QA List**          | For Quality Assurance (QA) related activities   |                                               |
+|                      | subscribe to this list. You can opt out         | - `QA Subscribe`_                             |
+|                      | of the email list at any time by unsubscribing. | - `QA Unsubscribe`_                           |
+|                      | A simple email is all it takes!                 | - `Mailing list archives`_                    |
++----------------------+-------------------------------------------------+-----------------------------------------------+
+| **Community List**   | For all discussions related to the Ceph User    |                                               |
+|                      | Committee and other community topics. You can   | - `Community Subscribe`_                      |
+|                      | opt out of the email list at any time by        | - `Community Unsubscribe`_                    |
+|                      | unsubscribing. A simple email is all it takes!  | - `Mailing list archives`_                    |
++----------------------+-------------------------------------------------+-----------------------------------------------+
+| **Bug Tracker**      | You can help keep Ceph production worthy by     | http://tracker.ceph.com/projects/ceph         |
+|                      | filing and tracking bugs, and providing feature |                                               |
+|                      | requests using the Bug Tracker_.                |                                               |
++----------------------+-------------------------------------------------+-----------------------------------------------+
+| **Source Code**      | If you would like to participate in             |                                               |
+|                      | development, bug fixing, or if you just want    | - http://github.com/ceph/ceph                 |
+|                      | the very latest code for Ceph, you can get it   | - http://download.ceph.com/tarballs/          |
+|                      | at http://github.com. See `Ceph Source Code`_   |                                               |
+|                      | for details on cloning from github.             |                                               |
++----------------------+-------------------------------------------------+-----------------------------------------------+
+| **Ceph Calendar**    | Learn about upcoming Ceph events.               | https://ceph.io/contribute/                   |
++----------------------+-------------------------------------------------+-----------------------------------------------+
+
+
+
+.. _Devel Subscribe: mailto:dev-request@ceph.io?body=subscribe
+.. _Devel Unsubscribe: mailto:dev-request@ceph.io?body=unsubscribe
+.. _User Subscribe: mailto:ceph-users-request@ceph.io?body=subscribe
+.. _User Unsubscribe: mailto:ceph-users-request@ceph.io?body=unsubscribe
+.. _Community Subscribe: mailto:ceph-community-join@lists.ceph.com
+.. _Community Unsubscribe: mailto:ceph-community-leave@lists.ceph.com
+.. _Commit Subscribe: mailto:ceph-commit-join@lists.ceph.com
+.. _Commit Unsubscribe: mailto:ceph-commit-leave@lists.ceph.com
+.. _QA Subscribe: mailto:ceph-qa-join@lists.ceph.com
+.. _QA Unsubscribe: mailto:ceph-qa-leave@lists.ceph.com
+.. _Devel Archives: https://lists.ceph.io/hyperkitty/list/dev@ceph.io/
+.. _User Archives: https://lists.ceph.io/hyperkitty/list/ceph-users@ceph.io/
+.. _Mailing list archives: http://lists.ceph.com/
+.. _Blog: http://ceph.com/community/blog/
+.. _Tracker: http://tracker.ceph.com/
+.. _Ceph Source Code: http://github.com/ceph/ceph
+
diff --git a/doc/start/hardware-recommendations.rst b/doc/start/hardware-recommendations.rst
new file mode 100644
index 000000000..bf8eca4ba
--- /dev/null
+++ b/doc/start/hardware-recommendations.rst
@@ -0,0 +1,509 @@
+.. _hardware-recommendations:
+
+==========================
+ Hardware Recommendations
+==========================
+
+Ceph was designed to run on commodity hardware, which makes building and
+maintaining petabyte-scale data clusters economically feasible. 
+When planning out your cluster hardware, you will need to balance a number 
+of considerations, including failure domains and potential performance
+issues. Hardware planning should include distributing Ceph daemons and 
+other processes that use Ceph across many hosts. Generally, we recommend 
+running Ceph daemons of a specific type on a host configured for that type 
+of daemon. We recommend using other hosts for processes that utilize your 
+data cluster (e.g., OpenStack, CloudStack, etc).
+
+
+.. tip:: Check out the `Ceph blog`_ too.
+
+
+CPU
+===
+
+CephFS metadata servers (MDS) are CPU-intensive. CephFS metadata servers (MDS)
+should therefore have quad-core (or better) CPUs and high clock rates (GHz). OSD
+nodes need enough processing power to run the RADOS service, to calculate data
+placement with CRUSH, to replicate data, and to maintain their own copies of the
+cluster map.
+
+The requirements of one Ceph cluster are not the same as the requirements of
+another, but here are some general guidelines. 
+
+In earlier versions of Ceph, we would make hardware recommendations based on
+the number of cores per OSD, but this cores-per-OSD metric is no longer as
+useful a metric as the number of cycles per IOP and the number of IOPs per OSD.
+For example, for NVMe drives, Ceph can easily utilize five or six cores on real
+clusters and up to about fourteen cores on single OSDs in isolation. So cores
+per OSD are no longer as pressing a concern as they were. When selecting
+hardware, select for IOPs per core.
+
+Monitor nodes and manager nodes have no heavy CPU demands and require only
+modest processors. If your host machines will run CPU-intensive processes in
+addition to Ceph daemons, make sure that you have enough processing power to
+run both the CPU-intensive processes and the Ceph daemons. (OpenStack Nova is
+one such example of a CPU-intensive process.) We recommend that you run
+non-Ceph CPU-intensive processes on separate hosts (that is, on hosts that are
+not your monitor and manager nodes) in order to avoid resource contention.
+
+RAM
+===
+
+Generally, more RAM is better.  Monitor / manager nodes for a modest cluster
+might do fine with 64GB; for a larger cluster with hundreds of OSDs 128GB
+is a reasonable target.  There is a memory target for BlueStore OSDs that
+defaults to 4GB.  Factor in a prudent margin for the operating system and
+administrative tasks (like monitoring and metrics) as well as increased
+consumption during recovery:  provisioning ~8GB per BlueStore OSD
+is advised.
+
+Monitors and managers (ceph-mon and ceph-mgr)
+---------------------------------------------
+
+Monitor and manager daemon memory usage generally scales with the size of the
+cluster.  Note that at boot-time and during topology changes and recovery these
+daemons will need more RAM than they do during steady-state operation, so plan
+for peak usage. For very small clusters, 32 GB suffices. For clusters of up to,
+say, 300 OSDs go with 64GB. For clusters built with (or which will grow to)
+even more OSDs you should provision 128GB. You may also want to consider
+tuning the following settings:
+
+* `mon_osd_cache_size`
+* `rocksdb_cache_size`
+
+
+Metadata servers (ceph-mds)
+---------------------------
+
+The metadata daemon memory utilization depends on how much memory its cache is
+configured to consume.  We recommend 1 GB as a minimum for most systems.  See
+``mds_cache_memory``.
+
+Memory
+======
+
+Bluestore uses its own memory to cache data rather than relying on the
+operating system's page cache. In Bluestore you can adjust the amount of memory
+that the OSD attempts to consume by changing the `osd_memory_target`
+configuration option.
+
+- Setting the `osd_memory_target` below 2GB is typically not
+  recommended (Ceph may fail to keep the memory consumption under 2GB and 
+  this may cause extremely slow performance).
+
+- Setting the memory target between 2GB and 4GB typically works but may result
+  in degraded performance: metadata may be read from disk during IO unless the
+  active data set is relatively small.
+
+- 4GB is the current default `osd_memory_target` size. This default
+  was chosen for typical use cases, and is intended to balance memory
+  requirements and OSD performance.
+
+- Setting the `osd_memory_target` higher than 4GB can improve
+  performance when there many (small) objects or when large (256GB/OSD 
+  or more) data sets are processed.
+
+.. important:: OSD memory autotuning is "best effort". Although the OSD may
+   unmap memory to allow the kernel to reclaim it, there is no guarantee that
+   the kernel will actually reclaim freed memory within a specific time
+   frame. This applies especially in older versions of Ceph, where transparent
+   huge pages can prevent the kernel from reclaiming memory that was freed from
+   fragmented huge pages. Modern versions of Ceph disable transparent huge
+   pages at the application level to avoid this, but that does not
+   guarantee that the kernel will immediately reclaim unmapped memory. The OSD
+   may still at times exceed its memory target. We recommend budgeting 
+   approximately 20% extra memory on your system to prevent OSDs from going OOM
+   (**O**\ut **O**\f **M**\emory) during temporary spikes or due to delay in
+   the kernel reclaiming freed pages. That 20% value might be more or less than
+   needed, depending on the exact configuration of the system.
+
+When using the legacy FileStore back end, the page cache is used for caching
+data, so no tuning is normally needed. When using the legacy FileStore backend,
+the OSD memory consumption is related to the number of PGs per daemon in the
+system.
+
+
+Data Storage
+============
+
+Plan your data storage configuration carefully. There are significant cost and
+performance tradeoffs to consider when planning for data storage. Simultaneous
+OS operations and simultaneous requests from multiple daemons for read and
+write operations against a single drive can slow performance.
+
+Hard Disk Drives
+----------------
+
+OSDs should have plenty of storage drive space for object data. We recommend a
+minimum disk drive size of 1 terabyte. Consider the cost-per-gigabyte advantage
+of larger disks. We recommend dividing the price of the disk drive by the
+number of gigabytes to arrive at a cost per gigabyte, because larger drives may
+have a significant impact on the cost-per-gigabyte. For example, a 1 terabyte
+hard disk priced at $75.00 has a cost of $0.07 per gigabyte (i.e., $75 / 1024 =
+0.0732). By contrast, a 3 terabyte disk priced at $150.00 has a cost of $0.05
+per gigabyte (i.e., $150 / 3072 = 0.0488). In the foregoing example, using the
+1 terabyte disks would generally increase the cost per gigabyte by
+40%--rendering your cluster substantially less cost efficient.
+
+.. tip:: Running multiple OSDs on a single SAS / SATA drive
+   is **NOT** a good idea.  NVMe drives, however, can achieve
+   improved performance by being split into two or more OSDs.
+
+.. tip:: Running an OSD and a monitor or a metadata server on a single 
+   drive is also **NOT** a good idea.
+
+.. tip:: With spinning disks, the SATA and SAS interface increasingly
+   becomes a bottleneck at larger capacities. See also the `Storage Networking 
+   Industry Association's Total Cost of Ownership calculator`_.
+
+
+Storage drives are subject to limitations on seek time, access time, read and
+write times, as well as total throughput. These physical limitations affect
+overall system performance--especially during recovery. We recommend using a
+dedicated (ideally mirrored) drive for the operating system and software, and
+one drive for each Ceph OSD Daemon you run on the host (modulo NVMe above).
+Many "slow OSD" issues (when they are not attributable to hardware failure)
+arise from running an operating system and multiple OSDs on the same drive.
+
+It is technically possible to run multiple Ceph OSD Daemons per SAS / SATA
+drive, but this will lead to resource contention and diminish overall
+throughput.
+
+To get the best performance out of Ceph, run the following on separate drives:
+(1) operating systems, (2) OSD data, and (3) BlueStore db.  For more
+information on how to effectively use a mix of fast drives and slow drives in
+your Ceph cluster, see the `block and block.db`_ section of the Bluestore
+Configuration Reference.
+
+Solid State Drives
+------------------
+
+Ceph performance can be improved by using solid-state drives (SSDs). This
+reduces random access time and reduces latency while accelerating throughput. 
+
+SSDs cost more per gigabyte than do hard disk drives, but SSDs often offer
+access times that are, at a minimum, 100 times faster than hard disk drives.
+SSDs avoid hotspot issues and bottleneck issues within busy clusters, and
+they may offer better economics when TCO is evaluated holistically.
+
+SSDs do not have moving mechanical parts, so they are not necessarily subject
+to the same types of limitations as hard disk drives. SSDs do have significant
+limitations though. When evaluating SSDs, it is important to consider the
+performance of sequential reads and writes.
+
+.. important:: We recommend exploring the use of SSDs to improve performance. 
+   However, before making a significant investment in SSDs, we **strongly
+   recommend** reviewing the performance metrics of an SSD and testing the
+   SSD in a test configuration in order to gauge performance. 
+
+Relatively inexpensive SSDs may appeal to your sense of economy. Use caution.
+Acceptable IOPS are not the only factor to consider when selecting an SSD for
+use with Ceph. 
+
+SSDs have historically been cost prohibitive for object storage, but emerging
+QLC drives are closing the gap, offering greater density with lower power
+consumption and less power spent on cooling. HDD OSDs may see a significant
+performance improvement by offloading WAL+DB onto an SSD.
+
+To get a better sense of the factors that determine the cost of storage, you
+might use the `Storage Networking Industry Association's Total Cost of
+Ownership calculator`_
+
+Partition Alignment
+~~~~~~~~~~~~~~~~~~~
+
+When using SSDs with Ceph, make sure that your partitions are properly aligned.
+Improperly aligned partitions suffer slower data transfer speeds than do
+properly aligned partitions. For more information about proper partition
+alignment and example commands that show how to align partitions properly, see
+`Werner Fischer's blog post on partition alignment`_.
+
+CephFS Metadata Segregation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+One way that Ceph accelerates CephFS file system performance is by segregating
+the storage of CephFS metadata from the storage of the CephFS file contents.
+Ceph provides a default ``metadata`` pool for CephFS metadata. You will never
+have to create a pool for CephFS metadata, but you can create a CRUSH map
+hierarchy for your CephFS metadata pool that points only to SSD storage media.
+See :ref:`CRUSH Device Class<crush-map-device-class>` for details.
+
+
+Controllers
+-----------
+
+Disk controllers (HBAs) can have a significant impact on write throughput.
+Carefully consider your selection of HBAs to ensure that they do not create a
+performance bottleneck. Notably, RAID-mode (IR) HBAs may exhibit higher latency
+than simpler "JBOD" (IT) mode HBAs. The RAID SoC, write cache, and battery
+backup can substantially increase hardware and maintenance costs. Some RAID
+HBAs can be configured with an IT-mode "personality".
+
+.. tip:: The `Ceph blog`_ is often an excellent source of information on Ceph
+   performance issues. See `Ceph Write Throughput 1`_ and `Ceph Write 
+   Throughput 2`_ for additional details.
+
+
+Benchmarking
+------------
+
+BlueStore opens block devices in O_DIRECT and uses fsync frequently to ensure
+that data is safely persisted to media. You can evaluate a drive's low-level
+write performance using ``fio``. For example, 4kB random write performance is
+measured as follows:
+
+.. code-block:: console
+
+  # fio --name=/dev/sdX --ioengine=libaio --direct=1 --fsync=1 --readwrite=randwrite --blocksize=4k --runtime=300
+
+Write Caches
+------------
+
+Enterprise SSDs and HDDs normally include power loss protection features which
+use multi-level caches to speed up direct or synchronous writes.  These devices
+can be toggled between two caching modes -- a volatile cache flushed to
+persistent media with fsync, or a non-volatile cache written synchronously.
+
+These two modes are selected by either "enabling" or "disabling" the write
+(volatile) cache.  When the volatile cache is enabled, Linux uses a device in
+"write back" mode, and when disabled, it uses "write through".
+
+The default configuration (normally caching enabled) may not be optimal, and
+OSD performance may be dramatically increased in terms of increased IOPS and
+decreased commit_latency by disabling the write cache.
+
+Users are therefore encouraged to benchmark their devices with ``fio`` as
+described earlier and persist the optimal cache configuration for their
+devices.
+
+The cache configuration can be queried with ``hdparm``, ``sdparm``,
+``smartctl`` or by reading the values in ``/sys/class/scsi_disk/*/cache_type``,
+for example:
+
+.. code-block:: console
+
+  # hdparm -W /dev/sda
+
+  /dev/sda:
+   write-caching =  1 (on)
+
+  # sdparm --get WCE /dev/sda
+      /dev/sda: ATA       TOSHIBA MG07ACA1  0101
+  WCE           1  [cha: y]
+  # smartctl -g wcache /dev/sda
+  smartctl 7.1 2020-04-05 r5049 [x86_64-linux-4.18.0-305.19.1.el8_4.x86_64] (local build)
+  Copyright (C) 2002-19, Bruce Allen, Christian Franke, www.smartmontools.org
+
+  Write cache is:   Enabled
+
+  # cat /sys/class/scsi_disk/0\:0\:0\:0/cache_type
+  write back
+
+The write cache can be disabled with those same tools:
+
+.. code-block:: console
+
+  # hdparm -W0 /dev/sda
+
+  /dev/sda:
+   setting drive write-caching to 0 (off)
+   write-caching =  0 (off)
+
+  # sdparm --clear WCE /dev/sda
+      /dev/sda: ATA       TOSHIBA MG07ACA1  0101
+  # smartctl -s wcache,off /dev/sda
+  smartctl 7.1 2020-04-05 r5049 [x86_64-linux-4.18.0-305.19.1.el8_4.x86_64] (local build)
+  Copyright (C) 2002-19, Bruce Allen, Christian Franke, www.smartmontools.org
+
+  === START OF ENABLE/DISABLE COMMANDS SECTION ===
+  Write cache disabled
+
+Normally, disabling the cache using ``hdparm``, ``sdparm``, or ``smartctl``
+results in the cache_type changing automatically to "write through". If this is
+not the case, you can try setting it directly as follows. (Users should note
+that setting cache_type also correctly persists the caching mode of the device
+until the next reboot):
+
+.. code-block:: console
+
+  # echo "write through" > /sys/class/scsi_disk/0\:0\:0\:0/cache_type
+
+  # hdparm -W /dev/sda
+
+  /dev/sda:
+   write-caching =  0 (off)
+
+.. tip:: This udev rule (tested on CentOS 8) will set all SATA/SAS device cache_types to "write
+  through":
+
+  .. code-block:: console
+
+    # cat /etc/udev/rules.d/99-ceph-write-through.rules
+    ACTION=="add", SUBSYSTEM=="scsi_disk", ATTR{cache_type}:="write through"
+
+.. tip:: This udev rule (tested on CentOS 7) will set all SATA/SAS device cache_types to "write
+  through":
+
+  .. code-block:: console
+
+    # cat /etc/udev/rules.d/99-ceph-write-through-el7.rules
+    ACTION=="add", SUBSYSTEM=="scsi_disk", RUN+="/bin/sh -c 'echo write through > /sys/class/scsi_disk/$kernel/cache_type'"
+
+.. tip:: The ``sdparm`` utility can be used to view/change the volatile write
+  cache on several devices at once:
+
+  .. code-block:: console
+
+    # sdparm --get WCE /dev/sd*
+        /dev/sda: ATA       TOSHIBA MG07ACA1  0101
+    WCE           0  [cha: y]
+        /dev/sdb: ATA       TOSHIBA MG07ACA1  0101
+    WCE           0  [cha: y]
+    # sdparm --clear WCE /dev/sd*
+        /dev/sda: ATA       TOSHIBA MG07ACA1  0101
+        /dev/sdb: ATA       TOSHIBA MG07ACA1  0101
+
+Additional Considerations
+-------------------------
+
+You typically will run multiple OSDs per host, but you should ensure that the
+aggregate throughput of your OSD drives doesn't exceed the network bandwidth
+required to service a client's need to read or write data. You should also
+consider what percentage of the overall data the cluster stores on each host. If
+the percentage on a particular host is large and the host fails, it can lead to
+problems such as exceeding the ``full ratio``,  which causes Ceph to halt
+operations as a safety precaution that prevents data loss.
+
+When you run multiple OSDs per host, you also need to ensure that the kernel
+is up to date. See `OS Recommendations`_ for notes on ``glibc`` and
+``syncfs(2)`` to ensure that your hardware performs as expected when running
+multiple OSDs per host.
+
+
+Networks
+========
+
+Provision at least 10 Gb/s networking in your racks.
+
+Speed
+-----
+
+It takes three hours to replicate 1 TB of data across a 1 Gb/s network and it
+takes thirty hours to replicate 10 TB across a 1 Gb/s network. But it takes only
+twenty minutes to replicate 1 TB across a 10 Gb/s network, and it takes
+only one hour to replicate 10 TB across a 10 Gb/s network. 
+
+Cost
+----
+
+The larger the Ceph cluster, the more common OSD failures will be.
+The faster that a placement group (PG) can recover from a ``degraded`` state to
+an ``active + clean`` state, the better. Notably, fast recovery minimizes
+the liklihood of multiple, overlapping failures that can cause data to become
+temporarily unavailable or even lost. Of course, when provisioning your
+network, you will have to balance price against performance. 
+
+Some deployment tools employ VLANs to make hardware and network cabling more
+manageable. VLANs that use the 802.1q protocol require VLAN-capable NICs and
+switches. The added expense of this hardware may be offset by the operational
+cost savings on network setup and maintenance. When using VLANs to handle VM
+traffic between the cluster and compute stacks (e.g., OpenStack, CloudStack,
+etc.), there is additional value in using 10 Gb/s Ethernet or better; 40 Gb/s or
+25/50/100 Gb/s networking as of 2022 is common for production clusters.
+
+Top-of-rack (TOR) switches also need fast and redundant uplinks to spind
+spine switches / routers, often at least 40 Gb/s.
+
+
+Baseboard Management Controller (BMC)
+-------------------------------------
+
+Your server chassis should have a Baseboard Management Controller (BMC).
+Well-known examples are iDRAC (Dell), CIMC (Cisco UCS), and iLO (HPE).
+Administration and deployment tools may also use BMCs extensively, especially
+via IPMI or Redfish, so consider the cost/benefit tradeoff of an out-of-band
+network for security and administration.  Hypervisor SSH access, VM image uploads,
+OS image installs, management sockets, etc. can impose significant loads on a network.
+Running three networks may seem like overkill, but each traffic path represents
+a potential capacity, throughput and/or performance bottleneck that you should
+carefully consider before deploying a large scale data cluster.
+ 
+
+Failure Domains
+===============
+
+A failure domain is any failure that prevents access to one or more OSDs. That
+could be a stopped daemon on a host; a disk failure, an OS crash, a
+malfunctioning NIC, a failed power supply, a network outage, a power outage,
+and so forth. When planning out your hardware needs, you must balance the
+temptation to reduce costs by placing too many responsibilities into too few
+failure domains, and the added costs of isolating every potential failure
+domain.
+
+
+Minimum Hardware Recommendations
+================================
+
+Ceph can run on inexpensive commodity hardware. Small production clusters
+and development clusters can run successfully with modest hardware.
+
++--------------+----------------+-----------------------------------------+
+|  Process     | Criteria       | Minimum Recommended                     |
++==============+================+=========================================+
+| ``ceph-osd`` | Processor      | - 1 core minimum                        |
+|              |                | - 1 core per 200-500 MB/s               |
+|              |                | - 1 core per 1000-3000 IOPS             |
+|              |                |                                         |
+|              |                | * Results are before replication.       |
+|              |                | * Results may vary with different       |
+|              |                |   CPU models and Ceph features.         |
+|              |                |   (erasure coding, compression, etc)    |
+|              |                | * ARM processors specifically may       |
+|              |                |   require additional cores.             |
+|              |                | * Actual performance depends on many    |
+|              |                |   factors including drives, net, and    |
+|              |                |   client throughput and latency.        |
+|              |                |   Benchmarking is highly recommended.   |
+|              +----------------+-----------------------------------------+
+|              | RAM            | - 4GB+ per daemon (more is better)      |
+|              |                | - 2-4GB often functions (may be slow)   |
+|              |                | - Less than 2GB not recommended         |
+|              +----------------+-----------------------------------------+
+|              | Volume Storage |  1x storage drive per daemon            |
+|              +----------------+-----------------------------------------+
+|              | DB/WAL         |  1x SSD partition per daemon (optional) |
+|              +----------------+-----------------------------------------+
+|              | Network        |  1x 1GbE+ NICs (10GbE+ recommended)     |
++--------------+----------------+-----------------------------------------+
+| ``ceph-mon`` | Processor      | - 2 cores minimum                       |
+|              +----------------+-----------------------------------------+
+|              | RAM            |  2-4GB+ per daemon                      |
+|              +----------------+-----------------------------------------+
+|              | Disk Space     |  60 GB per daemon                       |
+|              +----------------+-----------------------------------------+
+|              | Network        |  1x 1GbE+ NICs                          |
++--------------+----------------+-----------------------------------------+
+| ``ceph-mds`` | Processor      | - 2 cores minimum                       |
+|              +----------------+-----------------------------------------+
+|              | RAM            |  2GB+ per daemon                        |
+|              +----------------+-----------------------------------------+
+|              | Disk Space     |  1 MB per daemon                        |
+|              +----------------+-----------------------------------------+
+|              | Network        |  1x 1GbE+ NICs                          |
++--------------+----------------+-----------------------------------------+
+
+.. tip:: If you are running an OSD with a single disk, create a
+   partition for your volume storage that is separate from the partition
+   containing the OS. Generally, we recommend separate disks for the
+   OS and the volume storage.
+
+
+
+.. _block and block.db: https://docs.ceph.com/en/latest/rados/configuration/bluestore-config-ref/#block-and-block-db
+.. _Ceph blog: https://ceph.com/community/blog/
+.. _Ceph Write Throughput 1: http://ceph.com/community/ceph-performance-part-1-disk-controller-write-throughput/
+.. _Ceph Write Throughput 2: http://ceph.com/community/ceph-performance-part-2-write-throughput-without-ssd-journals/
+.. _Mapping Pools to Different Types of OSDs: ../../rados/operations/crush-map#placing-different-pools-on-different-osds
+.. _OS Recommendations: ../os-recommendations
+.. _Storage Networking Industry Association's Total Cost of Ownership calculator: https://www.snia.org/forums/cmsi/programs/TCOcalc
+.. _Werner Fischer's blog post on partition alignment: https://www.thomas-krenn.com/en/wiki/Partition_Alignment_detailed_explanation
diff --git a/doc/start/intro.rst b/doc/start/intro.rst
new file mode 100644
index 000000000..d05ccbf7a
--- /dev/null
+++ b/doc/start/intro.rst
@@ -0,0 +1,89 @@
+===============
+ Intro to Ceph
+===============
+
+Whether you want to provide :term:`Ceph Object Storage` and/or
+:term:`Ceph Block Device` services to :term:`Cloud Platforms`, deploy
+a :term:`Ceph File System` or use Ceph for another purpose, all
+:term:`Ceph Storage Cluster` deployments begin with setting up each
+:term:`Ceph Node`, your network, and the Ceph Storage Cluster. A Ceph
+Storage Cluster requires at least one Ceph Monitor, Ceph Manager, and
+Ceph OSD (Object Storage Daemon). The Ceph Metadata Server is also
+required when running Ceph File System clients.
+
+.. ditaa::
+
+            +---------------+ +------------+ +------------+ +---------------+
+            |      OSDs     | | Monitors   | |  Managers  | |      MDSs     |
+            +---------------+ +------------+ +------------+ +---------------+
+
+- **Monitors**: A :term:`Ceph Monitor` (``ceph-mon``) maintains maps
+  of the cluster state, including the monitor map, manager map, the
+  OSD map, the MDS map, and the CRUSH map.  These maps are critical 
+  cluster state required for Ceph daemons to coordinate with each other.  
+  Monitors are also responsible for managing authentication between 
+  daemons and clients.  At least three monitors are normally required 
+  for redundancy and high availability.
+
+- **Managers**: A :term:`Ceph Manager` daemon (``ceph-mgr``) is
+  responsible for keeping track of runtime metrics and the current
+  state of the Ceph cluster, including storage utilization, current
+  performance metrics, and system load.  The Ceph Manager daemons also
+  host python-based modules to manage and expose Ceph cluster
+  information, including a web-based :ref:`mgr-dashboard` and
+  `REST API`_.  At least two managers are normally required for high
+  availability.
+
+- **Ceph OSDs**: An Object Storage Daemon (:term:`Ceph OSD`,
+  ``ceph-osd``) stores data, handles data replication, recovery,
+  rebalancing, and provides some monitoring information to Ceph
+  Monitors and Managers by checking other Ceph OSD Daemons for a
+  heartbeat. At least three Ceph OSDs are normally required for 
+  redundancy and high availability.
+
+- **MDSs**: A :term:`Ceph Metadata Server` (MDS, ``ceph-mds``) stores
+  metadata on behalf of the :term:`Ceph File System` (i.e., Ceph Block
+  Devices and Ceph Object Storage do not use MDS). Ceph Metadata
+  Servers allow POSIX file system users to execute basic commands (like
+  ``ls``, ``find``, etc.) without placing an enormous burden on the
+  Ceph Storage Cluster.
+
+Ceph stores data as objects within logical storage pools. Using the
+:term:`CRUSH` algorithm, Ceph calculates which placement group (PG) should
+contain the object, and which OSD should store the placement group.  The
+CRUSH algorithm enables the Ceph Storage Cluster to scale, rebalance, and
+recover dynamically.
+
+.. _REST API: ../../mgr/restful
+
+.. raw:: html
+
+	<style type="text/css">div.body h3{margin:5px 0px 0px 0px;}</style>
+	<table cellpadding="10"><colgroup><col width="50%"><col width="50%"></colgroup><tbody valign="top"><tr><td><h3>Recommendations</h3>
+	
+To begin using Ceph in production, you should review our hardware
+recommendations and operating system recommendations. 
+
+.. toctree::
+   :maxdepth: 2
+
+   Hardware Recommendations <hardware-recommendations>
+   OS Recommendations <os-recommendations>
+
+
+.. raw:: html 
+
+	</td><td><h3>Get Involved</h3>
+
+   You can avail yourself of help or contribute documentation, source 
+   code or bugs by getting involved in the Ceph community.
+
+.. toctree::
+   :maxdepth: 2
+
+   get-involved
+   documenting-ceph
+
+.. raw:: html
+
+	</td></tr></tbody></table>
diff --git a/doc/start/os-recommendations.rst b/doc/start/os-recommendations.rst
new file mode 100644
index 000000000..64887c925
--- /dev/null
+++ b/doc/start/os-recommendations.rst
@@ -0,0 +1,65 @@
+====================
+ OS Recommendations
+====================
+
+Ceph Dependencies
+=================
+
+As a general rule, we recommend deploying Ceph on newer releases of Linux. 
+We also recommend deploying on releases with long-term support.
+
+Linux Kernel
+------------
+
+- **Ceph Kernel Client**
+
+  If you are using the kernel client to map RBD block devices or mount
+  CephFS, the general advice is to use a "stable" or "longterm
+  maintenance" kernel series provided by either http://kernel.org or
+  your Linux distribution on any client hosts.
+
+  For RBD, if you choose to *track* long-term kernels, we currently recommend
+  4.x-based "longterm maintenance" kernel series or later:
+
+  - 4.19.z
+  - 4.14.z
+  - 5.x
+
+  For CephFS, see the section about `Mounting CephFS using Kernel Driver`_
+  for kernel version guidance.
+
+  Older kernel client versions may not support your `CRUSH tunables`_ profile
+  or other newer features of the Ceph cluster, requiring the storage cluster
+  to be configured with those features disabled.
+
+
+Platforms
+=========
+
+The charts below show how Ceph's requirements map onto various Linux
+platforms.  Generally speaking, there is very little dependence on
+specific distributions outside of the kernel and system initialization
+package (i.e., sysvinit, systemd).
+
++--------------+--------+------------------------+--------------------------------+-------------------+-----------------+
+| Release Name | Tag    | CentOS                 | Ubuntu                         | OpenSUSE :sup:`C` | Debian :sup:`C` |
++==============+========+========================+================================+===================+=================+
+| Quincy       | 17.2.z | 8 :sup:`A`             | 20.04 :sup:`A`                 | 15.3              | 11              |
++--------------+--------+------------------------+--------------------------------+-------------------+-----------------+
+| Pacific      | 16.2.z | 8 :sup:`A`             | 18.04 :sup:`C`, 20.04 :sup:`A` | 15.2              | 10, 11          |
++--------------+--------+------------------------+--------------------------------+-------------------+-----------------+
+| Octopus      | 15.2.z | 7 :sup:`B` 8 :sup:`A`  | 18.04 :sup:`C`, 20.04 :sup:`A` | 15.2              | 10              |
++--------------+--------+------------------------+--------------------------------+-------------------+-----------------+
+
+- **A**: Ceph provides packages and has done comprehensive tests on the software in them.
+- **B**: Ceph provides packages and has done basic tests on the software in them.
+- **C**: Ceph provides packages only. No tests have been done on these releases.
+
+.. note::
+   **For Centos 7 Users** 
+   
+   ``Btrfs`` is no longer tested on Centos 7 in the Octopus release. We recommend using ``bluestore`` instead.
+
+.. _CRUSH Tunables: ../../rados/operations/crush-map#tunables
+
+.. _Mounting CephFS using Kernel Driver: ../../cephfs/mount-using-kernel-driver#which-kernel-version
diff --git a/doc/start/quick-rbd.rst b/doc/start/quick-rbd.rst
new file mode 100644
index 000000000..c1cf77098
--- /dev/null
+++ b/doc/start/quick-rbd.rst
@@ -0,0 +1,69 @@
+==========================
+ Block Device Quick Start
+==========================
+
+Ensure your :term:`Ceph Storage Cluster` is in an ``active + clean`` state
+before working with the :term:`Ceph Block Device`.
+
+.. note:: The Ceph Block Device is also known as :term:`RBD` or :term:`RADOS`
+   Block Device.
+
+
+.. ditaa::
+
+           /------------------\         /----------------\
+           |    Admin Node    |         |   ceph-client  |
+           |                  +-------->+ cCCC           |
+           |    ceph-deploy   |         |      ceph      |
+           \------------------/         \----------------/
+
+
+You may use a virtual machine for your ``ceph-client`` node, but do not 
+execute the following procedures on the same physical node as your Ceph 
+Storage Cluster nodes (unless you use a VM). See `FAQ`_ for details.
+
+Create a Block Device Pool
+==========================
+
+#. On the admin node, use the ``ceph`` tool to `create a pool`_
+   (we recommend the name 'rbd').
+
+#. On the admin node, use the ``rbd`` tool to initialize the pool for use by RBD::
+
+        rbd pool init <pool-name>
+
+Configure a Block Device
+========================
+
+#. On the ``ceph-client`` node, create a block device image. :: 
+
+	rbd create foo --size 4096 --image-feature layering [-m {mon-IP}] [-k /path/to/ceph.client.admin.keyring] [-p {pool-name}]
+
+#. On the ``ceph-client`` node, map the image to a block device. :: 
+
+	sudo rbd map foo --name client.admin [-m {mon-IP}] [-k /path/to/ceph.client.admin.keyring] [-p {pool-name}]
+	
+#. Use the block device by creating a file system on the ``ceph-client`` 
+   node. :: 
+
+	sudo mkfs.ext4 -m0 /dev/rbd/{pool-name}/foo
+	
+	This may take a few moments.
+	
+#. Mount the file system on the ``ceph-client`` node. ::
+
+	sudo mkdir /mnt/ceph-block-device
+	sudo mount /dev/rbd/{pool-name}/foo /mnt/ceph-block-device
+	cd /mnt/ceph-block-device
+
+#. Optionally configure the block device to be automatically mapped and mounted
+   at boot (and unmounted/unmapped at shutdown) - see the `rbdmap manpage`_.
+
+
+See `block devices`_ for additional details.
+
+.. _create a pool: ../../rados/operations/pools/#create-a-pool
+.. _block devices: ../../rbd
+.. _FAQ: http://wiki.ceph.com/How_Can_I_Give_Ceph_a_Try
+.. _OS Recommendations: ../os-recommendations
+.. _rbdmap manpage: ../../man/8/rbdmap
diff --git a/doc/start/rgw.conf b/doc/start/rgw.conf
new file mode 100644
index 000000000..e1bee9986
--- /dev/null
+++ b/doc/start/rgw.conf
@@ -0,0 +1,30 @@
+FastCgiExternalServer /var/www/s3gw.fcgi -socket /tmp/radosgw.sock
+
+
+<VirtualHost *:80>
+
+	ServerName {fqdn}
+	<!--Remove the comment. Add a server alias with *.{fqdn} for S3 subdomains-->
+	<!--ServerAlias *.{fqdn}-->
+	ServerAdmin {email.address}
+	DocumentRoot /var/www
+	RewriteEngine On
+	RewriteRule ^/([a-zA-Z0-9-_.]*)([/]?.*) /s3gw.fcgi?page=$1&params=$2&%{QUERY_STRING} [E=HTTP_AUTHORIZATION:%{HTTP:Authorization},L]
+
+	<IfModule mod_fastcgi.c>
+   	<Directory /var/www>
+			Options +ExecCGI
+			AllowOverride All
+			SetHandler fastcgi-script
+			Order allow,deny
+			Allow from all
+			AuthBasicAuthoritative Off
+		</Directory>
+	</IfModule>
+
+	AllowEncodedSlashes On
+	ErrorLog /var/log/apache2/error.log
+	CustomLog /var/log/apache2/access.log combined
+	ServerSignature Off
+
+</VirtualHost>
\ No newline at end of file